From 8b2df58266cbe67df1fec255fbee81f456a5cccd Mon Sep 17 00:00:00 2001 From: Alex Maldonado Date: Mon, 12 Oct 2026 01:07:50 -0400 Subject: [PATCH] feat: documents and session tokens --- .gitattributes | 2 + .github/workflows/check.yml | 16 +-- .gitignore | 6 +- .markdownlint-cli2.jsonc | 5 + CHANGELOG.md | 19 +-- CONTRIBUTING.md | 15 ++- DECISIONS.md | 61 --------- LICENSE.md | 5 +- Makefile | 13 -- README.md | 98 +++++++++----- context/README.md | 14 ++ docs/README.md | 17 +++ docs/concepts.md | 262 ++++++++++++++++++++++++++++++++++++ docs/examples.md | 221 ++++++++++++++++++++++++++++++ docs/faq.md | 45 +++++++ docs/glossary.md | 118 ++++++++++++++++ docs/overview.md | 92 +++++++++++++ docs/rationale.md | 150 +++++++++++++++++++++ docs/walkthrough.md | 126 +++++++++++++++++ fixtures/README.md | 60 +++++++-- fixtures/src/README.md | 3 +- pixi.toml | 34 +++++ registry/README.md | 36 +++++ schemas/README.md | 47 +++++++ spec/00-introduction.md | 25 ++-- spec/05-hashes.md | 2 +- spec/08-fidelity.md | 4 +- spec/11-corrections.md | 2 +- spec/12-package.md | 2 +- spec/13-database.md | 2 +- spec/14-rights.md | 8 +- spec/21-drafts.md | 2 +- spec/22-conformance.md | 6 +- spec/README.md | 18 ++- sql/README.md | 40 ++++++ tools/README.md | 25 +++- tools/check.py | 52 ++++++- tools/load.py | 38 ++++++ vectors/README.md | 31 +++-- 39 files changed, 1523 insertions(+), 199 deletions(-) create mode 100644 .markdownlint-cli2.jsonc delete mode 100644 DECISIONS.md delete mode 100644 Makefile create mode 100644 context/README.md create mode 100644 docs/README.md create mode 100644 docs/concepts.md create mode 100644 docs/examples.md create mode 100644 docs/faq.md create mode 100644 docs/glossary.md create mode 100644 docs/overview.md create mode 100644 docs/rationale.md create mode 100644 docs/walkthrough.md create mode 100644 pixi.toml create mode 100644 registry/README.md create mode 100644 schemas/README.md create mode 100644 sql/README.md create mode 100644 tools/load.py diff --git a/.gitattributes b/.gitattributes index b902239..7d23aca 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,3 +3,5 @@ vectors/** -text diff fixtures/** -text diff fixtures/**/*.png binary +# The pixi lock file: generated, highlighted as YAML, never merged line by line. +pixi.lock merge=binary linguist-language=YAML linguist-generated=true diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index c0540da..b9fd041 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -1,21 +1,21 @@ # Forgejo and Gitea Actions both read .github/workflows when their own folder is absent. +# On Forgejo, if the runner's default action mirror lacks setup-pixi, write it as +# https://github.com/prefix-dev/setup-pixi@v0.9.4. name: check on: [push, pull_request] jobs: check: strategy: + fail-fast: false matrix: os: [ubuntu-latest, macos-latest] + environment: [default, py310] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: prefix-dev/setup-pixi@v0.9.4 with: - python-version: "3.12" - - name: Second schema validator - run: python3 -m pip install jsonschema referencing + pixi-version: v0.69.0 + environments: ${{ matrix.environment }} - name: Check the repository - run: python3 tools/check.py - - name: Lint Markdown - if: matrix.os == 'ubuntu-latest' - run: npx --yes markdownlint-cli@0.48.0 '**/*.md' + run: pixi run --environment ${{ matrix.environment }} check diff --git a/.gitignore b/.gitignore index 1d37372..c1c6f93 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,8 @@ __pycache__/ *.pyc -.venv/ *.db +*.db.assets/ + +# pixi environments +.pixi/* +!.pixi/config.toml diff --git a/.markdownlint-cli2.jsonc b/.markdownlint-cli2.jsonc new file mode 100644 index 0000000..d5c089b --- /dev/null +++ b/.markdownlint-cli2.jsonc @@ -0,0 +1,5 @@ +{ + // Which files markdownlint-cli2 checks; the rules themselves are in .markdownlint.jsonc. + "globs": ["**/*.md"], + "ignores": [".pixi/**", "node_modules/**"] +} diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b3e6d9..843e719 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,17 @@ # Changelog -Every change to a computed value is a breaking change and needs a major version of the format it touches; see [spec/00-introduction.md](spec/00-introduction.md). +A change to any computed value is a breaking change and needs a new major version of the format it touches; see [spec/00-introduction.md](spec/00-introduction.md). ## 0.1.0 -The first release, written for level 3 so that formats never change as the studio moves up. +The first release. -- Chapters 0 to 22, covering every format the design names. -- 37 JSON Schemas, restricted to a keyword subset that independent validators read alike. -- Tables for SQLite, Dolt and PostgreSQL, a team store, and the SQLite bundle with FTS5. -- The JSON-LD context for layers, and registries of formats, rules, stamp rules and stage defaults. -- Vectors for canonical JSON, normalization, IDs, notation, both projections, offsets, the five values, addresses, keyed hashes, page fingerprints, Merkle roots, signatures, spot-checks, error bounds, alignment, every correction, anchor resolution, rights and grading. -- Packages for the three M1 fixtures (two sections of a converted chemistry paper with a multi-level-header table, an open-access paper and a short Typst draft), a checks-only export, and a tiny document; records of every other format; and seventeen invalid packages. +- The specification, in chapters 0 to 22, covering every format. +- 37 JSON Schemas, using a keyword subset that independent validators read alike. +- Tables for SQLite, Dolt and PostgreSQL, a team's store, and the SQLite bundle with full-text search. +- The JSON-LD context for layers, and registries of formats, rules, download-stamp rules and default stage requirements. +- Test vectors for every computed value, from canonical JSON to anchor resolution and grading. +- Example packages (a converted chemistry paper with corrections and receipts, its checks-only export, an open-access paper, a Typst draft and a tiny document), a record of every other format, and seventeen broken packages. +- Guides for newcomers: an overview, the concepts, a walkthrough of the example paper, runnable examples, the rationale, a FAQ and a glossary. +- Tools that build and check everything, and one that loads packages into SQLite. +- A pixi workspace with tasks for checking, building, linting and loading, and an environment for the oldest supported Python. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fe55e2b..525da13 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,23 +3,30 @@ ## One fix, one test Every change to a rule comes with the vector or fixture that shows it. -Change the oracle in `tools/oracle/` and the chapter together, run `python3 tools/build.py` to regenerate the vectors and fixtures, and commit the regenerated files with the change. -`python3 tools/check.py` must pass. +Change the oracle in `tools/oracle/` and the chapter together, run `pixi run build` to regenerate the vectors and fixtures, and commit the regenerated files with the change. +`pixi run check` must pass. +Without pixi, run `python3 tools/build.py` and `python3 tools/check.py`, and lint the Markdown with markdownlint-cli2. A fixture that breaks something belongs in `fixtures/invalid/`, with the rule it breaks in `expect.json`, built by `tools/build.py` like the others. + +## The environment + +`pixi.toml` pins the development environment: Python, the `jsonschema` package and markdownlint-cli2, from conda-forge. +Commit `pixi.lock` whenever pixi updates it, so that CI and every contributor solve the same environment. +`pixi run -e py310 check` runs the checks on the oldest supported Python. Rule IDs are stable: a rule's meaning never changes, and a retired ID is never reused. ## Versions A change that makes a valid file invalid, or changes any computed value, is breaking: it needs a new major version of the format and a migration in `scienting-core`. Adding an optional property, a kind or an enumeration value is a minor change. -Record each change in [CHANGELOG.md](CHANGELOG.md), and each departure from the design in [DECISIONS.md](DECISIONS.md). +Record each change in [CHANGELOG.md](CHANGELOG.md), and explain any rule that a reader may question in [docs/rationale.md](docs/rationale.md). ## Prose US English, one sentence per line, so a change to a sentence is a one-line diff. Prefer plain words to abbreviations; write "for example", not "e.g.". -`tools/check.py` lints both, and `markdownlint` checks the Markdown. +`tools/check.py` lints both, and `pixi run lint` checks the Markdown with markdownlint-cli2. ## Data diff --git a/DECISIONS.md b/DECISIONS.md deleted file mode 100644 index 62f1b74..0000000 --- a/DECISIONS.md +++ /dev/null @@ -1,61 +0,0 @@ -# Decisions - -In the style of Lode's `DECISIONS.md`. -Each entry says what `spec` 0.1 does where the design leaves a choice open, refines a rule so that two implementations compute the same bytes, or departs from the design's wording. -Entries marked *provisional* wait for the maintainer; everything else is adopted for 0.1. - -## Refinements and departures - -| ID | Decision | Why | -| --- | --- | --- | -| D1 | The repository is `varve-spec`; every identifier inside stays `scienting-*`. | The design keeps product names out of technical identifiers. | -| D2 | Offsets and quotations index the stored plain projection; `own_hash` and `node_hash` cover stored values exactly, without normalization; only the text values normalize. | When `node_hash` is equal, every offset is still exact, and a renderer slices stored text directly. | -| D3 | The kind list names `title`, `authors`, `abstract`, `keywords`, `notes`, `acknowledgments`, `appendix`, `figure-note` and `table-note` explicitly. | The design uses them as children but omits them from the list it says refuses all others. | -| D4 | After its heading, a section's blocks and subsections may interleave. | Papers put text after subsections; the design's "then nested section" would refuse them. | -| D5 | Simple notation includes `.` and `,`. | Otherwise `$-1.5$` is complex and never agrees with the text "−1.5". | -| D6 | The strict projection marks `cite`, `xref` and `fnmark` runs with U+FFF9 and U+FFFB. | Citation structure must not vanish: "10" and citation 3 is not "103". The design is silent. | -| D7 | Outside notation, a hyphen-minus or en dash before a digit becomes U+2212. | The design's "a hyphen before a digit" read for text as well as notation; "10-20" and "10–20" then agree. | -| D8 | The lossy projection deletes whitespace and dashes rather than collapsing them. | OCR merges and splits words; the key only proposes alignments, so recall matters. | -| D9 | Container projections join their text leaves with a space; [3.6](spec/03-nodes.md#36-the-plain-projection) defines the leaves. | The design says "concatenated" without a separator or a leaf set. | -| D10 | Anchors always carry `own_hash`; anchors on containers without content carry their `leaves`. | The relabel rule needs `own_hash` on every anchor, and the "half of the original leaves" rule needs the original leaves. | -| D11 | Quotations over 200 code points become `exact_hash` and `length`; `prefix` and `suffix` hold at most 32. | The open decision on the cap, adopted provisionally, without losing reattachment of long sentences. *Provisional.* | -| D12 | When a quote has several hits, `prefix` and `suffix` pick one. | As W3C text-quote selectors do; the design says only "exactly once". | -| D13 | A tenth operation, `set`, changes one structural attribute; `place_after` positions inserts and moves; `split` trims whitespace at the cut; `merge` joins runs with one space by default. | Spans, roles and header paths are dropped by readings and need a correction; `after` was needed for a hash. | -| D14 | Applied corrections record `before`, `after`, `path`, `to_path`, `touched`, `created`, `retired` and `from`. | Drift and relabel reattachment become computable from the record, without replaying history. | -| D15 | Drift's share is touched leaves over current leaves and touched leaves together, four decimals, half up; `high` is strictly above the threshold. | The design gives the idea; this gives one number. | -| D16 | A row's `header_path` is not part of any hash. | The design gives rows no content of their own; so a header-path fix does not stale cell anchors. | -| D17 | Fingerprint text deletes whitespace and control and format characters. | PDF extractors disagree about where they insert spaces. | -| D18 | SQL names avoid words reserved in SQLite, PostgreSQL or MySQL: `doc_key`, `work_key`, `idx`, `made_at`, `signer`, `before_hash`, `after_hash`, `ref_node`. | Dolt is MySQL-compatible, and `key`, `index`, `before` and `by` are reserved there. | -| D19 | The tables store every property of every record, JSON as canonical text, so that unload gives the package back byte for byte, with assets by SHA-256 in `package_files`. | The design requires a byte-identical round trip. | -| D20 | Keyed hashing: HKDF salts `scienting-team-key/1` and `scienting-page-key/1`, info `scienting-hash/1` or the export's context, `key_id` from a domain-separated SHA-256, and page hashes rather than fingerprints in exports. | The design's open decision on the scheme, made concrete. *Provisional.* | -| D21 | Receipts may attest F1 to F3, may cover `records` as well as nodes, and are re-issued under a key with `export_of`. | The same mechanism signs annotations and digitizations; plain hashes cannot leave the team. | -| D22 | Spot-check samples rank node IDs by SHA-256 of the seed and the ID; bounds are one-sided at 95%, reported to six places, rounded up. | One deterministic sample and one string for every implementation. | -| D23 | Checks-only packages also omit corrections, suspects, datasets, figure text and CSL records. | Each carries fragments of the text. | -| D24 | Fidelity is recorded on leaves only, and F3 comes only from receipts. | A receipt is the only record of a person's check. | -| D25 | A policy resolves only unknown uses; an expired basis makes every use false. | A team's decision should never silently widen an explicit `false`. | -| D26 | Backend conditions are a closed vocabulary, and an unknown answer fails a condition. | A gate that guesses is not a gate. | - -## Still open - -| ID | Decision | Default here | Waits for | -| --- | --- | --- | --- | -| O1 | The roster: `scienting-roster/1` is defined here; Lode already verifies a roster. | This format. | Reconciling with Lode's at M4. | -| O2 | The manifest's exact field names against `lode-cache/1`. | `{format, doc, key, files: [{path, sha256, size}]}`. | Comparing with Lode's implementation. | -| O3 | Package extension and media type. | `.sct.zip`, `application/vnd.scienting.doc+zip`. | The maintainer; fixed forever once shared. | -| O4 | Node ID form. | Opaque. | The maintainer. | -| O5 | Text normalization for `text_hash`. | NFC and whitespace. | The maintainer; existing Lode extracts keep Lode's rule. | -| O6 | Canonical math, stage 2. | Stage 1: complex runs compared by source, run by run. | H4. | -| O7 | Quotation cap. | 200 code points. | Counsel and robustness measurements. | -| O8 | Drift threshold, spot-check sample, digit rule, target set, receipt scopes, container strictness. | As the design proposes. | The validation tests of the design's 16.1. | -| O9 | Stage-to-rights defaults and the combining rule. | As the design proposes, in `registry/stages/`. | Counsel. | -| O10 | Stamp-line rules. | Four published patterns and a margin rule. | Pairs of real downloads, kept out of the repository. | -| O11 | License of the tools. | Apache-2.0. | The maintainer. | - -## Known limits of 0.1 - -- The strict projection removes spaces inside notation but keeps them in text, so `$T = 300$` from LaTeX and "T = 300" from a PDF disagree and become a `word` suspect; M3 should measure how often. -- Page fingerprints depend on extracted text lines; the vectors test the function, and the converter and the reader must use extractors that agree on the corpus. -- The oracle verifies Ed25519 only; SSH, OpenPGP and Sigstore signatures are specified by reference. -- CI executes the SQL on SQLite only; Dolt and PostgreSQL are covered by the portability rules and the reserved-word check, not by execution. -- There is no zipped package among the fixtures yet. -- Vectors were computed with Unicode 15.0; normalization is stable across versions, but Script and General Category assignments for newly encoded characters are not. diff --git a/LICENSE.md b/LICENSE.md index debf9d2..dc63081 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -2,14 +2,13 @@ | Paths | License | SPDX | | --- | --- | --- | -| `spec/`, `README.md`, `DECISIONS.md`, `CONTRIBUTING.md`, `CHANGELOG.md` and the other prose | Creative Commons Attribution 4.0 International | `CC-BY-4.0` | +| `spec/`, `docs/`, `README.md`, `CONTRIBUTING.md`, `CHANGELOG.md` and the other prose | Creative Commons Attribution 4.0 International | `CC-BY-4.0` | | `schemas/`, `sql/`, `context/`, `registry/`, `vectors/`, `fixtures/` | Creative Commons Zero 1.0 Universal | `CC0-1.0` | | `tools/` | Apache License 2.0 | `Apache-2.0` | The legal texts are at , and . -Before the first public release, add their verbatim texts under `LICENSES/`, as the REUSE specification describes; `reuse download --all` fetches them. -The license of the tools is an open decision of the design (Apache-2.0 or Prosperity 3.0.0); Apache-2.0 is used here because the tools exist only to check the formats. +The formats are open so that anyone may implement them, in any product, without asking. All text in the fixtures is synthetic, written for them. License fields inside fixtures describe fictional works, not the fixtures. diff --git a/Makefile b/Makefile deleted file mode 100644 index 6f9f807..0000000 --- a/Makefile +++ /dev/null @@ -1,13 +0,0 @@ -.PHONY: check quick build lint - -check: lint - python3 tools/check.py - -quick: - python3 tools/check.py --quick - -build: - python3 tools/build.py - -lint: - @if command -v markdownlint >/dev/null 2>&1; then markdownlint '**/*.md'; else echo "markdownlint not installed; skipped"; fi diff --git a/README.md b/README.md index d03fdf7..bb43e6c 100644 --- a/README.md +++ b/README.md @@ -1,52 +1,76 @@ # varve-spec -The specifications of the Scientific Computing Studio's data formats: documents and their packages, anchors, receipts, rights, layers, exercises, datasets, digitizations, views, projects and bundles. -This is the repository the design calls `spec`, the baseline every other part implements. +**The data formats behind Lode, paperpack and Varve: how a team's scientific papers are stored once, corrected once, addressed down to the sentence, and kept.** -`spec` defines data, and data outlives code. -Every file any part writes is defined here, and a conforming file can be read without the part that wrote it. -Formats are named `scienting-/`, never after a product, so a product can be renamed without touching data. +A research team usually cares about a few dozen papers it understands completely, not the world's literature. +Those papers deserve better than a folder of PDFs: every sentence, table cell and equation should have a permanent address; every correction should be recorded with its evidence; every note, claim and extracted number should point at exactly what it came from; and everyone should be able to see how far each piece was checked, and by whom. -## What is here +This repository defines the files that make that possible. +It holds no application. +paperpack writes these files, Lode reads them to check claims, and Varve stores, renders and annotates them; any other program can do the same, because every format is documented here and readable without the studio's code. -| Path | Holds | -| --- | --- | -| [spec/](spec/README.md) | The normative chapters. Start with [spec/00-introduction.md](spec/00-introduction.md). | -| [schemas/](schemas/) | JSON Schemas (draft 2020-12) for every file and record. | -| [sql/](sql/) | Tables for SQLite, Dolt and PostgreSQL, and the SQLite bundle with full-text search. | -| [context/](context/) | The JSON-LD context that makes layers W3C Web Annotations. | -| [registry/](registry/) | Formats and media types, validation rule IDs, download-stamp rules and the default stage requirements. | -| [vectors/](vectors/README.md) | Test vectors for every computed value. | -| [fixtures/](fixtures/README.md) | Valid packages and records, invalid packages with the rule each breaks, and the test keys that signed them. | -| [tools/](tools/README.md) | The non-normative oracle that computed the vectors, the builder and the checker. | -| [DECISIONS.md](DECISIONS.md) | Where this text refines or departs from the design, and the decisions still open. | - -## Check it - -Python 3.10 or later, with nothing installed: - -```sh -python3 tools/check.py +```text +cornell1995second#k3m9q2a@4f1a0c[12:57] +└──── document ──┘└ node ┘└ pin ┘└span┘ ``` -It validates every schema, record and package, refuses every invalid package for the stated rule, recomputes every vector, rebuilds the fixtures and vectors and compares them byte for byte, loads every package into SQLite and unloads it byte for byte, and lints the prose. -With `jsonschema` installed it also validates every record a second time with that library. -`make check` does the same and runs `markdownlint` when it is available. +That string names code points 12 to 57 of one sentence, as it read when its hash began `4f1a0c`. +It survives reconversion with a better tool, a typo fixed elsewhere in the paper, and a teammate's differently stamped download of the same PDF. -## Implement it +## Start here -An implementation conforms to the chapters and the vectors. -`scienting-core` (Python, standard library only) is the reference implementation of the shared rules, and `core-ts` the browser's; each runs this suite in its CI. -The oracle in `tools/` is not a library and is not meant to be imported: it exists so that no vector is typed by hand. +| If you want to | Read | +| --- | --- | +| Understand what the project is and how the parts fit | [docs/overview.md](docs/overview.md) | +| Learn the ideas one at a time, each with an example | [docs/concepts.md](docs/concepts.md) | +| Follow one paper from its first reading to a signed record | [docs/walkthrough.md](docs/walkthrough.md) | +| Run something in the next five minutes | [docs/examples.md](docs/examples.md) | +| Implement a reader or writer | [spec/](spec/README.md), then [vectors/](vectors/README.md) | +| Know why a rule is the way it is | [docs/rationale.md](docs/rationale.md) | +| Look up a term | [docs/glossary.md](docs/glossary.md) | -## Status +## What is in the repository -Version 0.1.0. -The formats of milestone M1 are `candidate`; datasets, digitizations, views, projects, bundles and the draft formats are `provisional` until their milestones; see [registry/formats.json](registry/formats.json). +| Folder | Holds | +| --- | --- | +| [spec/](spec/README.md) | The specification, in 23 chapters. Normative. | +| [schemas/](schemas/README.md) | JSON Schemas for every file and record. | +| [sql/](sql/README.md) | Tables for SQLite, Dolt and PostgreSQL, and the SQLite bundle with full-text search. | +| [context/](context/README.md) | The JSON-LD context that makes annotation layers W3C Web Annotations. | +| [registry/](registry/README.md) | The list of formats, the rule IDs validators report, download-stamp rules, and default rights requirements. | +| [vectors/](vectors/README.md) | Test vectors: inputs and the exact outputs every implementation must compute. | +| [fixtures/](fixtures/README.md) | Example packages and records, deliberately broken packages, and the test keys that signed them. | +| [tools/](tools/README.md) | The programs that build and check everything above. | +| [docs/](docs/README.md) | Guides for people getting to know the project. | + +## Try it + +With [pixi](https://pixi.sh), one command sets up a pinned environment and checks everything: + +```sh +pixi run check +``` + +The checker validates every schema, record and package, confirms that every broken package is refused for the stated reason, recomputes every test vector, rebuilds everything and compares it byte for byte, loads every package into SQLite and back, and lints the prose and the links. +It ends with `All checks pass.` +`pixi task list` shows the other tasks: `quick`, `build`, `lint` and `load`. + +The tools need nothing but Python 3.10 or later, so without pixi the same check is `python3 tools/check.py`; pixi adds the `jsonschema` package as a second schema validator and markdownlint for the Markdown. + +To poke at a real document, load the example paper into SQLite and look around; [docs/examples.md](docs/examples.md) has queries to start from. + +```sh +pixi run load --sqlite lab.db fixtures/packages/hale2026harmonic +``` + +## Formats + +Every format is named `scienting-/`, such as `scienting-doc/1` or `scienting-anchor/1`, and never after a product, so products can be renamed without touching data. +[registry/formats.json](registry/formats.json) lists them all with their media types, schemas and chapters. ## Licenses -The chapters and other prose are licensed CC BY 4.0. -Schemas, SQL, the JSON-LD context, the registries, the vectors and the fixtures are dedicated to the public domain under CC0 1.0, so anyone may implement the formats. -The tools are licensed Apache-2.0. +The specification and other prose are CC BY 4.0, so anyone may implement the formats. +Schemas, SQL, the JSON-LD context, the registries, the vectors and the fixtures are CC0 1.0. +The tools are Apache-2.0. See [LICENSE.md](LICENSE.md). diff --git a/context/README.md b/context/README.md new file mode 100644 index 0000000..29295c5 --- /dev/null +++ b/context/README.md @@ -0,0 +1,14 @@ +# JSON-LD context + +`scienting-1.jsonld` is the context that, together with the W3C Web Annotation context, makes every annotation layer valid JSON-LD. +It is published at `https://scient.ing/ns/scienting/1.jsonld`, and every layer and annotation lists both: + +```json +{"@context": ["http://www.w3.org/ns/anno.jsonld", "https://scient.ing/ns/scienting/1.jsonld"]} +``` + +The context maps the terms the formats add to annotations (`kind`, `status`, `audience`, `receipt`, `verdict`, `AnchorSelector`, `anchor` and others) to IRIs under `https://scient.ing/ns/scienting#`. +`anchor` and `exercise` are JSON literals, so a JSON-LD processor keeps a `scienting-anchor/1` record intact instead of expanding it. + +A Web Annotation tool that knows nothing of this context still reads the annotation's body and its `TextQuoteSelector`; the context matters only to tools that process the data as linked data. +[Chapter 18](../spec/18-layers.md) defines layers and annotations. diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..947281c --- /dev/null +++ b/docs/README.md @@ -0,0 +1,17 @@ +# Guides + +These guides explain the project to someone meeting it for the first time. +The [specification](../spec/README.md) says exactly what each format is; the guides say what it is for, with examples drawn from the fixtures. + +| Guide | For | +| --- | --- | +| [Overview](overview.md) | What the project is, who it serves, and how its parts fit together. | +| [Concepts](concepts.md) | The ideas behind the formats, one at a time, each with a real example. | +| [Walkthrough](walkthrough.md) | One paper, from a flawed first reading to a corrected, reviewed and signed document. | +| [Examples](examples.md) | Things to run: checking hashes by hand, reading a package with a few lines of Python, querying a database. | +| [Rationale](rationale.md) | Why the formats look the way they do, and the questions they leave open. | +| [FAQ](faq.md) | Short answers to the questions people ask first. | +| [Glossary](glossary.md) | Every term, in a sentence. | + +A good order for a first afternoon is the overview, then the walkthrough with the fixture open beside it, then the examples. +Read the concepts when a term in the walkthrough needs more than the glossary gives. diff --git a/docs/concepts.md b/docs/concepts.md new file mode 100644 index 0000000..c9966a6 --- /dev/null +++ b/docs/concepts.md @@ -0,0 +1,262 @@ +# Concepts + +Each section introduces one idea, shows it in the example paper `hale2026harmonic` (a synthetic chemistry paper in [fixtures/packages/](../fixtures/packages/)), and links to the chapter that defines it exactly. + +## Works, versions, witnesses and documents + +Four things are easy to confuse, so the formats keep them apart. + +| Object | In the example | +| --- | --- | +| A **work** is the paper across all its versions. | Hale and Osei's paper on harmonic bond parameters, `doi:10.5555/example.hale.2026`. | +| A **version** is one expression of it: a preprint, the version of record, an erratum, the Supporting Information. | The version of record. | +| A **witness** is one file of a version, identified by its SHA-256. | The publisher's PDF, witness `qptsanv`. | +| A **document** is the text established for one version, from one or more witnesses. | `hale2026harmonic`. | + +A document's key is its work's key, with a suffix for anything but the version of record: `hale2026harmonic.si` would be its Supporting Information. +See [chapter 2](../spec/02-model.md). + +## A document is a tree of nodes + +Every part of the text is a node: the document, its sections, headings, paragraphs and sentences, figures and their panels and captions, tables with their rows and cells, equations, footnotes and references. +The example has 75 nodes; its outline is: + +```text +bzqvbhd 1 Introduction +qn08tj9 equation (1) +f0tj6ca 2 Methods +ajfvq4s figure Figure 1 +gm5z4rx table Table 1 +``` + +`nodes.jsonl` holds one node per line, in reading order, each with its kind, parent, position, content and hashes. +A sentence's content is a list of **runs**: plain text, styled text, inline math, citations, cross-references, footnote marks, links, and display equations the sentence runs through. +This is one table cell, shortened: + +```json +{"id":"nkm3xn5","kind":"cell","parent":"zhe4g0k","ord":1, + "attrs":{"column":"…","role":"data","rowspan":1,"colspan":1}, + "content":[{"kind":"text","text":"310.0"}], + "fidelity":"F1","text_hash":"…","match_hash":"…","align_key":"…","own_hash":"…","node_hash":"21f13b…"} +``` + +See [chapter 3](../spec/03-nodes.md). + +## IDs are names, not positions + +`nkm3xn5` says nothing about where the cell is. +That is deliberate: an ID that encoded a page or an ordinal would change when a better converter ordered blocks differently. +IDs are minted once from the document key, the node's kind, its text and an occurrence count, so a fresh conversion of the same input gives the same IDs, and then they never change. + +When two nodes merge, one ID is retired and points at its successor; when an artifact is deleted, its ID is retired with no successor. +Legacy IDs, such as the block IDs older tools used, live on as aliases: in the example, `b-p01-text-003` still leads to sentence `f929eq9`. +See [chapter 4](../spec/04-identity.md). + +## Addresses + +An address names a node, and optionally pins it to a state and narrows it to a span: + +| Address | Names | +| --- | --- | +| `hale2026harmonic#f929eq9` | The sentence about water, whatever it says now. | +| `hale2026harmonic#f929eq9@cd6cdb` | That sentence as it read when its `node_hash` began `cd6cdb`. | +| `hale2026harmonic#f929eq9@cd6cdb[29:44]` | Code points 29 to 44 of it in that state: "first test case". | +| `smithlab/hale2026harmonic#f929eq9` | The same sentence in the `smithlab` store. | + +On the web the ID is the fragment, so a link to `…/smithlab/hale2026harmonic#f929eq9` lands on the sentence. +See [chapter 6](../spec/06-anchors.md#61-addresses). + +## Five hashes, five jobs + +Every node carries five content values. + +| Value | Covers | Answers | +| --- | --- | --- | +| `text_hash` | The text exactly, normalized. | Is this the same quotation, character for character? | +| `match_hash` | The text with notation folded to one form. | Do two witnesses really agree? | +| `align_key` | The text folded loosely. | Is this probably the same sentence? | +| `own_hash` | The node's own content, without its children. | Has this section itself changed, as opposed to something inside it? | +| `node_hash` | The node and everything below it. | Has anything here changed at all? | + +The middle two exist because one hash cannot do both of their jobs. +Chemistry writes the same thing many ways, and every one of these sentences has the same `match_hash`: + +| As stored | Strict projection | +| --- | --- | +| `The solvent is $H_2O$.` | `The solvent is H_{2}O.` | +| `The solvent is H$_2$O.` | `The solvent is H_{2}O.` | +| `The solvent is H` and a subscript `2`, then `O.` | `The solvent is H_{2}O.` | +| `The solvent is H₂O.` | `The solvent is H_{2}O.` | +| `The solvent is $H_2 O$.` in Typst | `The solvent is H_{2}O.` | + +But OCR often drops a superscript, and "103" must never be certified as agreeing with 10³. +So "A rate of 103 per second." and "A rate of 10³ per second." share an `align_key`, which proposes that they are the same sentence, while their `match_hash` values differ, which raises a suspect of kind `number` for a person to resolve. +The same care keeps "The energy" followed by citation 3 from matching the text "The energy3". +See [chapter 5](../spec/05-hashes.md) and [vectors/projections.jsonl](../vectors/projections.jsonl). + +## Anchors + +An anchor is how anything points into a document: a Lode extract, an annotation, a dataset value, a draft's citation. +It is a pinned address with enough context to find its text again after the document changes: + +```json +{"format":"scienting-anchor/1","store":"smithlab","doc":"hale2026harmonic","node":"f929eq9","kind":"sentence", + "own_hash":"d5b2eb…","node_hash":"cd6cdb…","span":[29,44], + "quote":{"exact":"first test case","prefix":"Water, H$_2$O, served as the ","suffix":", with a bending band near 1595 "}, + "rev":12} +``` + +`rev` is the document's revision when the anchor was made, counted in corrections rather than clock time. +When the document changes, an anchor resolves one of five ways: + +| Result | Means | In the example | +| --- | --- | --- | +| `current` | Nothing it points at has changed. | A note on the water sentence. | +| `changed-below` | A container is unchanged in itself, but something inside it changed; a drift figure says how much. | A note on Table 1 after a row was inserted. | +| `moved` | Its node was merged into another that still holds the quote. | A note on half of a sentence that was wrongly split, then rejoined. | +| `reattached` | Its node changed, but the quote is still there exactly once, so it is re-pinned. | A note on "treated as zero" in a sentence whose superscript was fixed. | +| `stale` | It can no longer be trusted; a person must re-anchor it. | A note on the cell that read "3l0.0" before it was corrected to "310.0". | + +See [chapter 6](../spec/06-anchors.md); the [walkthrough](walkthrough.md#5-what-happened-to-the-notes) shows all five. + +## Corrections + +A document is never edited in place. +Every fix is a correction: an operation with its evidence, its author and a reason, appended to `corrections.jsonl`. + +```json +{"format":"scienting-correction/1","index":1,"op":"replace","target":"nkm3xn5", + "find":"3l0.0","replace":"310.0", + "evidence":[{"kind":"crop","sha256":"9b6fe6…"}],"author":{"key":"ana"}, + "reason":"OCR read the digit 1 as the letter l.","at":"2026-10-02T10:00:00Z", + "before":"2fd1e9…","after":"21f13b…","touched":["nkm3xn5"]} +``` + +The operations are `replace`, `insert`, `delete`, `split`, `merge`, `move`, `reclassify`, `relabel` and `set`. +A gate refuses any correction that would change a digit without a crop of the page as evidence, or that would let a digit change anywhere it did not declare. +See [chapter 11](../spec/11-corrections.md). + +## Fidelity + +Every leaf has a level that says how far its text was checked. + +| Level | Means | +| --- | --- | +| F0 converted | One reading, not corroborated. | +| F1 resolved | Every doubt about it was answered against the page image. | +| F2 corroborated | An independent witness agrees exactly, or the text was born digital. | +| F3 proofread | A named person compared it with the page and signed for its current text. | + +A level counts only at the node's current hash, so a correction drops a proofread sentence back until someone signs for it again. +The example reports 28 leaves at F3, 12 at F2, 4 at F1 and 1 at F0. +See [chapter 8](../spec/08-fidelity.md). + +## Receipts and spot-checks + +Nobody signs three thousand nodes one at a time. +A reviewer signs a **receipt**: one signature over a Merkle root of everything reviewed in a sitting, with the evidence looked at. +In the example, the reviewer `ana` signs one receipt for 23 nodes: Table 1's cells, the equation and the sentences around it. + +Signing once per sitting invites rubber-stamping, so a second reviewer spot-checks a sample. +The sample is chosen by the checker's Ed25519 signature over the receipt, which neither reviewer can steer, and anyone can recompute it. +Across samples, the record states a bound on each reviewer's error rate: zero errors in 10 sampled nodes bounds the rate below 26%, and zero errors in 300 bounds it below 1%. + +A spot-check that finds an error suspends the receipt; a revocation withdraws it. +Either way the nodes fall back to the level they had without it. +See [chapter 8](../spec/08-fidelity.md#83-receipts). + +## Page fingerprints + +Publishers stamp each download with the institution and time, so two teammates' copies of one paper rarely share a SHA-256. +Each page therefore also has a fingerprint: a hash of its size, its text with the stamp lines removed, and its images. +Two downloads of the same page stamped "Downloaded via UNIVERSITY OF EXAMPLE on October 11, 2026" and "Downloaded via EXAMPLE PHARMA INC on March 2, 2027" share the fingerprint `6272e6f3…`; change one digit of the text and it becomes `b545d285…`. +Geometry and digitizations point at fingerprints, so a re-download keeps every box and every value read off a plot. +See [chapter 7](../spec/07-geometry.md). + +## Keeping text private + +A team may annotate a paper it may not share. +A **checks-only** package keeps every ID, label, page and box, but withholds the text, and keys every hash, because a plain hash of a short text such as "310.0" is recovered by hashing guesses. +Inside the team, values are plain SHA-256; exported to another of the team's machines, they are HMACs under the team's key; in a public layer, the key is derived from the paper's page fingerprints, so only someone holding the paper can check them. +[fixtures/packages/hale2026harmonic.checks-only](../fixtures/packages/hale2026harmonic.checks-only) is the example paper exported this way. +See [chapter 5.8](../spec/05-hashes.md#58-keyed-hashes). + +## Rights + +What a team may do with a paper depends on how it holds the paper. +Every witness carries a rights record: the license as stated, the basis (an open license, a subscription, a collective license, the team's own work), and which uses that allows, including AI uses by purpose and by where the processing runs. +Every processing stage declares what it needs and runs only when the rights allow it. + +The example PDF is held on a subscription that allows text mining and internal AI use on the team's own machines. +So conversion by a local model is allowed, conversion by an outside service is refused for lacking `ai/internal/outside`, and model review in the team's cloud is refused for lacking `ai/internal/tenancy?` (the question mark means unknown) until a project owner signs a policy that settles it. +See [chapter 14](../spec/14-rights.md). + +## Layers and annotations + +Everything people and models add is an annotation, grouped in layers such as "Novice glosses" or "Journal club, March". +Annotations are W3C Web Annotations whose selector is an anchor, so any Web Annotation tool can read them. +This gloss, shortened, explains a phrase for a first-year student: + +```json +{"type":"Annotation","motivation":"describing","status":"confirmed", + "body":{"type":"TextualBody","kind":"gloss","audience":"novice", + "value":"A *harmonic* term treats a bond like a spring: the energy grows with the square of the stretch."}, + "target":{"type":"SpecificResource","source":"urn:scienting:doc:smithlab/hale2026harmonic", + "selector":[{"type":"AnchorSelector","anchor":{"node":"sxgcc9h","span":[111,130],"…":"…"}}, + {"type":"TextQuoteSelector","exact":"bond force constant"}]}} +``` + +One rule decides what is a claim and what is a layer: a statement true independently of the paper belongs in Lode as a checked claim; a statement about reading the paper belongs in a layer. +Model-written text is always a `generated` annotation, never source. +See [chapter 18](../spec/18-layers.md). + +## Exercises + +Because every sentence and cell has an address, the answer to "find the evidence" is an address, and an address can be graded. +An exercise asks a student to find the cell giving the Set A bond constant for CT–CT; a student who points at `nkm3xn5` scores 1 of 1 with "That is the cell.", and one who points at the Set B cell scores 0 with "Look in Table 1, row CT–CT." +Students submit under pseudonyms, and a key whose anchors have gone stale is never used to grade. +See [chapter 19](../spec/19-exercises.md). + +## Datasets, digitizations and views + +Numbers live in prose, in tables and inside figures; all three should end up as data a team can plot and cite. +A dataset is a Frictionless Data Package with units, uncertainty, and an origin for every value. +The example's Table 1 becomes a CSV whose every value is anchored to the cell that prints it, digits exactly as printed: + +```text +key,bond,set,k_r,r_eq +ct-ct-a,CT–CT,A,310.0,1.526 +ct-ct-b,CT–CT,B,305.2,1.5260 +``` + +A digitization records how values were read off a plot (axes, calibration, series, uncertainty) and counts only once a person confirms it against an overlay. +A view is a Vega-Lite specification that names its data by dataset ID and hash. +See chapters [15](../spec/15-datasets.md), [16](../spec/16-digitization.md) and [17](../spec/17-views.md). + +## Packages, databases and bundles + +A document travels as a package: a folder of canonical JSON files. + +```text +hale2026harmonic/ + MANIFEST.json every other file, with its SHA-256; its own hash pins the package + document.json the work, version, metadata, policy, root hash and fidelity + nodes.jsonl one node per line, in reading order + witnesses.json each witness and its rights + pages.jsonl each page and its fingerprint + locations.jsonl where each node is on each page + references.jsonl parsed references + corrections.jsonl every correction, in order + receipts.jsonl receipts and revocations + ids.jsonl retired IDs and legacy aliases + roster.json the keys that signed + provenance.json tools, versions and where each stage ran + assets/ images, by SHA-256 + datasets/ datasets reported from the document + derived/ MathML and other renderings, rebuildable +``` + +Every line is canonical JSON, so a corrected sentence is a one-line diff, and every file has exactly one valid byte sequence. +The same data loads into tables that run on SQLite, Dolt and PostgreSQL, and unloads back to the same bytes. +A bundle is one SQLite file with every table and a full-text index, which a reader or an MCP server can query offline. +See chapters [12](../spec/12-package.md) and [13](../spec/13-database.md). diff --git a/docs/examples.md b/docs/examples.md new file mode 100644 index 0000000..fd062cb --- /dev/null +++ b/docs/examples.md @@ -0,0 +1,221 @@ +# Examples + +Things to run, each with the output it gives. +Run them from the repository's root. +With [pixi](https://pixi.sh), `pixi run ` sets up the environment first; every command also works with plain Python 3.10 or later and standard command-line tools. + +## Run the checks + +```sh +pixi run check +``` + +`pixi run check` lints the Markdown and then runs `python tools/check.py`, which you can also run directly. +Each section prints how many checks it ran, and the last line is `All checks pass.` +`pixi run quick`, or `--quick`, skips the slowest part: rebuilding everything and comparing it byte for byte. + +## Recompute a hash by hand + +A sentence's `text_hash` is SHA-256 of its text, after Unicode NFC and with whitespace collapsed. +For a sentence of plain text, that is a one-liner: + +```sh +printf '%s' 'The two parameter sets agree to within 2% for every bond.' | sha256sum +``` + +```text +175bfc8867816a9761fd5bb9ffc956df6d9e7f203870961a5485fc1912cb663a - +``` + +The same value is in the package: + +```sh +grep -o '"id":"j06tfhn".*' fixtures/packages/hale2026harmonic/nodes.jsonl | grep -o '"text_hash":"[0-9a-f]*"' +``` + +```text +"text_hash":"175bfc8867816a9761fd5bb9ffc956df6d9e7f203870961a5485fc1912cb663a" +``` + +## Check a package's manifest + +Every file of a package is listed in `MANIFEST.json` with its SHA-256, and the manifest's own hash pins the whole package. + +```sh +cd fixtures/packages/okafor2025open +sha256sum nodes.jsonl +grep -o '"path":"nodes.jsonl","sha256":"[0-9a-f]*"' MANIFEST.json +``` + +```text +0fccd6492121fb91ebd1d32487e23ca5b30ad92627e7d3a0801d77549e1a2f94 nodes.jsonl +"path":"nodes.jsonl","sha256":"0fccd6492121fb91ebd1d32487e23ca5b30ad92627e7d3a0801d77549e1a2f94" +``` + +## Read a package with nothing but a JSON parser + +Packages are plain JSON Lines. +Save this script as `outline.py`; it prints a document's sections, figures, tables and equations. + +```python +import json, sys + +pkg = sys.argv[1] +nodes = [json.loads(line) for line in open(pkg + "/nodes.jsonl", encoding="utf-8")] +by_id = {n["id"]: n for n in nodes} +children = {} +for n in nodes: + children.setdefault(n["parent"], []).append(n) + + +def text(n): + """The plain projection of a node with runs.""" + out = [] + for r in n["content"]: + if r["kind"] == "math": + out.append("$" + r["source"] + "$") + elif r["kind"] == "equation": + out.append("$$" + by_id[r["node"]]["content"]["source"] + "$$") + else: + out.append(r["text"]) + return "".join(out) + + +for n in nodes: + if n["kind"] == "section": + heading = children[n["id"]][0] + print(n["id"], n.get("label", ""), " ".join(text(s) for s in children[heading["id"]])) + elif n["kind"] in ("figure", "table", "equation"): + print(n["id"], " ", n["kind"], n.get("label", "")) +``` + +```sh +python3 outline.py fixtures/packages/hale2026harmonic +``` + +```text +bzqvbhd 1 Introduction +qn08tj9 equation (1) +f0tj6ca 2 Methods +ajfvq4s figure Figure 1 +gm5z4rx table Table 1 +``` + +## Follow an address + +`hale2026harmonic#f929eq9[29:44]` names code points 29 to 44 of sentence `f929eq9`. +With `text()` from the script above: + +```python +print(text(by_id["f929eq9"])[29:44]) +``` + +```text +first test case +``` + +Offsets count Unicode code points; in Python, string indexes already do. +In JavaScript, convert to UTF-16 units first; [vectors/offsets.jsonl](../vectors/offsets.jsonl) gives both for text with characters outside the Basic Multilingual Plane. + +## Load packages into SQLite + +```sh +pixi run load --sqlite lab.db fixtures/packages/hale2026harmonic fixtures/packages/okafor2025open fixtures/packages/smithlab2027charges +``` + +Without pixi, run `python3 tools/load.py` with the same arguments. + +```text +hale2026harmonic: 75 nodes +okafor2025open: 15 nodes +smithlab2027charges: 15 nodes +``` + +The tables are those of [sql/doc.sql](../sql/doc.sql) and [sql/store.sql](../sql/store.sql), and the same statements run on Dolt and PostgreSQL. +Here are four queries, in Python's `sqlite3` so that nothing else is needed. + +```python +import json, sqlite3 + +db = sqlite3.connect("lab.db") + +# Sentences the converter left below F2. Receipts can raise them; see below. +for row in db.execute("SELECT id, fidelity FROM nodes WHERE doc_key = 'hale2026harmonic' " + "AND kind = 'sentence' AND fidelity IN ('F0', 'F1') ORDER BY seq"): + print(row) + +# Every sentence that cites reference 2: the citation graph is sentence-level. +for row in db.execute("SELECT c.sentence FROM citations c JOIN refs r " + "ON r.doc_key = c.doc_key AND r.node = c.ref_node " + "WHERE c.doc_key = 'hale2026harmonic' AND r.label = '2'"): + print(row) + +# Who signed what. +for row in db.execute("SELECT r.signer, r.scope, COUNT(*) FROM receipts r " + "JOIN receipt_nodes rn ON rn.receipt = r.id GROUP BY r.id ORDER BY r.made_at"): + print(row) + +# Table 1's body, row by row. +rows = db.execute("SELECT r.id FROM nodes r JOIN nodes b ON b.doc_key = r.doc_key AND b.id = r.parent " + "WHERE r.doc_key = 'hale2026harmonic' AND r.kind = 'row' AND b.kind = 'table-body' " + "ORDER BY r.seq").fetchall() +for (row,) in rows: + cells = db.execute("SELECT content FROM nodes WHERE doc_key = 'hale2026harmonic' AND parent = ? ORDER BY ord", + (row,)).fetchall() + print(" | ".join("".join(r.get("text", "") for r in json.loads(c)) for (c,) in cells)) +``` + +```text +('e4fkseg', 'F1') +('r1gacps', 'F0') +('eajq93t', 'F0') +('f929eq9',) +('ana', 'targeted:lode', 3) +('ana', 'targeted:lode', 23) +('ana', 'section:bzqvbhd', 2) +('cara', 'page:2', 3) +('ana', 'targeted:captions', 3) +('ben', 'qa:eqgbbpwb6725v', 10) +('ben', 'qa:gena76nkb6cnc', 3) +Bond parameters +CT–CT | 310.0 | 1.526 | 305.2 | 1.5260 +CT–HC | 340.0 | 1.090 | 338.7 | 1.0900 +OW–HW | 553.0 | 0.9572 | 548.1 | 0.9570 +``` + +The first query reads the level the converter recorded, not the final one. +Receipts by `ana` raise `e4fkseg` and `r1gacps` to F3; `eajq93t` stays at F0 because the only receipt that covered it was revoked. +Computing the final level means checking each receipt's signature, revocations and spot-checks, as [chapter 8](../spec/08-fidelity.md#82-levels) describes. + +## Test your implementation against a vector + +Every vector file has one case per line, with its inputs and expected outputs. +A harness for the projections looks like this, with your functions in place of the comments: + +```python +import json + +for line in open("vectors/projections.jsonl", encoding="utf-8"): + case = json.loads(line) + runs = case["runs"] + # assert my_plain(runs) == case["plain"] + # assert my_strict(runs) == case["strict"] + # assert my_lossy(runs) == case["lossy"] + # assert my_sha256(case["strict"]) == case["match_hash"] + print(case["id"], case["strict"]) +``` + +[vectors/README.md](../vectors/README.md) lists every file and what it tests. + +## Break something on purpose + +Change one digit in a fixture and run the checks: + +```sh +sed -i 's/1718 cm/1719 cm/' fixtures/packages/okafor2025open/nodes.jsonl +pixi run quick +``` + +The checker reports the package's manifest hash no longer matches, and that unloading it from the database no longer gives the same bytes. +Restore the file with `git checkout fixtures/packages/okafor2025open/nodes.jsonl`. +The broken packages in [fixtures/invalid/](../fixtures/invalid/) are this kind of change, each with the rule it breaks. diff --git a/docs/faq.md b/docs/faq.md new file mode 100644 index 0000000..0e8e66d --- /dev/null +++ b/docs/faq.md @@ -0,0 +1,45 @@ +# FAQ + +**Can I read these files without the studio's software?** +Yes. +Every file is JSON or JSON Lines, every format is specified in [spec/](../spec/README.md), and [the examples](examples.md) read a package with a few lines of Python. + +**Why not keep using PDFs and page numbers?** +A page number names a region, not a statement, and it changes between a preprint and the version of record. +An address names a sentence or a cell, survives reconversion, and can be checked against the exact text. + +**Why are numbers stored as strings?** +Because databases and JSON libraries rewrite numbers: `310.0` becomes `310` and `1.5260` becomes `1.526`, and the printed digits are part of what a paper says. + +**What happens when a better converter comes along?** +Its reading is aligned with the established document sentence by sentence. +Sentences it reads the same keep their IDs; every difference becomes a suspect for review; nothing changes until a correction passes the gate. + +**Do I need receipts, rights records and spot-checks for a personal library?** +No. +One person can leave receipts out entirely and record only a license. +The fields are defined so that a personal library loads into a team's store later without conversion. + +**Is a model allowed to correct a document?** +A model may propose corrections and answer suspects in a review session, under the same gate as a person, and its corrections record the model and the run. +Only a named person can sign a receipt. + +**Why are annotations W3C Web Annotations?** +So existing tools can read them. +A tool that does not know the anchor selector falls back to the text-quote selector beside it. + +**What is the difference between a claim and an annotation?** +A statement true independently of the paper, such as "K_r is the bond force constant of the AMBER energy", is a claim and belongs in Lode, where it is checked. +A statement about reading the paper, such as "the authors pass quickly over the scaling here", is an annotation in a layer, attributed but not checked. + +**Can a team share notes on a paywalled paper?** +Yes, without sharing the paper: a checks-only package or a public layer carries IDs, offsets and keyed hashes, and is shown over each reader's own copy. +Whether a team may do so is a question for its own counsel; nothing here is legal advice. + +**How do I add a node kind or a run kind?** +Propose it with the vector or fixture that shows it; a new kind is a minor version. +See [CONTRIBUTING.md](../CONTRIBUTING.md). + +**What are the files in `tools/` for?** +They compute and check every vector and fixture so that no expected value is typed by hand. +They are not a library; implementations follow the specification. diff --git a/docs/glossary.md b/docs/glossary.md new file mode 100644 index 0000000..1bc90f6 --- /dev/null +++ b/docs/glossary.md @@ -0,0 +1,118 @@ +# Glossary + +Address +: A string naming a node, optionally pinned to a hash and narrowed to a span: `key#id@hash6[start:end]`. + +Alias +: A legacy ID kept so that old references still lead to the node that now holds their text. + +Align key +: A hash of a node's lossy projection, used only to propose that two readings hold the same sentence. + +Anchor +: The stored form of a pinned address, with its quotation and context, that resolves as current, changed below, moved, reattached or stale. + +Asset store +: Where a team keeps files by SHA-256: witnesses, images, crops and withheld text. + +Bundle +: One SQLite file with every table and a full-text index, which readers, servers and MCP servers query. + +Changed below +: The resolution of an anchor on a container that is unchanged in itself while something inside it changed. + +Correction +: An operation on an established document, with its evidence and author; never edited, only appended. + +Document +: The text established for one version of one work, as a tree of nodes. + +Drift +: How much has changed below a container since an anchor on it was made, counted from the corrections. + +Exercise +: An annotation whose body holds a prompt, an answer key of anchors or a value, and a grading rule. + +Fidelity level +: How far a leaf's text was checked against its source, from F0 converted to F3 proofread. + +Keyed hash +: A content value computed as an HMAC under a team key or a key derived from page fingerprints, so a short text cannot be recovered by hashing guesses. + +Layer +: A named set of annotations with one purpose, audience, visibility and license. + +Leaf +: A node that holds text itself: a sentence, cell, reference, author line, keyword line, equation, equation line, code or algorithm. + +Lossy projection +: A node's text with scripts flattened, citation marks, whitespace and dashes removed, and case folded; used for proposing alignments. + +Match hash +: A hash of a node's strict projection; equal match hashes are what corroboration requires. + +Node +: An addressable part of a document: a section, paragraph, sentence, figure, panel, caption, table, row, cell, equation, footnote, reference and the rest. + +Own hash +: A hash of a node's own content without its children, used to judge anchors on containers. + +Package +: A document as a folder of canonical JSON files with a manifest whose hash pins them all. + +Page fingerprint +: A hash of a page's size, its text with download stamps removed, and its images, so that differently stamped downloads share their pages. + +Plain projection +: A node's text as one string, on which offsets, quotations and text hashes are defined. + +Policy +: A project owner's signed decision that settles rights a record leaves unknown. + +Project +: A team's curated set of documents, with its members, layers, data, reading paths and drafts. + +Receipt +: One signature over a Merkle root of what a person reviewed in a sitting, which sets the level of what it lists. + +Revision +: The index of the last correction applied to a document; anchors and receipts record it. + +Revocation +: A signed record that withdraws a receipt, or every receipt by one signer in a period. + +Rights record +: A witness's license, the basis on which the team holds it, and the uses that basis allows. + +Roster +: The list of keys that may sign, with their owners and purposes. + +Spot-check +: A second reviewer's check of a sample of a receipt's nodes, drawn from a seed nobody can steer. + +Stamp rule +: A pattern that recognizes a publisher's download stamp in a page's text, so that fingerprints ignore it. + +Strict projection +: A node's text with notation folded to one canonical form and structure kept; used for certifying agreement. + +Suspect +: A doubtful reading raised for a person or a review session to answer. + +Targeted F3 +: Proofreading the nodes a team's claims, quotations and datasets rest on, rather than every node. + +Text policy +: Whether a package carries its text (`full`) or withholds it (`checks-only`). + +Verdict +: A judgment of whether a cited passage supports a citing sentence: supports, partly, does not, or cannot tell. + +Version +: One expression of a work: a preprint, the version of record, an erratum, the Supporting Information. + +Witness +: One file of one version of a work, identified by its SHA-256. + +Work +: A piece of scholarship across all its versions. diff --git a/docs/overview.md b/docs/overview.md new file mode 100644 index 0000000..2a3ea88 --- /dev/null +++ b/docs/overview.md @@ -0,0 +1,92 @@ +# Overview + +## The idea + +Most tools for scientific literature aggregate: they index hundreds of millions of papers and answer questions over all of them, shallowly. +This project does the opposite. +A team working on a focused question cares more about fifty papers it understands completely than about the world's literature. +Those fifty papers deserve to be converted once, corrected once, split down to the sentence, annotated by the people who know them, linked to checked claims, and kept. + +What that work produces answers two questions scientists ask every day. + +1. **Does the cited passage say what is claimed?** + A citation to a real paper can still be inaccurate, incomplete or beside the point, and a reader needs to see the exact sentence it rests on. +2. **Where did this number come from, and how far was it checked?** + A force constant in a table, a value read off a plot, a parameter in a model: each should lead back to the cell or sentence it came from, with a record of who compared it with the page. + +A model can summarize a paper, but it cannot produce a signed statement that a person checked a passage, IDs that never move, or a store its owner controls. +Those are what these formats are for, and they are worth most at the scale of one team. + +## Who it is for + +| User | Example | +| --- | --- | +| One researcher | A thesis student, a science writer or fact-checker, a Wikipedia editor checking a citation, a scientist keeping a deep personal library. | +| An academic lab | A research group, a thesis committee, a journal club reading one annotated paper together. | +| A course | An instructor whose students learn to check whether a source supports a claim, with answers that are addresses and can be graded. | +| An industrial team | A computational chemistry or materials group that needs to show which passage of which paper each parameter came from. | + +"Team" throughout means roughly five to twenty people with a few dozen to a few hundred papers that matter. +Everything is designed for that scale: hundreds of documents and millions of sentences, not hundreds of millions of papers. + +## The parts + +The formats in this repository are the meeting point of several programs. +No program imports another's code; they meet only through these files. + +```text + spec (this repository) + │ + ┌─────────────────┼──────────────────┐ + core core-ts typst-scienting + ┌──────┬──┴─────┬────────┐ │ +paperpack digitize lode varve varve-web +``` + +| Part | Does | +| --- | --- | +| **spec** | Defines every format: prose, schemas, tables, test vectors and fixtures. | +| **core** | `scienting-core`, the reference implementation of the shared rules in Python, standard library only: canonical JSON, hashing, IDs, addresses, anchors, packages, receipts and rights checks. | +| **core-ts** | The same rules in TypeScript, for the browser. | +| **paperpack** | The converter. It turns a PDF, Supporting Information, JATS XML, LaTeX, HTML, EPUB or a Typst project into a package, raises doubtful readings as suspects, and applies corrections only when they pass its gate. | +| **digitize** | Reads values off plots: calibrated axes, traced series, an overlay a person confirms. | +| **Lode** | The store of checked claims. Its extracts point at sentences through anchors, so a claim can verify itself against the words it rests on. | +| **Varve** | The team workspace: a store a team can clone, a server, a reader, an editor and an MCP server that lets a team's own models quote the corpus by address. | +| **typst-scienting** | A Typst package that lets a draft cite sentences, claims and datasets. | + +A varve is one year's pair of sediment layers on a lake bed; geologists count them because each layer is laid down once and kept. +That is the product's idea: every reading a team does leaves a layer, dated and signed, over a text that never has to be redone. + +## How a paper moves through the system + +1. **Acquire.** A witness arrives (the publisher's PDF, say) with a rights record that says how the team holds it and what that allows. +2. **Read and reconcile.** paperpack reads it with one or more backends, compares the readings with each other and with the PDF's text layer, and fingerprints every page so that differently stamped downloads are recognized as the same pages. +3. **Segment and mint.** Text is split into sentences, and every section, paragraph, sentence, figure, panel, table, row, cell, equation, footnote and reference gets a permanent ID and five content hashes. +4. **Review.** Disagreements become suspects with page crops; a person or a model session answers them as corrections, and the gate refuses any correction that changes a digit without image evidence. +5. **Package.** The result is a package: a folder of canonical JSON files with a manifest whose hash pins the whole. +6. **Use.** The package loads into a database, a reader renders it with layers of annotations, Lode anchors claims to it, datasets anchor values to its cells, and drafts cite its sentences. +7. **Check and sign.** People proofread what matters and sign receipts; second reviewers spot-check them; every level counts only at the current text. + +[The walkthrough](walkthrough.md) follows one paper through every step. + +## Principles + +| Principle | Means | +| --- | --- | +| Quality over quantity | A small corpus processed to a stated and checkable fidelity beats a large one processed to an unknown one. | +| Do the work once | Conversion, correction, segmentation and digitization are recorded and kept; nothing is reparsed to be fixed again. | +| Address everything | Every section, paragraph, sentence, figure, panel, caption, table, cell, equation, footnote and reference has a permanent ID and a content hash. | +| Keep source and commentary apart | What a source says is stored as the source said it; everything people or models add is stored beside it and points at it. | +| Count only what was done | A level says how far something was checked, and it falls back when the checked content changes. | +| Models propose, people sign | Models draft, segment, gloss and digitize; only a named person confirms. | +| The team owns its data | Every store is a database the team can clone, and every format is documented and readable without the studio's code. | + +## What it is not + +It is not a search engine over all literature, a citation index, a recommendation feed or a chat-with-any-PDF tool. +Tools may import from Crossref, OpenAlex, PubMed Central or arXiv, but never depend on them to work. + +## Where next + +- [Concepts](concepts.md) explains each idea with an example. +- [The specification](../spec/README.md) defines each format exactly. diff --git a/docs/rationale.md b/docs/rationale.md new file mode 100644 index 0000000..1086673 --- /dev/null +++ b/docs/rationale.md @@ -0,0 +1,150 @@ +# Rationale + +Why the formats look the way they do. +Each choice below is one a reader is likely to question; the specification states the rule, and this page gives the reason. + +## Identity + +**IDs carry no meaning.** +An ID that encodes a page, an ordinal or a kind changes when a better converter orders blocks differently, or when a paragraph turns out to be a caption. +An opaque name minted from the text survives both, and a fresh conversion of the same input still gets the same IDs. + +**IDs never move, and retired IDs leave a trail.** +Everything a team builds points at IDs: notes, claims, dataset values, citations in drafts. +So a merge records its successor, a deletion records that there is none, and legacy IDs live on as aliases. + +**Revisions are counted, not timed.** +Anchors are made on different machines and packages are imported later, and clocks disagree; a correction index does not. + +## Hashes and projections + +**Five values, not one.** +Quotations need every character; certifying that two witnesses agree needs notation folded but structure kept; proposing that two sentences are the same needs something looser still; and containers need a hash of themselves apart from what they hold. +No single hash does all four jobs. + +**Only the strict projection certifies.** +OCR drops superscripts, so a projection loose enough to match "103" with 10³ would certify a thousandfold error. +The lossy key may propose that match; only the strict hash may accept it. + +**Citation marks stay marked in the strict projection.** +Otherwise "10" followed by citation 3 would agree with the text "103". + +**Decimal points and commas count as simple notation.** +Otherwise `$-1.5$` in a LaTeX source would be complex math and could never agree with "−1.5" in the PDF. + +**A hyphen or en dash before a digit is a minus sign in the strict projection.** +Text layers and OCR disagree about dashes in "10–20" and "−5"; folding them before digits makes witnesses agree without touching "CT–CT". + +**The lossy key deletes whitespace and dashes.** +OCR merges and splits words and line-break hyphens; a key that only proposes alignments should forgive that. + +**Offsets index the stored text, and structural hashes cover it exactly.** +When `node_hash` is unchanged, every stored offset is still exact, and a renderer can slice the stored text without normalizing it first. +Only the three text values normalize. + +**Hashes that leave the team are keyed.** +A plain hash of "310.0" is recovered by hashing guesses, and overlapping anchors can rebuild passages. +Inside a team, plain hashes are fine; anything exported is an HMAC under the team's key, or under a key derived from the paper's own pages, which only someone holding the paper can compute. + +## Anchors + +**Every anchor carries `own_hash`, and anchors on containers without content carry their leaves.** +Telling a relabel from a real change needs the anchored `own_hash`, and deciding whether a merged paragraph still holds "at least half" of what was anchored needs the original leaves. + +**Long quotations become a hash and a length.** +Quotations are capped at 200 code points to keep anchors small and to limit how much text an anchor carries; a hashed window can still be found again in the node. + +**Prefix and suffix pick between repeated quotes.** +"The value" may occur twice in a sentence; the context around it, as in W3C text-quote selectors, decides which one was meant. + +**`changed-below` reports drift.** +A note on a section stays valid when a typo inside it is fixed, but not forever: once a large share of what is inside has changed, the note is flagged for review. + +**A row's header path is not hashed.** +Rows have no content of their own, so inserting a dropped row-group header does not stale the anchors on every cell beneath it. + +## Corrections + +**Corrections are appended, never edited.** +A document's history is its first reading plus its corrections, so anyone can see what changed, when, why and on what evidence. + +**There is a `set` operation.** +Readings drop spans, roles and header paths as often as they drop words, and those need a correction too. + +**Applied corrections record what they touched.** +With `path`, `to_path` and `touched` in each record, drift is computed from the records alone, with no need to replay history. + +**Digits need image evidence.** +A wrong digit is the most expensive error a scientific text can carry and the easiest for a model to introduce, so no digit changes without a crop of the page. + +## Geometry + +**Pages have fingerprints.** +Publishers stamp each download, so file hashes cannot identify a paper across teammates' copies; fingerprints with the stamps removed can. + +**Fingerprint text deletes whitespace entirely.** +PDF text extractors disagree about where to insert spaces, and the fingerprint must come out the same in a converter and in a browser. + +## Review + +**People sign receipts, not nodes.** +Nobody signs three thousand times, so one signature covers a Merkle root of everything reviewed in a sitting. + +**Spot-check samples come from a signature nobody can steer.** +A seed the reviewer could influence could be ground until the sample fell on careful work. +The checker's deterministic Ed25519 signature over the receipt is unpredictable to the reviewer and unchoosable by the checker, and anyone can recompute it. + +**Error rates are reported as bounds.** +One clean sample of ten proves little; a Clopper–Pearson bound across all of a reviewer's samples says what can actually be claimed. + +**Levels fall back.** +A level counts only at the current text, so a correction never leaves a stale "proofread" behind. + +## Rights + +**Rights are recorded per witness, not per paper.** +What a team may do depends on how it obtained the paper: a subscription, a mining license, an open license, its own work. + +**AI uses have two axes.** +Reviewers ask not only whether a model may see the text, but where it runs and whether the text is kept there. + +**Unknown means no.** +A stage whose rights are unknown does not run; a project owner settles the question with a signed policy, so the decision is the team's and is on the record. +A policy fills in unknowns and never overrides an explicit answer. + +**Backend licenses are part of the gate.** +A converter's code and its model weights can be licensed differently, and the weights may exclude commercial or competing use. + +## Packages and databases + +**Canonical JSON, one record per line.** +A corrected sentence is a one-line diff, and every file has exactly one valid byte sequence, so a hash of the manifest pins the whole package. + +**Checks-only packages drop everything that carries text.** +Corrections carry their find and replace strings, datasets carry values, figure text and reference records carry text; a checks-only export omits them all. + +**SQL avoids reserved words in every engine.** +The same tables run on SQLite, Dolt (MySQL-compatible) and PostgreSQL, where `key`, `index`, `before` and `by` are reserved; hence `doc_key`, `idx`, `before_hash` and `signer`. + +**Loading and unloading are exact inverses.** +A team can keep its documents in a database and still hand anyone the package, byte for byte. + +## Questions the formats leave open + +These are settled for now, but a future version may revisit them. + +| Question | Current answer | +| --- | --- | +| Should node IDs carry a kind prefix, such as `s-k3m9q2a`, to help people debugging? | No: opaque IDs survive reclassification. | +| Should `text_hash` normalize to NFC, or only collapse whitespace as Lode's extracts do? | NFC and whitespace; existing Lode extracts keep Lode's rule. | +| How long may a quotation in an anchor be? | 200 code points. | +| How is complex math compared? | By its source, run by run; comparing canonical MathML is a possible later addition. | +| What extension and media type do zipped packages use? | `.sct.zip` and `application/vnd.scienting.doc+zip`. | +| Which drift threshold, spot-check rate and digit rule should teams use? | A quarter of a container's leaves; 5% of a receipt's nodes, at least ten; and `all`, `anchored-sections` or `none`, set per project. | +| How should rights combine when a document comes from witnesses held on different bases? | The most restrictive witness governs each use. | + +## Limits worth knowing + +- Notation drops spaces and text keeps them, so `$T = 300$` from a LaTeX source and "T = 300" from a PDF disagree, and become a `word` suspect for review. +- Page fingerprints are computed from extracted text lines; two programs that should agree on fingerprints must extract lines the same way. +- The roster is defined here and is meant to be the same one Lode verifies; the manifest is meant to have the shape of Lode's `lode-cache/1`. diff --git a/docs/walkthrough.md b/docs/walkthrough.md new file mode 100644 index 0000000..59a343c --- /dev/null +++ b/docs/walkthrough.md @@ -0,0 +1,126 @@ +# Walkthrough: one paper, start to finish + +This guide follows the example paper `hale2026harmonic` from a flawed first reading to a corrected, reviewed and signed document. +The paper is synthetic, written for the fixtures, but everything that happens to it happens to real papers. +Open [fixtures/packages/hale2026harmonic/](../fixtures/packages/hale2026harmonic/) beside this page: every ID below is in it. +The hand-written source of the first reading is [fixtures/src/hale2026harmonic.json](../fixtures/src/hale2026harmonic.json). + +## 1. A PDF arrives + +The team downloads the publisher's PDF. +It becomes a witness, identified by its SHA-256, with the short ID `qptsanv`, and a rights record: the library's subscription allows reading, text mining, and AI processing on the team's own machines, until August 31, 2027; whether it allows processing in the team's cloud is unknown. + +The publisher stamped the download in the margin with the institution and the time. +Fingerprinting each page removes those lines first (the rules `downloaded-via` and `sharing-guidelines` fired on both pages), so page 101's fingerprint is `6272e6f3…` in this copy and in a colleague's download stamped a year later. +See [`pages.jsonl`](../fixtures/packages/hale2026harmonic/pages.jsonl). + +## 2. The first reading, with its mistakes + +A converter reads the PDF into sections, paragraphs, sentences, a figure, a table, an equation, a footnote and references, and mints a permanent ID for every node. +Like every reading, this one has mistakes: + +| What the reading did | What the page says | +| --- | --- | +| A cell read `3l0.0`. | `310.0`, with the digit 1. | +| A sentence read `Values below 10-3 kcal mol⁻¹…`. | A superscript: 10⁻³. | +| Two sentences ran together: "…treated as zero. Table 1 lists…". | Two sentences. | +| One sentence was cut in two at a line break: "…and the energies were" and "fitted by least squares." | One sentence. | +| A paragraph was split at a column break. | One paragraph. | +| The running head "J. Example Chem. 2026, 14, 101–109" became a paragraph. | A running head, not text. | +| The equation's number read `(l)` and the figure's `Figure l`. | `(1)` and `Figure 1`. | +| The CT–HC row merged two cells into "338.7 1.0900". | Two cells. | +| The OW–HW row was missing. | It is there. | +| "Set B" spanned one column. | It spans two. | + +The doubtful places become suspects with crops of the page, for a person to answer. +The converter does not guess, and nothing changes until a correction passes the gate. + +## 3. Twelve corrections + +Each fix is appended to [`corrections.jsonl`](../fixtures/packages/hale2026harmonic/corrections.jsonl), numbered, with its evidence, its author and its reason. + +| # | Operation | On | What it did | +| --- | --- | --- | --- | +| 1 | `replace` | cell `nkm3xn5` | `3l0.0` to `310.0`. A digit changed, so the gate required a crop, and it has one. | +| 2 | `replace` | sentence `e4fkseg` | `10-3` to the runs `10` and superscript `−3`. | +| 3 | `split` | sentence `e4fkseg` | Split before "Table 1"; the first part keeps `e4fkseg`, the second is new: `r1gacps`. | +| 4 | `merge` | sentence `eajq93t` | Joined with `e9rh611`, which is retired with `eajq93t` as its successor. | +| 5 | `merge` | paragraph `zyqr0yc` | Joined with `g44k023`, retired the same way. | +| 6 | `delete` | paragraph `nm7vyzk` | The running head is gone; its IDs are retired with no successor. | +| 7 | `relabel` | equation `qn08tj9` | `(l)` to `(1)`. | +| 8 | `delete` | row `tfjge3c` | The garbled CT–HC row and its four cells. | +| 9 | `insert` | row `fbg2mwq` | The CT–HC row, read again: five new cells. | +| 10 | `insert` | row `j0x5twd` | The missing OW–HW row. | +| 11 | `set` | cell `vtszvw9` | Set B's `colspan` from 1 to 2. | +| 12 | `relabel` | figure `ajfvq4s` | `Figure l` to `Figure 1`. | + +The document is now at revision 12. +Notice what did not happen: no other cell, sentence or row changed its ID or its hashes. +Inserting a row adds a row node; the table's own hash covers only its label and column definitions, so every other cell, and every value already taken from them, is untouched. + +## 4. IDs that never move + +After twelve corrections, the sentence about water is still `f929eq9`, the corrected cell is still `nkm3xn5`, and the sentence that lost its superscript is still `e4fkseg`. +The retired IDs are recorded in [`ids.jsonl`](../fixtures/packages/hale2026harmonic/ids.jsonl), so anything that pointed at them can follow the trail. +So is a legacy alias: `b-p01-text-003`, the ID an older tool gave the water sentence, still leads to `f929eq9`. + +If a better converter comes along next year, its reading is aligned with this document sentence by sentence; every sentence it reads the same way keeps its ID, and every difference becomes a suspect rather than a silent change. + +## 5. What happened to the notes + +Before any correction, at revision 0, the team had already made seventeen anchors: notes, extracts and citations pointing into the first reading. +Here is how each resolves at revision 12 ([vectors/resolve.jsonl](../vectors/resolve.jsonl) records them all). + +| Anchor on | Result | Why | +| --- | --- | --- | +| "H$_2$O" in the water sentence | `current` | Nothing in that sentence changed. | +| The cell `1.5260` | `current` | Untouched, though its table changed around it. | +| The water sentence, through the alias `b-p01-text-003` | `current` | The alias leads to the same node. | +| The cell that read `3l0.0` | `stale` | Its text changed and "3l0.0" is no longer in it; a person must look again, which is right, because the number changed. | +| "treated as zero" | `reattached` | The sentence changed twice, but the quote occurs in it exactly once, so the anchor is re-pinned. | +| "fitted constants" | `stale` | After the split, those words are in the new sentence `r1gacps`, not in `e4fkseg`. | +| "least squares", in the half-sentence `e9rh611` | `moved` | That ID was merged into `eajq93t`, which still holds the quote. | +| The paragraph `g44k023` | `moved` | Merged into `zyqr0yc`, which holds all of its sentences. | +| The running head | `stale` | Deleted, with no successor. | +| The equation | `reattached` | Only its number changed. | +| The figure | `reattached` | Only its label changed, by a `relabel`. | +| The CT–HC cell `1.090` in the garbled row | `reattached` | Its row was deleted, but exactly one cell in Table 1 has the same row and column headers and the same text: the new `e0n07n0`. | +| Section 2 | `changed-below`, drift high | The section itself is unchanged, but 53% of what is in it was touched. | +| Table 1 | `changed-below`, drift high | Rows were replaced and added. | +| The table body | `changed-below`, drift high | 75% of its cells were touched. | +| The paragraph whose broken sentence was rejoined | `changed-below`, drift high | Two of the three sentences it has held were touched. | +| The CT–CT row | `changed-below`, drift low | One of its five cells was corrected. | + +A reader shows each note accordingly, and a Lode check, which is about exact words, accepts only `current`, `moved` and `reattached`. + +## 6. Review and signatures + +Now people check what matters. +The receipts are in [`receipts.jsonl`](../fixtures/packages/hale2026harmonic/receipts.jsonl), and the keys that signed them in [`roster.json`](../fixtures/packages/hale2026harmonic/roster.json). + +- **Before the corrections**, `ana` had signed receipt `c5cx7n4tga6y9` for three cells at revision 0. + Two of them never changed, so the receipt still counts for them; for the corrected cell it no longer does, because its hash moved on. +- **After the corrections**, `ana` signs `eqgbbpwb6725v` at F3 for 23 nodes: the cells of Table 1's three data rows, its four column headers, the equation and the sentences around it. + `ben` spot-checks it with receipt `p38t0s91trb6j`: his signature over the receipt picks 10 nodes nobody could have chosen, and he finds no errors. +- `ana` signs `zaxky8c1y12tc` for two sentences of section 1, including the new `r1gacps`. +- `cara` signs `adtrpq41np7sd` for three sentences on page 2, then revokes it herself (`q0yg0w6ygwwxh`): she had compared them with the crops of another paper. + Those sentences fall back to their earlier levels, which is why `eajq93t` is still F0. +- `ana` signs `gena76nkb6cnc` for the figure's caption; `ben`'s spot-check `qab588b6xy94y` finds a missing period in `nmm4md0`, which suspends the receipt until she reviews the sitting again. + +The document's report, in [`document.json`](../fixtures/packages/hale2026harmonic/document.json), counts the result: 28 leaves at F3, 12 at F2, 4 at F1 and 1 at F0. + +## 7. Data out of the table + +Table 1 becomes a dataset, `ds-hale-bond-params`, inside the package. +Its CSV keeps every digit as printed (`1.526` and `1.5260` stay different), and every value has an anchor to the cell that prints it, so a claim that rests on a value can show the cell. +All twelve value anchors resolve as `current`; the checker confirms each CSV value equals its cell's text. + +## 8. Sharing without the text + +The team wants to share its notes with a collaborator's machine but not the paper. +[`hale2026harmonic.checks-only`](../fixtures/packages/hale2026harmonic.checks-only/) is the same document with every leaf's text withheld and every hash replaced by an HMAC under the team's key. +IDs, labels, pages and boxes survive, so annotations still land in the right place; only someone holding the key and the text can check the values. + +## 9. Your turn + +[The examples](examples.md) show how to recompute one of these hashes with `sha256sum`, print the outline with a few lines of Python, and load the whole package into SQLite to query it. diff --git a/fixtures/README.md b/fixtures/README.md index e35855f..323e727 100644 --- a/fixtures/README.md +++ b/fixtures/README.md @@ -1,17 +1,53 @@ # Fixtures -All text here is synthetic and dedicated CC0; license fields describe fictional works. +Example data for learning the formats and for testing implementations. +All text here is synthetic, written for these files and dedicated CC0; license fields inside the fixtures describe fictional works. -| Path | Is | +## Packages + +| Package | What it shows | | --- | --- | -| `packages/hale2026harmonic/` | Two sections of a converted chemistry paper, read once with errors and fixed by twelve corrections: a digit, a lost superscript, a missed boundary, a false boundary, a split paragraph, a running head, two labels, a garbled row replaced, a dropped row inserted, and a header span. It has a multi-level-header table, a display equation inside a sentence, a footnote, a figure with figure text and a real image asset, references, legacy aliases, geometry on fingerprinted pages, receipts (one partly invalidated by a correction, one spot-checked clean, one revoked, one suspended), a roster, a reported dataset and derived MathML. | -| `packages/hale2026harmonic.checks-only/` | The same document exported to another machine of the team: text withheld, values under the team key in `keys/`. | -| `packages/okafor2025open/` | An open-access paper with a JATS witness and a PDF witness, corroborated. | -| `packages/smithlab2027charges/` | A short Typst draft that cites sentences of `hale2026harmonic` by address. | -| `packages/tiny2026example/` | A small document; the base of the correction vectors and the invalid packages. | -| `records/` | A valid record of every other format: anchors (span, leaf, cell, paragraph and keyed), rights, a policy, backends, profiles, a layer with its annotations and receipt, submissions, grades, a project, a digitization with its confirmation and dataset, a view, a QA report, a snapshot and a draft's sentence IDs. | -| `invalid//` | A package that breaks one rule, and `expect.json` naming it. | -| `keys/` | The Ed25519 test keys that signed the fixtures, and the checks-only export's team secret. Public; never use them for anything else. | -| `src/` | The hand-written sources of the three M1 documents ([src/README.md](src/README.md)). | +| `packages/hale2026harmonic/` | The main example, followed step by step in [the walkthrough](../docs/walkthrough.md): a chemistry paper read once with mistakes and fixed by twelve corrections. It has a table with two levels of column headers and grouped rows, an equation inside a sentence, a footnote, a figure with text inside its panels and a real image, references, legacy aliases, boxes on fingerprinted pages, receipts in every state (counting, partly invalidated, spot-checked, revoked and suspended), the roster that signed them, a dataset taken from the table, and derived MathML. | +| `packages/hale2026harmonic.checks-only/` | The same document prepared for another of the team's machines: text withheld, every hash an HMAC under the team key in `keys/`. | +| `packages/okafor2025open/` | An open-access paper established from a JATS file and a PDF that agree, so every sentence is corroborated. | +| `packages/smithlab2027charges/` | A short draft written in Typst, whose sentences cite sentences of the main example by address. | +| `packages/tiny2026example/` | A small document, the starting point of the correction vectors and the broken packages. | -Everything except `src/` is generated by `python3 tools/build.py`; do not edit it by hand. +## Records + +`records/` holds a valid example of every other format: anchors (a span, a leaf, a cell, a paragraph, and one keyed for a public layer), rights records, a team policy, backend declarations, team profiles, a layer with its annotations and the receipt that confirms them, submissions and grades, a project, a digitization with its confirmation and the dataset it yields, a view, a QA report, a draft snapshot and a draft's sentence IDs. + +## Broken packages + +Each folder in `invalid/` holds a package that breaks exactly one rule, and `expect.json` names it. +A conforming implementation refuses every one of them and should report that rule. + +| Case | What is wrong | Rule | +| --- | --- | --- | +| `alias-dangling/` | An alias whose node does not exist. | `doc.id.successor` | +| `bad-id/` | IDs use Crockford base32, which has no i, l, o or u. | `package.schema` | +| `checks-only-plain/` | A checks-only package that still carries text and plain hashes. | `doc.policy.withheld` | +| `cite-not-reference/` | A cite run names a node that is not a reference. | `doc.run.cite` | +| `empty-optional-file/` | An optional JSONL file with no records. | `package.file.empty` | +| `hash-mismatch/` | A node_hash that does not recompute. | `doc.hash.node-hash` | +| `heading-not-first/` | A heading must be a section's first child. | `doc.tree.child` | +| `location-unknown-page/` | A location on a page no witness has. | `doc.location.page` | +| `manifest-hash/` | A file changed after the manifest was written. | `package.manifest.hash` | +| `not-canonical/` | A line that is valid JSON but not canonical JSON. | `package.file.canonical` | +| `receipt-root/` | A receipt whose Merkle root does not recompute. | `doc.receipt.root` | +| `receipt-signature/` | A receipt changed after it was signed. | `doc.receipt.signature` | +| `root-mismatch/` | document.json's root is not the root node's node_hash. | `doc.hash.root` | +| `style-sup-sub/` | A run cannot be both a subscript and a superscript. | `doc.run.style` | +| `text-edited/` | Text changed without its hashes. | `doc.hash.text-hash` | +| `unknown-kind/` | Kinds outside the list are refused. | `package.schema` | +| `unlisted-file/` | A file the manifest does not list. | `package.manifest.unlisted` | + +## Keys + +`keys/` holds the Ed25519 secret keys that signed the fixtures' receipts, and the team secret of the checks-only package. +They are published so that anyone can reproduce the signatures; never use them for anything else. + +## Sources + +`src/` holds the hand-written descriptions of the three main documents, as first read; see [src/README.md](src/README.md). +Everything else here is generated by `python3 tools/build.py`, so change the sources or the builder, never the generated files. diff --git a/fixtures/src/README.md b/fixtures/src/README.md index 981ec33..69fe7e8 100644 --- a/fixtures/src/README.md +++ b/fixtures/src/README.md @@ -1,6 +1,7 @@ # Fixture sources -Each source describes a document as first read, errors included, in a shorthand that `tools/build.py` expands; the build mints IDs, applies the listed corrections, lays out synthetic geometry, signs the listed receipts and writes the package. +Each source describes a document as a converter first read it, mistakes included, in a shorthand that `tools/build.py` expands. +The build mints IDs, applies the listed corrections, lays out synthetic geometry, signs the listed receipts and writes the package. The shorthand is tooling, not a format. | Shorthand | Becomes | diff --git a/pixi.toml b/pixi.toml new file mode 100644 index 0000000..be1dd95 --- /dev/null +++ b/pixi.toml @@ -0,0 +1,34 @@ +# The development environment for the scienting specifications. +# +# pixi run check check everything, after linting the Markdown +# pixi task list every task, with what it does +# +# The tools need nothing but Python's standard library; pixi adds a pinned Python, +# the jsonschema package as a second schema validator, and markdownlint-cli2. + +[workspace] +name = "varve-spec" +version = "0.1.0" +description = "The scienting data formats: specification, JSON Schemas, SQL, test vectors and fixtures" +authors = ["Scientific Computing Studio "] +license = "CC-BY-4.0 AND CC0-1.0 AND Apache-2.0" +license-file = "LICENSE.md" +readme = "README.md" +homepage = "https://scient.ing" +repository = "https://git.scient.ing/varve/varve-spec" +documentation = "https://git.scient.ing/varve/varve-spec/src/branch/main/docs/README.md" +channels = ["conda-forge"] +platforms = ["linux-64", "linux-aarch64", "osx-64", "osx-arm64"] +requires-pixi = ">=0.40" + +[dependencies] +python = ">=3.14,<3.15" +jsonschema = ">=4.26.0" +markdownlint-cli2 = ">=0.23.3" + +[tasks] +check = { cmd = "python tools/check.py", depends-on = ["lint"], description = "Check schemas, records, packages, vectors, a byte-for-byte rebuild, the SQL round trip, prose and links" } +quick = { cmd = "python tools/check.py --quick", description = "Check everything except the byte-for-byte rebuild" } +build = { cmd = "python tools/build.py", description = "Rebuild fixtures/ and vectors/ from fixtures/src/ and the oracle" } +lint = { cmd = "markdownlint-cli2", description = "Lint every Markdown file with markdownlint" } +load = { cmd = "python tools/load.py", description = "Load packages into SQLite: pixi run load --sqlite lab.db PACKAGE..." } diff --git a/registry/README.md b/registry/README.md new file mode 100644 index 0000000..1ca3e98 --- /dev/null +++ b/registry/README.md @@ -0,0 +1,36 @@ +# Registry + +Lists that tools read at run time. + +| File | Holds | +| --- | --- | +| `formats.json` | Every format: its identifier, media type, schema, status and chapter. | +| `rules.json` | Every validation rule ID, with what it means. | +| `stamps/scienting-stamps-1.json` | The published rules that recognize publishers' download stamps, so page fingerprints ignore them. | +| `stages/scienting-stages-default.json` | The rights each processing stage needs, as a default a team may replace. | + +## Rule IDs + +Validators report failures as `path:line: rule: message`, for example: + +```text +nodes.jsonl:7: doc.hash.node-hash: node_hash does not recompute +``` + +A rule's meaning never changes, and a retired rule's ID is never reused, so tools and people can rely on the IDs. + +## Adding stamp rules + +When two downloads of one paper differ only in a stamp line that no published rule recognizes, record a rule in a file of your own, marked as learned: + +```json +{"format": "scienting-stamps/1", "id": "smithlab-learned", "learned": true, + "rules": [{"id": "licensed-to", "pattern": "^This copy is licensed to .+", "where": "margin", "margin": 36}]} +``` + +Patterns use only syntax that ECMAScript and Python read alike, and match a line's normalized text, with ASCII letters compared without case. + +## Replacing the stage defaults + +The stage requirements are configuration, not code: licenses define text and data mining differently, so a team copies `scienting-stages-default.json`, adjusts it with its counsel, and points its project's settings at the copy. +[Chapter 14](../spec/14-rights.md) explains how stages are checked. diff --git a/schemas/README.md b/schemas/README.md new file mode 100644 index 0000000..5b7663c --- /dev/null +++ b/schemas/README.md @@ -0,0 +1,47 @@ +# Schemas + +JSON Schemas (draft 2020-12) for every file and record the formats define. +Each schema's `$id` is `https://scient.ing/ns/schemas/`, and schemas refer to each other by those names, so a validator that loads the whole folder resolves every reference. + +## Which schema for which file + +| Topic | Schemas | +| --- | --- | +| Shared definitions | `common.schema.json`: hashes, IDs, keys, addresses, times, decimals, boxes, licenses, signatures, evidence, quotes. | +| A package | `manifest.schema.json`, `document.schema.json`, `node.schema.json`, `runs.schema.json`, `witnesses.schema.json`, `provenance.schema.json`, `page.schema.json`, `location.schema.json`, `reference.schema.json`, `id-entry.schema.json`, `correction.schema.json`, `receipt.schema.json`, `revocation.schema.json`, `roster.schema.json`, `suspect.schema.json`, `mathml.schema.json` | +| Pointing into documents | `anchor.schema.json` | +| Rights | `rights.schema.json`, `policy.schema.json`, `stages.schema.json`, `backend.schema.json`, `profile.schema.json`, `stamps.schema.json` | +| Review | `qa-report.schema.json` | +| Annotation and teaching | `layer.schema.json`, `annotation.schema.json`, `exercise.schema.json`, `submission.schema.json`, `grade.schema.json` | +| Data | `dataset.schema.json`, `value-anchor.schema.json`, `digitization.schema.json`, `view.schema.json` | +| Teams and drafts | `project.schema.json`, `snapshot.schema.json`, `draft-ids.schema.json` | + +A record that travels on its own carries a `format` such as `scienting-anchor/1`; [registry/formats.json](../registry/formats.json) maps each format to its schema. + +## Validating + +With the `jsonschema` package (4.18 or later): + +```python +import json, pathlib +from jsonschema import Draft202012Validator +from referencing import Registry, Resource + +schemas = {p.name: json.loads(p.read_text()) for p in pathlib.Path("schemas").glob("*.json")} +registry = Registry().with_resources((s["$id"], Resource.from_contents(s)) for s in schemas.values()) +validator = Draft202012Validator(schemas["anchor.schema.json"], registry=registry) +validator.validate(json.load(open("fixtures/records/anchor-span.json"))) +``` + +Any draft 2020-12 validator works the same way; `python3 tools/check.py` validates every fixture with the repository's own validator and, when installed, with `jsonschema` too. + +## What a schema does not check + +A schema checks shape: required properties, types, patterns, enumerations. +It cannot check that a hash recomputes, that a node's children are allowed, that a correction's ID derives from its content, or that a receipt's signature verifies. +Those are the rules of [spec/22-conformance.md](../spec/22-conformance.md), each with an ID in [registry/rules.json](../registry/rules.json). + +## Writing schemas + +The schemas use only the keywords listed in [spec/22-conformance.md](../spec/22-conformance.md#223-schemas), and their patterns avoid `\d`, `\w`, `\s`, `\b` and lookbehind, so that validators in every language read them alike. +Optional properties are omitted rather than set to null, except where a schema says a property is nullable. diff --git a/spec/00-introduction.md b/spec/00-introduction.md index b6387e6..a3bc29a 100644 --- a/spec/00-introduction.md +++ b/spec/00-introduction.md @@ -2,13 +2,11 @@ ## 0.1 Scope -These specifications define the data the Scientific Computing Studio's parts write and read: documents and their packages, anchors, receipts, rights records, layers, exercises, datasets, digitizations, views, projects and bundles. +These specifications define the data that tools for reading, correcting, annotating and checking scientific papers exchange: documents and their packages, anchors, receipts, rights records, layers, exercises, datasets, digitizations, views, projects and bundles. They define data, not programs. -A conforming file can be read, checked and loaded without any studio code, and every rule here is stated so that a second implementation, in any language, computes the same bytes. +A conforming file can be read, checked and loaded without any particular program, and every rule is stated so that a second implementation, in any language, computes the same bytes. -The design document, *The Scientific Computing Studio: design of Lode, paperpack and Varve*, explains why each format exists. -This text says exactly what each format is. -Where the two differ, this text governs, and [DECISIONS.md](../DECISIONS.md) records why it differs. +The [guides](../docs/README.md) explain what each format is for, and the [rationale](../docs/rationale.md) why its rules are what they are. ## 0.2 Conformance language @@ -39,16 +37,16 @@ The specification as a whole is versioned with semantic versioning; this is vers ## 0.4 Status -Each format has one of three statuses. +Each format has one of three statuses, listed in [registry/formats.json](../registry/formats.json). | Status | Meaning | | --- | --- | -| `candidate` | Complete, with vectors and fixtures, and ready to implement. Changes before 1.0 need a recorded decision. | -| `provisional` | Complete, but its milestone has not come, and it may change with less ceremony. | +| `candidate` | Complete, with vectors and fixtures, and ready to implement. | +| `provisional` | Complete, but expected to change as implementations gain experience with it. | | `reserved` | Named and shaped, but not yet specified in full. | -The design builds in levels, and level 1 implements a subset. -Fields that level 1 does not use, such as rights beyond the license, receipts and spot-checks, are defined now so that formats never change when a team moves up a level. +Some fields matter only to some users: a personal library has no use for spot-checks, and a team that reads only open-access papers rarely needs a policy. +Those fields are defined all the same, so that files move between a personal library and a team's store without conversion. A consumer that does not use a field MUST still preserve it when it rewrites a file. ## 0.5 How the repository is arranged @@ -62,7 +60,8 @@ A consumer that does not use a field MUST still preserve it when it rewrites a f | `registry/` | Formats, rule IDs, stamp rules and the default stage requirements. | Yes | | `vectors/` | Test vectors, one case per line. | Yes | | `fixtures/` | Valid packages and records, invalid packages, and the keys that signed them. | Yes | -| `tools/` | The oracle that computed the vectors, the builder and the checker. | No | +| `tools/` | The programs that compute and check the vectors and fixtures, and load packages into SQLite. | No | +| `docs/` | Guides for people getting to know the formats. | No | -The oracle exists so that no vector is typed by hand. -It is not a library: implementations conform to the chapters and the vectors, not to the oracle, and `scienting-core` is the reference implementation. +The programs in `tools/` compute every expected value, so that none is typed by hand. +They are not a library: implementations conform to the chapters and the vectors, and `scienting-core` is the reference implementation of the shared rules. diff --git a/spec/05-hashes.md b/spec/05-hashes.md index 6b026ed..a33d76e 100644 --- a/spec/05-hashes.md +++ b/spec/05-hashes.md @@ -64,7 +64,7 @@ Marking citations matters for the same reason: "10" followed by citation 3 must Each complex run also yields a complex key, `notation:source`, with the source in NFC and every whitespace character removed. A node's complex keys are listed in order. Two readings agree only when their `match_hash` values are equal and their complex keys are equal, run by run; this is stage 1 of canonical math. -Stage 2, a later minor version, will replace U+FFFC with canonical MathML; it changes `match_hash` and `align_key` values but no `text_hash`, `own_hash` or `node_hash`, so no anchor goes stale. +A later minor version may add stage 2, which replaces U+FFFC with canonical MathML; that would change `match_hash` and `align_key` values but no `text_hash`, `own_hash` or `node_hash`, so no anchor would go stale. ## 5.4 Simple notation diff --git a/spec/08-fidelity.md b/spec/08-fidelity.md index 3299f82..a3828b8 100644 --- a/spec/08-fidelity.md +++ b/spec/08-fidelity.md @@ -91,7 +91,7 @@ It is valid when its signature verifies and its signer is the revoked receipt's The roster, `scienting-roster/1`, lists the keys that may sign: each with an `id`, a `person`, a `scheme`, its `public` key (Base64 for Ed25519), its `purposes` (`sign`, `spot-check`, `export`), `registered_at` and `revoked_at`. A package MAY carry the roster its receipts need as `roster.json`, so its receipts verify without the store. -The roster is meant to match the one Lode already verifies; aligning the two is recorded in [DECISIONS.md](../DECISIONS.md). +The roster is meant to be the same one Lode verifies; see the [rationale](../docs/rationale.md#limits-worth-knowing). ## 8.8 Spot-checks @@ -109,7 +109,7 @@ Anyone can recompute the sample; `spotcheck.jsonl` does. ## 8.9 Reviewer error bounds -One sample proves little; what a buyer wants is a bound on each reviewer's error rate across all their samples. +One sample proves little; what someone relying on the work needs is a bound on each reviewer's error rate across all their samples. With *k* errors in *n* sampled nodes, the bound is the one-sided Clopper–Pearson upper confidence bound at confidence *c*, by default 0.95: the rate *p* at which the probability of at most *k* errors in *n* is 1 − *c*, or 1 when *k* = *n*. It is reported to six decimal places, rounded up. With no errors the bound is about 3/*n*, so zero errors in 300 sampled nodes bounds a rate below 1%; `bounds.jsonl` gives cases. diff --git a/spec/11-corrections.md b/spec/11-corrections.md index dfbedef..0c38e98 100644 --- a/spec/11-corrections.md +++ b/spec/11-corrections.md @@ -20,7 +20,7 @@ Each has an `index`, counted from 1 in the order corrections are applied, and th | `set` | `target`, `attr`, `value` | Sets one structural attribute: a section's `level`, a table's `columns`, a cell's `column`, `rowspan`, `colspan` or `role`, a row's `header_path`, a list's `ordered`, an item's `marker` or a theorem's `variant`. | Unchanged. | A leaf's editable strings are the `text` of its `text`, `link`, `cite`, `xref` and `fnmark` runs and the `source` of its `math` runs, each searched on its own; an equation's `source`; a verbatim node's `text`. -`set` is an addition to the design's eight operations, needed to fix table structure; see [DECISIONS.md](../DECISIONS.md). +`set` exists because readings drop spans, roles and header paths; see the [rationale](../docs/rationale.md#corrections). Runs are made canonical after every operation, and the result MUST satisfy every rule of [3](03-nodes.md). ## 11.2 Every correction records diff --git a/spec/12-package.md b/spec/12-package.md index f66dc91..42f9f95 100644 --- a/spec/12-package.md +++ b/spec/12-package.md @@ -67,4 +67,4 @@ Nothing derived is content, and a consumer may delete and rebuild it. ## 12.8 Media type and extension -The proposed extension is `.sct.zip` and the media type `application/vnd.scienting.doc+zip`; both wait for a decision ([DECISIONS.md](../DECISIONS.md)) and are fixed forever once packages are shared. +A zipped package has the extension `.sct.zip` and the media type `application/vnd.scienting.doc+zip`. diff --git a/spec/13-database.md b/spec/13-database.md index 70264b4..f27cbb6 100644 --- a/spec/13-database.md +++ b/spec/13-database.md @@ -9,7 +9,7 @@ Both run unchanged on SQLite, Dolt and PostgreSQL, under these rules: 2. Enumerations are `VARCHAR` with `CHECK (… IN (…))`. 3. JSON values are canonical JSON in `TEXT` columns; booleans are `INTEGER` 0 or 1. 4. Every table has a primary key, and key columns are `VARCHAR` with a length, as MySQL-compatible engines require. -5. No table, column or index is named with a word reserved in SQLite, PostgreSQL or MySQL; so the design's `key`, `index`, `at`, `by`, `before` and `after` are `doc_key`, `idx`, `made_at`, `signer`, `before_hash` and `after_hash`. +5. No table, column or index is named with a word reserved in SQLite, PostgreSQL or MySQL; so the formats' `key`, `index`, `at`, `by`, `before` and `after` are `doc_key`, `idx`, `made_at`, `signer`, `before_hash` and `after_hash`. ## 13.2 Tables diff --git a/spec/14-rights.md b/spec/14-rights.md index d616469..cbcff35 100644 --- a/spec/14-rights.md +++ b/spec/14-rights.md @@ -2,7 +2,7 @@ What a team may do with a paper depends on how it holds the paper, and AI processing is now a use that licenses name. Every witness carries a rights record, every stage declares the uses it needs, and a stage runs only when the rights allow them. -Nothing here is legal advice; the defaults are proposals for counsel to set. +Nothing here is legal advice: the defaults are a starting point that each team reviews with its own counsel. ## 14.1 The rights record @@ -33,7 +33,7 @@ A policy, `scienting-policy/1`, is a project owner's signed decision: `applies_t It applies from `at` until `valid_until`. A node established from several witnesses takes the most restrictive of their uses: `false` if any is false, else `"unknown"` if any is unknown, else `true`. -That combining rule is proposed for counsel to confirm. +This is the default combining rule. ## 14.3 Stages @@ -41,13 +41,13 @@ That combining rule is proposed for counsel to confirm. A stage may run on a processor when every listed use is `true`, the processor is among those allowed, if any are listed, and the pair `ai//` is `true`. When a check fails, the refusal lists what is lacking: a use or pair that is `false` by name, one that is unknown with a trailing `?`, and a disallowed processor as `processor:`. -[registry/stages/scienting-stages-default.json](../registry/stages/scienting-stages-default.json) holds the design's proposed defaults; a team replaces the file, not the code. +[registry/stages/scienting-stages-default.json](../registry/stages/scienting-stages-default.json) holds a default set; a team replaces the file, not the code. Rights are checked when a stage runs and again whenever text is served. Every derived artifact records the rights record and use it was made under, so it can be listed and purged when a basis ends. ## 14.4 Backends and profiles -A backend's code and weights can be licensed differently, and the weights decide whether a paying team may use it. +A backend's code and weights can be licensed differently, and the weight license may exclude commercial or competing use. `scienting-backend/1` declares a backend's `code_license`, its `weights` with their licenses, and the `conditions` they impose: | Condition | Met when the team profile | diff --git a/spec/21-drafts.md b/spec/21-drafts.md index 1bc9d68..e8a75a0 100644 --- a/spec/21-drafts.md +++ b/spec/21-drafts.md @@ -14,4 +14,4 @@ Authors rewrite constantly, and IDs must survive without cluttering the source. IDs live in a sidecar, `ids.jsonl` in the Typst project, `scienting-draft-ids/1`: one line per sentence with its `id`, its `text_hash`, its `section` and its `ord`, and `pinned` when the author fixed the ID to a label. Each conversion aligns the new sentences with the previous revision, exact text first and then by edit distance within the section; an edited sentence keeps its ID and gets a new hash, so its annotations reattach or go stale as [6.5](06-anchors.md#65-resolution) says. -Both formats are provisional until the Typst package exists. +Both formats are `provisional` ([0.4](00-introduction.md#04-status)). diff --git a/spec/22-conformance.md b/spec/22-conformance.md index 6b0e078..b4794d0 100644 --- a/spec/22-conformance.md +++ b/spec/22-conformance.md @@ -48,6 +48,8 @@ Rule IDs are stable: a rule's meaning never changes, and a retired rule's ID is | `doc.witness.`, `doc.page.`, `doc.location.`, `doc.reference.` | Witnesses and geometry. | | `grade.` | Grading. | -## 22.5 Freezing 1.0 +## 22.5 Stability -These specifications become 1.0 when the formats have been written by at least three producers (conversion, hand-built fixtures, and a structured witness such as JATS) and read by at least three consumers (databases, Lode and the reader), and at least one paying buyer has used a delivery built on them. +Within a major version of a format, a valid file stays valid, and every value the chapters define is computed the same way. +Rule IDs, enumeration values and format identifiers are never reused with another meaning. +A `candidate` format changes only with a recorded reason and a new or changed vector; a `provisional` one may change more freely, and says so in [registry/formats.json](../registry/formats.json). diff --git a/spec/README.md b/spec/README.md index 8a0c07c..c464db6 100644 --- a/spec/README.md +++ b/spec/README.md @@ -1,8 +1,22 @@ # The scienting specifications -These chapters are the normative text of `spec` 0.1. +This is the specification of the scienting formats: exactly what each file holds and how every value in it is computed. +For what the formats are for, start with the [guides](../docs/README.md); this text assumes you know why and tells you how. Schemas, SQL, the JSON-LD context, the registries, the vectors and the fixtures are normative too, and the chapters say which. -Where a chapter and a vector disagree, the disagreement is a bug in this repository; report it. +Where a chapter and a vector disagree, the disagreement is a bug; please report it. + +## Reading paths + +| If you are writing | Read | +| --- | --- | +| Anything at all | 0, 1, 3, 4 and 5: conventions, nodes, identity and the content values. | +| A reader of packages | Then 2, 6, 7 and 12: the model, anchors, geometry and the package. | +| A converter | Then 9, 10 and 11: figures and tables, references, and corrections with the gate. | +| A store or a server | Then 8, 13 and 14: receipts, the database form, and rights. | +| A tool for annotation, teaching or data | Then 15 to 21. | +| A validator | Chapter 22, with [registry/rules.json](../registry/rules.json). | + +## Chapters | Chapter | Subject | | --- | --- | diff --git a/sql/README.md b/sql/README.md new file mode 100644 index 0000000..86f99f7 --- /dev/null +++ b/sql/README.md @@ -0,0 +1,40 @@ +# SQL + +Table definitions for the database form of packages, a team's store, and the bundle. + +| File | Holds | Runs on | +| --- | --- | --- | +| `doc.sql` | Documents: works, documents, witnesses, nodes, pages, locations, references, citations, corrections, IDs, receipts, revocations, suspects and package files. | SQLite, Dolt, PostgreSQL | +| `store.sql` | What a team adds: roster keys, projects, members, reading paths, layers, annotations, datasets, digitizations, views, policies, submissions and grades. Load `doc.sql` first. | SQLite, Dolt, PostgreSQL | +| `bundle-sqlite.sql` | The bundle's extra tables: metadata, a full-text index (FTS5) and room for embeddings. Load both files above first. | SQLite | + +## Creating a database + +```sh +sqlite3 lab.db < sql/doc.sql && sqlite3 lab.db < sql/store.sql +psql -d lab -f sql/doc.sql && psql -d lab -f sql/store.sql +dolt sql < sql/doc.sql && dolt sql < sql/store.sql +``` + +To load packages into SQLite with nothing installed, use `python3 tools/load.py --sqlite lab.db PACKAGE…`; [docs/examples.md](../docs/examples.md#load-packages-into-sqlite) has queries to try. + +## Conventions + +- No foreign keys: validators check references, so the same file runs everywhere. +- Enumerations are `VARCHAR` with a `CHECK`; JSON values are canonical JSON in `TEXT`; booleans are `INTEGER` 0 or 1. +- No table, column or index name is reserved in any of the three engines, so the formats' `key`, `index`, `at`, `by`, `before` and `after` become `doc_key`, `idx`, `made_at`, `signer`, `before_hash` and `after_hash`. +- Each row keeps every property of its record, so a package loaded into the tables unloads back to the same bytes. +- `citations`, `receipt_nodes` and `annotation_targets` are derived on load, for queries. + +## A first query + +Every sentence that cites a given reference, which is the citation's context: + +```sql +SELECT c.sentence +FROM citations c +JOIN refs r ON r.doc_key = c.doc_key AND r.node = c.ref_node +WHERE c.doc_key = 'hale2026harmonic' AND r.label = '2'; +``` + +[Chapter 13](../spec/13-database.md) defines the mapping from files to tables. diff --git a/tools/README.md b/tools/README.md index 159f029..ee68c3a 100644 --- a/tools/README.md +++ b/tools/README.md @@ -1,11 +1,24 @@ # Tools -Non-normative, standard library only, licensed Apache-2.0. +Small programs that build and check the repository, and one for loading packages into SQLite. +They use only the Python standard library and are licensed Apache-2.0. -| Tool | Does | +| Program | Does | | --- | --- | -| `oracle/` | A compact implementation of every rule, used to compute and check the vectors and fixtures. It is not a library; implementations conform to the chapters, and `scienting-core` is the reference implementation. | -| `build.py` | Builds `fixtures/` (except `src/`) and `vectors/` from `fixtures/src/` and the cases it defines. Deterministic; `--out DIR` builds elsewhere. | -| `check.py` | Checks the whole repository; `--quick` skips the determinism rebuild. Uses `jsonschema` as a second validator when installed. | +| `check.py` | Checks everything: schemas, records, packages, the broken packages, every vector, a byte-for-byte rebuild, the SQL round trip and the prose. `--quick` skips the rebuild. | +| `build.py` | Builds `fixtures/` and `vectors/` from the hand-written sources in `fixtures/src/` and the cases it defines. `--out DIR` builds into another folder. | +| `load.py` | Loads packages into a SQLite database with the tables of `sql/`: `python3 tools/load.py --sqlite lab.db PACKAGE…`. | +| `oracle/` | A compact implementation of every rule, which `build.py` uses to compute expected values and `check.py` uses to recompute them. | -The oracle's JSON Schema validator supports exactly the keyword subset of [spec/22-conformance.md](../spec/22-conformance.md#223-schemas) and refuses schemas that use anything else. +## When to run what + +Each program is also a pixi task, so `pixi run check`, `pixi run quick`, `pixi run build` and `pixi run load` work from anywhere in the repository; `pixi run check` lints the Markdown first. + +Run `pixi run check` before every commit. +Run `pixi run build` after changing a fixture source, a rule in `oracle/`, or a case in `build.py`, then commit the regenerated files with the change; the check fails if they are out of date. + +## About the oracle + +The oracle exists so that no expected value in this repository is typed by hand. +It favors clarity over speed, its Ed25519 is the slow reference code of RFC 8032, and it is not a library to build on: implementations follow the [specification](../spec/README.md) and test themselves against the [vectors](../vectors/README.md). +Its JSON Schema validator supports exactly the keyword subset the schemas use, and refuses any schema that uses more. diff --git a/tools/check.py b/tools/check.py index 70a94aa..1849688 100644 --- a/tools/check.py +++ b/tools/check.py @@ -31,6 +31,8 @@ from tools.oracle.text import cp_to_utf16, letter_first, mint_node_id, norm, rec from tools.oracle.tree import Refusal # noqa: E402 FAILS = [] +# Folders that are not part of the repository: version control, environments and caches. +SKIP_DIRS = {".git", ".pixi", "node_modules", "__pycache__"} COUNTS = {} @@ -55,6 +57,7 @@ def js(path): STORE = Store(p("schemas")) +SCHEMA_FILES = sorted(f for f in os.listdir(p("schemas")) if f.endswith(".schema.json")) FORMAT_SCHEMA = {f["format"]: f["schema"] for f in js(p("registry", "formats.json"))["formats"]} @@ -63,7 +66,7 @@ def schema_errors(name, rec): def check_schemas(): - for f in sorted(os.listdir(p("schemas"))): + for f in SCHEMA_FILES: s = STORE.load(f) ok("schemas", s.get("$id", "").endswith("/" + f), "%s: $id must end with its file name" % f) for f in js(p("registry", "formats.json"))["formats"]: @@ -73,7 +76,7 @@ def check_schemas(): import jsonschema # noqa: F401 from referencing import Registry, Resource reg = Registry().with_resources( - [(STORE.load(f)["$id"], Resource.from_contents(STORE.load(f))) for f in os.listdir(p("schemas"))]) + [(STORE.load(f)["$id"], Resource.from_contents(STORE.load(f))) for f in SCHEMA_FILES]) COUNTS["second-validator"] = 0 globals()["SECOND"] = (jsonschema, reg) except ImportError: @@ -460,7 +463,7 @@ ABBREV = re.compile(r"\b(e\.g|i\.e|et al|cf|vs|Fig|Eq|Ref|ca|No|St|Dr|Mr|Ms|al)\ def check_prose(): mds = [] for d, dirs, fs in os.walk(ROOT): - dirs[:] = [x for x in dirs if x not in (".git", "node_modules", "__pycache__")] + dirs[:] = [x for x in dirs if x not in SKIP_DIRS] mds += [os.path.join(d, f) for f in fs if f.endswith(".md")] for path in sorted(mds): rel = os.path.relpath(path, ROOT) @@ -479,6 +482,48 @@ def check_prose(): ok("prose", not SENTENCE_BREAK.search(body), "%s:%d: one sentence per line" % (rel, i)) +def _slug(heading): + """A heading's anchor, as GitHub, Gitea and Forgejo compute it.""" + t = re.sub(r"[`*]", "", heading.strip().lower()) + t = "".join(ch for ch in t if ch.isalnum() or ch in " -_") + return t.replace(" ", "-") + + +def check_links(): + anchors = {} + mds = [] + for d, dirs, fs in os.walk(ROOT): + dirs[:] = [x for x in dirs if x not in SKIP_DIRS] + mds += [os.path.join(d, f) for f in fs if f.endswith(".md")] + for path in mds: + seen, fence = {}, False + for ln in open(path, encoding="utf-8"): + if ln.startswith("```"): + fence = not fence + elif not fence and ln.startswith("#"): + base = _slug(ln.lstrip("#")) + n = seen.get(base, 0) + seen[base] = n + 1 + anchors.setdefault(path, set()).add(base if n == 0 else "%s-%d" % (base, n)) + for path in sorted(mds): + fence = False + for i, ln in enumerate(open(path, encoding="utf-8"), 1): + if ln.startswith("```"): + fence = not fence + continue + if fence: + continue + for target in re.findall(r"\]\(([^)\s]+)\)", re.sub(r"`[^`]*`", "", ln)): + if re.match(r"^[a-z]+:", target): + continue + file, _, frag = target.partition("#") + dest = os.path.normpath(os.path.join(os.path.dirname(path), file)) if file else path + rel = os.path.relpath(path, ROOT) + ok("links", os.path.exists(dest), "%s:%d: no such file %s" % (rel, i, file)) + if frag and dest.endswith(".md") and os.path.exists(dest): + ok("links", frag in anchors.get(dest, ()), "%s:%d: no heading #%s in %s" % (rel, i, frag, file or "this file")) + + def main(): check_schemas() check_registry() @@ -490,6 +535,7 @@ def main(): check_determinism() check_sql() check_prose() + check_links() for k in sorted(COUNTS): print("%-17s %5d checks" % (k, COUNTS[k])) if FAILS: diff --git a/tools/load.py b/tools/load.py new file mode 100644 index 0000000..b14aae4 --- /dev/null +++ b/tools/load.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Load packages into a SQLite database with the tables of sql/doc.sql and sql/store.sql. + +Non-normative tooling, standard library only. +Usage: python3 tools/load.py --sqlite lab.db PACKAGE [PACKAGE ...] +Files without tables of their own (assets, datasets, derived files) are copied by SHA-256 into an +asset store folder beside the database, named .assets/. +""" + +import argparse +import os +import sqlite3 +import sys + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) + +from tools.oracle import db # noqa: E402 + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--sqlite", required=True, help="the database file; created with the tables if new") + ap.add_argument("packages", nargs="+", help="package folders") + a = ap.parse_args() + new = not os.path.exists(a.sqlite) + conn = sqlite3.connect(a.sqlite) + if new: + db.create(conn, os.path.join(ROOT, "sql")) + for p in a.packages: + key = db.load(conn, p, a.sqlite + ".assets") + n = conn.execute("SELECT COUNT(*) FROM nodes WHERE doc_key = ?", (key,)).fetchone()[0] + print("%s: %d nodes" % (key, n)) + conn.commit() + + +if __name__ == "__main__": + main() diff --git a/vectors/README.md b/vectors/README.md index 78b6676..1d85382 100644 --- a/vectors/README.md +++ b/vectors/README.md @@ -1,27 +1,30 @@ # Test vectors -Each file holds one case per line, in canonical JSON, with an `id`, the inputs, and the expected outputs. -Every value was computed by the oracle in `tools/` and is checked again by `tools/check.py`; none is typed by hand. +Inputs and the exact outputs every implementation must compute. +Each file holds one case per line, in canonical JSON, with an `id`, its inputs, and its expected outputs, usually under `expect`. +Run them in your own test suite; [docs/examples.md](../docs/examples.md#test-your-implementation-against-a-vector) shows the shape of a harness. | File | Tests | Chapter | | --- | --- | --- | | `canonical-json.jsonl` | `input_json`, parsed with every number as an IEEE 754 double, and its canonical `output`. | [1.1](../spec/01-conventions.md#11-canonical-json) | | `normalize.jsonl` | Normalization for hashing, and the SHA-256 of the result. | [1.4](../spec/01-conventions.md#14-text-normalization-for-hashing) | -| `ids.jsonl` | Minting payloads and node IDs; letter-first encodings of digests. | [4](../spec/04-identity.md) | -| `notation.jsonl` | Each source's canonical linear form, or null when it is complex. | [5.4](../spec/05-hashes.md#54-simple-notation) | +| `ids.jsonl` | Minting payloads and node IDs, and letter-first encodings of digests. | [4](../spec/04-identity.md) | +| `notation.jsonl` | Each math source's canonical linear form, or null when it is complex. | [5.4](../spec/05-hashes.md#54-simple-notation) | | `projections.jsonl` | One-sentence runs: plain, normalized, strict and lossy projections, complex keys and the three text values. | [5.3](../spec/05-hashes.md#53-the-strict-projection) | -| `projection-groups.jsonl` | Which cases must share, or must differ in, a value. | [5.3](../spec/05-hashes.md#53-the-strict-projection) | +| `projection-groups.jsonl` | Which cases must share, or must differ in, a value: H₂O in six spellings, cm⁻¹, 10⁻³, K_r, ligatures, the micro sign, "103" against 10³. | [5.3](../spec/05-hashes.md#53-the-strict-projection) | | `offsets.jsonl` | Code-point spans and their UTF-16 equivalents. | [1.5](../spec/01-conventions.md#15-character-offsets) | | `hashes.jsonl` | Every node of a small document, its own content and its five values. | [5.1](../spec/05-hashes.md#51-the-five-values) | | `addresses.jsonl` | How addresses parse; null for invalid ones. | [6.1](../spec/06-anchors.md#61-addresses) | -| `keyed.jsonl` | The team key and its ID, an HMAC text hash, page hashes, a page key, and what a non-holder can and cannot check. | [5.8](../spec/05-hashes.md#58-keyed-hashes) | -| `fingerprints.jsonl` | Page inputs, the stamp rules that removed lines, the fingerprint text and the fingerprint. | [7.1](../spec/07-geometry.md#71-page-fingerprints) | -| `merkle.jsonl` | Receipt items and their RFC 6962 roots. | [8.5](../spec/08-fidelity.md#85-merkle-roots) | -| `signatures.jsonl` | The RFC 8032 test vector, and a receipt's record hash, payload and signature. | [8.4](../spec/08-fidelity.md#84-record-hashes-and-signatures) | +| `keyed.jsonl` | The team key and its ID, an HMAC, page hashes, a page-derived key, and what someone without the paper cannot check. | [5.8](../spec/05-hashes.md#58-keyed-hashes) | +| `fingerprints.jsonl` | Pages as text lines, the stamp rules that fired, and the fingerprint; two downloads that must agree, and a change that must not. | [7.1](../spec/07-geometry.md#71-page-fingerprints) | +| `merkle.jsonl` | Receipt items and their Merkle roots. | [8.5](../spec/08-fidelity.md#85-merkle-roots) | +| `signatures.jsonl` | The RFC 8032 test vector, and a receipt's record hash, signing payload and signature. | [8.4](../spec/08-fidelity.md#84-record-hashes-and-signatures) | | `spotcheck.jsonl` | Seed payloads, seeds, sample sizes and samples. | [8.8](../spec/08-fidelity.md#88-spot-checks) | -| `bounds.jsonl` | Clopper–Pearson bounds as reported. | [8.9](../spec/08-fidelity.md#89-reviewer-error-bounds) | -| `alignment.jsonl` | Two readings, their pairs and their suspects. | [5.6](../spec/05-hashes.md#56-alignment) | -| `corrections.jsonl` | Each operation and refusal on a small document: the state before, the correction, and the record and state after, or the rule. | [11](../spec/11-corrections.md) | -| `resolve.jsonl` | Anchors made at revision 0 and how each resolves against a fixture package. | [6.5](../spec/06-anchors.md#65-resolution) | +| `bounds.jsonl` | Reviewer error bounds as reported. | [8.9](../spec/08-fidelity.md#89-reviewer-error-bounds) | +| `alignment.jsonl` | Two readings of a document, their pairs and their suspects. | [5.6](../spec/05-hashes.md#56-alignment) | +| `corrections.jsonl` | Each operation and refusal on a small document: the state before, the correction, and the record and state after, or the rule broken. | [11](../spec/11-corrections.md) | +| `resolve.jsonl` | Anchors made on the example paper's first reading, and how each resolves after its corrections. | [6.5](../spec/06-anchors.md#65-resolution) | | `rights.jsonl` | Stage checks with policies and expiry, the combining rule, and backend conditions. | [14](../spec/14-rights.md) | -| `grading.jsonl` | Exercises, submissions and grades. | [19.4](../spec/19-exercises.md#194-grading) | +| `grading.jsonl` | Exercises, submissions and the grades they earn. | [19.4](../spec/19-exercises.md#194-grading) | + +Vectors that refer to a package (`resolve.jsonl` and `grading.jsonl`) name it by path; resolve against the files in that package, not against anything else.