From 30d310f64d9610b1cfef9598f612ddf898f352ee Mon Sep 17 00:00:00 2001 From: Alex Maldonado Date: Mon, 19 Oct 2026 04:23:49 -0400 Subject: [PATCH] Publish the package and its documentation, and move to the papers organization Add the Zensical documentation of the Python package (docs/package, built by the docs workflow), merge the standard publish workflow with the setuptools-scm tag check, add the SPDX header rule to ruff, restructure the README, and point every repository and registry address at the Gitea organization papers. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq --- .gitea/workflows/docs.yml | 56 +++++++ .gitea/workflows/publish.yml | 40 +++-- .ruff.toml | 6 + CHANGELOG.md | 14 ++ CONTRIBUTING.md | 18 ++- LICENSE.md | 4 +- README.md | 180 +++++++++++++++++----- deploy/README.md | 22 ++- docs/comparison.md | 2 +- docs/concepts.md | 2 +- docs/examples.md | 6 +- docs/faq.md | 2 +- docs/glossary.md | 2 +- docs/overview.md | 2 +- docs/package/about/changelog.md | 8 + docs/package/about/index.md | 36 +++++ docs/package/about/license.md | 14 ++ docs/package/about/llm-assistance.md | 14 ++ docs/package/guides/check-and-build.md | 51 ++++++ docs/package/guides/index.md | 12 ++ docs/package/guides/load-into-sqlite.md | 46 ++++++ docs/package/guides/use-the-library.md | 66 ++++++++ docs/package/guides/validate-a-package.md | 43 ++++++ docs/package/index.md | 22 +++ docs/package/reference/api.md | 70 +++++++++ docs/package/reference/cli.md | 82 ++++++++++ docs/package/reference/index.md | 6 + docs/package/start/index.md | 9 ++ docs/package/start/install.md | 74 +++++++++ docs/package/start/quick-start.md | 59 +++++++ docs/rationale.md | 2 +- docs/walkthrough.md | 2 +- fixtures/README.md | 2 +- pyproject.toml | 8 +- spec/README.md | 2 +- tools/README.md | 4 +- zensical.toml | 100 ++++++++++++ 37 files changed, 1005 insertions(+), 83 deletions(-) create mode 100644 .gitea/workflows/docs.yml create mode 100644 docs/package/about/changelog.md create mode 100644 docs/package/about/index.md create mode 100644 docs/package/about/license.md create mode 100644 docs/package/about/llm-assistance.md create mode 100644 docs/package/guides/check-and-build.md create mode 100644 docs/package/guides/index.md create mode 100644 docs/package/guides/load-into-sqlite.md create mode 100644 docs/package/guides/use-the-library.md create mode 100644 docs/package/guides/validate-a-package.md create mode 100644 docs/package/index.md create mode 100644 docs/package/reference/api.md create mode 100644 docs/package/reference/cli.md create mode 100644 docs/package/reference/index.md create mode 100644 docs/package/start/index.md create mode 100644 docs/package/start/install.md create mode 100644 docs/package/start/quick-start.md create mode 100644 zensical.toml diff --git a/.gitea/workflows/docs.yml b/.gitea/workflows/docs.yml new file mode 100644 index 0000000..b15921c --- /dev/null +++ b/.gitea/workflows/docs.yml @@ -0,0 +1,56 @@ +# Builds the Zensical documentation strictly (a broken link or a page missing from the navigation fails the job) and, +# on main, copies it to /srv/www/docs/paperspec, which serves https://docs.scient.ing/paperspec/. +# The linux-base runner mounts the host's /srv/www into its job containers, so publishing is a local copy. +name: docs +on: + push: + branches: [main, dev] + paths: + - "docs/**" + - "zensical.toml" + - "README.md" + - "CHANGELOG.md" + - "src/**" + - ".gitea/workflows/docs.yml" + workflow_dispatch: +permissions: + contents: read +concurrency: + group: docs-${{ gitea.ref }} + cancel-in-progress: true +defaults: + run: + shell: bash +env: + SITE_ROOT: /srv/www/docs + SITE_SLUG: paperspec +jobs: + docs: + runs-on: linux-base + container: + image: git.scient.ing/infra/linux-base:26.10.7 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Build the documentation, strictly + run: pixi run --locked -e docs docs-build + - name: Publish the documentation + if: github.ref == 'refs/heads/main' + run: | + if [ ! -d "$SITE_ROOT" ]; then + echo "$SITE_ROOT is not mounted on this runner, so the docs were built but not published." + exit 0 + fi + case "$SITE_SLUG" in + ''|*/*|.*) echo "SITE_SLUG must be a plain folder name, not '$SITE_SLUG'." >&2; exit 1 ;; + esac + if [ ! -f build/docs/index.html ] || [ ! -f build/docs/404.html ]; then + echo "build/docs has no index.html or 404.html, so it was not published." >&2 + exit 1 + fi + # Same copy as the other studio sites: unchanged files keep their Last-Modified and ETag, and the + # changes apply together at the end of the copy. + rsync --recursive --links --perms --chmod=D755,F644 --checksum --delay-updates --delete-delay --stats \ + build/docs/ "$SITE_ROOT/$SITE_SLUG/" diff --git a/.gitea/workflows/publish.yml b/.gitea/workflows/publish.yml index 6b915c2..832801a 100644 --- a/.gitea/workflows/publish.yml +++ b/.gitea/workflows/publish.yml @@ -1,16 +1,18 @@ -# Publishes the paperspec Python package to this instance's PyPI registry when a version tag (v0.1.1) is -# pushed; see deploy/README.md for the secrets it needs and how to install from the registry. -# The package version comes from the tag through setuptools-scm, so the job refuses to upload unless the -# built version equals the tag. Run it by hand only on a tag, for example after adding the secrets. +# Publishes the paperspec wheel and sdist to this Gitea instance's PyPI registry when a version tag (v1.2.3) is pushed, +# and creates the Gitea release with both files attached. The version is the tag's: setuptools-scm computes it +# from git (hence the full-history checkout), and the job refuses to upload unless the built version equals the +# tag, so tag the commit you mean to release. Run it by hand only on a tag. +# Needs the organization secrets PACKAGE_USER and PACKAGE_TOKEN (a token with the write:package scope). +# Install side: README.md ("Installing") and deploy/README.md name the registry index. name: publish on: push: - tags: ["v*"] + tags: ["v[0-9]*"] workflow_dispatch: permissions: contents: read concurrency: - group: publish-${{ github.ref }} + group: publish-${{ gitea.ref }} cancel-in-progress: false defaults: run: @@ -23,14 +25,14 @@ jobs: timeout-minutes: 30 env: REGISTRY: ${{ github.server_url }}/api/packages/${{ github.repository_owner }}/pypi + API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }} steps: - name: Require a version tag run: | if [[ "$GITHUB_REF" != refs/tags/v* ]]; then - echo "Publish runs on a version tag such as v0.1.1; $GITHUB_REF is not one." + echo "Publish runs on a version tag such as v1.2.3; $GITHUB_REF is not one." exit 1 fi - # The whole history, so setuptools-scm finds the tag and computes the release version. - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -38,7 +40,7 @@ jobs: - name: Check the repository run: pixi run --locked check - name: Build the sdist and wheel - run: pixi run --locked -e dev dist + run: pixi run --locked -e release dist - name: Confirm the built version is the tag run: | tag="${GITHUB_REF_NAME#v}" @@ -50,7 +52,7 @@ jobs: exit 1 fi - name: Check the package metadata - run: pixi run --locked -e dev twine check --strict dist/* + run: pixi run --locked -e release twine check --strict dist/* # --skip-existing makes a rerun on the same tag succeed instead of failing on the files already there. - name: Upload to the package registry env: @@ -62,5 +64,21 @@ jobs: echo "The secrets PACKAGE_USER and PACKAGE_TOKEN are not set; deploy/README.md says how." exit 1 fi - pixi run --locked -e dev twine upload --non-interactive --skip-existing dist/* + pixi run --locked -e release twine upload --non-interactive --skip-existing dist/* echo "Published to $REGISTRY: pip install --index-url $REGISTRY/simple/ paperspec" + - name: Create the release and attach the files + env: + TOKEN: ${{ secrets.RELEASE_TOKEN || secrets.GITHUB_TOKEN }} + run: | + # A release is a "## 0.1.0" or "### 0.1.1" heading in CHANGELOG.md; its notes run to the next heading. + notes="$(awk -v v="${GITHUB_REF_NAME#v}" ' + /^##+ / { if (found) exit; if (index($0, v)) found = 1; next } + found { print }' CHANGELOG.md)" + body="$(printf '%s\n' "${notes:-See CHANGELOG.md.}" | jq -Rs .)" + release_id="$(curl -fsS -X POST "$API/releases" \ + -H "Authorization: token $TOKEN" -H "Content-Type: application/json" \ + -d "{\"tag_name\": \"$GITHUB_REF_NAME\", \"name\": \"$GITHUB_REF_NAME\", \"body\": $body}" | jq .id)" + for f in dist/*; do + curl -fsS -X POST "$API/releases/$release_id/assets?name=$(basename "$f")" \ + -H "Authorization: token $TOKEN" -F "attachment=@$f" + done diff --git a/.ruff.toml b/.ruff.toml index c54b9f2..805612a 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -47,6 +47,7 @@ select = [ "PIE", # misc lints "INP", # implicit namespace packages "RUF", # ruff-specific + "CPY001", # every Python file opens with the license header (preview rule, so selected by code) ] ignore = [ "D105", # magic-method docstrings @@ -90,6 +91,11 @@ dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" "**/__init__.py" = ["F401"] # re-exports "src/paperspec/cli/*.py" = ["T201"] # paperspec-specific: the commands are a package, and they print by design +[lint.flake8-copyright] +# paperspec-specific: the code is Apache-2.0 (LICENSE.md), not Prosperity, so the header is an SPDX identifier. +notice-rgx = "SPDX-License-Identifier: Apache-2\\.0" +min-file-size = 0 + [lint.pycodestyle] # The formatter wraps code at 88; allow unbreakable strings/URLs up to 100. max-line-length = 100 diff --git a/CHANGELOG.md b/CHANGELOG.md index bd36a8b..502c5b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -135,3 +135,17 @@ It is a minor change in the sense of [spec/00](spec/00-introduction.md): one new - `fixtures/records/data/`, on the main example, with `vectors/data.jsonl` (35 cases): a valid case for each kind of data, annotations that break no rule and one or more that break each. The layer ties the caption of Table 1 to its dataset as force-field parameters for ethane, water and the CT–CT pattern, the caption of Figure 1a to the dataset read from it as an energy profile of ethane, and the sentence that introduces Table 1 to the same dataset as a model's proposal. The example project holds the layer, which changes `project.json`; no other fixture or vector changes. + +Tooling and code quality only; no computed value, vector, fixture, schema or SQL file changes. + +- The Python code meets the studio's standard: `ruff format`, `ruff check` with the full rule set, and `ty check` pass on `src/paperspec`, every function is annotated and documented, and `tests/` holds a pytest suite, driven by the vectors and fixtures, with branch coverage that fails under 90%. + `paperspec check` (with the byte-for-byte rebuild) and `paperspec stress` give the same results as before. +- Fixed: `structures.scan` raised an unrelated `ValueError` ("unsupported format character") for a malformed ring closure such as `C%1`, because its message was an invalid format string; the message is now "% needs two digits at N". +- Fixed: `paperspec site --out /` passed the check that the output folder does not contain the repository and went on to delete the folder it was given; it is now refused as a usage error. +- The pixi tasks follow the standard (`fmt`, `fmt-check`, `lint` for ruff, `typecheck`, `test`, `cov`, `hooks`); the Markdown lint is `lint-md`, `paperspec check` is also `verify`, and `check` runs them all. + The build environment is `release` (was `dev`), and `docs` builds the package documentation. +- The package documentation, for the Python package and its command, is built with Zensical from `docs/package/` and published at ; the format documentation stays in `docs/` and on . +- The repository lives in the Gitea organization `papers`: the repository and registry addresses change from `infra` to `papers`. +- The publish workflow also creates the Gitea release, with the files attached and the notes taken from this changelog; CI runs the studio's checks (`.gitea/workflows/ci.yml`) on `main`, `dev` and pull requests. +- Python files open with an `SPDX-License-Identifier: Apache-2.0` header, which ruff and a test check. +- The checker and the site builder skip `work/`, caches and build output. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fc01a41..6e741a3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,7 +1,7 @@ # Contributing This page is for anyone who wants to fix the specification, change a format or add an example. -Propose a change as a pull request on the repository at . +Propose a change as a pull request on the repository at . ## What a change needs @@ -16,10 +16,18 @@ The schemas, SQL, registries, vectors and fixtures then follow the chapter ([spe Change the oracle in [src/paperspec/](src/paperspec/) to apply the rule, and add the case that shows it to [src/paperspec/cli/build.py](src/paperspec/cli/build.py): a vector, a fixture or a broken package in `fixtures/invalid/`. Then run `pixi run build`, which rewrites `fixtures/` and `vectors/`. 4. **Run the checks.** - `pixi run check` must end with `All checks pass.` - Without pixi, install the checkout with `pip install -e ".[tools]"` on Python 3.14, then run `paperspec check`. + `pixi run check` must pass and end with `All checks pass.` + Without pixi, install the checkout with `pip install -e ".[tools]" --group dev` on Python 3.14, then run `paperspec check` and the commands in [How the checks run](#how-the-checks-run). 5. **Record the change** in [CHANGELOG.md](CHANGELOG.md). +## How the checks run + +The Python code follows the studio's standard: ruff formats and lints it with `.ruff.toml`, ty checks its types with `ty.toml`, and pytest tests it with `.pytest.ini` and `.coveragerc`. +`pixi run check` runs, in order, `fmt-check` (ruff format), `lint` (ruff check), `typecheck` (ty), `lint-md` (markdownlint), `test` (pytest) and `verify` (`paperspec check`, the repository's own checker). +`pixi run fmt` and `pixi run lint-fix` apply the formatter and the safe lint fixes, and `pixi run hooks` installs the git hooks that run the same tools before each commit. +The tests in `tests/` take their data from the vectors and fixtures, so a rule change that a vector or a fixture covers shows up in both the suite and `paperspec check`. +CI runs `fmt-check`, `lint`, `typecheck` and `cov` (the suite with branch coverage, which fails under 90%), then `lint-md`, `verify` and `stress`. + ## Versions A change that makes a valid file invalid, or that changes a computed value, is a breaking change and needs a new major version of the format, with a migration. @@ -40,7 +48,7 @@ A release that changes only the code takes the next patch version and records it Pushing the tag publishes the package to the Gitea package registry ([deploy/](deploy/README.md#publish-the-python-package)). Before tagging: -1. `pixi run check` passes on Linux and macOS, which the workflow in `.gitea/workflows/` runs on every push. +1. `pixi run check` passes, on macOS as well as on Linux, where the workflow `.gitea/workflows/ci.yml` runs the checks on every push to `main` and `dev` and on every pull request. The check also confirms that every count the prose states (formats, schemas, vectors, broken packages) matches the repository, and that `pixi.toml`, the registries and `src/paperspec/cli/build.py` state the same version. 2. `paperspec stress` passes with its default seed, and with one or two others. 3. The website is published from the release ([deploy/](deploy/README.md)), and the JSON-LD context at `https://paperspec.scient.ing/context/scienting-1.jsonld` and a schema at its `$id` both answer with their media types. @@ -51,7 +59,7 @@ Before tagging: ## Writing The checker enforces two rules on every Markdown file: one sentence per line, and US spelling. -`pixi run lint` runs markdownlint with the repository's settings. +`pixi run lint-md` runs markdownlint with the repository's settings. Define each term where it first appears, because the guides and READMEs are written for newcomers as well as implementers. All fixture text is synthetic, written for the fixtures. diff --git a/LICENSE.md b/LICENSE.md index 9399809..57ac570 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -4,10 +4,12 @@ | --- | --- | --- | | `spec/`, `docs/`, `README.md`, `CONTRIBUTING.md`, `CHANGELOG.md` and the other prose | Creative Commons Attribution 4.0 International | `CC-BY-4.0` | | `schemas/`, `sql/`, `context/`, `registry/`, `vectors/`, `fixtures/` | Creative Commons Zero 1.0 Universal | `CC0-1.0` | -| `src/` (the `paperspec` package and its commands), `tools/` | Apache License 2.0 | `Apache-2.0` | +| `src/` (the `paperspec` package and its commands), `tests/`, `tools/` | Apache License 2.0 | `Apache-2.0` | The legal texts are at , and . +Source Code: + The formats are open so that anyone may implement them, in any product, without asking. All text in the fixtures is synthetic, written for them. diff --git a/README.md b/README.md index fac9ddf..4d6be45 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This page is for three readers. A newcomer should read straight through; each term is explained where it first appears. The tables below point an implementer, who will write a reader, writer, converter or checker, to the chapters, test vectors and fixtures that define exact behavior. The sections on status and on differences give an expert in the neighboring standards the version, the maturity of each format and the comparison. -The documentation and every file of the specification are published at , and the repository is at . +The specification and every file of the formats are published at , the documentation of the Python package is at , and the repository is at . A research team usually cares about a few dozen papers it understands completely, not the world's literature. Those papers deserve better than a folder of PDFs. @@ -37,19 +37,6 @@ If a different converter reads the paper, each sentence it reads identically kee Nothing changes until a correction with its evidence is accepted ([spec/04](spec/04-identity.md), [spec/05 §5.6](spec/05-hashes.md#56-alignment)). The address grammar is in [spec/06 §6.1](spec/06-anchors.md#61-addresses), and [vectors/addresses.jsonl](vectors/addresses.jsonl) holds valid and invalid examples. -## Start here - -| If you want to | Read | -| --- | --- | -| Understand what the project is and how the parts fit | [docs/overview.md](docs/overview.md) | -| Learn the ideas one at a time, each with an example | [docs/concepts.md](docs/concepts.md) | -| Follow one paper from its first reading to a signed document | [docs/walkthrough.md](docs/walkthrough.md) | -| Run something in the next five minutes | [docs/examples.md](docs/examples.md) | -| Implement a reader or writer | [spec/](spec/README.md), then [vectors/](vectors/README.md) | -| Know why a rule is the way it is | [docs/rationale.md](docs/rationale.md) | -| Look up a term | [docs/glossary.md](docs/glossary.md) | -| Compare with neighboring standards | [docs/comparison.md](docs/comparison.md) | - ## What is in the repository | Folder | Holds | @@ -63,36 +50,103 @@ The address grammar is in [spec/06 §6.1](spec/06-anchors.md#61-addresses), and | [fixtures/](fixtures/README.md) | Example packages and records, deliberately broken packages, and the test keys that signed them. | | [tools/](tools/README.md) | How to run the commands that build and check everything above. | | [src/paperspec/](src/paperspec/) | The reference implementation of every rule and those commands, installable as the Python package `paperspec`. | -| [docs/](docs/README.md) | Guides for people getting to know the project. | -| [deploy/](deploy/README.md) | How the website at is built and served. | +| [tests/](tests/) | The pytest suite for the reference implementation, which takes its data from the vectors and fixtures. | +| [docs/](docs/README.md) | Guides to the formats for people getting to know the project, and the Python package's documentation in `docs/package/`. | +| [deploy/](deploy/README.md) | How the website at is built and served, and how the package is published. | -## Try it +## Installing -With [pixi](https://pixi.sh), one command sets up a pinned environment and checks everything: +The formats need no installation: they are files. +The Python package `paperspec` needs Python 3.14 and, for the `paperspec` command, the `tools` extra (`jsonschema`, `referencing` and `cryptography`). + +### From the studio's package registry + +The registry does not mirror PyPI, so an installer needs two indexes: the registry for `paperspec`, and PyPI for the packages it depends on. +Tie `paperspec` to the registry wherever the installer allows it, so that a package of the same name on PyPI is never chosen instead. +`[CHECK: no version tag has been pushed, so the registry holds no release yet]` + +With pixi, in `pixi.toml`: + +```toml +[pypi-dependencies] +paperspec = { version = ">=0.1.1", extras = ["tools"], index = "https://git.scient.ing/api/packages/papers/pypi/simple" } +``` + +With uv, in `pyproject.toml`: + +```toml +[[tool.uv.index]] +name = "scienting" +url = "https://git.scient.ing/api/packages/papers/pypi/simple" +explicit = true + +[tool.uv.sources] +paperspec = { index = "scienting" } +``` + +With pip, which cannot tie a package to an index and takes the highest version from either: + +```sh +pip install --index-url https://git.scient.ing/api/packages/papers/pypi/simple/ --extra-index-url https://pypi.org/simple/ "paperspec[tools]" +``` + +[deploy/](deploy/README.md#install-from-the-registry) says what a private registry needs. + +### From a checkout + +```sh +git clone https://git.scient.ing/papers/paperspec +pixi install +pixi run check +``` + +Without pixi, install the checkout as the package with `pip install -e ".[tools]" --group dev` on Python 3.14, then run `paperspec check`. + +## Quick start + +With [pixi](https://pixi.sh), one command sets up a pinned environment and runs every check: ```sh pixi run check ``` -The checker validates every schema, record and package. +It first checks the Python code: formatting and lint with ruff, types with ty, and the pytest suite. +Then it lints the Markdown. +Last it runs `paperspec check`, the repository's own checker, which you can also run alone with `pixi run verify`. +That checker validates every schema, record and package. It confirms that every broken package is refused for the stated reason. It recomputes every test vector, and it rebuilds everything and compares it byte for byte. -Last, it loads every package into SQLite and back, and checks the prose and the links. -Pixi lints the Markdown first. +It also loads every package into SQLite and back, and checks the prose and the links. The run ends with `All checks pass.` -`pixi task list` shows the other tasks: `quick`, `build`, `lint`, `load`, `validate` and `stress`. -`pixi run site` builds the website into `_site/` ([deploy/](deploy/README.md)). -The commands need Python 3.14 and the package's `tools` extra (`jsonschema`, `referencing` and `cryptography`), which pixi installs along with markdownlint for the Markdown. -Without pixi, install the checkout as the package with `pip install -e ".[tools]"` on Python 3.14, then run `paperspec check`. -`paperspec --help` lists the commands, and each command's `--help` gives advice and examples ([tools/](tools/README.md)). - -To explore a worked package, load the example paper (a synthetic chemistry paper with 12 corrections) into SQLite and query it; [docs/examples.md](docs/examples.md) has queries to start from. +To validate the example paper, a synthetic chemistry paper with 12 corrections, and load it into SQLite to query it: ```sh +pixi run validate fixtures/packages/hale2026harmonic-damb pixi run load --sqlite lab.db fixtures/packages/hale2026harmonic-damb ``` +[docs/examples.md](docs/examples.md) has queries to start from. +`paperspec --help` lists the commands, and each command's `--help` gives advice and examples ([tools/](tools/README.md)). +`pixi run site` builds the website into `_site/` ([deploy/](deploy/README.md)). + +## Documentation + +The documentation of the Python package and its command is at : how to install it, guides for validating a package, loading it into SQLite, using the library and checking the repository, the command line and the modules. +Its source is in `docs/package/`, and `pixi run docs-build` builds it. +The guides to the formats are at : + +| If you want to | Read | +| --- | --- | +| Understand what the project is and how the parts fit | [docs/overview.md](docs/overview.md) | +| Learn the ideas one at a time, each with an example | [docs/concepts.md](docs/concepts.md) | +| Follow one paper from its first reading to a signed document | [docs/walkthrough.md](docs/walkthrough.md) | +| Run something in the next five minutes | [docs/examples.md](docs/examples.md) | +| Implement a reader or writer | [spec/](spec/README.md), then [vectors/](vectors/README.md) | +| Know why a rule is the way it is | [docs/rationale.md](docs/rationale.md) | +| Look up a term | [docs/glossary.md](docs/glossary.md) | +| Compare with neighboring standards | [docs/comparison.md](docs/comparison.md) | + ## Format names Every format is named `scienting-/`, such as `scienting-doc/1` or `scienting-anchor/1`. @@ -101,19 +155,6 @@ A format is named after what it holds, never after a product or a repository, an Schemas use the identifier base `https://paperspec.scient.ing/schemas/`. [registry/formats.json](registry/formats.json) lists every format with its media type, schema, status and chapter. -## Status and limits - -This is version 0.1.0. -Of the 32 formats, 19 are `candidate` and 13 are `provisional`. -The `provisional` formats are project, bundle, dataset, digitization, view, snapshot, draft-ids, exercise, submission, grade, checklist, log-entry and checkpoint; where those are described, they are marked as such. -The repository includes a reference checker, test vectors and fixtures. - -The formats are built for a team-scale corpus of hundreds of documents, not hundreds of millions of papers. -They do not replace publisher interchange formats. -They do not fix what a converter reads wrongly; they record it and make it checkable. -Not every addressable thing is a node with five hashes (the content values `text_hash`, `match_hash`, `align_key`, `own_hash` and `node_hash`, explained in [docs/concepts.md](docs/concepts.md)). -Table columns and figure-text items have IDs and are not nodes ([spec/04 §4.2](spec/04-identity.md), [spec/09](spec/09-figures-tables-equations.md)). - ## How it differs from neighboring standards - JATS is the XML format in which publishers and archives exchange journal content ([NISO Z39.96-2024](https://www.niso.org/publications/z3996-2024-jats)). @@ -129,9 +170,64 @@ Table columns and figure-text items have IDs and are not nodes ([spec/04 §4.2]( It identifies items by position in a list, such as `#/texts/12` ([schema](https://raw.githubusercontent.com/docling-project/docling-core/main/docs/DoclingDocument.json)). Here, IDs do not depend on position, and a second converter's reading is aligned with the first and compared. +## Development + +The pixi tasks: + +| Task | Does | +| --- | --- | +| `fmt`, `fmt-check` | Format the code with ruff, or check the formatting. | +| `lint`, `lint-fix` | Lint with ruff, or lint and apply the safe fixes. | +| `typecheck` | Type-check `src/` with ty. | +| `test`, `cov` | Run the pytest suite, or run it with branch coverage that fails under 90%. | +| `check` | Every check CI runs, then the repository's own checker (`verify`). | +| `hooks` | Install the git pre-commit hooks. | +| `docs-build`, `docs-serve` | Build the package documentation strictly into `build/docs`, or preview it. | +| `dist` | Build the sdist and wheel into `dist/` (the `release` environment). | + +Work happens on the `dev` branch. +A release is a tag `vX.Y.Z` on the commit to release, and the package version is the tag, which setuptools-scm reads. +Add the release's section to [CHANGELOG.md](CHANGELOG.md), tag the commit and push the tag; CI checks the repository, builds the package, confirms the built version is the tag's, publishes it to the package registry and creates the release ([CONTRIBUTING.md](CONTRIBUTING.md#releasing), [deploy/](deploy/README.md#publish-the-python-package)). + +## Built with LLM assistance + +Large language models (Anthropic's Claude, through Claude Code) write much of this code, its tests and its documentation. +A person reviews every change, `pixi run check` gates it, and each assisted commit carries a `Co-Authored-By` trailer. + +## Status and known limits + +This is version 0.1.0. +Of the 32 formats, 19 are `candidate` and 13 are `provisional`. +The `provisional` formats are project, bundle, dataset, digitization, view, snapshot, draft-ids, exercise, submission, grade, checklist, log-entry and checkpoint; where those are described, they are marked as such. +The repository includes a reference checker, test vectors and fixtures. + +The formats are built for a team-scale corpus of hundreds of documents, not hundreds of millions of papers. +They do not replace publisher interchange formats. +They do not fix what a converter reads wrongly; they record it and make it checkable. +Not every addressable thing is a node with five hashes (the content values `text_hash`, `match_hash`, `align_key`, `own_hash` and `node_hash`, explained in [docs/concepts.md](docs/concepts.md)). +Table columns and figure-text items have IDs and are not nodes ([spec/04 §4.2](spec/04-identity.md), [spec/09](spec/09-figures-tables-equations.md)). + +The Python package is an oracle, not a production reader or writer: it favors clarity over speed, and its VRF and ML-DSA-65 are written out in plain Python ([tools/](tools/README.md#about-the-oracle)). +It needs Python 3.14, the only version with the Unicode 16.0 character data the specification pins, and refuses to import on any other. +Only `paperspec validate` and `paperspec load` work anywhere the package is installed; `check`, `build`, `stress` and `site` work on a checkout of this repository. + ## Licenses The specification and other prose are CC BY 4.0, so anyone may implement the formats. Schemas, SQL, the JSON-LD context, the registries, the vectors and the fixtures are CC0 1.0. -The code, the `paperspec` package in `src/` with its commands, is Apache-2.0. +The code, the `paperspec` package in `src/` with its commands and its tests, is Apache-2.0, and its Python files open with an SPDX header that says so. +This repository does not use the studio's Prosperity license, because the formats are meant to be implemented by anyone without asking. See [LICENSE.md](LICENSE.md). + +## Credits + +The reference implementation leaves the cryptography and the schema validation to established packages: [cryptography](https://cryptography.io) for Ed25519 and HKDF, and [jsonschema](https://python-jsonschema.dev) with [referencing](https://referencing.readthedocs.io) for JSON Schema. +Canonical JSON follows [RFC 8785](https://www.rfc-editor.org/rfc/rfc8785). +The verifiable random function follows [RFC 9381](https://www.rfc-editor.org/rfc/rfc9381), and ML-DSA-65 follows [FIPS 204](https://csrc.nist.gov/pubs/fips/204/final); both are checked against those standards' own test vectors. +The annotation layers build on the [W3C Web Annotation Data Model](https://www.w3.org/TR/annotation-model/). +[pixi](https://pixi.sh), [ruff](https://docs.astral.sh/ruff/), [ty](https://docs.astral.sh/ty/), [pytest](https://pytest.org) and [markdownlint-cli2](https://github.com/DavidAnson/markdownlint-cli2) run the checks, and [Zensical](https://zensical.org) builds the package documentation. + +## Contributing + +Propose a change as a pull request; [CONTRIBUTING.md](CONTRIBUTING.md) says what a change needs, from the chapter to the vectors. +`[CHECK: the studio's contributor license agreement does not exist yet; CONTRIBUTING.md invites pull requests as it did before]` diff --git a/deploy/README.md b/deploy/README.md index bd934a7..2fc6ee8 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -56,16 +56,16 @@ The environment in `.pixi/envs` is cached between runs under the hash of `pixi.l ## Publish the Python package The Python package `paperspec` holds the reference implementation and the `paperspec` command. -The workflow `.gitea/workflows/publish.yml` publishes it to the PyPI registry of the Gitea instance, `https://git.scient.ing/api/packages/infra/pypi`, when a version tag such as `v0.1.1` is pushed. -It checks the repository, builds the sdist and the wheel with `pixi run -e dev dist`, confirms that the version setuptools-scm computed is the tag's, checks the metadata with twine, and uploads both files. +The workflow `.gitea/workflows/publish.yml` publishes it to the PyPI registry of the Gitea instance, `https://git.scient.ing/api/packages/papers/pypi`, when a version tag such as `v0.1.1` is pushed. +It checks the repository with `pixi run check` (the code checks, the Markdown lint and `paperspec check`), builds the sdist and the wheel with `pixi run -e release dist`, confirms that the version setuptools-scm computed is the tag's, checks the metadata with twine, uploads both files, and creates the Gitea release with the files attached and the notes taken from the section of `CHANGELOG.md` whose heading contains the version. The package version is the version of this code; the formats keep the version that `CHANGELOG.md` and `pixi.toml` state. ### Set it up once -1. Pick the account the workflow publishes as, a bot account or your own, and make it a member of a team in the `infra` organization with write access to packages. +1. Pick the account the workflow publishes as, a bot account or your own, and make it a member of a team in the `papers` organization with write access to packages. 2. Signed in as that account, create an access token under Settings, Applications, with the `write:package` scope. 3. In this repository's Settings, Actions, Secrets, add `PACKAGE_USER` with the account's user name and `PACKAGE_TOKEN` with the token. - Organization secrets work as well, for every repository in `infra`. + Organization secrets work as well, for every repository in `papers`. Gitea refuses secret names that start with `GITEA_` or `GITHUB_`. 4. After the first upload, link the package to this repository in the package's settings, so it shows on the repository's Packages tab. @@ -91,7 +91,7 @@ With pixi, in `pixi.toml`: ```toml [pypi-dependencies] -paperspec = { version = ">=0.1.1", extras = ["tools"], index = "https://git.scient.ing/api/packages/infra/pypi/simple" } +paperspec = { version = ">=0.1.1", extras = ["tools"], index = "https://git.scient.ing/api/packages/papers/pypi/simple" } ``` With uv, in `pyproject.toml`: @@ -99,7 +99,7 @@ With uv, in `pyproject.toml`: ```toml [[tool.uv.index]] name = "scienting" -url = "https://git.scient.ing/api/packages/infra/pypi/simple" +url = "https://git.scient.ing/api/packages/papers/pypi/simple" explicit = true [tool.uv.sources] @@ -109,12 +109,20 @@ paperspec = { index = "scienting" } With pip, which cannot tie a package to an index and takes the highest version from either: ```sh -pip install --index-url https://git.scient.ing/api/packages/infra/pypi/simple/ --extra-index-url https://pypi.org/simple/ "paperspec[tools]" +pip install --index-url https://git.scient.ing/api/packages/papers/pypi/simple/ --extra-index-url https://pypi.org/simple/ "paperspec[tools]" ``` If the organization or the package is private, the installer also needs a token with the `read:package` scope, for example in a `~/.netrc` entry for `git.scient.ing`. A git pin to a commit, as `pixi.toml` and `pyproject.toml` can also express, keeps working and needs no registry. +## Publish the package documentation + +The pages for the Python package and its command are in `docs/package/` and are built with Zensical, separately from the format site above. +The workflow `.gitea/workflows/docs.yml` builds them strictly with `pixi run -e docs docs-build` into `build/docs` on every push to `main` and `dev`, and on `main` copies them to `/srv/www/docs/paperspec`, which serves . +A broken link, or a page missing from the navigation in `zensical.toml`, fails the build. +The copy is local, like the site's: the runner mounts the host's `/srv/www`. +Serving `/srv/www/docs` at `docs.scient.ing` is the same kind of Caddy entry as [Caddyfile](Caddyfile), and not part of this repository. + ## Distributing document packages A zipped package is an ordinary file, and the site serves the fixtures as such. diff --git a/docs/comparison.md b/docs/comparison.md index 2ac8c57..67a6cee 100644 --- a/docs/comparison.md +++ b/docs/comparison.md @@ -4,7 +4,7 @@ A newcomer, such as a student who knows what a scientific paper and a citation a An implementer, who will write a reader, writer, converter or checker, gets the chapter that defines each behavior and the places where the formats differ from a neighbor in ways that cause bugs. An expert, such as a researcher, standards author or reviewer who knows the neighboring standards, gets stated limits, the status of every format, and where the other standard is the better choice. -The repository is at . +The repository is at . This page was checked against the repository, at the version given under "Limits and status". Every statement about another standard links to the page it was read on, and those pages were read on 2026-10-12, so versions and dates are as of that day and may have moved. Where the checked sources say nothing, this page says "not seen", "not specified" or "not in scope", and each phrase keeps one meaning: "not in scope" means the standard's own stated purpose excludes the feature, "not seen" means it was not found in the pages read (which is not proof it is absent), and "not specified" means the page read does not define it. diff --git a/docs/concepts.md b/docs/concepts.md index 164bed3..af83063 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -17,7 +17,7 @@ The provisional ones are the project, bundle, dataset, digitization, view, snaps Format identifiers such as `scienting-anchor/1` and `scienting-correction/1` begin with `scienting-`, which is the formats' namespace. The repository's name, paperspec, is a separate matter, and the formats are named after what they hold. -The repository is at . +The repository is at . ## Hashes and canonical JSON diff --git a/docs/examples.md b/docs/examples.md index a20f87f..26cc195 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -7,7 +7,7 @@ The last sections point implementers (anyone writing a reader, writer, converter This page makes no comparison with the neighboring standards. The formats are version 0.1.0, and each is `candidate` or `provisional` in `registry/formats.json`. -Run every command from the root of the repository, . +Run every command from the root of the repository, . With [pixi](https://pixi.sh), `pixi run ` sets up the environment first. Every command also works without pixi once the checkout is installed with `pip install -e ".[tools]"` on Python 3.14, given standard command-line tools (the `sed` command in the last section is the GNU version of `sed`). @@ -19,10 +19,10 @@ This shows how to confirm that the repository's specification, fixtures and test pixi run check ``` -`pixi run check` lints the Markdown, then runs `paperspec check`, the repository's reference checker. +`pixi run check` checks the Python code and lints the Markdown, then runs `paperspec check`, the repository's reference checker. Each section prints how many checks it ran, and the last line is `All checks pass.` `pixi run quick`, or `paperspec check --quick`, skips the slowest part, which rebuilds everything and compares it byte for byte. -Without pixi, the Markdown lint is a separate step (`markdownlint-cli2`); `paperspec check` does not run it. +Without pixi, the code checks and the Markdown lint (`markdownlint-cli2`) are separate steps; `paperspec check` does not run them. ## Recompute a hash by hand diff --git a/docs/faq.md b/docs/faq.md index 89ce1de..f14409e 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -3,7 +3,7 @@ This page answers questions about paperspec, a set of file formats that give every sentence and table cell of a scientific paper a permanent address. Newcomers, implementers of a reader, writer, converter or checker, and experts who know JATS (Journal Article Tag Suite), TEI (Text Encoding Initiative) or W3C Web Annotations can all use it. Each answer gives the plain statement first, then the precise one, then the chapter that defines it. -The repository is , and the formats are at version 0.1.0. +The repository is , and the formats are at version 0.1.0. ## Can I read these files without special software? diff --git a/docs/glossary.md b/docs/glossary.md index 0cb8bad..64ec38b 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -1,6 +1,6 @@ # Glossary -A scientific paper changes after it is published: errata appear, readings get corrected, annotations pile up. paperspec is a set of file formats for scientific papers that keeps each sentence findable through those changes (repository: ). +A scientific paper changes after it is published: errata appear, readings get corrected, annotations pile up. paperspec is a set of file formats for scientific papers that keeps each sentence findable through those changes (repository: ). This glossary defines its terms for readers from newcomer to expert, implementers included. Each entry gives the plain meaning, then the precise one, then the chapter that defines it. diff --git a/docs/overview.md b/docs/overview.md index 1a2d92b..f355997 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -25,7 +25,7 @@ It cannot give you a signed statement that a named person checked a passage, IDs The formats supply those three, and they are worth most at the scale of one team. paperspec is a set of file formats for scientific papers that have been converted, split into addressable pieces and checked. -The repository is at . +The repository is at . It is version 0.1.0 and defines 32 formats; in [registry/formats.json](../registry/formats.json), 19 are `candidate` and 13 are `provisional`. ## Who it is for diff --git a/docs/package/about/changelog.md b/docs/package/about/changelog.md new file mode 100644 index 0000000..e4fcb3b --- /dev/null +++ b/docs/package/about/changelog.md @@ -0,0 +1,8 @@ +# Changelog + +The changelog is one file, `CHANGELOG.md` at the root of the repository, so there is a single copy to keep in sync. +It is published with the format documentation at , and the source is at . + +Its top-level headings are the versions of the formats, and a release of the Python package that changes only the code is a subsection under the current format version, as `### 0.1.1` sits under `## 0.1.0`. +The package version comes from the git tag; the format version is the one `CHANGELOG.md` and `pixi.toml` state. +When a release is tagged, the publish workflow takes the release notes from the section whose heading contains the version. diff --git a/docs/package/about/index.md b/docs/package/about/index.md new file mode 100644 index 0000000..c19c6f9 --- /dev/null +++ b/docs/package/about/index.md @@ -0,0 +1,36 @@ +# About + +## Status + +The formats are at version 0.1.0. +Of the 32 formats, 19 are `candidate` and 13 are `provisional`; `registry/formats.json` in the repository lists each with its media type, schema, status and chapter. +The Python package is at an alpha stage (the `Development Status :: 3 - Alpha` classifier). +Its version comes from git tags, so a tag `v0.1.1` builds as 0.1.1 and later commits as 0.1.2.devN. + +## Design commitments + +- **The code defines the expected values.** + No expected value in a vector or fixture is typed by hand: the oracle computes it, and `paperspec check` rebuilds everything and compares it byte for byte. +- **Standard library first.** + Every module imports with the standard library alone. + Schema validation and the Ed25519 and HKDF functions import `jsonschema`, `referencing` and `cryptography` on first use, through the package's extras. +- **Clarity over speed.** + The oracle is a slow, plain program. + The verifiable random function (RFC 9381) and ML-DSA-65 (FIPS 204) are written out in Python and checked against those standards' own test vectors. +- **One Python.** + The specification pins the Unicode 16.0 character data that normalization and the character classes come from, so the package refuses to import on a Python with another version. + Python 3.14 is the only one that matches. + +## Known limits and unfinished parts + +This documentation describes what the code does. +Where an older guide, a doc comment or the `--help` text says something different, the code wins, and this section lists the differences. + +- The package is an oracle, not a production reader or writer. + It has not been measured for speed or for large corpora; the formats are built for a team-scale corpus of hundreds of documents. +- Only `validate` and `load` work on an installed package. + `check`, `build`, `stress` and `site` need a checkout of the repository. +- `paperspec validate --correction` reports a proposals file that is not valid JSON with a Python traceback, not a one-line message. +- The Python API has no generated reference yet: [the API page](../reference/api.md) is written by hand from the code's docstrings. +- The package is not on the studio's registry until a version tag is pushed. `[CHECK: no tag exists yet]` +- Thirteen formats are `provisional` and may still change. diff --git a/docs/package/about/license.md b/docs/package/about/license.md new file mode 100644 index 0000000..dfedf73 --- /dev/null +++ b/docs/package/about/license.md @@ -0,0 +1,14 @@ +# License + +paperspec is not under the studio's usual license. +Because the formats are meant to be implemented by anyone, in any product, without asking, each path of the repository keeps the license that `LICENSE.md` gives it. + +| Paths | License | SPDX | +| --- | --- | --- | +| `spec/`, `docs/`, `README.md`, `CONTRIBUTING.md`, `CHANGELOG.md` and the other prose | Creative Commons Attribution 4.0 International | `CC-BY-4.0` | +| `schemas/`, `sql/`, `context/`, `registry/`, `vectors/`, `fixtures/` | Creative Commons Zero 1.0 Universal | `CC0-1.0` | +| `src/` (the `paperspec` package and its commands), `tests/`, `tools/` | Apache License 2.0 | `Apache-2.0` | + +Python source files open with an `SPDX-License-Identifier: Apache-2.0` header, which ruff checks. +All text in the fixtures is synthetic, and the keys in `fixtures/keys/` are public test keys that must never sign anything else. +The full text is in `LICENSE.md` at the root of the repository, . diff --git a/docs/package/about/llm-assistance.md b/docs/package/about/llm-assistance.md new file mode 100644 index 0000000..a68bf7a --- /dev/null +++ b/docs/package/about/llm-assistance.md @@ -0,0 +1,14 @@ +# LLM assistance + +Large language models (Anthropic's Claude, through Claude Code) write much of this code, its tests and its documentation, and we do not hide it. +The repository is built so that this is safe to rely on. + +- **A person reviews every change.** A model proposes a commit; it does not decide one. +- **Checks gate the code.** `pixi run check` runs the formatter, the linter with the full rule set, the type checker, the Markdown lint, the pytest suite and the repository's own checker, `paperspec check`. + CI runs the same, with branch coverage that fails under 90%, and the stress test. +- **The oracle checks itself.** Every expected value in the vectors and fixtures is computed by the code and rebuilt byte for byte, so a change that alters a computed value shows up as a changed file in the diff. +- **Assisted commits are marked.** Each carries a `Co-Authored-By` trailer naming the model, and a `Claude-Session` trailer linking the session. +- **Claude's working material stays out of the repository.** Plans, notes and review reports live in an untracked `work/` folder. + +The specification is the authority. +Where a model's code and a chapter disagree, the chapter wins and the code is the bug. diff --git a/docs/package/guides/check-and-build.md b/docs/package/guides/check-and-build.md new file mode 100644 index 0000000..caea120 --- /dev/null +++ b/docs/package/guides/check-and-build.md @@ -0,0 +1,51 @@ +# Check and rebuild the repository + +Four commands work on a checkout of the paperspec repository, not on an installed package: `check`, `build`, `stress` and `site`. +The checkout is the one an editable install points at, or else the one around the current folder. + +## Check everything + +```sh +paperspec check +``` + +The checker validates every schema, record and package, confirms that every broken package is refused for the stated reason, recomputes every test vector, rebuilds everything and compares it byte for byte, loads every package into SQLite and back, and checks the prose, the links and the website. +Each section prints how many checks it ran, and a passing run ends with `All checks pass.` +The exit status is 0 when every check passed and 1 otherwise. + +`paperspec check --quick` skips the byte-for-byte rebuild, the slowest part. +The command does not run the Python code checks or the Markdown lint; `pixi run check` runs those first ([Install](../start/install.md#the-pixi-tasks)). + +## Rebuild the vectors and fixtures + +No expected value in the repository is typed by hand. +The vectors and fixtures are computed from the hand-written sources in `fixtures/src/` and the cases the builder defines in `src/paperspec/cli/build.py`. +After you change a rule in the code, a fixture source or a case, rebuild and commit the regenerated files with the change: + +```sh +paperspec build +``` + +`paperspec build --out DIR` builds elsewhere, so you can compare with `diff -r DIR/vectors vectors` before overwriting anything. +The build is deterministic, and `paperspec check` proves it by rebuilding into a temporary folder and comparing every file. + +## Stress the oracle + +```sh +paperspec stress --rounds 5 +``` + +The vectors pin values; `stress` checks the invariants the chapters state, such as canonical JSON round trips and the zip reader never writing outside its folder, on random inputs from a seeded generator. +A failure prints its seed, so `paperspec stress --seed SEED` reproduces it. +The defaults are the seed 20261014 and 200 cases per property. +Before a release, run it with the default seed and with one or two others. + +## Build the website + +```sh +paperspec site --out /tmp/paperspec-site +``` + +The site mirrors the repository: each file is served at its own path, each Markdown file gets a rendered page beside it, and the schemas are at the addresses in their `$id`. +It needs only the standard library. +The pages you are reading are a separate site, built with Zensical by `pixi run -e docs docs-build`. diff --git a/docs/package/guides/index.md b/docs/package/guides/index.md new file mode 100644 index 0000000..576ae77 --- /dev/null +++ b/docs/package/guides/index.md @@ -0,0 +1,12 @@ +# Guides + +Each guide takes one task from start to finish, with commands that were run against the repository's synthetic fixtures. + +| Guide | For | +| --- | --- | +| [Validate a package](validate-a-package.md) | Check a package folder or a zipped package against every rule, and try a proposed correction without writing it. | +| [Load packages into SQLite](load-into-sqlite.md) | Put packages in a database you can query, or build a bundle with full-text search. | +| [Use the library](use-the-library.md) | Compute hashes and identifiers, read a package and run the gate from Python. | +| [Check and rebuild the repository](check-and-build.md) | Run the repository's checks, regenerate the vectors and fixtures, stress the oracle and build the website. | + +The formats themselves are explained in the guides at . diff --git a/docs/package/guides/load-into-sqlite.md b/docs/package/guides/load-into-sqlite.md new file mode 100644 index 0000000..bae02ee --- /dev/null +++ b/docs/package/guides/load-into-sqlite.md @@ -0,0 +1,46 @@ +# Load packages into SQLite + +`paperspec load` validates packages and loads them into a SQLite database with the tables of `sql/doc.sql` and `sql/store.sql`. +A node, a reference, a correction and a receipt each get rows you can join and query. + +## Load a package + +```sh +paperspec load --sqlite lab.db fixtures/packages/hale2026harmonic-damb +``` + +```text +hale2026harmonic-damb: 75 nodes +``` + +- Every package is validated first, and nothing is loaded if any fails. + Each failure is printed as `package: where: rule: message`, as `paperspec validate` prints it. +- A new file gets the tables, and an existing one takes more packages, so you can add papers to the same database later. +- Files with no table of their own (assets, datasets, derived files) are copied by SHA-256 into a folder beside the database, `lab.db.assets/`, which must travel with it. +- The full and checks-only forms of one document share its key, so load them into separate databases. + +## Query it + +```sh +python -m sqlite3 lab.db "SELECT kind, COUNT(*) FROM nodes GROUP BY kind ORDER BY 2 DESC, kind LIMIT 3" +``` + +```text +('cell', 23) +('sentence', 18) +('row', 6) +``` + +The guide at has queries to start from: the sentences still below fidelity F2, the citation graph at sentence level, and who signed what. + +## Build a bundle with full-text search + +```sh +paperspec load --sqlite search.db --bundle --built-from REV paper/ other/ +``` + +A bundle is a `scienting-bundle/1` file: it also gets the tables of `sql/bundle-sqlite.sql`, the full-text search index and `bundle_meta`. +It is built once, into a file that does not exist yet. +`--bundle` needs `--built-from`, which names the store revision the packages come from, and the command prints the number of search rows it added. +`--store` names the store, and defaults to the packages' common store. +`--built-from` and `--store` are refused without `--bundle`. diff --git a/docs/package/guides/use-the-library.md b/docs/package/guides/use-the-library.md new file mode 100644 index 0000000..a7eed21 --- /dev/null +++ b/docs/package/guides/use-the-library.md @@ -0,0 +1,66 @@ +# Use the library + +The modules of `paperspec` compute every value the formats define, so a program that writes or reads packages can import them instead of reimplementing the specification. +Every module imports with the standard library alone. +Schema validation needs `paperspec[schema]`, and Ed25519 and HKDF need `paperspec[crypto]`; `paperspec[tools]` installs both ([Install](../start/install.md)). + +## Canonical JSON and hashes + +`jcs.dumps` writes the canonical JSON the formats hash and store: sorted keys, no spaces, and numbers written as an ECMAScript engine would. +`text.sha256_hex` hashes a string, and `text.norm` applies the normalization the five hashes start from. + +```python +from paperspec import jcs, text + +print(jcs.dumps({"b": 1.0, "a": [2, "é"]})) +print(text.norm("a b\n c")) +print(text.sha256_hex("The two parameter sets agree to within 2% for every bond.")) +``` + +```text +{"a":[2,"é"],"b":1} +a b c +175bfc8867816a9761fd5bb9ffc956df6d9e7f203870961a5485fc1912cb663a +``` + +## Validate a package + +`package.validate` checks a package folder against every rule and returns a report whose `errors` list is empty when the package is valid. +The schema store reads the schemas from wherever the package carries them, in an installed wheel or at the root of a checkout. + +```python +from paperspec import package, resources +from paperspec.schema import Store + +store = Store(resources.path("schemas")) +report = package.validate("fixtures/packages/hale2026harmonic-damb", store) +print(bool(report), report.errors) +``` + +```text +True [] +``` + +Each entry of `errors` is a dictionary with `rule`, `where` and `message`, the three fields `paperspec validate` prints. + +## Read a package and run the gate + +`package.read_state` reads a package folder into a state that holds its nodes, and `corrections.gate` runs a proposed correction against it without changing it. +It returns the state the correction would produce and the record it would append, or raises `tree.Refusal`, which names the rule broken. + +```python +from paperspec import corrections, package, resources +from paperspec.schema import Store + +state = package.read_state("fixtures/packages/tiny2026tiny-j4qm") +store = Store(resources.path("schemas")) +new_state, record = corrections.gate(state, proposal, store) +``` + +Here `proposal` is a parsed correction such as the one in [Validate a package](validate-a-package.md). + +## Check your own code against the vectors + +The vectors travel with the package as the data subpackage `paperspec.vectors`, and `resources.path("vectors")` finds the folder in an installed wheel and in an editable install alike. +Four of them, `resolve.jsonl`, `grading.jsonl`, `targets.jsonl` and `traces.jsonl`, name a fixture package, and the fixtures stay in the repository. +The [reference](../reference/api.md) lists the modules. diff --git a/docs/package/guides/validate-a-package.md b/docs/package/guides/validate-a-package.md new file mode 100644 index 0000000..9fd925c --- /dev/null +++ b/docs/package/guides/validate-a-package.md @@ -0,0 +1,43 @@ +# Validate a package + +A package is a folder, or a zipped package (`.sct.zip`), that holds one document. +`paperspec validate` checks it against every rule of the specification's conformance chapter and works anywhere the package is installed with its `tools` extra. + +## Check one or several packages + +```sh +paperspec validate fixtures/packages/hale2026harmonic-damb +paperspec validate fixtures/packages/tiny2026tiny-j4qm.sct.zip fixtures/packages/tiny2026tiny-j4qm +``` + +Each valid package prints `PACKAGE: valid` and the exit status is 0. +A failure goes to standard error as `package: where: rule: message`, the exit status is 1, and nothing else prints. +A usage error, such as no package at all, exits with status 2. + +A zipped package is unpacked under the specification's path rules before anything else happens: a path that leaves the folder, a symbolic link and an oversize archive are refused. +`--max-size BYTES` sets the largest size an archive may expand to; the default is 1 GiB. + +## Read a refusal + +The rule ID names the rule that failed. +`https://paperspec.scient.ing/registry/rules.json` lists every ID with the chapter of the specification that states it, which is where to read why. +The folder `fixtures/invalid/` holds one package for each of many rules; its `expect.json` says which rules the package must trigger. + +## Try a correction before making it + +A correction changes one node of a document and records why. +Before a tool writes one, it can run the proposal through the gate, which needs only the package's current state: + +```sh +paperspec validate --correction fix.json fixtures/packages/tiny2026tiny-j4qm +``` + +`fix.json` holds one proposal as a JSON object, or several as JSON Lines, applied in order: + +```json +{"format":"scienting-correction/1","op":"emend","target":"j663d4h","find":"second","replace":"next","kind":"wording","status":"proposed","evidence":[{"kind":"person","by":"ana","note":"The editor prefers a plainer word."}],"author":{"key":"ana"},"reason":"The authors misuse the ordinal; the paragraph follows the first.","at":"2026-10-03T10:00:00Z"} +``` + +The command prints the record the correction would append to `corrections.jsonl`, with its computed `index`, `id`, `path`, `before` and `after`, and writes nothing. +A refused proposal prints `file:N node: rule: message` on standard error and stops: with this proposal, changing `second` to `2nd` is refused with `correction.digits`, because an emendation changes no digit. +`--correction` takes exactly one package, and it refuses a package that withholds its text. diff --git a/docs/package/index.md b/docs/package/index.md new file mode 100644 index 0000000..fb24a43 --- /dev/null +++ b/docs/package/index.md @@ -0,0 +1,22 @@ +# paperspec + +paperspec is the reference implementation of the scienting formats, the files in which a team stores its scientific papers once, corrects them once and addresses them down to the sentence. +This site documents the Python package `paperspec` and its command. +The formats themselves, with the specification, schemas, test vectors and fixtures, are published at . + +The package is an oracle: a plain program whose answers count as the expected ones. +Every hash, identifier, canonical JSON string, package check and database form in the repository is computed by this code, and no expected value in a test vector is typed by hand. +It favors clarity over speed, and it is not a production reader or writer. + +## Who it is for + +- **A developer writing a reader, writer, converter or store** installs the package to validate their output against every rule, and to compute the values their own code must match. +- **A researcher or editor with a package** uses the `paperspec` command to validate it, to try a proposed correction without writing it, and to load it into SQLite and query it. +- **A maintainer of the formats** uses the command to rebuild the vectors and fixtures, and to check the whole repository. + +## Where to go next + +- [Get started](start/index.md): install the package and run a first command. +- [Guides](guides/index.md): validate a package, load it into SQLite, use the library from Python, and check and rebuild the repository. +- [Reference](reference/index.md): every command and option, and the library's modules. +- [About](about/index.md): status, design commitments and known limits. diff --git a/docs/package/reference/api.md b/docs/package/reference/api.md new file mode 100644 index 0000000..0077851 --- /dev/null +++ b/docs/package/reference/api.md @@ -0,0 +1,70 @@ +# Python API + +The package `paperspec` is a set of small modules, one for each part of the specification, and each opens with a docstring that says what it is for. +This page is written by hand from those docstrings and lists the modules and the functions most programs call; `help(paperspec.MODULE)` has the rest. +The base package needs only the standard library. +`schema` needs `paperspec[schema]`, and the Ed25519 and HKDF functions of `crypto` need `paperspec[crypto]`; both import their packages on first use. + +Two aliases are shared by the type annotations: `jsontypes.Record`, one parsed JSON object, and `jsontypes.StrPath`, a path given as text or as a path object. +A function that takes a path returns text unless its docstring says otherwise. +A rule that is broken raises `tree.Refusal`, whose `rule` attribute names the rule as the specification's conformance chapter lists it. + +`paperspec.resources.path(name)` returns the folder `schemas`, `sql` or `vectors`, in an installed wheel and in a checkout alike. + +## Text, hashes and canonical JSON + +| Module | For | Main functions | +| --- | --- | --- | +| `text` | Normalization, IDs and small helpers shared by the rest. | `norm`, `sha256_hex`, `collapse_ws`, `letter_first`, `mint_node_id`, `record_id`, `cp_to_utf16` | +| `jcs` | Canonical JSON, RFC 8785. | `dumps`, `dumpb`, `loads_line` | +| `hashing` | The plain, strict and lossy projections of a node, and its five content values. | `compute`, `plain`, `strict`, `lossy`, `own_content`, `meta_hash`, `Hasher`, `Index` | +| `notation` | Simple math notation and its canonical linear form. | `fold_math`, `complex_key`, `fold_plain_text` | +| `keys` | Work and document keys. | `work_key`, `unique_work_key`, `document_key`, `split_key` | + +## The document + +| Module | For | Main functions | +| --- | --- | --- | +| `model` | The node tree of `scienting-doc/1`. | `Doc`, `anchor_class`, the tables of allowed children and attributes | +| `tree` | Establishing a document from a nested tree, and the structural checks. | `establish`, `mint_ids`, `validate_structure`, `validate_references`, `Refusal` | +| `anchors` | Addresses, anchors and their resolution. | `parse_address`, `format_address`, `make_anchor`, `resolve` | +| `align` | Alignment of two readings of one paper. | `align`, `classify`, `similarity` | +| `fingerprint` | Page fingerprints and download-stamp rules. | `fingerprint`, `is_stamp` | +| `targets` | The target set of targeted F3. | `target_set` | + +## Packages, corrections and the database + +| Module | For | Main functions | +| --- | --- | --- | +| `package` | Writing, reading and validating a package folder or a zipped package. | `validate`, `validate_zip`, `read_state`, `write`, `write_zip`, `read_zip`, `read_json`, `read_jsonl` | +| `corrections` | The correction gate and the state it works on. | `gate`, `apply`, `State` | +| `schema` | JSON Schema validation with the keyword subset the schemas are restricted to. | `Store`, `validate`, `lint` | +| `db` | The database form. | `create`, `create_bundle`, `load`, `unload`, `index` | + +## Signatures, receipts and the log + +| Module | For | Main functions | +| --- | --- | --- | +| `crypto` | Keyed hashing, Merkle roots and proofs, Ed25519, HKDF and `did:key`. | `merkle_root`, `inclusion_path`, `verify_inclusion`, `consistency_proof`, `verify_consistency`, `ed25519_sign`, `ed25519_verify`, `did_key` | +| `receipts` | Receipts, revocations, levels and spot-check sampling. | `seal`, `verify_signature`, `statuses`, `effective_levels`, `sample`, `upper_bound` | +| `log` | The team's transparency log and its signed checkpoints. | `sign_checkpoint`, `verify_checkpoint`, `write_log`, `read_log` | +| `vrf` | ECVRF-EDWARDS25519-SHA512-TAI, RFC 9381. | `prove`, `verify`, `proof_to_hash` | +| `mldsa` | ML-DSA-65, FIPS 204, in plain Python. | `keygen`, `sign`, `verify` | + +## Layers, rights and grading + +| Module | For | Main functions | +| --- | --- | --- | +| `rights` | Rights records, policies and stage checks. | `effective`, `combine`, `check`, `backend_allowed` | +| `traces` | Traced citations. | `resolve`, `problems`, `disputed` | +| `structures` | Chemical structures tied to words. | `scan`, `problems` | +| `sequences` | Biological sequences tied to words. | `residues`, `problems` | +| `data` | Data tied to words. | `problems` | +| `grade` | Grading a submission against an exercise key. | `grade`, `key_anchors` | + +The `problems` functions of the four annotation modules return the rules an annotation breaks, as a list of `{"rule", "message"}` records in the order checked. + +## The command + +`paperspec.cli` holds the subcommands of the `paperspec` command, one module each, with a `run(args)` function that returns the exit status; see [Command line](cli.md). +`cli.checkout()` finds the repository root that `check`, `build`, `stress` and `site` work on. diff --git a/docs/package/reference/cli.md b/docs/package/reference/cli.md new file mode 100644 index 0000000..b61a7f7 --- /dev/null +++ b/docs/package/reference/cli.md @@ -0,0 +1,82 @@ +# Command line + +One command, `paperspec`, with six subcommands. +`python -m paperspec` runs the same program. +`paperspec --version` prints the package version and the Unicode version it pins, and `paperspec COMMAND --help` prints a command's options, advice and examples. + +| Command | Works | Does | +| --- | --- | --- | +| [`validate`](#validate) | anywhere the package is installed with its `tools` extra | Validate packages, or run proposed corrections through the gate. | +| [`load`](#load) | anywhere the package is installed with its `tools` extra | Validate packages and load them into a SQLite database. | +| [`check`](#check) | in a checkout | Check the whole repository. | +| [`build`](#build) | in a checkout | Rebuild `fixtures/` and `vectors/` from their sources. | +| [`stress`](#stress) | in a checkout | Stress the oracle with random inputs. | +| [`site`](#site) | in a checkout | Build the website into `_site/`. | + +A misspelled command gets a suggestion, and an option a command does not know is refused. +"In a checkout" means the repository an editable install points at, or else the one around the current folder. + +## validate + +```text +paperspec validate [-h] [--correction PROPOSALS] [--max-size BYTES] PACKAGE [PACKAGE ...] +``` + +| Option | Meaning | +| --- | --- | +| `PACKAGE` | A package folder or a zipped package (`.sct.zip`), unpacked under the specification's path rules first. | +| `--correction PROPOSALS` | A JSON file holding one proposed correction, or a JSON Lines file of several, run through the gate against the one package given. Each record that would be appended to `corrections.jsonl` is printed, until one is refused. Nothing is written. | +| `--max-size BYTES` | Refuse a zipped package that expands past this many bytes (default 1073741824). | + +Each failure is printed to standard error as `package: where: rule: message`. +Exit status: 0 when everything is valid, 1 when something is refused, 2 for a usage error. + +## load + +```text +paperspec load [-h] --sqlite DATABASE [--bundle] [--built-from REV] [--store STORE] [--max-size BYTES] PACKAGE [PACKAGE ...] +``` + +| Option | Meaning | +| --- | --- | +| `PACKAGE` | A package folder or a zipped package. Every package is validated first, and nothing is loaded if any fails. | +| `--sqlite DATABASE` | The database file, created with the tables of `sql/` if it does not exist. Files with no table of their own are copied into `DATABASE.assets/`. | +| `--bundle` | Build a new `scienting-bundle/1` file with the full-text search index. Needs `--built-from`. | +| `--built-from REV` | With `--bundle`: the store revision the packages come from. | +| `--store STORE` | With `--bundle`: the store's name (default: the packages' common store). | +| `--max-size BYTES` | As for `validate`. | + +## check + +```text +paperspec check [-h] [--quick] +``` + +`--quick` skips the byte-for-byte rebuild. +Exit status: 0 when every check passed, 1 otherwise. + +## build + +```text +paperspec build [-h] [--out DIR] +``` + +Without `--out` it rewrites `fixtures/` and `vectors/` in the checkout; with it, it builds into `DIR` instead. + +## stress + +```text +paperspec stress [-h] [--seed SEED] [--rounds ROUNDS] +``` + +`--seed` sets the random seed (default 20261014) and `--rounds` the cases per property (default 200). +Exit status: 0 when every property held, 1 otherwise. + +## site + +```text +paperspec site [-h] [--out DIR] [--base-url URL] +``` + +`--out` sets the output folder, which must not be the repository or contain it (default `_site/` in the checkout). +`--base-url` sets the site's address for canonical links (default `https://paperspec.scient.ing`). diff --git a/docs/package/reference/index.md b/docs/package/reference/index.md new file mode 100644 index 0000000..a0ddcec --- /dev/null +++ b/docs/package/reference/index.md @@ -0,0 +1,6 @@ +# Reference + +- [Command line](cli.md): the `paperspec` command, its six subcommands and every option. +- [Python API](api.md): the modules of the package and what each is for. + +The formats the code implements are specified at , and the JSON Schemas, SQL and test vectors are at the same address. diff --git a/docs/package/start/index.md b/docs/package/start/index.md new file mode 100644 index 0000000..f15020f --- /dev/null +++ b/docs/package/start/index.md @@ -0,0 +1,9 @@ +# Get started + +Two pages take you from nothing to a result. + +1. [Install](install.md) the package from the studio's package registry, or work from a checkout. +2. [Quick start](quick-start.md) validates the example paper and loads it into a database. + +The package needs Python 3.14. +It is the only version with the Unicode 16.0 character data the specification pins, and the package refuses to import on any other. diff --git a/docs/package/start/install.md b/docs/package/start/install.md new file mode 100644 index 0000000..a55e6db --- /dev/null +++ b/docs/package/start/install.md @@ -0,0 +1,74 @@ +# Install + +The package needs Python 3.14. +Its base install has no dependencies: every module imports with the standard library alone. +Three extras add what some modules need: + +| Extra | Adds | Needed by | +| --- | --- | --- | +| `schema` | `jsonschema`, `referencing` | schema validation, so `package.validate` and `corrections.gate` | +| `crypto` | `cryptography` | Ed25519 signatures and HKDF in `crypto`, which the signature and receipt checks use | +| `tools` | both of the above | the `paperspec` command, and any program that validates packages | + +## From the studio's package registry + +The registry does not mirror PyPI, so an installer needs two indexes: the registry for `paperspec`, and PyPI for `jsonschema`, `referencing` and `cryptography`. +Tie `paperspec` to the registry wherever the installer allows it, so that a package of the same name on PyPI is never chosen instead. +`[CHECK: no version tag has been pushed yet, so the registry holds no release until the first tag is published]` + +With pixi, in `pixi.toml`: + +```toml +[pypi-dependencies] +paperspec = { version = ">=0.1.1", extras = ["tools"], index = "https://git.scient.ing/api/packages/papers/pypi/simple" } +``` + +With uv, in `pyproject.toml`: + +```toml +[[tool.uv.index]] +name = "scienting" +url = "https://git.scient.ing/api/packages/papers/pypi/simple" +explicit = true + +[tool.uv.sources] +paperspec = { index = "scienting" } +``` + +With pip, which cannot tie a package to an index and takes the highest version from either: + +```sh +pip install --index-url https://git.scient.ing/api/packages/papers/pypi/simple/ --extra-index-url https://pypi.org/simple/ "paperspec[tools]" +``` + +If the organization or the package is private, the installer also needs a token with the `read:package` scope, for example in a `~/.netrc` entry for `git.scient.ing`. + +## From a checkout + +Clone the repository and let pixi set up a pinned environment, with the package installed editable: + +```sh +git clone https://git.scient.ing/papers/paperspec +pixi install +pixi run check +``` + +`pixi run check` runs the formatter, the linter, the type checker, the Markdown lint, the tests and the repository's own checker, and ends with `All checks pass.` +Without pixi, install the checkout with `pip install -e ".[tools]" --group dev` on Python 3.14. + +## The pixi tasks + +| Task | Does | +| --- | --- | +| `check` | Every check CI runs, in order: `fmt-check`, `lint`, `typecheck`, `lint-md`, `test`, `verify`. | +| `verify` | `paperspec check`, the repository's own checker. | +| `quick` | `paperspec check --quick`, which skips the byte-for-byte rebuild. | +| `fmt`, `fmt-check` | Format the code with ruff, or check the formatting. | +| `lint`, `lint-fix` | Lint with ruff, or lint and apply the safe fixes. | +| `typecheck` | Type-check `src/` with ty. | +| `test`, `cov` | Run the pytest suite, or run it with branch coverage that fails under 90%. | +| `lint-md` | Lint every Markdown file with markdownlint. | +| `build`, `load`, `validate`, `stress`, `site` | The commands of the same names. | +| `hooks` | Install the git pre-commit hooks. | +| `docs-build`, `docs-serve` | Build these pages strictly into `build/docs`, or preview them. | +| `dist` | Build the sdist and wheel into `dist/`. | diff --git a/docs/package/start/quick-start.md b/docs/package/start/quick-start.md new file mode 100644 index 0000000..0ff866a --- /dev/null +++ b/docs/package/start/quick-start.md @@ -0,0 +1,59 @@ +# Quick start + +This page validates the repository's example paper, shows what a refusal looks like, and loads the paper into a database. +Run every command from the root of a checkout (see [Install](install.md)); the example is a synthetic chemistry paper in `fixtures/packages/hale2026harmonic-damb`. + +## Validate a package + +```sh +paperspec validate fixtures/packages/hale2026harmonic-damb +``` + +```text +fixtures/packages/hale2026harmonic-damb: valid +``` + +The exit status is 0. +A broken package prints one line per failure, as `package: where: rule: message`, and exits with status 1. +The folder `fixtures/invalid/` holds packages that each break one named rule: + +```sh +paperspec validate fixtures/invalid/text-edited/package +``` + +```text +fixtures/invalid/text-edited/package: s0fn2fj: doc.hash.text-hash: text_hash does not recompute +fixtures/invalid/text-edited/package: s0fn2fj: doc.hash.match-hash: match_hash does not recompute +… +``` + +Each rule ID, such as `doc.hash.text-hash`, is listed in with the chapter of the specification that states it. + +## Load it into SQLite + +```sh +paperspec load --sqlite lab.db fixtures/packages/hale2026harmonic-damb +``` + +```text +hale2026harmonic-damb: 75 nodes +``` + +The command validates the package first, creates `lab.db` with the tables of `sql/`, and loads the package. +Query the result with any SQLite client; Python's own works: + +```sh +python -m sqlite3 lab.db "SELECT kind, COUNT(*) FROM nodes GROUP BY kind ORDER BY 2 DESC, kind LIMIT 3" +``` + +```text +('cell', 23) +('sentence', 18) +('row', 6) +``` + +## Where next + +- [Validate a package](../guides/validate-a-package.md), including how to try a correction without writing it. +- [Load packages into SQLite](../guides/load-into-sqlite.md). +- [Use the library](../guides/use-the-library.md) from Python. diff --git a/docs/rationale.md b/docs/rationale.md index f971162..737ad47 100644 --- a/docs/rationale.md +++ b/docs/rationale.md @@ -10,7 +10,7 @@ Experts find the cost of each choice and, where a neighboring standard chose dif The formats are at version 0.1.0. Of the 29 in `registry/formats.json`, 19 are `candidate` and 13 are `provisional`, and the provisional ones are named where they appear. -The repository is paperspec, at . +The repository is paperspec, at . The `scienting-` prefix in format names such as `scienting-anchor/1` is the formats' namespace, separate from the repository's name. ## Identity diff --git a/docs/walkthrough.md b/docs/walkthrough.md index 641dd95..2dcc8bd 100644 --- a/docs/walkthrough.md +++ b/docs/walkthrough.md @@ -7,7 +7,7 @@ Its problems are modeled on the failures that the specification's rules address. Three readers can use the guide, and each gets something different. A newcomer knows what a paper and a citation are, not hashes or JSON, so every term is defined where it first appears. An implementer who will write a reader, writer, converter or checker gets the chapter and test vector for each step. -An expert gets the status: [paperspec](https://git.scient.ing/infra/paperspec) is at version 0.1.0, and each format is `candidate` or `provisional` in [`registry/formats.json`](../registry/formats.json) (19 candidate, 13 provisional). +An expert gets the status: [paperspec](https://git.scient.ing/papers/paperspec) is at version 0.1.0, and each format is `candidate` or `provisional` in [`registry/formats.json`](../registry/formats.json) (19 candidate, 13 provisional). The `scienting-` prefix on format names, as in `scienting-doc/1`, is the formats' namespace. It has nothing to do with the repository's name. diff --git a/fixtures/README.md b/fixtures/README.md index a5587d9..b030ba2 100644 --- a/fixtures/README.md +++ b/fixtures/README.md @@ -1,7 +1,7 @@ # Fixtures A reader, writer, converter or checker of stored papers needs files to test against, and a newcomer needs examples of what a stored paper looks like. -This folder holds that example data of paperspec (), version 0.1.0. +This folder holds that example data of paperspec (), version 0.1.0. All text is synthetic, written for these files and dedicated CC0; license fields inside the fixtures describe fictional works. Three readers use these files. diff --git a/pyproject.toml b/pyproject.toml index cf55f23..2afcc95 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,9 +43,11 @@ tools = ["paperspec[crypto,schema]"] paperspec = "paperspec.cli.main:main" [project.urls] -Homepage = "https://paperspec.scient.ing" -Repository = "https://git.scient.ing/infra/paperspec" -Documentation = "https://paperspec.scient.ing/docs/" +Homepage = "https://docs.scient.ing/paperspec/" +Documentation = "https://docs.scient.ing/paperspec/" +Specification = "https://paperspec.scient.ing" +Repository = "https://git.scient.ing/papers/paperspec" +Changelog = "https://git.scient.ing/papers/paperspec/src/branch/main/CHANGELOG.md" # PEP 735 dependency groups (pip >= 25.1: `pip install -e ".[tools]" --group dev`). # pixi.toml mirrors these for conda-forge environments. The tests need the `tools` extra as well. diff --git a/spec/README.md b/spec/README.md index d8ed0cd..c7386f6 100644 --- a/spec/README.md +++ b/spec/README.md @@ -2,7 +2,7 @@ This specification states exactly what each paperspec file holds and how every value in it is computed. The schemas, SQL, JSON-LD context, registries, vectors and fixtures are normative too, and each chapter says which of them it relies on. -A chapter and a vector that disagree signal a bug, so please report it in the repository, . +A chapter and a vector that disagree signal a bug, so please report it in the repository, . Two readers will get the most from it: implementers who build a reader, writer, converter or checker, and experts who want the precise rules. Newcomers should start with the [guides](../docs/README.md), which explain what the formats are for. diff --git a/tools/README.md b/tools/README.md index 25096ce..5cf82ba 100644 --- a/tools/README.md +++ b/tools/README.md @@ -32,7 +32,7 @@ The other four work on a checkout of this repository: the one an editable instal Each command is also a pixi task. Pixi sets up a pinned Python and the other packages the repository uses, and installs this checkout as the package, editable, so a change to the code takes effect at once. The commands `pixi run check`, `pixi run quick`, `pixi run build` and `pixi run load` then work from anywhere in the repository. -`pixi run check` lints the Markdown first. +`pixi run check` runs the code checks and the Markdown lint first, then `paperspec check` (also `pixi run verify`); [CONTRIBUTING.md](../CONTRIBUTING.md#how-the-checks-run) lists them. If you do not use pixi, install the checkout editable with its extras on Python 3.14, then run the commands from the repository root: @@ -81,6 +81,6 @@ The package needs Python 3.14, the only version with the Unicode 16.0 character Importing any of its modules needs only the standard library. Schema validation (`schema`, and with it `package.validate` and `corrections.gate`) needs `paperspec[schema]` (`jsonschema` and `referencing`), and HKDF and Ed25519 in `crypto`, which the signature and receipt checks use, need `paperspec[crypto]`. `paperspec[tools]` installs both, and is what a program that validates packages needs. -The package version comes from git tags through setuptools-scm, and `pixi run -e dev dist` builds the sdist and the wheel into `dist/`. +The package version comes from git tags through setuptools-scm, and `pixi run -e release dist` builds the sdist and the wheel into `dist/`. When `scienting-core` exists it takes this role behind the same vectors, and the import changes without any computed value changing. Independent implementations follow the [specification](../spec/README.md) and test themselves against the [vectors](../vectors/README.md). diff --git a/zensical.toml b/zensical.toml new file mode 100644 index 0000000..c55e8da --- /dev/null +++ b/zensical.toml @@ -0,0 +1,100 @@ +# The documentation site (Zensical): https://docs.scient.ing/paperspec/ +# `pixi run -e docs docs-build` builds it strictly into build/docs; `docs-serve` previews it. + +[project] +site_name = "paperspec" +site_description = "The Python package and the commands of paperspec, the reference implementation of the scienting formats" +site_url = "https://docs.scient.ing/paperspec/" +repo_url = "https://git.scient.ing/papers/paperspec" +repo_name = "papers/paperspec" +copyright = "Scientific Computing Studio, LLC" +# paperspec-specific: docs/ already holds the guides to the formats, which `paperspec site` renders on +# https://paperspec.scient.ing, so these pages for the Python package live in a subfolder of it. +docs_dir = "docs/package" +site_dir = "build/docs" +use_directory_urls = true + +# Every page appears here, so a page nobody links to fails the strict build. +nav = [ + { "Home" = "index.md" }, + { "Get started" = [ + "start/index.md", + "start/install.md", + "start/quick-start.md", + ] }, + { "Guides" = [ + "guides/index.md", + "guides/validate-a-package.md", + "guides/load-into-sqlite.md", + "guides/use-the-library.md", + "guides/check-and-build.md", + ] }, + { "Reference" = [ + "reference/index.md", + "reference/cli.md", + "reference/api.md", + ] }, + { "About" = [ + "about/index.md", + "about/changelog.md", + "about/llm-assistance.md", + "about/license.md", + ] }, +] + +[project.theme] +language = "en" +font = false # system fonts: no request to a font host +features = [ + "content.code.annotate", + "content.code.copy", + "content.tabs.link", + "content.tooltips", + "navigation.footer", + "navigation.indexes", + "navigation.instant", + "navigation.instant.prefetch", + "navigation.path", + "navigation.sections", + "navigation.tabs", + "navigation.top", + "navigation.tracking", + "search.highlight", + "toc.follow", +] + +[[project.theme.palette]] +media = "(prefers-color-scheme)" +toggle.icon = "lucide/sun-moon" +toggle.name = "Switch to light mode" + +[[project.theme.palette]] +media = "(prefers-color-scheme: light)" +scheme = "default" +toggle.icon = "lucide/sun" +toggle.name = "Switch to dark mode" + +[[project.theme.palette]] +media = "(prefers-color-scheme: dark)" +scheme = "slate" +toggle.icon = "lucide/moon" +toggle.name = "Switch to system preference" + +[project.markdown_extensions] +abbr = {} +admonition = {} +attr_list = {} +def_list = {} +footnotes = {} +md_in_html = {} +toc.permalink = true +pymdownx.betterem = {} +pymdownx.details = {} +pymdownx.highlight.anchor_linenums = true +pymdownx.highlight.line_spans = "__span" +pymdownx.highlight.pygments_lang_class = true +pymdownx.inlinehilite = {} +pymdownx.keys = {} +pymdownx.superfences = {} +pymdownx.tabbed.alternate_style = true +pymdownx.tasklist.custom_checkbox = true