diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index 7d34f8e..1ddd009 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -18,4 +18,9 @@ jobs: - name: Check the repository run: pixi run check - name: Stress the reference implementation +<<<<<<< HEAD run: pixi run stress +||||||| parent of 4fa326d (First implementation (#1)) +======= + run: pixi run python tools/stress.py +>>>>>>> 4fa326d (First implementation (#1)) diff --git a/.gitignore b/.gitignore index 580ba05..fc2e5a8 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ __pycache__/ *.db *.db.assets/ +<<<<<<< HEAD # the package version, written by setuptools-scm when the package is built or installed /src/paperspec/_version.py @@ -762,6 +763,765 @@ FodyWeavers.xsd # End of https://www.toptal.com/developers/gitignore/api/osx,python,pycharm,windows,visualstudio,visualstudiocode,zsh,jetbrains,jupyternotebooks # The Visual Studio rules above ignore every packages/ and log/ folder; these hold fixtures that `paperspec check` +||||||| parent of 4fa326d (First implementation (#1)) +======= +# the website, built by tools/site.py +/_site/ + +# pixi environments +.pixi/* +!.pixi/config.toml + + +# Created by https://www.toptal.com/developers/gitignore/api/osx,python,pycharm,windows,visualstudio,visualstudiocode,zsh,jetbrains,jupyternotebooks +# Edit at https://www.toptal.com/developers/gitignore?templates=osx,python,pycharm,windows,visualstudio,visualstudiocode,zsh,jetbrains,jupyternotebooks + +### JetBrains ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# SonarLint plugin +.idea/sonarlint/ + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### JetBrains Patch ### + +# Sonarlint plugin +# https://plugins.jetbrains.com/plugin/7973-sonarlint +.idea/**/sonarlint/ + +# SonarQube Plugin +# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin +.idea/**/sonarIssues.xml + +# Markdown Navigator plugin +# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced +.idea/**/markdown-navigator.xml +.idea/**/markdown-navigator-enh.xml +.idea/**/markdown-navigator/ + +# Cache file creation bug +# See https://youtrack.jetbrains.com/issue/JBR-2257 +.idea/$CACHE_FILE$ + +# CodeStream plugin +# https://plugins.jetbrains.com/plugin/12206-codestream +.idea/codestream.xml + +# Azure Toolkit for IntelliJ plugin +# https://plugins.jetbrains.com/plugin/8053-azure-toolkit-for-intellij +.idea/**/azureSettings.xml + +### JupyterNotebooks ### +# gitignore template for Jupyter Notebooks +# website: http://jupyter.org/ + +.ipynb_checkpoints +*/.ipynb_checkpoints/* + +# IPython +profile_default/ +ipython_config.py + +### OSX ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +.claude + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +### Python Patch ### +# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration +poetry.toml + +# ruff +.ruff_cache/ + +# LSP config files +pyrightconfig.json + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +!.vscode/*.code-snippets + +# Local History for Visual Studio Code +.history/ + +# Built Visual Studio Code Extensions +*.vsix + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history +.ionide + +### Windows ### +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +### Zsh ### +# Zsh compiled script + zrecompile backup +*.zwc +*.zwc.old + +# Zsh completion-optimization dumpfile +*zcompdump* + +# Zsh history +.zsh_history + +# Zsh sessions +.zsh_sessions + +# Zsh zcalc history +.zcalc_history + +# A popular plugin manager's files +._zinit +.zinit_lstupd + +# zdharma/zshelldoc tool's files +zsdoc/data + +# robbyrussell/oh-my-zsh/plugins/per-directory-history plugin's files +# (when set-up to store the history in the local directory) +.directory_history + +# MichaelAquilina/zsh-autoswitch-virtualenv plugin's files +# (for Zsh plugins using Python) + +# Zunit tests' output +/tests/_output/* +!/tests/_output/.gitkeep + +### VisualStudio ### +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. +## +## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore + +# User-specific files +*.rsuser +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Mono auto generated files +mono_crash.* + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +[Ww][Ii][Nn]32/ +[Aa][Rr][Mm]/ +[Aa][Rr][Mm]64/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ +[Ll]ogs/ + +# Visual Studio 2015/2017 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# Visual Studio 2017 auto generated files +Generated\ Files/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUnit +*.VisualState.xml +TestResult.xml +nunit-*.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# Benchmark Results +BenchmarkDotNet.Artifacts/ + +# .NET Core +project.lock.json +project.fragment.lock.json +artifacts/ + +# ASP.NET Scaffolding +ScaffoldingReadMe.txt + +# StyleCop +StyleCopReport.xml + +# Files built by Visual Studio +*_i.c +*_p.c +*_h.h +*.ilk +*.meta +*.obj +*.iobj +*.pch +*.pdb +*.ipdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*_wpftmp.csproj +*.tlog +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# Visual Studio Trace Files +*.e2e + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# AxoCover is a Code Coverage Tool +.axoCover/* +!.axoCover/settings.json + +# Coverlet is a free, cross platform Code Coverage Tool +coverage*.json +coverage*.xml +coverage*.info + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# Note: Comment the next line if you want to checkin your web deploy settings, +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# NuGet Symbol Packages +*.snupkg +# The packages folder can be ignored because of Package Restore +**/[Pp]ackages/* +# except build/, which is used as an MSBuild target. +!**/[Pp]ackages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/[Pp]ackages/repositories.config +# NuGet v3's project.json files produces more ignorable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt +*.appx +*.appxbundle +*.appxupload + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!?*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +orleans.codegen.cs + +# Including strong name files can present a security risk +# (https://github.com/github/gitignore/pull/2483#issue-259490424) +#*.snk + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm +ServiceFabricBackup/ +*.rptproj.bak + +# SQL Server files +*.mdf +*.ldf +*.ndf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings +*.rptproj.rsuser +*- [Bb]ackup.rdl +*- [Bb]ackup ([0-9]).rdl +*- [Bb]ackup ([0-9][0-9]).rdl + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat +node_modules/ + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio 6 auto-generated project file (contains which files were open etc.) +*.vbp + +# Visual Studio 6 workspace and project file (working project files containing files to include in project) +*.dsw +*.dsp + +# Visual Studio 6 technical files + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# CodeRush personal settings +.cr/personal + +# Python Tools for Visual Studio (PTVS) +*.pyc + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Tabs Studio +*.tss + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs + +# OpenCover UI analysis results +OpenCover/ + +# Azure Stream Analytics local run output +ASALocalRun/ + +# MSBuild Binary and Structured Log +*.binlog + +# NVidia Nsight GPU debugger configuration file +*.nvuser + +# MFractors (Xamarin productivity tool) working folder +.mfractor/ + +# Local History for Visual Studio +.localhistory/ + +# Visual Studio History (VSHistory) files +.vshistory/ + +# BeatPulse healthcheck temp database +healthchecksdb + +# Backup folder for Package Reference Convert tool in Visual Studio 2017 +MigrationBackup/ + +# Ionide (cross platform F# VS Code tools) working folder +.ionide/ + +# Fody - auto-generated XML schema +FodyWeavers.xsd + +# VS Code files for those working on multiple tools +*.code-workspace + +# Local History for Visual Studio Code + +# Windows Installer files from build outputs + +# JetBrains Rider +*.sln.iml + +### VisualStudio Patch ### +# Additional files built by Visual Studio + +# End of https://www.toptal.com/developers/gitignore/api/osx,python,pycharm,windows,visualstudio,visualstudiocode,zsh,jetbrains,jupyternotebooks + +# The Visual Studio rules above ignore every packages/ and log/ folder; these hold fixtures that tools/check.py +>>>>>>> 4fa326d (First implementation (#1)) # compares byte for byte, so keep them. !/fixtures/packages/* !/fixtures/records/log/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e2c3fd..8587581 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The first release. - Guides for newcomers: an overview, the concepts, a walkthrough of the example paper, runnable examples, the rationale, a FAQ and a glossary. - Tools that build and check everything, validate packages and zipped packages and run proposed corrections through the gate, load packages into SQLite or a bundle, and stress the reference implementation with random inputs. - A pixi workspace pinned to Python 3.14, with tasks for checking, building, linting, validating and loading. +<<<<<<< HEAD ### 0.1.1 @@ -35,3 +36,6 @@ Packaging, commands and lazy imports only; the formats stay at 0.1.0, and no com - `src/` is Apache-2.0, as `tools/` was, and the package metadata states `Apache-2.0 AND CC0-1.0` for the code and the data it carries. - The pixi workspace installs the package editable, and a `dev` environment builds the sdist and the wheel with `pixi run -e dev dist`. - Pushing a version tag publishes the package to the Gitea package registry through `.gitea/workflows/publish.yml` ([deploy/](deploy/README.md#publish-the-python-package)); a release tag versions the package, and [CONTRIBUTING.md](CONTRIBUTING.md#releasing) says how that relates to the format version. +||||||| parent of 4fa326d (First implementation (#1)) +======= +>>>>>>> 4fa326d (First implementation (#1)) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fc01a41..df5f5de 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,6 +13,7 @@ The schemas, SQL, registries, vectors and fixtures then follow the chapter ([spe 2. **Change the schemas, SQL and registries** that the rule shapes. A new refusal gets a rule ID in [registry/rules.json](registry/rules.json). 3. **Compute, never type, the expected values.** +<<<<<<< HEAD Change the oracle in [src/paperspec/](src/paperspec/) to apply the rule, and add the case that shows it to [src/paperspec/cli/build.py](src/paperspec/cli/build.py): a vector, a fixture or a broken package in `fixtures/invalid/`. Then run `pixi run build`, which rewrites `fixtures/` and `vectors/`. 4. **Run the checks.** @@ -60,3 +61,50 @@ The keys in `fixtures/keys/` are public test keys and must never sign anything e ## Licenses Each path keeps the license that [LICENSE.md](LICENSE.md) gives it: prose CC-BY-4.0, the normative data CC0-1.0, and the code in `src/` and `tools/` Apache-2.0. +||||||| parent of 4fa326d (First implementation (#1)) +======= + Change the oracle in [tools/oracle/](tools/oracle/) to apply the rule, and add the case that shows it to [tools/build.py](tools/build.py): a vector, a fixture or a broken package in `fixtures/invalid/`. + Then run `pixi run build`, which rewrites `fixtures/` and `vectors/`. +4. **Run the checks.** + `pixi run check` must end with `All checks pass.` + Without pixi, run `python3 tools/check.py`, with Python 3.14 and two packages, `jsonschema` and `cryptography` installed. +5. **Record the change** in [CHANGELOG.md](CHANGELOG.md). + +## Versions + +A change that makes a valid file invalid, or that changes a computed value, is a breaking change and needs a new major version of the format, with a migration. +Adding an optional property, a node kind, a run kind or an enumeration value is a minor change ([spec/00 §0.3](spec/00-introduction.md#03-format-identifiers-and-versions)). +A rebuild that changes any file under `vectors/` or any hash in `fixtures/` is a computed value changing, so say in the pull request why it must. + +## New node kinds and run kinds + +Propose a kind with the vector or fixture that shows it. +A node kind needs its row in [chapter 3](spec/03-nodes.md), the node schema, and its entry in the `CHILDREN` table of `tools/oracle/model.py`. +A run kind needs its row in [section 3.5](spec/03-nodes.md#35-inline-runs) and the node schema, and its text in the plain projection ([3.6](spec/03-nodes.md#36-the-plain-projection)) and in the strict and lossy projections ([5.3](spec/05-hashes.md#53-the-strict-projection), [5.5](spec/05-hashes.md#55-the-lossy-projection)). + +## Releasing + +A release is a tag on `main` whose version is the top heading of [CHANGELOG.md](CHANGELOG.md). +Before tagging: + +1. `pixi run check` passes on Linux and macOS, which the workflow in `.gitea/workflows/` runs on every push. + The check also confirms that every count the prose states (formats, schemas, vectors, broken packages) matches the repository, and that `pixi.toml`, the registries and `tools/build.py` state the same version. +2. `python3 tools/stress.py` passes with its default seed, and with one or two others. +3. The website is published from the release ([deploy/](deploy/README.md)), and the JSON-LD context at `https://paperspec.scient.ing/context/scienting-1.jsonld` and a schema at its `$id` both answer with their media types. + Layers claim to be W3C Web Annotations, and a JSON-LD processor that cannot fetch the context cannot read them. +4. The decisions that freeze on release are made: enumeration values and format identifiers never change meaning afterwards ([spec/22 §22.5](spec/22-conformance.md#225-stability)), so a name worth changing is changed now. +5. The CHANGELOG entry describes the release in the terms of the chapters it touches. + +## Writing + +The checker enforces two rules on every Markdown file: one sentence per line, and US spelling. +`pixi run lint` runs markdownlint with the repository's settings. +Define each term where it first appears, because the guides and READMEs are written for newcomers as well as implementers. + +All fixture text is synthetic, written for the fixtures. +The keys in `fixtures/keys/` are public test keys and must never sign anything else. + +## Licenses + +Each path keeps the license that [LICENSE.md](LICENSE.md) gives it: prose CC-BY-4.0, the normative data CC0-1.0, and `tools/` Apache-2.0. +>>>>>>> 4fa326d (First implementation (#1)) diff --git a/LICENSE.md b/LICENSE.md index 9399809..d9c2909 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -4,7 +4,12 @@ | --- | --- | --- | | `spec/`, `docs/`, `README.md`, `CONTRIBUTING.md`, `CHANGELOG.md` and the other prose | Creative Commons Attribution 4.0 International | `CC-BY-4.0` | | `schemas/`, `sql/`, `context/`, `registry/`, `vectors/`, `fixtures/` | Creative Commons Zero 1.0 Universal | `CC0-1.0` | +<<<<<<< HEAD | `src/` (the `paperspec` package and its commands), `tools/` | Apache License 2.0 | `Apache-2.0` | +||||||| parent of 4fa326d (First implementation (#1)) +======= +| `tools/` | Apache License 2.0 | `Apache-2.0` | +>>>>>>> 4fa326d (First implementation (#1)) The legal texts are at , and . diff --git a/README.md b/README.md index fac9ddf..664fd3f 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,7 @@ The address grammar is in [spec/06 §6.1](spec/06-anchors.md#61-addresses), and | [registry/](registry/README.md) | The list of formats, the rule IDs validators report, download-stamp rules, and the default rights each processing stage requires. | | [vectors/](vectors/README.md) | Test vectors: inputs and the exact outputs every implementation must compute. | | [fixtures/](fixtures/README.md) | Example packages and records, deliberately broken packages, and the test keys that signed them. | +<<<<<<< HEAD | [tools/](tools/README.md) | How to run the commands that build and check everything above. | | [src/paperspec/](src/paperspec/) | The reference implementation of every rule and those commands, installable as the Python package `paperspec`. | | [docs/](docs/README.md) | Guides for people getting to know the project. | @@ -86,6 +87,34 @@ The run ends with `All checks pass.` The commands need Python 3.14 and the package's `tools` extra (`jsonschema`, `referencing` and `cryptography`), which pixi installs along with markdownlint for the Markdown. Without pixi, install the checkout as the package with `pip install -e ".[tools]"` on Python 3.14, then run `paperspec check`. `paperspec --help` lists the commands, and each command's `--help` gives advice and examples ([tools/](tools/README.md)). +||||||| parent of 4fa326d (First implementation (#1)) +Version 0.1.0. +The formats of milestone M1 are `candidate`; datasets, digitizations, views, projects, bundles and the draft formats are `provisional` until their milestones; see [registry/formats.json](registry/formats.json). +======= +| [tools/](tools/README.md) | The programs that build and check everything above. | +| [docs/](docs/README.md) | Guides for people getting to know the project. | +| [deploy/](deploy/README.md) | How the website at is built and served. | + +## Try it + +With [pixi](https://pixi.sh), one command sets up a pinned environment and checks everything: + +```sh +pixi run check +``` + +The checker validates every schema, record and package. +It confirms that every broken package is refused for the stated reason. +It recomputes every test vector, and it rebuilds everything and compares it byte for byte. +Last, it loads every package into SQLite and back, and checks the prose and the links. +Pixi lints the Markdown first. +The run ends with `All checks pass.` +`pixi task list` shows the other tasks: `quick`, `build`, `lint`, `load`, `validate` and `stress`. +`pixi run python tools/site.py` builds the website into `_site/` ([deploy/](deploy/README.md)). + +The tools need Python 3.14 and two packages, `jsonschema` and `cryptography`, which pixi installs along with markdownlint for the Markdown. +Without pixi, install those two with pip and run `python3 tools/check.py`. +>>>>>>> 4fa326d (First implementation (#1)) To explore a worked package, load the example paper (a synthetic chemistry paper with 12 corrections) into SQLite and query it; [docs/examples.md](docs/examples.md) has queries to start from. @@ -133,5 +162,13 @@ Table columns and figure-text items have IDs and are not nodes ([spec/04 §4.2]( The specification and other prose are CC BY 4.0, so anyone may implement the formats. Schemas, SQL, the JSON-LD context, the registries, the vectors and the fixtures are CC0 1.0. +<<<<<<< HEAD The code, the `paperspec` package in `src/` with its commands, is Apache-2.0. +||||||| parent of 4fa326d (First implementation (#1)) +The chapters and other prose are licensed CC BY 4.0. +Schemas, SQL, the JSON-LD context, the registries, the vectors and the fixtures are dedicated to the public domain under CC0 1.0, so anyone may implement the formats. +The tools are licensed Apache-2.0. +======= +The tools are Apache-2.0. +>>>>>>> 4fa326d (First implementation (#1)) See [LICENSE.md](LICENSE.md). diff --git a/context/README.md b/context/README.md index e78c3dc..f067857 100644 --- a/context/README.md +++ b/context/README.md @@ -96,12 +96,21 @@ Three of the nine fixture annotations have only an `AnchorSelector` (the section - Reader: Follow [chapter 6](../spec/06-anchors.md) to resolve an anchor. The file `vectors/resolve.jsonl` resolves seventeen anchors against `fixtures/packages/hale2026harmonic-damb` and gives every outcome. - JSON-LD processor: Use a JSON-LD 1.1 processor; the context relies on `@json` and `@version` 1.1. +<<<<<<< HEAD - Checker: `schemas/annotation.schema.json` and `schemas/layer.schema.json` validate the files. `paperspec check` also checks that each annotation's ID derives from its content and that the receipt at `fixtures/records/layer/receipt.json` covers each `confirmed` annotation. The traps: - No test vector pins the context, and nothing here runs a JSON-LD processor on it; `paperspec check` and `paperspec site` read only its term IRIs. +||||||| parent of 4fa326d (First implementation (#1)) +======= +- Checker: `schemas/annotation.schema.json` and `schemas/layer.schema.json` validate the files. `tools/check.py` also checks that each annotation's ID derives from its content and that the receipt at `fixtures/records/layer/receipt.json` covers each `confirmed` annotation. + +The traps: + +- No test vector pins the context, and nothing in `tools/` reads `scienting-1.jsonld`. +>>>>>>> 4fa326d (First implementation (#1)) The schemas require only that `@context` be an array of at least two non-empty strings; they do not check the URLs. A file with a wrong context can pass the checker and still expand wrongly. - `@context` is part of an annotation's content hash (the hash covers everything except `status`, `receipt` and `modified`). diff --git a/deploy/Caddyfile b/deploy/Caddyfile index 467d427..7bbc525 100644 --- a/deploy/Caddyfile +++ b/deploy/Caddyfile @@ -1,4 +1,9 @@ +<<<<<<< HEAD # Serves the paperspec website built by `paperspec site` (`pixi run site`, the `_site/` folder), copied to /srv/www/paperspec. +||||||| parent of 4fa326d (First implementation (#1)) +======= +# Serves the paperspec website built by `python3 tools/site.py` (the `_site/` folder), copied to /srv/www/paperspec. +>>>>>>> 4fa326d (First implementation (#1)) # The site mirrors the repository, so every schema is served at its $id and the JSON-LD context at the address # layers name; this file sets the media types those addresses need and lets any page fetch them. paperspec.scient.ing { diff --git a/deploy/README.md b/deploy/README.md index bd934a7..5172942 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1,3 +1,4 @@ +<<<<<<< HEAD # Deploying the website and the Python package This folder holds what it takes to serve the paperspec website at . @@ -116,6 +117,64 @@ If the organization or the package is private, the installer also needs a token A git pin to a commit, as `pixi.toml` and `pyproject.toml` can also express, keeps working and needs no registry. ## Distributing document packages +||||||| parent of 4fa326d (First implementation (#1)) +======= +# Deploying the website + +This folder holds what it takes to serve the paperspec website at . +The site is static: `tools/site.py` builds it into `_site/`, and any web server can serve that folder. + +## What the site serves + +The site mirrors the repository, so a file at `schemas/anchor.schema.json` in the repository is served at `https://paperspec.scient.ing/schemas/anchor.schema.json`. +Three addresses that the formats themselves name depend on that: + +| Address | What is there | +| --- | --- | +| `https://paperspec.scient.ing/schemas/` | Each JSON Schema, at the address in its `$id`. | +| `https://paperspec.scient.ing/context/scienting-1.jsonld` | The JSON-LD context every layer and annotation lists. | +| `https://paperspec.scient.ing/vocab` | The vocabulary page, whose fragments are the IRIs of the context's terms. | + +Beside every Markdown file the builder writes a rendered page: `README.md` becomes its folder's `index.html`, and `X.md` becomes `X.html`. +A folder without a README gets a listing page, so every fixture and vector can be browsed. +The pages need no server-side code; search runs in the browser over a small index of headings, `search.json`. + +## Build it + +```sh +pixi run python tools/site.py # writes _site/ +pixi run python tools/site.py --out /tmp/paperspec-site +``` + +`pixi run check` builds the site into a temporary folder and checks that every Markdown file has a page, that every internal link and fragment resolves, that no Markdown is left unrendered, and that each schema's `$id`, the context's address and the vocabulary's IRIs are files of the site. + +To look at it locally, serve the folder with any static server, for example: + +```sh +pixi run python -m http.server --directory _site 8000 +``` + +## Serve it with Caddy + +[Caddyfile](Caddyfile) serves `/srv/www/paperspec` at `paperspec.scient.ing`. +It sets the media types the data needs (`application/ld+json` for the context, `application/schema+json` for schemas, `application/jsonl` for vectors and package files), allows every file to be fetched from any origin, and serves `404.html` for missing paths. +Add it to the server's Caddy configuration, for example with an `import` line, and reload Caddy. + +## Publish from CI + +The workflow `.gitea/workflows/site.yml` runs on every push to `main`. +It installs the environment from `pixi.lock`, runs `pixi run quick`, builds the site, and copies it with `rsync` to `/srv/www/paperspec`, the folder Caddy serves. +A failed check stops the publish, so the site never serves a release that does not pass its own checks. + +The copy is local: the `linux-base` runner starts each job in a container with the host's `/srv/www` mounted at the same path, so no key or secret is involved. +On a runner without that mount the workflow still builds and checks the site, and says that it did not publish. +`SITE_ROOT` and `SITE_SLUG` at the top of the workflow name the folder. + +Only changed files are copied, so every other file keeps its `Last-Modified` and `ETag` and caches stay valid across releases. +The environment in `.pixi/envs` is cached between runs under the hash of `pixi.lock`, so a fresh container installs it only when the lock file changes. + +## Distributing packages +>>>>>>> 4fa326d (First implementation (#1)) A zipped package is an ordinary file, and the site serves the fixtures as such. For a team's own packages, the Gitea instance speaks the OCI distribution protocol, so packages can be pushed to and pulled from its package registry as OCI artifacts with ORAS, and signed there with cosign; the media type of the artifact is the package's, `application/vnd.scienting.doc+zip`. diff --git a/docs/comparison.md b/docs/comparison.md index e103377..be80dc6 100644 --- a/docs/comparison.md +++ b/docs/comparison.md @@ -428,8 +428,14 @@ Keep using it as a reading and distribution format. *Status.* The specification is version 0.1.0. `registry/formats.json` lists 32 formats: 19 are `candidate` (complete, with vectors and fixtures, ready to implement) and 13 are `provisional` (complete; expected to change as implementations gain experience): project, bundle, dataset, digitization, view, snapshot, draft-ids, exercise, submission, grade, checklist, log-entry and checkpoint ([spec/00 §0.4](../spec/00-introduction.md#04-status)). +<<<<<<< HEAD The repository holds 24 chapters, 42 JSON Schemas, 25 vector files, five valid fixture packages, one of them also zipped, and 31 invalid ones, each naming the rule it breaks, and a reference checker, `paperspec check`. Licenses: prose CC BY 4.0; schemas, SQL, the JSON-LD context, registries, vectors and fixtures CC0 1.0; the code in `src/` Apache-2.0 ([README.md](../README.md#licenses)). +||||||| parent of 4fa326d (First implementation (#1)) +======= +The repository holds 24 chapters, 42 JSON Schemas, 25 vector files, five valid fixture packages, one of them also zipped, and 31 invalid ones, each naming the rule it breaks, and a reference checker, `tools/check.py`. +Licenses: prose CC BY 4.0; schemas, SQL, the JSON-LD context, registries, vectors and fixtures CC0 1.0; tools Apache-2.0 ([README.md](../README.md#licenses)). +>>>>>>> 4fa326d (First implementation (#1)) *No adoption claim.* The repository names no outside adopter and no independent implementation, and this page claims none. diff --git a/docs/examples.md b/docs/examples.md index dba488e..41e8c8b 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -9,6 +9,7 @@ The formats are version 0.1.0, and each is `candidate` or `provisional` in `regi Run every command from the root of the repository, . With [pixi](https://pixi.sh), `pixi run ` sets up the environment first. +<<<<<<< HEAD Every command also works without pixi once the checkout is installed with `pip install -e ".[tools]"` on Python 3.14, given standard command-line tools (the `sed` command in the last section is the GNU version of `sed`). ## Run the checks @@ -155,6 +156,155 @@ writer2027fixed-ynr7.v3: 15 nodes ``` The tables are those of [sql/doc.sql](../sql/doc.sql) and [sql/store.sql](../sql/store.sql), which run unchanged on Dolt and PostgreSQL as well ([chapter 13](../spec/13-database.md)); `paperspec check` exercises them on SQLite only, and this repository has not yet run them on Dolt or PostgreSQL. +||||||| parent of 4fa326d (First implementation (#1)) +======= +Every command also works without pixi, with Python 3.14 and two packages, `jsonschema` and `cryptography` installed and standard command-line tools (the `sed` command in the last section is the GNU version of `sed`). + +## Run the checks + +This shows how to confirm that the repository's specification, fixtures and test vectors agree with each other. + +```sh +pixi run check +``` + +`pixi run check` lints the Markdown, then runs `python tools/check.py`, the repository's reference checker. +Each section prints how many checks it ran, and the last line is `All checks pass.` +`pixi run quick`, or `python3 tools/check.py --quick`, skips the slowest part, which rebuilds everything and compares it byte for byte. +Without pixi, the Markdown lint is a separate step (`markdownlint-cli2`); `tools/check.py` does not run it. + +## Recompute a hash by hand + +This shows that a hash in a package is not magic: you can reproduce it with a standard tool. +A hash is a fixed-length fingerprint of some text; change one character and the fingerprint changes. +A sentence's `text_hash` is the SHA-256 of its text after two steps. +The text is converted to Unicode NFC (a standard form that writes each accented or composed character one way), and each run of whitespace is collapsed to one space, as [chapter 1](../spec/01-conventions.md#14-text-normalization-for-hashing) defines. +For a sentence with math or citations, the text is its plain projection ([chapter 5](../spec/05-hashes.md#51-the-five-values)). +For a sentence of plain text, the whole computation is a one-liner: + +```sh +printf '%s' 'The two parameter sets agree to within 2% for every bond.' | sha256sum +``` + +```text +175bfc8867816a9761fd5bb9ffc956df6d9e7f203870961a5485fc1912cb663a - +``` + +The same value is in the package, in the `nodes.jsonl` of `hale2026harmonic-damb`: + +```sh +grep -o '"id":"v3063qv".*' fixtures/packages/hale2026harmonic-damb/nodes.jsonl | grep -o '"text_hash":"[0-9a-f]*"' +``` + +```text +"text_hash":"175bfc8867816a9761fd5bb9ffc956df6d9e7f203870961a5485fc1912cb663a" +``` + +## Check a package's manifest + +This shows how a reader detects that a package's files have changed. +A package is a folder of JSON (JavaScript Object Notation, a plain-text format for structured data) files describing one paper. +`MANIFEST.json` lists every other file of a package with its SHA-256 and size, and the SHA-256 of the manifest itself pins the whole package ([chapter 12](../spec/12-package.md#123-the-manifest)). + +```sh +cd fixtures/packages/okafor2025solvent-acth +sha256sum nodes.jsonl +grep -o '"path":"nodes.jsonl","sha256":"[0-9a-f]*"' MANIFEST.json +``` + +```text +09c9b99eed97987e51dd0f85461ad9ad8768a0769ba7facfb7e0879d742660a3 nodes.jsonl +"path":"nodes.jsonl","sha256":"09c9b99eed97987e51dd0f85461ad9ad8768a0769ba7facfb7e0879d742660a3" +``` + +## Read a package with nothing but a JSON parser + +This shows that a package needs no special software to read. +JSON Lines is a text format with one JSON object per line, here one node (a section, sentence, figure, table or equation) per line of `nodes.jsonl`. +Save this script as `outline.py`. +It prints a document's sections, figures, tables and equations. + +```python +import json, sys + +pkg = sys.argv[1] +nodes = [json.loads(line) for line in open(pkg + "/nodes.jsonl", encoding="utf-8")] +by_id = {n["id"]: n for n in nodes} +children = {} +for n in nodes: + children.setdefault(n["parent"], []).append(n) + + +def text(n): + """The plain projection of a node with runs.""" + out = [] + for r in n["content"]: + if r["kind"] == "math": + out.append("$" + r["source"] + "$") + elif r["kind"] == "equation": + out.append("$$" + by_id[r["node"]]["content"]["source"] + "$$") + else: + out.append(r["text"]) + return "".join(out) + + +for n in nodes: + if n["kind"] == "section": + heading = children[n["id"]][0] + print(n["id"], n.get("label", ""), " ".join(text(s) for s in children[heading["id"]])) + elif n["kind"] in ("figure", "table", "equation"): + print(n["id"], " ", n["kind"], n.get("label", "")) +``` + +```sh +python3 outline.py fixtures/packages/hale2026harmonic-damb +``` + +```text +p36k8t6 1 Introduction +tkxsmts equation (1) +d400rz3 2 Methods +v7cx6qs figure Figure 1 +hj4s7pb table Table 1 +``` + +## Follow an address + +This shows how a citation points inside a sentence. +An address names a node, and optionally a span of its text ([chapter 6](../spec/06-anchors.md#61-addresses)). +`hale2026harmonic-damb#hca6dc5[29:44]` names code points 29 to 44 of sentence `hca6dc5`. +With `text()` from the script above: + +```python +print(text(by_id["hca6dc5"])[29:44]) +``` + +```text +first test case +``` + +Offsets count Unicode code points, one per character; in Python, string indexes already do. +JavaScript strings index UTF-16 units, so convert first. +For text with characters outside the Basic Multilingual Plane the two counts differ, and [vectors/offsets.jsonl](../vectors/offsets.jsonl) gives both. + +## Load packages into SQLite + +This shows that the same packages can live in a database and be queried with SQL. + +```sh +pixi run load --sqlite lab.db fixtures/packages/hale2026harmonic-damb fixtures/packages/okafor2025solvent-acth fixtures/packages/writer2027fixed-ynr7.v3 +``` + +Without pixi, run `python3 tools/load.py` with the same arguments. + +```text +hale2026harmonic-damb: 75 nodes +okafor2025solvent-acth: 15 nodes +writer2027fixed-ynr7.v3: 15 nodes +``` + +The tables are those of [sql/doc.sql](../sql/doc.sql) and [sql/store.sql](../sql/store.sql), which run unchanged on Dolt and PostgreSQL as well ([chapter 13](../spec/13-database.md)); `tools/check.py` exercises them on SQLite only, and this repository has not yet run them on Dolt or PostgreSQL. +>>>>>>> 4fa326d (First implementation (#1)) Here are four queries, in Python's `sqlite3` so that nothing else is needed. ```python diff --git a/docs/faq.md b/docs/faq.md index c433c2a..98b1c32 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -107,6 +107,7 @@ A `candidate` format is complete, with vectors and fixtures, and ready to implem A `provisional` one is complete and expected to change as implementations gain experience. The provisional formats are dataset, digitization, view, project, bundle, snapshot, draft-ids, exercise, submission, grade, checklist, log-entry and checkpoint. +<<<<<<< HEAD The repository holds a reference checker (`paperspec check`), 25 vector files, five valid packages, one of them also as a zip, and 31 deliberately broken packages ([spec/00 §0.4](../spec/00-introduction.md#04-status), [spec/22](../spec/22-conformance.md)). ## What does it not do? @@ -130,3 +131,28 @@ See [CONTRIBUTING.md](../CONTRIBUTING.md). `paperspec check` recomputes and checks every one, and `paperspec load` loads packages into SQLite. The oracle's modules in `src/paperspec/` compute every expected value, so that none is typed by hand. The other repositories of the project install them as the Python package `paperspec`; independent implementations follow the specification ([tools/README.md](../tools/README.md)). +||||||| parent of 4fa326d (First implementation (#1)) +======= +The repository holds a reference checker (`tools/check.py`), 25 vector files, five valid packages, one of them also as a zip, and 31 deliberately broken packages ([spec/00 §0.4](../spec/00-introduction.md#04-status), [spec/22](../spec/22-conformance.md)). + +## What does it not do? + +It is built for a team-scale corpus of hundreds of documents, not hundreds of millions of papers. +It does not replace publisher interchange formats; it reads JATS as a witness. +It does not fix what a converter reads wrongly. +It records the reading and raises disagreements as suspects, and a correction is checkable. +It is not a search engine, a citation index or a recommendation feed ([docs/overview.md](overview.md)). + +## How do I add a node kind or a run kind? + +A node kind is a type of document part, such as `sentence`; a run kind is a type of inline content within a sentence, such as `math`. +Propose either with the vector or fixture that shows it. +A new kind is a minor version ([spec/00 §0.3](../spec/00-introduction.md#03-format-identifiers-and-versions)). +See [CONTRIBUTING.md](../CONTRIBUTING.md). + +## What are the files in `tools/` for? + +`build.py` builds the fixtures and vectors. `check.py` recomputes and checks every one, and `load.py` loads packages into SQLite. +The `oracle/` modules compute every expected value, so that none is typed by hand. +They are not a library; implementations follow the specification ([tools/README.md](../tools/README.md)). +>>>>>>> 4fa326d (First implementation (#1)) diff --git a/docs/overview.md b/docs/overview.md index abdba08..e0369e9 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -54,6 +54,7 @@ It appears inside hashed data and test vectors, and it is separate from the repo | [spec/](../spec/README.md) | The specification, in 24 chapters (00 to 23). Normative. | | [schemas/](../schemas/README.md) | 42 JSON (plain-text data format) Schemas for the files and records. | | [registry/](../registry/README.md) | The list of formats, the rule IDs a checker reports, download-stamp rules, and default rights requirements. | +<<<<<<< HEAD | [tools/](../tools/README.md) | How to run the reference commands: `paperspec check` checks everything, `paperspec build` rebuilds the fixtures and vectors, and `paperspec load` loads packages into SQLite. | | [src/paperspec/](../src/paperspec/) | The oracle, a compact implementation of every rule, which the other repositories of the project install as the Python package `paperspec`. | | [vectors/](../vectors/README.md) | 25 files of test vectors: inputs and the exact outputs every implementation must compute. | @@ -75,6 +76,29 @@ The repository ships reference tools only for the checker and store roles. | Converter | Turns a witness (a PDF, JATS XML, the article format publishers use, a LaTeX source archive, an HTML page, an EPUB or a Typst project) into a package and raises doubtful readings as suspects (flagged disagreements awaiting review). | | Checker | Recomputes hashes, validates files against their schemas and reports broken rules by rule ID. `paperspec check` does this for the repository. | | Store | Keeps packages and records in a database a team can clone. `paperspec load` loads packages into SQLite. | +||||||| parent of 4fa326d (First implementation (#1)) +======= +| [tools/](../tools/README.md) | Reference programs: `check.py` checks everything, `build.py` rebuilds the fixtures and vectors, `load.py` loads packages into SQLite, and `oracle/` is a compact implementation of every rule. | +| [vectors/](../vectors/README.md) | 25 files of test vectors: inputs and the exact outputs every implementation must compute. | +| [fixtures/](../fixtures/README.md) | Five valid example packages, one of them also as a zip, 31 deliberately broken ones that each break one named rule, example records, and the test keys that signed them. All text is synthetic. | +| [sql/](../sql/README.md) | Tables for SQLite, Dolt and PostgreSQL, and a SQLite bundle with full-text search. | +| [context/](../context/README.md) | The JSON-LD context that makes annotation layers W3C Web Annotations. | +| [docs/](README.md) | Guides, including this page. | + +## Roles + +The formats serve five roles. +A role is a job someone could implement. +The repository ships reference tools only for the checker and store roles. + +| Role | Does | +| --- | --- | +| Reader | Opens a package, resolves addresses and anchors, and shows text, annotations and each passage's fidelity level. | +| Writer | Produces files in these formats: nodes, corrections, receipts, annotations, datasets. | +| Converter | Turns a witness (a PDF, JATS XML, the article format publishers use, a LaTeX source archive, an HTML page, an EPUB or a Typst project) into a package and raises doubtful readings as suspects (flagged disagreements awaiting review). | +| Checker | Recomputes hashes, validates files against their schemas and reports broken rules by rule ID. `tools/check.py` does this for the repository. | +| Store | Keeps packages and records in a database a team can clone. `tools/load.py` loads packages into SQLite. | +>>>>>>> 4fa326d (First implementation (#1)) The [specification's reading paths](../spec/README.md) say which chapters each role needs. diff --git a/docs/rationale.md b/docs/rationale.md index f971162..5ff6a99 100644 --- a/docs/rationale.md +++ b/docs/rationale.md @@ -112,7 +112,12 @@ Only the three text values, `text_hash`, `match_hash` and `align_key`, normalize ### The character data is pinned to one Unicode version Normalization and General Categories change between Unicode versions, and both enter every hash. +<<<<<<< HEAD The chapters pin Unicode 16.0, the reference implementation refuses to import on another version, and moving to a later one is a change to every text-hashing format ([spec/01 §1.4](../spec/01-conventions.md#14-text-normalization-for-hashing)). +||||||| parent of 4fa326d (First implementation (#1)) +======= +The chapters pin Unicode 16.0, the reference tools refuse to run on another version, and moving to a later one is a change to every text-hashing format ([spec/01 §1.4](../spec/01-conventions.md#14-text-normalization-for-hashing)). +>>>>>>> 4fa326d (First implementation (#1)) ### Hashes that leave the team are keyed diff --git a/docs/walkthrough.md b/docs/walkthrough.md index 5c17d1a..28dcd5d 100644 --- a/docs/walkthrough.md +++ b/docs/walkthrough.md @@ -13,6 +13,7 @@ It has nothing to do with the repository's name. Open [fixtures/packages/hale2026harmonic-damb/](../fixtures/packages/hale2026harmonic-damb/) beside this page, because every ID below is in it. A package is a folder of files, mostly JSON Lines (`.jsonl`: one JSON record per line). +<<<<<<< HEAD The hand-written source of the first reading is [fixtures/src/hale2026harmonic.json](../fixtures/src/hale2026harmonic.json); no converter ran on it, and [`paperspec build`](../src/paperspec/cli/build.py) expands it into the package. Each step says what a converter, a person or a checker does. @@ -167,6 +168,163 @@ Its CSV keeps every digit as printed (`1.526` and `1.5260` stay different), and The descriptor types the fields as numbers; how a generic Data Package tool treats `1.5260` was not verified. All twelve value anchors resolve as `current`. The reference checker, [`paperspec check`](../src/paperspec/cli/check.py), confirms each CSV value equals its cell's text. +||||||| parent of 4fa326d (First implementation (#1)) +======= +The hand-written source of the first reading is [fixtures/src/hale2026harmonic.json](../fixtures/src/hale2026harmonic.json); no converter ran on it, and [`tools/build.py`](../tools/build.py) expands it into the package. +Each step says what a converter, a person or a checker does. + +## 1. A PDF arrives + +The team downloads the publisher's PDF. +The file becomes a witness, identified by its SHA-256 (a 64-digit hash that changes if any byte changes), with the short ID `qptsanv` ([spec/02](../spec/02-model.md)). +A person records its rights, meaning what the team may do with the file ([spec/14](../spec/14-rights.md)). +The library's subscription allows reading, text mining, and AI processing on the team's own machines for internal use, until August 31, 2027. +Whether it allows processing in the team's cloud is unknown. + +The publisher stamped the download in the margin with the institution and the time, so two downloads rarely share a SHA-256. +A page fingerprint avoids the problem of downloads that differ in their stamps. +It is a hash of a page's text, size and images after the stamp lines are removed ([spec/07](../spec/07-geometry.md)), and the rules `downloaded-via` and `sharing-guidelines` fired on both pages (see [`pages.jsonl`](../fixtures/packages/hale2026harmonic-damb/pages.jsonl)). +Page 101's fingerprint is `6272e6f3…` in this copy, stamped October 11, 2026, and in a second download from another institution, stamped March 2, 2027 ([vectors/fingerprints.jsonl](../vectors/fingerprints.jsonl)). + +## 2. The first reading, with its mistakes + +A converter reads the PDF into sections, paragraphs, sentences, a figure, a table, an equation, a footnote and references. +Each of these is a node, and each node gets a permanent ID: seven characters minted once from the document key, the node's kind, a hash of its text and an occurrence count, then frozen ([spec/04](../spec/04-identity.md)). +An ID encodes no page, order or kind, so it keeps naming the node after the text is corrected. + +Like every reading, this one has mistakes: + +| What the reading did | What the page says | +| --- | --- | +| A cell read `3l0.0`. | `310.0`, with the digit 1. | +| A sentence read `Values below 10-3 kcal mol⁻¹…`. | A superscript: 10⁻³. | +| Two sentences ran together: "…treated as zero. Table 1 lists…". | Two sentences. | +| One sentence was cut in two at a line break: "…and the energies were" and "fitted by least squares." | One sentence. | +| A paragraph was split at a column break. | One paragraph. | +| The running head "J. Example Chem. 2026, 14, 101–109" became a paragraph. | A running head, not text. | +| The equation's number read `(l)` and the figure's `Figure l`. | `(1)` and `Figure 1`. | +| The CT–HC row merged two cells into "338.7 1.0900". | Two cells. | +| The OW–HW row was missing. | It is there. | +| "Set B" spanned one column. | It spans two. | + +The converter does not guess. +Each doubtful place becomes a suspect, with a crop (an image cut from the page) for a person to answer ([spec/05 §5.7](../spec/05-hashes.md)). +Unresolved suspects travel in `suspects.jsonl`, which this package does not need. +Nothing changes until a correction passes the gate ([spec/11](../spec/11-corrections.md)). + +## 3. Twelve corrections + +Each fix is appended to [`corrections.jsonl`](../fixtures/packages/hale2026harmonic-damb/corrections.jsonl), numbered, with its evidence (a crop, or a person who read the page), its author (`ana` throughout) and its reason. + +| # | Operation | On | What it did | +| --- | --- | --- | --- | +| 1 | `replace` | cell `c20cna6` | `3l0.0` to `310.0`. A digit changed, so the gate required a crop, and it has one. | +| 2 | `replace` | sentence `ec5jc93` | `10-3` to the runs `10` and superscript `−3`. | +| 3 | `split` | sentence `ec5jc93` | Split before "Table 1"; the first part keeps `ec5jc93`, the second is new: `a4jrz2t`. | +| 4 | `merge` | sentence `xjns57z` | Joined with `ayea71r`, which is retired with `xjns57z` as its successor. | +| 5 | `merge` | paragraph `gwzbj3s` | Joined with `ncx2v4f`, retired the same way. | +| 6 | `delete` | paragraph `v8e4z6z` | The running head is gone; its IDs are retired with no successor. | +| 7 | `relabel` | equation `tkxsmts` | `(l)` to `(1)`. | +| 8 | `delete` | row `rncchma` | The garbled CT–HC row and its four cells. | +| 9 | `insert` | row `kx2kz2y` | The CT–HC row, read again: five new cells. | +| 10 | `insert` | row `ke0jxpn` | The missing OW–HW row. | +| 11 | `set` | cell `xyy7x6c` | Set B's `colspan` from 1 to 2. | +| 12 | `relabel` | figure `v7cx6qs` | `Figure l` to `Figure 1`. | + +The document is now at revision 12. +No uncorrected cell, sentence or row changed its ID or its hashes. +Only corrected nodes and their ancestors got new `node_hash` values; the sentence `rr20tev` is one of the ancestors, because it holds the relabeled equation. +Inserting a row adds a row node. +The table's own hash (`own_hash`) covers only its label and column definitions, so every other cell, and every value already taken from them, is untouched. + +## 4. IDs that never move + +The sentence about water is still `hca6dc5` after twelve corrections, the corrected cell is still `c20cna6`, and the sentence that lost its superscript is still `ec5jc93`. +The retired IDs are recorded in [`ids.jsonl`](../fixtures/packages/hale2026harmonic-damb/ids.jsonl), so anything that pointed at them can follow the trail. +Legacy aliases are recorded there too: `b-p01-text-003`, the ID an older tool gave the water sentence, still leads to `hca6dc5`. + +A fresh conversion of the same input mints the same IDs. +A reading by a different converter is aligned with this document sentence by sentence ([spec/05 §5.6](../spec/05-hashes.md), pinned by [vectors/alignment.jsonl](../vectors/alignment.jsonl)). +Sentences the alignment pairs keep their IDs, and every difference becomes a suspect rather than a silent change. +Sentences it cannot pair are recorded as dropped or added. + +## 5. What happened to the notes + +An anchor is the stored pointer behind a note or citation. +It holds a node's ID, the node's hashes when the anchor was made, the revision, and, for part of a sentence, the quoted words with context ([spec/06](../spec/06-anchors.md)). +The team had already made seventeen anchors pointing into the first reading at revision 0, before any correction. + +Resolving an anchor gives one of five results: + +- `current`: nothing changed. +- `changed-below`: the node is unchanged, and something under it changed. +- `moved`: the node was merged into a successor that still holds the quote. +- `reattached`: the node changed, and the quote or the cell's headers now find exactly one place. +- `stale`: nothing is found, so a person must look. + +A `changed-below` result carries a drift, the share of the leaves under the node, counting retired ones, that corrections touched. +Above a quarter the drift is `high`. +The table shows how each anchor resolves at revision 12 ([vectors/resolve.jsonl](../vectors/resolve.jsonl) records them all). + +| Anchor on | Result | Why | +| --- | --- | --- | +| "H$_2$O" in the water sentence | `current` | Nothing in that sentence changed. | +| The cell `1.5260` | `current` | Untouched, though its table changed around it. | +| The water sentence, through the alias `b-p01-text-003` | `current` | The alias leads to the same node. | +| The cell that read `3l0.0` | `stale` | Its text changed and "3l0.0" is no longer in it; a person must look again. | +| "treated as zero" | `reattached` | The sentence changed twice, and the quote occurs in it exactly once, so the anchor is re-pinned. | +| "fitted constants" | `stale` | After the split, those words are in the new sentence `a4jrz2t`, not in `ec5jc93`. | +| "least squares", in the half-sentence `ayea71r` | `moved` | That ID was merged into `xjns57z`, which still holds the quote. | +| The paragraph `ncx2v4f` | `moved` | Merged into `gwzbj3s`, which holds all of its sentences. | +| The running head | `stale` | Deleted, with no successor. | +| The equation | `reattached` | Only its number changed. | +| The figure | `reattached` | Only its label changed, by a `relabel`. | +| The CT–HC cell `1.090` in the garbled row | `reattached` | Its row was deleted, and exactly one cell in Table 1 has the same row and column headers and the same text: the new `b5gd0h1`. | +| Section 2 | `changed-below`, drift high | The section itself is unchanged, and 53% of what it holds or held was touched. | +| Table 1 | `changed-below`, drift high | Rows were replaced and added. | +| The table body | `changed-below`, drift high | 75% of the cells it holds or held were touched. | +| The paragraph whose broken sentence was rejoined | `changed-below`, drift high | Two of the three sentences it has held were touched. | +| The CT–CT row | `changed-below`, drift low | One of its five cells was corrected. | + +A reader shows each note by its result. +A checker of exact words accepts only `current`, `moved` and `reattached`. + +The [W3C Web Annotation Data Model](https://www.w3.org/TR/annotation-model/) is the general W3C Recommendation for annotating web resources. paperspec's annotation layers build on it, with a custom `AnchorSelector` followed by a standard `TextQuoteSelector` as a fallback. +It has no permanent IDs for sentences inside the source, no hashes of the target text, and no results such as `moved` or drift. +See [comparison.md](comparison.md). + +## 6. Review and signatures + +A receipt is a signed record that a person compared a set of nodes with the page. +It lifts a node to F3, the top fidelity level (F0 is one reading; F1, every doubtful place answered; F2, an independent witness agrees) ([spec/08](../spec/08-fidelity.md)). +One signature covers a Merkle tree, a hash over everything reviewed in a sitting, built as in RFC 6962 (a Request for Comments standard; replaced by [RFC 9162](https://www.rfc-editor.org/rfc/rfc9162), which defines the same tree). +The receipts are in [`receipts.jsonl`](../fixtures/packages/hale2026harmonic-damb/receipts.jsonl), and the keys that signed them in [`roster.json`](../fixtures/packages/hale2026harmonic-damb/roster.json). + +- **Before the corrections**, `ana` had signed receipt `npc15pn6675z5` for three cells at revision 0. + Two of them never changed, so the receipt still counts for them. +For the corrected cell it no longer counts, because the cell's hash moved on. +- After the corrections, `ana` signs `y2zw8ctg5s8dp` at F3 for 23 nodes: the cells of Table 1's three data rows (15), its four column headers, the equation, and three sentences (`ec5jc93`, `hca6dc5` and `rr20tev`, the one holding the equation). + `ben` spot-checks it with receipt `mg18668nzxvap`. +His signature, made with his separate spot-check key `ben-spot`, picks 10 nodes nobody could have chosen, and he finds no errors. +- `ana` signs `mvjmb94nz7rdh` for two sentences of section 1, including the new `a4jrz2t`. +- `cara` signs `q465jpwz8afhe` for three sentences on page 2, then revokes it herself (`pcbhtpwvysgq6`) because she had compared them with the crops of another paper. + Those sentences fall back to their earlier levels, which is why `xjns57z` is still F0. +- `ana` signs `gx2s84z9jf9xv` for the figure's three caption sentences. `ben`'s spot-check `yt70fcc3qm0xb` finds a missing period in `caehhp6`, which suspends the receipt until she reviews the sitting again and signs a new one. + +The document's report, in [`document.json`](../fixtures/packages/hale2026harmonic-damb/document.json), counts the result over 45 leaves: 27 at F3, 13 at F2, 4 at F1 and 1 at F0. +The 27 are the 23 nodes of `y2zw8ctg5s8dp`, the 2 of `mvjmb94nz7rdh`, and 2 of the 3 caption sentences. +Those two still count because Ben's spot-check is itself a valid receipt, and he compared them with the page and found them right. +The third, `caehhp6`, stays at F2: a spot-check never raises a node it found in error ([spec/08 §8.2](../spec/08-fidelity.md#82-levels)). + +## 7. Data out of the table + +Table 1 becomes a dataset, `ds-hale-bond-params`, inside the package ([spec/15](../spec/15-datasets.md)). +It is a [Frictionless Data Package](https://datapackage.org/standard/data-package/) with units, uncertainty and an origin added, and the format is `provisional`. +Its CSV keeps every digit as printed (`1.526` and `1.5260` stay different), and every value has an anchor to the cell that prints it, so a claim that rests on a value can show the cell. +The descriptor types the fields as numbers; how a generic Data Package tool treats `1.5260` was not verified. +All twelve value anchors resolve as `current`. +The reference checker, [`tools/check.py`](../tools/check.py), confirms each CSV value equals its cell's text. +>>>>>>> 4fa326d (First implementation (#1)) ## 8. Sharing without the text diff --git a/fixtures/README.md b/fixtures/README.md index 79b3e62..9f9d8a2 100644 --- a/fixtures/README.md +++ b/fixtures/README.md @@ -35,6 +35,7 @@ A conforming reader accepts each one and recomputes every hash it contains. ## Records `records/` holds a valid example of every other format that is a JSON file of its own. +<<<<<<< HEAD Three formats have their example elsewhere: the stage requirements (`scienting-stages/1`) and the stamp rules (`scienting-stamps/1`) are in [registry/](../registry/README.md), and a bundle (`scienting-bundle/1`) is a SQLite file that `paperspec load --bundle` builds from the packages. Each record is a single file, or a folder where a format needs several files. @@ -110,3 +111,81 @@ Use them for nothing else. `src/` holds the hand-written descriptions of the three main documents, as first read; see [src/README.md](src/README.md). Everything else here is generated by `paperspec build`, so change the sources or the builder, never the generated files. +||||||| parent of 4fa326d (First implementation (#1)) +======= +Three formats have their example elsewhere: the stage requirements (`scienting-stages/1`) and the stamp rules (`scienting-stamps/1`) are in [registry/](../registry/README.md), and a bundle (`scienting-bundle/1`) is a SQLite file that `tools/load.py --bundle` builds from the packages. +Each record is a single file, or a folder where a format needs several files. + +| Files | What they are for | +| --- | --- | +| `anchor-span.json`, `anchor-leaf.json`, `anchor-cell.json`, `anchor-paragraph.json` | An anchor is an address pinned to a hash, with a quotation and context, so it can be found again after the text changes. These four anchors point at a span of a sentence, a sentence, a table cell and a paragraph of the main example. | +| `anchor-keyed.json` | An anchor that leaves the team, as in a public layer: its table context is stored as keyed hashes, so the labels cannot be read from it ([spec/06 section 6.3](../spec/06-anchors.md#63-keyed-anchors)). | +| `rights-open.json`, `rights-subscription.json`, `rights-collective.json` | A witness's license and what uses it allows, under an open license, a subscription and a collective license. | +| `policy.json` | A signed team decision that settles a use the rights record leaves unknown. | +| `backend-docling.json`, `backend-examplevendor.json` | The code and weight licenses of a conversion tool and the conditions attached to them. | +| `profile-acme.json`, `profile-smithlab.json` | What those conditions ask about a team: its purpose, its funding and whom it competes with. | +| `roster.json` | The keys that may sign, the role of each key's holder in the team, and what each key may sign. Ana is the owner, so she may revoke anyone's receipt. | +| `layer/` | A layer of annotations on the main example (`layer.json`, `annotations.jsonl`) and the receipt that confirms them (`receipt.json`). | +| `submissions/`, `grades/` | A student's answers to exercises in the layer, and the grader's result for each. | +| `suspect.json` | A doubtful reading raised for review: the converter's reading of an exponent in the main example, before the first correction answered it. | +| `qa-report.json` | Reviewer error bounds for a delivery that covers two of the main example's receipts, signed by the team's owner key. | +| `checklist/` | A conformance checklist (`checklist.json`) for an accessible HTML export of the main example, checked against WCAG 2.1 AA criterion by criterion, and the receipt that covers it at its hash (`receipt.json`). The format is `provisional`. | +| `proof-inclusion.json`, `proof-consistency.json` | Two RFC 9162 proofs: that one node was covered by the main example's first receipt without showing the rest, and that revision 12 of its correction history extends revision 8 rather than rewriting it. | +| `checkpoint.json` | A signed checkpoint of the team's transparency log over the main example's receipts, revocations and roster, as the parsed form of its signed note, cosigned by a witness. The format is `provisional`. | +| `log/` | The same log as the static files a server would publish: `checkpoint` and the hash and entries tiles ([spec/23 §23.4](../spec/23-log.md#234-serving-a-log)). | +| `receipt-ml-dsa.json` | A receipt signed with ML-DSA-65 (FIPS 204) by `dana`, whose roster key is post-quantum. | +| `project.json`, `digitization.json`, `digitization-confirmation.json`, `datasets/`, `view.vl.json`, `snapshot.json`, `draft-ids.jsonl` | A team project; values read off the main example's figure by hand, the receipt that confirms them, and the dataset they yield; a Vega-Lite chart pinned to that dataset; what the draft cites, pinned; and the sentence IDs of the draft's Typst project. These formats are `provisional`. | + +## Broken packages + +Each folder in `invalid/` holds a `package/` folder, or a `package.sct.zip` archive, that breaks exactly one rule, built from `packages/tiny2026example/` by one edit, and an `expect.json` that names the rule. +A conforming implementation refuses every one of them and should report that rule. +`registry/rules.json` lists every rule ID with its meaning, and a validator reports each failure as `path:line: rule: message` ([spec/22 section 22.2](../spec/22-conformance.md#222-validating-a-package)). + +| Case | What is wrong | Rule | +| --- | --- | --- | +| `alias-dangling/` | An alias whose node does not exist. | `doc.id.successor` | +| `bad-id/` | An ID that uses a letter Crockford base32 lacks (i, l, o or u). | `package.schema` | +| `checks-only-plain/` | A checks-only package that still carries text and plain hashes. | `doc.policy.withheld` | +| `cite-not-reference/` | A cite run names a node that is not a reference. | `doc.run.cite` | +| `counts-wrong/` | document.json's counts are not the number of nodes and leaves. | `doc.counts` | +| `crlf/` | A file whose lines end in CR LF, which is not the canonical form. | `package.file.canonical` | +| `empty-optional-file/` | An optional JSONL file with no records. | `package.file.empty` | +| `hash-mismatch/` | A node_hash that does not recompute. | `doc.hash.node-hash` | +| `heading-not-first/` | A heading must be a section's first child. | `doc.tree.child` | +| `history-wrong/` | document.json's history is not the root over the corrections. | `doc.history` | +| `key-suffix/` | A work key whose hash part is not the hash of the work's DOI. | `doc.key.suffix` | +| `key-work/` | A document key whose suffix is not the version's. | `doc.key.work` | +| `location-unknown-page/` | A location on a page no witness has. | `doc.location.page` | +| `manifest-hash/` | A file changed after the manifest was written. | `package.manifest.hash` | +| `manifest-key/` | The manifest names another key than document.json. | `package.manifest.key` | +| `missing-required-file/` | A required file is missing. | `package.file.required` | +| `not-canonical/` | A line that is valid JSON but not canonical JSON (the fixed form of RFC 8785, which gives equal data identical bytes). | `package.file.canonical` | +| `receipt-root/` | A receipt whose Merkle root (one hash that summarizes a list of hashes) does not recompute. | `doc.receipt.root` | +| `receipt-signature/` | A receipt changed after it was signed. | `doc.receipt.signature` | +| `receipt-unsorted-nodes/` | A receipt whose nodes are not sorted by ID. | `doc.receipt.nodes` | +| `retired-live/` | An ID that is both live and retired. | `doc.id.retired-live` | +| `rev-without-corrections/` | document.json's rev is not the number of corrections. | `doc.correction.rev` | +| `root-mismatch/` | document.json's root is not the root node's node_hash. | `doc.hash.root` | +| `signer-mismatch/` | A receipt whose signature was made with another key than the signer it names. | `doc.receipt.signature` | +| `style-sup-sub/` | A run cannot be both a subscript and a superscript. | `doc.run.style` | +| `text-edited/` | Text changed without its hashes. | `doc.hash.text-hash` | +| `unknown-kind/` | Kinds outside the list are refused. | `package.schema` | +| `unlisted-file/` | A file the manifest does not list. | `package.manifest.unlisted` | +| `unsafe-path/` | The manifest lists a path that leaves the package. | `package.path.unsafe` | +| `zip-symlink/` | A zipped package with an entry that is a symbolic link. | `package.zip.entry` | +| `zip-traversal/` | A zipped package with an entry whose path leaves the package folder. | `package.zip.entry` | + +## Keys + +`keys/` holds the Ed25519 secret keys (`ana`, `ben`, `cara`, `wit`, the witness, and `ben-spot`, whose VRF proof picks the nodes of a spot-check), the seed of `dana`'s ML-DSA-65 key, which signed the fixtures' receipts, and the team secret of the checks-only package. +Ed25519 is a signature scheme: the secret key makes a signature, and the public key in a roster verifies it. +These are published test keys. +Anyone can reproduce the signatures, so a signature made with these keys proves nothing about who made it. +Use them for nothing else. + +## Sources + +`src/` holds the hand-written descriptions of the three main documents, as first read; see [src/README.md](src/README.md). +Everything else here is generated by `python3 tools/build.py`, so change the sources or the builder, never the generated files. +>>>>>>> 4fa326d (First implementation (#1)) diff --git a/fixtures/src/README.md b/fixtures/src/README.md index 4fe0bf4..422cbc1 100644 --- a/fixtures/src/README.md +++ b/fixtures/src/README.md @@ -1,6 +1,11 @@ # Fixture sources +<<<<<<< HEAD Each source describes a document as a converter first read it, mistakes included, in a shorthand that `paperspec build` expands. +||||||| parent of 4fa326d (First implementation (#1)) +======= +Each source describes a document as a converter first read it, mistakes included, in a shorthand that `tools/build.py` expands. +>>>>>>> 4fa326d (First implementation (#1)) The build mints IDs, applies the listed corrections, lays out synthetic geometry, signs the listed receipts and writes the package. The shorthand is tooling, not a format. diff --git a/pixi.toml b/pixi.toml index ae22273..6bb3bd1 100644 --- a/pixi.toml +++ b/pixi.toml @@ -20,6 +20,7 @@ markdownlint-cli2 = ">=0.23.3" nodejs = ">=22" cryptography = ">=46" +<<<<<<< HEAD # The reference implementation in src/paperspec, installed editable so that `import paperspec` and the # `paperspec` command work in every pixi environment; without pixi, `pip install -e ".[tools]"` does the same. [pypi-dependencies] @@ -51,3 +52,17 @@ load = { cmd = "paperspec load", description = "Load packages into SQLite: pixi validate = { cmd = "paperspec validate", description = "Validate packages, or run proposed corrections through the gate" } stress = { cmd = "paperspec stress", description = "Stress the oracle with random inputs: pixi run stress --seed N" } site = { cmd = "paperspec site", description = "Build the website into _site/" } +||||||| parent of 4fa326d (First implementation (#1)) +======= +[tasks] +check = { cmd = "python tools/check.py", depends-on = [ + "lint", +], description = "Check schemas, records, packages, vectors, a byte-for-byte rebuild, the SQL round trip, prose and links" } +quick = { cmd = "python tools/check.py --quick", description = "Check everything except the byte-for-byte rebuild" } +build = { cmd = "python tools/build.py", description = "Rebuild fixtures/ and vectors/ from fixtures/src/ and the oracle" } +lint = { cmd = "markdownlint-cli2", description = "Lint every Markdown file with markdownlint" } +load = { cmd = "python tools/load.py", description = "Load packages into SQLite: pixi run load --sqlite lab.db PACKAGE..." } +validate = { cmd = "python tools/validate.py", description = "Validate packages, or run proposed corrections through the gate" } +stress = { cmd = "python tools/stress.py", description = "Stress the oracle with random inputs: pixi run stress -- --seed N" } +site = { cmd = "python tools/site.py", description = "Build the website into _site/" } +>>>>>>> 4fa326d (First implementation (#1)) diff --git a/registry/README.md b/registry/README.md index 06cc306..9d186e8 100644 --- a/registry/README.md +++ b/registry/README.md @@ -43,7 +43,12 @@ A rule's meaning never changes, and a retired rule's ID is never reused, so tool A tool can match on the `rule` field, whose meaning is looked up in `rules.json`, and treat the message as text for people. [Chapter 22, section 22.4](../spec/22-conformance.md#224-rules) lists the ID prefixes. Each directory in `fixtures/invalid/` has an `expect.json` that names the rule it breaks. +<<<<<<< HEAD The reference checker (`paperspec check`) fails if that rule is not in `rules.json`. +||||||| parent of 4fa326d (First implementation (#1)) +======= +The reference checker (`tools/check.py`) fails if that rule is not in `rules.json`. +>>>>>>> 4fa326d (First implementation (#1)) ## Adding stamp rules diff --git a/schemas/README.md b/schemas/README.md index f07f386..53fcb28 100644 --- a/schemas/README.md +++ b/schemas/README.md @@ -64,7 +64,12 @@ The `Registry` maps that name to the file's contents, so nothing is fetched over Never load a schema alone. Any draft 2020-12 validator works the same way: give it every schema under its `$id`, then validate against the one you need. +<<<<<<< HEAD `paperspec check` validates every fixture with `jsonschema` this way, and checks that every schema keeps to the keyword subset. +||||||| parent of 4fa326d (First implementation (#1)) +======= +`python3 tools/check.py` validates every fixture with `jsonschema` this way, and checks that every schema keeps to the keyword subset. +>>>>>>> 4fa326d (First implementation (#1)) ## What a schema does not check diff --git a/spec/00-introduction.md b/spec/00-introduction.md index 90393f3..b6bc007 100644 --- a/spec/00-introduction.md +++ b/spec/00-introduction.md @@ -82,9 +82,18 @@ The other folders help people use the specification. | `registry/` | Formats, rule IDs, stamp rules and the default stage requirements. | Yes | | `vectors/` | Test vectors, one case per line. | Yes | | `fixtures/` | Valid packages and records, invalid packages, and the keys that signed them. | Yes | +<<<<<<< HEAD | `src/` | The Python package `paperspec`: the reference implementation, and the `paperspec` command that computes and checks the vectors and fixtures and loads packages into SQLite. | No | | `tools/` | How to run those commands. | No | | `docs/` | Guides for people getting to know the formats. | No | The reference implementation in `src/paperspec/` computes every expected value, so that none is typed by hand. Other repositories of the project import it, but implementations conform to the chapters and the vectors, not to it. +||||||| parent of 4fa326d (First implementation (#1)) +======= +| `tools/` | The programs that compute and check the vectors and fixtures, and load packages into SQLite. | No | +| `docs/` | Guides for people getting to know the formats. | No | + +The programs in `tools/` compute every expected value, so that none is typed by hand. +They are not a library, and implementations conform to the chapters and the vectors. +>>>>>>> 4fa326d (First implementation (#1)) diff --git a/spec/01-conventions.md b/spec/01-conventions.md index f2a291b..97e7a0d 100644 --- a/spec/01-conventions.md +++ b/spec/01-conventions.md @@ -46,7 +46,12 @@ Stored text is kept exactly as witnessed; normalization applies only when a valu The vectors in `normalize.jsonl` cover the cases, including the angstrom sign, which NFC turns into Å. Normalization forms, General Categories and the `White_Space` property are those of Unicode 16.0. +<<<<<<< HEAD A later Unicode version can change a decomposition or a category, and with it a hash, so an implementation states the Unicode version its character data comes from, the reference implementation refuses to import on another, and moving the formats to a later version is a change to every format that hashes text ([0.3](00-introduction.md#03-format-identifiers-and-versions)). +||||||| parent of 4fa326d (First implementation (#1)) +======= +A later Unicode version can change a decomposition or a category, and with it a hash, so an implementation states the Unicode version its character data comes from, the reference tools refuse to run on another, and moving the formats to a later version is a change to every format that hashes text ([0.3](00-introduction.md#03-format-identifiers-and-versions)). +>>>>>>> 4fa326d (First implementation (#1)) ## 1.5 Character offsets diff --git a/spec/11-corrections.md b/spec/11-corrections.md index 533abcc..38a4969 100644 --- a/spec/11-corrections.md +++ b/spec/11-corrections.md @@ -67,7 +67,12 @@ A correction is refused, naming the rule ([22](22-conformance.md)), when: 11. the result breaks a rule of [3](03-nodes.md). The gate needs only the document's current state, so a tool can run it on a proposal before applying anything, and get back the record the correction would append or the rule it breaks. +<<<<<<< HEAD `paperspec validate --correction` does this against a package. +||||||| parent of 4fa326d (First implementation (#1)) +======= +`tools/validate.py --correction` does this against a package. +>>>>>>> 4fa326d (First implementation (#1)) Digits are characters of General Category Nd in the plain projection. A package's corrections are also checked as a whole: indexes run from 1 without gaps, each `id` derives from its record, `rev` equals their number, every retired ID is in `ids.jsonl`, and the last correction of each leaf ends at its current `node_hash`. diff --git a/spec/22-conformance.md b/spec/22-conformance.md index 79bfa62..67f30c7 100644 --- a/spec/22-conformance.md +++ b/spec/22-conformance.md @@ -6,7 +6,12 @@ | --- | --- | --- | | `vectors/*.jsonl` | Inputs and expected outputs, one case per line. | Computes every expected output from its input. | | `fixtures/packages/` | Valid packages: a converted chemistry paper with corrections, receipts, a multi-level-header table and a dataset; its checks-only export; an open-access paper with two witnesses; a Typst draft; a tiny document, as a folder and as the zip of [12.2](12-package.md#122-paths). | Accepts each, and recomputes every value it contains. | +<<<<<<< HEAD | `fixtures/records/` | A valid record of every other format that is a JSON file of its own; the stage requirements and stamp rules are in `registry/`, and a bundle is built by `paperspec load --bundle`. | Accepts each. | +||||||| parent of 4fa326d (First implementation (#1)) +======= +| `fixtures/records/` | A valid record of every other format that is a JSON file of its own; the stage requirements and stamp rules are in `registry/`, and a bundle is built by `tools/load.py --bundle`. | Accepts each. | +>>>>>>> 4fa326d (First implementation (#1)) | `fixtures/invalid/` | Invalid packages, each with `expect.json` naming the rule it breaks. | Refuses each, and SHOULD name that rule. | | `fixtures/keys/` | The test keys that signed the fixtures and the team secret of the checks-only export. | Never uses them for anything else. | diff --git a/sql/README.md b/sql/README.md index a238a69..fbd80b7 100644 --- a/sql/README.md +++ b/sql/README.md @@ -31,6 +31,7 @@ dolt sql < sql/doc.sql && dolt sql < sql/store.sql The PostgreSQL database `lab` and the Dolt repository must already exist. The `sqlite3` line runs as shown; the `psql` and `dolt` lines follow the portability rules of [chapter 13](../spec/13-database.md#131-portability) but have not been run on those engines in this repository. +<<<<<<< HEAD To load packages into SQLite without pixi, install the package with its `tools` extra on Python 3.14, then run `paperspec load --sqlite lab.db PACKAGE…`. The file [docs/examples.md](../docs/examples.md#load-packages-into-sqlite) has queries to try. The loader copies files that have no table (assets, dataset files, derived files) by SHA-256 into `lab.db.assets/`. @@ -59,6 +60,36 @@ On `fixtures/packages/hale2026harmonic-damb` it returns one row, `hca6dc5`: the - Round trip: each row keeps every property of its record, so unloading restores the same bytes and the same manifest hash. Unloading copies the files with no table from the asset store, which must travel with the database. `paperspec check` tests this for every fixture package. +||||||| parent of 4fa326d (First implementation (#1)) +======= +To load packages into SQLite without pixi, run `python3 tools/load.py --sqlite lab.db PACKAGE…` with Python 3.14 and two packages, `jsonschema` and `cryptography` installed. +The file [docs/examples.md](../docs/examples.md#load-packages-into-sqlite) has queries to try. +The loader copies files that have no table (assets, dataset files, derived files) by SHA-256 into `lab.db.assets/`. + +## A first query + +The query below finds every sentence that cites a given reference, which is the citation's context. +It returns sentence IDs, and `nodes.content` holds each sentence's text. + +```sql +SELECT c.sentence +FROM citations c +JOIN refs r ON r.doc_key = c.doc_key AND r.node = c.ref_node +WHERE c.doc_key = 'hale2026harmonic-damb' AND r.label = '2'; +``` + +On `fixtures/packages/hale2026harmonic-damb` it returns one row, `hca6dc5`: the sentence ending "served as the first test case, with a bending band near 1595 cm⁻¹." + +## Conventions + +- Foreign keys: none. + Validators check references, so the same file runs everywhere. +- Types: enumerations are `VARCHAR` with a `CHECK`, JSON values are canonical JSON in `TEXT`, and booleans are `INTEGER` 0 or 1. +- Names: no table, column or index name is reserved in SQLite, PostgreSQL or MySQL (Dolt is MySQL-compatible). + The formats' `key`, `index`, `at`, `by`, `before` and `after` become `doc_key`, `idx`, `made_at`, `signer`, `before_hash` and `after_hash`. +- Round trip: each row keeps every property of its record, so unloading restores the same bytes and the same manifest hash. + Unloading copies the files with no table from the asset store, which must travel with the database. `tools/check.py` tests this for every fixture package. +>>>>>>> 4fa326d (First implementation (#1)) - Derived tables: `citations`, `receipt_nodes` and `annotation_targets` are built on load for queries and are not part of the package. ## Status diff --git a/tools/README.md b/tools/README.md index 76e7621..9ebe662 100644 --- a/tools/README.md +++ b/tools/README.md @@ -4,6 +4,7 @@ This page is for anyone who wants to run the repository's checks, rebuild its ex Newcomers get each term defined where it first appears. Implementers get the commands and the traps. +<<<<<<< HEAD These commands are the repository's reference implementation. They apply every rule of the specification, so the rest of the repository can be checked against them. Two build and check the repository, one validates packages and proposed corrections, one loads packages into SQLite, a database stored in a single file, one stresses the reference implementation with random inputs, and one builds the website. @@ -84,3 +85,71 @@ Schema validation (`schema`, and with it `package.validate` and `corrections.gat The package version comes from git tags through setuptools-scm, and `pixi run -e dev dist` builds the sdist and the wheel into `dist/`. When `scienting-core` exists it takes this role behind the same vectors, and the import changes without any computed value changing. Independent implementations follow the [specification](../spec/README.md) and test themselves against the [vectors](../vectors/README.md). +||||||| parent of 4fa326d (First implementation (#1)) +======= +These programs are the repository's reference implementation. +They apply every rule of the specification, so the rest of the repository can be checked against them. +Two build and check the repository, one validates packages and proposed corrections, and one loads packages into SQLite, a database stored in a single file. +They need Python 3.14 and two packages, `jsonschema` and `cryptography`, and are licensed Apache-2.0. + +A package is the folder of files that holds one document, such as `fixtures/packages/hale2026harmonic`. +A vector is a test case: an input and the exact output every implementation must compute (see [the vectors](../vectors/README.md)). +A fixture is an example file or package under `fixtures/`. + +| Program | Does | +| --- | --- | +| `check.py` | Checks everything: schemas, records, packages and the zipped package, the broken packages, every vector, a byte-for-byte rebuild, the SQL (a database query language) round trip, the counts the prose states, and the prose. `--quick` skips the rebuild. | +| `build.py` | Builds `fixtures/` and `vectors/` from the hand-written sources in `fixtures/src/` and the cases it defines. `--out DIR` builds into another folder. | +| `validate.py` | Validates packages, as folders or as zipped packages (`.sct.zip`, unpacked under the rules of [spec/12 §12.2](../spec/12-package.md#122-paths)): `python3 tools/validate.py PACKAGE [PACKAGE ...]`. With `--correction PROPOSALS` it runs proposed corrections through the gate against one package's current state, and prints the record each would append or the rule it breaks, without writing anything ([spec/11 §11.4](../spec/11-corrections.md#114-the-gate)). | +| `load.py` | Validates packages, folders or zipped, then loads them into a SQLite database with the tables of `sql/`: `python3 tools/load.py --sqlite lab.db PACKAGE [PACKAGE ...]`. With `--bundle --built-from REV` it builds a new bundle instead, with the full-text search index ([spec/13 §13.5](../spec/13-database.md#135-the-bundle)). | +| `site.py` | Builds the website into `_site/`: every file of the repository at its own path, and each Markdown file rendered as a page beside it, with a vocabulary page, folder listings and a search index. It uses only the standard library. `pixi run check` builds it into a temporary folder and checks every link ([deploy/](../deploy/README.md)). | +| `stress.py` | Stresses the oracle with random inputs from a seeded generator: `python3 tools/stress.py [--seed N] [--rounds N]`. It checks the invariants the chapters state rather than the vectors' values, such as canonical JSON round trips, corrections keeping the five values consistent with a full recomputation, anchors always resolving to one of the five outcomes, the zip reader never writing outside its folder, and a mutated package always being refused. | +| `oracle/` | A compact implementation of every rule. `build.py` uses it to compute expected values, and `check.py` uses it to recompute them. | + +## When to run what + +Each program is also a pixi task. +Pixi sets up a pinned Python and the other packages the repository uses. +The commands `pixi run check`, `pixi run quick`, `pixi run build` and `pixi run load` then work from anywhere in the repository. +`pixi run check` lints the Markdown first. + +Run the programs directly from the repository root if you do not use pixi: + +```sh +python3 tools/check.py # everything +python3 tools/check.py --quick # everything except the rebuild +python3 tools/build.py --out /tmp/rebuilt +python3 tools/validate.py fixtures/packages/hale2026harmonic +python3 tools/load.py --sqlite lab.db fixtures/packages/hale2026harmonic +``` + +A passing check ends with `All checks pass.` +The load command prints `hale2026harmonic: 75 nodes`, one line per package, and writes `lab.db` with a folder `lab.db.assets/` beside it for files that have no table of their own. +It validates every package first and loads nothing if one fails, printing each failure as `package: where: rule: message`. +The full and checks-only forms of one document share its key, so they go into separate databases. +Each program prints its options with `--help` and refuses options it does not know. +`build.py` rebuilds `fixtures/` and `vectors/` in place unless you pass `--out`. + +Run `pixi run check` before every commit. +Run `pixi run build` after you change a fixture source, a rule in `oracle/` or a case in `build.py`, and commit the regenerated files with the change. +The check rebuilds into a temporary folder and compares every file byte for byte, so it fails on out-of-date files. + +## About the oracle + +An oracle is a reference computation: a slow, plain program whose answers count as the expected ones. +It is not a production reader or writer. +The oracle exists so that no expected value in this repository is typed by hand. + +It favors clarity, and leaves the cryptography and schema validation to established packages. +Ed25519 (a digital signature scheme) and HKDF come from `cryptography`, and JSON Schema (a format that describes the shape of JSON data) validation from `jsonschema`. +The VRF of RFC 9381 (`oracle/vrf.py`) and ML-DSA-65 of FIPS 204 (`oracle/mldsa.py`) are written out in plain Python, slow and clear, and checked against the standards' own test vectors. +The oracle still refuses any schema that uses a keyword outside the subset the schemas are restricted to. +Canonical JSON goes through Python's built-in JSON encoder, and falls back to an exact RFC 8785 serializer for the values that encoder writes differently: floating-point numbers, integers beyond 2^53 and some non-ASCII member names. + +Hashes are kept incrementally. +A node's five values depend only on its own subtree, so after a correction the oracle rehashes only the nodes that changed and their ancestors, and a container's projections are its children's cached projections joined. +`check.py` compares the result after every correction with a slow computation that projects each node from its whole subtree. + +Do not build on it as a library. +Implementations follow the [specification](../spec/README.md) and test themselves against the [vectors](../vectors/README.md). +>>>>>>> 4fa326d (First implementation (#1)) diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/build.py b/tools/build.py new file mode 100644 index 0000000..93e33ef --- /dev/null +++ b/tools/build.py @@ -0,0 +1,1869 @@ +#!/usr/bin/env python3 +"""Build every fixture and test vector from fixtures/src and the cases below. + +Non-normative tooling. +The build is deterministic: run it twice and the output is byte-identical; `tools/check.py` verifies that. +Usage: python3 tools/build.py [--out DIR] (default: the repository itself) +""" + +import base64 +import copy +import hashlib +import json +import os +import shutil +import struct +import sys +import zipfile +import zlib +from fractions import Fraction + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) + +from tools.oracle import align as AL # noqa: E402 +from tools.oracle import anchors as AN # noqa: E402 +from tools.oracle import corrections as CO # noqa: E402 +from tools.oracle import crypto, fingerprint as FP, grade as GR, hashing, jcs, receipts as RC, rights as RI # noqa: E402 +from tools.oracle import keys as K, log as LG, mldsa, targets as TG, vrf # noqa: E402 +from tools.oracle.model import LEAVES, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS, Doc # noqa: E402 +from tools.oracle.notation import fold_math # noqa: E402 +from tools.oracle.package import ZIP_EXTENSION, history_items, jsonl, line, witness_id, write, write_zip # noqa: E402 +from tools.oracle.schema import Store # noqa: E402 +from tools.oracle.text import letter_first, mint_node_id, norm, record_id, sha256_hex # noqa: E402 +from tools.oracle.tree import Refusal, _flatten, mint_ids, validate_structure # noqa: E402 + +VERSION = "0.1.0" +STORE = "smithlab" +SCHEMAS = Store(os.path.join(ROOT, "schemas")) + + +def H(s): + return sha256_hex(s) + + +def secret(name): + return hashlib.sha256(("paperspec fixture key: " + name).encode()).digest() + + +KEYS = { + "ana": {"name": "Ana Example", "role": "owner", "purposes": ["sign"]}, + "ben": {"name": "Ben Example", "role": "editor", "purposes": ["sign"]}, + "ben-spot": {"name": "Ben Example", "role": "editor", "purposes": ["spot-check"]}, + "cara": {"name": "Cara Example", "role": "reader", "purposes": ["sign"]}, + "dana": {"name": "Dana Example", "role": "editor", "purposes": ["sign"], "scheme": "ml-dsa-65"}, + "wit": {"name": "Open Witness (fictional)", "role": "guest", "purposes": ["witness"]}, +} +TEAM_SECRET = hashlib.sha256(b"paperspec fixture team secret: smithlab").digest() +# NIST ACVP ML-DSA-keyGen-FIPS204, parameter set ML-DSA-65, test case 26: the seed and the public key it gives. +NIST_SEED = "a991fd42b071d49c48ae3e75c647459e0daad1e1ba356a04801912d3294bcff8" +NIST_PK = "36db0b5dce98bd190cb139e80b71b49c7d7040b71c5a1f3412c46bde939192b1b57ccb88ac2714c1240cb0eb62c689e031aea3d9f3eb3ed7bfa45931d288dcae3413199b31a7032560dce8a61e195d13a1440615c2f3aa7dd28c5b1b742bfa400052186721f13d3df9dcfaee348b10d66913c7148913e085e1a4a03c659398dadc6a8e0e0c1a7f9f44d30436db90fd65a6ab8f36137338255653baae8da21526a333426dbd9f76cce0f43212643e854d772018b35ce726bcaaa5ab0651baf8c122e13929bb35b6e4963df2595fdc7237cdaa7234bf776b07f353ccdba12ad3e025138e3492d7f8e929db55dc23e23075f66d57a10492e6a10ae7b758acc2291ca18ba1ca07a5b574ab6d8aac18b9524990ad2f110225b7d82f696300a660a166ad35b3c57ecbab77117c79656fa8ae2a19a7defb2afd2af54683d043be0f933b8eae0d591448ed55d00068cd9fe10b067fcfaac53aedb1e9b667e36c4e30231f85c7aa0a474af2fa4776226f4479555e155528d78b98183cbdf7fae4e7301140f163eb71e991d15fad4a0d2f25a5a62fa2e9bcc823cc2927662e40c538213ded9e2df508e911e4924e507a50861fbb050edbbf56d937206f8fbc6f4cead4cd10d06b73aadcd4aa39703a7a2bffae68b7baa47341b699da9f3b167d4d90efee0a07ee3529a3b5e8648b9cb07ee973e1d8dccf1d16e95092c4a0184ccb4902d6086d9f444aca5fa45f43ca91b351e82585989ffcbd6d2c3471d6b8593aa46f29d0dd9b44e8aa4d8f9a0bc886bb7982c56aab11e23bfdbd8bc674732fadacacae25fa416b2d0cc7743827293336507da4b14c1f0aa2e929af975466dadc89a016f33a0cca2d5c08114cf04b02358805a772536432c44dbe9886130d2d3a0ffa0e175875a2207686f5e562b879eb2957573ab706b942468c20cc69bc566d29d9f151f3cfae71cc97ce4a30722d4679fc1c089b5009935931ee60aac5496b0fd5f24e514c0e20fa1dcc7729184a50fc85fad1d2f32f715fbf55666e49f5a19761f2dd1aa5d1a33c7916eb6a794981c0334176abc493ef30d9eaeaad42e705989dcfcdeb578529a700bd14076a348a2062d6483cc63cd7f55136587aac0e531a06eb2de74e61cfcbbce18f2ada5a741f683bf101f71432ee659dd1508e0c8fb2400e0ccbe435da3466d543d3ef5ba369e125c0b84d855efe6d4a22ff929a7a7a984e448d23871e09b88a0bc3f3b7da55dd2edfb5a6daa102819ff50cd4dccd0a95d2f27354668065d4a56c31fb18b92b2a8db2c6453baa9333aea6eabb1bd6411d584dd5900262057a707f81cc5137dbdd9ac1079bb98da78a8e4bd1b2e0546c2b3d956fcc280d37d855e31f1e4315b387a742280f057f3219eae512884ae7ec4d2e3a72265b1d0163fbcbf616b2e289b0eaf9c63437d50b7b50ce408f5b4562f2abf510c19f5e8a0ace264db6e0f2a69a7d0b4a5e62a2b964f08c8ffe9c295f5773bdd7fab054a13822d428fae28ae5e4adc9d9f6e4dffc457a3e49f0bcf62b32961c4667b60960452afd917fdd00d954fa30c8533e5629f90af85948dc1bad889f91832ddf9b738254c9e7939726c37ab4557c2ce363c1391816c467537b471e5985e8084c277b62be514922d352e20689eabb3eb91c343f36e77b152d5e85afd088f4d02e7024b248a7420f58c7ebbeb480cae39b56164f5acd37a4f56b3db6e1cc6b7c8c96cd3c44a69d9ac99175257bab7fd83c5b574b5c9702c0fd13a5b176c60f82d2dfff50c2af25d96e0f8d27ec818d499e479b9642aed4a4a0e6af5f14cc5e1299eabae055ef3c763d1e350e2d76e92cee47a4233368466a298afb4ca108a325d2a4f8b79f21ee7349c1c186ecd7897f9886cf27ec01b05388870484867f84bae2c016d04a3762241907c4de207798dd125a2cebb6c2982f779e04117bdd65cd7ff0361a59d3ec05f6d903b6d15554beefb6d40d96a0d4b37ae76c69c1b9592088b7db878f95abedcb5fd5423ed93df1b27d01a4dc9f4438e7c55f35b0aeb7395b08e1ecbf15cb2b61d043c0454aeaef2d487093fa0d7de3fc6caf084b6a0f15a5cb05d9340d4e6763983dc45b7828539c77a60d5e081a03fe29949d916392b6d989b4c8c047e3635a76ba88ac18a7a18ccff5c7e06b02a43d2dff169fa449739e382bd020e0963c14a9acae6b6561c722d2bda183f33ee6a904df0207f5b098e56335cc063f9640c4997f593218d502f6b382354c73979e93c4b1b2471965ffa5a0dc8ee8eca9f5697e7ef08dc0eefd9ad75cd4122194b450201dfd73cda46a7b2478df66129fbb9c75b774213f9615bd990f8d07501fed440fd25d6cb912b8ecfa678d887a4ee28677e6e0491d49efc7a3b34b9815c5c22983da280d0afde2324f5281bc8b796ddacfd82723bbd9aa34b0c96075b36848591e47b80086897846fa76d092bc8bc6200837bfb5545039f8602b7ea49f63c0c3b8317eeeb7612f8e818dde09e43c7da76fd2ff6847906a45da3d993e8eaed9fb3b1e579d8cc6900c89522aaeb0b4a80da1e66ab8dfd62dfe4e4d77a3a77e5bd669207c70aa8537dd6d80a647b0420d79531a7456052c3c989f0f08de3d343c40067680b39ece95a17aac8a622d1d5d95b38ca0f11d94e5b0a7634eef4055517ace79f0df1d7c172e0246abb2ab6b135ee1a38a3b84f86fd7c3caf178cd4446d0b554256ad45c657e1192070aba7df480f489ebdf9753a79ccbc6aa893913c5f1271f1c6035" + + +def public_key(kid): + scheme = KEYS[kid].get("scheme", "ed25519") + return mldsa.public_key(secret(kid)) if scheme == "ml-dsa-65" else crypto.ed25519_public(secret(kid)) + + +def seal(rec, kid): + """Sign a record with a fixture key under its scheme.""" + if KEYS[kid].get("scheme", "ed25519") == "ml-dsa-65": + return RC.seal_mldsa(rec, mldsa.keygen(secret(kid))[1], kid) + return RC.seal(rec, secret(kid), kid) + + +def roster(): + keys = [] + for kid, k in KEYS.items(): + scheme = k.get("scheme", "ed25519") + pub = public_key(kid) + keys.append({"id": kid, "person": {"name": k["name"]}, "role": k["role"], "scheme": scheme, + "public": base64.b64encode(pub).decode(), "did": crypto.did_key(scheme, pub), + "purposes": k["purposes"], "registered_at": "2026-09-01T00:00:00Z", "revoked_at": None}) + return {"format": "scienting-roster/1", "store": STORE, "keys": keys} + + +ROSTER = roster() +ROSTER_BY_ID = {k["id"]: k for k in ROSTER["keys"]} + + +def png(w, h, rgb): + """A tiny valid PNG, so a fixture can carry a real image asset.""" + raw = b"".join(b"\x00" + bytes(rgb) * w for _ in range(h)) + + def chunk(t, d): + return struct.pack(">I", len(d)) + t + d + struct.pack(">I", zlib.crc32(t + d) & 0xFFFFFFFF) + + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw, 9)) + chunk(b"IEND", b"")) + + +# --- the shorthand of fixtures/src ------------------------------------------------ + + +def runs_of(parts, kids): + out = [] + for p in parts: + if isinstance(p, str): + out.append({"kind": "text", "text": p}) + elif "sup" in p or "sub" in p or "em" in p: + st = next(k for k in ("sup", "sub", "em") if k in p) + out.append({"kind": "text", "text": p[st], "style": [st]}) + elif "m" in p: + out.append({"kind": "math", "notation": "tex", "source": p["m"]}) + elif "t" in p: + out.append({"kind": "math", "notation": "typst", "source": p["t"]}) + elif "cite" in p: + out.append({"kind": "cite", "refs": p["cite"], "text": p["text"]}) + elif "xref" in p: + out.append({"kind": "xref", "target": p["xref"], "text": p["text"]}) + elif "fn" in p: + out.append({"kind": "fnmark", "footnote": p["fn"], "text": p["text"]}) + elif "link" in p: + out.append({"kind": "link", "url": p["link"], "text": p["text"]}) + elif "eq" in p: + e = p["eq"] + kids.append(_math_node("equation", e)) + out.append({"kind": "equation", "child": len(kids) - 1}) + else: + raise ValueError(p) + return out + + +def _math_node(kind, e): + n = {"kind": kind, "content": {"notation": "typst" if "typst" in e else "tex", "source": e.get("tex", e.get("typst"))}} + if "label" in e: + n["label"] = e["label"] + for k in ("name", "fidelity"): + if k in e: + n["_" + k if k == "name" else k] = e[k] + return n + + +def expand(s, page=None): + if isinstance(s, str): + s = {"s": [s]} + page = s.get("page", page) + if "s" in s: + kids = [] + n = {"kind": "sentence", "content": runs_of(s["s"], kids), "children": kids} + elif "p" in s: + n = {"kind": "paragraph", "children": [expand(x, page) for x in s["p"]]} + elif "row" in s: + n = {"kind": "row", "children": [expand(x, page) for x in s["row"]]} + if "header_path" in s: + n["attrs"] = {"header_path": s["header_path"]} + elif "c" in s: + n = {"kind": "cell", "content": runs_of(s["c"], []), + "attrs": {"column": s["col"], "rowspan": s.get("rowspan", 1), "colspan": s.get("colspan", 1), + "role": s.get("role", "data")}} + n["fidelity"] = s.get("fidelity", "F1") + else: + n = {"kind": s["kind"]} + if "label" in s: + n["label"] = s["label"] + kids = [] + if s["kind"] == "section": + n["attrs"] = {"level": s["level"]} + if "title" in s: + kids.append({"kind": "heading", "children": [expand({"s": [s["title"]], "fidelity": "F2"}, page)]}) + if s["kind"] == "table": + n["attrs"] = {"columns": [{"id": c["name"], "header_path": c["header_path"]} for c in s["columns"]]} + if s["kind"] == "panel": + n["attrs"] = {} + if "figure_text" in s: + n["attrs"]["figure_text"] = [dict(t) for t in s["figure_text"]] + if s["kind"] in ("reference", "authors", "keywords"): + n["content"] = runs_of(s["runs"], []) + if s["kind"] in MATH_KINDS: + n = _math_node(s["kind"], s) + for k in ("csl", "resolution"): + if k in s: + n["_" + k] = s[k] + kids += [expand(x, page) for x in s.get("children", [])] + n["children"] = kids + if "name" in s: + n["_name"] = s["name"] + if "fidelity" in s and n["kind"] in LEAVES: + n["fidelity"] = s["fidelity"] + n["_page"] = page + for c in n.get("children", []): + if c.get("_page") is None: + c["_page"] = page + return n + + +class Names: + """Local names (`@x`) and the IDs minted for them, per document, registered under the source's name.""" + + registry = {} + + def __init__(self, name, key): + self.name = name + self.key = key + self.ids = {} + Names.registry[name] = self + + def resolve(self, v): + if isinstance(v, str) and v.startswith("ext:"): + parts = v.split(":", 2) + other = Names.registry[parts[1]] + if len(parts) == 2: # `ext:`: the document, qualified by the store + return STORE + "/" + other.key + return other.key + "#" + other.ids[parts[2]] + if isinstance(v, str) and v.startswith("@"): + return self.ids[v] + if isinstance(v, list): + return [self.resolve(x) for x in v] + return v + + +def resolve_refs(nodes, names): + for n in nodes: + if n["kind"] in RUN_KINDS: + out = [] + for r in n["content"]: + r = dict(r) + if r["kind"] == "cite": + r["refs"] = names.resolve(r["refs"]) + elif r["kind"] == "xref": + r["target"] = names.resolve(r["target"]) + elif r["kind"] == "fnmark": + r["footnote"] = names.resolve(r["footnote"]) + out.append(r) + n["content"] = out + a = n.get("attrs") + if a: + if "header_path" in a: + a["header_path"] = names.resolve(a["header_path"]) + if "column" in a: + a["column"] = names.resolve(a["column"]) + if "columns" in a: + a["columns"] = [{"id": names.resolve(c["id"]), "header_path": names.resolve(c["header_path"])} + for c in a["columns"]] + + +def mint_extras(key, nodes, names, used): + """Columns and figure-text items get IDs from the same minting rule, kinds `column` and `figure-text`.""" + doc = Doc(key, nodes) + seen = {} + + def mint(kind, th): + occ = seen.get((kind, th), 0) + seen[(kind, th)] = occ + 1 + nid = mint_node_id(key, kind, th, occ) + while nid in used: + occ += 1 + nid = mint_node_id(key, kind, th, occ) + used.add(nid) + return nid + + for n in nodes: + a = n.get("attrs", {}) + for c in a.get("columns", []): + if c["id"].startswith("@"): + th = H(norm(" ".join(hashing.plain(doc, names.ids[h]) for h in c["header_path"]))) + names.ids[c["id"]] = mint("column", th) + for t in a.get("figure_text", []): + t["id"] = mint("figure-text", H(norm(t["text"]))) + if "name" in t: + names.ids[t.pop("name")] = t["id"] + + +def flatten_spec(spec): + flat = [] + _flatten(copy.deepcopy(spec), None, 0, flat) + return flat + + +def establish(key, spec, names, used): + flat = flatten_spec(spec) + nodes = mint_ids(key, flat, used) + side = {} + for f, n in zip(flat, nodes): + if "_name" in f: + names.ids[f["_name"]] = n["id"] + side[n["id"]] = {k: v for k, v in f.items() if k.startswith("_")} + mint_extras(key, nodes, names, used) + resolve_refs(nodes, names) + return nodes, side + + +# --- geometry ----------------------------------------------------------------- + + +def page_layout(doc, side, src, wit_id, fps_override=None): + """Synthetic but consistent geometry: lines per page, fingerprints and boxes.""" + P = src["pages"] + size = P["size"] + npages = len(P["labels"]) + stamps = json.load(open(os.path.join(ROOT, "registry", "stamps", "scienting-stamps-1.json")))["rules"] + lines = {p: [] for p in range(1, npages + 1)} + boxes = {} + y = {p: 72.0 for p in lines} + for p, rh in P.get("running_head", {}).items(): + lines[int(p)].append({"text": rh, "box": [72.0, 40.0, 400.0, 50.0]}) + for nid in doc.preorder(): + n = doc.nodes[nid] + pg = side.get(nid, {}).get("_page") or 1 + if n["kind"] in ("sentence", "cell", "reference", "authors", "keywords") or ( + n["kind"] == "equation" and doc.nodes[n["parent"]]["kind"] != "sentence"): + text = norm(hashing.plain(doc, nid)) + lines[pg].append({"text": text}) + bx = [] + rest = len(text) + while True: + w = min(rest, 80) + bx.append([72.0, y[pg], round(72.0 + 5.85 * w, 2), round(y[pg] + 11.8, 2)]) + y[pg] = round(y[pg] + 13.95, 2) + rest -= 80 + if rest <= 0: + break + boxes[nid] = (pg, bx) + elif n["kind"] == "panel": + boxes[nid] = (pg, [[72.0, y[pg], 300.0, round(y[pg] + 150.0, 2)]]) + y[pg] = round(y[pg] + 160.0, 2) + pages = [] + images = {} + for nid, s in side.items(): + if "_image" in s: + images.setdefault(boxes[nid][0], []).append(s["_image"]) + for p in range(1, npages + 1): + ls = list(lines[p]) + if "stamp" in P: + ls.append({"text": P["stamp"], "box": [8.0, 200.0, 20.0, 600.0]}) + ls.append({"text": P["stamp2"], "box": [580.0, 200.0, 592.0, 600.0]}) + page_in = {"size": size, "lines": ls, "images": images.get(p, [])} + fp, removed = FP.fingerprint(page_in, stamps) + rec = {"witness": wit_id, "page": p, "page_fp": fp, "size": size, "page_label": P["labels"][p - 1], + "text_layer": True} + if removed: + rec["stamps_removed"] = removed + pages.append((rec, page_in)) + return pages, boxes + + +# --- packages ------------------------------------------------------------------- + + +def rights_record(kind): + base = {"format": "scienting-rights/1", "at": "2026-09-15T12:00:00Z"} + if kind == "open": + base.update({"license": {"spdx": "CC-BY-4.0", "url": "https://creativecommons.org/licenses/by/4.0/"}, + "basis": "open-license", + "uses": {"read": True, "tdm": True, "redistribute": True, "quote-public": True, + "accessible-copy": True, "ai": {p: True for p in RI.AI_PAIRS}}, + "asserted_by": {"by": "crossref", "evidence": [ + {"kind": "crossref-license", "doi": "10.5555/example.okafor.2025", + "url": "https://creativecommons.org/licenses/by/4.0/"}]}, + "valid_until": None}) + elif kind == "own": + base.update({"license": {"spdx": "CC-BY-4.0"}, "basis": "own-work", + "uses": {"read": True, "tdm": True, "redistribute": True, "quote-public": True, + "accessible-copy": True, "ai": {p: True for p in RI.AI_PAIRS}}, + "asserted_by": {"by": "ana", "evidence": [{"kind": "statement", "note": "The team's own draft."}]}, + "valid_until": None}) + else: + base.update({"license": {"text": "Publisher's subscription terms (fictional)."}, "basis": "subscription", + "uses": {"read": True, "tdm": True, "redistribute": False, "quote-public": False, + "accessible-copy": "unknown", + "ai": {"internal/local": True, "internal/tenancy": "unknown", + "internal/outside-no-retention": "unknown", "internal/outside": False, + "external/local": False, "external/tenancy": False, + "external/outside-no-retention": False, "external/outside": False}}, + "asserted_by": {"by": "ana", "evidence": [ + {"kind": "statement", "note": "The library's subscription terms, as the team read them (fictional)."}]}, + "valid_until": "2027-08-31T23:59:59Z"}) + return base + + +def crop_sha(name): + return H("crop of " + name + " at 300 dpi (synthetic)") + + +def node_record(n, values): + rec = {k: n[k] for k in ("id", "kind", "parent", "ord")} + for k in ("label", "attrs", "content", "withheld", "fidelity", "prov"): + if k in n and n[k] not in (None, {}, []): + rec[k] = n[k] + rec.update(values[n["id"]]) + return rec + + +def seal_receipts(spec, names, state_by_rev, final_state, key, wit_sha): + """Build and sign the receipts and revocations a source lists.""" + out = [] + by_name = {} + final_doc = final_state.doc() + final_vals = hashing.compute(final_doc) + for r in spec: + if "revoke" in r: + rec = {"format": "scienting-revocation/1", "receipt": by_name[r["revoke"]]["id"], "reason": r["reason"], + "by": r["by"], "at": r["at"]} + rec = RC.seal(rec, secret(r["by"]), r["by"]) + elif "qa_of" in r: + orig = by_name[r["qa_of"]] + seed = RC.make_seed(orig, secret(r["key"])) + sample = RC.sample(orig, seed) + hashes = dict((nid, h) for nid, h in orig["nodes"]) + errs = [{"node": names.ids[e["node"]], **{k: v for k, v in e.items() if k != "node"}} for e in r["errors"]] + rec = {"format": "scienting-receipt/1", "doc": key, "rev": orig["rev"], "level": orig["level"], + "scope": "qa:" + orig["id"], "nodes": [[nid, hashes[nid]] for nid in sample], + "evidence": [{"kind": "crop", "node": nid, "sha256": crop_sha(nid)} for nid in sample], + "by": r["by"], "at": r["at"], + "qa": {"of": orig["id"], "key": r["key"], "seed": base64.b64encode(seed).decode(), "rate": "0.05", + "minimum": 10, "sample": sample, "errors": errs}} + rec = RC.seal(rec, secret(r["by"]), r["by"]) + else: + rev = r.get("rev", final_state.rev) + st = state_by_rev[rev] + vals = hashing.compute(st.doc()) if rev != final_state.rev else final_vals + ids = sorted(names.ids[x] for x in r["nodes"]) + scope = r["scope"] + if scope.startswith("section:@"): + scope = "section:" + names.ids[scope[8:]] + rec = {"format": "scienting-receipt/1", "doc": key, "rev": rev, "level": r["level"], "scope": scope, + "nodes": [[i, vals[i]["node_hash"]] for i in ids], + "evidence": [{"kind": "crop", "node": i, "sha256": crop_sha(i)} for i in ids] + + [{"kind": "witness", "sha256": wit_sha}], + "by": r["by"], "at": r["at"]} + rec = RC.seal(rec, secret(r["by"]), r["by"]) + by_name[r["name"]] = rec + out.append(rec) + out.sort(key=lambda x: (x["at"], x["id"])) + return out, by_name + + +def build_document(src, used=None, extra_files=None): + # The keys are minted, never written by hand (spec/02 §2.4): the work key from the work's identity and its + # metadata, the document key from the work key and the version. + d = src["document"] + wkey = K.work_key(d["work"].get("same_as"), d["meta"]) + key = K.document_key(wkey, d["version"]) + d["key"], d["work"]["key"] = key, wkey + names = Names(src.get("name", key), key) + used = used if used is not None else set() + spec = expand(src["tree"]) + nodes, side = establish(key, spec, names, used) + aliases = [{"id": a, "kind": "alias", "successor": names.ids[t]} + for a, t in sorted(src.get("legacy_aliases", {}).items())] + state = CO.State(key, nodes, aliases) + states = {0: state} + records = [] + for c in src.get("corrections", []): + c = copy.deepcopy(c) + crop = c.pop("crop", None) + new_name = c.pop("new_name", None) + before = c.pop("before", None) + c["evidence"] = [{"kind": "crop", "sha256": crop_sha(names.ids.get(crop, crop))}] if crop else [ + {"kind": "person", "by": "ana", "note": "Read against the page."}] + c["author"] = {"key": "ana"} + for k in ("target", "parent", "place_after", "with"): + if k in c: + c[k] = names.resolve(c[k]) + if "node" in c: + nspec = expand(c["node"]) + flat = flatten_spec(nspec) + pre = mint_ids(key, flat, state.used_ids()) + for f, n in zip(flat, pre): + if "_name" in f: + names.ids[f["_name"]] = n["id"] + c["node"] = _strip_private(nspec, names) + if before is not None: + d = state.doc() + c["offset"] = hashing.plain(d, c["target"]).index(before) + state, rec = CO.apply(state, c, store=SCHEMAS) + if new_name: + names.ids[new_name] = rec["created"][0] + records.append(rec) + states[state.rev] = state + # A corrected leaf was checked against the page image: it is F1 at most until corroborated again. + final = state + touched = set() + for rec in records: + if any(e["kind"] == "crop" for e in rec["evidence"]): + touched |= set(rec["touched"]) | set(rec.get("created", [])) + for n in final.nodes: + if n["id"] in touched and n["kind"] in LEAVES: + n["fidelity"] = "F1" + for n in final.nodes: + if n["kind"] in LEAVES and "fidelity" not in n: + n["fidelity"] = "F1" + return key, names, side, final, states, records + + +def _strip_private(spec, names): + out = {k: v for k, v in spec.items() if not k.startswith("_") and k != "children"} + if out["kind"] == "row" and "attrs" in out: + out["attrs"] = {"header_path": names.resolve(out["attrs"]["header_path"])} + if out["kind"] == "cell": + out["attrs"] = dict(out["attrs"], column=names.resolve(out["attrs"]["column"])) + if spec.get("children"): + out["children"] = [_strip_private(c, names) for c in spec["children"]] + return out + + +def package_files(src, key, names, side, final, records, receipts=None, roster_rec=None, witnesses=None, + pages=None, boxes=None, extra=None, absent=()): + doc = final.doc() + vals = hashing.compute(doc) + nodes = [node_record(doc.nodes[nid], vals) for nid in doc.preorder()] + files = {} + files["nodes.jsonl"] = jsonl(nodes) + if pages: + files["pages.jsonl"] = jsonl([p for p, _ in sorted(pages, key=lambda x: (x[0]["witness"], x[0]["page"]))]) + locs = [] + fps = {p["page"]: p for p, _ in pages} + pos = {nid: i for i, nid in enumerate(doc.preorder())} + for nid, (pg, bx) in boxes.items(): + if nid in doc.nodes: + locs.append({"node": nid, "page_fp": fps[pg]["page_fp"], "page": pg, + "page_label": fps[pg]["page_label"], "boxes": bx}) + locs.sort(key=lambda l: (pos[l["node"]], l["page"], l["page_fp"])) + if locs: + files["locations.jsonl"] = jsonl(locs) + refs = [] + for nid in doc.preorder(): + s = side.get(nid, {}) + if doc.nodes[nid]["kind"] == "reference": + r = {"node": nid, "label": doc.nodes[nid].get("label")} + if "_csl" in s: + r["csl"] = s["_csl"] + res = s.get("_resolution", {"type": "none"}) + r["resolution"] = {k: names.resolve(v) for k, v in res.items()} + refs.append({k: v for k, v in r.items() if v is not None}) + if refs: + files["references.jsonl"] = jsonl(refs) + if records: + files["corrections.jsonl"] = jsonl(records) + ids = sorted(final.ids, key=lambda e: (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"])) + if ids: + files["ids.jsonl"] = jsonl(ids) + if receipts: + files["receipts.jsonl"] = jsonl(receipts) + if roster_rec: + files["roster.json"] = (jcs.dumps(roster_rec) + "\n").encode() + files["witnesses.json"] = (jcs.dumps({"witnesses": witnesses}) + "\n").encode() + files["provenance.json"] = (jcs.dumps({ + "tools": [{"name": "paperspec tools/build.py", "version": VERSION}], + "stages": [{"stage": "hand", "processor": "local", "tool": "paperspec tools/build.py"}], + "note": "Built by hand for the conformance suite; every reading is synthetic."}) + "\n").encode() + levels = RC.effective_levels(nodes, vals, receipts or [], ROSTER_BY_ID if roster_rec else {}) + d = dict(src["document"]) + d = {"format": "scienting-doc/1", **d} + d.update({"hashing": {"scheme": "plain", "algorithm": "sha-256"}, "root": vals[doc.root]["node_hash"], "rev": final.rev, + "history": crypto.history_root(history_items(records or [])), + "counts": {"nodes": len(nodes), "leaves": sum(1 for n in nodes if n["kind"] in LEAVES)}, + "fidelity": RC.distribution(levels)}) + d["meta_hash"] = hashing.meta_hash(d) + files["document.json"] = (jcs.dumps(d) + "\n").encode() + if extra: + files.update(extra) + return files, nodes, vals, d + + +def build_hale(out): + src = json.load(open(os.path.join(ROOT, "fixtures", "src", "hale2026harmonic.json"))) + src["name"] = "hale2026harmonic" + image = png(4, 3, (40, 90, 160)) + image_sha = sha256_hex(image) + key, names, side, final, states, records = build_document(src) + side[names.ids["@pa"]]["_image"] = image_sha + doc = final.doc() + pa = doc.nodes[names.ids["@pa"]] + pa["attrs"]["image"] = {"sha256": image_sha, "media_type": "image/png", "native": True} + pa["attrs"]["crop"] = {"sha256": crop_sha("panel a"), "media_type": "image/png", "dpi": 300} + P = src["pages"] + wsha = H(P["witness"]["sha_seed"]) + wid = witness_id(wsha) + for n in final.nodes: + if n["kind"] in LEAVES: + n["prov"] = {"witnesses": [wid], "backend": "docling (illustrative)"} + if n.get("fidelity") == "F2": + n["prov"]["corroborated_by"] = [wid] + pages, boxes = page_layout(final.doc(), side, src, wid) + p2 = [p for p, _ in pages if p["page"] == 2][0] + for t in pa["attrs"]["figure_text"]: + t["page_fp"] = p2["page_fp"] + t["boxes"] = [[90.0, 300.0, 140.0, 310.0]] + witnesses = [{"id": wid, "sha256": wsha, "size": P["witness"]["size"], "kind": "pdf", "media_type": "application/pdf", + "pages": len(P["labels"]), "text_layer": True, "version_label": "version of record", + "origin": {"source": "publisher", "doi": "10.5555/example.hale.2026", "retrieved_at": "2026-10-01T08:00:00Z"}, + "rights": rights_record("subscription")}] + receipts, by_name = seal_receipts(src["receipts"], names, states, final, key, wsha) + # The dataset reported from Table 1. + final_doc = final.doc() + vals = hashing.compute(final_doc) + rows = [("ct-ct", "CT–CT", "@cc"), ("ct-hc", "CT–HC", "@ch2"), ("ow-hw", "OW–HW", "@ow")] + csv = ["key,bond,set,k_r,r_eq"] + anchors = [] + for rk, bond, pre in rows: + for st, (kc, rc) in (("A", ("_ak", "_ar")), ("B", ("_bk", "_br"))): + kid, rid = names.ids[pre + kc], names.ids[pre + rc] + kv, rv = norm(hashing.plain(final_doc, kid)), norm(hashing.plain(final_doc, rid)) + row_key = rk + "-" + st.lower() + csv.append("%s,%s,%s,%s,%s" % (row_key, bond, st, kv, rv)) + for field, nid in (("k_r", kid), ("r_eq", rid)): + anchors.append({"resource": "bond-parameters", "row": row_key, "field": field, + "anchor": AN.make_anchor(final_doc, vals, nid, final.rev, store=STORE)}) + ds_id = "ds-hale-bond-params" + datapackage = {"format": "scienting-dataset/1", "id": ds_id, "name": "hale-bond-params", + "title": "Harmonic bond parameters reported in Table 1", + "resources": [{"name": "bond-parameters", "path": "data.csv", "format": "csv", "encoding": "utf-8", + "schema": {"fields": [ + {"name": "key", "type": "string"}, {"name": "bond", "type": "string"}, + {"name": "set", "type": "string"}, + {"name": "k_r", "type": "number", "unit": "kcal / mol / angstrom ** 2"}, + {"name": "r_eq", "type": "number", "unit": "angstrom", + "uncertainty": {"kind": "unknown"}}], + "primaryKey": "key"}, + "origin": {"kind": "reported", "anchors": "anchors.jsonl"}}]} + eq = names.ids["@eq1"] + mathml = ('Ebond=12' + 'Kr(r−r' + 'eq)2') + extra = { + "assets/%s.png" % image_sha: image, + "datasets/%s/datapackage.json" % ds_id: (jcs.dumps(datapackage) + "\n").encode(), + "datasets/%s/data.csv" % ds_id: ("\n".join(csv) + "\n").encode(), + "datasets/%s/anchors.jsonl" % ds_id: jsonl(anchors), + "derived/mathml.jsonl": jsonl([{"node": eq, "run": None, "tool": {"name": "hand", "version": "0"}, + "mathml": mathml}]), + } + absent = [{"path": "assets/%s.png" % crop_sha("panel a"), "sha256": crop_sha("panel a"), "size": 18213, + "present": False}] + files, nodes, vals, d = package_files(src, key, names, side, final, records, receipts, ROSTER, witnesses, pages, + boxes, extra, absent) + write(os.path.join(out, "fixtures", "packages", key), files, absent) + return {"src": src, "key": key, "names": names, "states": states, "final": final, "records": records, + "receipts": receipts, "by_name": by_name, "pages": pages, "files": files, "nodes": nodes, "values": vals, + "document": d, "witness": witnesses[0], "absent": absent} + + +def build_checks_only(out, hale): + """The same document exported to another machine of the team: text withheld, hashes keyed.""" + final = hale["final"] + doc = final.doc() + k = crypto.team_key(TEAM_SECRET) + keyed = hashing.compute(doc, hashing.Hasher(k)) + nodes = [] + for nid in doc.preorder(): + n = copy.deepcopy(doc.nodes[nid]) + if n["kind"] in LEAVES: + n.pop("content", None) + n["withheld"] = True + if n["kind"] == "panel": + n["attrs"].pop("figure_text", None) + nodes.append(node_record(n, keyed)) + f = dict(hale["files"]) + for p in ("corrections.jsonl", "receipts.jsonl", "roster.json", "derived/mathml.jsonl"): + f.pop(p, None) + for p in [p for p in f if p.startswith("datasets/")]: + f.pop(p) + f["nodes.jsonl"] = jsonl(nodes) + refs = [{"node": r["node"], "label": r["label"], "resolution": r["resolution"]} + for r in map(json.loads, f["references.jsonl"].decode().splitlines())] + f["references.jsonl"] = jsonl(refs) + d = dict(hale["document"]) + d.update({"policy": "checks-only", "hashing": {"scheme": "hmac-team", "algorithm": "sha-256", "key_id": crypto.key_id(k)}, + "root": keyed[doc.root]["node_hash"], "history": crypto.EMPTY_ROOT}) + d["meta_hash"] = hashing.meta_hash(d, hashing.Hasher(k)) + f["document.json"] = (jcs.dumps(d) + "\n").encode() + write(os.path.join(out, "fixtures", "packages", hale["key"] + ".checks-only"), f, hale["absent"]) + + +def build_simple(out, name, used=None): + src = json.load(open(os.path.join(ROOT, "fixtures", "src", name + ".json"))) + src["name"] = name + key, names, side, final, states, records = build_document(src) + witnesses = [] + pages = boxes = None + if "pages" in src: + P = src["pages"] + wsha = H(P["witness"]["sha_seed"]) + wid = witness_id(wsha) + jsha = H(P["jats"]["sha_seed"]) + jid = witness_id(jsha) + for n in final.nodes: + if n["kind"] in LEAVES: + n["prov"] = {"witnesses": [jid, wid], "backend": "jats reader (illustrative)", "corroborated_by": [wid]} + pages, boxes = page_layout(final.doc(), side, src, wid) + witnesses = [ + {"id": jid, "sha256": jsha, "size": P["jats"]["size"], "kind": "jats", "media_type": "application/xml", + "version_label": "version of record", + "origin": {"source": "repository", "doi": "10.5555/example.okafor.2025"}, "rights": rights_record("open")}, + {"id": wid, "sha256": wsha, "size": P["witness"]["size"], "kind": "pdf", "media_type": "application/pdf", + "pages": len(P["labels"]), "text_layer": True, "version_label": "version of record", + "origin": {"source": "publisher", "doi": "10.5555/example.okafor.2025"}, "rights": rights_record("open")}] + else: + T = src["typst"] + tsha = H(T["sha_seed"]) + tid = witness_id(tsha) + for n in final.nodes: + if n["kind"] in LEAVES: + n["prov"] = {"witnesses": [tid], "backend": "typst reader (illustrative)"} + witnesses = [{"id": tid, "sha256": tsha, "size": T["size"], "kind": "typst", "media_type": "application/zip", + "version_label": "draft 3", "origin": {"source": "team"}, "rights": rights_record("own")}] + files, nodes, vals, d = package_files(src, key, names, side, final, records, None, None, witnesses, pages, boxes) + write(os.path.join(out, "fixtures", "packages", key), files) + return {"key": key, "names": names, "final": final, "files": files, "values": vals, "src": src} + + +# --- the tiny document used for corrections vectors and invalid packages ----------------- + +TINY = {"name": "tiny2026example", + "document": {"kind": "article", "work": {"same_as": ["doi:10.5555/example.tiny.2026"]}, + "version": {"type": "version-of-record"}, "relations": [], + "meta": {"title": "A tiny example", "author": [{"family": "Tiny", "given": "T."}], + "issued": {"date-parts": [[2026]]}}, + "language": "en", "license": {"spdx": "CC0-1.0"}, "policy": "full"}, + "tree": {"kind": "document", "children": [ + {"kind": "section", "name": "@sec", "label": "1", "level": 1, "title": "Results", "children": [ + {"name": "@p1", "p": [ + {"name": "@a", "s": ["The value is 42.0 at 300 K."], "fidelity": "F2"}, + {"name": "@b", "s": ["See ", {"xref": "@t", "text": "Table 1"}, " and the note.", {"cite": ["@r"], "text": "1"}], + "fidelity": "F2"}]}, + {"name": "@p2", "p": [{"name": "@c", "s": ["A second paragraph."], "fidelity": "F2"}, + {"name": "@d", "s": ["It ends here."], "fidelity": "F2"}]}, + {"kind": "table", "name": "@t", "label": "Table 1", + "columns": [{"name": "@cx", "header_path": ["@hx"]}, {"name": "@cy", "header_path": ["@hy"]}], + "children": [ + {"kind": "table-head", "children": [{"row": [ + {"name": "@hx", "c": ["x"], "role": "column-header", "col": "@cx"}, + {"name": "@hy", "c": ["y"], "role": "column-header", "col": "@cy"}]}]}, + {"kind": "table-body", "name": "@tb", "children": [ + {"name": "@row1", "header_path": ["@x1"], "row": [ + {"name": "@x1", "c": ["1"], "role": "row-header", "col": "@cx"}, + {"name": "@y1", "c": ["2.5"], "col": "@cy"}]}]}]}]}, + {"kind": "back", "children": [{"kind": "references", "children": [ + {"kind": "reference", "name": "@r", "label": "1", "runs": ["1. An Example, 2020."], + "resolution": {"type": "none"}, "fidelity": "F2"}]}]}]}} + + +def tiny_package(out_dir, receipts=True): + src = copy.deepcopy(TINY) + key, names, side, final, states, records = build_document(src) + wsha = H("tiny2026example: a file (synthetic)") + wit = [{"id": witness_id(wsha), "sha256": wsha, "size": 1000, "kind": "pdf", "media_type": "application/pdf", + "pages": 1, "origin": {"source": "author"}, "rights": rights_record("own")}] + recs = None + if receipts: + recs, _ = seal_receipts([{"name": "t1", "by": "ana", "level": "F3", "scope": "all", "at": "2026-10-11T12:00:00Z", + "nodes": ["@a", "@b", "@c", "@d", "@x1", "@y1", "@hx", "@hy", "@r"]}], + names, states, final, key, wsha) + files, nodes, vals, d = package_files(src, key, names, side, final, records, recs, ROSTER if receipts else None, + wit) + if out_dir: + write(out_dir, files) + return {"key": key, "names": names, "final": final, "files": files, "values": vals} + + +# --- vectors -------------------------------------------------------------------- + + +def wv(out, name, cases): + path = os.path.join(out, "vectors", name) + with open(path, "wb") as f: + f.write(jsonl(cases)) + + +def sentence_doc(runs, key="vectors"): + """A one-sentence document, so projections of runs can be computed with the node functions.""" + nodes = [{"id": "aaaaaaa", "kind": "document", "parent": None, "ord": 0}, + {"id": "bbbbbbb", "kind": "section", "parent": "aaaaaaa", "ord": 0, "attrs": {"level": 1}}, + {"id": "ccccccc", "kind": "paragraph", "parent": "bbbbbbb", "ord": 0}, + {"id": "ddddddd", "kind": "sentence", "parent": "ccccccc", "ord": 0, "content": runs}] + return Doc(key, nodes), "ddddddd" + + +def projection_case(cid, runs, note=None): + doc, s = sentence_doc(runs) + strict, cx = hashing.strict(doc, s) + lossy = hashing.lossy_from_strict(strict) + plain = hashing.plain(doc, s) + c = {"id": cid, "runs": runs, "plain": plain, "normalized": norm(plain), "strict": strict, "lossy": lossy, + "complex": cx, "text_hash": H(norm(plain)), "match_hash": H(strict), "align_key": H(lossy)} + if note: + c["note"] = note + return c + + +T = lambda t, **kw: dict({"kind": "text", "text": t}, **({"style": kw["style"]} if "style" in kw else {})) # noqa: E731 +M = lambda s, n="tex": {"kind": "math", "notation": n, "source": s} # noqa: E731 + + +def parse_doubles(text): + """Parse JSON as an ECMAScript engine does: every number is an IEEE 754 double.""" + return json.loads(text, parse_int=lambda x: int(x) if abs(int(x)) <= 2**53 else float(x)) + + +def vectors(out, hale, tiny): + # Canonical JSON. + cj = [ + ("sort-utf16", '{"\\u20ac": 1, "\\ud83d\\ude00": 2, "a": 3, "\\ufb01": 4}'), + ("numbers", '[0, -0.0, 1.0, 100.0, 72.10, 1E21, 1e-7, 0.000001, 123456789012345678901, -2.5e-5, 4.50e2]'), + ("escapes", '{"s": "tab\\tnl\\ncr\\rq\\"bs\\\\ctl\\u0001del\\u007fnbsp\\u00a0slash\\/"}'), + ("nested", '{ "b" : [ true , false , null ] , "a" : { "z" : "" , "y" : [ ] } }'), + ("strings-stay-unnormalized", '{"cafe\\u0301": "e\\u0301", "caf\\u00e9": "\\u00e9"}'), + ] + wv(out, "canonical-json.jsonl", [{"id": i, "input_json": t, "output": jcs.dumps(parse_doubles(t))} for i, t in cj]) + # Normalization for hashing. + ns = [("nfc", "cafe\u0301"), ("collapse", "a \t\n b"), ("trim", "\u00a0 a b \u3000"), + ("angstrom-sign", "1.5 \u212b"), ("nbsp-thin", "300\u2009K and 10\u00a0mol"), + ("zero-width-kept", "a\u200bb"), ("ligature-kept", "\ufb01t"), ("micro-kept", "5 \u00b5m"), + ("line-separator", "one\u2028two")] + wv(out, "normalize.jsonl", [{"id": i, "input": s, "output": norm(s), "sha256": H(norm(s))} for i, s in ns]) + # IDs. + idc = [] + for doc_key, kind, th, occ in [("cornell1995second", "sentence", H("A sentence."), 0), + ("cornell1995second", "sentence", H("A sentence."), 1), + ("cornell1995second.si", "sentence", H("A sentence."), 0), + (hale["key"], "cell", H("310.0"), 0), + (hale["key"], "column", H("Set A $K_r$ (kcal mol⁻¹ Å⁻²)"), 0)]: + payload = jcs.dumps(["scienting-id/1", doc_key, kind, th, occ]) + idc.append({"id": "mint-%s-%s-%d" % (doc_key, kind, occ), "doc": doc_key, "kind": kind, "text_hash": th, + "occurrence": occ, "payload": payload, "sha256": H(payload), + "node_id": mint_node_id(doc_key, kind, th, occ)}) + for hx in ("00" * 32, "ff" * 32, H("x")): + idc.append({"id": "letter-first-" + hx[:8], "digest": hx, "node_id": letter_first(bytes.fromhex(hx), 7), + "record_id": record_id(hx)}) + wv(out, "ids.jsonl", idc) + # Notation. + notes = [("tex", "H_2O"), ("tex", "_2"), ("tex", "\\mathrm{H}_{2}\\mathrm{O}"), ("typst", "H_2 O"), + ("typst", "upright(H)_2 upright(O)"), ("tex", "\\mathrm{cm}^{-1}"), ("typst", "\"cm\"^(-1)"), + ("tex", "10^{-3}"), ("typst", "10^(-3)"), ("tex", "K_r"), ("tex", "K_{\\mathrm{r}}"), ("typst", "K_r"), + ("tex", "\\alpha_i"), ("tex", "\\alpha_{i}"), ("tex", "x'_1"), ("tex", "x^{\\prime}"), + ("tex", "T = 300\\,\\mathrm{K}"), ("tex", "-1.5"), ("tex", "\\epsilon"), ("typst", "epsilon.alt"), + ("tex", "a \\cdot b"), ("typst", "a dot b"), ("tex", "a \\times b \\pm c"), ("tex", "\\left(x\\right)^2"), + ("typst", "(a+b)^2"), ("tex", "\\frac{a}{b}"), ("typst", "a/b"), ("tex", "\\sqrt{2}"), + ("tex", "\\mathbf{F}"), ("tex", "{ab}_2"), ("tex", "x_1_2"), ("tex", "\\Delta G^\\circ"), + ("tex", "E_{\\mathrm{bond}} = \\frac{1}{2} K_r (r - r_{\\mathrm{eq}})^2"), ("typst", "E = K_r (r - r_\"eq\")^2"), + # A script after an empty group applies to the atom before it, or to an empty base (the isotope idiom). + ("tex", "{}^{14}C"), ("tex", "x{}^2"), ("tex", "{}_2"), ("tex", "\\left)x\\right("), + # Letters are the Latin and Greek blocks: a fullwidth or modifier letter is complex, a Coptic letter too. + ("tex", "Ａ"), ("tex", "ʰ"), ("tex", "Ϣ"), ("tex", "ἀω")] + wv(out, "notation.jsonl", [{"id": "%s:%s" % (n, s), "notation": n, "source": s, "canonical": fold_math(n, s)} + for n, s in notes]) + # Projections. + pc = [ + projection_case("h2o-math", [T("The solvent is "), M("H_2O"), T(".")]), + projection_case("h2o-split-math", [T("The solvent is H"), M("_2"), T("O.")]), + projection_case("h2o-mathrm", [T("The solvent is "), M("\\mathrm{H}_{2}\\mathrm{O}"), T(".")]), + projection_case("h2o-sub", [T("The solvent is H"), T("2", style=["sub"]), T("O.")]), + projection_case("h2o-unicode", [T("The solvent is H\u2082O.")]), + projection_case("h2o-typst", [T("The solvent is "), M("H_2 O", "typst"), T(".")]), + projection_case("wavenumber-unicode", [T("A band near 1595 cm\u207b\u00b9.")]), + projection_case("wavenumber-sup", [T("A band near 1595 cm"), T("-1", style=["sup"]), T(".")]), + projection_case("wavenumber-math", [T("A band near 1595 "), M("\\mathrm{cm}^{-1}"), T(".")]), + projection_case("wavenumber-typst", [T("A band near 1595 "), M("\"cm\"^(-1)", "typst"), T(".")]), + projection_case("milli-unicode", [T("Values below 10\u207b\u00b3 were dropped.")]), + projection_case("milli-sup", [T("Values below 10"), T("\u22123", style=["sup"]), T(" were dropped.")]), + projection_case("milli-math", [T("Values below "), M("10^{-3}"), T(" were dropped.")]), + projection_case("kr-math", [T("The constant "), M("K_r"), T(" is large.")]), + projection_case("kr-mathrm", [T("The constant "), M("K_{\\mathrm{r}}"), T(" is large.")]), + projection_case("kr-sub", [T("The constant K"), T("r", style=["sub"]), T(" is large.")]), + projection_case("kr-unicode", [T("The constant K\u1d63 is large.")]), + projection_case("ligature", [T("The \ufb01t is good.")]), + projection_case("no-ligature", [T("The fit is good.")]), + projection_case("micro-sign", [T("A 5 \u00b5m film.")]), + projection_case("greek-mu", [T("A 5 \u03bcm film.")]), + projection_case("lost-superscript", [T("A rate of 103 per second.")], "OCR lost the superscript."), + projection_case("superscript", [T("A rate of 10\u00b3 per second.")]), + projection_case("cite-mark", [T("The energy"), {"kind": "cite", "refs": [], "text": "3"}, T(" is low.")]), + projection_case("cite-as-text", [T("The energy3 is low.")]), + projection_case("complex", [T("The energy is "), M("\\frac{1}{2} k x^2"), T(" exactly.")]), + projection_case("hyphen-before-digit", [T("pages 10-20 and CT\u2013CT with 6-31G*")]), + projection_case("en-dash-before-digit", [T("pages 10\u201320 and CT-CT with 6\u201331G*")]), + projection_case("astral", [T("Let "), M("\U0001d465"), T(" vary.")]), + ] + groups = [ + {"id": "h2o", "same": "match_hash", + "cases": ["h2o-math", "h2o-split-math", "h2o-mathrm", "h2o-sub", "h2o-unicode", "h2o-typst"]}, + {"id": "wavenumber", "same": "match_hash", + "cases": ["wavenumber-unicode", "wavenumber-sup", "wavenumber-math", "wavenumber-typst"]}, + {"id": "milli", "same": "match_hash", "cases": ["milli-unicode", "milli-sup", "milli-math"]}, + {"id": "kr", "same": "match_hash", "cases": ["kr-math", "kr-mathrm", "kr-sub", "kr-unicode"]}, + {"id": "ligature", "same": "match_hash", "cases": ["ligature", "no-ligature"]}, + {"id": "micro", "same": "match_hash", "cases": ["micro-sign", "greek-mu"]}, + {"id": "103-align", "same": "align_key", "cases": ["lost-superscript", "superscript"]}, + {"id": "103-match", "differ": "match_hash", "cases": ["lost-superscript", "superscript"]}, + {"id": "cite-structure", "differ": "match_hash", "cases": ["cite-mark", "cite-as-text"]}, + {"id": "dashes-align", "same": "align_key", "cases": ["hyphen-before-digit", "en-dash-before-digit"]}, + {"id": "dashes-text", "differ": "text_hash", "cases": ["hyphen-before-digit", "en-dash-before-digit"]}, + ] + byid = {c["id"]: c for c in pc} + for g in groups: + vals = {byid[c][g.get("same") or g.get("differ")] for c in g["cases"]} + assert (len(vals) == 1) if "same" in g else (len(vals) == len(g["cases"])), g + wv(out, "projections.jsonl", pc) + wv(out, "projection-groups.jsonl", groups) + # Offsets. + off = [] + for cid, runs, span in [("bmp", [T("A sentence about H"), M("_2"), T("O.")], [18, 24]), + ("astral", [T("Let "), M("\U0001d465"), T(" vary with "), M("\U0001d466"), T(".")], [4, 7])]: + doc, s = sentence_doc(runs) + p = hashing.plain(doc, s) + from tools.oracle.text import cp_to_utf16 + off.append({"id": cid, "runs": runs, "plain": p, "span": span, "exact": p[span[0]:span[1]], + "utf16_span": [cp_to_utf16(p, span[0]), cp_to_utf16(p, span[1])], "code_points": len(p), + "utf16_units": cp_to_utf16(p, len(p))}) + wv(out, "offsets.jsonl", off) + # Five values on a small tree: the tiny document. + tnodes = [{k: v for k, v in n.items() if k not in ("text_hash", "match_hash", "align_key", "own_hash", "node_hash")} + for n in map(json.loads, tiny["files"]["nodes.jsonl"].decode().splitlines())] + tdoc = Doc(tiny["key"], tnodes) + tv = hashing.compute(tdoc) + own = {nid: hashing.own_content(tdoc, nid, lambda x: tv[x]["text_hash"]) for nid in tdoc.order} + wv(out, "hashes.jsonl", [{"id": "tiny2026example", "doc": tiny["key"], "nodes": tnodes, + "own_content": own, "values": tv}]) + # Addresses. The example keys are real work keys: Cornell et al. (1995) by its DOI, a fictional preprint. + cornell = "cornell1995second-" + K.suffix("doi:10.1021/ja00124a002") + dong = "dong2026machine-" + K.suffix("doi:10.5555/example.dong.2026") + ad = [("plain", cornell + "#k3m9q2a"), ("pinned", cornell + "#k3m9q2a@4f1a0c"), + ("span", cornell + "#k3m9q2a@4f1a0c[12:57]"), ("store", "smithlab/" + cornell + "#k3m9q2a"), + ("span-no-pin", dong + ".v2#k3m9q2a[0:0]"), ("si", cornell + ".si#m2x8p4d"), + ("alias", cornell + "#b-p03-text-004"), ("bad-reversed", cornell + "#k3m9q2a[57:12]"), + ("bad-pin", cornell + "#k3m9q2a@4F1A0C"), ("bad-leading-zero", "k-0000#k3m9q2a[01:2]"), + ("bad-no-node", cornell + "#"), ("bad-upper-key", "Cornell1995-k3f9#k3m9q2a"), + ("bad-two-stores", "a/b/c-0000#k3m9q2a"), ("bad-pin-length", "k-0000#k3m9q2a@4f1a"), + ("bad-no-hash", "cornell1995second#k3m9q2a"), ("bad-hash-alphabet", "cornell1995second-k3fi#k3m9q2a"), + ("bad-short-hash", "cornell1995second-k3f#k3m9q2a")] + wv(out, "addresses.jsonl", [{"id": i, "address": a, "parsed": AN.parse_address(a)} for i, a in ad]) + # Keyed hashes. + k = crypto.team_key(TEAM_SECRET) + fps = [p["page_fp"] for p, _ in hale["pages"]] + pk = crypto.page_key(fps, "layer:smithlab/kp4m2x7q") + text = "310.0" + keyed = [ + {"id": "team-key", "secret": TEAM_SECRET.hex(), "key": k.hex(), "key_id": crypto.key_id(k)}, + {"id": "team-hmac-text", "key": k.hex(), "text": text, "text_hash_plain": H(text), + "text_hash_keyed": hashing.Hasher(k)(text)}, + {"id": "page-hashes", "page_fps": fps, "page_hashes": [crypto.page_hash(f) for f in fps]}, + {"id": "page-key", "page_fps": fps, "context": "layer:smithlab/kp4m2x7q", "key": pk.hex(), + "quote": "310.0", "quote_mac": hashing.Hasher(pk)(norm(text))}, + {"id": "non-holder", "note": "Without the pages, a guess hashes to the plain value, which the export never " + "carries; with one page changed, the key differs.", + "guess": text, "guess_sha256": H(text), + "wrong_page_fps": [fps[0], H("another page")], + "wrong_key": crypto.page_key([fps[0], H("another page")], "layer:smithlab/kp4m2x7q").hex(), + "quote_mac": hashing.Hasher(pk)(norm(text))}, + ] + assert keyed[4]["wrong_key"] != pk.hex() and keyed[4]["guess_sha256"] != keyed[4]["quote_mac"] + wv(out, "keyed.jsonl", keyed) + # Fingerprints: two downloads of one page, stamped differently. + stamps = json.load(open(os.path.join(ROOT, "registry", "stamps", "scienting-stamps-1.json")))["rules"] + _, p1 = hale["pages"][0] + body = [l for l in p1["lines"] if "box" not in l] + acs1 = dict(p1) + acs2 = dict(p1, lines=body + [ + {"text": "Downloaded via EXAMPLE PHARMA INC on March 2, 2027 at 09:14:55 (UTC).", "box": [8.0, 210.0, 20.0, 590.0]}, + {"text": "See https://pubs.acs.org/sharingguidelines for options on how to legitimately share published articles.", + "box": [580.0, 200.0, 592.0, 600.0]}]) + wiley = {"size": [595.276, 841.89], "images": [H("figure 2 image stream (synthetic)")], "lines": [ + {"text": "Results"}, {"text": "The fitted slope is 0.42 \u00b1 0.03 per kelvin."}, + {"text": "Downloaded from https://onlinelibrary.wiley.com/doi/10.5555/example.wiley by Example University, " + "Wiley Online Library on [11/10/2026]. See the Terms and Conditions (https://onlinelibrary.wiley.com/" + "terms-and-conditions) on Wiley Online Library for rules of use.", "box": [580.0, 100.0, 592.0, 740.0]}]} + wiley2 = dict(wiley, lines=wiley["lines"][:2] + [ + {"text": "Downloaded from https://onlinelibrary.wiley.com/doi/10.5555/example.wiley by Example Pharma Inc, " + "Wiley Online Library on [02/03/2027].", "box": [580.0, 100.0, 592.0, 740.0]}, + {"text": "See the Terms and Conditions (https://onlinelibrary.wiley.com/terms-and-conditions) on Wiley Online " + "Library for rules of use.", "box": [580.0, 60.0, 592.0, 99.0]}]) + changed = dict(acs1, lines=[dict(l, text=l["text"].replace("1595", "1594")) for l in acs1["lines"]]) + scan = {"size": [612.4, 791.6], "lines": [], "images": [H("scanned page image (synthetic)")]} + fpc = [] + for cid, pg in [("acs-download-1", acs1), ("acs-download-2", acs2), ("acs-changed-text", changed), + ("wiley-download-1", wiley), ("wiley-download-2", wiley2), ("scanned", scan)]: + fp, removed = FP.fingerprint(pg, stamps) + kept = [l["text"] for l in pg["lines"] if not FP.is_stamp(l, pg["size"], stamps)] + fpc.append({"id": cid, "page": pg, "rules": "scienting-stamps-1", "removed": removed, + "text": FP.fp_text(" ".join(kept)), "page_fp": fp}) + assert fpc[0]["page_fp"] == fpc[1]["page_fp"] != fpc[2]["page_fp"] and fpc[3]["page_fp"] == fpc[4]["page_fp"] + wv(out, "fingerprints.jsonl", fpc) + # Merkle roots. + mk = [] + for n in (1, 2, 3, 4, 5, 7): + items = [["node", mint_node_id("m", "sentence", H(str(i)), 0), H("h%d" % i)] for i in range(n)] + mk.append({"id": "items-%d" % n, "items": items, "root": crypto.merkle_root(items)}) + wv(out, "merkle.jsonl", mk) + # Signatures. + r1 = hale["by_name"]["r1"] + sk = bytes.fromhex("9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60") + wv(out, "signatures.jsonl", [ + {"id": "rfc8032-test-1", "secret": sk.hex(), "public": crypto.ed25519_public(sk).hex(), "message": "", + "signature": crypto.ed25519_sign(sk, b"").hex()}, + {"id": "receipt-r1", "receipt": r1, "record_hash": RC.record_hash(r1), + "payload": RC.signing_payload(r1).decode(), "public": ROSTER_BY_ID["ana"]["public"], + "record_id": r1["id"]}, + {"id": "did-key-ed25519", "scheme": "ed25519", "public": crypto.ed25519_public(sk).hex(), + "did": crypto.did_key("ed25519", crypto.ed25519_public(sk))}, + {"id": "did-key-ml-dsa-65", "scheme": "ml-dsa-65", "public": mldsa.public_key(sk).hex(), + "did": crypto.did_key("ml-dsa-65", mldsa.public_key(sk))}, + # FIPS 204: NIST's ACVP key-generation case 26 for ML-DSA-65 (seed to public key), and our own + # deterministic signature over the empty message for the RFC 8032 test key used as a seed. + {"id": "ml-dsa-65-nist-keygen-26", + "seed": NIST_SEED, + "public": NIST_PK}, + {"id": "ml-dsa-65-sign-empty", "seed": sk.hex(), "public": mldsa.public_key(sk).hex(), "message": "", + "signature": mldsa.sign(mldsa.keygen(sk)[1], b"").hex()}]) + # Spot-checks. + q1 = hale["by_name"]["q1"] + wv(out, "spotcheck.jsonl", [{"id": "q1", "receipt": r1, "seed_payload": RC.seed_payload(r1).decode(), + "checker_public": ROSTER_BY_ID["ben-spot"]["public"], "seed": q1["qa"]["seed"], + "beta": vrf.proof_to_hash(base64.b64decode(q1["qa"]["seed"])).hex(), + "n": len(r1["nodes"]), "rate": "0.05", "minimum": 10, + "size": RC.sample_size(len(r1["nodes"])), "sample": q1["qa"]["sample"]}] + + [{"id": "size-%d" % n, "n": n, "rate": "0.05", "minimum": 10, "size": RC.sample_size(n)} + for n in (3, 10, 199, 200, 201, 3000)]) + # Bounds. + wv(out, "bounds.jsonl", [{"id": "k%d-n%d" % (kk, nn), "errors": kk, "sampled": nn, "confidence": "0.95", + "bound": RC.bound_string(kk, nn), "approx": round(RC.upper_bound(kk, nn), 9)} + for kk, nn in [(0, 10), (0, 300), (1, 10), (1, 300), (2, 50), (5, 100), (0, 1), (10, 10)]]) + # Keys (spec/02 §2.4): Scholar parts, identities and suffixes, including the cases paperstack pins. + attn = {"title": "Attention is all you need", "author": [{"family": "Vaswani", "given": "Ashish"}], + "issued": {"date-parts": [[2017]]}} + kc = [] + for cid, same_as, meta in [ + ("doi", ["doi:10.48550/arXiv.1706.03762"], attn), + ("openalex", ["openalex:W2963403868"], attn), + ("doi-before-arxiv", ["arxiv:1706.03762", "doi:10.48550/arXiv.1706.03762"], attn), + ("arxiv-only", ["arxiv:1706.03762"], attn), + ("url-is-not-an-identity", ["url:https://example.org/paper"], attn), + ("meta", [], {"title": "On the Origin of Species", "author": [{"family": "Darwin", "given": "Charles"}], + "issued": {"date-parts": [[1859]]}}), + ("doi-case", ["doi:10.1109/CVPR.2016.90"], {"title": "Deep Residual Learning for Image Recognition", + "author": [{"family": "He", "given": "Kaiming"}], + "issued": {"date-parts": [[2016]]}}), + ("accents", [], {"title": "A new method", "author": [{"literal": "M\u00fcller, J\u00fcrgen"}], + "issued": {"date-parts": [[2001]]}}), + ("apostrophe", [], {"title": "Don't stop", "author": [{"literal": "Conan O'Brien"}], + "issued": {"date-parts": [[2020]]}}), + ("name-suffix", [], {"title": "3D printing of bone", "author": [{"literal": "A. Smith Jr."}], + "issued": {"date-parts": [[2015]]}}), + ("only-stop-words", [], {"title": "The of a", "author": [{"family": "Lee", "given": "Jo"}], + "issued": {"date-parts": [[2019]]}}), + ("no-year", [], {"title": "Untitled", "author": [{"family": "Lee"}]}), + ("no-author", [], {"title": "Fluids", "issued": {"date-parts": [[2020]]}}), + ("non-latin-author", [], {"title": "Fluids", "author": [{"family": "\u5f20\u4f1f"}], "issued": {"date-parts": [[2020]]}}), + ("hale", hale["src"]["document"]["work"]["same_as"], hale["src"]["document"]["meta"]), + ]: + ident = K.identity(same_as, meta) + kc.append({"id": cid, "same_as": same_as, "meta": meta, "scholar": K.scholar_part(meta), "identity": ident, + "payload": K.SCHEME + ":" + ident, "sha256": H(K.SCHEME + ":" + ident), "suffix": K.suffix(ident), + "suffix_8": K.suffix(ident, 8), "key": K.work_key(same_as, meta)}) + hk = hale["document"]["work"]["key"] + for cid, version in [("version-of-record", {"type": "version-of-record"}), ("supplement", {"type": "supplement"}), + ("preprint-v2", {"type": "preprint", "seq": 2}), ("draft-v3", {"type": "draft", "seq": 3}), + ("erratum", {"type": "erratum"}), ("accepted-no-seq", {"type": "accepted"})]: + kc.append({"id": "document-" + cid, "work_key": hk, "version": version, "document_key": K.document_key(hk, version)}) + kc.append({"id": "collision", "same_as": attn and ["doi:10.48550/arXiv.1706.03762"], "meta": attn, + "taken": [K.work_key(["doi:10.48550/arXiv.1706.03762"], attn)], + "key": K.unique_work_key(["doi:10.48550/arXiv.1706.03762"], attn, {K.work_key(["doi:10.48550/arXiv.1706.03762"], attn)})}) + wv(out, "keys.jsonl", kc) + # Proofs (RFC 9162): inclusion for every item of the Merkle vectors, consistency for every prefix, and the + # example paper's correction history. + pf = [{"id": "empty-history", "kind": "root", "items": [], "root": crypto.history_root([])}] + for n in (1, 2, 3, 4, 5, 7): + items = [["node", mint_node_id("m", "sentence", H(str(i)), 0), H("h%d" % i)] for i in range(n)] + root = crypto.merkle_root(items) + for i in range(n): + pf.append({"id": "inclusion-%d-of-%d" % (i, n), "kind": "inclusion", "items": items, "index": i, "size": n, + "root": root, "leaf": crypto.leaf_hash(items[i]), "path": crypto.inclusion_path(items, i)}) + for m in range(n + 1): + pf.append({"id": "consistency-%d-to-%d" % (m, n), "kind": "consistency", "items": items, "from": m, "to": n, + "from_root": crypto.history_root(items[:m]), "to_root": root, + "path": crypto.consistency_proof(items, m)}) + hist = history_items(hale["records"]) + pf.append({"id": "hale-history-8-to-12", "kind": "consistency", "items": hist, "from": 8, "to": len(hist), + "from_root": crypto.history_root(hist[:8]), "to_root": crypto.history_root(hist), + "path": crypto.consistency_proof(hist, 8)}) + pf.append({"id": "hale-history-correction-5", "kind": "inclusion", "items": hist, "index": 4, "size": len(hist), + "root": crypto.history_root(hist), "leaf": crypto.leaf_hash(hist[4]), "path": crypto.inclusion_path(hist, 4)}) + wv(out, "proofs.jsonl", pf) + # The VRF (RFC 9381, ECVRF-EDWARDS25519-SHA512-TAI): the RFC's Example 16, two more inputs for its key, and + # the spot-check seed of the example paper's first receipt. + vv = [] + for cid, alpha in (("rfc9381-example-16", b""), ("rfc9381-key-alpha-72", b"\x72"), ("rfc9381-key-alpha-af82", b"\xaf\x82")): + pi = vrf.prove(sk, alpha) + vv.append({"id": cid, "secret": sk.hex(), "public": vrf.public_key(sk).hex(), "alpha": alpha.hex(), + "proof": pi.hex(), "beta": vrf.proof_to_hash(pi).hex()}) + seed = base64.b64decode(q1["qa"]["seed"]) + vv.append({"id": "ben-spot-seed-q1", "public": base64.b64decode(ROSTER_BY_ID["ben-spot"]["public"]).hex(), + "alpha": RC.seed_payload(r1).hex(), "proof": seed.hex(), "beta": vrf.proof_to_hash(seed).hex()}) + bad = bytearray(seed) + bad[40] ^= 1 + vv.append({"id": "tampered-challenge", "public": base64.b64decode(ROSTER_BY_ID["ben-spot"]["public"]).hex(), + "alpha": RC.seed_payload(r1).hex(), "proof": bytes(bad).hex(), "beta": None}) + vv.append({"id": "other-input", "public": base64.b64decode(ROSTER_BY_ID["ben-spot"]["public"]).hex(), + "alpha": (RC.seed_payload(r1) + b"x").hex(), "proof": seed.hex(), "beta": None}) + wv(out, "vrf.jsonl", vv) + # A checkpoint of the team's log (spec/23): every receipt and revocation of the example paper, then the roster. + entries = [LG.entry(r) for r in hale["receipts"]] + [LG.entry(ROSTER)] + note = LG.sign_checkpoint("scient.ing/log/smithlab", entries, [("ana", secret("ana")), ("wit", secret("wit"))]) + wv(out, "checkpoint.jsonl", [{"id": "smithlab", "origin": "scient.ing/log/smithlab", "entries": entries, + "root": LG.root(entries), "signers": ["ana", "wit"], "note": note, + "tiles": sorted(LG.tiles(entries))}]) + # Alignment. + def item(i, runs, container, context="paragraph"): + doc, s = sentence_doc(runs) + st, cx = hashing.strict(doc, s) + return {"id": i, "runs": runs, "strict": st, "lossy": hashing.lossy_from_strict(st), "complex": cx, + "container": container, "context": context} + A = [item("a1", [T("Methods")], "h", "heading"), + item("a2", [T("Every bond was scanned in steps of 0.01 \u00c5 around its minimum.")], "p1"), + item("a3", [T("where "), M("\\frac{a}{b}"), T(" is the bond length.")], "p1"), + item("a4", [T("The fitted rate is 10\u00b3 per second at room temperature.")], "p1"), + item("a5", [T("where "), M("\\frac{c}{d}"), T(" is the bond length.")], "p2"), + item("a6", [T("This sentence was dropped by the second reading entirely.")], "p2"), + item("a7", [T("The last sentence of the section is unique and long enough.")], "p2")] + B = [item("b1", [T("Methods")], "x", "paragraph"), + item("b2", [T("Every bond was scanned in steps of 0.01 \u00c5 around its minimum.")], "q1"), + item("b3", [T("where "), M("\\frac{a}{b}"), T(" is the bond length.")], "q1"), + item("b4", [T("The fitted rate is 103 per second at room temperature.")], "q2"), + item("b5", [T("where "), M("\\frac{c}{e}"), T(" is the bond length.")], "q2"), + item("b6", [T("A sentence only the second reading has, also long.")], "q2"), + item("b7", [T("The last sentence of the section is unique and long enough.")], "q2")] + wv(out, "alignment.jsonl", [{"id": "two-readings", "a": A, "b": B, "expect": AL.align(A, B)}]) + # Corrections, op by op, on the tiny document. + wv(out, "corrections.jsonl", corrections_vectors()) + # Anchor resolution against the hale package. + wv(out, "resolve.jsonl", resolve_vectors(hale)) + # Rights. + wv(out, "rights.jsonl", rights_vectors(hale)) + + +def corrections_vectors(): + tiny = tiny_package(None, receipts=False) + st0 = tiny["final"] + nm = tiny["names"].ids + crop = [{"kind": "crop", "sha256": crop_sha("tiny")}] + person = [{"kind": "person", "by": "ana"}] + base = {"author": {"key": "ana"}, "reason": "Vector.", "at": "2026-10-11T12:00:00Z"} + cases = [ + ("replace-digit", {"op": "replace", "target": nm["@a"], "find": "42.0", "replace": "42.5", "evidence": crop}), + ("replace-digit-no-crop", {"op": "replace", "target": nm["@a"], "find": "42.0", "replace": "42.5", "evidence": person}), + ("replace-word", {"op": "replace", "target": nm["@c"], "find": "second", "replace": "later", "evidence": person}), + ("replace-not-unique", {"op": "replace", "target": nm["@a"], "find": "0", "replace": "1", "evidence": crop}), + ("replace-runs", {"op": "replace", "target": nm["@a"], "find": "300 K", "replace": [ + {"kind": "math", "notation": "tex", "source": "300\\,\\mathrm{K}"}], "evidence": crop}), + ("insert-text", {"op": "insert", "target": nm["@d"], "run": 0, "offset": 8, "text": "only ", "evidence": crop}), + ("insert-text-no-crop", {"op": "insert", "target": nm["@d"], "run": 0, "offset": 8, "text": "only ", + "evidence": person}), + ("insert-text-bad-run", {"op": "insert", "target": nm["@d"], "run": 7, "offset": 0, "text": "x", + "evidence": crop}), + ("insert-node", {"op": "insert", "parent": nm["@p2"], "place_after": nm["@c"], + "node": {"kind": "sentence", "content": [{"kind": "text", "text": "A dropped sentence."}]}, + "evidence": crop}), + ("delete", {"op": "delete", "target": nm["@p2"], "evidence": person}), + ("delete-digits-no-crop", {"op": "delete", "target": nm["@p1"], "evidence": person}), + ("split", {"op": "split", "target": nm["@a"], "offset": 13, "evidence": person}), + ("split-inside-xref", {"op": "split", "target": nm["@b"], "offset": 6, "evidence": person}), + ("merge", {"op": "merge", "target": nm["@c"], "with": nm["@d"], "joiner": " ", "evidence": person}), + ("merge-not-adjacent", {"op": "merge", "target": nm["@a"], "with": nm["@c"], "evidence": person}), + ("move", {"op": "move", "target": nm["@d"], "parent": nm["@p1"], "place_after": None, "evidence": person}), + ("move-after-itself", {"op": "move", "target": nm["@d"], "parent": nm["@p2"], "place_after": nm["@d"], + "evidence": person}), + ("reclassify", {"op": "reclassify", "target": nm["@p2"], "kind": "footnote", "evidence": person}), + ("reclassify-blocks", {"op": "reclassify", "target": nm["@p2"], "kind": "quote", "evidence": person}), + ("reclassify-bad", {"op": "reclassify", "target": nm["@a"], "kind": "paragraph", "evidence": person}), + ("relabel-section", {"op": "relabel", "target": nm["@sec"], "label": "2", "evidence": crop}), + ("relabel-remove", {"op": "relabel", "target": nm["@sec"], "label": None, "evidence": crop}), + ("relabel-digit-no-crop", {"op": "relabel", "target": nm["@t"], "label": "Table 2", "evidence": person}), + ("set-colspan", {"op": "set", "target": nm["@y1"], "attr": "colspan", "value": 2, "evidence": person}), + ("set-bad-value", {"op": "set", "target": nm["@y1"], "attr": "colspan", "value": "2", "evidence": person}), + ("set-not-settable", {"op": "set", "target": nm["@y1"], "attr": "level", "value": 2, "evidence": person}), + ("insert-node-junk", {"op": "insert", "parent": nm["@p2"], "place_after": nm["@c"], + "node": {"kind": "sentence", "content": [{"kind": "text", "text": "Junk."}], "foo": 1}, + "evidence": crop}), + ("delete-header-cell", {"op": "delete", "target": nm["@hx"], "evidence": person}), + # A model proposed this one, so its author names the model, by weights and prompt, and the run. + ("replace-by-model", {"op": "replace", "target": nm["@c"], "find": "second", "replace": "later", "evidence": person, + "author": {"model": MODEL, "run": "illustrative-run-7"}}), + ] + out = [] + vals0 = hashing.compute(st0.doc()) + before = [dict(n, **vals0[n["id"]]) for n in st0.nodes] + for cid, c in cases: + c = dict(base, **c) + case = {"id": cid, "doc": st0.key, "before": {"nodes": before, "ids": [], "rev": 0}, "correction": c} + try: + st1, rec = CO.apply(st0, c, store=SCHEMAS) + v1 = hashing.compute(st1.doc()) + case["expect"] = {"record": rec, "nodes": [dict(n, **v1[n["id"]]) for n in st1.nodes], "ids": st1.ids} + except Refusal as e: + case["expect"] = {"refusal": e.rule} + out.append(case) + return out + + +def resolve_vectors(hale): + src, names, states, final = hale["src"], hale["names"], hale["states"], hale["final"] + s0 = states[0] + d0 = s0.doc() + v0 = hashing.compute(d0) + out = [] + for cid, a in src["anchors_at_rev0"].items(): + nid = names.ids[a["node"]] + if a.get("container_of"): + nid = d0.nodes[nid]["parent"] + span = None + if "quote" in a: + t = hashing.plain(d0, nid) + i = t.index(a["quote"]) + span = [i, i + len(a["quote"])] + anchor = AN.make_anchor(d0, v0, nid, 0, span=span, store=STORE) + if "alias" in a: + anchor["node"] = a["alias"] + res = AN.resolve(final, anchor) + out.append({"id": cid, "package": "fixtures/packages/" + hale["key"], "anchor": anchor, "expect": res}) + return out + + +def rights_vectors(hale): + out = OUT + w = hale["witness"] + stages = json.load(open(os.path.join(ROOT, "registry", "stages", "scienting-stages-default.json")))["stages"] + policy = json.load(open(os.path.join(out, "fixtures", "records", "policy.json"))) + cases = [] + for cid, stage, proc, now, pols in [ + ("convert-local", "convert-model", "local", "2026-10-11T00:00:00Z", []), + ("convert-outside", "convert-model", "outside", "2026-10-11T00:00:00Z", []), + ("review-tenancy-unknown", "review-model", "tenancy", "2026-10-11T00:00:00Z", []), + ("review-tenancy-policy", "review-model", "tenancy", "2026-10-11T00:00:00Z", [policy]), + ("index", "index", "local", "2026-10-11T00:00:00Z", []), + ("embed-outside-refused", "embed", "outside-no-retention", "2026-10-11T00:00:00Z", [policy]), + ("publish-quotes", "publish-quotes", "local", "2026-10-11T00:00:00Z", []), + ("read-expired", "read", "local", "2027-09-01T00:00:00Z", []), + ("accessible-copy-unknown", "accessible-copy", "local", "2026-10-11T00:00:00Z", []), + ]: + uses, expired = RI.effective(w, pols, now) + ok, lacking = RI.check(stages[stage], proc, uses) + cases.append({"id": cid, "rights": w["rights"], "origin": w["origin"], "witness_sha256": w["sha256"], + "policies": pols, "stage": stage, "requires": stages[stage], "processor": proc, "now": now, + "expect": {"allowed": ok, "lacking": lacking, "expired": expired}}) + oa = rights_record("open") + sub = w["rights"] + cases.append({"id": "combine", "rights": [oa, sub], + "expect": RI.combine([RI.flat(oa), RI.flat(sub)])}) + backends = [json.load(open(os.path.join(out, "fixtures", "records", "backend-%s.json" % b))) for b in ("docling", "examplevendor")] + profiles = [json.load(open(os.path.join(out, "fixtures", "records", "profile-%s.json" % p))) for p in ("smithlab", "acme")] + for b in backends: + for p in profiles: + ok, failed = RI.backend_allowed(b, p) + cases.append({"id": "backend-%s-%s" % (b["name"], p["team"]), "backend": b, "profile": p, + "expect": {"allowed": ok, "failed": failed}}) + return cases + + +# --- records ------------------------------------------------------------------------ + + +def wr(out, rel, rec): + path = os.path.join(out, "fixtures", "records", rel) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "wb") as f: + f.write((jcs.dumps(rec) + "\n").encode() if not isinstance(rec, bytes) else rec) + + +CTX = ["http://www.w3.org/ns/anno.jsonld", "https://paperspec.scient.ing/context/scienting-1.jsonld"] + + +def ann_hash(a): + return H(jcs.dumpb({k: v for k, v in a.items() if k not in ("status", "receipt", "modified")})) + + +TAG = "tag:scient.ing,2026:" +# A model's provenance: enough to name the weights and the instructions it ran with (all illustrative). +MODEL = {"name": "example-model", "provider": "Example Models (illustrative)", "version": "2026-09", + "weights": sha256_hex("illustrative weights"), "prompt": sha256_hex("illustrative review prompt"), + "parameters": sha256_hex("illustrative sampling parameters"), "processor": "local"} + + +def records(out, hale): + names, final = hale["names"], hale["final"] + doc = final.doc() + vals = hashing.compute(doc) + rev = final.rev + I = names.ids + HALE = hale["key"] + QHALE = STORE + "/" + HALE + OKAFOR = Names.registry["okafor2025open"].key + DRAFT = Names.registry["smithlab2027charges"].key + + def anchor(name, quote=None): + nid = I[name] + span = None + if quote: + t = hashing.plain(doc, nid) + i = t.index(quote) + span = [i, i + len(quote)] + return AN.make_anchor(doc, vals, nid, rev, span=span, store=STORE) + + wr(out, "roster.json", ROSTER) + # The doubtful reading behind the first correction, as the converter raised it before anyone answered it. + sus = {"format": "scienting-suspect/1", "kind": "number", "node": I["@s4"], "a": "10-3", "b": "10\u22123", + "crop": crop_sha(I["@s4"]), + "note": "The text layer reads a hyphen on the line; the page image shows a raised minus three."} + sus = dict(sus, id=record_id(sha256_hex(jcs.dumpb(sus)))) + wr(out, "suspect.json", sus) + wr(out, "anchor-span.json", anchor("@s3", "first test case")) + wr(out, "anchor-cell.json", anchor("@cc_ak")) + wr(out, "anchor-paragraph.json", AN.make_anchor(doc, vals, doc.nodes[I["@s3"]]["parent"], rev, store=STORE)) + long_anchor = AN.make_anchor(doc, vals, I["@s2"], rev, store=STORE) + wr(out, "anchor-leaf.json", long_anchor) + # A public layer's anchor on a paywalled paper: keyed by the pages, no text. + fps = [p["page_fp"] for p, _ in hale["pages"]] + layer_id = TAG + "layer:smithlab/kp4m2x7q" + pk = crypto.page_key(fps, "layer:smithlab/kp4m2x7q") + hk = hashing.Hasher(pk) + ka = anchor("@cc_ak") + keyed_vals = hashing.compute(doc, hk) + ka = {"format": "scienting-anchor/1", "store": STORE, "doc": ka["doc"], "node": ka["node"], "kind": "cell", + "own_hash": keyed_vals[ka["node"]]["own_hash"], "node_hash": keyed_vals[ka["node"]]["node_hash"], + "quote": {"mac": hk(norm(ka["quote"]["exact"])), "length": len(ka["quote"]["exact"])}, + "context": {"table_label_mac": hk(norm(ka["context"]["table_label"])), + "row_path_mac": [hk(norm(x)) for x in ka["context"]["row_path"]], + "column_path_mac": [hk(norm(x)) for x in ka["context"]["column_path"]]}, + "rev": rev, "keyed": {"scheme": "hkdf-pages", "algorithm": "sha-256", "context": "layer:smithlab/kp4m2x7q", + "pages": [crypto.page_hash(f) for f in fps]}} + wr(out, "anchor-keyed.json", ka) + # Rights, policy, backends, profiles. + wr(out, "rights-open.json", rights_record("open")) + wr(out, "rights-subscription.json", rights_record("subscription")) + ccc = rights_record("subscription") + ccc.update({"license": {"text": "Annual collective license with internal AI re-use (fictional)."}, + "basis": "collective-license", "valid_until": "2027-06-30T23:59:59Z"}) + wr(out, "rights-collective.json", ccc) + policy = RC.seal({"format": "scienting-policy/1", "project": "smithlab-charges", + "applies_to": {"basis": ["subscription"], "doi_prefixes": ["10.5555/"]}, + "uses": {"ai": {"internal/tenancy": True}}, + "statement": "Our institutional agreement covers internal AI use in our own cloud account for " + "these papers (fictional).", + "by": "ana", "at": "2026-10-05T09:00:00Z", "valid_until": "2027-08-31T23:59:59Z"}, + secret("ana"), "ana") + wr(out, "policy.json", policy) + wr(out, "backend-docling.json", { + "format": "scienting-backend/1", "name": "docling", "version": "illustrative", "stage": "read", + "processor": "local", "code_license": {"spdx": "MIT"}, + "weights": [{"name": "layout model", "license": {"spdx": "Apache-2.0"}}], "conditions": []}) + wr(out, "backend-examplevendor.json", { + "format": "scienting-backend/1", "name": "examplevendor", "version": "1.0", "stage": "read", + "processor": "local", "code_license": {"spdx": "Apache-2.0"}, + "weights": [{"name": "example weights", "license": {"text": "A fictional weight license with conditions."}}], + "conditions": [{"id": "revenue-cap", "kind": "revenue-cap", "usd": 5000000}, + {"id": "no-competition", "kind": "no-competition", "vendor": "examplevendor"}, + {"id": "purpose", "kind": "purpose", "allowed": ["research", "personal", "education", "startup"]}]}) + wr(out, "profile-smithlab.json", {"format": "scienting-profile/1", "team": "smithlab", "purpose": "research", + "revenue_or_funding_usd": {"at_most": 2000000}, "competes_with": [], + "recorded_by": "ana", "at": "2026-10-01T00:00:00Z"}) + wr(out, "profile-acme.json", {"format": "scienting-profile/1", "team": "acme", "purpose": "commercial", + "revenue_or_funding_usd": {"at_most": None}, "competes_with": None, + "recorded_by": "acme-admin", "at": "2026-10-01T00:00:00Z"}) + # A layer with one annotation of each common kind. + creator = {"id": TAG + "key:ana", "type": "Person", "name": "Ana Example"} + model = {"id": TAG + "software:example-model", "type": "Software", "name": "A model (illustrative)"} + + def target(a): + return {"type": "SpecificResource", "source": TAG + "doc:" + QHALE, + "selector": [{"type": "AnchorSelector", "anchor": a}] + + ([{"type": "TextQuoteSelector", "exact": a["quote"]["exact"], + "prefix": a["quote"].get("prefix", ""), "suffix": a["quote"].get("suffix", "")}] + if "quote" in a and "exact" in a["quote"] else [])} + + def ann(body, tgt, motivation, status="proposed", who=creator, created="2026-10-11T19:00:00Z"): + a = {"@context": CTX, "type": "Annotation", "layer": layer_id, "motivation": motivation, "creator": who, + "created": created, "body": body, "target": tgt} + a["id"] = TAG + "annotation:smithlab/" + record_id(H(jcs.dumpb(a))) + a["status"] = status + return a + + ex_body = {"format": "scienting-exercise/1", "exercise": "anchor", + "prompt": "Find the table cell that gives the Set A bond constant for the CT–CT bond.", + "key": {"elements": [{"anchors": [anchor("@cc_ak")], "points": "1", + "feedback": {"hit": "That is the cell.", "miss": "Look in Table 1, row CT–CT."}}]}, + "rule": {"match": "same-node"}} + check_body = {"format": "scienting-exercise/1", "exercise": "check", + "prompt": "Does the cited passage support the claim that harmonic terms are the most common choice?", + "key": {"verdict": {"value": "supports", "points": "1"}, + "elements": [{"anchors": [anchor("@s1", "the most common description")], "points": "1"}]}, + "rule": {"match": "overlap", "min_overlap": "0.5"}} + anns = [ + ann({"type": "TextualBody", "kind": "gloss", "audience": "novice", "format": "text/markdown", "language": "en", + "value": "A *harmonic* term treats a bond like a spring: the energy grows with the square of the stretch."}, + target(anchor("@s2", "bond force constant")), "describing", "confirmed"), + ann({"type": "SpecificResource", "kind": "claim-link", "source": "lode:amber/k-bond", "pin": "illustrative-pin", + "role": "states"}, target(anchor("@s2")), "linking", "confirmed"), + ann({"type": "TextualBody", "kind": "comment", "value": "The step size matters more than the text admits."}, + target(anchor("@s6a", "steps of 0.01")), "commenting"), + ann({"type": "TextualBody", "kind": "question", "for": "group", + "value": "Why fit to a scan first and to frequencies second?"}, + target(AN.make_anchor(doc, vals, I["@sec2"], rev, store=STORE)), "questioning"), + ann({"type": "TextualBody", "kind": "citation-context", "reference": I["@r2"], "verdict": "cannot-tell", + "reason": "value", "value": "Cited for the band position; the reference is not in the library."}, + target(anchor("@s3", "1595")), "assessing"), + ann({"type": "TextualBody", "kind": "generated", "value": "Two panels: an energy scan and its fit residual.", + "generator": {"model": MODEL, "run": "illustrative-run-1"}}, + target(AN.make_anchor(doc, vals, I["@fig1"], rev, store=STORE)), "describing", "proposed", model), + ann({"type": "TextualBody", "kind": "exercise", "value": ex_body["prompt"], "exercise": ex_body}, + target(AN.make_anchor(doc, vals, I["@tab1"], rev, store=STORE)), "assessing", "confirmed"), + ann({"type": "TextualBody", "kind": "exercise", "value": check_body["prompt"], "exercise": check_body}, + target(anchor("@s1")), "assessing", "confirmed"), + # A person's description of the figure, which an accessible export may use because it is confirmed. + ann({"type": "TextualBody", "kind": "description", "role": "alt", "language": "en", + "value": "Energy along the C–C bond stretch (panel a) and the fit residual (panel b)."}, + target(AN.make_anchor(doc, vals, I["@fig1"], rev, store=STORE)), "describing", "confirmed"), + ] + anns.append(ann({"type": "TextualBody", "kind": "comment", "value": "Agreed; see the residual panel."}, + {"type": "SpecificResource", "source": anns[2]["id"]}, "replying", created="2026-10-11T19:05:00Z")) + confirmed = [a for a in anns if a["status"] == "confirmed"] + lr = RC.seal({"format": "scienting-receipt/1", "doc": HALE, "rev": rev, "level": "F3", + "scope": "targeted:layer", "nodes": [], + "records": sorted([["annotation", a["id"].rsplit("/", 1)[1], ann_hash(a)] for a in confirmed]), + "evidence": [], "by": "ana", "at": "2026-10-11T19:30:00Z"}, secret("ana"), "ana") + for a in confirmed: + a["receipt"] = lr["id"] + wr(out, "layer/layer.json", {"@context": CTX, "format": "scienting-layer/1", "id": layer_id, + "type": "AnnotationCollection", "label": "Smith lab reading of Hale and Osei", + "purpose": "Glosses, claim links and exercises for new students.", + "audience": "student", "visibility": "team", "license": {"spdx": "CC-BY-4.0"}, + "creator": creator, "created": "2026-10-11T19:00:00Z", "project": "smithlab-charges", + "docs": [QHALE], "total": len(anns)}) + wr(out, "layer/annotations.jsonl", jsonl(anns)) + wr(out, "layer/receipt.json", lr) + # Submissions and grades. + ex_ann = anns[6] + sub_ok = {"format": "scienting-submission/1", "exercise": ex_ann["id"], "doc": QHALE, + "rev": rev, "student": "s-7k2m", "answers": {"anchors": [anchor("@cc_ak")]}, + "submitted_at": "2026-10-20T10:00:00Z"} + sub_ok["id"] = record_id(H(jcs.dumpb(sub_ok))) + sub_miss = dict(sub_ok, student="s-9q4p", answers={"anchors": [anchor("@cc_bk")]}) + sub_miss.pop("id") + sub_miss["id"] = record_id(H(jcs.dumpb(sub_miss))) + ch_ann = anns[7] + sub_check = {"format": "scienting-submission/1", "exercise": ch_ann["id"], "doc": QHALE, + "rev": rev, "student": "s-7k2m", + "answers": {"verdict": "supports", "anchors": [anchor("@s1", "most common description of covalent")]}, + "submitted_at": "2026-10-20T10:05:00Z"} + sub_check["id"] = record_id(H(jcs.dumpb(sub_check))) + grades = [] + for name, sub, body in (("anchor-hit", sub_ok, ex_body), ("anchor-miss", sub_miss, ex_body), + ("check-overlap", sub_check, check_body)): + g = GR.grade(body, sub, final) + wr(out, "submissions/%s.json" % name, sub) + wr(out, "grades/%s.json" % name, g) + grades.append({"id": name, "package": "fixtures/packages/" + hale["key"], "exercise": body, + "submission": sub, "expect": g}) + wv(out, "grading.jsonl", grades) + # Project. + wr(out, "project.json", { + "format": "scienting-project/1", "key": "smithlab-charges", "store": STORE, "name": "Charges and bonds", + "members": [{"person": "Ana Example", "role": "owner", "keys": ["ana"]}, + {"person": "Ben Example", "role": "editor", "keys": ["ben", "ben-spot"]}, + {"person": "Cara Example", "role": "reader", "keys": ["cara"]}], + "documents": [{"doc": QHALE, "reason": "The bond constants we start from.", + "tags": ["parameters"], "status": {"Ben Example": "read", "Cara Example": "reading"}}, + {"doc": OKAFOR, "reason": "Solvent shifts for the validation set.", "tags": ["spectra"]}], + "paths": [{"id": "onboarding", "title": "First week", "steps": [ + {"type": "document", "ref": QHALE, "note": "Read the introduction."}, + {"type": "node", "ref": QHALE + "#" + I["@tab1"], "note": "Then the table."}, + {"type": "claim", "ref": "lode:amber/k-bond"}]}], + "layers": [layer_id], "datasets": ["ds-hale-bond-params", "ds-hale-ctct-scan"], "views": ["ctct-scan"], + "drafts": [STORE + "/" + DRAFT], "claims": {"lode": "amber"}, + "settings": {"digit_rule": "anchored-sections", "drift_threshold": "0.25", + "spot_check": {"rate": "0.05", "minimum": 10}, "rights_unknown": "refuse", + "stages": "scienting-stages-default", + "log": {"origin": "scient.ing/log/smithlab", "witnesses": ["wit"], "threshold": 1}}}) + # A digitization of panel a, and the dataset it yields. + p2 = hale["pages"][1][0] + pa = I["@pa"] + ft = {t["text"]: t["id"] for t in doc.nodes[pa]["attrs"]["figure_text"]} + pts = [((110.0, 360.0), ("1.45", "3.10")), ((150.0, 330.0), ("1.50", "0.40")), ((190.0, 300.0), ("1.55", "2.80"))] + dg = {"format": "scienting-digitization/1", "doc": QHALE, + "target": {"anchor": AN.make_anchor(doc, vals, pa, rev, store=STORE), "page_fp": p2["page_fp"], + "region": [72.0, 260.0, 300.0, 410.0], "witness": hale["witness"]["sha256"]}, + "axes": [{"id": "x", "scale": "linear", "unit": "angstrom", + "calibration": [{"at": [90.0, 400.0], "value": "1.4"}, {"at": [210.0, 400.0], "value": "1.6"}], + "check": {"at": [150.0, 400.0], "value": "1.5"}, "label": {"panel": pa, "figure_text": ft["r (Å)"]}}, + {"id": "y", "scale": "linear", "unit": "kcal / mol", + "calibration": [{"at": [80.0, 400.0], "value": "0"}, {"at": [80.0, 280.0], "value": "4"}], + "label": {"panel": pa, "figure_text": ft["E (kcal mol⁻¹)"]}}], + "series": [{"id": "scan", "legend": "scan", "mark": {"color": "#285aa0", "marker": "circle"}, "kind": "points", + "values": [{"at": list(at), "value": {"x": x, "y": y}, + "uncertainty": {"x": "0.01", "y": "0.05", "resolution": "0.01", "marker": "0.02", + "calibration": "0.01"}} for at, (x, y) in pts]}], + "method": {"kind": "manual", "tool": {"name": "hand", "version": "0"}}, + "verification": {"image": "", "residual": "0.4", "max_residual": "0.9"}, + "confirmation": None, "dataset": "ds-hale-ctct-scan", "rev": rev} + dg["verification"]["image"] = crop_sha("overlay of panel a") + dg["id"] = record_id(H(jcs.dumpb(dg))) + wr(out, "digitization.json", dg) + body = {k: v for k, v in dg.items() if k != "confirmation"} + conf = RC.seal({"format": "scienting-receipt/1", "doc": HALE, "rev": rev, "level": "F3", + "scope": "targeted:digitization", "nodes": [], + "records": [["digitization", dg["id"], H(jcs.dumpb(body))]], + "evidence": [{"kind": "crop", "node": pa, "sha256": crop_sha("overlay of panel a")}], + "by": "ana", "at": "2026-10-11T20:00:00Z"}, secret("ana"), "ana") + wr(out, "digitization-confirmation.json", conf) + ds = {"format": "scienting-dataset/1", "id": "ds-hale-ctct-scan", "name": "hale-ctct-scan", + "title": "Energy along the C–C stretch, read from Figure 1a", + "resources": [{"name": "scan", "path": "data.csv", "format": "csv", "encoding": "utf-8", + "schema": {"fields": [{"name": "point", "type": "string"}, + {"name": "r", "type": "number", "unit": "angstrom", + "uncertainty": {"field": "r_u", "kind": "resolution"}}, + {"name": "r_u", "type": "number", "unit": "angstrom"}, + {"name": "e", "type": "number", "unit": "kcal / mol", + "uncertainty": {"field": "e_u", "kind": "resolution"}}, + {"name": "e_u", "type": "number", "unit": "kcal / mol"}], + "primaryKey": "point"}, + "origin": {"kind": "digitized", "digitization": dg["id"], "anchors": "anchors.jsonl"}}]} + csvd = "point,r,r_u,e,e_u\n" + "".join("p%d,%s,0.01,%s,0.05\n" % (i, x, y) for i, (_, (x, y)) in enumerate(pts)) + va = [{"resource": "scan", "row": "p%d" % i, "field": f, "digitization": dg["id"], "series": "scan", "point": i} + for i in range(len(pts)) for f in ("r", "e")] + wr(out, "datasets/ds-hale-ctct-scan/datapackage.json", ds) + wr(out, "datasets/ds-hale-ctct-scan/data.csv", csvd.encode()) + wr(out, "datasets/ds-hale-ctct-scan/anchors.jsonl", jsonl(va)) + ds_files = sorted([["anchors.jsonl", H(jsonl(va)), len(jsonl(va))], ["data.csv", H(csvd), len(csvd.encode())], + ["datapackage.json", H(jcs.dumps(ds) + "\n"), len((jcs.dumps(ds) + "\n").encode())]]) + ds_hash = H(jcs.dumpb({"format": "scienting-dataset/1", "files": ds_files})) + wr(out, "view.vl.json", { + "$schema": "https://vega.github.io/schema/vega-lite/v5.json", + "usermeta": {"scienting": {"format": "scienting-view/1", "id": "ctct-scan", + "data": [{"name": "ds-hale-ctct-scan", "dataset": "ds-hale-ctct-scan", + "hash": ds_hash, "resource": "scan"}]}}, + "data": {"name": "ds-hale-ctct-scan"}, "mark": {"type": "point"}, + "encoding": {"x": {"field": "r", "type": "quantitative", "title": "r (Å)"}, + "y": {"field": "e", "type": "quantitative", "title": "E (kcal/mol)"}}}) + # QA report. + q1 = hale["by_name"]["q1"] + q2 = hale["by_name"]["q2"] + n = len(q1["qa"]["sample"]) + len(q2["qa"]["sample"]) + k = len(q1["qa"]["errors"]) + len(q2["qa"]["errors"]) + wr(out, "qa-report.json", RC.seal({ + "format": "scienting-qa-report/1", "delivery": "smithlab evidence pilot (fictional)", + "docs": [QHALE], "receipts": [hale["by_name"]["r1"]["id"], hale["by_name"]["r4"]["id"]], + "reviewers": [{"key": "ana", "sampled": n, "errors": k, "confidence": "0.95", "method": "clopper-pearson", + "bound": RC.bound_string(k, n)}], + "by": "ana", "at": "2026-10-12T10:00:00Z"}, secret("ana"), "ana")) + # A checklist for an accessible HTML export of the paper, and the receipt that covers it (spec/08 §8.11). + html_sha = H("accessible HTML export of %s (synthetic)" % HALE) + checker = {"name": "example-checker", "version": "1.0"} + checklist = {"format": "scienting-checklist/1", "doc": QHALE, "rev": rev, + "standard": {"name": "WCAG 2.1", "level": "AA", "url": "https://www.w3.org/TR/WCAG21/"}, + "export": {"kind": "html", "sha256": html_sha, "size": 48213}, + "items": [ + {"criterion": "1.1.1", "result": "pass", "method": "manual", + "note": "Every figure has a confirmed description in the layer."}, + {"criterion": "1.2.1", "result": "not-applicable", "method": "manual", "note": "No audio or video."}, + {"criterion": "1.3.1", "result": "pass", "method": "manual", + "note": "Headings, lists and table headers are marked up; each cell names its header path."}, + {"criterion": "1.3.2", "result": "pass", "method": "automated", "tool": checker}, + {"criterion": "1.4.3", "result": "pass", "method": "automated", "tool": checker}, + {"criterion": "1.4.5", "result": "pass", "method": "manual", + "note": "Equations are MathML, never images of text."}, + {"criterion": "2.4.2", "result": "pass", "method": "automated", "tool": checker}, + {"criterion": "3.1.1", "result": "pass", "method": "automated", "tool": checker}], + "by": "ana", "at": "2026-10-12T11:00:00Z"} + checklist_hash = H(jcs.dumpb(checklist)) + checklist["id"] = record_id(checklist_hash) + wr(out, "checklist/checklist.json", checklist) + wr(out, "checklist/receipt.json", RC.seal({ + "format": "scienting-receipt/1", "doc": HALE, "rev": rev, "level": "F3", + "scope": "targeted:checklist", "nodes": [], "records": [["checklist", checklist["id"], checklist_hash]], + "evidence": [{"kind": "source-file", "sha256": html_sha, "note": "The exported HTML."}], + "by": "ana", "at": "2026-10-12T11:30:00Z"}, secret("ana"), "ana")) + # The target set of targeted F3 (spec/08 §8.10) under each digit rule: the dataset's cells, the sentence a + # confirmed claim link anchors, and the panel a digitization reads from are the anchored nodes. + dataset_anchors = [json.loads(x) for x in + hale["files"]["datasets/ds-hale-bond-params/anchors.jsonl"].decode().splitlines()] + anchored = sorted({a["anchor"]["node"] for a in dataset_anchors} | {I["@s2"], pa}) + wv(out, "targets.jsonl", [{"id": "hale-" + rule, "package": "fixtures/packages/" + hale["key"], + "anchored": anchored, "digit_rule": rule, + "expect": {"nodes": TG.target_set(doc, anchored, rule)}} + for rule in TG.DIGIT_RULES]) + # Proofs (spec/08 §8.5, spec/11 §11.5): one node of the first receipt was covered, and revision 12 extends + # revision 8 of the correction history. + r1 = hale["by_name"]["r1"] + items = crypto.receipt_items(r1) + wr(out, "proof-inclusion.json", {"format": "scienting-proof/1", "kind": "inclusion", "receipt": r1["id"], + "item": items[3], "index": 3, "size": len(items), "root": r1["root"], + "path": crypto.inclusion_path(items, 3)}) + hist = history_items(hale["records"]) + wr(out, "proof-consistency.json", {"format": "scienting-proof/1", "kind": "consistency", "doc": QHALE, + "from_rev": 8, "to_rev": len(hist), "from_root": crypto.history_root(hist[:8]), + "to_root": hale["document"]["history"], "path": crypto.consistency_proof(hist, 8)}) + # A checkpoint of the team's log (spec/23), as the parsed form of its signed note. + entries = [LG.entry(r) for r in hale["receipts"]] + [LG.entry(ROSTER)] + signers = [("ana", secret("ana")), ("wit", secret("wit"))] + note = LG.write_log(entries, "scient.ing/log/smithlab", signers, os.path.join(out, "fixtures", "records", "log")) + parsed = LG.parse_checkpoint(note) + wr(out, "checkpoint.json", {"format": LG.FORMAT, "origin": parsed["origin"], "size": parsed["size"], + "root": parsed["root"], "note": note, + "signatures": [{"name": n, "key_hash": kh.hex(), "value": base64.b64encode(sig).decode()} + for n, kh, sig in parsed["signatures"]]}) + # A receipt signed with ML-DSA-65 (FIPS 204) by dana: two sentences read against the page, at F2. + two = sorted([I["@s1"], I["@s2"]]) + wr(out, "receipt-ml-dsa.json", seal({"format": "scienting-receipt/1", "doc": HALE, "rev": rev, "level": "F2", + "scope": "targeted:ml-dsa", "nodes": [[n, vals[n]["node_hash"]] for n in two], + "evidence": [{"kind": "person", "by": "dana"}], "by": "dana", + "at": "2026-10-13T09:00:00Z"}, "dana")) + # Draft snapshot and sentence IDs. + s5 = I["@s5"] + snap_anchor = AN.make_anchor(doc, vals, s5, rev, store=STORE) + wr(out, "snapshot.json", {"format": "scienting-snapshot/1", "store": STORE, "made_at": "2026-10-11T21:00:00Z", + "quotes": [{"address": AN.format_address(STORE, HALE, s5, vals[s5]["node_hash"]), + "anchor": snap_anchor, "text": norm(hashing.plain(doc, s5)), + "fidelity": "F3", "result": "current"}], + "claims": [{"ref": "amber/k-bond", "pin": "illustrative-pin", "level": "illustrative", + "typst": "K_r"}], + "datasets": [{"id": "ds-hale-ctct-scan", "hash": ds_hash}]}) + draft = Names.registry["smithlab2027charges"] + dd = Doc(DRAFT, [json.loads(x) for x in open(os.path.join(out, "fixtures", "packages", DRAFT, "nodes.jsonl"))]) + lines = [] + for nid in dd.preorder(): + if dd.nodes[nid]["kind"] == "sentence": + sec = next((a for a in reversed(dd.ancestors(nid)) if dd.nodes[a]["kind"] == "section"), None) + lines.append({"id": nid, "text_hash": dd.nodes[nid]["text_hash"], "section": sec, + "ord": dd.nodes[nid]["ord"]}) + lines[-1]["pinned"] = "energy-sentence" + wr(out, "draft-ids.jsonl", jsonl(lines)) + + +# --- invalid packages --------------------------------------------------------------- + + +def invalid(out, tiny): + base = os.path.join(out, "fixtures", "invalid") + files = tiny["files"] + nodes = [json.loads(x) for x in files["nodes.jsonl"].decode().splitlines()] + nm = tiny["names"].ids + + def edit_nodes(fn): + ns = copy.deepcopy(nodes) + fn(ns) + return jsonl(ns) + + def by(ns, name): + return next(n for n in ns if n["id"] == nm[name]) + + def case(name, rule, change, note, raw=None): + f = dict(files) + f.update(change) + d = os.path.join(base, name, "package") + write(d, f) + if raw: + raw(d) + with open(os.path.join(base, name, "expect.json"), "wb") as fh: + fh.write((jcs.dumps({"rule": rule, "note": note}) + "\n").encode()) + + case("hash-mismatch", "doc.hash.node-hash", {"nodes.jsonl": edit_nodes(lambda ns: by(ns, "@a").update(node_hash="0" * 64))}, + "A node_hash that does not recompute.") + case("text-edited", "doc.hash.text-hash", {"nodes.jsonl": edit_nodes( + lambda ns: by(ns, "@a").update(content=[{"kind": "text", "text": "The value is 42.5 at 300 K."}]))}, + "Text changed without its hashes.") + case("unknown-kind", "package.schema", {"nodes.jsonl": edit_nodes(lambda ns: by(ns, "@p2").update(kind="aside"))}, + "Kinds outside the list are refused.") + case("heading-not-first", "doc.tree.child", {"nodes.jsonl": edit_nodes(lambda ns: _swap_first(ns, nm["@sec"]))}, + "A heading must be a section's first child.") + case("cite-not-reference", "doc.run.cite", {"nodes.jsonl": edit_nodes( + lambda ns: by(ns, "@b").update(content=[r if r["kind"] != "cite" else dict(r, refs=[nm["@a"]]) + for r in by(ns, "@b")["content"]]))}, + "A cite run names a node that is not a reference.") + case("unlisted-file", "package.manifest.unlisted", {}, "A file the manifest does not list.", + raw=lambda d: open(os.path.join(d, "notes.txt"), "w").write("stray\n")) + case("manifest-hash", "package.manifest.hash", {}, "A file changed after the manifest was written.", + raw=lambda d: open(os.path.join(d, "provenance.json"), "a").write(" ")) + case("not-canonical", "package.file.canonical", {}, "A line that is valid JSON but not canonical JSON.", + raw=lambda d: _rewrite_noncanonical(d)) + case("root-mismatch", "doc.hash.root", {"document.json": _doc_edit(files, root="1" * 64)}, + "document.json's root is not the root node's node_hash.") + case("receipt-root", "doc.receipt.root", {"receipts.jsonl": _receipt_edit(files, root="2" * 64)}, + "A receipt whose Merkle root does not recompute.") + case("receipt-signature", "doc.receipt.signature", {"receipts.jsonl": _receipt_edit(files, at="2026-10-11T12:00:01Z")}, + "A receipt changed after it was signed.") + case("checks-only-plain", "doc.policy.withheld", {"document.json": _doc_edit(files, policy="checks-only")}, + "A checks-only package that still carries text and plain hashes.") + case("bad-id", "package.schema", {"nodes.jsonl": edit_nodes(lambda ns: _rename(ns, nm["@d"], "i1lo0uu"))}, + "IDs use Crockford base32, which has no i, l, o or u.") + case("empty-optional-file", "package.file.empty", {"suspects.jsonl": b""}, "An optional JSONL file with no records.") + case("alias-dangling", "doc.id.successor", {"ids.jsonl": jsonl([{"id": "b-p01-text-001", "kind": "alias", + "successor": "zzzzzzz"}])}, + "An alias whose node does not exist.") + case("location-unknown-page", "doc.location.page", {"locations.jsonl": jsonl([ + {"node": nm["@a"], "page_fp": "3" * 64, "page": 1, "boxes": [[72, 72, 200, 84]]}])}, + "A location on a page no witness has.") + case("style-sup-sub", "doc.run.style", {"nodes.jsonl": edit_nodes(lambda ns: by(ns, "@c").update( + content=[{"kind": "text", "text": "A second paragraph.", "style": ["sub", "sup"]}]))}, + "A run cannot be both a subscript and a superscript.") + # Zipped packages that break spec/12 §12.2: the tiny package's files plus one bad entry. + tiny_dir = os.path.join(out, "fixtures", "packages", tiny["key"]) + entries = [] + for rel in sorted(os.path.relpath(os.path.join(d, f), tiny_dir).replace(os.sep, "/") + for d, _, fs in os.walk(tiny_dir) for f in fs): + with open(os.path.join(tiny_dir, rel), "rb") as fh: + entries.append((tiny["key"] + "/" + rel, fh.read(), 0o644)) + + def zip_case(name, rule, note, extra): + d = os.path.join(base, name) + os.makedirs(d) + with zipfile.ZipFile(os.path.join(d, "package" + ZIP_EXTENSION), "w") as zf: + for ename, data, mode in entries + extra: + zi = zipfile.ZipInfo(ename, date_time=(1980, 1, 1, 0, 0, 0)) + zi.compress_type = zipfile.ZIP_STORED + zi.create_system = 3 + zi.external_attr = mode << 16 + zf.writestr(zi, data) + with open(os.path.join(d, "expect.json"), "wb") as fh: + fh.write((jcs.dumps({"rule": rule, "note": note}) + "\n").encode()) + + zip_case("zip-traversal", "package.zip.entry", "An archive entry whose path leaves the package folder.", + [(tiny["key"] + "/../escape.txt", b"outside\n", 0o644)]) + zip_case("zip-symlink", "package.zip.entry", "An archive entry that is a symbolic link.", + [(tiny["key"] + "/assets/link", b"document.json", 0o120777)]) + # Rules the audit found no broken package for. + case("missing-required-file", "package.file.required", {"provenance.json": None}, "A required file is missing.") + case("counts-wrong", "doc.counts", {"document.json": _doc_edit(files, counts={"nodes": 1, "leaves": 1})}, + "document.json's counts are not the number of nodes and leaves.") + case("rev-without-corrections", "doc.correction.rev", {"document.json": _doc_edit(files, rev=3)}, + "document.json's rev is not the number of corrections.") + case("receipt-unsorted-nodes", "doc.receipt.nodes", {"receipts.jsonl": _receipt_edit( + files, nodes=list(reversed([json.loads(x) for x in files["receipts.jsonl"].decode().splitlines()][0]["nodes"])))}, + "A receipt whose nodes are not sorted by ID.") + case("retired-live", "doc.id.retired-live", {"ids.jsonl": jsonl([ + {"id": nm["@a"], "kind": "retired", "successor": None, "node_kind": "sentence", "rev": 1}])}, + "An ID that is both live and retired.") + case("manifest-key", "package.manifest.key", {}, "The manifest names another key than document.json.", + raw=lambda d: _manifest_edit(d, key="other2026example-0000")) + case("unsafe-path", "package.path.unsafe", {}, "The manifest lists a path that leaves the package.", + raw=lambda d: _manifest_edit(d, files=[{"path": "../escape.txt", "sha256": "0" * 64, "size": 1}] + + json.load(open(os.path.join(d, "MANIFEST.json")))["files"])) + case("crlf", "package.file.canonical", {}, "A file whose lines end in CR LF, which is not the canonical form.", + raw=lambda d: _crlf(d)) + signer = [json.loads(x) for x in files["receipts.jsonl"].decode().splitlines()][0] + case("signer-mismatch", "doc.receipt.signature", {"receipts.jsonl": jsonl([ + RC.seal(dict(signer, by="cara"), secret("ana"), "ana")])}, + "A receipt whose signature was made with another key than the signer it names.") + case("history-wrong", "doc.history", {"document.json": _doc_edit(files, history="4" * 64)}, + "document.json's history is not the root over the corrections.") + d0 = json.loads(files["document.json"]) + wk = d0["work"]["key"] + bad = wk[:-4] + "zzzz" + case("key-suffix", "doc.key.suffix", {"document.json": _doc_edit(files, key=bad, work=dict(d0["work"], key=bad))}, + "A work key whose hash part is not the hash of the work's DOI.") + case("key-work", "doc.key.work", {"document.json": _doc_edit(files, key=wk + ".si")}, + "A document key whose suffix is not the version's.") + + +def _manifest_edit(d, **kw): + p = os.path.join(d, "MANIFEST.json") + m = json.load(open(p, encoding="utf-8")) + m.update(kw) + open(p, "w", encoding="utf-8").write(jcs.dumps(m) + "\n") + + +def _crlf(d): + p = os.path.join(d, "provenance.json") + data = open(p, "rb").read().replace(b"\n", b"\r\n") + open(p, "wb").write(data) + m = json.load(open(os.path.join(d, "MANIFEST.json"), encoding="utf-8")) + for f in m["files"]: + if f["path"] == "provenance.json": + f["sha256"], f["size"] = H(data), len(data) + open(os.path.join(d, "MANIFEST.json"), "w", encoding="utf-8").write(jcs.dumps(m) + "\n") + + +def _swap_first(ns, sec): + kids = sorted([n for n in ns if n["parent"] == sec], key=lambda n: n["ord"]) + kids[0]["ord"], kids[1]["ord"] = kids[1]["ord"], kids[0]["ord"] + d = Doc("x", ns) + order = d.preorder() + ns.sort(key=lambda n: order.index(n["id"])) + + +def _rename(ns, old, new): + for n in ns: + if n["id"] == old: + n["id"] = new + + +def _doc_edit(files, **kw): + d = json.loads(files["document.json"]) + d.update(kw) + return (jcs.dumps(d) + "\n").encode() + + +def _receipt_edit(files, **kw): + rs = [json.loads(x) for x in files["receipts.jsonl"].decode().splitlines()] + rs[0].update(kw) + return jsonl(rs) + + +def _rewrite_noncanonical(d): + p = os.path.join(d, "nodes.jsonl") + lines = open(p).read().splitlines() + lines[0] = json.dumps(json.loads(lines[0]), indent=None, separators=(", ", ": ")) + data = ("\n".join(lines) + "\n").encode() + open(p, "wb").write(data) + m = json.load(open(os.path.join(d, "MANIFEST.json"))) + for f in m["files"]: + if f["path"] == "nodes.jsonl": + f["sha256"], f["size"] = H(data), len(data) + open(os.path.join(d, "MANIFEST.json"), "w").write(jcs.dumps(m) + "\n") + + +# --- main ----------------------------------------------------------------------------- + + +OUT = ROOT + + +def main(out): + global OUT + OUT = out + for sub in ("fixtures/packages", "fixtures/records", "fixtures/invalid", "fixtures/keys"): + p = os.path.join(out, sub) + if os.path.exists(p): + shutil.rmtree(p) + os.makedirs(p) + vec = os.path.join(out, "vectors") + os.makedirs(vec, exist_ok=True) + for f in os.listdir(vec): + if f.endswith(".jsonl"): + os.remove(os.path.join(vec, f)) + for kid in KEYS: + with open(os.path.join(out, "fixtures", "keys", kid + ".ed25519.hex"), "w") as f: + f.write(secret(kid).hex() + "\n") + with open(os.path.join(out, "fixtures", "keys", "smithlab-team-secret.hex"), "w") as f: + f.write(TEAM_SECRET.hex() + "\n") + Names.registry.clear() + hale = build_hale(out) + build_checks_only(out, hale) + build_simple(out, "okafor2025open") + build_simple(out, "smithlab2027charges") + tiny = tiny_package(None) + tiny_dir = os.path.join(out, "fixtures", "packages", tiny["key"]) + write(tiny_dir, tiny["files"]) + # The same package zipped as spec/12 §12.2 says a producer writes a byte-identical archive. + write_zip(tiny_dir, tiny_dir + ZIP_EXTENSION) + records(out, hale) + vectors(out, hale, tiny) + invalid(out, tiny) + + +def cli(): + import argparse + ap = argparse.ArgumentParser( + description="Build every fixture and test vector from fixtures/src and the oracle. The build is " + "deterministic, and tools/check.py rebuilds into a temporary folder to prove it.") + ap.add_argument("--out", metavar="DIR", default=ROOT, + help="build into DIR instead of rewriting fixtures/ and vectors/ in this repository") + out = os.path.abspath(ap.parse_args().out) + if out != ROOT: + for sub in ("registry", "fixtures/src"): + dst = os.path.join(out, sub) + if not os.path.exists(dst): + shutil.copytree(os.path.join(ROOT, sub), dst) + main(out) + + +if __name__ == "__main__": + cli() diff --git a/tools/check.py b/tools/check.py new file mode 100644 index 0000000..d43a72e --- /dev/null +++ b/tools/check.py @@ -0,0 +1,1206 @@ +#!/usr/bin/env python3 +"""Check the whole repository. Exit status 0 means every check passed. + +Non-normative tooling; it needs Python 3.14 and the packages pixi installs (`jsonschema`, `cryptography`). +Usage: python3 tools/check.py [--quick] (--quick skips the determinism rebuild) +""" + +import base64 +import filecmp +import json +import os +import re +import sqlite3 +import subprocess +import sys +import shutil +import tempfile +import zipfile +from fractions import Fraction + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) + +from tools.oracle import align as AL # noqa: E402 +from tools.oracle import anchors as AN # noqa: E402 +from tools.oracle import corrections as CO # noqa: E402 +from tools.oracle import crypto, db, fingerprint as FP, grade as GR, hashing, jcs, receipts as RC, rights as RI # noqa: E402 +from tools.oracle import package as PK # noqa: E402 +from tools.oracle import keys as K, log as LG, mldsa, targets as TG, vrf # noqa: E402 +import unicodedata # noqa: E402 +from tools.oracle.model import Doc # noqa: E402 +from tools.oracle.notation import fold_math # noqa: E402 +from tools.oracle.schema import Store, validate as sv # noqa: E402 +from jsonschema import Draft202012Validator # noqa: E402 +from jsonschema.exceptions import SchemaError # noqa: E402 +from tools.oracle.text import cp_to_utf16, letter_first, mint_node_id, norm, record_id, sha256_hex # noqa: E402 +from tools.oracle.tree import Refusal # noqa: E402 + +FAILS = [] +# Folders that are not part of the repository: version control, environments and caches. +SKIP_DIRS = {".git", ".pixi", "node_modules", "__pycache__", "_site", ".claude"} +COUNTS = {} + + +def ok(section, cond, msg): + COUNTS[section] = COUNTS.get(section, 0) + 1 + if not cond: + FAILS.append("%s: %s" % (section, msg)) + + +def p(*parts): + return os.path.join(ROOT, *parts) + + +def jl(path): + with open(path, encoding="utf-8") as f: + return [json.loads(x) for x in f] + + +def js(path): + with open(path, encoding="utf-8") as f: + return json.load(f) + + +STORE = Store(p("schemas")) + + +def fixture_key(stem): + """The key the rule of spec/02 §2.4 mints for a fixture source, which names its package folder.""" + if stem == "tiny2026example": + from tools.build import TINY + d = TINY["document"] + else: + d = js(p("fixtures", "src", stem + ".json"))["document"] + return K.document_key(K.work_key(d["work"].get("same_as"), d["meta"]), d["version"]) + + +HALE = fixture_key("hale2026harmonic") +TINY_KEY = fixture_key("tiny2026example") +SCHEMA_FILES = sorted(f for f in os.listdir(p("schemas")) if f.endswith(".schema.json")) +FORMAT_SCHEMA = {f["format"]: f["schema"] for f in js(p("registry", "formats.json"))["formats"]} + + +def schema_errors(name, rec): + return sv(STORE, name, rec) + + +def check_schemas(): + for f in SCHEMA_FILES: + s = STORE.load(f) + ok("schemas", s.get("$id", "").endswith("/" + f), "%s: $id must end with its file name" % f) + try: + Draft202012Validator.check_schema(s) + meta = True + except SchemaError: + meta = False + ok("schemas", meta, "%s is not a valid draft 2020-12 schema" % f) + for f in js(p("registry", "formats.json"))["formats"]: + ok("schemas", os.path.exists(p("schemas", f["schema"])) or os.path.exists(p(f["schema"])), + "%s names a missing schema %s" % (f["format"], f["schema"])) + + +def valid(section, name, rec, where): + e = schema_errors(name, rec) + ok(section, not e, "%s: %s" % (where, e[:3])) + + +def check_registry(): + valid("registry", "stamps.schema.json", js(p("registry", "stamps", "scienting-stamps-1.json")), "stamps") + valid("registry", "stages.schema.json", js(p("registry", "stages", "scienting-stages-default.json")), "stages") + for rule in js(p("registry", "stamps", "scienting-stamps-1.json"))["rules"]: + re.compile(rule["pattern"]) + rules = {r["id"] for r in js(p("registry", "rules.json"))["rules"]} + for f in sorted(os.listdir(p("tools", "oracle"))): + if f.endswith(".py"): + for rule in re.findall(r'(?:Refusal|rep\.add)\(\s*"([a-z][a-z0-9.-]+)"', open(p("tools", "oracle", f)).read()): + if not rule.endswith("."): + ok("registry", rule in rules, "tools/oracle/%s raises %s, which is not in rules.json" % (f, rule)) + for v in ("text-hash", "match-hash", "align-key", "own-hash", "node-hash"): + ok("registry", "doc.hash." + v in rules, "doc.hash.%s is not in rules.json" % v) + # The character data the hashes depend on is pinned to one Unicode version (spec/01 §1.4). + ok("registry", unicodedata.unidata_version == "16.0.0", + "the tools run on Unicode %s, but the specification pins 16.0" % unicodedata.unidata_version) + for d in sorted(os.listdir(p("fixtures", "invalid"))): + ok("registry", js(p("fixtures", "invalid", d, "expect.json"))["rule"] in rules, "%s: rule is not in rules.json" % d) + + +def check_records(): + R = p("fixtures", "records") + roster = {k["id"]: k for k in js(os.path.join(R, "roster.json"))["keys"]} + for dirpath, _, files in os.walk(R): + if os.path.relpath(dirpath, R).split(os.sep)[0] == "log": + continue # the log's tiles and checkpoint are checked by check_log + for f in sorted(files): + path = os.path.join(dirpath, f) + rel = os.path.relpath(path, R) + if f.endswith(".csv"): + continue + if f.endswith(".jsonl"): + sch = {"annotations.jsonl": "annotation.schema.json", "draft-ids.jsonl": "draft-ids.schema.json", + "anchors.jsonl": "value-anchor.schema.json"}[f] + for i, rec in enumerate(jl(path), 1): + valid("records", sch, rec, "%s:%d" % (rel, i)) + continue + rec = js(path) + if f == "view.vl.json": + sch = "view.schema.json" + elif f == "layer.json": + sch = "layer.schema.json" + else: + sch = FORMAT_SCHEMA[rec["format"]] + valid("records", sch, rec, rel) + if rec.get("format") == "scienting-suspect/1": + ok("records", rec["id"] == record_id(sha256_hex(jcs.dumpb({k: v for k, v in rec.items() if k != "id"}))), + "%s: id does not derive" % rel) + if rec.get("signature"): + ok("records", RC.verify_signature(rec, roster), "%s: signature does not verify" % rel) + ok("records", rec["id"] == record_id(RC.record_hash(rec)), "%s: id does not derive" % rel) + ok("records", rec.get("by") == rec["signature"]["key"], "%s: by differs from the signing key" % rel) + ok("records", rec["format"] in RC.SIGNED_FORMATS, "%s: %s is not a signed format" % (rel, rec["format"])) + # A QA report is signed (spec/08 §8.9); a checklist is covered by its receipt at its hash (spec/08 §8.11). + ok("records", "signature" in js(os.path.join(R, "qa-report.json")), "the QA report is not signed") + cl = js(os.path.join(R, "checklist", "checklist.json")) + clr = js(os.path.join(R, "checklist", "receipt.json")) + cl_hash = sha256_hex(jcs.dumpb({k: v for k, v in cl.items() if k != "id"})) + ok("records", cl["id"] == record_id(cl_hash), "checklist id does not derive from its content hash") + ok("records", ["checklist", cl["id"], cl_hash] in clr["records"], "the checklist is not covered at its hash") + ok("records", RC.receipt_status(clr, [clr], roster) == "valid", "the checklist receipt is not valid") + ok("records", {i["result"] for i in cl["items"]} <= {"pass", "fail", "not-applicable"}, "checklist results") + # Proofs and the checkpoint (spec/08 §8.5, spec/11 §11.5, spec/23). + hale_receipts = {r["id"]: r for r in jl(p("fixtures", "packages", HALE, "receipts.jsonl"))} + pi = js(os.path.join(R, "proof-inclusion.json")) + rcp = hale_receipts.get(pi["receipt"]) + ok("records", rcp is not None and pi["item"] in crypto.receipt_items(rcp) and pi["root"] == rcp["root"] + and crypto.verify_inclusion(pi["root"], crypto.leaf_hash(pi["item"]), pi["index"], pi["size"], pi["path"]), + "the inclusion proof does not verify against its receipt") + pc = js(os.path.join(R, "proof-consistency.json")) + hist = PK.history_items(jl(p("fixtures", "packages", HALE, "corrections.jsonl"))) + ok("records", pc["to_root"] == js(p("fixtures", "packages", HALE, "document.json"))["history"] + and pc["from_root"] == crypto.history_root(hist[:pc["from_rev"]]) and pc["to_rev"] == len(hist) + and crypto.verify_consistency(pc["from_root"], pc["to_root"], pc["from_rev"], pc["to_rev"], pc["path"]), + "the consistency proof does not verify against the example's history") + cp = js(os.path.join(R, "checkpoint.json")) + note, verified = LG.verify_checkpoint(cp["note"], roster) + ok("records", note and note["origin"] == cp["origin"] and note["size"] == cp["size"] and note["root"] == cp["root"] + and verified == [sg["name"] for sg in cp["signatures"]], "the checkpoint note does not verify") + entries = [LG.entry(r) for r in hale_receipts.values()] + [LG.entry(js(os.path.join(R, "roster.json")))] + ok("records", LG.root(entries) == cp["root"], "the checkpoint root is not the root over the example's receipts") + # A project's settings.stages holds the id of a stage file. + stage_ids = {js(p("registry", "stages", f))["id"] for f in os.listdir(p("registry", "stages")) if f.endswith(".json")} + stages = js(os.path.join(R, "project.json")).get("settings", {}).get("stages") + ok("records", stages is None or stages in stage_ids, "project settings.stages names no stage file: %s" % stages) + # Confirmed annotations are covered by the layer receipt at their content hash. + lr = js(os.path.join(R, "layer", "receipt.json")) + items = {(k, i): h for k, i, h in lr["records"]} + for a in jl(os.path.join(R, "layer", "annotations.jsonl")): + sel = a["target"].get("selector", []) + anchors = [x["anchor"] for x in sel if x.get("type") == "AnchorSelector"] + for x in sel: + if x.get("type") == "TextQuoteSelector": + q = anchors[0].get("quote", {}) if anchors else {} + same = all(x.get(k, "") == q.get(k, "") for k in ("exact", "prefix", "suffix")) + ok("records", "exact" in q and same and set(x) <= {"type", "exact", "prefix", "suffix"}, + "a TextQuoteSelector must repeat the anchor's plain quote: " + a["id"]) + body = {k: v for k, v in a.items() if k not in ("status", "receipt", "modified")} + ok("records", a["id"].rsplit("/", 1)[1] == record_id(sha256_hex(jcs.dumpb( + {k: v for k, v in body.items() if k != "id"}))), "annotation id does not derive: " + a["id"]) + if a["status"] == "confirmed": + ok("records", items.get(("annotation", a["id"].rsplit("/", 1)[1])) == sha256_hex(jcs.dumpb(body)), + "a confirmed annotation is not covered at its hash: " + a["id"]) + ok("records", lr["root"] == crypto.merkle_root(crypto.receipt_items(lr)), "layer receipt root") + # Every figure of the main example has a confirmed description, which is what an accessible export needs. + anns = jl(os.path.join(R, "layer", "annotations.jsonl")) + described = {x["anchor"]["node"] for a in anns if a["body"]["kind"] == "description" and a["status"] == "confirmed" + for x in a["target"].get("selector", []) if x.get("type") == "AnchorSelector"} + figures = {n["id"] for n in jl(p("fixtures", "packages", HALE, "nodes.jsonl")) if n["kind"] == "figure"} + ok("records", figures and figures <= described, "a figure of the example has no confirmed description: %s" + % sorted(figures - described)) + ok("records", any(a["body"]["kind"] == "generated" and a["status"] == "proposed" for a in anns), + "the layer has no proposed generated annotation to contrast with the confirmed description") + + +def state_of(pkg): + return PK.read_state(pkg) + + +def check_packages(): + team_key = crypto.team_key(bytes.fromhex(open(p("fixtures", "keys", "smithlab-team-secret.hex")).read().strip())) + pk = p("fixtures", "packages") + for d in sorted(os.listdir(pk)): + if not os.path.isdir(os.path.join(pk, d)): + continue # the zipped form is checked by check_zips + rep = PK.validate(os.path.join(pk, d), STORE, team_key if d.endswith("checks-only") else None) + ok("packages", bool(rep), "%s: %s" % (d, rep.errors[:4])) + # The checks-only export carries exactly the HMAC values of the full document. + full = state_of(os.path.join(pk, HALE)).doc() + keyed = hashing.compute(full, hashing.Hasher(team_key)) + for n in jl(os.path.join(pk, HALE + ".checks-only", "nodes.jsonl")): + ok("packages", all(n[k] == keyed[n["id"]][k] for k in keyed[n["id"]]), "keyed values of " + n["id"]) + # Every reported value in the hale dataset is anchored to the cell that prints it, and resolves as current. + hp = os.path.join(pk, HALE) + st = state_of(hp) + ds = os.path.join(hp, "datasets", "ds-hale-bond-params") + desc = js(os.path.join(ds, "datapackage.json")) + valid("packages", "dataset.schema.json", desc, "ds-hale-bond-params") + rows = {} + with open(os.path.join(ds, "data.csv"), encoding="utf-8") as f: + head = f.readline().strip().split(",") + for ln in f: + vals = dict(zip(head, ln.rstrip("\n").split(","))) + rows[vals["key"]] = vals + doc = st.doc() + for va in jl(os.path.join(ds, "anchors.jsonl")): + valid("packages", "value-anchor.schema.json", va, "anchors.jsonl") + r = AN.resolve(st, va["anchor"]) + ok("packages", r["result"] == "current", "value anchor %s/%s is %s" % (va["row"], va["field"], r["result"])) + ok("packages", norm(hashing.plain(doc, r["node"])) == rows[va["row"]][va["field"]], + "CSV value differs from its cell: %s/%s" % (va["row"], va["field"])) + for m in jl(os.path.join(hp, "derived", "mathml.jsonl")): + valid("packages", "mathml.schema.json", m, "derived/mathml.jsonl") + + +def _zip(path, entries): + """Write a zip of (name, bytes, mode) entries the way a producer does (spec/12 §12.2).""" + with zipfile.ZipFile(path, "w") as zf: + for name, data, mode in entries: + zi = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + zi.compress_type = zipfile.ZIP_STORED + zi.create_system = 3 + zi.external_attr = mode << 16 + zf.writestr(zi, data) + + +def check_zips(): + """Zipped packages (spec/12 §12.2 and §12.8). + + The fixture archive unpacks to the folder it was made from byte for byte, validates, keeps the producer's + byte-identical rules, and is reproduced by write_zip; crafted archives are refused with the rule named. + """ + pk = p("fixtures", "packages") + zips = [f for f in sorted(os.listdir(pk)) if f.endswith(PK.ZIP_EXTENSION)] + ok("zips", len(zips) >= 1, "no zipped package among the fixtures") + for z in zips: + key = z[:-len(PK.ZIP_EXTENSION)] + folder = os.path.join(pk, key) + rep = PK.validate_zip(os.path.join(pk, z), STORE) + ok("zips", bool(rep), "%s: %s" % (z, rep.errors[:4])) + with zipfile.ZipFile(os.path.join(pk, z)) as zf: + infos = zf.infolist() + names = [i.filename for i in infos] + ok("zips", names == sorted(names, key=lambda n: n.encode("utf-8")), "%s: entries are sorted by path" % z) + ok("zips", all(i.date_time == (1980, 1, 1, 0, 0, 0) and i.extra == b"" and not i.is_dir() + and i.compress_type in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED) for i in infos), + "%s: entries are dated 1980-01-01 00:00, carry no extra field and are stored or deflated" % z) + ok("zips", {n.partition("/")[0] for n in names} == {key}, "%s: one top-level folder named by the key" % z) + ok("zips", key + "/MANIFEST.json" in names, "%s: the manifest is an entry" % z) + with tempfile.TemporaryDirectory() as tmp: + root = PK.read_zip(os.path.join(pk, z), tmp) + ok("zips", _same_tree(folder, root), "%s: unpacks to its folder byte for byte" % z) + again = os.path.join(tmp, "again.zip") + PK.write_zip(root, again) + ok("zips", filecmp.cmp(again, os.path.join(pk, z), shallow=False), "%s: write_zip reproduces it" % z) + # Crafted archives, built from the tiny package. + tiny = p("fixtures", "packages", TINY_KEY) + entries = [] + for rel in sorted(os.path.relpath(os.path.join(d, f), tiny).replace(os.sep, "/") + for d, _, fs in os.walk(tiny) for f in fs): + with open(os.path.join(tiny, rel), "rb") as fh: + entries.append((TINY_KEY + "/" + rel, fh.read(), 0o644)) + cases = [ + ("two-folders", entries + [("other2026/x.txt", b"x", 0o644)], "package.zip.entry"), + ("duplicate", entries + [entries[0]], "package.zip.entry"), + ("folder-not-a-key", [("Tiny Example/" + n.partition("/")[2], d, m) for n, d, m in entries], "package.zip.entry"), + ("folder-not-the-key", [("other2026example/" + n.partition("/")[2], d, m) for n, d, m in entries], + "package.zip.entry"), + ("dot-dot", entries + [(TINY_KEY + "/assets/../../x", b"x", 0o644)], "package.zip.entry"), + ("symlink", entries + [(TINY_KEY + "/assets/link", b"document.json", 0o120777)], "package.zip.entry"), + ("no-folder", [(n.partition("/")[2], d, m) for n, d, m in entries], "package.zip.entry"), + ("directory-entries", entries + [(TINY_KEY + "/assets/", b"", 0o755)], None), + ] + with tempfile.TemporaryDirectory() as tmp: + for name, ents, rule in cases: + path = os.path.join(tmp, name + ".zip") + with __import__("warnings").catch_warnings(): + __import__("warnings").simplefilter("ignore") + _zip(path, ents) + rules = {e["rule"] for e in PK.validate_zip(path, STORE).errors} + ok("zips", (rule in rules) if rule else not rules, "%s: expected %s, got %s" % (name, rule, sorted(rules))) + ok("zips", {e["rule"] for e in PK.validate_zip(os.path.join(tmp, "directory-entries.zip"), STORE, limit=100).errors} + == {"package.zip.size"}, "an archive past the size limit is refused with package.zip.size") + with open(os.path.join(tmp, "not-a.zip"), "wb") as fh: + fh.write(b"PK\x03\x04 this is not an archive") + ok("zips", {e["rule"] for e in PK.validate_zip(os.path.join(tmp, "not-a.zip"), STORE).errors} + == {"package.zip.entry"}, "bytes that are not an archive are refused") + _zip(os.path.join(tmp, "empty.zip"), []) + ok("zips", {e["rule"] for e in PK.validate_zip(os.path.join(tmp, "empty.zip"), STORE).errors} + == {"package.zip.entry"}, "an empty archive is refused") + outside = [f for f in os.listdir(tmp) if not f.endswith(".zip")] + ok("zips", not outside, "reading archives wrote outside their folders: %s" % outside) + + +def check_owner_revocation(): + """A key the roster makes an owner may revoke anyone's receipt (spec/08 §8.7); any other key may not. + + No fixture has such a revocation, so this signs one: Ben revokes a receipt of Ana's, once with the + roster role owner and once as the editor he is. + """ + hp = p("fixtures", "packages", HALE) + revoker = "ben" + secret = bytes.fromhex(open(p("fixtures", "keys", revoker + ".ed25519.hex")).read().strip()) + roster_rec = js(os.path.join(hp, "roster.json")) + as_owner = dict(roster_rec, keys=[dict(k, role="owner") if k["id"] == revoker else k for k in roster_rec["keys"]]) + owner_roster = {k["id"]: k for k in as_owner["keys"]} + nodes = jl(os.path.join(hp, "nodes.jsonl")) + vals = {n["id"]: n for n in nodes} + document = js(os.path.join(hp, "document.json")) + recs, fidelity = jl(os.path.join(hp, "receipts.jsonl")), None + for target in recs: + if target["format"] != "scienting-receipt/1" or target["by"] == revoker: + continue + rev = RC.seal({"format": "scienting-revocation/1", "receipt": target["id"], "reason": "Owner check.", + "by": revoker, "at": "2026-10-13T00:00:00Z"}, secret, revoker) + trial = sorted(recs + [rev], key=lambda r: (r["at"], r["id"])) + fidelity = RC.distribution(RC.effective_levels(nodes, vals, trial, owner_roster)) + if fidelity != document["fidelity"]: + recs = trial + break + ok("packages", fidelity != document["fidelity"], "no receipt of the hale package matters to its levels") + document["fidelity"] = fidelity + manifest = js(os.path.join(hp, "MANIFEST.json")) + files, absent = {}, [f for f in manifest["files"] if f.get("present", True) is False] + for f in manifest["files"]: + if f.get("present", True): + with open(os.path.join(hp, f["path"]), "rb") as fh: + files[f["path"]] = fh.read() + files["receipts.jsonl"] = PK.jsonl(recs) + files["document.json"] = PK.line(document).encode("utf-8") + for roster, passes in ((as_owner, True), (roster_rec, False)): + files["roster.json"] = PK.line(roster).encode("utf-8") + with tempfile.TemporaryDirectory() as tmp: + PK.write(tmp, files, absent) + rules = {e["rule"] for e in PK.validate(tmp, STORE).errors} + role = next(k["role"] for k in roster["keys"] if k["id"] == revoker) + ok("packages", ("doc.fidelity.report" not in rules) == passes and rules <= {"doc.fidelity.report"}, + "a revocation by a key with the role %s: %s" % (role, sorted(rules))) + + +def check_unsafe_paths(): + """Loading and writing refuse manifest paths that leave the package, even without a prior validation.""" + with tempfile.TemporaryDirectory() as tmp: + pkg = os.path.join(tmp, "pkg") + shutil.copytree(p("fixtures", "packages", TINY_KEY), pkg) + manifest = js(os.path.join(pkg, "MANIFEST.json")) + for bad in ("../outside", "/etc/outside", "assets/../../outside"): + m = dict(manifest, files=[{"path": bad, "sha256": "0" * 64, "size": 1}] + manifest["files"]) + with open(os.path.join(pkg, "MANIFEST.json"), "w", encoding="utf-8") as f: + f.write(PK.line(m)) + conn = sqlite3.connect(":memory:") + db.create(conn, p("sql")) + try: + db.load(conn, pkg, os.path.join(tmp, "assets")) + rule = None + except Refusal as e: + rule = e.rule + ok("packages", rule == "package.path.unsafe", "load accepted the manifest path %r" % bad) + try: + PK.write(os.path.join(tmp, "out"), {bad: b"x", "document.json": b"{}"}) + rule = None + except Refusal as e: + rule = e.rule + ok("packages", rule == "package.path.unsafe", "write accepted the path %r" % bad) + ok("packages", not os.path.exists(os.path.join(tmp, "outside")), "a file was written outside the package") + + +def reference_values(doc, hasher=hashing.PLAIN): + """The five values the slow way: every node projected from its whole subtree, as spec/05 §5.1 reads. + + The oracle's hashing.Index joins its children's cached projections and rehashes only what a correction + changed; this independent computation must always agree with it. + """ + values = {} + for nid in reversed(doc.preorder()): + st, _ = hashing.strict(doc, nid) + own = hasher(jcs.dumpb(hashing.own_content(doc, nid, lambda x: values[x]["text_hash"]))) + values[nid] = {"text_hash": hasher(norm(hashing.plain(doc, nid))), "match_hash": hasher(st), + "align_key": hasher(hashing.lossy_from_strict(st)), "own_hash": own, + "node_hash": hasher(jcs.dumpb({"own": own, "children": [ + values[c]["node_hash"] for c in doc.kids(nid)]}))} + return values + + +def check_incremental(): + """After every correction of every fixture source, the incrementally kept values equal the slow ones.""" + from tools.build import Names, build_document + team = hashing.Hasher(crypto.team_key(bytes.fromhex( + open(p("fixtures", "keys", "smithlab-team-secret.hex")).read().strip()))) + Names.registry.clear() + # Sorted, the main example comes first, so the draft's references to its sentences resolve. + for f in sorted(os.listdir(p("fixtures", "src"))): + if not f.endswith(".json"): + continue + src = js(p("fixtures", "src", f)) + src["name"] = f[:-len(".json")] # the name other sources refer to, as the builder sets it + _, _, _, _, states, _ = build_document(src) + keyed = None + for rev in sorted(states): + st = states[rev] + ok("packages", st.index().values == reference_values(st.doc()), + "%s rev %d: incremental values differ from the full computation" % (f, rev)) + # The keyed values, carried forward from revision 0 by updates alone. + keyed = hashing.Index.build(st.doc(), team) if keyed is None else keyed.update(st.doc()) + ok("packages", keyed.values == reference_values(st.doc(), team), + "%s rev %d: incremental keyed values differ from the full computation" % (f, rev)) + Names.registry.clear() + + +def check_gate(): + """The gate runs on a proposal against a package's current state and leaves that state unchanged.""" + st = state_of(p("fixtures", "packages", HALE)) + before = json.dumps(st.nodes, sort_keys=True) + target = next(n for n in st.nodes if n["kind"] == "sentence" and "Each bond" in hashing.plain(st.doc(), n["id"])) + base = {"format": "scienting-correction/1", "evidence": [{"kind": "person", "by": "ana"}], "author": {"key": "ana"}, + "reason": "Gate check.", "at": "2026-10-14T09:00:00Z", "status": "proposed"} + st1, rec = CO.gate(st, dict(base, op="replace", target=target["id"], find="Each bond", replace="Every bond"), STORE) + ok("packages", rec["index"] == st.rev + 1 and st1.rev == st.rev + 1, "the gate's record has the next index") + ok("packages", json.dumps(st.nodes, sort_keys=True) == before, "the gate changed the state it was given") + valid("packages", "correction.schema.json", rec, "gate record") + crop = [{"kind": "crop", "sha256": "0" * 64}] + cell = next(n for n in st.nodes if n["kind"] == "cell") + section = next(n for n in st.nodes if n["kind"] == "section" and "label" in n) + header = next(c["header_path"][0] for n in st.nodes if n["kind"] == "table" for c in n["attrs"]["columns"]) + for bad, rule in (({"op": "replace", "target": target["id"]}, "correction.schema"), + (dict(base, op="replace", target=target["id"], find="0.01", replace="0.02"), "correction.digits"), + (dict(base, op="insert", target=target["id"], run=9, offset=0, text="x", evidence=crop), + "correction.insert"), + # Rule 11 of spec/11 §11.4: the result keeps to chapter 3, attribute shapes included. + (dict(base, op="set", target=cell["id"], attr="colspan", value="2"), "doc.node.attrs"), + (dict(base, op="set", target=cell["id"], attr="role", value="boss"), "doc.node.attrs"), + (dict(base, op="insert", parent=target["parent"], place_after=target["id"], evidence=crop, + node={"kind": "sentence", "content": [{"kind": "text", "text": "Junk."}], "foo": 1}), + "doc.node.schema"), + (dict(base, op="delete", target=header), "doc.table.header")): + try: + CO.gate(st, bad, STORE) + got = None + except Refusal as e: + got = e.rule + ok("packages", got == rule, "the gate gave %s for a proposal that breaks %s" % (got, rule)) + # A relabel to null removes the label instead of writing null (spec/03 §3.1). + st2, _ = CO.gate(st, dict(base, op="relabel", target=section["id"], label=None, evidence=crop), STORE) + ok("packages", "label" not in next(n for n in st2.nodes if n["id"] == section["id"]), "relabel to null drops the label") + + +def check_invalid(): + base = p("fixtures", "invalid") + for d in sorted(os.listdir(base)): + expect = js(os.path.join(base, d, "expect.json"))["rule"] + zipped = os.path.join(base, d, "package" + PK.ZIP_EXTENSION) + rep = PK.validate_zip(zipped, STORE) if os.path.exists(zipped) else PK.validate(os.path.join(base, d, "package"), STORE) + rules = {e["rule"] for e in rep.errors} + ok("invalid", expect in rules, "%s: expected %s, got %s" % (d, expect, sorted(rules) or "no error")) + + +def V(name): + return jl(p("vectors", name)) + + +ROSTER_BY_ID_CHECK = {k["id"]: k for k in js(p("fixtures", "records", "roster.json"))["keys"]} + + +def check_vectors(): + from tools.build import parse_doubles, projection_case, sentence_doc + for c in V("canonical-json.jsonl"): + ok("vectors", jcs.dumps(parse_doubles(c["input_json"])) == c["output"], "canonical-json " + c["id"]) + for c in V("normalize.jsonl"): + ok("vectors", norm(c["input"]) == c["output"] and sha256_hex(c["output"]) == c["sha256"], "normalize " + c["id"]) + for c in V("ids.jsonl"): + if "digest" in c: + d = bytes.fromhex(c["digest"]) + ok("vectors", letter_first(d, 7) == c["node_id"] and record_id(c["digest"]) == c["record_id"], c["id"]) + else: + ok("vectors", mint_node_id(c["doc"], c["kind"], c["text_hash"], c["occurrence"]) == c["node_id"] + and jcs.dumps(["scienting-id/1", c["doc"], c["kind"], c["text_hash"], c["occurrence"]]) == c["payload"], + c["id"]) + for c in V("notation.jsonl"): + ok("vectors", fold_math(c["notation"], c["source"]) == c["canonical"], "notation " + c["id"]) + byid = {} + for c in V("projections.jsonl"): + again = projection_case(c["id"], c["runs"], c.get("note")) + ok("vectors", again == c, "projection " + c["id"]) + byid[c["id"]] = c + for g in V("projection-groups.jsonl"): + field = g.get("same") or g.get("differ") + values = {byid[x][field] for x in g["cases"]} + ok("vectors", len(values) == 1 if "same" in g else len(values) == len(g["cases"]), "group " + g["id"]) + for c in V("offsets.jsonl"): + doc, s = sentence_doc(c["runs"]) + t = hashing.plain(doc, s) + ok("vectors", t == c["plain"] and t[c["span"][0]:c["span"][1]] == c["exact"] + and [cp_to_utf16(t, x) for x in c["span"]] == c["utf16_span"], "offsets " + c["id"]) + ok("vectors", t.encode("utf-16-le")[2 * c["utf16_span"][0]:2 * c["utf16_span"][1]].decode("utf-16-le") + == c["exact"], "offsets utf16 " + c["id"]) + for c in V("hashes.jsonl"): + d = Doc(c["doc"], c["nodes"]) + vals = hashing.compute(d) + ok("vectors", vals == c["values"], "hashes " + c["id"]) + for nid, o in c["own_content"].items(): + ok("vectors", sha256_hex(jcs.dumpb(o)) == vals[nid]["own_hash"], "own content " + nid) + for c in V("addresses.jsonl"): + ok("vectors", AN.parse_address(c["address"]) == c["parsed"], "address " + c["id"]) + if c["parsed"]: + q = c["parsed"] + ok("vectors", AN.format_address(q["store"], q["key"], q["node"], q["pin"], q["span"]) == c["address"], + "address round trip " + c["id"]) + kv = {c["id"]: c for c in V("keyed.jsonl")} + k = crypto.team_key(bytes.fromhex(kv["team-key"]["secret"])) + ok("vectors", k.hex() == kv["team-key"]["key"] and crypto.key_id(k) == kv["team-key"]["key_id"], "team key") + ok("vectors", hashing.Hasher(k)(kv["team-hmac-text"]["text"]) == kv["team-hmac-text"]["text_hash_keyed"], "hmac") + ok("vectors", [crypto.page_hash(f) for f in kv["page-hashes"]["page_fps"]] == kv["page-hashes"]["page_hashes"], + "page hashes") + pkk = crypto.page_key(kv["page-key"]["page_fps"], kv["page-key"]["context"]) + ok("vectors", pkk.hex() == kv["page-key"]["key"] and hashing.Hasher(pkk)(kv["page-key"]["quote"]) + == kv["page-key"]["quote_mac"], "page key") + nh = kv["non-holder"] + ok("vectors", sha256_hex(nh["guess"]) != nh["quote_mac"] and crypto.page_key(nh["wrong_page_fps"], kv["page-key"][ + "context"]).hex() == nh["wrong_key"] != kv["page-key"]["key"], "non-holder") + stamps = js(p("registry", "stamps", "scienting-stamps-1.json"))["rules"] + fpv = {c["id"]: c for c in V("fingerprints.jsonl")} + for c in fpv.values(): + fp, removed = FP.fingerprint(c["page"], stamps) + ok("vectors", fp == c["page_fp"] and removed == c["removed"], "fingerprint " + c["id"]) + ok("vectors", fpv["acs-download-1"]["page_fp"] == fpv["acs-download-2"]["page_fp"], "ACS downloads share a page") + ok("vectors", fpv["wiley-download-1"]["page_fp"] == fpv["wiley-download-2"]["page_fp"], "Wiley downloads share") + ok("vectors", fpv["acs-download-1"]["page_fp"] != fpv["acs-changed-text"]["page_fp"], "a changed page differs") + for c in V("merkle.jsonl"): + ok("vectors", crypto.merkle_root(c["items"]) == c["root"], "merkle " + c["id"]) + for c in V("signatures.jsonl"): + if "did" in c: + ok("vectors", crypto.did_key(c["scheme"], bytes.fromhex(c["public"])) == c["did"], c["id"]) + elif c["id"].startswith("ml-dsa-65"): + pk, sk = mldsa.keygen(bytes.fromhex(c["seed"])) + ok("vectors", pk.hex() == c["public"], c["id"] + ": key generation") + if "signature" in c: + sig = bytes.fromhex(c["signature"]) + ok("vectors", mldsa.sign(sk, bytes.fromhex(c["message"])) == sig and mldsa.verify(pk, bytes.fromhex(c["message"]), sig) + and not mldsa.verify(pk, b"x", sig), c["id"] + ": signature") + elif "secret" in c: + sk = bytes.fromhex(c["secret"]) + ok("vectors", crypto.ed25519_public(sk).hex() == c["public"] + and crypto.ed25519_sign(sk, c["message"].encode()).hex() == c["signature"], c["id"]) + else: + r = c["receipt"] + ok("vectors", RC.record_hash(r) == c["record_hash"] and RC.signing_payload(r).decode() == c["payload"] + and crypto.ed25519_verify(base64.b64decode(c["public"]), c["payload"].encode(), + base64.b64decode(r["signature"]["value"])), c["id"]) + for c in V("spotcheck.jsonl"): + ok("vectors", RC.sample_size(c["n"], c["rate"], c["minimum"]) == c["size"], "sample size " + c["id"]) + if "seed" in c: + seed = base64.b64decode(c["seed"]) + beta = vrf.verify(base64.b64decode(c["checker_public"]), c["seed_payload"].encode(), seed) + ok("vectors", beta is not None and beta.hex() == c["beta"] + and RC.sample(c["receipt"], seed, c["rate"], c["minimum"]) == c["sample"], "spot-check " + c["id"]) + for c in V("bounds.jsonl"): + ok("vectors", RC.bound_string(c["errors"], c["sampled"], c["confidence"]) == c["bound"], "bound " + c["id"]) + ok("vectors", Fraction(next(c for c in V("bounds.jsonl") if c["id"] == "k0-n300")["bound"]) < Fraction(1, 100), + "zero errors in 300 bounds the rate below 1%") + for c in V("alignment.jsonl"): + ok("vectors", AL.align(c["a"], c["b"]) == c["expect"], "alignment " + c["id"]) + for c in V("corrections.jsonl"): + st = CO.State(c["doc"], [{k: v for k, v in n.items() if k not in ( + "text_hash", "match_hash", "align_key", "own_hash", "node_hash")} for n in c["before"]["nodes"]]) + try: + st1, rec = CO.apply(st, c["correction"], store=STORE) + v1 = hashing.compute(st1.doc()) + ok("vectors", st1.index().values == reference_values(st1.doc()), + "correction %s: incremental values differ from the full computation" % c["id"]) + got = {"record": rec, "nodes": [dict(n, **v1[n["id"]]) for n in st1.nodes], "ids": st1.ids} + except Refusal as e: + got = {"refusal": e.rule} + ok("vectors", got == c["expect"], "correction " + c["id"]) + states = {} + for c in V("resolve.jsonl"): + st = states.setdefault(c["package"], state_of(p(c["package"]))) + valid("vectors", "anchor.schema.json", c["anchor"], "resolve " + c["id"]) + ok("vectors", AN.resolve(st, c["anchor"]) == c["expect"], "resolve " + c["id"]) + results = {c["expect"]["result"] for c in V("resolve.jsonl")} + ok("vectors", results == {"current", "changed-below", "moved", "reattached", "stale"}, "all five results occur") + for c in V("rights.jsonl"): + if "stage" in c: + w = {"rights": c["rights"], "origin": c["origin"], "sha256": c["witness_sha256"]} + uses, expired = RI.effective(w, c["policies"], c["now"]) + allowed, lacking = RI.check(c["requires"], c["processor"], uses) + ok("vectors", {"allowed": allowed, "lacking": lacking, "expired": expired} == c["expect"], "rights " + c["id"]) + elif "backend" in c: + a, f = RI.backend_allowed(c["backend"], c["profile"]) + ok("vectors", {"allowed": a, "failed": f} == c["expect"], "rights " + c["id"]) + else: + ok("vectors", RI.combine([RI.flat(r) for r in c["rights"]]) == c["expect"], "rights " + c["id"]) + for c in V("grading.jsonl"): + st = states.setdefault(c["package"], state_of(p(c["package"]))) + ok("vectors", GR.grade(c["exercise"], c["submission"], st) == c["expect"], "grading " + c["id"]) + sets = {} + for c in V("targets.jsonl"): + st = states.setdefault(c["package"], state_of(p(c["package"]))) + got = TG.target_set(st.doc(), c["anchored"], c["digit_rule"]) + ok("vectors", got == c["expect"]["nodes"], "targets " + c["id"]) + sets[c["digit_rule"]] = {n for n, _ in got} + d = st.doc() + ok("vectors", all(d.nodes[n]["kind"] in TG.MATH_KINDS for n, r in got if r == "equation") + and all(d.nodes[n]["kind"] in TG.LEAVES for n, _ in got), "targets %s: only leaves, equations by kind" % c["id"]) + ok("vectors", set(sets) == set(TG.DIGIT_RULES) and sets["none"] <= sets["anchored-sections"] <= sets["all"] + and sets["none"] < sets["all"], "the three digit rules nest") + for c in V("keys.jsonl"): + if "version" in c: + ok("vectors", K.document_key(c["work_key"], c["version"]) == c["document_key"], "keys " + c["id"]) + elif "taken" in c: + ok("vectors", K.unique_work_key(c["same_as"], c["meta"], set(c["taken"])) == c["key"] + and len(c["key"].rsplit("-", 1)[1]) == 6, "keys " + c["id"]) + else: + ident = K.identity(c["same_as"], c["meta"]) + ok("vectors", ident == c["identity"] and K.scholar_part(c["meta"]) == c["scholar"] + and K.SCHEME + ":" + ident == c["payload"] and sha256_hex(c["payload"]) == c["sha256"] + and K.suffix(ident) == c["suffix"] and K.suffix(ident, 8) == c["suffix_8"] + and c["suffix_8"].startswith(c["suffix"]) and K.work_key(c["same_as"], c["meta"]) == c["key"] + and K.WORK_KEY_RE.match(c["key"]) and c["key"] == c["scholar"] + "-" + c["suffix"], "keys " + c["id"]) + for c in V("proofs.jsonl"): + items = c["items"] + if c["kind"] == "root": + ok("vectors", crypto.history_root(items) == c["root"] == crypto.EMPTY_ROOT, "proofs " + c["id"]) + elif c["kind"] == "inclusion": + ok("vectors", crypto.inclusion_path(items, c["index"]) == c["path"] + and crypto.leaf_hash(items[c["index"]]) == c["leaf"] + and crypto.verify_inclusion(c["root"], c["leaf"], c["index"], c["size"], c["path"]), "proofs " + c["id"]) + other = (c["index"] + 1) % c["size"] + ok("vectors", c["size"] == 1 or not crypto.verify_inclusion(c["root"], c["leaf"], other, c["size"], c["path"]), + "proofs %s: the path proves another index" % c["id"]) + else: + ok("vectors", crypto.consistency_proof(items, c["from"]) == c["path"] + and crypto.history_root(items[:c["from"]]) == c["from_root"] + and crypto.verify_consistency(c["from_root"], c["to_root"], c["from"], c["to"], c["path"]), "proofs " + c["id"]) + ok("vectors", c["from"] == 0 or not crypto.verify_consistency(c["from_root"], "0" * 64, c["from"], c["to"], c["path"]), + "proofs %s: a wrong root passes" % c["id"]) # every tree extends the empty one, so m = 0 proves nothing + for c in V("vrf.jsonl"): + pub, alpha, proof = bytes.fromhex(c["public"]), bytes.fromhex(c["alpha"]), bytes.fromhex(c["proof"]) + if "secret" in c: + sk = bytes.fromhex(c["secret"]) + ok("vectors", vrf.public_key(sk) == pub == crypto.ed25519_public(sk) and vrf.prove(sk, alpha) == proof, + "vrf %s: prove" % c["id"]) + beta = vrf.verify(pub, alpha, proof) + ok("vectors", (beta.hex() if beta else None) == c["beta"], "vrf %s: verify" % c["id"]) + if c["beta"]: + ok("vectors", vrf.proof_to_hash(proof).hex() == c["beta"], "vrf %s: proof_to_hash" % c["id"]) + ok("vectors", any(c["id"] == "rfc9381-example-16" and c["proof"].startswith("8657106690b5526245a92b003bb079cc") + and c["beta"].startswith("90cf1df3b703cce59e2a35b925d41116") for c in V("vrf.jsonl")), + "the RFC 9381 example is not reproduced") + from tools.build import secret + for c in V("checkpoint.jsonl"): + ok("vectors", LG.root(c["entries"]) == c["root"], "checkpoint %s: root" % c["id"]) + signers = [(n, secret(n)) for n in c["signers"]] + ok("vectors", LG.sign_checkpoint(c["origin"], c["entries"], signers) == c["note"], "checkpoint %s: note" % c["id"]) + note, verified = LG.verify_checkpoint(c["note"], ROSTER_BY_ID_CHECK) + ok("vectors", note and note["size"] == len(c["entries"]) and note["root"] == c["root"] and verified == c["signers"], + "checkpoint %s: verify" % c["id"]) + bad = c["note"].replace(c["origin"], c["origin"] + "x", 1) + ok("vectors", LG.verify_checkpoint(bad, ROSTER_BY_ID_CHECK)[1] == [], "checkpoint %s: a changed note verifies" % c["id"]) + ok("vectors", sorted(LG.tiles(c["entries"])) == c["tiles"], "checkpoint %s: tile paths" % c["id"]) + + +def check_determinism(): + with tempfile.TemporaryDirectory() as tmp: + subprocess.run([sys.executable, p("tools", "build.py"), "--out", tmp], check=True) + for sub in ("fixtures", "vectors"): + # Files the repository ignores, such as macOS's .DS_Store, are not part of the build. + cmp = filecmp.dircmp(p(sub), os.path.join(tmp, sub), ignore=filecmp.DEFAULT_IGNORES + [".DS_Store"]) + diffs = [] + + def walk(c, prefix): + for f in c.left_only + c.right_only + c.diff_files + c.funny_files: + if f != "README.md" and not (prefix == "fixtures/" and f == "src"): + diffs.append(prefix + f) + for f in c.common_files: + if not filecmp.cmp(os.path.join(c.left, f), os.path.join(c.right, f), shallow=False): + diffs.append(prefix + f) + for name, s in c.subdirs.items(): + walk(s, prefix + name + "/") + + walk(cmp, sub + "/") + ok("determinism", not diffs, "rebuild differs: %s" % sorted(set(diffs))[:5]) + + +RESERVED = set(""" +ABORT ACTION ADD AFTER ALL ALTER ALWAYS ANALYZE AND ANY ARRAY AS ASC ASENSITIVE AT ATTACH AUTHORIZATION AUTOINCREMENT +BEFORE BEGIN BETWEEN BIGINT BINARY BLOB BOTH BY CALL CASCADE CASE CAST CHANGE CHAR CHARACTER CHECK COLLATE COLUMN +COMMIT CONDITION CONFLICT CONSTRAINT CONTINUE CONVERT CREATE CROSS CUBE CUME_DIST CURRENT CURRENT_DATE CURRENT_ROLE +CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR DATABASE DATABASES DEC DECIMAL DECLARE DEFAULT DEFERRABLE DEFERRED +DELAYED DELETE DENSE_RANK DESC DESCRIBE DETACH DETERMINISTIC DISTINCT DISTINCTROW DIV DO DOUBLE DROP DUAL EACH ELSE +ELSEIF EMPTY ENCLOSED END ESCAPE ESCAPED EXCEPT EXCLUDE EXCLUSIVE EXISTS EXIT EXPLAIN FAIL FALSE FETCH FILTER FIRST +FIRST_VALUE FLOAT FOLLOWING FOR FORCE FOREIGN FREEZE FROM FULL FULLTEXT FUNCTION GENERATED GET GLOB GRANT GROUP +GROUPING GROUPS HAVING HIGH_PRIORITY IF IGNORE ILIKE IMMEDIATE IN INDEX INDEXED INFILE INITIALLY INNER INOUT +INSENSITIVE INSERT INSTEAD INT INTEGER INTERSECT INTERVAL INTO IS ISNULL ITERATE JOIN JSON_TABLE KEY KEYS KILL LAG LAST +LAST_VALUE LATERAL LEAD LEADING LEAVE LEFT LIKE LIMIT LINEAR LINES LOAD LOCALTIME LOCALTIMESTAMP LOCK LONG LOOP +LOW_PRIORITY MATCH MATERIALIZED MAXVALUE MOD MODIFIES NATURAL NO NOT NOTHING NOTNULL NTH_VALUE NTILE NULL NULLS NUMERIC +OF OFFSET ON ONLY OPTIMIZE OPTION OPTIONALLY OR ORDER OTHERS OUT OUTER OVER OVERLAPS PARTITION PERCENT_RANK PLACING PLAN +PRAGMA PRECEDING PRECISION PRIMARY PROCEDURE PURGE QUERY RAISE RANGE RANK READ READS REAL RECURSIVE REFERENCES REGEXP +REINDEX RELEASE RENAME REPEAT REPLACE REQUIRE RESIGNAL RESTRICT RETURN RETURNING REVOKE RIGHT RLIKE ROLLBACK ROW ROWS +ROW_NUMBER SAVEPOINT SCHEMA SCHEMAS SELECT SENSITIVE SEPARATOR SESSION_USER SET SHOW SIGNAL SIMILAR SMALLINT SOME +SPATIAL SPECIFIC SQL SQLEXCEPTION SQLSTATE SQLWARNING SSL STARTING STORED STRAIGHT_JOIN SYMMETRIC SYSTEM SYSTEM_USER +TABLE TABLESAMPLE TEMP TEMPORARY TERMINATED THEN TIES TINYINT TO TRAILING TRANSACTION TRIGGER TRUE UNBOUNDED UNDO +UNION UNIQUE UNLOCK UNSIGNED UPDATE USAGE USE USER USING UTC_DATE UTC_TIME UTC_TIMESTAMP VACUUM VALUES VARBINARY +VARCHAR VARCHARACTER VARIADIC VARYING VERBOSE VIEW VIRTUAL WHEN WHERE WHILE WINDOW WITH WITHOUT WRITE XOR ZEROFILL +""".split()) + + +def check_sql(): + ddl = "" + for f in ("doc.sql", "store.sql", "bundle-sqlite.sql"): + ddl += open(p("sql", f), encoding="utf-8").read() + "\n" + for m in re.finditer(r"CREATE (?:VIRTUAL )?TABLE (\w+)\s*(?:USING fts5)?\(\n(.*?)\n\);", ddl, re.S): + names = [m.group(1)] + [ln.strip().split()[0] for ln in m.group(2).splitlines() + if ln.strip() and not ln.strip().startswith(("PRIMARY", "--", "'", "tokenize"))] + for n in names: + ok("sql", n.upper() not in RESERVED, "%s uses the reserved word %s" % (m.group(1), n)) + for m in re.finditer(r"CREATE INDEX (\w+) ON", ddl): + ok("sql", m.group(1).upper() not in RESERVED, "index name " + m.group(1)) + ok("sql", "FOREIGN KEY" not in ddl and "REFERENCES" not in ddl, "no foreign keys") + with tempfile.TemporaryDirectory() as tmp: + conn = sqlite3.connect(os.path.join(tmp, "bundle.db")) + db.create_bundle(conn, p("sql")) + assets = os.path.join(tmp, "assets") + pk = p("fixtures", "packages") + for d in sorted(os.listdir(pk)): + if not os.path.isdir(os.path.join(pk, d)): + continue + key = db.load(conn, os.path.join(pk, d), assets) if not d.endswith("checks-only") else None + if key is None: + # A second load of the same key goes to its own database. + c2 = sqlite3.connect(":memory:") + db.create_bundle(c2, p("sql")) + key = db.load(c2, os.path.join(pk, d), assets) + ok("sql", db.index(c2, key) == 0, "%s: withheld text was indexed" % d) + h = db.unload(c2, key, os.path.join(tmp, "out", d), assets) + else: + db.index(conn, key) + h = db.unload(conn, key, os.path.join(tmp, "out", d), assets) + same = _same_tree(os.path.join(pk, d), os.path.join(tmp, "out", d)) + ok("sql", same, "%s: unload is not byte-identical" % d) + with open(os.path.join(pk, d, "MANIFEST.json"), "rb") as f: + ok("sql", sha256_hex(f.read()) == h, "%s: package hash differs after unload" % d) + check_bundle(conn) + # Store records satisfy the store tables' CHECK constraints. + R = p("fixtures", "records") + lay = js(os.path.join(R, "layer", "layer.json")) + conn.execute("INSERT INTO layers VALUES (?,?,?,?,?,?,?,?)", + (lay["id"], lay.get("project"), lay["label"], lay["purpose"], lay.get("audience"), + lay["visibility"], jcs.dumps(lay["license"]), jcs.dumps(lay))) + for a in jl(os.path.join(R, "layer", "annotations.jsonl")): + conn.execute("INSERT INTO annotations VALUES (?,?,?,?,?,?,?,?)", + (a["id"], a["layer"], a["body"]["kind"], a["status"], a.get("receipt"), a["creator"]["id"], + a["created"], jcs.dumps(a))) + prj = js(os.path.join(R, "project.json")) + conn.execute("INSERT INTO projects VALUES (?,?,?,?,?,?,?)", + (prj["key"], prj.get("store"), prj["name"], prj.get("description"), jcs.dumps(prj.get("claims")), + jcs.dumps(prj.get("settings")), jcs.dumps(prj))) + for m in prj["members"]: + conn.execute("INSERT INTO members VALUES (?,?,?,?)", (prj["key"], m["person"], m["role"], jcs.dumps(m["keys"]))) + for k in js(os.path.join(R, "roster.json"))["keys"]: + conn.execute("INSERT INTO roster_keys VALUES (?,?,?,?,?,?,?,?,?,?)", + ("smithlab", k["id"], jcs.dumps(k["person"]), k["role"], k["scheme"], k["public"], k.get("did"), + jcs.dumps(k["purposes"]), k["registered_at"], k["revoked_at"])) + dg = js(os.path.join(R, "digitization.json")) + conn.execute("INSERT INTO digitizations VALUES (?,?,?,?,?,?,?,?)", + (dg["id"], dg["doc"], dg["target"]["anchor"]["node"], dg["target"].get("page_fp"), + dg["method"]["kind"], dg["confirmation"], dg.get("dataset"), jcs.dumps(dg))) + conn.commit() + ok("sql", True, "store records loaded") + + +def check_bundle(conn): + """The bundle's search index and metadata (spec/13 §13.5), built by the oracle's db.index and db.write_meta.""" + units = dict(conn.execute("SELECT unit, COUNT(*) FROM search GROUP BY unit").fetchall()) + for u in ("sentence", "cell", "figure-text"): + ok("sql", units.get(u, 0) > 0, "the search index has no %s rows" % u) + # No fixture has an equation outside a sentence, so a small document shows both kinds of equation. + nodes = [{"id": "d", "kind": "document", "parent": None, "ord": 0}, + {"id": "p", "kind": "paragraph", "parent": "d", "ord": 0}, + {"id": "s", "kind": "sentence", "parent": "p", "ord": 0, + "content": [{"kind": "text", "text": "Then "}, {"kind": "equation", "node": "e1"}]}, + {"id": "e1", "kind": "equation", "parent": "s", "ord": 0, "content": {"notation": "tex", "source": "a=b"}}, + {"id": "e2", "kind": "equation", "parent": "p", "ord": 1, "content": {"notation": "tex", "source": "c=d"}}] + got = [(n, u, c) for _, n, u, c, _ in db.search_rows("k", nodes)] + ok("sql", got == [("s", "sentence", "paragraph"), ("e2", "equation", "paragraph")], + "search rows for a sentence with an inline equation and a display equation: %s" % got) + ok("sql", set(units) <= {"sentence", "cell", "equation", "figure-text"}, "unknown search units %s" % sorted(units)) + expect = conn.execute( + "SELECT COUNT(*) FROM nodes n LEFT JOIN nodes q ON q.doc_key = n.doc_key AND q.id = n.parent " + "WHERE n.withheld IS NULL AND (n.kind IN ('sentence', 'cell') OR (n.kind = 'equation' AND q.kind <> 'sentence'))" + ).fetchone()[0] + ok("sql", sum(v for u, v in units.items() if u != "figure-text") == expect, + "the search index does not have one row per sentence, cell and equation outside a sentence") + inner = conn.execute( + "SELECT COUNT(*) FROM search s JOIN nodes n ON n.doc_key = s.doc_key AND n.id = s.node " + "JOIN nodes q ON q.doc_key = n.doc_key AND q.id = n.parent WHERE n.kind = 'equation' AND q.kind = 'sentence'" + ).fetchone()[0] + ok("sql", inner == 0, "an equation inside a sentence is indexed apart from its sentence") + bad = conn.execute( + "SELECT COUNT(*) FROM search s JOIN nodes n ON n.doc_key = s.doc_key AND n.id = s.node " + "LEFT JOIN nodes q ON q.doc_key = n.doc_key AND q.id = n.parent " + "WHERE s.unit <> 'figure-text' AND s.context IS NOT q.kind").fetchone()[0] + ok("sql", bad == 0, "a search row's context is not the kind of its node's parent") + hits = conn.execute("SELECT node FROM search WHERE search MATCH 'scanned'").fetchall() + ok("sql", len(hits) == 1, "full-text search finds the merged sentence") + hits = conn.execute("SELECT context FROM search WHERE unit = 'sentence' AND search MATCH 'Methods'").fetchall() + ok("sql", ("heading",) in hits, "a heading sentence is indexed with the context heading") + stores = [r[0] for r in conn.execute("SELECT DISTINCT store FROM documents WHERE store IS NOT NULL")] + db.write_meta(conn, stores[0] if len(stores) == 1 else "", "fixtures/packages") + meta = dict(conn.execute("SELECT name, value FROM bundle_meta").fetchall()) + ok("sql", {"format", "store", "built_from", "spec"} <= set(meta), "bundle_meta lacks a required name") + ok("sql", meta["format"] in FORMAT_SCHEMA, "bundle_meta names an unregistered format") + top = re.search(r"^## (\S+)", open(p("CHANGELOG.md"), encoding="utf-8").read(), re.M).group(1) + ok("sql", meta["spec"] == top, "bundle_meta spec %s is not the changelog's latest version %s" % (meta["spec"], top)) + + +def _same_tree(a, b): + fa = sorted(os.path.relpath(os.path.join(d, f), a) for d, _, fs in os.walk(a) for f in fs) + fb = sorted(os.path.relpath(os.path.join(d, f), b) for d, _, fs in os.walk(b) for f in fs) + return fa == fb and all(filecmp.cmp(os.path.join(a, f), os.path.join(b, f), shallow=False) for f in fa) + + +UK = re.compile(r"\b(\w*(?:colour|behaviour|favour|honour|labour|neighbour|centre|metre|litre|fibre|analys(?:e|ed|es|ing)|" + r"catalogue|licence|defence|offence|organis\w*|normalis\w*|recognis\w*|optimis\w*|minimis\w*|" + r"maximis\w*|standardis\w*|serialis\w*|canonicalis\w*|initialis\w*|summaris\w*|prioritis\w*|" + r"emphasis(?:e|ed|es|ing)|realis(?:e|ed|es|ing)|travell\w*|modell(?:ed|ing)|labell(?:ed|ing)|" + r"cancell(?:ed|ing)|signall(?:ed|ing)|programme|grey|whilst|amongst|towards|judgement|ageing|" + r"artefact)\w*)\b", re.I) +SENTENCE_BREAK = re.compile(r"[a-z0-9)\]*_`'\"]\. [A-Z]") +ABBREV = re.compile(r"\b(e\.g|i\.e|et al|cf|vs|Fig|Eq|Ref|ca|No|St|Dr|Mr|Ms|al)\. [A-Z]") + + +def check_prose(): + mds = [] + for d, dirs, fs in os.walk(ROOT): + dirs[:] = [x for x in dirs if x not in SKIP_DIRS] + mds += [os.path.join(d, f) for f in fs if f.endswith(".md")] + for path in sorted(mds): + rel = os.path.relpath(path, ROOT) + fence = False + for i, ln in enumerate(open(path, encoding="utf-8"), 1): + if ln.startswith("```"): + fence = not fence + continue + if fence: + continue + text = re.sub(r"`[^`]*`", "``", ln) + m = UK.search(text) + ok("prose", not m, "%s:%d: UK spelling %r" % (rel, i, m.group(0) if m else "")) + if not text.lstrip().startswith("|"): + body = ABBREV.sub("", re.sub(r"^\s*(#+\s+)?([0-9]+(\.[0-9]+)*\.?\s+)?(\*\*[^*]+\*\*\s*)?", "", text)) + ok("prose", not SENTENCE_BREAK.search(body), "%s:%d: one sentence per line" % (rel, i)) + + +def _slug(heading): + """A heading's anchor, as GitHub, Gitea and Forgejo compute it.""" + t = re.sub(r"[`*]", "", heading.strip().lower()) + t = "".join(ch for ch in t if ch.isalnum() or ch in " -_") + return t.replace(" ", "-") + + +def check_links(): + anchors = {} + mds = [] + for d, dirs, fs in os.walk(ROOT): + dirs[:] = [x for x in dirs if x not in SKIP_DIRS] + mds += [os.path.join(d, f) for f in fs if f.endswith(".md")] + for path in mds: + seen, fence = {}, False + for ln in open(path, encoding="utf-8"): + if ln.startswith("```"): + fence = not fence + elif not fence and ln.startswith("#"): + base = _slug(ln.lstrip("#")) + n = seen.get(base, 0) + seen[base] = n + 1 + anchors.setdefault(path, set()).add(base if n == 0 else "%s-%d" % (base, n)) + for path in sorted(mds): + fence = False + for i, ln in enumerate(open(path, encoding="utf-8"), 1): + if ln.startswith("```"): + fence = not fence + continue + if fence: + continue + for target in re.findall(r"\]\(([^)\s]+)\)", re.sub(r"`[^`]*`", "", ln)): + if re.match(r"^[a-z]+:", target): + continue + file, _, frag = target.partition("#") + dest = os.path.normpath(os.path.join(os.path.dirname(path), file)) if file else path + rel = os.path.relpath(path, ROOT) + ok("links", os.path.exists(dest), "%s:%d: no such file %s" % (rel, i, file)) + if frag and dest.endswith(".md") and os.path.exists(dest): + ok("links", frag in anchors.get(dest, ()), "%s:%d: no heading #%s in %s" % (rel, i, frag, file or "this file")) + + +WORD_NUMBERS = {w: i for i, w in enumerate( + "zero one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen sixteen " + "seventeen eighteen nineteen twenty".split())} +WORD_NUMBERS.update({"twenty-" + w: 20 + i for i, w in enumerate("one two three four five six seven eight nine".split(), 1)}) +# A count: digits or a number word, not the tail of a section number such as "22.3". +NUM = r"(?.assets/. +""" + +import argparse +import os +import sqlite3 +import sys +import tempfile + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) + +from tools.oracle import db, package # noqa: E402 +from tools.oracle.schema import Store # noqa: E402 + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--sqlite", required=True, help="the database file; created with the tables if new") + ap.add_argument("--bundle", action="store_true", help="build a new scienting-bundle/1 file with search") + ap.add_argument("--built-from", help="with --bundle: the store revision the packages come from") + ap.add_argument("--store", help="with --bundle: the store's name (default: the packages' common store)") + ap.add_argument("--max-size", metavar="BYTES", type=int, default=package.ZIP_LIMIT, + help="refuse a zipped package that expands past this many bytes (default %d)" % package.ZIP_LIMIT) + ap.add_argument("packages", nargs="+", help="package folders or zipped packages") + a = ap.parse_args() + if a.bundle and (os.path.exists(a.sqlite) or not a.built_from): + ap.error("--bundle needs --built-from and a database file that does not exist yet") + store, bad = Store(os.path.join(ROOT, "schemas")), False + with tempfile.TemporaryDirectory() as tmp: + folders = [] + for i, p in enumerate(a.packages): + if package.is_zip(p): + try: + folders.append(package.read_zip(p, os.path.join(tmp, str(i)), a.max_size)) + except package.Refusal as e: + print("%s: %s: %s" % (p, e.rule, e.message), file=sys.stderr) + bad = True + continue + else: + folders.append(p) + for e in package.validate(folders[-1], store).errors: + print("%s: %s: %s: %s" % (p, e["where"], e["rule"], e["message"]), file=sys.stderr) + bad = True + if bad: + sys.exit("Nothing was loaded.") + new = not os.path.exists(a.sqlite) + conn = sqlite3.connect(a.sqlite) + if a.bundle: + db.create_bundle(conn, os.path.join(ROOT, "sql")) + elif new: + db.create(conn, os.path.join(ROOT, "sql")) + for p in folders: + key = db.load(conn, p, a.sqlite + ".assets") + n = conn.execute("SELECT COUNT(*) FROM nodes WHERE doc_key = ?", (key,)).fetchone()[0] + if a.bundle: + print("%s: %d nodes, %d search rows" % (key, n, db.index(conn, key))) + else: + print("%s: %d nodes" % (key, n)) + if a.bundle: + stores = {r[0] for r in conn.execute("SELECT DISTINCT store FROM documents WHERE store IS NOT NULL")} + store_name = a.store or (stores.pop() if len(stores) == 1 else None) + if store_name is None: + sys.exit("The packages name no single store; give one with --store.") + db.write_meta(conn, store_name, a.built_from) + conn.commit() + + +if __name__ == "__main__": + main() diff --git a/tools/oracle/__init__.py b/tools/oracle/__init__.py new file mode 100644 index 0000000..9288a99 --- /dev/null +++ b/tools/oracle/__init__.py @@ -0,0 +1,13 @@ +"""The reference computations behind every expected value in this repository; see tools/README.md.""" + +import importlib.util +import sys + +if sys.version_info < (3, 14): + sys.exit("The paperspec tools need Python 3.14 or later; this is Python %d.%d." % sys.version_info[:2]) + +_missing = [m for m in ("cryptography", "jsonschema") if importlib.util.find_spec(m) is None] +if _missing: + sys.exit("The paperspec tools need %s %s; run them through pixi, or install %s with pip." + % ("the package" if len(_missing) == 1 else "the packages", " and ".join(_missing), + "it" if len(_missing) == 1 else "them")) diff --git a/tools/oracle/align.py b/tools/oracle/align.py new file mode 100644 index 0000000..9fae18d --- /dev/null +++ b/tools/oracle/align.py @@ -0,0 +1,127 @@ +"""Whole-document sentence alignment, in the manner of a patience diff. + +Non-normative; see spec/05-hashes.md, "Alignment". +Each reading is a list of items: {"id", "strict", "lossy", "complex", "container", "context"}. +""" + +import unicodedata + +from .hashing import prose_length + +MIN_PROSE = 20 + + +def _numeric_skeleton(strict): + keep = [] + i, n = 0, len(strict) + while i < n: + ch = strict[i] + if ch in "_^" and i + 1 < n and strict[i + 1] == "{": + keep.append(strict[i:i + 2]) + i += 2 + continue + if unicodedata.category(ch) == "Nd" or ch in "\u2212.,+}": + keep.append(ch) + i += 1 + return "".join(keep) + + +def classify(a, b): + """`None` when two paired sentences agree; else the suspect kind.""" + if a["strict"] == b["strict"] and a["complex"] == b["complex"]: + return None + if _numeric_skeleton(a["strict"]) != _numeric_skeleton(b["strict"]): + return "number" + if a["complex"] != b["complex"]: + return "equation" + return "word" + + +def _lis(pairs): + """Longest subsequence increasing in B among pairs sorted by A; lexicographically smallest A on ties.""" + n = len(pairs) + if not n: + return [] + best = [1] * n + nxt = [-1] * n + for i in range(n - 1, -1, -1): + for j in range(i + 1, n): + if pairs[j][1] > pairs[i][1] and best[j] + 1 > best[i]: + best[i] = best[j] + 1 + nxt[i] = j + top = max(best) + i = next(k for k in range(n) if best[k] == top) + out = [] + while i != -1: + out.append(pairs[i]) + i = nxt[i] + return out + + +def _lcs(A, B, a0, a1, b0, b1): + """LCS over align keys of A[a0:a1] and B[b0:b1], as index pairs, with the spec's tie rule.""" + n, m = a1 - a0, b1 - b0 + L = [[0] * (m + 1) for _ in range(n + 1)] + for i in range(n - 1, -1, -1): + for j in range(m - 1, -1, -1): + if A[a0 + i]["lossy"] == B[b0 + j]["lossy"]: + L[i][j] = L[i + 1][j + 1] + 1 + else: + L[i][j] = max(L[i + 1][j], L[i][j + 1]) + out = [] + i = j = 0 + while i < n and j < m: + if A[a0 + i]["lossy"] == B[b0 + j]["lossy"]: + out.append((a0 + i, b0 + j)) + i += 1 + j += 1 + elif L[i + 1][j] >= L[i][j + 1]: + i += 1 + else: + j += 1 + return out + + +def align(A, B): + """Align reading B to reading A. Returns {"pairs": [...], "suspects": [...]}.""" + count_a, count_b = {}, {} + for x in A: + count_a[x["lossy"]] = count_a.get(x["lossy"], 0) + 1 + for x in B: + count_b[x["lossy"]] = count_b.get(x["lossy"], 0) + 1 + pos_b = {x["lossy"]: j for j, x in enumerate(B)} + cands = [ + (i, pos_b[x["lossy"]]) for i, x in enumerate(A) + if count_a[x["lossy"]] == 1 and count_b.get(x["lossy"]) == 1 and prose_length(x["lossy"]) >= MIN_PROSE + ] + anchors = _lis(cands) + pairs = [] + prev_a, prev_b = 0, 0 + for ai, bi in anchors + [(len(A), len(B))]: + pairs += _lcs(A, B, prev_a, ai, prev_b, bi) + if ai < len(A): + pairs.append((ai, bi)) + prev_a, prev_b = ai + 1, bi + 1 + out_pairs, suspects = [], [] + for ai, bi in pairs: + kind = classify(A[ai], B[bi]) + out_pairs.append({"a": A[ai]["id"], "b": B[bi]["id"], "agree": kind is None}) + if kind: + suspects.append({"kind": kind, "a": A[ai]["id"], "b": B[bi]["id"]}) + if A[ai]["context"] != B[bi]["context"]: + suspects.append({"kind": "structure", "a": A[ai]["id"], "b": B[bi]["id"]}) + for (a1, b1), (a2, b2) in zip(pairs, pairs[1:]): + if a2 == a1 + 1 and b2 == b1 + 1: + same_a = A[a1]["container"] == A[a2]["container"] + same_b = B[b1]["container"] == B[b2]["container"] + if same_a != same_b: + suspects.append({"kind": "join", "a": A[a2]["id"], "b": B[b2]["id"]}) + paired_a = {a for a, _ in pairs} + paired_b = {b for _, b in pairs} + for i, x in enumerate(A): + if i not in paired_a: + suspects.append({"kind": "dropped", "a": x["id"], "b": None}) + for j, x in enumerate(B): + if j not in paired_b: + suspects.append({"kind": "added", "a": None, "b": x["id"]}) + return {"pairs": out_pairs, "suspects": suspects} diff --git a/tools/oracle/anchors.py b/tools/oracle/anchors.py new file mode 100644 index 0000000..3f8f930 --- /dev/null +++ b/tools/oracle/anchors.py @@ -0,0 +1,271 @@ +"""Addresses, anchors and their resolution. + +Non-normative; see spec/06-anchors.md. +""" + +import re +from fractions import Fraction + +from . import hashing, jcs +from .model import CONTAINERS_WITH_CONTENT, LEAVES, RUN_KINDS, Doc, anchor_class +from .text import norm + +ADDRESS_RE = re.compile( + r"^(?:(?P[a-z0-9][a-z0-9._-]*)/)?" + r"(?P[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}(?:\.[a-z0-9][a-z0-9-]*)*)" + r"#(?P[a-z0-9][a-z0-9._-]*)" + r"(?:@(?P[0-9a-f]{6}))?" + r"(?:\[(?P0|[1-9][0-9]*):(?P0|[1-9][0-9]*)\])?$" +) +QUOTE_CAP = 200 +CONTEXT_CAP = 32 + + +def parse_address(s): + m = ADDRESS_RE.match(s) + if not m: + return None + d = m.groupdict() + out = {"store": d["store"], "key": d["key"], "node": d["node"], "pin": d["pin"], "span": None} + if d["start"] is not None: + a, b = int(d["start"]), int(d["end"]) + if a > b: + return None + out["span"] = [a, b] + return out + + +def format_address(store, key, node, pin=None, span=None): + s = (store + "/" if store else "") + key + "#" + node + if pin: + s += "@" + pin[:6] + if span is not None: + s += "[%d:%d]" % tuple(span) + return s + + +def cell_context(doc, cell_id): + """Table label and the text of the row and column header paths of a cell.""" + cell = doc.nodes[cell_id] + row = doc.nodes[cell["parent"]] + table_id = doc.ancestors(cell_id)[-3] + table = doc.nodes[table_id] + cols = {c["id"]: c for c in table.get("attrs", {}).get("columns", [])} + col = cols.get(cell.get("attrs", {}).get("column"), {"header_path": []}) + + def txt(nid): + return norm(hashing.plain(doc, nid)) + + return { + "table_label": table.get("label"), + "row_path": [txt(x) for x in row.get("attrs", {}).get("header_path", [])], + "column_path": [txt(x) for x in col["header_path"]], + } + + +def make_anchor(doc, values, node, rev, span=None, store=None): + n = doc.nodes[node] + a = {"format": "scienting-anchor/1"} + if store: + a["store"] = store + a.update({"doc": doc.key, "node": node, "kind": n["kind"], "own_hash": values[node]["own_hash"], + "node_hash": values[node]["node_hash"]}) + if n["kind"] in LEAVES: + text = hashing.plain(doc, node) + if span is not None: + a["span"] = list(span) + exact = text[span[0]:span[1]] + pre, suf = text[max(0, span[0] - CONTEXT_CAP):span[0]], text[span[1]:span[1] + CONTEXT_CAP] + else: + exact, pre, suf = text, "", "" + q = {} + if len(exact) <= QUOTE_CAP: + q["exact"] = exact + else: + q["exact_hash"] = hashing.PLAIN(norm(exact)) + q["length"] = len(exact) + if pre: + q["prefix"] = pre + if suf: + q["suffix"] = suf + a["quote"] = q + if n["kind"] == "cell": + a["context"] = cell_context(doc, node) + elif anchor_class(n["kind"]) == "container-without-content": + a["leaves"] = [[x, values[x]["match_hash"]] for x in doc.leaves(node)] + a["rev"] = rev + return a + + +def _quote_hits(text, q): + """Start positions where the quote matches `text`.""" + if "exact" in q: + ex = q["exact"] + hits = [i for i in range(len(text) - len(ex) + 1) if text.startswith(ex, i)] + length = len(ex) + else: + length = q["length"] + hits = [i for i in range(len(text) - length + 1) + if hashing.PLAIN(norm(text[i:i + length])) == q["exact_hash"]] + if len(hits) > 1 and ("prefix" in q or "suffix" in q): + pre, suf = q.get("prefix", ""), q.get("suffix", "") + hits = [i for i in hits if text[:i].endswith(pre) and text[i + length:].startswith(suf)] + return hits, length + + +def _drift(state, doc, anchor, nid, threshold): + leaves = set(doc.leaves(nid)) + touched = set() + count = 0 + for c in state.corrections: + if c["index"] <= anchor["rev"]: + continue + inside = nid in (c.get("path") or []) or nid in (c.get("to_path") or []) + t = set(c.get("touched", [])) + if inside and t: + count += 1 + touched |= t + denom = len(leaves | touched) + share = Fraction(len(touched), denom) if denom else Fraction(0) + level = "none" if count == 0 else ("high" if share > threshold else "low") + return {"count": count, "share": _decimal(share), "level": level} + + +def _decimal(fr): + """A share as a decimal string with four places, rounded half up.""" + v = (fr * 10000 + Fraction(1, 2)).__floor__() + return "%d.%04d" % (v // 10000, v % 10000) + + +def _follow(state, nid): + """Follow aliases and retirements. Returns (live id or None, chain, via).""" + live = {n["id"] for n in state.nodes} + table = {e["id"]: e for e in state.ids} + chain, via = [], None + seen = set() + while nid not in live: + if nid in seen or nid not in table: + return None, chain, via + seen.add(nid) + e = table[nid] + if e["kind"] == "retired" or via is None: + via = e["kind"] + chain.append(nid) + nid = e.get("successor") + if nid is None: + return None, chain, via + return nid, chain, via + + +def _relabel_reverted_own(state, doc, nid, anchor): + """The own_hash the node had before the relabel corrections since the anchor, or None when anything else + changed the node or what is below it since (spec/06 §6.5: its *only* changes were relabel corrections).""" + since = [c for c in state.corrections if c["index"] > anchor["rev"] and ( + c.get("target") == nid or nid in (c.get("path") or []) or nid in (c.get("to_path") or []))] + rel = [c for c in since if c.get("target") == nid] + if not rel or any(c["op"] != "relabel" or c.get("target") != nid for c in since): + return None + n = dict(doc.nodes[nid], label=rel[0]["from"]) + tmp = Doc(doc.key, [n if x["id"] == nid else x for x in [doc.nodes[i] for i in doc.order]]) + return hashing.PLAIN(jcs.dumpb(hashing.own_content(tmp, nid, lambda h: hashing.PLAIN( + norm(hashing.plain(tmp, h)))))) + + +def resolve(state, anchor, threshold=Fraction(1, 4)): + doc = state.doc() + values = hashing.compute(doc) + kind = anchor.get("kind") + original = anchor["node"] + nid, chain, via = _follow(state, original) + live_kind = doc.nodes[nid]["kind"] if nid else None + table = {e["id"]: e for e in state.ids} + # The kind falls back to the retired entry the chain reached (spec/06 §6.5); an alias carries no kind. + retired_kind = next((table[x].get("node_kind") for x in reversed(chain) + if table.get(x, {}).get("kind") == "retired"), None) + kind = kind or live_kind or retired_kind + cls = anchor_class(kind) if kind else "leaf" + is_span = "span" in anchor + result = {"node": nid} + retired = via == "retired" + + def out(res, **kw): + r = {"result": res, "node": kw.pop("node", nid)} + if r["node"]: + r["node_hash"] = values[r["node"]]["node_hash"] + if chain: + r["via"] = chain + r.update(kw) + return r + + # The cell rule: a retired cell is found again by its table label and header paths. + def cell_rule(): + ctx = anchor.get("context") + q = anchor.get("quote") + if kind != "cell" or not ctx or not q: + return None + found = [] + for x in doc.order: + if doc.nodes[x]["kind"] != "cell": + continue + if cell_context(doc, x) != ctx: + continue + hits, ln = _quote_hits(hashing.plain(doc, x), q) + if len(hits) == 1: + found.append((x, hits[0], ln)) + if len(found) == 1: + x, i, ln = found[0] + kw = {"node": x} + if is_span: + kw["span"] = [i, i + ln] + return out("reattached", **kw) + return None + + if nid is None: + return cell_rule() or out("stale", node=None, reason="retired without a successor" if chain + else "unknown node") + + if cls == "leaf": + text = hashing.plain(doc, nid) + if not retired and values[nid]["node_hash"] == anchor["node_hash"]: + return out("current", **({"span": anchor["span"]} if is_span else {})) + q = anchor.get("quote") + if retired: + if q: + hits, ln = _quote_hits(text, q) + if (is_span and len(hits) == 1) or (not is_span and hits): + return out("moved", **({"span": [hits[0], hits[0] + ln]} if is_span else {})) + return cell_rule() or out("stale", reason="the successor does not contain the quote") + if q: + hits, ln = _quote_hits(text, q) + if len(hits) == 1: + return out("reattached", **({"span": [hits[0], hits[0] + ln]} if is_span else {})) + own = _relabel_reverted_own(state, doc, nid, anchor) + if own is not None and own == anchor["own_hash"]: + return out("reattached", reason="relabel") + return out("stale", reason="the node changed and the quote does not occur exactly once") + + if cls == "container-with-content": + if values[nid]["own_hash"] == anchor["own_hash"]: + if retired: + return out("moved") + if values[nid]["node_hash"] == anchor["node_hash"]: + return out("current") + return out("changed-below", drift=_drift(state, doc, anchor, nid, threshold)) + if not retired: + own = _relabel_reverted_own(state, doc, nid, anchor) + if own is not None and own == anchor["own_hash"]: + return out("reattached", reason="relabel") + return out("stale", reason="the node's own content changed") + + # Container without content. + if retired: + orig = anchor.get("leaves", []) + now = set(doc.leaves(nid)) + now_match = {values[x]["match_hash"] for x in now} + held = sum(1 for i, m in orig if i in now or m in now_match) + if 2 * held >= len(orig): + return out("moved") + return out("stale", reason="the successor holds fewer than half of the original leaves") + if values[nid]["node_hash"] == anchor["node_hash"]: + return out("current") + return out("changed-below", drift=_drift(state, doc, anchor, nid, threshold)) diff --git a/tools/oracle/corrections.py b/tools/oracle/corrections.py new file mode 100644 index 0000000..496abc5 --- /dev/null +++ b/tools/oracle/corrections.py @@ -0,0 +1,454 @@ +"""Corrections on an established document. + +Non-normative; see spec/11-corrections.md. +""" + +import copy + +from . import hashing +from .model import ATTRS, LABELLED, LEAVES, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS, Doc, CHILDREN +from .tree import Refusal, canonical_runs, mint_ids, validate_references, validate_structure, _flatten +from . import jcs +from .schema import validate as schema_validate +from .text import digits, record_id, sha256_hex + +MERGEABLE = RUN_KINDS | {"paragraph", "list", "item", "quote", "caption", "figure-note", "table-note"} +SETTABLE = { + "section": {"level"}, + "table": {"columns"}, + "cell": {"column", "rowspan", "colspan", "role"}, + "row": {"header_path"}, + "list": {"ordered"}, + "item": {"marker"}, + "theorem": {"variant"}, +} +EDITABLE_RUN_FIELD = {"text": "text", "link": "text", "cite": "text", "xref": "text", "fnmark": "text", "math": "source"} + + +class State: + """The current state of a document: nodes, the ID table and the corrections applied.""" + + def __init__(self, key, nodes, ids=None, corrections=None, indexes=None): + self.key = key + self.nodes = [dict(n) for n in nodes] + self.ids = [dict(e) for e in (ids or [])] + self.corrections = [dict(c) for c in (corrections or [])] + self._indexes = dict(indexes or {}) + + @property + def rev(self): + return len(self.corrections) + + def doc(self): + return Doc(self.key, self.nodes) + + def index(self, hasher=hashing.PLAIN): + """The node values under `hasher`, kept with the state so the next correction rehashes only what changes.""" + if hasher.key not in self._indexes: + self._indexes[hasher.key] = hashing.Index.build(self.doc(), hasher) + return self._indexes[hasher.key] + + def used_ids(self): + return {n["id"] for n in self.nodes} | {e["id"] for e in self.ids} + + +def _strings(node): + """Editable strings of a leaf: (path, value) pairs.""" + k = node["kind"] + if k in RUN_KINDS: + out = [] + for i, r in enumerate(node["content"]): + f = EDITABLE_RUN_FIELD.get(r["kind"]) + if f: + out.append(((i, f), r[f])) + return out + if k in MATH_KINDS: + return [(("source",), node["content"]["source"])] + if k in VERBATIM_KINDS: + return [(("text",), node["content"]["text"])] + return [] + + +def _occurrences(s, find): + return [i for i in range(len(s) - len(find) + 1) if s.startswith(find, i)] + + +def _set_string(node, path, value): + if len(path) == 2: + i, f = path + node["content"][i] = dict(node["content"][i], **{f: value}) + else: + node["content"] = dict(node["content"], **{path[0]: value}) + + +def _has_crop(c): + return any(e.get("kind") == "crop" for e in c.get("evidence", [])) + + +def _rebuild(nodes): + """Recompute document order and `ord` from parent links, keeping sibling order as listed.""" + by_id = {n["id"]: n for n in nodes} + kids = {} + root = None + for n in nodes: + if n.get("parent") is None: + root = n["id"] + else: + kids.setdefault(n["parent"], []).append(n["id"]) + for p in kids: + kids[p].sort(key=lambda c: by_id[c]["ord"]) + out = [] + stack = [root] + while stack: + x = stack.pop() + out.append(by_id[x]) + for i, c in enumerate(kids.get(x, [])): + by_id[c]["ord"] = i + stack.extend(reversed(kids.get(x, []))) + return out + + +def _subtree(doc, nid): + return doc.preorder(nid) + + +def _place(nodes, nid, parent, after): + """Give `nid` a fractional position under `parent` after sibling `after` (None: first).""" + by_id = {n["id"]: n for n in nodes} + sibs = sorted([n for n in nodes if n.get("parent") == parent and n["id"] != nid], key=lambda n: n["ord"]) + for i, s in enumerate(sibs): + s["ord"] = 2 * i + 2 + if after is None: + pos = 0 + else: + if after not in by_id or by_id[after].get("parent") != parent: + raise Refusal("correction.target", "`place_after` is not a child of the parent") + pos = by_id[after]["ord"] + 1 + by_id[nid]["parent"] = parent + by_id[nid]["ord"] = pos + + +def gate(state, proposal, store, hasher=hashing.PLAIN): + """Run the gate (spec/11 §11.4) on a proposed correction, leaving `state` unchanged. + + Returns the state the correction would produce and the record it would append to corrections.jsonl, as apply + does. Raises Refusal naming the rule broken. `store` is the schema store that holds correction.schema.json. + """ + errors = schema_validate(store, "correction.schema.json", proposal) + if errors: + raise Refusal("correction.schema", "; ".join(errors[:3])) + return apply(state, proposal, hasher, store) + + +def _is_index(v): + return isinstance(v, int) and not isinstance(v, bool) and v >= 0 + + +def id_order(e): + """The order of ids.jsonl (spec/04 §4.4): aliases first by ID, then retirements by revision and ID.""" + return (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"]) + + +def _node_rule(error): + """The rule a node-schema error breaks, from the property the error names.""" + path = error.split(":", 1)[0] + for prefix, rule in (("/attrs", "doc.node.attrs"), ("/label", "doc.node.label"), ("/content", "doc.node.content")): + if path.startswith(prefix): + return rule + return "doc.node.schema" + + +def apply(state, c, hasher=hashing.PLAIN, store=None): + """Apply correction `c` to `state` and return the recorded correction. Raises Refusal. + + With a schema `store`, every node the correction changed or created is also validated against + node.schema.json, which is how the gate enforces the attribute shapes of spec/03 §3.4 (rule 11 of §11.4). + """ + c = copy.deepcopy(c) + doc = state.doc() + before_ix = state.index(hasher) + before_vals = before_ix.values + nodes = [copy.deepcopy(n) for n in state.nodes] + by_id = {n["id"]: n for n in nodes} + op = c["op"] + target = c.get("target") + if op != "insert" or "target" in c: + if target not in by_id: + raise Refusal("correction.target", "no live node %s" % target) + if target == doc.root and op in ("delete", "move", "merge", "reclassify"): + raise Refusal("correction.target", "the root cannot be " + op + "d") + record = {"format": "scienting-correction/1", "index": state.rev + 1} + retired, created = [], [] + path = doc.ancestors(target) if target in by_id else None + touched = [] + + if op == "replace": + node = by_id[target] + find, repl = c.get("find", ""), c["replace"] + if not find: + raise Refusal("correction.find", "find must not be empty") + hits = [] + for pth, s in _strings(node): + if "run" in c and (len(pth) != 2 or pth[0] != c["run"]): + continue + hits += [(pth, s, i) for i in _occurrences(s, find)] + if len(hits) != 1: + raise Refusal("correction.find", "find occurs %d times, not exactly once" % len(hits), target) + pth, s, i = hits[0] + if isinstance(repl, str): + _set_string(node, pth, s[:i] + repl + s[i + len(find):]) + else: + if len(pth) != 2 or node["content"][pth[0]]["kind"] != "text": + raise Refusal("correction.replace", "runs may replace text only inside a text run", target) + if any(r["kind"] == "equation" for r in repl): + raise Refusal("correction.replace", "equation runs are added by insert", target) + run = node["content"][pth[0]] + style = {"style": run["style"]} if "style" in run else {} + mid = [dict({"kind": "text", "text": s[:i]}, **style)] + list(repl) + [ + dict({"kind": "text", "text": s[i + len(find):]}, **style) + ] + node["content"] = node["content"][: pth[0]] + mid + node["content"][pth[0] + 1:] + if node["kind"] in RUN_KINDS: + node["content"] = canonical_runs(node["content"]) + touched = [target] + + elif op == "insert" and "node" in c: + if not _has_crop(c): + raise Refusal("correction.evidence", "an insert needs crop evidence") + parent = c["parent"] + if parent not in by_id: + raise Refusal("correction.target", "no live parent %s" % parent) + flat = [] + _flatten(copy.deepcopy(c["node"]), None, 0, flat) + new = mint_ids(state.key, flat, state.used_ids()) + new[0]["parent"] = parent + nodes.extend(new) + _place(nodes, new[0]["id"], parent, c.get("place_after")) + nodes = _rebuild(nodes) + created = [n["id"] for n in new] + tmp = Doc(state.key, nodes) + touched = [x for x in tmp.leaves(new[0]["id"])] + path = tmp.ancestors(new[0]["id"]) + target = new[0]["id"] + + elif op == "insert": + if not _has_crop(c): + raise Refusal("correction.evidence", "an insert needs crop evidence") + node = by_id[target] + if node["kind"] not in RUN_KINDS: + raise Refusal("correction.insert", "text is inserted into runs", target) + run, offset = c.get("run"), c.get("offset") + if not _is_index(run) or run >= len(node["content"]) or not _is_index(offset): + raise Refusal("correction.insert", "bad run or offset", target) + r = node["content"][run] + f = EDITABLE_RUN_FIELD.get(r["kind"]) + if not f or offset > len(r[f]) or not isinstance(c.get("text"), str): + raise Refusal("correction.insert", "bad run or offset", target) + node["content"][run] = dict(r, **{f: r[f][:offset] + c["text"] + r[f][offset:]}) + node["content"] = canonical_runs(node["content"]) + touched = [target] + + elif op == "delete": + gone = _subtree(doc, target) + touched = [x for x in gone if doc.nodes[x]["kind"] in LEAVES] + nodes = _rebuild([n for n in nodes if n["id"] not in set(gone)]) + retired = [(x, None) for x in gone] + + elif op == "split": + node = by_id[target] + if node["kind"] != "sentence": + raise Refusal("correction.split", "only sentences are split", target) + k = c.get("offset") + first, second, pos = [], [], 0 + total = len(hashing.plain_runs(node["content"], doc)) + if not _is_index(k) or not 0 < k < total: + raise Refusal("correction.split", "offset must fall inside the sentence", target) + for r in node["content"]: + ln = len(hashing.plain_runs([r], doc)) + if pos + ln <= k: + first.append(r) + elif pos >= k: + second.append(r) + elif r["kind"] == "text": + cut = k - pos + first.append(dict(r, text=r["text"][:cut])) + second.append(dict(r, text=r["text"][cut:])) + else: + raise Refusal("correction.split", "the offset falls inside a %s run" % r["kind"], target) + pos += ln + _strip_edge(first, end=True) + _strip_edge(second, end=False) + first, second = canonical_runs(first), canonical_runs(second) + if not first or not second: + raise Refusal("correction.split", "both parts must keep text", target) + new_spec = {"kind": "sentence", "content": second} + eq_second = [r["node"] for r in second if r["kind"] == "equation"] + flat = [dict(new_spec, _parent=None, ord=0)] + tmp_nodes = mint_ids(state.key, [dict(flat[0], content=[ + {"kind": "text", "text": hashing.plain_runs(second, doc)}])], state.used_ids()) + new_id = tmp_nodes[0]["id"] + node["content"] = first + new_node = {"id": new_id, "kind": "sentence", "parent": node["parent"], "ord": 0, "content": second} + if "fidelity" in node: + new_node["fidelity"] = node["fidelity"] + nodes.append(new_node) + _place(nodes, new_id, node["parent"], target) + for i, e in enumerate(eq_second): + by_id[e]["parent"] = new_id + by_id[e]["ord"] = i + nodes = _rebuild(nodes) + created = [new_id] + touched = [target, new_id] + + elif op == "merge": + other = c["with"] + node = by_id[target] + if other not in by_id: + raise Refusal("correction.target", "no live node %s" % other) + o = by_id[other] + if node["kind"] != o["kind"] or node["kind"] not in MERGEABLE: + raise Refusal("correction.merge", "merge joins two nodes of one mergeable kind", target) + if o.get("parent") != node.get("parent") or o["ord"] != node["ord"] + 1: + raise Refusal("correction.merge", "merge joins a node with its next sibling", target) + if node["kind"] in RUN_KINDS: + joiner = c.get("joiner", " ") + mid = [{"kind": "text", "text": joiner}] if joiner else [] + node["content"] = canonical_runs(node["content"] + mid + o["content"]) + touched = [target, other] + else: + touched = [x for x in doc.leaves(other) if x != other] + base = len(doc.kids(target)) + for i, kid in enumerate(doc.kids(other)): + by_id[kid]["parent"] = target + by_id[kid]["ord"] = base + i + nodes = _rebuild([n for n in nodes if n["id"] != other]) + retired = [(other, target)] + + elif op == "move": + parent = c["parent"] + if parent not in by_id or parent in set(_subtree(doc, target)) or c.get("place_after") == target: + raise Refusal("correction.move", "bad destination", target) + _place(nodes, target, parent, c.get("place_after")) + nodes = _rebuild(nodes) + touched = [x for x in doc.leaves(target)] + record["to_path"] = Doc(state.key, nodes).ancestors(target) + + elif op == "reclassify": + node = by_id[target] + new_kind = c["kind"] + old = node["kind"] + same_shape = (old in RUN_KINDS) == (new_kind in RUN_KINDS) and (old in MATH_KINDS) == ( + new_kind in MATH_KINDS) and (old in VERBATIM_KINDS) == (new_kind in VERBATIM_KINDS) + if new_kind not in CHILDREN or not same_shape: + raise Refusal("correction.reclassify", "%s cannot become %s" % (old, new_kind), target) + node["kind"] = new_kind + if new_kind not in LABELLED: + node.pop("label", None) + # Each kind carries only its own attributes (spec/03 §3.4). + kept = {k: v for k, v in node.get("attrs", {}).items() if k in ATTRS.get(new_kind, ())} + if kept: + node["attrs"] = kept + else: + node.pop("attrs", None) + record["from"] = old + touched = [x for x in doc.leaves(target)] + + elif op == "relabel": + node = by_id[target] + if node["kind"] not in LABELLED: + raise Refusal("correction.relabel", "%s takes no label" % node["kind"], target) + record["from"] = node.get("label") + if c.get("label") is None: + node.pop("label", None) # optional properties are omitted, never written as null (spec/03 §3.1) + else: + node["label"] = c["label"] + touched = [target] if node["kind"] in LEAVES else [] + + elif op == "set": + node = by_id[target] + attr = c["attr"] + if attr not in SETTABLE.get(node["kind"], ()): + raise Refusal("correction.set", "%s cannot be set on %s" % (attr, node["kind"]), target) + attrs = dict(node.get("attrs", {})) + record["from"] = attrs.get(attr) + attrs[attr] = c["value"] + node["attrs"] = attrs + touched = [target] if node["kind"] in LEAVES else [] + + else: + raise Refusal("correction.op", "unknown operation " + op) + + after_doc = Doc(state.key, nodes) + validate_structure(after_doc) + validate_references(after_doc) + after_ix = before_ix.update(after_doc) + after_vals = after_ix.values + if store is not None: + changed = {x for x in set(touched) | set(created) | {target} if x in after_doc.nodes} + for nid in sorted(changed): + errors = schema_validate(store, "node.schema.json", dict(after_doc.nodes[nid], **after_vals[nid])) + if errors: + raise Refusal(_node_rule(errors[0]), errors[0], nid) + + # Digits: a change to a digit needs crop evidence, and no leaf outside `touched` may change its digits. + needs_crop = False + if op == "replace": + rep = c["replace"] if isinstance(c["replace"], str) else hashing.plain_runs(c["replace"], doc) + needs_crop = digits(c["find"]) != digits(rep) + elif op == "delete": + needs_crop = any(digits(before_ix.plain(x)) for x in touched) + elif op == "relabel": + needs_crop = digits(record.get("from") or "") != digits(c.get("label") or "") + if needs_crop and not _has_crop(c): + raise Refusal("correction.digits", "a digit changed without crop evidence", target) + for nid in doc.leaves(doc.root): + if nid in after_ix.entries and nid not in touched: + if digits(before_ix.plain(nid)) != digits(after_ix.plain(nid)): + raise Refusal("correction.digits", "an undeclared digit change", nid) + + for k in ("op", "target", "find", "replace", "run", "offset", "text", "parent", "place_after", "node", "with", + "joiner", "kind", "label", "attr", "value", "evidence", "author", "reason", "at"): + if k in c: + record[k] = c[k] + if op == "insert" and "node" in c: + record["target"] = target + record["before"] = before_vals[target]["node_hash"] if target in before_vals else None + record["after"] = after_vals[target]["node_hash"] if target in after_vals else None + record["path"] = path + record["touched"] = sorted(set(touched)) + if created: + record["created"] = created + if retired: + record["retired"] = [r[0] for r in retired] + record["id"] = record_id(sha256_hex(jcs.dumpb({k: v for k, v in record.items() if k != "id"}))) + + old_kinds = {n["id"]: n["kind"] for n in state.nodes} + new_ids = list(state.ids) + for rid, succ in retired: + new_ids.append({"id": rid, "kind": "retired", "successor": succ, "node_kind": old_kinds[rid], + "rev": record["index"]}) + new_ids.sort(key=id_order) + new_state = State(state.key, [after_doc.nodes[x] for x in after_doc.preorder()], new_ids, + state.corrections + [record], {hasher.key: after_ix}) + return new_state, record + + +def _strip_edge(runs, end): + """Remove the whitespace at one edge of a split part, across as many text runs as it spans (spec/11 §11.1).""" + from .text import is_ws + while runs: + i = len(runs) - 1 if end else 0 + r = runs[i] + if r["kind"] != "text": + return + t = r["text"] + if end: + while t and is_ws(t[-1]): + t = t[:-1] + else: + while t and is_ws(t[0]): + t = t[1:] + if t: + runs[i] = dict(r, text=t) + return + del runs[i] diff --git a/tools/oracle/crypto.py b/tools/oracle/crypto.py new file mode 100644 index 0000000..aab1c3e --- /dev/null +++ b/tools/oracle/crypto.py @@ -0,0 +1,249 @@ +"""Keyed hashing, key derivation, Merkle roots and Ed25519. + +Non-normative; see spec/05-hashes.md and spec/08-fidelity.md. +HKDF and Ed25519 come from the `cryptography` package. +""" + +import hashlib + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey +from cryptography.hazmat.primitives.kdf.hkdf import HKDF + +from . import jcs + + +def hkdf(ikm, salt, info, length=32): + """HKDF-SHA-256, RFC 5869.""" + return HKDF(algorithm=hashes.SHA256(), length=length, salt=salt, info=info).derive(ikm) + + +def team_key(secret): + return hkdf(secret, b"scienting-team-key/1", b"scienting-hash/1") + + +def key_id(key): + return hashlib.sha256(b"scienting-key-id/1" + key).hexdigest()[:16] + + +def page_hash(page_fp_hex): + return hashlib.sha256(b"scienting-page-hash/1" + bytes.fromhex(page_fp_hex)).hexdigest() + + +def page_key(page_fps_hex, context): + ikm = b"".join(bytes.fromhex(h) for h in page_fps_hex) + return hkdf(ikm, b"scienting-page-key/1", context.encode("utf-8")) + + +# --- Merkle trees (RFC 6962 hashing) ------------------------------------------ + + +def _leaf(data): + return hashlib.sha256(b"\x00" + data).digest() + + +def _node(left, right): + return hashlib.sha256(b"\x01" + left + right).digest() + + +def _mth(leaves): + n = len(leaves) + if n == 1: + return leaves[0] + k = 1 + while k * 2 < n: + k *= 2 + return _node(_mth(leaves[:k]), _mth(leaves[k:])) + + +def merkle_root(items): + """Root over receipt items: each item is a list such as ["node", id, hash].""" + if not items: + raise ValueError("a Merkle root needs at least one item") + return _mth([_leaf(jcs.dumpb(list(it))) for it in items]).hex() + + +def receipt_items(receipt): + items = [["node", nid, h] for nid, h in receipt.get("nodes", [])] + items += [list(r) for r in receipt.get("records", [])] + return items + + +EMPTY_ROOT = hashlib.sha256(b"").hexdigest() + + +def history_root(items): + """Root over an append-only history, with RFC 9162's hash of the empty list for no items.""" + if not items: + return EMPTY_ROOT + return _mth([_leaf(jcs.dumpb(list(it))) for it in items]).hex() + + +def leaf_hash(item): + return _leaf(jcs.dumpb(list(item))).hex() + + +# --- RFC 9162 proofs (§2.1.3 and §2.1.4) --------------------------------------- + + +def _path(m, leaves): + """PATH(m, D[n]): the inclusion path for the leaf at index m, as RFC 9162 §2.1.3.1 defines it.""" + n = len(leaves) + if n == 1: + return [] + k = 1 + while k * 2 < n: + k *= 2 + if m < k: + return _path(m, leaves[:k]) + [_mth(leaves[k:])] + return _path(m - k, leaves[k:]) + [_mth(leaves[:k])] + + +def inclusion_path(items, index): + """The audit path of `items[index]` in the tree over `items`, as hex hashes, leaf-most first.""" + if not 0 <= index < len(items): + raise ValueError("index outside the tree") + return [h.hex() for h in _path(index, [_leaf(jcs.dumpb(list(it))) for it in items])] + + +def verify_inclusion(root, leaf, index, size, path): + """Whether `path` proves that the leaf hash `leaf` is item `index` of a tree of `size` with `root` + (RFC 9162 §2.1.3.2). Never raises.""" + try: + if not 0 <= index < size: + return False + fn, sn = index, size - 1 + r = bytes.fromhex(leaf) + for p in path: + p = bytes.fromhex(p) + if sn == 0: + return False + if fn & 1 or fn == sn: + r = _node(p, r) + while fn & 1 == 0 and fn != 0: + fn >>= 1 + sn >>= 1 + else: + r = _node(r, p) + fn >>= 1 + sn >>= 1 + return sn == 0 and r.hex() == root + except (ValueError, TypeError): + return False + + +def _subproof(m, leaves, b): + n = len(leaves) + if m == n: + return [] if b else [_mth(leaves)] + k = 1 + while k * 2 < n: + k *= 2 + if m <= k: + return _subproof(m, leaves[:k], b) + [_mth(leaves[k:])] + return _subproof(m - k, leaves[k:], False) + [_mth(leaves[:k])] + + +def consistency_proof(items, m): + """PROOF(m, D[n]): what shows the tree over the first m items is a prefix of the tree over all n + (RFC 9162 §2.1.4.1), as hex hashes; empty when m is 0 or n.""" + n = len(items) + if not 0 <= m <= n: + raise ValueError("m outside the tree") + if m == 0 or m == n: + return [] + return [h.hex() for h in _subproof(m, [_leaf(jcs.dumpb(list(it))) for it in items], True)] + + +def verify_consistency(root_m, root_n, m, n, proof): + """Whether `proof` shows the tree of size m with `root_m` is a prefix of the tree of size n with `root_n` + (RFC 9162 §2.1.4.2). Never raises.""" + try: + if m == 0: + return not proof and root_m == EMPTY_ROOT + if m == n: + return not proof and root_m == root_n + if not 0 < m < n or not proof: + return False + proof = [bytes.fromhex(p) for p in proof] + fn, sn = m - 1, n - 1 + while fn & 1: + fn >>= 1 + sn >>= 1 + if fn == 0: + fr = sr = bytes.fromhex(root_m) + else: + fr = sr = proof[0] + proof = proof[1:] + for c in proof: + if sn == 0: + return False + if fn & 1 or fn == sn: + fr = _node(c, fr) + sr = _node(c, sr) + while fn & 1 == 0 and fn != 0: + fn >>= 1 + sn >>= 1 + else: + sr = _node(sr, c) + fn >>= 1 + sn >>= 1 + return sn == 0 and fr.hex() == root_m and sr.hex() == root_n + except (ValueError, TypeError): + return False + + +# --- Ed25519 (RFC 8032) ------------------------------------------------------- + + +def ed25519_public(secret): + """The 32-byte public key of a 32-byte Ed25519 secret.""" + return Ed25519PrivateKey.from_private_bytes(secret).public_key().public_bytes_raw() + + +def ed25519_sign(secret, msg): + return Ed25519PrivateKey.from_private_bytes(secret).sign(msg) + + +def ed25519_verify(public, msg, signature): + """True when `signature` is a valid Ed25519 signature of `msg` under `public`; never raises.""" + if len(public) != 32 or len(signature) != 64: + return False + try: + Ed25519PublicKey.from_public_bytes(public).verify(signature, msg) + except (InvalidSignature, ValueError): + return False + return True + + +# --- did:key (W3C did:key method, multicodec and base58btc) --------------------------- + +BASE58 = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz" +MULTICODEC = {"ed25519": 0xED, "ml-dsa-65": 0x1207} + + +def _varint(n): + out = bytearray() + while True: + b = n & 0x7F + n >>= 7 + out.append(b | (0x80 if n else 0)) + if not n: + return bytes(out) + + +def base58btc(data): + n = int.from_bytes(data, "big") + out = "" + while n: + n, r = divmod(n, 58) + out = BASE58[r] + out + return "1" * (len(data) - len(data.lstrip(b"\x00"))) + out + + +def did_key(scheme, public): + """The did:key of a public key: `z` and the base58btc of the multicodec prefix and the key bytes.""" + if scheme not in MULTICODEC: + raise ValueError("no multicodec code for " + scheme) + return "did:key:z" + base58btc(_varint(MULTICODEC[scheme]) + public) diff --git a/tools/oracle/db.py b/tools/oracle/db.py new file mode 100644 index 0000000..39f8fdd --- /dev/null +++ b/tools/oracle/db.py @@ -0,0 +1,319 @@ +"""The database form: loading a package into tables and unloading it again. + +Non-normative; see spec/13-database.md. +Loading is a pure function of the package, and unloading gives back the same bytes. +""" + +import json +import os +import shutil + +from . import hashing, jcs +from .model import Doc +from .package import SHA256_RE, _all_files, read_json, read_jsonl, safe_path, write +from .text import norm, sha256_hex +from .tree import Refusal + +SPEC_VERSION = "0.1.0" +BUNDLE_FORMAT = "scienting-bundle/1" + +STRUCTURED = {"MANIFEST.json", "document.json", "witnesses.json", "provenance.json", "roster.json", "nodes.jsonl", + "pages.jsonl", "locations.jsonl", "references.jsonl", "corrections.jsonl", "receipts.jsonl", + "ids.jsonl", "suspects.jsonl"} + + +def J(v): + return None if v is None else jcs.dumps(v) + + +def P(s): + return None if s is None else json.loads(s) + + +def B(v): + return None if v is None else (1 if v else 0) + + +def _opt(d, k, v): + if v is not None: + d[k] = v + + +def ins(conn, into, values): + conn.execute("INSERT %s VALUES (%s)" % (into, ",".join("?" * len(values))), values) + + +def create(conn, sql_dir): + for name in ("doc.sql", "store.sql"): + with open(os.path.join(sql_dir, name), encoding="utf-8") as f: + conn.executescript(f.read()) + + +def create_bundle(conn, sql_dir): + """The tables of a bundle: those of create(), then sql/bundle-sqlite.sql (spec/13 §13.5).""" + create(conn, sql_dir) + with open(os.path.join(sql_dir, "bundle-sqlite.sql"), encoding="utf-8") as f: + conn.executescript(f.read()) + + +def search_rows(key, nodes): + """The bundle's search rows for one document, in document order (spec/13 §13.5). + + One row per sentence, cell, equation outside a sentence and figure-text item, as + (doc_key, node, unit, context, text). For a node, `context` is the kind of its parent; a figure-text item + takes its panel as `node` and its `role` as `context`. Withheld nodes are skipped. + """ + doc = Doc(key, nodes) + out = [] + for nid in doc.order: + n = doc.nodes[nid] + kind = n["kind"] + parent = doc.nodes[n["parent"]]["kind"] if n.get("parent") else None + if n.get("withheld"): + continue + if kind in ("sentence", "cell") or (kind == "equation" and parent != "sentence"): + out.append((key, nid, kind, parent, norm(hashing.plain(doc, nid)))) + elif kind == "panel": + for item in (n.get("attrs") or {}).get("figure_text", []): + out.append((key, nid, "figure-text", item["role"], norm(item["text"]))) + return out + + +def index(conn, key): + """Fill the bundle's `search` table for the loaded document `key`.""" + nodes = [] + for n in _q(conn, "SELECT id, kind, parent, ord, attrs, content, withheld FROM nodes WHERE doc_key = ? ORDER BY seq", + (key,)): + rec = {"id": n["id"], "kind": n["kind"], "parent": n["parent"], "ord": n["ord"]} + _opt(rec, "attrs", P(n["attrs"])) + _opt(rec, "content", P(n["content"])) + if n["withheld"]: + rec["withheld"] = True + nodes.append(rec) + rows = search_rows(key, nodes) + conn.executemany("INSERT INTO search VALUES (?, ?, ?, ?, ?)", rows) + return len(rows) + + +def write_meta(conn, store, built_from, extra=None): + """Write `bundle_meta`: format, store, built_from and spec, then any `extra` pairs.""" + pairs = {"format": BUNDLE_FORMAT, "store": store, "built_from": built_from, "spec": SPEC_VERSION} + pairs.update(extra or {}) + conn.executemany("INSERT INTO bundle_meta VALUES (?, ?)", sorted(pairs.items())) + + +def load(conn, root, asset_store): + """Load the package at `root`. Files without tables go to `asset_store`, by SHA-256. + + Validate the package first (package.validate); this only refuses paths and hashes that are unsafe to use. + """ + _all_files(root) # refuses a symbolic link anywhere in the package before anything is copied + manifest = read_json(os.path.join(root, "MANIFEST.json")) + listed = {f["path"]: f for f in manifest["files"]} + for path, f in listed.items(): + if not safe_path(path): + raise Refusal("package.path.unsafe", "unsafe path %r" % path) + if not isinstance(f.get("sha256"), str) or not SHA256_RE.fullmatch(f["sha256"]): + raise Refusal("package.manifest.schema", "%s: sha256 is not 64 lowercase hex digits" % path) + + def jl(name): + return read_jsonl(os.path.join(root, name)) if name in listed else [] + + d = read_json(os.path.join(root, "document.json")) + key = d["key"] + with open(os.path.join(root, "MANIFEST.json"), "rb") as f: + package_hash = sha256_hex(f.read()) + prov = read_json(os.path.join(root, "provenance.json")) + roster = read_json(os.path.join(root, "roster.json")) if "roster.json" in listed else None + conn.execute("INSERT OR IGNORE INTO works VALUES (?, ?)", (d["work"]["key"], J(d["work"]["same_as"]))) + ins(conn, "INTO documents", (key, d.get("store"), J(d.get("aliases")), d["kind"], J(d["work"]), J(d["version"]), J(d["relations"]), + J(d["meta"]), d["language"], J(d.get("license")), d["policy"], J(d["hashing"]), d["root"], d["meta_hash"], + d["history"], + d["rev"], J(d["counts"]), J(d["fidelity"]), J(prov), J(roster), package_hash)) + for i, w in enumerate(read_json(os.path.join(root, "witnesses.json"))["witnesses"]): + ins(conn, "INTO witnesses", (key, w["id"], i, w["sha256"], w["size"], w["kind"], w["media_type"], w.get("pages"), + B(w.get("text_layer")), w.get("version_label"), J(w["origin"]), J(w["rights"]))) + for seq, n in enumerate(jl("nodes.jsonl")): + ins(conn, "INTO nodes", (key, n["id"], seq, n["kind"], n["parent"], n["ord"], n.get("label"), J(n.get("attrs")), + J(n.get("content")), B(n.get("withheld")), n.get("fidelity"), J(n.get("prov")), + n["text_hash"], n["match_hash"], n["align_key"], n["own_hash"], n["node_hash"])) + if isinstance(n.get("content"), list): + ords = 0 + for r in n["content"]: + if r["kind"] == "cite": + for ref in r["refs"]: + ins(conn, "OR IGNORE INTO citations", (key, n["id"], ref, ords)) + ords += 1 + for p in jl("pages.jsonl"): + ins(conn, "INTO pages", (key, p["witness"], p["page"], p["page_fp"], J(p["size"]), p.get("page_label"), + B(p["text_layer"]), J(p.get("stamps_removed")))) + for l in jl("locations.jsonl"): + ins(conn, "INTO locations", (key, l["node"], l["page_fp"], l["page"], l.get("page_label"), J(l["boxes"]), B(l.get("approx")))) + for r in jl("references.jsonl"): + ins(conn, "INTO refs", (key, r["node"], r.get("label"), J(r.get("csl")), J(r["resolution"]))) + main = ("format", "index", "id", "op", "target", "evidence", "author", "reason", "at", "before", "after", "path", + "touched") + for c in jl("corrections.jsonl"): + details = {k: v for k, v in c.items() if k not in main} + ins(conn, "INTO corrections", (key, c["index"], c["id"], c["op"], c.get("target"), J(c["evidence"]), J(c["author"]), + c["reason"], c["at"], c["before"], c["after"], J(c["path"]), J(c["touched"]), J(details))) + for e in jl("ids.jsonl"): + ins(conn, "INTO node_ids", (key, e["id"], e["kind"], e["successor"], e.get("node_kind"), e.get("rev"))) + for r in jl("receipts.jsonl"): + if r["format"] == "scienting-receipt/1": + ins(conn, "INTO receipts", (key, r["id"], r["level"], r["scope"], r["rev"], r["root"], J(r["nodes"]), + J(r.get("records")), J(r["evidence"]), r["by"], r["at"], J(r.get("qa")), + r.get("export_of"), J(r.get("keyed")), J(r["signature"]))) + for nid, h in r["nodes"]: + ins(conn, "INTO receipt_nodes", (r["id"], key, nid, h)) + else: + per = r.get("period") or {} + ins(conn, "INTO revocations", (key, r["id"], r.get("receipt"), r.get("signer"), per.get("from"), per.get("to"), + r["reason"], r["by"], r["at"], J(r["signature"]))) + for s in jl("suspects.jsonl"): + ins(conn, "INTO suspects", (key, s["id"], s["kind"], s.get("node"), J(s))) + os.makedirs(asset_store, exist_ok=True) + for path, f in listed.items(): + if path in STRUCTURED: + continue + present = f.get("present", True) + ins(conn, "INTO package_files", (key, path, f["sha256"], f["size"], 1 if present else 0)) + if present: + shutil.copyfile(os.path.join(root, path), os.path.join(asset_store, f["sha256"])) + return key + + +def _q(conn, sql, args): + cur = conn.execute(sql, args) + cols = [c[0] for c in cur.description] + return [dict(zip(cols, row)) for row in cur.fetchall()] + + +def unload(conn, key, out, asset_store): + """Write the package for `key` from the tables into `out`. Returns the package hash.""" + files = {} + d = _q(conn, "SELECT * FROM documents WHERE doc_key = ?", (key,))[0] + doc = {"format": "scienting-doc/1", "key": key} + _opt(doc, "store", d["store"]) + _opt(doc, "aliases", P(d["aliases"])) + doc.update({"kind": d["kind"], "work": P(d["work"]), "version": P(d["version"]), "relations": P(d["relations"]), + "meta": P(d["meta"]), "language": d["language"]}) + _opt(doc, "license", P(d["license"])) + doc.update({"policy": d["policy"], "hashing": P(d["hashing"]), "root": d["root_hash"], + "meta_hash": d["meta_hash"], "history": d["history"], "rev": d["rev"], "counts": P(d["counts"]), + "fidelity": P(d["fidelity"])}) + + def put(name, rec): + files[name] = (jcs.dumps(rec) + "\n").encode("utf-8") + + def putl(name, recs): + if recs: + files[name] = "".join(jcs.dumps(r) + "\n" for r in recs).encode("utf-8") + + put("document.json", doc) + put("provenance.json", P(d["provenance"])) + if d["roster"] is not None: + put("roster.json", P(d["roster"])) + ws = [] + for w in _q(conn, "SELECT * FROM witnesses WHERE doc_key = ? ORDER BY ord", (key,)): + rec = {"id": w["id"], "sha256": w["sha256"], "size": w["size"], "kind": w["kind"], "media_type": w["media_type"]} + _opt(rec, "pages", w["pages"]) + _opt(rec, "text_layer", None if w["text_layer"] is None else bool(w["text_layer"])) + _opt(rec, "version_label", w["version_label"]) + rec.update({"origin": P(w["origin"]), "rights": P(w["rights"])}) + ws.append(rec) + put("witnesses.json", {"witnesses": ws}) + nodes = [] + for n in _q(conn, "SELECT * FROM nodes WHERE doc_key = ? ORDER BY seq", (key,)): + rec = {"id": n["id"], "kind": n["kind"], "parent": n["parent"], "ord": n["ord"]} + _opt(rec, "label", n["label"]) + _opt(rec, "attrs", P(n["attrs"])) + _opt(rec, "content", P(n["content"])) + if n["withheld"]: + rec["withheld"] = True + _opt(rec, "fidelity", n["fidelity"]) + _opt(rec, "prov", P(n["prov"])) + for h in ("text_hash", "match_hash", "align_key", "own_hash", "node_hash"): + rec[h] = n[h] + nodes.append(rec) + putl("nodes.jsonl", nodes) + seq = {n["id"]: i for i, n in enumerate(nodes)} + pages = [] + for p in _q(conn, "SELECT * FROM pages WHERE doc_key = ? ORDER BY witness, page", (key,)): + rec = {"witness": p["witness"], "page": p["page"], "page_fp": p["page_fp"], "size": P(p["size"])} + _opt(rec, "page_label", p["page_label"]) + rec["text_layer"] = bool(p["text_layer"]) + _opt(rec, "stamps_removed", P(p["stamps_removed"])) + pages.append(rec) + putl("pages.jsonl", pages) + locs = [] + for l in _q(conn, "SELECT * FROM locations WHERE doc_key = ?", (key,)): + rec = {"node": l["node"], "page_fp": l["page_fp"], "page": l["page"]} + _opt(rec, "page_label", l["page_label"]) + rec["boxes"] = P(l["boxes"]) + if l["approx"]: + rec["approx"] = True + locs.append(rec) + locs.sort(key=lambda r: (seq[r["node"]], r["page"], r["page_fp"])) + putl("locations.jsonl", locs) + refs = [] + for r in _q(conn, "SELECT * FROM refs WHERE doc_key = ?", (key,)): + rec = {"node": r["node"]} + _opt(rec, "label", r["label"]) + _opt(rec, "csl", P(r["csl"])) + rec["resolution"] = P(r["resolution"]) + refs.append(rec) + refs.sort(key=lambda r: seq[r["node"]]) + putl("references.jsonl", refs) + cors = [] + for c in _q(conn, "SELECT * FROM corrections WHERE doc_key = ? ORDER BY idx", (key,)): + rec = {"format": "scienting-correction/1", "index": c["idx"], "id": c["id"], "op": c["op"]} + _opt(rec, "target", c["target"]) + rec.update({"evidence": P(c["evidence"]), "author": P(c["author"]), "reason": c["reason"], "at": c["made_at"], + "before": c["before_hash"], "after": c["after_hash"], "path": P(c["path"]), + "touched": P(c["touched"])}) + rec.update(P(c["details"])) + cors.append(rec) + putl("corrections.jsonl", cors) + ids = [] + for e in _q(conn, "SELECT * FROM node_ids WHERE doc_key = ?", (key,)): + rec = {"id": e["id"], "kind": e["kind"], "successor": e["successor"]} + _opt(rec, "node_kind", e["node_kind"]) + _opt(rec, "rev", e["rev"]) + ids.append(rec) + ids.sort(key=lambda e: (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"])) + putl("ids.jsonl", ids) + recs = [] + for r in _q(conn, "SELECT * FROM receipts WHERE doc_key = ?", (key,)): + rec = {"format": "scienting-receipt/1", "id": r["id"], "doc": key, "rev": r["rev"], "level": r["level"], + "scope": r["scope"], "nodes": P(r["nodes"])} + _opt(rec, "records", P(r["records"])) + rec.update({"root": r["root"], "evidence": P(r["evidence"]), "by": r["signer"], "at": r["made_at"]}) + _opt(rec, "qa", P(r["qa"])) + _opt(rec, "export_of", r["export_of"]) + _opt(rec, "keyed", P(r["keyed"])) + rec["signature"] = P(r["signature"]) + recs.append(rec) + for r in _q(conn, "SELECT * FROM revocations WHERE doc_key = ?", (key,)): + rec = {"format": "scienting-revocation/1", "id": r["id"]} + _opt(rec, "receipt", r["receipt"]) + _opt(rec, "signer", r["revoked_signer"]) + if r["period_from"] is not None: + rec["period"] = {"from": r["period_from"], "to": r["period_to"]} + rec.update({"reason": r["reason"], "by": r["signer"], "at": r["made_at"], "signature": P(r["signature"])}) + recs.append(rec) + recs.sort(key=lambda r: (r["at"], r["id"])) + putl("receipts.jsonl", recs) + sus = [P(s["record"]) for s in _q(conn, "SELECT * FROM suspects WHERE doc_key = ?", (key,))] + sus.sort(key=lambda s: s["id"]) + putl("suspects.jsonl", sus) + absent = [] + for f in _q(conn, "SELECT * FROM package_files WHERE doc_key = ?", (key,)): + if f["present"]: + if not SHA256_RE.fullmatch(f["sha256"]): + raise Refusal("package.manifest.schema", "%s: sha256 is not 64 lowercase hex digits" % f["path"]) + with open(os.path.join(asset_store, f["sha256"]), "rb") as fh: + files[f["path"]] = fh.read() + else: + absent.append({"path": f["path"], "sha256": f["sha256"], "size": f["size"], "present": False}) + return write(out, files, absent) diff --git a/tools/oracle/fingerprint.py b/tools/oracle/fingerprint.py new file mode 100644 index 0000000..d7b7ce1 --- /dev/null +++ b/tools/oracle/fingerprint.py @@ -0,0 +1,66 @@ +"""Page fingerprints, scienting-page/1, and download-stamp rules, scienting-stamps/1. + +Non-normative; see spec/07-geometry.md. +""" + +import math +import re +import unicodedata + +from . import jcs +from .notation import fold_plain_text +from .text import is_ws, norm, sha256_hex + + +def round_point(x): + """Round half up to a whole point.""" + return int(math.floor(x + 0.5)) + + +def _in_margin(box, size, margin): + x0, y0, x1, y1 = box + w, h = size + return x1 <= margin or x0 >= w - margin or y1 <= margin or y0 >= h - margin + + +def is_stamp(line, size, rules): + text = norm(line["text"]) + for r in rules: + if r.get("where", "any") == "margin": + if "box" not in line or not _in_margin(line["box"], size, r.get("margin", 36)): + continue + if re.search(r["pattern"], text, re.IGNORECASE | re.ASCII): + return r["id"] + return None + + +def _strip(s): + return "".join(c for c in s if not is_ws(c) and unicodedata.category(c) not in ("Cc", "Cf")) + + +def fp_text(text): + """Steps 3 and 4 of spec/07 §7.1: whitespace and control characters go first, so that "10 -20" and "10-20" + fold alike; then scripts and dashes fold as in the strict projection, then NFKC.""" + return _strip(unicodedata.normalize("NFKC", fold_plain_text(_strip(text)))) + + +def fingerprint(page, rules): + """`page`: {"size": [w, h], "lines": [{"text", "box"?}], "images": [sha256...]}. + + Returns (fingerprint, removed stamp rule IDs in line order). + """ + size = page["size"] + kept, removed = [], [] + for line in page.get("lines", []): + rid = is_stamp(line, size, rules) + if rid: + removed.append(rid) + else: + kept.append(line["text"]) + payload = { + "format": "scienting-page/1", + "size": [round_point(size[0]), round_point(size[1])], + "text": fp_text(" ".join(kept)), + "images": sorted(page.get("images", [])), + } + return sha256_hex(jcs.dumpb(payload)), removed diff --git a/tools/oracle/grade.py b/tools/oracle/grade.py new file mode 100644 index 0000000..42f972c --- /dev/null +++ b/tools/oracle/grade.py @@ -0,0 +1,93 @@ +"""Grading a scienting-submission/1 against a scienting-exercise/1 key. + +Non-normative; see spec/19-exercises.md. +""" + +from decimal import Decimal +from fractions import Fraction + +from . import anchors as A +from .tree import Refusal + + +def dec(d): + """A decimal string, exactly as the sum came out (spec/19 §19.4: points are decimal strings, summed exactly).""" + return format(Decimal(d), "f") + + +def _zero_like(d): + """Zero with the scale of `d`, so a missed element worth "0.5" scores "0.0".""" + return Decimal(0).quantize(Decimal(d)) + + +def _resolved(state, anchor): + r = A.resolve(state, anchor) + if r["result"] == "stale": + return None + return r["node"], r.get("span") + + +def _match(state, rule, student, key): + s, k = _resolved(state, student), _resolved(state, key) + if not s or not k or s[0] != k[0]: + return False + if rule.get("match") == "overlap" and k[1] is not None: + if s[1] is None: + return True + lo, hi = max(s[1][0], k[1][0]), min(s[1][1], k[1][1]) + size = k[1][1] - k[1][0] + return size > 0 and Fraction(max(0, hi - lo), size) >= Fraction(rule.get("min_overlap", "0.5")) + return True + + +def _any(state, rule, students, keys): + return any(_match(state, rule, s, k) for s in students for k in keys) + + +def key_anchors(body): + key = body["key"] + out = [] + for el in key.get("elements", []) + key.get("chain", []) + key.get("errors", []): + out += el["anchors"] + return out + + +def grade(body, submission, state): + for a in key_anchors(body): + if A.resolve(state, a)["result"] == "stale": + raise Refusal("grade.key-stale", "an anchor in the key is stale") + kind, key, rule, ans = body["exercise"], body["key"], body.get("rule", {}), submission["answers"] + items = [] + + def item(name, pts, got, fb): + items.append({"element": name, "points": dec(pts if got else _zero_like(pts)), "max": dec(pts), "feedback": fb}) + + if kind in ("anchor", "check"): + if kind == "check": + v = key["verdict"] + ok = ans.get("verdict") == v["value"] + item("verdict", v["points"], ok, v.get("feedback", {}).get("hit" if ok else "miss", "")) + for i, el in enumerate(key["elements"]): + ok = _any(state, rule, ans.get("anchors", []), el["anchors"]) + item("element:%d" % i, el["points"], ok, el.get("feedback", {}).get("hit" if ok else "miss", "")) + elif kind == "trace": + chain = ans.get("chain", []) + for i, link in enumerate(key["chain"]): + ok = i < len(chain) and _any(state, rule, chain[i], link["anchors"]) + item("link:%d" % i, link["points"], ok, link.get("feedback", {}).get("hit" if ok else "miss", "")) + elif kind == "reproduce": + ok = abs(Decimal(ans["value"]) - Decimal(key["value"])) <= Decimal(key["tolerance"]) + item("value", key["points"], ok, key.get("feedback", {}).get("hit" if ok else "miss", "")) + elif kind == "verify": + flags = ans.get("flags", []) + for i, err in enumerate(key["errors"]): + hit = [f for f in flags if _any(state, rule, [f["anchor"]], err["anchors"])] + ok = any(f["verdict"] == err["verdict"] for f in hit) + fb = err.get("feedback", {}).get("hit" if ok else "miss", "") + item("error:%d" % i, err["points"], ok, fb) + else: + raise Refusal("grade.kind", "unknown exercise kind " + kind) + score = sum((Decimal(x["points"]) for x in items), Decimal(0)) + total = sum((Decimal(x["max"]) for x in items), Decimal(0)) + return {"format": "scienting-grade/1", "submission": submission["id"], "exercise": submission["exercise"], + "score": dec(score), "max": dec(total), "items": items} diff --git a/tools/oracle/hashing.py b/tools/oracle/hashing.py new file mode 100644 index 0000000..016ac87 --- /dev/null +++ b/tools/oracle/hashing.py @@ -0,0 +1,320 @@ +"""The plain, strict and lossy projections, and the five content values. + +Non-normative; see spec/05-hashes.md. +""" + +import hashlib +import hmac +import unicodedata + +from . import jcs +from .model import LABELLED, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS +from .notation import complex_key, fold_math, fold_plain_text, fold_styled_script +from .text import collapse_ws, is_ws, norm, remove_ws, trim + +OBJ = "\ufffc" +MARK_OPEN = "\ufff9" +MARK_CLOSE = "\ufffb" + + +class Hasher: + """SHA-256, or HMAC-SHA-256 under a key: every content value uses the same function.""" + + def __init__(self, key=None): + self.key = key + + def __call__(self, data): + if isinstance(data, str): + data = data.encode("utf-8") + if self.key is None: + return hashlib.sha256(data).hexdigest() + return hmac.new(self.key, data, hashlib.sha256).hexdigest() + + +PLAIN = Hasher() + + +# --- runs -------------------------------------------------------------------- + + +def _eq_source(doc, run): + eq = doc.nodes[run["node"]] + return eq["content"]["source"], eq["content"]["notation"] + + +def plain_runs(runs, doc=None): + out = [] + for r in runs: + k = r["kind"] + if k in ("text", "link", "cite", "xref", "fnmark"): + out.append(r["text"]) + elif k == "math": + out.append("$" + r["source"] + "$") + elif k == "equation": + src, _ = _eq_source(doc, r) + out.append("$$" + src + "$$") + else: + raise ValueError("run kind " + k) + return "".join(out) + + +def _strict_math(notation, source, complexes): + folded = fold_math(notation, source) + if folded is None: + complexes.append(complex_key(notation, source)) + return OBJ + return folded + + +def _finish_strict(s): + s = unicodedata.normalize("NFKC", s) + return trim(collapse_ws(s)) + + +def strict_runs_raw(runs, doc, complexes): + out = [] + for r in runs: + k = r["kind"] + if k == "text": + st = r.get("style", []) + if "sup" in st: + out.append(fold_styled_script("^", r["text"])) + elif "sub" in st: + out.append(fold_styled_script("_", r["text"])) + else: + out.append(fold_plain_text(r["text"])) + elif k == "link": + out.append(fold_plain_text(r["text"])) + elif k == "math": + out.append(_strict_math(r["notation"], r["source"], complexes)) + elif k == "equation": + src, notation = _eq_source(doc, r) + out.append(_strict_math(notation, src, complexes)) + elif k in ("cite", "xref", "fnmark"): + out.append(MARK_OPEN + remove_ws(unicodedata.normalize("NFC", r["text"])) + MARK_CLOSE) + else: + raise ValueError("run kind " + k) + return "".join(out) + + +def lossy_from_strict(s): + out = [] + stack = [] + i, n = 0, len(s) + while i < n: + ch = s[i] + if ch in "_^" and i + 1 < n and s[i + 1] == "{": + stack.append("s") + i += 2 + continue + if ch == "{": + stack.append("t") + out.append(ch) + elif ch == "}": + if stack: + if stack.pop() == "t": + out.append(ch) + else: + out.append(ch) + elif ch == MARK_OPEN: + j = s.find(MARK_CLOSE, i) + i = n if j < 0 else j + 1 + continue + else: + out.append(ch) + i += 1 + t = unicodedata.normalize("NFKC", "".join(out).casefold()) + return "".join( + c for c in t if not is_ws(c) and unicodedata.category(c) != "Pd" and c != "\u00ad" + ) + + +def prose_length(lossy): + return sum(1 for c in lossy if c != OBJ) + + +# --- nodes ------------------------------------------------------------------- + + +def plain(doc, nid): + n = doc.nodes[nid] + k = n["kind"] + if k in RUN_KINDS: + return plain_runs(n["content"], doc) + if k in MATH_KINDS: + return "$$" + n["content"]["source"] + "$$" + if k in VERBATIM_KINDS: + return n["content"]["text"] + return " ".join(plain(doc, x) for x in doc.text_leaves(nid)) + + +def strict(doc, nid): + """The strict canonical projection and the node's complex math keys, in order.""" + n = doc.nodes[nid] + k = n["kind"] + complexes = [] + if k in RUN_KINDS: + s = strict_runs_raw(n["content"], doc, complexes) + elif k in MATH_KINDS: + s = _strict_math(n["content"]["notation"], n["content"]["source"], complexes) + elif k in VERBATIM_KINDS: + s = n["content"]["text"] + else: + parts = [] + for x in doc.text_leaves(nid): + p, c = strict(doc, x) + parts.append(p) + complexes.extend(c) + return _finish_strict(" ".join(parts)), complexes + return _finish_strict(s), complexes + + +def lossy(doc, nid): + return lossy_from_strict(strict(doc, nid)[0]) + + +def _runs_for_own(runs): + out = [] + for r in runs: + if r["kind"] == "equation": + out.append({"kind": "equation"}) + else: + out.append(r) + return out + + +def own_content(doc, nid, text_hash_of=None): + n = doc.nodes[nid] + k = n["kind"] + o = {"kind": k} + attrs = n.get("attrs", {}) + if k in LABELLED: + o["label"] = n.get("label") + if k in RUN_KINDS: + o["runs"] = _runs_for_own(n["content"]) + if k in MATH_KINDS: + o["notation"] = n["content"]["notation"] + o["source"] = n["content"]["source"] + if k in VERBATIM_KINDS: + o["language"] = n["content"].get("language") + o["text"] = n["content"]["text"] + if k == "section": + o["level"] = attrs["level"] + heads = [c for c in doc.kids(nid) if doc.nodes[c]["kind"] == "heading"] + o["heading"] = text_hash_of(heads[0]) if heads else None + if k == "table": + o["columns"] = attrs.get("columns", []) + if k == "cell": + o["column"] = attrs.get("column") + o["rowspan"] = attrs.get("rowspan", 1) + o["colspan"] = attrs.get("colspan", 1) + o["role"] = attrs.get("role", "data") + if k == "theorem": + o["variant"] = attrs.get("variant") + return o + + +class _Entry: + """One node's cached work: its five values, and the projections its parent joins.""" + + __slots__ = ("sig", "values", "plain", "joined_plain", "joined_strict", "complexes") + + def __init__(self, sig, values, plain, joined_plain, joined_strict, complexes): + self.sig = sig + self.values = values + self.plain = plain + # What this node gives a container's projections: its text leaves joined, or None when it has none. + self.joined_plain = joined_plain + self.joined_strict = joined_strict + self.complexes = complexes + + +def _sig(doc, nid): + """Everything a node's values depend on besides its children's values.""" + n = doc.nodes[nid] + return (n["kind"], n.get("label"), n.get("attrs"), n.get("content"), tuple(doc.kids(nid))) + + +class Index: + """The five values of every node of one document, kept so that a change rehashes only what it touched. + + A node's values depend only on its own subtree, so after a change only the nodes whose own record or + children changed, and their ancestors, are rehashed. A container's projections are its children's + cached projections joined, so no container walks its subtree again. + """ + + def __init__(self, hasher=PLAIN): + self.hasher = hasher + self.entries = {} + self.values = {} + + @classmethod + def build(cls, doc, hasher=PLAIN): + ix = cls(hasher) + ix._rehash(doc, set(doc.nodes)) + return ix + + def update(self, doc): + """A new Index for `doc`, a later state of this one's document; this Index is left unchanged.""" + ix = Index(self.hasher) + ix.entries = {nid: e for nid, e in self.entries.items() if nid in doc.nodes} + ix.values = {nid: v for nid, v in self.values.items() if nid in doc.nodes} + dirty = {nid for nid in doc.nodes if nid not in ix.entries or ix.entries[nid].sig != _sig(doc, nid)} + for nid in list(dirty): + p = doc.nodes[nid].get("parent") + while p is not None and p not in dirty: + dirty.add(p) + p = doc.nodes[p].get("parent") + ix._rehash(doc, dirty) + return ix + + def plain(self, nid): + """The node's plain projection.""" + return self.entries[nid].plain + + def _rehash(self, doc, dirty): + h = self.hasher + for nid in reversed(doc.preorder()): + if nid not in dirty: + continue + n = doc.nodes[nid] + k = n["kind"] + kids = doc.kids(nid) + if k in RUN_KINDS or k in MATH_KINDS or k in VERBATIM_KINDS: + p = plain(doc, nid) + st, cx = strict(doc, nid) + # An equation line is never a text leaf; a sentence's equations are part of its own runs. + joined = (None, None, []) if k == "equation-line" else (p, st, cx) + else: + ps, ss, cx = [], [], [] + for c in kids: + e = self.entries[c] + if e.joined_plain is not None: + ps.append(e.joined_plain) + ss.append(e.joined_strict) + cx.extend(e.complexes) + p = " ".join(ps) + st = _finish_strict(" ".join(ss)) + joined = (p, " ".join(ss), cx) if ps else (None, None, []) + own = h(jcs.dumpb(own_content(doc, nid, lambda x: self.values[x]["text_hash"]))) + self.values[nid] = { + "text_hash": h(norm(p)), + "match_hash": h(st), + "align_key": h(lossy_from_strict(st)), + "own_hash": own, + "node_hash": h(jcs.dumpb({"own": own, "children": [self.values[c]["node_hash"] for c in kids]})), + } + self.entries[nid] = _Entry(_sig(doc, nid), self.values[nid], p, *joined) + + +def compute(doc, hasher=PLAIN): + """The five content values of every node, bottom up. Returns {id: {...}}.""" + return Index.build(doc, hasher).values + + +def meta_object(document): + return {k: document.get(k) for k in ("work", "version", "relations", "meta", "language", "license")} + + +def meta_hash(document, hasher=PLAIN): + return hasher(jcs.dumpb(meta_object(document))) diff --git a/tools/oracle/jcs.py b/tools/oracle/jcs.py new file mode 100644 index 0000000..c4d394f --- /dev/null +++ b/tools/oracle/jcs.py @@ -0,0 +1,155 @@ +"""Canonical JSON, RFC 8785 (JSON Canonicalization Scheme). + +Non-normative. +The normative rule is spec/01-conventions.md. +Most values serialize through the standard library's C encoder, which escapes strings exactly as RFC 8785 does. +Values it would write differently take the exact path below: floats, integers beyond 2^53, and member names +outside the Basic Multilingual Plane, which RFC 8785 sorts by UTF-16 code units rather than by code points. +""" + +import json +import math + +_C = json.JSONEncoder(ensure_ascii=False, separators=(",", ":"), sort_keys=True, allow_nan=False) +_EXACT_INT = 2**53 + +_ESC = { + 0x08: "\\b", + 0x09: "\\t", + 0x0A: "\\n", + 0x0C: "\\f", + 0x0D: "\\r", + 0x22: '\\"', + 0x5C: "\\\\", +} + + +def _string(s): + out = ['"'] + for ch in s: + cp = ord(ch) + if 0xD800 <= cp <= 0xDFFF: + raise ValueError("lone surrogate in string") + if cp in _ESC: + out.append(_ESC[cp]) + elif cp < 0x20: + out.append("\\u%04x" % cp) + else: + out.append(ch) + out.append('"') + return "".join(out) + + +def _number(x): + """ECMAScript Number.prototype.toString, as RFC 8785 requires.""" + if isinstance(x, bool): + raise TypeError("bool is not a number") + if isinstance(x, int): + if abs(x) > 2**53: + raise ValueError("integer outside the IEEE 754 exact range") + return str(x) + if not math.isfinite(x): + raise ValueError("NaN and Infinity are not JSON") + if x == 0: + return "0" + if x < 0: + return "-" + _number(-x) + # Shortest round-trip digits, as ECMAScript requires. + r = repr(x) + if "e" in r or "E" in r: + mant, exp = r.lower().split("e") + exp = int(exp) + else: + mant, exp = r, 0 + if "." in mant: + ip, fp = mant.split(".") + else: + ip, fp = mant, "" + digits = (ip + fp).lstrip("0") + # Position of the decimal point relative to the start of `digits`. + lead_zeros = len(ip + fp) - len((ip + fp).lstrip("0")) + n = len(ip) - lead_zeros + exp + digits = digits.rstrip("0") or "0" + k = len(digits) + if k <= n <= 21: + return digits + "0" * (n - k) + if 0 < n <= 21: + return digits[:n] + "." + digits[n:] + if -6 < n <= 0: + return "0." + "0" * (-n) + digits + e = n - 1 + sign = "+" if e >= 0 else "-" + if k == 1: + return digits + "e" + sign + str(abs(e)) + return digits[0] + "." + digits[1:] + "e" + sign + str(abs(e)) + + +def _key(s): + # RFC 8785 sorts member names by their UTF-16 code units. + return s.encode("utf-16-be") + + +def _fast(v): + """True when the C encoder writes `v` exactly as RFC 8785 does.""" + t = type(v) + if t is str or v is None or t is bool: + return True + if t is int: + return -_EXACT_INT <= v <= _EXACT_INT + if t is dict: + for k, x in v.items(): + if type(k) is not str or (not k.isascii() and max(k) > "\uffff") or not _fast(x): + return False + return True + if t is list or t is tuple: + return all(_fast(x) for x in v) + return False + + +def dumps(value): + """Serialize `value` to canonical JSON text.""" + if _fast(value): + text = _C.encode(value) + if not text.isascii(): + try: + text.encode("utf-8") + except UnicodeEncodeError: + raise ValueError("lone surrogate in string") from None + return text + return _exact(value) + + +def _exact(value): + if value is None: + return "null" + if value is True: + return "true" + if value is False: + return "false" + if isinstance(value, str): + return _string(value) + if isinstance(value, (int, float)): + return _number(value) + if isinstance(value, (list, tuple)): + return "[" + ",".join(dumps(v) for v in value) + "]" + if isinstance(value, dict): + for k in value: + if not isinstance(k, str): + raise TypeError("object keys must be strings") + items = sorted(value.items(), key=lambda kv: _key(kv[0])) + return "{" + ",".join(_string(k) + ":" + dumps(v) for k, v in items) + "}" + raise TypeError("not JSON: %r" % type(value)) + + +def dumpb(value): + """Canonical JSON as UTF-8 bytes.""" + return dumps(value).encode("utf-8") + + +def loads_line(data): + """Parse one canonical JSON line of bytes, refusing a line that is not canonical.""" + import json as _json + value = _json.loads(data.decode("utf-8")) + if dumpb(value) != data: + raise ValueError("not canonical JSON") + return value diff --git a/tools/oracle/keys.py b/tools/oracle/keys.py new file mode 100644 index 0000000..9fcd17b --- /dev/null +++ b/tools/oracle/keys.py @@ -0,0 +1,161 @@ +"""Work and document keys: a Scholar-style citation key plus a short hash of the work's identity. + +Non-normative; see spec/02-model.md §2.4. +A work key looks like `vaswani2017attention-k3f9`. The first part is what Google Scholar would call the work; the +characters after the hyphen come from a hash of the work's identity, so two works Scholar would name alike never +share a key. A document key is the work key with a suffix for anything other than the version of record. +""" + +import hashlib +import re +import unicodedata + +from .text import ALPHABET + +SCHEME = "scienting-key/1" +SUFFIX_LENGTH = 4 +MAX_SUFFIX = 51 +IDENTIFIER_ORDER = ("doi", "arxiv", "isbn", "pmid", "pmcid", "openalex") +WORK_KEY_RE = re.compile(r"^[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}$") +DOCUMENT_KEY_RE = re.compile(r"^[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}(\.[a-z0-9][a-z0-9-]*)*$") +STOP_WORDS = frozenset("a an the of on in for to and or at by with from as into onto about toward towards via " + "is are be its it this that these those".split()) +NAME_SUFFIXES = frozenset(["jr", "sr", "ii", "iii", "iv", "v", "phd", "md"]) +_YEAR = re.compile(r"^\d{4}$") +_DOI_PREFIX = re.compile(r"^(?:doi:|https?://(?:dx\.|www\.)?doi\.org/)+", re.IGNORECASE) + + +def fold(text): + """Lower-case ASCII with accents removed: `Müller` gives `muller`.""" + raw = text if isinstance(text, str) else "" + return unicodedata.normalize("NFKD", raw).encode("ascii", "ignore").decode().lower() + + +def title_words(title): + """A title's words, folded, with apostrophes removed and other punctuation splitting.""" + t = re.sub(r"['’`]", "", fold(title)) + return re.findall(r"[a-z0-9]+", t) + + +def family_name(name): + """A best guess at the family name in a display name: the part before a comma, else the last word that is + not a suffix such as `Jr.` or `III`.""" + name = (name or "").strip() + if "," in name: + head, _, tail = name.partition(",") + if tail.strip().rstrip(".").lower() not in NAME_SUFFIXES or " " not in head.strip(): + return head.strip() + name = head + parts = name.split() + while len(parts) > 1 and parts[-1].rstrip(".").lower() in NAME_SUFFIXES: + parts.pop() + return parts[-1] if parts else "" + + +def first_family(meta): + """The first author's family name from a CSL-JSON item, as the key spells it, or `anon`.""" + authors = meta.get("author") or [] + if not authors: + return "anon" + a = authors[0] + name = a.get("family") or family_name(a.get("literal") or a.get("name") or "") + fam = re.sub(r"[^a-z]", "", fold(name)) + return fam or "anon" + + +def year_of(meta): + """The year of a CSL-JSON item's `issued` date, when it is a four-digit year, else the empty string.""" + parts = (meta.get("issued") or {}).get("date-parts") or [] + y = parts[0][0] if parts and parts[0] else None + s = str(y) if isinstance(y, int) and not isinstance(y, bool) else (y if isinstance(y, str) else "") + return s if _YEAR.match(s) else "" + + +def scholar_part(meta): + """Family name, year and the first title word that is not a stop word: `vaswani2017attention`.""" + word = next((w for w in title_words(meta.get("title")) if w not in STOP_WORDS), "") + return first_family(meta) + year_of(meta) + word + + +def norm_doi(value): + """A DOI in bare form with ASCII letters lower-cased (DOI Handbook 2.4).""" + s = _DOI_PREFIX.sub("", value.strip()).strip() + return s.translate(str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz")) + + +def identity(same_as, meta): + """What the key's hash is computed over: the first identifier of IDENTIFIER_ORDER among `same_as`, else the + folded metadata `meta:|<year>|<family>`.""" + by_kind = {} + for s in same_as or []: + kind, _, value = s.partition(":") + by_kind.setdefault(kind, value) + for kind in IDENTIFIER_ORDER: + if kind in by_kind and by_kind[kind]: + value = norm_doi(by_kind[kind]) if kind == "doi" else by_kind[kind].strip() + return "%s:%s" % (kind, value) + return "meta:%s|%s|%s" % (" ".join(title_words(meta.get("title"))), year_of(meta), first_family(meta)) + + +def suffix(ident, length=SUFFIX_LENGTH): + """The leading 5 × `length` bits of SHA-256 of `scienting-key/1:` and the identity, in Crockford base32.""" + if not 1 <= length <= MAX_SUFFIX: + raise ValueError("a key suffix is 1 to %d characters" % MAX_SUFFIX) + digest = int.from_bytes(hashlib.sha256((SCHEME + ":" + ident).encode("utf-8")).digest(), "big") + bits = digest >> (256 - 5 * length) + return "".join(ALPHABET[(bits >> (5 * (length - 1 - i))) & 31] for i in range(length)) + + +def work_key(same_as, meta, length=SUFFIX_LENGTH): + return scholar_part(meta) + "-" + suffix(identity(same_as, meta), length) + + +def unique_work_key(same_as, meta, taken): + """The work key, lengthened by two characters at a time while another identity already holds it.""" + for length in range(SUFFIX_LENGTH, MAX_SUFFIX + 1, 2): + key = work_key(same_as, meta, length) + if key not in taken: + return key + raise ValueError("no free key for " + identity(same_as, meta)) + + +def version_suffix(version): + """The document key's suffix for a version: nothing for the version of record, `.si` for Supporting + Information, `.v2` for a preprint or draft with a sequence number, else the version type.""" + t = version.get("type") + if t == "version-of-record": + return "" + if t == "supplement": + return ".si" + seq = version.get("seq") + if t in ("preprint", "draft", "submitted", "accepted") and isinstance(seq, int): + return ".v%d" % seq + return "." + t + + +def document_key(wkey, version): + return wkey + version_suffix(version) + + +def split_key(dkey): + """A document key's work key and its suffix (with the leading dot, or empty).""" + m = WORK_KEY_RE.match(dkey.split(".", 1)[0]) if dkey else None + if not m or not DOCUMENT_KEY_RE.match(dkey): + raise ValueError("%r is not a document key" % dkey) + head, dot, tail = dkey.partition(".") + return head, (dot + tail if dot else "") + + +def suffix_of(wkey): + return wkey.rsplit("-", 1)[1] + + +def suffix_agrees(wkey, same_as, meta): + """Whether a stored work key's hash part is what the identity gives: always checked when the identity is an + identifier such as a DOI, which never changes; a metadata identity may have been corrected since, so it + is not held to the key.""" + ident = identity(same_as, meta) + if ident.startswith("meta:"): + return True + sfx = suffix_of(wkey) + return sfx == suffix(ident, len(sfx)) diff --git a/tools/oracle/log.py b/tools/oracle/log.py new file mode 100644 index 0000000..32b0fd3 --- /dev/null +++ b/tools/oracle/log.py @@ -0,0 +1,204 @@ +"""A team's transparency log of receipts, revocations and roster changes, and its signed checkpoints. + +Non-normative; see spec/23-log.md. +Entries are the items of an RFC 9162 tree (the same tree as receipts and the correction history), and a +checkpoint is a signed note in the format transparency logs share (C2SP `signed-note` and `tlog-checkpoint`): +an origin line, the tree size, the root hash in Base64, an empty line, and one signature line per signer. +""" + +import base64 +import hashlib +import os + +from . import crypto, jcs + +ALGORITHM_ED25519 = 0x01 +FORMAT = "scienting-checkpoint/1" + + +def entry(record): + """The log item for a signed record: its kind, its ID and its record hash.""" + kinds = {"scienting-receipt/1": "receipt", "scienting-revocation/1": "revocation", "scienting-roster/1": "roster"} + kind = kinds.get(record.get("format")) + if kind is None: + raise ValueError("only receipts, revocations and rosters are logged") + if kind == "roster": + return ["roster", record.get("store", ""), hashlib.sha256(jcs.dumpb(record)).hexdigest()] + body = {k: v for k, v in record.items() if k not in ("id", "signature")} + return [kind, record["id"], hashlib.sha256(jcs.dumpb(body)).hexdigest()] + + +def root(entries): + return crypto.history_root(entries) + + +def key_hash(name, public): + """The four-byte key ID of a signed-note signer: SHA-256 of the name, a newline, the algorithm byte and the + public key, truncated.""" + return hashlib.sha256(name.encode("utf-8") + b"\n" + bytes([ALGORITHM_ED25519]) + public).digest()[:4] + + +def _b64(text): + """Canonical standard Base64 only: the bytes, or None when the text is not exactly their encoding.""" + try: + raw = base64.b64decode(text, validate=True) + except (ValueError, TypeError): + return None + return raw if base64.b64encode(raw).decode("ascii") == text else None + + +def note_body(origin, size, root_hex): + return "%s\n%d\n%s\n" % (origin, size, base64.b64encode(bytes.fromhex(root_hex)).decode("ascii")) + + +def sign_checkpoint(origin, entries, signers): + """A checkpoint note over `entries`, signed by each (name, secret) in `signers`.""" + body = note_body(origin, len(entries), root(entries)) + lines = [] + for name, secret in signers: + public = crypto.ed25519_public(secret) + sig = crypto.ed25519_sign(secret, body.encode("utf-8")) + lines.append("— %s %s" % (name, base64.b64encode(key_hash(name, public) + sig).decode("ascii"))) + return body + "\n" + "".join(ln + "\n" for ln in lines) + + +def parse_checkpoint(text): + """The origin, size, root (hex) and [(name, key hash, signature)] of a note, or None when malformed.""" + try: + body, _, sigs = text.partition("\n\n") + origin, size, root_b64 = body.split("\n") + rt = _b64(root_b64) + if rt is None or len(rt) != 32 or not size.isdigit() or str(int(size)) != size or not origin: + return None + out = [] + for ln in sigs.splitlines(): + if not ln.startswith("— "): + return None + _, name, blob = ln.split(" ", 2) + raw = _b64(blob) + if raw is None or len(raw) != 68: + return None + out.append((name, raw[:4], raw[4:])) + return {"origin": origin, "size": int(size), "root": rt.hex(), "signatures": out} + except (ValueError, TypeError): + return None + + +def verify_checkpoint(text, roster): + """The signers of the note whose Ed25519 signatures verify under the roster's keys of those names. + + Returns (parsed note, [names verified]); the note is None when malformed. Never raises. + """ + note = parse_checkpoint(text) + if note is None: + return None, [] + body = note_body(note["origin"], note["size"], note["root"]).encode("utf-8") + verified = [] + for name, kh, sig in note["signatures"]: + key = roster.get(name) + if not key or key.get("scheme") != "ed25519": + continue + try: + public = base64.b64decode(key["public"], validate=True) + except (ValueError, TypeError): + continue + if key_hash(name, public) == kh and crypto.ed25519_verify(public, body, sig): + verified.append(name) + return note, verified + + +# --- acceptance (spec/23 §23.3) --------------------------------------------------------------------------- + + +def accept_checkpoint(text, roster, settings): + """Whether a checkpoint note is one the project accepts: its origin is the project's log, a team key with the + `sign` purpose and the role `owner` signed it, and at least `threshold` of the project's witnesses cosigned it. + + Returns (accepted, reasons); `settings` is the project's `log` setting. Never raises. + """ + note, verified = verify_checkpoint(text, roster) + reasons = [] + if note is None: + return False, ["the note is malformed"] + if note["origin"] != settings.get("origin"): + reasons.append("the origin is not the project's log") + team = [n for n in verified if roster[n].get("role") == "owner" and "sign" in roster[n].get("purposes", [])] + if not team: + reasons.append("no owner key with the sign purpose signed it") + witnesses = [n for n in verified if n in settings.get("witnesses", []) and "witness" in roster[n].get("purposes", [])] + if len(witnesses) < settings.get("threshold", 0): + reasons.append("%d of %d witnesses cosigned" % (len(witnesses), settings.get("threshold", 0))) + return not reasons, reasons + + +# --- tiles (spec/23 §23.4) -------------------------------------------------------------------------------- + +TILE_HEIGHT = 8 +TILE_WIDTH = 1 << TILE_HEIGHT + + +def tile_path(level, index, width=None): + """The path of a tile: `tile/<level>/<N>`, with N as 3-digit groups, all but the last prefixed `x`, and + `.p/<W>` for a partial tile of W entries; `level` is a number or `entries`.""" + parts, n = ["%03d" % (index % 1000)], index // 1000 + while n: + parts.insert(0, "x%03d" % (n % 1000)) + n //= 1000 + path = "tile/%s/%s" % (level, "/".join(parts)) + return path + (".p/%d" % width if width is not None else "") + + +def tiles(entries): + """Every tile of a log over `entries`, as {path: bytes}: hash tiles per level (32-byte hashes of the complete + subtrees of 256^level entries, 256 to a tile) and entries tiles (canonical JSON Lines, 256 to a tile).""" + out = {} + leaves = [crypto._leaf(jcs.dumpb(list(e))) for e in entries] + level = 0 + while True: + span = TILE_WIDTH ** level + nodes = [crypto._mth(leaves[i * span:(i + 1) * span]) for i in range(len(leaves) // span)] + if not nodes: + break + for t in range(len(nodes) // TILE_WIDTH): + out[tile_path(level, t)] = b"".join(nodes[t * TILE_WIDTH:(t + 1) * TILE_WIDTH]) + rest = len(nodes) % TILE_WIDTH + if rest: + out[tile_path(level, len(nodes) // TILE_WIDTH, rest)] = b"".join(nodes[-rest:]) + level += 1 + for t in range(len(entries) // TILE_WIDTH): + out[tile_path("entries", t)] = b"".join(jcs.dumps(list(e)).encode("utf-8") + b"\n" + for e in entries[t * TILE_WIDTH:(t + 1) * TILE_WIDTH]) + rest = len(entries) % TILE_WIDTH + if rest: + out[tile_path("entries", len(entries) // TILE_WIDTH, rest)] = b"".join( + jcs.dumps(list(e)).encode("utf-8") + b"\n" for e in entries[-rest:]) + return out + + +def write_log(entries, origin, signers, out_dir): + """Write a log as static files: `checkpoint` and every tile. Returns the checkpoint note.""" + note = sign_checkpoint(origin, entries, signers) + for path, data in tiles(entries).items(): + full = os.path.join(out_dir, *path.split("/")) + os.makedirs(os.path.dirname(full), exist_ok=True) + with open(full, "wb") as f: + f.write(data) + with open(os.path.join(out_dir, "checkpoint"), "w", encoding="utf-8") as f: + f.write(note) + return note + + +def read_log(log_dir): + """The entries and the checkpoint note of a log written by `write_log`, read from its entries tiles.""" + with open(os.path.join(log_dir, "checkpoint"), encoding="utf-8") as f: + note = f.read() + parsed = parse_checkpoint(note) + if parsed is None: + raise ValueError("the checkpoint is malformed") + size = parsed["size"] + entries = [] + for t in range(size // TILE_WIDTH + (1 if size % TILE_WIDTH else 0)): + width = size % TILE_WIDTH if (t + 1) * TILE_WIDTH > size else None + with open(os.path.join(log_dir, *tile_path("entries", t, width).split("/")), "rb") as f: + entries += [json_line for json_line in (jcs.loads_line(ln) for ln in f.read().split(b"\n") if ln)] + return entries, note diff --git a/tools/oracle/mldsa.py b/tools/oracle/mldsa.py new file mode 100644 index 0000000..a7baaaa --- /dev/null +++ b/tools/oracle/mldsa.py @@ -0,0 +1,446 @@ +"""ML-DSA-65, the module-lattice digital signature algorithm of FIPS 204, in plain Python. + +Non-normative; see spec/08-fidelity.md §8.4. +This is the reference form of the standard, slow and clear, for the signature scheme `ml-dsa-65`: key generation +from a 32-byte seed, the deterministic signing variant (the hedging bytes are zero), and verification. Keys and +signatures are the standard's byte encodings, so a production verifier can use any FIPS 204 library. The pure +(non pre-hashed) variant with an empty context is used, as the standard's ML-DSA.Sign defines it. +""" + +import hashlib + +Q = 8380417 +N = 256 +D = 13 +ZETA = 1753 +K, L = 6, 5 +ETA = 4 +TAU = 49 +LAMBDA = 192 +GAMMA1 = 1 << 19 +GAMMA2 = (Q - 1) // 32 +BETA = TAU * ETA +OMEGA = 55 +PK_SIZE = 32 + 32 * K * (23 - D) +SK_SIZE = 32 + 32 + 64 + 32 * ((L + K) * 4 + D * K) +SIG_SIZE = LAMBDA // 4 + L * 32 * 20 + OMEGA + K +F_INV256 = 8347681 # 256^-1 mod q + + +def _bitrev8(m): + return int("{:08b}".format(m)[::-1], 2) + + +ZETAS = [pow(ZETA, _bitrev8(m), Q) for m in range(256)] + + +def _shake256(data, n): + return hashlib.shake_256(data).digest(n) + + +def _shake128(data, n): + return hashlib.shake_128(data).digest(n) + + +# --- bit packing (FIPS 204 §7.1) ---------------------------------------------------------------------------- + + +def _bitlen(x): + return x.bit_length() + + +def simple_bit_pack(w, b): + """Coefficients in [0, b] as bitlen(b)-bit little-endian fields.""" + bits = _bitlen(b) + acc = 0 + for i, c in enumerate(w): + acc |= c << (bits * i) + return acc.to_bytes(32 * bits, "little") + + +def simple_bit_unpack(v, b): + bits = _bitlen(b) + acc = int.from_bytes(v, "little") + mask = (1 << bits) - 1 + return [(acc >> (bits * i)) & mask for i in range(N)] + + +def bit_pack(w, a, b): + """Coefficients in [-a, b] stored as b - w_i in bitlen(a + b) bits.""" + bits = _bitlen(a + b) + acc = 0 + for i, c in enumerate(w): + acc |= (b - c) << (bits * i) + return acc.to_bytes(32 * bits, "little") + + +def bit_unpack(v, a, b): + bits = _bitlen(a + b) + acc = int.from_bytes(v, "little") + mask = (1 << bits) - 1 + return [b - ((acc >> (bits * i)) & mask) for i in range(N)] + + +def hint_bit_pack(h): + y = bytearray(OMEGA + K) + index = 0 + for i in range(K): + for j in range(N): + if h[i][j]: + y[index] = j + index += 1 + y[OMEGA + i] = index + return bytes(y) + + +def hint_bit_unpack(y): + h = [[0] * N for _ in range(K)] + index = 0 + for i in range(K): + if y[OMEGA + i] < index or y[OMEGA + i] > OMEGA: + return None + first = index + while index < y[OMEGA + i]: + if index > first and y[index - 1] >= y[index]: + return None + h[i][y[index]] = 1 + index += 1 + if any(y[i] != 0 for i in range(index, OMEGA)): + return None + return h + + +# --- arithmetic --------------------------------------------------------------------------------------------- + + +def _mod_pm(r, m): + """r mod± m: the representative in (-m/2, m/2].""" + r %= m + return r - m if r > m // 2 else r + + +def power2round(r): + rp = r % Q + r0 = _mod_pm(rp, 1 << D) + return (rp - r0) >> D, r0 + + +def decompose(r): + rp = r % Q + r0 = _mod_pm(rp, 2 * GAMMA2) + if rp - r0 == Q - 1: + return 0, r0 - 1 + return (rp - r0) // (2 * GAMMA2), r0 + + +def high_bits(r): + return decompose(r)[0] + + +def low_bits(r): + return decompose(r)[1] + + +def make_hint(z, r): + return 1 if high_bits(r) != high_bits(r + z) else 0 + + +def use_hint(h, r): + m = (Q - 1) // (2 * GAMMA2) + r1, r0 = decompose(r) + if h and r0 > 0: + return (r1 + 1) % m + if h and r0 <= 0: + return (r1 - 1) % m + return r1 + + +def _inf_norm(w): + return max(abs(_mod_pm(c, Q)) for c in w) + + +def ntt(w): + w = list(w) + m, length = 0, 128 + while length >= 1: + start = 0 + while start < N: + m += 1 + z = ZETAS[m] + for j in range(start, start + length): + t = z * w[j + length] % Q + w[j + length] = (w[j] - t) % Q + w[j] = (w[j] + t) % Q + start += 2 * length + length //= 2 + return w + + +def ntt_inv(w): + w = list(w) + m, length = 256, 1 + while length < N: + start = 0 + while start < N: + m -= 1 + z = (-ZETAS[m]) % Q + for j in range(start, start + length): + t = w[j] + w[j] = (t + w[j + length]) % Q + w[j + length] = z * (t - w[j + length]) % Q + start += 2 * length + length *= 2 + return [c * F_INV256 % Q for c in w] + + +def _mul(a, b): + return [x * y % Q for x, y in zip(a, b)] + + +def _add(a, b): + return [(x + y) % Q for x, y in zip(a, b)] + + +def _sub(a, b): + return [(x - y) % Q for x, y in zip(a, b)] + + +# --- sampling (FIPS 204 §7.3) ------------------------------------------------------------------------------- + + +def _coeff_from_three_bytes(b0, b1, b2): + z = ((b2 & 0x7F) << 16) | (b1 << 8) | b0 + return z if z < Q else None + + +def rej_ntt_poly(rho): + out, j, n = [], 0, 3 * 300 + stream = _shake128(rho, n) + pos = 0 + while j < N: + if pos + 3 > len(stream): + n *= 2 + stream = _shake128(rho, n) + c = _coeff_from_three_bytes(stream[pos], stream[pos + 1], stream[pos + 2]) + pos += 3 + if c is not None: + out.append(c) + j += 1 + return out + + +def _coeff_from_half_byte(b): + return 4 - b if b < 9 else None # eta = 4 + + +def rej_bounded_poly(rho): + out, j, n = [], 0, 200 + stream = _shake256(rho, n) + pos = 0 + while j < N: + if pos >= len(stream): + n *= 2 + stream = _shake256(rho, n) + z = stream[pos] + pos += 1 + z0 = _coeff_from_half_byte(z & 15) + z1 = _coeff_from_half_byte(z >> 4) + if z0 is not None: + out.append(z0) + j += 1 + if z1 is not None and j < N: + out.append(z1) + j += 1 + return out + + +def expand_a(rho): + return [[rej_ntt_poly(rho + bytes([s, r])) for s in range(L)] for r in range(K)] + + +def expand_s(rho_prime): + s1 = [rej_bounded_poly(rho_prime + r.to_bytes(2, "little")) for r in range(L)] + s2 = [rej_bounded_poly(rho_prime + (r + L).to_bytes(2, "little")) for r in range(K)] + return s1, s2 + + +def expand_mask(rho, mu): + c = 1 + _bitlen(GAMMA1 - 1) + return [bit_unpack(_shake256(rho + (mu + r).to_bytes(2, "little"), 32 * c), GAMMA1 - 1, GAMMA1) for r in range(L)] + + +def sample_in_ball(rho): + c = [0] * N + n = 8 + TAU * 4 + stream = _shake256(rho, n) + signs = int.from_bytes(stream[:8], "little") + pos = 8 + for i in range(N - TAU, N): + while True: + if pos >= len(stream): + n *= 2 + stream = _shake256(rho, n) + j = stream[pos] + pos += 1 + if j <= i: + break + c[i] = c[j] + c[j] = 1 if (signs >> (i + TAU - N)) & 1 == 0 else Q - 1 + return c + + +# --- encodings (FIPS 204 §7.2) ------------------------------------------------------------------------------ + + +def pk_encode(rho, t1): + return rho + b"".join(simple_bit_pack(t, (1 << (23 - D)) - 1) for t in t1) + + +def pk_decode(pk): + rho, rest = pk[:32], pk[32:] + step = 32 * (23 - D) + return rho, [simple_bit_unpack(rest[i * step:(i + 1) * step], (1 << (23 - D)) - 1) for i in range(K)] + + +def sk_encode(rho, k, tr, s1, s2, t0): + out = rho + k + tr + out += b"".join(bit_pack(s, ETA, ETA) for s in s1) + out += b"".join(bit_pack(s, ETA, ETA) for s in s2) + out += b"".join(bit_pack(t, (1 << (D - 1)) - 1, 1 << (D - 1)) for t in t0) + return out + + +def sk_decode(sk): + rho, k, tr = sk[:32], sk[32:64], sk[64:128] + pos = 128 + s1, s2, t0 = [], [], [] + for _ in range(L): + s1.append(bit_unpack(sk[pos:pos + 128], ETA, ETA)) + pos += 128 + for _ in range(K): + s2.append(bit_unpack(sk[pos:pos + 128], ETA, ETA)) + pos += 128 + for _ in range(K): + t0.append(bit_unpack(sk[pos:pos + 32 * D], (1 << (D - 1)) - 1, 1 << (D - 1))) + pos += 32 * D + return rho, k, tr, s1, s2, t0 + + +def sig_encode(c_tilde, z, h): + return c_tilde + b"".join(bit_pack(zi, GAMMA1 - 1, GAMMA1) for zi in z) + hint_bit_pack(h) + + +def sig_decode(sigma): + c_tilde = sigma[:LAMBDA // 4] + pos = LAMBDA // 4 + z = [] + for _ in range(L): + z.append(bit_unpack(sigma[pos:pos + 32 * 20], GAMMA1 - 1, GAMMA1)) + pos += 32 * 20 + h = hint_bit_unpack(sigma[pos:]) + return c_tilde, z, h + + +def w1_encode(w1): + return b"".join(simple_bit_pack(w, (Q - 1) // (2 * GAMMA2) - 1) for w in w1) + + +# --- the scheme (FIPS 204 §6) ------------------------------------------------------------------------------- + + +def _matvec(a_hat, v_hat): + out = [] + for i in range(K): + acc = [0] * N + for j in range(L): + acc = _add(acc, _mul(a_hat[i][j], v_hat[j])) + out.append(acc) + return out + + +def keygen(seed): + """ML-DSA.KeyGen_internal: (public key, secret key) bytes for a 32-byte seed.""" + if len(seed) != 32: + raise ValueError("the seed is 32 bytes") + h = _shake256(seed + bytes([K, L]), 128) + rho, rho_prime, kk = h[:32], h[32:96], h[96:] + a_hat = expand_a(rho) + s1, s2 = expand_s(rho_prime) + t = [_add(ntt_inv(row), s2i) for row, s2i in zip(_matvec(a_hat, [ntt(s) for s in s1]), s2)] + t1, t0 = [], [] + for poly in t: + hi, lo = zip(*(power2round(c) for c in poly)) + t1.append(list(hi)) + t0.append(list(lo)) + pk = pk_encode(rho, t1) + tr = _shake256(pk, 64) + return pk, sk_encode(rho, kk, tr, s1, s2, t0) + + +def public_key(seed): + return keygen(seed)[0] + + +def _message_representative(message, ctx=b""): + if len(ctx) > 255: + raise ValueError("the context is at most 255 bytes") + return bytes([0, len(ctx)]) + ctx + message + + +def sign(sk, message, ctx=b"", rnd=bytes(32)): + """ML-DSA.Sign with the deterministic variant (rnd = 0): the 3309-byte signature.""" + rho, kk, tr, s1, s2, t0 = sk_decode(sk) + s1_hat = [ntt(s) for s in s1] + s2_hat = [ntt(s) for s in s2] + t0_hat = [ntt(t) for t in t0] + a_hat = expand_a(rho) + mu = _shake256(tr + _message_representative(message, ctx), 64) + rho_pp = _shake256(kk + rnd + mu, 64) + kappa = 0 + while True: + y = expand_mask(rho_pp, kappa) + w = [ntt_inv(row) for row in _matvec(a_hat, [ntt(yi) for yi in y])] + w1 = [[high_bits(c) for c in wi] for wi in w] + c_tilde = _shake256(mu + w1_encode(w1), LAMBDA // 4) + c = sample_in_ball(c_tilde) + c_hat = ntt(c) + cs1 = [ntt_inv(_mul(c_hat, s)) for s in s1_hat] + cs2 = [ntt_inv(_mul(c_hat, s)) for s in s2_hat] + z = [_add(yi, ci) for yi, ci in zip(y, cs1)] + r0 = [[low_bits(c) for c in _sub(wi, ci)] for wi, ci in zip(w, cs2)] + kappa += L + if max(_inf_norm(zi) for zi in z) >= GAMMA1 - BETA or max(max(abs(c) for c in r) for r in r0) >= GAMMA2 - BETA: + continue + ct0 = [ntt_inv(_mul(c_hat, t)) for t in t0_hat] + h = [] + for i in range(K): + base = _add(_sub(w[i], cs2[i]), ct0[i]) + h.append([make_hint((-ct0[i][j]) % Q, base[j]) for j in range(N)]) + if max(_inf_norm(t) for t in ct0) >= GAMMA2 or sum(sum(hi) for hi in h) > OMEGA: + continue + z_signed = [[_mod_pm(c, Q) for c in zi] for zi in z] + return sig_encode(c_tilde, z_signed, h) + + +def verify(pk, message, signature, ctx=b""): + """ML-DSA.Verify: True for a valid signature of `message` under `pk`. Never raises.""" + try: + if len(pk) != PK_SIZE or len(signature) != SIG_SIZE or len(ctx) > 255: + return False + rho, t1 = pk_decode(pk) + c_tilde, z, h = sig_decode(signature) + if h is None: + return False + a_hat = expand_a(rho) + tr = _shake256(pk, 64) + mu = _shake256(tr + _message_representative(message, ctx), 64) + c_hat = ntt(sample_in_ball(c_tilde)) + az = _matvec(a_hat, [ntt(zi) for zi in z]) + ct1 = [_mul(c_hat, ntt([c << D for c in t])) for t in t1] + w_approx = [ntt_inv(_sub(a, b)) for a, b in zip(az, ct1)] + w1 = [[use_hint(h[i][j], w_approx[i][j]) for j in range(N)] for i in range(K)] + if max(_inf_norm(zi) for zi in z) >= GAMMA1 - BETA: + return False + return _shake256(mu + w1_encode(w1), LAMBDA // 4) == c_tilde + except (ValueError, TypeError, IndexError): + return False diff --git a/tools/oracle/model.py b/tools/oracle/model.py new file mode 100644 index 0000000..9a85991 --- /dev/null +++ b/tools/oracle/model.py @@ -0,0 +1,162 @@ +"""The node tree of scienting-doc/1. + +Non-normative; see spec/03-nodes.md. +""" + +BLOCKS = frozenset( + ["paragraph", "equation", "figure", "table", "list", "quote", "code", "algorithm", "theorem", "footnote"] +) + +# Allowed child kinds for each node kind. +CHILDREN = { + "document": frozenset(["front", "section", "back"]), + "front": frozenset(["title", "authors", "abstract", "keywords", "notes"]), + "title": frozenset(["sentence"]), + "authors": frozenset(), + "abstract": BLOCKS | {"section"}, + "keywords": frozenset(), + "notes": frozenset(["footnote", "paragraph"]), + "section": BLOCKS | {"heading", "section"}, + "heading": frozenset(["sentence"]), + "paragraph": frozenset(["sentence"]), + "sentence": frozenset(["equation"]), + "equation": frozenset(["equation-line"]), + "equation-line": frozenset(), + "figure": frozenset(["panel", "caption", "figure-note"]), + "panel": frozenset(), + "caption": frozenset(["sentence"]), + "figure-note": frozenset(["sentence"]), + "table": frozenset(["caption", "table-head", "table-body", "table-note"]), + "table-head": frozenset(["row"]), + "table-body": frozenset(["row"]), + "table-note": frozenset(["sentence"]), + "row": frozenset(["cell"]), + "cell": frozenset(), + "list": frozenset(["item"]), + "item": BLOCKS, + "quote": BLOCKS, + "code": frozenset(), + "algorithm": frozenset(), + "theorem": BLOCKS, + "footnote": frozenset(["sentence"]), + "back": frozenset(["acknowledgments", "references", "appendix", "notes"]), + "acknowledgments": BLOCKS, + "references": frozenset(["reference"]), + "reference": frozenset(), + "appendix": frozenset(["section"]), +} +KINDS = frozenset(CHILDREN) + +# Kinds that must have at least one child. +NONEMPTY = frozenset( + ["title", "heading", "paragraph", "caption", "figure-note", "table-note", "table-head", "table-body", + "row", "list", "item", "quote", "theorem", "footnote", "appendix"] +) +# Kinds that occur at most once among their siblings. +AT_MOST_ONCE = frozenset( + ["front", "back", "title", "authors", "abstract", "keywords", "caption", "table-head", "table-body", + "acknowledgments", "references"] +) + +RUN_KINDS = frozenset(["sentence", "cell", "reference", "authors", "keywords"]) +MATH_KINDS = frozenset(["equation", "equation-line"]) +VERBATIM_KINDS = frozenset(["code", "algorithm"]) +LEAVES = RUN_KINDS | MATH_KINDS | VERBATIM_KINDS +CONTAINERS_WITH_CONTENT = frozenset(["section", "figure", "panel", "table", "theorem", "footnote"]) +CONTAINERS_WITHOUT_CONTENT = KINDS - LEAVES - CONTAINERS_WITH_CONTENT +LABELLED = frozenset( + ["section", "figure", "panel", "table", "theorem", "footnote", "equation", "equation-line", "reference", + "code", "algorithm"] +) + +# The attributes each kind may carry (spec/03 §3.4); kinds absent here carry none. +ATTRS = { + "section": frozenset(["level"]), + "table": frozenset(["columns"]), + "cell": frozenset(["column", "rowspan", "colspan", "role"]), + "row": frozenset(["header_path"]), + "list": frozenset(["ordered"]), + "item": frozenset(["marker"]), + "theorem": frozenset(["variant"]), + "panel": frozenset(["image", "crop", "figure_text"]), +} + +RUNS = frozenset(["text", "math", "cite", "xref", "fnmark", "link", "equation"]) +STYLES = frozenset(["em", "strong", "sup", "sub", "sc", "mono"]) + +LEVELS = ["F0", "F1", "F2", "F3"] + + +def anchor_class(kind): + if kind in LEAVES: + return "leaf" + if kind in CONTAINERS_WITH_CONTENT: + return "container-with-content" + return "container-without-content" + + +class Doc: + """A document: nodes by ID with parent and order, plus document order.""" + + def __init__(self, key, nodes): + self.key = key + self.nodes = {} + self.order = [] + self.children = {} + for n in nodes: + self.nodes[n["id"]] = n + self.order.append(n["id"]) + self.children.setdefault(n["id"], []) + for nid in self.order: + p = self.nodes[nid].get("parent") + if p is not None: + self.children.setdefault(p, []).append(nid) + for p, kids in self.children.items(): + kids.sort(key=lambda c: self.nodes[c]["ord"]) + roots = [nid for nid in self.order if self.nodes[nid].get("parent") is None] + self.root = roots[0] if roots else None + + def kids(self, nid): + return self.children.get(nid, []) + + def preorder(self, nid=None): + nid = self.root if nid is None else nid + out = [] + stack = [nid] + while stack: + x = stack.pop() + out.append(x) + stack.extend(reversed(self.kids(x))) + return out + + def ancestors(self, nid): + out = [] + p = self.nodes[nid].get("parent") + while p is not None: + out.append(p) + p = self.nodes[p].get("parent") + return list(reversed(out)) + + def text_leaves(self, nid): + """The text-bearing nodes a container's projections are made of, in document order.""" + out = [] + stack = [nid] + while stack: + x = stack.pop() + n = self.nodes[x] + k = n["kind"] + if k in RUN_KINDS or k in VERBATIM_KINDS or (k == "equation" and self._parent_kind(x) != "sentence"): + out.append(x) + continue + if k == "equation-line" or k == "equation": + continue + stack.extend(reversed(self.kids(x))) + return out + + def leaves(self, nid): + """Every leaf below or at `nid`, including equations inside sentences and equation lines.""" + return [x for x in self.preorder(nid) if self.nodes[x]["kind"] in LEAVES] + + def _parent_kind(self, nid): + p = self.nodes[nid].get("parent") + return None if p is None else self.nodes[p]["kind"] diff --git a/tools/oracle/notation.py b/tools/oracle/notation.py new file mode 100644 index 0000000..5a675ce --- /dev/null +++ b/tools/oracle/notation.py @@ -0,0 +1,481 @@ +"""Simple notation and its canonical linear form. + +Non-normative; the rules are spec/05-hashes.md, "The strict projection". +A math source or a styled script either folds to one canonical linear form, or is complex. +""" + +import unicodedata + +from .text import collapse_ws, is_ws, remove_ws, trim + +MINUS = "\u2212" +DOT = "\u22c5" +PRIME = "\u2032" + +# Characters that fold to one canonical character inside notation. +CHAR_FOLD = { + "-": MINUS, + "\u2013": MINUS, + "\u2212": MINUS, + "\u00b7": DOT, + "\u2219": DOT, + "\u22c5": DOT, + "'": PRIME, + "\u2032": PRIME, +} + +OPERATORS = set("+=/<>.,()") | {MINUS, DOT, PRIME, "\u00d7", "\u2264", "\u2265", "\u00b1"} + +TEX_GREEK = { + "alpha": "\u03b1", "beta": "\u03b2", "gamma": "\u03b3", "delta": "\u03b4", + "epsilon": "\u03f5", "varepsilon": "\u03b5", "zeta": "\u03b6", "eta": "\u03b7", + "theta": "\u03b8", "vartheta": "\u03d1", "iota": "\u03b9", "kappa": "\u03ba", + "varkappa": "\u03f0", "lambda": "\u03bb", "mu": "\u03bc", "nu": "\u03bd", + "xi": "\u03be", "pi": "\u03c0", "varpi": "\u03d6", "rho": "\u03c1", + "varrho": "\u03f1", "sigma": "\u03c3", "varsigma": "\u03c2", "tau": "\u03c4", + "upsilon": "\u03c5", "phi": "\u03d5", "varphi": "\u03c6", "chi": "\u03c7", + "psi": "\u03c8", "omega": "\u03c9", + "Gamma": "\u0393", "Delta": "\u0394", "Theta": "\u0398", "Lambda": "\u039b", + "Xi": "\u039e", "Pi": "\u03a0", "Sigma": "\u03a3", "Upsilon": "\u03a5", + "Phi": "\u03a6", "Psi": "\u03a8", "Omega": "\u03a9", +} +TEX_SYMBOLS = { + "times": "\u00d7", "cdot": DOT, "pm": "\u00b1", "le": "\u2264", "leq": "\u2264", + "ge": "\u2265", "geq": "\u2265", "lt": "<", "gt": ">", "prime": PRIME, + "AA": "\u00c5", +} +TEX_WRAPPERS = {"mathrm", "text", "textrm", "mathup"} +TEX_DECLARATIONS = {"rm"} +TEX_SPACES = {"quad", "qquad", "enspace", "thinspace", "medspace", "thickspace"} +TEX_SPACE_SYMBOLS = {",", ";", ":", "!", " "} + +TYPST_GREEK = { + "alpha": "\u03b1", "beta": "\u03b2", "gamma": "\u03b3", "delta": "\u03b4", + "epsilon": "\u03b5", "zeta": "\u03b6", "eta": "\u03b7", "theta": "\u03b8", + "iota": "\u03b9", "kappa": "\u03ba", "lambda": "\u03bb", "mu": "\u03bc", + "nu": "\u03bd", "xi": "\u03be", "omicron": "\u03bf", "pi": "\u03c0", + "rho": "\u03c1", "sigma": "\u03c3", "tau": "\u03c4", "upsilon": "\u03c5", + "phi": "\u03c6", "chi": "\u03c7", "psi": "\u03c8", "omega": "\u03c9", + "Alpha": "\u0391", "Beta": "\u0392", "Gamma": "\u0393", "Delta": "\u0394", + "Epsilon": "\u0395", "Zeta": "\u0396", "Eta": "\u0397", "Theta": "\u0398", + "Iota": "\u0399", "Kappa": "\u039a", "Lambda": "\u039b", "Mu": "\u039c", + "Nu": "\u039d", "Xi": "\u039e", "Omicron": "\u039f", "Pi": "\u03a0", + "Rho": "\u03a1", "Sigma": "\u03a3", "Tau": "\u03a4", "Upsilon": "\u03a5", + "Phi": "\u03a6", "Chi": "\u03a7", "Psi": "\u03a8", "Omega": "\u03a9", + "epsilon.alt": "\u03f5", "theta.alt": "\u03d1", "phi.alt": "\u03d5", + "pi.alt": "\u03d6", "rho.alt": "\u03f1", "sigma.alt": "\u03c2", + "kappa.alt": "\u03f0", "beta.alt": "\u03d0", +} +TYPST_SYMBOLS = { + "plus": "+", "minus": MINUS, "times": "\u00d7", "dot": DOT, "dot.op": DOT, + "dot.c": DOT, "plus.minus": "\u00b1", "lt": "<", "gt": ">", "lt.eq": "\u2264", + "gt.eq": "\u2265", "eq": "=", "prime": PRIME, "slash": "/", +} +TYPST_WRAPPERS = {"upright"} +TYPST_SPACES = {"thin", "med", "thick", "quad", "wide", "space"} + + +class NotSimple(Exception): + pass + + +# The Latin and Greek letter blocks of spec/05 \u00a75.4: Basic Latin through Latin Extended-B, Greek and Coptic without +# the Coptic letters, Latin Extended Additional, and Greek Extended. +LETTER_RANGES = ((0x0041, 0x024F), (0x0370, 0x03E1), (0x03F0, 0x03FF), (0x1E00, 0x1EFF), (0x1F00, 0x1FFF)) + + +def is_letter(ch): + """A letter of General Category L in the Latin or Greek blocks, or the micro sign.""" + if ch == "\u00b5": + return True + if not unicodedata.category(ch).startswith("L"): + return False + cp = ord(ch) + return any(lo <= cp <= hi for lo, hi in LETTER_RANGES) + + +def fold_char(ch): + return CHAR_FOLD.get(ch, ch) + + +def is_simple_char(ch): + ch = fold_char(ch) + return is_letter(ch) or unicodedata.category(ch) == "Nd" or ch in OPERATORS + + +def script_type(ch): + d = unicodedata.decomposition(ch) + if d.startswith("<super>"): + return "^", "".join(chr(int(h, 16)) for h in d.split()[1:]) + if d.startswith("<sub>"): + return "_", "".join(chr(int(h, 16)) for h in d.split()[1:]) + return None, None + + +class Atom: + __slots__ = ("base", "primes", "sub", "sup") + + def __init__(self, base=""): + self.base = base + self.primes = 0 + self.sub = None + self.sup = None + + +def _canon(atoms): + out = [] + for a in atoms: + if a.sup is not None and a.sup and all( + x.base == PRIME and x.sub is None and x.sup is None and x.primes == 0 for x in a.sup + ): + a.primes += len(a.sup) + a.sup = None + s = a.base + PRIME * a.primes + if a.sub is not None: + s += "_{" + _canon(a.sub) + "}" + if a.sup is not None: + s += "^{" + _canon(a.sup) + "}" + out.append(s) + return "".join(out) + + +def _attach(atoms, op, arg): + if not atoms: + atoms.append(Atom("")) + a = atoms[-1] + if op == "_": + if a.sub is not None: + raise NotSimple("double subscript") + a.sub = arg + else: + if a.sup is not None: + raise NotSimple("double superscript") + a.sup = arg + + +def _char_atom(ch): + if not is_simple_char(ch): + raise NotSimple("character %r" % ch) + return Atom(fold_char(ch)) + + +# --- TeX --------------------------------------------------------------------- + + +def _tex_tokens(src): + toks = [] + i, n = 0, len(src) + while i < n: + ch = src[i] + if ch == "\\": + j = i + 1 + if j < n and src[j].isascii() and src[j].isalpha(): + while j < n and src[j].isascii() and src[j].isalpha(): + j += 1 + toks.append(("cw", src[i + 1:j])) + while j < n and is_ws(src[j]): + j += 1 + i = j + continue + if j < n: + toks.append(("cs", src[j])) + i = j + 1 + continue + raise NotSimple("trailing backslash") + if is_ws(ch) or ch == "~": + i += 1 + continue + toks.append(("ch", ch)) + i += 1 + return toks + + +class _TexParser: + def __init__(self, toks): + self.t = toks + self.i = 0 + + def peek(self): + return self.t[self.i] if self.i < len(self.t) else None + + def take(self): + tok = self.peek() + self.i += 1 + return tok + + def seq(self, closing): + atoms = [] + while True: + tok = self.peek() + if tok is None: + if closing: + raise NotSimple("unclosed group") + return atoms + if tok == ("ch", "}"): + if not closing: + raise NotSimple("unbalanced brace") + self.take() + return atoms + self.take() + kind, val = tok + if kind == "ch" and val in "_^": + _attach(atoms, val, self.arg()) + elif kind == "ch" and val == "'": + if not atoms: + atoms.append(Atom("")) + atoms[-1].primes += 1 + elif kind == "ch" and val == "{": + inner = self.seq(True) + nxt = self.peek() + if nxt in (("ch", "_"), ("ch", "^")) and len(inner) > 1: + raise NotSimple("script on a multi-atom group") + atoms.extend(inner) # an empty group leaves a following script to the previous atom or an empty base + else: + atoms.extend(self.command_or_char(kind, val)) + + def command_or_char(self, kind, val): + if kind == "ch": + return [_char_atom(val)] + if kind == "cs": + if val in TEX_SPACE_SYMBOLS: + return [] + raise NotSimple("control symbol \\" + val) + if val in TEX_GREEK: + return [Atom(TEX_GREEK[val])] + if val in TEX_SYMBOLS: + return [Atom(TEX_SYMBOLS[val])] + if val in TEX_SPACES or val in TEX_DECLARATIONS: + return [] + if val in TEX_WRAPPERS: + return self.arg() + if val in ("left", "right"): + tok = self.take() + if tok == ("ch", "(" if val == "left" else ")"): + return [Atom(tok[1])] + raise NotSimple("\\left or \\right with another delimiter") + raise NotSimple("command \\" + val) + + def arg(self): + tok = self.take() + if tok is None: + raise NotSimple("missing argument") + kind, val = tok + if tok == ("ch", "{"): + return self.seq(True) + if kind == "ch" and val in "}_^'": + raise NotSimple("bad argument") + return self.command_or_char(kind, val) + + +def tex_atoms(src): + return _TexParser(_tex_tokens(src)).seq(False) + + +# --- Typst ------------------------------------------------------------------- + + +def _typst_tokens(src): + toks = [] + i, n = 0, len(src) + while i < n: + ch = src[i] + if is_ws(ch): + i += 1 + continue + if ch == '"': + j = i + 1 + buf = [] + while j < n and src[j] != '"': + if src[j] == "\\": + raise NotSimple("escape in string") + buf.append(src[j]) + j += 1 + if j >= n: + raise NotSimple("unclosed string") + toks.append(("str", "".join(buf))) + i = j + 1 + continue + if unicodedata.category(ch).startswith("L"): + j = i + while j < n and ( + unicodedata.category(src[j]).startswith("L") + or (src[j] == "." and j + 1 < n and src[j + 1].isascii() and src[j + 1].isalpha() and j > i) + ): + j += 1 + toks.append(("id", src[i:j])) + i = j + continue + if unicodedata.category(ch) == "Nd": + j = i + while j < n and unicodedata.category(src[j]) == "Nd": + j += 1 + if j + 1 < n and src[j] == "." and unicodedata.category(src[j + 1]) == "Nd": + j += 1 + while j < n and unicodedata.category(src[j]) == "Nd": + j += 1 + toks.append(("num", src[i:j])) + i = j + continue + two = src[i:i + 2] + if two in ("<=", ">="): + toks.append(("ch", "\u2264" if two == "<=" else "\u2265")) + i += 2 + continue + if two in ("->", "=>", "!=", "<-", "==", ":="): + raise NotSimple("operator " + two) + toks.append(("ch", ch)) + i += 1 + return toks + + +class _TypstParser: + def __init__(self, toks): + self.t = toks + self.i = 0 + + def peek(self): + return self.t[self.i] if self.i < len(self.t) else None + + def take(self): + tok = self.peek() + self.i += 1 + return tok + + def seq(self, closing): + atoms = [] + while True: + tok = self.peek() + if tok is None: + if closing: + raise NotSimple("unclosed parenthesis") + return atoms + if closing and tok == ("ch", ")"): + self.take() + return atoms + self.take() + kind, val = tok + if kind == "ch" and val in "_^": + _attach(atoms, val, self.arg()) + elif kind == "ch" and val == "'": + if not atoms: + atoms.append(Atom("")) + atoms[-1].primes += 1 + elif kind == "ch" and val == "(": + atoms.append(Atom("(")) + atoms.extend(self.seq(True)) + atoms.append(Atom(")")) + else: + atoms.extend(self.single(kind, val)) + + def single(self, kind, val): + if kind == "num": + return [Atom(c) for c in val] + if kind == "str": + return [_char_atom(c) for c in remove_ws(val)] + if kind == "id": + if len(val) == 1: + return [_char_atom(val)] + if val in TYPST_GREEK: + return [Atom(TYPST_GREEK[val])] + if val in TYPST_SYMBOLS: + return [Atom(TYPST_SYMBOLS[val])] + if val in TYPST_SPACES: + return [] + if val in TYPST_WRAPPERS and self.peek() == ("ch", "("): + self.take() + return self.seq(True) + raise NotSimple("identifier " + val) + if val in "/*#&[]{}\\$!:;|~@": + raise NotSimple("character %r" % val) + return [_char_atom(val)] + + def arg(self): + tok = self.take() + if tok is None: + raise NotSimple("missing argument") + kind, val = tok + if tok == ("ch", "("): + return self.seq(True) + if kind == "ch" and val in ")_^'": + raise NotSimple("bad argument") + return self.single(kind, val) + + +def typst_atoms(src): + return _TypstParser(_typst_tokens(src)).seq(False) + + +# --- public ------------------------------------------------------------------ + + +def fold_math(notation, source): + """Return the canonical linear form of a math source, or None when it is complex.""" + src = unicodedata.normalize("NFC", source) + try: + atoms = tex_atoms(src) if notation == "tex" else typst_atoms(src) + return _canon(atoms) + except NotSimple: + return None + + +def complex_key(notation, source): + return notation + ":" + remove_ws(unicodedata.normalize("NFC", source)) + + +def _fold_script_chars(chars): + """Fold a run of Unicode script characters of one type, or None if any base is not simple.""" + op = None + bases = [] + for ch in chars: + t, b = script_type(ch) + if t is None or (op is not None and t != op): + return None + op = t + bases.append(b) + base = "".join(bases) + if not all(is_simple_char(c) for c in base): + return None + return op + "{" + "".join(fold_char(c) for c in base) + "}" + + +def _fold_chars(s, in_notation): + """Fold the characters of text: script characters always, operators only inside notation.""" + out = [] + i, n = 0, len(s) + while i < n: + ch = s[i] + t, b = script_type(ch) + if t is not None: + j = i + while j < n and script_type(s[j])[0] == t: + j += 1 + folded = _fold_script_chars(s[i:j]) + out.append(folded if folded is not None else s[i:j]) + i = j + continue + if in_notation: + out.append(fold_char(ch)) + elif ch in ("-", "\u2013") and i + 1 < n and unicodedata.category(s[i + 1]) == "Nd": + out.append(MINUS) + else: + out.append(ch) + i += 1 + return "".join(out) + + +def fold_plain_text(text): + return _fold_chars(unicodedata.normalize("NFC", text), in_notation=False) + + +def fold_styled_script(op, text): + """The canonical form of text styled as a subscript (`_`) or superscript (`^`).""" + t = unicodedata.normalize("NFC", text) + s = remove_ws(t) + simple = all(is_simple_char(c) or script_type(c)[0] is not None for c in s) + if simple: + inner = _fold_chars(s, in_notation=True) + if op == "^" and inner and all(c == PRIME for c in inner): + return inner + else: + inner = trim(collapse_ws(t)) + return op + "{" + inner + "}" diff --git a/tools/oracle/package.py b/tools/oracle/package.py new file mode 100644 index 0000000..9efe310 --- /dev/null +++ b/tools/oracle/package.py @@ -0,0 +1,628 @@ +"""Packages: writing, reading and validating a scienting-doc/1 package folder. + +Non-normative; see spec/12-package.md and spec/22-conformance.md. +""" + +import base64 +import binascii +import hashlib +import json +import os +import posixpath +import re +import stat +import tempfile +import unicodedata +import zipfile + +from . import crypto, hashing, jcs, keys as K, receipts as R, vrf +from .anchors import ADDRESS_RE +from .model import LEAVES, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS, Doc +from .schema import validate as schema_validate +from .text import KEY_RE, letter_first, record_id, sha256_hex +from .tree import Refusal, validate_references, validate_structure + +# The reference tools' own limit on a package's total size (spec/12 §12.2 leaves the limit to the consumer). +SIZE_LIMIT = 1 << 30 + +JSONL = { + "nodes.jsonl": "node.schema.json", + "pages.jsonl": "page.schema.json", + "locations.jsonl": "location.schema.json", + "references.jsonl": "reference.schema.json", + "corrections.jsonl": "correction.schema.json", + "receipts.jsonl": None, + "ids.jsonl": "id-entry.schema.json", + "suspects.jsonl": "suspect.schema.json", +} +JSON = { + "document.json": "document.schema.json", + "witnesses.json": "witnesses.schema.json", + "provenance.json": "provenance.schema.json", + "roster.json": "roster.schema.json", +} +REQUIRED = ["document.json", "nodes.jsonl", "witnesses.json", "provenance.json"] + + +def line(rec): + return jcs.dumps(rec) + "\n" + + +def jsonl(recs): + return "".join(line(r) for r in recs).encode("utf-8") + + +def witness_id(sha): + return letter_first(bytes.fromhex(sha), 7) + + +SHA256_RE = re.compile(r"[0-9a-f]{64}") + + +def safe_path(p): + if not isinstance(p, str) or not p or p.startswith("/") or "\\" in p or "\x00" in p: + return False + parts = p.split("/") + return all(x not in ("", ".", "..") for x in parts) and posixpath.normpath(p) == p + + +def write(root, files, absent=()): + """Write `files` ({path: bytes}) and a MANIFEST.json listing them and the `absent` entries. + + Returns the package hash. + """ + for path in list(files) + [a["path"] for a in absent]: + if not safe_path(path) or path == "MANIFEST.json": + raise Refusal("package.path.unsafe", "unsafe path %r" % path) + os.makedirs(root, exist_ok=True) + entries = [dict(a) for a in absent] + for path in sorted(files, key=lambda p: p.encode("utf-8")): + data = files[path] + if data is None: + continue + full = os.path.join(root, path) + os.makedirs(os.path.dirname(full), exist_ok=True) + with open(full, "wb") as f: + f.write(data) + entries.append({"path": path, "sha256": sha256_hex(data), "size": len(data)}) + entries.sort(key=lambda e: e["path"].encode("utf-8")) + doc = json.loads(files["document.json"]) + manifest = {"format": "scienting-package/1", "doc": "scienting-doc/1", "key": doc["key"], "files": entries} + mb = (jcs.dumps(manifest) + "\n").encode("utf-8") + with open(os.path.join(root, "MANIFEST.json"), "wb") as f: + f.write(mb) + return sha256_hex(mb) + + +def _refuse_constant(name): + raise ValueError("JSON has no canonical form for " + name) + + +def _parse_int(text): + """Read a JSON integer as an ECMAScript engine would: exactly within 2^53, as a double beyond (spec/01 §1.1).""" + v = int(text) + return v if abs(v) <= 2**53 else float(v) + + +def _canonical_bytes(data, where): + """Parse one canonical JSON value given as bytes; refuse anything else (spec/01 §1.1, §1.9). + + Numbers are read as IEEE 754 doubles, as the canonical form writes them, and the comparison is on bytes, so a + carriage return, a byte order mark, a lone surrogate, NaN or invalid UTF-8 is refused, not silently translated. + """ + try: + rec = json.loads(data.decode("utf-8"), parse_constant=_refuse_constant, parse_int=_parse_int) + canon = jcs.dumps(rec).encode("utf-8") + except (UnicodeDecodeError, ValueError, TypeError, OverflowError, RecursionError) as e: + raise Refusal("package.file.canonical", "%s: not canonical JSON (%s)" % (where, type(e).__name__)) + if canon != data: + raise Refusal("package.file.canonical", "%s: not canonical JSON" % where) + return rec + + +def read_jsonl(path): + with open(path, "rb") as f: + data = f.read() + if data and not data.endswith(b"\n"): + raise Refusal("package.file.newline", "%s: every line ends with a newline" % path) + return [_canonical_bytes(ln, "%s:%d" % (path, i)) for i, ln in enumerate(data.split(b"\n")[:-1], 1)] + + +def read_json(path): + with open(path, "rb") as f: + data = f.read() + if not data.endswith(b"\n"): + raise Refusal("package.file.canonical", "%s: the file does not end with one newline" % path) + return _canonical_bytes(data[:-1], path) + + +class Report: + def __init__(self): + self.errors = [] + + def add(self, rule, where, message): + self.errors.append({"rule": rule, "where": where, "message": message}) + + def __bool__(self): + return not self.errors + + +def _all_files(root): + """Every file under `root` by its exact name, refusing any symbolic link on the way (spec/12 §12.2).""" + out = [] + for d, dirs, fs in os.walk(root): + for name in dirs + fs: + if os.path.islink(os.path.join(d, name)): + rel = os.path.relpath(os.path.join(d, name), root).replace(os.sep, "/") + raise Refusal("package.path.unsafe", "%s: symbolic links are refused" % rel) + for f in fs: + out.append(os.path.relpath(os.path.join(d, f), root).replace(os.sep, "/")) + return sorted(out) + + +def _hash_file(full, budget): + """SHA-256 and size of a file read in chunks, refusing a read past `budget` bytes.""" + h, n = hashlib.sha256(), 0 + with open(full, "rb") as fh: + while True: + chunk = fh.read(1 << 20) + if not chunk: + break + n += len(chunk) + if n > budget: + raise Refusal("package.size", "the package's files exceed the limit of %d bytes" % SIZE_LIMIT) + h.update(chunk) + return h.hexdigest(), n + + +def validate(root, store, key=None, limit=SIZE_LIMIT): + """Validate the package at `root`. `store` is a schema store; `key` a hashing key for keyed packages; + `limit` the most bytes the validator reads before refusing.""" + rep = Report() + try: + _validate(root, store, key, rep, limit) + except Refusal as e: + rep.add(e.rule, e.node or "", e.message) + return rep + + +def _validate(root, store, key, rep, limit=SIZE_LIMIT): + mpath = os.path.join(root, "MANIFEST.json") + if not os.path.lexists(mpath): + raise Refusal("package.manifest.missing", "MANIFEST.json is missing") + actual = set(_all_files(root)) + manifest = read_json(mpath) + for e in schema_validate(store, "manifest.schema.json", manifest): + rep.add("package.manifest.schema", "MANIFEST.json", e) + if not rep: + return + listed = {} + paths = [f["path"] for f in manifest["files"]] + if paths != sorted(paths, key=lambda p: p.encode("utf-8")) or len(set(paths)) != len(paths): + rep.add("package.manifest.order", "MANIFEST.json", "files must be sorted by path and unique") + folded = {} + budget = limit + for f in manifest["files"]: + p = f["path"] + listed[p] = f + if not safe_path(p) or p == "MANIFEST.json": + rep.add("package.path.unsafe", p, "unsafe path") + continue + fold = unicodedata.normalize("NFC", p).casefold() + if fold in folded: + rep.add("package.manifest.order", p, "collides with %s under case folding" % folded[fold]) + folded[fold] = p + if f.get("present", True) is False: + if not p.startswith("assets/"): + rep.add("package.manifest.absent", p, "only assets may be listed without their bytes") + if p in actual: + rep.add("package.manifest.absent", p, "a file listed as absent is present") + continue + if p not in actual: + rep.add("package.manifest.missing-file", p, "listed file is missing") + continue + sha, size = _hash_file(os.path.join(root, p), budget) + budget -= size + if sha != f["sha256"] or size != f["size"]: + rep.add("package.manifest.hash", p, "SHA-256 or size does not match the manifest") + for p in sorted(actual): + if p != "MANIFEST.json" and p not in listed: + rep.add("package.manifest.unlisted", p, "file is not listed in the manifest") + for p in REQUIRED: + if p not in listed: + rep.add("package.file.required", p, "required file is missing") + if not rep: + return + + data = {} + for p, sch in JSON.items(): + if p in listed: + data[p] = read_json(os.path.join(root, p)) + for e in schema_validate(store, sch, data[p]): + rep.add("package.schema", p, e) + for p, sch in JSONL.items(): + if p in listed: + data[p] = read_jsonl(os.path.join(root, p)) + for i, rec in enumerate(data[p], 1): + s = sch + if p == "receipts.jsonl": + s = "receipt.schema.json" if rec.get("format") == "scienting-receipt/1" else "revocation.schema.json" + for e in schema_validate(store, s, rec): + rep.add("package.schema", "%s:%d" % (p, i), e) + if p != "nodes.jsonl" and not data[p]: + rep.add("package.file.empty", p, "an optional JSONL file with no records is omitted") + else: + data[p] = [] + if not rep: + return + + document = data["document.json"] + if document["key"] != manifest["key"]: + rep.add("package.manifest.key", "MANIFEST.json", "key differs from document.json") + # Keys (spec/02 §2.4): the document key is the work key and a version suffix, and the work key's hash part is + # the hash of the work's identifier when it has one. + try: + wkey, sfx = K.split_key(document["key"]) + except ValueError: + wkey, sfx = None, None + if wkey != document["work"]["key"] or sfx != K.version_suffix(document["version"]): + rep.add("doc.key.work", "document.json", "the document key is not the work key with the version's suffix") + elif not K.suffix_agrees(wkey, document["work"].get("same_as"), document.get("meta") or {}): + rep.add("doc.key.suffix", "document.json", "the work key's hash part is not the hash of the work's identifier") + nodes = data["nodes.jsonl"] + doc = Doc(document["key"], nodes) + try: + validate_structure(doc) + except Refusal as e: + rep.add(e.rule, "nodes.jsonl " + (e.node or ""), e.message) + return + + # Policy and hashing. + withheld = [n["id"] for n in nodes if n.get("withheld")] + scheme = document["hashing"]["scheme"] + if document["hashing"].get("algorithm") != "sha-256": + rep.add("doc.hash.algorithm", "document.json", "the hash algorithm is not one this validator implements") + if document["policy"] == "full" and withheld: + rep.add("doc.policy.withheld", "nodes.jsonl", "a full package withholds no text") + if document["policy"] == "checks-only": + bearing = [n["id"] for n in nodes if n["kind"] in LEAVES] + if set(bearing) != set(withheld): + rep.add("doc.policy.withheld", "nodes.jsonl", "a checks-only package withholds every leaf's content") + if scheme == "plain": + rep.add("doc.policy.keyed", "document.json", "a checks-only package carries keyed hashes") + if scheme == "hmac-team" and key is not None and document["hashing"]["key_id"] != crypto.key_id(key): + rep.add("doc.hash.key", "document.json", "key_id does not match the key supplied") + + # References between nodes, and the counts. + kinds = {n["id"]: n["kind"] for n in nodes} + try: + validate_references(doc) + except Refusal as e: + rep.add(e.rule, e.node or "", e.message) + leaves_n = sum(1 for n in nodes if n["kind"] in LEAVES) + if document.get("counts") != {"nodes": len(nodes), "leaves": leaves_n}: + rep.add("doc.counts", "document.json", "counts do not recompute") + + # Hashes. + hasher = hashing.PLAIN if scheme == "plain" else (hashing.Hasher(key) if key else None) + if hasher is not None and not withheld: + vals = hashing.compute(doc, hasher) + for n in nodes: + for k, v in vals[n["id"]].items(): + if n[k] != v: + rep.add("doc.hash." + k.replace("_", "-"), n["id"], "%s does not recompute" % k) + if vals[doc.root]["node_hash"] != document["root"]: + rep.add("doc.hash.root", "document.json", "root does not equal the root node's node_hash") + if hashing.meta_hash(document, hasher) != document["meta_hash"]: + rep.add("doc.hash.meta", "document.json", "meta_hash does not recompute") + else: + vals = {n["id"]: {k: n[k] for k in ("text_hash", "match_hash", "align_key", "own_hash", "node_hash")} + for n in nodes} + if n_root_hash(nodes, doc) != document["root"]: + rep.add("doc.hash.root", "document.json", "root does not equal the root node's node_hash") + + # IDs. + live = set(kinds) + ids = data["ids.jsonl"] + known = set(live) + for i, e in enumerate(ids, 1): + if e["id"] in live: + rep.add("doc.id.retired-live", "ids.jsonl:%d" % i, "%s is both live and in the ID table" % e["id"]) + known.add(e["id"]) + for i, e in enumerate(ids, 1): + s = e.get("successor") + if s is not None and s not in known: + rep.add("doc.id.successor", "ids.jsonl:%d" % i, "successor %s is unknown" % s) + if e["kind"] == "alias" and s not in live: + rep.add("doc.id.alias", "ids.jsonl:%d" % i, "an alias names a live node") + order = sorted(ids, key=lambda e: (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"])) + if order != ids: + rep.add("doc.order.ids", "ids.jsonl", "aliases first by ID, then retirements by revision and ID") + + # Corrections. + cors = data["corrections.jsonl"] + for i, c in enumerate(cors, 1): + if c.get("index") != i: + rep.add("doc.correction.index", "corrections.jsonl:%d" % i, "indexes count from 1 without gaps") + if c["id"] != record_id(sha256_hex(jcs.dumpb({k: v for k, v in c.items() if k != "id"}))): + rep.add("doc.correction.id", "corrections.jsonl:%d" % i, "id does not derive from the record") + if document["policy"] == "checks-only" and (cors or data["suspects.jsonl"]): + rep.add("doc.policy.withheld", "corrections.jsonl", "a checks-only package carries no corrections or suspects") + if document["policy"] == "full" and document["rev"] != len(cors): + rep.add("doc.correction.rev", "document.json", "rev must equal the number of corrections") + if document.get("history") != crypto.history_root(history_items(cors)): + rep.add("doc.history", "document.json", "history is not the root over the corrections") + retired_by = {e["id"]: e for e in ids if e["kind"] == "retired"} + for c in cors: + for r in c.get("retired", []): + if r not in retired_by: + rep.add("doc.id.retired-missing", c["id"], "%s was retired but is not in ids.jsonl" % r) + if not withheld and hasher is not None: + last = {} + for c in cors: + if c.get("target") in live and kinds[c["target"]] in LEAVES: + last[c["target"]] = c + for x in c.get("touched", []): + if x != c.get("target"): + last.pop(x, None) + for nid, c in last.items(): + if c.get("after") and c["after"] != vals[nid]["node_hash"] and not doc.kids(nid): + rep.add("doc.correction.after", c["id"], "the last correction of %s does not end at its hash" % nid) + + # Geometry. + wits = {w["id"]: w for w in data["witnesses.json"]["witnesses"]} + for w in wits.values(): + if w["id"] != witness_id(w["sha256"]): + rep.add("doc.witness.id", w["id"], "a witness ID derives from its SHA-256") + fps = set() + for i, p in enumerate(data["pages.jsonl"], 1): + if p["witness"] not in wits: + rep.add("doc.page.witness", "pages.jsonl:%d" % i, "unknown witness") + fps.add(p["page_fp"]) + porder = sorted(data["pages.jsonl"], key=lambda p: (p["witness"], p["page"])) + if porder != data["pages.jsonl"]: + rep.add("doc.order.pages", "pages.jsonl", "pages sort by witness and page") + pos = {nid: i for i, nid in enumerate(doc.order)} + for i, loc in enumerate(data["locations.jsonl"], 1): + if loc["node"] not in live: + rep.add("doc.location.node", "locations.jsonl:%d" % i, "unknown node") + if loc["page_fp"] not in fps: + rep.add("doc.location.page", "locations.jsonl:%d" % i, "page_fp is not a page of any witness") + lorder = sorted(data["locations.jsonl"], key=lambda l: (pos.get(l["node"], -1), l["page"], l["page_fp"])) + if lorder != data["locations.jsonl"]: + rep.add("doc.order.locations", "locations.jsonl", "locations sort by node order, then page") + for i, ref in enumerate(data["references.jsonl"], 1): + if kinds.get(ref["node"]) != "reference": + rep.add("doc.reference.node", "references.jsonl:%d" % i, "names no reference node") + rorder = sorted(data["references.jsonl"], key=lambda r: pos.get(r["node"], -1)) + if rorder != data["references.jsonl"]: + rep.add("doc.order.references", "references.jsonl", "references follow document order") + + sorder = sorted(data["suspects.jsonl"], key=lambda x: x["id"]) + if sorder != data["suspects.jsonl"]: + rep.add("doc.order.suspects", "suspects.jsonl", "suspects sort by ID") + + # Receipts and fidelity. + roster = {k["id"]: k for k in data.get("roster.json", {}).get("keys", [])} + for k in roster.values(): + if k.get("did") and k["scheme"] in crypto.MULTICODEC: + pub = R._b64(k.get("public", "")) + if pub is None or crypto.did_key(k["scheme"], pub) != k["did"]: + rep.add("doc.roster.did", k["id"], "the did:key is not the one the public key determines") + recs = data["receipts.jsonl"] + rorder = sorted(recs, key=lambda r: (r["at"], r["id"])) + if rorder != recs: + rep.add("doc.order.receipts", "receipts.jsonl", "receipts and revocations sort by time, then ID") + by_id = {r["id"]: r for r in recs} + for r in recs: + if r["id"] != record_id(R.record_hash(r)): + rep.add("doc.receipt.id", r["id"], "id does not derive from the record") + if r["format"] == "scienting-receipt/1": + if r["doc"] != document["key"]: + rep.add("doc.receipt.doc", r["id"], "receipt names another document") + if r["root"] != crypto.merkle_root(crypto.receipt_items(r)): + rep.add("doc.receipt.root", r["id"], "Merkle root does not recompute") + ids_listed = [x[0] for x in r["nodes"]] + if ids_listed != sorted(ids_listed) or len(set(ids_listed)) != len(ids_listed): + rep.add("doc.receipt.nodes", r["id"], "nodes sort by ID without repeats") + if roster: + signer = roster.get(r["by"]) + if not R.verify_signature(r, roster): + rep.add("doc.receipt.signature", r["id"], "signature does not verify") + elif r["signature"]["key"] != r["by"] or not signer or not R.key_valid_at(signer, r["at"]): + rep.add("doc.receipt.signature", r["id"], "the signing key is not the signer's key, valid when signed") + qa = r.get("qa") + if qa: + orig = by_id.get(qa["of"]) + if not orig or orig["format"] != "scienting-receipt/1": + rep.add("doc.qa.of", r["id"], "the spot-checked receipt is not a receipt in this package") + continue + if r["scope"] != "qa:" + qa["of"]: + rep.add("doc.qa.of", r["id"], "the scope names another receipt than qa.of") + if orig["at"] > r["at"]: + rep.add("doc.qa.of", r["id"], "a spot-check is signed after the receipt it checks") + orig_hashes = dict(orig["nodes"]) + if [n for n, _ in r["nodes"]] != qa["sample"] or any(orig_hashes.get(n) != h for n, h in r["nodes"]): + rep.add("doc.qa.sample", r["id"], "the receipt's nodes are not the sampled nodes at the original's hashes") + if not {e["node"] for e in qa["errors"]} <= set(qa["sample"]): + rep.add("doc.qa.sample", r["id"], "an error names a node outside the sample") + try: + seed = base64.b64decode(qa["seed"], validate=True) + except (binascii.Error, ValueError): + rep.add("doc.qa.seed", r["id"], "the seed is not Base64") + continue + if vrf.proof_to_hash(seed) is None: + rep.add("doc.qa.seed", r["id"], "the seed is not a VRF proof") + continue + ck = roster.get(qa["key"]) + if roster: + if not ck or ck["scheme"] != "ed25519" or "spot-check" not in ck["purposes"]: + rep.add("doc.qa.key", r["id"], "the seed key is not a registered Ed25519 spot-check key") + elif not ck["registered_at"] <= orig["at"]: + rep.add("doc.qa.key", r["id"], "the spot-check key was registered after the receipt") + elif vrf.verify(R._b64(ck["public"]) or b"", R.seed_payload(orig), seed) is None: + rep.add("doc.qa.seed", r["id"], "the seed is not the checker's VRF proof for the receipt") + if qa["sample"] != R.sample(orig, seed, qa["rate"], qa["minimum"]): + rep.add("doc.qa.sample", r["id"], "the sample does not follow from the seed") + if not withheld and (roster or not recs): + # Without a roster no receipt can be judged valid, so a package that carries receipts but no roster is + # checked against its store instead; one with neither reports the leaves' own levels. + if R.distribution(R.effective_levels(nodes, vals, recs, roster)) != document["fidelity"]: + rep.add("doc.fidelity.report", "document.json", "the fidelity distribution does not recompute") + + +def read_state(root): + """The current state of the package at `root`, for the gate (spec/11 §11.4): nodes without their values, + the ID table and the corrections applied so far. Validate the package first.""" + from .corrections import State + present = {f["path"] for f in read_json(os.path.join(root, "MANIFEST.json"))["files"]} + + def jl(name): + return read_jsonl(os.path.join(root, name)) if name in present else [] + + values = ("text_hash", "match_hash", "align_key", "own_hash", "node_hash") + nodes = [{k: v for k, v in n.items() if k not in values} for n in jl("nodes.jsonl")] + return State(read_json(os.path.join(root, "document.json"))["key"], nodes, jl("ids.jsonl"), jl("corrections.jsonl")) + + +def n_root_hash(nodes, doc): + return {n["id"]: n for n in nodes}[doc.root]["node_hash"] + + +def history_items(corrections): + """The items of a document's history tree (spec/11 §11.5): one per correction, in order.""" + return [["correction", c["index"], c["id"]] for c in corrections] + + +# --- zipped packages (spec/12 §12.2 and §12.8) ----------------------------------------- + +ZIP_EXTENSION = ".sct.zip" +# The reference tools' own limit on an archive's expanded size (spec/12 §12.2 leaves the limit to the consumer). +ZIP_LIMIT = 1 << 30 +_ZIP_DATE = (1980, 1, 1, 0, 0, 0) +_FLAG_ENCRYPTED = 0x1 +_FLAG_UTF8 = 0x800 + + +def is_zip(path): + return os.path.isfile(path) and path.lower().endswith(".zip") + + +def write_zip(root, path): + """Zip the package folder at `root` the way spec/12 §12.2 says a producer writes a byte-identical archive. + + Entries sit under one top-level folder named by the key, sorted by path, stored, dated 1980-01-01 00:00, + with no extra fields and no directory entries. Returns the SHA-256 of the archive. + """ + key = read_json(os.path.join(root, "document.json"))["key"] + files = _all_files(root) + for p in files: + if p != "MANIFEST.json" and not safe_path(p): + raise Refusal("package.path.unsafe", "unsafe path %r" % p) + with zipfile.ZipFile(path, "w") as zf: + for p in sorted(files, key=lambda x: x.encode("utf-8")): + zi = zipfile.ZipInfo(key + "/" + p, date_time=_ZIP_DATE) + zi.compress_type = zipfile.ZIP_STORED + zi.create_system = 3 + zi.external_attr = 0o644 << 16 + with open(os.path.join(root, p), "rb") as f: + zf.writestr(zi, f.read()) + with open(path, "rb") as f: + return sha256_hex(f.read()) + + +def _zip_entries(zf, limit): + """Check every entry of an open archive as spec/12 §12.2 requires; return the top-level folder.""" + infos = zf.infolist() + if not infos: + raise Refusal("package.zip.entry", "the archive has no entries") + folder, seen, total = None, set(), 0 + for zi in infos: + name = zi.filename + if not (zi.flag_bits & _FLAG_UTF8) and any(ord(ch) > 0x7F for ch in name): + raise Refusal("package.zip.entry", "%r: entry name is not marked as UTF-8" % name) + if zi.flag_bits & _FLAG_ENCRYPTED: + raise Refusal("package.zip.entry", "%s: encrypted entries are refused" % name) + top, sep, rest = name.partition("/") + if not sep or not K.DOCUMENT_KEY_RE.match(top): + raise Refusal("package.zip.entry", "%s: every entry sits under one folder named by the key" % name) + if folder is None: + folder = top + elif top != folder: + raise Refusal("package.zip.entry", "%s: entries sit under more than one top-level folder" % name) + if name in seen: + raise Refusal("package.zip.entry", "%s: duplicate entry" % name) + seen.add(name) + if zi.is_dir(): + if rest != "" and not safe_path(rest.rstrip("/")): + raise Refusal("package.zip.entry", "%s: unsafe path" % name) + continue + if stat.S_ISLNK(zi.external_attr >> 16): + raise Refusal("package.zip.entry", "%s: symbolic links are refused" % name) + if zi.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED): + raise Refusal("package.zip.entry", "%s: entries are stored or deflated" % name) + if rest != "MANIFEST.json" and not safe_path(rest): + raise Refusal("package.zip.entry", "%s: unsafe path" % name) + total += zi.file_size + if total > limit: + raise Refusal("package.zip.size", "the archive expands past the limit of %d bytes" % limit) + return folder + + +def read_zip(path, dest, limit=ZIP_LIMIT): + """Unpack a zipped package into the empty or new folder `dest`, refusing what spec/12 §12.2 refuses. + + Returns the package folder, `dest/<key>`. Every byte written is counted against each entry's declared size + and against `limit`, so a header that lies about a size is refused too. + """ + try: + zf = zipfile.ZipFile(path) + except (zipfile.BadZipFile, OSError) as e: + raise Refusal("package.zip.entry", "not a zip archive: %s" % e) + with zf: + folder = _zip_entries(zf, limit) + root = os.path.join(dest, folder) + os.makedirs(root) + written = 0 + for zi in zf.infolist(): + if zi.is_dir(): + continue + rest = zi.filename.partition("/")[2] + full = os.path.join(root, rest) + os.makedirs(os.path.dirname(full), exist_ok=True) + try: + with zf.open(zi) as src, open(full, "wb") as dst: + n = 0 + while True: + chunk = src.read(1 << 16) + if not chunk: + break + n += len(chunk) + written += len(chunk) + if n > zi.file_size or written > limit: + raise Refusal("package.zip.size", "%s: holds more than its header declares" % zi.filename) + dst.write(chunk) + except (zipfile.BadZipFile, RuntimeError, OSError) as e: + raise Refusal("package.zip.entry", "%s: %s" % (zi.filename, e)) + return root + + +def validate_zip(path, store, key=None, limit=ZIP_LIMIT): + """Validate a zipped package: unpack it under spec/12 §12.2's rules, then validate the folder.""" + with tempfile.TemporaryDirectory() as tmp: + try: + root = read_zip(path, tmp, limit) + except Refusal as e: + rep = Report() + rep.add(e.rule, e.node or "", e.message) + return rep + rep = validate(root, store, key) + mpath = os.path.join(root, "MANIFEST.json") + if os.path.exists(mpath): + try: + mkey = read_json(mpath).get("key") + except (Refusal, ValueError): + mkey = None + if mkey != os.path.basename(root): + rep.add("package.zip.entry", os.path.basename(root), "the top-level folder is not named by the key") + return rep diff --git a/tools/oracle/receipts.py b/tools/oracle/receipts.py new file mode 100644 index 0000000..05d9465 --- /dev/null +++ b/tools/oracle/receipts.py @@ -0,0 +1,284 @@ +"""Receipts, revocations, levels, spot-check sampling and reviewer error bounds. + +Non-normative; see spec/08-fidelity.md. +""" + +import base64 +import binascii +import hashlib +import math +from fractions import Fraction +from math import comb + +from . import crypto, jcs, mldsa, vrf +from .model import LEVELS +from .text import record_id, sha256_hex + +SIGNED_FORMATS = ("scienting-receipt/1", "scienting-revocation/1", "scienting-policy/1", "scienting-qa-report/1") + + +def record_hash(rec): + """SHA-256 of the canonical JSON of a signed record without its `id` and `signature`.""" + return sha256_hex(jcs.dumpb({k: v for k, v in rec.items() if k not in ("id", "signature")})) + + +def signing_payload(rec): + return jcs.dumpb({"format": rec["format"], "hash": record_hash(rec)}) + + +def seal(rec, secret, key_id): + """Fill in a record's root (for receipts), ID and Ed25519 signature.""" + rec = dict(rec) + if rec["format"] == "scienting-receipt/1": + rec["root"] = crypto.merkle_root(crypto.receipt_items(rec)) + rec.pop("id", None) + rec.pop("signature", None) + sig = crypto.ed25519_sign(secret, signing_payload(rec)) + rec["id"] = record_id(record_hash(rec)) + rec["signature"] = {"scheme": "ed25519", "key": key_id, "value": base64.b64encode(sig).decode("ascii")} + return rec + + +def _b64(s): + """Strict Base64 decoding; None for anything that is not Base64.""" + try: + return base64.b64decode(s, validate=True) + except (binascii.Error, ValueError, TypeError): + return None + + +def verify_signature(rec, roster): + """Whether the record's signature verifies under the roster key it names, for the schemes the tools implement + (`ed25519` and `ml-dsa-65`); any other scheme does not verify. Never raises.""" + sig = rec.get("signature") or {} + if not isinstance(sig, dict): + return False + key = roster.get(sig.get("key")) + if not key or sig.get("scheme") != key.get("scheme"): + return False + pub, value = _b64(key.get("public", "")), _b64(sig.get("value", "")) + if pub is None or value is None: + return False + try: + if sig["scheme"] == "ed25519": + return len(pub) == 32 and len(value) == 64 and crypto.ed25519_verify(pub, signing_payload(rec), value) + if sig["scheme"] == "ml-dsa-65": + return mldsa.verify(pub, signing_payload(rec), value) + except (ValueError, TypeError): + return False + return False + + +def seal_mldsa(rec, sk, key_id): + """Fill in a record's root (for receipts), ID and ML-DSA-65 signature, deterministically.""" + rec = dict(rec) + if rec["format"] == "scienting-receipt/1": + rec["root"] = crypto.merkle_root(crypto.receipt_items(rec)) + rec.pop("id", None) + rec.pop("signature", None) + sig = mldsa.sign(sk, signing_payload(rec)) + rec["id"] = record_id(record_hash(rec)) + rec["signature"] = {"scheme": "ml-dsa-65", "key": key_id, "value": base64.b64encode(sig).decode("ascii")} + return rec + + +def key_valid_at(key, at): + return key["registered_at"] <= at and (key.get("revoked_at") is None or at < key["revoked_at"]) + + +# --- spot-checks -------------------------------------------------------------- + + +def seed_payload(receipt): + return jcs.dumpb({"format": "scienting-spot-check/1", "receipt": record_hash(receipt)}) + + +def make_seed(receipt, checker_secret): + """The seed: the checker's VRF proof (RFC 9381, ECVRF-EDWARDS25519-SHA512-TAI) for the receipt's payload.""" + return vrf.prove(checker_secret, seed_payload(receipt)) + + +def sample_size(n, rate="0.05", minimum=10): + return min(n, max(minimum, math.ceil(Fraction(rate) * n))) + + +def sample(receipt, seed, rate="0.05", minimum=10): + """The sample a seed selects: ranked by SHA-256 of the proof's VRF output followed by the node ID.""" + beta = vrf.proof_to_hash(seed) + if beta is None: + raise ValueError("the seed is not a VRF proof") + ids = [nid for nid, _ in receipt["nodes"]] + m = sample_size(len(ids), rate, minimum) + ranked = sorted(ids, key=lambda nid: (hashlib.sha256(beta + nid.encode("utf-8")).hexdigest(), nid)) + return sorted(ranked[:m]) + + +def upper_bound(k, n, confidence="0.95"): + """One-sided Clopper-Pearson upper bound on an error rate, as a float.""" + if n <= 0: + raise ValueError("n must be positive") + if k >= n: + return 1.0 + alpha = 1.0 - float(Fraction(confidence)) + + def cdf(p): + if p <= 0: + return 1.0 + if p >= 1: + return 0.0 + lp, lq = math.log(p), math.log1p(-p) + total = 0.0 + for i in range(k + 1): + total += math.exp(math.lgamma(n + 1) - math.lgamma(i + 1) - math.lgamma(n - i + 1) + i * lp + + (n - i) * lq) + return total + + lo, hi = 0.0, 1.0 + for _ in range(200): + mid = (lo + hi) / 2 + if cdf(mid) > alpha: + lo = mid + else: + hi = mid + return hi + + +def bound_string(k, n, confidence="0.95"): + """The bound as the spec reports it: six decimal places, rounded up, computed exactly. + + The reported value is the least multiple of 10^-6 at which the probability of at most k errors in n is + at most 1 - c, found by bisection with integer arithmetic, so no floating point enters the result. + """ + if n <= 0: + raise ValueError("n must be positive") + if k >= n: + return "1.000000" + alpha = 1 - Fraction(confidence) + scale = 10**6 + + def tail_at_most(m): + # P[X <= k] at p = m / scale, compared with alpha, both scaled by scale**n and alpha's denominator. + total = sum(comb(n, i) * m**i * (scale - m)**(n - i) for i in range(k + 1)) + return total * alpha.denominator <= alpha.numerator * scale**n + + lo, hi = 0, scale # the tail at 0 is 1 (above alpha); at 1 it is 0 (k < n) + while hi - lo > 1: + mid = (lo + hi) // 2 + if tail_at_most(mid): + hi = mid + else: + lo = mid + return "%d.%06d" % (hi // scale, hi % scale) + + +# --- levels ------------------------------------------------------------------- + + +def revoked(receipt, revocations): + for r in revocations: + if r.get("receipt") == receipt["id"]: + return True + if r.get("signer") == receipt["by"] and r.get("period"): + p = r["period"] + if p["from"] <= receipt["at"] <= p["to"]: + return True + return False + + +def valid_revocations(revocations, receipts_by_id, roster): + """The revocations signed by the receipt's signer, the signer they name, or a key the roster makes an owner.""" + owners = {k for k, v in roster.items() if v.get("role") == "owner"} + out = [] + for r in revocations: + if not verify_signature(r, roster): + continue + signer = r["signature"]["key"] + allowed = set(owners) + if r.get("receipt") in receipts_by_id: + allowed.add(receipts_by_id[r["receipt"]]["by"]) + if r.get("signer"): + allowed.add(r["signer"]) + if signer in allowed: + out.append(r) + return out + + +def _base_status(rec, roster, valid_revs): + """`invalid`, `revoked` or `valid` for one receipt, before spot-checks are considered (spec/08 §8.6).""" + try: + if rec.get("root") != crypto.merkle_root(crypto.receipt_items(rec)): + return "invalid" + if rec["id"] != record_id(record_hash(rec)) or not verify_signature(rec, roster): + return "invalid" + key = roster.get(rec["by"]) + if not key or not key_valid_at(key, rec["at"]) or rec["signature"]["key"] != rec["by"]: + return "invalid" + except (KeyError, TypeError, ValueError): + return "invalid" + return "revoked" if revoked(rec, valid_revs) else "valid" + + +def statuses(records, roster): + """The status of every receipt in `records` (spec/08 §8.6), computed once. + + A receipt is suspended when a valid spot-check of it found an error, and a spot-check's own validity may + depend on its own spot-check, so statuses are settled from the end of each chain of spot-checks; a chain + that loops back on itself settles nothing, and its receipts are invalid. + """ + receipts = [x for x in records if x.get("format") == "scienting-receipt/1"] + revs = [x for x in records if x.get("format") == "scienting-revocation/1"] + by_id = {x["id"]: x for x in receipts} + valid_revs = valid_revocations(revs, by_id, roster) + base = {rec["id"]: _base_status(rec, roster, valid_revs) for rec in receipts} + checkers = {} + for q in receipts: + qa = q.get("qa") + if isinstance(qa, dict) and qa.get("errors") and base.get(q["id"]) == "valid" and qa.get("of") in by_id: + checkers.setdefault(qa["of"], []).append(q["id"]) + final = {rid: s for rid, s in base.items() if s != "valid" or rid not in checkers} + pending = set(base) - set(final) + progress = True + while pending and progress: + progress = False + for rid in sorted(pending): + qs = checkers[rid] + if all(q in final for q in qs): + final[rid] = "suspended" if any(final[q] == "valid" for q in qs) else "valid" + pending.discard(rid) + progress = True + for rid in pending: + final[rid] = "invalid" + return final + + +def receipt_status(rec, records, roster): + """`valid`, `revoked`, `suspended`, or `invalid` for one receipt among `records`.""" + return statuses(records, roster).get(rec.get("id"), "invalid") + + +def effective_levels(nodes, values, records, roster): + """Every leaf's level: its recorded level, raised by valid receipts that list it at its current hash. + + A spot-check's `qa:` receipt does not raise the nodes it found in error (spec/08 §8.2). + """ + level = {n["id"]: n.get("fidelity") for n in nodes if "fidelity" in n} + status = statuses(records, roster) + for rec in records: + if rec["format"] != "scienting-receipt/1": + continue + if status.get(rec["id"]) != "valid": + continue + wrong = {e["node"] for e in (rec.get("qa") or {}).get("errors", [])} + for nid, h in rec.get("nodes", []): + if nid in level and nid not in wrong and values[nid]["node_hash"] == h: + if LEVELS.index(rec["level"]) > LEVELS.index(level[nid]): + level[nid] = rec["level"] + return level + + +def distribution(levels): + counts = {lv: 0 for lv in LEVELS} + for lv in levels.values(): + counts[lv] += 1 + lowest = next((lv for lv in LEVELS if counts[lv]), None) + return {"lowest": lowest, "counts": counts} diff --git a/tools/oracle/rights.py b/tools/oracle/rights.py new file mode 100644 index 0000000..9d9a3bb --- /dev/null +++ b/tools/oracle/rights.py @@ -0,0 +1,106 @@ +"""Rights records, policies, stage checks and backend conditions. + +Non-normative; see spec/14-rights.md. +""" + +USES = ("read", "tdm", "redistribute", "quote-public", "accessible-copy") +PURPOSES = ("internal", "external") +PROCESSORS = ("local", "tenancy", "outside-no-retention", "outside") +AI_PAIRS = tuple(p + "/" + q for p in PURPOSES for q in PROCESSORS) + + +def _value(v): + return v if v in (True, False) else "unknown" + + +def flat(rights): + """A rights record's uses as one flat map: the five uses plus ai/<purpose>/<processor>.""" + uses = rights.get("uses", {}) + out = {u: _value(uses.get(u, "unknown")) for u in USES} + ai = uses.get("ai", {}) + for pair in AI_PAIRS: + out["ai/" + pair] = _value(ai.get(pair, "unknown")) + return out + + +def _policy_applies(policy, witness, now): + if policy.get("valid_until") and now >= policy["valid_until"]: + return False + if policy.get("at") and now < policy["at"]: + return False + a = policy.get("applies_to", {}) + r = witness["rights"] + if "basis" in a and r.get("basis") not in a["basis"]: + return False + if "license" in a and (r.get("license") or {}).get("spdx") not in a["license"]: + return False + if "doi_prefixes" in a: + doi = (witness.get("origin") or {}).get("doi") or "" + if not any(doi.startswith(p) for p in a["doi_prefixes"]): + return False + if "witnesses" in a and witness["sha256"] not in a["witnesses"]: + return False + return True + + +def effective(witness, policies, now): + """The witness's uses at `now`, with unknowns resolved by applicable policies, or all false if expired.""" + r = witness["rights"] + f = flat(r) + if r.get("valid_until") and now >= r["valid_until"]: + return {k: False for k in f}, True + for p in policies: + if not _policy_applies(p, witness, now): + continue + pf = flat({"uses": p.get("uses", {})}) + for k, v in pf.items(): + if f[k] == "unknown" and v != "unknown": + f[k] = v + return f, False + + +def combine(flats): + """The most restrictive of several witnesses' uses: false over unknown over true.""" + out = {} + for k in flats[0]: + vals = [f[k] for f in flats] + out[k] = False if False in vals else ("unknown" if "unknown" in vals else True) + return out + + +def check(stage, processor, uses): + """Whether a stage may run on a processor under `uses`. Returns (allowed, lacking).""" + lacking = [] + for u in stage.get("uses", []): + if uses[u] is not True: + lacking.append(u if uses[u] is False else u + "?") + ai = stage.get("ai") + if ai: + allowed_here = ai.get("processors") + if allowed_here and processor not in allowed_here: + lacking.append("processor:" + processor) + k = "ai/" + ai["purpose"] + "/" + processor + if uses[k] is not True: + lacking.append(k if uses[k] is False else k + "?") + return (not lacking), lacking + + +def backend_allowed(backend, profile): + """Whether a team profile meets every condition a backend's licenses impose. Returns (ok, failed).""" + failed = [] + for cond in backend.get("conditions", []): + k = cond["kind"] + if k == "revenue-cap": + r = profile.get("revenue_or_funding_usd") + if r is None or r.get("at_most") is None or r["at_most"] > cond["usd"]: + failed.append(cond["id"]) + elif k == "no-competition": + comp = profile.get("competes_with") + if comp is None or cond["vendor"] in comp: + failed.append(cond["id"]) + elif k == "purpose": + if profile.get("purpose") not in cond["allowed"]: + failed.append(cond["id"]) + else: + failed.append(cond["id"]) + return (not failed), failed diff --git a/tools/oracle/schema.py b/tools/oracle/schema.py new file mode 100644 index 0000000..7bba4b8 --- /dev/null +++ b/tools/oracle/schema.py @@ -0,0 +1,89 @@ +"""JSON Schema (draft 2020-12) validation with the `jsonschema` package, and the lint of the keyword subset. + +Non-normative. +The spec's schemas are restricted to the keywords below (spec/22-conformance.md) so that every 2020-12 +validator reads them alike; `lint` refuses any schema that leaves the subset. +""" + +import json +import os + +from jsonschema import Draft202012Validator +from referencing import Registry, Resource + +KEYWORDS = { + "$schema", "$id", "$ref", "$defs", "$comment", "title", "description", "examples", "default", "deprecated", + "type", "enum", "const", "properties", "required", "additionalProperties", "patternProperties", + "propertyNames", "minProperties", "maxProperties", "dependentRequired", "items", "prefixItems", "minItems", + "maxItems", "uniqueItems", "contains", "minLength", "maxLength", "pattern", "format", "minimum", "maximum", + "exclusiveMinimum", "exclusiveMaximum", "oneOf", "anyOf", "allOf", "not", "if", "then", "else", +} +FORBIDDEN_PATTERN_TOKENS = ("\\d", "\\w", "\\s", "\\b", "\\D", "\\W", "\\S", "\\B", "(?<", "(?P") + + +class SchemaError(Exception): + pass + + +class Store: + """The schemas of one folder, linted on load, with a compiled validator per schema.""" + + def __init__(self, root): + self.root = root + self.docs = {} + self._registry = None + self._validators = {} + + def registry(self): + if self._registry is None: + names = sorted(f for f in os.listdir(self.root) if f.endswith(".schema.json")) + self._registry = Registry().with_resources( + [(self.load(f)["$id"], Resource.from_contents(self.load(f))) for f in names]) + return self._registry + + def validator(self, name): + if name not in self._validators: + self._validators[name] = Draft202012Validator(self.load(name), registry=self.registry()) + return self._validators[name] + + def load(self, name): + if name not in self.docs: + with open(os.path.join(self.root, name), encoding="utf-8") as f: + self.docs[name] = json.load(f) + lint(self.docs[name], name) + return self.docs[name] + + def deref(self, base, ref): + name, _, frag = ref.partition("#") + name = name or base + node = self.load(name) + if frag: + for part in frag.lstrip("/").split("/"): + node = node[part.replace("~1", "/").replace("~0", "~")] + return name, node + + +def lint(schema, where): + """Refuse keywords outside the subset, and regex syntax that differs between dialects.""" + if isinstance(schema, dict): + for k, v in schema.items(): + if k not in KEYWORDS: + raise SchemaError("%s: keyword %s is outside the subset" % (where, k)) + if k == "pattern" and any(t in v for t in FORBIDDEN_PATTERN_TOKENS): + raise SchemaError("%s: pattern %r uses dialect-dependent syntax" % (where, v)) + if k in ("properties", "patternProperties", "$defs"): + for kk, vv in v.items(): + if k == "patternProperties" and any(t in kk for t in FORBIDDEN_PATTERN_TOKENS): + raise SchemaError("%s: pattern %r uses dialect-dependent syntax" % (where, kk)) + lint(vv, where + "/" + kk) + elif k in ("items", "additionalProperties", "propertyNames", "not", "if", "then", "else", "contains"): + lint(v, where + "/" + k) + elif k in ("oneOf", "anyOf", "allOf", "prefixItems"): + for i, s in enumerate(v): + lint(s, "%s/%s/%d" % (where, k, i)) + + +def validate(store, name, instance): + """Return a list of error strings, `path: message`, in path order; empty when the instance is valid.""" + errors = sorted(store.validator(name).iter_errors(instance), key=lambda e: [str(x) for x in e.absolute_path]) + return ["%s: %s" % ("/" + "/".join(str(x) for x in e.absolute_path), e.message) for e in errors] diff --git a/tools/oracle/targets.py b/tools/oracle/targets.py new file mode 100644 index 0000000..007fada --- /dev/null +++ b/tools/oracle/targets.py @@ -0,0 +1,69 @@ +"""The target set of targeted F3: the nodes a team's claims, data and quotations rest on. + +Non-normative; see spec/08-fidelity.md §8.10. +""" + +from .model import LEAVES, MATH_KINDS +from .text import is_digit + +DIGIT_RULES = ("all", "anchored-sections", "none") +# The kinds that count as the section a node sits in: a section, or the front or back matter outside any. +SECTION_KINDS = frozenset(["section", "front", "back"]) +DIGIT_KINDS = frozenset(["sentence", "cell"]) +DIGIT_RUNS = frozenset(["text", "math"]) + + +def has_digit_run(node): + """Whether a `text` or `math` run of the node holds a digit (General Category Nd).""" + for r in node.get("content") or []: + if not isinstance(r, dict) or r.get("kind") not in DIGIT_RUNS: + continue + s = r.get("text") if r["kind"] == "text" else r.get("source") + if any(is_digit(ch) for ch in s or ""): + return True + return False + + +def enclosing_section(doc, nid): + """The nearest ancestor-or-self that is a section, or the front or back matter, else the root.""" + if doc.nodes[nid]["kind"] in SECTION_KINDS: + return nid + for a in reversed(doc.ancestors(nid)): + if doc.nodes[a]["kind"] in SECTION_KINDS: + return a + return doc.root + + +def target_set(doc, anchored, digit_rule="all"): + """The target set as `[[node, reason], …]` in document order, with `reason` the first that applies of + `anchored`, `equation` and `digits`. + + `anchored` lists the live nodes that extracts, dataset values, confirmed claim links, draft quotations or + pinned citations anchor; a container among them contributes its leaves. Every equation and equation line + is a target. Under `all`, so is every sentence or cell with a digit in a text or math run; under + `anchored-sections`, only those in a section that holds an anchored node; under `none`, none. + """ + if digit_rule not in DIGIT_RULES: + raise ValueError("digit rule must be one of %s" % ", ".join(DIGIT_RULES)) + anchored = set(anchored) + unknown = sorted(a for a in anchored if a not in doc.nodes) + if unknown: + raise ValueError("anchored nodes are not live: %s" % ", ".join(unknown)) + reasons = {} + for nid in sorted(anchored): + if doc.nodes[nid]["kind"] in LEAVES: + reasons[nid] = "anchored" + else: + for leaf in doc.leaves(nid): + reasons.setdefault(leaf, "anchored") + for nid in doc.preorder(): + if doc.nodes[nid]["kind"] in MATH_KINDS: + reasons.setdefault(nid, "equation") + if digit_rule != "none": + sections = {enclosing_section(doc, a) for a in anchored} + for nid in doc.preorder(): + n = doc.nodes[nid] + if n["kind"] in DIGIT_KINDS and has_digit_run(n): + if digit_rule == "all" or enclosing_section(doc, nid) in sections: + reasons.setdefault(nid, "digits") + return [[nid, reasons[nid]] for nid in doc.preorder() if nid in reasons] diff --git a/tools/oracle/text.py b/tools/oracle/text.py new file mode 100644 index 0000000..1c3ef85 --- /dev/null +++ b/tools/oracle/text.py @@ -0,0 +1,105 @@ +"""Text normalization, IDs, keys and small shared helpers. + +Non-normative; see spec/01-conventions.md and spec/04-identity.md. +""" + +import hashlib +import re +import unicodedata + +from . import jcs + +# Unicode White_Space=yes (PropList.txt), the set every rule in the spec means by "whitespace". +WHITE_SPACE = frozenset( + [0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x20, 0x85, 0xA0, 0x1680] + + list(range(0x2000, 0x200B)) + + [0x2028, 0x2029, 0x202F, 0x205F, 0x3000] +) + + +def is_ws(ch): + return ord(ch) in WHITE_SPACE + + +def collapse_ws(s): + out = [] + prev_ws = False + for ch in s: + if is_ws(ch): + if not prev_ws: + out.append(" ") + prev_ws = True + else: + out.append(ch) + prev_ws = False + return "".join(out) + + +def trim(s): + a, b = 0, len(s) + while a < b and is_ws(s[a]): + a += 1 + while b > a and is_ws(s[b - 1]): + b -= 1 + return s[a:b] + + +def remove_ws(s): + return "".join(ch for ch in s if not is_ws(ch)) + + +def norm(s): + """The hashing normalization: NFC, whitespace runs to one U+0020, trimmed.""" + return trim(collapse_ws(unicodedata.normalize("NFC", s))) + + +def sha256_hex(b): + if isinstance(b, str): + b = b.encode("utf-8") + return hashlib.sha256(b).hexdigest() + + +def is_digit(ch): + return unicodedata.category(ch) == "Nd" + + +def digits(s): + return "".join(ch for ch in s if is_digit(ch)) + + +# Crockford base32, lowercase. +ALPHABET = "0123456789abcdefghjkmnpqrstvwxyz" +LETTERS = "abcdefghjkmnpqrstvwxyz" +assert len(ALPHABET) == 32 and len(LETTERS) == 22 + +NODE_ID_RE = re.compile(r"^[a-hjkmnp-tv-z][0-9a-hjkmnp-tv-z]{6}$") +RECORD_ID_RE = re.compile(r"^[a-hjkmnp-tv-z][0-9a-hjkmnp-tv-z]{6,25}$") +KEY_RE = re.compile(r"^[a-z0-9][a-z0-9._-]*$") +HASH_RE = re.compile(r"^[0-9a-f]{64}$") + + +def letter_first(digest, length): + """Encode a digest as a letter-first Crockford base32 ID of `length` characters.""" + first = LETTERS[digest[0] % 22] + bits = int.from_bytes(digest[1:], "big") + nbits = 8 * (len(digest) - 1) + out = [first] + for i in range(length - 1): + shift = nbits - 5 * (i + 1) + out.append(ALPHABET[(bits >> shift) & 31]) + return "".join(out) + + +def mint_node_id(doc_key, kind, text_hash, occurrence): + d = hashlib.sha256(jcs.dumpb(["scienting-id/1", doc_key, kind, text_hash, occurrence])).digest() + return letter_first(d, 7) + + +def record_id(record_hash_hex): + """A record's ID: 13 letter-first base32 characters of its record hash.""" + return letter_first(bytes.fromhex(record_hash_hex), 13) + + +def cp_to_utf16(s, offset): + """Convert a code point offset in `s` to a UTF-16 code unit offset.""" + return sum(2 if ord(ch) > 0xFFFF else 1 for ch in s[:offset]) diff --git a/tools/oracle/tree.py b/tools/oracle/tree.py new file mode 100644 index 0000000..f4a1fa8 --- /dev/null +++ b/tools/oracle/tree.py @@ -0,0 +1,249 @@ +"""Establishing a document from a nested tree, minting IDs, and structural checks. + +Non-normative; see spec/03-nodes.md and spec/04-identity.md. +""" + +import copy + +from . import hashing +from .model import ( + AT_MOST_ONCE, CHILDREN, KINDS, LABELLED, LEAVES, MATH_KINDS, NONEMPTY, RUN_KINDS, RUNS, STYLES, + VERBATIM_KINDS, Doc, +) +from .text import NODE_ID_RE, mint_node_id, norm + + +class Refusal(Exception): + """A rule was broken. `rule` names it, as spec/22-conformance.md lists.""" + + def __init__(self, rule, message, node=None): + super().__init__("%s: %s" % (rule, message)) + self.rule = rule + self.message = message + self.node = node + + +def canonical_runs(runs): + """Drop empty text runs and join adjacent text runs with the same styles.""" + out = [] + for r in runs: + r = dict(r) + if r["kind"] == "text": + if "style" in r: + r["style"] = sorted(set(r["style"])) + if not r["style"]: + del r["style"] + if r["text"] == "": + continue + if out and out[-1]["kind"] == "text" and out[-1].get("style") == r.get("style"): + out[-1] = dict(out[-1], text=out[-1]["text"] + r["text"]) + continue + out.append(r) + return out + + +def _flatten(spec, parent, ord_, out): + node = {k: v for k, v in spec.items() if k != "children"} + node["_parent"] = parent + node["ord"] = ord_ + out.append(node) + idx = len(out) - 1 + for i, c in enumerate(spec.get("children", [])): + _flatten(c, idx, i, out) + return idx + + +def mint_ids(doc_key, flat, used, preassigned=None): + """Mint IDs for flat nodes in document order. `flat` nodes carry `_parent` indices.""" + preassigned = preassigned or {} + tmp = [] + for i, n in enumerate(flat): + m = {k: v for k, v in n.items() if not k.startswith("_")} + m["id"] = "tmp%d" % i + m["parent"] = None if n["_parent"] is None else "tmp%d" % n["_parent"] + tmp.append(m) + _resolve_child_refs(tmp) + doc = Doc(doc_key, tmp) + seen = {} + ids = {} + for i, n in enumerate(tmp): + if i in preassigned: + ids[n["id"]] = preassigned[i] + used.add(preassigned[i]) + continue + th = hashing.PLAIN(norm(hashing.plain(doc, n["id"]))) + k = (n["kind"], th) + occ = seen.get(k, 0) + seen[k] = occ + 1 + nid = mint_node_id(doc_key, n["kind"], th, occ) + while nid in used: + occ += 1 + nid = mint_node_id(doc_key, n["kind"], th, occ) + used.add(nid) + ids[n["id"]] = nid + result = [] + for n in tmp: + m = dict(n) + m["id"] = ids[n["id"]] + m["parent"] = None if n["parent"] is None else ids[n["parent"]] + if m["kind"] in RUN_KINDS: + m["content"] = [ + dict(r, node=ids[r["node"]]) if r["kind"] == "equation" else r for r in m["content"] + ] + result.append(m) + return result + + +def _resolve_child_refs(tmp): + """Equation runs in a spec name their child by position: {"kind": "equation", "child": i}.""" + by_parent = {} + for n in tmp: + by_parent.setdefault(n["parent"], []).append(n) + for n in tmp: + if n["kind"] in RUN_KINDS: + kids = sorted(by_parent.get(n["id"], []), key=lambda c: c["ord"]) + runs = [] + for r in n["content"]: + if r["kind"] == "equation" and "child" in r: + r = {"kind": "equation", "node": kids[r["child"]]["id"]} + runs.append(r) + n["content"] = canonical_runs(runs) + + +def establish(doc_key, spec, used=None): + """Turn a nested spec into flat node records with minted IDs, in document order.""" + flat = [] + _flatten(copy.deepcopy(spec), None, 0, flat) + return mint_ids(doc_key, flat, used if used is not None else set()) + + +def with_values(doc_key, nodes, hasher=hashing.PLAIN): + doc = Doc(doc_key, nodes) + vals = hashing.compute(doc, hasher) + out = [] + for nid in doc.preorder(): + n = dict(doc.nodes[nid]) + for k in ("text_hash", "match_hash", "align_key", "own_hash", "node_hash"): + n.pop(k, None) + n.update(vals[nid]) + out.append(n) + return out + + +def validate_structure(doc): + """Check the tree against the child rules of spec/03-nodes.md.""" + if doc.root is None: + raise Refusal("doc.tree.root", "no root node") + roots = [x for x in doc.order if doc.nodes[x].get("parent") is None] + if len(roots) != 1 or doc.nodes[doc.root]["kind"] != "document": + raise Refusal("doc.tree.root", "exactly one root, of kind document, is required") + if doc.preorder() != doc.order: + raise Refusal("doc.tree.order", "nodes are not in document order") + seen = set() + for nid in doc.order: + n = doc.nodes[nid] + if nid in seen: + raise Refusal("doc.id.unique", "duplicate ID " + nid, nid) + seen.add(nid) + if not NODE_ID_RE.match(nid): + raise Refusal("doc.id.form", "ID is not seven letter-first base32 characters", nid) + k = n["kind"] + if k not in KINDS: + raise Refusal("doc.node.kind", "unknown kind " + k, nid) + kids = doc.kids(nid) + if [doc.nodes[c]["ord"] for c in kids] != list(range(len(kids))): + raise Refusal("doc.tree.ord", "child order is not 0..n-1", nid) + counts = {} + for i, c in enumerate(kids): + ck = doc.nodes[c]["kind"] + if ck not in CHILDREN[k]: + raise Refusal("doc.tree.child", "%s may not contain %s" % (k, ck), c) + counts[ck] = counts.get(ck, 0) + 1 + if ck == "heading" and i != 0: + raise Refusal("doc.tree.child", "a heading must be a section's first child", c) + for ck, cnt in counts.items(): + if cnt > 1 and (ck in AT_MOST_ONCE or ck == "heading"): + raise Refusal("doc.tree.child", "%s may occur once in %s" % (ck, k), nid) + if k == "table" and counts.get("table-body", 0) != 1: + raise Refusal("doc.tree.child", "a table has exactly one table-body", nid) + if k in NONEMPTY and not kids: + raise Refusal("doc.tree.empty", "%s must have children" % k, nid) + if n.get("label") is not None and k not in LABELLED: + raise Refusal("doc.node.label", "%s does not take a label" % k, nid) + _check_content(doc, nid) + + +def _check_content(doc, nid): + n = doc.nodes[nid] + k = n["kind"] + if n.get("withheld"): + if "content" in n: + raise Refusal("doc.policy.withheld", "a withheld node keeps no content", nid) + return + if k in RUN_KINDS: + runs = n.get("content") + if not isinstance(runs, list) or not runs: + raise Refusal("doc.node.content", "runs are required", nid) + if canonical_runs(runs) != runs: + raise Refusal("doc.run.canonical", "runs are not in canonical form", nid) + eq_refs = [] + for r in runs: + if r["kind"] not in RUNS: + raise Refusal("doc.run.kind", "unknown run kind " + r["kind"], nid) + st = r.get("style", []) + if any(s not in STYLES for s in st) or ("sup" in st and "sub" in st): + raise Refusal("doc.run.style", "bad style set", nid) + if r["kind"] == "equation": + eq_refs.append(r["node"]) + kids = doc.kids(nid) + if eq_refs != kids: + raise Refusal("doc.run.equation", "equation runs must name the sentence's children in order", nid) + elif k in MATH_KINDS: + c = n.get("content") or {} + if c.get("notation") not in ("tex", "typst") or not isinstance(c.get("source"), str): + raise Refusal("doc.node.content", "equations need a notation and a source", nid) + elif k in VERBATIM_KINDS: + c = n.get("content") or {} + if not isinstance(c.get("text"), str): + raise Refusal("doc.node.content", "verbatim nodes need text", nid) + elif "content" in n: + raise Refusal("doc.node.content", "%s holds no content" % k, nid) + if k == "section" and not isinstance(n.get("attrs", {}).get("level"), int): + raise Refusal("doc.node.attrs", "a section needs a level", nid) + + +def validate_references(doc): + """Check the references between nodes: cite, fnmark and xref runs, and the header paths of tables and rows + (spec/03 §3.4–3.5, spec/09 §9.2). Raises Refusal.""" + kinds = {nid: doc.nodes[nid]["kind"] for nid in doc.order} + for nid in doc.order: + n = doc.nodes[nid] + if n["kind"] in RUN_KINDS and not n.get("withheld"): + for r in n.get("content") or []: + if r["kind"] == "cite": + for ref in r["refs"]: + if kinds.get(ref) != "reference": + raise Refusal("doc.run.cite", "cite names %s, which is not a reference" % ref, nid) + if r["kind"] == "fnmark" and r.get("footnote") is not None and kinds.get(r["footnote"]) != "footnote": + raise Refusal("doc.run.fnmark", "fnmark names a node that is not a footnote", nid) + if r["kind"] == "xref" and r.get("target") and "#" not in r["target"] and r["target"] not in kinds: + raise Refusal("doc.run.xref", "xref names no node in this document", nid) + if n["kind"] == "table": + for col in n.get("attrs", {}).get("columns", []): + for h in col["header_path"]: + if kinds.get(h) != "cell": + raise Refusal("doc.table.header", "a header path names %s, not a cell" % h, nid) + if n["kind"] == "row": + for h in n.get("attrs", {}).get("header_path", []): + if kinds.get(h) != "cell": + raise Refusal("doc.table.header", "a row header path names %s, not a cell" % h, nid) + if n["kind"] == "cell": + col = n.get("attrs", {}).get("column") + table = next((a for a in reversed(doc.ancestors(nid)) if kinds[a] == "table"), None) + columns = {c["id"] for c in doc.nodes[table].get("attrs", {}).get("columns", [])} if table else set() + if col is not None and columns and col not in columns: + raise Refusal("doc.table.header", "a cell names a column its table does not define", nid) + + +def leaf_kinds(): + return LEAVES diff --git a/tools/oracle/vrf.py b/tools/oracle/vrf.py new file mode 100644 index 0000000..da9d695 --- /dev/null +++ b/tools/oracle/vrf.py @@ -0,0 +1,179 @@ +"""ECVRF-EDWARDS25519-SHA512-TAI, the verifiable random function of RFC 9381 over the Ed25519 curve. + +Non-normative; see spec/08-fidelity.md §8.8. +A VRF proof has exactly one valid value for a key and an input, where an Ed25519 signature has many, so a +spot-check seed that is a VRF proof cannot be ground by the checker. The arithmetic is the reference form of +RFC 8032 in plain Python, slow and clear; the keys are ordinary Ed25519 keys, so the roster needs nothing new. +The RFC's Example 16 is pinned in vectors/vrf.jsonl. +""" + +import hashlib + +SUITE = b"\x03" +COFACTOR = 8 +P = 2**255 - 19 +Q = 2**252 + 27742317777372353535851937790883648493 +D = (-121665 * pow(121666, P - 2, P)) % P +_I = pow(2, (P - 1) // 4, P) + + +def _inv(x): + return pow(x, P - 2, P) + + +def _recover_x(y, sign): + if y >= P: + return None + x2 = (y * y - 1) * _inv(D * y * y + 1) % P + if x2 == 0: + return None if sign else 0 + x = pow(x2, (P + 3) // 8, P) + if (x * x - x2) % P != 0: + x = x * _I % P + if (x * x - x2) % P != 0: + return None + if (x & 1) != sign: + x = P - x + return x + + +# Points in extended homogeneous coordinates (X, Y, Z, T) with x = X/Z, y = Y/Z, x*y = T/Z (RFC 8032 §5.1.4). +G_Y = 4 * _inv(5) % P +G_X = _recover_x(G_Y, 0) +G = (G_X, G_Y, 1, G_X * G_Y % P) +IDENTITY = (0, 1, 1, 0) + + +def _add(a, b): + x1, y1, z1, t1 = a + x2, y2, z2, t2 = b + A = (y1 - x1) * (y2 - x2) % P + B = (y1 + x1) * (y2 + x2) % P + C = t1 * 2 * D * t2 % P + Dd = z1 * 2 * z2 % P + E, F, Gg, H = B - A, Dd - C, Dd + C, B + A + return (E * F % P, Gg * H % P, F * Gg % P, E * H % P) + + +def _mul(s, pt): + out, acc = IDENTITY, pt + while s > 0: + if s & 1: + out = _add(out, acc) + acc = _add(acc, acc) + s >>= 1 + return out + + +def _equal(a, b): + return (a[0] * b[2] - b[0] * a[2]) % P == 0 and (a[1] * b[2] - b[1] * a[2]) % P == 0 + + +def point_to_string(pt): + z = _inv(pt[2]) + x, y = pt[0] * z % P, pt[1] * z % P + return (y | ((x & 1) << 255)).to_bytes(32, "little") + + +def string_to_point(s): + """The point a 32-byte string encodes, or None when it is not a valid encoding (RFC 8032 §5.1.3).""" + if len(s) != 32: + return None + y = int.from_bytes(s, "little") + sign = y >> 255 + y &= (1 << 255) - 1 + x = _recover_x(y, sign) + if x is None: + return None + return (x, y, 1, x * y % P) + + +def _secret_scalar(sk): + h = hashlib.sha512(sk).digest() + a = bytearray(h[:32]) + a[0] &= 248 + a[31] &= 63 + a[31] |= 64 + return int.from_bytes(a, "little"), h[32:] + + +def public_key(sk): + x, _ = _secret_scalar(sk) + return point_to_string(_mul(x, G)) + + +def encode_to_curve(pk_string, alpha): + """ECVRF_encode_to_curve_try_and_increment (RFC 9381 §5.4.1.1).""" + for ctr in range(256): + h = hashlib.sha512(SUITE + b"\x01" + pk_string + alpha + bytes([ctr]) + b"\x00").digest() + pt = string_to_point(h[:32]) + if pt is not None: + pt = _mul(COFACTOR, pt) + if not _equal(pt, IDENTITY): + return pt + raise ValueError("no curve point found") + + +def _challenge(points): + """ECVRF_challenge_generation (RFC 9381 §5.4.3): the first 16 octets of the hash, as a little-endian integer.""" + s = SUITE + b"\x02" + b"".join(point_to_string(p) for p in points) + b"\x00" + return int.from_bytes(hashlib.sha512(s).digest()[:16], "little") + + +def prove(sk, alpha): + """ECVRF_prove: the 80-byte proof Gamma || c || s for the 32-byte Ed25519 secret `sk` and input `alpha`.""" + x, suffix = _secret_scalar(sk) + y = _mul(x, G) + h = encode_to_curve(point_to_string(y), alpha) + h_string = point_to_string(h) + gamma = _mul(x, h) + k = int.from_bytes(hashlib.sha512(suffix + h_string).digest(), "little") % Q + c = _challenge([y, h, gamma, _mul(k, G), _mul(k, h)]) + s = (k + c * x) % Q + return point_to_string(gamma) + c.to_bytes(16, "little") + s.to_bytes(32, "little") + + +def decode_proof(pi): + if len(pi) != 80: + return None + gamma = string_to_point(pi[:32]) + if gamma is None: + return None + c = int.from_bytes(pi[32:48], "little") + s = int.from_bytes(pi[48:80], "little") + if s >= Q: + return None + return gamma, c, s + + +def proof_to_hash(pi): + """ECVRF_proof_to_hash: the 64-byte output beta of a proof (RFC 9381 §5.2), or None for a malformed proof.""" + d = decode_proof(pi) + if d is None: + return None + gamma = d[0] + return hashlib.sha512(SUITE + b"\x03" + point_to_string(_mul(COFACTOR, gamma)) + b"\x00").digest() + + +def _neg(pt): + return ((-pt[0]) % P, pt[1], pt[2], (-pt[3]) % P) + + +def verify(pk_string, alpha, pi): + """ECVRF_verify: beta when `pi` is the proof for `alpha` under the public key, else None. Never raises.""" + try: + y = string_to_point(pk_string) + if y is None or _equal(_mul(COFACTOR, y), IDENTITY): + return None + d = decode_proof(pi) + if d is None: + return None + gamma, c, s = d + h = encode_to_curve(pk_string, alpha) + u = _add(_mul(s, G), _neg(_mul(c, y))) + v = _add(_mul(s, h), _neg(_mul(c, gamma))) + if _challenge([y, h, gamma, u, v]) != c: + return None + return proof_to_hash(pi) + except (ValueError, TypeError): + return None diff --git a/tools/site.py b/tools/site.py new file mode 100644 index 0000000..841ee00 --- /dev/null +++ b/tools/site.py @@ -0,0 +1,636 @@ +#!/usr/bin/env python3 +"""Build the paperspec website: every file of the repository, and its Markdown rendered as documentation. + +Non-normative tooling, standard library only. +Usage: python3 tools/site.py [--out DIR] [--base-url URL] +The output folder (default `_site/`) mirrors the repository, so each file is served at its own path: a schema at +the URL in its `$id`, the JSON-LD context at the address layers name, and every vector, fixture and table as it is. +Beside each Markdown file the builder writes a page, `README.md` becoming its folder's `index.html` and `X.md` +becoming `X.html`; a folder with no README gets a listing. It also writes the vocabulary page that the context's +term IRIs point at, a search index, a 404 page and one stylesheet. Links between Markdown files are rewritten to +the pages, so the site works from any host, and `deploy/Caddyfile` serves it with the right media types. +""" + +import argparse +import html +import json +import os +import re +import shutil +import sys + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SKIP_DIRS = {".git", ".pixi", ".claude", "_site", "__pycache__", "node_modules", ".gitea", ".vscode", ".idea"} +SKIP_FILES = {".DS_Store"} +NAME = "paperspec" +REPO = "https://git.scient.ing/infra/paperspec" +BASE_URL = "https://paperspec.scient.ing" + + +def version(): + with open(os.path.join(ROOT, "CHANGELOG.md"), encoding="utf-8") as f: + m = re.search(r"^## (\S+)", f.read(), re.M) + return m.group(1) if m else "unreleased" + + +# --- inline Markdown ------------------------------------------------------------------------------------------ + +def slug(heading): + """A heading's anchor as GitHub, Gitea and Forgejo compute it (the same rule tools/check.py uses).""" + t = re.sub(r"[`*]", "", heading.strip().lower()) + t = "".join(ch for ch in t if ch.isalnum() or ch in " -_") + return t.replace(" ", "-") + + +def rewrite_href(target): + """A Markdown link target as a site link: `X.md` becomes `X.html`, `README.md` its folder.""" + if re.match(r"^[a-z][a-z0-9+.-]*:", target) or target.startswith(("/", "#")): + return target + path, hashmark, frag = target.partition("#") + if path.endswith(".md"): + head, base = os.path.split(path) + if base == "README.md": + path = (head + "/") if head else "./" + else: + path = path[:-3] + ".html" + return path + hashmark + frag + + +def inline(text): + """Render the inline Markdown the documentation uses: code spans, autolinks, links, strong and emphasis.""" + stash = [] + + def keep(s): + stash.append(s) + return "\x00%d\x00" % (len(stash) - 1) + + text = re.sub(r"(`+)(.+?)\1", lambda m: keep("<code>%s</code>" % html.escape(m.group(2).strip(), quote=False)), text) + text = re.sub(r"<(https?://[^>\s]+)>", + lambda m: keep('<a href="%s">%s</a>' % (html.escape(m.group(1)), html.escape(m.group(1)))), text) + text = html.escape(text, quote=False) + + def link(m): + label, target = m.group(1), html.unescape(m.group(2)) + return keep('<a href="%s">%s</a>' % (html.escape(rewrite_href(target)), emphasis(label))) + + text = re.sub(r"\[([^\]]+)\]\(([^)\s]+)\)", link, text) + text = emphasis(text) + while "\x00" in text: + text = re.sub(r"\x00(\d+)\x00", lambda m: stash[int(m.group(1))], text) + return text + + +def emphasis(text): + text = re.sub(r"\*\*(?=\S)(.+?)(?<=\S)\*\*", r"<strong>\1</strong>", text) + return re.sub(r"(?<![\w*])\*(?=[^\s*])(.+?)(?<=[^\s*])\*(?![\w*])", r"<em>\1</em>", text) + + +def plain(text): + """Inline Markdown as plain text, for titles and the search index.""" + text = re.sub(r"`+([^`]+)`+", r"\1", text) + text = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", text) + return re.sub(r"\*+([^*]+)\*+", r"\1", text).strip() + + +# --- block Markdown ------------------------------------------------------------------------------------------- + +LIST_RE = re.compile(r"^( *)([-*]|\d+\.)(?: +(.*)|$)") +FENCE_RE = re.compile(r"^( *)```(\S*)\s*$") +HEADING_RE = re.compile(r"^(#{1,6}) +(.*?)\s*#*\s*$") + + +def indent_of(line): + return len(line) - len(line.lstrip(" ")) + + +def starts_block(line): + return bool(HEADING_RE.match(line) or FENCE_RE.match(line) or LIST_RE.match(line) or line.startswith("|") + or line.startswith(": ")) + + +class Renderer: + def __init__(self): + self.ids = {} + self.headings = [] + self.title = None + + def heading_id(self, text): + base = slug(text) + n = self.ids.get(base, 0) + self.ids[base] = n + 1 + return base if n == 0 else "%s-%d" % (base, n) + + def blocks(self, lines): + out, i = [], 0 + while i < len(lines): + line = lines[i] + if not line.strip(): + i += 1 + continue + m = FENCE_RE.match(line) + if m: + pad = len(m.group(1)) + body, i = [], i + 1 + while i < len(lines) and not re.match(r"^ *```\s*$", lines[i]): + body.append(lines[i][pad:] if lines[i][:pad].strip() == "" else lines[i]) + i += 1 + lang = m.group(2) + cls = ' class="language-%s"' % html.escape(lang) if lang else "" + out.append("<pre><code%s>%s</code></pre>" % (cls, html.escape("\n".join(body), quote=False))) + i += 1 + continue + m = HEADING_RE.match(line) + if m: + level, text = len(m.group(1)), m.group(2) + hid = self.heading_id(text) + if level == 1 and self.title is None: + self.title = plain(text) + self.headings.append({"level": level, "text": plain(text), "id": hid}) + anchor = '<a class="anchor" href="#%s" aria-label="Link to this section">#</a>' % hid if level > 1 else "" + out.append('<h%d id="%s">%s%s</h%d>' % (level, hid, inline(text), anchor, level)) + i += 1 + continue + if line.startswith("|"): + rows = [] + while i < len(lines) and lines[i].startswith("|"): + rows.append(lines[i]) + i += 1 + out.append(self.table(rows)) + continue + if i + 1 < len(lines) and lines[i + 1].startswith(": ") and not starts_block(line): + items = [] + while i + 1 < len(lines) and lines[i].strip() and lines[i + 1].startswith(": "): + term, i = lines[i], i + 1 + desc = [] + while i < len(lines) and lines[i].strip(): + desc.append(lines[i][4:] if lines[i].startswith(": ") else lines[i].strip()) + i += 1 + items.append("<dt>%s</dt><dd>%s</dd>" % (inline(term.strip()), inline("\n".join(desc)))) + while i < len(lines) and not lines[i].strip(): + i += 1 + out.append("<dl>%s</dl>" % "".join(items)) + continue + m = LIST_RE.match(line) + if m: + html_list, i = self.list(lines, i) + out.append(html_list) + continue + para = [] + while i < len(lines) and lines[i].strip() and not (para and starts_block(lines[i])): + para.append(lines[i].strip()) + i += 1 + out.append("<p>%s</p>" % inline("\n".join(para))) + return "\n".join(out) + + def list(self, lines, i): + first = LIST_RE.match(lines[i]) + base = len(first.group(1)) + ordered = first.group(2)[0].isdigit() + items = [] + while i < len(lines): + m = LIST_RE.match(lines[i]) + if not m or len(m.group(1)) != base or m.group(2)[0].isdigit() != ordered: + break + content_col = base + len(m.group(2)) + 1 + body = [m.group(3) or ""] if m.group(3) else [] + i += 1 + while i < len(lines): + ln = lines[i] + if not ln.strip(): + nxt = next((x for x in lines[i + 1:] if x.strip()), None) + if nxt is not None and indent_of(nxt) >= content_col - 1 and indent_of(nxt) > base: + body.append("") + i += 1 + continue + break + if indent_of(ln) > base: + body.append(ln[min(indent_of(ln), content_col):]) + i += 1 + continue + if body and not starts_block(ln): + body.append(ln.strip()) # a lazy continuation of the item's paragraph + i += 1 + continue + break + inner = self.blocks(body) + if inner.startswith("<p>") and inner.count("<p>") == 1 and inner.endswith("</p>"): + inner = inner[3:-4] + items.append("<li>%s</li>" % inner) + while i < len(lines) and not lines[i].strip(): + nxt = next((x for x in lines[i:] if x.strip()), None) + m2 = LIST_RE.match(nxt) if nxt else None + if m2 and len(m2.group(1)) == base: + i += 1 + else: + break + tag = "ol" if ordered else "ul" + start = int(first.group(2)[:-1]) if ordered else 1 + attr = ' start="%d"' % start if start != 1 else "" + return "<%s%s>%s</%s>" % (tag, attr, "".join(items), tag), i + + def table(self, rows): + def cells(row): + row = row.strip() + if row.startswith("|"): + row = row[1:] + if row.endswith("|"): + row = row[:-1] + out, cur, tick = [], "", False + for ch in row: + if ch == "`": + tick = not tick + if ch == "|" and not tick: + out.append(cur.strip()) + cur = "" + else: + cur += ch + out.append(cur.strip()) + return out + + head = cells(rows[0]) + body = [cells(r) for r in rows[2:]] if len(rows) > 1 and re.match(r"^\|[\s:|-]+\|?$", rows[1]) else \ + [cells(r) for r in rows[1:]] + thead = "<tr>%s</tr>" % "".join("<th>%s</th>" % inline(c) for c in head) + tbody = "".join("<tr>%s</tr>" % "".join("<td>%s</td>" % inline(c) for c in r) for r in body) + return '<div class="table-wrap"><table><thead>%s</thead><tbody>%s</tbody></table></div>' % (thead, tbody) + + +def render_markdown(text): + r = Renderer() + body = r.blocks(text.split("\n")) + return body, r + + +# --- pages ---------------------------------------------------------------------------------------------------- + +NAV = [ + ("Start", [("README.md", "Home"), ("docs/overview.md", "Overview"), ("docs/concepts.md", "Concepts"), + ("docs/walkthrough.md", "Walkthrough"), ("docs/examples.md", "Examples"), ("docs/faq.md", "FAQ"), + ("docs/glossary.md", "Glossary"), ("docs/rationale.md", "Rationale"), + ("docs/comparison.md", "Comparison")]), + ("Specification", None), + ("Reference", [("schemas/README.md", "Schemas"), ("registry/README.md", "Registry"), + ("vectors/README.md", "Test vectors"), ("fixtures/README.md", "Fixtures"), + ("sql/README.md", "SQL"), ("context/README.md", "JSON-LD context"), ("vocab", "Vocabulary"), + ("tools/README.md", "Tools")]), + ("Project", [("CHANGELOG.md", "Changelog"), ("CONTRIBUTING.md", "Contributing"), ("LICENSE.md", "Licenses")]), +] + + +def page_path(md_rel): + """Where the page for a Markdown file goes, relative to the site root.""" + head, base = os.path.split(md_rel) + if base == "README.md": + return os.path.join(head, "index.html") + return md_rel[:-3] + ".html" + + +def page_url(md_rel): + if md_rel == "vocab": + return "vocab/" + p = page_path(md_rel) + return p[:-len("index.html")] if p.endswith("index.html") else p + + +def rel(from_page, to_url): + """A relative link from the page at `from_page` to the site URL `to_url` (both relative to the root).""" + start = os.path.dirname(from_page) or "." + target = to_url or "./" + trailing = target.endswith("/") + r = os.path.relpath(target.rstrip("/") or ".", start) + if r == ".": + return "./" + return r + "/" if trailing else r + + +def spec_nav(): + out = [] + for f in sorted(os.listdir(os.path.join(ROOT, "spec"))): + if re.match(r"\d\d-.*\.md$", f): + with open(os.path.join(ROOT, "spec", f), encoding="utf-8") as fh: + m = re.search(r"^# (.*)$", fh.read(), re.M) + out.append(("spec/" + f, plain(m.group(1)) if m else f)) + return [("spec/README.md", "Contents")] + out + + +def nav_html(page, here_md): + """The sidebar for the page at `page`, marking `here_md` (a Markdown path, `vocab`, or None) as current.""" + current = page_url(here_md) if here_md else None + parts = [] + for group, entries in NAV: + entries = spec_nav() if entries is None else entries + links = [] + for md, label in entries: + url = page_url(md) + cur = ' aria-current="page"' if current == url else "" + links.append('<li><a href="%s"%s>%s</a></li>' % (html.escape(rel(page, url)), cur, html.escape(label))) + parts.append('<section><h2>%s</h2><ul>%s</ul></section>' % (html.escape(group), "".join(links))) + return "".join(parts) + + +CSS = """\ +:root { + --bg: #fbfaf8; --surface: #f2efea; --text: #1d1c1a; --muted: #5d5a54; --rule: #dedad3; + --accent: #1f5f8b; --accent-soft: #e3edf4; --code-bg: #efece6; --mark: #fff3c4; + --sans: system-ui, -apple-system, "Segoe UI", Roboto, sans-serif; + --mono: ui-monospace, "SF Mono", Menlo, Consolas, monospace; +} +@media (prefers-color-scheme: dark) { + :root:not([data-theme="light"]) { + --bg: #161615; --surface: #1f1f1d; --text: #e9e6e0; --muted: #a6a29a; --rule: #34332f; + --accent: #7fb6dc; --accent-soft: #1d2c36; --code-bg: #24231f; --mark: #4a3f12; + } +} +:root[data-theme="dark"] { + --bg: #161615; --surface: #1f1f1d; --text: #e9e6e0; --muted: #a6a29a; --rule: #34332f; + --accent: #7fb6dc; --accent-soft: #1d2c36; --code-bg: #24231f; --mark: #4a3f12; +} +* { box-sizing: border-box; } +html { -webkit-text-size-adjust: 100%; } +body { margin: 0; background: var(--bg); color: var(--text); font: 16px/1.6 var(--sans); } +a { color: var(--accent); text-underline-offset: 2px; } +header.top { position: sticky; top: 0; z-index: 2; display: flex; gap: 1rem; align-items: center; + padding: .6rem 1rem; background: var(--bg); border-bottom: 1px solid var(--rule); } +header.top .brand { font-weight: 650; color: var(--text); text-decoration: none; letter-spacing: -.01em; } +header.top .version { color: var(--muted); font-size: .85rem; } +header.top .spacer { flex: 1; } +header.top input { width: min(18rem, 40vw); padding: .35rem .6rem; border: 1px solid var(--rule); border-radius: 6px; + background: var(--surface); color: var(--text); font: inherit; font-size: .9rem; } +.results { position: absolute; right: 1rem; top: 3rem; width: min(28rem, calc(100vw - 2rem)); max-height: 60vh; + overflow: auto; background: var(--bg); border: 1px solid var(--rule); border-radius: 8px; padding: .3rem 0; + box-shadow: 0 8px 24px rgba(0,0,0,.12); } +.results a { display: block; padding: .35rem .8rem; text-decoration: none; color: var(--text); } +.results a:hover, .results a:focus { background: var(--accent-soft); outline: none; } +.results small { display: block; color: var(--muted); } +.layout { display: grid; grid-template-columns: 16rem minmax(0, 1fr); max-width: 76rem; margin: 0 auto; } +nav.side { position: sticky; top: 3.2rem; align-self: start; max-height: calc(100vh - 3.2rem); overflow: auto; + padding: 1.2rem 1rem 2rem; font-size: .9rem; border-right: 1px solid var(--rule); } +nav.side h2 { font-size: .72rem; text-transform: uppercase; letter-spacing: .08em; color: var(--muted); + margin: 1.2rem 0 .3rem; } +nav.side ul { list-style: none; margin: 0; padding: 0; } +nav.side a { display: block; padding: .15rem .4rem; border-radius: 4px; text-decoration: none; color: var(--text); } +nav.side a:hover { background: var(--surface); } +nav.side a[aria-current] { background: var(--accent-soft); color: var(--accent); font-weight: 600; } +main { padding: 1.5rem 2.5rem 4rem; min-width: 0; } +article { max-width: 48rem; } +h1, h2, h3, h4 { line-height: 1.25; letter-spacing: -.01em; } +h1 { font-size: 2rem; margin: .2rem 0 1rem; } +h2 { font-size: 1.4rem; margin: 2.2rem 0 .6rem; padding-top: .4rem; border-top: 1px solid var(--rule); } +h3 { font-size: 1.12rem; margin: 1.6rem 0 .4rem; } +h1, h2, h3, h4, h5, h6 { scroll-margin-top: 4rem; } +.anchor { margin-left: .4rem; color: var(--muted); text-decoration: none; opacity: 0; font-weight: 400; } +h2:hover .anchor, h3:hover .anchor, h4:hover .anchor, .anchor:focus { opacity: 1; } +:target { background: var(--mark); } +p, ul, ol, dl, pre, .table-wrap { margin: 0 0 1rem; } +li + li { margin-top: .25rem; } +code { font: .88em/1.4 var(--mono); background: var(--code-bg); padding: .1em .3em; border-radius: 4px; + overflow-wrap: anywhere; } +pre { background: var(--code-bg); padding: .8rem 1rem; border-radius: 6px; overflow-x: auto; } +pre code { background: none; padding: 0; font-size: .85rem; overflow-wrap: normal; } +.table-wrap { overflow-x: auto; } +table { border-collapse: collapse; font-size: .92rem; min-width: 60%; } +th, td { text-align: left; vertical-align: top; padding: .45rem .7rem; border-bottom: 1px solid var(--rule); } +th { background: var(--surface); font-weight: 600; } +dt { font-weight: 650; margin-top: 1rem; } +dd { margin: .2rem 0 0 1.2rem; } +.meta { margin-top: 3rem; padding-top: 1rem; border-top: 1px solid var(--rule); color: var(--muted); font-size: .85rem; } +.meta a { color: var(--muted); } +.listing td:last-child { text-align: right; color: var(--muted); font-variant-numeric: tabular-nums; } +.crumbs { color: var(--muted); font-size: .9rem; margin-bottom: .5rem; } +details.menu { display: none; } +@media (max-width: 52rem) { + .layout { display: block; } + nav.side { display: none; } + details.menu { display: block; padding: .5rem 16px; border-bottom: 1px solid var(--rule); } + details.menu summary { cursor: pointer; font-weight: 600; } + details.menu nav.side { display: block; position: static; max-height: none; border: 0; padding: .5rem 0; } + main { padding: 1rem 16px 3rem; } + header.top { padding: .5rem 16px; } + header.top input { width: 9rem; } + h1 { font-size: 1.6rem; } +} +""" + +SEARCH_JS = """\ +(function () { + var input = document.getElementById("search"), box = document.getElementById("results"), index = null; + if (!input) return; + var root = document.documentElement.getAttribute("data-root") || ""; + function load(cb) { + if (index) return cb(); + fetch(root + "search.json").then(function (r) { return r.json(); }).then(function (j) { index = j; cb(); }) + .catch(function () { index = []; cb(); }); + } + function esc(s) { return s.replace(/[&<>"]/g, function (c) { return {"&": "&", "<": "<", ">": ">", '"': """}[c]; }); } + function show() { + var q = input.value.trim().toLowerCase(); + if (q.length < 2) { box.hidden = true; return; } + var hits = []; + index.forEach(function (p) { + if (p.title.toLowerCase().indexOf(q) >= 0) hits.push({url: p.url, text: p.title, page: ""}); + p.headings.forEach(function (h) { + if (h.text.toLowerCase().indexOf(q) >= 0) hits.push({url: p.url + "#" + h.id, text: h.text, page: p.title}); + }); + }); + hits = hits.slice(0, 25); + box.innerHTML = hits.length ? hits.map(function (h) { + return '<a href="' + root + esc(h.url) + '">' + esc(h.text) + (h.page ? "<small>" + esc(h.page) + "</small>" : "") + "</a>"; + }).join("") : '<a aria-disabled="true">No matching heading</a>'; + box.hidden = false; + } + input.addEventListener("input", function () { load(show); }); + input.addEventListener("keydown", function (e) { + if (e.key === "Escape") { box.hidden = true; input.blur(); } + if (e.key === "Enter") { var a = box.querySelector("a[href]"); if (a) location.href = a.href; } + }); + document.addEventListener("click", function (e) { if (!box.contains(e.target) && e.target !== input) box.hidden = true; }); + document.addEventListener("keydown", function (e) { + if (e.key === "/" && document.activeElement !== input) { e.preventDefault(); input.focus(); } + }); +})(); +""" + + +def layout(page, here_md, title, body, source=None, base_url=BASE_URL): + """A full page. `page` is the output path relative to the site root; `here_md` the Markdown it renders.""" + root = rel(page, "") + root = "" if root == "./" else root + ("" if root.endswith("/") else "/") + if page == "404.html": + root = "/" # served at whatever path was missing, so its links start from the site root + canonical = base_url.rstrip("/") + "/" + (page[:-len("index.html")] if page.endswith("index.html") else page) + nav = nav_html(page, here_md) + meta = "" + if source: + meta = ('<p class="meta">Source: <a href="%s/src/branch/main/%s">%s</a> · <a href="%s%s">raw</a></p>' + % (REPO, html.escape(source), html.escape(source), root, html.escape(source))) + return """<!doctype html> +<html lang="en" data-root="{root}"> +<head> +<meta charset="utf-8"> +<meta name="viewport" content="width=device-width, initial-scale=1"> +<title>{title} + + + + + +
+{name}{version} + + + +
+
Menu
+
+ +
+{body} +{meta} +
+
+ + + +""".format(root=root, root_or_dot=root or "./", title=html.escape(title), canonical=html.escape(canonical), + name=NAME, version=html.escape(version()), nav=nav, body=body, meta=meta) + + +def size_text(n): + for unit in ("B", "KB", "MB"): + if n < 1024 or unit == "MB": + return ("%d %s" % (n, unit)) if unit == "B" else ("%.1f %s" % (n, unit)) + n /= 1024.0 + + +def listing(out, d, base_url): + """A page listing the files and folders of `d` (relative to the site root), for folders with no README.""" + full = os.path.join(out, d) + entries = sorted(os.listdir(full)) + rows = [] + if d: + rows.append('../') + for e in entries: + if e == "index.html": + continue + p = os.path.join(full, e) + if os.path.isdir(p): + rows.append('%s/' % (html.escape(e), html.escape(e))) + elif not (e.endswith(".html") and os.path.exists(os.path.join(full, e[:-5] + ".md"))): + rows.append('%s%s' + % (html.escape(e), html.escape(e), size_text(os.path.getsize(p)))) + crumbs = ['%s' % (rel(os.path.join(d, "index.html"), ""), NAME)] + parts = d.split("/") if d else [] + for i, part in enumerate(parts): + crumbs.append('%s' % (rel(os.path.join(d, "index.html"), "/".join(parts[:i + 1]) + "/"), + html.escape(part))) + body = ('

%s

%s/

' + '%s
NameSize
' + % (" / ".join(crumbs), html.escape(d or NAME), "".join(rows))) + with open(os.path.join(full, "index.html"), "w", encoding="utf-8") as f: + f.write(layout(os.path.join(d, "index.html"), None, (d or NAME) + "/ · " + NAME, body, base_url=base_url)) + + +def vocab_page(out, base_url): + """The page the context's term IRIs point at: one entry per IRI fragment of the `sct` namespace.""" + with open(os.path.join(ROOT, "context", "scienting-1.jsonld"), encoding="utf-8") as f: + ctx = json.load(f)["@context"] + prefix = ctx.get("sct", "") + rows = [] + for term, value in ctx.items(): + if term.startswith("@") or term in ("sct", "as"): + continue + iri = value if isinstance(value, str) else value.get("@id", "") + if not iri.startswith("sct:"): + continue + local = iri[4:] + notes = [] + if isinstance(value, dict): + if value.get("@type") == "@id": + notes.append("an IRI") + if value.get("@type") == "@json": + notes.append("a JSON literal, kept intact") + if value.get("@container"): + notes.append("a set") + rows.append('%s%s%s%s' + % (html.escape(local), html.escape(term), html.escape(prefix), html.escape(local), + html.escape(", ".join(notes)))) + body = ("

Vocabulary

The terms the JSON-LD context " + 'scienting-1.jsonld adds to the W3C Web ' + 'Annotation vocabulary. Each term's IRI is a fragment of this page; ' + 'the context's guide explains how layers use them.

' + '
%s' + "
TermIRIValue
" % "".join(rows)) + os.makedirs(os.path.join(out, "vocab"), exist_ok=True) + with open(os.path.join(out, "vocab", "index.html"), "w", encoding="utf-8") as f: + f.write(layout("vocab/index.html", "vocab", "Vocabulary · " + NAME, body, base_url=base_url)) + return {"title": "Vocabulary", "url": "vocab/", "headings": []} + + +def build(out, base_url=BASE_URL): + if os.path.exists(out): + shutil.rmtree(out) + os.makedirs(out) + mds = [] + for d, dirs, fs in os.walk(ROOT): + dirs[:] = sorted(x for x in dirs if x not in SKIP_DIRS and not (x.startswith(".") and x != ".well-known")) + reld = os.path.relpath(d, ROOT) + reld = "" if reld == "." else reld.replace(os.sep, "/") + if os.path.abspath(d) == os.path.abspath(out): + continue + for f in sorted(fs): + if f in SKIP_FILES or f.startswith("."): + continue + src = os.path.join(d, f) + if os.path.islink(src): + continue + dst = os.path.join(out, reld, f) + os.makedirs(os.path.dirname(dst), exist_ok=True) + shutil.copyfile(src, dst) + if f.endswith(".md"): + mds.append((reld + "/" + f) if reld else f) + index = [] + for md in mds: + with open(os.path.join(ROOT, md), encoding="utf-8") as f: + body, r = render_markdown(f.read()) + title = r.title or md + page = page_path(md) + full_title = (title + " · " + NAME) if md != "README.md" else NAME + " · " + title + with open(os.path.join(out, page), "w", encoding="utf-8") as f: + f.write(layout(page, md, full_title, body, source=md, base_url=base_url)) + index.append({"title": title, "url": page_url(md), + "headings": [{"text": h["text"], "id": h["id"]} for h in r.headings if h["level"] > 1]}) + index.append(vocab_page(out, base_url)) + for d, dirs, fs in os.walk(out): + reld = os.path.relpath(d, out) + reld = "" if reld == "." else reld.replace(os.sep, "/") + if "index.html" not in fs: + listing(out, reld, base_url) + with open(os.path.join(out, "404.html"), "w", encoding="utf-8") as f: + f.write(layout("404.html", None, "Not found · " + NAME, + '

Not found

No file of the specification has this address. ' + 'Start from the home page or search the headings above.

' + % html.escape(base_url.rstrip("/")), base_url=base_url)) + with open(os.path.join(out, "site.css"), "w", encoding="utf-8") as f: + f.write(CSS) + with open(os.path.join(out, "search.js"), "w", encoding="utf-8") as f: + f.write(SEARCH_JS) + with open(os.path.join(out, "search.json"), "w", encoding="utf-8") as f: + json.dump(sorted(index, key=lambda p: p["url"]), f, ensure_ascii=False, separators=(",", ":")) + f.write("\n") + return len(mds) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--out", default=os.path.join(ROOT, "_site"), help="the output folder (default _site/)") + ap.add_argument("--base-url", default=BASE_URL, help="the site's address, for canonical links (default %s)" % BASE_URL) + a = ap.parse_args() + out = os.path.abspath(a.out) + if out == ROOT or ROOT.startswith(out + os.sep): + ap.error("the output folder must not be the repository or contain it") + n = build(out, a.base_url) + print("Wrote %d pages and every file of the repository to %s" % (n, out)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/stress.py b/tools/stress.py new file mode 100644 index 0000000..e46c40d --- /dev/null +++ b/tools/stress.py @@ -0,0 +1,732 @@ +#!/usr/bin/env python3 +"""Stress the reference implementation with random inputs. Exit status 0 means every property held. + +Non-normative tooling; it needs Python 3.14 and the packages pixi installs (`jsonschema`, `cryptography`). +Usage: python3 tools/stress.py [--seed N] [--rounds N] +The vectors pin values; this program checks the invariants the chapters state, over inputs drawn from a seeded +random generator, so a failure is reproducible from the seed it prints. Canonical JSON round-trips and sorts keys +by UTF-16 units; normalization is idempotent; notation folding never raises; random corrections keep the five +values equal to a full recomputation, never reuse an ID and apply deterministically; anchors made before those +corrections always resolve to one of the five outcomes; Merkle roots agree with a second implementation; +spot-check samples do not depend on input order; error bounds are the least six-decimal value with the stated tail; +differently stamped pages share a fingerprint; the zip reader and the package validator refuse or accept mutated +input without raising and never write outside their folders; rights combine the same in any order; and +spot-check chains hundreds deep settle without recursion. +""" + +import argparse +import copy +import json +import os +import random +import shutil +import sys +import tempfile +import time +import zipfile +from fractions import Fraction +from math import comb + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) + +from tools.oracle import align as AL # noqa: E402 +from tools.oracle import anchors as AN # noqa: E402 +from tools.oracle import corrections as CO # noqa: E402 +from tools.oracle import crypto, fingerprint as FP, hashing, jcs, receipts as RC, rights as RI, targets as TG # noqa: E402 +from tools.oracle import keys as K, log as LG, mldsa, vrf # noqa: E402 +from tools.oracle import package as PK # noqa: E402 +from tools.oracle.model import LABELLED, LEAVES, Doc # noqa: E402 +from tools.oracle.notation import fold_math # noqa: E402 +from tools.oracle.text import NODE_ID_RE, RECORD_ID_RE, letter_first, norm, record_id, sha256_hex, is_ws # noqa: E402 +from tools.oracle.tree import Refusal, validate_structure # noqa: E402 +from tools.build import ROSTER_BY_ID, SCHEMAS, secret, sentence_doc, tiny_package # noqa: E402 +from tools.check import HALE, STORE, TINY_KEY, reference_values # noqa: E402 + +FAILS = [] +COUNTS = {} +RULES = {r["id"] for r in json.load(open(os.path.join(ROOT, "registry", "rules.json"), encoding="utf-8"))["rules"]} + + +def ok(section, cond, msg): + COUNTS[section] = COUNTS.get(section, 0) + 1 + if not cond and len(FAILS) < 200: + FAILS.append("%s: %s" % (section, msg)) + + +TEXT_POOL = list("abcdefghijklmnopqrstuvwxyzABCDEFXYZ0123456789 .,;:-()[]") + [ + "é", "Å", "fi", "µ", "−", "–", " ", "\t", "\n", "
", "€", + "\U0001d49c", "\U0001f600", "é", "H₂O", "10³", "cm⁻¹", " "] +KEY_POOL = list("abcxyz") + ["€", "\U0001f600", "fi", "A", "é", "", "a b", "\u0001"] + + +def rand_text(rng, lo=0, hi=12, pool=TEXT_POOL): + return "".join(rng.choice(pool) for _ in range(rng.randint(lo, hi))) + + +def rand_json(rng, depth=0): + r = rng.random() + if depth > 3 or r < 0.4: + choice = rng.randint(0, 6) + if choice == 0: + return None + if choice == 1: + return rng.random() < 0.5 + if choice == 2: + return rng.randint(-2**53, 2**53) + if choice == 3: + return rng.uniform(-1, 1) * 10.0 ** rng.randint(-12, 24) + if choice == 4: + return rng.choice([0, -0.0, 1e21, 1e-7, 0.000001, 123456789.125, 5e-324, 1.7976931348623157e308]) + return rand_text(rng) + if r < 0.7: + return [rand_json(rng, depth + 1) for _ in range(rng.randint(0, 4))] + return {rand_text(rng, 0, 3, KEY_POOL): rand_json(rng, depth + 1) for _ in range(rng.randint(0, 4))} + + +def _as_doubles(text): + """Parse JSON as an ECMAScript engine does, which is how canonical JSON is read (vectors/README.md).""" + return json.loads(text, parse_int=lambda x: int(x) if abs(int(x)) <= 2**53 else float(x)) + + +def stress_canonical_json(rng, rounds): + for _ in range(rounds): + v = rand_json(rng) + try: + s = jcs.dumps(v) + except ValueError as e: + ok("canonical-json", False, "dumps refused a plain value: %r (%s)" % (v, e)) + continue + ok("canonical-json", jcs.dumpb(v) == s.encode("utf-8"), "dumpb is dumps encoded") + back = _as_doubles(s) + ok("canonical-json", back == v, "round trip changed the value: %r" % (v,)) + ok("canonical-json", jcs.dumps(back) == s, "canonical form is not idempotent for %r" % (v,)) + pairs = _as_doubles(s) if not s.startswith(("{", "[")) else json.loads(s, object_pairs_hook=lambda kv: kv) + + def ordered(x): + if isinstance(x, list) and x and isinstance(x[0], tuple): + keys = [k for k, _ in x] + return keys == sorted(keys, key=lambda k: k.encode("utf-16-be")) and all(ordered(v) for _, v in x) + if isinstance(x, list): + return all(ordered(y) for y in x) + return True + ok("canonical-json", ordered(pairs), "object members are not sorted by UTF-16 code units in %s" % s[:80]) + + +def stress_normalization(rng, rounds): + for _ in range(rounds): + s = rand_text(rng, 0, 30) + n = norm(s) + ok("normalize", norm(n) == n, "normalization is not idempotent for %r" % s) + ok("normalize", not n or (not is_ws(n[0]) and not is_ws(n[-1])), "normalized text keeps edge whitespace") + ok("normalize", " " not in n and all(ch == " " or not is_ws(ch) for ch in n), + "normalized text keeps a whitespace run or a non-space whitespace: %r" % n) + + +TEX_TOKENS = list("xyzKrHO0123456789+-=()'") + ["_", "^", "{", "}", "\\alpha", "\\mathrm{H}", "\\frac{a}{b}", " ", + "\\,", "\\cdot", "H_2O", "^{14}", "{}", "\\left(", "\\right)", "\\\\", + "µ", "−", "''", "_{", "^{"] +TYPST_TOKENS = list("xyzKr0123456789+-=()'") + ["_", "^", "(", ")", "alpha", "upright(H)", "\"cm\"", " ", "dot", + "plus.minus", "<=", "/", "epsilon.alt", "_(", "^(", "\"a\\b\""] + + +def stress_notation(rng, rounds): + for _ in range(rounds): + for notation, pool in (("tex", TEX_TOKENS), ("typst", TYPST_TOKENS)): + s = "".join(rng.choice(pool) for _ in range(rng.randint(0, 8))) + try: + a = fold_math(notation, s) + b = fold_math(notation, s) + except Exception as e: # noqa: BLE001 + ok("notation", False, "fold_math raised %r on %s %r" % (e, notation, s)) + continue + ok("notation", a == b and (a is None or isinstance(a, str)), "fold_math is not deterministic on %r" % s) + ok("notation", a is None or not any(is_ws(ch) for ch in a), "a canonical form holds whitespace: %r" % a) + ok("notation", a is None or "$" not in a, "a canonical form holds a dollar sign: %r" % a) + + +def _leaf_ids(doc, kinds): + return [nid for nid in doc.preorder() if doc.nodes[nid]["kind"] in kinds] + + +def random_correction(rng, st, doc): + """One random proposal against `st`, or None.""" + sentences = _leaf_ids(doc, {"sentence"}) + cells = _leaf_ids(doc, {"cell"}) + op = rng.choice(["replace", "replace", "split", "merge", "move", "relabel", "set", "delete", "reclassify", + "insert-text", "insert-node"]) + evidence = [{"kind": "crop", "sha256": "0" * 64}] if rng.random() < 0.8 else [{"kind": "person", "by": "ana"}] + c = {"op": op.split("-")[0], "evidence": evidence} + if op == "replace" and (sentences or cells): + nid = rng.choice(sentences + cells) + text = hashing.plain(doc, nid) + if not text: + return None + i = rng.randrange(len(text)) + j = min(len(text), i + rng.randint(1, 6)) + c.update(target=nid, find=text[i:j], replace=rand_text(rng, 0, 8)) + elif op == "split" and sentences: + nid = rng.choice(sentences) + c.update(target=nid, offset=rng.randint(0, max(1, len(hashing.plain(doc, nid))))) + elif op == "merge" and len(sentences) > 1: + nid = rng.choice(sentences) + sibs = doc.kids(doc.nodes[nid]["parent"]) + k = sibs.index(nid) + other = sibs[k + 1] if k + 1 < len(sibs) else rng.choice(sentences) + c.update(target=nid, **{"with": other}, joiner=rng.choice([" ", ""])) + elif op == "move" and sentences: + paragraphs = _leaf_ids(doc, {"paragraph"}) + if not paragraphs: + return None + c.update(target=rng.choice(sentences), parent=rng.choice(paragraphs), place_after=None) + elif op == "relabel": + labelled = [nid for nid in doc.preorder() if doc.nodes[nid]["kind"] in LABELLED] + if not labelled: + return None + c.update(target=rng.choice(labelled), label=rng.choice([None, "2", "Table 7", "a", rand_text(rng, 1, 5)])) + elif op == "set" and cells: + c.update(target=rng.choice(cells), attr=rng.choice(["colspan", "rowspan", "role", "column", "level"]), + value=rng.choice([1, 2, 3, "2", -1, "data", "row-header", "boss", None])) + elif op == "delete" and len(doc.order) > 8: + c.update(target=rng.choice(doc.order[1:])) + elif op == "reclassify": + c.update(target=rng.choice(doc.order), kind=rng.choice(["footnote", "quote", "paragraph", "caption", "section"])) + elif op == "insert-text" and sentences: + nid = rng.choice(sentences) + c.update(target=nid, run=rng.randint(0, len(doc.nodes[nid]["content"])), offset=rng.randint(0, 20), + text=rand_text(rng, 1, 6)) + elif op == "insert-node": + paragraphs = _leaf_ids(doc, {"paragraph"}) + if not paragraphs: + return None + parent = rng.choice(paragraphs) + kids = doc.kids(parent) + node = {"kind": "sentence", "content": [{"kind": "text", "text": rand_text(rng, 1, 20) or "x"}]} + if rng.random() < 0.2: + node["foo"] = 1 + c.update(parent=parent, place_after=rng.choice(kids + [None]), node=node) + else: + return None + return c + + +def stress_corrections(rng, rounds): + """A random walk of corrections from the tiny document; returns the states it passed through.""" + tiny = tiny_package(None, receipts=False) + st = tiny["final"] + base = {"author": {"key": "ana"}, "reason": "Stress.", "at": "2026-10-11T12:00:00Z"} + seen = {n["id"] for n in st.nodes} + history, accepted = [st], 0 + for _ in range(rounds): + doc = st.doc() + c = random_correction(rng, st, doc) + if c is None: + continue + c = dict(base, **c) + try: + st1, rec = CO.apply(st, copy.deepcopy(c), store=SCHEMAS) + except Refusal as e: + ok("corrections", e.rule in RULES, "a refusal names the unregistered rule %s" % e.rule) + continue + except Exception as e: # noqa: BLE001 + ok("corrections", False, "apply raised %r on %s" % (e, json.dumps(c)[:160])) + continue + accepted += 1 + try: + validate_structure(st1.doc()) + ok("corrections", st1.index().values == reference_values(st1.doc()), + "incremental values differ from the full computation after %s" % c["op"]) + except Refusal as e: + ok("corrections", False, "an accepted correction left the tree invalid: %s" % e.rule) + new_ids = {n["id"] for n in st1.nodes} - {n["id"] for n in st.nodes} + ok("corrections", not (new_ids & seen), "an ID was reused: %s" % sorted(new_ids & seen)) + seen |= new_ids + table = {e["id"] for e in st1.ids} + ok("corrections", set(rec.get("retired", [])) <= table, "a retired ID is not in the ID table") + ok("corrections", st1.ids == sorted(st1.ids, key=CO.id_order), "ids are not in the order of spec/04 §4.4") + ok("corrections", rec["id"] == record_id(sha256_hex(jcs.dumpb({k: v for k, v in rec.items() if k != "id"}))), + "the record's id does not derive from its content") + ok("corrections", rec["index"] == st.rev + 1 == st1.rev, "the record's index is not the next revision") + st2, rec2 = CO.apply(st, copy.deepcopy(c), store=SCHEMAS) + ok("corrections", rec2 == rec and st2.nodes == st1.nodes and st2.ids == st1.ids, "apply is not deterministic") + try: + _, rec3 = CO.gate(st, copy.deepcopy(c), SCHEMAS) + ok("corrections", rec3 == rec, "the gate records something else than apply") + except Refusal as e: + ok("corrections", e.rule == "correction.schema", "the gate refused an applied correction with %s" % e.rule) + st = st1 + history.append(st) + ok("corrections", accepted >= rounds // 10, "too few random corrections were accepted: %d of %d" % (accepted, rounds)) + return history + + +def stress_anchors(rng, history, rounds): + st0, final = history[0], history[-1] + doc0, vals0 = st0.doc(), hashing.compute(st0.doc()) + outcomes = {"current", "changed-below", "moved", "reattached", "stale"} + seen_outcomes = set() + anchors = [] + for nid in doc0.preorder(): + anchors.append(AN.make_anchor(doc0, vals0, nid, 0, store="smithlab")) + if doc0.nodes[nid]["kind"] in LEAVES: + t = hashing.plain(doc0, nid) + if len(t) > 2: + i = rng.randrange(len(t) - 1) + j = rng.randint(i + 1, len(t)) + anchors.append(AN.make_anchor(doc0, vals0, nid, 0, span=[i, j], store="smithlab")) + for _ in range(rounds // 4): + a = copy.deepcopy(rng.choice(anchors)) + a["node"] = letter_first(os.urandom(32), 7) if rng.random() < 0.5 else a["node"] + a.pop("kind", None) + anchors.append(a) + for state in (final, history[len(history) // 2], history[min(1, len(history) - 1)]): + values = hashing.compute(state.doc()) + for a in anchors: + try: + r = AN.resolve(state, a) + except Exception as e: # noqa: BLE001 + ok("anchors", False, "resolve raised %r on %s" % (e, json.dumps(a)[:120])) + continue + ok("anchors", r["result"] in outcomes, "unknown outcome %s" % r.get("result")) + seen_outcomes.add(r["result"]) + if r["result"] != "stale" or r.get("node"): + ok("anchors", r.get("node") in state.doc().nodes, "a non-stale result names no live node") + if r["result"] == "current": + ok("anchors", values[r["node"]]["node_hash"] == a["node_hash"], "current with another node_hash") + if "span" in r: + t = hashing.plain(state.doc(), r["node"]) + ok("anchors", 0 <= r["span"][0] <= r["span"][1] <= len(t), "a span outside the node's text") + ok("anchors", {"current", "stale"} <= seen_outcomes, "the random walk produced only %s" % sorted(seen_outcomes)) + + +def _merkle_reference(items): + leaves = [sha256_hex(b"\x00" + jcs.dumpb(x)) for x in items] + + def mth(ls): + if len(ls) == 1: + return ls[0] + k = 1 + while 2 * k < len(ls): + k *= 2 + return sha256_hex(b"\x01" + bytes.fromhex(mth(ls[:k])) + bytes.fromhex(mth(ls[k:]))) + return mth(leaves) + + +def stress_merkle(rng, rounds): + """Receipt items are lists, `["node", id, hash]` or `[kind, record id, hash]` (spec/08 §8.5).""" + for _ in range(rounds): + n = rng.randint(1, 40) + items = [[rng.choice(["node", "annotation", "digitization", "checklist"]), letter_first(os.urandom(32), 7), + sha256_hex(os.urandom(8))] for _ in range(n)] + if rng.random() < 0.3: + items.append([rand_text(rng), [rand_text(rng)], rng.randint(0, 2**53)]) + ok("merkle", crypto.merkle_root(items) == _merkle_reference(items), "Merkle roots differ for %d items" % len(items)) + + +def stress_spotcheck(rng, rounds): + for _ in range(rounds): + n = rng.randint(1, 60) + ids = sorted({letter_first(os.urandom(32), 7) for _ in range(n)}) + receipt = {"format": "scienting-receipt/1", "nodes": [[i, sha256_hex(i)] for i in ids]} + seed = RC.make_seed(receipt, os.urandom(32)) + rate, minimum = rng.choice(["0.05", "0.1", "0.5", "1"]), rng.randint(1, 15) + sample = RC.sample(receipt, seed, rate, minimum) + try: + RC.sample(receipt, os.urandom(64), rate, minimum) + ok("spot-check", False, "random bytes were accepted as a seed") + except ValueError: + pass + size = RC.sample_size(len(ids), rate, minimum) + ok("spot-check", len(sample) == size == min(len(ids), max(minimum, -(-Fraction(rate) * len(ids) // 1))), + "sample size %d is not the rule's for n=%d" % (len(sample), len(ids))) + ok("spot-check", sample == sorted(sample) and set(sample) <= set(ids), "sample is not sorted or not from the receipt") + shuffled = {"nodes": rng.sample(receipt["nodes"], len(receipt["nodes"]))} + ok("spot-check", RC.sample(shuffled, seed, rate, minimum) == sample, "the sample depends on input order") + + +def stress_bounds(rng, rounds): + alpha = Fraction(5, 100) + for _ in range(rounds // 4): + n = rng.randint(1, 120) + k = rng.randint(0, n) + b = RC.bound_string(k, n) + ok("bounds", len(b.split(".")[1]) == 6, "bound has not six decimals: %s" % b) + if k >= n: + ok("bounds", b == "1.000000", "k = n must give 1") + continue + p = Fraction(b) + + def tail(q): + return sum(comb(n, i) * q**i * (1 - q)**(n - i) for i in range(k + 1)) + ok("bounds", tail(p) <= alpha, "the tail at the bound is above alpha for k=%d n=%d" % (k, n)) + if p > 0: + ok("bounds", tail(p - Fraction(1, 10**6)) > alpha, "the bound is not the least six-decimal value for k=%d n=%d" % (k, n)) + if k + 1 < n: + ok("bounds", Fraction(RC.bound_string(k + 1, n)) >= p, "the bound is not monotone in k") + + +def stress_fingerprints(rng, rounds): + rules = json.load(open(os.path.join(ROOT, "registry", "stamps", "scienting-stamps-1.json"), encoding="utf-8"))["rules"] + for _ in range(rounds): + lines = [{"text": rand_text(rng, 1, 40)} for _ in range(rng.randint(1, 8))] + page = {"size": [612.0, 792.0], "lines": lines, "images": [sha256_hex(rand_text(rng)) for _ in range(rng.randint(0, 2))]} + stamped = copy.deepcopy(page) + stamped["lines"].insert(rng.randint(0, len(lines)), + {"text": "Downloaded via EXAMPLE UNIV on %s" % rand_text(rng, 1, 10, list("abcXYZ0123456789-:"))}) + fp, removed = FP.fingerprint(page, rules) + fp2, removed2 = FP.fingerprint(stamped, rules) + ok("fingerprints", fp == fp2 and len(removed2) == len(removed) + 1, "a stamped download has another fingerprint") + digit_lines = [i for i, l in enumerate(lines) if any(ch in "0123456789" for ch in l["text"])] + if digit_lines: + changed = copy.deepcopy(page) + i = rng.choice(digit_lines) + t = changed["lines"][i]["text"] + j = next(j for j, ch in enumerate(t) if ch in "0123456789") + changed["lines"][i]["text"] = t[:j] + str((int(t[j]) + 1) % 10) + t[j + 1:] + ok("fingerprints", FP.fingerprint(changed, rules)[0] != fp, "a changed digit kept the fingerprint") + spaced = copy.deepcopy(page) + spaced["lines"] = [{"text": " ".join(l["text"])} for l in spaced["lines"]] + ok("fingerprints", FP.fingerprint(spaced, rules)[0] == fp, "inserted spaces changed the fingerprint") + + +def stress_zip(rng, rounds): + src = os.path.join(ROOT, "fixtures", "packages", TINY_KEY + PK.ZIP_EXTENSION) + with open(src, "rb") as f: + good = f.read() + accepted = 0 + with tempfile.TemporaryDirectory() as tmp: + for i in range(rounds): + data = bytearray(good) + mode = rng.random() + if mode < 0.6: + for _ in range(rng.randint(1, 3)): + data[rng.randrange(len(data))] = rng.randrange(256) + elif mode < 0.8: + data = data[:rng.randrange(len(data))] + else: + data = data[:rng.randrange(len(data))] + os.urandom(rng.randint(1, 64)) + data[rng.randrange(len(data)):] + path = os.path.join(tmp, "z%d.zip" % i) + with open(path, "wb") as f: + f.write(data) + try: + rep = PK.validate_zip(path, STORE) + except Exception as e: # noqa: BLE001 + ok("zip", False, "validate_zip raised %r on mutation %d" % (e, i)) + continue + accepted += bool(rep) + dest = os.path.join(tmp, "d%d" % i) + try: + PK.read_zip(path, dest) + except Refusal as e: + ok("zip", e.rule in RULES, "read_zip names the unregistered rule %s" % e.rule) + except Exception as e: # noqa: BLE001 + ok("zip", False, "read_zip raised %r on mutation %d" % (e, i)) + stray = [f for f in os.listdir(tmp) if not (f.startswith("z") and f.endswith(".zip")) and not f.startswith("d")] + ok("zip", not stray, "the zip reader wrote outside its folder: %s" % stray) + ok("zip", accepted <= rounds // 20 + 1, "%d of %d mutated archives were accepted" % (accepted, rounds)) + + +def stress_packages(rng, rounds): + src = os.path.join(ROOT, "fixtures", "packages", TINY_KEY) + files = sorted(os.path.relpath(os.path.join(d, f), src) for d, _, fs in os.walk(src) for f in fs) + with tempfile.TemporaryDirectory() as tmp: + for i in range(rounds): + pkg = os.path.join(tmp, "p%d" % i) + shutil.copytree(src, pkg) + mode = rng.choice(["flip", "flip", "truncate", "delete", "add", "rename-case", "symlink"]) + target = rng.choice(files) + full = os.path.join(pkg, target) + if mode == "flip": + data = bytearray(open(full, "rb").read()) + if data: + k = rng.randrange(len(data)) + data[k] = (data[k] + rng.randrange(1, 256)) % 256 + open(full, "wb").write(data) + elif mode == "truncate": + data = open(full, "rb").read() + open(full, "wb").write(data[:rng.randrange(len(data) + 1)]) + elif mode == "delete": + os.remove(full) + elif mode == "add": + open(os.path.join(pkg, rand_text(rng, 1, 8, list("abcxyz")) + ".txt"), "wb").write(b"stray\n") + elif mode == "rename-case": + if target.lower() == target.upper(): + continue + os.rename(full, os.path.join(pkg, target.upper())) + else: + os.remove(full) + os.symlink(os.path.join(src, target), full) + try: + rep = PK.validate(pkg, STORE) + except Exception as e: # noqa: BLE001 + ok("packages", False, "validate raised %r after %s of %s" % (e, mode, target)) + continue + ok("packages", not rep, "a package mutated by %s of %s validated" % (mode, target)) + ok("packages", all(e["rule"] in RULES for e in rep.errors), "an unregistered rule was reported") + shutil.rmtree(pkg) + + +def stress_rights(rng, rounds): + tri = [True, False, "unknown"] + keys = list(RI.USES) + ["ai/" + p for p in RI.AI_PAIRS] + for _ in range(rounds): + flats = [{k: rng.choice(tri) for k in keys} for _ in range(rng.randint(1, 4))] + combined = RI.combine(flats) + shuffled = RI.combine(rng.sample(flats, len(flats))) + ok("rights", combined == shuffled, "combining depends on order") + for k in keys: + vals = [f[k] for f in flats] + want = False if False in vals else ("unknown" if "unknown" in vals else True) + ok("rights", combined[k] == want, "combining %s gave %r for %r" % (k, combined[k], vals)) + + +def _align_items(rng, prefix, texts): + items = [] + for i, t in enumerate(texts): + runs = [{"kind": "text", "text": t}] + doc, s = sentence_doc(runs) + strict, cx = hashing.strict(doc, s) + items.append({"id": "%s%d" % (prefix, i), "container": "p%d" % (i // 3), "context": "paragraph", "runs": runs, + "strict": strict, "lossy": hashing.lossy_from_strict(strict), "complex": cx}) + return items + + +def stress_alignment(rng, rounds): + for _ in range(rounds // 4): + texts = [rand_text(rng, 5, 60) or "x" for _ in range(rng.randint(1, 12))] + a = _align_items(rng, "a", texts) + try: + same = AL.align(a, _align_items(rng, "b", texts)) + b_texts = [t for t in texts if rng.random() > 0.3] + [rand_text(rng, 5, 30)] + diff = AL.align(a, _align_items(rng, "b", b_texts)) + ok("alignment", AL.align(a, _align_items(rng, "b", b_texts)) == diff, "alignment is not deterministic") + except Exception as e: # noqa: BLE001 + ok("alignment", False, "align raised %r" % (e,)) + continue + ok("alignment", len(same["pairs"]) == len(a) and all(p["agree"] for p in same["pairs"]) + and not [s for s in same["suspects"] if s["kind"] not in ("structure", "join")], + "aligning two equal readings left disagreements or unpaired sentences: %s" % same["suspects"][:3]) + ids_a = {x["id"] for x in a} + ok("alignment", all(p["a"] in ids_a for p in diff["pairs"]), "a pair names an unknown sentence") + paired = {p["a"] for p in diff["pairs"]} + dropped = {s["a"] for s in diff["suspects"] if s["kind"] == "dropped"} + ok("alignment", paired | dropped == ids_a and not (paired & dropped), + "every sentence of a reading is paired or dropped, not both or neither") + + +def stress_addresses(rng, rounds): + for _ in range(rounds): + store = rng.choice([None, "smithlab", "a1", "x.y-z"]) + key = rng.choice([HALE, "k-0000", "a1-aaaa.si", "x-k3f9qd.v2"]) + node = letter_first(os.urandom(32), 7) + pin = sha256_hex(rand_text(rng))[:6] if rng.random() < 0.5 else None + span = sorted([rng.randint(0, 50), rng.randint(0, 50)]) if rng.random() < 0.5 else None + s = AN.format_address(store, key, node, pin, span) + parsed = AN.parse_address(s) + ok("addresses", parsed and AN.format_address(parsed["store"], parsed["key"], parsed["node"], parsed["pin"], + parsed["span"]) == s, "address does not round trip: %s" % s) + junk = rand_text(rng, 0, 20) + try: + AN.parse_address(junk) + except Exception as e: # noqa: BLE001 + ok("addresses", False, "parse_address raised %r on %r" % (e, junk)) + for _ in range(rounds): + d = os.urandom(32) + ok("ids", NODE_ID_RE.match(letter_first(d, 7)) and RECORD_ID_RE.match(record_id(d.hex())), "ID form") + + +def stress_targets(rng, rounds): + st = PK.read_state(os.path.join(ROOT, "fixtures", "packages", HALE)) + doc = st.doc() + for _ in range(rounds // 4): + anchored = rng.sample(doc.order, rng.randint(0, 6)) + sets = {} + for rule in TG.DIGIT_RULES: + got = TG.target_set(doc, anchored, rule) + ids = [n for n, _ in got] + pos = {nid: i for i, nid in enumerate(doc.order)} + ok("targets", ids == sorted(ids, key=pos.get) and len(set(ids)) == len(ids), "targets are not in document order") + ok("targets", all(doc.nodes[n]["kind"] in LEAVES for n in ids), "a target is not a leaf") + sets[rule] = set(ids) + ok("targets", sets["none"] <= sets["anchored-sections"] <= sets["all"], "the digit rules do not nest") + try: + TG.target_set(doc, anchored + ["zzzzzzz"], "all") + ok("targets", False, "an unknown anchored node was accepted") + except ValueError: + ok("targets", True, "") + + +def stress_receipt_chains(rng, rounds): + """Chains of spot-checks hundreds deep settle without recursion, and alternate as spec/08 §8.6 implies.""" + depth = max(50, rounds) + ids = sorted({letter_first(os.urandom(32), 7) for _ in range(20)}) + rec = RC.seal({"format": "scienting-receipt/1", "doc": HALE, "rev": 0, "level": "F3", "scope": "all", + "nodes": [[i, sha256_hex(i)] for i in ids], "evidence": [], "by": "ana", + "at": "2026-10-11T12:00:00Z"}, secret("ana"), "ana") + records = [rec] + for i in range(depth): + orig = records[-1] + seed = RC.make_seed(orig, secret("ben-spot")) + sample = RC.sample(orig, seed) + hashes = dict(orig["nodes"]) + q = RC.seal({"format": "scienting-receipt/1", "doc": HALE, "rev": 0, "level": "F3", + "scope": "qa:" + orig["id"], "nodes": [[n, hashes[n]] for n in sample], "evidence": [], + "by": "ben", "at": "2026-10-11T12:%02d:%02dZ" % (i // 60 % 60, i % 60), + "qa": {"of": orig["id"], "key": "ben-spot", "seed": __import__("base64").b64encode(seed).decode(), + "rate": "0.05", "minimum": 10, "sample": sample, "errors": [{"node": sample[0]}]}}, + secret("ben"), "ben") + records.append(q) + t0 = time.time() + try: + status = RC.statuses(records, ROSTER_BY_ID) + except RecursionError: + ok("receipts", False, "a chain of %d spot-checks overflowed the stack" % depth) + return + ok("receipts", time.time() - t0 < 30, "statuses of a %d-deep chain took too long" % depth) + # The last spot-check nobody checks is valid; each receipt it checks is suspended; and so on, alternating. + want = "valid" + for r in reversed(records): + ok("receipts", status[r["id"]] == want, "receipt at depth %d is %s, not %s" % (records.index(r), status[r["id"]], want)) + want = "suspended" if want == "valid" else "valid" + + +def stress_keys(rng, rounds): + """Keys (spec/02 §2.4): the pattern always holds, the suffix is a prefix of longer ones, lengthening settles a + collision without touching the first key, and a key's hash part depends on the identity alone.""" + words = ["Attention", "On", "the", "Origin", "of", "Species", "H\u2082O", "3D", "Self-consistent", "\u00c9tude", "\u5f20", "don't", "a"] + for _ in range(rounds): + meta = {"title": " ".join(rng.choice(words) for _ in range(rng.randint(0, 6))), + "author": [{"family": rng.choice(["M\u00fcller", "O'Brien", "\u5f20\u4f1f", "", "de la Cruz", "Smith Jr."])}] if rng.random() < 0.8 else [], + "issued": {"date-parts": [[rng.choice([2017, 1859, 99, 20261, None])]]} if rng.random() < 0.8 else {}} + same_as = [rng.choice(["doi:10.1/X.Y", "doi:10.1/x.y", "arxiv:1706.03762", "url:https://e.org/p", "openalex:W1"])] if rng.random() < 0.6 else [] + key = K.work_key(same_as, meta) + ok("keys", K.WORK_KEY_RE.match(key) and K.DOCUMENT_KEY_RE.match(key), "a minted key breaks the pattern: %r" % key) + ident = K.identity(same_as, meta) + ok("keys", K.suffix(ident, 8).startswith(K.suffix(ident)), "a longer suffix does not extend the shorter") + longer = K.unique_work_key(same_as, meta, {key}) + ok("keys", longer != key and longer.startswith(key) and len(longer) == len(key) + 2, "lengthening does not extend the key") + if same_as and same_as[0].lower().startswith("doi:"): + other = dict(meta, title="Another title entirely") + ok("keys", K.suffix_of(K.work_key(same_as, other)) == K.suffix_of(key), "the hash part depends on more than the identity") + for v in ({"type": "version-of-record"}, {"type": "supplement"}, {"type": "preprint", "seq": rng.randint(1, 9)}, {"type": "erratum"}): + dk = K.document_key(key, v) + ok("keys", K.DOCUMENT_KEY_RE.match(dk) and K.split_key(dk)[0] == key, "a document key does not split back") + + +def stress_vrf(rng, rounds): + """The VRF proves and verifies for random keys and inputs, and a single flipped byte never verifies.""" + for _ in range(max(5, rounds // 20)): + sk = os.urandom(32) + pk = vrf.public_key(sk) + alpha = os.urandom(rng.randint(0, 40)) + pi = vrf.prove(sk, alpha) + beta = vrf.verify(pk, alpha, pi) + ok("vrf", beta is not None and beta == vrf.proof_to_hash(pi) and len(pi) == 80 and len(beta) == 64, "a proof does not verify") + ok("vrf", vrf.prove(sk, alpha) == pi, "proving is not deterministic") + bad = bytearray(pi) + bad[rng.randrange(80)] ^= 1 << rng.randrange(8) + ok("vrf", vrf.verify(pk, alpha, bytes(bad)) is None, "a tampered proof verifies") + ok("vrf", vrf.verify(pk, alpha + b"x", pi) is None and vrf.verify(vrf.public_key(os.urandom(32)), alpha, pi) is None, + "a proof verifies for another input or key") + ok("vrf", vrf.verify(pk, alpha, os.urandom(80)) is None and vrf.verify(os.urandom(32), alpha, pi) is None, "garbage verifies") + + +def stress_proofs(rng, rounds): + """RFC 9162 inclusion and consistency proofs verify for every index and prefix of random trees, and fail for a + wrong index, root or size.""" + for _ in range(rounds // 4): + n = rng.randint(1, 40) + items = [["node", letter_first(os.urandom(32), 7), sha256_hex(os.urandom(4))] for _ in range(n)] + root = crypto.history_root(items) + for i in range(n): + path = crypto.inclusion_path(items, i) + leaf = crypto.leaf_hash(items[i]) + ok("proofs", crypto.verify_inclusion(root, leaf, i, n, path), "inclusion fails for %d of %d" % (i, n)) + ok("proofs", n == 1 or not crypto.verify_inclusion(root, leaf, (i + 1) % n, n, path), "inclusion passes for another index") + ok("proofs", not crypto.verify_inclusion(sha256_hex(b"x"), leaf, i, n, path), "inclusion passes for another root") + # A wrong size is not always detectable from a path: RFC 9162 commits to the content through the + # root, and two sizes with one tree shape verify alike. The root is what a verifier must trust. + for m in range(n + 1): + proof = crypto.consistency_proof(items, m) + ok("proofs", crypto.verify_consistency(crypto.history_root(items[:m]), root, m, n, proof), "consistency fails %d to %d" % (m, n)) + if 0 < m < n: + ok("proofs", not crypto.verify_consistency(sha256_hex(b"x"), root, m, n, proof) + and not crypto.verify_consistency(crypto.history_root(items[:m]), sha256_hex(b"y"), m, n, proof), + "a consistency proof passes with a wrong root") + ok("proofs", not crypto.verify_consistency(crypto.history_root(items[:1]), root, 1, n, ["zz"]), "garbage path verifies") + + +def stress_checkpoints(rng, rounds): + """Checkpoint notes parse back to what was signed and reject edits anywhere.""" + for _ in range(rounds // 10): + entries = [[rng.choice(["receipt", "revocation"]), letter_first(os.urandom(32), 13), sha256_hex(os.urandom(4))] + for _ in range(rng.randint(0, 20))] + note = LG.sign_checkpoint("scient.ing/log/t", entries, [("ana", secret("ana")), ("ben", secret("ben"))]) + parsed, verified = LG.verify_checkpoint(note, ROSTER_BY_ID) + ok("checkpoints", parsed and parsed["size"] == len(entries) and parsed["root"] == LG.root(entries) and verified == ["ana", "ben"], + "a checkpoint does not verify") + k = rng.randrange(len(note)) + bad = note[:k] + chr((ord(note[k]) + 1) % 128 or 1) + note[k + 1:] + p2, v2 = LG.verify_checkpoint(bad, ROSTER_BY_ID) + # An edit in the body loses every signer; an edit in one signature line loses that signer. + ok("checkpoints", p2 is None or len(v2) < len(verified), "an edited note still fully verifies: %r" % bad[max(0, k - 5):k + 5]) + + +def stress_mldsa(rng, rounds): + """ML-DSA-65 signs deterministically and verifies; a flipped bit anywhere, another message or another key fails.""" + for _ in range(max(3, rounds // 50)): + seed = os.urandom(32) + pk, sk = mldsa.keygen(seed) + msg = os.urandom(rng.randint(0, 100)) + sig = mldsa.sign(sk, msg) + ok("ml-dsa", len(pk) == mldsa.PK_SIZE and len(sk) == mldsa.SK_SIZE and len(sig) == mldsa.SIG_SIZE, "sizes") + ok("ml-dsa", mldsa.verify(pk, msg, sig) and mldsa.sign(sk, msg) == sig, "a signature does not verify or is not deterministic") + bad = bytearray(sig) + bad[rng.randrange(len(bad))] ^= 1 << rng.randrange(8) + ok("ml-dsa", not mldsa.verify(pk, msg, bytes(bad)), "a tampered signature verifies") + ok("ml-dsa", not mldsa.verify(pk, msg + b"x", sig) and not mldsa.verify(mldsa.keygen(os.urandom(32))[0], msg, sig), + "a signature verifies for another message or key") + ok("ml-dsa", not mldsa.verify(pk, msg, os.urandom(mldsa.SIG_SIZE)) and not mldsa.verify(os.urandom(mldsa.PK_SIZE), msg, sig), + "garbage verifies") + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--seed", type=int, default=20261014, help="the random seed (default 20261014)") + ap.add_argument("--rounds", type=int, default=200, help="cases per property (default 200)") + a = ap.parse_args() + rng = random.Random(a.seed) + print("seed %d, %d rounds" % (a.seed, a.rounds)) + stress_canonical_json(rng, a.rounds) + stress_normalization(rng, a.rounds) + stress_notation(rng, a.rounds) + history = stress_corrections(rng, a.rounds) + stress_anchors(rng, history, a.rounds) + stress_merkle(rng, a.rounds) + stress_spotcheck(rng, a.rounds) + stress_bounds(rng, a.rounds) + stress_fingerprints(rng, a.rounds) + stress_zip(rng, a.rounds) + stress_packages(rng, a.rounds // 4) + stress_rights(rng, a.rounds) + stress_alignment(rng, a.rounds) + stress_addresses(rng, a.rounds) + stress_targets(rng, a.rounds) + stress_receipt_chains(rng, a.rounds) + stress_keys(rng, a.rounds) + stress_vrf(rng, a.rounds) + stress_proofs(rng, a.rounds) + stress_checkpoints(rng, a.rounds) + stress_mldsa(rng, a.rounds) + for k in sorted(COUNTS): + print("%-17s %6d checks" % (k, COUNTS[k])) + if FAILS: + print("\n%d failures (seed %d):" % (len(FAILS), a.seed)) + for f in FAILS[:60]: + print(" " + f) + sys.exit(1) + print("\nEvery property held.") + + +if __name__ == "__main__": + main() diff --git a/tools/validate.py b/tools/validate.py new file mode 100755 index 0000000..abafff3 --- /dev/null +++ b/tools/validate.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Validate packages, or run the correction gate on proposals against a package. + +Non-normative tooling. +Usage: python3 tools/validate.py PACKAGE [PACKAGE ...] + python3 tools/validate.py --correction PROPOSALS PACKAGE +A package is a folder, or a zipped package (`.sct.zip`), which is unpacked under the rules of spec/12 §12.2 +before it is checked. Each is checked as spec/22 §22.2 describes, and each failure printed as +`package: where: rule: message`. +With --correction, PROPOSALS is a JSON file holding one proposed correction, or a JSON Lines file of several +applied in order. Each runs through the gate (spec/11 §11.4) against the package's current state, and the record +it would append to corrections.jsonl is printed, until one is refused. Nothing is written. +Exit status: 0 when everything is valid, 1 when something is refused, 2 for a usage error. +""" + +import argparse +import json +import os +import sys +import tempfile + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) + +from tools.oracle import corrections, jcs, package # noqa: E402 +from tools.oracle.schema import Store # noqa: E402 +from tools.oracle.tree import Refusal # noqa: E402 + + +def read_proposals(path): + with open(path, encoding="utf-8") as f: + text = f.read() + try: + return [json.loads(text)] + except json.JSONDecodeError: + return [json.loads(ln) for ln in text.splitlines() if ln.strip()] + + +def validate_one(path, store, limit): + """The validation report of a package folder or a zipped package.""" + if package.is_zip(path): + return package.validate_zip(path, store, limit=limit) + return package.validate(path, store) + + +def main(): + lines = __doc__.splitlines() + ap = argparse.ArgumentParser(description=lines[0], epilog="\n".join(lines[5:]), + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--correction", metavar="PROPOSALS", + help="a JSON or JSON Lines file of proposed corrections to run through the gate") + ap.add_argument("--max-size", metavar="BYTES", type=int, default=package.ZIP_LIMIT, + help="refuse a zipped package that expands past this many bytes (default %d)" % package.ZIP_LIMIT) + ap.add_argument("packages", nargs="+", metavar="PACKAGE", help="package folders or zipped packages") + a = ap.parse_args() + if a.correction and len(a.packages) != 1: + ap.error("--correction checks proposals against exactly one package") + store, bad = Store(os.path.join(ROOT, "schemas")), False + for p in a.packages: + for e in validate_one(p, store, a.max_size).errors: + print("%s: %s: %s: %s" % (p, e["where"], e["rule"], e["message"]), file=sys.stderr) + bad = True + if bad: + return 1 + if not a.correction: + for p in a.packages: + print("%s: valid" % p) + return 0 + pkg = a.packages[0] + with tempfile.TemporaryDirectory() as tmp: + if package.is_zip(pkg): + pkg = package.read_zip(pkg, tmp, a.max_size) + state = package.read_state(pkg) + if any(n.get("withheld") for n in state.nodes): + print("%s: doc.policy.withheld: a package that withholds its text takes no corrections" % a.packages[0], + file=sys.stderr) + return 1 + for i, proposal in enumerate(read_proposals(a.correction), 1): + try: + state, record = corrections.gate(state, proposal, store) + except Refusal as e: + where = "%s:%d" % (a.correction, i) + (" " + e.node if e.node else "") + print("%s: %s: %s" % (where, e.rule, e.message), file=sys.stderr) + return 1 + print(jcs.dumps(record)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/vectors/README.md b/vectors/README.md index 430fdd2..e5a3262 100644 --- a/vectors/README.md +++ b/vectors/README.md @@ -73,6 +73,7 @@ Some files change the procedure: Compute the field for each listed case and compare across the group. - `fingerprints.jsonl` needs the download-stamp rules in [registry/stamps/scienting-stamps-1.json](../registry/stamps/scienting-stamps-1.json), which each case names in `rules`. - `resolve.jsonl`, `grading.jsonl` and `targets.jsonl` name a package by a path from the repository root. +<<<<<<< HEAD All three use `fixtures/packages/hale2026harmonic-damb`. Several files also mix more than one kind of case, told apart by their fields. @@ -137,6 +138,73 @@ The specification has 32 formats: 19 are `candidate` and 13 are `provisional` in A candidate format changes only with a recorded reason and a new or changed vector ([22.5](../spec/22-conformance.md#225-stability)). A passing run shows one thing: for these inputs, your implementation computes the same values as the reference implementation in `src/paperspec/`. +||||||| parent of 4fa326d (First implementation (#1)) +======= + All three use `fixtures/packages/hale2026harmonic`. + +Several files also mix more than one kind of case, told apart by their fields. +`ids.jsonl` has minting cases and letter-first cases, `keyed.jsonl` and `signatures.jsonl` have a case for each derivation or record, `spotcheck.jsonl` has sample-size cases and one full sample, and `rights.jsonl` has stage checks (a `stage` field), backend checks (a `backend` field) and one combining case (neither). + +The reference checker in [tools/](../tools/README.md) runs all twenty-five files, together with the rest of the repository's checks: + +```sh +python3 tools/check.py --quick +``` + +It needs Python 3.14 and two packages, `jsonschema` and `cryptography`, and has no option to run one file. +It tests the reference implementation in `tools/oracle/` against the vectors. +It does not test yours. + +## What each file tests + +The table below uses a few terms that the chapters define: + +- A code point is one Unicode character; UTF-16 is an encoding that writes some characters as two units, so a JavaScript string offset can differ from a code-point offset ([1.5](../spec/01-conventions.md#15-character-offsets)). +- An HMAC is a hash computed under a secret key, so that someone without the key cannot recompute it ([5.8](../spec/05-hashes.md#58-keyed-hashes)). +- A Merkle root is one hash that commits to a list of items by hashing them in a binary tree ([8.5](../spec/08-fidelity.md#85-merkle-roots)). +- A page fingerprint is a hash of a page's text and images with download stamps removed, so two differently stamped downloads of one paper share it ([7.1](../spec/07-geometry.md#71-page-fingerprints)). +- The strict and lossy projections are two canonical forms of a sentence's text, one to certify that two readings agree and one to propose that they might ([5.3](../spec/05-hashes.md#53-the-strict-projection)). + +[docs/glossary.md](../docs/glossary.md) defines the rest. + +| File | Tests | Chapter | +| --- | --- | --- | +| `canonical-json.jsonl` | `input_json`, parsed with every number as an IEEE 754 double, and its canonical `output`. | [1.1](../spec/01-conventions.md#11-canonical-json) | +| `normalize.jsonl` | Normalization for hashing, and the SHA-256 of the result. | [1.4](../spec/01-conventions.md#14-text-normalization-for-hashing) | +| `ids.jsonl` | Minting payloads and node IDs, and letter-first encodings of digests. | [4](../spec/04-identity.md) | +| `notation.jsonl` | Each math source's canonical linear form, or null when it is complex. | [5.4](../spec/05-hashes.md#54-simple-notation) | +| `projections.jsonl` | One-sentence runs: plain, normalized, strict and lossy projections, complex keys and the three text values. | [5.3](../spec/05-hashes.md#53-the-strict-projection) | +| `projection-groups.jsonl` | Which cases must share, or must differ in, a value: H₂O in six spellings, cm⁻¹, 10⁻³, K_r, ligatures, the micro sign, "103" against 10³. | [5.3](../spec/05-hashes.md#53-the-strict-projection) | +| `offsets.jsonl` | Code-point spans and their UTF-16 equivalents. | [1.5](../spec/01-conventions.md#15-character-offsets) | +| `hashes.jsonl` | Every node of a small document, its own content and its five values. | [5.1](../spec/05-hashes.md#51-the-five-values) | +| `addresses.jsonl` | How addresses parse; null for invalid ones. | [6.1](../spec/06-anchors.md#61-addresses) | +| `keyed.jsonl` | The team key and its ID, an HMAC, page hashes, a page-derived key, and what someone without the paper cannot check. | [5.8](../spec/05-hashes.md#58-keyed-hashes) | +| `fingerprints.jsonl` | Pages as text lines, the stamp rules that fired, and the fingerprint; two downloads that must agree, and a change that must not. | [7.1](../spec/07-geometry.md#71-page-fingerprints) | +| `merkle.jsonl` | Receipt items and their Merkle roots. | [8.5](../spec/08-fidelity.md#85-merkle-roots) | +| `signatures.jsonl` | The RFC 8032 test vector; a receipt's record hash, signing payload and signature; `did:key` identifiers for an Ed25519 and an ML-DSA-65 key; and ML-DSA-65 (FIPS 204): NIST's key-generation case and a deterministic signature. | [8.4](../spec/08-fidelity.md#84-record-hashes-and-signatures) | +| `spotcheck.jsonl` | Seed payloads, seeds, sample sizes and samples. | [8.8](../spec/08-fidelity.md#88-spot-checks) | +| `bounds.jsonl` | Reviewer error bounds as reported. | [8.9](../spec/08-fidelity.md#89-reviewer-error-bounds) | +| `alignment.jsonl` | Two readings of a document, their pairs and their suspects. | [5.6](../spec/05-hashes.md#56-alignment) | +| `corrections.jsonl` | Each operation and refusal on a small document: the state before, the correction, and the record and state after, or the rule broken. | [11](../spec/11-corrections.md) | +| `resolve.jsonl` | Anchors made on the example paper's first reading, and how each resolves after its corrections. | [6.5](../spec/06-anchors.md#65-resolution) | +| `rights.jsonl` | Stage checks with policies and expiry, the combining rule, and backend conditions. | [14](../spec/14-rights.md) | +| `grading.jsonl` | Exercises, submissions and the grades they earn. | [19.4](../spec/19-exercises.md#194-grading) | +| `targets.jsonl` | The target set of targeted F3 for the example paper under each digit rule, given the anchored nodes. | [8.10](../spec/08-fidelity.md#810-targeted-f3) | +| `keys.jsonl` | Work keys: Scholar parts, identities, hash suffixes and document keys, with accented names, missing years and a collision. | [2.4](../spec/02-model.md#24-work-and-document-keys) | +| `proofs.jsonl` | RFC 9162 inclusion paths for every item of small trees, consistency proofs for every prefix, the empty root, and the example paper's history. | [8.5](../spec/08-fidelity.md#85-merkle-roots), [11.5](../spec/11-corrections.md#115-the-history-root) | +| `vrf.jsonl` | The VRF of spot-check seeds: RFC 9381's Example 16, two more inputs for its key, the example paper's seed, and proofs that must not verify (`beta` null). Bytes are hex. | [8.8](../spec/08-fidelity.md#88-spot-checks) | +| `checkpoint.jsonl` | A transparency-log checkpoint: the entries, their root, the signed note and its signers. | [23](../spec/23-log.md) | + +Resolve those paths against the files in that package only. + +## What the vectors prove, and what they do not + +The specification, and the vectors with it, are at version 0.1.0 ([0.3](../spec/00-introduction.md#03-format-identifiers-and-versions)). +The specification has 32 formats: 19 are `candidate` and 13 are `provisional` in [registry/formats.json](../registry/formats.json). +A candidate format changes only with a recorded reason and a new or changed vector ([22.5](../spec/22-conformance.md#225-stability)). + +A passing run shows one thing: for these inputs, your implementation computes the same values as the reference implementation in `tools/oracle/`. +>>>>>>> 4fa326d (First implementation (#1)) That code computed the expected outputs, and `pixi run build` regenerates them ([tools/](../tools/README.md)). The vectors hold 311 cases in all, some of them small: `hashes.jsonl` has one document, `alignment.jsonl` one pair of readings, `offsets.jsonl` two cases.