Add the pytest suite and fix the bugs it found
The suite takes its data from the vectors and fixtures: 3635 tests, 97.9% branch coverage, and paperspec check and paperspec stress give the same results as before. Fixed: receipts.verify_signature raised KeyError for a record without a format; traces.DOC_TAG matched a tag followed by a line feed; paperspec load and paperspec check left SQLite connections open; the site builder copied the output folder's subfolders into itself. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
This commit is contained in:
76 files changed
+15564
-80
No files matched your search
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
+18
-16
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
@@ -26,7 +26,7 @@ import unicodedata
|
||||
import warnings
|
||||
import zipfile
|
||||
from collections.abc import Generator, Sequence
|
||||
from contextlib import contextmanager
|
||||
from contextlib import closing, contextmanager
|
||||
from fractions import Fraction
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
@@ -1013,13 +1013,13 @@ def check_unsafe_paths() -> None:
|
||||
],
|
||||
)
|
||||
(pkg / "MANIFEST.json").write_text(package.line(m), encoding="utf-8")
|
||||
conn = sqlite3.connect(":memory:")
|
||||
db.create(conn, repo_path("sql"))
|
||||
try:
|
||||
db.load(conn, pkg, tmp / "assets")
|
||||
rule = None
|
||||
except Refusal as e:
|
||||
rule = e.rule
|
||||
with closing(sqlite3.connect(":memory:")) as conn:
|
||||
db.create(conn, repo_path("sql"))
|
||||
try:
|
||||
db.load(conn, pkg, tmp / "assets")
|
||||
rule = None
|
||||
except Refusal as e:
|
||||
rule = e.rule
|
||||
ok(
|
||||
"packages",
|
||||
rule == "package.path.unsafe",
|
||||
@@ -2869,8 +2869,10 @@ def check_sql() -> None:
|
||||
"body_kind differs from the schema's body kinds: "
|
||||
f"{sorted(sql_kinds ^ schema_kinds)}",
|
||||
)
|
||||
with temp_dir() as tmp:
|
||||
conn = sqlite3.connect(tmp / "bundle.db")
|
||||
with (
|
||||
temp_dir() as tmp,
|
||||
closing(sqlite3.connect(tmp / "bundle.db")) as conn,
|
||||
):
|
||||
db.create_bundle(conn, repo_path("sql"))
|
||||
assets = tmp / "assets"
|
||||
pk = repo_path("fixtures", "packages")
|
||||
@@ -2882,11 +2884,11 @@ def check_sql() -> None:
|
||||
)
|
||||
if key is None:
|
||||
# A second load of the same key goes to its own database.
|
||||
c2 = sqlite3.connect(":memory:")
|
||||
db.create_bundle(c2, repo_path("sql"))
|
||||
key = db.load(c2, pk / d, assets)
|
||||
ok("sql", db.index(c2, key) == 0, f"{d}: withheld text was indexed")
|
||||
h = db.unload(c2, key, tmp / "out" / d, assets)
|
||||
with closing(sqlite3.connect(":memory:")) as c2:
|
||||
db.create_bundle(c2, repo_path("sql"))
|
||||
key = db.load(c2, pk / d, assets)
|
||||
ok("sql", db.index(c2, key) == 0, f"{d}: withheld text was indexed")
|
||||
h = db.unload(c2, key, tmp / "out" / d, assets)
|
||||
else:
|
||||
db.index(conn, key)
|
||||
h = db.unload(conn, key, tmp / "out" / d, assets)
|
||||
|
||||
+29
-26
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
@@ -21,6 +21,7 @@ import argparse
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
|
||||
from paperspec import db, package, resources
|
||||
@@ -80,30 +81,32 @@ def run(a: argparse.Namespace) -> int:
|
||||
"paperspec validate checks one without loading it."
|
||||
)
|
||||
new = not Path(a.sqlite).exists()
|
||||
conn = sqlite3.connect(a.sqlite)
|
||||
if a.bundle:
|
||||
db.create_bundle(conn, resources.path("sql"))
|
||||
elif new:
|
||||
db.create(conn, resources.path("sql"))
|
||||
for p in folders:
|
||||
key = db.load(conn, p, a.sqlite + ".assets")
|
||||
n = conn.execute(
|
||||
"SELECT COUNT(*) FROM nodes WHERE doc_key = ?", (key,)
|
||||
).fetchone()[0]
|
||||
with closing(sqlite3.connect(a.sqlite)) as conn:
|
||||
if a.bundle:
|
||||
print(f"{key}: {n} nodes, {db.index(conn, key)} search rows")
|
||||
else:
|
||||
print(f"{key}: {n} nodes")
|
||||
if a.bundle:
|
||||
stores = {
|
||||
r[0]
|
||||
for r in conn.execute(
|
||||
"SELECT DISTINCT store FROM documents WHERE store IS NOT NULL"
|
||||
)
|
||||
}
|
||||
store_name = a.store or (stores.pop() if len(stores) == 1 else None)
|
||||
if store_name is None:
|
||||
sys.exit("The packages name no single store; give one with --store.")
|
||||
db.write_meta(conn, store_name, a.built_from)
|
||||
conn.commit()
|
||||
db.create_bundle(conn, resources.path("sql"))
|
||||
elif new:
|
||||
db.create(conn, resources.path("sql"))
|
||||
for p in folders:
|
||||
key = db.load(conn, p, a.sqlite + ".assets")
|
||||
n = conn.execute(
|
||||
"SELECT COUNT(*) FROM nodes WHERE doc_key = ?", (key,)
|
||||
).fetchone()[0]
|
||||
if a.bundle:
|
||||
print(f"{key}: {n} nodes, {db.index(conn, key)} search rows")
|
||||
else:
|
||||
print(f"{key}: {n} nodes")
|
||||
if a.bundle:
|
||||
stores = {
|
||||
r[0]
|
||||
for r in conn.execute(
|
||||
"SELECT DISTINCT store FROM documents WHERE store IS NOT NULL"
|
||||
)
|
||||
}
|
||||
store_name = a.store or (stores.pop() if len(stores) == 1 else None)
|
||||
if store_name is None:
|
||||
sys.exit(
|
||||
"The packages name no single store; give one with --store."
|
||||
)
|
||||
db.write_meta(conn, store_name, a.built_from)
|
||||
conn.commit()
|
||||
return 0
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
@@ -872,6 +872,7 @@ def build(out: StrPath, base_url: str = BASE_URL) -> int:
|
||||
rel_dir = d.relative_to(root).as_posix()
|
||||
reld = "" if rel_dir == "." else rel_dir
|
||||
if d.resolve() == out_resolved:
|
||||
dirs.clear()
|
||||
continue
|
||||
for f in sorted(fs):
|
||||
if f in SKIP_FILES or f.startswith("."):
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
@@ -93,7 +93,7 @@ def verify_signature(rec: Record, roster: Mapping[str, Record]) -> bool:
|
||||
)
|
||||
if sig["scheme"] == "ml-dsa-65":
|
||||
return mldsa.verify(pub, signing_payload(rec), value)
|
||||
except ValueError, TypeError:
|
||||
except ValueError, TypeError, KeyError:
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
@@ -28,7 +28,7 @@ RESULTS = ("current", "changed", "stale", "unlinked", "misresolved")
|
||||
DOC_TAG = re.compile(
|
||||
r"^tag:scient\.ing,2026:doc:(?P<store>[a-z0-9][a-z0-9._-]*)/"
|
||||
r"(?P<key>[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}"
|
||||
r"(?:\.[a-z0-9][a-z0-9-]*)*)$"
|
||||
r"(?:\.[a-z0-9][a-z0-9-]*)*)\Z"
|
||||
)
|
||||
# The statuses of a reply that still stands: a withdrawn or rejected reply disputes nothing.
|
||||
LIVE = ("proposed", "confirmed")
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scientific Computing Studio
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Shared test helpers.
|
||||
|
||||
The repository's own vectors and fixtures are the test data: the vectors hold the exact
|
||||
outputs the specification requires, and `fixtures/` holds the packages and records the
|
||||
checks read.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
VECTORS = ROOT / "vectors"
|
||||
FIXTURES = ROOT / "fixtures"
|
||||
|
||||
|
||||
def read_jsonl(path: Path) -> list[dict]:
|
||||
"""Parse a JSON Lines file into a list of records."""
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in path.read_text(encoding="utf-8").splitlines()
|
||||
if line
|
||||
]
|
||||
|
||||
|
||||
def vector_cases(name: str) -> list[dict]:
|
||||
"""The cases of `vectors/<name>.jsonl`."""
|
||||
return read_jsonl(VECTORS / f"{name}.jsonl")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def root() -> Path:
|
||||
"""The repository root, where `paperspec check` and `paperspec build` work."""
|
||||
return ROOT
|
||||
@@ -0,0 +1,130 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Helpers shared by the package, corrections and command line tests.
|
||||
|
||||
They read the repository's fixtures, write edited copies of a package under a temporary
|
||||
folder, and run `paperspec` commands in process.
|
||||
"""
|
||||
|
||||
import json
|
||||
import shutil
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
from conftest import FIXTURES, ROOT
|
||||
from paperspec import crypto, jcs, package, resources
|
||||
from paperspec.schema import Store
|
||||
|
||||
PACKAGES = FIXTURES / "packages"
|
||||
INVALID = FIXTURES / "invalid"
|
||||
HALE = PACKAGES / "hale2026harmonic-damb"
|
||||
CHECKS_ONLY = PACKAGES / "hale2026harmonic-damb.checks-only"
|
||||
TINY = PACKAGES / "tiny2026tiny-j4qm"
|
||||
TINY_ZIP = PACKAGES / "tiny2026tiny-j4qm.sct.zip"
|
||||
|
||||
# Folders and files that are not part of the repository and are not copied.
|
||||
COPY_IGNORE = shutil.ignore_patterns(
|
||||
".git",
|
||||
".pixi",
|
||||
".claude",
|
||||
".hypothesis",
|
||||
".pytest_cache",
|
||||
".ruff_cache",
|
||||
"__pycache__",
|
||||
"_site",
|
||||
"work",
|
||||
"tests",
|
||||
"htmlcov",
|
||||
"*.egg-info",
|
||||
".DS_Store",
|
||||
)
|
||||
|
||||
|
||||
def store() -> Store:
|
||||
"""The schema store of the installed package."""
|
||||
return Store(resources.path("schemas"))
|
||||
|
||||
|
||||
def team_key() -> bytes:
|
||||
"""The HMAC key of the checks-only fixture, derived from the published team secret."""
|
||||
secret = (FIXTURES / "keys" / "smithlab-team-secret.hex").read_text().strip()
|
||||
return crypto.team_key(bytes.fromhex(secret))
|
||||
|
||||
|
||||
def package_dirs() -> list[Path]:
|
||||
"""The valid package folders of `fixtures/packages/`, by name."""
|
||||
return sorted(p for p in PACKAGES.iterdir() if p.is_dir())
|
||||
|
||||
|
||||
def key_for(path: Path) -> bytes | None:
|
||||
"""The hashing key a fixture package needs to validate: the team key for a checks-only one."""
|
||||
return team_key() if path.name.endswith("checks-only") else None
|
||||
|
||||
|
||||
def read_package(path: Path) -> tuple[dict[str, bytes], list[dict]]:
|
||||
"""The files of a package folder by path, and its manifest entries for absent files."""
|
||||
manifest = json.loads((path / "MANIFEST.json").read_text(encoding="utf-8"))
|
||||
files = {}
|
||||
absent = []
|
||||
for entry in manifest["files"]:
|
||||
if entry.get("present", True) is False:
|
||||
absent.append(entry)
|
||||
else:
|
||||
files[entry["path"]] = (path / entry["path"]).read_bytes()
|
||||
return files, absent
|
||||
|
||||
|
||||
def encode(path: str, value) -> bytes:
|
||||
"""A parsed package file as the canonical bytes a producer writes."""
|
||||
if path.endswith(".jsonl"):
|
||||
return package.jsonl(value)
|
||||
return package.line(value).encode("utf-8")
|
||||
|
||||
|
||||
def decode(path: str, data: bytes):
|
||||
"""A package file's records: a list for a JSON Lines file, the value for a JSON file."""
|
||||
if path.endswith(".jsonl"):
|
||||
return [json.loads(line) for line in data.split(b"\n") if line]
|
||||
return json.loads(data)
|
||||
|
||||
|
||||
def edit_package(
|
||||
dest: Path, source: Path, edits: dict[str, Callable | bytes | None]
|
||||
) -> Path:
|
||||
"""Write a copy of package `source` at `dest/<name>` with `edits` applied, and a fresh manifest.
|
||||
|
||||
Each edit is keyed by a package path. A function takes the file's parsed records and returns the
|
||||
new ones; bytes replace the file as they are; None removes the file.
|
||||
"""
|
||||
files, absent = read_package(source)
|
||||
for path, change in edits.items():
|
||||
if change is None:
|
||||
files.pop(path, None)
|
||||
elif isinstance(change, bytes):
|
||||
files[path] = change
|
||||
else:
|
||||
files[path] = encode(path, change(decode(path, files[path])))
|
||||
root = dest / source.name
|
||||
package.write(root, files, absent)
|
||||
return root
|
||||
|
||||
|
||||
def rules_of(report) -> set[str]:
|
||||
"""The rule IDs of a validation report."""
|
||||
return {e["rule"] for e in report.errors}
|
||||
|
||||
|
||||
def copy_repository(dest: Path) -> Path:
|
||||
"""A copy of the repository's tracked content under `dest/paperspec`, for the commands that work on a checkout."""
|
||||
root = dest / "paperspec"
|
||||
shutil.copytree(ROOT, root, ignore=COPY_IGNORE)
|
||||
return root
|
||||
|
||||
|
||||
def canonical_line(value) -> str:
|
||||
"""One canonical JSON line, as the package files hold records."""
|
||||
return jcs.dumps(value) + "\n"
|
||||
@@ -0,0 +1,175 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Helpers for the tests of the document model, hashing and projection modules.
|
||||
|
||||
Vector readers that follow the procedure in `vectors/README.md`, small documents built
|
||||
from nested specs, fixture package states, and the slow reference computation of the five
|
||||
content values that the incremental index must always agree with.
|
||||
"""
|
||||
|
||||
import json
|
||||
from functools import cache
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from conftest import FIXTURES, ROOT, VECTORS
|
||||
from paperspec import hashing, jcs
|
||||
from paperspec.corrections import State
|
||||
from paperspec.model import Doc
|
||||
from paperspec.package import read_state
|
||||
from paperspec.text import norm
|
||||
from paperspec.tree import establish
|
||||
|
||||
HALE = "fixtures/packages/hale2026harmonic-damb"
|
||||
TINY = FIXTURES / "packages" / "tiny2026tiny-j4qm"
|
||||
VALUE_FIELDS = ("text_hash", "match_hash", "align_key", "own_hash", "node_hash")
|
||||
|
||||
|
||||
def read_cases(name: str) -> list[dict]:
|
||||
"""The cases of `vectors/<name>.jsonl`, split at U+000A only as the README requires."""
|
||||
text = (VECTORS / f"{name}.jsonl").read_text(encoding="utf-8")
|
||||
return [json.loads(line) for line in text.split("\n") if line]
|
||||
|
||||
|
||||
def case_params(cases: list[dict]) -> list:
|
||||
"""Pytest parameters for vector cases, each named by its `id`."""
|
||||
return [pytest.param(c, id=c["id"]) for c in cases]
|
||||
|
||||
|
||||
def vectors(name: str) -> list:
|
||||
"""Parametrization list for a vector file."""
|
||||
return case_params(read_cases(name))
|
||||
|
||||
|
||||
def parse_doubles(text: str):
|
||||
"""Parse JSON as an ECMAScript engine does: every number is an IEEE 754 double."""
|
||||
return json.loads(
|
||||
text, parse_int=lambda x: int(x) if abs(int(x)) <= 2**53 else float(x)
|
||||
)
|
||||
|
||||
|
||||
def sentence_doc(runs: list[dict]) -> tuple[Doc, str]:
|
||||
"""A one-sentence document, and the sentence's ID."""
|
||||
nodes = [
|
||||
{"id": "aaaaaaa", "kind": "document", "parent": None, "ord": 0},
|
||||
{
|
||||
"id": "bbbbbbb",
|
||||
"kind": "section",
|
||||
"parent": "aaaaaaa",
|
||||
"ord": 0,
|
||||
"attrs": {"level": 1},
|
||||
},
|
||||
{"id": "ccccccc", "kind": "paragraph", "parent": "bbbbbbb", "ord": 0},
|
||||
{
|
||||
"id": "ddddddd",
|
||||
"kind": "sentence",
|
||||
"parent": "ccccccc",
|
||||
"ord": 0,
|
||||
"content": runs,
|
||||
},
|
||||
]
|
||||
return Doc("vectors", nodes), "ddddddd"
|
||||
|
||||
|
||||
def text_run(text: str, **extra) -> dict:
|
||||
"""A text run."""
|
||||
return {"kind": "text", "text": text, **extra}
|
||||
|
||||
|
||||
def math_run(source: str, notation: str = "tex") -> dict:
|
||||
"""A math run."""
|
||||
return {"kind": "math", "notation": notation, "source": source}
|
||||
|
||||
|
||||
def sentence(*runs: dict, children: list | None = None) -> dict:
|
||||
"""A sentence spec for `build_doc`."""
|
||||
spec = {"kind": "sentence", "content": list(runs)}
|
||||
if children:
|
||||
spec["children"] = children
|
||||
return spec
|
||||
|
||||
|
||||
def paragraph(*sentences: dict) -> dict:
|
||||
"""A paragraph spec holding sentences."""
|
||||
return {"kind": "paragraph", "children": list(sentences)}
|
||||
|
||||
|
||||
def section(*children: dict, level: int = 1, label: str | None = None) -> dict:
|
||||
"""A section spec; the first child is its heading when it is a `heading`."""
|
||||
spec = {"kind": "section", "attrs": {"level": level}, "children": list(children)}
|
||||
if label is not None:
|
||||
spec["label"] = label
|
||||
return spec
|
||||
|
||||
|
||||
def heading(text: str) -> dict:
|
||||
"""A heading spec of one sentence."""
|
||||
return {"kind": "heading", "children": [sentence(text_run(text))]}
|
||||
|
||||
|
||||
def document(*children: dict) -> dict:
|
||||
"""A document spec."""
|
||||
return {"kind": "document", "children": list(children)}
|
||||
|
||||
|
||||
def build_doc(spec: dict, key: str = "test2026doc-aaaa") -> Doc:
|
||||
"""Establish a nested spec as a document with minted IDs."""
|
||||
return Doc(key, establish(key, spec))
|
||||
|
||||
|
||||
def simple_doc(*sentence_texts: str) -> Doc:
|
||||
"""A document of one section and one paragraph holding a sentence for each text."""
|
||||
para = paragraph(*(sentence(text_run(t)) for t in sentence_texts))
|
||||
return build_doc(document(section(heading("Head"), para)))
|
||||
|
||||
|
||||
def ids_of_kind(doc: Doc, kind: str) -> list[str]:
|
||||
"""The IDs of a document's nodes of one kind, in document order."""
|
||||
return [n for n in doc.preorder() if doc.nodes[n]["kind"] == kind]
|
||||
|
||||
|
||||
@cache
|
||||
def package_state(rel: str) -> State:
|
||||
"""The state of a fixture package named by a path from the repository root."""
|
||||
return read_state(str(ROOT / rel))
|
||||
|
||||
|
||||
def package_nodes(path: Path) -> list[dict]:
|
||||
"""The node records of a package folder, values included."""
|
||||
lines = (path / "nodes.jsonl").read_text(encoding="utf-8").split("\n")
|
||||
return [json.loads(x) for x in lines if x]
|
||||
|
||||
|
||||
def bare(nodes: list[dict]) -> list[dict]:
|
||||
"""Node records without their five content values."""
|
||||
return [{k: v for k, v in n.items() if k not in VALUE_FIELDS} for n in nodes]
|
||||
|
||||
|
||||
def reference_values(doc: Doc, hasher=hashing.PLAIN) -> dict:
|
||||
"""The five values the slow way: every node projected from its whole subtree."""
|
||||
values = {}
|
||||
for nid in reversed(doc.preorder()):
|
||||
strict, _ = hashing.strict(doc, nid)
|
||||
own = hasher(
|
||||
jcs.dumpb(hashing.own_content(doc, nid, lambda x: values[x]["text_hash"]))
|
||||
)
|
||||
values[nid] = {
|
||||
"text_hash": hasher(norm(hashing.plain(doc, nid))),
|
||||
"match_hash": hasher(strict),
|
||||
"align_key": hasher(hashing.lossy_from_strict(strict)),
|
||||
"own_hash": own,
|
||||
"node_hash": hasher(
|
||||
jcs.dumpb(
|
||||
{
|
||||
"own": own,
|
||||
"children": [values[c]["node_hash"] for c in doc.kids(nid)],
|
||||
}
|
||||
)
|
||||
),
|
||||
}
|
||||
return values
|
||||
@@ -0,0 +1,119 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Helpers for the tests of the signing, receipt, log, trace and annotation-body modules."""
|
||||
|
||||
import base64
|
||||
import functools
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
from hypothesis import settings
|
||||
|
||||
from conftest import FIXTURES, ROOT
|
||||
from paperspec import crypto, mldsa, receipts
|
||||
from paperspec.schema import Store, validate
|
||||
|
||||
PROPS = settings(derandomize=True, database=None, deadline=None, max_examples=50)
|
||||
"""The hypothesis settings of these tests: deterministic, nothing written to the checkout."""
|
||||
|
||||
HALE = "hale2026harmonic-damb"
|
||||
DRAFT = "writer2027fixed-ynr7.v3"
|
||||
RECORDS = FIXTURES / "records"
|
||||
|
||||
|
||||
@functools.cache
|
||||
def schema_store():
|
||||
"""The schemas of the repository, linted and compiled once for the whole run."""
|
||||
return Store(str(ROOT / "schemas"))
|
||||
|
||||
|
||||
@functools.cache
|
||||
def rule_chapters():
|
||||
"""The chapter of each rule in `registry/rules.json`, by rule ID."""
|
||||
text = (ROOT / "registry" / "rules.json").read_text(encoding="utf-8")
|
||||
return {r["id"]: r["chapter"] for r in json.loads(text)["rules"]}
|
||||
|
||||
|
||||
def accepted_by_schema(annotation):
|
||||
"""Whether `schemas/annotation.schema.json` accepts `annotation`."""
|
||||
return validate(schema_store(), "annotation.schema.json", annotation) == []
|
||||
|
||||
|
||||
def secret(name):
|
||||
"""The 32-byte secret the fixtures derive for the key called `name`."""
|
||||
return hashlib.sha256(("paperspec fixture key: " + name).encode()).digest()
|
||||
|
||||
|
||||
def roster_key(key_id, role="editor", scheme="ed25519", purposes=("sign",), **extra):
|
||||
"""A roster entry for the fixture key `key_id`, registered before every fixture date."""
|
||||
if scheme == "ml-dsa-65":
|
||||
public = mldsa.public_key(secret(key_id))
|
||||
else:
|
||||
public = crypto.ed25519_public(secret(key_id))
|
||||
entry = {
|
||||
"id": key_id,
|
||||
"role": role,
|
||||
"scheme": scheme,
|
||||
"public": base64.b64encode(public).decode("ascii"),
|
||||
"purposes": list(purposes),
|
||||
"registered_at": "2026-09-01T00:00:00Z",
|
||||
"revoked_at": None,
|
||||
}
|
||||
entry.update(extra)
|
||||
return entry
|
||||
|
||||
|
||||
def roster_of(*keys):
|
||||
"""A roster keyed by ID from the entries made by `roster_key`."""
|
||||
return {k["id"]: k for k in keys}
|
||||
|
||||
|
||||
def fixture_roster():
|
||||
"""The roster of `fixtures/records/roster.json`, keyed by key ID."""
|
||||
keys = json.loads((RECORDS / "roster.json").read_text(encoding="utf-8"))["keys"]
|
||||
return {k["id"]: k for k in keys}
|
||||
|
||||
|
||||
def fixture_receipts():
|
||||
"""The signed records of the main example's `receipts.jsonl`, in file order."""
|
||||
path = FIXTURES / "packages" / HALE / "receipts.jsonl"
|
||||
return [json.loads(x) for x in path.read_text(encoding="utf-8").splitlines()]
|
||||
|
||||
|
||||
def receipt(nodes, by="ana", at="2026-10-11T12:00:00Z", level="F3", **extra):
|
||||
"""An unsigned receipt over `nodes`, a list of (node ID, node hash) pairs."""
|
||||
rec = {
|
||||
"format": "scienting-receipt/1",
|
||||
"doc": HALE,
|
||||
"rev": 0,
|
||||
"level": level,
|
||||
"scope": "all",
|
||||
"nodes": [list(n) for n in nodes],
|
||||
"evidence": [],
|
||||
"by": by,
|
||||
"at": at,
|
||||
}
|
||||
rec.update(extra)
|
||||
return rec
|
||||
|
||||
|
||||
def sealed(rec, signer=None):
|
||||
"""`rec` sealed with the fixture secret of its `by` (or of `signer`)."""
|
||||
name = signer or rec["by"]
|
||||
return receipts.seal(rec, secret(name), name)
|
||||
|
||||
|
||||
def revocation(by, at="2026-10-11T13:00:00Z", **target):
|
||||
"""A revocation signed by the fixture key `by`; `target` gives `receipt`, or `signer` and `period`."""
|
||||
rec = {
|
||||
"format": "scienting-revocation/1",
|
||||
"reason": "Revoked for a test.",
|
||||
"by": by,
|
||||
"at": at,
|
||||
}
|
||||
rec.update(target)
|
||||
return receipts.seal(rec, secret(by), by)
|
||||
@@ -0,0 +1,284 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Whole-document sentence alignment (spec/05 §5.6)."""
|
||||
|
||||
import pytest
|
||||
from hypothesis import given, settings, strategies as st
|
||||
|
||||
from helpers_lib1 import vectors
|
||||
from paperspec import align
|
||||
from paperspec.hashing import OBJ, lossy_from_strict
|
||||
|
||||
PROPERTIES = settings(derandomize=True, max_examples=150, database=None, deadline=None)
|
||||
|
||||
LONG = [
|
||||
"The first long sentence about bond constants.",
|
||||
"A second sentence describing the fitting procedure.",
|
||||
"The third sentence reports how charges were derived.",
|
||||
"Another sentence that covers the validation set used.",
|
||||
"Finally this sentence closes the paragraph for good.",
|
||||
]
|
||||
|
||||
|
||||
def item(ident, text, container="p1", context="paragraph", complex_keys=()):
|
||||
"""A reading item whose strict text is `text`."""
|
||||
return {
|
||||
"id": ident,
|
||||
"strict": text,
|
||||
"lossy": lossy_from_strict(text),
|
||||
"complex": list(complex_keys),
|
||||
"container": container,
|
||||
"context": context,
|
||||
}
|
||||
|
||||
|
||||
def reading(prefix, texts, **kw):
|
||||
"""A reading of one paragraph from texts, with IDs `<prefix>1`, `<prefix>2`, ..."""
|
||||
return [item(f"{prefix}{i}", t, **kw) for i, t in enumerate(texts, 1)]
|
||||
|
||||
|
||||
def pairs_of(result):
|
||||
return [(p["a"], p["b"]) for p in result["pairs"]]
|
||||
|
||||
|
||||
def kinds_of(result):
|
||||
return [(s["kind"], s["a"], s["b"]) for s in result["suspects"]]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", vectors("alignment"))
|
||||
def test_alignment_vector(case):
|
||||
assert align.align(case["a"], case["b"]) == case["expect"]
|
||||
|
||||
|
||||
def test_identical_readings_agree_everywhere():
|
||||
a = reading("a", LONG)
|
||||
result = align.align(a, reading("b", LONG))
|
||||
assert pairs_of(result) == [(f"a{i}", f"b{i}") for i in range(1, 6)]
|
||||
assert all(p["agree"] for p in result["pairs"])
|
||||
assert result["suspects"] == []
|
||||
|
||||
|
||||
def test_two_empty_readings_give_nothing():
|
||||
assert align.align([], []) == {"pairs": [], "suspects": []}
|
||||
|
||||
|
||||
def test_a_sentence_only_in_the_first_reading_is_dropped():
|
||||
result = align.align(reading("a", LONG), reading("b", LONG[:2] + LONG[3:]))
|
||||
assert kinds_of(result) == [("dropped", "a3", None)]
|
||||
assert len(result["pairs"]) == 4
|
||||
|
||||
|
||||
def test_a_sentence_only_in_the_second_reading_is_added():
|
||||
result = align.align(reading("a", LONG[:2] + LONG[3:]), reading("b", LONG))
|
||||
assert kinds_of(result) == [("added", None, "b3")]
|
||||
|
||||
|
||||
def test_everything_is_dropped_or_added_when_nothing_is_shared():
|
||||
a = reading("a", ["Alpha beta gamma delta epsilon zeta eta."])
|
||||
b = reading("b", ["Completely unrelated words fill this row."])
|
||||
result = align.align(a, b)
|
||||
assert result["pairs"] == []
|
||||
assert kinds_of(result) == [("dropped", "a1", None), ("added", None, "b1")]
|
||||
|
||||
|
||||
def test_a_changed_number_is_a_number_suspect():
|
||||
a = reading("a", ["The fitted rate is 103 per second at room temperature."])
|
||||
b = reading("b", ["The fitted rate is 108 per second at room temperature."])
|
||||
result = align.align(a, b)
|
||||
assert result["pairs"] == [{"a": "a1", "b": "b1", "agree": False}]
|
||||
assert kinds_of(result) == [("number", "a1", "b1")]
|
||||
|
||||
|
||||
def test_a_changed_word_is_a_word_suspect():
|
||||
a = reading("a", ["The solvent was heated slowly to the boiling point."])
|
||||
b = reading("b", ["The solvent was heated quickly to the boiling point."])
|
||||
assert kinds_of(align.align(a, b)) == [("word", "a1", "b1")]
|
||||
|
||||
|
||||
def test_a_changed_equation_with_the_same_text_is_an_equation_suspect():
|
||||
text = f"where {OBJ} is the bond length in the model."
|
||||
a = [item("a1", text, complex_keys=["tex:\\frac{a}{b}"])]
|
||||
b = [item("b1", text, complex_keys=["tex:\\frac{c}{d}"])]
|
||||
result = align.align(a, b)
|
||||
assert result["pairs"] == [{"a": "a1", "b": "b1", "agree": False}]
|
||||
assert kinds_of(result) == [("equation", "a1", "b1")]
|
||||
|
||||
|
||||
def test_a_changed_context_is_a_structure_suspect_even_when_the_text_agrees():
|
||||
a = reading("a", LONG[:1], context="paragraph")
|
||||
b = reading("b", LONG[:1], context="heading")
|
||||
result = align.align(a, b)
|
||||
assert result["pairs"][0]["agree"] is True
|
||||
assert kinds_of(result) == [("structure", "a1", "b1")]
|
||||
|
||||
|
||||
def test_a_paragraph_break_that_appears_or_vanishes_is_a_join_suspect():
|
||||
a = [item("a1", LONG[0], "p1"), item("a2", LONG[1], "p1")]
|
||||
b = [item("b1", LONG[0], "q1"), item("b2", LONG[1], "q2")]
|
||||
assert kinds_of(align.align(a, b)) == [("join", "a2", "b2")]
|
||||
assert kinds_of(align.align(b, a)) == [("join", "b2", "a2")]
|
||||
|
||||
|
||||
def test_neighbours_in_matching_containers_are_not_a_join():
|
||||
a = [item("a1", LONG[0], "p1"), item("a2", LONG[1], "p2")]
|
||||
b = [item("b1", LONG[0], "q1"), item("b2", LONG[1], "q2")]
|
||||
assert kinds_of(align.align(a, b)) == []
|
||||
|
||||
|
||||
def test_short_sentences_pair_only_by_equal_text():
|
||||
a = reading("a", ["Rate is 10.", "Next one."])
|
||||
b = reading("b", ["Rate is 11.", "Next one."])
|
||||
result = align.align(a, b)
|
||||
assert pairs_of(result) == [("a2", "b2")]
|
||||
assert kinds_of(result) == [("dropped", "a1", None), ("added", None, "b1")]
|
||||
|
||||
|
||||
def test_a_sentence_that_changed_a_little_is_paired_and_one_that_changed_a_lot_is_not():
|
||||
base = "Every bond was scanned in steps of small size around its minimum."
|
||||
little = base.replace("small", "tiny")
|
||||
a = reading("a", [LONG[0], base, LONG[2]])
|
||||
near = align.align(a, reading("b", [LONG[0], little, LONG[2]]))
|
||||
assert pairs_of(near) == [("a1", "b1"), ("a2", "b2"), ("a3", "b3")]
|
||||
far = align.align(
|
||||
a, reading("b", [LONG[0], "Zebras graze quietly beyond.....", LONG[2]])
|
||||
)
|
||||
assert pairs_of(far) == [("a1", "b1"), ("a3", "b3")]
|
||||
assert kinds_of(far) == [("dropped", "a2", None), ("added", None, "b2")]
|
||||
|
||||
|
||||
def test_a_moved_sentence_is_a_drop_and_an_add_not_a_pair():
|
||||
a = reading("a", LONG[:3])
|
||||
b = reading("b", [LONG[1], LONG[2], LONG[0]])
|
||||
result = align.align(a, b)
|
||||
assert pairs_of(result) == [("a2", "b1"), ("a3", "b2")]
|
||||
assert kinds_of(result) == [("dropped", "a1", None), ("added", None, "b3")]
|
||||
|
||||
|
||||
def test_a_repeated_sentence_does_not_anchor_but_still_pairs():
|
||||
a = reading("a", [LONG[0], LONG[1], LONG[1], LONG[2]])
|
||||
b = reading("b", [LONG[0], LONG[1], LONG[1], LONG[2]])
|
||||
result = align.align(a, b)
|
||||
assert pairs_of(result) == [(f"a{i}", f"b{i}") for i in range(1, 5)]
|
||||
assert result["suspects"] == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("a", "b", "kind"),
|
||||
[
|
||||
({"strict": "x", "complex": []}, {"strict": "x", "complex": []}, None),
|
||||
({"strict": "x 1", "complex": []}, {"strict": "x 2", "complex": []}, "number"),
|
||||
(
|
||||
{"strict": "x_{1}", "complex": []},
|
||||
{"strict": "x^{1}", "complex": []},
|
||||
"number",
|
||||
),
|
||||
({"strict": "1.5", "complex": []}, {"strict": "15", "complex": []}, "number"),
|
||||
(
|
||||
{"strict": "a \N{MINUS SIGN} 1", "complex": []},
|
||||
{"strict": "a 1", "complex": []},
|
||||
"number",
|
||||
),
|
||||
({"strict": "ab", "complex": []}, {"strict": "ac", "complex": []}, "word"),
|
||||
(
|
||||
{"strict": "ab", "complex": ["k"]},
|
||||
{"strict": "ab", "complex": []},
|
||||
"equation",
|
||||
),
|
||||
(
|
||||
{"strict": "ab 1", "complex": ["k"]},
|
||||
{"strict": "ac 1", "complex": []},
|
||||
"equation",
|
||||
),
|
||||
(
|
||||
{"strict": "a 1", "complex": ["k"]},
|
||||
{"strict": "a 2", "complex": ["k"]},
|
||||
"number",
|
||||
),
|
||||
({"strict": "٣", "complex": []}, {"strict": "3", "complex": []}, "number"),
|
||||
],
|
||||
)
|
||||
def test_classify(a, b, kind):
|
||||
assert align.classify(a, b) == kind
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("a", "b", "length"),
|
||||
[
|
||||
("", "", 0),
|
||||
("abc", "", 0),
|
||||
("abc", "abc", 3),
|
||||
("abcbdab", "bdcaba", 4),
|
||||
("AGGTAB", "GXTXAYB", 4),
|
||||
("abc", "xyz", 0),
|
||||
("aaaa", "aa", 2),
|
||||
("\U0001d465y", "y\U0001d465", 1),
|
||||
],
|
||||
)
|
||||
def test_lcs_length(a, b, length):
|
||||
assert align.lcs_length(a, b) == align.lcs_length(b, a) == length
|
||||
|
||||
|
||||
def _lcs_table(a, b):
|
||||
table = [[0] * (len(b) + 1) for _ in range(len(a) + 1)]
|
||||
for i in range(len(a) - 1, -1, -1):
|
||||
for j in range(len(b) - 1, -1, -1):
|
||||
if a[i] == b[j]:
|
||||
table[i][j] = table[i + 1][j + 1] + 1
|
||||
else:
|
||||
table[i][j] = max(table[i + 1][j], table[i][j + 1])
|
||||
return table[0][0]
|
||||
|
||||
|
||||
@given(st.text(alphabet="abc", max_size=14), st.text(alphabet="abc", max_size=14))
|
||||
@PROPERTIES
|
||||
def test_lcs_length_is_the_textbook_dynamic_programme(a, b):
|
||||
assert align.lcs_length(a, b) == _lcs_table(a, b)
|
||||
|
||||
|
||||
@given(st.text(max_size=30), st.text(max_size=30))
|
||||
@PROPERTIES
|
||||
def test_similarity_is_symmetric_and_between_zero_and_one(a, b):
|
||||
assert align.similarity(a, b) == align.similarity(b, a)
|
||||
assert 0.0 <= align.similarity(a, b) <= 1.0
|
||||
assert align.similarity(a, a) == 1.0
|
||||
|
||||
|
||||
def test_similarity_is_twice_the_common_length_over_the_total():
|
||||
assert align.similarity("", "") == 1.0
|
||||
assert align.similarity("abc", "") == 0.0
|
||||
assert align.similarity("abcd", "abxd") == 0.75
|
||||
assert align.similarity("kitten", "sitting") == pytest.approx(2 * 4 / 13)
|
||||
|
||||
|
||||
SENTENCES = [*LONG, "Short.", "Also short.", "Rate 10 here.", "Rate 11 here."]
|
||||
|
||||
|
||||
@given(
|
||||
st.lists(st.sampled_from(SENTENCES), max_size=7),
|
||||
st.lists(st.sampled_from(SENTENCES), max_size=7),
|
||||
)
|
||||
@PROPERTIES
|
||||
def test_pairs_increase_in_both_readings_and_every_sentence_is_accounted_for(ta, tb):
|
||||
a, b = reading("a", ta), reading("b", tb)
|
||||
result = align.align(a, b)
|
||||
ai = [int(p["a"][1:]) for p in result["pairs"]]
|
||||
bi = [int(p["b"][1:]) for p in result["pairs"]]
|
||||
assert ai == sorted(set(ai))
|
||||
assert bi == sorted(set(bi))
|
||||
dropped = {s["a"] for s in result["suspects"] if s["kind"] == "dropped"}
|
||||
added = {s["b"] for s in result["suspects"] if s["kind"] == "added"}
|
||||
assert dropped == {x["id"] for x in a} - {p["a"] for p in result["pairs"]}
|
||||
assert added == {x["id"] for x in b} - {p["b"] for p in result["pairs"]}
|
||||
|
||||
|
||||
@given(st.lists(st.sampled_from(SENTENCES), max_size=7))
|
||||
@PROPERTIES
|
||||
def test_a_reading_aligned_with_itself_has_no_suspects(texts):
|
||||
result = align.align(reading("a", texts), reading("b", texts))
|
||||
assert len(result["pairs"]) == len(texts)
|
||||
assert all(p["agree"] for p in result["pairs"])
|
||||
assert result["suspects"] == []
|
||||
@@ -0,0 +1,822 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Addresses, anchors and their resolution (spec/06)."""
|
||||
|
||||
from fractions import Fraction
|
||||
|
||||
import pytest
|
||||
from hypothesis import given, settings, strategies as st
|
||||
|
||||
from helpers_lib1 import (
|
||||
HALE,
|
||||
build_doc,
|
||||
case_params,
|
||||
document,
|
||||
heading,
|
||||
ids_of_kind,
|
||||
package_state,
|
||||
paragraph,
|
||||
read_cases,
|
||||
sentence,
|
||||
text_run,
|
||||
vectors,
|
||||
)
|
||||
from paperspec import anchors, hashing
|
||||
from paperspec.corrections import State
|
||||
from paperspec.model import LEAVES
|
||||
from paperspec.text import norm
|
||||
|
||||
PROPERTIES = settings(derandomize=True, max_examples=200, database=None, deadline=None)
|
||||
|
||||
FIRST = "Harmonic bond terms remain the most common description of covalent stretching."
|
||||
SECOND = "Every bond was scanned in steps of small size around its minimum energy."
|
||||
THIRD = "The charges came from a two-stage restrained fit to the potential."
|
||||
FOURTH = "Torsions were refit against a second, larger set of conformers."
|
||||
|
||||
KEYS = st.builds(
|
||||
lambda head, tail, sfx: f"{head}-{tail}" + "".join(f".{s}" for s in sfx),
|
||||
st.from_regex(r"[a-z][a-z0-9]{0,8}", fullmatch=True),
|
||||
st.text(alphabet="0123456789abcdefghjkmnpqrstvwxyz", min_size=4, max_size=12),
|
||||
st.lists(st.from_regex(r"[a-z0-9][a-z0-9-]{0,4}", fullmatch=True), max_size=2),
|
||||
)
|
||||
NAMES = st.from_regex(r"[a-z0-9][a-z0-9._-]{0,10}", fullmatch=True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", vectors("addresses"))
|
||||
def test_address_vector_parses(case):
|
||||
assert anchors.parse_address(case["address"]) == case["parsed"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case", case_params([c for c in read_cases("addresses") if c["parsed"]])
|
||||
)
|
||||
def test_address_vector_formats_back(case):
|
||||
parts = case["parsed"]
|
||||
got = anchors.format_address(
|
||||
parts["store"], parts["key"], parts["node"], parts["pin"], parts["span"]
|
||||
)
|
||||
assert got == case["address"]
|
||||
|
||||
|
||||
@given(
|
||||
st.one_of(st.none(), NAMES),
|
||||
KEYS,
|
||||
NAMES,
|
||||
st.one_of(st.none(), st.text(alphabet="0123456789abcdef", min_size=6, max_size=6)),
|
||||
st.one_of(
|
||||
st.none(), st.tuples(st.integers(0, 10**6), st.integers(0, 10**6)).map(sorted)
|
||||
),
|
||||
)
|
||||
@PROPERTIES
|
||||
def test_a_formatted_address_parses_back(store, key, node, pin, span):
|
||||
address = anchors.format_address(store, key, node, pin, span)
|
||||
assert anchors.parse_address(address) == {
|
||||
"store": store,
|
||||
"key": key,
|
||||
"node": node,
|
||||
"pin": pin,
|
||||
"span": span,
|
||||
}
|
||||
|
||||
|
||||
@given(st.text(max_size=40))
|
||||
@PROPERTIES
|
||||
def test_parsing_any_text_gives_an_address_or_none(text):
|
||||
parsed = anchors.parse_address(text)
|
||||
assert parsed is None or anchors.format_address(**parsed) == text
|
||||
|
||||
|
||||
def test_format_address_keeps_six_characters_of_a_longer_pin():
|
||||
assert (
|
||||
anchors.format_address(None, "a-bcde", "n1", "0123456789abcdef")
|
||||
== "a-bcde#n1@012345"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("store", "pin", "span", "tail"),
|
||||
[
|
||||
("", None, None, ""),
|
||||
(None, "", None, ""),
|
||||
(None, None, [3, 9], "[3:9]"),
|
||||
(None, None, (0, 0), "[0:0]"),
|
||||
],
|
||||
)
|
||||
def test_format_address_leaves_out_what_is_empty(store, pin, span, tail):
|
||||
assert (
|
||||
anchors.format_address(store, "a-bcde", "n1", pin, span) == "a-bcde#n1" + tail
|
||||
)
|
||||
|
||||
|
||||
# --- making anchors -----------------------------------------------------------------
|
||||
|
||||
|
||||
def _tree():
|
||||
return build_doc(
|
||||
document(
|
||||
{
|
||||
"kind": "section",
|
||||
"label": "1",
|
||||
"attrs": {"level": 1},
|
||||
"children": [
|
||||
heading("Methods"),
|
||||
paragraph(sentence(text_run(FIRST)), sentence(text_run(SECOND))),
|
||||
paragraph(sentence(text_run(THIRD)), sentence(text_run(FOURTH))),
|
||||
],
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _state(doc, ids=(), corrections=()):
|
||||
return State(doc.key, [doc.nodes[n] for n in doc.order], ids, corrections)
|
||||
|
||||
|
||||
def _anchor(state, nid, rev=0, span=None, store=None):
|
||||
doc = state.doc()
|
||||
return anchors.make_anchor(doc, hashing.compute(doc), nid, rev, span, store)
|
||||
|
||||
|
||||
def _edit(state, texts, drop=(), ids=(), corrections=(), move=None):
|
||||
"""A later state: sentence texts replaced, nodes dropped, optionally one node re-parented."""
|
||||
nodes = []
|
||||
for original in state.nodes:
|
||||
if original["id"] in drop:
|
||||
continue
|
||||
n = dict(original)
|
||||
if n["id"] in texts:
|
||||
n["content"] = [text_run(texts[n["id"]])]
|
||||
if move and n["id"] == move[0]:
|
||||
n["parent"], n["ord"] = move[1], move[2]
|
||||
nodes.append(n)
|
||||
return State(
|
||||
state.key,
|
||||
nodes,
|
||||
list(state.ids) + list(ids),
|
||||
list(state.corrections) + list(corrections),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tree():
|
||||
doc = _tree()
|
||||
state = _state(doc)
|
||||
sentences = ids_of_kind(doc, "sentence")
|
||||
paragraphs = ids_of_kind(doc, "paragraph")
|
||||
return {
|
||||
"state": state,
|
||||
"section": ids_of_kind(doc, "section")[0],
|
||||
"heading_sentence": sentences[0],
|
||||
"s1": sentences[1],
|
||||
"s2": sentences[2],
|
||||
"s3": sentences[3],
|
||||
"s4": sentences[4],
|
||||
"p1": paragraphs[0],
|
||||
"p2": paragraphs[1],
|
||||
}
|
||||
|
||||
|
||||
class TestMakeAnchor:
|
||||
def test_a_leaf_anchor_holds_its_hashes_and_the_whole_text_as_the_quote(self, tree):
|
||||
state = tree["state"]
|
||||
doc = state.doc()
|
||||
values = hashing.compute(doc)
|
||||
a = _anchor(state, tree["s1"], rev=7, store="smithlab")
|
||||
assert a == {
|
||||
"format": "scienting-anchor/1",
|
||||
"store": "smithlab",
|
||||
"doc": doc.key,
|
||||
"node": tree["s1"],
|
||||
"kind": "sentence",
|
||||
"own_hash": values[tree["s1"]]["own_hash"],
|
||||
"node_hash": values[tree["s1"]]["node_hash"],
|
||||
"quote": {"exact": FIRST},
|
||||
"rev": 7,
|
||||
}
|
||||
|
||||
def test_without_a_store_there_is_no_store_member(self, tree):
|
||||
assert "store" not in _anchor(tree["state"], tree["s1"])
|
||||
|
||||
def test_a_span_gives_the_quote_with_its_context(self, tree):
|
||||
start = FIRST.index("description")
|
||||
span = [start, start + len("description")]
|
||||
a = _anchor(tree["state"], tree["s1"], span=span)
|
||||
assert a["span"] == span
|
||||
assert a["quote"] == {
|
||||
"exact": "description",
|
||||
"prefix": FIRST[max(0, start - 32) : start],
|
||||
"suffix": FIRST[span[1] : span[1] + 32],
|
||||
}
|
||||
assert len(a["quote"]["prefix"]) == 32
|
||||
|
||||
def test_a_span_at_the_ends_leaves_out_the_empty_context(self, tree):
|
||||
a = _anchor(tree["state"], tree["s1"], span=[0, 8])
|
||||
assert a["quote"] == {"exact": FIRST[:8], "suffix": FIRST[8:40]}
|
||||
end = len(FIRST)
|
||||
b = _anchor(tree["state"], tree["s1"], span=(end - 5, end))
|
||||
assert b["span"] == [end - 5, end]
|
||||
assert b["quote"] == {"exact": FIRST[-5:], "prefix": FIRST[end - 37 : end - 5]}
|
||||
|
||||
def test_a_long_quote_is_held_by_its_hash_and_length(self):
|
||||
long_text = "word " * 50 + "end."
|
||||
doc = build_doc(
|
||||
document(
|
||||
{
|
||||
"kind": "section",
|
||||
"attrs": {"level": 1},
|
||||
"children": [
|
||||
heading("H"),
|
||||
paragraph(sentence(text_run(long_text))),
|
||||
],
|
||||
}
|
||||
)
|
||||
)
|
||||
state = _state(doc)
|
||||
(nid,) = [
|
||||
n for n in ids_of_kind(doc, "sentence") if len(hashing.plain(doc, n)) > 200
|
||||
]
|
||||
a = _anchor(state, nid)
|
||||
assert a["quote"] == {
|
||||
"exact_hash": hashing.PLAIN(long_text),
|
||||
"length": len(long_text),
|
||||
}
|
||||
short = _anchor(state, nid, span=[0, 200])
|
||||
assert short["quote"]["exact"] == long_text[:200]
|
||||
over = _anchor(state, nid, span=[0, 201])
|
||||
assert over["quote"]["length"] == 201
|
||||
assert over["quote"]["exact_hash"] == hashing.PLAIN(norm(long_text[:201]))
|
||||
|
||||
def test_a_cell_anchor_carries_the_table_label_and_header_paths(self):
|
||||
state = package_state(HALE)
|
||||
doc = state.doc()
|
||||
cell = next(
|
||||
n for n in ids_of_kind(doc, "cell") if hashing.plain(doc, n) == "310.0"
|
||||
)
|
||||
a = _anchor(state, cell, store="smithlab")
|
||||
assert a["context"] == {
|
||||
"table_label": "Table 1",
|
||||
"row_path": ["Bond parameters", "CT\N{EN DASH}CT"],
|
||||
"column_path": ["Set A", "$K_r$ (kcal mol⁻¹ Å⁻²)"],
|
||||
}
|
||||
assert a["quote"] == {"exact": "310.0"}
|
||||
|
||||
def test_a_container_with_content_has_no_quote_and_no_leaves(self, tree):
|
||||
a = _anchor(tree["state"], tree["section"])
|
||||
assert a["kind"] == "section"
|
||||
assert not {"quote", "leaves", "span", "context"} & set(a)
|
||||
|
||||
def test_a_container_without_content_lists_its_leaves_with_their_match_hashes(
|
||||
self, tree
|
||||
):
|
||||
state = tree["state"]
|
||||
values = hashing.compute(state.doc())
|
||||
a = _anchor(state, tree["p1"])
|
||||
assert a["leaves"] == [
|
||||
[tree["s1"], values[tree["s1"]]["match_hash"]],
|
||||
[tree["s2"], values[tree["s2"]]["match_hash"]],
|
||||
]
|
||||
assert "quote" not in a
|
||||
|
||||
def test_the_whole_document_is_a_container_without_content(self, tree):
|
||||
state = tree["state"]
|
||||
a = _anchor(state, state.doc().root)
|
||||
assert len(a["leaves"]) == 5
|
||||
|
||||
|
||||
class TestResolveAllNodes:
|
||||
@pytest.mark.parametrize("package", [HALE, "fixtures/packages/tiny2026tiny-j4qm"])
|
||||
def test_an_anchor_on_any_node_of_the_unchanged_document_is_current(self, package):
|
||||
state = package_state(package)
|
||||
doc = state.doc()
|
||||
values = hashing.compute(doc)
|
||||
for nid in doc.order:
|
||||
a = anchors.make_anchor(doc, values, nid, state.rev)
|
||||
got = anchors.resolve(state, a)
|
||||
assert got == {
|
||||
"result": "current",
|
||||
"node": nid,
|
||||
"node_hash": values[nid]["node_hash"],
|
||||
}
|
||||
|
||||
def test_a_span_anchor_on_any_leaf_is_current_with_its_span(self):
|
||||
state = package_state(HALE)
|
||||
doc = state.doc()
|
||||
values = hashing.compute(doc)
|
||||
for nid in doc.order:
|
||||
if doc.nodes[nid]["kind"] not in LEAVES:
|
||||
continue
|
||||
size = len(hashing.plain(doc, nid))
|
||||
span = [size // 3, size // 3 + size // 4]
|
||||
a = anchors.make_anchor(doc, values, nid, state.rev, span)
|
||||
got = anchors.resolve(state, a)
|
||||
assert got["result"] == "current"
|
||||
assert got["span"] == span
|
||||
|
||||
|
||||
# --- resolution ---------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", vectors("resolve"))
|
||||
def test_resolve_vector(case):
|
||||
state = package_state(case["package"])
|
||||
assert anchors.resolve(state, case["anchor"]) == case["expect"]
|
||||
|
||||
|
||||
def test_the_vectors_reach_all_five_results():
|
||||
results = {c["expect"]["result"] for c in read_cases("resolve")}
|
||||
assert results == {"current", "changed-below", "moved", "reattached", "stale"}
|
||||
|
||||
|
||||
class TestResolveLeaf:
|
||||
def test_an_edit_that_keeps_the_quote_once_reattaches_the_anchor(self, tree):
|
||||
anchor = _anchor(tree["state"], tree["s1"])
|
||||
later = _edit(tree["state"], {tree["s1"]: FIRST + " It holds widely."})
|
||||
got = anchors.resolve(later, anchor)
|
||||
assert got["result"] == "reattached"
|
||||
assert got["node"] == tree["s1"]
|
||||
assert "span" not in got
|
||||
|
||||
def test_a_reattached_span_anchor_reports_the_new_span(self, tree):
|
||||
start = FIRST.index("covalent")
|
||||
anchor = _anchor(tree["state"], tree["s1"], span=[start, start + 8])
|
||||
later = _edit(tree["state"], {tree["s1"]: "Indeed, " + FIRST})
|
||||
assert anchors.resolve(later, anchor) == {
|
||||
"result": "reattached",
|
||||
"node": tree["s1"],
|
||||
"node_hash": hashing.compute(later.doc())[tree["s1"]]["node_hash"],
|
||||
"span": [start + 8, start + 16],
|
||||
}
|
||||
|
||||
def test_an_edit_that_removes_the_quote_makes_the_anchor_stale(self, tree):
|
||||
anchor = _anchor(tree["state"], tree["s1"])
|
||||
later = _edit(tree["state"], {tree["s1"]: "Something else entirely."})
|
||||
got = anchors.resolve(later, anchor)
|
||||
assert got["result"] == "stale"
|
||||
assert (
|
||||
got["reason"]
|
||||
== "the node changed and the quote does not occur exactly once"
|
||||
)
|
||||
assert got["node"] == tree["s1"]
|
||||
|
||||
def test_a_quote_without_context_that_occurs_twice_is_stale(self, tree):
|
||||
text = "The cat sat. The cat ran."
|
||||
base = _edit(tree["state"], {tree["s1"]: text})
|
||||
start = text.rindex("cat")
|
||||
anchor = _anchor(base, tree["s1"], span=[start, start + 3])
|
||||
anchor["quote"] = {"exact": "cat"}
|
||||
got = anchors.resolve(_edit(base, {tree["s1"]: text + " More."}), anchor)
|
||||
assert got["result"] == "stale"
|
||||
|
||||
def test_the_prefix_and_suffix_pick_one_of_several_occurrences(self, tree):
|
||||
lead = "Start of the sentence runs on for quite some while here. "
|
||||
text = lead + "The cat sat down. The cat ran off home."
|
||||
base = _edit(tree["state"], {tree["s1"]: text})
|
||||
start = text.rindex("cat")
|
||||
anchor = _anchor(base, tree["s1"], span=[start, start + 3])
|
||||
changed = _edit(base, {tree["s1"]: text.replace("Start", "Begin")})
|
||||
got = anchors.resolve(changed, anchor)
|
||||
assert got["result"] == "reattached"
|
||||
assert got["span"] == [start, start + 3]
|
||||
|
||||
def test_a_long_quote_is_found_again_by_its_hash(self, tree):
|
||||
body = " ".join(f"word{i}" for i in range(60))
|
||||
base = _edit(tree["state"], {tree["s1"]: "Lead. " + body + " Tail."})
|
||||
anchor = _anchor(base, tree["s1"], span=[6, 6 + len(body)])
|
||||
assert "exact_hash" in anchor["quote"]
|
||||
got = anchors.resolve(
|
||||
_edit(base, {tree["s1"]: "New lead. " + body + " Tail."}), anchor
|
||||
)
|
||||
assert got["result"] == "reattached"
|
||||
assert got["span"] == [10, 10 + len(body)]
|
||||
|
||||
def test_an_anchor_without_a_kind_takes_the_kind_of_the_live_node(self, tree):
|
||||
anchor = _anchor(tree["state"], tree["s1"])
|
||||
del anchor["kind"]
|
||||
assert anchors.resolve(tree["state"], anchor)["result"] == "current"
|
||||
|
||||
def test_an_anchor_without_a_quote_on_a_changed_leaf_is_stale(self, tree):
|
||||
anchor = _anchor(tree["state"], tree["s1"])
|
||||
del anchor["quote"]
|
||||
later = _edit(tree["state"], {tree["s1"]: "Changed."})
|
||||
assert anchors.resolve(later, anchor)["result"] == "stale"
|
||||
|
||||
|
||||
class TestIdentifiersThatMoved:
|
||||
def test_an_unknown_node_is_stale(self, tree):
|
||||
anchor = dict(_anchor(tree["state"], tree["s1"]), node="zzzzzzz")
|
||||
assert anchors.resolve(tree["state"], anchor) == {
|
||||
"result": "stale",
|
||||
"node": None,
|
||||
"reason": "unknown node",
|
||||
}
|
||||
|
||||
def test_a_retired_node_without_a_successor_is_stale(self, tree):
|
||||
anchor = _anchor(tree["state"], tree["s1"])
|
||||
gone = _edit(
|
||||
tree["state"],
|
||||
{},
|
||||
drop={tree["s1"]},
|
||||
ids=[{"id": tree["s1"], "kind": "retired", "node_kind": "sentence"}],
|
||||
)
|
||||
assert anchors.resolve(gone, anchor) == {
|
||||
"result": "stale",
|
||||
"node": None,
|
||||
"via": [tree["s1"]],
|
||||
"reason": "retired without a successor",
|
||||
}
|
||||
|
||||
def test_a_chain_of_aliases_that_loops_is_stale(self, tree):
|
||||
anchor = dict(_anchor(tree["state"], tree["s1"]), node="aaaaaaa")
|
||||
ids = [
|
||||
{"id": "aaaaaaa", "kind": "alias", "successor": "bbbbbbb"},
|
||||
{"id": "bbbbbbb", "kind": "alias", "successor": "aaaaaaa"},
|
||||
]
|
||||
got = anchors.resolve(_edit(tree["state"], {}, ids=ids), anchor)
|
||||
assert got["result"] == "stale"
|
||||
assert got["via"] == ["aaaaaaa", "bbbbbbb"]
|
||||
assert got["reason"] == "retired without a successor"
|
||||
|
||||
def test_an_alias_is_followed_and_reported(self, tree):
|
||||
anchor = dict(_anchor(tree["state"], tree["s1"]), node="aaaaaaa")
|
||||
ids = [{"id": "aaaaaaa", "kind": "alias", "successor": tree["s1"]}]
|
||||
got = anchors.resolve(_edit(tree["state"], {}, ids=ids), anchor)
|
||||
assert got["result"] == "current"
|
||||
assert got["node"] == tree["s1"]
|
||||
assert got["via"] == ["aaaaaaa"]
|
||||
|
||||
def test_a_retired_leaf_moves_to_a_successor_that_holds_the_quote(self, tree):
|
||||
anchor = _anchor(tree["state"], tree["s1"])
|
||||
ids = [
|
||||
{
|
||||
"id": tree["s1"],
|
||||
"kind": "retired",
|
||||
"node_kind": "sentence",
|
||||
"successor": tree["s2"],
|
||||
}
|
||||
]
|
||||
later = _edit(
|
||||
tree["state"],
|
||||
{tree["s2"]: SECOND + " " + FIRST},
|
||||
drop={tree["s1"]},
|
||||
ids=ids,
|
||||
)
|
||||
got = anchors.resolve(later, anchor)
|
||||
assert got["result"] == "moved"
|
||||
assert got["node"] == tree["s2"]
|
||||
assert got["via"] == [tree["s1"]]
|
||||
|
||||
def test_a_moved_span_anchor_reports_where_the_span_went(self, tree):
|
||||
start = FIRST.index("common")
|
||||
anchor = _anchor(tree["state"], tree["s1"], span=[start, start + 6])
|
||||
ids = [
|
||||
{
|
||||
"id": tree["s1"],
|
||||
"kind": "retired",
|
||||
"node_kind": "sentence",
|
||||
"successor": tree["s2"],
|
||||
}
|
||||
]
|
||||
successor = "Before. " + FIRST
|
||||
later = _edit(
|
||||
tree["state"], {tree["s2"]: successor}, drop={tree["s1"]}, ids=ids
|
||||
)
|
||||
got = anchors.resolve(later, anchor)
|
||||
assert got["result"] == "moved"
|
||||
assert got["span"] == [start + 8, start + 14]
|
||||
|
||||
def test_a_retired_leaf_whose_successor_lacks_the_quote_is_stale(self, tree):
|
||||
anchor = _anchor(tree["state"], tree["s1"])
|
||||
ids = [
|
||||
{
|
||||
"id": tree["s1"],
|
||||
"kind": "retired",
|
||||
"node_kind": "sentence",
|
||||
"successor": tree["s2"],
|
||||
}
|
||||
]
|
||||
later = _edit(tree["state"], {}, drop={tree["s1"]}, ids=ids)
|
||||
got = anchors.resolve(later, anchor)
|
||||
assert got["result"] == "stale"
|
||||
assert got["reason"] == "the successor does not contain the quote"
|
||||
assert got["node"] == tree["s2"]
|
||||
|
||||
def test_a_retired_span_anchor_needs_exactly_one_hit(self, tree):
|
||||
start = 4
|
||||
anchor = _anchor(tree["state"], tree["s1"], span=[start, start + 4])
|
||||
quote = anchor["quote"]["exact"]
|
||||
ids = [
|
||||
{
|
||||
"id": tree["s1"],
|
||||
"kind": "retired",
|
||||
"node_kind": "sentence",
|
||||
"successor": tree["s2"],
|
||||
}
|
||||
]
|
||||
twice = _edit(
|
||||
tree["state"],
|
||||
{tree["s2"]: f"{quote} and {quote}"},
|
||||
drop={tree["s1"]},
|
||||
ids=ids,
|
||||
)
|
||||
anchor["quote"] = {"exact": quote}
|
||||
assert anchors.resolve(twice, anchor)["result"] == "stale"
|
||||
whole = dict(anchor)
|
||||
del whole["span"]
|
||||
assert anchors.resolve(twice, whole)["result"] == "moved"
|
||||
|
||||
def test_the_kind_comes_from_the_retired_entry_when_the_anchor_has_none(self, tree):
|
||||
anchor = _anchor(tree["state"], tree["p1"])
|
||||
del anchor["kind"]
|
||||
ids = [
|
||||
{
|
||||
"id": tree["p1"],
|
||||
"kind": "retired",
|
||||
"node_kind": "paragraph",
|
||||
"successor": tree["p2"],
|
||||
}
|
||||
]
|
||||
later = _edit(
|
||||
tree["state"],
|
||||
{},
|
||||
drop={tree["p1"], tree["s2"]},
|
||||
ids=ids,
|
||||
move=(tree["s1"], tree["p2"], 5),
|
||||
)
|
||||
assert anchors.resolve(later, anchor)["result"] == "moved"
|
||||
|
||||
|
||||
class TestResolveContainers:
|
||||
def test_a_section_whose_leaves_changed_is_changed_below_with_its_drift(self, tree):
|
||||
anchor = _anchor(tree["state"], tree["section"])
|
||||
correction = {
|
||||
"index": 1,
|
||||
"op": "replace",
|
||||
"path": [tree["section"]],
|
||||
"touched": [tree["s1"]],
|
||||
}
|
||||
later = _edit(
|
||||
tree["state"], {tree["s1"]: "Changed text."}, corrections=[correction]
|
||||
)
|
||||
got = anchors.resolve(later, anchor)
|
||||
assert got["result"] == "changed-below"
|
||||
assert got["drift"] == {"count": 1, "share": "0.2000", "level": "low"}
|
||||
|
||||
def test_drift_is_high_when_the_touched_share_passes_the_threshold(self, tree):
|
||||
anchor = _anchor(tree["state"], tree["section"])
|
||||
corrections = [
|
||||
{
|
||||
"index": 1,
|
||||
"path": [tree["section"]],
|
||||
"touched": [tree["s1"], tree["s2"]],
|
||||
},
|
||||
{
|
||||
"index": 2,
|
||||
"to_path": [tree["section"]],
|
||||
"touched": [tree["s2"], tree["s3"]],
|
||||
},
|
||||
{"index": 3, "path": ["other"], "touched": [tree["s4"]]},
|
||||
{"index": 4, "path": [tree["section"]], "touched": []},
|
||||
]
|
||||
later = _edit(
|
||||
tree["state"],
|
||||
{tree["s1"]: "x one", tree["s3"]: "y two"},
|
||||
corrections=corrections,
|
||||
)
|
||||
got = anchors.resolve(later, anchor)
|
||||
assert got["drift"] == {"count": 2, "share": "0.6000", "level": "high"}
|
||||
|
||||
def test_corrections_before_the_anchor_do_not_count(self, tree):
|
||||
anchor = _anchor(tree["state"], tree["section"], rev=2)
|
||||
corrections = [
|
||||
{"index": 1, "path": [tree["section"]], "touched": [tree["s1"]]},
|
||||
{"index": 2, "path": [tree["section"]], "touched": [tree["s2"]]},
|
||||
{"index": 3, "path": [tree["section"]], "touched": [tree["s3"]]},
|
||||
]
|
||||
later = _edit(tree["state"], {tree["s3"]: "z"}, corrections=corrections)
|
||||
assert anchors.resolve(later, anchor)["drift"]["count"] == 1
|
||||
|
||||
def test_the_threshold_can_be_given(self, tree):
|
||||
anchor = _anchor(tree["state"], tree["section"])
|
||||
corrections = [{"index": 1, "path": [tree["section"]], "touched": [tree["s1"]]}]
|
||||
later = _edit(tree["state"], {tree["s1"]: "x"}, corrections=corrections)
|
||||
assert anchors.resolve(later, anchor)["drift"]["level"] == "low"
|
||||
assert (
|
||||
anchors.resolve(later, anchor, threshold=Fraction(1, 10))["drift"]["level"]
|
||||
== "high"
|
||||
)
|
||||
assert (
|
||||
anchors.resolve(later, anchor, threshold=Fraction(1, 5))["drift"]["level"]
|
||||
== "low"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("touched", "share"),
|
||||
[
|
||||
(["s1"], "0.5000"),
|
||||
(["s1", "s2", "gone"], "1.0000"),
|
||||
(["s1", "gone"], "0.6667"),
|
||||
(["gone"], "0.3333"),
|
||||
],
|
||||
)
|
||||
def test_a_share_is_written_with_four_places_rounded_half_up(
|
||||
self, tree, touched, share
|
||||
):
|
||||
anchor = _anchor(tree["state"], tree["p1"])
|
||||
ids = [tree.get(t, t) for t in touched]
|
||||
corrections = [{"index": 1, "path": [tree["p1"]], "touched": ids}]
|
||||
later = _edit(tree["state"], {tree["s1"]: "changed"}, corrections=corrections)
|
||||
assert anchors.resolve(later, anchor)["drift"]["share"] == share
|
||||
|
||||
def test_drift_is_none_when_no_correction_touched_the_node(self, tree):
|
||||
anchor = _anchor(tree["state"], tree["p1"])
|
||||
later = _edit(tree["state"], {tree["s1"]: "changed"})
|
||||
assert anchors.resolve(later, anchor)["drift"] == {
|
||||
"count": 0,
|
||||
"share": "0.0000",
|
||||
"level": "none",
|
||||
}
|
||||
|
||||
def test_a_section_whose_own_content_changed_is_stale(self, tree):
|
||||
anchor = _anchor(tree["state"], tree["section"])
|
||||
nodes = [
|
||||
dict(n, label="2") if n["id"] == tree["section"] else n
|
||||
for n in tree["state"].nodes
|
||||
]
|
||||
later = State(tree["state"].key, nodes)
|
||||
got = anchors.resolve(later, anchor)
|
||||
assert got["result"] == "stale"
|
||||
assert got["reason"] == "the node's own content changed"
|
||||
|
||||
def test_a_relabel_alone_reattaches_the_anchor(self, tree):
|
||||
anchor = _anchor(tree["state"], tree["section"])
|
||||
nodes = [
|
||||
dict(n, label="2") if n["id"] == tree["section"] else n
|
||||
for n in tree["state"].nodes
|
||||
]
|
||||
relabel = {
|
||||
"index": 1,
|
||||
"op": "relabel",
|
||||
"target": tree["section"],
|
||||
"from": "1",
|
||||
"to": "2",
|
||||
}
|
||||
got = anchors.resolve(
|
||||
State(tree["state"].key, nodes, corrections=[relabel]), anchor
|
||||
)
|
||||
assert got["result"] == "reattached"
|
||||
assert got["reason"] == "relabel"
|
||||
|
||||
def test_a_relabel_beside_another_change_to_the_node_does_not_reattach(self, tree):
|
||||
anchor = _anchor(tree["state"], tree["section"])
|
||||
nodes = [
|
||||
dict(n, label="2") if n["id"] == tree["section"] else n
|
||||
for n in tree["state"].nodes
|
||||
]
|
||||
other = {
|
||||
"index": 2,
|
||||
"op": "replace",
|
||||
"path": [tree["section"]],
|
||||
"touched": [tree["s1"]],
|
||||
}
|
||||
relabel = {"index": 1, "op": "relabel", "target": tree["section"], "from": "1"}
|
||||
got = anchors.resolve(
|
||||
State(tree["state"].key, nodes, corrections=[relabel, other]), anchor
|
||||
)
|
||||
assert got["result"] == "stale"
|
||||
|
||||
def test_a_relabel_to_a_different_original_label_is_stale(self, tree):
|
||||
anchor = _anchor(tree["state"], tree["section"])
|
||||
nodes = [
|
||||
dict(n, label="3") if n["id"] == tree["section"] else n
|
||||
for n in tree["state"].nodes
|
||||
]
|
||||
relabel = {
|
||||
"index": 1,
|
||||
"op": "relabel",
|
||||
"target": tree["section"],
|
||||
"from": "2",
|
||||
"to": "3",
|
||||
}
|
||||
assert (
|
||||
anchors.resolve(
|
||||
State(tree["state"].key, nodes, corrections=[relabel]), anchor
|
||||
)["result"]
|
||||
== "stale"
|
||||
)
|
||||
|
||||
def test_a_paragraph_is_current_or_changed_below(self, tree):
|
||||
anchor = _anchor(tree["state"], tree["p1"])
|
||||
assert anchors.resolve(tree["state"], anchor)["result"] == "current"
|
||||
later = _edit(tree["state"], {tree["s1"]: "changed"})
|
||||
assert anchors.resolve(later, anchor)["result"] == "changed-below"
|
||||
|
||||
def test_a_retired_section_with_the_same_own_content_moved(self, tree):
|
||||
anchor = _anchor(tree["state"], tree["section"])
|
||||
successor = dict(tree["state"].doc().nodes[tree["section"]], id="sec2222")
|
||||
nodes = [
|
||||
dict(n, parent="sec2222") if n.get("parent") == tree["section"] else n
|
||||
for n in tree["state"].nodes
|
||||
]
|
||||
nodes = [n for n in nodes if n["id"] != tree["section"]]
|
||||
nodes.insert(1, successor)
|
||||
ids = [
|
||||
{
|
||||
"id": tree["section"],
|
||||
"kind": "retired",
|
||||
"node_kind": "section",
|
||||
"successor": "sec2222",
|
||||
}
|
||||
]
|
||||
got = anchors.resolve(State(tree["state"].key, nodes, ids), anchor)
|
||||
assert got["result"] == "moved"
|
||||
assert got["node"] == "sec2222"
|
||||
|
||||
def test_a_retired_section_with_different_own_content_is_stale(self, tree):
|
||||
anchor = _anchor(tree["state"], tree["section"])
|
||||
successor = dict(
|
||||
tree["state"].doc().nodes[tree["section"]], id="sec2222", label="9"
|
||||
)
|
||||
nodes = [
|
||||
dict(n, parent="sec2222") if n.get("parent") == tree["section"] else n
|
||||
for n in tree["state"].nodes
|
||||
]
|
||||
nodes = [n for n in nodes if n["id"] != tree["section"]]
|
||||
nodes.insert(1, successor)
|
||||
ids = [
|
||||
{
|
||||
"id": tree["section"],
|
||||
"kind": "retired",
|
||||
"node_kind": "section",
|
||||
"successor": "sec2222",
|
||||
}
|
||||
]
|
||||
got = anchors.resolve(State(tree["state"].key, nodes, ids), anchor)
|
||||
assert got["result"] == "stale"
|
||||
assert got["reason"] == "the node's own content changed"
|
||||
|
||||
def test_a_retired_paragraph_moves_when_half_its_leaves_are_held(self, tree):
|
||||
anchor = _anchor(tree["state"], tree["p1"])
|
||||
ids = [
|
||||
{
|
||||
"id": tree["p1"],
|
||||
"kind": "retired",
|
||||
"node_kind": "paragraph",
|
||||
"successor": tree["p2"],
|
||||
}
|
||||
]
|
||||
later = _edit(
|
||||
tree["state"],
|
||||
{},
|
||||
drop={tree["p1"], tree["s2"]},
|
||||
ids=ids,
|
||||
move=(tree["s1"], tree["p2"], 5),
|
||||
)
|
||||
got = anchors.resolve(later, anchor)
|
||||
assert got["result"] == "moved"
|
||||
assert got["node"] == tree["p2"]
|
||||
|
||||
def test_a_retired_paragraph_holds_a_leaf_that_was_rewritten_with_the_same_match_hash(
|
||||
self, tree
|
||||
):
|
||||
anchor = _anchor(tree["state"], tree["p1"])
|
||||
ids = [
|
||||
{
|
||||
"id": tree["p1"],
|
||||
"kind": "retired",
|
||||
"node_kind": "paragraph",
|
||||
"successor": tree["p2"],
|
||||
}
|
||||
]
|
||||
later = _edit(
|
||||
tree["state"],
|
||||
{tree["s3"]: FIRST},
|
||||
drop={tree["p1"], tree["s1"], tree["s2"]},
|
||||
ids=ids,
|
||||
)
|
||||
assert anchors.resolve(later, anchor)["result"] == "moved"
|
||||
|
||||
def test_a_retired_paragraph_is_stale_when_fewer_than_half_its_leaves_are_held(
|
||||
self, tree
|
||||
):
|
||||
anchor = _anchor(tree["state"], tree["p1"])
|
||||
ids = [
|
||||
{
|
||||
"id": tree["p1"],
|
||||
"kind": "retired",
|
||||
"node_kind": "paragraph",
|
||||
"successor": tree["p2"],
|
||||
}
|
||||
]
|
||||
later = _edit(
|
||||
tree["state"], {}, drop={tree["p1"], tree["s1"], tree["s2"]}, ids=ids
|
||||
)
|
||||
got = anchors.resolve(later, anchor)
|
||||
assert got["result"] == "stale"
|
||||
assert (
|
||||
got["reason"]
|
||||
== "the successor holds fewer than half of the original leaves"
|
||||
)
|
||||
@@ -0,0 +1,110 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""`paperspec build`: the fixtures and vectors it writes are the committed ones, byte for byte."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from conftest import FIXTURES, VECTORS
|
||||
from helpers_cli import copy_repository
|
||||
from paperspec.cli import build, main as cli_main
|
||||
|
||||
# Files that are hand-written or local, not output of the build.
|
||||
NOT_BUILT = {"README.md", ".DS_Store"}
|
||||
|
||||
|
||||
def committed(folder: Path, *, skip_src: bool = False) -> list[str]:
|
||||
"""The built files under `folder`, by path relative to it."""
|
||||
files = []
|
||||
for path in sorted(folder.rglob("*")):
|
||||
rel = path.relative_to(folder)
|
||||
if not path.is_file() or path.name in NOT_BUILT:
|
||||
continue
|
||||
if skip_src and rel.parts[0] == "src":
|
||||
continue
|
||||
files.append(rel.as_posix())
|
||||
return files
|
||||
|
||||
|
||||
FIXTURE_FILES = committed(FIXTURES, skip_src=True)
|
||||
VECTOR_FILES = committed(VECTORS)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def rebuilt(tmp_path_factory):
|
||||
out = tmp_path_factory.mktemp("rebuilt")
|
||||
status = cli_main.main(["build", "--out", str(out)])
|
||||
assert status == 0
|
||||
return out
|
||||
|
||||
|
||||
class TestBuildElsewhere:
|
||||
@pytest.mark.parametrize("name", VECTOR_FILES)
|
||||
def test_every_vector_file_is_rebuilt_byte_for_byte(self, rebuilt, name):
|
||||
assert (rebuilt / "vectors" / name).read_bytes() == (
|
||||
VECTORS / name
|
||||
).read_bytes()
|
||||
|
||||
@pytest.mark.parametrize("name", FIXTURE_FILES)
|
||||
def test_every_fixture_file_is_rebuilt_byte_for_byte(self, rebuilt, name):
|
||||
assert (rebuilt / "fixtures" / name).read_bytes() == (
|
||||
FIXTURES / name
|
||||
).read_bytes()
|
||||
|
||||
def test_it_builds_exactly_the_committed_vector_files(self, rebuilt):
|
||||
assert committed(rebuilt / "vectors") == VECTOR_FILES
|
||||
|
||||
def test_it_builds_exactly_the_committed_fixture_files(self, rebuilt):
|
||||
assert committed(rebuilt / "fixtures", skip_src=True) == FIXTURE_FILES
|
||||
|
||||
def test_the_sources_the_cases_read_are_copied_beside_the_output(self, rebuilt):
|
||||
for name in committed(FIXTURES / "src"):
|
||||
assert (rebuilt / "fixtures" / "src" / name).read_bytes() == (
|
||||
FIXTURES / "src" / name
|
||||
).read_bytes()
|
||||
assert (rebuilt / "registry" / "rules.json").is_file()
|
||||
|
||||
def test_a_second_build_replaces_what_the_first_left(self, rebuilt, tmp_path):
|
||||
out = tmp_path / "again"
|
||||
(out / "vectors").mkdir(parents=True)
|
||||
(out / "vectors" / "stale.jsonl").write_text("{}\n")
|
||||
(out / "vectors" / "kept.txt").write_text("kept\n")
|
||||
(out / "fixtures" / "packages" / "stale").mkdir(parents=True)
|
||||
(out / "fixtures" / "packages" / "stale" / "file").write_text("x")
|
||||
assert cli_main.main(["build", "--out", str(out)]) == 0
|
||||
assert not (out / "vectors" / "stale.jsonl").exists()
|
||||
assert not (out / "fixtures" / "packages" / "stale").exists()
|
||||
assert (out / "vectors" / "kept.txt").read_text() == "kept\n"
|
||||
assert committed(out / "vectors") == sorted([*VECTOR_FILES, "kept.txt"])
|
||||
|
||||
def test_it_prints_nothing_when_it_builds_elsewhere(self, tmp_path, capsys):
|
||||
assert cli_main.main(["build", "--out", str(tmp_path / "quiet")]) == 0
|
||||
assert capsys.readouterr() == ("", "")
|
||||
|
||||
|
||||
class TestBuildInPlace:
|
||||
def test_rebuilding_a_checkout_leaves_it_as_it_was(
|
||||
self, tmp_path, monkeypatch, capsys
|
||||
):
|
||||
root = copy_repository(tmp_path)
|
||||
before = {n: (root / "vectors" / n).read_bytes() for n in VECTOR_FILES}
|
||||
before |= {
|
||||
"fixtures/" + n: (root / "fixtures" / n).read_bytes() for n in FIXTURE_FILES
|
||||
}
|
||||
(root / "vectors" / "normalize.jsonl").write_text("{}\n")
|
||||
monkeypatch.setattr(build, "ROOT", root)
|
||||
status = cli_main.main(["build"])
|
||||
out = capsys.readouterr().out
|
||||
assert status == 0
|
||||
assert out.startswith(f"Rebuilt fixtures/ and vectors/ in {root}.")
|
||||
assert "git diff" in out
|
||||
after = {n: (root / "vectors" / n).read_bytes() for n in VECTOR_FILES}
|
||||
after |= {
|
||||
"fixtures/" + n: (root / "fixtures" / n).read_bytes() for n in FIXTURE_FILES
|
||||
}
|
||||
assert after == before
|
||||
@@ -0,0 +1,157 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""`paperspec check`: the repository passes its own checker, and the report names a failure."""
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import io
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from conftest import ROOT
|
||||
from paperspec.cli import build, check, main as cli_main
|
||||
|
||||
SECTIONS = [
|
||||
"counts",
|
||||
"data",
|
||||
"docs",
|
||||
"entities",
|
||||
"invalid",
|
||||
"links",
|
||||
"log",
|
||||
"packages",
|
||||
"prose",
|
||||
"records",
|
||||
"registry",
|
||||
"schemas",
|
||||
"site",
|
||||
"sql",
|
||||
"traces",
|
||||
"vectors",
|
||||
"zips",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def clean_state(monkeypatch):
|
||||
"""Empty the module's counters and failures, and put them back afterwards."""
|
||||
monkeypatch.setattr(check, "COUNTS", {})
|
||||
monkeypatch.setattr(check, "FAILS", [])
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def quick_report():
|
||||
"""The status and output of `paperspec check --quick` on this repository, run once."""
|
||||
|
||||
out = io.StringIO()
|
||||
check.COUNTS.clear()
|
||||
check.FAILS.clear()
|
||||
with contextlib.redirect_stdout(out):
|
||||
status = cli_main.main(["check", "--quick"])
|
||||
return status, out.getvalue()
|
||||
|
||||
|
||||
class TestQuickCheck:
|
||||
def test_the_repository_passes(self, quick_report):
|
||||
status, out = quick_report
|
||||
assert status == 0, out
|
||||
assert out.rstrip().endswith("All checks pass.")
|
||||
|
||||
@pytest.mark.parametrize("section", SECTIONS)
|
||||
def test_every_section_counts_its_checks(self, quick_report, section):
|
||||
_, out = quick_report
|
||||
match = re.search(rf"^{section} +(\d+) checks$", out, re.M)
|
||||
assert match is not None, f"no line for {section}"
|
||||
assert int(match[1]) > 0
|
||||
|
||||
def test_it_skips_the_rebuild(self, quick_report):
|
||||
_, out = quick_report
|
||||
assert "determinism" not in out
|
||||
|
||||
|
||||
class TestDeterminism:
|
||||
@pytest.fixture
|
||||
def in_process_build(self, clean_state, monkeypatch):
|
||||
"""Run the subprocess `python -m paperspec build --out DIR` as a call, so no interpreter is spawned."""
|
||||
calls = []
|
||||
|
||||
def fake_run(argv, *, check):
|
||||
calls.append(argv)
|
||||
assert argv[1:4] == ["-m", "paperspec", "build"]
|
||||
assert build.run(argparse.Namespace(out=argv[-1])) in (0, None)
|
||||
|
||||
monkeypatch.setattr(check.subprocess, "run", fake_run)
|
||||
return calls
|
||||
|
||||
def test_a_rebuild_reproduces_fixtures_and_vectors(self, in_process_build):
|
||||
check.check_determinism()
|
||||
assert check.FAILS == []
|
||||
assert check.COUNTS == {"determinism": 2}
|
||||
assert len(in_process_build) == 1
|
||||
|
||||
def test_a_differing_file_is_reported(self, in_process_build, monkeypatch):
|
||||
real_build = build.run
|
||||
|
||||
def tampered(args):
|
||||
real_build(args)
|
||||
(Path(args.out) / "vectors" / "hashes.jsonl").write_text("{}\n")
|
||||
|
||||
monkeypatch.setattr(build, "run", tampered)
|
||||
check.check_determinism()
|
||||
assert len(check.FAILS) == 1
|
||||
assert check.FAILS[0].startswith("determinism: rebuild differs")
|
||||
assert "vectors/hashes.jsonl" in check.FAILS[0]
|
||||
|
||||
|
||||
class TestReport:
|
||||
@pytest.fixture
|
||||
def no_checks(self, clean_state, monkeypatch):
|
||||
"""Replace every check by a no-op, so `run` only reports."""
|
||||
for name in dir(check):
|
||||
if name.startswith("check_"):
|
||||
monkeypatch.setattr(check, name, lambda: None)
|
||||
|
||||
def test_a_clean_run_counts_and_passes(self, no_checks, capsys):
|
||||
check.ok("demo", True, "unused")
|
||||
assert check.run(argparse.Namespace(quick=True)) == 0
|
||||
out = capsys.readouterr().out
|
||||
assert re.search(r"^demo +1 checks$", out, re.M)
|
||||
assert out.rstrip().endswith("All checks pass.")
|
||||
|
||||
def test_a_failure_is_listed_and_fails_the_run(self, no_checks, capsys):
|
||||
check.ok("demo", False, "it broke")
|
||||
assert check.run(argparse.Namespace(quick=True)) == 1
|
||||
out = capsys.readouterr().out
|
||||
assert "1 failures:" in out
|
||||
assert " demo: it broke" in out
|
||||
assert "All checks pass." not in out
|
||||
|
||||
def test_a_determinism_failure_says_to_rebuild(self, no_checks, capsys):
|
||||
check.ok("determinism", False, "rebuild differs")
|
||||
assert check.run(argparse.Namespace(quick=True)) == 1
|
||||
assert "run paperspec build" in capsys.readouterr().out
|
||||
|
||||
def test_it_lists_sixty_failures_and_counts_the_rest(self, no_checks, capsys):
|
||||
for i in range(63):
|
||||
check.ok("demo", False, f"case {i}")
|
||||
assert check.run(argparse.Namespace(quick=True)) == 1
|
||||
out = capsys.readouterr().out
|
||||
assert " demo: case 59" in out
|
||||
assert "demo: case 60" not in out
|
||||
assert "and 3 more" in out
|
||||
|
||||
def test_the_full_check_runs_the_rebuild(self, no_checks, monkeypatch, capsys):
|
||||
ran = []
|
||||
monkeypatch.setattr(check, "check_determinism", lambda: ran.append(True))
|
||||
assert check.run(argparse.Namespace(quick=False)) == 0
|
||||
assert ran == [True]
|
||||
|
||||
|
||||
def test_the_checkout_is_this_repository():
|
||||
assert ROOT == check.ROOT
|
||||
@@ -0,0 +1,279 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""The `paperspec` command: parsing, help, usage errors, exit statuses and `python -m paperspec`."""
|
||||
|
||||
import runpy
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
from conftest import ROOT
|
||||
from paperspec import UNICODE_VERSION, __version__, cli
|
||||
from paperspec.cli import main as cli_main
|
||||
from paperspec.package import ZIP_LIMIT
|
||||
|
||||
COMMANDS = ["validate", "load", "check", "build", "stress", "site"]
|
||||
|
||||
|
||||
def run_main(argv, capsys):
|
||||
"""Run `paperspec ARGV`, returning its exit status and the text it printed."""
|
||||
try:
|
||||
status = cli_main.main(argv)
|
||||
except SystemExit as stop:
|
||||
status = stop.code
|
||||
captured = capsys.readouterr()
|
||||
return status, captured.out, captured.err
|
||||
|
||||
|
||||
class TestParser:
|
||||
def test_knows_every_command(self):
|
||||
_, subs = cli_main.parser()
|
||||
assert list(subs) == COMMANDS
|
||||
assert list(cli_main.COMMANDS) == COMMANDS
|
||||
|
||||
def test_validate_takes_packages_a_proposal_file_and_a_size_limit(self):
|
||||
top, _ = cli_main.parser()
|
||||
args = top.parse_args(
|
||||
[
|
||||
"validate",
|
||||
"--correction",
|
||||
"fix.json",
|
||||
"--max-size",
|
||||
"9",
|
||||
"a",
|
||||
"b.sct.zip",
|
||||
]
|
||||
)
|
||||
assert args.command == "validate"
|
||||
assert args.correction == "fix.json"
|
||||
assert args.max_size == 9
|
||||
assert args.packages == ["a", "b.sct.zip"]
|
||||
|
||||
def test_the_size_limit_defaults_to_the_zip_limit(self):
|
||||
top, _ = cli_main.parser()
|
||||
assert top.parse_args(["validate", "a"]).max_size == ZIP_LIMIT
|
||||
assert top.parse_args(["load", "--sqlite", "x.db", "a"]).max_size == ZIP_LIMIT
|
||||
|
||||
def test_load_takes_the_bundle_options(self):
|
||||
top, _ = cli_main.parser()
|
||||
args = top.parse_args(
|
||||
[
|
||||
"load",
|
||||
"--sqlite",
|
||||
"x.db",
|
||||
"--bundle",
|
||||
"--built-from",
|
||||
"4fa326d",
|
||||
"--store",
|
||||
"lab",
|
||||
"p",
|
||||
]
|
||||
)
|
||||
assert (args.sqlite, args.bundle, args.built_from, args.store) == (
|
||||
"x.db",
|
||||
True,
|
||||
"4fa326d",
|
||||
"lab",
|
||||
)
|
||||
|
||||
def test_check_has_a_quick_flag(self):
|
||||
top, _ = cli_main.parser()
|
||||
assert top.parse_args(["check"]).quick is False
|
||||
assert top.parse_args(["check", "--quick"]).quick is True
|
||||
|
||||
def test_build_and_site_take_an_output_folder(self):
|
||||
top, _ = cli_main.parser()
|
||||
assert top.parse_args(["build"]).out is None
|
||||
assert top.parse_args(["build", "--out", "o"]).out == "o"
|
||||
args = top.parse_args(["site", "--out", "o", "--base-url", "http://x"])
|
||||
assert (args.out, args.base_url) == ("o", "http://x")
|
||||
|
||||
def test_stress_takes_a_seed_and_a_round_count(self):
|
||||
top, _ = cli_main.parser()
|
||||
args = top.parse_args(["stress"])
|
||||
assert (args.seed, args.rounds) == (20261014, 200)
|
||||
args = top.parse_args(["stress", "--seed", "7", "--rounds", "5"])
|
||||
assert (args.seed, args.rounds) == (7, 5)
|
||||
|
||||
|
||||
class TestHelp:
|
||||
def test_the_top_help_lists_the_commands_and_where_they_work(self, capsys):
|
||||
status, out, _ = run_main(["--help"], capsys)
|
||||
assert status == 0
|
||||
for name in COMMANDS:
|
||||
assert name in out
|
||||
assert "where each command works" in out
|
||||
assert cli_main.SITE in out
|
||||
|
||||
@pytest.mark.parametrize("command", COMMANDS)
|
||||
def test_every_command_prints_its_own_help(self, command, capsys):
|
||||
status, out, _ = run_main([command, "--help"], capsys)
|
||||
assert status == 0
|
||||
assert out.startswith(f"usage: paperspec {command}")
|
||||
assert "examples:" in out
|
||||
|
||||
def test_validate_help_names_the_exit_statuses(self, capsys):
|
||||
_, out, _ = run_main(["validate", "--help"], capsys)
|
||||
assert "Exit status: 0 when everything is valid" in out
|
||||
assert str(ZIP_LIMIT) in out
|
||||
|
||||
def test_the_version_names_the_package_and_unicode(self, capsys):
|
||||
status, out, _ = run_main(["--version"], capsys)
|
||||
assert status == 0
|
||||
assert out.strip() == f"paperspec {__version__} (Unicode {UNICODE_VERSION})"
|
||||
|
||||
def test_no_command_prints_the_help_and_exits_2(self, capsys):
|
||||
status, out, _ = run_main([], capsys)
|
||||
assert status == 2
|
||||
assert "usage: paperspec" in out
|
||||
assert "examples:" in out
|
||||
|
||||
|
||||
class TestUsageErrors:
|
||||
def test_an_unknown_command_is_refused(self, capsys):
|
||||
status, _, err = run_main(["frobnicate"], capsys)
|
||||
assert status == 2
|
||||
assert "invalid choice: 'frobnicate'" in err
|
||||
|
||||
def test_a_misspelled_command_is_suggested(self, capsys):
|
||||
_, _, err = run_main(["validat", "x"], capsys)
|
||||
assert "validate" in err
|
||||
|
||||
def test_an_unknown_option_is_refused(self, capsys):
|
||||
status, _, err = run_main(["check", "--fast"], capsys)
|
||||
assert status == 2
|
||||
assert "unrecognized arguments: --fast" in err
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("argv", "missing"),
|
||||
[
|
||||
(["validate"], "PACKAGE"),
|
||||
(["load", "x"], "--sqlite"),
|
||||
(["load", "--sqlite", "x.db"], "PACKAGE"),
|
||||
],
|
||||
)
|
||||
def test_a_missing_argument_is_refused(self, argv, missing, capsys):
|
||||
status, _, err = run_main(argv, capsys)
|
||||
assert status == 2
|
||||
assert missing in err
|
||||
|
||||
def test_a_size_limit_must_be_a_number(self, capsys):
|
||||
status, _, err = run_main(["validate", "--max-size", "big", "x"], capsys)
|
||||
assert status == 2
|
||||
assert "invalid int value: 'big'" in err
|
||||
|
||||
def test_a_usage_error_of_a_command_prints_its_usage_and_exits_2(self, capsys):
|
||||
status, _, err = run_main(
|
||||
["validate", "--correction", "c.json", "a", "b"], capsys
|
||||
)
|
||||
assert status == 2
|
||||
assert err.startswith("usage: paperspec validate")
|
||||
assert "--correction checks proposals against exactly one package" in err
|
||||
|
||||
|
||||
class TestRun:
|
||||
def test_a_command_that_returns_nothing_exits_0(self, monkeypatch, capsys):
|
||||
monkeypatch.setattr("paperspec.cli.validate.run", lambda args: None)
|
||||
assert run_main(["validate", "x"], capsys)[0] == 0
|
||||
|
||||
def test_a_command_returns_its_own_status(self, monkeypatch, capsys):
|
||||
monkeypatch.setattr("paperspec.cli.validate.run", lambda args: 1)
|
||||
assert run_main(["validate", "x"], capsys)[0] == 1
|
||||
|
||||
def test_an_interrupt_exits_130(self, monkeypatch, capsys):
|
||||
def interrupted(args):
|
||||
raise KeyboardInterrupt
|
||||
|
||||
monkeypatch.setattr("paperspec.cli.validate.run", interrupted)
|
||||
assert run_main(["validate", "x"], capsys)[0] == 130
|
||||
|
||||
def test_python_dash_m_runs_the_same_program(self, monkeypatch, capsys):
|
||||
monkeypatch.setattr(sys, "argv", ["paperspec", "--version"])
|
||||
with pytest.raises(SystemExit) as stop:
|
||||
runpy.run_module("paperspec", run_name="__main__")
|
||||
assert stop.value.code == 0
|
||||
assert capsys.readouterr().out.startswith(f"paperspec {__version__}")
|
||||
|
||||
def test_python_dash_m_exits_with_the_status_of_main(self, monkeypatch, capsys):
|
||||
monkeypatch.setattr(sys, "argv", ["paperspec"])
|
||||
with pytest.raises(SystemExit) as stop:
|
||||
runpy.run_module("paperspec", run_name="__main__")
|
||||
assert stop.value.code == 2
|
||||
assert "usage: paperspec" in capsys.readouterr().out
|
||||
|
||||
|
||||
class TestRequire:
|
||||
def test_installed_packages_pass(self):
|
||||
assert cli.require("json", "os") is None
|
||||
|
||||
def test_one_missing_package_is_named_with_what_to_install(self):
|
||||
with pytest.raises(SystemExit) as stop:
|
||||
cli.require("json", "no_such_package_here")
|
||||
message = str(stop.value.code)
|
||||
assert "needs the package no_such_package_here" in message
|
||||
assert "paperspec[tools]" in message
|
||||
assert cli.INSTALL in message
|
||||
|
||||
def test_several_missing_packages_are_listed(self):
|
||||
with pytest.raises(SystemExit) as stop:
|
||||
cli.require("no_such_a", "no_such_b")
|
||||
assert "the packages no_such_a and no_such_b" in str(stop.value.code)
|
||||
|
||||
|
||||
class FakeCheckout:
|
||||
"""A folder with the four marks of a paperspec checkout."""
|
||||
|
||||
def __init__(self, root):
|
||||
self.root = root
|
||||
(root / "spec").mkdir(parents=True)
|
||||
(root / "registry").mkdir()
|
||||
(root / "fixtures" / "src").mkdir(parents=True)
|
||||
(root / "pyproject.toml").write_text("")
|
||||
|
||||
|
||||
class TestCheckout:
|
||||
def test_an_editable_install_uses_the_checkout_it_is_in(self):
|
||||
assert cli.checkout() == str(ROOT)
|
||||
|
||||
def test_an_installed_wheel_uses_the_nearest_checkout_above_the_current_folder(
|
||||
self, tmp_path, monkeypatch
|
||||
):
|
||||
site = tmp_path / "site-packages" / "paperspec" / "cli"
|
||||
site.mkdir(parents=True)
|
||||
monkeypatch.setattr(cli, "__file__", str(site / "__init__.py"))
|
||||
repo = tmp_path / "repo"
|
||||
FakeCheckout(repo)
|
||||
inner = repo / "docs" / "deep"
|
||||
inner.mkdir(parents=True)
|
||||
monkeypatch.chdir(inner)
|
||||
assert cli.checkout() == str(repo.resolve())
|
||||
|
||||
def test_outside_any_checkout_it_says_what_to_do(self, tmp_path, monkeypatch):
|
||||
site = tmp_path / "site-packages" / "paperspec" / "cli"
|
||||
site.mkdir(parents=True)
|
||||
monkeypatch.setattr(cli, "__file__", str(site / "__init__.py"))
|
||||
elsewhere = tmp_path / "elsewhere"
|
||||
elsewhere.mkdir()
|
||||
monkeypatch.chdir(elsewhere)
|
||||
with pytest.raises(SystemExit) as stop:
|
||||
cli.checkout()
|
||||
message = str(stop.value.code)
|
||||
assert "is not inside one" in message
|
||||
assert cli.REPOSITORY in message
|
||||
assert "paperspec validate and paperspec load work anywhere" in message
|
||||
|
||||
def test_a_folder_missing_one_mark_is_not_a_checkout(self, tmp_path, monkeypatch):
|
||||
site = tmp_path / "site-packages" / "paperspec" / "cli"
|
||||
site.mkdir(parents=True)
|
||||
monkeypatch.setattr(cli, "__file__", str(site / "__init__.py"))
|
||||
whole = tmp_path / "whole"
|
||||
FakeCheckout(whole)
|
||||
partial = whole / "partial"
|
||||
FakeCheckout(partial)
|
||||
(partial / "registry").rmdir()
|
||||
monkeypatch.chdir(partial)
|
||||
assert cli.checkout() == str(whole.resolve())
|
||||
@@ -0,0 +1,678 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""`paperspec site`: the Markdown renderer, the page layout and the folder the build writes."""
|
||||
|
||||
import json
|
||||
import pathlib
|
||||
import re
|
||||
|
||||
import pytest
|
||||
|
||||
from conftest import ROOT
|
||||
from paperspec.cli import main as cli_main, site
|
||||
|
||||
GUIDE = """\
|
||||
# Guide
|
||||
|
||||
Intro with `code <b>`, <https://example.org/x> and a [link](other.md#part) to **bold**
|
||||
and *emphasis* text.
|
||||
|
||||
## Part
|
||||
|
||||
### Part
|
||||
|
||||
1. first
|
||||
2. second
|
||||
- nested a
|
||||
- nested b
|
||||
|
||||
3. third
|
||||
|
||||
- bullet
|
||||
|
||||
with a second paragraph
|
||||
- tail
|
||||
lazy line
|
||||
|
||||
5. five
|
||||
|
||||
| Name | Value |
|
||||
| --- | --- |
|
||||
| `a|b` | 1 |
|
||||
| c | 2 |
|
||||
|
||||
Term
|
||||
: Definition of the term
|
||||
continued
|
||||
|
||||
```python
|
||||
if a < b:
|
||||
print("x")
|
||||
```
|
||||
|
||||
closing paragraph
|
||||
"""
|
||||
|
||||
|
||||
class TestInline:
|
||||
@pytest.mark.parametrize(
|
||||
("source", "html"),
|
||||
[
|
||||
("plain text", "plain text"),
|
||||
("a *b* c", "a <em>b</em> c"),
|
||||
("a **b** c", "a <strong>b</strong> c"),
|
||||
(
|
||||
"**bold *and em* inside**",
|
||||
"<strong>bold <em>and em</em> inside</strong>",
|
||||
),
|
||||
("2*3*4", "2*3*4"),
|
||||
("a * b * c", "a * b * c"),
|
||||
("`x < y`", "<code>x < y</code>"),
|
||||
("``a`b``", "<code>a`b</code>"),
|
||||
("`*not em*`", "<code>*not em*</code>"),
|
||||
("a & b < c", "a & b < c"),
|
||||
(
|
||||
"<https://e.org/a?b=1&c=2>",
|
||||
'<a href="https://e.org/a?b=1&c=2">https://e.org/a?b=1&c=2</a>',
|
||||
),
|
||||
("[text](https://e.org)", '<a href="https://e.org">text</a>'),
|
||||
("[text](docs/a.md#frag)", '<a href="docs/a.html#frag">text</a>'),
|
||||
("[**b**](README.md)", '<a href="./"><strong>b</strong></a>'),
|
||||
("[`c`](a.md)", '<a href="a.html"><code>c</code></a>'),
|
||||
],
|
||||
)
|
||||
def test_renders_the_constructs_the_documentation_uses(self, source, html):
|
||||
assert site.inline(source) == html
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("source", "text"),
|
||||
[
|
||||
("`code` and [link](x.md) and **strong**", "code and link and strong"),
|
||||
(" padded ", "padded"),
|
||||
("*emphasis*", "emphasis"),
|
||||
],
|
||||
)
|
||||
def test_plain_drops_the_markup(self, source, text):
|
||||
assert site.plain(source) == text
|
||||
|
||||
|
||||
class TestLinks:
|
||||
@pytest.mark.parametrize(
|
||||
("target", "href"),
|
||||
[
|
||||
("https://example.org/a.md", "https://example.org/a.md"),
|
||||
("mailto:a@b.c", "mailto:a@b.c"),
|
||||
("/abs/path.md", "/abs/path.md"),
|
||||
("#frag", "#frag"),
|
||||
("a.md", "a.html"),
|
||||
("dir/b.md#sec", "dir/b.html#sec"),
|
||||
("README.md", "./"),
|
||||
("dir/README.md", "dir/"),
|
||||
("dir/README.md#x", "dir/#x"),
|
||||
("file.json", "file.json"),
|
||||
("../up.md", "../up.html"),
|
||||
],
|
||||
)
|
||||
def test_markdown_targets_become_page_links(self, target, href):
|
||||
assert site.rewrite_href(target) == href
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("heading", "anchor"),
|
||||
[
|
||||
("Hello, World!", "hello-world"),
|
||||
("`code` and **bold**", "code-and-bold"),
|
||||
("Part 2.1 \u2013 Results", "part-21--results"),
|
||||
("snake_case name", "snake_case-name"),
|
||||
(" Padded ", "padded"),
|
||||
("Ångström units", "ångström-units"),
|
||||
],
|
||||
)
|
||||
def test_slug_follows_the_forge_rule(self, heading, anchor):
|
||||
assert site.slug(heading) == anchor
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("md", "path", "url"),
|
||||
[
|
||||
("README.md", "index.html", ""),
|
||||
("docs/overview.md", "docs/overview.html", "docs/overview.html"),
|
||||
("docs/README.md", "docs/index.html", "docs/"),
|
||||
("CHANGELOG.md", "CHANGELOG.html", "CHANGELOG.html"),
|
||||
],
|
||||
)
|
||||
def test_where_a_page_goes(self, md, path, url):
|
||||
assert site.page_path(md) == path
|
||||
assert site.page_url(md) == url
|
||||
|
||||
def test_the_vocabulary_page_has_its_own_address(self):
|
||||
assert site.page_url("vocab") == "vocab/"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("source", "target", "link"),
|
||||
[
|
||||
("index.html", "", "./"),
|
||||
("index.html", "docs/a.html", "docs/a.html"),
|
||||
("docs/a.html", "docs/b.html", "b.html"),
|
||||
("docs/a.html", "", "../"),
|
||||
("docs/a.html", "docs/", "./"),
|
||||
("docs/deep/a.html", "docs/", "../"),
|
||||
("docs/a.html", "schemas/x.json", "../schemas/x.json"),
|
||||
("docs/a.html", "vocab/", "../vocab/"),
|
||||
],
|
||||
)
|
||||
def test_relative_links_between_pages(self, source, target, link):
|
||||
assert site.rel(source, target) == link
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("size", "text"),
|
||||
[
|
||||
(0, "0 B"),
|
||||
(1023, "1023 B"),
|
||||
(1024, "1.0 KB"),
|
||||
(1536, "1.5 KB"),
|
||||
(5 * 1024**2, "5.0 MB"),
|
||||
(5 * 1024**3, "5120.0 MB"),
|
||||
],
|
||||
)
|
||||
def test_sizes_read_in_the_nearest_unit(self, size, text):
|
||||
assert site.size_text(size) == text
|
||||
|
||||
|
||||
class TestBlocks:
|
||||
def render(self, text):
|
||||
body, renderer = site.render_markdown(text)
|
||||
return body, renderer
|
||||
|
||||
def test_a_heading_gets_an_id_and_the_first_h1_is_the_title(self):
|
||||
body, renderer = self.render("# Title\n\n## Sub *one*\n")
|
||||
assert body == (
|
||||
'<h1 id="title">Title</h1>\n'
|
||||
'<h2 id="sub-one">Sub <em>one</em>'
|
||||
'<a class="anchor" href="#sub-one" aria-label="Link to this section">#</a></h2>'
|
||||
)
|
||||
assert renderer.title == "Title"
|
||||
assert renderer.headings == [
|
||||
{"level": 1, "text": "Title", "id": "title"},
|
||||
{"level": 2, "text": "Sub one", "id": "sub-one"},
|
||||
]
|
||||
|
||||
def test_equal_headings_get_numbered_ids_like_the_forge_does(self):
|
||||
_, renderer = self.render("## Part\n\n### Part\n\n## Part\n")
|
||||
assert [h["id"] for h in renderer.headings] == ["part", "part-1", "part-2"]
|
||||
|
||||
def test_a_page_without_an_h1_has_no_title(self):
|
||||
_, renderer = self.render("## Only\n")
|
||||
assert renderer.title is None
|
||||
|
||||
def test_only_the_first_h1_names_the_page(self):
|
||||
_, renderer = self.render("# One\n\n# Two\n")
|
||||
assert renderer.title == "One"
|
||||
|
||||
def test_consecutive_lines_form_one_paragraph(self):
|
||||
body, _ = self.render("one\ntwo\n\nthree\n")
|
||||
assert body == "<p>one\ntwo</p>\n<p>three</p>"
|
||||
|
||||
def test_a_fenced_block_keeps_its_text_and_names_its_language(self):
|
||||
body, _ = self.render("```python\nif a < b:\n pass\n```\n")
|
||||
assert (
|
||||
body
|
||||
== '<pre><code class="language-python">if a < b:\n pass</code></pre>'
|
||||
)
|
||||
|
||||
def test_a_fence_without_a_language_has_no_class(self):
|
||||
body, _ = self.render("```\n# not a heading\n```\n")
|
||||
assert body == "<pre><code># not a heading</code></pre>"
|
||||
|
||||
def test_an_indented_fence_is_dedented(self):
|
||||
body, _ = self.render(" ```\n a\n b\n ```\n")
|
||||
assert body == "<pre><code>a\n b</code></pre>"
|
||||
|
||||
def test_a_table_has_a_head_and_a_body(self):
|
||||
body, _ = self.render("| A | B |\n| --- | :-: |\n| `x|y` | 2 |\n")
|
||||
assert body == (
|
||||
'<div class="table-wrap"><table><thead><tr><th>A</th><th>B</th></tr></thead>'
|
||||
"<tbody><tr><td><code>x|y</code></td><td>2</td></tr></tbody></table></div>"
|
||||
)
|
||||
|
||||
def test_rows_may_leave_out_the_closing_pipe(self):
|
||||
body, _ = self.render("| A | B\n| --- | ---\n| c | d\n")
|
||||
assert body == (
|
||||
'<div class="table-wrap"><table><thead><tr><th>A</th><th>B</th></tr></thead>'
|
||||
"<tbody><tr><td>c</td><td>d</td></tr></tbody></table></div>"
|
||||
)
|
||||
|
||||
def test_a_table_without_a_rule_row_has_only_body_rows_after_the_first(self):
|
||||
body, _ = self.render("| A | B |\n| c | d |\n")
|
||||
assert "<thead><tr><th>A</th><th>B</th></tr></thead>" in body
|
||||
assert "<td>c</td><td>d</td>" in body
|
||||
|
||||
def test_a_definition_list(self):
|
||||
body, _ = self.render("Term\n: Meaning\n more\n\nOther\n: Second\n")
|
||||
assert body == (
|
||||
"<dl><dt>Term</dt><dd>Meaning\nmore</dd><dt>Other</dt><dd>Second</dd></dl>"
|
||||
)
|
||||
|
||||
def test_a_bullet_list(self):
|
||||
body, _ = self.render("- a\n- b\n")
|
||||
assert body == "<ul><li>a</li><li>b</li></ul>"
|
||||
|
||||
def test_an_ordered_list_keeps_its_start(self):
|
||||
body, _ = self.render("3. c\n4. d\n")
|
||||
assert body == '<ol start="3"><li>c</li><li>d</li></ol>'
|
||||
|
||||
def test_a_nested_list(self):
|
||||
body, _ = self.render("- a\n - b\n - c\n- d\n")
|
||||
assert body == (
|
||||
"<ul><li><p>a</p>\n<ul><li>b</li><li>c</li></ul></li><li>d</li></ul>"
|
||||
)
|
||||
|
||||
def test_an_item_with_two_paragraphs(self):
|
||||
body, _ = self.render("- a\n\n more\n- b\n")
|
||||
assert body == "<ul><li><p>a</p>\n<p>more</p></li><li>b</li></ul>"
|
||||
|
||||
def test_an_unindented_line_continues_the_item(self):
|
||||
body, _ = self.render("- a\nlazy\n")
|
||||
assert body == "<ul><li>a\nlazy</li></ul>"
|
||||
|
||||
def test_a_blank_line_between_items_keeps_one_list(self):
|
||||
body, _ = self.render("- a\n\n- b\n")
|
||||
assert body == "<ul><li>a</li><li>b</li></ul>"
|
||||
|
||||
def test_a_list_of_another_kind_starts_a_new_list(self):
|
||||
body, _ = self.render("- a\n1. b\n")
|
||||
assert body == "<ul><li>a</li></ul>\n<ol><li>b</li></ol>"
|
||||
|
||||
def test_a_block_ends_a_paragraph(self):
|
||||
body, _ = self.render("text\n# Heading\n")
|
||||
assert body == '<p>text</p>\n<h1 id="heading">Heading</h1>'
|
||||
|
||||
def test_empty_input_renders_nothing(self):
|
||||
assert self.render("")[0] == ""
|
||||
assert self.render("\n\n \n")[0] == ""
|
||||
|
||||
def test_the_guide_renders_every_construct_once(self):
|
||||
body, renderer = self.render(GUIDE)
|
||||
assert body.count("<table>") == 1
|
||||
assert body.count("<dl>") == 1
|
||||
assert body.count("<pre>") == 1
|
||||
assert body.count("<ol") == 2
|
||||
assert '<ol start="5"><li>five</li></ol>' in body
|
||||
assert '<a href="other.html#part">link</a>' in body
|
||||
assert renderer.title == "Guide"
|
||||
assert [h["id"] for h in renderer.headings] == ["guide", "part", "part-1"]
|
||||
|
||||
|
||||
class TestPages:
|
||||
def test_a_page_is_a_full_document_with_its_canonical_address(self):
|
||||
page = site.layout("docs/a.html", "docs/a.md", "A · paperspec", "<p>body</p>")
|
||||
assert page.startswith("<!doctype html>")
|
||||
assert (
|
||||
'<link rel="canonical" href="https://paperspec.scient.ing/docs/a.html">'
|
||||
in page
|
||||
)
|
||||
assert 'data-root="../"' in page
|
||||
assert '<link rel="stylesheet" href="../site.css">' in page
|
||||
assert "<title>A · paperspec</title>" in page
|
||||
assert "<p>body</p>" in page
|
||||
|
||||
def test_a_folder_page_is_canonical_at_the_folder(self):
|
||||
page = site.layout(
|
||||
"docs/index.html", None, "Docs", "", base_url="http://localhost:8000/"
|
||||
)
|
||||
assert 'href="http://localhost:8000/docs/"' in page
|
||||
|
||||
def test_the_home_page_links_from_the_root(self):
|
||||
page = site.layout("index.html", "README.md", "Home", "")
|
||||
assert 'data-root=""' in page
|
||||
assert '<a class="brand" href="./">paperspec</a>' in page
|
||||
assert 'href="https://paperspec.scient.ing/"' in page
|
||||
|
||||
def test_the_not_found_page_links_from_the_site_root(self):
|
||||
page = site.layout("404.html", None, "Not found", "")
|
||||
assert 'data-root="/"' in page
|
||||
assert 'href="/site.css"' in page
|
||||
|
||||
def test_a_source_page_links_to_its_markdown_and_the_repository(self):
|
||||
page = site.layout("docs/a.html", "docs/a.md", "A", "", source="docs/a.md")
|
||||
assert f"{site.REPO}/src/branch/main/docs/a.md" in page
|
||||
assert '<a href="../docs/a.md">raw</a>' in page
|
||||
|
||||
def test_titles_and_sources_are_escaped(self):
|
||||
page = site.layout("a.html", None, "A <b> & c", "", source="x&y.md")
|
||||
assert "<title>A <b> & c</title>" in page
|
||||
assert "x&y.md" in page
|
||||
|
||||
def test_the_sidebar_marks_the_current_page(self):
|
||||
nav = site.nav_html("docs/faq.html", "docs/faq.md")
|
||||
assert nav.count('aria-current="page"') == 1
|
||||
assert '<a href="faq.html" aria-current="page">FAQ</a>' in nav
|
||||
assert '<a href="overview.html">Overview</a>' in nav
|
||||
assert "<h2>Specification</h2>" in nav
|
||||
|
||||
def test_the_sidebar_of_a_listing_marks_nothing(self):
|
||||
assert "aria-current" not in site.nav_html("docs/index.html", None)
|
||||
|
||||
def test_the_vocabulary_page_is_marked_in_the_sidebar(self):
|
||||
nav = site.nav_html("vocab/index.html", "vocab")
|
||||
assert '<a href="./" aria-current="page">Vocabulary</a>' in nav
|
||||
|
||||
def test_the_specification_menu_lists_every_chapter(self):
|
||||
entries = site.spec_nav()
|
||||
assert entries[0] == ("spec/README.md", "Contents")
|
||||
chapters = sorted(p.name for p in (ROOT / "spec").glob("[0-9][0-9]-*.md"))
|
||||
assert [e[0] for e in entries[1:]] == ["spec/" + c for c in chapters]
|
||||
assert all(label for _, label in entries)
|
||||
|
||||
|
||||
class MiniSite:
|
||||
"""A small checkout the site builder can work on."""
|
||||
|
||||
def __init__(self, root):
|
||||
self.root = root
|
||||
self.write("CHANGELOG.md", "# Changelog\n\n## 2.3.4\n\n- change\n")
|
||||
self.write(
|
||||
"README.md",
|
||||
"# Home\n\nSee [guide](docs/guide.md#part) and [data](data/x.json).\n",
|
||||
)
|
||||
self.write("docs/guide.md", GUIDE)
|
||||
self.write("docs/notitle.md", "no heading here\n")
|
||||
self.write("spec/README.md", "# Spec\n")
|
||||
self.write("spec/01-first.md", "# First chapter\n")
|
||||
self.write("spec/notes.md", "# Not a chapter\n")
|
||||
self.write("data/x.json", '{"a":1}\n')
|
||||
self.write("assets/logo.svg", "<svg/>")
|
||||
self.write(".secret", "hidden")
|
||||
self.write(".DS_Store", "junk")
|
||||
self.write("node_modules/dep/index.md", "# skipped\n")
|
||||
self.write("_site/old.html", "old")
|
||||
self.write("sub.egg-info/PKG", "x")
|
||||
self.write(".well-known/security.txt", "contact")
|
||||
self.write(".hidden/inside.md", "# hidden\n")
|
||||
self.write(
|
||||
"context/scienting-1.jsonld",
|
||||
json.dumps(
|
||||
{
|
||||
"@context": {
|
||||
"@version": 1.1,
|
||||
"sct": "https://paperspec.scient.ing/vocab#",
|
||||
"as": "https://www.w3.org/ns/activitystreams#",
|
||||
"plain": "sct:plain",
|
||||
"linked": {"@id": "sct:linked", "@type": "@id"},
|
||||
"blob": {"@id": "sct:blob", "@type": "@json"},
|
||||
"many": {"@id": "sct:many", "@container": "@set"},
|
||||
"foreign": "as:foreign",
|
||||
}
|
||||
}
|
||||
),
|
||||
)
|
||||
pathlib.Path(self.root / "link.md").symlink_to(self.root / "README.md")
|
||||
|
||||
def write(self, rel, text):
|
||||
path = self.root / rel
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mini(tmp_path, monkeypatch):
|
||||
checkout = tmp_path / "checkout"
|
||||
checkout.mkdir()
|
||||
MiniSite(checkout)
|
||||
monkeypatch.setattr(site, "ROOT", str(checkout))
|
||||
return checkout
|
||||
|
||||
|
||||
class TestBuildMini:
|
||||
def build(self, tmp_path, **kwargs):
|
||||
out = tmp_path / "out"
|
||||
count = site.build(str(out), **kwargs)
|
||||
return out, count
|
||||
|
||||
def test_every_markdown_file_gets_a_page_and_is_counted(self, mini, tmp_path):
|
||||
out, count = self.build(tmp_path)
|
||||
pages = {
|
||||
"index.html",
|
||||
"docs/guide.html",
|
||||
"docs/notitle.html",
|
||||
"spec/index.html",
|
||||
"spec/01-first.html",
|
||||
"spec/notes.html",
|
||||
"CHANGELOG.html",
|
||||
}
|
||||
assert count == len(pages)
|
||||
for page in pages:
|
||||
assert (out / page).is_file()
|
||||
|
||||
def test_every_file_is_copied_to_its_own_path(self, mini, tmp_path):
|
||||
out, _ = self.build(tmp_path)
|
||||
assert (out / "data" / "x.json").read_text() == '{"a":1}\n'
|
||||
assert (out / "assets" / "logo.svg").is_file()
|
||||
assert (out / "docs" / "guide.md").read_text() == GUIDE
|
||||
assert (out / ".well-known" / "security.txt").is_file()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path",
|
||||
[
|
||||
".secret",
|
||||
".DS_Store",
|
||||
"node_modules",
|
||||
"_site",
|
||||
"sub.egg-info",
|
||||
".hidden",
|
||||
"link.md",
|
||||
"link.html",
|
||||
],
|
||||
)
|
||||
def test_hidden_generated_and_linked_files_are_left_out(self, mini, tmp_path, path):
|
||||
out, _ = self.build(tmp_path)
|
||||
assert not (out / path).exists()
|
||||
|
||||
def test_the_output_folder_inside_the_checkout_is_not_copied_into_itself(
|
||||
self, mini
|
||||
):
|
||||
out = mini / "public"
|
||||
out.mkdir()
|
||||
(out / "stale.html").write_text("old")
|
||||
site.build(str(out))
|
||||
assert not (out / "public").exists()
|
||||
|
||||
def test_an_output_folder_inside_the_checkout_is_emptied_first(self, mini):
|
||||
out = mini / "public"
|
||||
out.mkdir()
|
||||
(out / "stale.html").write_text("old")
|
||||
site.build(str(out))
|
||||
assert not (out / "stale.html").exists()
|
||||
assert (out / "index.html").is_file()
|
||||
|
||||
def test_links_between_markdown_files_point_at_pages(self, mini, tmp_path):
|
||||
out, _ = self.build(tmp_path)
|
||||
home = (out / "index.html").read_text()
|
||||
assert 'href="docs/guide.html#part"' in home
|
||||
assert 'href="data/x.json"' in home
|
||||
|
||||
def test_the_title_comes_from_the_first_heading(self, mini, tmp_path):
|
||||
out, _ = self.build(tmp_path)
|
||||
assert "<title>paperspec · Home</title>" in (out / "index.html").read_text()
|
||||
assert (
|
||||
"<title>Guide · paperspec</title>" in (out / "docs/guide.html").read_text()
|
||||
)
|
||||
assert (
|
||||
"<title>docs/notitle.md · paperspec</title>"
|
||||
in (out / "docs/notitle.html").read_text()
|
||||
)
|
||||
|
||||
def test_the_version_is_the_changelogs_first_heading(self, mini, tmp_path):
|
||||
out, _ = self.build(tmp_path)
|
||||
assert '<span class="version">2.3.4</span>' in (out / "index.html").read_text()
|
||||
|
||||
def test_a_changelog_without_a_heading_is_unreleased(self, mini):
|
||||
(mini / "CHANGELOG.md").write_text("nothing yet\n")
|
||||
assert site.version() == "unreleased"
|
||||
|
||||
def test_a_folder_with_a_readme_is_its_index_and_others_get_a_listing(
|
||||
self, mini, tmp_path
|
||||
):
|
||||
out, _ = self.build(tmp_path)
|
||||
assert '<h1 id="spec">Spec</h1>' in (out / "spec" / "index.html").read_text()
|
||||
listing = (out / "data" / "index.html").read_text()
|
||||
assert "<h1>data/</h1>" in listing
|
||||
assert '<a href="x.json">x.json</a>' in listing
|
||||
assert "<td>8 B</td>" in listing
|
||||
assert '<a href="../">../</a>' in listing
|
||||
|
||||
def test_a_listing_names_folders_and_hides_pages_rendered_from_markdown(
|
||||
self, mini, tmp_path
|
||||
):
|
||||
out, _ = self.build(tmp_path)
|
||||
listing = (out / "docs" / "index.html").read_text()
|
||||
assert '<a href="guide.md">guide.md</a>' in listing
|
||||
assert "guide.html" not in listing.split("<tbody>")[1]
|
||||
top = (out / "index.html").read_text()
|
||||
assert '<h1 id="home">Home</h1>' in top
|
||||
|
||||
def test_the_root_listing_is_not_needed_when_a_readme_exists(self, mini, tmp_path):
|
||||
out, _ = self.build(tmp_path)
|
||||
assert '<h1 id="home">Home</h1>' in (out / "index.html").read_text()
|
||||
assert '<a href="../">' not in (out / "index.html").read_text()
|
||||
|
||||
def test_a_root_without_a_readme_gets_a_listing_named_for_the_site(
|
||||
self, mini, tmp_path
|
||||
):
|
||||
(mini / "README.md").unlink()
|
||||
out, _ = self.build(tmp_path)
|
||||
listing = (out / "index.html").read_text()
|
||||
assert "<h1>paperspec/</h1>" in listing
|
||||
assert '<a href="../">' not in listing
|
||||
|
||||
def test_the_vocabulary_page_lists_each_term_of_the_namespace(self, mini, tmp_path):
|
||||
out, _ = self.build(tmp_path)
|
||||
page = (out / "vocab" / "index.html").read_text()
|
||||
assert '<tr id="plain"><td><code>plain</code></td>' in page
|
||||
assert "<code>https://paperspec.scient.ing/vocab#plain</code>" in page
|
||||
assert "an IRI" in page
|
||||
assert "a JSON literal, kept intact" in page
|
||||
assert "a set" in page
|
||||
assert 'id="foreign"' not in page
|
||||
assert 'id="sct"' not in page
|
||||
|
||||
def test_the_search_index_lists_pages_headings_and_the_vocabulary(
|
||||
self, mini, tmp_path
|
||||
):
|
||||
out, _ = self.build(tmp_path)
|
||||
index = json.loads((out / "search.json").read_text())
|
||||
assert [e["url"] for e in index] == sorted(e["url"] for e in index)
|
||||
guide = next(e for e in index if e["url"] == "docs/guide.html")
|
||||
assert guide["title"] == "Guide"
|
||||
assert {"text": "Part", "id": "part"} in guide["headings"]
|
||||
assert {"text": "Guide", "id": "guide"} not in guide["headings"]
|
||||
assert {"title": "Vocabulary", "url": "vocab/", "headings": []} in index
|
||||
assert any(e["title"] == "docs/notitle.md" for e in index)
|
||||
|
||||
def test_it_writes_the_stylesheet_the_script_and_the_not_found_page(
|
||||
self, mini, tmp_path
|
||||
):
|
||||
out, _ = self.build(tmp_path)
|
||||
assert (out / "site.css").read_text() == site.CSS
|
||||
assert (out / "search.js").read_text() == site.SEARCH_JS
|
||||
missing = (out / "404.html").read_text()
|
||||
assert "<h1>Not found</h1>" in missing
|
||||
assert f'href="{site.BASE_URL}/"' in missing
|
||||
|
||||
def test_the_base_url_goes_into_every_canonical_link(self, mini, tmp_path):
|
||||
out, _ = self.build(tmp_path, base_url="http://localhost:8000")
|
||||
for page in out.rglob("*.html"):
|
||||
text = page.read_text()
|
||||
if page.name == "404.html" or '<link rel="canonical"' not in text:
|
||||
continue
|
||||
assert 'rel="canonical" href="http://localhost:8000/' in text
|
||||
|
||||
def test_a_second_build_replaces_the_first(self, mini, tmp_path):
|
||||
out, _ = self.build(tmp_path)
|
||||
(out / "stale.txt").write_text("old")
|
||||
site.build(str(out))
|
||||
assert not (out / "stale.txt").exists()
|
||||
|
||||
|
||||
class TestRun:
|
||||
def test_writes_the_site_and_says_where(self, mini, tmp_path, capsys):
|
||||
out = tmp_path / "public"
|
||||
status = cli_main.main(["site", "--out", str(out)])
|
||||
assert status == 0
|
||||
assert capsys.readouterr().out == (
|
||||
f"Wrote 7 pages and every file of the repository to {out}\n"
|
||||
)
|
||||
assert (out / "index.html").is_file()
|
||||
|
||||
def test_the_default_output_is_site_in_the_checkout(self, mini, capsys):
|
||||
assert cli_main.main(["site"]) == 0
|
||||
assert (mini / "_site" / "index.html").is_file()
|
||||
assert str(mini / "_site") in capsys.readouterr().out
|
||||
|
||||
def test_the_base_url_option_reaches_the_pages(self, mini, tmp_path):
|
||||
out = tmp_path / "public"
|
||||
cli_main.main(
|
||||
["site", "--out", str(out), "--base-url", "http://localhost:9000"]
|
||||
)
|
||||
assert 'href="http://localhost:9000/"' in (out / "index.html").read_text()
|
||||
|
||||
@pytest.mark.parametrize("where", ["checkout", "parent"])
|
||||
def test_the_output_cannot_be_the_checkout_or_contain_it(self, mini, where, capsys):
|
||||
out = mini if where == "checkout" else mini.parent
|
||||
with pytest.raises(SystemExit) as stop:
|
||||
cli_main.main(["site", "--out", str(out)])
|
||||
assert stop.value.code == 2
|
||||
err = capsys.readouterr().err
|
||||
assert err.startswith("usage: paperspec site")
|
||||
assert "must not be the repository or contain it" in err
|
||||
assert (mini / "README.md").is_file()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def repository_site(tmp_path_factory):
|
||||
out = tmp_path_factory.mktemp("site") / "site"
|
||||
assert cli_main.main(["site", "--out", str(out)]) == 0
|
||||
return out
|
||||
|
||||
|
||||
class TestRepositorySite:
|
||||
def test_every_markdown_file_has_a_page(self, repository_site):
|
||||
skipped = {
|
||||
".git",
|
||||
".pixi",
|
||||
"node_modules",
|
||||
"__pycache__",
|
||||
"_site",
|
||||
"work",
|
||||
"tests",
|
||||
}
|
||||
for md in ROOT.rglob("*.md"):
|
||||
rel = md.relative_to(ROOT)
|
||||
if skipped & set(rel.parts) or any(p.startswith(".") for p in rel.parts):
|
||||
continue
|
||||
assert (repository_site / site.page_path(rel.as_posix())).is_file(), rel
|
||||
|
||||
def test_each_schema_is_served_at_its_id(self, repository_site):
|
||||
for path in sorted((ROOT / "schemas").glob("*.schema.json")):
|
||||
schema_id = json.loads(path.read_text())["$id"]
|
||||
assert schema_id.startswith(site.BASE_URL + "/")
|
||||
served = repository_site / schema_id[len(site.BASE_URL) + 1 :]
|
||||
assert served.read_bytes() == path.read_bytes()
|
||||
|
||||
def test_every_vocabulary_term_has_an_anchor(self, repository_site):
|
||||
context = json.loads((ROOT / "context" / "scienting-1.jsonld").read_text())
|
||||
page = (repository_site / "vocab" / "index.html").read_text()
|
||||
ids = set(re.findall(r'<tr id="([^"]+)"', page))
|
||||
for term, value in context["@context"].items():
|
||||
if term.startswith("@"):
|
||||
continue
|
||||
iri = value if isinstance(value, str) else value.get("@id", "")
|
||||
if iri.startswith("sct:"):
|
||||
assert iri[4:] in ids
|
||||
|
||||
def test_the_search_index_is_valid_json_sorted_by_address(self, repository_site):
|
||||
index = json.loads((repository_site / "search.json").read_text())
|
||||
assert [e["url"] for e in index] == sorted(e["url"] for e in index)
|
||||
assert all({"title", "url", "headings"} <= set(e) for e in index)
|
||||
@@ -0,0 +1,100 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""`paperspec stress`: the properties hold on the oracle, and a failure is reported with its seed."""
|
||||
|
||||
import argparse
|
||||
|
||||
import pytest
|
||||
|
||||
from paperspec.cli import main as cli_main, stress
|
||||
|
||||
SEED = 20261014
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def clean_state(monkeypatch):
|
||||
"""Empty the module's counters and failures, and put them back afterwards."""
|
||||
monkeypatch.setattr(stress, "COUNTS", {})
|
||||
monkeypatch.setattr(stress, "FAILS", [])
|
||||
|
||||
|
||||
def run_stress(argv, capsys):
|
||||
"""Run `paperspec stress ARGV`, returning its exit status and what it printed."""
|
||||
status = cli_main.main(["stress", *argv])
|
||||
return status, capsys.readouterr().out
|
||||
|
||||
|
||||
class TestProperties:
|
||||
def test_every_property_holds_for_a_few_rounds(self, clean_state, capsys):
|
||||
status, out = run_stress(["--seed", str(SEED), "--rounds", "8"], capsys)
|
||||
assert status == 0, out
|
||||
assert out.startswith(f"seed {SEED}, 8 rounds")
|
||||
assert out.rstrip().endswith("Every property held.")
|
||||
|
||||
def test_it_reports_a_count_for_each_property(self, clean_state, capsys):
|
||||
_, out = run_stress(["--seed", "1", "--rounds", "4"], capsys)
|
||||
sections = {line.split()[0] for line in out.splitlines() if " checks" in line}
|
||||
assert {"canonical-json", "corrections", "merkle", "vrf", "ml-dsa"} <= sections
|
||||
|
||||
def test_the_same_seed_gives_the_same_report(
|
||||
self, clean_state, capsys, monkeypatch
|
||||
):
|
||||
first = run_stress(["--seed", "7", "--rounds", "4"], capsys)
|
||||
monkeypatch.setattr(stress, "COUNTS", {})
|
||||
second = run_stress(["--seed", "7", "--rounds", "4"], capsys)
|
||||
assert first == second
|
||||
|
||||
|
||||
class TestOk:
|
||||
def test_a_true_condition_is_counted_and_not_a_failure(self, clean_state):
|
||||
stress.ok("demo", True, "never reported")
|
||||
assert stress.COUNTS == {"demo": 1}
|
||||
assert stress.FAILS == []
|
||||
|
||||
def test_a_false_condition_is_counted_and_recorded_with_its_section(
|
||||
self, clean_state
|
||||
):
|
||||
stress.ok("demo", False, "it broke")
|
||||
assert stress.COUNTS == {"demo": 1}
|
||||
assert stress.FAILS == ["demo: it broke"]
|
||||
|
||||
|
||||
class TestFailureReport:
|
||||
@pytest.fixture
|
||||
def only(self, clean_state, monkeypatch):
|
||||
"""Replace every property by a no-op, and return a function that swaps one back."""
|
||||
for name in dir(stress):
|
||||
if name.startswith("stress_"):
|
||||
monkeypatch.setattr(stress, name, lambda *args: None)
|
||||
monkeypatch.setattr(stress, "stress_corrections", lambda *args: [])
|
||||
|
||||
def failing(count):
|
||||
def property_(*args):
|
||||
for i in range(count):
|
||||
stress.ok("demo", False, f"case {i}")
|
||||
|
||||
monkeypatch.setattr(stress, "stress_merkle", property_)
|
||||
|
||||
return failing
|
||||
|
||||
def test_a_failure_prints_the_seed_and_how_to_reproduce_it(self, only, capsys):
|
||||
only(2)
|
||||
args = argparse.Namespace(seed=5, rounds=3)
|
||||
assert stress.run(args) == 1
|
||||
out = capsys.readouterr().out
|
||||
assert "2 failures (seed 5):" in out
|
||||
assert " demo: case 0" in out
|
||||
assert "paperspec stress --seed 5 --rounds 3" in out
|
||||
assert "Every property held." not in out
|
||||
|
||||
def test_it_lists_sixty_failures_and_counts_the_rest(self, only, capsys):
|
||||
only(65)
|
||||
assert stress.run(argparse.Namespace(seed=1, rounds=1)) == 1
|
||||
out = capsys.readouterr().out
|
||||
assert " demo: case 59" in out
|
||||
assert "demo: case 60" not in out
|
||||
assert "and 5 more" in out
|
||||
@@ -0,0 +1,431 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""`paperspec validate` and `paperspec load`: exit statuses, messages and what lands in the database."""
|
||||
|
||||
import gc
|
||||
import json
|
||||
import sqlite3
|
||||
from contextlib import closing
|
||||
|
||||
import pytest
|
||||
|
||||
from helpers_cli import (
|
||||
CHECKS_ONLY,
|
||||
HALE,
|
||||
INVALID,
|
||||
PACKAGES,
|
||||
TINY,
|
||||
TINY_ZIP,
|
||||
edit_package,
|
||||
store,
|
||||
)
|
||||
from paperspec import corrections, jcs, package
|
||||
from paperspec.cli import main as cli_main
|
||||
|
||||
OKAFOR = PACKAGES / "okafor2025solvent-acth"
|
||||
PROPOSAL = {
|
||||
"format": "scienting-correction/1",
|
||||
"evidence": [{"kind": "person", "by": "ana"}],
|
||||
"author": {"key": "ana"},
|
||||
"reason": "Test.",
|
||||
"at": "2026-10-14T09:00:00Z",
|
||||
"status": "proposed",
|
||||
"op": "replace",
|
||||
"target": "mhp0at8",
|
||||
"find": "harmonic",
|
||||
"replace": "quadratic",
|
||||
}
|
||||
|
||||
|
||||
def run_main(argv, capsys):
|
||||
"""Run `paperspec ARGV`, returning its exit status and the text it printed."""
|
||||
try:
|
||||
status = cli_main.main([str(a) for a in argv])
|
||||
except SystemExit as stop:
|
||||
status = stop.code
|
||||
captured = capsys.readouterr()
|
||||
return status, captured.out, captured.err
|
||||
|
||||
|
||||
def snapshot(folder):
|
||||
return {
|
||||
p.relative_to(folder): p.read_bytes() for p in folder.rglob("*") if p.is_file()
|
||||
}
|
||||
|
||||
|
||||
def write_proposals(path, proposals, *, lines=False):
|
||||
if lines:
|
||||
path.write_text("".join(json.dumps(p) + "\n" for p in proposals))
|
||||
else:
|
||||
path.write_text(json.dumps(proposals[0]))
|
||||
return path
|
||||
|
||||
|
||||
def invalid_packages():
|
||||
cases = sorted(INVALID.iterdir(), key=lambda p: p.name)
|
||||
return [
|
||||
(
|
||||
case.name,
|
||||
next(p for p in (case / "package", case / "package.sct.zip") if p.exists()),
|
||||
)
|
||||
for case in cases
|
||||
]
|
||||
|
||||
|
||||
class TestValidate:
|
||||
def test_a_valid_folder_is_reported_valid(self, capsys):
|
||||
status, out, err = run_main(["validate", TINY], capsys)
|
||||
assert (status, out, err) == (0, f"{TINY}: valid\n", "")
|
||||
|
||||
def test_several_packages_are_each_reported(self, capsys):
|
||||
status, out, _ = run_main(["validate", TINY, TINY_ZIP, HALE], capsys)
|
||||
assert status == 0
|
||||
assert out.splitlines() == [f"{p}: valid" for p in (TINY, TINY_ZIP, HALE)]
|
||||
|
||||
@pytest.mark.parametrize(("name", "path"), invalid_packages())
|
||||
def test_a_broken_package_fails_with_its_rule(self, name, path, capsys):
|
||||
expected = json.loads((path.parent / "expect.json").read_text())["rule"]
|
||||
status, out, err = run_main(["validate", path], capsys)
|
||||
assert status == 1
|
||||
assert out == ""
|
||||
failures = [line.split(": ", 3) for line in err.splitlines()]
|
||||
assert all(line[0] == str(path) for line in failures)
|
||||
assert expected in {line[2] for line in failures}
|
||||
|
||||
def test_a_failure_names_where_the_rule_and_the_message(self, capsys):
|
||||
path = INVALID / "counts-wrong" / "package"
|
||||
_, _, err = run_main(["validate", path], capsys)
|
||||
assert err == f"{path}: document.json: doc.counts: counts do not recompute\n"
|
||||
|
||||
def test_one_failure_among_valid_packages_reports_no_valid_line(self, capsys):
|
||||
broken = INVALID / "counts-wrong" / "package"
|
||||
status, out, err = run_main(["validate", TINY, broken, HALE], capsys)
|
||||
assert status == 1
|
||||
assert out == ""
|
||||
assert str(broken) in err
|
||||
assert str(HALE) not in err
|
||||
|
||||
def test_each_failure_names_its_own_package(self, capsys):
|
||||
first = INVALID / "counts-wrong" / "package"
|
||||
second = INVALID / "crlf" / "package"
|
||||
_, _, err = run_main(["validate", first, second], capsys)
|
||||
assert any(line.startswith(f"{first}: ") for line in err.splitlines())
|
||||
assert any(line.startswith(f"{second}: ") for line in err.splitlines())
|
||||
|
||||
def test_a_zip_past_the_size_limit_is_refused(self, capsys):
|
||||
status, _, err = run_main(["validate", "--max-size", "100", TINY_ZIP], capsys)
|
||||
assert status == 1
|
||||
assert f"{TINY_ZIP}: : package.zip.size: " in err
|
||||
|
||||
def test_the_size_limit_does_not_count_folders(self, capsys):
|
||||
status, out, _ = run_main(["validate", "--max-size", "100", TINY], capsys)
|
||||
assert (status, out) == (0, f"{TINY}: valid\n")
|
||||
|
||||
|
||||
class TestValidateCorrections:
|
||||
def test_prints_the_record_a_proposal_would_append(self, tmp_path, capsys):
|
||||
proposals = write_proposals(tmp_path / "fix.json", [PROPOSAL])
|
||||
status, out, err = run_main(
|
||||
["validate", "--correction", proposals, HALE], capsys
|
||||
)
|
||||
assert (status, err) == (0, "")
|
||||
state = package.read_state(HALE)
|
||||
_, record = corrections.gate(state, PROPOSAL, store())
|
||||
assert out == jcs.dumps(record) + "\n"
|
||||
assert json.loads(out)["index"] == 13
|
||||
|
||||
def test_writes_nothing(self, tmp_path, capsys):
|
||||
before = snapshot(HALE)
|
||||
proposals = write_proposals(tmp_path / "fix.json", [PROPOSAL])
|
||||
run_main(["validate", "--correction", proposals, HALE], capsys)
|
||||
assert snapshot(HALE) == before
|
||||
|
||||
def test_applies_a_file_of_proposals_in_order(self, tmp_path, capsys):
|
||||
second = dict(PROPOSAL, find="quadratic", replace="harmonic")
|
||||
proposals = write_proposals(
|
||||
tmp_path / "fixes.jsonl", [PROPOSAL, second], lines=True
|
||||
)
|
||||
status, out, _ = run_main(["validate", "--correction", proposals, HALE], capsys)
|
||||
assert status == 0
|
||||
records = [json.loads(line) for line in out.splitlines()]
|
||||
assert [r["index"] for r in records] == [13, 14]
|
||||
assert records[1]["before"] == records[0]["after"]
|
||||
|
||||
def test_blank_lines_between_proposals_are_ignored(self, tmp_path, capsys):
|
||||
path = tmp_path / "fixes.jsonl"
|
||||
path.write_text(json.dumps(PROPOSAL) + "\n\n" + json.dumps(PROPOSAL) + "\n")
|
||||
status, out, err = run_main(["validate", "--correction", path, HALE], capsys)
|
||||
assert status == 1
|
||||
assert len(out.splitlines()) == 1
|
||||
assert f"{path}:2 mhp0at8: correction.find: " in err
|
||||
|
||||
def test_stops_at_the_first_refusal_and_names_it(self, tmp_path, capsys):
|
||||
refused = dict(PROPOSAL, find="absent words")
|
||||
proposals = write_proposals(
|
||||
tmp_path / "fixes.jsonl", [PROPOSAL, refused], lines=True
|
||||
)
|
||||
status, out, err = run_main(
|
||||
["validate", "--correction", proposals, HALE], capsys
|
||||
)
|
||||
assert status == 1
|
||||
assert len(out.splitlines()) == 1
|
||||
assert err == (
|
||||
f"{proposals}:2 mhp0at8: correction.find: find occurs 0 times, not exactly once\n"
|
||||
)
|
||||
|
||||
def test_a_refusal_without_a_node_names_only_the_line(self, tmp_path, capsys):
|
||||
broken = {k: v for k, v in PROPOSAL.items() if k != "replace"}
|
||||
proposals = write_proposals(tmp_path / "fix.json", [broken])
|
||||
status, out, err = run_main(
|
||||
["validate", "--correction", proposals, HALE], capsys
|
||||
)
|
||||
assert (status, out) == (1, "")
|
||||
assert err.startswith(f"{proposals}:1: correction.schema: ")
|
||||
|
||||
def test_works_on_a_zipped_package(self, tmp_path, capsys):
|
||||
proposal = dict(PROPOSAL, target=_first_sentence(TINY), find="The", replace="A")
|
||||
proposals = write_proposals(tmp_path / "fix.json", [proposal])
|
||||
status, out, err = run_main(
|
||||
["validate", "--correction", proposals, TINY_ZIP], capsys
|
||||
)
|
||||
assert (status, err) == (0, "")
|
||||
assert json.loads(out)["op"] == "replace"
|
||||
|
||||
def test_a_package_that_withholds_its_text_takes_no_corrections(
|
||||
self, tmp_path, capsys
|
||||
):
|
||||
proposals = write_proposals(tmp_path / "fix.json", [PROPOSAL])
|
||||
status, out, err = run_main(
|
||||
["validate", "--correction", proposals, CHECKS_ONLY], capsys
|
||||
)
|
||||
assert (status, out) == (1, "")
|
||||
assert err == (
|
||||
f"{CHECKS_ONLY}: doc.policy.withheld: "
|
||||
"a package that withholds its text takes no corrections\n"
|
||||
)
|
||||
|
||||
def test_an_invalid_package_is_reported_before_any_proposal_runs(
|
||||
self, tmp_path, capsys
|
||||
):
|
||||
broken = INVALID / "counts-wrong" / "package"
|
||||
proposals = write_proposals(tmp_path / "fix.json", [PROPOSAL])
|
||||
status, out, err = run_main(
|
||||
["validate", "--correction", proposals, broken], capsys
|
||||
)
|
||||
assert (status, out) == (1, "")
|
||||
assert "doc.counts" in err
|
||||
|
||||
def test_one_package_only(self, tmp_path, capsys):
|
||||
proposals = write_proposals(tmp_path / "fix.json", [PROPOSAL])
|
||||
status, _, err = run_main(
|
||||
["validate", "--correction", proposals, HALE, TINY], capsys
|
||||
)
|
||||
assert status == 2
|
||||
assert "exactly one package" in err
|
||||
assert err.startswith("usage: paperspec validate")
|
||||
|
||||
|
||||
def _first_sentence(root):
|
||||
nodes = [
|
||||
json.loads(line) for line in (root / "nodes.jsonl").read_text().splitlines()
|
||||
]
|
||||
return next(
|
||||
n["id"]
|
||||
for n in nodes
|
||||
if n["kind"] == "sentence" and n["content"][0].get("text", "").startswith("The")
|
||||
)
|
||||
|
||||
|
||||
def tables(path):
|
||||
with closing(sqlite3.connect(path)) as conn:
|
||||
return {
|
||||
r[0]
|
||||
for r in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table'")
|
||||
}
|
||||
|
||||
|
||||
def count(path, sql, args=()):
|
||||
with closing(sqlite3.connect(path)) as conn:
|
||||
return conn.execute(sql, args).fetchone()[0]
|
||||
|
||||
|
||||
def meta(path):
|
||||
with closing(sqlite3.connect(path)) as conn:
|
||||
return dict(conn.execute("SELECT name, value FROM bundle_meta"))
|
||||
|
||||
|
||||
# `paperspec load` leaves its SQLite connection open; see test_load_closes_its_database.
|
||||
LEAKS = pytest.mark.filterwarnings(
|
||||
"ignore::ResourceWarning", "ignore::pytest.PytestUnraisableExceptionWarning"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def collect_garbage():
|
||||
"""Finalize the connections a test left open while that test's warning filters apply."""
|
||||
yield
|
||||
gc.collect()
|
||||
|
||||
|
||||
@LEAKS
|
||||
class TestLoad:
|
||||
def test_load_closes_its_database(self, tmp_path, capsys, monkeypatch):
|
||||
opened = []
|
||||
connect = sqlite3.connect
|
||||
|
||||
def tracking(*args, **kwargs):
|
||||
conn = connect(*args, **kwargs)
|
||||
opened.append(conn)
|
||||
return conn
|
||||
|
||||
monkeypatch.setattr(sqlite3, "connect", tracking)
|
||||
run_main(["load", "--sqlite", tmp_path / "lab.db", TINY], capsys)
|
||||
monkeypatch.undo()
|
||||
try:
|
||||
for conn in opened:
|
||||
with pytest.raises(sqlite3.ProgrammingError):
|
||||
conn.execute("SELECT 1")
|
||||
finally:
|
||||
for conn in opened:
|
||||
conn.close()
|
||||
|
||||
def test_loads_a_package_and_counts_its_nodes(self, tmp_path, capsys):
|
||||
db = tmp_path / "lab.db"
|
||||
status, out, err = run_main(["load", "--sqlite", db, HALE], capsys)
|
||||
assert (status, err) == (0, "")
|
||||
assert out == "hale2026harmonic-damb: 75 nodes\n"
|
||||
assert {"documents", "nodes", "corrections"} <= tables(db)
|
||||
assert (
|
||||
count(db, "SELECT COUNT(*) FROM nodes WHERE doc_key = ?", (HALE.name,))
|
||||
== 75
|
||||
)
|
||||
assert "bundle_meta" not in tables(db)
|
||||
|
||||
def test_files_without_a_table_go_to_the_asset_store(self, tmp_path, capsys):
|
||||
db = tmp_path / "lab.db"
|
||||
run_main(["load", "--sqlite", db, HALE], capsys)
|
||||
stored = list((tmp_path / "lab.db.assets").rglob("*"))
|
||||
assert any(p.is_file() for p in stored)
|
||||
|
||||
def test_a_later_load_adds_to_the_same_database(self, tmp_path, capsys):
|
||||
db = tmp_path / "lab.db"
|
||||
run_main(["load", "--sqlite", db, HALE], capsys)
|
||||
status, out, _ = run_main(["load", "--sqlite", db, TINY, OKAFOR], capsys)
|
||||
assert status == 0
|
||||
assert [line.split(":")[0] for line in out.splitlines()] == [
|
||||
TINY.name,
|
||||
OKAFOR.name,
|
||||
]
|
||||
assert count(db, "SELECT COUNT(*) FROM documents") == 3
|
||||
|
||||
def test_loads_a_zipped_package(self, tmp_path, capsys):
|
||||
db = tmp_path / "lab.db"
|
||||
status, out, _ = run_main(["load", "--sqlite", db, TINY_ZIP], capsys)
|
||||
assert status == 0
|
||||
assert out.startswith(f"{TINY.name}: ")
|
||||
|
||||
def test_loads_nothing_when_a_package_is_broken(self, tmp_path, capsys):
|
||||
db = tmp_path / "lab.db"
|
||||
broken = INVALID / "counts-wrong" / "package"
|
||||
status, out, err = run_main(["load", "--sqlite", db, HALE, broken], capsys)
|
||||
assert str(status).startswith("Nothing was loaded")
|
||||
assert out == ""
|
||||
assert f"{broken}: document.json: doc.counts: counts do not recompute" in err
|
||||
assert not db.exists()
|
||||
|
||||
def test_a_zip_that_cannot_be_unpacked_is_reported_by_rule(self, tmp_path, capsys):
|
||||
db = tmp_path / "lab.db"
|
||||
archive = INVALID / "zip-traversal" / "package.sct.zip"
|
||||
status, _, err = run_main(["load", "--sqlite", db, archive], capsys)
|
||||
assert str(status).startswith("Nothing was loaded")
|
||||
assert f"{archive}: package.zip.entry: " in err
|
||||
assert not db.exists()
|
||||
|
||||
def test_a_zip_past_the_size_limit_is_reported(self, tmp_path, capsys):
|
||||
db = tmp_path / "lab.db"
|
||||
status, _, err = run_main(
|
||||
["load", "--sqlite", db, "--max-size", "100", TINY_ZIP], capsys
|
||||
)
|
||||
assert str(status).startswith("Nothing was loaded")
|
||||
assert "package.zip.size" in err
|
||||
|
||||
def test_flags_that_go_with_a_bundle_need_one(self, tmp_path, capsys):
|
||||
for flag in (["--built-from", "abc"], ["--store", "lab"]):
|
||||
status, _, err = run_main(
|
||||
["load", "--sqlite", tmp_path / "x.db", *flag, TINY], capsys
|
||||
)
|
||||
assert status == 2
|
||||
assert "--built-from and --store go with --bundle" in err
|
||||
|
||||
def test_a_bundle_needs_the_revision_it_was_built_from(self, tmp_path, capsys):
|
||||
status, _, err = run_main(
|
||||
["load", "--sqlite", tmp_path / "x.db", "--bundle", TINY], capsys
|
||||
)
|
||||
assert status == 2
|
||||
assert "--bundle needs --built-from REV" in err
|
||||
|
||||
def test_a_bundle_is_built_once_into_a_new_file(self, tmp_path, capsys):
|
||||
db = tmp_path / "x.db"
|
||||
db.write_bytes(b"")
|
||||
status, _, err = run_main(
|
||||
["load", "--sqlite", db, "--bundle", "--built-from", "abc", TINY], capsys
|
||||
)
|
||||
assert status == 2
|
||||
assert f"--bundle builds a new file, and {db} exists" in err
|
||||
|
||||
|
||||
@LEAKS
|
||||
class TestLoadBundle:
|
||||
def test_builds_a_bundle_with_search_rows_and_meta(self, tmp_path, capsys):
|
||||
db = tmp_path / "search.db"
|
||||
status, out, err = run_main(
|
||||
["load", "--sqlite", db, "--bundle", "--built-from", "4fa326d", HALE],
|
||||
capsys,
|
||||
)
|
||||
assert (status, err) == (0, "")
|
||||
key, detail = out.strip().split(": ")
|
||||
assert key == HALE.name
|
||||
nodes, rows = (int(part.split()[0]) for part in detail.split(", "))
|
||||
assert nodes == 75
|
||||
assert rows > 0
|
||||
assert count(db, "SELECT COUNT(*) FROM search") == rows
|
||||
assert meta(db)["built_from"] == "4fa326d"
|
||||
assert meta(db)["store"] == "smithlab"
|
||||
|
||||
def test_names_the_store_from_the_option(self, tmp_path, capsys):
|
||||
db = tmp_path / "search.db"
|
||||
status, _, _ = run_main(
|
||||
[
|
||||
"load",
|
||||
"--sqlite",
|
||||
db,
|
||||
"--bundle",
|
||||
"--built-from",
|
||||
"abc",
|
||||
"--store",
|
||||
"lab",
|
||||
TINY,
|
||||
],
|
||||
capsys,
|
||||
)
|
||||
assert status == 0
|
||||
assert meta(db)["store"] == "lab"
|
||||
|
||||
def test_packages_of_two_stores_need_one_named(self, tmp_path, capsys):
|
||||
other = edit_package(
|
||||
tmp_path / "other",
|
||||
OKAFOR,
|
||||
{"document.json": lambda d: dict(d, store="elsewhere")},
|
||||
)
|
||||
db = tmp_path / "search.db"
|
||||
status, _, _ = run_main(
|
||||
["load", "--sqlite", db, "--bundle", "--built-from", "abc", HALE, other],
|
||||
capsys,
|
||||
)
|
||||
assert (
|
||||
str(status) == "The packages name no single store; give one with --store."
|
||||
)
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,442 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Keyed hashing, Merkle trees, RFC 9162 proofs, Ed25519 and did:key (paperspec.crypto)."""
|
||||
|
||||
import hashlib
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
from hypothesis import given, strategies as st
|
||||
|
||||
from conftest import vector_cases
|
||||
from helpers_lib2 import PROPS
|
||||
from paperspec import crypto, jcs
|
||||
|
||||
SHA256_EMPTY = hashlib.sha256(b"").hexdigest()
|
||||
ITEM = st.tuples(
|
||||
st.sampled_from(["node", "annotation", "digitization"]),
|
||||
st.text(alphabet="abcdefghjkmnpqrstvwxyz0123456789", min_size=1, max_size=8),
|
||||
st.binary(min_size=4, max_size=4).map(lambda b: hashlib.sha256(b).hexdigest()),
|
||||
).map(list)
|
||||
TREES = st.lists(ITEM, min_size=1, max_size=24)
|
||||
|
||||
|
||||
def mth(items):
|
||||
"""The Merkle tree hash of RFC 6962 §2.1, written out from the definition."""
|
||||
leaves = [hashlib.sha256(b"\x00" + jcs.dumpb(i)).digest() for i in items]
|
||||
|
||||
def tree(ls):
|
||||
if len(ls) == 1:
|
||||
return ls[0]
|
||||
k = 1 << ((len(ls) - 1).bit_length() - 1)
|
||||
return hashlib.sha256(b"\x01" + tree(ls[:k]) + tree(ls[k:])).digest()
|
||||
|
||||
return tree(leaves).hex()
|
||||
|
||||
|
||||
def base58_decode(text):
|
||||
"""The bytes a base58btc string encodes, with its leading `1` characters as zero bytes."""
|
||||
n = 0
|
||||
for ch in text:
|
||||
n = n * 58 + crypto.BASE58.index(ch)
|
||||
body = n.to_bytes((n.bit_length() + 7) // 8, "big")
|
||||
return b"\x00" * (len(text) - len(text.lstrip("1"))) + body
|
||||
|
||||
|
||||
# --- keyed hashes and keys ----------------------------------------------------------------
|
||||
|
||||
|
||||
KEYED = {c["id"]: c for c in vector_cases("keyed")}
|
||||
|
||||
|
||||
def test_team_key_and_its_id():
|
||||
case = KEYED["team-key"]
|
||||
key = crypto.team_key(bytes.fromhex(case["secret"]))
|
||||
assert key.hex() == case["key"]
|
||||
assert crypto.key_id(key) == case["key_id"]
|
||||
|
||||
|
||||
def test_hkdf_rfc5869_case_1():
|
||||
okm = crypto.hkdf(
|
||||
bytes.fromhex("0b" * 22),
|
||||
bytes(range(13)),
|
||||
bytes(range(0xF0, 0xFA)),
|
||||
length=42,
|
||||
)
|
||||
assert okm.hex() == (
|
||||
"3cb25f25faacd57a90434f64d0362f2a2d2d0a90cf1a5a4c5db02d56ecc4c5bf"
|
||||
"34007208d5b887185865"
|
||||
)
|
||||
|
||||
|
||||
def test_page_hashes():
|
||||
case = KEYED["page-hashes"]
|
||||
assert [crypto.page_hash(fp) for fp in case["page_fps"]] == case["page_hashes"]
|
||||
|
||||
|
||||
def test_page_key_depends_on_every_page_and_the_context():
|
||||
case = KEYED["page-key"]
|
||||
key = crypto.page_key(case["page_fps"], case["context"])
|
||||
assert key.hex() == case["key"]
|
||||
other = KEYED["non-holder"]
|
||||
wrong = crypto.page_key(other["wrong_page_fps"], case["context"])
|
||||
assert wrong.hex() == other["wrong_key"] != case["key"]
|
||||
assert crypto.page_key(case["page_fps"], case["context"] + "x") != key
|
||||
assert crypto.page_key(case["page_fps"][::-1], case["context"]) != key
|
||||
|
||||
|
||||
def test_missing_cryptography_package_names_the_extra(monkeypatch):
|
||||
monkeypatch.setitem(sys.modules, "cryptography", None)
|
||||
with pytest.raises(ImportError, match=r"paperspec\[crypto\]"):
|
||||
crypto.hkdf(b"k", b"salt", b"info")
|
||||
with pytest.raises(ImportError, match="cryptography"):
|
||||
crypto.ed25519_public(bytes(32))
|
||||
|
||||
|
||||
# --- Merkle roots ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", vector_cases("merkle"), ids=lambda c: c["id"])
|
||||
def test_merkle_root_vectors(case):
|
||||
assert crypto.merkle_root(case["items"]) == case["root"]
|
||||
|
||||
|
||||
def test_merkle_root_of_nothing_is_refused():
|
||||
with pytest.raises(ValueError, match="at least one item"):
|
||||
crypto.merkle_root([])
|
||||
|
||||
|
||||
def test_history_root_of_nothing_is_the_hash_of_the_empty_string():
|
||||
assert crypto.history_root([]) == crypto.EMPTY_ROOT == SHA256_EMPTY
|
||||
|
||||
|
||||
def test_history_root_agrees_with_merkle_root_on_a_nonempty_list():
|
||||
items = [["node", "k0yqkzd", "00" * 32], ["node", "h9haqzy", "11" * 32]]
|
||||
assert crypto.history_root(items) == crypto.merkle_root(items)
|
||||
|
||||
|
||||
def test_a_single_item_root_is_its_leaf_hash():
|
||||
item = ["node", "k0yqkzd", "ab" * 32]
|
||||
assert crypto.merkle_root([item]) == crypto.leaf_hash(item)
|
||||
|
||||
|
||||
def test_leaf_and_node_hashes_are_domain_separated():
|
||||
left, right = ["node", "a", "00" * 32], ["node", "b", "11" * 32]
|
||||
root = crypto.merkle_root([left, right])
|
||||
forged_leaf = bytes.fromhex(crypto.leaf_hash(left) + crypto.leaf_hash(right))
|
||||
assert hashlib.sha256(b"\x00" + forged_leaf).hexdigest() != root
|
||||
|
||||
|
||||
def test_receipt_items_lists_nodes_before_records():
|
||||
receipt = {
|
||||
"nodes": [["n1", "h1"], ["n2", "h2"]],
|
||||
"records": [("annotation", "a1", "h3")],
|
||||
}
|
||||
assert crypto.receipt_items(receipt) == [
|
||||
["node", "n1", "h1"],
|
||||
["node", "n2", "h2"],
|
||||
["annotation", "a1", "h3"],
|
||||
]
|
||||
|
||||
|
||||
def test_receipt_items_of_a_receipt_without_either_is_empty():
|
||||
assert crypto.receipt_items({}) == []
|
||||
|
||||
|
||||
@given(TREES)
|
||||
@PROPS
|
||||
def test_merkle_root_follows_the_rfc_6962_definition(items):
|
||||
assert crypto.merkle_root(items) == mth(items) == crypto.history_root(items)
|
||||
|
||||
|
||||
# --- inclusion and consistency proofs -----------------------------------------------------------
|
||||
|
||||
PROOFS = vector_cases("proofs")
|
||||
|
||||
|
||||
def test_proof_vectors_cover_every_kind():
|
||||
assert {c["kind"] for c in PROOFS} == {"root", "inclusion", "consistency"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case", [c for c in PROOFS if c["kind"] == "root"], ids=lambda c: c["id"]
|
||||
)
|
||||
def test_root_vectors(case):
|
||||
assert crypto.history_root(case["items"]) == case["root"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case", [c for c in PROOFS if c["kind"] == "inclusion"], ids=lambda c: c["id"]
|
||||
)
|
||||
def test_inclusion_vectors(case):
|
||||
items, index = case["items"], case["index"]
|
||||
assert crypto.history_root(items) == case["root"]
|
||||
assert crypto.inclusion_path(items, index) == case["path"]
|
||||
assert crypto.leaf_hash(items[index]) == case["leaf"]
|
||||
assert crypto.verify_inclusion(
|
||||
case["root"], case["leaf"], index, case["size"], case["path"]
|
||||
)
|
||||
if case["size"] > 1:
|
||||
other = (index + 1) % case["size"]
|
||||
assert not crypto.verify_inclusion(
|
||||
case["root"], case["leaf"], other, case["size"], case["path"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case", [c for c in PROOFS if c["kind"] == "consistency"], ids=lambda c: c["id"]
|
||||
)
|
||||
def test_consistency_vectors(case):
|
||||
items, m = case["items"], case["from"]
|
||||
assert crypto.history_root(items[:m]) == case["from_root"]
|
||||
assert crypto.history_root(items) == case["to_root"]
|
||||
assert crypto.consistency_proof(items, m) == case["path"]
|
||||
assert crypto.verify_consistency(
|
||||
case["from_root"], case["to_root"], m, case["to"], case["path"]
|
||||
)
|
||||
if m > 0:
|
||||
assert not crypto.verify_consistency(
|
||||
case["from_root"], "0" * 64, m, case["to"], case["path"]
|
||||
)
|
||||
|
||||
|
||||
def make_items(n):
|
||||
return [
|
||||
["node", f"n{i}", hashlib.sha256(str(i).encode()).hexdigest()] for i in range(n)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("index", [-1, 5])
|
||||
def test_inclusion_path_outside_the_tree_is_refused(index):
|
||||
with pytest.raises(ValueError, match="outside the tree"):
|
||||
crypto.inclusion_path(make_items(5), index)
|
||||
|
||||
|
||||
def test_inclusion_path_of_an_empty_tree_is_refused():
|
||||
with pytest.raises(ValueError, match="outside the tree"):
|
||||
crypto.inclusion_path([], 0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m", [-1, 6])
|
||||
def test_consistency_proof_beyond_the_tree_is_refused(m):
|
||||
with pytest.raises(ValueError, match="outside the tree"):
|
||||
crypto.consistency_proof(make_items(5), m)
|
||||
|
||||
|
||||
def test_consistency_proof_is_empty_for_the_empty_prefix_and_the_whole_tree():
|
||||
items = make_items(6)
|
||||
assert crypto.consistency_proof(items, 0) == []
|
||||
assert crypto.consistency_proof(items, 6) == []
|
||||
|
||||
|
||||
@given(TREES)
|
||||
@PROPS
|
||||
def test_every_inclusion_proof_verifies_and_only_at_its_own_index(items):
|
||||
root, size = crypto.history_root(items), len(items)
|
||||
for i, item in enumerate(items):
|
||||
path = crypto.inclusion_path(items, i)
|
||||
leaf = crypto.leaf_hash(item)
|
||||
assert crypto.verify_inclusion(root, leaf, i, size, path)
|
||||
assert not crypto.verify_inclusion(SHA256_EMPTY, leaf, i, size, path)
|
||||
if size > 1 and len(set(map(str, items))) == size:
|
||||
assert not crypto.verify_inclusion(root, leaf, (i + 1) % size, size, path)
|
||||
|
||||
|
||||
@given(TREES)
|
||||
@PROPS
|
||||
def test_every_prefix_is_consistent_with_the_whole(items):
|
||||
n, root = len(items), crypto.history_root(items)
|
||||
for m in range(n + 1):
|
||||
proof = crypto.consistency_proof(items, m)
|
||||
root_m = crypto.history_root(items[:m])
|
||||
assert crypto.verify_consistency(root_m, root, m, n, proof)
|
||||
if 0 < m < n:
|
||||
assert not crypto.verify_consistency(SHA256_EMPTY, root, m, n, proof)
|
||||
assert not crypto.verify_consistency(root_m, SHA256_EMPTY, m, n, proof)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("index", "size"), [(-1, 4), (4, 4), (0, 0)], ids=["negative", "at-size", "empty"]
|
||||
)
|
||||
def test_inclusion_outside_the_tree_does_not_verify(index, size):
|
||||
items = make_items(4)
|
||||
leaf = crypto.leaf_hash(items[0])
|
||||
root = crypto.history_root(items)
|
||||
assert not crypto.verify_inclusion(root, leaf, index, size, [])
|
||||
|
||||
|
||||
def test_inclusion_with_a_path_too_long_or_too_short_does_not_verify():
|
||||
items = make_items(4)
|
||||
root = crypto.history_root(items)
|
||||
path = crypto.inclusion_path(items, 2)
|
||||
leaf = crypto.leaf_hash(items[2])
|
||||
assert crypto.verify_inclusion(root, leaf, 2, 4, path)
|
||||
assert not crypto.verify_inclusion(root, leaf, 2, 4, [*path, path[0]])
|
||||
assert not crypto.verify_inclusion(root, leaf, 2, 4, path[:-1])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", [["zz"], [None], "not-a-list", None, 7], ids=repr)
|
||||
def test_inclusion_verification_never_raises_on_garbage(path):
|
||||
items = make_items(3)
|
||||
assert not crypto.verify_inclusion(
|
||||
crypto.history_root(items), crypto.leaf_hash(items[0]), 0, 3, path
|
||||
)
|
||||
|
||||
|
||||
def test_inclusion_verification_never_raises_on_a_garbage_leaf():
|
||||
assert not crypto.verify_inclusion("00" * 32, "not hex", 0, 1, [])
|
||||
|
||||
|
||||
def test_consistency_of_the_empty_prefix_needs_the_empty_root_and_no_proof():
|
||||
root = crypto.history_root(make_items(3))
|
||||
assert crypto.verify_consistency(crypto.EMPTY_ROOT, root, 0, 3, [])
|
||||
assert not crypto.verify_consistency(crypto.EMPTY_ROOT, root, 0, 3, [root])
|
||||
assert not crypto.verify_consistency(root, root, 0, 3, [])
|
||||
|
||||
|
||||
def test_consistency_of_equal_sizes_needs_equal_roots_and_no_proof():
|
||||
root = crypto.history_root(make_items(3))
|
||||
other = crypto.history_root(make_items(4))
|
||||
assert crypto.verify_consistency(root, root, 3, 3, [])
|
||||
assert not crypto.verify_consistency(root, other, 3, 3, [])
|
||||
assert not crypto.verify_consistency(root, root, 3, 3, [root])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("m", "n"), [(4, 3), (-1, 3)], ids=["prefix-longer", "negative-prefix"]
|
||||
)
|
||||
def test_consistency_with_impossible_sizes_does_not_verify(m, n):
|
||||
root = crypto.history_root(make_items(3))
|
||||
assert not crypto.verify_consistency(root, root, m, n, [root])
|
||||
|
||||
|
||||
def test_consistency_without_a_proof_does_not_verify():
|
||||
items = make_items(5)
|
||||
assert not crypto.verify_consistency(
|
||||
crypto.history_root(items[:2]), crypto.history_root(items), 2, 5, []
|
||||
)
|
||||
|
||||
|
||||
def test_consistency_with_extra_proof_hashes_does_not_verify():
|
||||
items = make_items(5)
|
||||
proof = crypto.consistency_proof(items, 3)
|
||||
args = (crypto.history_root(items[:3]), crypto.history_root(items), 3, 5)
|
||||
assert crypto.verify_consistency(*args, proof)
|
||||
assert not crypto.verify_consistency(*args, [*proof, proof[0]])
|
||||
assert not crypto.verify_consistency(*args, proof[:-1])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("proof", [["zz"], [None], None], ids=repr)
|
||||
def test_consistency_verification_never_raises_on_garbage(proof):
|
||||
items = make_items(4)
|
||||
assert not crypto.verify_consistency(
|
||||
crypto.history_root(items[:1]), crypto.history_root(items), 1, 4, proof
|
||||
)
|
||||
|
||||
|
||||
# --- Ed25519 ------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
RFC8032 = vector_cases("signatures")[0]
|
||||
|
||||
|
||||
def test_ed25519_matches_rfc_8032_test_1():
|
||||
secret = bytes.fromhex(RFC8032["secret"])
|
||||
message = RFC8032["message"].encode()
|
||||
assert crypto.ed25519_public(secret).hex() == RFC8032["public"]
|
||||
signature = crypto.ed25519_sign(secret, message)
|
||||
assert signature.hex() == RFC8032["signature"]
|
||||
assert crypto.ed25519_verify(bytes.fromhex(RFC8032["public"]), message, signature)
|
||||
|
||||
|
||||
def test_ed25519_rejects_a_changed_message_signature_or_key():
|
||||
secret = bytes.fromhex(RFC8032["secret"])
|
||||
public = crypto.ed25519_public(secret)
|
||||
signature = crypto.ed25519_sign(secret, b"message")
|
||||
flipped = bytes([signature[0] ^ 1]) + signature[1:]
|
||||
other = crypto.ed25519_public(bytes(32))
|
||||
assert crypto.ed25519_verify(public, b"message", signature)
|
||||
assert not crypto.ed25519_verify(public, b"messagf", signature)
|
||||
assert not crypto.ed25519_verify(public, b"message", flipped)
|
||||
assert not crypto.ed25519_verify(other, b"message", signature)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("public", "signature"),
|
||||
[(bytes(31), bytes(64)), (bytes(33), bytes(64)), (bytes(32), bytes(63))],
|
||||
ids=["short-key", "long-key", "short-signature"],
|
||||
)
|
||||
def test_ed25519_rejects_wrong_lengths(public, signature):
|
||||
assert not crypto.ed25519_verify(public, b"message", signature)
|
||||
|
||||
|
||||
def test_ed25519_rejects_bytes_that_encode_no_point():
|
||||
assert not crypto.ed25519_verify(b"\xff" * 32, b"message", bytes(64))
|
||||
|
||||
|
||||
@given(st.binary(min_size=32, max_size=32), st.binary(max_size=64))
|
||||
@PROPS
|
||||
def test_ed25519_signatures_verify_and_are_deterministic(secret, message):
|
||||
public = crypto.ed25519_public(secret)
|
||||
signature = crypto.ed25519_sign(secret, message)
|
||||
assert len(public) == 32
|
||||
assert len(signature) == 64
|
||||
assert signature == crypto.ed25519_sign(secret, message)
|
||||
assert crypto.ed25519_verify(public, message, signature)
|
||||
assert not crypto.ed25519_verify(public, message + b"\x00", signature)
|
||||
|
||||
|
||||
# --- did:key --------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[c for c in vector_cases("signatures") if "did" in c],
|
||||
ids=lambda c: c["id"],
|
||||
)
|
||||
def test_did_key_vectors(case):
|
||||
did = crypto.did_key(case["scheme"], bytes.fromhex(case["public"]))
|
||||
assert did == case["did"]
|
||||
|
||||
|
||||
def test_did_key_for_an_unknown_scheme_is_refused():
|
||||
with pytest.raises(ValueError, match="no multicodec code for rsa"):
|
||||
crypto.did_key("rsa", bytes(32))
|
||||
|
||||
|
||||
def test_did_key_decodes_to_the_multicodec_prefix_and_the_key():
|
||||
public = crypto.ed25519_public(bytes(32))
|
||||
did = crypto.did_key("ed25519", public)
|
||||
assert did.startswith("did:key:z6Mk")
|
||||
assert base58_decode(did.removeprefix("did:key:z")) == b"\xed\x01" + public
|
||||
|
||||
|
||||
def test_ml_dsa_multicodec_is_a_two_byte_varint():
|
||||
did = crypto.did_key("ml-dsa-65", b"\x07" * 4)
|
||||
assert base58_decode(did.removeprefix("did:key:z")) == b"\x87\x24" + b"\x07" * 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("data", "text"),
|
||||
[
|
||||
(b"", ""),
|
||||
(b"\x00", "1"),
|
||||
(b"\x00\x00\x01", "112"),
|
||||
(b"hello world", "StV1DL6CwTryKyV"),
|
||||
(b"\x00\x00hello", "11Cn8eVZg"),
|
||||
],
|
||||
)
|
||||
def test_base58btc_known_encodings(data, text):
|
||||
assert crypto.base58btc(data) == text
|
||||
|
||||
|
||||
@given(st.binary(max_size=40))
|
||||
@PROPS
|
||||
def test_base58btc_round_trips(data):
|
||||
text = crypto.base58btc(data)
|
||||
assert set(text) <= set(crypto.BASE58)
|
||||
assert base58_decode(text) == data
|
||||
@@ -0,0 +1,420 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Data tied to words: kinds of data, quantities, subjects and datasets (paperspec.data)."""
|
||||
|
||||
import copy
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from hypothesis import given, strategies as st
|
||||
|
||||
from conftest import FIXTURES, read_jsonl, vector_cases
|
||||
from helpers_lib2 import (
|
||||
HALE,
|
||||
PROPS,
|
||||
RECORDS,
|
||||
accepted_by_schema,
|
||||
rule_chapters,
|
||||
schema_store,
|
||||
)
|
||||
from paperspec import data
|
||||
|
||||
CASES = vector_cases("data")
|
||||
RULES = (
|
||||
"layer.data.type",
|
||||
"layer.data.quantity",
|
||||
"layer.data.subject",
|
||||
"layer.data.dataset",
|
||||
)
|
||||
LAYER = read_jsonl(RECORDS / "data" / "annotations.jsonl")
|
||||
BOND_PARAMS = json.loads(
|
||||
(
|
||||
FIXTURES
|
||||
/ "packages"
|
||||
/ HALE
|
||||
/ "datasets"
|
||||
/ "ds-hale-bond-params"
|
||||
/ "datapackage.json"
|
||||
).read_text(encoding="utf-8")
|
||||
)
|
||||
DESCRIPTORS = {
|
||||
"ds-hale-bond-params": BOND_PARAMS,
|
||||
"ds-hale-ctct-scan": json.loads(
|
||||
(RECORDS / "datasets" / "ds-hale-ctct-scan" / "datapackage.json").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
),
|
||||
}
|
||||
TABLE = next(c for c in CASES if c["id"] == "valid-a-table-against-its-dataset")[
|
||||
"annotation"
|
||||
]
|
||||
WATER = "XLYOFNOQVPJJNP-UHFFFAOYSA-N"
|
||||
PEPTIDE = "protein:5807fcbf6d5567ca"
|
||||
|
||||
|
||||
def edited(annotation, **body):
|
||||
"""A copy of `annotation` whose body has the given fields replaced; None removes a field."""
|
||||
out = copy.deepcopy(annotation)
|
||||
for key, value in body.items():
|
||||
if value is None:
|
||||
out["body"].pop(key, None)
|
||||
else:
|
||||
out["body"][key] = value
|
||||
return out
|
||||
|
||||
|
||||
def messages(annotation, dataset=None):
|
||||
"""The (rule, message) pairs `problems` reports for `annotation`."""
|
||||
return [(x["rule"], x["message"]) for x in data.problems(annotation, dataset)]
|
||||
|
||||
|
||||
# --- the vocabulary ---------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_the_kinds_of_data_are_those_of_the_annotation_schema():
|
||||
bodies = schema_store().load("annotation.schema.json")["properties"]["body"][
|
||||
"oneOf"
|
||||
]
|
||||
body = next(b for b in bodies if b["properties"]["kind"]["const"] == "data")
|
||||
assert set(data.DATA_TYPES) == set(body["properties"]["data_type"]["enum"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", data.DATA_TYPES)
|
||||
def test_each_kind_of_data_is_described(kind):
|
||||
assert data.DATA_TYPES[kind].endswith(".")
|
||||
|
||||
|
||||
def test_the_roles_and_subject_kinds():
|
||||
assert data.ROLES == ("reports", "uses")
|
||||
assert set(data.SUBJECT_KINDS) == {"structure", "sequence"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rule", RULES)
|
||||
def test_the_rules_are_registered_in_the_layers_chapter(rule):
|
||||
assert rule_chapters()[rule] == "spec/18-layers.md"
|
||||
|
||||
|
||||
# --- vectors ----------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", CASES, ids=lambda c: c["id"])
|
||||
def test_data_vectors(case):
|
||||
got = data.problems(case["annotation"], case.get("dataset"))
|
||||
assert got == case["expect"]
|
||||
assert accepted_by_schema(case["annotation"]) is case["schema"]
|
||||
assert {x["rule"] for x in got} <= set(rule_chapters())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", CASES, ids=lambda c: c["id"])
|
||||
def test_a_vector_is_named_for_the_rule_it_breaks(case):
|
||||
stem = case["id"].split("-", 1)[0]
|
||||
rules = {x["rule"] for x in case["expect"]}
|
||||
if stem == "valid":
|
||||
assert rules == set()
|
||||
else:
|
||||
assert "layer.data." + stem in rules
|
||||
|
||||
|
||||
def test_every_rule_is_broken_by_some_vector():
|
||||
assert {x["rule"] for c in CASES for x in c["expect"]} == set(RULES)
|
||||
|
||||
|
||||
def test_every_kind_of_data_has_a_valid_vector():
|
||||
ids = {c["id"] for c in CASES}
|
||||
assert {"valid-kind-" + kind for kind in data.DATA_TYPES} <= ids
|
||||
|
||||
|
||||
# --- the fixture layer -------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("annotation", LAYER, ids=lambda a: a["id"].rsplit("/", 1)[1])
|
||||
def test_the_data_layer_breaks_no_rule_alone_or_against_its_dataset(annotation):
|
||||
assert annotation["body"]["kind"] == "data"
|
||||
assert data.problems(annotation) == []
|
||||
descriptor = DESCRIPTORS.get(annotation["body"].get("dataset"))
|
||||
assert data.problems(annotation, descriptor) == []
|
||||
|
||||
|
||||
def test_the_data_layers_datasets_are_held_by_the_project():
|
||||
project = json.loads((RECORDS / "project.json").read_text(encoding="utf-8"))
|
||||
named = {a["body"]["dataset"] for a in LAYER if "dataset" in a["body"]}
|
||||
assert named <= set(project["datasets"])
|
||||
|
||||
|
||||
# --- type and role -----------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_an_annotation_of_another_kind_is_refused():
|
||||
with pytest.raises(ValueError, match="not a data annotation"):
|
||||
data.problems({"body": {"kind": "comment"}})
|
||||
with pytest.raises(ValueError, match="not a data annotation"):
|
||||
data.problems({"body": {}})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["spectroscopy", "Spectrum", None, ""])
|
||||
def test_a_kind_of_data_that_is_not_listed_is_refused(value):
|
||||
got = messages(edited(TABLE, data_type=value))
|
||||
assert [r for r, _ in got] == ["layer.data.type"]
|
||||
assert got[0][1].startswith("the kind of data is not one of abundance, benchmark")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("role", ["names", None, ""])
|
||||
def test_a_role_that_is_not_listed_is_refused(role):
|
||||
assert messages(edited(TABLE, role=role)) == [
|
||||
("layer.data.type", "the role is not one of reports, uses")
|
||||
]
|
||||
|
||||
|
||||
def test_both_a_wrong_kind_and_a_wrong_role_are_reported_in_that_order():
|
||||
got = messages(edited(TABLE, data_type="nope", role="nope"))
|
||||
assert [m.split(" is not")[0] for _, m in got] == ["the kind of data", "the role"]
|
||||
|
||||
|
||||
# --- quantities ----------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_field_may_be_given_once():
|
||||
quantities = [{"field": "a", "name": "A"}, {"field": "b", "name": "B"}]
|
||||
assert data.problems(edited(TABLE, quantities=quantities, dataset=None)) == []
|
||||
|
||||
|
||||
def test_a_repeated_field_is_reported_at_every_repeat():
|
||||
quantities = [{"field": "a"}, {"field": "b"}, {"field": "a"}, {"field": "a"}]
|
||||
got = messages(edited(TABLE, quantities=quantities, dataset=None))
|
||||
assert got == [
|
||||
("layer.data.quantity", "quantity 2 repeats the field a"),
|
||||
("layer.data.quantity", "quantity 3 repeats the field a"),
|
||||
]
|
||||
|
||||
|
||||
def test_quantities_without_a_field_repeat_the_empty_field():
|
||||
got = messages(edited(TABLE, quantities=[{}, {}], dataset=None))
|
||||
assert got == [("layer.data.quantity", "quantity 1 repeats the field ")]
|
||||
|
||||
|
||||
# --- subjects ------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"subject",
|
||||
[
|
||||
{"kind": "structure", "identity": WATER},
|
||||
{"kind": "structure", "identity": "smiles:0123456789ab"},
|
||||
{"kind": "structure", "identity": "smarts:0123456789ab"},
|
||||
{"kind": "sequence", "identity": PEPTIDE},
|
||||
{"kind": "sequence", "identity": "dna:0123456789abcdef"},
|
||||
{"kind": "sequence", "identity": "rna:0123456789abcdef"},
|
||||
],
|
||||
)
|
||||
def test_subjects_take_the_identities_of_the_structure_and_sequence_bodies(subject):
|
||||
assert data.problems(edited(TABLE, subjects=[subject], dataset=None)) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("subject", "fragment"),
|
||||
[
|
||||
(
|
||||
{"kind": "reaction", "identity": WATER},
|
||||
"is of a kind that is not structure or sequence",
|
||||
),
|
||||
({"identity": WATER}, "is of a kind that is not structure or sequence"),
|
||||
(
|
||||
{"kind": "structure", "identity": PEPTIDE},
|
||||
"is not the form of a structure identity",
|
||||
),
|
||||
(
|
||||
{"kind": "sequence", "identity": WATER},
|
||||
"is not the form of a sequence identity",
|
||||
),
|
||||
(
|
||||
{"kind": "structure", "identity": "smiles:0123456789a"},
|
||||
"not the form of a structure",
|
||||
),
|
||||
(
|
||||
{"kind": "structure", "identity": "smiles:0123456789ABCDEF"},
|
||||
"not the form of a structure",
|
||||
),
|
||||
({"kind": "structure", "identity": WATER + "X"}, "not the form of a structure"),
|
||||
({"kind": "structure", "identity": ""}, "not the form of a structure"),
|
||||
(
|
||||
{"kind": "sequence", "identity": "protein:0123456789abcdef0"},
|
||||
"not the form of a sequence",
|
||||
),
|
||||
(
|
||||
{"kind": "sequence", "identity": "peptide:0123456789abcdef"},
|
||||
"not the form of a sequence",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_a_subject_of_the_wrong_kind_or_form_is_refused(subject, fragment):
|
||||
got = messages(edited(TABLE, subjects=[subject], dataset=None))
|
||||
assert [r for r, _ in got] == ["layer.data.subject"]
|
||||
assert fragment in got[0][1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("suffix", ["\n", " ", "\x00"])
|
||||
def test_an_identity_is_matched_whole_so_a_trailing_character_is_refused(suffix):
|
||||
for identity, kind in ((WATER, "structure"), (PEPTIDE, "sequence")):
|
||||
subject = {"kind": kind, "identity": identity + suffix}
|
||||
got = messages(edited(TABLE, subjects=[subject], dataset=None))
|
||||
assert [r for r, _ in got] == ["layer.data.subject"]
|
||||
|
||||
|
||||
def test_a_subject_listed_twice_is_reported_at_its_second_place_only():
|
||||
first = {"kind": "structure", "identity": WATER}
|
||||
other = {"kind": "sequence", "identity": PEPTIDE}
|
||||
got = messages(edited(TABLE, subjects=[first, other, first], dataset=None))
|
||||
assert got == [("layer.data.subject", "subject 2 repeats the structure " + WATER)]
|
||||
|
||||
|
||||
def test_the_same_identity_under_two_kinds_is_not_a_repeat():
|
||||
subjects = [
|
||||
{"kind": "structure", "identity": WATER},
|
||||
{"kind": "sequence", "identity": WATER},
|
||||
]
|
||||
got = messages(edited(TABLE, subjects=subjects, dataset=None))
|
||||
assert [m for _, m in got] == [
|
||||
f"subject 1: {WATER} is not the form of a sequence identity"
|
||||
]
|
||||
|
||||
|
||||
def test_a_wrong_subject_that_is_repeated_is_reported_twice():
|
||||
bad = {"kind": "structure", "identity": "ethane"}
|
||||
got = messages(edited(TABLE, subjects=[bad, bad], dataset=None))
|
||||
assert [m.split(":")[0].split(" repeats")[0] for _, m in got] == [
|
||||
"subject 0",
|
||||
"subject 1",
|
||||
"subject 1",
|
||||
]
|
||||
|
||||
|
||||
@given(
|
||||
st.text(alphabet="ABCDEFGHIJKLMNOPQRSTUVWXYZ", min_size=14, max_size=14),
|
||||
st.text(alphabet="ABCDEFGHIJKLMNOPQRSTUVWXYZ", min_size=10, max_size=10),
|
||||
st.sampled_from("ABCDEFGHIJKLMNOPQRSTUVWXYZ"),
|
||||
)
|
||||
@PROPS
|
||||
def test_any_inchikey_shaped_identity_is_a_structure_and_never_a_sequence(
|
||||
first, second, last
|
||||
):
|
||||
identity = f"{first}-{second}-{last}"
|
||||
assert data.SUBJECT_KINDS["structure"].fullmatch(identity)
|
||||
assert not data.SUBJECT_KINDS["sequence"].fullmatch(identity)
|
||||
assert not data.SUBJECT_KINDS["structure"].fullmatch(identity + "-")
|
||||
|
||||
|
||||
# --- datasets ---------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"dataset", ["Table 1", "ds-", "DS-a", "ds-A", "ds-a b", "ds-a\n", "", "dataset-a"]
|
||||
)
|
||||
def test_a_dataset_that_is_no_id_is_refused(dataset):
|
||||
got = messages(edited(TABLE, dataset=dataset))
|
||||
assert got == [("layer.data.dataset", f"{dataset} is not the ID of a dataset")]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dataset", ["ds-a", "ds-0", "ds-hale.bond_params-2", "ds-a-"])
|
||||
def test_a_dataset_id_starts_with_ds_and_a_lower_case_letter_or_digit(dataset):
|
||||
assert data.problems(edited(TABLE, dataset=dataset, quantities=None)) == []
|
||||
|
||||
|
||||
def test_without_a_dataset_there_is_nothing_to_compare():
|
||||
annotation = edited(TABLE, dataset=None)
|
||||
assert data.problems(annotation, BOND_PARAMS) == []
|
||||
|
||||
|
||||
def test_without_a_descriptor_only_the_id_is_checked():
|
||||
assert data.problems(TABLE) == []
|
||||
assert data.problems(TABLE, None) == []
|
||||
|
||||
|
||||
def test_the_main_examples_bond_table_agrees_with_its_descriptor():
|
||||
assert TABLE["body"]["dataset"] == BOND_PARAMS["id"]
|
||||
assert data.problems(TABLE, BOND_PARAMS) == []
|
||||
|
||||
|
||||
def test_another_dataset_than_the_descriptor_is_named_in_the_message():
|
||||
got = messages(TABLE, DESCRIPTORS["ds-hale-ctct-scan"])
|
||||
assert got[0] == (
|
||||
"layer.data.dataset",
|
||||
"the dataset is ds-hale-ctct-scan, not ds-hale-bond-params",
|
||||
)
|
||||
|
||||
|
||||
def test_a_field_the_dataset_lacks_is_reported_by_quantity():
|
||||
quantities = [{"field": "k_r"}, {"field": "nope", "name": "Nothing"}]
|
||||
got = messages(edited(TABLE, quantities=quantities), BOND_PARAMS)
|
||||
assert got == [
|
||||
(
|
||||
"layer.data.dataset",
|
||||
"quantity 1 names the field nope, which ds-hale-bond-params does not have",
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_a_unit_that_differs_from_the_datasets_is_reported_with_both():
|
||||
quantities = [
|
||||
{"field": "k_r", "unit": "N / m"},
|
||||
{"field": "r_eq", "unit": "angstrom"},
|
||||
]
|
||||
got = messages(edited(TABLE, quantities=quantities), BOND_PARAMS)
|
||||
assert got == [
|
||||
(
|
||||
"layer.data.dataset",
|
||||
"quantity 0 gives k_r the unit N / m, but ds-hale-bond-params gives it kcal / mol / angstrom ** 2",
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_a_quantity_without_a_unit_takes_the_datasets():
|
||||
quantities = [{"field": "k_r"}, {"field": "r_eq"}]
|
||||
assert data.problems(edited(TABLE, quantities=quantities), BOND_PARAMS) == []
|
||||
|
||||
|
||||
def test_a_unit_given_for_a_field_that_has_none_in_the_dataset_is_a_difference():
|
||||
dataset = {"id": "ds-x", "resources": [{"schema": {"fields": [{"name": "point"}]}}]}
|
||||
annotation = edited(
|
||||
TABLE, dataset="ds-x", quantities=[{"field": "point", "unit": "m"}]
|
||||
)
|
||||
assert messages(annotation, dataset) == [
|
||||
(
|
||||
"layer.data.dataset",
|
||||
"quantity 0 gives point the unit m, but ds-x gives it None",
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_the_fields_of_every_resource_of_the_dataset_count():
|
||||
dataset = {
|
||||
"id": "ds-x",
|
||||
"resources": [
|
||||
{"schema": {"fields": [{"name": "a", "unit": "m"}]}},
|
||||
{"name": "no schema"},
|
||||
{"schema": {"fields": [{"name": "b", "unit": "s"}]}},
|
||||
],
|
||||
}
|
||||
annotation = edited(
|
||||
TABLE,
|
||||
dataset="ds-x",
|
||||
quantities=[{"field": "a", "unit": "m"}, {"field": "b", "unit": "s"}],
|
||||
)
|
||||
assert data.problems(annotation, dataset) == []
|
||||
|
||||
|
||||
def test_a_descriptor_without_resources_has_no_fields():
|
||||
annotation = edited(TABLE, dataset="ds-x", quantities=[{"field": "a"}])
|
||||
got = messages(annotation, {"id": "ds-x"})
|
||||
assert got == [
|
||||
("layer.data.dataset", "quantity 0 names the field a, which ds-x does not have")
|
||||
]
|
||||
|
||||
|
||||
def test_a_descriptor_with_another_id_reports_the_mismatch_before_the_fields():
|
||||
annotation = edited(TABLE, dataset="ds-x", quantities=[{"field": "a"}])
|
||||
got = messages(annotation, {"id": "ds-y"})
|
||||
assert [m.split(" ")[0] for _, m in got] == ["the", "quantity"]
|
||||
@@ -0,0 +1,564 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""The database form: loading a package into tables and unloading it again (paperspec.db)."""
|
||||
|
||||
import collections
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import sqlite3
|
||||
|
||||
import pytest
|
||||
|
||||
from conftest import FIXTURES, ROOT, read_jsonl
|
||||
from helpers_lib2 import HALE, RECORDS
|
||||
from paperspec import db, jcs, package, resources
|
||||
from paperspec.tree import Refusal
|
||||
|
||||
PACKAGES = sorted(p.name for p in (FIXTURES / "packages").iterdir() if p.is_dir())
|
||||
SQL = resources.path("sql")
|
||||
TINY = "tiny2026tiny-j4qm"
|
||||
ROW_COUNTS = {
|
||||
"nodes.jsonl": "SELECT COUNT(*) FROM nodes",
|
||||
"pages.jsonl": "SELECT COUNT(*) FROM pages",
|
||||
"locations.jsonl": "SELECT COUNT(*) FROM locations",
|
||||
"references.jsonl": "SELECT COUNT(*) FROM refs",
|
||||
"corrections.jsonl": "SELECT COUNT(*) FROM corrections",
|
||||
"ids.jsonl": "SELECT COUNT(*) FROM node_ids",
|
||||
"suspects.jsonl": "SELECT COUNT(*) FROM suspects",
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def conn():
|
||||
"""An empty in-memory bundle database."""
|
||||
connection = sqlite3.connect(":memory:")
|
||||
db.create_bundle(connection, SQL)
|
||||
yield connection
|
||||
connection.close()
|
||||
|
||||
|
||||
def tree(root):
|
||||
"""Every file under `root` by relative path, with its bytes."""
|
||||
return {
|
||||
p.relative_to(root).as_posix(): p.read_bytes()
|
||||
for p in root.rglob("*")
|
||||
if p.is_file()
|
||||
}
|
||||
|
||||
|
||||
def column(conn, sql, *args):
|
||||
"""The first column of every row `sql` returns."""
|
||||
return [row[0] for row in conn.execute(sql, args)]
|
||||
|
||||
|
||||
def read_jsonl_bytes(data):
|
||||
"""The records of JSON Lines `data`, split at line feeds only."""
|
||||
return [json.loads(x) for x in data.decode("utf-8").split("\n") if x]
|
||||
|
||||
|
||||
def jsonl_of(name, filename):
|
||||
"""The records of a fixture package's JSON Lines file, or nothing when it has none."""
|
||||
path = FIXTURES / "packages" / name / filename
|
||||
return read_jsonl(path) if path.exists() else []
|
||||
|
||||
|
||||
# --- loading and unloading ----------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", PACKAGES)
|
||||
def test_a_package_unloads_to_the_bytes_it_was_loaded_from(conn, tmp_path, name):
|
||||
source = FIXTURES / "packages" / name
|
||||
assets = tmp_path / "assets"
|
||||
key = db.load(conn, str(source), str(assets))
|
||||
db.index(conn, key)
|
||||
package_hash = db.unload(conn, key, str(tmp_path / "out"), str(assets))
|
||||
assert key == json.loads((source / "document.json").read_bytes())["key"]
|
||||
assert tree(tmp_path / "out") == tree(source)
|
||||
assert (
|
||||
package_hash
|
||||
== hashlib.sha256((source / "MANIFEST.json").read_bytes()).hexdigest()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", PACKAGES)
|
||||
def test_loading_fills_a_row_for_each_record(conn, tmp_path, name):
|
||||
db.load(conn, str(FIXTURES / "packages" / name), str(tmp_path / "assets"))
|
||||
for filename, count in ROW_COUNTS.items():
|
||||
found = conn.execute(count).fetchone()[0]
|
||||
assert found == len(jsonl_of(name, filename)), filename
|
||||
receipts = jsonl_of(name, "receipts.jsonl")
|
||||
kinds = collections.Counter(r["format"] for r in receipts)
|
||||
assert (
|
||||
conn.execute("SELECT COUNT(*) FROM receipts").fetchone()[0]
|
||||
== kinds["scienting-receipt/1"]
|
||||
)
|
||||
assert (
|
||||
conn.execute("SELECT COUNT(*) FROM revocations").fetchone()[0]
|
||||
== kinds["scienting-revocation/1"]
|
||||
)
|
||||
witnesses = json.loads(
|
||||
(FIXTURES / "packages" / name / "witnesses.json").read_bytes()
|
||||
)["witnesses"]
|
||||
assert conn.execute("SELECT COUNT(*) FROM witnesses").fetchone()[0] == len(
|
||||
witnesses
|
||||
)
|
||||
|
||||
|
||||
def test_the_main_examples_receipts_revocation_and_nodes_are_in_their_tables(
|
||||
conn, tmp_path
|
||||
):
|
||||
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets"))
|
||||
receipts = [
|
||||
r
|
||||
for r in jsonl_of(HALE, "receipts.jsonl")
|
||||
if r["format"] == "scienting-receipt/1"
|
||||
]
|
||||
listed = sum(len(r["nodes"]) for r in receipts)
|
||||
assert (
|
||||
conn.execute(
|
||||
"SELECT COUNT(*) FROM receipt_nodes WHERE doc_key = ?", (key,)
|
||||
).fetchone()[0]
|
||||
== listed
|
||||
)
|
||||
revoked = conn.execute("SELECT receipt, signer FROM revocations").fetchall()
|
||||
assert revoked == [("q465jpwz8afhe", "cara")]
|
||||
works = conn.execute("SELECT work_key FROM works").fetchall()
|
||||
assert works == [
|
||||
(
|
||||
json.loads((FIXTURES / "packages" / HALE / "document.json").read_bytes())[
|
||||
"work"
|
||||
]["key"],
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_each_citation_of_a_sentence_is_stored_once_with_its_run_order(conn, tmp_path):
|
||||
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets"))
|
||||
expected = set()
|
||||
for n in jsonl_of(HALE, "nodes.jsonl"):
|
||||
if isinstance(n.get("content"), list):
|
||||
run = 0
|
||||
for r in n["content"]:
|
||||
if r["kind"] == "cite":
|
||||
expected |= {(n["id"], ref, run) for ref in r["refs"]}
|
||||
run += 1
|
||||
found = set(
|
||||
conn.execute(
|
||||
"SELECT sentence, ref_node, ord FROM citations WHERE doc_key = ?", (key,)
|
||||
)
|
||||
)
|
||||
assert found == expected
|
||||
assert found
|
||||
|
||||
|
||||
def test_files_without_tables_are_stored_by_their_hash_and_absent_ones_are_listed(
|
||||
conn, tmp_path
|
||||
):
|
||||
name = HALE + ".checks-only"
|
||||
source = FIXTURES / "packages" / name
|
||||
assets = tmp_path / "assets"
|
||||
key = db.load(conn, str(source), str(assets))
|
||||
manifest = json.loads((source / "MANIFEST.json").read_bytes())["files"]
|
||||
rows = {
|
||||
p: (s, z, pr)
|
||||
for p, s, z, pr in conn.execute(
|
||||
"SELECT path, sha256, size, present FROM package_files WHERE doc_key = ?",
|
||||
(key,),
|
||||
)
|
||||
}
|
||||
absent = [f for f in manifest if f.get("present") is False]
|
||||
assert absent
|
||||
stored = {p.name for p in assets.iterdir()}
|
||||
for f in manifest:
|
||||
if f["path"] in db.STRUCTURED:
|
||||
assert f["path"] not in rows
|
||||
continue
|
||||
present = f.get("present", True)
|
||||
assert rows[f["path"]] == (f["sha256"], f["size"], 1 if present else 0)
|
||||
assert (f["sha256"] in stored) is present
|
||||
|
||||
|
||||
def test_a_second_load_of_one_document_into_one_database_is_refused(conn, tmp_path):
|
||||
source = str(FIXTURES / "packages" / TINY)
|
||||
db.load(conn, source, str(tmp_path / "assets"))
|
||||
with pytest.raises(sqlite3.IntegrityError):
|
||||
db.load(conn, source, str(tmp_path / "assets"))
|
||||
|
||||
|
||||
def test_two_documents_of_one_work_share_one_works_row(conn, tmp_path):
|
||||
second = tmp_path / "second"
|
||||
shutil.copytree(FIXTURES / "packages" / TINY, second)
|
||||
document = json.loads((second / "document.json").read_bytes())
|
||||
document["key"] += ".v2"
|
||||
(second / "document.json").write_bytes((jcs.dumps(document) + "\n").encode("utf-8"))
|
||||
assets = str(tmp_path / "assets")
|
||||
db.load(conn, str(FIXTURES / "packages" / TINY), assets)
|
||||
assert db.load(conn, str(second), assets) == TINY + ".v2"
|
||||
assert column(conn, "SELECT doc_key FROM documents ORDER BY doc_key") == [
|
||||
TINY,
|
||||
TINY + ".v2",
|
||||
]
|
||||
assert column(conn, "SELECT COUNT(*) FROM works") == [1]
|
||||
|
||||
|
||||
def test_suspects_approximate_locations_and_period_revocations_survive_a_round_trip(
|
||||
conn, tmp_path
|
||||
):
|
||||
files = {
|
||||
k: v
|
||||
for k, v in tree(FIXTURES / "packages" / HALE).items()
|
||||
if k != "MANIFEST.json"
|
||||
}
|
||||
locations = read_jsonl_bytes(files["locations.jsonl"])
|
||||
locations[0]["approx"] = True
|
||||
files["locations.jsonl"] = package.jsonl(locations)
|
||||
receipts = read_jsonl_bytes(files["receipts.jsonl"])
|
||||
receipts.append(
|
||||
{
|
||||
"at": "2026-12-01T00:00:00Z",
|
||||
"by": "ana",
|
||||
"format": "scienting-revocation/1",
|
||||
"id": "zzzzzzzzzzzzz",
|
||||
"period": {"from": "2026-10-01T00:00:00Z", "to": "2026-10-02T00:00:00Z"},
|
||||
"reason": "A period of a key.",
|
||||
"signature": {"key": "ana", "scheme": "ed25519", "value": "AAAA"},
|
||||
"signer": "ben",
|
||||
}
|
||||
)
|
||||
files["receipts.jsonl"] = package.jsonl(receipts)
|
||||
suspect = json.loads((RECORDS / "suspect.json").read_bytes())
|
||||
files["suspects.jsonl"] = package.jsonl([suspect])
|
||||
source = tmp_path / "source"
|
||||
package.write(str(source), files)
|
||||
assets = tmp_path / "assets"
|
||||
key = db.load(conn, str(source), str(assets))
|
||||
assert conn.execute("SELECT kind, node FROM suspects").fetchall() == [
|
||||
(suspect["kind"], suspect["node"])
|
||||
]
|
||||
assert conn.execute(
|
||||
"SELECT period_from, period_to, revoked_signer FROM revocations WHERE id = 'zzzzzzzzzzzzz'"
|
||||
).fetchall() == [("2026-10-01T00:00:00Z", "2026-10-02T00:00:00Z", "ben")]
|
||||
db.unload(conn, key, str(tmp_path / "out"), str(assets))
|
||||
assert tree(tmp_path / "out") == tree(source)
|
||||
|
||||
|
||||
# --- refusals --------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def edit_manifest(root, edit):
|
||||
"""Rewrite the manifest of the package at `root` after `edit(manifest)` has changed it."""
|
||||
path = root / "MANIFEST.json"
|
||||
manifest = json.loads(path.read_bytes())
|
||||
edit(manifest)
|
||||
path.write_bytes((jcs.dumps(manifest) + "\n").encode("utf-8"))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def copy_of_tiny(tmp_path):
|
||||
root = tmp_path / "package"
|
||||
shutil.copytree(FIXTURES / "packages" / TINY, root)
|
||||
return root
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path", ["../evil", "/abs", "a//b", "a/./b", "a\\b", "", "nodes.jsonl/.."]
|
||||
)
|
||||
def test_a_manifest_that_lists_an_unsafe_path_is_refused(
|
||||
conn, tmp_path, copy_of_tiny, path
|
||||
):
|
||||
edit_manifest(
|
||||
copy_of_tiny,
|
||||
lambda m: m["files"].append({"path": path, "sha256": "0" * 64, "size": 0}),
|
||||
)
|
||||
with pytest.raises(Refusal) as excinfo:
|
||||
db.load(conn, str(copy_of_tiny), str(tmp_path / "assets"))
|
||||
assert excinfo.value.rule == "package.path.unsafe"
|
||||
assert not (tmp_path / "assets").exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"sha", ["xyz", "A" * 64, "0" * 63, "0" * 65, 5, None, ""], ids=repr
|
||||
)
|
||||
def test_a_manifest_hash_that_is_not_64_lowercase_hex_digits_is_refused(
|
||||
conn, tmp_path, copy_of_tiny, sha
|
||||
):
|
||||
edit_manifest(copy_of_tiny, lambda m: m["files"][0].update(sha256=sha))
|
||||
with pytest.raises(Refusal) as excinfo:
|
||||
db.load(conn, str(copy_of_tiny), str(tmp_path / "assets"))
|
||||
assert excinfo.value.rule == "package.manifest.schema"
|
||||
|
||||
|
||||
def test_a_hash_with_a_final_line_feed_is_refused(conn, tmp_path, copy_of_tiny):
|
||||
edit_manifest(copy_of_tiny, lambda m: m["files"][0].update(sha256="0" * 64 + "\n"))
|
||||
with pytest.raises(Refusal) as excinfo:
|
||||
db.load(conn, str(copy_of_tiny), str(tmp_path / "assets"))
|
||||
assert excinfo.value.rule == "package.manifest.schema"
|
||||
|
||||
|
||||
def test_a_symbolic_link_anywhere_in_the_package_is_refused(
|
||||
conn, tmp_path, copy_of_tiny
|
||||
):
|
||||
(copy_of_tiny / "link").symlink_to(copy_of_tiny / "nodes.jsonl")
|
||||
with pytest.raises(Refusal) as excinfo:
|
||||
db.load(conn, str(copy_of_tiny), str(tmp_path / "assets"))
|
||||
assert excinfo.value.rule == "package.path.unsafe"
|
||||
assert conn.execute("SELECT COUNT(*) FROM documents").fetchone()[0] == 0
|
||||
|
||||
|
||||
def test_a_stored_hash_that_is_not_a_hash_is_refused_when_unloading(conn, tmp_path):
|
||||
assets = tmp_path / "assets"
|
||||
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(assets))
|
||||
conn.execute("UPDATE package_files SET sha256 = 'not a hash' WHERE present = 1")
|
||||
with pytest.raises(Refusal) as excinfo:
|
||||
db.unload(conn, key, str(tmp_path / "out"), str(assets))
|
||||
assert excinfo.value.rule == "package.manifest.schema"
|
||||
|
||||
|
||||
def test_unloading_without_a_stored_file_fails(conn, tmp_path):
|
||||
assets = tmp_path / "assets"
|
||||
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(assets))
|
||||
next(assets.iterdir()).unlink()
|
||||
with pytest.raises(FileNotFoundError):
|
||||
db.unload(conn, key, str(tmp_path / "out"), str(assets))
|
||||
|
||||
|
||||
# --- the bundle: search rows, the index and metadata --------------------------------------------------------------------
|
||||
|
||||
|
||||
def expected_units(nodes):
|
||||
"""The search units the chapter gives for `nodes`: sentences, cells, equations outside a sentence, figure text."""
|
||||
parent = {n["id"]: n["kind"] for n in nodes}
|
||||
units = collections.Counter()
|
||||
for n in nodes:
|
||||
if n.get("withheld"):
|
||||
continue
|
||||
if n["kind"] in ("sentence", "cell") or (
|
||||
n["kind"] == "equation" and parent.get(n["parent"]) != "sentence"
|
||||
):
|
||||
units[n["kind"]] += 1
|
||||
elif n["kind"] == "panel":
|
||||
units["figure-text"] += len((n.get("attrs") or {}).get("figure_text", []))
|
||||
return units
|
||||
|
||||
|
||||
def test_the_index_holds_one_row_per_unit_of_the_main_example(conn, tmp_path):
|
||||
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets"))
|
||||
added = db.index(conn, key)
|
||||
found = collections.Counter(
|
||||
dict(conn.execute("SELECT unit, COUNT(*) FROM search GROUP BY unit"))
|
||||
)
|
||||
assert found == expected_units(jsonl_of(HALE, "nodes.jsonl"))
|
||||
assert added == sum(found.values()) > 0
|
||||
assert {"sentence", "cell", "figure-text"} <= set(found)
|
||||
|
||||
|
||||
def test_text_that_is_withheld_is_never_indexed(conn, tmp_path):
|
||||
name = HALE + ".checks-only"
|
||||
nodes = jsonl_of(name, "nodes.jsonl")
|
||||
assert any(n.get("withheld") for n in nodes)
|
||||
key = db.load(conn, str(FIXTURES / "packages" / name), str(tmp_path / "assets"))
|
||||
assert db.index(conn, key) == 0
|
||||
assert conn.execute("SELECT COUNT(*) FROM search").fetchone()[0] == 0
|
||||
assert sum(expected_units(nodes).values()) == 0
|
||||
|
||||
|
||||
def test_the_index_finds_words_and_says_where_they_are(conn, tmp_path):
|
||||
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets"))
|
||||
db.index(conn, key)
|
||||
hits = conn.execute(
|
||||
"SELECT node, unit FROM search WHERE search MATCH 'scanned'"
|
||||
).fetchall()
|
||||
assert len(hits) == 1
|
||||
contexts = column(
|
||||
conn,
|
||||
"SELECT context FROM search WHERE unit = 'sentence' AND search MATCH 'Methods'",
|
||||
)
|
||||
assert "heading" in contexts
|
||||
|
||||
|
||||
def nodes_with_equations():
|
||||
"""A document with an equation inside a sentence and another outside one."""
|
||||
return [
|
||||
{"id": "d", "kind": "document", "parent": None, "ord": 0},
|
||||
{"id": "p", "kind": "paragraph", "parent": "d", "ord": 0},
|
||||
{
|
||||
"id": "s",
|
||||
"kind": "sentence",
|
||||
"parent": "p",
|
||||
"ord": 0,
|
||||
"content": [
|
||||
{"kind": "text", "text": "Then "},
|
||||
{"kind": "equation", "node": "e1"},
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "e1",
|
||||
"kind": "equation",
|
||||
"parent": "s",
|
||||
"ord": 0,
|
||||
"content": {"notation": "tex", "source": "a=b"},
|
||||
},
|
||||
{
|
||||
"id": "e2",
|
||||
"kind": "equation",
|
||||
"parent": "p",
|
||||
"ord": 1,
|
||||
"content": {"notation": "tex", "source": "c=d"},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def test_an_equation_is_a_unit_only_outside_a_sentence():
|
||||
rows = db.search_rows("k", nodes_with_equations())
|
||||
assert [(n, unit, context) for _, n, unit, context, _ in rows] == [
|
||||
("s", "sentence", "paragraph"),
|
||||
("e2", "equation", "paragraph"),
|
||||
]
|
||||
assert all(row[0] == "k" for row in rows)
|
||||
|
||||
|
||||
def test_search_text_is_the_normalized_plain_projection():
|
||||
nodes = [
|
||||
{"id": "d", "kind": "document", "parent": None, "ord": 0},
|
||||
{
|
||||
"id": "s",
|
||||
"kind": "sentence",
|
||||
"parent": "d",
|
||||
"ord": 0,
|
||||
"content": [{"kind": "text", "text": "Hello \t world\n"}],
|
||||
},
|
||||
]
|
||||
assert db.search_rows("k", nodes) == [
|
||||
("k", "s", "sentence", "document", "Hello world")
|
||||
]
|
||||
|
||||
|
||||
def test_figure_text_takes_its_panel_as_the_node_and_its_role_as_the_context():
|
||||
nodes = [
|
||||
{"id": "d", "kind": "document", "parent": None, "ord": 0},
|
||||
{"id": "f", "kind": "figure", "parent": "d", "ord": 0},
|
||||
{
|
||||
"id": "pn",
|
||||
"kind": "panel",
|
||||
"parent": "f",
|
||||
"ord": 0,
|
||||
"attrs": {
|
||||
"figure_text": [
|
||||
{"role": "label", "text": "r eq"},
|
||||
{"role": "axis", "text": "E"},
|
||||
]
|
||||
},
|
||||
},
|
||||
{"id": "pn2", "kind": "panel", "parent": "f", "ord": 1},
|
||||
]
|
||||
assert db.search_rows("k", nodes) == [
|
||||
("k", "pn", "figure-text", "label", "r eq"),
|
||||
("k", "pn", "figure-text", "axis", "E"),
|
||||
]
|
||||
|
||||
|
||||
def test_a_withheld_node_is_skipped():
|
||||
nodes = nodes_with_equations()
|
||||
nodes[2]["withheld"] = True
|
||||
assert [n for _, n, _, _, _ in db.search_rows("k", nodes)] == ["e2"]
|
||||
|
||||
|
||||
def test_the_metadata_names_the_format_the_store_the_source_and_the_spec(conn):
|
||||
db.write_meta(conn, "smithlab", "fixtures/packages")
|
||||
rows = conn.execute("SELECT name, value FROM bundle_meta").fetchall()
|
||||
assert rows == [
|
||||
("built_from", "fixtures/packages"),
|
||||
("format", "scienting-bundle/1"),
|
||||
("spec", db.SPEC_VERSION),
|
||||
("store", "smithlab"),
|
||||
]
|
||||
|
||||
|
||||
def test_extra_metadata_is_added_and_may_replace_a_default(conn):
|
||||
db.write_meta(conn, "s", "b", {"embedding": "none", "spec": "9.9.9"})
|
||||
meta = dict(conn.execute("SELECT name, value FROM bundle_meta"))
|
||||
assert meta == {
|
||||
"built_from": "b",
|
||||
"embedding": "none",
|
||||
"format": db.BUNDLE_FORMAT,
|
||||
"spec": "9.9.9",
|
||||
"store": "s",
|
||||
}
|
||||
|
||||
|
||||
def test_the_specification_version_is_the_changelogs_latest():
|
||||
changelog = (ROOT / "CHANGELOG.md").read_text(encoding="utf-8")
|
||||
top = next(
|
||||
line[3:].split()[0] for line in changelog.splitlines() if line.startswith("## ")
|
||||
)
|
||||
assert top == db.SPEC_VERSION
|
||||
|
||||
|
||||
def test_the_bundle_format_is_a_registered_format():
|
||||
formats = json.loads(
|
||||
(ROOT / "registry" / "formats.json").read_text(encoding="utf-8")
|
||||
)["formats"]
|
||||
assert db.BUNDLE_FORMAT in {f["format"] for f in formats}
|
||||
|
||||
|
||||
def test_a_database_without_the_bundle_tables_has_no_search_index():
|
||||
plain = sqlite3.connect(":memory:")
|
||||
try:
|
||||
db.create(plain, SQL)
|
||||
tables = set(
|
||||
column(plain, "SELECT name FROM sqlite_master WHERE type = 'table'")
|
||||
)
|
||||
assert {"documents", "nodes", "receipts", "annotations", "layers"} <= tables
|
||||
assert not {"search", "bundle_meta"} & tables
|
||||
finally:
|
||||
plain.close()
|
||||
|
||||
|
||||
def test_the_bundle_tables_add_the_search_index_and_the_metadata(conn):
|
||||
tables = set(column(conn, "SELECT name FROM sqlite_master WHERE type IN ('table')"))
|
||||
assert {"search", "bundle_meta", "documents", "annotations"} <= tables
|
||||
|
||||
|
||||
# --- what the columns hold ---------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_structured_columns_hold_canonical_json_and_absent_ones_hold_null(
|
||||
conn, tmp_path
|
||||
):
|
||||
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets"))
|
||||
stored = dict(
|
||||
conn.execute("SELECT id, content FROM nodes WHERE doc_key = ?", (key,))
|
||||
)
|
||||
nodes = jsonl_of(HALE, "nodes.jsonl")
|
||||
assert any("content" not in n for n in nodes)
|
||||
for n in nodes:
|
||||
want = jcs.dumps(n["content"]) if "content" in n else None
|
||||
assert stored[n["id"]] == want, n["id"]
|
||||
|
||||
|
||||
def test_flags_are_stored_as_one_zero_or_null(conn, tmp_path):
|
||||
name = HALE + ".checks-only"
|
||||
key = db.load(conn, str(FIXTURES / "packages" / name), str(tmp_path / "assets"))
|
||||
withheld = dict(
|
||||
conn.execute("SELECT id, withheld FROM nodes WHERE doc_key = ?", (key,))
|
||||
)
|
||||
for n in jsonl_of(name, "nodes.jsonl"):
|
||||
assert withheld[n["id"]] == (1 if n.get("withheld") else None)
|
||||
witnesses = json.loads(
|
||||
(FIXTURES / "packages" / name / "witnesses.json").read_bytes()
|
||||
)["witnesses"]
|
||||
layers = dict(
|
||||
conn.execute("SELECT id, text_layer FROM witnesses WHERE doc_key = ?", (key,))
|
||||
)
|
||||
for w in witnesses:
|
||||
flag = w.get("text_layer")
|
||||
assert layers[w["id"]] == (None if flag is None else int(flag))
|
||||
pages = conn.execute(
|
||||
"SELECT text_layer FROM pages WHERE doc_key = ?", (key,)
|
||||
).fetchall()
|
||||
assert {row[0] for row in pages} <= {0, 1}
|
||||
@@ -0,0 +1,230 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Page fingerprints and download-stamp rules (spec/07 §7.1)."""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from hypothesis import given, settings, strategies as st
|
||||
|
||||
from conftest import ROOT
|
||||
from helpers_lib1 import read_cases, vectors
|
||||
from paperspec import fingerprint as fp
|
||||
from paperspec.text import HASH_RE
|
||||
|
||||
PROPERTIES = settings(derandomize=True, max_examples=100, database=None, deadline=None)
|
||||
|
||||
RULES = json.loads(
|
||||
(ROOT / "registry" / "stamps" / "scienting-stamps-1.json").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
)["rules"]
|
||||
SIZE = [595, 842]
|
||||
|
||||
|
||||
def page(lines, size=None, images=()):
|
||||
"""A page of text lines, each a string or a line object."""
|
||||
return {
|
||||
"size": size or SIZE,
|
||||
"lines": [{"text": x} if isinstance(x, str) else x for x in lines],
|
||||
"images": list(images),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", vectors("fingerprints"))
|
||||
def test_fingerprint_vector(case):
|
||||
fingerprint, removed = fp.fingerprint(case["page"], RULES)
|
||||
assert fingerprint == case["page_fp"]
|
||||
assert removed == case["removed"]
|
||||
|
||||
|
||||
def test_two_downloads_of_one_paper_share_a_fingerprint_and_a_change_does_not():
|
||||
by_id = {c["id"]: c["page_fp"] for c in read_cases("fingerprints")}
|
||||
assert by_id["acs-download-1"] == by_id["acs-download-2"]
|
||||
assert by_id["wiley-download-1"] == by_id["wiley-download-2"]
|
||||
assert by_id["acs-download-1"] != by_id["acs-changed-text"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("x", "whole"),
|
||||
[
|
||||
(0, 0),
|
||||
(0.49, 0),
|
||||
(0.5, 1),
|
||||
(1.5, 2),
|
||||
(2.5, 3),
|
||||
(595.276, 595),
|
||||
(841.89, 842),
|
||||
(-0.5, 0),
|
||||
(-1.5, -1),
|
||||
],
|
||||
)
|
||||
def test_round_point_rounds_half_up(x, whole):
|
||||
assert fp.round_point(x) == whole
|
||||
|
||||
|
||||
class TestIsStamp:
|
||||
def test_a_line_matching_a_rule_gives_the_rule_id(self):
|
||||
line = {"text": "Downloaded via UNIVERSITY OF EXAMPLE on October 11, 2026"}
|
||||
assert fp.is_stamp(line, SIZE, RULES) == "downloaded-via"
|
||||
|
||||
def test_the_text_is_normalized_and_matched_without_regard_to_ascii_case(self):
|
||||
line = {
|
||||
"text": " downloaded FROM https://example.org\N{NO-BREAK SPACE}on May 1 "
|
||||
}
|
||||
assert fp.is_stamp(line, SIZE, RULES) == "downloaded-from-on"
|
||||
|
||||
def test_a_line_that_merely_contains_a_stamp_phrase_is_kept(self):
|
||||
assert (
|
||||
fp.is_stamp({"text": "We Downloaded via a tool on Monday"}, SIZE, RULES)
|
||||
is None
|
||||
)
|
||||
assert fp.is_stamp({"text": "Introduction"}, SIZE, RULES) is None
|
||||
|
||||
def test_the_first_rule_that_matches_wins(self):
|
||||
rules = [
|
||||
{"id": "first", "pattern": "^a"},
|
||||
{"id": "second", "pattern": "^ab"},
|
||||
]
|
||||
assert fp.is_stamp({"text": "abc"}, SIZE, rules) == "first"
|
||||
|
||||
def test_no_rules_remove_nothing(self):
|
||||
assert fp.is_stamp({"text": "Downloaded via x on y"}, SIZE, []) is None
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("box", "expected"),
|
||||
[
|
||||
([8, 200, 20, 600], "margin-downloaded"),
|
||||
([300, 5, 400, 36], "margin-downloaded"),
|
||||
([559, 100, 580, 200], "margin-downloaded"),
|
||||
([100, 806, 200, 830], "margin-downloaded"),
|
||||
([100, 100, 200, 200], None),
|
||||
([37, 37, 558, 805], None),
|
||||
],
|
||||
)
|
||||
def test_a_margin_rule_needs_a_box_wholly_inside_one_outer_margin(
|
||||
self, box, expected
|
||||
):
|
||||
line = {"text": "Downloaded by someone", "box": box}
|
||||
assert fp.is_stamp(line, SIZE, RULES) == expected
|
||||
|
||||
def test_a_margin_rule_does_not_apply_to_a_line_without_a_box(self):
|
||||
assert fp.is_stamp({"text": "Downloaded by someone"}, SIZE, RULES) is None
|
||||
|
||||
def test_a_margin_rule_can_name_its_own_margin(self):
|
||||
rules = [{"id": "wide", "pattern": "^x", "where": "margin", "margin": 100}]
|
||||
inside = {"text": "x", "box": [90, 300, 100, 310]}
|
||||
outside = {"text": "x", "box": [90, 300, 101, 310]}
|
||||
assert fp.is_stamp(inside, SIZE, rules) == "wide"
|
||||
assert fp.is_stamp(outside, SIZE, rules) is None
|
||||
|
||||
def test_a_margin_rule_defaults_to_36_points(self):
|
||||
rules = [{"id": "m", "pattern": "^x", "where": "margin"}]
|
||||
assert fp.is_stamp({"text": "x", "box": [20, 300, 36, 310]}, SIZE, rules) == "m"
|
||||
assert (
|
||||
fp.is_stamp({"text": "x", "box": [20, 300, 37, 310]}, SIZE, rules) is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "folded"),
|
||||
[
|
||||
("10 -20", "10\N{MINUS SIGN}20"),
|
||||
("10-20", "10\N{MINUS SIGN}20"),
|
||||
("a \t b\N{NO-BREAK SPACE}c", "abc"),
|
||||
("a\u200bbc", "abc"),
|
||||
("a\x00b", "ab"),
|
||||
("H₂O", "H_{2}O"),
|
||||
("x²", "x^{2}"),
|
||||
("find", "find"),
|
||||
("é", "é"),
|
||||
("", ""),
|
||||
],
|
||||
)
|
||||
def test_fp_text(text, folded):
|
||||
assert fp.fp_text(text) == folded
|
||||
|
||||
|
||||
def test_a_split_number_and_a_whole_one_fingerprint_alike():
|
||||
assert (
|
||||
fp.fingerprint(page(["10 -20"]), [])[0]
|
||||
== fp.fingerprint(page(["10-20"]), [])[0]
|
||||
)
|
||||
|
||||
|
||||
class TestFingerprint:
|
||||
def test_it_is_a_sha256_and_reports_the_removed_rules_in_line_order(self):
|
||||
lines = [
|
||||
"Body line.",
|
||||
{"text": "Downloaded from https://x.org on 1 May", "box": [1, 1, 9, 9]},
|
||||
"Another line.",
|
||||
{"text": "Downloaded via A on B"},
|
||||
]
|
||||
fingerprint, removed = fp.fingerprint(page(lines), RULES)
|
||||
assert HASH_RE.match(fingerprint)
|
||||
assert removed == ["downloaded-from-on", "downloaded-via"]
|
||||
|
||||
def test_stamp_lines_do_not_change_the_fingerprint(self):
|
||||
plain, _ = fp.fingerprint(page(["One.", "Two."]), RULES)
|
||||
stamped, removed = fp.fingerprint(
|
||||
page(["Downloaded via X on Y", "One.", "Downloaded via Z on W", "Two."]),
|
||||
RULES,
|
||||
)
|
||||
assert stamped == plain
|
||||
assert removed == ["downloaded-via", "downloaded-via"]
|
||||
|
||||
def test_white_space_and_line_breaks_do_not_change_the_fingerprint(self):
|
||||
a = fp.fingerprint(page(["The quick brown", "fox jumps."]), [])[0]
|
||||
b = fp.fingerprint(page(["The quick", "brown fox", " jumps. "]), [])[0]
|
||||
assert a == b
|
||||
|
||||
def test_the_order_of_the_text_matters(self):
|
||||
a = fp.fingerprint(page(["One.", "Two."]), [])[0]
|
||||
b = fp.fingerprint(page(["Two.", "One."]), [])[0]
|
||||
assert a != b
|
||||
|
||||
def test_the_order_of_the_images_does_not_matter_but_their_set_does(self):
|
||||
h1, h2 = "1" * 64, "2" * 64
|
||||
both = fp.fingerprint(page(["t"], images=[h1, h2]), [])[0]
|
||||
assert both == fp.fingerprint(page(["t"], images=[h2, h1]), [])[0]
|
||||
assert both != fp.fingerprint(page(["t"], images=[h1]), [])[0]
|
||||
assert both != fp.fingerprint(page(["t"]), [])[0]
|
||||
|
||||
def test_the_size_is_rounded_to_whole_points(self):
|
||||
a = fp.fingerprint(page(["t"], size=[595.276, 841.89]), [])[0]
|
||||
assert a == fp.fingerprint(page(["t"], size=[595, 842]), [])[0]
|
||||
assert a != fp.fingerprint(page(["t"], size=[612, 792]), [])[0]
|
||||
|
||||
def test_a_page_without_lines_or_images_still_has_a_fingerprint(self):
|
||||
bare = {"size": SIZE}
|
||||
fingerprint, removed = fp.fingerprint(bare, RULES)
|
||||
assert fingerprint == fp.fingerprint(page([]), RULES)[0]
|
||||
assert removed == []
|
||||
|
||||
def test_a_page_of_stamps_alone_is_the_empty_page(self):
|
||||
stamped, removed = fp.fingerprint(page(["Downloaded via X on Y"]), RULES)
|
||||
assert stamped == fp.fingerprint(page([]), RULES)[0]
|
||||
assert removed == ["downloaded-via"]
|
||||
|
||||
def test_a_changed_character_changes_the_fingerprint(self):
|
||||
a = fp.fingerprint(page(["near 1595 cm"]), [])[0]
|
||||
assert a != fp.fingerprint(page(["near 1594 cm"]), [])[0]
|
||||
|
||||
|
||||
@given(
|
||||
st.lists(st.text(alphabet="abc 1-2", min_size=1, max_size=12), max_size=5),
|
||||
st.integers(min_value=0, max_value=5),
|
||||
)
|
||||
@PROPERTIES
|
||||
def test_inserting_a_stamp_anywhere_leaves_the_fingerprint_unchanged(lines, position):
|
||||
stamp = "Downloaded via Someone on Some Day"
|
||||
at = min(position, len(lines))
|
||||
stamped = [*lines[:at], stamp, *lines[at:]]
|
||||
expected = fp.fingerprint(page(lines), RULES)[0]
|
||||
got, removed = fp.fingerprint(page(stamped), RULES)
|
||||
assert got == expected
|
||||
assert "downloaded-via" in removed
|
||||
@@ -0,0 +1,444 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Grading a submission against an exercise key (spec/19 §19.4)."""
|
||||
|
||||
from decimal import Decimal
|
||||
from functools import cache
|
||||
from typing import ClassVar
|
||||
|
||||
import pytest
|
||||
|
||||
from helpers_lib1 import HALE, package_state, vectors
|
||||
from paperspec import anchors, grade as grading, hashing
|
||||
from paperspec.tree import Refusal
|
||||
|
||||
|
||||
@cache
|
||||
def _sentences():
|
||||
"""Plain text of three sentences of the example paper, and their IDs."""
|
||||
doc = package_state(HALE).doc()
|
||||
wanted = {
|
||||
"terms": "Harmonic bond terms remain",
|
||||
"water": "Water, H$_2$O",
|
||||
"table": "Table 1 lists",
|
||||
}
|
||||
found = {}
|
||||
for nid in doc.preorder():
|
||||
if doc.nodes[nid]["kind"] != "sentence":
|
||||
continue
|
||||
text = hashing.plain(doc, nid)
|
||||
for name, start in wanted.items():
|
||||
if text.startswith(start):
|
||||
found[name] = (nid, text)
|
||||
return found
|
||||
|
||||
|
||||
def anchor(name, span=None):
|
||||
"""An anchor on a sentence of the example paper, current at its latest state."""
|
||||
state = package_state(HALE)
|
||||
doc = state.doc()
|
||||
return anchors.make_anchor(
|
||||
doc, hashing.compute(doc), _sentences()[name][0], state.rev, span, "smithlab"
|
||||
)
|
||||
|
||||
|
||||
def span_of(name, quote):
|
||||
"""The span of `quote` within a sentence's plain text."""
|
||||
text = _sentences()[name][1]
|
||||
start = text.index(quote)
|
||||
return [start, start + len(quote)]
|
||||
|
||||
|
||||
def submission(answers, ident="s1"):
|
||||
return {"id": ident, "exercise": "ex:1", "answers": answers}
|
||||
|
||||
|
||||
def element(name, points="1", **feedback):
|
||||
"""A key element anchored on a sentence."""
|
||||
out = {"anchors": [anchor(name)], "points": points}
|
||||
if feedback:
|
||||
out["feedback"] = feedback
|
||||
return out
|
||||
|
||||
|
||||
def run(body, answers):
|
||||
return grading.grade(body, submission(answers), package_state(HALE))
|
||||
|
||||
|
||||
def scores(grade):
|
||||
return [(i["element"], i["points"]) for i in grade["items"]]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", vectors("grading"))
|
||||
def test_grading_vector(case):
|
||||
state = package_state(case["package"])
|
||||
assert grading.grade(case["exercise"], case["submission"], state) == case["expect"]
|
||||
|
||||
|
||||
class TestDecimalStrings:
|
||||
@pytest.mark.parametrize(
|
||||
("value", "text"),
|
||||
[
|
||||
("1", "1"),
|
||||
("0.50", "0.50"),
|
||||
("1E+1", "10"),
|
||||
(Decimal("2.50"), "2.50"),
|
||||
("0.0", "0.0"),
|
||||
("1e-3", "0.001"),
|
||||
],
|
||||
)
|
||||
def test_dec_writes_the_decimal_as_a_plain_string(self, value, text):
|
||||
assert grading.dec(value) == text
|
||||
|
||||
def test_a_missed_element_scores_zero_with_the_scale_of_its_points(self):
|
||||
body = {"exercise": "anchor", "key": {"elements": [element("terms", "0.50")]}}
|
||||
grade = run(body, {"anchors": [anchor("water")]})
|
||||
assert grade["items"][0]["points"] == "0.00"
|
||||
assert grade["items"][0]["max"] == "0.50"
|
||||
assert grade["score"] == "0.00"
|
||||
assert grade["max"] == "0.50"
|
||||
|
||||
def test_points_are_summed_exactly(self):
|
||||
body = {
|
||||
"exercise": "anchor",
|
||||
"key": {
|
||||
"elements": [
|
||||
element("terms", "0.1"),
|
||||
element("water", "0.2"),
|
||||
element("table", "0.4"),
|
||||
]
|
||||
},
|
||||
}
|
||||
grade = run(body, {"anchors": [anchor("terms"), anchor("water")]})
|
||||
assert grade["score"] == "0.3"
|
||||
assert grade["max"] == "0.7"
|
||||
assert scores(grade) == [
|
||||
("element:0", "0.1"),
|
||||
("element:1", "0.2"),
|
||||
("element:2", "0.0"),
|
||||
]
|
||||
|
||||
|
||||
class TestAnchorExercise:
|
||||
BODY: ClassVar[dict] = {
|
||||
"exercise": "anchor",
|
||||
"key": {
|
||||
"elements": [
|
||||
element("terms", "2", hit="Right.", miss="Wrong."),
|
||||
element("water", "1"),
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
def test_the_grade_names_the_submission_and_its_exercise(self):
|
||||
grade = run(self.BODY, {"anchors": []})
|
||||
assert grade["format"] == "scienting-grade/1"
|
||||
assert grade["submission"] == "s1"
|
||||
assert grade["exercise"] == "ex:1"
|
||||
|
||||
def test_each_element_is_scored_by_whether_any_answer_hits_it(self):
|
||||
grade = run(self.BODY, {"anchors": [anchor("water"), anchor("terms")]})
|
||||
assert grade["score"] == "3"
|
||||
assert grade["max"] == "3"
|
||||
assert [i["feedback"] for i in grade["items"]] == ["Right.", ""]
|
||||
|
||||
def test_a_miss_gives_the_miss_feedback(self):
|
||||
grade = run(self.BODY, {"anchors": [anchor("table")]})
|
||||
assert grade["score"] == "0"
|
||||
assert [i["feedback"] for i in grade["items"]] == ["Wrong.", ""]
|
||||
|
||||
def test_no_answer_scores_nothing(self):
|
||||
assert run(self.BODY, {})["score"] == "0"
|
||||
|
||||
def test_one_answer_may_hit_several_elements(self):
|
||||
body = {
|
||||
"exercise": "anchor",
|
||||
"key": {"elements": [element("terms"), element("terms", "3")]},
|
||||
}
|
||||
grade = run(body, {"anchors": [anchor("terms")]})
|
||||
assert grade["score"] == "4"
|
||||
|
||||
def test_an_element_may_list_several_acceptable_anchors(self):
|
||||
two = {"anchors": [anchor("terms"), anchor("water")], "points": "1"}
|
||||
body = {"exercise": "anchor", "key": {"elements": [two]}}
|
||||
assert run(body, {"anchors": [anchor("water")]})["score"] == "1"
|
||||
assert run(body, {"anchors": [anchor("table")]})["score"] == "0"
|
||||
|
||||
def test_a_stale_answer_hits_nothing(self):
|
||||
stale = dict(anchor("terms"), node="zzzzzzz")
|
||||
assert run(self.BODY, {"anchors": [stale]})["score"] == "0"
|
||||
|
||||
def test_an_answer_whose_hash_moved_but_whose_quote_is_found_still_hits(self):
|
||||
moved = dict(anchor("terms"), node_hash="0" * 64)
|
||||
assert run(self.BODY, {"anchors": [moved]})["score"] == "2"
|
||||
|
||||
|
||||
class TestMatchRules:
|
||||
TEXT = "Harmonic bond terms remain the most common description of covalent"
|
||||
|
||||
def key_span(self):
|
||||
return span_of("terms", "the most common description")
|
||||
|
||||
def body(self, rule=None):
|
||||
key = {"anchors": [anchor("terms", self.key_span())], "points": "1"}
|
||||
out = {"exercise": "anchor", "key": {"elements": [key]}}
|
||||
if rule is not None:
|
||||
out["rule"] = rule
|
||||
return out
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("quote", "points"),
|
||||
[
|
||||
("the most common description", "1"),
|
||||
("most common description of covalent", "1"),
|
||||
("the most common", "1"),
|
||||
("the most", "0"),
|
||||
("terms remain the", "0"),
|
||||
("covalent stretching", "0"),
|
||||
],
|
||||
)
|
||||
def test_overlap_needs_half_the_key_span_by_default(self, quote, points):
|
||||
answer = anchor("terms", span_of("terms", quote))
|
||||
body = self.body({"match": "overlap"})
|
||||
assert run(body, {"anchors": [answer]})["score"] == points
|
||||
|
||||
def test_the_least_overlap_can_be_set(self):
|
||||
answer = anchor("terms", span_of("terms", "the most"))
|
||||
lenient = self.body({"match": "overlap", "min_overlap": "0.25"})
|
||||
assert run(lenient, {"anchors": [answer]})["score"] == "1"
|
||||
tight = self.body({"match": "overlap", "min_overlap": "1"})
|
||||
assert run(tight, {"anchors": [answer]})["score"] == "0"
|
||||
|
||||
def test_an_answer_without_a_span_covers_the_whole_node(self):
|
||||
body = self.body({"match": "overlap"})
|
||||
assert run(body, {"anchors": [anchor("terms")]})["score"] == "1"
|
||||
|
||||
def test_an_overlap_rule_on_a_key_without_a_span_needs_only_the_node(self):
|
||||
body = {
|
||||
"exercise": "anchor",
|
||||
"rule": {"match": "overlap"},
|
||||
"key": {"elements": [element("terms")]},
|
||||
}
|
||||
answer = anchor("terms", span_of("terms", "the most"))
|
||||
assert run(body, {"anchors": [answer]})["score"] == "1"
|
||||
|
||||
def test_same_node_ignores_spans(self):
|
||||
answer = anchor("terms", span_of("terms", "covalent stretching"))
|
||||
for rule in (None, {"match": "same-node"}, {}):
|
||||
assert run(self.body(rule), {"anchors": [answer]})["score"] == "1"
|
||||
|
||||
def test_an_answer_on_another_node_never_matches(self):
|
||||
answer = anchor("water", [0, 5])
|
||||
assert (
|
||||
run(self.body({"match": "overlap"}), {"anchors": [answer]})["score"] == "0"
|
||||
)
|
||||
|
||||
def test_an_empty_key_span_cannot_be_overlapped(self):
|
||||
key = {"anchors": [anchor("terms", [5, 5])], "points": "1"}
|
||||
body = {
|
||||
"exercise": "anchor",
|
||||
"rule": {"match": "overlap"},
|
||||
"key": {"elements": [key]},
|
||||
}
|
||||
answer = anchor("terms", [0, 20])
|
||||
assert run(body, {"anchors": [answer]})["score"] == "0"
|
||||
|
||||
|
||||
class TestCheckExercise:
|
||||
BODY: ClassVar[dict] = {
|
||||
"exercise": "check",
|
||||
"key": {
|
||||
"verdict": {
|
||||
"value": "supports",
|
||||
"points": "1",
|
||||
"feedback": {"hit": "Yes.", "miss": "No."},
|
||||
},
|
||||
"elements": [element("terms")],
|
||||
},
|
||||
}
|
||||
|
||||
def test_the_verdict_comes_first_and_is_scored_on_its_own(self):
|
||||
grade = run(self.BODY, {"verdict": "supports", "anchors": [anchor("terms")]})
|
||||
assert scores(grade) == [("verdict", "1"), ("element:0", "1")]
|
||||
assert grade["items"][0]["feedback"] == "Yes."
|
||||
assert grade["score"] == "2"
|
||||
|
||||
def test_a_wrong_verdict_loses_only_its_points(self):
|
||||
grade = run(self.BODY, {"verdict": "refutes", "anchors": [anchor("terms")]})
|
||||
assert scores(grade) == [("verdict", "0"), ("element:0", "1")]
|
||||
assert grade["items"][0]["feedback"] == "No."
|
||||
|
||||
def test_no_verdict_is_wrong(self):
|
||||
assert run(self.BODY, {"anchors": [anchor("terms")]})["score"] == "1"
|
||||
|
||||
def test_a_verdict_without_feedback_has_empty_feedback(self):
|
||||
body = {
|
||||
"exercise": "check",
|
||||
"key": {"verdict": {"value": "supports", "points": "1"}, "elements": []},
|
||||
}
|
||||
assert run(body, {"verdict": "supports"})["items"][0]["feedback"] == ""
|
||||
|
||||
|
||||
class TestTraceExercise:
|
||||
BODY: ClassVar[dict] = {
|
||||
"exercise": "trace",
|
||||
"key": {
|
||||
"chain": [
|
||||
{
|
||||
"anchors": [anchor("terms")],
|
||||
"points": "1",
|
||||
"feedback": {"hit": "First."},
|
||||
},
|
||||
{"anchors": [anchor("water")], "points": "2"},
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
def test_each_link_is_matched_by_the_answers_at_its_position(self):
|
||||
chain = [[anchor("terms")], [anchor("water")]]
|
||||
grade = run(self.BODY, {"chain": chain})
|
||||
assert scores(grade) == [("link:0", "1"), ("link:1", "2")]
|
||||
assert grade["items"][0]["feedback"] == "First."
|
||||
|
||||
def test_an_answer_at_the_wrong_position_does_not_count(self):
|
||||
grade = run(self.BODY, {"chain": [[anchor("water")], [anchor("terms")]]})
|
||||
assert grade["score"] == "0"
|
||||
|
||||
def test_a_chain_that_stops_early_misses_the_rest(self):
|
||||
grade = run(self.BODY, {"chain": [[anchor("terms")]]})
|
||||
assert scores(grade) == [("link:0", "1"), ("link:1", "0")]
|
||||
|
||||
def test_a_link_may_be_answered_by_several_anchors(self):
|
||||
grade = run(self.BODY, {"chain": [[anchor("table"), anchor("terms")], []]})
|
||||
assert scores(grade) == [("link:0", "1"), ("link:1", "0")]
|
||||
|
||||
def test_a_missing_chain_scores_nothing(self):
|
||||
assert run(self.BODY, {})["score"] == "0"
|
||||
|
||||
|
||||
class TestReproduceExercise:
|
||||
def body(self, value="3.14", tolerance="0.01", points="2"):
|
||||
return {
|
||||
"exercise": "reproduce",
|
||||
"key": {
|
||||
"value": value,
|
||||
"tolerance": tolerance,
|
||||
"points": points,
|
||||
"feedback": {"hit": "Close.", "miss": "Off."},
|
||||
},
|
||||
}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("answer", "points"),
|
||||
[
|
||||
("3.14", "2"),
|
||||
("3.15", "2"),
|
||||
("3.13", "2"),
|
||||
("3.1501", "0"),
|
||||
("3.1299", "0"),
|
||||
("-3.14", "0"),
|
||||
("3.140000", "2"),
|
||||
],
|
||||
)
|
||||
def test_the_value_must_be_within_the_tolerance_inclusive(self, answer, points):
|
||||
grade = run(self.body(), {"value": answer})
|
||||
assert scores(grade) == [("value", points)]
|
||||
|
||||
def test_values_are_compared_exactly_in_decimal(self):
|
||||
assert run(self.body("0.3", "0"), {"value": "0.30"})["score"] == "2"
|
||||
assert (
|
||||
run(self.body("0.3", "0"), {"value": "0.30000000000000004"})["score"] == "0"
|
||||
)
|
||||
|
||||
def test_feedback_follows_the_outcome(self):
|
||||
assert run(self.body(), {"value": "3.14"})["items"][0]["feedback"] == "Close."
|
||||
assert run(self.body(), {"value": "9"})["items"][0]["feedback"] == "Off."
|
||||
|
||||
|
||||
class TestVerifyExercise:
|
||||
BODY: ClassVar[dict] = {
|
||||
"exercise": "verify",
|
||||
"key": {
|
||||
"errors": [
|
||||
{
|
||||
"anchors": [anchor("terms")],
|
||||
"verdict": "wrong-number",
|
||||
"points": "1",
|
||||
"feedback": {"hit": "Found.", "miss": "Missed."},
|
||||
},
|
||||
{"anchors": [anchor("water")], "verdict": "wrong-unit", "points": "1"},
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
def flag(self, name, verdict):
|
||||
return {"anchor": anchor(name), "verdict": verdict}
|
||||
|
||||
def test_an_error_is_found_by_a_flag_on_its_node_with_its_verdict(self):
|
||||
flags = [self.flag("terms", "wrong-number"), self.flag("water", "wrong-unit")]
|
||||
grade = run(self.BODY, {"flags": flags})
|
||||
assert scores(grade) == [("error:0", "1"), ("error:1", "1")]
|
||||
assert grade["items"][0]["feedback"] == "Found."
|
||||
|
||||
def test_the_wrong_verdict_on_the_right_node_misses(self):
|
||||
grade = run(self.BODY, {"flags": [self.flag("terms", "wrong-unit")]})
|
||||
assert scores(grade) == [("error:0", "0"), ("error:1", "0")]
|
||||
assert grade["items"][0]["feedback"] == "Missed."
|
||||
|
||||
def test_any_one_of_several_flags_on_the_node_may_carry_the_verdict(self):
|
||||
flags = [self.flag("terms", "other"), self.flag("terms", "wrong-number")]
|
||||
assert run(self.BODY, {"flags": flags})["score"] == "1"
|
||||
|
||||
def test_a_flag_on_another_node_finds_nothing(self):
|
||||
assert (
|
||||
run(self.BODY, {"flags": [self.flag("table", "wrong-number")]})["score"]
|
||||
== "0"
|
||||
)
|
||||
|
||||
def test_no_flags_find_nothing(self):
|
||||
assert run(self.BODY, {})["score"] == "0"
|
||||
|
||||
|
||||
class TestRefusals:
|
||||
def test_a_stale_anchor_in_the_key_refuses_the_grade(self):
|
||||
stale = dict(anchor("terms"), node="zzzzzzz")
|
||||
body = {
|
||||
"exercise": "anchor",
|
||||
"key": {"elements": [{"anchors": [stale], "points": "1"}]},
|
||||
}
|
||||
with pytest.raises(Refusal, match=r"grade\.key-stale") as excinfo:
|
||||
run(body, {"anchors": []})
|
||||
assert excinfo.value.rule == "grade.key-stale"
|
||||
|
||||
def test_every_part_of_the_key_is_checked_for_stale_anchors(self):
|
||||
stale = dict(anchor("terms"), node="zzzzzzz")
|
||||
for part, shape in (
|
||||
("elements", {"verdict": {"value": "x", "points": "1"}}),
|
||||
("chain", {}),
|
||||
("errors", {}),
|
||||
):
|
||||
key = dict(shape, **{part: [{"anchors": [stale], "points": "1"}]})
|
||||
with pytest.raises(Refusal, match="key-stale"):
|
||||
run({"exercise": "check", "key": key}, {})
|
||||
|
||||
def test_an_unknown_kind_is_refused(self):
|
||||
with pytest.raises(
|
||||
Refusal, match=r"grade\.kind: unknown exercise kind essay"
|
||||
) as excinfo:
|
||||
run({"exercise": "essay", "key": {}}, {})
|
||||
assert excinfo.value.rule == "grade.kind"
|
||||
|
||||
|
||||
def test_key_anchors_lists_elements_then_chain_then_errors():
|
||||
a, b, c = ({"node": x} for x in "abc")
|
||||
key = {
|
||||
"elements": [{"anchors": [a]}],
|
||||
"chain": [{"anchors": [b]}],
|
||||
"errors": [{"anchors": [c, a]}],
|
||||
}
|
||||
assert grading.key_anchors({"key": key}) == [a, b, c, a]
|
||||
assert grading.key_anchors({"key": {}}) == []
|
||||
@@ -0,0 +1,694 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""The projections and the five content values (spec/03 §3.6, spec/05 §5.1 to §5.3)."""
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import unicodedata
|
||||
from typing import ClassVar
|
||||
|
||||
import pytest
|
||||
from hypothesis import given, settings, strategies as st
|
||||
|
||||
from conftest import FIXTURES
|
||||
from helpers_lib1 import (
|
||||
VALUE_FIELDS,
|
||||
bare,
|
||||
build_doc,
|
||||
document,
|
||||
heading,
|
||||
ids_of_kind,
|
||||
math_run,
|
||||
package_nodes,
|
||||
paragraph,
|
||||
read_cases,
|
||||
reference_values,
|
||||
section,
|
||||
sentence,
|
||||
sentence_doc,
|
||||
simple_doc,
|
||||
text_run,
|
||||
vectors,
|
||||
)
|
||||
from paperspec import hashing, jcs
|
||||
from paperspec.hashing import (
|
||||
MARK_CLOSE,
|
||||
MARK_OPEN,
|
||||
OBJ,
|
||||
PLAIN,
|
||||
Hasher,
|
||||
Index,
|
||||
as_printed,
|
||||
compute,
|
||||
lossy_from_strict,
|
||||
meta_hash,
|
||||
meta_object,
|
||||
own_content,
|
||||
plain_runs,
|
||||
prose_length,
|
||||
)
|
||||
from paperspec.model import Doc
|
||||
from paperspec.text import is_ws, norm, sha256_hex
|
||||
|
||||
PROPERTIES = settings(derandomize=True, max_examples=100, database=None, deadline=None)
|
||||
PACKAGES = sorted(
|
||||
p.name
|
||||
for p in (FIXTURES / "packages").iterdir()
|
||||
if p.is_dir() and not p.name.endswith(".checks-only")
|
||||
)
|
||||
|
||||
|
||||
def _hash(s):
|
||||
return hashlib.sha256(s.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
class TestHasher:
|
||||
def test_plain_hasher_is_sha256_of_text_or_bytes(self):
|
||||
assert PLAIN("abc") == hashlib.sha256(b"abc").hexdigest()
|
||||
assert PLAIN(b"abc") == PLAIN("abc")
|
||||
assert Hasher().key is None
|
||||
|
||||
def test_keyed_hasher_is_hmac_sha256(self):
|
||||
keyed = Hasher(b"Jefe")
|
||||
expected = "5bdcc146bf60754e6a042426089575c75a003f089d2739839dec58b964ec3843"
|
||||
assert keyed("what do ya want for nothing?") == expected
|
||||
assert keyed(b"what do ya want for nothing?") == expected
|
||||
assert keyed("x") == hmac.new(b"Jefe", b"x", hashlib.sha256).hexdigest()
|
||||
|
||||
def test_keyed_and_plain_values_differ(self):
|
||||
assert Hasher(b"k")("x") != PLAIN("x")
|
||||
assert Hasher(b"k")("x") != Hasher(b"l")("x")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", vectors("projections"))
|
||||
def test_projection_vector(case):
|
||||
doc, nid = sentence_doc(case["runs"])
|
||||
strict, complexes = hashing.strict(doc, nid)
|
||||
lossy = lossy_from_strict(strict)
|
||||
plain = hashing.plain(doc, nid)
|
||||
assert plain == case["plain"]
|
||||
assert norm(plain) == case["normalized"]
|
||||
assert strict == case["strict"]
|
||||
assert lossy == case["lossy"]
|
||||
assert complexes == case["complex"]
|
||||
assert hashing.lossy(doc, nid) == lossy
|
||||
assert _hash(norm(plain)) == case["text_hash"]
|
||||
assert _hash(strict) == case["match_hash"]
|
||||
assert _hash(lossy) == case["align_key"]
|
||||
printed = hashing.printed(doc, nid)
|
||||
assert printed == case.get("printed", plain)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", vectors("projections"))
|
||||
def test_projection_vector_values_are_those_the_index_keeps(case):
|
||||
doc, nid = sentence_doc(case["runs"])
|
||||
values = compute(doc)[nid]
|
||||
assert values["text_hash"] == case["text_hash"]
|
||||
assert values["match_hash"] == case["match_hash"]
|
||||
assert values["align_key"] == case["align_key"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("group", vectors("projection-groups"))
|
||||
def test_projection_group(group):
|
||||
by_id = {c["id"]: c for c in read_cases("projections")}
|
||||
field = group.get("same") or group.get("differ")
|
||||
values = {by_id[x][field] for x in group["cases"]}
|
||||
if "same" in group:
|
||||
assert len(values) == 1
|
||||
else:
|
||||
assert len(values) == len(group["cases"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", vectors("offsets"))
|
||||
def test_offsets_vector_plain_text(case):
|
||||
doc, nid = sentence_doc(case["runs"])
|
||||
assert hashing.plain(doc, nid) == case["plain"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", vectors("hashes"))
|
||||
def test_hashes_vector(case):
|
||||
doc = Doc(case["doc"], case["nodes"])
|
||||
values = compute(doc)
|
||||
assert values == case["values"]
|
||||
for nid, own in case["own_content"].items():
|
||||
assert sha256_hex(jcs.dumpb(own)) == values[nid]["own_hash"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", vectors("hashes"))
|
||||
def test_hashes_vector_agree_with_the_slow_computation(case):
|
||||
doc = Doc(case["doc"], case["nodes"])
|
||||
assert reference_values(doc) == case["values"]
|
||||
|
||||
|
||||
def test_a_math_form_enters_no_content_value():
|
||||
case = next(c for c in read_cases("hashes") if c["id"] == "math-form")
|
||||
|
||||
def unformed(node):
|
||||
content = node.get("content")
|
||||
if isinstance(content, dict):
|
||||
content = {k: v for k, v in content.items() if k != "form"}
|
||||
elif isinstance(content, list):
|
||||
content = [{k: v for k, v in r.items() if k != "form"} for r in content]
|
||||
return dict(node, content=content) if "content" in node else node
|
||||
|
||||
assert sum(json.dumps(n).count('"form"') for n in case["nodes"]) == 2
|
||||
stripped = Doc(case["doc"], [unformed(n) for n in case["nodes"]])
|
||||
assert compute(stripped) == case["values"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("package", PACKAGES)
|
||||
def test_fixture_values_are_the_stored_values(package):
|
||||
nodes = package_nodes(FIXTURES / "packages" / package)
|
||||
values = compute(Doc(package, bare(nodes)))
|
||||
for n in nodes:
|
||||
assert {k: n[k] for k in VALUE_FIELDS} == values[n["id"]]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("package", PACKAGES)
|
||||
def test_fixture_values_agree_with_the_slow_computation(package):
|
||||
nodes = bare(package_nodes(FIXTURES / "packages" / package))
|
||||
doc = Doc(package, nodes)
|
||||
assert Index.build(doc).values == reference_values(doc)
|
||||
|
||||
|
||||
def test_keyed_values_follow_the_hasher():
|
||||
doc = Doc("p", bare(package_nodes(FIXTURES / "packages" / "tiny2026tiny-j4qm")))
|
||||
keyed = Hasher(b"team secret")
|
||||
values = compute(doc, keyed)
|
||||
assert values == reference_values(doc, keyed)
|
||||
plain_values = compute(doc)
|
||||
assert all(values[n][f] != plain_values[n][f] for n in values for f in VALUE_FIELDS)
|
||||
|
||||
|
||||
class TestProjections:
|
||||
def test_plain_projection_marks_math_and_equation_runs(self):
|
||||
doc = build_doc(
|
||||
document(
|
||||
section(
|
||||
heading("H"),
|
||||
paragraph(
|
||||
sentence(
|
||||
text_run("Energy "),
|
||||
math_run("E"),
|
||||
text_run(" obeys "),
|
||||
{"kind": "equation", "child": 0},
|
||||
text_run("."),
|
||||
children=[
|
||||
{
|
||||
"kind": "equation",
|
||||
"content": {"notation": "tex", "source": "E=mc^2"},
|
||||
}
|
||||
],
|
||||
)
|
||||
),
|
||||
)
|
||||
)
|
||||
)
|
||||
(sent,) = [
|
||||
n for n in ids_of_kind(doc, "sentence") if "E=mc" in hashing.plain(doc, n)
|
||||
]
|
||||
assert hashing.plain(doc, sent) == "Energy $E$ obeys $$E=mc^2$$."
|
||||
strict, complexes = hashing.strict(doc, sent)
|
||||
assert strict == "Energy E obeys E=mc^{2}."
|
||||
assert complexes == []
|
||||
|
||||
def test_a_complex_equation_leaves_an_object_mark_and_a_key(self):
|
||||
runs = [text_run("see "), math_run("\\frac{a}{b}"), text_run(" now")]
|
||||
doc, nid = sentence_doc(runs)
|
||||
strict, complexes = hashing.strict(doc, nid)
|
||||
assert strict == f"see {OBJ} now"
|
||||
assert complexes == ["tex:\\frac{a}{b}"]
|
||||
assert OBJ in hashing.lossy(doc, nid)
|
||||
|
||||
def test_cite_xref_and_fnmark_runs_are_marked_and_dropped_from_the_lossy_form(self):
|
||||
runs = [
|
||||
text_run("Known"),
|
||||
{"kind": "cite", "refs": ["aaaaaaa"], "text": " [ 1 ] "},
|
||||
text_run(" and"),
|
||||
{"kind": "xref", "target": "bbbbbbb", "text": "Fig. 2"},
|
||||
{"kind": "fnmark", "text": "3"},
|
||||
]
|
||||
doc, nid = sentence_doc(runs)
|
||||
strict, _ = hashing.strict(doc, nid)
|
||||
assert (
|
||||
strict
|
||||
== f"Known{MARK_OPEN}[1]{MARK_CLOSE} and{MARK_OPEN}Fig.2{MARK_CLOSE}{MARK_OPEN}3{MARK_CLOSE}"
|
||||
)
|
||||
assert hashing.lossy(doc, nid) == "knownand"
|
||||
assert hashing.plain(doc, nid) == "Known [ 1 ] andFig. 23"
|
||||
|
||||
def test_a_link_run_is_plain_text(self):
|
||||
doc, nid = sentence_doc(
|
||||
[text_run("See "), {"kind": "link", "text": "the site"}]
|
||||
)
|
||||
assert hashing.plain(doc, nid) == "See the site"
|
||||
assert hashing.strict(doc, nid)[0] == "See the site"
|
||||
|
||||
def test_styled_text_folds_as_a_script(self):
|
||||
runs = [
|
||||
text_run("H"),
|
||||
text_run("2", style=["sub"]),
|
||||
text_run("O at 10"),
|
||||
text_run("-3", style=["sup"]),
|
||||
text_run("\N{PRIME}", style=["sup"]),
|
||||
]
|
||||
doc, nid = sentence_doc(runs)
|
||||
assert hashing.strict(doc, nid)[0] == "H_{2}O at 10^{\N{MINUS SIGN}3}\N{PRIME}"
|
||||
|
||||
def test_an_unknown_run_kind_is_refused_by_every_projection(self):
|
||||
doc, nid = sentence_doc([{"kind": "image", "text": "x"}])
|
||||
with pytest.raises(ValueError, match="run kind image"):
|
||||
hashing.plain(doc, nid)
|
||||
with pytest.raises(ValueError, match="run kind image"):
|
||||
hashing.strict(doc, nid)
|
||||
|
||||
def test_a_container_joins_the_projections_of_its_text_leaves(self):
|
||||
doc = simple_doc("First one.", "Second one.")
|
||||
para = ids_of_kind(doc, "paragraph")[0]
|
||||
assert hashing.plain(doc, para) == "First one. Second one."
|
||||
assert hashing.strict(doc, para) == ("First one. Second one.", [])
|
||||
assert hashing.printed(doc, para) == "First one. Second one."
|
||||
assert hashing.plain(doc, doc.root) == "Head First one. Second one."
|
||||
|
||||
def test_verbatim_and_equation_nodes_project_their_source(self):
|
||||
doc = build_doc(
|
||||
document(
|
||||
section(
|
||||
heading("H"),
|
||||
{"kind": "code", "content": {"text": "x = 1"}},
|
||||
{
|
||||
"kind": "equation",
|
||||
"content": {"notation": "tex", "source": "a_1"},
|
||||
"children": [
|
||||
{
|
||||
"kind": "equation-line",
|
||||
"content": {"notation": "tex", "source": "a_1"},
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
)
|
||||
)
|
||||
(code,) = ids_of_kind(doc, "code")
|
||||
(eq,) = ids_of_kind(doc, "equation")
|
||||
(line,) = ids_of_kind(doc, "equation-line")
|
||||
assert hashing.plain(doc, code) == "x = 1"
|
||||
assert hashing.strict(doc, code) == ("x = 1", [])
|
||||
assert hashing.printed(doc, code) == "x = 1"
|
||||
assert hashing.plain(doc, eq) == "$$a_1$$"
|
||||
assert hashing.strict(doc, eq) == ("a_{1}", [])
|
||||
assert hashing.printed(doc, eq) == "$$a_1$$"
|
||||
assert hashing.plain(doc, line) == "$$a_1$$"
|
||||
values = compute(doc)
|
||||
assert values[eq]["text_hash"] == values[line]["text_hash"]
|
||||
section_id = ids_of_kind(doc, "section")[0]
|
||||
assert hashing.plain(doc, section_id) == "H x = 1 $$a_1$$"
|
||||
|
||||
|
||||
class TestAsPrinted:
|
||||
def test_runs_without_printed_are_returned_as_they_are(self):
|
||||
runs = [text_run("a"), math_run("x"), text_run("b", style=["em"])]
|
||||
assert as_printed(runs) == runs
|
||||
|
||||
def test_printed_replaces_the_text_and_rejoins_the_split_run(self):
|
||||
runs = [
|
||||
text_run("The "),
|
||||
text_run("cat", printed="kat"),
|
||||
text_run(" sat"),
|
||||
]
|
||||
assert as_printed(runs) == [text_run("The kat sat")]
|
||||
|
||||
def test_the_input_runs_are_not_changed(self):
|
||||
runs = [text_run("The "), text_run("cat", printed="kat")]
|
||||
as_printed(runs)
|
||||
assert runs[1] == text_run("cat", printed="kat")
|
||||
|
||||
def test_runs_of_other_styles_or_kinds_are_not_joined(self):
|
||||
runs = [
|
||||
text_run("a "),
|
||||
text_run("b", style=["em"], printed="c"),
|
||||
text_run(" d"),
|
||||
math_run("x"),
|
||||
text_run("e", printed="f"),
|
||||
]
|
||||
assert as_printed(runs) == [
|
||||
text_run("a "),
|
||||
text_run("c", style=["em"]),
|
||||
text_run(" d"),
|
||||
math_run("x"),
|
||||
text_run("f"),
|
||||
]
|
||||
|
||||
def test_only_the_run_after_a_printed_one_is_joined(self):
|
||||
runs = [text_run("a"), text_run("b"), text_run("c", printed="d"), text_run("e")]
|
||||
assert as_printed(runs) == [text_run("a"), text_run("bde")]
|
||||
|
||||
def test_the_strict_projection_reads_the_runs_as_printed(self):
|
||||
runs = [text_run("H"), text_run("2", printed="3", style=["sub"]), text_run("O")]
|
||||
doc, nid = sentence_doc(runs)
|
||||
assert hashing.printed(doc, nid) == "H3O"
|
||||
assert hashing.strict(doc, nid)[0] == "H_{3}O"
|
||||
assert hashing.plain(doc, nid) == "H2O"
|
||||
|
||||
|
||||
class TestPlainRuns:
|
||||
def test_unknown_kinds_are_refused(self):
|
||||
with pytest.raises(ValueError, match="run kind sticker"):
|
||||
plain_runs([{"kind": "sticker"}])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("strict", "lossy"),
|
||||
[
|
||||
("H_{2}O", "h2o"),
|
||||
("x^{2} + y_{ab}", "x2+yab"),
|
||||
("{a}", "{a}"),
|
||||
("a}", "a}"),
|
||||
("_{a{b}c}", "a{b}c"),
|
||||
("_ {a}", "_{a}"),
|
||||
("a_", "a_"),
|
||||
(f"see{MARK_OPEN}12{MARK_CLOSE}now", "seenow"),
|
||||
(f"see{MARK_OPEN}12", "see"),
|
||||
(f"see {OBJ} now", f"see{OBJ}now"),
|
||||
("a-b\N{HYPHEN}c\N{EN DASH}de", "abcde"),
|
||||
("a\N{MINUS SIGN}b", "a\N{MINUS SIGN}b"),
|
||||
("Straße", "strasse"),
|
||||
("find", "find"),
|
||||
("\N{MICRO SIGN}m", "μm"),
|
||||
("CafÉ Au\N{NO-BREAK SPACE}Lait", "caféaulait"),
|
||||
("", ""),
|
||||
],
|
||||
)
|
||||
def test_lossy_from_strict(strict, lossy):
|
||||
assert lossy_from_strict(strict) == lossy
|
||||
|
||||
|
||||
@given(st.text())
|
||||
@PROPERTIES
|
||||
def test_lossy_projection_has_no_white_space_dashes_or_annotation_marks(s):
|
||||
lossy = lossy_from_strict(s)
|
||||
assert not any(is_ws(c) or unicodedata.category(c) == "Pd" for c in lossy)
|
||||
assert "" not in lossy
|
||||
|
||||
|
||||
def test_prose_length_does_not_count_object_marks():
|
||||
assert prose_length("abc") == 3
|
||||
assert prose_length(f"a{OBJ}b{OBJ}") == 2
|
||||
assert prose_length("") == 0
|
||||
|
||||
|
||||
class TestOwnContent:
|
||||
def test_a_sentence_keeps_its_runs_but_not_the_form_of_a_math_run(self):
|
||||
runs = [text_run("x "), dict(math_run("a"), form="inline"), text_run("!")]
|
||||
doc, nid = sentence_doc(runs)
|
||||
own = own_content(doc, nid)
|
||||
assert own == {
|
||||
"kind": "sentence",
|
||||
"runs": [text_run("x "), math_run("a"), text_run("!")],
|
||||
}
|
||||
|
||||
def test_a_section_names_the_text_hash_of_its_heading(self):
|
||||
doc = build_doc(
|
||||
document(
|
||||
section(heading("Intro"), paragraph(sentence(text_run("t")))),
|
||||
section(paragraph(sentence(text_run("u")))),
|
||||
)
|
||||
)
|
||||
first, second = ids_of_kind(doc, "section")
|
||||
head = ids_of_kind(doc, "heading")[0]
|
||||
assert own_content(doc, first, lambda x: f"hash of {x}") == {
|
||||
"kind": "section",
|
||||
"label": None,
|
||||
"level": 1,
|
||||
"heading": f"hash of {head}",
|
||||
}
|
||||
assert own_content(doc, second, lambda x: "unused")["heading"] is None
|
||||
|
||||
def test_a_label_is_part_of_the_own_content_of_a_labelled_kind(self):
|
||||
doc = build_doc(
|
||||
document(
|
||||
section(heading("H"), paragraph(sentence(text_run("t"))), label="2.1")
|
||||
)
|
||||
)
|
||||
sec = ids_of_kind(doc, "section")[0]
|
||||
assert own_content(doc, sec, lambda x: "h")["label"] == "2.1"
|
||||
para = ids_of_kind(doc, "paragraph")[0]
|
||||
assert "label" not in own_content(doc, para)
|
||||
|
||||
def test_a_table_carries_its_columns_and_a_cell_its_attributes_with_defaults(self):
|
||||
nodes = [
|
||||
{"id": "aaaaaaa", "kind": "document", "parent": None, "ord": 0},
|
||||
{"id": "bbbbbbb", "kind": "table", "parent": "aaaaaaa", "ord": 0},
|
||||
{
|
||||
"id": "ccccccc",
|
||||
"kind": "table",
|
||||
"parent": "aaaaaaa",
|
||||
"ord": 1,
|
||||
"attrs": {"columns": [{"id": "k", "header_path": []}]},
|
||||
},
|
||||
{"id": "ddddddd", "kind": "cell", "parent": "bbbbbbb", "ord": 0},
|
||||
{
|
||||
"id": "eeeeeee",
|
||||
"kind": "cell",
|
||||
"parent": "bbbbbbb",
|
||||
"ord": 1,
|
||||
"attrs": {
|
||||
"column": "k",
|
||||
"rowspan": 2,
|
||||
"colspan": 3,
|
||||
"role": "row-header",
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": "fffffff",
|
||||
"kind": "theorem",
|
||||
"parent": "aaaaaaa",
|
||||
"ord": 2,
|
||||
"attrs": {"variant": "lemma"},
|
||||
},
|
||||
{"id": "ggggggg", "kind": "theorem", "parent": "aaaaaaa", "ord": 3},
|
||||
{
|
||||
"id": "hhhhhhh",
|
||||
"kind": "code",
|
||||
"parent": "aaaaaaa",
|
||||
"ord": 4,
|
||||
"content": {"text": "x"},
|
||||
},
|
||||
{
|
||||
"id": "iiiiiii",
|
||||
"kind": "code",
|
||||
"parent": "aaaaaaa",
|
||||
"ord": 5,
|
||||
"label": "L",
|
||||
"content": {"text": "y", "language": "python"},
|
||||
},
|
||||
]
|
||||
doc = Doc(
|
||||
"t", [dict(n, content=[]) if n["kind"] == "cell" else n for n in nodes]
|
||||
)
|
||||
assert own_content(doc, "bbbbbbb")["columns"] == []
|
||||
assert own_content(doc, "ccccccc")["columns"] == [
|
||||
{"id": "k", "header_path": []}
|
||||
]
|
||||
assert own_content(doc, "ddddddd") == {
|
||||
"kind": "cell",
|
||||
"runs": [],
|
||||
"column": None,
|
||||
"rowspan": 1,
|
||||
"colspan": 1,
|
||||
"role": "data",
|
||||
}
|
||||
assert own_content(doc, "eeeeeee") == {
|
||||
"kind": "cell",
|
||||
"runs": [],
|
||||
"column": "k",
|
||||
"rowspan": 2,
|
||||
"colspan": 3,
|
||||
"role": "row-header",
|
||||
}
|
||||
assert own_content(doc, "fffffff") == {
|
||||
"kind": "theorem",
|
||||
"label": None,
|
||||
"variant": "lemma",
|
||||
}
|
||||
assert own_content(doc, "ggggggg")["variant"] is None
|
||||
assert own_content(doc, "hhhhhhh") == {
|
||||
"kind": "code",
|
||||
"label": None,
|
||||
"language": None,
|
||||
"text": "x",
|
||||
}
|
||||
assert own_content(doc, "iiiiiii") == {
|
||||
"kind": "code",
|
||||
"label": "L",
|
||||
"language": "python",
|
||||
"text": "y",
|
||||
}
|
||||
|
||||
def test_an_equation_run_is_named_by_kind_alone(self):
|
||||
doc = build_doc(
|
||||
document(
|
||||
section(
|
||||
heading("H"),
|
||||
paragraph(
|
||||
sentence(
|
||||
{"kind": "equation", "child": 0},
|
||||
children=[
|
||||
{
|
||||
"kind": "equation",
|
||||
"content": {"notation": "tex", "source": "z"},
|
||||
}
|
||||
],
|
||||
)
|
||||
),
|
||||
)
|
||||
)
|
||||
)
|
||||
sent = next(n for n in ids_of_kind(doc, "sentence") if doc.kids(n))
|
||||
assert own_content(doc, sent)["runs"] == [{"kind": "equation"}]
|
||||
|
||||
|
||||
def _two_sentence_doc(first, second):
|
||||
return build_doc(
|
||||
document(
|
||||
section(
|
||||
heading("Head"),
|
||||
paragraph(sentence(text_run(first)), sentence(text_run(second))),
|
||||
paragraph(sentence(text_run("Untouched."))),
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _with_text(doc, nid, text):
|
||||
nodes = [dict(doc.nodes[x]) for x in doc.order]
|
||||
for n in nodes:
|
||||
if n["id"] == nid:
|
||||
n["content"] = [text_run(text)]
|
||||
return Doc(doc.key, nodes)
|
||||
|
||||
|
||||
class TestIndex:
|
||||
def test_build_gives_the_values_of_compute(self):
|
||||
doc = _two_sentence_doc("One.", "Two.")
|
||||
index = Index.build(doc)
|
||||
assert index.values == compute(doc)
|
||||
assert set(index.values) == set(doc.nodes)
|
||||
|
||||
def test_plain_is_the_plain_projection(self):
|
||||
doc = _two_sentence_doc("One.", "Two.")
|
||||
index = Index.build(doc)
|
||||
para = ids_of_kind(doc, "paragraph")[0]
|
||||
assert index.plain(para) == hashing.plain(doc, para) == "One. Two."
|
||||
assert index.plain(doc.root) == hashing.plain(doc, doc.root)
|
||||
|
||||
def test_update_after_an_edit_equals_a_rebuild_and_leaves_the_old_index_alone(self):
|
||||
doc = _two_sentence_doc("One.", "Two.")
|
||||
old = Index.build(doc)
|
||||
before = {k: dict(v) for k, v in old.values.items()}
|
||||
target = ids_of_kind(doc, "sentence")[1]
|
||||
after = _with_text(doc, target, "Changed.")
|
||||
new = old.update(after)
|
||||
assert new.values == Index.build(after).values == reference_values(after)
|
||||
assert old.values == before
|
||||
assert new.values[target]["text_hash"] != old.values[target]["text_hash"]
|
||||
untouched = ids_of_kind(doc, "sentence")[-1]
|
||||
assert new.values[untouched] == old.values[untouched]
|
||||
assert new.values[doc.root]["node_hash"] != old.values[doc.root]["node_hash"]
|
||||
assert new.plain(doc.root) == hashing.plain(after, after.root)
|
||||
|
||||
def test_an_edit_changes_exactly_the_ancestors_of_the_edited_leaf(self):
|
||||
doc = _two_sentence_doc("One.", "Two.")
|
||||
target = ids_of_kind(doc, "sentence")[1]
|
||||
after = _with_text(doc, target, "Changed.")
|
||||
old = Index.build(doc)
|
||||
new = old.update(after).values
|
||||
old = old.values
|
||||
changed = {n for n in old if old[n]["node_hash"] != new[n]["node_hash"]}
|
||||
assert changed == {target, *doc.ancestors(target)}
|
||||
|
||||
def test_update_forgets_deleted_nodes_and_learns_new_ones(self):
|
||||
doc = _two_sentence_doc("One.", "Two.")
|
||||
old = Index.build(doc)
|
||||
gone = ids_of_kind(doc, "paragraph")[1]
|
||||
removed = {gone, *doc.preorder(gone)}
|
||||
smaller = Doc(doc.key, [doc.nodes[x] for x in doc.order if x not in removed])
|
||||
assert set(old.update(smaller).values) == set(smaller.nodes)
|
||||
assert old.update(smaller).values == reference_values(smaller)
|
||||
assert Index.build(smaller).update(doc).values == old.values
|
||||
|
||||
def test_a_keyed_index_keeps_its_hasher_through_updates(self):
|
||||
keyed = Hasher(b"team")
|
||||
doc = _two_sentence_doc("One.", "Two.")
|
||||
target = ids_of_kind(doc, "sentence")[0]
|
||||
after = _with_text(doc, target, "Other.")
|
||||
new = Index.build(doc, keyed).update(after)
|
||||
assert new.hasher is keyed
|
||||
assert new.values == reference_values(after, keyed)
|
||||
|
||||
def test_an_equation_line_adds_nothing_to_its_containers_text(self):
|
||||
doc = build_doc(
|
||||
document(
|
||||
section(
|
||||
heading("H"),
|
||||
{
|
||||
"kind": "equation",
|
||||
"content": {"notation": "tex", "source": "q"},
|
||||
"children": [
|
||||
{
|
||||
"kind": "equation-line",
|
||||
"content": {"notation": "tex", "source": "q"},
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
)
|
||||
)
|
||||
eq = ids_of_kind(doc, "equation")[0]
|
||||
index = Index.build(doc)
|
||||
assert index.plain(eq) == "$$q$$"
|
||||
assert index.values == reference_values(doc)
|
||||
|
||||
|
||||
class TestMetaHash:
|
||||
DOCUMENT: ClassVar[dict] = {
|
||||
"work": {"same_as": ["doi:10.1/x"]},
|
||||
"version": {"type": "version-of-record"},
|
||||
"relations": [],
|
||||
"meta": {"title": "T"},
|
||||
"language": "en",
|
||||
"license": {"spdx": "CC0-1.0"},
|
||||
"policy": "full",
|
||||
"key": "ignored",
|
||||
}
|
||||
|
||||
def test_meta_object_takes_six_members_and_nulls_the_missing(self):
|
||||
obj = meta_object(self.DOCUMENT)
|
||||
assert list(obj) == [
|
||||
"work",
|
||||
"version",
|
||||
"relations",
|
||||
"meta",
|
||||
"language",
|
||||
"license",
|
||||
]
|
||||
assert meta_object({}) == dict.fromkeys(obj)
|
||||
|
||||
def test_meta_hash_is_the_hash_of_the_canonical_object(self):
|
||||
expected = _hash(jcs.dumps(meta_object(self.DOCUMENT)))
|
||||
assert meta_hash(self.DOCUMENT) == expected
|
||||
keyed = Hasher(b"k")
|
||||
assert meta_hash(self.DOCUMENT, keyed) == keyed(
|
||||
jcs.dumpb(meta_object(self.DOCUMENT))
|
||||
)
|
||||
assert meta_hash(self.DOCUMENT, keyed) != expected
|
||||
|
||||
def test_members_outside_the_six_do_not_change_the_hash(self):
|
||||
other = dict(self.DOCUMENT, policy="withheld", key="another")
|
||||
assert meta_hash(other) == meta_hash(self.DOCUMENT)
|
||||
assert meta_hash(dict(self.DOCUMENT, language="fr")) != meta_hash(self.DOCUMENT)
|
||||
@@ -0,0 +1,240 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Canonical JSON: the vectors, the refusals of spec/01 §1.1, and round-trip properties."""
|
||||
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
|
||||
import pytest
|
||||
from hypothesis import given, settings, strategies as st
|
||||
|
||||
from helpers_lib1 import parse_doubles, vectors
|
||||
from paperspec import jcs
|
||||
|
||||
PROPERTIES = settings(derandomize=True, max_examples=200, database=None, deadline=None)
|
||||
|
||||
JSON_VALUES = st.recursive(
|
||||
st.none()
|
||||
| st.booleans()
|
||||
| st.integers(min_value=-(2**53), max_value=2**53)
|
||||
| st.floats(allow_nan=False, allow_infinity=False)
|
||||
| st.text(),
|
||||
lambda children: (
|
||||
st.lists(children, max_size=4)
|
||||
| st.dictionaries(st.text(), children, max_size=4)
|
||||
),
|
||||
max_leaves=12,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", vectors("canonical-json"))
|
||||
def test_canonical_json_vector(case):
|
||||
assert jcs.dumps(parse_doubles(case["input_json"])) == case["output"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("number", "text"),
|
||||
[
|
||||
(0.0, "0"),
|
||||
(1.0, "1"),
|
||||
(100.0, "100"),
|
||||
(1.5, "1.5"),
|
||||
(-2.5, "-2.5"),
|
||||
(0.1 + 0.2, "0.30000000000000004"),
|
||||
(4.35, "4.35"),
|
||||
(1e20, "100000000000000000000"),
|
||||
(1e21, "1e+21"),
|
||||
(1.2345678901234568e22, "1.2345678901234568e+22"),
|
||||
(123456789012345680000.0, "123456789012345680000"),
|
||||
(0.000001, "0.000001"),
|
||||
(1e-7, "1e-7"),
|
||||
(1.5e-7, "1.5e-7"),
|
||||
(-2.5e-5, "-0.000025"),
|
||||
(5e-324, "5e-324"),
|
||||
(1.7976931348623157e308, "1.7976931348623157e+308"),
|
||||
(2.0**53, "9007199254740992"),
|
||||
(2.0**60, "1152921504606847000"),
|
||||
],
|
||||
)
|
||||
def test_floats_are_written_as_ecmascript_writes_them(number, text):
|
||||
assert jcs.dumps(number) == text
|
||||
assert jcs.dumps([number, "x"]) == f'[{text},"x"]'
|
||||
|
||||
|
||||
def test_negative_zero_is_written_as_zero():
|
||||
assert jcs.dumps(-0.0) == "0"
|
||||
assert jcs.dumps([-0.0, "x"]) == '[0,"x"]'
|
||||
|
||||
|
||||
@pytest.mark.parametrize("number", [2**53, -(2**53), 0, -1, 12345])
|
||||
def test_integers_in_the_exact_range_are_written_in_full(number):
|
||||
assert jcs.dumps(number) == str(number)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("number", [2**53 + 1, -(2**53) - 1, 10**30])
|
||||
def test_integers_beyond_the_exact_range_are_refused(number):
|
||||
with pytest.raises(ValueError, match="integer outside the IEEE 754 exact range"):
|
||||
jcs.dumps(number)
|
||||
with pytest.raises(ValueError, match="integer outside the IEEE 754 exact range"):
|
||||
jcs.dumps([1.5, number])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("number", [math.nan, math.inf, -math.inf])
|
||||
def test_nan_and_infinity_have_no_canonical_form(number):
|
||||
with pytest.raises(ValueError, match="NaN and Infinity are not JSON"):
|
||||
jcs.dumps(number)
|
||||
with pytest.raises(ValueError, match="NaN and Infinity are not JSON"):
|
||||
jcs.dumps({"a": [number]})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"lone",
|
||||
[chr(0xD800), chr(0xDFFF), "a" + chr(0xDC00) + "b"],
|
||||
ids=["high", "low", "inside"],
|
||||
)
|
||||
def test_a_lone_surrogate_has_no_canonical_form(lone):
|
||||
with pytest.raises(ValueError, match="lone surrogate"):
|
||||
jcs.dumps(lone)
|
||||
with pytest.raises(ValueError, match="lone surrogate"):
|
||||
jcs.dumps({lone: 1})
|
||||
with pytest.raises(ValueError, match="lone surrogate"):
|
||||
jcs.dumps([1.5, lone])
|
||||
with pytest.raises(ValueError, match="lone surrogate"):
|
||||
jcs.dumpb([lone])
|
||||
|
||||
|
||||
def test_a_surrogate_pair_held_as_two_code_points_is_two_lone_surrogates():
|
||||
with pytest.raises(ValueError, match="lone surrogate"):
|
||||
jcs.dumps(chr(0xD83D) + chr(0xDE00))
|
||||
|
||||
|
||||
def test_object_keys_must_be_strings():
|
||||
with pytest.raises(TypeError, match="object keys must be strings"):
|
||||
jcs.dumps({1: "a"})
|
||||
with pytest.raises(TypeError, match="object keys must be strings"):
|
||||
jcs.dumps({"a": 1.5, 2: 3})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [{1, 2}, b"bytes", object(), 1 + 2j])
|
||||
def test_values_outside_json_are_refused(value):
|
||||
with pytest.raises(TypeError, match="not JSON"):
|
||||
jcs.dumps(value)
|
||||
with pytest.raises(TypeError, match="not JSON"):
|
||||
jcs.dumps([1.5, value])
|
||||
|
||||
|
||||
def test_literals_and_containers():
|
||||
assert jcs.dumps(None) == "null"
|
||||
assert jcs.dumps(True) == "true"
|
||||
assert jcs.dumps(False) == "false"
|
||||
assert jcs.dumps([]) == "[]"
|
||||
assert jcs.dumps({}) == "{}"
|
||||
assert jcs.dumps((1, "a")) == '[1,"a"]'
|
||||
assert jcs.dumps([1.5, True, None, False]) == "[1.5,true,null,false]"
|
||||
assert jcs.dumps([1.5, (2, 3)]) == "[1.5,[2,3]]"
|
||||
|
||||
|
||||
def test_members_are_sorted_by_utf16_code_units_not_code_points():
|
||||
# U+FF5E sorts after the astral U+1F600 in code points, before it in UTF-16.
|
||||
value = {"\U0001f600": 1, "\N{FULLWIDTH TILDE}": 2, "": 3}
|
||||
assert jcs.dumps(value) == '{"\U0001f600":1,"":3,"\N{FULLWIDTH TILDE}":2}'
|
||||
assert sorted(value) == ["", "\N{FULLWIDTH TILDE}", "\U0001f600"]
|
||||
|
||||
|
||||
def test_members_are_sorted_in_nested_objects_and_with_floats_present():
|
||||
value = {"b": {"z": 1, "a": 2.5}, "a": [{"y": 1, "x": 2}]}
|
||||
assert jcs.dumps(value) == '{"a":[{"x":2,"y":1}],"b":{"a":2.5,"z":1}}'
|
||||
|
||||
|
||||
def test_strings_use_the_short_escapes_and_lower_case_unicode_escapes():
|
||||
s = '\b\t\n\f\r"\\\x01\x1f\x7f/\N{LINE SEPARATOR}'
|
||||
expected = '"\\b\\t\\n\\f\\r\\"\\\\\\u0001\\u001f\x7f/\N{LINE SEPARATOR}"'
|
||||
assert jcs.dumps(s) == expected
|
||||
assert jcs.dumps([1.5, s]) == "[1.5," + expected + "]"
|
||||
|
||||
|
||||
def test_non_ascii_text_is_written_as_it_is():
|
||||
assert jcs.dumps("café \U0001f600") == '"café \U0001f600"'
|
||||
assert jcs.dumpb("é") == b'"\xc3\xa9"'
|
||||
|
||||
|
||||
def test_dumpb_is_the_utf8_of_dumps():
|
||||
value = {"k": ["é", 1.25]}
|
||||
assert jcs.dumpb(value) == jcs.dumps(value).encode("utf-8")
|
||||
|
||||
|
||||
@given(JSON_VALUES)
|
||||
@PROPERTIES
|
||||
def test_canonical_text_parses_back_to_the_value(value):
|
||||
assert parse_doubles(jcs.dumps(value)) == value
|
||||
|
||||
|
||||
@given(JSON_VALUES)
|
||||
@PROPERTIES
|
||||
def test_canonical_json_is_a_fixed_point(value):
|
||||
once = jcs.dumps(value)
|
||||
assert jcs.dumps(parse_doubles(once)) == once
|
||||
|
||||
|
||||
@given(st.text())
|
||||
@PROPERTIES
|
||||
def test_strings_agree_whichever_path_writes_them(s):
|
||||
assert jcs.dumps([1.5, s]) == "[1.5," + jcs.dumps(s) + "]"
|
||||
|
||||
|
||||
@given(st.dictionaries(st.text(), st.integers(min_value=0, max_value=9), max_size=6))
|
||||
@PROPERTIES
|
||||
def test_members_come_out_in_utf16_order(value):
|
||||
keys = list(json.loads(jcs.dumps(value)))
|
||||
assert keys == sorted(value, key=lambda k: k.encode("utf-16-be"))
|
||||
|
||||
|
||||
@given(st.floats(allow_nan=False, allow_infinity=False))
|
||||
@PROPERTIES
|
||||
def test_a_float_is_written_in_the_shortest_form_that_reads_back_exactly(x):
|
||||
text = jcs.dumps(x)
|
||||
assert float(text) == x
|
||||
assert re.fullmatch(r"-?(0|[1-9][0-9]*)(\.[0-9]+)?(e[+-][0-9]+)?", text)
|
||||
|
||||
|
||||
@given(st.floats(min_value=-(2.0**53), max_value=2.0**53).filter(float.is_integer))
|
||||
@PROPERTIES
|
||||
def test_a_whole_float_is_written_like_an_integer(x):
|
||||
assert jcs.dumps(x) == str(int(x))
|
||||
|
||||
|
||||
def test_loads_line_reads_canonical_lines():
|
||||
line = b'{"a":[1,2.5,null],"b":"\xc3\xa9"}'
|
||||
assert jcs.loads_line(line) == {"a": [1, 2.5, None], "b": "é"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"line",
|
||||
[
|
||||
b'{"b":1,"a":2}',
|
||||
b'{"a": 1}',
|
||||
b'{"a":1.0}',
|
||||
b'{"a":1e2}',
|
||||
b'{"a":"\\u00e9"}',
|
||||
b'{"a":"\\/"}',
|
||||
b" 1",
|
||||
],
|
||||
)
|
||||
def test_loads_line_refuses_a_line_that_is_not_canonical(line):
|
||||
with pytest.raises(ValueError, match="not canonical JSON"):
|
||||
jcs.loads_line(line)
|
||||
|
||||
|
||||
def test_loads_line_refuses_bytes_that_are_not_utf8():
|
||||
with pytest.raises(UnicodeDecodeError):
|
||||
jcs.loads_line(b'"\xff"')
|
||||
|
||||
|
||||
def test_loads_line_refuses_text_that_is_not_json():
|
||||
with pytest.raises(json.JSONDecodeError):
|
||||
jcs.loads_line(b"{")
|
||||
@@ -0,0 +1,332 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Work keys and document keys (spec/02 §2.4)."""
|
||||
|
||||
from typing import ClassVar
|
||||
|
||||
import pytest
|
||||
from hypothesis import given, settings, strategies as st
|
||||
|
||||
from helpers_lib1 import case_params, read_cases
|
||||
from paperspec import keys
|
||||
from paperspec.text import ALPHABET, sha256_hex
|
||||
|
||||
PROPERTIES = settings(derandomize=True, max_examples=150, database=None, deadline=None)
|
||||
|
||||
META = {
|
||||
"author": [{"family": "Vaswani", "given": "Ashish"}],
|
||||
"issued": {"date-parts": [[2017]]},
|
||||
"title": "Attention is all you need",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case", case_params([c for c in read_cases("keys") if "version" not in c])
|
||||
)
|
||||
def test_key_vector(case):
|
||||
if "taken" in case:
|
||||
key = keys.unique_work_key(case["same_as"], case["meta"], set(case["taken"]))
|
||||
assert key == case["key"]
|
||||
assert len(key.rsplit("-", 1)[1]) == 6
|
||||
return
|
||||
ident = keys.identity(case["same_as"], case["meta"])
|
||||
assert ident == case["identity"]
|
||||
assert keys.scholar_part(case["meta"]) == case["scholar"]
|
||||
assert keys.SCHEME + ":" + ident == case["payload"]
|
||||
assert sha256_hex(case["payload"]) == case["sha256"]
|
||||
assert keys.suffix(ident) == case["suffix"]
|
||||
assert keys.suffix(ident, 8) == case["suffix_8"]
|
||||
assert case["suffix_8"].startswith(case["suffix"])
|
||||
assert keys.work_key(case["same_as"], case["meta"]) == case["key"]
|
||||
assert keys.WORK_KEY_RE.match(case["key"])
|
||||
assert case["key"] == case["scholar"] + "-" + case["suffix"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case", case_params([c for c in read_cases("keys") if "version" in c])
|
||||
)
|
||||
def test_document_key_vector(case):
|
||||
assert keys.document_key(case["work_key"], case["version"]) == case["document_key"]
|
||||
work, suffix = keys.split_key(case["document_key"])
|
||||
assert work == case["work_key"]
|
||||
assert work + suffix == case["document_key"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "folded"),
|
||||
[
|
||||
("Müller", "muller"),
|
||||
("Ångström", "angstrom"),
|
||||
("张伟", ""),
|
||||
("ALL CAPS", "all caps"),
|
||||
("fi", "fi"),
|
||||
("", ""),
|
||||
(None, ""),
|
||||
(5, ""),
|
||||
],
|
||||
)
|
||||
def test_fold(text, folded):
|
||||
assert keys.fold(text) == folded
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("title", "words"),
|
||||
[
|
||||
("Don't stop", ["dont", "stop"]),
|
||||
("Don\N{RIGHT SINGLE QUOTATION MARK}t stop", ["dont", "stop"]),
|
||||
(
|
||||
"Self-assembly of 3D-printed bone",
|
||||
["self", "assembly", "of", "3d", "printed", "bone"],
|
||||
),
|
||||
(" ", []),
|
||||
(None, []),
|
||||
("Étude: cafés", ["etude", "cafes"]),
|
||||
],
|
||||
)
|
||||
def test_title_words(title, words):
|
||||
assert keys.title_words(title) == words
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("name", "family"),
|
||||
[
|
||||
("Ada Lovelace", "Lovelace"),
|
||||
("Lovelace, Ada", "Lovelace"),
|
||||
("A. Smith Jr.", "Smith"),
|
||||
("Smith III", "Smith"),
|
||||
("John Smith, Jr.", "Smith"),
|
||||
("Smith, Jr.", "Smith"),
|
||||
("Smith, John", "Smith"),
|
||||
("Madonna", "Madonna"),
|
||||
("Jr.", "Jr."),
|
||||
("", ""),
|
||||
(None, ""),
|
||||
(" Plato ", "Plato"),
|
||||
],
|
||||
)
|
||||
def test_family_name(name, family):
|
||||
assert keys.family_name(name) == family
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("meta", "family"),
|
||||
[
|
||||
({}, "anon"),
|
||||
({"author": []}, "anon"),
|
||||
({"author": [{"family": "O'Brien"}]}, "obrien"),
|
||||
({"author": [{"literal": "Conan O'Brien"}]}, "obrien"),
|
||||
({"author": [{"name": "Grace Hopper"}]}, "hopper"),
|
||||
({"author": [{"family": "张伟"}]}, "anon"),
|
||||
({"author": [{}]}, "anon"),
|
||||
({"author": [{"family": "van der Waals"}]}, "vanderwaals"),
|
||||
({"author": [{"family": "Kim"}, {"family": "Lee"}]}, "kim"),
|
||||
],
|
||||
)
|
||||
def test_first_family(meta, family):
|
||||
assert keys.first_family(meta) == family
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("meta", "year"),
|
||||
[
|
||||
({"issued": {"date-parts": [[2017, 3, 1]]}}, "2017"),
|
||||
({"issued": {"date-parts": [["2017"]]}}, "2017"),
|
||||
({"issued": {"date-parts": [[17]]}}, ""),
|
||||
({"issued": {"date-parts": [[20170]]}}, ""),
|
||||
({"issued": {"date-parts": [[True]]}}, ""),
|
||||
({"issued": {"date-parts": [[None]]}}, ""),
|
||||
({"issued": {"date-parts": [[]]}}, ""),
|
||||
({"issued": {"date-parts": []}}, ""),
|
||||
({"issued": {}}, ""),
|
||||
({"issued": None}, ""),
|
||||
({}, ""),
|
||||
],
|
||||
)
|
||||
def test_year_of(meta, year):
|
||||
assert keys.year_of(meta) == year
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("title", "scholar"),
|
||||
[
|
||||
("The theory of everything", "vaswani2017theory"),
|
||||
("A of the", "vaswani2017"),
|
||||
("", "vaswani2017"),
|
||||
("Attention is all you need", "vaswani2017attention"),
|
||||
("On the origin", "vaswani2017origin"),
|
||||
],
|
||||
)
|
||||
def test_scholar_part_skips_stop_words(title, scholar):
|
||||
assert keys.scholar_part(dict(META, title=title)) == scholar
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "bare"),
|
||||
[
|
||||
("10.1000/ABC", "10.1000/abc"),
|
||||
("doi:10.1000/ABC", "10.1000/abc"),
|
||||
("DOI:10.1000/abc", "10.1000/abc"),
|
||||
("https://doi.org/10.1000/abc", "10.1000/abc"),
|
||||
("http://dx.doi.org/10.1000/abc", "10.1000/abc"),
|
||||
("https://www.doi.org/10.1000/abc", "10.1000/abc"),
|
||||
(" doi:doi:10.1000/abc ", "10.1000/abc"),
|
||||
("10.1000/É", "10.1000/É"),
|
||||
],
|
||||
)
|
||||
def test_norm_doi(value, bare):
|
||||
assert keys.norm_doi(value) == bare
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("same_as", "identity"),
|
||||
[
|
||||
(["arxiv:1706.03762"], "arxiv:1706.03762"),
|
||||
(["isbn: 978-3-16 "], "isbn:978-3-16"),
|
||||
(["pmid:1", "arxiv:2"], "arxiv:2"),
|
||||
(["openalex:W1", "pmcid:PMC1"], "pmcid:PMC1"),
|
||||
(["doi:", "arxiv:1706.03762"], "arxiv:1706.03762"),
|
||||
(["doi:10.1/A", "doi:10.1/B"], "doi:10.1/a"),
|
||||
(
|
||||
["url:https://example.org", "title:x"],
|
||||
"meta:attention is all you need|2017|vaswani",
|
||||
),
|
||||
(None, "meta:attention is all you need|2017|vaswani"),
|
||||
([], "meta:attention is all you need|2017|vaswani"),
|
||||
(["noprefix"], "meta:attention is all you need|2017|vaswani"),
|
||||
],
|
||||
)
|
||||
def test_identity(same_as, identity):
|
||||
assert keys.identity(same_as, META) == identity
|
||||
|
||||
|
||||
@pytest.mark.parametrize("length", [1, 4, 8, 51])
|
||||
def test_suffix_has_the_asked_length_and_alphabet(length):
|
||||
out = keys.suffix("doi:10.1/x", length)
|
||||
assert len(out) == length
|
||||
assert set(out) <= set(ALPHABET)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("length", [0, -1, 52, 100])
|
||||
def test_suffix_length_is_one_to_fifty_one(length):
|
||||
with pytest.raises(ValueError, match="a key suffix is 1 to 51 characters"):
|
||||
keys.suffix("doi:10.1/x", length)
|
||||
|
||||
|
||||
@given(st.text(max_size=40), st.integers(min_value=1, max_value=50))
|
||||
@PROPERTIES
|
||||
def test_a_shorter_suffix_is_a_prefix_of_a_longer_one(ident, length):
|
||||
assert keys.suffix(ident, length + 1).startswith(keys.suffix(ident, length))
|
||||
|
||||
|
||||
def test_unique_work_key_lengthens_by_two_while_taken():
|
||||
same_as = ["doi:10.1/x"]
|
||||
first = keys.work_key(same_as, META, 4)
|
||||
second = keys.work_key(same_as, META, 6)
|
||||
third = keys.work_key(same_as, META, 8)
|
||||
assert keys.unique_work_key(same_as, META, set()) == first
|
||||
assert keys.unique_work_key(same_as, META, {first}) == second
|
||||
assert keys.unique_work_key(same_as, META, {first, second}) == third
|
||||
assert keys.unique_work_key(same_as, META, {second}) == first
|
||||
|
||||
|
||||
def test_unique_work_key_gives_up_when_every_length_is_taken():
|
||||
same_as = ["doi:10.1/x"]
|
||||
taken = {keys.work_key(same_as, META, n) for n in range(4, 52, 2)}
|
||||
with pytest.raises(ValueError, match=r"no free key for doi:10\.1/x"):
|
||||
keys.unique_work_key(same_as, META, taken)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("version", "suffix"),
|
||||
[
|
||||
({"type": "version-of-record"}, ""),
|
||||
({"type": "supplement"}, ".si"),
|
||||
({"type": "preprint", "seq": 2}, ".v2"),
|
||||
({"type": "draft", "seq": 10}, ".v10"),
|
||||
({"type": "submitted", "seq": 1}, ".v1"),
|
||||
({"type": "accepted", "seq": 3}, ".v3"),
|
||||
({"type": "accepted"}, ".accepted"),
|
||||
({"type": "preprint", "seq": "2"}, ".preprint"),
|
||||
({"type": "erratum", "seq": 4}, ".erratum"),
|
||||
],
|
||||
)
|
||||
def test_version_suffix(version, suffix):
|
||||
assert keys.version_suffix(version) == suffix
|
||||
assert keys.document_key("a2020b-abcd", version) == "a2020b-abcd" + suffix
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("dkey", "parts"),
|
||||
[
|
||||
("vaswani2017attention-a241", ("vaswani2017attention-a241", "")),
|
||||
("vaswani2017attention-a241.v2", ("vaswani2017attention-a241", ".v2")),
|
||||
("vaswani2017attention-a241.si.x-y", ("vaswani2017attention-a241", ".si.x-y")),
|
||||
],
|
||||
)
|
||||
def test_split_key(dkey, parts):
|
||||
assert keys.split_key(dkey) == parts
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"dkey",
|
||||
[
|
||||
"",
|
||||
"vaswani",
|
||||
"Vaswani2017-a241",
|
||||
"vaswani2017-a24",
|
||||
"vaswani2017-a24i",
|
||||
"vaswani2017-a241.",
|
||||
"vaswani2017-a241..v2",
|
||||
"vaswani2017-a241.V2",
|
||||
"2017vaswani-a241",
|
||||
"a-b-c0de",
|
||||
],
|
||||
)
|
||||
def test_split_key_refuses_what_is_not_a_document_key(dkey):
|
||||
with pytest.raises(ValueError, match="is not a document key"):
|
||||
keys.split_key(dkey)
|
||||
|
||||
|
||||
def test_suffix_of_takes_the_part_after_the_last_hyphen():
|
||||
assert keys.suffix_of("vaswani2017attention-a241") == "a241"
|
||||
assert keys.suffix_of("a-b-c0de") == "c0de"
|
||||
|
||||
|
||||
class TestSuffixAgrees:
|
||||
DOI: ClassVar[list] = ["doi:10.5555/example"]
|
||||
|
||||
def test_an_identifier_identity_is_held_to_the_key(self):
|
||||
key = keys.work_key(self.DOI, META)
|
||||
assert keys.suffix_agrees(key, self.DOI, META)
|
||||
long_key = keys.work_key(self.DOI, META, 8)
|
||||
assert keys.suffix_agrees(long_key, self.DOI, META)
|
||||
|
||||
def test_a_different_hash_part_disagrees(self):
|
||||
key = keys.work_key(self.DOI, META)
|
||||
other = key[:-1] + ("0" if key[-1] != "0" else "1")
|
||||
assert not keys.suffix_agrees(other, self.DOI, META)
|
||||
|
||||
def test_a_metadata_identity_is_not_held_to_the_key(self):
|
||||
assert keys.suffix_agrees("vaswani2017attention-zzzz", [], META)
|
||||
|
||||
|
||||
@given(
|
||||
st.text(max_size=30),
|
||||
st.text(max_size=20),
|
||||
st.one_of(st.none(), st.integers(min_value=0, max_value=99999)),
|
||||
)
|
||||
@PROPERTIES
|
||||
def test_any_metadata_gives_a_valid_work_key_and_document_key(title, family, year):
|
||||
meta = {"title": title, "author": [{"family": family}]}
|
||||
if year is not None:
|
||||
meta["issued"] = {"date-parts": [[year]]}
|
||||
key = keys.work_key([], meta)
|
||||
assert keys.WORK_KEY_RE.match(key)
|
||||
document = keys.document_key(key, {"type": "preprint", "seq": 2})
|
||||
assert keys.DOCUMENT_KEY_RE.match(document)
|
||||
assert keys.split_key(document) == (key, ".v2")
|
||||
assert keys.suffix_agrees(key, [], meta)
|
||||
@@ -0,0 +1,72 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Every source file carries the license header; ruff checks the Python files, this checks the rest.
|
||||
|
||||
paperspec is an exception to Scienting's Prosperity header: its code is Apache-2.0 (LICENSE.md), so the header
|
||||
is an SPDX identifier, and the schemas, SQL and vectors, which are CC0-1.0 data, carry none.
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
NOTICE = "SPDX-License-Identifier: Apache-2.0"
|
||||
SOURCE = "Source Code: https://git.scient.ing/papers/paperspec"
|
||||
|
||||
# Comment syntax by file suffix. Data, lock files, generated output and the normative CC0 data (schemas, SQL,
|
||||
# vectors, fixtures) are not source files and are not listed.
|
||||
COMMENT = {
|
||||
".py": "#",
|
||||
".sh": "#",
|
||||
".js": "//",
|
||||
".ts": "//",
|
||||
}
|
||||
# Paths (relative, forward slashes) that are generated or vendored, never source.
|
||||
EXCLUDED = ("src/paperspec/_version.py",)
|
||||
|
||||
|
||||
def tracked_sources() -> list[Path]:
|
||||
"""Return the tracked and untracked-but-not-ignored files whose suffix names a source language."""
|
||||
out = subprocess.run(
|
||||
[
|
||||
"git",
|
||||
"-C",
|
||||
str(ROOT),
|
||||
"ls-files",
|
||||
"--cached",
|
||||
"--others",
|
||||
"--exclude-standard",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
).stdout.splitlines()
|
||||
return [
|
||||
ROOT / name
|
||||
for name in out
|
||||
if Path(name).suffix in COMMENT
|
||||
and not name.startswith(EXCLUDED)
|
||||
and (ROOT / name).is_file()
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path", tracked_sources(), ids=lambda p: str(p.relative_to(ROOT))
|
||||
)
|
||||
def test_file_has_the_license_header(path: Path) -> None:
|
||||
"""The first lines (after a shebang) are the SPDX notice and name this repository as the source."""
|
||||
lines = path.read_text(encoding="utf-8").splitlines()
|
||||
if lines and lines[0].startswith("#!"):
|
||||
lines = lines[1:]
|
||||
head = "\n".join(lines[:8])
|
||||
marker = COMMENT[path.suffix]
|
||||
assert f"{marker} {NOTICE}" in head, (
|
||||
f"{path.relative_to(ROOT)} lacks the license header"
|
||||
)
|
||||
assert f"{marker} {SOURCE}" in head
|
||||
@@ -0,0 +1,525 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""The transparency log: entries, signed checkpoints and tiles (paperspec.log)."""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from hypothesis import given, settings, strategies as st
|
||||
|
||||
from conftest import vector_cases
|
||||
from helpers_lib2 import (
|
||||
PROPS,
|
||||
RECORDS,
|
||||
fixture_receipts,
|
||||
fixture_roster,
|
||||
receipt,
|
||||
revocation,
|
||||
roster_key,
|
||||
roster_of,
|
||||
sealed,
|
||||
secret,
|
||||
)
|
||||
from paperspec import crypto, jcs, log
|
||||
|
||||
CHECKPOINT = vector_cases("checkpoint")[0]
|
||||
ORIGIN = "scient.ing/log/test"
|
||||
ROSTER = roster_of(
|
||||
roster_key("ana", role="owner"),
|
||||
roster_key("ben", role="owner"),
|
||||
roster_key("cara", role="editor"),
|
||||
roster_key("w1", role="guest", purposes=("witness",)),
|
||||
roster_key("w2", role="guest", purposes=("witness",)),
|
||||
)
|
||||
SETTINGS = {"origin": ORIGIN, "threshold": 1, "witnesses": ["w1", "w2"]}
|
||||
|
||||
|
||||
def entries_of(n):
|
||||
"""`n` log entries."""
|
||||
return [
|
||||
["receipt", f"r{i:04d}", hashlib.sha256(str(i).encode()).hexdigest()]
|
||||
for i in range(n)
|
||||
]
|
||||
|
||||
|
||||
def note_of(signers, entries=None, origin=ORIGIN):
|
||||
"""A checkpoint over `entries` signed by the fixture keys named in `signers`."""
|
||||
pairs = [(name, secret(name)) for name in signers]
|
||||
if entries is None:
|
||||
entries = entries_of(3)
|
||||
return log.sign_checkpoint(origin, entries, pairs)
|
||||
|
||||
|
||||
# --- entries ---------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_receipt_entry_names_its_kind_id_and_record_hash():
|
||||
rec = sealed(receipt([("nodea", "aa" * 32)]))
|
||||
body = {k: v for k, v in rec.items() if k not in ("id", "signature")}
|
||||
assert log.entry(rec) == [
|
||||
"receipt",
|
||||
rec["id"],
|
||||
hashlib.sha256(jcs.dumpb(body)).hexdigest(),
|
||||
]
|
||||
|
||||
|
||||
def test_a_revocation_entry_is_of_its_own_kind():
|
||||
rev = revocation("ana", receipt="abc")
|
||||
assert log.entry(rev)[:2] == ["revocation", rev["id"]]
|
||||
|
||||
|
||||
def test_a_roster_entry_names_the_store_and_hashes_the_whole_roster():
|
||||
roster = json.loads((RECORDS / "roster.json").read_text(encoding="utf-8"))
|
||||
assert log.entry(roster) == [
|
||||
"roster",
|
||||
roster.get("store", ""),
|
||||
hashlib.sha256(jcs.dumpb(roster)).hexdigest(),
|
||||
]
|
||||
assert (
|
||||
log.entry({"format": "scienting-roster/1", "store": "smithlab"})[1]
|
||||
== "smithlab"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"record",
|
||||
[{}, {"format": "scienting-policy/1", "id": "x"}, {"format": None}],
|
||||
ids=["no-format", "unlogged-format", "null-format"],
|
||||
)
|
||||
def test_other_records_are_not_logged(record):
|
||||
with pytest.raises(ValueError, match="only receipts, revocations and rosters"):
|
||||
log.entry(record)
|
||||
|
||||
|
||||
def test_the_root_of_the_entries_is_the_history_root():
|
||||
entries = entries_of(5)
|
||||
assert log.root(entries) == crypto.history_root(entries)
|
||||
assert log.root([]) == crypto.EMPTY_ROOT
|
||||
|
||||
|
||||
# --- checkpoints -----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_checkpoint_vector_root_note_verification_and_tiles():
|
||||
case = CHECKPOINT
|
||||
roster = fixture_roster()
|
||||
signers = [(name, secret(name)) for name in case["signers"]]
|
||||
assert log.root(case["entries"]) == case["root"]
|
||||
assert log.sign_checkpoint(case["origin"], case["entries"], signers) == case["note"]
|
||||
note, verified = log.verify_checkpoint(case["note"], roster)
|
||||
assert note["size"] == len(case["entries"])
|
||||
assert note["root"] == case["root"]
|
||||
assert note["origin"] == case["origin"]
|
||||
assert verified == case["signers"]
|
||||
assert sorted(log.tiles(case["entries"])) == case["tiles"]
|
||||
|
||||
|
||||
def test_a_checkpoint_with_a_changed_origin_verifies_no_signer():
|
||||
case = CHECKPOINT
|
||||
bad = case["note"].replace(case["origin"], case["origin"] + "x", 1)
|
||||
note, verified = log.verify_checkpoint(bad, fixture_roster())
|
||||
assert note is not None
|
||||
assert verified == []
|
||||
|
||||
|
||||
def test_the_note_body_is_the_origin_the_size_and_the_root_in_base64():
|
||||
entries = entries_of(4)
|
||||
note = note_of([], entries)
|
||||
root = bytes.fromhex(log.root(entries))
|
||||
assert note == f"{ORIGIN}\n4\n{base64.b64encode(root).decode()}\n\n"
|
||||
assert log.note_body(ORIGIN, 4, log.root(entries)) + "\n" == note
|
||||
|
||||
|
||||
def test_a_signature_line_is_the_name_and_the_key_hash_and_signature_in_base64():
|
||||
note = note_of(["ana"])
|
||||
line = note.rstrip("\n").split("\n")[-1]
|
||||
dash, name, blob = line.split(" ")
|
||||
raw = base64.b64decode(blob)
|
||||
public = crypto.ed25519_public(secret("ana"))
|
||||
assert (dash, name) == ("—", "ana")
|
||||
assert raw[:4] == log.key_hash("ana", public)
|
||||
body = log.note_body(ORIGIN, 3, log.root(entries_of(3))).encode()
|
||||
assert crypto.ed25519_verify(public, body, raw[4:])
|
||||
|
||||
|
||||
def test_the_key_hash_binds_the_name_and_the_key():
|
||||
one, other = crypto.ed25519_public(bytes(32)), crypto.ed25519_public(b"\x01" * 32)
|
||||
assert len(log.key_hash("ana", one)) == 4
|
||||
assert log.key_hash("ana", one) != log.key_hash("ben", one)
|
||||
assert log.key_hash("ana", one) != log.key_hash("ana", other)
|
||||
expected = hashlib.sha256(b"ana\n\x01" + one).digest()[:4]
|
||||
assert log.key_hash("ana", one) == expected
|
||||
|
||||
|
||||
def test_a_note_parses_to_its_parts():
|
||||
entries = entries_of(3)
|
||||
parsed = log.parse_checkpoint(note_of(["ana", "w1"], entries))
|
||||
assert parsed["origin"] == ORIGIN
|
||||
assert parsed["size"] == 3
|
||||
assert parsed["root"] == log.root(entries)
|
||||
assert [name for name, _, _ in parsed["signatures"]] == ["ana", "w1"]
|
||||
assert all(len(kh) == 4 and len(sig) == 64 for _, kh, sig in parsed["signatures"])
|
||||
|
||||
|
||||
def test_a_note_with_no_signature_is_well_formed_and_verifies_no_signer():
|
||||
note = note_of([])
|
||||
assert log.parse_checkpoint(note)["signatures"] == []
|
||||
parsed, verified = log.verify_checkpoint(note, ROSTER)
|
||||
assert parsed["size"] == 3
|
||||
assert verified == []
|
||||
|
||||
|
||||
def test_a_note_of_an_empty_log_is_well_formed():
|
||||
parsed = log.parse_checkpoint(note_of(["ana"], entries=[]))
|
||||
assert parsed["size"] == 0
|
||||
assert parsed["root"] == crypto.EMPTY_ROOT
|
||||
|
||||
|
||||
GOOD = note_of(["ana", "w1"])
|
||||
BODY, _, SIGS = GOOD.partition("\n\n")
|
||||
LINES = SIGS.split("\n")[:-1]
|
||||
ROOT_B64 = BODY.split("\n")[2]
|
||||
FORGED_ROOT = base64.b64encode(bytes(32)).decode()
|
||||
|
||||
|
||||
def with_body(origin=ORIGIN, size="3", root=ROOT_B64, sigs=SIGS):
|
||||
"""A note assembled from the given parts, none of them checked."""
|
||||
return f"{origin}\n{size}\n{root}\n\n{sigs}"
|
||||
|
||||
|
||||
MALFORMED = {
|
||||
"empty": "",
|
||||
"no-blank-line": BODY + "\n",
|
||||
"two-lines": f"{ORIGIN}\n3\n\n{SIGS}",
|
||||
"four-lines": with_body(root=ROOT_B64 + "\nextra"),
|
||||
"empty-origin": with_body(origin=""),
|
||||
"size-with-leading-zero": with_body(size="03"),
|
||||
"size-negative": with_body(size="-3"),
|
||||
"size-not-a-number": with_body(size="three"),
|
||||
"size-superscript": with_body(size="²"),
|
||||
"size-arabic-indic": with_body(size="٣"),
|
||||
"size-empty": with_body(size=""),
|
||||
"root-not-base64": with_body(root="!!!!"),
|
||||
"root-too-short": with_body(root=base64.b64encode(bytes(31)).decode()),
|
||||
"root-too-long": with_body(root=base64.b64encode(bytes(33)).decode()),
|
||||
"root-not-canonical": with_body(root=ROOT_B64[:-2] + "B="),
|
||||
"root-unpadded": with_body(root=ROOT_B64.rstrip("=")),
|
||||
"signatures-without-final-newline": GOOD[:-1],
|
||||
"signature-line-without-dash": with_body(sigs=SIGS.replace("— ", "- ", 1)),
|
||||
"signature-without-name": with_body(sigs="— \n"),
|
||||
"signature-without-blob": with_body(sigs="— ana\n"),
|
||||
"signature-blob-not-base64": with_body(sigs="— ana !!!!\n"),
|
||||
"signature-blob-too-short": with_body(
|
||||
sigs=f"— ana {base64.b64encode(bytes(67)).decode()}\n"
|
||||
),
|
||||
"signature-blob-too-long": with_body(
|
||||
sigs=f"— ana {base64.b64encode(bytes(69)).decode()}\n"
|
||||
),
|
||||
"blank-line-between-signatures": with_body(
|
||||
sigs=LINES[0] + "\n\n" + LINES[1] + "\n"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", MALFORMED)
|
||||
def test_malformed_notes_do_not_parse(name):
|
||||
assert log.parse_checkpoint(MALFORMED[name]) is None
|
||||
assert log.verify_checkpoint(MALFORMED[name], ROSTER) == (None, [])
|
||||
|
||||
|
||||
def test_a_note_with_a_different_root_is_well_formed_but_signed_by_nobody():
|
||||
parsed, verified = log.verify_checkpoint(with_body(root=FORGED_ROOT), ROSTER)
|
||||
assert parsed["root"] == "00" * 32
|
||||
assert verified == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"separator",
|
||||
["\x0b", "\x0c", "\x1c", "\x1d", "\x1e", "\x85", chr(0x2028), chr(0x2029), "\r"],
|
||||
ids=lambda s: f"U+{ord(s):04X}",
|
||||
)
|
||||
def test_only_the_line_feed_ends_a_line_of_a_note(separator):
|
||||
assert log.parse_checkpoint(GOOD[:-1] + separator) is None
|
||||
assert log.parse_checkpoint(with_body(sigs=separator.join(LINES) + "\n")) is None
|
||||
|
||||
|
||||
def test_a_note_with_text_after_its_signatures_is_malformed():
|
||||
assert log.parse_checkpoint(GOOD + "extra\n") is None
|
||||
|
||||
|
||||
def test_verification_checks_each_signer_against_the_roster():
|
||||
note = note_of(["ana", "ben", "cara", "w1"])
|
||||
_, verified = log.verify_checkpoint(note, ROSTER)
|
||||
assert verified == ["ana", "ben", "cara", "w1"]
|
||||
|
||||
|
||||
def test_a_signer_the_roster_does_not_hold_is_skipped():
|
||||
_, verified = log.verify_checkpoint(note_of(["ana", "zed"]), ROSTER)
|
||||
assert verified == ["ana"]
|
||||
|
||||
|
||||
def test_a_signer_whose_key_is_not_ed25519_is_skipped():
|
||||
roster = {**ROSTER, "ana": {**ROSTER["ana"], "scheme": "ml-dsa-65"}}
|
||||
assert log.verify_checkpoint(note_of(["ana", "ben"]), roster)[1] == ["ben"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"public", ["not base64!", None, base64.b64encode(bytes(31)).decode()], ids=repr
|
||||
)
|
||||
def test_a_signer_whose_public_key_is_unusable_is_skipped(public):
|
||||
roster = {**ROSTER, "ana": {**ROSTER["ana"], "public": public}}
|
||||
assert log.verify_checkpoint(note_of(["ana", "ben"]), roster)[1] == ["ben"]
|
||||
|
||||
|
||||
def test_a_signer_under_another_key_is_skipped():
|
||||
roster = {**ROSTER, "ana": roster_key("ben")}
|
||||
assert log.verify_checkpoint(note_of(["ana", "ben"]), roster)[1] == ["ben"]
|
||||
|
||||
|
||||
def test_a_signature_line_moved_to_another_name_does_not_verify():
|
||||
note = note_of(["ana", "ben"])
|
||||
swapped = note.replace("— ana ", "— zed ", 1)
|
||||
assert log.verify_checkpoint(swapped, ROSTER)[1] == ["ben"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("position", range(0, 200, 7))
|
||||
def test_an_edit_anywhere_in_a_note_costs_a_signer_or_the_whole_note(position):
|
||||
note = note_of(["ana", "ben"])
|
||||
edited = (
|
||||
note[:position] + chr((ord(note[position]) % 126) + 1) + note[position + 1 :]
|
||||
)
|
||||
parsed, verified = log.verify_checkpoint(edited, ROSTER)
|
||||
assert parsed is None or len(verified) < 2
|
||||
|
||||
|
||||
NAMES = st.text(
|
||||
alphabet="abcdefghijklmnopqrstuvwxyz0123456789-", min_size=1, max_size=8
|
||||
)
|
||||
|
||||
|
||||
@given(
|
||||
st.text(alphabet="abcdefghijklmnopqrstuvwxyz/.-_ ", min_size=1, max_size=30),
|
||||
st.integers(0, 12),
|
||||
st.lists(st.sampled_from(["ana", "ben", "cara"]), unique=True, max_size=3),
|
||||
)
|
||||
@settings(PROPS, max_examples=30)
|
||||
def test_a_signed_note_parses_back_and_every_signer_verifies(origin, size, signers):
|
||||
entries = entries_of(size)
|
||||
note = note_of(signers, entries, origin=origin)
|
||||
parsed, verified = log.verify_checkpoint(note, ROSTER)
|
||||
assert parsed["origin"] == origin
|
||||
assert parsed["size"] == size
|
||||
assert parsed["root"] == log.root(entries)
|
||||
assert verified == signers
|
||||
|
||||
|
||||
# --- acceptance --------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_the_example_projects_log_accepts_its_checkpoint():
|
||||
note = (RECORDS / "log" / "checkpoint").read_text(encoding="utf-8")
|
||||
project = json.loads((RECORDS / "project.json").read_text(encoding="utf-8"))
|
||||
settings_log = project["settings"]["log"]
|
||||
roster = fixture_roster()
|
||||
assert log.accept_checkpoint(note, roster, settings_log) == (True, [])
|
||||
assert not log.accept_checkpoint(note, roster, dict(settings_log, threshold=2))[0]
|
||||
assert not log.accept_checkpoint(note, roster, dict(settings_log, origin="other"))[
|
||||
0
|
||||
]
|
||||
|
||||
|
||||
def test_a_checkpoint_from_an_owner_and_enough_witnesses_is_accepted():
|
||||
note = note_of(["ana", "w1"])
|
||||
assert log.accept_checkpoint(note, ROSTER, SETTINGS) == (True, [])
|
||||
|
||||
|
||||
def test_a_malformed_note_is_not_accepted():
|
||||
assert log.accept_checkpoint("junk", ROSTER, SETTINGS) == (
|
||||
False,
|
||||
["the note is malformed"],
|
||||
)
|
||||
|
||||
|
||||
def test_a_checkpoint_of_another_log_is_not_accepted():
|
||||
note = note_of(["ana", "w1"], origin="scient.ing/log/other")
|
||||
accepted, reasons = log.accept_checkpoint(note, ROSTER, SETTINGS)
|
||||
assert not accepted
|
||||
assert reasons == ["the origin is not the project's log"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"signers", [["w1"], ["cara", "w1"]], ids=["witness-only", "editor-only"]
|
||||
)
|
||||
def test_a_checkpoint_without_an_owner_key_is_not_accepted(signers):
|
||||
accepted, reasons = log.accept_checkpoint(note_of(signers), ROSTER, SETTINGS)
|
||||
assert not accepted
|
||||
assert reasons == ["no owner key with the sign purpose signed it"]
|
||||
|
||||
|
||||
def test_an_owner_key_without_the_sign_purpose_does_not_count():
|
||||
roster = {
|
||||
**ROSTER,
|
||||
"ana": roster_key("ana", role="owner", purposes=("spot-check",)),
|
||||
}
|
||||
accepted, reasons = log.accept_checkpoint(note_of(["ana", "w1"]), roster, SETTINGS)
|
||||
assert not accepted
|
||||
assert reasons == ["no owner key with the sign purpose signed it"]
|
||||
|
||||
|
||||
def test_witnesses_must_reach_the_threshold_and_be_the_projects_own():
|
||||
settings_two = dict(SETTINGS, threshold=2)
|
||||
one = log.accept_checkpoint(note_of(["ana", "w1"]), ROSTER, settings_two)
|
||||
assert one[0] is False
|
||||
assert one[1] == ["1 of 2 witnesses cosigned"]
|
||||
assert log.accept_checkpoint(
|
||||
note_of(["ana", "w1", "w2"]), ROSTER, settings_two
|
||||
) == (True, [])
|
||||
unlisted = dict(SETTINGS, witnesses=["w2"])
|
||||
assert not log.accept_checkpoint(note_of(["ana", "w1"]), ROSTER, unlisted)[0]
|
||||
|
||||
|
||||
def test_a_witness_key_needs_the_witness_purpose():
|
||||
roster = {**ROSTER, "w1": roster_key("w1", role="guest", purposes=("sign",))}
|
||||
accepted, reasons = log.accept_checkpoint(note_of(["ana", "w1"]), roster, SETTINGS)
|
||||
assert not accepted
|
||||
assert reasons == ["0 of 1 witnesses cosigned"]
|
||||
|
||||
|
||||
def test_settings_without_witnesses_or_a_threshold_ask_for_none():
|
||||
assert log.accept_checkpoint(note_of(["ana"]), ROSTER, {"origin": ORIGIN}) == (
|
||||
True,
|
||||
[],
|
||||
)
|
||||
|
||||
|
||||
def test_every_failing_condition_is_reported():
|
||||
note = note_of(["cara"], origin="scient.ing/log/other")
|
||||
accepted, reasons = log.accept_checkpoint(note, ROSTER, SETTINGS)
|
||||
assert not accepted
|
||||
assert len(reasons) == 3
|
||||
|
||||
|
||||
# --- tile paths --------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("level", "index", "width", "path"),
|
||||
[
|
||||
(0, 0, None, "tile/0/000"),
|
||||
(0, 7, None, "tile/0/007"),
|
||||
(0, 1234067, None, "tile/0/x001/x234/067"),
|
||||
(3, 1000, None, "tile/3/x001/000"),
|
||||
("entries", 5, 9, "tile/entries/005.p/9"),
|
||||
(1, 999, 255, "tile/1/999.p/255"),
|
||||
(0, 1000000, None, "tile/0/x001/x000/000"),
|
||||
],
|
||||
)
|
||||
def test_tile_paths_group_the_index_in_threes(level, index, width, path):
|
||||
assert log.tile_path(level, index, width) == path
|
||||
|
||||
|
||||
# --- tiles ---------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_log_of_nine_entries_has_one_partial_hash_tile_and_one_of_entries():
|
||||
entries = entries_of(9)
|
||||
tiles = log.tiles(entries)
|
||||
assert sorted(tiles) == ["tile/0/000.p/9", "tile/entries/000.p/9"]
|
||||
leaves = [crypto.leaf_hash(e) for e in entries]
|
||||
assert tiles["tile/0/000.p/9"] == bytes.fromhex("".join(leaves))
|
||||
assert tiles["tile/entries/000.p/9"] == b"".join(
|
||||
jcs.dumpb(e) + b"\n" for e in entries
|
||||
)
|
||||
|
||||
|
||||
def test_a_log_of_nothing_has_no_tiles():
|
||||
assert log.tiles([]) == {}
|
||||
|
||||
|
||||
def test_a_log_of_two_tiles_and_a_bit_has_full_partial_and_upper_tiles():
|
||||
entries = entries_of(2 * log.TILE_WIDTH + 44)
|
||||
tiles = log.tiles(entries)
|
||||
assert sorted(tiles) == [
|
||||
"tile/0/000",
|
||||
"tile/0/001",
|
||||
"tile/0/002.p/44",
|
||||
"tile/1/000.p/2",
|
||||
"tile/entries/000",
|
||||
"tile/entries/001",
|
||||
"tile/entries/002.p/44",
|
||||
]
|
||||
assert len(tiles["tile/0/000"]) == 32 * log.TILE_WIDTH
|
||||
assert len(tiles["tile/1/000.p/2"]) == 64
|
||||
first = crypto.history_root(entries[: log.TILE_WIDTH])
|
||||
second = crypto.history_root(entries[log.TILE_WIDTH : 2 * log.TILE_WIDTH])
|
||||
assert tiles["tile/1/000.p/2"] == bytes.fromhex(first + second)
|
||||
assert tiles["tile/entries/001"].count(b"\n") == log.TILE_WIDTH
|
||||
|
||||
|
||||
def test_a_log_of_exactly_one_tile_has_no_partial_tiles():
|
||||
tiles = log.tiles(entries_of(log.TILE_WIDTH))
|
||||
assert sorted(tiles) == ["tile/0/000", "tile/1/000.p/1", "tile/entries/000"]
|
||||
|
||||
|
||||
def test_the_example_logs_tiles_are_those_the_oracle_writes():
|
||||
entries, note = log.read_log(str(RECORDS / "log"))
|
||||
on_disk = {
|
||||
p.relative_to(RECORDS / "log").as_posix(): p.read_bytes()
|
||||
for p in (RECORDS / "log").rglob("*")
|
||||
if p.is_file() and p.name != "checkpoint"
|
||||
}
|
||||
assert log.tiles(entries) == on_disk
|
||||
want = [log.entry(r) for r in fixture_receipts()]
|
||||
want.append(
|
||||
log.entry(json.loads((RECORDS / "roster.json").read_text(encoding="utf-8")))
|
||||
)
|
||||
assert entries == want
|
||||
assert log.verify_checkpoint(note, fixture_roster())[1] == ["ana", "wit"]
|
||||
|
||||
|
||||
# --- writing and reading a log -------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"size", [0, 1, 9, 255, 256, 257, 600], ids=lambda n: f"{n}-entries"
|
||||
)
|
||||
def test_a_written_log_reads_back(tmp_path, size):
|
||||
entries = entries_of(size)
|
||||
signers = [("ana", secret("ana"))]
|
||||
note = log.write_log(entries, ORIGIN, signers, str(tmp_path))
|
||||
assert (tmp_path / "checkpoint").read_text(encoding="utf-8") == note
|
||||
read, read_note = log.read_log(str(tmp_path))
|
||||
assert read == entries
|
||||
assert read_note == note
|
||||
assert log.verify_checkpoint(read_note, ROSTER) == (
|
||||
log.parse_checkpoint(note),
|
||||
["ana"],
|
||||
)
|
||||
|
||||
|
||||
def test_a_written_log_holds_every_tile_and_the_checkpoint(tmp_path):
|
||||
entries = entries_of(300)
|
||||
log.write_log(entries, ORIGIN, [("ana", secret("ana"))], str(tmp_path))
|
||||
written = {
|
||||
p.relative_to(tmp_path).as_posix() for p in tmp_path.rglob("*") if p.is_file()
|
||||
}
|
||||
assert written == set(log.tiles(entries)) | {"checkpoint"}
|
||||
|
||||
|
||||
def test_reading_a_log_with_a_malformed_checkpoint_is_refused(tmp_path):
|
||||
(tmp_path / "checkpoint").write_text("junk\n", encoding="utf-8")
|
||||
with pytest.raises(ValueError, match="checkpoint is malformed"):
|
||||
log.read_log(str(tmp_path))
|
||||
|
||||
|
||||
def test_reading_a_log_with_a_missing_tile_fails(tmp_path):
|
||||
log.write_log(entries_of(9), ORIGIN, [("ana", secret("ana"))], str(tmp_path))
|
||||
(tmp_path / "tile" / "entries" / "000.p" / "9").unlink()
|
||||
with pytest.raises(FileNotFoundError):
|
||||
log.read_log(str(tmp_path))
|
||||
@@ -0,0 +1,399 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""ML-DSA-65 of FIPS 204 in plain Python (paperspec.mldsa)."""
|
||||
|
||||
import pytest
|
||||
from hypothesis import given, settings, strategies as st
|
||||
|
||||
from conftest import vector_cases
|
||||
from helpers_lib2 import PROPS
|
||||
from paperspec import mldsa
|
||||
|
||||
CASES = {c["id"]: c for c in vector_cases("signatures") if c["id"].startswith("ml-dsa")}
|
||||
SEED = bytes.fromhex(CASES["ml-dsa-65-sign-empty"]["seed"])
|
||||
POLY = st.lists(st.integers(0, mldsa.Q - 1), min_size=mldsa.N, max_size=mldsa.N)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def keys():
|
||||
return mldsa.keygen(SEED)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def signature(keys):
|
||||
return mldsa.sign(keys[1], b"a message")
|
||||
|
||||
|
||||
def negacyclic_product(a, b):
|
||||
"""The product of two polynomials modulo x^256 + 1 and q, by the schoolbook method."""
|
||||
out = [0] * mldsa.N
|
||||
for i, x in enumerate(a):
|
||||
for j, y in enumerate(b):
|
||||
k = i + j
|
||||
if k < mldsa.N:
|
||||
out[k] += x * y
|
||||
else:
|
||||
out[k - mldsa.N] -= x * y
|
||||
return [c % mldsa.Q for c in out]
|
||||
|
||||
|
||||
# --- vectors and sizes ----------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", CASES.values(), ids=list(CASES))
|
||||
def test_key_generation_matches_the_vector(case):
|
||||
public, _ = mldsa.keygen(bytes.fromhex(case["seed"]))
|
||||
assert public.hex() == case["public"]
|
||||
assert mldsa.public_key(bytes.fromhex(case["seed"])) == public
|
||||
|
||||
|
||||
def test_signature_matches_the_vector_and_verifies(keys):
|
||||
case = CASES["ml-dsa-65-sign-empty"]
|
||||
public, secret = keys
|
||||
message = bytes.fromhex(case["message"])
|
||||
signature = mldsa.sign(secret, message)
|
||||
assert signature.hex() == case["signature"]
|
||||
assert mldsa.verify(public, message, signature)
|
||||
assert not mldsa.verify(public, b"x", signature)
|
||||
|
||||
|
||||
def test_key_and_signature_sizes_are_those_of_ml_dsa_65(keys, signature):
|
||||
public, secret = keys
|
||||
assert (len(public), len(secret), len(signature)) == (1952, 4032, 3309)
|
||||
assert (mldsa.PK_SIZE, mldsa.SK_SIZE, mldsa.SIG_SIZE) == (1952, 4032, 3309)
|
||||
|
||||
|
||||
def test_signing_is_deterministic(keys, signature):
|
||||
assert mldsa.sign(keys[1], b"a message") == signature
|
||||
|
||||
|
||||
def test_the_hedging_bytes_change_the_signature_but_not_its_validity(keys):
|
||||
public, secret = keys
|
||||
hedged = mldsa.sign(secret, b"a message", rnd=bytes([7]) * 32)
|
||||
assert hedged != mldsa.sign(secret, b"a message")
|
||||
assert mldsa.verify(public, b"a message", hedged)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("length", [0, 31, 33])
|
||||
def test_key_generation_needs_a_32_byte_seed(length):
|
||||
with pytest.raises(ValueError, match="32 bytes"):
|
||||
mldsa.keygen(bytes(length))
|
||||
|
||||
|
||||
# --- verification refusals --------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_signature_does_not_verify_under_another_key_or_message(keys, signature):
|
||||
public, _ = keys
|
||||
other, _ = mldsa.keygen(bytes(32))
|
||||
assert mldsa.verify(public, b"a message", signature)
|
||||
assert not mldsa.verify(other, b"a message", signature)
|
||||
assert not mldsa.verify(public, b"a message ", signature)
|
||||
assert not mldsa.verify(public, b"", signature)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"position",
|
||||
[0, 23, 24, 1000, 3000, 3200, 3309 - 56, 3309 - 7, 3308],
|
||||
ids=lambda p: f"byte-{p}",
|
||||
)
|
||||
def test_a_signature_with_a_changed_bit_does_not_verify(keys, signature, position):
|
||||
bad = bytearray(signature)
|
||||
bad[position] ^= 0x04
|
||||
assert not mldsa.verify(keys[0], b"a message", bytes(bad))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("length", [0, 3308, 3310])
|
||||
def test_a_signature_of_the_wrong_length_does_not_verify(keys, signature, length):
|
||||
padded = signature + bytes(8)
|
||||
assert not mldsa.verify(keys[0], b"a message", padded[:length])
|
||||
|
||||
|
||||
def test_a_key_of_the_wrong_length_does_not_verify(keys, signature):
|
||||
assert not mldsa.verify(keys[0][:-1], b"a message", signature)
|
||||
assert not mldsa.verify(keys[0] + b"\x00", b"a message", signature)
|
||||
|
||||
|
||||
def test_verification_never_raises_on_the_wrong_types(keys, signature):
|
||||
assert not mldsa.verify(keys[0], None, signature)
|
||||
assert not mldsa.verify(None, b"a message", signature)
|
||||
assert not mldsa.verify(keys[0], b"a message", None)
|
||||
|
||||
|
||||
def test_a_signature_with_a_response_out_of_bounds_does_not_verify(keys, signature):
|
||||
c_tilde, z, hint = mldsa.sig_decode(signature)
|
||||
z[0][0] = mldsa.GAMMA1 - 1
|
||||
forged = mldsa.sig_encode(c_tilde, z, hint)
|
||||
assert len(forged) == mldsa.SIG_SIZE
|
||||
assert not mldsa.verify(keys[0], b"a message", forged)
|
||||
|
||||
|
||||
def test_a_signature_with_a_malformed_hint_does_not_verify(keys, signature):
|
||||
tail = mldsa.SIG_SIZE - mldsa.OMEGA - mldsa.K
|
||||
bad = signature[:tail] + bytes([0xFF] * mldsa.OMEGA) + bytes(mldsa.K)
|
||||
assert not mldsa.verify(keys[0], b"a message", bad)
|
||||
|
||||
|
||||
# --- contexts -----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_context_is_part_of_what_is_signed(keys):
|
||||
public, secret = keys
|
||||
signature = mldsa.sign(secret, b"a message", ctx=b"paper")
|
||||
assert mldsa.verify(public, b"a message", signature, ctx=b"paper")
|
||||
assert not mldsa.verify(public, b"a message", signature)
|
||||
assert not mldsa.verify(public, b"a message", signature, ctx=b"papers")
|
||||
|
||||
|
||||
def test_a_context_of_256_bytes_is_refused(keys):
|
||||
public, secret = keys
|
||||
longest = bytes(255)
|
||||
signature = mldsa.sign(secret, b"m", ctx=longest)
|
||||
assert mldsa.verify(public, b"m", signature, ctx=longest)
|
||||
with pytest.raises(ValueError, match="at most 255 bytes"):
|
||||
mldsa.sign(secret, b"m", ctx=bytes(256))
|
||||
assert not mldsa.verify(public, b"m", signature, ctx=bytes(256))
|
||||
|
||||
|
||||
@given(st.binary(max_size=80))
|
||||
@settings(PROPS, max_examples=6)
|
||||
def test_a_signature_of_any_message_verifies_and_only_for_it(keys, message):
|
||||
public, secret = keys
|
||||
signature = mldsa.sign(secret, message)
|
||||
assert mldsa.verify(public, message, signature)
|
||||
assert not mldsa.verify(public, message + b"\x00", signature)
|
||||
|
||||
|
||||
@given(st.binary(min_size=32, max_size=32))
|
||||
@settings(PROPS, max_examples=4)
|
||||
def test_any_seed_gives_a_key_that_verifies_its_own_signature(seed):
|
||||
public, secret = mldsa.keygen(seed)
|
||||
assert (len(public), len(secret)) == (mldsa.PK_SIZE, mldsa.SK_SIZE)
|
||||
assert mldsa.verify(public, b"m", mldsa.sign(secret, b"m"))
|
||||
|
||||
|
||||
# --- encodings ----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_key_encodings_round_trip(keys):
|
||||
public, secret = keys
|
||||
rho, t1 = mldsa.pk_decode(public)
|
||||
assert mldsa.pk_encode(rho, t1) == public
|
||||
assert len(t1) == mldsa.K
|
||||
assert all(0 <= c < 1 << 10 for t in t1 for c in t)
|
||||
assert mldsa.sk_encode(*mldsa.sk_decode(secret)) == secret
|
||||
|
||||
|
||||
def test_signature_encoding_round_trips(signature):
|
||||
c_tilde, z, hint = mldsa.sig_decode(signature)
|
||||
assert len(c_tilde) == mldsa.LAMBDA // 4
|
||||
assert len(z) == mldsa.L
|
||||
assert sum(map(sum, hint)) <= mldsa.OMEGA
|
||||
assert mldsa.sig_encode(c_tilde, z, hint) == signature
|
||||
|
||||
|
||||
@given(POLY, st.sampled_from([1, 255, 1023, (mldsa.Q - 1) // 32 - 1]))
|
||||
@PROPS
|
||||
def test_simple_bit_packing_round_trips(poly, bound):
|
||||
clipped = [c % (bound + 1) for c in poly]
|
||||
assert (
|
||||
mldsa.simple_bit_unpack(mldsa.simple_bit_pack(clipped, bound), bound) == clipped
|
||||
)
|
||||
|
||||
|
||||
@given(st.lists(st.integers(-4, 4), min_size=256, max_size=256))
|
||||
@PROPS
|
||||
def test_bit_packing_of_small_coefficients_round_trips(poly):
|
||||
assert mldsa.bit_unpack(mldsa.bit_pack(poly, 4, 4), 4, 4) == poly
|
||||
|
||||
|
||||
@given(st.lists(st.integers(-(2**19) + 1, 2**19), min_size=256, max_size=256))
|
||||
@PROPS
|
||||
def test_bit_packing_of_masks_round_trips(poly):
|
||||
packed = mldsa.bit_pack(poly, mldsa.GAMMA1 - 1, mldsa.GAMMA1)
|
||||
assert len(packed) == 32 * 20
|
||||
assert mldsa.bit_unpack(packed, mldsa.GAMMA1 - 1, mldsa.GAMMA1) == poly
|
||||
|
||||
|
||||
@given(
|
||||
st.sets(st.tuples(st.integers(0, mldsa.K - 1), st.integers(0, 255)), max_size=55)
|
||||
)
|
||||
@PROPS
|
||||
def test_hints_round_trip(positions):
|
||||
hint = [[0] * mldsa.N for _ in range(mldsa.K)]
|
||||
for row, col in positions:
|
||||
hint[row][col] = 1
|
||||
packed = mldsa.hint_bit_pack(hint)
|
||||
assert len(packed) == mldsa.OMEGA + mldsa.K
|
||||
assert mldsa.hint_bit_unpack(packed) == hint
|
||||
|
||||
|
||||
def hint_bytes(counts, indexes=()):
|
||||
"""A packed hint with the given cumulative counts per row and leading index bytes."""
|
||||
packed = bytearray(mldsa.OMEGA + mldsa.K)
|
||||
packed[: len(indexes)] = bytes(indexes)
|
||||
packed[mldsa.OMEGA :] = bytes(counts)
|
||||
return bytes(packed)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"packed",
|
||||
[
|
||||
hint_bytes([2, 2, 2, 2, 2, 2], [9, 3]),
|
||||
hint_bytes([2, 2, 2, 2, 2, 2], [4, 4]),
|
||||
hint_bytes([2, 1, 1, 1, 1, 1], [3, 4]),
|
||||
hint_bytes([mldsa.OMEGA + 1] * mldsa.K),
|
||||
hint_bytes([0, 0, 0, 0, 0, 0], [5]),
|
||||
],
|
||||
ids=[
|
||||
"indexes-descend",
|
||||
"index-repeats",
|
||||
"counts-shrink",
|
||||
"count-above-omega",
|
||||
"unused-byte-set",
|
||||
],
|
||||
)
|
||||
def test_malformed_hints_are_refused(packed):
|
||||
assert mldsa.hint_bit_unpack(packed) is None
|
||||
|
||||
|
||||
# --- arithmetic ---------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given(POLY)
|
||||
@PROPS
|
||||
def test_the_number_theoretic_transform_inverts(poly):
|
||||
assert mldsa.ntt_inv(mldsa.ntt(poly)) == poly
|
||||
|
||||
|
||||
@given(POLY, POLY)
|
||||
@settings(PROPS, max_examples=6)
|
||||
def test_the_transform_turns_products_into_negacyclic_convolutions(a, b):
|
||||
product = mldsa.ntt_inv(
|
||||
[x * y % mldsa.Q for x, y in zip(mldsa.ntt(a), mldsa.ntt(b), strict=True)]
|
||||
)
|
||||
assert product == negacyclic_product(a, b)
|
||||
|
||||
|
||||
@given(st.integers(0, mldsa.Q - 1))
|
||||
@PROPS
|
||||
def test_power2round_splits_a_residue_into_high_and_low_parts(r):
|
||||
high, low = mldsa.power2round(r)
|
||||
assert high * 2**mldsa.D + low == r
|
||||
assert -(2 ** (mldsa.D - 1)) < low <= 2 ** (mldsa.D - 1)
|
||||
|
||||
|
||||
@given(st.integers(0, mldsa.Q - 1))
|
||||
@PROPS
|
||||
def test_decompose_splits_a_residue_around_multiples_of_two_gamma2(r):
|
||||
high, low = mldsa.decompose(r)
|
||||
step = 2 * mldsa.GAMMA2
|
||||
assert (high * step + low) % mldsa.Q == r
|
||||
assert -mldsa.GAMMA2 <= low <= mldsa.GAMMA2
|
||||
assert 0 <= high < (mldsa.Q - 1) // step
|
||||
assert (mldsa.high_bits(r), mldsa.low_bits(r)) == (high, low)
|
||||
|
||||
|
||||
def test_decompose_folds_the_top_residue_into_the_lowest_high_part():
|
||||
assert mldsa.decompose(mldsa.Q - 1) == (0, -1)
|
||||
|
||||
|
||||
@given(st.integers(-mldsa.GAMMA2, mldsa.GAMMA2), st.integers(0, mldsa.Q - 1))
|
||||
@PROPS
|
||||
def test_a_hint_recovers_the_high_bits_of_a_sum(z, r):
|
||||
hint = mldsa.make_hint(z, r)
|
||||
assert hint in (0, 1)
|
||||
assert mldsa.use_hint(hint, r) == mldsa.high_bits(r + z)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("hint", "r", "expected"),
|
||||
[(0, 10, 0), (1, 10, 1), (1, mldsa.GAMMA2 + 1, 0), (1, 0, 15)],
|
||||
ids=["no-hint", "positive-low-part-up", "negative-low-part-down", "zero-wraps"],
|
||||
)
|
||||
def test_use_hint_moves_the_high_part_toward_the_low_part(hint, r, expected):
|
||||
assert mldsa.use_hint(hint, r) == expected
|
||||
|
||||
|
||||
# --- sampling -----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given(st.binary(min_size=34, max_size=34))
|
||||
@settings(PROPS, max_examples=10)
|
||||
def test_rejection_sampling_gives_reduced_coefficients(rho):
|
||||
poly = mldsa.rej_ntt_poly(rho)
|
||||
assert len(poly) == mldsa.N
|
||||
assert all(0 <= c < mldsa.Q for c in poly)
|
||||
assert poly == mldsa.rej_ntt_poly(rho)
|
||||
|
||||
|
||||
@given(st.binary(min_size=66, max_size=66))
|
||||
@settings(PROPS, max_examples=10)
|
||||
def test_bounded_sampling_gives_small_coefficients(rho):
|
||||
poly = mldsa.rej_bounded_poly(rho)
|
||||
assert len(poly) == mldsa.N
|
||||
assert all(-mldsa.ETA <= c <= mldsa.ETA for c in poly)
|
||||
|
||||
|
||||
@given(st.binary(min_size=48, max_size=48))
|
||||
@settings(PROPS, max_examples=10)
|
||||
def test_sample_in_ball_has_exactly_tau_signed_ones(rho):
|
||||
ball = mldsa.sample_in_ball(rho)
|
||||
assert len(ball) == mldsa.N
|
||||
assert sorted(c for c in ball if c) == sorted(
|
||||
c for c in ball if c in (1, mldsa.Q - 1)
|
||||
)
|
||||
assert sum(1 for c in ball if c) == mldsa.TAU
|
||||
|
||||
|
||||
def test_the_matrix_and_secret_vectors_have_the_shapes_of_ml_dsa_65():
|
||||
matrix = mldsa.expand_a(bytes(32))
|
||||
s1, s2 = mldsa.expand_s(bytes(64))
|
||||
assert (len(matrix), len(matrix[0])) == (mldsa.K, mldsa.L)
|
||||
assert (len(s1), len(s2)) == (mldsa.L, mldsa.K)
|
||||
assert len(mldsa.expand_mask(bytes(64), 0)) == mldsa.L
|
||||
assert mldsa.expand_mask(bytes(64), 0) != mldsa.expand_mask(bytes(64), mldsa.L)
|
||||
|
||||
|
||||
def test_ntt_sampling_asks_for_more_bytes_when_the_stream_runs_out(monkeypatch):
|
||||
def stream(data, n):
|
||||
return (
|
||||
b"\xff" * n
|
||||
if n <= 900
|
||||
else b"\xff" * 900 + b"\x05\x00\x00" * ((n - 900) // 3)
|
||||
)
|
||||
|
||||
monkeypatch.setattr(mldsa, "_shake128", stream)
|
||||
assert mldsa.rej_ntt_poly(bytes(34)) == [5] * mldsa.N
|
||||
|
||||
|
||||
def test_bounded_sampling_asks_for_more_bytes_when_the_stream_runs_out(monkeypatch):
|
||||
def stream(data, n):
|
||||
return b"\xff" * n if n <= 200 else b"\xff" * 200 + bytes(n - 200)
|
||||
|
||||
monkeypatch.setattr(mldsa, "_shake256", stream)
|
||||
assert mldsa.rej_bounded_poly(bytes(66)) == [mldsa.ETA] * mldsa.N
|
||||
|
||||
|
||||
def test_bounded_sampling_stops_at_the_last_coefficient_mid_byte(monkeypatch):
|
||||
def stream(data, n):
|
||||
return b"\x0f" + bytes(n - 1)
|
||||
|
||||
monkeypatch.setattr(mldsa, "_shake256", stream)
|
||||
assert mldsa.rej_bounded_poly(bytes(66)) == [mldsa.ETA] * mldsa.N
|
||||
|
||||
|
||||
def test_ball_sampling_asks_for_more_bytes_when_the_stream_runs_out(monkeypatch):
|
||||
first = 8 + mldsa.TAU * 4
|
||||
|
||||
def stream(data, n):
|
||||
return bytes(8) + b"\xff" * (first - 8) + bytes(n - first)
|
||||
|
||||
monkeypatch.setattr(mldsa, "_shake256", stream)
|
||||
ball = mldsa.sample_in_ball(bytes(48))
|
||||
assert sum(1 for c in ball if c) == mldsa.TAU
|
||||
assert set(ball) == {0, 1}
|
||||
@@ -0,0 +1,251 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""The node kinds, their rules and the document structure that walks the tree (spec/03)."""
|
||||
|
||||
import pytest
|
||||
|
||||
from helpers_lib1 import (
|
||||
HALE,
|
||||
build_doc,
|
||||
document,
|
||||
heading,
|
||||
ids_of_kind,
|
||||
math_run,
|
||||
package_state,
|
||||
paragraph,
|
||||
section,
|
||||
sentence,
|
||||
text_run,
|
||||
)
|
||||
from paperspec import model
|
||||
from paperspec.model import Doc
|
||||
|
||||
|
||||
class TestKindTables:
|
||||
def test_every_kind_has_a_row_in_the_child_table(self):
|
||||
assert set(model.CHILDREN) == model.KINDS
|
||||
assert "document" in model.KINDS
|
||||
|
||||
def test_every_allowed_child_is_a_known_kind(self):
|
||||
for kind, kids in model.CHILDREN.items():
|
||||
assert kids <= model.KINDS, kind
|
||||
|
||||
def test_every_kind_but_the_document_can_be_somebodys_child(self):
|
||||
children = set().union(*model.CHILDREN.values())
|
||||
assert model.KINDS - children == {"document"}
|
||||
|
||||
def test_leaves_and_containers_with_content_are_kinds_and_do_not_overlap(self):
|
||||
assert model.LEAVES <= model.KINDS
|
||||
assert model.CONTAINERS_WITH_CONTENT <= model.KINDS
|
||||
assert not (model.LEAVES & model.CONTAINERS_WITH_CONTENT)
|
||||
assert {"document", "paragraph", "row", "list"} <= (
|
||||
model.KINDS - model.LEAVES - model.CONTAINERS_WITH_CONTENT
|
||||
)
|
||||
|
||||
def test_the_leaf_kinds_are_runs_equations_and_verbatim_text(self):
|
||||
assert model.LEAVES == model.RUN_KINDS | model.MATH_KINDS | model.VERBATIM_KINDS
|
||||
assert not (model.RUN_KINDS & model.MATH_KINDS)
|
||||
assert not (model.RUN_KINDS & model.VERBATIM_KINDS)
|
||||
assert {"equation", "equation-line"} == model.MATH_KINDS
|
||||
assert {"code", "algorithm"} == model.VERBATIM_KINDS
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"table",
|
||||
[
|
||||
model.NONEMPTY,
|
||||
model.AT_MOST_ONCE,
|
||||
model.LABELLED,
|
||||
model.CONTAINERS_WITH_CONTENT,
|
||||
set(model.ATTRS),
|
||||
],
|
||||
)
|
||||
def test_every_rule_table_names_known_kinds(self, table):
|
||||
assert set(table) <= model.KINDS
|
||||
|
||||
def test_a_kind_that_must_have_children_may_have_some(self):
|
||||
for kind in model.NONEMPTY:
|
||||
assert model.CHILDREN[kind], kind
|
||||
|
||||
def test_levels_run_from_f0_to_f3(self):
|
||||
assert model.LEVELS == ["F0", "F1", "F2", "F3"]
|
||||
|
||||
def test_run_kinds_and_styles(self):
|
||||
assert {
|
||||
"text",
|
||||
"math",
|
||||
"cite",
|
||||
"xref",
|
||||
"fnmark",
|
||||
"link",
|
||||
"equation",
|
||||
} == model.RUNS
|
||||
assert {"em", "strong", "sup", "sub", "sc", "mono"} == model.STYLES
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kind", "cls"),
|
||||
[
|
||||
("sentence", "leaf"),
|
||||
("cell", "leaf"),
|
||||
("equation", "leaf"),
|
||||
("equation-line", "leaf"),
|
||||
("code", "leaf"),
|
||||
("reference", "leaf"),
|
||||
("section", "container-with-content"),
|
||||
("table", "container-with-content"),
|
||||
("figure", "container-with-content"),
|
||||
("footnote", "container-with-content"),
|
||||
("paragraph", "container-without-content"),
|
||||
("document", "container-without-content"),
|
||||
("row", "container-without-content"),
|
||||
("list", "container-without-content"),
|
||||
],
|
||||
)
|
||||
def test_anchor_class(self, kind, cls):
|
||||
assert model.anchor_class(kind) == cls
|
||||
|
||||
def test_every_kind_has_an_anchor_class(self):
|
||||
classes = {model.anchor_class(k) for k in model.KINDS}
|
||||
assert classes == {
|
||||
"leaf",
|
||||
"container-with-content",
|
||||
"container-without-content",
|
||||
}
|
||||
|
||||
|
||||
def _nodes():
|
||||
"""document > section > [heading > sentence, paragraph > sentence, equation > line]."""
|
||||
return [
|
||||
{"id": "ddddddd", "kind": "document", "parent": None, "ord": 0},
|
||||
{"id": "sssssss", "kind": "section", "parent": "ddddddd", "ord": 0},
|
||||
{"id": "ppppppp", "kind": "paragraph", "parent": "sssssss", "ord": 1},
|
||||
{"id": "hhhhhhh", "kind": "heading", "parent": "sssssss", "ord": 0},
|
||||
{"id": "1111111", "kind": "sentence", "parent": "hhhhhhh", "ord": 0},
|
||||
{"id": "2222222", "kind": "sentence", "parent": "ppppppp", "ord": 0},
|
||||
{"id": "3333333", "kind": "equation", "parent": "2222222", "ord": 0},
|
||||
{"id": "eeeeeee", "kind": "equation", "parent": "sssssss", "ord": 2},
|
||||
{"id": "lllllll", "kind": "equation-line", "parent": "eeeeeee", "ord": 0},
|
||||
{"id": "ccccccc", "kind": "code", "parent": "sssssss", "ord": 3},
|
||||
]
|
||||
|
||||
|
||||
class TestDoc:
|
||||
def test_children_come_in_ord_order_not_listing_order(self):
|
||||
doc = Doc("k", _nodes())
|
||||
assert doc.kids("sssssss") == ["hhhhhhh", "ppppppp", "eeeeeee", "ccccccc"]
|
||||
assert doc.order[2] == "ppppppp"
|
||||
|
||||
def test_the_root_is_the_first_node_without_a_parent(self):
|
||||
doc = Doc("k", _nodes())
|
||||
assert doc.root == "ddddddd"
|
||||
assert doc.key == "k"
|
||||
|
||||
def test_an_empty_document_has_no_root_and_no_preorder(self):
|
||||
doc = Doc("k", [])
|
||||
assert doc.root is None
|
||||
assert doc.order == []
|
||||
assert doc.preorder() == []
|
||||
assert doc.kids("anything") == []
|
||||
|
||||
def test_a_leaf_has_no_kids_and_an_unknown_id_has_none(self):
|
||||
doc = Doc("k", _nodes())
|
||||
assert doc.kids("1111111") == []
|
||||
assert doc.kids("zzzzzzz") == []
|
||||
|
||||
def test_preorder_visits_a_node_before_its_children_in_order(self):
|
||||
doc = Doc("k", _nodes())
|
||||
assert doc.preorder() == [
|
||||
"ddddddd",
|
||||
"sssssss",
|
||||
"hhhhhhh",
|
||||
"1111111",
|
||||
"ppppppp",
|
||||
"2222222",
|
||||
"3333333",
|
||||
"eeeeeee",
|
||||
"lllllll",
|
||||
"ccccccc",
|
||||
]
|
||||
|
||||
def test_preorder_of_a_subtree(self):
|
||||
doc = Doc("k", _nodes())
|
||||
assert doc.preorder("ppppppp") == ["ppppppp", "2222222", "3333333"]
|
||||
assert doc.preorder("1111111") == ["1111111"]
|
||||
|
||||
def test_ancestors_run_from_the_root_down_to_the_parent(self):
|
||||
doc = Doc("k", _nodes())
|
||||
assert doc.ancestors("3333333") == ["ddddddd", "sssssss", "ppppppp", "2222222"]
|
||||
assert doc.ancestors("ddddddd") == []
|
||||
|
||||
def test_text_leaves_are_the_nodes_whose_text_a_container_is_made_of(self):
|
||||
doc = Doc("k", _nodes())
|
||||
assert doc.text_leaves("ddddddd") == [
|
||||
"1111111",
|
||||
"2222222",
|
||||
"eeeeeee",
|
||||
"ccccccc",
|
||||
]
|
||||
assert doc.text_leaves("ppppppp") == ["2222222"]
|
||||
|
||||
def test_an_equation_inside_a_sentence_is_part_of_the_sentence(self):
|
||||
doc = Doc("k", _nodes())
|
||||
assert "3333333" not in doc.text_leaves("ddddddd")
|
||||
assert doc.text_leaves("3333333") == []
|
||||
|
||||
def test_an_equation_line_is_never_a_text_leaf(self):
|
||||
doc = Doc("k", _nodes())
|
||||
assert doc.text_leaves("lllllll") == []
|
||||
assert "lllllll" not in doc.text_leaves("eeeeeee")
|
||||
assert doc.text_leaves("eeeeeee") == ["eeeeeee"]
|
||||
|
||||
def test_a_leaf_is_its_own_text_leaf(self):
|
||||
doc = Doc("k", _nodes())
|
||||
assert doc.text_leaves("ccccccc") == ["ccccccc"]
|
||||
assert doc.text_leaves("1111111") == ["1111111"]
|
||||
|
||||
def test_leaves_include_equations_in_sentences_and_equation_lines(self):
|
||||
doc = Doc("k", _nodes())
|
||||
assert doc.leaves("ddddddd") == [
|
||||
"1111111",
|
||||
"2222222",
|
||||
"3333333",
|
||||
"eeeeeee",
|
||||
"lllllll",
|
||||
"ccccccc",
|
||||
]
|
||||
assert doc.leaves("ppppppp") == ["2222222", "3333333"]
|
||||
assert doc.leaves("hhhhhhh") == ["1111111"]
|
||||
assert doc.leaves("1111111") == ["1111111"]
|
||||
|
||||
def test_the_fixture_documents_are_walked_consistently(self):
|
||||
doc = package_state(HALE).doc()
|
||||
order = doc.preorder()
|
||||
assert order == doc.order
|
||||
assert set(order) == set(doc.nodes)
|
||||
for nid in doc.text_leaves(doc.root):
|
||||
assert doc.nodes[nid]["kind"] in model.LEAVES
|
||||
assert set(doc.text_leaves(doc.root)) <= set(doc.leaves(doc.root))
|
||||
|
||||
def test_a_built_document_keeps_its_structure(self):
|
||||
doc = build_doc(
|
||||
document(
|
||||
section(
|
||||
heading("H"),
|
||||
paragraph(
|
||||
sentence(text_run("a")), sentence(text_run("b"), math_run("x"))
|
||||
),
|
||||
)
|
||||
)
|
||||
)
|
||||
sentences = ids_of_kind(doc, "sentence")
|
||||
assert len(sentences) == 3
|
||||
para = ids_of_kind(doc, "paragraph")[0]
|
||||
assert doc.kids(para) == sentences[1:]
|
||||
assert doc.ancestors(sentences[1]) == [
|
||||
doc.root,
|
||||
ids_of_kind(doc, "section")[0],
|
||||
para,
|
||||
]
|
||||
@@ -0,0 +1,455 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Simple notation and its canonical linear form (spec/05 §5.4)."""
|
||||
|
||||
import pytest
|
||||
from hypothesis import given, settings, strategies as st
|
||||
|
||||
from helpers_lib1 import vectors
|
||||
from paperspec import notation as nt
|
||||
|
||||
PROPERTIES = settings(derandomize=True, max_examples=300, database=None, deadline=None)
|
||||
|
||||
MINUS = "\N{MINUS SIGN}"
|
||||
PRIME = "\N{PRIME}"
|
||||
DOT = "⋅"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", vectors("notation"))
|
||||
def test_notation_vector(case):
|
||||
assert nt.fold_math(case["notation"], case["source"]) == case["canonical"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("source", "canonical"),
|
||||
[
|
||||
("", ""),
|
||||
(" ", ""),
|
||||
("a b", "ab"),
|
||||
("a~b", "ab"),
|
||||
("a\\,b\\;c\\:d\\!e\\ f", "abcdef"),
|
||||
("a\\quad b\\qquad c", "abc"),
|
||||
("\\rm x", "x"),
|
||||
("\\mathrm x", "x"),
|
||||
("\\text{ab}", "ab"),
|
||||
("\\textrm{a}_1", "a_{1}"),
|
||||
("\\mathup{c}", "c"),
|
||||
("x'", "x" + PRIME),
|
||||
("x''", "x" + PRIME * 2),
|
||||
("x^{\\prime\\prime}", "x" + PRIME * 2),
|
||||
("x^{\\prime}_1", "x" + PRIME + "_{1}"),
|
||||
("x'_1", "x" + PRIME + "_{1}"),
|
||||
("'", PRIME),
|
||||
("x_{i}^{2}", "x_{i}^{2}"),
|
||||
("x^2_i", "x_{i}^{2}"),
|
||||
("{a}_2", "a_{2}"),
|
||||
("{a}{b}", "ab"),
|
||||
("{}", ""),
|
||||
("x{}_2", "x_{2}"),
|
||||
("\\alpha\\beta", "αβ"),
|
||||
("\\varepsilon\\epsilon", "εϵ"),
|
||||
("\\Omega", "Ω"),
|
||||
("\\AA", "Å"),
|
||||
("a \\le b \\geq c", "a≤b≥c"),
|
||||
("a\\lt b\\gt c", "a<b>c"),
|
||||
("a \\cdot b \\times c \\pm d", "a" + DOT + "b\N{MULTIPLICATION SIGN}c±d"),
|
||||
("a-b\N{EN DASH}c", "a" + MINUS + "b" + MINUS + "c"),
|
||||
("a·b∙c", "a" + DOT + "b" + DOT + "c"),
|
||||
("\\left(x\\right)", "(x)"),
|
||||
("(a,b)/", "(a,b)/"),
|
||||
("3.14", "3.14"),
|
||||
("٣", "٣"),
|
||||
("\N{MICRO SIGN}", "\N{MICRO SIGN}"),
|
||||
("é", "é"),
|
||||
("x_{{a}}", "x_{a}"),
|
||||
],
|
||||
)
|
||||
def test_tex_folds_to_the_canonical_linear_form(source, canonical):
|
||||
assert nt.fold_math("tex", source) == canonical
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"source",
|
||||
[
|
||||
"a\\",
|
||||
"a}",
|
||||
"{a",
|
||||
"x_{a",
|
||||
"\\{",
|
||||
"\\}",
|
||||
"\\é",
|
||||
"\\foo",
|
||||
"\\frac12",
|
||||
"\\left[x\\right]",
|
||||
"\\left(x\\right]",
|
||||
"\\right(",
|
||||
"\\left",
|
||||
"x^",
|
||||
"x_",
|
||||
"x^}",
|
||||
"x^^2",
|
||||
"x_^2",
|
||||
"x^'",
|
||||
"\\mathrm",
|
||||
"\\text",
|
||||
"x_1_2",
|
||||
"x^1^2",
|
||||
"{ab}_2",
|
||||
"{ab}^2",
|
||||
"∫",
|
||||
"a=b#",
|
||||
"$",
|
||||
"&",
|
||||
"\N{CYRILLIC SMALL LETTER A}",
|
||||
"\N{FULLWIDTH LATIN CAPITAL LETTER A}",
|
||||
"ʰ",
|
||||
],
|
||||
)
|
||||
def test_tex_outside_the_simple_forms_is_complex(source):
|
||||
assert nt.fold_math("tex", source) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("source", "canonical"),
|
||||
[
|
||||
("", ""),
|
||||
("a b", "ab"),
|
||||
("x_1", "x_{1}"),
|
||||
("x_(12)", "x_{12}"),
|
||||
("x^12", "x^{12}"),
|
||||
("x^(a+b)", "x^{a+b}"),
|
||||
("x'", "x" + PRIME),
|
||||
("x''_1", "x" + PRIME * 2 + "_{1}"),
|
||||
("alpha beta", "αβ"),
|
||||
("Omega", "Ω"),
|
||||
("epsilon epsilon.alt", "εϵ"),
|
||||
("theta.alt sigma.alt", "ϑς"),
|
||||
("a plus b minus c", "a+b" + MINUS + "c"),
|
||||
("a times b", "a\N{MULTIPLICATION SIGN}b"),
|
||||
("a dot.op b dot.c c", "a" + DOT + "b" + DOT + "c"),
|
||||
("a plus.minus b", "a±b"),
|
||||
("a lt b gt c eq d", "a<b>c=d"),
|
||||
("a lt.eq b gt.eq c", "a≤b≥c"),
|
||||
("a <= b >= c", "a≤b≥c"),
|
||||
("a prime", "a" + PRIME),
|
||||
("a slash b", "a/b"),
|
||||
("a thin b med c thick d quad e wide f space g", "abcdefg"),
|
||||
('"cm"^(-1)', "cm^{" + MINUS + "1}"),
|
||||
('"a b"', "ab"),
|
||||
("upright(H)_2", "H_{2}"),
|
||||
("1.5", "1.5"),
|
||||
("x1.5", "x1.5"),
|
||||
("12", "12"),
|
||||
("(a)", "(a)"),
|
||||
("((a))", "((a))"),
|
||||
("a)", "a)"),
|
||||
("a-b", "a" + MINUS + "b"),
|
||||
("\N{MICRO SIGN}", "\N{MICRO SIGN}"),
|
||||
],
|
||||
)
|
||||
def test_typst_folds_to_the_canonical_linear_form(source, canonical):
|
||||
assert nt.fold_math("typst", source) == canonical
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"source",
|
||||
[
|
||||
"(a",
|
||||
"((a)",
|
||||
"x^",
|
||||
"x_",
|
||||
"x^)",
|
||||
"x^_2",
|
||||
"x^'",
|
||||
"x_1_2",
|
||||
"x^1^2",
|
||||
"a / b",
|
||||
"a * b",
|
||||
"a # b",
|
||||
"a & b",
|
||||
"a [b]",
|
||||
"a {b}",
|
||||
"a \\ b",
|
||||
"a $ b",
|
||||
"a ! b",
|
||||
"a : b",
|
||||
"a ; b",
|
||||
"a | b",
|
||||
"a ~ b",
|
||||
"a @ b",
|
||||
"a -> b",
|
||||
"a => b",
|
||||
"a != b",
|
||||
"a <- b",
|
||||
"a == b",
|
||||
"a := b",
|
||||
'"unclosed',
|
||||
'"a\\"b"',
|
||||
"foo",
|
||||
"cos x",
|
||||
"a.b",
|
||||
"upright",
|
||||
"upright x",
|
||||
"frac(a, b)",
|
||||
'"∫"',
|
||||
"∫",
|
||||
],
|
||||
)
|
||||
def test_typst_outside_the_simple_forms_is_complex(source):
|
||||
assert nt.fold_math("typst", source) is None
|
||||
|
||||
|
||||
def test_a_notation_other_than_tex_is_read_as_typst():
|
||||
assert nt.fold_math("typst", "x_1") == nt.fold_math("other", "x_1") == "x_{1}"
|
||||
assert nt.fold_math("other", "\\frac{a}{b}") is None
|
||||
|
||||
|
||||
def test_the_source_is_composed_before_it_is_read():
|
||||
assert nt.fold_math("tex", "é") == nt.fold_math("tex", "é") == "é"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("notation", "source", "key"),
|
||||
[
|
||||
("tex", "\\frac{a}{b}", "tex:\\frac{a}{b}"),
|
||||
("tex", "\\frac { a } { b }", "tex:\\frac{a}{b}"),
|
||||
("typst", "a / b", "typst:a/b"),
|
||||
("tex", "é \n x", "tex:éx"),
|
||||
("typst", "", "typst:"),
|
||||
],
|
||||
)
|
||||
def test_complex_key_is_the_notation_and_the_source_without_white_space(
|
||||
notation, source, key
|
||||
):
|
||||
assert nt.complex_key(notation, source) == key
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("ch", "expected"),
|
||||
[
|
||||
("a", True),
|
||||
("Z", True),
|
||||
("é", True),
|
||||
("ɐ", False),
|
||||
("\N{GREEK CAPITAL LETTER ALPHA}", True),
|
||||
("ω", True),
|
||||
("Ϣ", False),
|
||||
("ϵ", True),
|
||||
("ẞ", True),
|
||||
("ἀ", True),
|
||||
("\N{MICRO SIGN}", True),
|
||||
("\N{CYRILLIC SMALL LETTER A}", False),
|
||||
("中", False),
|
||||
("1", False),
|
||||
("+", False),
|
||||
("ª", True),
|
||||
("ʰ", False),
|
||||
],
|
||||
)
|
||||
def test_is_letter_is_latin_or_greek_letters_and_the_micro_sign(ch, expected):
|
||||
assert nt.is_letter(ch) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("ch", "expected"),
|
||||
[
|
||||
("x", True),
|
||||
("7", True),
|
||||
("٣", True),
|
||||
("-", True),
|
||||
("\N{EN DASH}", True),
|
||||
("'", True),
|
||||
("+", True),
|
||||
("(", True),
|
||||
(".", True),
|
||||
("\N{MULTIPLICATION SIGN}", True),
|
||||
("±", True),
|
||||
("*", False),
|
||||
("{", False),
|
||||
("\\", False),
|
||||
("∫", False),
|
||||
("²", False),
|
||||
(" ", False),
|
||||
],
|
||||
)
|
||||
def test_is_simple_char(ch, expected):
|
||||
assert nt.is_simple_char(ch) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("ch", "folded"),
|
||||
[
|
||||
("-", MINUS),
|
||||
("\N{EN DASH}", MINUS),
|
||||
("\N{MINUS SIGN}", MINUS),
|
||||
("·", DOT),
|
||||
("∙", DOT),
|
||||
("⋅", DOT),
|
||||
("'", PRIME),
|
||||
(PRIME, PRIME),
|
||||
("a", "a"),
|
||||
("—", "—"),
|
||||
],
|
||||
)
|
||||
def test_fold_char(ch, folded):
|
||||
assert nt.fold_char(ch) == folded
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("ch", "expected"),
|
||||
[
|
||||
("²", ("^", "2")),
|
||||
("³", ("^", "3")),
|
||||
("⁴", ("^", "4")),
|
||||
("⁺", ("^", "+")),
|
||||
("₂", ("_", "2")),
|
||||
("ₙ", ("_", "n")),
|
||||
("℠", ("^", "SM")),
|
||||
("2", (None, None)),
|
||||
("a", (None, None)),
|
||||
("½", (None, None)),
|
||||
],
|
||||
)
|
||||
def test_script_type(ch, expected):
|
||||
assert nt.script_type(ch) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "folded"),
|
||||
[
|
||||
("10³ per s", "10^{3} per s"),
|
||||
("H₂O", "H_{2}O"),
|
||||
("cm⁻¹", "cm^{\N{MINUS SIGN}1}"),
|
||||
("x²⁺", "x^{2+}"),
|
||||
("x²₂", "x^{2}_{2}"),
|
||||
("ᴬᴮ", "^{AB}"),
|
||||
("aʴ", "aʴ"),
|
||||
("ʴ²", "ʴ²"),
|
||||
("ʰ", "^{h}"),
|
||||
("10-3", "10" + MINUS + "3"),
|
||||
("10\N{EN DASH}3", "10" + MINUS + "3"),
|
||||
("a-b", "a-b"),
|
||||
("a - 3", "a - 3"),
|
||||
("1-", "1-"),
|
||||
("-", "-"),
|
||||
("é", "é"),
|
||||
("plain text", "plain text"),
|
||||
("", ""),
|
||||
],
|
||||
)
|
||||
def test_fold_plain_text(text, folded):
|
||||
assert nt.fold_plain_text(text) == folded
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("op", "text", "folded"),
|
||||
[
|
||||
("^", "2", "^{2}"),
|
||||
("_", "2", "_{2}"),
|
||||
("_", " i j ", "_{ij}"),
|
||||
("^", "-1", "^{" + MINUS + "1}"),
|
||||
("^", "\N{PRIME}", "\N{PRIME}"),
|
||||
("^", "\N{PRIME}\N{PRIME}", "\N{PRIME}\N{PRIME}"),
|
||||
("_", "\N{PRIME}", "_{\N{PRIME}}"),
|
||||
("^", "", "^{}"),
|
||||
("^", "a b c?", "^{a b c?}"),
|
||||
("^", " a b ? ", "^{a b ?}"),
|
||||
("_", "x²", "_{x^{2}}"),
|
||||
("^", "\N{GREEK SMALL LETTER ALPHA}", "^{\N{GREEK SMALL LETTER ALPHA}}"),
|
||||
("^", "\N{CYRILLIC SMALL LETTER A}", "^{\N{CYRILLIC SMALL LETTER A}}"),
|
||||
],
|
||||
)
|
||||
def test_fold_styled_script(op, text, folded):
|
||||
assert nt.fold_styled_script(op, text) == folded
|
||||
|
||||
|
||||
def test_a_styled_script_and_a_script_character_fold_alike():
|
||||
assert nt.fold_styled_script("^", "3") == nt.fold_plain_text("³")
|
||||
assert nt.fold_styled_script("_", "2") == nt.fold_plain_text("₂")
|
||||
|
||||
|
||||
TEX_PIECES = [
|
||||
"a",
|
||||
"b",
|
||||
"x",
|
||||
"1",
|
||||
"2",
|
||||
"_",
|
||||
"^",
|
||||
"{",
|
||||
"}",
|
||||
"'",
|
||||
"-",
|
||||
"+",
|
||||
" ",
|
||||
"\\alpha",
|
||||
"\\mathrm",
|
||||
"\\left(",
|
||||
"\\right)",
|
||||
"\\,",
|
||||
"\\frac",
|
||||
"\\prime",
|
||||
"(",
|
||||
")",
|
||||
"=",
|
||||
]
|
||||
TYPST_PIECES = [
|
||||
"a",
|
||||
"b",
|
||||
"1",
|
||||
"2",
|
||||
"_",
|
||||
"^",
|
||||
"(",
|
||||
")",
|
||||
"'",
|
||||
"-",
|
||||
"+",
|
||||
" ",
|
||||
"alpha",
|
||||
"upright",
|
||||
'"',
|
||||
'"cm"',
|
||||
"<=",
|
||||
"->",
|
||||
".",
|
||||
"/",
|
||||
"dot.op",
|
||||
]
|
||||
|
||||
|
||||
@given(st.text())
|
||||
@PROPERTIES
|
||||
def test_any_source_gives_a_form_or_none_never_an_error(source):
|
||||
for notation in ("tex", "typst"):
|
||||
result = nt.fold_math(notation, source)
|
||||
assert result is None or isinstance(result, str)
|
||||
|
||||
|
||||
@given(st.lists(st.sampled_from(TEX_PIECES), max_size=14).map("".join))
|
||||
@PROPERTIES
|
||||
def test_a_tex_form_is_stable_under_folding_again(source):
|
||||
form = nt.fold_math("tex", source)
|
||||
if form is not None:
|
||||
assert nt.fold_math("tex", form) == form
|
||||
|
||||
|
||||
@given(st.lists(st.sampled_from(TYPST_PIECES), max_size=14).map("".join))
|
||||
@PROPERTIES
|
||||
def test_a_typst_form_contains_no_white_space(source):
|
||||
form = nt.fold_math("typst", source)
|
||||
if form is not None:
|
||||
assert not any(c.isspace() for c in form)
|
||||
|
||||
|
||||
@given(st.lists(st.sampled_from(TEX_PIECES), max_size=14).map("".join))
|
||||
@PROPERTIES
|
||||
def test_white_space_does_not_change_a_tex_form(source):
|
||||
spaced = " ".join(source)
|
||||
plain = nt.fold_math("tex", source.replace(" ", ""))
|
||||
if plain is not None and "\\" not in source:
|
||||
assert nt.fold_math("tex", spaced) == plain
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,856 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Receipts: signing, statuses, revocations, spot-check samples, error bounds and levels (paperspec.receipts)."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
from fractions import Fraction
|
||||
from math import comb
|
||||
|
||||
import pytest
|
||||
from hypothesis import given, settings, strategies as st
|
||||
|
||||
from conftest import FIXTURES, vector_cases
|
||||
from helpers_lib2 import (
|
||||
HALE,
|
||||
PROPS,
|
||||
RECORDS,
|
||||
fixture_receipts,
|
||||
fixture_roster,
|
||||
receipt,
|
||||
revocation,
|
||||
roster_key,
|
||||
roster_of,
|
||||
sealed,
|
||||
secret,
|
||||
)
|
||||
from paperspec import crypto, mldsa, package, receipts as rcp, vrf
|
||||
from paperspec.text import record_id
|
||||
|
||||
NODES = [(f"node{c}", c * 64) for c in "abcdef"]
|
||||
ROSTER = roster_of(
|
||||
roster_key("ana", role="owner"),
|
||||
roster_key("ben"),
|
||||
roster_key("cara", role="reader"),
|
||||
roster_key("ben-spot", purposes=("spot-check",)),
|
||||
roster_key("dana", scheme="ml-dsa-65"),
|
||||
)
|
||||
SIGNATURES = {c["id"]: c for c in vector_cases("signatures")}
|
||||
SPOTCHECKS = vector_cases("spotcheck")
|
||||
|
||||
|
||||
def status_of(rec, records=None, roster=ROSTER):
|
||||
"""The status of `rec` among `records` (itself, by default)."""
|
||||
return rcp.receipt_status(rec, records or [rec], roster)
|
||||
|
||||
|
||||
# --- record hashes and signatures -----------------------------------------------------------------
|
||||
|
||||
|
||||
def test_receipt_vector_record_hash_payload_and_signature():
|
||||
case = SIGNATURES["receipt-r1"]
|
||||
rec = case["receipt"]
|
||||
assert rcp.record_hash(rec) == case["record_hash"]
|
||||
assert rcp.signing_payload(rec).decode() == case["payload"]
|
||||
assert record_id(rcp.record_hash(rec)) == case["record_id"] == rec["id"]
|
||||
roster = {"ana": {"scheme": "ed25519", "public": case["public"]}}
|
||||
assert rcp.verify_signature(rec, roster)
|
||||
|
||||
|
||||
def test_record_hash_ignores_id_signature_and_key_order():
|
||||
rec = sealed(receipt(NODES))
|
||||
shuffled = dict(reversed(list(rec.items())))
|
||||
assert rcp.record_hash(shuffled) == rcp.record_hash(rec)
|
||||
assert rcp.record_hash(
|
||||
{k: v for k, v in rec.items() if k != "id"}
|
||||
) == rcp.record_hash(rec)
|
||||
assert rcp.record_hash(dict(rec, at="2026-10-11T12:00:01Z")) != rcp.record_hash(rec)
|
||||
|
||||
|
||||
def test_the_signing_payload_names_the_format_and_the_hash():
|
||||
rec = receipt(NODES)
|
||||
payload = json.loads(rcp.signing_payload(rec))
|
||||
assert payload == {"format": rec["format"], "hash": rcp.record_hash(rec)}
|
||||
|
||||
|
||||
def test_signed_formats_are_the_four_signed_records():
|
||||
assert rcp.SIGNED_FORMATS == (
|
||||
"scienting-receipt/1",
|
||||
"scienting-revocation/1",
|
||||
"scienting-policy/1",
|
||||
"scienting-qa-report/1",
|
||||
)
|
||||
|
||||
|
||||
def test_seal_fills_in_the_root_the_id_and_a_signature_that_verifies():
|
||||
rec = sealed(receipt(NODES))
|
||||
assert rec["root"] == crypto.merkle_root(crypto.receipt_items(rec))
|
||||
assert rec["id"] == record_id(rcp.record_hash(rec))
|
||||
assert rec["signature"]["scheme"] == "ed25519"
|
||||
assert rec["signature"]["key"] == "ana"
|
||||
assert len(base64.b64decode(rec["signature"]["value"])) == 64
|
||||
assert rcp.verify_signature(rec, ROSTER)
|
||||
|
||||
|
||||
def test_sealing_a_sealed_receipt_again_changes_nothing():
|
||||
rec = sealed(receipt(NODES))
|
||||
assert sealed(rec) == rec
|
||||
|
||||
|
||||
def test_sealing_does_not_change_its_argument():
|
||||
rec = receipt(NODES)
|
||||
before = json.dumps(rec, sort_keys=True)
|
||||
sealed(rec)
|
||||
assert json.dumps(rec, sort_keys=True) == before
|
||||
|
||||
|
||||
def test_sealing_a_revocation_adds_no_root():
|
||||
rev = revocation("ana", receipt="abc")
|
||||
assert "root" not in rev
|
||||
assert rcp.verify_signature(rev, ROSTER)
|
||||
|
||||
|
||||
def test_the_ml_dsa_fixture_receipt_verifies_and_is_reproduced_by_sealing():
|
||||
rec = json.loads((RECORDS / "receipt-ml-dsa.json").read_text(encoding="utf-8"))
|
||||
roster = fixture_roster()
|
||||
assert rec["signature"]["scheme"] == "ml-dsa-65"
|
||||
assert rcp.verify_signature(rec, roster)
|
||||
assert not rcp.verify_signature(dict(rec, at="2026-10-13T09:00:01Z"), roster)
|
||||
resealed = rcp.seal_mldsa(rec, mldsa.keygen(secret("dana"))[1], "dana")
|
||||
assert resealed == rec
|
||||
|
||||
|
||||
def test_an_ml_dsa_receipt_is_valid_under_a_roster_that_holds_its_key():
|
||||
secret_key = mldsa.keygen(secret("dana"))[1]
|
||||
rec = rcp.seal_mldsa(receipt(NODES, by="dana", level="F2"), secret_key, "dana")
|
||||
assert rec["root"] == crypto.merkle_root(crypto.receipt_items(rec))
|
||||
assert status_of(rec) == "valid"
|
||||
|
||||
|
||||
def test_an_ml_dsa_revocation_has_no_root_and_verifies():
|
||||
revocation_record = {
|
||||
"format": "scienting-revocation/1",
|
||||
"receipt": "abc",
|
||||
"reason": "Revoked for a test.",
|
||||
"by": "dana",
|
||||
"at": "2026-10-11T13:00:00Z",
|
||||
}
|
||||
rec = rcp.seal_mldsa(revocation_record, mldsa.keygen(secret("dana"))[1], "dana")
|
||||
assert "root" not in rec
|
||||
assert rec["signature"]["scheme"] == "ml-dsa-65"
|
||||
assert rcp.verify_signature(rec, ROSTER)
|
||||
|
||||
|
||||
def verifies(rec, roster=ROSTER):
|
||||
"""Whether the signature of `rec` verifies under `roster`."""
|
||||
return rcp.verify_signature(rec, roster)
|
||||
|
||||
|
||||
def signature_with(rec, **changes):
|
||||
"""`rec` with fields of its signature replaced; a value of None removes the field."""
|
||||
sig = {**rec["signature"], **changes}
|
||||
return dict(rec, signature={k: v for k, v in sig.items() if v is not None})
|
||||
|
||||
|
||||
def test_a_signature_does_not_verify_for_a_changed_record():
|
||||
rec = sealed(receipt(NODES))
|
||||
assert verifies(rec)
|
||||
assert not verifies(dict(rec, level="F2"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tamper",
|
||||
[
|
||||
lambda r: {k: v for k, v in r.items() if k != "signature"},
|
||||
lambda r: dict(r, signature=None),
|
||||
lambda r: dict(r, signature="ana"),
|
||||
lambda r: signature_with(r, key="nobody"),
|
||||
lambda r: signature_with(r, key=None),
|
||||
lambda r: signature_with(r, key="ben"),
|
||||
lambda r: signature_with(r, scheme="ml-dsa-65"),
|
||||
lambda r: signature_with(r, scheme="rsa"),
|
||||
lambda r: signature_with(r, value="not base64!"),
|
||||
lambda r: signature_with(r, value=None),
|
||||
lambda r: signature_with(r, value=7),
|
||||
lambda r: signature_with(r, value=base64.b64encode(bytes(63)).decode()),
|
||||
lambda r: signature_with(r, value=base64.b64encode(bytes(64)).decode()),
|
||||
],
|
||||
ids=[
|
||||
"no-signature",
|
||||
"signature-null",
|
||||
"signature-not-an-object",
|
||||
"unknown-key",
|
||||
"no-key",
|
||||
"another-key",
|
||||
"scheme-differs-from-the-roster",
|
||||
"unimplemented-scheme",
|
||||
"value-not-base64",
|
||||
"no-value",
|
||||
"value-not-text",
|
||||
"value-too-short",
|
||||
"value-zeros",
|
||||
],
|
||||
)
|
||||
def test_signatures_that_cannot_verify_do_not(tamper):
|
||||
assert not verifies(tamper(sealed(receipt(NODES))))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"key",
|
||||
[
|
||||
{"scheme": "ed25519", "public": "not base64!"},
|
||||
{"scheme": "ed25519", "public": base64.b64encode(bytes(31)).decode()},
|
||||
{"scheme": "ed25519"},
|
||||
{"scheme": "ml-dsa-65", "public": base64.b64encode(bytes(32)).decode()},
|
||||
{"scheme": "rsa", "public": base64.b64encode(bytes(32)).decode()},
|
||||
{},
|
||||
],
|
||||
ids=[
|
||||
"public-not-base64",
|
||||
"short-public",
|
||||
"no-public",
|
||||
"short-ml-dsa",
|
||||
"rsa",
|
||||
"empty",
|
||||
],
|
||||
)
|
||||
def test_a_roster_key_that_cannot_verify_does_not(key):
|
||||
rec = sealed(receipt(NODES))
|
||||
scheme = key.get("scheme", "ed25519")
|
||||
rec = signature_with(rec, scheme=scheme)
|
||||
assert not verifies(rec, {"ana": key})
|
||||
|
||||
|
||||
def test_a_record_that_has_no_canonical_form_does_not_verify():
|
||||
rec = sealed(receipt(NODES))
|
||||
assert not verifies(dict(rec, extra=float("nan")))
|
||||
|
||||
|
||||
def test_a_record_without_a_format_does_not_verify_and_does_not_raise():
|
||||
rec = sealed(receipt(NODES))
|
||||
assert not verifies({k: v for k, v in rec.items() if k != "format"})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("at", "valid"),
|
||||
[
|
||||
("2026-08-31T23:59:59Z", False),
|
||||
("2026-09-01T00:00:00Z", True),
|
||||
("2026-10-01T00:00:00Z", True),
|
||||
("2026-10-15T00:00:00Z", False),
|
||||
("2026-12-01T00:00:00Z", False),
|
||||
],
|
||||
)
|
||||
def test_a_key_is_valid_from_registration_until_revocation(at, valid):
|
||||
key = {
|
||||
"registered_at": "2026-09-01T00:00:00Z",
|
||||
"revoked_at": "2026-10-15T00:00:00Z",
|
||||
}
|
||||
assert rcp.key_valid_at(key, at) is valid
|
||||
|
||||
|
||||
def test_a_key_without_a_revocation_date_stays_valid():
|
||||
key = {"registered_at": "2026-09-01T00:00:00Z"}
|
||||
assert rcp.key_valid_at(key, "2099-01-01T00:00:00Z")
|
||||
assert rcp.key_valid_at({**key, "revoked_at": None}, "2099-01-01T00:00:00Z")
|
||||
|
||||
|
||||
# --- spot-checks ------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
FULL_SAMPLE = next(c for c in SPOTCHECKS if "seed" in c)
|
||||
|
||||
|
||||
def test_the_spot_check_vector_reproduces_seed_payload_beta_and_sample():
|
||||
case = FULL_SAMPLE
|
||||
seed = base64.b64decode(case["seed"])
|
||||
checker = base64.b64decode(case["checker_public"])
|
||||
assert rcp.seed_payload(case["receipt"]).decode() == case["seed_payload"]
|
||||
assert (
|
||||
vrf.verify(checker, case["seed_payload"].encode(), seed).hex() == case["beta"]
|
||||
)
|
||||
assert (
|
||||
rcp.sample(case["receipt"], seed, case["rate"], case["minimum"])
|
||||
== case["sample"]
|
||||
)
|
||||
assert len(case["sample"]) == case["size"]
|
||||
|
||||
|
||||
def test_the_seed_of_the_spot_check_vector_is_the_checkers_vrf_proof():
|
||||
case = FULL_SAMPLE
|
||||
seed = rcp.make_seed(case["receipt"], secret("ben-spot"))
|
||||
assert base64.b64encode(seed).decode() == case["seed"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case", [c for c in SPOTCHECKS if "seed" not in c], ids=lambda c: c["id"]
|
||||
)
|
||||
def test_sample_size_vectors(case):
|
||||
assert rcp.sample_size(case["n"], case["rate"], case["minimum"]) == case["size"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("n", "rate", "minimum", "size"),
|
||||
[
|
||||
(0, "0.05", 10, 0),
|
||||
(5, "0.05", 10, 5),
|
||||
(400, "0.05", 10, 20),
|
||||
(401, "0.05", 10, 21),
|
||||
(7, "1", 1, 7),
|
||||
(9, "1/3", 1, 3),
|
||||
(100, "0.5", 100, 100),
|
||||
],
|
||||
)
|
||||
def test_sample_size_is_the_ceiling_of_the_rate_bounded_by_the_minimum_and_n(
|
||||
n, rate, minimum, size
|
||||
):
|
||||
assert rcp.sample_size(n, rate, minimum) == size
|
||||
|
||||
|
||||
def spot_check_receipt(n):
|
||||
"""A receipt over `n` nodes, for sampling."""
|
||||
return {"nodes": [[f"n{i:03d}", f"{i:064x}"] for i in range(n)]}
|
||||
|
||||
|
||||
def test_sample_depends_on_the_seed_and_not_on_the_order_of_the_nodes():
|
||||
rec = spot_check_receipt(60)
|
||||
seed = rcp.make_seed(rec, secret("ben-spot"))
|
||||
picked = rcp.sample(rec, seed)
|
||||
reordered = {"nodes": rec["nodes"][::-1]}
|
||||
assert rcp.sample(reordered, seed) == picked
|
||||
assert picked == sorted(picked)
|
||||
assert len(set(picked)) == len(picked) == 10
|
||||
other = rcp.make_seed(rec, secret("ben"))
|
||||
assert rcp.sample(rec, other) != picked
|
||||
|
||||
|
||||
def test_sample_takes_every_node_of_a_small_receipt():
|
||||
rec = spot_check_receipt(4)
|
||||
seed = rcp.make_seed(rec, secret("ben-spot"))
|
||||
assert rcp.sample(rec, seed) == [n for n, _ in rec["nodes"]]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("seed", [b"", bytes(79), b"x" * 81, b"\xff" * 80])
|
||||
def test_a_seed_that_is_no_vrf_proof_is_refused(seed):
|
||||
with pytest.raises(ValueError, match="not a VRF proof"):
|
||||
rcp.sample(spot_check_receipt(12), seed)
|
||||
|
||||
|
||||
@given(
|
||||
st.integers(1, 80),
|
||||
st.sampled_from(["0.05", "0.1", "0.5", "1"]),
|
||||
st.integers(1, 15),
|
||||
)
|
||||
@settings(PROPS, max_examples=25)
|
||||
def test_a_sample_has_the_size_the_rule_gives(n, rate, minimum):
|
||||
rec = spot_check_receipt(n)
|
||||
seed = rcp.make_seed(rec, secret("ben-spot"))
|
||||
picked = rcp.sample(rec, seed, rate, minimum)
|
||||
assert len(picked) == rcp.sample_size(n, rate, minimum)
|
||||
assert set(picked) <= {nid for nid, _ in rec["nodes"]}
|
||||
|
||||
|
||||
# --- error bounds -----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
BOUNDS = vector_cases("bounds")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", BOUNDS, ids=lambda c: c["id"])
|
||||
def test_bound_vectors(case):
|
||||
assert (
|
||||
rcp.bound_string(case["errors"], case["sampled"], case["confidence"])
|
||||
== case["bound"]
|
||||
)
|
||||
approx = rcp.upper_bound(case["errors"], case["sampled"], case["confidence"])
|
||||
assert approx == pytest.approx(case["approx"], abs=1e-9)
|
||||
|
||||
|
||||
def test_zero_errors_in_300_bound_the_rate_below_one_percent():
|
||||
assert Fraction(rcp.bound_string(0, 300)) < Fraction(1, 100)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n", [0, -3])
|
||||
def test_bounds_need_a_positive_sample(n):
|
||||
with pytest.raises(ValueError, match="n must be positive"):
|
||||
rcp.bound_string(0, n)
|
||||
with pytest.raises(ValueError, match="n must be positive"):
|
||||
rcp.upper_bound(0, n)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("errors", [10, 11])
|
||||
def test_all_errors_bound_the_rate_by_one(errors):
|
||||
assert rcp.bound_string(errors, 10) == "1.000000"
|
||||
assert rcp.upper_bound(errors, 10) == 1.0
|
||||
|
||||
|
||||
def test_a_higher_confidence_gives_a_higher_bound():
|
||||
assert Fraction(rcp.bound_string(2, 50, "0.99")) > Fraction(
|
||||
rcp.bound_string(2, 50, "0.95")
|
||||
)
|
||||
assert rcp.upper_bound(2, 50, "0.99") > rcp.upper_bound(2, 50, "0.95")
|
||||
|
||||
|
||||
@given(st.integers(1, 60), st.integers(0, 60), st.sampled_from(["0.9", "0.95", "0.99"]))
|
||||
@settings(PROPS, max_examples=40)
|
||||
def test_the_bound_is_the_least_six_decimal_rate_whose_tail_is_within_alpha(
|
||||
n, k, confidence
|
||||
):
|
||||
k = min(k, n)
|
||||
text = rcp.bound_string(k, n, confidence)
|
||||
assert len(text.split(".")[1]) == 6
|
||||
if k == n:
|
||||
assert text == "1.000000"
|
||||
return
|
||||
alpha = 1 - Fraction(confidence)
|
||||
|
||||
def tail(p):
|
||||
return sum(comb(n, i) * p**i * (1 - p) ** (n - i) for i in range(k + 1))
|
||||
|
||||
bound = Fraction(text)
|
||||
assert tail(bound) <= alpha
|
||||
assert bound == 0 or tail(bound - Fraction(1, 10**6)) > alpha
|
||||
assert rcp.upper_bound(k, n, confidence) == pytest.approx(float(bound), abs=2e-6)
|
||||
|
||||
|
||||
@given(st.integers(2, 60), st.integers(0, 58))
|
||||
@settings(PROPS, max_examples=30)
|
||||
def test_one_more_error_never_lowers_the_bound(n, k):
|
||||
k = min(k, n - 2)
|
||||
assert Fraction(rcp.bound_string(k + 1, n)) >= Fraction(rcp.bound_string(k, n))
|
||||
|
||||
|
||||
# --- revocations -------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_revoked_by_the_id_of_the_receipt():
|
||||
rec = {"id": "r1", "by": "ana", "at": "2026-10-11T12:00:00Z"}
|
||||
assert rcp.revoked(rec, [{"receipt": "r1"}])
|
||||
assert not rcp.revoked(rec, [{"receipt": "r2"}])
|
||||
assert not rcp.revoked(rec, [])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("period", "hit"),
|
||||
[
|
||||
({"from": "2026-10-11T00:00:00Z", "to": "2026-10-12T00:00:00Z"}, True),
|
||||
({"from": "2026-10-11T12:00:00Z", "to": "2026-10-12T00:00:00Z"}, True),
|
||||
({"from": "2026-10-01T00:00:00Z", "to": "2026-10-11T12:00:00Z"}, True),
|
||||
({"from": "2026-10-11T12:00:01Z", "to": "2026-10-12T00:00:00Z"}, False),
|
||||
({"from": "2026-10-01T00:00:00Z", "to": "2026-10-11T11:59:59Z"}, False),
|
||||
(None, False),
|
||||
],
|
||||
ids=[
|
||||
"inside",
|
||||
"from-inclusive",
|
||||
"to-inclusive",
|
||||
"starts-after",
|
||||
"ends-before",
|
||||
"no-period",
|
||||
],
|
||||
)
|
||||
def test_revoked_by_the_signer_and_a_period(period, hit):
|
||||
rec = {"id": "r1", "by": "ana", "at": "2026-10-11T12:00:00Z"}
|
||||
assert rcp.revoked(rec, [{"signer": "ana", "period": period}]) is hit
|
||||
assert not rcp.revoked(rec, [{"signer": "ben", "period": period}])
|
||||
|
||||
|
||||
def test_only_revocations_signed_by_someone_entitled_to_them_count():
|
||||
own = sealed(receipt(NODES, by="ben"))
|
||||
by_id = {own["id"]: own}
|
||||
by_signer = revocation("ben", receipt=own["id"])
|
||||
by_owner = revocation("ana", receipt=own["id"])
|
||||
by_stranger = revocation("cara", receipt=own["id"])
|
||||
names_signer = revocation("ben", signer="ben", period={"from": "a", "to": "b"})
|
||||
names_other = revocation("cara", signer="ben", period={"from": "a", "to": "b"})
|
||||
forged = dict(by_owner, reason="Changed after signing.")
|
||||
unknown = revocation("zed", receipt=own["id"])
|
||||
revs = [
|
||||
by_signer,
|
||||
by_owner,
|
||||
by_stranger,
|
||||
names_signer,
|
||||
names_other,
|
||||
forged,
|
||||
unknown,
|
||||
]
|
||||
assert rcp.valid_revocations(revs, by_id, ROSTER) == [
|
||||
by_signer,
|
||||
by_owner,
|
||||
names_signer,
|
||||
]
|
||||
|
||||
|
||||
def test_an_owner_may_revoke_a_receipt_nobody_holds():
|
||||
rev = revocation("ana", receipt="vanished")
|
||||
assert rcp.valid_revocations([rev], {}, ROSTER) == [rev]
|
||||
assert (
|
||||
rcp.valid_revocations([revocation("ben", receipt="vanished")], {}, ROSTER) == []
|
||||
)
|
||||
|
||||
|
||||
# --- statuses ----------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_signed_receipt_of_a_registered_key_is_valid():
|
||||
assert status_of(sealed(receipt(NODES))) == "valid"
|
||||
|
||||
|
||||
def test_statuses_of_the_main_examples_receipts():
|
||||
records = fixture_receipts()
|
||||
status = rcp.statuses(records, fixture_roster())
|
||||
assert status == {
|
||||
"vskq18t5sxydz": "valid",
|
||||
"fhhvg9qrzqp0n": "valid",
|
||||
"mvjmb94nz7rdh": "valid",
|
||||
"q465jpwz8afhe": "revoked",
|
||||
"gx2s84z9jf9xv": "suspended",
|
||||
"ksx5sc327xp97": "valid",
|
||||
"yt70fcc3qm0xb": "valid",
|
||||
}
|
||||
|
||||
|
||||
def test_a_receipt_among_records_that_do_not_hold_it_is_invalid():
|
||||
rec = sealed(receipt(NODES))
|
||||
assert rcp.receipt_status(rec, [], ROSTER) == "invalid"
|
||||
assert rcp.receipt_status({}, [rec], ROSTER) == "invalid"
|
||||
|
||||
|
||||
def test_statuses_ignore_records_that_are_not_receipts():
|
||||
rec = sealed(receipt(NODES))
|
||||
assert rcp.statuses([{"format": "scienting-policy/1", "id": "p"}, rec], ROSTER) == {
|
||||
rec["id"]: "valid"
|
||||
}
|
||||
|
||||
|
||||
INVALID_RECEIPTS = {
|
||||
"changed-after-signing": lambda: dict(
|
||||
sealed(receipt(NODES)), at="2026-10-11T12:00:01Z"
|
||||
),
|
||||
"wrong-id": lambda: dict(sealed(receipt(NODES)), id="aaaaaaaaaaaaa"),
|
||||
"wrong-root": lambda: dict(sealed(receipt(NODES)), root="00" * 32),
|
||||
"no-root": lambda: {k: v for k, v in sealed(receipt(NODES)).items() if k != "root"},
|
||||
"signer-not-in-roster": lambda: sealed(receipt(NODES, by="zed")),
|
||||
"signed-by-another-key": lambda: sealed(receipt(NODES, by="ana"), signer="ben"),
|
||||
"no-time": lambda: sealed({k: v for k, v in receipt(NODES).items() if k != "at"}),
|
||||
"time-not-comparable": lambda: sealed(receipt(NODES, at=5)),
|
||||
"no-signer": lambda: sealed(
|
||||
{k: v for k, v in receipt(NODES).items() if k != "by"}, signer="ana"
|
||||
),
|
||||
"before-registration": lambda: sealed(receipt(NODES, at="2026-08-01T00:00:00Z")),
|
||||
"after-key-revocation": lambda: sealed(
|
||||
receipt(NODES, by="ben", at="2026-10-20T00:00:00Z")
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", INVALID_RECEIPTS)
|
||||
def test_receipts_that_break_a_condition_are_invalid(name):
|
||||
roster = {**ROSTER, "ben": roster_key("ben", revoked_at="2026-10-15T00:00:00Z")}
|
||||
assert status_of(INVALID_RECEIPTS[name](), roster=roster) == "invalid"
|
||||
|
||||
|
||||
def test_a_receipt_signed_before_its_key_was_revoked_stays_valid():
|
||||
roster = {**ROSTER, "ben": roster_key("ben", revoked_at="2026-10-15T00:00:00Z")}
|
||||
rec = sealed(receipt(NODES, by="ben", at="2026-10-14T00:00:00Z"))
|
||||
assert status_of(rec, roster=roster) == "valid"
|
||||
|
||||
|
||||
def test_a_receipt_is_revoked_by_a_valid_revocation_and_not_by_a_forged_one():
|
||||
rec = sealed(receipt(NODES))
|
||||
assert status_of(rec, [rec, revocation("ana", receipt=rec["id"])]) == "revoked"
|
||||
forged = dict(revocation("ana", receipt=rec["id"]), reason="Changed.")
|
||||
assert status_of(rec, [rec, forged]) == "valid"
|
||||
assert status_of(rec, [rec, revocation("cara", receipt=rec["id"])]) == "valid"
|
||||
|
||||
|
||||
def test_a_receipt_is_revoked_by_a_revocation_of_its_signer_over_a_period():
|
||||
rec = sealed(receipt(NODES, by="ben"))
|
||||
period = {"from": "2026-10-11T00:00:00Z", "to": "2026-10-12T00:00:00Z"}
|
||||
covering = revocation("ben", signer="ben", period=period)
|
||||
assert status_of(rec, [rec, covering]) == "revoked"
|
||||
elsewhere = revocation(
|
||||
"ben",
|
||||
signer="ben",
|
||||
period={"from": "2026-11-01T00:00:00Z", "to": "2026-11-02T00:00:00Z"},
|
||||
)
|
||||
assert status_of(rec, [rec, elsewhere]) == "valid"
|
||||
|
||||
|
||||
def test_the_revocation_of_a_revocation_signer_key_is_not_a_revocation_of_receipts():
|
||||
rec = sealed(receipt(NODES, by="ben"))
|
||||
bare = revocation("ben", signer="ben")
|
||||
assert status_of(rec, [rec, bare]) == "valid"
|
||||
|
||||
|
||||
def qa_receipt(of, by, found_errors, at):
|
||||
"""A receipt that records a spot-check of the receipt `of`; `found_errors` is a list of node IDs."""
|
||||
qa = {"of": of["id"], "errors": [{"node": n} for n in found_errors]}
|
||||
return sealed(
|
||||
receipt(NODES[:2], by=by, at=at, level="F1", scope="qa:" + of["id"], qa=qa)
|
||||
)
|
||||
|
||||
|
||||
def test_a_spot_check_that_found_an_error_suspends_the_receipt_it_checked():
|
||||
original = sealed(receipt(NODES))
|
||||
check = qa_receipt(original, "ben", ["nodea"], "2026-10-12T09:00:00Z")
|
||||
status = rcp.statuses([original, check], ROSTER)
|
||||
assert status == {original["id"]: "suspended", check["id"]: "valid"}
|
||||
|
||||
|
||||
def test_a_spot_check_that_found_nothing_suspends_nothing():
|
||||
original = sealed(receipt(NODES))
|
||||
check = qa_receipt(original, "ben", [], "2026-10-12T09:00:00Z")
|
||||
assert rcp.statuses([original, check], ROSTER)[original["id"]] == "valid"
|
||||
|
||||
|
||||
def test_a_spot_check_that_is_not_valid_suspends_nothing():
|
||||
original = sealed(receipt(NODES))
|
||||
stranger = qa_receipt(original, "zed", ["nodea"], "2026-10-12T09:00:00Z")
|
||||
revoked_check = qa_receipt(original, "ben", ["nodea"], "2026-10-12T10:00:00Z")
|
||||
records = [
|
||||
original,
|
||||
stranger,
|
||||
revoked_check,
|
||||
revocation("ana", receipt=revoked_check["id"]),
|
||||
]
|
||||
status = rcp.statuses(records, ROSTER)
|
||||
assert status[original["id"]] == "valid"
|
||||
assert status[stranger["id"]] == "invalid"
|
||||
assert status[revoked_check["id"]] == "revoked"
|
||||
|
||||
|
||||
def test_a_spot_check_of_a_receipt_that_is_not_there_suspends_nothing():
|
||||
original = sealed(receipt(NODES))
|
||||
check = qa_receipt(original, "ben", ["nodea"], "2026-10-12T09:00:00Z")
|
||||
assert rcp.statuses([check], ROSTER) == {check["id"]: "valid"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("qa", ["checked", None, ["errors"], 3], ids=repr)
|
||||
def test_a_receipt_whose_qa_is_not_an_object_is_an_ordinary_receipt(qa):
|
||||
rec = sealed(receipt(NODES, qa=qa))
|
||||
assert status_of(rec) == "valid"
|
||||
|
||||
|
||||
def test_chains_of_spot_checks_alternate_from_the_last_unchecked_one():
|
||||
chain = [sealed(receipt(NODES))]
|
||||
for i in range(7):
|
||||
chain.append(
|
||||
qa_receipt(chain[-1], "ben", ["nodea"], f"2026-10-12T09:{i:02d}:00Z")
|
||||
)
|
||||
status = rcp.statuses(chain, ROSTER)
|
||||
expected = ["valid", "suspended"] * 4
|
||||
assert [status[r["id"]] for r in reversed(chain)] == expected[:8]
|
||||
|
||||
|
||||
def test_two_spot_checks_of_one_receipt_suspend_it_if_either_is_valid():
|
||||
original = sealed(receipt(NODES))
|
||||
first = qa_receipt(original, "ben", ["nodea"], "2026-10-12T09:00:00Z")
|
||||
second = qa_receipt(original, "ana", ["nodeb"], "2026-10-12T10:00:00Z")
|
||||
both = rcp.statuses([original, first, second], ROSTER)
|
||||
assert both[original["id"]] == "suspended"
|
||||
third = qa_receipt(first, "ana", ["nodea"], "2026-10-12T11:00:00Z")
|
||||
fourth = qa_receipt(second, "ben", ["nodeb"], "2026-10-12T12:00:00Z")
|
||||
nested = rcp.statuses([original, first, second, third, fourth], ROSTER)
|
||||
assert nested[first["id"]] == "suspended"
|
||||
assert nested[second["id"]] == "suspended"
|
||||
assert nested[original["id"]] == "valid"
|
||||
|
||||
|
||||
# --- carried hashes and levels ----------------------------------------------------------------------------
|
||||
|
||||
|
||||
def values_of(**hashes):
|
||||
"""Node values holding only the `node_hash` of each node."""
|
||||
return {nid: {"node_hash": h} for nid, h in hashes.items()}
|
||||
|
||||
|
||||
def emend(target, before, after, **extra):
|
||||
"""A correction that changes `target` from the hash `before` to `after`."""
|
||||
return {"op": "emend", "target": target, "before": before, "after": after, **extra}
|
||||
|
||||
|
||||
def test_carried_hashes_without_corrections_are_the_current_ones():
|
||||
assert rcp.carried_hashes(values_of(a="h1", b="h2"), []) == {
|
||||
"a": {"h1"},
|
||||
"b": {"h2"},
|
||||
}
|
||||
|
||||
|
||||
def test_carried_hashes_follow_a_chain_of_corrections_that_keep_the_level():
|
||||
corrections = [emend("a", "h0", "h1"), emend("a", "h1", "h2")]
|
||||
assert rcp.carried_hashes(values_of(a="h2"), corrections) == {
|
||||
"a": {"h2", "h1", "h0"}
|
||||
}
|
||||
|
||||
|
||||
def test_relink_and_resolve_keep_the_level_like_emend():
|
||||
corrections = [
|
||||
{"op": "resolve", "target": "a", "before": "h0", "after": "h1"},
|
||||
{"op": "relink", "target": "a", "before": "h1", "after": "h2"},
|
||||
]
|
||||
assert rcp.carried_hashes(values_of(a="h2"), corrections)["a"] == {"h2", "h1", "h0"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"breaker",
|
||||
[
|
||||
{"op": "replace", "target": "a", "before": "h1", "after": "h2"},
|
||||
{"op": "merge", "target": "z", "touched": ["a"], "before": "x", "after": "y"},
|
||||
{"op": "split", "target": "z", "path": ["a"], "before": "x", "after": "y"},
|
||||
{"op": "delete", "target": "z", "to_path": ["a"], "before": "x", "after": "y"},
|
||||
{"op": "insert", "target": "z", "created": ["a"], "before": "x", "after": "y"},
|
||||
{"op": "merge", "target": "z", "retired": ["a"], "before": "x", "after": "y"},
|
||||
],
|
||||
ids=["replace", "touched", "path", "to-path", "created", "retired"],
|
||||
)
|
||||
def test_a_correction_that_does_not_keep_the_level_ends_the_chain(breaker):
|
||||
corrections = [emend("a", "h0", "h1"), breaker]
|
||||
assert rcp.carried_hashes(values_of(a="h2"), corrections) == {"a": {"h2"}}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"link",
|
||||
[
|
||||
emend("a", "h0", "other"),
|
||||
{"op": "emend", "target": "a", "after": "h2"},
|
||||
{"op": "emend", "target": "a", "before": "", "after": "h2"},
|
||||
emend("z", "h0", "h2", touched=["a"]),
|
||||
],
|
||||
ids=["after-differs", "no-before", "empty-before", "targets-another-node"],
|
||||
)
|
||||
def test_a_link_that_does_not_lead_to_the_hash_reached_ends_the_chain(link):
|
||||
assert rcp.carried_hashes(values_of(a="h2"), [link]) == {"a": {"h2"}}
|
||||
|
||||
|
||||
def test_corrections_of_nodes_without_values_are_ignored():
|
||||
corrections = [
|
||||
emend("gone", "h0", "h1"),
|
||||
{"op": "delete", "target": "gone", "path": None},
|
||||
]
|
||||
assert rcp.carried_hashes(values_of(a="h2"), corrections) == {"a": {"h2"}}
|
||||
|
||||
|
||||
def test_a_chain_of_one_node_is_independent_of_the_others():
|
||||
corrections = [
|
||||
emend("a", "h0", "h1"),
|
||||
emend("b", "k0", "k1"),
|
||||
emend("a", "h1", "h2"),
|
||||
]
|
||||
carried = rcp.carried_hashes(values_of(a="h2", b="k1"), corrections)
|
||||
assert carried == {"a": {"h2", "h1", "h0"}, "b": {"k1", "k0"}}
|
||||
|
||||
|
||||
NODES_AT = [
|
||||
{"id": "nodea", "fidelity": "F1"},
|
||||
{"id": "nodeb", "fidelity": "F2"},
|
||||
{"id": "nodec"},
|
||||
]
|
||||
VALUES = values_of(nodea="aa" * 32, nodeb="bb" * 32, nodec="cc" * 32)
|
||||
|
||||
|
||||
def levels(records, corrections=(), values=VALUES, roster=ROSTER):
|
||||
"""The effective levels of the three nodes of `NODES_AT`."""
|
||||
return rcp.effective_levels(NODES_AT, values, records, roster, corrections)
|
||||
|
||||
|
||||
def test_without_receipts_a_leaf_keeps_its_recorded_level_and_other_nodes_have_none():
|
||||
assert levels([]) == {"nodea": "F1", "nodeb": "F2"}
|
||||
|
||||
|
||||
def test_a_valid_receipt_at_the_current_hash_raises_the_level():
|
||||
rec = sealed(receipt([("nodea", "aa" * 32), ("nodeb", "bb" * 32)], level="F3"))
|
||||
assert levels([rec]) == {"nodea": "F3", "nodeb": "F3"}
|
||||
|
||||
|
||||
def test_a_receipt_never_lowers_a_level_and_ignores_nodes_without_one():
|
||||
rec = sealed(
|
||||
receipt(
|
||||
[("nodea", "aa" * 32), ("nodeb", "bb" * 32), ("nodec", "cc" * 32)],
|
||||
level="F1",
|
||||
)
|
||||
)
|
||||
assert levels([rec]) == {"nodea": "F1", "nodeb": "F2"}
|
||||
|
||||
|
||||
def test_a_receipt_for_an_old_hash_raises_nothing():
|
||||
rec = sealed(receipt([("nodea", "00" * 32)], level="F3"))
|
||||
assert levels([rec]) == {"nodea": "F1", "nodeb": "F2"}
|
||||
|
||||
|
||||
def test_a_receipt_for_an_emended_hash_still_counts():
|
||||
rec = sealed(receipt([("nodea", "00" * 32)], level="F3"))
|
||||
kept = [emend("nodea", "00" * 32, "aa" * 32)]
|
||||
replaced = [
|
||||
{"op": "replace", "target": "nodea", "before": "00" * 32, "after": "aa" * 32}
|
||||
]
|
||||
assert levels([rec], kept)["nodea"] == "F3"
|
||||
assert levels([rec], replaced)["nodea"] == "F1"
|
||||
|
||||
|
||||
def test_receipts_that_are_not_valid_raise_nothing():
|
||||
good = receipt([("nodea", "aa" * 32)], level="F3")
|
||||
tampered = dict(sealed(good), at="2026-10-11T12:00:01Z")
|
||||
stranger = sealed(dict(good, by="zed"))
|
||||
original = sealed(good)
|
||||
revoked = [original, revocation("ana", receipt=original["id"])]
|
||||
for records in ([tampered], [stranger], revoked):
|
||||
assert levels(records)["nodea"] == "F1"
|
||||
|
||||
|
||||
def test_a_suspended_receipt_raises_nothing():
|
||||
original = sealed(receipt([("nodea", "aa" * 32)], level="F3"))
|
||||
check = qa_receipt(original, "ben", ["nodeb"], "2026-10-12T09:00:00Z")
|
||||
assert levels([original, check])["nodea"] == "F1"
|
||||
|
||||
|
||||
def test_a_spot_check_does_not_raise_the_nodes_it_found_in_error():
|
||||
original = sealed(receipt(NODES))
|
||||
found = {"of": original["id"], "errors": [{"node": "nodea"}]}
|
||||
check = sealed(
|
||||
receipt(
|
||||
[("nodea", "aa" * 32), ("nodeb", "bb" * 32)],
|
||||
by="ben",
|
||||
at="2026-10-12T09:00:00Z",
|
||||
scope="qa:" + original["id"],
|
||||
qa=found,
|
||||
)
|
||||
)
|
||||
assert levels([original, check]) == {"nodea": "F1", "nodeb": "F3"}
|
||||
|
||||
|
||||
def test_revocations_among_the_records_are_not_read_as_receipts():
|
||||
original = sealed(receipt([("nodea", "aa" * 32)], level="F2"))
|
||||
assert levels([original, revocation("ana", receipt="x")])["nodea"] == "F2"
|
||||
|
||||
|
||||
def test_the_main_examples_levels_recompute_the_distribution_it_reports():
|
||||
root = FIXTURES / "packages" / HALE
|
||||
state = package.read_state(str(root))
|
||||
document = json.loads((root / "document.json").read_text(encoding="utf-8"))
|
||||
effective = rcp.effective_levels(
|
||||
state.nodes,
|
||||
state.index().values,
|
||||
fixture_receipts(),
|
||||
fixture_roster(),
|
||||
state.corrections,
|
||||
)
|
||||
assert rcp.distribution(effective) == document["fidelity"]
|
||||
|
||||
|
||||
def test_distribution_counts_each_level_and_names_the_lowest():
|
||||
assert rcp.distribution({"a": "F2", "b": "F3", "c": "F2"}) == {
|
||||
"lowest": "F2",
|
||||
"counts": {"F0": 0, "F1": 0, "F2": 2, "F3": 1},
|
||||
}
|
||||
|
||||
|
||||
def test_the_distribution_of_nothing_has_no_lowest_level():
|
||||
assert rcp.distribution({}) == {
|
||||
"lowest": None,
|
||||
"counts": {"F0": 0, "F1": 0, "F2": 0, "F3": 0},
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Where the schemas, SQL and vectors are on disk (paperspec.resources)."""
|
||||
|
||||
import pathlib
|
||||
|
||||
import pytest
|
||||
|
||||
from paperspec import resources
|
||||
|
||||
MARKERS = {
|
||||
"schemas": "annotation.schema.json",
|
||||
"sql": "doc.sql",
|
||||
"vectors": "proofs.jsonl",
|
||||
}
|
||||
|
||||
|
||||
def test_the_folders_are_the_three_the_package_names():
|
||||
assert resources.FOLDERS == ("schemas", "sql", "vectors")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", resources.FOLDERS)
|
||||
def test_each_folder_is_found_and_holds_its_files(name):
|
||||
folder = resources.path(name)
|
||||
assert pathlib.Path(folder).is_dir()
|
||||
assert (pathlib.Path(folder) / MARKERS[name]).is_file()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["tests", "", "SCHEMAS", "schemas/", "../schemas"])
|
||||
def test_any_other_name_is_refused_and_the_folders_are_listed(name):
|
||||
with pytest.raises(ValueError, match="there are schemas, sql, vectors"):
|
||||
resources.path(name)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def layout(tmp_path, monkeypatch):
|
||||
"""A fake install under `tmp_path/root/src/paperspec`, with the checkout root `tmp_path/root`."""
|
||||
package = tmp_path / "root" / "src" / "paperspec"
|
||||
package.mkdir(parents=True)
|
||||
monkeypatch.setattr(resources, "_PACKAGE", package)
|
||||
return tmp_path / "root"
|
||||
|
||||
|
||||
def test_a_folder_inside_the_package_is_found_first(layout):
|
||||
inside = layout / "src" / "paperspec" / "sql"
|
||||
inside.mkdir()
|
||||
(layout / "sql").mkdir()
|
||||
assert resources.path("sql") == str(inside)
|
||||
|
||||
|
||||
def test_a_folder_at_the_root_of_the_checkout_is_found_when_the_package_has_none(
|
||||
layout,
|
||||
):
|
||||
(layout / "vectors").mkdir()
|
||||
assert resources.path("vectors") == str(layout / "vectors")
|
||||
|
||||
|
||||
def test_a_folder_found_in_neither_place_is_an_error_naming_it(layout):
|
||||
(layout / "sql").mkdir()
|
||||
with pytest.raises(
|
||||
FileNotFoundError, match="the schemas folder is neither in the package"
|
||||
):
|
||||
resources.path("schemas")
|
||||
|
||||
|
||||
def test_a_file_in_place_of_a_folder_is_not_a_folder(layout):
|
||||
(layout / "schemas").write_text("not a folder", encoding="utf-8")
|
||||
with pytest.raises(FileNotFoundError):
|
||||
resources.path("schemas")
|
||||
@@ -0,0 +1,369 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Rights records, policies, stage checks and backend conditions (spec/14)."""
|
||||
|
||||
from typing import ClassVar
|
||||
|
||||
import pytest
|
||||
|
||||
from helpers_lib1 import case_params, read_cases
|
||||
from paperspec import rights
|
||||
|
||||
SHA = "a" * 64
|
||||
OTHER_SHA = "b" * 64
|
||||
NOW = "2026-10-11T00:00:00Z"
|
||||
ALL_KEYS = [
|
||||
*rights.USES,
|
||||
*(f"ai/{p}/{q}" for p in rights.PURPOSES for q in rights.PROCESSORS),
|
||||
]
|
||||
|
||||
|
||||
def witness(uses=None, **extra):
|
||||
"""A witness whose rights record holds `uses` and the extra record members."""
|
||||
record = {"basis": "subscription", "uses": uses or {}}
|
||||
record.update({k: v for k, v in extra.items() if k not in ("origin", "sha256")})
|
||||
return {
|
||||
"rights": record,
|
||||
"origin": extra.get("origin", {"doi": "10.5555/x"}),
|
||||
"sha256": extra.get("sha256", SHA),
|
||||
}
|
||||
|
||||
|
||||
def policy(uses, **applies):
|
||||
"""A policy granting `uses`, applying to the witnesses `applies` describes."""
|
||||
out = {"uses": uses}
|
||||
if applies:
|
||||
out["applies_to"] = applies
|
||||
return out
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case", case_params([c for c in read_cases("rights") if "stage" in c])
|
||||
)
|
||||
def test_stage_vector(case):
|
||||
w = {
|
||||
"rights": case["rights"],
|
||||
"origin": case["origin"],
|
||||
"sha256": case["witness_sha256"],
|
||||
}
|
||||
uses, expired = rights.effective(w, case["policies"], case["now"])
|
||||
allowed, lacking = rights.check(case["requires"], case["processor"], uses)
|
||||
assert {"allowed": allowed, "lacking": lacking, "expired": expired} == case[
|
||||
"expect"
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case", case_params([c for c in read_cases("rights") if "backend" in c])
|
||||
)
|
||||
def test_backend_vector(case):
|
||||
allowed, failed = rights.backend_allowed(case["backend"], case["profile"])
|
||||
assert {"allowed": allowed, "failed": failed} == case["expect"]
|
||||
|
||||
|
||||
def test_combining_vector():
|
||||
case = next(c for c in read_cases("rights") if c["id"] == "combine")
|
||||
assert rights.combine([rights.flat(r) for r in case["rights"]]) == case["expect"]
|
||||
|
||||
|
||||
class TestFlat:
|
||||
def test_an_empty_record_is_unknown_everywhere(self):
|
||||
flat = rights.flat({})
|
||||
assert list(flat) == ALL_KEYS
|
||||
assert set(flat.values()) == {"unknown"}
|
||||
|
||||
def test_there_are_five_uses_and_eight_ai_pairs(self):
|
||||
assert len(rights.USES) == 5
|
||||
assert len(rights.AI_PAIRS) == 8
|
||||
assert len(rights.flat({})) == 13
|
||||
|
||||
def test_values_are_kept_and_anything_but_a_boolean_is_unknown(self):
|
||||
flat = rights.flat(
|
||||
{
|
||||
"uses": {
|
||||
"read": True,
|
||||
"tdm": False,
|
||||
"redistribute": "unknown",
|
||||
"quote-public": "yes",
|
||||
"accessible-copy": None,
|
||||
"ai": {"internal/local": True, "external/outside": False},
|
||||
}
|
||||
}
|
||||
)
|
||||
assert flat["read"] is True
|
||||
assert flat["tdm"] is False
|
||||
assert flat["redistribute"] == "unknown"
|
||||
assert flat["quote-public"] == "unknown"
|
||||
assert flat["accessible-copy"] == "unknown"
|
||||
assert flat["ai/internal/local"] is True
|
||||
assert flat["ai/external/outside"] is False
|
||||
assert flat["ai/internal/tenancy"] == "unknown"
|
||||
|
||||
|
||||
class TestEffective:
|
||||
def test_without_policies_the_flat_uses_are_returned(self):
|
||||
w = witness({"read": True, "tdm": False})
|
||||
uses, expired = rights.effective(w, [], NOW)
|
||||
assert uses == rights.flat(w["rights"])
|
||||
assert expired is False
|
||||
|
||||
def test_an_expired_record_allows_nothing(self):
|
||||
w = witness({"read": True}, valid_until="2026-10-11T00:00:00Z")
|
||||
uses, expired = rights.effective(w, [policy({"tdm": True})], NOW)
|
||||
assert expired is True
|
||||
assert set(uses) == set(ALL_KEYS)
|
||||
assert set(uses.values()) == {False}
|
||||
|
||||
def test_a_record_is_current_until_the_instant_it_expires(self):
|
||||
w = witness({"read": True}, valid_until="2026-10-11T00:00:01Z")
|
||||
assert rights.effective(w, [], NOW)[1] is False
|
||||
assert rights.effective(w, [], "2026-10-11T00:00:01Z")[1] is True
|
||||
|
||||
def test_a_null_expiry_never_expires(self):
|
||||
w = witness({"read": True}, valid_until=None)
|
||||
assert rights.effective(w, [], "9999-01-01T00:00:00Z")[1] is False
|
||||
|
||||
def test_a_policy_settles_an_unknown_use(self):
|
||||
w = witness({"read": True})
|
||||
uses, _ = rights.effective(
|
||||
w, [policy({"tdm": True, "ai": {"internal/tenancy": True}})], NOW
|
||||
)
|
||||
assert uses["tdm"] is True
|
||||
assert uses["ai/internal/tenancy"] is True
|
||||
assert uses["redistribute"] == "unknown"
|
||||
|
||||
def test_a_policy_never_overrides_a_known_use(self):
|
||||
w = witness({"read": False, "tdm": True})
|
||||
uses, _ = rights.effective(w, [policy({"read": True, "tdm": False})], NOW)
|
||||
assert uses["read"] is False
|
||||
assert uses["tdm"] is True
|
||||
|
||||
def test_a_policy_can_settle_a_use_as_false(self):
|
||||
uses, _ = rights.effective(witness(), [policy({"read": False})], NOW)
|
||||
assert uses["read"] is False
|
||||
|
||||
def test_the_first_applicable_policy_to_speak_wins(self):
|
||||
policies = [policy({"read": True}), policy({"read": False, "tdm": True})]
|
||||
uses, _ = rights.effective(witness(), policies, NOW)
|
||||
assert uses["read"] is True
|
||||
assert uses["tdm"] is True
|
||||
|
||||
def test_a_policy_that_says_unknown_settles_nothing(self):
|
||||
uses, _ = rights.effective(witness(), [policy({"read": "unknown"})], NOW)
|
||||
assert uses["read"] == "unknown"
|
||||
|
||||
def test_a_policy_without_uses_changes_nothing(self):
|
||||
uses, _ = rights.effective(witness({"read": True}), [{"applies_to": {}}], NOW)
|
||||
assert uses == rights.flat({"uses": {"read": True}})
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("when", "applies"),
|
||||
[
|
||||
({"at": "2026-10-11T00:00:00Z"}, True),
|
||||
({"at": "2026-10-11T00:00:01Z"}, False),
|
||||
({"valid_until": "2026-10-11T00:00:00Z"}, False),
|
||||
({"valid_until": "2026-10-11T00:00:01Z"}, True),
|
||||
({"valid_until": None, "at": None}, True),
|
||||
],
|
||||
)
|
||||
def test_a_policy_applies_from_its_time_to_its_expiry(self, when, applies):
|
||||
p = dict(policy({"read": True}), **when)
|
||||
uses, _ = rights.effective(witness(), [p], NOW)
|
||||
assert (uses["read"] is True) is applies
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("applies", "w", "expected"),
|
||||
[
|
||||
({"basis": ["subscription"]}, witness(), True),
|
||||
({"basis": ["open-license"]}, witness(), False),
|
||||
({"license": ["CC-BY-4.0"]}, witness(license={"spdx": "CC-BY-4.0"}), True),
|
||||
({"license": ["CC-BY-4.0"]}, witness(license={"spdx": "MIT"}), False),
|
||||
({"license": ["CC-BY-4.0"]}, witness(license={"text": "terms"}), False),
|
||||
({"license": ["CC-BY-4.0"]}, witness(license=None), False),
|
||||
({"license": ["CC-BY-4.0"]}, witness(), False),
|
||||
({"doi_prefixes": ["10.5555/"]}, witness(), True),
|
||||
({"doi_prefixes": ["10.1/", "10.5555/"]}, witness(), True),
|
||||
({"doi_prefixes": ["10.1/"]}, witness(), False),
|
||||
({"doi_prefixes": ["10.5555/"]}, witness(origin={}), False),
|
||||
({"doi_prefixes": ["10.5555/"]}, witness(origin=None), False),
|
||||
({"doi_prefixes": ["10.5555/"]}, witness(origin={"doi": None}), False),
|
||||
({"witnesses": [SHA]}, witness(), True),
|
||||
({"witnesses": [OTHER_SHA]}, witness(), False),
|
||||
({}, witness(), True),
|
||||
(
|
||||
{"basis": ["subscription"], "doi_prefixes": ["10.1/"]},
|
||||
witness(),
|
||||
False,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_a_policy_applies_to_the_witnesses_it_describes(self, applies, w, expected):
|
||||
p = policy({"read": True})
|
||||
p["applies_to"] = applies
|
||||
uses, _ = rights.effective(w, [p], NOW)
|
||||
assert (uses["read"] is True) is expected
|
||||
|
||||
|
||||
class TestCombine:
|
||||
@pytest.mark.parametrize(
|
||||
("values", "combined"),
|
||||
[
|
||||
([True, True], True),
|
||||
([True, False], False),
|
||||
([True, "unknown"], "unknown"),
|
||||
([False, "unknown"], False),
|
||||
(["unknown", "unknown"], "unknown"),
|
||||
([False, False], False),
|
||||
([True], True),
|
||||
(["unknown", True, False], False),
|
||||
],
|
||||
)
|
||||
def test_false_beats_unknown_beats_true(self, values, combined):
|
||||
flats = [dict.fromkeys(ALL_KEYS, v) for v in values]
|
||||
assert set(rights.combine(flats).values()) == {combined}
|
||||
|
||||
def test_it_combines_each_use_separately(self):
|
||||
a = rights.flat({"uses": {"read": True, "tdm": True}})
|
||||
b = rights.flat({"uses": {"read": True, "tdm": False}})
|
||||
out = rights.combine([a, b])
|
||||
assert out["read"] is True
|
||||
assert out["tdm"] is False
|
||||
assert out["redistribute"] == "unknown"
|
||||
|
||||
def test_combining_one_record_is_that_record(self):
|
||||
flat = rights.flat({"uses": {"read": True, "tdm": False}})
|
||||
assert rights.combine([flat]) == flat
|
||||
|
||||
|
||||
class TestCheck:
|
||||
ALL_TRUE = dict.fromkeys(ALL_KEYS, True)
|
||||
|
||||
def test_a_stage_with_no_requirements_is_allowed(self):
|
||||
assert rights.check({}, "outside", {}) == (True, [])
|
||||
|
||||
def test_a_use_that_is_true_is_met(self):
|
||||
assert rights.check({"uses": ["read", "tdm"]}, "local", self.ALL_TRUE) == (
|
||||
True,
|
||||
[],
|
||||
)
|
||||
|
||||
def test_false_and_unknown_uses_are_lacking_and_unknown_is_marked(self):
|
||||
uses = dict(self.ALL_TRUE, tdm=False, read="unknown")
|
||||
allowed, lacking = rights.check(
|
||||
{"uses": ["tdm", "read", "redistribute"]}, "local", uses
|
||||
)
|
||||
assert allowed is False
|
||||
assert lacking == ["tdm", "read?"]
|
||||
|
||||
def test_an_ai_stage_needs_the_purpose_and_processor_pair(self):
|
||||
stage = {"ai": {"purpose": "external"}}
|
||||
assert rights.check(stage, "tenancy", self.ALL_TRUE) == (True, [])
|
||||
uses = dict(self.ALL_TRUE, **{"ai/external/tenancy": False})
|
||||
assert rights.check(stage, "tenancy", uses) == (False, ["ai/external/tenancy"])
|
||||
uses = dict(self.ALL_TRUE, **{"ai/external/tenancy": "unknown"})
|
||||
assert rights.check(stage, "tenancy", uses) == (False, ["ai/external/tenancy?"])
|
||||
|
||||
def test_a_processor_the_stage_does_not_allow_is_lacking_before_its_pair(self):
|
||||
stage = {"ai": {"purpose": "internal", "processors": ["local", "tenancy"]}}
|
||||
uses = dict(self.ALL_TRUE, **{"ai/internal/outside": "unknown"})
|
||||
assert rights.check(stage, "outside", uses) == (
|
||||
False,
|
||||
["processor:outside", "ai/internal/outside?"],
|
||||
)
|
||||
assert rights.check(stage, "local", uses) == (True, [])
|
||||
|
||||
def test_an_empty_list_of_allowed_processors_allows_any(self):
|
||||
stage = {"ai": {"purpose": "internal", "processors": []}}
|
||||
assert rights.check(stage, "outside", self.ALL_TRUE) == (True, [])
|
||||
|
||||
def test_lacking_lists_uses_before_the_ai_pair(self):
|
||||
stage = {"uses": ["tdm"], "ai": {"purpose": "internal"}}
|
||||
uses = dict(self.ALL_TRUE, tdm="unknown", **{"ai/internal/local": False})
|
||||
assert rights.check(stage, "local", uses)[1] == ["tdm?", "ai/internal/local"]
|
||||
|
||||
|
||||
class TestBackendAllowed:
|
||||
CAP: ClassVar[dict] = {"id": "cap", "kind": "revenue-cap", "usd": 100}
|
||||
COMPETE: ClassVar[dict] = {
|
||||
"id": "rival",
|
||||
"kind": "no-competition",
|
||||
"vendor": "acme",
|
||||
}
|
||||
PURPOSE: ClassVar[dict] = {
|
||||
"id": "why",
|
||||
"kind": "purpose",
|
||||
"allowed": ["research", "education"],
|
||||
}
|
||||
|
||||
def test_a_backend_without_conditions_is_allowed_to_anyone(self):
|
||||
assert rights.backend_allowed({}, {}) == (True, [])
|
||||
assert rights.backend_allowed(
|
||||
{"conditions": []}, {"purpose": "commercial"}
|
||||
) == (
|
||||
True,
|
||||
[],
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("profile", "passes"),
|
||||
[
|
||||
({"revenue_or_funding_usd": {"at_most": 100}}, True),
|
||||
({"revenue_or_funding_usd": {"at_most": 99}}, True),
|
||||
({"revenue_or_funding_usd": {"at_most": 101}}, False),
|
||||
({"revenue_or_funding_usd": {"at_most": None}}, False),
|
||||
({"revenue_or_funding_usd": {}}, False),
|
||||
({"revenue_or_funding_usd": None}, False),
|
||||
({}, False),
|
||||
],
|
||||
)
|
||||
def test_a_revenue_cap_needs_a_known_bound_within_it(self, profile, passes):
|
||||
assert rights.backend_allowed({"conditions": [self.CAP]}, profile) == (
|
||||
(True, []) if passes else (False, ["cap"])
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("competes_with", "passes"),
|
||||
[
|
||||
([], True),
|
||||
(["other"], True),
|
||||
(["other", "acme"], False),
|
||||
(["acme"], False),
|
||||
(None, False),
|
||||
],
|
||||
)
|
||||
def test_a_no_competition_condition_needs_a_known_list_without_the_vendor(
|
||||
self, competes_with, passes
|
||||
):
|
||||
profile = {"competes_with": competes_with}
|
||||
assert rights.backend_allowed({"conditions": [self.COMPETE]}, profile) == (
|
||||
(True, []) if passes else (False, ["rival"])
|
||||
)
|
||||
assert rights.backend_allowed({"conditions": [self.COMPETE]}, {}) == (
|
||||
False,
|
||||
["rival"],
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("purpose", "passes"),
|
||||
[("research", True), ("education", True), ("commercial", False), (None, False)],
|
||||
)
|
||||
def test_a_purpose_condition_needs_a_listed_purpose(self, purpose, passes):
|
||||
assert rights.backend_allowed(
|
||||
{"conditions": [self.PURPOSE]}, {"purpose": purpose}
|
||||
) == ((True, []) if passes else (False, ["why"]))
|
||||
|
||||
def test_a_condition_of_an_unknown_kind_always_fails(self):
|
||||
backend = {"conditions": [{"id": "novel", "kind": "from-the-future"}]}
|
||||
assert rights.backend_allowed(backend, {"purpose": "research"}) == (
|
||||
False,
|
||||
["novel"],
|
||||
)
|
||||
|
||||
def test_every_failed_condition_is_listed_in_order(self):
|
||||
backend = {"conditions": [self.PURPOSE, self.CAP, self.COMPETE]}
|
||||
profile = {"purpose": "commercial", "revenue_or_funding_usd": {"at_most": 5}}
|
||||
assert rights.backend_allowed(backend, profile) == (False, ["why", "rival"])
|
||||
@@ -0,0 +1,393 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""The schema store, the keyword-subset lint and validation (paperspec.schema)."""
|
||||
|
||||
import json
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
from jsonschema import Draft202012Validator
|
||||
|
||||
from conftest import ROOT, read_jsonl
|
||||
from helpers_lib2 import RECORDS, schema_store
|
||||
from paperspec import schema
|
||||
|
||||
SCHEMA_FILES = sorted(p.name for p in (ROOT / "schemas").glob("*.schema.json"))
|
||||
FORMATS = json.loads((ROOT / "registry" / "formats.json").read_text(encoding="utf-8"))[
|
||||
"formats"
|
||||
]
|
||||
|
||||
|
||||
def write_schemas(folder, **schemas):
|
||||
"""Write each keyword argument as `<name>.schema.json` under `folder`, and return the folder as text."""
|
||||
for name, body in schemas.items():
|
||||
full = {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": f"https://example.test/schemas/{name}.schema.json",
|
||||
**body,
|
||||
}
|
||||
(folder / (name + ".schema.json")).write_text(
|
||||
json.dumps(full), encoding="utf-8"
|
||||
)
|
||||
return str(folder)
|
||||
|
||||
|
||||
def object_schema(**properties):
|
||||
"""A closed object schema with the given properties."""
|
||||
return {"type": "object", "properties": properties, "additionalProperties": False}
|
||||
|
||||
|
||||
# --- the repository's schemas ---------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", SCHEMA_FILES)
|
||||
def test_a_repository_schema_is_in_the_subset_and_valid_draft_2020_12(name):
|
||||
body = schema_store().load(name)
|
||||
assert body["$id"].endswith("/" + name)
|
||||
Draft202012Validator.check_schema(body)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("entry", FORMATS, ids=lambda f: f["format"])
|
||||
def test_every_format_names_a_schema_that_exists(entry):
|
||||
assert (ROOT / "schemas" / entry["schema"]).exists() or (
|
||||
ROOT / entry["schema"]
|
||||
).exists()
|
||||
|
||||
|
||||
def test_the_repository_has_a_schema_for_every_fixture_record_format():
|
||||
by_format = {f["format"]: f["schema"] for f in FORMATS}
|
||||
for path in sorted(RECORDS.glob("*.json")):
|
||||
record = json.loads(path.read_text(encoding="utf-8"))
|
||||
if "format" in record:
|
||||
assert record["format"] in by_format, path.name
|
||||
|
||||
|
||||
def fixture_schema(path, record):
|
||||
"""The schema a record under `fixtures/records` is written for."""
|
||||
if path.name == "view.vl.json":
|
||||
return "view.schema.json"
|
||||
if path.name == "layer.json":
|
||||
return "layer.schema.json"
|
||||
return {f["format"]: f["schema"] for f in FORMATS}[record["format"]]
|
||||
|
||||
|
||||
FIXTURE_RECORDS = sorted(
|
||||
p
|
||||
for p in RECORDS.rglob("*.json")
|
||||
if "log" not in p.relative_to(RECORDS).parts
|
||||
and "datasets" not in p.relative_to(RECORDS).parts
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path", FIXTURE_RECORDS, ids=lambda p: p.relative_to(RECORDS).as_posix()
|
||||
)
|
||||
def test_a_fixture_record_is_valid_under_its_schema(path):
|
||||
record = json.loads(path.read_text(encoding="utf-8"))
|
||||
assert schema.validate(schema_store(), fixture_schema(path, record), record) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path",
|
||||
sorted(RECORDS.rglob("annotations.jsonl")),
|
||||
ids=lambda p: p.parent.name,
|
||||
)
|
||||
def test_every_annotation_of_the_fixture_layers_is_valid(path):
|
||||
for record in read_jsonl(path):
|
||||
assert schema.validate(schema_store(), "annotation.schema.json", record) == []
|
||||
|
||||
|
||||
# --- the lint -------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"body",
|
||||
[
|
||||
{"type": "string", "pattern": "^[a-z]+$"},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {"a": {"type": "integer", "minimum": 0}},
|
||||
"required": ["a"],
|
||||
},
|
||||
{"items": {"type": "string"}, "minItems": 1, "uniqueItems": True},
|
||||
{"additionalProperties": False},
|
||||
{"$defs": {"x": {"enum": [1, 2]}}, "$ref": "#/$defs/x"},
|
||||
{"oneOf": [{"const": 1}, {"const": 2}], "not": {"const": 3}},
|
||||
{"if": {"type": "string"}, "then": {"minLength": 1}, "else": {"type": "null"}},
|
||||
{"prefixItems": [{"type": "string"}], "contains": {"type": "integer"}},
|
||||
{
|
||||
"patternProperties": {"^x-": {"type": "string"}},
|
||||
"propertyNames": {"maxLength": 4},
|
||||
},
|
||||
{"description": "Free text may say \\d and (?<name>) without being a pattern."},
|
||||
{"type": "string", "format": "date-time", "default": "", "deprecated": True},
|
||||
True,
|
||||
["not", "a", "schema"],
|
||||
],
|
||||
ids=[
|
||||
"pattern",
|
||||
"properties",
|
||||
"items",
|
||||
"boolean-additional-properties",
|
||||
"defs-and-ref",
|
||||
"one-of-and-not",
|
||||
"if-then-else",
|
||||
"prefix-items-and-contains",
|
||||
"pattern-properties-and-names",
|
||||
"description-mentioning-syntax",
|
||||
"format-default-deprecated",
|
||||
"boolean-schema",
|
||||
"not-a-mapping",
|
||||
],
|
||||
)
|
||||
def test_lint_accepts_schemas_inside_the_subset(body):
|
||||
assert schema.lint(body, "ok.schema.json") is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("body", "where"),
|
||||
[
|
||||
({"unevaluatedProperties": False}, "w.json: keyword unevaluatedProperties"),
|
||||
(
|
||||
{"properties": {"a": {"dependentSchemas": {}}}},
|
||||
"w.json/a: keyword dependentSchemas",
|
||||
),
|
||||
({"$defs": {"d": {"$anchor": "x"}}}, "w.json/d: keyword $anchor"),
|
||||
(
|
||||
{"items": {"contentEncoding": "base64"}},
|
||||
"w.json/items: keyword contentEncoding",
|
||||
),
|
||||
(
|
||||
{"additionalProperties": {"unevaluatedItems": True}},
|
||||
"w.json/additionalProperties: keyword",
|
||||
),
|
||||
({"propertyNames": {"$dynamicRef": "#"}}, "w.json/propertyNames: keyword"),
|
||||
({"not": {"multipleOf": 2}}, "w.json/not: keyword multipleOf"),
|
||||
({"if": {"$vocabulary": {}}}, "w.json/if: keyword"),
|
||||
({"then": {"contentMediaType": "x"}}, "w.json/then: keyword"),
|
||||
({"else": {"readOnly": True}}, "w.json/else: keyword readOnly"),
|
||||
({"contains": {"minContains": 1}}, "w.json/contains: keyword minContains"),
|
||||
(
|
||||
{"oneOf": [{"type": "null"}, {"title": "t", "writeOnly": True}]},
|
||||
"w.json/oneOf/1: keyword writeOnly",
|
||||
),
|
||||
({"anyOf": [{"x": 1}]}, "w.json/anyOf/0: keyword x"),
|
||||
({"allOf": [{}, {}, {"unevaluatedItems": 1}]}, "w.json/allOf/2: keyword"),
|
||||
({"prefixItems": [{"dependentRequired2": 1}]}, "w.json/prefixItems/0: keyword"),
|
||||
],
|
||||
)
|
||||
def test_lint_refuses_keywords_outside_the_subset_and_says_where(body, where):
|
||||
with pytest.raises(schema.SchemaError) as excinfo:
|
||||
schema.lint(body, "w.json")
|
||||
assert where in str(excinfo.value)
|
||||
assert "outside the subset" in str(excinfo.value)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("token", schema.FORBIDDEN_PATTERN_TOKENS)
|
||||
def test_lint_refuses_regex_syntax_that_differs_between_dialects(token):
|
||||
with pytest.raises(schema.SchemaError, match="dialect-dependent syntax"):
|
||||
schema.lint({"pattern": "^a" + token + "z$"}, "w.json")
|
||||
with pytest.raises(schema.SchemaError, match="dialect-dependent syntax"):
|
||||
schema.lint({"patternProperties": {"^a" + token: {}}}, "w.json")
|
||||
|
||||
|
||||
def test_lint_looks_into_nested_patterns():
|
||||
body = {"properties": {"a": {"items": {"oneOf": [{"pattern": "\\d+"}]}}}}
|
||||
with pytest.raises(schema.SchemaError, match=r"w.json/a/items/oneOf/0"):
|
||||
schema.lint(body, "w.json")
|
||||
|
||||
|
||||
def test_the_forbidden_tokens_are_the_digit_word_space_and_boundary_classes_and_named_groups():
|
||||
assert set(schema.FORBIDDEN_PATTERN_TOKENS) == {
|
||||
"\\d",
|
||||
"\\w",
|
||||
"\\s",
|
||||
"\\b",
|
||||
"\\D",
|
||||
"\\W",
|
||||
"\\S",
|
||||
"\\B",
|
||||
"(?<",
|
||||
"(?P",
|
||||
}
|
||||
|
||||
|
||||
def test_the_keyword_subset_holds_the_keywords_the_chapter_allows():
|
||||
assert {
|
||||
"$ref",
|
||||
"$defs",
|
||||
"oneOf",
|
||||
"patternProperties",
|
||||
"dependentRequired",
|
||||
} <= schema.KEYWORDS
|
||||
assert (
|
||||
not {"unevaluatedProperties", "$anchor", "dependentSchemas"} & schema.KEYWORDS
|
||||
)
|
||||
|
||||
|
||||
# --- the store ---------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_load_reads_a_schema_once_and_lints_it(tmp_path):
|
||||
folder = write_schemas(tmp_path, a=object_schema(x={"type": "string"}))
|
||||
store = schema.Store(folder)
|
||||
first = store.load("a.schema.json")
|
||||
assert first["type"] == "object"
|
||||
(tmp_path / "a.schema.json").write_text("{}", encoding="utf-8")
|
||||
assert store.load("a.schema.json") is first
|
||||
|
||||
|
||||
def test_load_refuses_a_schema_outside_the_subset(tmp_path):
|
||||
folder = write_schemas(tmp_path, bad={"unevaluatedProperties": False})
|
||||
with pytest.raises(schema.SchemaError, match=r"bad\.schema\.json"):
|
||||
schema.Store(folder).load("bad.schema.json")
|
||||
|
||||
|
||||
def test_load_of_a_missing_schema_fails(tmp_path):
|
||||
with pytest.raises(FileNotFoundError):
|
||||
schema.Store(str(tmp_path)).load("nothing.schema.json")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def referencing_store(tmp_path):
|
||||
common = {
|
||||
"$defs": {
|
||||
"name": {"type": "string", "minLength": 2},
|
||||
"a/b": {"type": "integer"},
|
||||
"t~k": {"type": "null"},
|
||||
},
|
||||
}
|
||||
person = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"$ref": "common.schema.json#/$defs/name"},
|
||||
"age": {"type": "integer"},
|
||||
},
|
||||
"required": ["name"],
|
||||
"additionalProperties": False,
|
||||
}
|
||||
return schema.Store(write_schemas(tmp_path, common=common, person=person))
|
||||
|
||||
|
||||
def test_deref_follows_a_pointer_into_the_same_or_another_schema(referencing_store):
|
||||
name, node = referencing_store.deref(
|
||||
"person.schema.json", "common.schema.json#/$defs/name"
|
||||
)
|
||||
assert (name, node) == ("common.schema.json", {"type": "string", "minLength": 2})
|
||||
name, node = referencing_store.deref("person.schema.json", "#/properties/age")
|
||||
assert (name, node) == ("person.schema.json", {"type": "integer"})
|
||||
|
||||
|
||||
def test_deref_without_a_fragment_gives_the_whole_schema(referencing_store):
|
||||
name, node = referencing_store.deref("person.schema.json", "common.schema.json")
|
||||
assert name == "common.schema.json"
|
||||
assert node is referencing_store.load("common.schema.json")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("pointer", "expected"),
|
||||
[("#/$defs/a~1b", {"type": "integer"}), ("#/$defs/t~0k", {"type": "null"})],
|
||||
)
|
||||
def test_deref_unescapes_pointer_tokens(referencing_store, pointer, expected):
|
||||
assert referencing_store.deref("common.schema.json", pointer)[1] == expected
|
||||
|
||||
|
||||
def test_deref_of_a_missing_member_fails(referencing_store):
|
||||
with pytest.raises(KeyError):
|
||||
referencing_store.deref("common.schema.json", "#/$defs/nope")
|
||||
|
||||
|
||||
def test_validate_follows_references_between_the_schemas_of_a_store(referencing_store):
|
||||
assert (
|
||||
schema.validate(
|
||||
referencing_store, "person.schema.json", {"name": "Ana", "age": 3}
|
||||
)
|
||||
== []
|
||||
)
|
||||
got = schema.validate(referencing_store, "person.schema.json", {"name": "A"})
|
||||
assert got == ["/name: 'A' is too short"]
|
||||
|
||||
|
||||
def test_errors_are_reported_as_a_path_and_a_message_in_path_order(referencing_store):
|
||||
got = schema.validate(
|
||||
referencing_store, "person.schema.json", {"name": "A", "age": "x", "extra": 1}
|
||||
)
|
||||
assert got[0] == "/: Additional properties are not allowed ('extra' was unexpected)"
|
||||
assert got[1:] == ["/age: 'x' is not of type 'integer'", "/name: 'A' is too short"]
|
||||
|
||||
|
||||
def test_an_instance_of_the_wrong_shape_has_a_root_error(referencing_store):
|
||||
assert schema.validate(referencing_store, "person.schema.json", [1]) == [
|
||||
"/: [1] is not of type 'object'"
|
||||
]
|
||||
|
||||
|
||||
def test_array_indexes_appear_in_the_error_path(tmp_path):
|
||||
folder = write_schemas(
|
||||
tmp_path, list={"type": "array", "items": {"type": "integer"}}
|
||||
)
|
||||
got = schema.validate(schema.Store(folder), "list.schema.json", [1, "x", 3, "y"])
|
||||
assert got == [
|
||||
"/1: 'x' is not of type 'integer'",
|
||||
"/3: 'y' is not of type 'integer'",
|
||||
]
|
||||
|
||||
|
||||
def test_registry_and_validators_are_built_once(referencing_store):
|
||||
registry = referencing_store.registry()
|
||||
assert referencing_store.registry() is registry
|
||||
validator = referencing_store.validator("person.schema.json")
|
||||
assert referencing_store.validator("person.schema.json") is validator
|
||||
|
||||
|
||||
def test_the_registry_holds_every_schema_file_of_the_folder_by_its_id(
|
||||
referencing_store,
|
||||
):
|
||||
registry = referencing_store.registry()
|
||||
assert (
|
||||
registry.get_or_retrieve(
|
||||
"https://example.test/schemas/common.schema.json"
|
||||
).value
|
||||
is not None
|
||||
)
|
||||
assert len(registry) == 2
|
||||
|
||||
|
||||
def test_an_annotation_of_the_fixture_layer_validates_and_a_broken_one_does_not():
|
||||
annotation = read_jsonl(RECORDS / "layer" / "annotations.jsonl")[0]
|
||||
store = schema_store()
|
||||
assert schema.validate(store, "annotation.schema.json", annotation) == []
|
||||
broken = {k: v for k, v in annotation.items() if k != "creator"}
|
||||
errors = schema.validate(store, "annotation.schema.json", broken)
|
||||
assert errors
|
||||
assert any("creator" in e for e in errors)
|
||||
assert all(e.startswith("/") for e in errors)
|
||||
|
||||
|
||||
# --- the optional packages --------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("missing", ["jsonschema", "referencing"])
|
||||
def test_validation_without_the_packages_names_the_extra(
|
||||
tmp_path, monkeypatch, missing
|
||||
):
|
||||
folder = write_schemas(tmp_path, a={"type": "string"})
|
||||
monkeypatch.setitem(sys.modules, missing, None)
|
||||
store = schema.Store(folder)
|
||||
with pytest.raises(ImportError, match=r"paperspec\[schema\]"):
|
||||
store.validator("a.schema.json")
|
||||
with pytest.raises(ImportError, match=r"paperspec\[schema\]"):
|
||||
store.registry()
|
||||
|
||||
|
||||
def test_loading_and_linting_need_neither_package(tmp_path, monkeypatch):
|
||||
folder = write_schemas(tmp_path, a={"type": "string"})
|
||||
monkeypatch.setitem(sys.modules, "jsonschema", None)
|
||||
monkeypatch.setitem(sys.modules, "referencing", None)
|
||||
store = schema.Store(folder)
|
||||
assert store.load("a.schema.json")["type"] == "string"
|
||||
assert store.deref("a.schema.json", "#/type") == ("a.schema.json", "string")
|
||||
@@ -0,0 +1,339 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Biological sequences tied to words: identities and rules (paperspec.sequences)."""
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
|
||||
import pytest
|
||||
from hypothesis import given, settings, strategies as st
|
||||
|
||||
from conftest import read_jsonl, vector_cases
|
||||
from helpers_lib2 import PROPS, RECORDS, accepted_by_schema, rule_chapters
|
||||
from paperspec import sequences
|
||||
|
||||
CASES = vector_cases("sequences")
|
||||
IDENTITIES = [c for c in CASES if c["op"] == "identity"]
|
||||
PROBLEMS = [c for c in CASES if c["op"] == "problems"]
|
||||
RULES = (
|
||||
"layer.sequence.value",
|
||||
"layer.sequence.identity",
|
||||
"layer.sequence.region",
|
||||
"layer.sequence.variant",
|
||||
)
|
||||
BIOLOGY = read_jsonl(RECORDS / "biology" / "annotations.jsonl")
|
||||
PEPTIDE = next(c for c in PROBLEMS if c["id"] == "valid-regions")["annotation"]
|
||||
VALUE = PEPTIDE["body"]["value"]
|
||||
|
||||
|
||||
def edited(annotation, **body):
|
||||
"""A copy of `annotation` whose body has the given fields replaced; None removes a field."""
|
||||
out = copy.deepcopy(annotation)
|
||||
for key, value in body.items():
|
||||
if value is None:
|
||||
out["body"].pop(key, None)
|
||||
else:
|
||||
out["body"][key] = value
|
||||
return out
|
||||
|
||||
|
||||
def with_value(value, alphabet="protein", **body):
|
||||
"""The peptide annotation over `value`, with its identity made to match."""
|
||||
return edited(
|
||||
PEPTIDE,
|
||||
value=value,
|
||||
alphabet=alphabet,
|
||||
identity=sequences.identity(alphabet, value),
|
||||
regions=None,
|
||||
**body,
|
||||
)
|
||||
|
||||
|
||||
def messages(annotation):
|
||||
"""The (rule, message) pairs `problems` reports for `annotation`."""
|
||||
return [(x["rule"], x["message"]) for x in sequences.problems(annotation)]
|
||||
|
||||
|
||||
# --- identities -----------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", IDENTITIES, ids=lambda c: c["id"])
|
||||
def test_identity_vectors(case):
|
||||
assert sequences.identity(case["alphabet"], case["text"]) == case["expect"]
|
||||
|
||||
|
||||
def test_the_identity_is_the_alphabet_and_sixteen_hex_digits_of_the_residues_hash():
|
||||
digest = hashlib.sha256(b"MKV").hexdigest()[:16]
|
||||
assert sequences.identity("protein", "MKV") == "protein:" + digest
|
||||
assert sequences.identity("dna", "MKV") == "dna:" + digest
|
||||
|
||||
|
||||
def test_residues_drop_the_gaps_and_nothing_else():
|
||||
assert sequences.residues("AC-GT--") == "ACGT"
|
||||
assert sequences.residues("---") == ""
|
||||
assert sequences.residues("ac*") == "ac*"
|
||||
|
||||
|
||||
@given(st.text(alphabet="ACDEFGHIKLMNPQRSTVWY", min_size=1, max_size=40), st.data())
|
||||
@PROPS
|
||||
def test_gaps_change_the_numbering_of_nothing_and_the_identity_of_nothing(
|
||||
residues, data
|
||||
):
|
||||
positions = data.draw(st.lists(st.integers(0, len(residues)), max_size=6))
|
||||
gapped = residues
|
||||
for p in sorted(positions, reverse=True):
|
||||
gapped = gapped[:p] + "-" + gapped[p:]
|
||||
assert sequences.residues(gapped) == residues
|
||||
assert sequences.identity("protein", gapped) == sequences.identity(
|
||||
"protein", residues
|
||||
)
|
||||
|
||||
|
||||
def test_the_alphabets_allow_the_letters_the_chapter_lists():
|
||||
protein, dna, rna = (sequences.ALPHABETS[a] for a in ("protein", "dna", "rna"))
|
||||
assert set("ACDEFGHIKLMNPQRSTVWY") <= protein
|
||||
assert set("UO*-BXZJ") <= protein
|
||||
assert "U" not in dna
|
||||
assert "T" not in rna
|
||||
assert {"U", "*"}.isdisjoint(dna)
|
||||
assert all("-" in letters for letters in (protein, dna, rna))
|
||||
assert set("RYKMSWBDHVN") <= dna & rna
|
||||
|
||||
|
||||
# --- rules -----------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rule", RULES)
|
||||
def test_the_rules_are_registered_in_the_layers_chapter(rule):
|
||||
assert rule_chapters()[rule] == "spec/18-layers.md"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", PROBLEMS, ids=lambda c: c["id"])
|
||||
def test_problems_vectors(case):
|
||||
got = sequences.problems(case["annotation"])
|
||||
assert got == case["expect"]
|
||||
assert accepted_by_schema(case["annotation"]) is case["schema"]
|
||||
assert {x["rule"] for x in got} <= set(rule_chapters())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", PROBLEMS, ids=lambda c: c["id"])
|
||||
def test_a_vector_is_named_for_the_rule_it_breaks(case):
|
||||
stem = case["id"].split("-", 1)[0]
|
||||
rules = {x["rule"] for x in case["expect"]}
|
||||
if stem == "valid":
|
||||
assert rules == set()
|
||||
else:
|
||||
assert "layer.sequence." + stem in rules
|
||||
|
||||
|
||||
def test_every_rule_is_broken_by_some_vector():
|
||||
assert {x["rule"] for c in PROBLEMS for x in c["expect"]} == set(RULES)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("annotation", BIOLOGY, ids=lambda a: a["body"]["role"])
|
||||
def test_the_biology_layer_breaks_no_rule(annotation):
|
||||
assert annotation["body"]["kind"] == "sequence"
|
||||
assert sequences.problems(annotation) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"body", [{"kind": "comment"}, {}], ids=["other-kind", "no-kind"]
|
||||
)
|
||||
def test_an_annotation_of_another_kind_is_refused(body):
|
||||
with pytest.raises(ValueError, match="not a sequence annotation"):
|
||||
sequences.problems({"body": body})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("alphabet", [None, "peptide", "DNA"])
|
||||
def test_an_unknown_alphabet_is_the_only_problem_reported(alphabet):
|
||||
annotation = edited(PEPTIDE, alphabet=alphabet, identity="whatever")
|
||||
assert [r for r, _ in messages(annotation)] == ["layer.sequence.value"]
|
||||
assert "dna, protein, rna" in messages(annotation)[0][1]
|
||||
|
||||
|
||||
def test_letters_the_alphabet_lacks_are_listed_once_each_in_order():
|
||||
got = messages(with_value("EkZk*-?", "dna"))
|
||||
assert got == [
|
||||
("layer.sequence.value", "'*', '?', 'E', 'Z', 'k' is not a dna residue")
|
||||
]
|
||||
|
||||
|
||||
def test_a_value_of_gaps_only_has_no_residue():
|
||||
got = messages(with_value("---"))
|
||||
assert got == [("layer.sequence.value", "the value has no residue, only gaps")]
|
||||
|
||||
|
||||
def test_an_empty_value_has_no_residue_and_hashes_the_empty_string():
|
||||
assert messages(with_value("")) == [
|
||||
("layer.sequence.value", "the value has no residue, only gaps")
|
||||
]
|
||||
|
||||
|
||||
def test_a_wrong_identity_is_replaced_by_the_right_one_in_the_message():
|
||||
got = messages(edited(PEPTIDE, identity="protein:0000000000000000"))
|
||||
assert got == [
|
||||
(
|
||||
"layer.sequence.identity",
|
||||
"the identity is not " + sequences.identity("protein", VALUE),
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_a_missing_identity_breaks_the_identity_rule():
|
||||
assert [r for r, _ in messages(edited(PEPTIDE, identity=None))] == [
|
||||
"layer.sequence.identity"
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("region", "first_message"),
|
||||
[
|
||||
({"start": 50, "end": 49}, "region 0 starts at 50, after its end 49"),
|
||||
({"start": 41, "end": 50}, "region 0, 41-50, is outside the residues 42-76"),
|
||||
({"start": 50, "end": 77}, "region 0, 50-77, is outside the residues 42-76"),
|
||||
({"end": 50}, "region 0, 0-50, is outside the residues 42-76"),
|
||||
({"start": 50}, "region 0 starts at 50, after its end 0"),
|
||||
({}, "region 0, 0-0, is outside the residues 42-76"),
|
||||
],
|
||||
ids=["reversed", "before-first", "after-last", "no-start", "no-end", "empty"],
|
||||
)
|
||||
def test_regions_must_lie_within_the_residues(region, first_message):
|
||||
got = messages(edited(PEPTIDE, regions=[region]))
|
||||
assert got == [("layer.sequence.region", first_message)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"region",
|
||||
[{"start": 42, "end": 42}, {"start": 76, "end": 76}, {"start": 42, "end": 76}],
|
||||
)
|
||||
def test_regions_may_touch_either_end_or_one_residue(region):
|
||||
assert sequences.problems(edited(PEPTIDE, regions=[region])) == []
|
||||
|
||||
|
||||
def test_each_region_is_judged_and_numbered_on_its_own():
|
||||
regions = [
|
||||
{"start": 44, "end": 51},
|
||||
{"start": 90, "end": 91},
|
||||
{"start": 60, "end": 55},
|
||||
]
|
||||
got = messages(edited(PEPTIDE, regions=regions))
|
||||
assert [r for r, _ in got] == ["layer.sequence.region"] * 2
|
||||
assert got[0][1].startswith("region 1, 90-91")
|
||||
assert got[1][1].startswith("region 2 starts at 60")
|
||||
|
||||
|
||||
def test_a_sequence_without_a_start_is_numbered_from_one():
|
||||
annotation = edited(PEPTIDE, start=None, regions=[{"start": 1, "end": 35}])
|
||||
assert sequences.problems(annotation) == []
|
||||
outside = edited(PEPTIDE, start=None, regions=[{"start": 1, "end": 36}])
|
||||
assert messages(outside)[0][1] == "region 0, 1-36, is outside the residues 1-35"
|
||||
|
||||
|
||||
def test_gaps_are_not_numbered():
|
||||
value = "LSDED-FKAVFGMTRSAFANLPLWKQQNLKKEKGLF"
|
||||
annotation = with_value(value, variants=[{"position": 47, "ref": "F", "alt": "A"}])
|
||||
assert sequences.problems(annotation) == []
|
||||
shifted = with_value(value, variants=[{"position": 48, "ref": "F", "alt": "A"}])
|
||||
assert messages(shifted) == [
|
||||
(
|
||||
"layer.sequence.variant",
|
||||
"variant 0 says F is at 48, but the residue there is K",
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_a_variant_outside_the_residues_is_reported_with_its_numbers():
|
||||
variant = {"position": 100, "ref": "L", "alt": "A"}
|
||||
got = messages(edited(PEPTIDE, variants=[variant]))
|
||||
assert got == [
|
||||
("layer.sequence.variant", "variant 0 is at 100, outside the residues 42-76")
|
||||
]
|
||||
|
||||
|
||||
def test_a_variant_that_names_the_wrong_residue_says_what_is_there():
|
||||
variant = {"position": 42, "ref": "A", "alt": "C"}
|
||||
got = messages(edited(PEPTIDE, variants=[variant]))
|
||||
assert got == [
|
||||
(
|
||||
"layer.sequence.variant",
|
||||
"variant 0 says A is at 42, but the residue there is L",
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_a_variant_to_the_same_residue_is_no_change():
|
||||
variant = {"position": 42, "ref": "L", "alt": "L"}
|
||||
got = messages(edited(PEPTIDE, variants=[variant]))
|
||||
assert got == [("layer.sequence.variant", "variant 0 changes L at 42 to itself")]
|
||||
|
||||
|
||||
def test_a_variant_to_a_letter_the_alphabet_lacks_is_refused():
|
||||
variant = {"position": 42, "ref": "L", "alt": "z"}
|
||||
got = messages(edited(PEPTIDE, variants=[variant]))
|
||||
assert got == [
|
||||
("layer.sequence.variant", "variant 0: 'z' is not a protein residue")
|
||||
]
|
||||
|
||||
|
||||
def test_a_variant_may_delete_with_a_gap_or_stop_with_a_star():
|
||||
variants = [
|
||||
{"position": 42, "ref": "L", "alt": "-"},
|
||||
{"position": 43, "ref": "S", "alt": "*"},
|
||||
]
|
||||
assert sequences.problems(edited(PEPTIDE, variants=variants)) == []
|
||||
|
||||
|
||||
def test_a_variant_outside_the_residues_can_also_change_to_itself():
|
||||
variant = {"position": 0, "ref": "L", "alt": "L"}
|
||||
got = messages(edited(PEPTIDE, variants=[variant]))
|
||||
assert [r for r, _ in got] == ["layer.sequence.variant"] * 2
|
||||
|
||||
|
||||
def test_a_variant_without_fields_breaks_the_rule():
|
||||
got = messages(edited(PEPTIDE, variants=[{}]))
|
||||
assert [r for r, _ in got] == ["layer.sequence.variant"] * 2
|
||||
|
||||
|
||||
def test_each_variant_is_numbered_in_its_message():
|
||||
variants = [
|
||||
{"position": 42, "ref": "L", "alt": "A"},
|
||||
{"position": 43, "ref": "X", "alt": "A"},
|
||||
]
|
||||
got = messages(edited(PEPTIDE, variants=variants))
|
||||
assert [m.split(" ")[1] for _, m in got] == ["1"]
|
||||
|
||||
|
||||
@given(
|
||||
st.text(alphabet="ACDEFGHIKLMNPQRSTVWY", min_size=3, max_size=50),
|
||||
st.integers(1, 500),
|
||||
st.data(),
|
||||
)
|
||||
@settings(PROPS, max_examples=60)
|
||||
def test_a_sequence_whose_regions_and_variants_agree_with_its_residues_breaks_no_rule(
|
||||
residues, start, data
|
||||
):
|
||||
last = start + len(residues) - 1
|
||||
region_start = data.draw(st.integers(start, last))
|
||||
region_end = data.draw(st.integers(region_start, last))
|
||||
position = data.draw(st.integers(start, last))
|
||||
ref = residues[position - start]
|
||||
alt = data.draw(st.sampled_from(sorted(sequences.ALPHABETS["protein"] - {ref})))
|
||||
annotation = edited(
|
||||
PEPTIDE,
|
||||
value=residues,
|
||||
start=start,
|
||||
identity=sequences.identity("protein", residues),
|
||||
regions=[{"start": region_start, "end": region_end}],
|
||||
variants=[{"position": position, "ref": ref, "alt": alt}],
|
||||
)
|
||||
assert sequences.problems(annotation) == []
|
||||
past = edited(annotation, regions=[{"start": region_start, "end": last + 1}])
|
||||
assert [x["rule"] for x in sequences.problems(past)] == ["layer.sequence.region"]
|
||||
wrong = edited(
|
||||
annotation, variants=[{"position": position, "ref": alt, "alt": ref}]
|
||||
)
|
||||
assert [x["rule"] for x in sequences.problems(wrong)] == ["layer.sequence.variant"]
|
||||
@@ -0,0 +1,456 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Chemical structures tied to words: SMILES and SMARTS scanning, identities and rules (paperspec.structures)."""
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
|
||||
import pytest
|
||||
from hypothesis import given, settings, strategies as st
|
||||
|
||||
from conftest import read_jsonl, vector_cases
|
||||
from helpers_lib2 import PROPS, RECORDS, accepted_by_schema, rule_chapters
|
||||
from paperspec import structures
|
||||
|
||||
CASES = vector_cases("structures")
|
||||
SCANS = [c for c in CASES if c["op"] == "scan"]
|
||||
IDENTITIES = [c for c in CASES if c["op"] == "identity"]
|
||||
PROBLEMS = [c for c in CASES if c["op"] == "problems"]
|
||||
RULES = ("layer.structure.value", "layer.structure.identity", "layer.structure.term")
|
||||
CHEMISTRY = read_jsonl(RECORDS / "chemistry" / "annotations.jsonl")
|
||||
SMARTS_TERMS = next(c for c in PROBLEMS if c["id"] == "valid-a-smarts-with-terms")[
|
||||
"annotation"
|
||||
]
|
||||
SMILES_TERMS = next(c for c in PROBLEMS if c["id"] == "valid-a-smiles-with-terms")[
|
||||
"annotation"
|
||||
]
|
||||
|
||||
|
||||
def scanned(text, smarts=False):
|
||||
"""The atoms and sorted bonds `scan` reads from `text`, or None when it refuses the text."""
|
||||
try:
|
||||
count, bonds = structures.scan(text, smarts=smarts)
|
||||
except ValueError:
|
||||
return None
|
||||
return {"atoms": count, "bonds": [list(b) for b in sorted(bonds)]}
|
||||
|
||||
|
||||
def edited(annotation, **body):
|
||||
"""A copy of `annotation` whose body has the given fields replaced; None removes a field."""
|
||||
out = copy.deepcopy(annotation)
|
||||
for key, value in body.items():
|
||||
if value is None:
|
||||
out["body"].pop(key, None)
|
||||
else:
|
||||
out["body"][key] = value
|
||||
return out
|
||||
|
||||
|
||||
def rules_of(annotation):
|
||||
"""The rules `problems` reports for `annotation`, in order."""
|
||||
return [x["rule"] for x in structures.problems(annotation)]
|
||||
|
||||
|
||||
# --- scanning ------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", SCANS, ids=lambda c: c["id"])
|
||||
def test_scan_vectors(case):
|
||||
assert scanned(case["text"], smarts=case["format"] == "smarts") == case["expect"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "expect"),
|
||||
[
|
||||
("C=1CC=1", {"atoms": 3, "bonds": [[0, 1], [0, 2], [1, 2]]}),
|
||||
(
|
||||
"C1CC1C1CC1",
|
||||
{
|
||||
"atoms": 6,
|
||||
"bonds": [[0, 1], [0, 2], [1, 2], [2, 3], [3, 4], [3, 5], [4, 5]],
|
||||
},
|
||||
),
|
||||
("C1(CC1)C", {"atoms": 4, "bonds": [[0, 1], [0, 2], [0, 3], [1, 2]]}),
|
||||
("C(C)(C)C", {"atoms": 4, "bonds": [[0, 1], [0, 2], [0, 3]]}),
|
||||
("C.C.C", {"atoms": 3, "bonds": []}),
|
||||
("[13CH3][2H]", {"atoms": 2, "bonds": [[0, 1]]}),
|
||||
("[Fe+2]", {"atoms": 1, "bonds": []}),
|
||||
("[C@H](F)(Cl)Br", {"atoms": 4, "bonds": [[0, 1], [0, 2], [0, 3]]}),
|
||||
(
|
||||
"CC(=O)OC(=O)C",
|
||||
{"atoms": 7, "bonds": [[0, 1], [1, 2], [1, 3], [3, 4], [4, 5], [4, 6]]},
|
||||
),
|
||||
("C#N", {"atoms": 2, "bonds": [[0, 1]]}),
|
||||
("[[N]]", {"atoms": 1, "bonds": []}),
|
||||
],
|
||||
ids=[
|
||||
"ring-with-explicit-bond",
|
||||
"ring-numbers-reused",
|
||||
"ring-opened-before-a-branch",
|
||||
"branches-after-branches",
|
||||
"dots-separate-components",
|
||||
"isotopes",
|
||||
"charged-metal",
|
||||
"stereo-centre",
|
||||
"anhydride",
|
||||
"triple-bond",
|
||||
"nested-brackets",
|
||||
],
|
||||
)
|
||||
def test_scan_reads_further_smiles(text, expect):
|
||||
assert scanned(text) == expect
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "expect"),
|
||||
[
|
||||
("[C,N]C", {"atoms": 2, "bonds": [[0, 1]]}),
|
||||
("C!C", {"atoms": 2, "bonds": [[0, 1]]}),
|
||||
("[!C;R]~[#7]", {"atoms": 2, "bonds": [[0, 1]]}),
|
||||
("A@a", {"atoms": 2, "bonds": [[0, 1]]}),
|
||||
("C&C", {"atoms": 2, "bonds": [[0, 1]]}),
|
||||
],
|
||||
)
|
||||
def test_scan_reads_queries_only_in_smarts(text, expect):
|
||||
assert scanned(text, smarts=True) == expect
|
||||
assert scanned(text, smarts=False) is None
|
||||
|
||||
|
||||
def test_ring_closures_may_follow_a_bond_symbol_and_clear_the_pending_bond():
|
||||
assert scanned("C1=CC=C1") == {
|
||||
"atoms": 4,
|
||||
"bonds": [[0, 1], [0, 3], [1, 2], [2, 3]],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "message"),
|
||||
[
|
||||
("", "empty"),
|
||||
(" ", "empty"),
|
||||
(" C", "spaces at its ends"),
|
||||
("C\n", "spaces at its ends"),
|
||||
("C[N", "unclosed \\[ at 1"),
|
||||
("C[]", "empty bracket atom at 1"),
|
||||
("[C,N]", "a query inside the bracket atom at 0: that is SMARTS"),
|
||||
("[$(C)]", "that is SMARTS"),
|
||||
("=C", "a bond with no atom before it at 0"),
|
||||
("C.=C", "a bond with no atom before it at 2"),
|
||||
("1CC1", "a ring closure with no atom before it"),
|
||||
("%10C", "a ring closure with no atom before it"),
|
||||
("(C)C", "a branch with no atom before it at 0"),
|
||||
("C=(C)C", "a bond before \\( at 2: write it inside the branch"),
|
||||
("C)", "unmatched \\) at 1"),
|
||||
("C(C=)C", "a bond with no atom after it at 4"),
|
||||
("C()C", "an empty branch before 2"),
|
||||
("C(C", "unclosed \\("),
|
||||
("C=.C", "a bond with no atom after it at 2"),
|
||||
("C1CC", "ring closure 1 is never closed"),
|
||||
("C1C2C", "ring closure 1, 2 is never closed"),
|
||||
("CC=", "a bond with no atom after it"),
|
||||
("C?C", "'\\?' is not part of a SMILES at 1"),
|
||||
("C C", "' ' is not part of a SMILES at 1"),
|
||||
],
|
||||
)
|
||||
def test_scan_refuses_text_and_says_why(text, message):
|
||||
with pytest.raises(ValueError, match=message):
|
||||
structures.scan(text)
|
||||
|
||||
|
||||
def test_scan_names_the_dialect_in_a_stray_character_message():
|
||||
with pytest.raises(ValueError, match="is not part of a SMARTS at 1"):
|
||||
structures.scan("C?", smarts=True)
|
||||
with pytest.raises(ValueError, match="not part of a SMILES"):
|
||||
structures.scan("C?", smarts=False)
|
||||
|
||||
|
||||
def test_text_that_is_only_dots_has_no_atoms():
|
||||
with pytest.raises(ValueError, match="no atoms"):
|
||||
structures.scan("..")
|
||||
|
||||
|
||||
def test_a_percent_without_two_digits_says_so():
|
||||
with pytest.raises(ValueError, match="% needs two digits at 1"):
|
||||
structures.scan("C%1")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", ["C%", "C%1", "C%1C", "C%a1", "C%-1"])
|
||||
def test_a_ring_number_written_with_a_percent_needs_two_digits(text):
|
||||
assert scanned(text) is None
|
||||
|
||||
|
||||
@given(st.integers(1, 40))
|
||||
@PROPS
|
||||
def test_a_chain_has_a_bond_between_neighbours(n):
|
||||
got = scanned("C" * n)
|
||||
assert got == {"atoms": n, "bonds": [[i, i + 1] for i in range(n - 1)]}
|
||||
|
||||
|
||||
@given(st.integers(3, 40))
|
||||
@PROPS
|
||||
def test_a_ring_closes_its_last_atom_on_its_first(n):
|
||||
got = scanned("C1" + "C" * (n - 2) + "C1")
|
||||
assert got["atoms"] == n
|
||||
assert [0, n - 1] in got["bonds"]
|
||||
assert len(got["bonds"]) == n
|
||||
|
||||
|
||||
@given(st.integers(0, 6), st.integers(1, 5))
|
||||
@PROPS
|
||||
def test_branches_attach_to_the_atom_before_them(branches, length):
|
||||
got = scanned("C" + ("(" + "C" * length + ")") * branches)
|
||||
assert got["atoms"] == 1 + branches * length
|
||||
attached = [b for b in got["bonds"] if b[0] == 0]
|
||||
assert len(attached) == branches
|
||||
|
||||
|
||||
TEXT = st.text(alphabet="CNOcnoBrCl[]()=#-+@H:*~,;&!$/\\.%0123456789 aA", max_size=24)
|
||||
|
||||
|
||||
@given(TEXT, st.booleans())
|
||||
@settings(PROPS, max_examples=300)
|
||||
def test_scan_either_refuses_with_a_value_error_or_returns_consistent_bonds(
|
||||
text, smarts
|
||||
):
|
||||
try:
|
||||
count, bonds = structures.scan(text, smarts=smarts)
|
||||
except ValueError:
|
||||
return
|
||||
assert count >= 1
|
||||
assert all(0 <= i < j < count for i, j in bonds)
|
||||
|
||||
|
||||
@given(TEXT)
|
||||
@settings(PROPS, max_examples=300)
|
||||
def test_whatever_smiles_scans_scans_the_same_as_smarts(text):
|
||||
plain = scanned(text, smarts=False)
|
||||
if plain is not None:
|
||||
assert scanned(text, smarts=True) == plain
|
||||
|
||||
|
||||
# --- identities ---------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", IDENTITIES, ids=lambda c: c["id"])
|
||||
def test_identity_vectors(case):
|
||||
assert (
|
||||
structures.identity(case["format"], case["text"], case["inchikey"])
|
||||
== case["expect"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fmt", ["smiles", "smarts"])
|
||||
def test_without_an_inchikey_the_identity_is_the_format_and_a_hash_of_the_text(fmt):
|
||||
digest = hashlib.sha256(b"CCO").hexdigest()[:12]
|
||||
assert structures.identity(fmt, "CCO") == f"{fmt}:{digest}"
|
||||
assert structures.identity(fmt, "CCO", "") == f"{fmt}:{digest}"
|
||||
assert structures.identity(fmt, "CCO", None) == f"{fmt}:{digest}"
|
||||
|
||||
|
||||
def test_only_a_smiles_takes_an_inchikey():
|
||||
key = "OTMSDBZUPAUEDD-UHFFFAOYSA-N"
|
||||
assert structures.identity("smiles", "CC", key) == key
|
||||
assert structures.identity("smarts", "CC", key).startswith("smarts:")
|
||||
|
||||
|
||||
def test_two_spellings_of_one_molecule_have_two_text_identities():
|
||||
assert structures.identity("smiles", "OCC") != structures.identity("smiles", "CCO")
|
||||
|
||||
|
||||
def test_the_formats_and_their_media_types_are_inverse():
|
||||
assert structures.MEDIA == {
|
||||
"smiles": "chemical/x-daylight-smiles",
|
||||
"smarts": "chemical/x-daylight-smarts",
|
||||
}
|
||||
assert {structures.FORMATS[m]: m for m in structures.FORMATS} == structures.MEDIA
|
||||
|
||||
|
||||
# --- rules ---------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rule", RULES)
|
||||
def test_the_rules_are_registered_in_the_layers_chapter(rule):
|
||||
assert rule_chapters()[rule] == "spec/18-layers.md"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", PROBLEMS, ids=lambda c: c["id"])
|
||||
def test_problems_vectors(case):
|
||||
got = structures.problems(case["annotation"])
|
||||
assert got == case["expect"]
|
||||
assert accepted_by_schema(case["annotation"]) is case["schema"]
|
||||
assert {x["rule"] for x in got} <= set(rule_chapters())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", PROBLEMS, ids=lambda c: c["id"])
|
||||
def test_a_vector_is_named_for_the_rule_it_breaks(case):
|
||||
stem = case["id"].split("-", 1)[0]
|
||||
rules = {x["rule"] for x in case["expect"]}
|
||||
if stem == "valid":
|
||||
assert rules == set()
|
||||
else:
|
||||
assert "layer.structure." + stem in rules
|
||||
|
||||
|
||||
def test_every_rule_is_broken_by_some_vector():
|
||||
assert {x["rule"] for c in PROBLEMS for x in c["expect"]} == set(RULES)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("annotation", CHEMISTRY, ids=lambda a: a["body"]["label"])
|
||||
def test_the_chemistry_layer_breaks_no_rule(annotation):
|
||||
assert structures.problems(annotation) == []
|
||||
assert annotation["body"]["kind"] == "structure"
|
||||
|
||||
|
||||
def test_an_annotation_of_another_kind_is_refused():
|
||||
with pytest.raises(ValueError, match="not a structure annotation"):
|
||||
structures.problems({"body": {"kind": "comment"}})
|
||||
with pytest.raises(ValueError, match="not a structure annotation"):
|
||||
structures.problems({"body": {}})
|
||||
|
||||
|
||||
def test_a_missing_format_breaks_the_value_rule_and_says_which_formats_exist():
|
||||
got = structures.problems(edited(SMILES_TERMS, format=None))
|
||||
assert [x["rule"] for x in got] == ["layer.structure.value"]
|
||||
assert "chemical/x-daylight-smarts" in got[0]["message"]
|
||||
|
||||
|
||||
def test_a_scan_failure_is_reported_with_its_reason():
|
||||
got = structures.problems(
|
||||
edited(SMILES_TERMS, value="O(", identity="smiles:000000000000")
|
||||
)
|
||||
assert got[0]["rule"] == "layer.structure.value"
|
||||
assert got[0]["message"] == "the value is not a SMILES: unclosed ("
|
||||
|
||||
|
||||
def test_a_percent_value_breaks_only_the_value_rule():
|
||||
annotation = edited(
|
||||
SMILES_TERMS,
|
||||
value="C%1",
|
||||
terms=None,
|
||||
identity=structures.identity("smiles", "C%1"),
|
||||
)
|
||||
assert rules_of(annotation) == ["layer.structure.value"]
|
||||
|
||||
|
||||
def test_a_value_with_a_query_is_a_smarts_not_a_smiles():
|
||||
annotation = edited(
|
||||
SMILES_TERMS, value="[C,N]", identity="smiles:" + "0" * 12, terms=None
|
||||
)
|
||||
got = structures.problems(annotation)
|
||||
assert got[0]["rule"] == "layer.structure.value"
|
||||
assert "that is SMARTS" in got[0]["message"]
|
||||
|
||||
|
||||
def test_a_smarts_must_carry_a_smarts_identity():
|
||||
for identity in ("OTMSDBZUPAUEDD-UHFFFAOYSA-N", "smiles:2e2a923f840c", ""):
|
||||
got = structures.problems(edited(SMARTS_TERMS, identity=identity))
|
||||
assert [x["rule"] for x in got] == ["layer.structure.identity"]
|
||||
assert "has no InChIKey" in got[0]["message"]
|
||||
|
||||
|
||||
def test_an_identity_of_the_other_format_is_named_as_such():
|
||||
got = structures.problems(edited(SMILES_TERMS, identity="smarts:" + "0" * 12))
|
||||
assert [x["message"] for x in got] == [
|
||||
"the identity is a smarts one, the value a smiles"
|
||||
]
|
||||
|
||||
|
||||
def test_an_identity_that_is_the_hash_of_another_text_gives_the_right_one():
|
||||
right = structures.identity("smiles", "O([H])[H]")
|
||||
got = structures.problems(edited(SMILES_TERMS, identity="smiles:" + "0" * 12))
|
||||
assert got == [
|
||||
{
|
||||
"rule": "layer.structure.identity",
|
||||
"message": "the identity is not the SHA-256 of the value: " + right,
|
||||
}
|
||||
]
|
||||
assert rules_of(edited(SMILES_TERMS, identity=right)) == []
|
||||
|
||||
|
||||
def test_a_smiles_may_carry_an_inchikey_that_is_not_checked_against_the_text():
|
||||
assert rules_of(edited(SMILES_TERMS, identity="OTMSDBZUPAUEDD-UHFFFAOYSA-N")) == []
|
||||
|
||||
|
||||
def test_an_unknown_format_still_has_a_text_identity_checked_but_not_an_inchikey():
|
||||
wrong = edited(SMILES_TERMS, format="text/plain", identity="smarts:" + "0" * 12)
|
||||
assert rules_of(wrong) == ["layer.structure.value", "layer.structure.identity"]
|
||||
keyed = edited(SMILES_TERMS, format="text/plain")
|
||||
assert rules_of(keyed) == ["layer.structure.value"]
|
||||
|
||||
|
||||
def term(kind, atoms, **extra):
|
||||
"""A term of the given kind over `atoms`."""
|
||||
return {"kind": kind, "atoms": atoms, **extra}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "terms", "fragment"),
|
||||
[
|
||||
("CCCC", [term("bond", [0, 1, 2])], "a bond cites 2 atoms"),
|
||||
("CCCC", [term("angle", [0, 1])], "angle cites 3 atoms"),
|
||||
("CCCC", [term("dihedral", [0, 1, 2, 3])], "a dihedral cites ? atoms"),
|
||||
("CCCC", [{"atoms": [0, 1]}], "None cites ? atoms"),
|
||||
("CCCC", [term("bond", [1, 1])], "an atom is cited twice"),
|
||||
("CCCC", [term("bond", [0, 4])], "the value writes 4 atoms, numbered from 0"),
|
||||
("CCCC", [term("bond", [0, 2])], "the atoms are not joined as"),
|
||||
("CCCC", [term("angle", [0, 2, 1])], "the atoms are not joined as"),
|
||||
("CCCC", [term("torsion", [0, 1, 3, 2])], "the atoms are not joined as"),
|
||||
("CC(C)C", [term("improper", [1, 0, 2, 3])], None),
|
||||
("CC(C)C", [term("improper", [0, 1, 2, 3])], "the atoms are not joined as"),
|
||||
("CCCC", [term("bond", [0, 1], types=["C"])], "1 types for 2 atoms"),
|
||||
(
|
||||
"CCCC",
|
||||
[term("bond", [0, 1], units={"k": "x"}, values={})],
|
||||
"units for values that are not given: k",
|
||||
),
|
||||
(
|
||||
"CCCC",
|
||||
[term("bond", [0, 1], units={"b": "x", "a": "x"}, values={"c": "1"})],
|
||||
"a, b",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_terms_are_checked_against_the_atoms_and_bonds_the_text_writes(
|
||||
value, terms, fragment
|
||||
):
|
||||
annotation = edited(
|
||||
SMILES_TERMS,
|
||||
value=value,
|
||||
identity=structures.identity("smiles", value),
|
||||
terms=terms,
|
||||
)
|
||||
got = structures.problems(annotation)
|
||||
if fragment is None:
|
||||
assert got == []
|
||||
return
|
||||
assert {x["rule"] for x in got} == {"layer.structure.term"}
|
||||
assert fragment in got[0]["message"]
|
||||
|
||||
|
||||
def test_every_term_is_numbered_in_its_message():
|
||||
value = "CCC"
|
||||
terms = [term("bond", [0, 1]), term("bond", [0, 2]), term("angle", [0, 1])]
|
||||
annotation = edited(
|
||||
SMILES_TERMS,
|
||||
value=value,
|
||||
identity=structures.identity("smiles", value),
|
||||
terms=terms,
|
||||
)
|
||||
messages = [x["message"] for x in structures.problems(annotation)]
|
||||
assert len(messages) == 2
|
||||
assert messages[0].startswith("term 1:")
|
||||
assert messages[1].startswith("term 2:")
|
||||
|
||||
|
||||
def test_a_term_is_not_joined_against_a_value_that_failed_to_scan():
|
||||
annotation = edited(
|
||||
SMILES_TERMS,
|
||||
value="C(",
|
||||
identity=structures.identity("smiles", "C("),
|
||||
terms=[term("bond", [0, 1])],
|
||||
)
|
||||
assert rules_of(annotation) == ["layer.structure.value"]
|
||||
@@ -0,0 +1,318 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""The target set of targeted F3 (spec/08 §8.10)."""
|
||||
|
||||
import pytest
|
||||
|
||||
from helpers_lib1 import (
|
||||
build_doc,
|
||||
document,
|
||||
heading,
|
||||
ids_of_kind,
|
||||
math_run,
|
||||
package_state,
|
||||
paragraph,
|
||||
read_cases,
|
||||
sentence,
|
||||
text_run,
|
||||
vectors,
|
||||
)
|
||||
from paperspec import hashing, targets
|
||||
from paperspec.model import LEAVES, MATH_KINDS
|
||||
|
||||
|
||||
def _doc():
|
||||
"""Two sections, a table and an equation, with digits in some sentences."""
|
||||
return build_doc(
|
||||
document(
|
||||
{
|
||||
"kind": "front",
|
||||
"children": [
|
||||
{"kind": "title", "children": [sentence(text_run("Title 1"))]}
|
||||
],
|
||||
},
|
||||
{
|
||||
"kind": "section",
|
||||
"attrs": {"level": 1},
|
||||
"children": [
|
||||
heading("Intro"),
|
||||
paragraph(
|
||||
sentence(text_run("No numbers here.")),
|
||||
sentence(
|
||||
text_run("Three "), math_run("x_2"), text_run(" items.")
|
||||
),
|
||||
),
|
||||
{
|
||||
"kind": "equation",
|
||||
"content": {"notation": "tex", "source": "a=b"},
|
||||
"children": [
|
||||
{
|
||||
"kind": "equation-line",
|
||||
"content": {"notation": "tex", "source": "a=b"},
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"kind": "section",
|
||||
"attrs": {"level": 1},
|
||||
"children": [
|
||||
heading("Methods"),
|
||||
paragraph(
|
||||
sentence(text_run("We used 5 runs.")),
|
||||
sentence(text_run("Nothing numeric.")),
|
||||
),
|
||||
{
|
||||
"kind": "table",
|
||||
"children": [
|
||||
{
|
||||
"kind": "table-body",
|
||||
"children": [
|
||||
{
|
||||
"kind": "row",
|
||||
"children": [
|
||||
{
|
||||
"kind": "cell",
|
||||
"content": [text_run("3.5")],
|
||||
},
|
||||
{
|
||||
"kind": "cell",
|
||||
"content": [text_run("n/a")],
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _named(doc):
|
||||
"""IDs by a short name."""
|
||||
sentences = {hashing.plain(doc, s): s for s in ids_of_kind(doc, "sentence")}
|
||||
cells = ids_of_kind(doc, "cell")
|
||||
return {
|
||||
"title": sentences["Title 1"],
|
||||
"plain": sentences["No numbers here."],
|
||||
"three": next(s for t, s in sentences.items() if t.startswith("Three")),
|
||||
"five": sentences["We used 5 runs."],
|
||||
"none": sentences["Nothing numeric."],
|
||||
"num_cell": cells[0],
|
||||
"text_cell": cells[1],
|
||||
"eq": ids_of_kind(doc, "equation")[0],
|
||||
"line": ids_of_kind(doc, "equation-line")[0],
|
||||
"intro": ids_of_kind(doc, "section")[0],
|
||||
"methods": ids_of_kind(doc, "section")[1],
|
||||
"front": ids_of_kind(doc, "front")[0],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", vectors("targets"))
|
||||
def test_targets_vector(case):
|
||||
doc = package_state(case["package"]).doc()
|
||||
got = targets.target_set(doc, case["anchored"], case["digit_rule"])
|
||||
assert got == case["expect"]["nodes"]
|
||||
|
||||
|
||||
def test_the_three_digit_rules_nest_on_the_example_paper():
|
||||
cases = {c["digit_rule"]: c for c in read_cases("targets")}
|
||||
assert set(cases) == set(targets.DIGIT_RULES)
|
||||
sets = {rule: {n for n, _ in c["expect"]["nodes"]} for rule, c in cases.items()}
|
||||
assert sets["none"] <= sets["anchored-sections"] <= sets["all"]
|
||||
assert sets["none"] < sets["all"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", vectors("targets"))
|
||||
def test_targets_are_leaves_and_equations_are_equation_kinds(case):
|
||||
doc = package_state(case["package"]).doc()
|
||||
got = targets.target_set(doc, case["anchored"], case["digit_rule"])
|
||||
assert all(doc.nodes[n]["kind"] in LEAVES for n, _ in got)
|
||||
assert all(doc.nodes[n]["kind"] in MATH_KINDS for n, r in got if r == "equation")
|
||||
position = {nid: i for i, nid in enumerate(doc.preorder())}
|
||||
assert [position[n] for n, _ in got] == sorted(position[n] for n, _ in got)
|
||||
|
||||
|
||||
class TestTargetSet:
|
||||
def test_all_takes_every_sentence_and_cell_with_a_digit(self):
|
||||
doc = _doc()
|
||||
n = _named(doc)
|
||||
got = targets.target_set(doc, [], "all")
|
||||
assert got == [
|
||||
[n["title"], "digits"],
|
||||
[n["three"], "digits"],
|
||||
[n["eq"], "equation"],
|
||||
[n["line"], "equation"],
|
||||
[n["five"], "digits"],
|
||||
[n["num_cell"], "digits"],
|
||||
]
|
||||
|
||||
def test_none_takes_only_equations_and_anchored_leaves(self):
|
||||
doc = _doc()
|
||||
n = _named(doc)
|
||||
assert targets.target_set(doc, [], "none") == [
|
||||
[n["eq"], "equation"],
|
||||
[n["line"], "equation"],
|
||||
]
|
||||
assert targets.target_set(doc, [n["none"]], "none") == [
|
||||
[n["eq"], "equation"],
|
||||
[n["line"], "equation"],
|
||||
[n["none"], "anchored"],
|
||||
]
|
||||
|
||||
def test_anchored_sections_takes_digits_only_where_something_is_anchored(self):
|
||||
doc = _doc()
|
||||
n = _named(doc)
|
||||
got = targets.target_set(doc, [n["plain"]], "anchored-sections")
|
||||
assert got == [
|
||||
[n["plain"], "anchored"],
|
||||
[n["three"], "digits"],
|
||||
[n["eq"], "equation"],
|
||||
[n["line"], "equation"],
|
||||
]
|
||||
|
||||
def test_an_anchored_leaf_is_anchored_even_if_it_has_digits_or_is_an_equation(self):
|
||||
doc = _doc()
|
||||
n = _named(doc)
|
||||
got = dict(targets.target_set(doc, [n["three"], n["eq"]], "all"))
|
||||
assert got[n["three"]] == "anchored"
|
||||
assert got[n["eq"]] == "anchored"
|
||||
assert got[n["line"]] == "equation"
|
||||
|
||||
def test_an_anchored_container_contributes_its_leaves(self):
|
||||
doc = _doc()
|
||||
n = _named(doc)
|
||||
got = dict(targets.target_set(doc, [n["methods"]], "none"))
|
||||
leaves = set(doc.leaves(n["methods"]))
|
||||
assert {k for k, v in got.items() if v == "anchored"} == leaves
|
||||
assert got[n["num_cell"]] == "anchored"
|
||||
|
||||
def test_a_container_anchor_with_a_digit_rule_keeps_the_anchored_reason(self):
|
||||
doc = _doc()
|
||||
n = _named(doc)
|
||||
got = dict(targets.target_set(doc, [n["methods"]], "all"))
|
||||
assert got[n["five"]] == "anchored"
|
||||
assert got[n["three"]] == "digits"
|
||||
|
||||
def test_anchored_sections_counts_a_section_an_anchored_container_sits_in(self):
|
||||
doc = _doc()
|
||||
n = _named(doc)
|
||||
got = dict(targets.target_set(doc, [n["front"]], "anchored-sections"))
|
||||
assert got[n["title"]] == "anchored"
|
||||
assert n["five"] not in got
|
||||
assert n["three"] not in got
|
||||
|
||||
def test_the_result_is_in_document_order_whatever_the_order_of_anchors(self):
|
||||
doc = _doc()
|
||||
n = _named(doc)
|
||||
a = targets.target_set(doc, [n["five"], n["plain"], n["none"]], "none")
|
||||
b = targets.target_set(doc, [n["none"], n["plain"], n["five"]], "none")
|
||||
assert a == b
|
||||
order = doc.preorder()
|
||||
assert [order.index(x) for x, _ in a] == sorted(order.index(x) for x, _ in a)
|
||||
|
||||
def test_the_default_rule_is_all(self):
|
||||
doc = _doc()
|
||||
assert targets.target_set(doc, []) == targets.target_set(doc, [], "all")
|
||||
|
||||
def test_a_node_given_twice_counts_once(self):
|
||||
doc = _doc()
|
||||
n = _named(doc)
|
||||
assert targets.target_set(
|
||||
doc, [n["plain"], n["plain"]], "none"
|
||||
) == targets.target_set(doc, [n["plain"]], "none")
|
||||
|
||||
@pytest.mark.parametrize("rule", ["some", "", "ALL", None])
|
||||
def test_an_unknown_digit_rule_is_refused(self, rule):
|
||||
with pytest.raises(
|
||||
ValueError, match="digit rule must be one of all, anchored-sections, none"
|
||||
):
|
||||
targets.target_set(_doc(), [], rule)
|
||||
|
||||
def test_anchored_nodes_must_be_in_the_document(self):
|
||||
doc = _doc()
|
||||
n = _named(doc)
|
||||
with pytest.raises(
|
||||
ValueError, match="anchored nodes are not live: zzzzzzz, zzzzzzzz"
|
||||
):
|
||||
targets.target_set(doc, ["zzzzzzzz", n["plain"], "zzzzzzz"], "all")
|
||||
|
||||
|
||||
class TestHasDigitRun:
|
||||
@pytest.mark.parametrize(
|
||||
("content", "expected"),
|
||||
[
|
||||
([{"kind": "text", "text": "a1"}], True),
|
||||
([{"kind": "text", "text": "abc"}], False),
|
||||
([{"kind": "math", "source": "x_2"}], True),
|
||||
([{"kind": "math", "source": "x"}], False),
|
||||
([{"kind": "math", "source": None}], False),
|
||||
([{"kind": "text", "text": "٣"}], True),
|
||||
([{"kind": "text", "text": "²"}], False),
|
||||
([{"kind": "cite", "refs": ["a"], "text": "12"}], False),
|
||||
([{"kind": "xref", "text": "Table 1"}], False),
|
||||
([{"kind": "equation", "node": "aaaaaaa"}], False),
|
||||
([{"kind": "text", "text": "x"}, {"kind": "text", "text": "7"}], True),
|
||||
(["7", None, 7], False),
|
||||
([{"kind": "text"}], False),
|
||||
([], False),
|
||||
(None, False),
|
||||
],
|
||||
)
|
||||
def test_only_text_and_math_runs_count(self, content, expected):
|
||||
assert targets.has_digit_run({"content": content}) is expected
|
||||
|
||||
def test_a_node_without_content_has_no_digit_run(self):
|
||||
assert targets.has_digit_run({}) is False
|
||||
|
||||
|
||||
class TestEnclosingSection:
|
||||
def test_a_section_encloses_itself_and_what_it_holds(self):
|
||||
doc = _doc()
|
||||
n = _named(doc)
|
||||
assert targets.enclosing_section(doc, n["intro"]) == n["intro"]
|
||||
assert targets.enclosing_section(doc, n["three"]) == n["intro"]
|
||||
assert targets.enclosing_section(doc, n["num_cell"]) == n["methods"]
|
||||
|
||||
def test_front_matter_is_its_own_section(self):
|
||||
doc = _doc()
|
||||
n = _named(doc)
|
||||
assert targets.enclosing_section(doc, n["title"]) == n["front"]
|
||||
|
||||
def test_the_nearest_section_wins(self):
|
||||
doc = build_doc(
|
||||
document(
|
||||
{
|
||||
"kind": "section",
|
||||
"attrs": {"level": 1},
|
||||
"children": [
|
||||
heading("Outer"),
|
||||
{
|
||||
"kind": "section",
|
||||
"attrs": {"level": 2},
|
||||
"children": [
|
||||
heading("Inner"),
|
||||
paragraph(sentence(text_run("x"))),
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
)
|
||||
)
|
||||
outer, inner = ids_of_kind(doc, "section")
|
||||
leaf = ids_of_kind(doc, "paragraph")[0]
|
||||
assert targets.enclosing_section(doc, leaf) == inner
|
||||
assert targets.enclosing_section(doc, inner) == inner
|
||||
assert targets.enclosing_section(doc, outer) == outer
|
||||
|
||||
def test_a_node_outside_every_section_is_enclosed_by_the_root(self):
|
||||
doc = build_doc(document({"kind": "back", "children": []}))
|
||||
assert targets.enclosing_section(doc, doc.root) == doc.root
|
||||
@@ -0,0 +1,354 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Text normalization, node and record IDs, and offset conversion (spec/01, spec/04)."""
|
||||
|
||||
import hashlib
|
||||
import unicodedata
|
||||
from itertools import pairwise
|
||||
|
||||
import pytest
|
||||
from hypothesis import given, settings, strategies as st
|
||||
|
||||
from helpers_lib1 import read_cases, vectors
|
||||
from paperspec import text
|
||||
from paperspec.text import (
|
||||
ALPHABET,
|
||||
HASH_RE,
|
||||
KEY_RE,
|
||||
LETTERS,
|
||||
NODE_ID_RE,
|
||||
RECORD_ID_RE,
|
||||
collapse_ws,
|
||||
cp_to_utf16,
|
||||
digits,
|
||||
is_digit,
|
||||
is_ws,
|
||||
letter_first,
|
||||
mint_node_id,
|
||||
norm,
|
||||
record_id,
|
||||
remove_ws,
|
||||
sha256_hex,
|
||||
trim,
|
||||
)
|
||||
|
||||
PROPERTIES = settings(derandomize=True, max_examples=200, database=None, deadline=None)
|
||||
|
||||
WHITESPACE_CHARS = [
|
||||
"\t",
|
||||
"\n",
|
||||
"\x0b",
|
||||
"\x0c",
|
||||
"\r",
|
||||
" ",
|
||||
"\x85",
|
||||
"\xa0",
|
||||
"\N{OGHAM SPACE MARK}",
|
||||
"\N{EN QUAD}",
|
||||
"\N{FOUR-PER-EM SPACE}",
|
||||
"\N{HAIR SPACE}",
|
||||
"\N{LINE SEPARATOR}",
|
||||
"\N{PARAGRAPH SEPARATOR}",
|
||||
"\N{NARROW NO-BREAK SPACE}",
|
||||
"\N{MEDIUM MATHEMATICAL SPACE}",
|
||||
" ",
|
||||
]
|
||||
NOT_WHITESPACE_CHARS = ["\x1c", "\x1f", "", "\u200b", "", "", "a", "0"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", vectors("ids"))
|
||||
def test_ids_vector(case):
|
||||
if "digest" in case:
|
||||
digest = bytes.fromhex(case["digest"])
|
||||
assert letter_first(digest, 7) == case["node_id"]
|
||||
assert record_id(case["digest"]) == case["record_id"]
|
||||
else:
|
||||
got = mint_node_id(
|
||||
case["doc"], case["kind"], case["text_hash"], case["occurrence"]
|
||||
)
|
||||
assert got == case["node_id"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", vectors("normalize"))
|
||||
def test_normalize_vector(case):
|
||||
assert norm(case["input"]) == case["output"]
|
||||
assert sha256_hex(case["output"]) == case["sha256"]
|
||||
|
||||
|
||||
def test_line_separator_vector_is_one_case():
|
||||
cases = read_cases("normalize")
|
||||
assert len(cases) == 9
|
||||
assert any("\N{LINE SEPARATOR}" in c["input"] for c in cases)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ch", WHITESPACE_CHARS)
|
||||
def test_is_ws_accepts_unicode_white_space(ch):
|
||||
assert is_ws(ch)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ch", NOT_WHITESPACE_CHARS)
|
||||
def test_is_ws_rejects_other_characters(ch):
|
||||
assert not is_ws(ch)
|
||||
|
||||
|
||||
def test_white_space_set_has_the_twenty_five_code_points_of_proplist():
|
||||
assert len(text.WHITE_SPACE) == 25
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("raw", "collapsed"),
|
||||
[
|
||||
("a b", "a b"),
|
||||
("a \t\n b", "a b"),
|
||||
(" a", " a"),
|
||||
("a ", "a "),
|
||||
("\N{NO-BREAK SPACE}\N{EM SPACE}x", " x"),
|
||||
("", ""),
|
||||
("no-space", "no-space"),
|
||||
],
|
||||
)
|
||||
def test_collapse_ws_keeps_the_ends(raw, collapsed):
|
||||
assert collapse_ws(raw) == collapsed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("raw", "trimmed"),
|
||||
[
|
||||
(" a b ", "a b"),
|
||||
("\n\t", ""),
|
||||
("", ""),
|
||||
("a", "a"),
|
||||
("\u200ba\u200b", "\u200ba\u200b"),
|
||||
],
|
||||
)
|
||||
def test_trim(raw, trimmed):
|
||||
assert trim(raw) == trimmed
|
||||
|
||||
|
||||
def test_remove_ws_drops_every_white_space_character():
|
||||
assert remove_ws("a b\tc\N{NO-BREAK SPACE}d\N{LINE SEPARATOR}e") == "abcde"
|
||||
|
||||
|
||||
def test_norm_composes_before_collapsing():
|
||||
assert norm("café au\N{NO-BREAK SPACE}lait") == "café au lait"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", ["fi", "\N{MICRO SIGN}", "\u200b", "", "²"])
|
||||
def test_norm_does_not_apply_compatibility_folding(raw):
|
||||
assert norm(raw) == raw
|
||||
|
||||
|
||||
def test_norm_replaces_a_canonical_equivalent_but_not_a_compatibility_one():
|
||||
assert norm("\N{ANGSTROM SIGN}") == "Å"
|
||||
|
||||
|
||||
@given(st.text())
|
||||
@PROPERTIES
|
||||
def test_norm_is_idempotent(s):
|
||||
assert norm(norm(s)) == norm(s)
|
||||
|
||||
|
||||
@given(st.text())
|
||||
@PROPERTIES
|
||||
def test_norm_leaves_one_space_between_words_and_none_at_the_ends(s):
|
||||
out = norm(s)
|
||||
assert unicodedata.is_normalized("NFC", out)
|
||||
assert out == trim(out)
|
||||
assert not any(is_ws(a) and is_ws(b) for a, b in pairwise(out))
|
||||
assert all(not is_ws(c) or c == " " for c in out)
|
||||
|
||||
|
||||
@given(st.text())
|
||||
@PROPERTIES
|
||||
def test_norm_equals_nfc_of_the_whitespace_split_join(s):
|
||||
pieces = [p for p in _split_ws(unicodedata.normalize("NFC", s)) if p]
|
||||
assert norm(s) == " ".join(pieces)
|
||||
|
||||
|
||||
def _split_ws(s):
|
||||
pieces, cur = [], []
|
||||
for ch in s:
|
||||
if is_ws(ch):
|
||||
pieces.append("".join(cur))
|
||||
cur = []
|
||||
else:
|
||||
cur.append(ch)
|
||||
pieces.append("".join(cur))
|
||||
return pieces
|
||||
|
||||
|
||||
def test_sha256_hex_hashes_text_as_utf8_and_bytes_as_given():
|
||||
assert sha256_hex("café") == hashlib.sha256("café".encode()).hexdigest()
|
||||
assert sha256_hex(b"abc") == hashlib.sha256(b"abc").hexdigest()
|
||||
assert sha256_hex("abc") == sha256_hex(b"abc")
|
||||
assert HASH_RE.match(sha256_hex(""))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("ch", "expected"),
|
||||
[
|
||||
("7", True),
|
||||
("٣", True),
|
||||
("\N{FULLWIDTH DIGIT FIVE}", True),
|
||||
("²", False),
|
||||
("x", False),
|
||||
],
|
||||
)
|
||||
def test_is_digit_is_general_category_nd(ch, expected):
|
||||
assert is_digit(ch) is expected
|
||||
|
||||
|
||||
def test_digits_keeps_only_decimal_digits_in_order():
|
||||
assert digits("a1b٢2c²3") == "1٢23"
|
||||
|
||||
|
||||
def test_alphabet_is_crockford_base32_in_lower_case():
|
||||
assert len(ALPHABET) == 32
|
||||
assert len(set(ALPHABET)) == 32
|
||||
assert not set("ilou") & set(ALPHABET)
|
||||
assert "".join(sorted(ALPHABET)) == ALPHABET
|
||||
assert "".join(c for c in ALPHABET if c.isalpha()) == LETTERS
|
||||
assert len(LETTERS) == 22
|
||||
|
||||
|
||||
@given(st.binary(min_size=32, max_size=32))
|
||||
@PROPERTIES
|
||||
def test_letter_first_ids_have_the_node_form(digest):
|
||||
nid = letter_first(digest, 7)
|
||||
assert NODE_ID_RE.match(nid)
|
||||
assert nid[0] in LETTERS
|
||||
|
||||
|
||||
@given(st.binary(min_size=32, max_size=32))
|
||||
@PROPERTIES
|
||||
def test_record_ids_have_the_record_form(digest):
|
||||
rid = record_id(digest.hex())
|
||||
assert len(rid) == 13
|
||||
assert RECORD_ID_RE.match(rid)
|
||||
assert rid == letter_first(digest, 13)
|
||||
assert rid.startswith(letter_first(digest, 7))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("length", [1, 7, 26, 50])
|
||||
def test_letter_first_length_is_as_asked(length):
|
||||
assert len(letter_first(bytes(range(32)), length)) == length
|
||||
|
||||
|
||||
def test_letter_first_first_character_is_the_first_byte_modulo_22():
|
||||
assert letter_first(bytes([0]) + bytes(31), 3)[0] == LETTERS[0]
|
||||
assert letter_first(bytes([22]) + bytes(31), 3)[0] == LETTERS[0]
|
||||
assert letter_first(bytes([255]) + bytes(31), 3)[0] == LETTERS[255 % 22]
|
||||
|
||||
|
||||
def test_letter_first_reads_the_remaining_bytes_five_bits_at_a_time():
|
||||
digest = bytes([0]) + bytes([0b11111_000, 0b00_00001_0]) + bytes(29)
|
||||
assert (
|
||||
letter_first(digest, 4) == LETTERS[0] + ALPHABET[31] + ALPHABET[0] + ALPHABET[1]
|
||||
)
|
||||
|
||||
|
||||
@given(
|
||||
st.text(min_size=1, max_size=20),
|
||||
st.sampled_from(["sentence", "cell", "paragraph"]),
|
||||
st.text(alphabet="0123456789abcdef", min_size=64, max_size=64),
|
||||
st.integers(min_value=0, max_value=50),
|
||||
)
|
||||
@PROPERTIES
|
||||
def test_minted_ids_have_the_node_form(doc_key, kind, text_hash, occurrence):
|
||||
nid = mint_node_id(doc_key, kind, text_hash, occurrence)
|
||||
assert NODE_ID_RE.match(nid)
|
||||
assert nid == mint_node_id(doc_key, kind, text_hash, occurrence)
|
||||
|
||||
|
||||
def test_minted_ids_depend_on_every_field():
|
||||
base = mint_node_id("a2026b-aaaa", "sentence", "0" * 64, 0)
|
||||
assert mint_node_id("a2026b-aaab", "sentence", "0" * 64, 0) != base
|
||||
assert mint_node_id("a2026b-aaaa", "cell", "0" * 64, 0) != base
|
||||
assert mint_node_id("a2026b-aaaa", "sentence", "1" * 64, 0) != base
|
||||
assert mint_node_id("a2026b-aaaa", "sentence", "0" * 64, 1) != base
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"good", ["a000000", "m0666ac", "zzzzzzz", "b9wp3tt", "jvnys4s"]
|
||||
)
|
||||
def test_node_id_pattern_accepts(good):
|
||||
assert NODE_ID_RE.match(good)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bad",
|
||||
[
|
||||
"0000000",
|
||||
"a00000",
|
||||
"a0000000",
|
||||
"A000000",
|
||||
"i000000",
|
||||
"l000000",
|
||||
"o000000",
|
||||
"u000000",
|
||||
"a00000i",
|
||||
"a00000u",
|
||||
"a00000\n",
|
||||
"",
|
||||
],
|
||||
)
|
||||
def test_node_id_pattern_rejects(bad):
|
||||
assert not NODE_ID_RE.match(bad)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "valid"),
|
||||
[("a" * 7, True), ("a" * 26, True), ("a" * 6, False), ("a" * 27, False)],
|
||||
)
|
||||
def test_record_id_pattern_length_bounds(value, valid):
|
||||
assert bool(RECORD_ID_RE.match(value)) is valid
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "valid"),
|
||||
[
|
||||
("hale2026harmonic-damb", True),
|
||||
("0x", True),
|
||||
("a.b_c-d", True),
|
||||
("-a", False),
|
||||
(".a", False),
|
||||
("Ab", False),
|
||||
("", False),
|
||||
],
|
||||
)
|
||||
def test_key_pattern(value, valid):
|
||||
assert bool(KEY_RE.match(value)) is valid
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "valid"),
|
||||
[("0" * 64, True), ("a" * 64, True), ("A" * 64, False), ("0" * 63, False)],
|
||||
)
|
||||
def test_hash_pattern(value, valid):
|
||||
assert bool(HASH_RE.match(value)) is valid
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", vectors("offsets"))
|
||||
def test_offsets_vector(case):
|
||||
plain = case["plain"]
|
||||
assert len(plain) == case["code_points"]
|
||||
assert len(plain.encode("utf-16-le")) // 2 == case["utf16_units"]
|
||||
assert plain[case["span"][0] : case["span"][1]] == case["exact"]
|
||||
assert [cp_to_utf16(plain, x) for x in case["span"]] == case["utf16_span"]
|
||||
|
||||
|
||||
@given(st.text(), st.data())
|
||||
@PROPERTIES
|
||||
def test_cp_to_utf16_counts_code_units(s, data):
|
||||
offset = data.draw(st.integers(min_value=0, max_value=len(s)))
|
||||
assert cp_to_utf16(s, offset) == len(s[:offset].encode("utf-16-le")) // 2
|
||||
|
||||
|
||||
def test_cp_to_utf16_counts_astral_characters_twice():
|
||||
assert cp_to_utf16("a\U0001d465b", 0) == 0
|
||||
assert cp_to_utf16("a\U0001d465b", 2) == 3
|
||||
assert cp_to_utf16("a\U0001d465b", 3) == 4
|
||||
@@ -0,0 +1,403 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Traced citations: resolving a trace, the rules a trace breaks and disputes (paperspec.traces)."""
|
||||
|
||||
import copy
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from conftest import ROOT, read_jsonl, vector_cases
|
||||
from helpers_lib2 import DRAFT, HALE, RECORDS
|
||||
from paperspec import anchors, corrections, package, traces
|
||||
from paperspec.schema import Store
|
||||
|
||||
CASES = vector_cases("traces")
|
||||
RESOLVES = [c for c in CASES if c["op"] == "resolve"]
|
||||
PROBLEMS = [c for c in CASES if c["op"] == "problems"]
|
||||
LAYER = read_jsonl(RECORDS / "traces" / "annotations.jsonl")
|
||||
TRACES = [a for a in LAYER if a["body"]["kind"] == "trace"]
|
||||
MODEL_TRACE = next(
|
||||
a
|
||||
for a in TRACES
|
||||
if a["creator"]["type"] == "Software" and a["status"] == "confirmed"
|
||||
)
|
||||
PERSON_TRACE = next(a for a in TRACES if a["creator"]["type"] == "Person")
|
||||
SAME_AS = ["doi:10.5555/example.hale.2026"]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def schemas():
|
||||
return Store(str(ROOT / "schemas"))
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def states():
|
||||
"""The packages' states by path; tests copy a state before changing it."""
|
||||
cache = {}
|
||||
|
||||
def get(path):
|
||||
if path not in cache:
|
||||
cache[path] = package.read_state(str(ROOT / path))
|
||||
return cache[path]
|
||||
|
||||
return get
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def citing(states):
|
||||
return states("fixtures/packages/" + DRAFT)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def cited(states):
|
||||
return states("fixtures/packages/" + HALE)
|
||||
|
||||
|
||||
def evolve(state, steps, schemas):
|
||||
"""The state after the corrections `steps`, applied through the gate."""
|
||||
for step in steps:
|
||||
state, _ = corrections.apply(state, step, store=schemas)
|
||||
return state
|
||||
|
||||
|
||||
def snapshot(state):
|
||||
"""A comparable dump of the parts of a state that corrections change."""
|
||||
return json.dumps(
|
||||
[state.nodes, state.references, state.ids, state.corrections], sort_keys=True
|
||||
)
|
||||
|
||||
|
||||
def edited(annotation, **body):
|
||||
"""A copy of `annotation` whose body has the given fields replaced; None removes a field."""
|
||||
out = copy.deepcopy(annotation)
|
||||
for key, value in body.items():
|
||||
if value is None:
|
||||
out["body"].pop(key, None)
|
||||
else:
|
||||
out["body"][key] = value
|
||||
return out
|
||||
|
||||
|
||||
# --- the vectors ---------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", RESOLVES, ids=lambda c: c["id"])
|
||||
def test_resolve_vectors(case, states, schemas):
|
||||
start = states(case["citing"])
|
||||
before = snapshot(start)
|
||||
got = traces.resolve(
|
||||
case["annotation"],
|
||||
evolve(start, case["citing_corrections"], schemas),
|
||||
evolve(states(case["cited"]), case["cited_corrections"], schemas),
|
||||
case["same_as"],
|
||||
)
|
||||
assert got == case["expect"]
|
||||
assert snapshot(start) == before
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", RESOLVES, ids=lambda c: c["id"])
|
||||
def test_a_resolution_is_shaped_as_the_result_says(case):
|
||||
got = case["expect"]
|
||||
assert got["result"] in traces.RESULTS
|
||||
assert ("side" in got) == (got["result"] == "stale")
|
||||
assert ("reason" in got) == (got["result"] != "current")
|
||||
assert (got["open"] is None) == (got["cited"]["result"] == "stale")
|
||||
assert got["cites"] == [] or got["open"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", PROBLEMS, ids=lambda c: c["id"])
|
||||
def test_problems_vectors(case, states):
|
||||
got = traces.problems(
|
||||
case["annotation"],
|
||||
states(case["citing"]),
|
||||
states(case["cited"]) if case["cited"] else None,
|
||||
)
|
||||
assert got == case["expect"]
|
||||
|
||||
|
||||
def test_the_vectors_reach_every_outcome_and_both_rules():
|
||||
assert {c["expect"]["result"] for c in RESOLVES} == set(traces.RESULTS)
|
||||
broken = {x["rule"] for c in PROBLEMS for x in c["expect"]}
|
||||
assert broken == {"layer.trace.source", "layer.trace.target"}
|
||||
|
||||
|
||||
def test_one_vector_follows_a_citation_inside_the_passage():
|
||||
followed = [c for c in RESOLVES if c["expect"]["cites"]]
|
||||
assert [c["id"] for c in followed] == ["current-candidate-with-citation"]
|
||||
item = followed[0]["expect"]["cites"][0]
|
||||
assert set(item) == {"node", "run", "span", "text", "refs"}
|
||||
assert item["refs"][0]["resolution"]["type"] == "doi"
|
||||
|
||||
|
||||
# --- the fixture layer ------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("annotation", TRACES, ids=lambda a: a["id"].rsplit("/", 1)[1])
|
||||
def test_a_fixture_trace_breaks_no_rule_and_is_current(annotation, citing, cited):
|
||||
assert traces.problems(annotation) == []
|
||||
assert traces.problems(annotation, citing, cited) == []
|
||||
got = traces.resolve(annotation, citing, cited, SAME_AS)
|
||||
assert got["result"] == "current"
|
||||
assert got["open"] is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("annotation", TRACES, ids=lambda a: a["id"].rsplit("/", 1)[1])
|
||||
def test_a_fixture_traces_reference_resolves_to_its_source(annotation, citing):
|
||||
source = annotation["body"]["source"].split("doc:", 1)[1]
|
||||
record = next(
|
||||
r for r in citing.references if r["node"] == annotation["body"]["reference"]
|
||||
)
|
||||
assert record["resolution"] == {"type": "document", "doc": source}
|
||||
|
||||
|
||||
def test_the_layer_holds_three_traces_and_one_reply():
|
||||
assert len(TRACES) == 3
|
||||
assert len(LAYER) == 4
|
||||
|
||||
|
||||
def test_a_rejected_candidate_resolves_like_any_other_trace(citing, cited):
|
||||
rejected = next(a for a in TRACES if a["status"] == "rejected")
|
||||
assert traces.resolve(rejected, citing, cited, SAME_AS)["result"] == "current"
|
||||
|
||||
|
||||
# --- doc_tag and the helpers that follow replies --------------------------------------------------------------
|
||||
|
||||
|
||||
def test_doc_tag_is_the_tag_a_source_writes():
|
||||
assert (
|
||||
traces.doc_tag("smithlab", HALE) == "tag:scient.ing,2026:doc:smithlab/" + HALE
|
||||
)
|
||||
assert traces.doc_tag("smithlab", HALE) == MODEL_TRACE["body"]["source"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tag",
|
||||
[
|
||||
"tag:scient.ing,2026:doc:smithlab/hale2026harmonic-damb",
|
||||
"tag:scient.ing,2026:doc:a/hale2026harmonic-damb.v2",
|
||||
"tag:scient.ing,2026:doc:a.b-c_d/smith1999x-j4qm.v3.final",
|
||||
],
|
||||
)
|
||||
def test_a_document_tag_names_a_store_and_a_key(tag):
|
||||
assert traces.DOC_TAG.match(tag)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tag",
|
||||
[
|
||||
"hale2026harmonic-damb",
|
||||
"tag:scient.ing,2026:doc:smithlab",
|
||||
"tag:scient.ing,2026:doc:/hale2026harmonic-damb",
|
||||
"tag:scient.ing,2026:doc:smithlab/Hale2026harmonic-damb",
|
||||
"tag:scient.ing,2026:doc:smithlab/hale2026harmonic-damb/extra",
|
||||
"tag:scient.ing,2026:annotation:smithlab/hale2026harmonic-damb",
|
||||
],
|
||||
)
|
||||
def test_a_string_that_is_no_document_tag_does_not_match(tag):
|
||||
assert traces.DOC_TAG.match(tag) is None
|
||||
|
||||
|
||||
def test_a_document_tag_with_a_final_line_feed_does_not_match():
|
||||
assert traces.DOC_TAG.match(MODEL_TRACE["body"]["source"] + "\n") is None
|
||||
|
||||
|
||||
def test_only_the_model_trace_is_disputed():
|
||||
ids = [a["id"] for a in TRACES if traces.disputed(a["id"], LAYER)]
|
||||
assert ids == [MODEL_TRACE["id"]]
|
||||
assert len(traces.disputes(MODEL_TRACE["id"], LAYER)) == 1
|
||||
|
||||
|
||||
def test_a_dispute_is_an_assessing_reply_by_another_creator():
|
||||
reply = traces.disputes(MODEL_TRACE["id"], LAYER)[0]
|
||||
assert reply["motivation"] == "assessing"
|
||||
assert reply["creator"] != MODEL_TRACE["creator"]
|
||||
assert reply not in TRACES
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"change",
|
||||
[
|
||||
{"status": "withdrawn"},
|
||||
{"status": "rejected"},
|
||||
{"motivation": "replying"},
|
||||
{"motivation": None},
|
||||
],
|
||||
ids=["withdrawn", "rejected", "replying", "no-motivation"],
|
||||
)
|
||||
def test_a_reply_that_does_not_stand_or_does_not_assess_disputes_nothing(change):
|
||||
reply = traces.disputes(MODEL_TRACE["id"], LAYER)[0]
|
||||
assert not traces.disputed(MODEL_TRACE["id"], [{**reply, **change}])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", ["proposed", "confirmed"])
|
||||
def test_a_reply_that_stands_disputes(status):
|
||||
reply = traces.disputes(MODEL_TRACE["id"], LAYER)[0]
|
||||
assert traces.disputed(MODEL_TRACE["id"], [{**reply, "status": status}])
|
||||
|
||||
|
||||
def test_replies_are_the_annotations_that_target_an_annotation_without_a_selector():
|
||||
reply = traces.disputes(MODEL_TRACE["id"], LAYER)[0]
|
||||
assert traces.replies(MODEL_TRACE["id"], LAYER) == [reply]
|
||||
assert traces.replies(reply["id"], LAYER) == []
|
||||
assert traces.replies(PERSON_TRACE["id"], LAYER) == []
|
||||
|
||||
|
||||
def test_replies_skip_annotations_with_odd_targets():
|
||||
odd = [
|
||||
{"target": "a string"},
|
||||
{"target": None},
|
||||
{},
|
||||
{"target": {"source": "x", "selector": []}},
|
||||
]
|
||||
assert traces.replies("x", odd) == []
|
||||
|
||||
|
||||
# --- calls with the wrong states or the wrong annotation -----------------------------------------------------------
|
||||
|
||||
|
||||
def test_resolving_with_the_states_swapped_is_refused(citing, cited):
|
||||
with pytest.raises(ValueError, match="citing_state is"):
|
||||
traces.resolve(MODEL_TRACE, cited, citing)
|
||||
|
||||
|
||||
def test_resolving_with_the_citing_state_twice_is_refused(citing):
|
||||
with pytest.raises(ValueError, match="cited_state is"):
|
||||
traces.resolve(MODEL_TRACE, citing, citing)
|
||||
|
||||
|
||||
def test_problems_with_a_wrong_citing_state_is_refused(cited):
|
||||
with pytest.raises(ValueError, match="citing_state is not the document"):
|
||||
traces.problems(MODEL_TRACE, cited)
|
||||
|
||||
|
||||
def test_problems_with_a_wrong_cited_state_is_refused(citing):
|
||||
with pytest.raises(ValueError, match="cited_state is not the document"):
|
||||
traces.problems(MODEL_TRACE, citing, citing)
|
||||
|
||||
|
||||
def test_resolving_an_annotation_that_is_no_trace_is_refused(citing, cited):
|
||||
layer = read_jsonl(RECORDS / "layer" / "annotations.jsonl")
|
||||
with pytest.raises(ValueError, match="not a trace"):
|
||||
traces.resolve(layer[0], citing, cited)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("which", ["target", "body"])
|
||||
def test_resolving_a_trace_without_an_anchor_is_refused(citing, cited, which):
|
||||
trace = copy.deepcopy(MODEL_TRACE)
|
||||
holder = trace["target"] if which == "target" else trace["body"]
|
||||
holder["selector"] = [
|
||||
s for s in holder["selector"] if s["type"] != "AnchorSelector"
|
||||
]
|
||||
with pytest.raises(ValueError, match="needs an AnchorSelector"):
|
||||
traces.resolve(trace, citing, cited)
|
||||
|
||||
|
||||
def anchor_trace(annotation, state, node, on):
|
||||
"""`annotation` with the anchor of its `on` (target or body) made over `node` of `state`."""
|
||||
out = copy.deepcopy(annotation)
|
||||
anchor = anchors.make_anchor(
|
||||
state.doc(), state.index().values, node, state.rev, store="smithlab"
|
||||
)
|
||||
holder = out[on] if on == "target" else out["body"]
|
||||
holder["selector"] = [{"anchor": anchor, "type": "AnchorSelector"}]
|
||||
return out
|
||||
|
||||
|
||||
def test_the_citations_of_a_passage_are_those_of_its_leaves_that_have_runs(
|
||||
citing, cited
|
||||
):
|
||||
paragraph = "tgp1qey"
|
||||
trace = anchor_trace(MODEL_TRACE, cited, paragraph, "body")
|
||||
got = traces.resolve(trace, citing, cited, SAME_AS)
|
||||
assert got["result"] == "current"
|
||||
assert got["cited"]["node"] == paragraph
|
||||
assert [c["node"] for c in got["cites"]] == ["kea5026", "hca6dc5"]
|
||||
leaves = cited.doc().leaves(paragraph)
|
||||
assert {c["node"] for c in got["cites"]} < set(leaves)
|
||||
assert any(cited.doc().nodes[x]["kind"] == "equation" for x in leaves)
|
||||
|
||||
|
||||
def test_a_trace_whose_target_is_a_paragraph_no_longer_cites_the_reference(
|
||||
citing, cited
|
||||
):
|
||||
trace = anchor_trace(MODEL_TRACE, citing, "y4z5j53", "target")
|
||||
got = traces.resolve(trace, citing, cited, SAME_AS)
|
||||
assert got["citing"]["result"] == "current"
|
||||
assert got["result"] == "unlinked"
|
||||
assert rules_of(trace) == ["layer.trace.target"]
|
||||
|
||||
|
||||
# --- the rules, one edit at a time ------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def rules_of(annotation, *states):
|
||||
"""The rules `annotation` breaks, checked against `states`."""
|
||||
return [x["rule"] for x in traces.problems(annotation, *states)]
|
||||
|
||||
|
||||
def test_a_source_that_is_not_a_tag_breaks_the_source_rule():
|
||||
trace = edited(MODEL_TRACE, source="hale2026harmonic-damb")
|
||||
assert rules_of(trace) == ["layer.trace.source"]
|
||||
|
||||
|
||||
def test_a_missing_source_breaks_the_source_rule():
|
||||
assert rules_of(edited(MODEL_TRACE, source=None)) == ["layer.trace.source"]
|
||||
|
||||
|
||||
def test_an_anchor_in_another_store_than_the_source_breaks_the_source_rule():
|
||||
trace = copy.deepcopy(MODEL_TRACE)
|
||||
trace["body"]["selector"][0]["anchor"]["store"] = "elsewhere"
|
||||
got = traces.problems(trace)
|
||||
assert [x["rule"] for x in got] == ["layer.trace.source"]
|
||||
assert "not in the source" in got[0]["message"]
|
||||
|
||||
|
||||
def test_an_anchor_without_a_store_agrees_with_any_source_store():
|
||||
trace = copy.deepcopy(MODEL_TRACE)
|
||||
del trace["body"]["selector"][0]["anchor"]["store"]
|
||||
assert rules_of(trace) == []
|
||||
|
||||
|
||||
def test_a_quote_selector_that_differs_from_the_anchor_quote_breaks_the_source_rule():
|
||||
trace = copy.deepcopy(MODEL_TRACE)
|
||||
trace["body"]["selector"][1]["suffix"] = "something else"
|
||||
assert rules_of(trace) == ["layer.trace.source"]
|
||||
|
||||
|
||||
def test_a_quote_selector_beside_an_anchor_without_a_quote_breaks_the_source_rule():
|
||||
trace = copy.deepcopy(MODEL_TRACE)
|
||||
del trace["body"]["selector"][0]["anchor"]["quote"]
|
||||
assert rules_of(trace) == ["layer.trace.source"]
|
||||
|
||||
|
||||
def test_a_target_anchor_without_a_quote_breaks_the_target_rule():
|
||||
trace = copy.deepcopy(MODEL_TRACE)
|
||||
del trace["target"]["selector"][0]["anchor"]["quote"]
|
||||
assert rules_of(trace) == ["layer.trace.target"]
|
||||
|
||||
|
||||
def test_a_target_that_is_a_whole_document_breaks_the_target_rule():
|
||||
trace = copy.deepcopy(MODEL_TRACE)
|
||||
trace["target"] = {"source": trace["target"]["source"], "type": "SpecificResource"}
|
||||
assert rules_of(trace) == ["layer.trace.target"]
|
||||
|
||||
|
||||
def test_a_reference_that_is_not_in_the_citing_document_breaks_the_target_rule(citing):
|
||||
trace = edited(MODEL_TRACE, reference="zzzzzzz")
|
||||
assert rules_of(trace, citing) == ["layer.trace.target"]
|
||||
|
||||
|
||||
def test_both_rules_are_reported_in_the_order_checked():
|
||||
trace = edited(MODEL_TRACE, source="nowhere")
|
||||
del trace["target"]["selector"][0]["anchor"]["quote"]
|
||||
assert rules_of(trace) == ["layer.trace.source", "layer.trace.target"]
|
||||
|
||||
|
||||
def test_the_trace_alone_is_checked_without_states():
|
||||
assert traces.problems(MODEL_TRACE) == []
|
||||
assert traces.problems(PERSON_TRACE) == []
|
||||
@@ -0,0 +1,658 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Establishing a document from a nested tree, and the structural checks (spec/03, spec/04)."""
|
||||
|
||||
import copy
|
||||
from itertools import pairwise
|
||||
|
||||
import pytest
|
||||
from hypothesis import given, settings, strategies as st
|
||||
|
||||
from conftest import FIXTURES
|
||||
from helpers_lib1 import (
|
||||
bare,
|
||||
build_doc,
|
||||
document,
|
||||
heading,
|
||||
math_run,
|
||||
package_nodes,
|
||||
paragraph,
|
||||
section,
|
||||
sentence,
|
||||
text_run,
|
||||
)
|
||||
from paperspec import hashing, tree
|
||||
from paperspec.model import Doc
|
||||
from paperspec.text import NODE_ID_RE, mint_node_id
|
||||
from paperspec.tree import Refusal
|
||||
|
||||
PROPERTIES = settings(derandomize=True, max_examples=100, database=None, deadline=None)
|
||||
PACKAGES = sorted(
|
||||
p.name
|
||||
for p in (FIXTURES / "packages").iterdir()
|
||||
if p.is_dir() and not p.name.endswith(".checks-only")
|
||||
)
|
||||
|
||||
|
||||
def _tiny():
|
||||
"""The records of the small fixture document, without their values."""
|
||||
return bare(package_nodes(FIXTURES / "packages" / "tiny2026tiny-j4qm"))
|
||||
|
||||
|
||||
def _first(nodes, kind):
|
||||
return next(n for n in nodes if n["kind"] == kind)
|
||||
|
||||
|
||||
def _doc(nodes):
|
||||
return Doc("tiny2026tiny-j4qm", nodes)
|
||||
|
||||
|
||||
def _established(spec):
|
||||
return tree.establish("tiny2026tiny-j4qm", spec)
|
||||
|
||||
|
||||
def _problem(nodes, check=tree.validate_structure):
|
||||
with pytest.raises(Refusal) as excinfo:
|
||||
check(_doc(nodes))
|
||||
return excinfo.value
|
||||
|
||||
|
||||
class TestRefusal:
|
||||
def test_it_carries_the_rule_the_message_and_the_node(self):
|
||||
refusal = Refusal("doc.tree.root", "no root node", "abcdefg")
|
||||
assert refusal.rule == "doc.tree.root"
|
||||
assert refusal.message == "no root node"
|
||||
assert refusal.node == "abcdefg"
|
||||
assert str(refusal) == "doc.tree.root: no root node"
|
||||
|
||||
def test_the_node_is_optional(self):
|
||||
assert Refusal("r", "m").node is None
|
||||
|
||||
def test_it_is_an_exception(self):
|
||||
with pytest.raises(Refusal, match="r: m"):
|
||||
raise Refusal("r", "m")
|
||||
|
||||
|
||||
class TestCanonicalRuns:
|
||||
def test_empty_text_runs_are_dropped(self):
|
||||
assert tree.canonical_runs([text_run(""), text_run("a"), text_run("")]) == [
|
||||
text_run("a")
|
||||
]
|
||||
|
||||
def test_adjacent_text_runs_of_one_style_are_joined(self):
|
||||
runs = [
|
||||
text_run("a"),
|
||||
text_run("b"),
|
||||
text_run("c", style=["em"]),
|
||||
text_run("d", style=["em"]),
|
||||
]
|
||||
assert tree.canonical_runs(runs) == [
|
||||
text_run("ab"),
|
||||
text_run("cd", style=["em"]),
|
||||
]
|
||||
|
||||
def test_a_run_between_two_text_runs_keeps_them_apart(self):
|
||||
runs = [text_run("a"), math_run("x"), text_run("b")]
|
||||
assert tree.canonical_runs(runs) == runs
|
||||
|
||||
def test_styles_are_sorted_without_repeats_and_an_empty_set_is_dropped(self):
|
||||
runs = [text_run("a", style=["sup", "em", "em"]), text_run("b", style=[])]
|
||||
assert tree.canonical_runs(runs) == [
|
||||
text_run("a", style=["em", "sup"]),
|
||||
text_run("b"),
|
||||
]
|
||||
|
||||
def test_equal_style_sets_in_another_order_are_joined(self):
|
||||
runs = [
|
||||
text_run("a", style=["strong", "em"]),
|
||||
text_run("b", style=["em", "strong"]),
|
||||
]
|
||||
assert tree.canonical_runs(runs) == [text_run("ab", style=["em", "strong"])]
|
||||
|
||||
def test_a_printed_equal_to_the_text_records_nothing(self):
|
||||
runs = [text_run("a", printed="a"), text_run("b")]
|
||||
assert tree.canonical_runs(runs) == [text_run("ab")]
|
||||
|
||||
def test_a_run_with_a_printed_text_is_never_joined(self):
|
||||
runs = [
|
||||
text_run("a"),
|
||||
text_run("b", printed="c"),
|
||||
text_run("d"),
|
||||
text_run("e", printed="f"),
|
||||
]
|
||||
assert tree.canonical_runs(runs) == runs
|
||||
|
||||
def test_the_input_is_not_changed(self):
|
||||
runs = [text_run("a", style=["sup", "em"]), text_run("b", style=["sup", "em"])]
|
||||
before = copy.deepcopy(runs)
|
||||
tree.canonical_runs(runs)
|
||||
assert runs == before
|
||||
|
||||
def test_canonical_runs_are_a_fixed_point(self):
|
||||
runs = [
|
||||
text_run("a"),
|
||||
text_run(""),
|
||||
text_run("b", style=["sub"]),
|
||||
text_run("c", printed="d"),
|
||||
]
|
||||
once = tree.canonical_runs(runs)
|
||||
assert tree.canonical_runs(once) == once
|
||||
|
||||
@given(
|
||||
st.lists(
|
||||
st.builds(
|
||||
text_run,
|
||||
st.text(alphabet="ab ", max_size=3),
|
||||
style=st.lists(st.sampled_from(["em", "sup", "sub"]), max_size=2),
|
||||
),
|
||||
max_size=6,
|
||||
)
|
||||
)
|
||||
@PROPERTIES
|
||||
def test_joining_keeps_the_text_and_leaves_no_two_alike_neighbours(self, runs):
|
||||
out = tree.canonical_runs(runs)
|
||||
assert "".join(r["text"] for r in out) == "".join(r["text"] for r in runs)
|
||||
assert all(r["text"] for r in out)
|
||||
assert all(a.get("style") != b.get("style") for a, b in pairwise(out))
|
||||
|
||||
|
||||
class TestEstablish:
|
||||
SPEC = document(
|
||||
section(
|
||||
heading("Results"),
|
||||
paragraph(sentence(text_run("Same.")), sentence(text_run("Same."))),
|
||||
label="1",
|
||||
)
|
||||
)
|
||||
|
||||
def test_it_returns_flat_records_in_document_order_with_parents_and_ord(self):
|
||||
nodes = tree.establish("test2026doc-aaaa", self.SPEC)
|
||||
doc = Doc("test2026doc-aaaa", nodes)
|
||||
assert [n["id"] for n in nodes] == doc.preorder()
|
||||
assert [n["kind"] for n in nodes] == [
|
||||
"document",
|
||||
"section",
|
||||
"heading",
|
||||
"sentence",
|
||||
"paragraph",
|
||||
"sentence",
|
||||
"sentence",
|
||||
]
|
||||
assert nodes[0]["parent"] is None
|
||||
assert [n["ord"] for n in nodes] == [0, 0, 0, 0, 1, 0, 1]
|
||||
assert nodes[3]["parent"] == nodes[2]["id"]
|
||||
assert all("children" not in n and "_parent" not in n for n in nodes)
|
||||
|
||||
def test_every_id_has_the_node_form_and_is_unique(self):
|
||||
nodes = tree.establish("test2026doc-aaaa", self.SPEC)
|
||||
ids = [n["id"] for n in nodes]
|
||||
assert all(NODE_ID_RE.match(i) for i in ids)
|
||||
assert len(set(ids)) == len(ids)
|
||||
|
||||
def test_identical_sentences_get_the_next_occurrence(self):
|
||||
nodes = tree.establish("test2026doc-aaaa", self.SPEC)
|
||||
same = [n["id"] for n in nodes if n["kind"] == "sentence"][1:]
|
||||
text_hash = hashing.PLAIN("Same.")
|
||||
assert same == [
|
||||
mint_node_id("test2026doc-aaaa", "sentence", text_hash, 0),
|
||||
mint_node_id("test2026doc-aaaa", "sentence", text_hash, 1),
|
||||
]
|
||||
|
||||
def test_minting_is_deterministic_and_depends_on_the_document_key(self):
|
||||
a = tree.establish("test2026doc-aaaa", self.SPEC)
|
||||
assert tree.establish("test2026doc-aaaa", self.SPEC) == a
|
||||
other = tree.establish("test2026doc-aaab", self.SPEC)
|
||||
assert [n["id"] for n in other] != [n["id"] for n in a]
|
||||
|
||||
def test_the_spec_is_not_changed(self):
|
||||
spec = copy.deepcopy(self.SPEC)
|
||||
tree.establish("test2026doc-aaaa", spec)
|
||||
assert spec == self.SPEC
|
||||
|
||||
def test_ids_already_used_are_skipped(self):
|
||||
key = "test2026doc-aaaa"
|
||||
plain = tree.establish(key, self.SPEC)
|
||||
used = {plain[3]["id"]}
|
||||
again = tree.establish(key, self.SPEC, used)
|
||||
assert again[3]["id"] != plain[3]["id"]
|
||||
assert again[3]["id"] in used
|
||||
assert len({n["id"] for n in again}) == len(again)
|
||||
|
||||
def test_the_used_set_gains_every_minted_id(self):
|
||||
used = set()
|
||||
nodes = tree.establish("test2026doc-aaaa", self.SPEC, used)
|
||||
assert used == {n["id"] for n in nodes}
|
||||
|
||||
def test_the_text_of_a_container_decides_its_id(self):
|
||||
a = tree.establish(
|
||||
"k-aaaa",
|
||||
document(
|
||||
section(heading("A"), paragraph(sentence(text_run("x"))), label="1")
|
||||
),
|
||||
)
|
||||
b = tree.establish(
|
||||
"k-aaaa",
|
||||
document(
|
||||
section(heading("B"), paragraph(sentence(text_run("x"))), label="1")
|
||||
),
|
||||
)
|
||||
assert a[1]["id"] != b[1]["id"]
|
||||
assert a[4]["id"] == b[4]["id"]
|
||||
|
||||
def test_runs_are_put_in_canonical_form(self):
|
||||
spec = document(
|
||||
section(
|
||||
heading("H"),
|
||||
paragraph(sentence(text_run("a"), text_run(""), text_run("b"))),
|
||||
)
|
||||
)
|
||||
nodes = tree.establish("k-aaaa", spec)
|
||||
(made,) = [
|
||||
n
|
||||
for n in nodes
|
||||
if n["kind"] == "sentence" and n["content"][0]["text"] == "ab"
|
||||
]
|
||||
assert made["content"] == [text_run("ab")]
|
||||
|
||||
def test_an_equation_run_names_its_child_by_position_and_gets_its_id(self):
|
||||
spec = document(
|
||||
section(
|
||||
heading("H"),
|
||||
paragraph(
|
||||
sentence(
|
||||
text_run("see "),
|
||||
{"kind": "equation", "child": 1},
|
||||
{"kind": "equation", "child": 0},
|
||||
children=[
|
||||
{
|
||||
"kind": "equation",
|
||||
"content": {"notation": "tex", "source": "a"},
|
||||
},
|
||||
{
|
||||
"kind": "equation",
|
||||
"content": {"notation": "tex", "source": "b"},
|
||||
},
|
||||
],
|
||||
)
|
||||
),
|
||||
)
|
||||
)
|
||||
nodes = tree.establish("k-aaaa", spec)
|
||||
doc = Doc("k-aaaa", nodes)
|
||||
host = next(
|
||||
n for n in nodes if n["kind"] == "sentence" and len(n["content"]) == 3
|
||||
)
|
||||
first, second = doc.kids(host["id"])
|
||||
assert host["content"] == [
|
||||
text_run("see "),
|
||||
{"kind": "equation", "node": second},
|
||||
{"kind": "equation", "node": first},
|
||||
]
|
||||
|
||||
def test_an_established_document_passes_the_structural_checks(self):
|
||||
doc = Doc("test2026doc-aaaa", tree.establish("test2026doc-aaaa", self.SPEC))
|
||||
tree.validate_structure(doc)
|
||||
tree.validate_references(doc)
|
||||
|
||||
|
||||
class TestMintIds:
|
||||
def _flat(self):
|
||||
return [
|
||||
{"kind": "document", "_parent": None, "ord": 0},
|
||||
{"kind": "section", "_parent": 0, "ord": 0, "attrs": {"level": 1}},
|
||||
{"kind": "paragraph", "_parent": 1, "ord": 0},
|
||||
{
|
||||
"kind": "sentence",
|
||||
"_parent": 2,
|
||||
"ord": 0,
|
||||
"content": [text_run("Hello.")],
|
||||
},
|
||||
]
|
||||
|
||||
def test_it_mints_for_each_flat_node(self):
|
||||
used = set()
|
||||
nodes = tree.mint_ids("k-aaaa", self._flat(), used)
|
||||
assert len(nodes) == 4
|
||||
assert nodes[0]["parent"] is None
|
||||
assert nodes[3]["parent"] == nodes[2]["id"]
|
||||
assert used == {n["id"] for n in nodes}
|
||||
assert all("_parent" not in n for n in nodes)
|
||||
|
||||
def test_preassigned_ids_are_used_as_given(self):
|
||||
used = set()
|
||||
nodes = tree.mint_ids("k-aaaa", self._flat(), used, {3: "zzzzzzz"})
|
||||
assert nodes[3]["id"] == "zzzzzzz"
|
||||
assert "zzzzzzz" in used
|
||||
|
||||
def test_a_preassigned_id_is_the_parent_of_the_nodes_below(self):
|
||||
nodes = tree.mint_ids("k-aaaa", self._flat(), set(), {2: "yyyyyyy"})
|
||||
assert nodes[2]["id"] == "yyyyyyy"
|
||||
assert nodes[3]["parent"] == "yyyyyyy"
|
||||
|
||||
|
||||
class TestValidateStructure:
|
||||
@pytest.mark.parametrize("package", PACKAGES)
|
||||
def test_the_fixture_documents_pass(self, package):
|
||||
nodes = bare(package_nodes(FIXTURES / "packages" / package))
|
||||
doc = Doc(package, nodes)
|
||||
tree.validate_structure(doc)
|
||||
tree.validate_references(doc)
|
||||
|
||||
def test_no_nodes_means_no_root(self):
|
||||
refusal = _problem([])
|
||||
assert refusal.rule == "doc.tree.root"
|
||||
assert refusal.message == "no root node"
|
||||
|
||||
def test_exactly_one_root_of_kind_document_is_required(self):
|
||||
extra = {"id": "zzzzzzz", "kind": "document", "parent": None, "ord": 0}
|
||||
assert _problem([*_tiny(), extra]).rule == "doc.tree.root"
|
||||
only_section = [{"id": "sssssss", "kind": "section", "parent": None, "ord": 0}]
|
||||
refusal = _problem(only_section)
|
||||
assert refusal.rule == "doc.tree.root"
|
||||
assert "kind document" in refusal.message
|
||||
|
||||
def test_nodes_must_be_listed_in_document_order(self):
|
||||
nodes = _tiny()
|
||||
nodes[3], nodes[4] = nodes[4], nodes[3]
|
||||
assert _problem(nodes).rule == "doc.tree.order"
|
||||
|
||||
def test_an_id_must_have_seven_letter_first_characters(self):
|
||||
nodes = _tiny()
|
||||
leaf = next(n for n in nodes if n["kind"] == "cell")
|
||||
leaf["id"] = "A" + leaf["id"][1:]
|
||||
refusal = _problem(nodes)
|
||||
assert refusal.rule == "doc.id.form"
|
||||
assert refusal.node == leaf["id"]
|
||||
|
||||
def test_an_id_that_repeats_breaks_the_child_order_or_the_listing(self):
|
||||
nodes = _tiny()
|
||||
first = next(n for n in nodes if n["kind"] == "sentence")
|
||||
at = nodes.index(first) + 1
|
||||
beside = [*nodes[:at], copy.deepcopy(first), *nodes[at:]]
|
||||
assert _problem(beside).rule == "doc.tree.ord"
|
||||
later = copy.deepcopy(first)
|
||||
later["ord"] = 5
|
||||
assert _problem([*nodes, later]).rule == "doc.tree.order"
|
||||
|
||||
def test_child_order_must_count_up_from_zero(self):
|
||||
nodes = _tiny()
|
||||
sentence_node = next(n for n in nodes if n["kind"] == "sentence")
|
||||
sentence_node["ord"] = 4
|
||||
assert _problem(nodes).rule == "doc.tree.ord"
|
||||
|
||||
def test_a_node_may_only_contain_the_kinds_its_parent_allows(self):
|
||||
nodes = _tiny()
|
||||
row = _first(nodes, "row")
|
||||
row["kind"] = "paragraph"
|
||||
refusal = _problem(nodes)
|
||||
assert refusal.rule == "doc.tree.child"
|
||||
assert "may not contain" in refusal.message
|
||||
|
||||
def test_a_heading_must_be_first_in_its_section(self):
|
||||
spec = document(section(paragraph(sentence(text_run("x"))), heading("H")))
|
||||
refusal = _problem(_established(spec))
|
||||
assert refusal.rule == "doc.tree.child"
|
||||
assert "first child" in refusal.message
|
||||
|
||||
def test_a_kind_that_may_occur_once_is_refused_twice(self):
|
||||
title = {"kind": "title", "children": [sentence(text_run("T"))]}
|
||||
spec = document(
|
||||
{"kind": "front", "children": [title]},
|
||||
{"kind": "front", "children": [title]},
|
||||
)
|
||||
refusal = _problem(_established(spec))
|
||||
assert refusal.rule == "doc.tree.child"
|
||||
assert refusal.message == "front may occur once in document"
|
||||
|
||||
def test_a_table_has_exactly_one_body(self):
|
||||
refusal = _problem(_established(document(section({"kind": "table"}))))
|
||||
assert refusal.rule == "doc.tree.child"
|
||||
assert refusal.message == "a table has exactly one table-body"
|
||||
body = {
|
||||
"kind": "table-body",
|
||||
"children": [
|
||||
{
|
||||
"kind": "row",
|
||||
"children": [{"kind": "cell", "content": [text_run("1")]}],
|
||||
}
|
||||
],
|
||||
}
|
||||
two = document(section({"kind": "table", "children": [body, body]}))
|
||||
assert (
|
||||
"table-body may occur once in table" in _problem(_established(two)).message
|
||||
)
|
||||
|
||||
def test_a_kind_that_needs_children_may_not_be_empty(self):
|
||||
spec = document(section(heading("H"), {"kind": "paragraph"}))
|
||||
refusal = _problem(_established(spec))
|
||||
assert refusal.rule == "doc.tree.empty"
|
||||
assert refusal.message == "paragraph must have children"
|
||||
|
||||
def test_only_labelled_kinds_take_a_label(self):
|
||||
nodes = _tiny()
|
||||
_first(nodes, "paragraph")["label"] = "7"
|
||||
assert _problem(nodes).rule == "doc.node.label"
|
||||
|
||||
def test_a_withheld_node_keeps_no_content_and_needs_none(self):
|
||||
nodes = _tiny()
|
||||
sentence_node = _first(nodes, "sentence")
|
||||
sentence_node["withheld"] = True
|
||||
assert _problem(nodes).rule == "doc.policy.withheld"
|
||||
del sentence_node["content"]
|
||||
tree.validate_structure(_doc(nodes))
|
||||
|
||||
def test_runs_are_required_of_a_run_kind(self):
|
||||
for broken in ({"content": []}, {"content": "text"}, {"content": None}):
|
||||
nodes = _tiny()
|
||||
_first(nodes, "sentence").update(broken)
|
||||
assert _problem(nodes).rule == "doc.node.content"
|
||||
nodes = _tiny()
|
||||
del _first(nodes, "sentence")["content"]
|
||||
assert _problem(nodes).rule == "doc.node.content"
|
||||
|
||||
def test_runs_must_be_in_canonical_form(self):
|
||||
nodes = _tiny()
|
||||
_first(nodes, "sentence")["content"] = [text_run("a"), text_run("b")]
|
||||
assert _problem(nodes).rule == "doc.run.canonical"
|
||||
|
||||
def test_a_run_must_be_of_a_known_kind(self):
|
||||
nodes = _tiny()
|
||||
_first(nodes, "sentence")["content"] = [{"kind": "image", "text": "x"}]
|
||||
refusal = _problem(nodes)
|
||||
assert refusal.rule == "doc.run.kind"
|
||||
assert "image" in refusal.message
|
||||
|
||||
@pytest.mark.parametrize("style", [["bold"], ["sub", "sup"], ["em", "wavy"]])
|
||||
def test_styles_must_be_known_and_not_both_scripts(self, style):
|
||||
nodes = _tiny()
|
||||
_first(nodes, "sentence")["content"] = [text_run("a", style=style)]
|
||||
assert _problem(nodes).rule == "doc.run.style"
|
||||
|
||||
def test_equation_runs_must_name_the_sentences_children_in_order(self):
|
||||
nodes = _tiny()
|
||||
_first(nodes, "sentence")["content"] = [{"kind": "equation", "node": "zzzzzzz"}]
|
||||
assert _problem(nodes).rule == "doc.run.equation"
|
||||
|
||||
def test_a_child_equation_needs_a_run_that_names_it(self):
|
||||
nodes = _tiny()
|
||||
host = _first(nodes, "sentence")
|
||||
nodes.insert(
|
||||
nodes.index(host) + 1,
|
||||
{
|
||||
"id": "eqqqqqq",
|
||||
"kind": "equation",
|
||||
"parent": host["id"],
|
||||
"ord": 0,
|
||||
"content": {"notation": "tex", "source": "x"},
|
||||
},
|
||||
)
|
||||
assert _problem(nodes).rule == "doc.run.equation"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"content",
|
||||
[
|
||||
None,
|
||||
{},
|
||||
{"source": "x"},
|
||||
{"notation": "tex"},
|
||||
{"notation": "mathml", "source": "x"},
|
||||
{"notation": "tex", "source": 1},
|
||||
],
|
||||
)
|
||||
def test_an_equation_needs_a_known_notation_and_a_source(self, content):
|
||||
doc = build_doc(
|
||||
document(
|
||||
section(
|
||||
heading("H"),
|
||||
{"kind": "equation", "content": {"notation": "tex", "source": "x"}},
|
||||
)
|
||||
)
|
||||
)
|
||||
nodes = [dict(doc.nodes[n]) for n in doc.order]
|
||||
eq = _first(nodes, "equation")
|
||||
if content is None:
|
||||
del eq["content"]
|
||||
else:
|
||||
eq["content"] = content
|
||||
with pytest.raises(Refusal) as excinfo:
|
||||
tree.validate_structure(Doc(doc.key, nodes))
|
||||
assert excinfo.value.rule == "doc.node.content"
|
||||
|
||||
def test_a_verbatim_node_needs_text(self):
|
||||
doc = build_doc(
|
||||
document(section(heading("H"), {"kind": "code", "content": {"text": "x"}}))
|
||||
)
|
||||
nodes = [dict(doc.nodes[n]) for n in doc.order]
|
||||
tree.validate_structure(Doc(doc.key, nodes))
|
||||
_first(nodes, "code")["content"] = {"text": 5}
|
||||
with pytest.raises(Refusal) as excinfo:
|
||||
tree.validate_structure(Doc(doc.key, nodes))
|
||||
assert excinfo.value.rule == "doc.node.content"
|
||||
|
||||
def test_a_container_holds_no_content(self):
|
||||
nodes = _tiny()
|
||||
_first(nodes, "paragraph")["content"] = [text_run("x")]
|
||||
refusal = _problem(nodes)
|
||||
assert refusal.rule == "doc.node.content"
|
||||
assert "holds no content" in refusal.message
|
||||
|
||||
def test_a_section_needs_an_integer_level(self):
|
||||
for attrs in ({}, {"level": "1"}, {"level": 1.5}):
|
||||
nodes = _tiny()
|
||||
_first(nodes, "section")["attrs"] = attrs
|
||||
assert _problem(nodes).rule == "doc.node.attrs"
|
||||
nodes = _tiny()
|
||||
del _first(nodes, "section")["attrs"]
|
||||
assert _problem(nodes).rule == "doc.node.attrs"
|
||||
|
||||
|
||||
class TestValidateReferences:
|
||||
def test_a_citation_must_name_a_reference(self):
|
||||
nodes = _tiny()
|
||||
cite = next(
|
||||
r
|
||||
for n in nodes
|
||||
if n["kind"] == "sentence"
|
||||
for r in n["content"]
|
||||
if r["kind"] == "cite"
|
||||
)
|
||||
cite["refs"] = [_first(nodes, "paragraph")["id"]]
|
||||
refusal = _problem(nodes, tree.validate_references)
|
||||
assert refusal.rule == "doc.run.cite"
|
||||
assert refusal.node is not None
|
||||
|
||||
def test_a_citation_of_an_unknown_node_is_refused(self):
|
||||
nodes = _tiny()
|
||||
cite = next(
|
||||
r
|
||||
for n in nodes
|
||||
if n["kind"] == "sentence"
|
||||
for r in n["content"]
|
||||
if r["kind"] == "cite"
|
||||
)
|
||||
cite["refs"] = ["zzzzzzz"]
|
||||
assert _problem(nodes, tree.validate_references).rule == "doc.run.cite"
|
||||
|
||||
def test_a_footnote_mark_must_name_a_footnote_when_it_names_anything(self):
|
||||
nodes = _tiny()
|
||||
host = _first(nodes, "sentence")
|
||||
sentence_id = host["id"]
|
||||
host["content"] = [
|
||||
text_run("a"),
|
||||
{"kind": "fnmark", "footnote": sentence_id, "text": "1"},
|
||||
]
|
||||
assert _problem(nodes, tree.validate_references).rule == "doc.run.fnmark"
|
||||
host["content"] = [
|
||||
text_run("a"),
|
||||
{"kind": "fnmark", "footnote": None, "text": "1"},
|
||||
]
|
||||
tree.validate_references(_doc(nodes))
|
||||
host["content"] = [text_run("a"), {"kind": "fnmark", "text": "1"}]
|
||||
tree.validate_references(_doc(nodes))
|
||||
|
||||
def test_a_cross_reference_must_name_a_node_here_or_carry_an_address(self):
|
||||
nodes = _tiny()
|
||||
host = _first(nodes, "sentence")
|
||||
table = _first(nodes, "table")["id"]
|
||||
host["content"] = [{"kind": "xref", "target": "zzzzzzz", "text": "Fig. 9"}]
|
||||
assert _problem(nodes, tree.validate_references).rule == "doc.run.xref"
|
||||
for target in (table, "other2026paper-abcd#aaaaaaa", "", None):
|
||||
host["content"] = [{"kind": "xref", "target": target, "text": "Table 1"}]
|
||||
tree.validate_references(_doc(nodes))
|
||||
|
||||
def test_a_header_path_of_a_table_must_name_cells(self):
|
||||
nodes = _tiny()
|
||||
table = _first(nodes, "table")
|
||||
table["attrs"]["columns"][0]["header_path"] = [_first(nodes, "row")["id"]]
|
||||
refusal = _problem(nodes, tree.validate_references)
|
||||
assert refusal.rule == "doc.table.header"
|
||||
assert refusal.node == table["id"]
|
||||
|
||||
def test_a_header_path_of_a_row_must_name_cells(self):
|
||||
nodes = _tiny()
|
||||
row = next(
|
||||
n
|
||||
for n in nodes
|
||||
if n["kind"] == "row" and "header_path" in n.get("attrs", {})
|
||||
)
|
||||
row["attrs"]["header_path"] = ["zzzzzzz"]
|
||||
assert _problem(nodes, tree.validate_references).rule == "doc.table.header"
|
||||
|
||||
def test_a_cell_may_only_name_a_column_its_table_defines(self):
|
||||
nodes = _tiny()
|
||||
cell = next(n for n in nodes if n["kind"] == "cell")
|
||||
cell["attrs"]["column"] = "nosuchc"
|
||||
refusal = _problem(nodes, tree.validate_references)
|
||||
assert refusal.rule == "doc.table.header"
|
||||
assert refusal.node == cell["id"]
|
||||
|
||||
def test_a_cell_outside_any_table_may_name_any_column(self):
|
||||
doc = build_doc(
|
||||
document(
|
||||
section(
|
||||
heading("H"),
|
||||
{
|
||||
"kind": "list",
|
||||
"children": [
|
||||
{
|
||||
"kind": "item",
|
||||
"children": [paragraph(sentence(text_run("x")))],
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
)
|
||||
)
|
||||
nodes = [dict(doc.nodes[n]) for n in doc.order]
|
||||
tree.validate_references(Doc(doc.key, nodes))
|
||||
|
||||
def test_withheld_runs_are_not_checked(self):
|
||||
nodes = _tiny()
|
||||
host = _first(nodes, "sentence")
|
||||
host["content"] = [{"kind": "cite", "refs": ["zzzzzzz"], "text": "1"}]
|
||||
host["withheld"] = True
|
||||
tree.validate_references(_doc(nodes))
|
||||
@@ -0,0 +1,197 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""The ECVRF of RFC 9381 over Edwards25519 (paperspec.vrf)."""
|
||||
|
||||
import pytest
|
||||
from hypothesis import given, settings, strategies as st
|
||||
|
||||
from conftest import vector_cases
|
||||
from helpers_lib2 import PROPS
|
||||
from paperspec import crypto, vrf
|
||||
|
||||
CASES = vector_cases("vrf")
|
||||
PROVABLE = [c for c in CASES if "secret" in c]
|
||||
BASE_POINT = bytes.fromhex("58" + "66" * 31)
|
||||
IDENTITY_ENCODING = bytes([1]) + bytes(31)
|
||||
|
||||
|
||||
def hexes(case):
|
||||
"""The public key, input and proof of a case as bytes."""
|
||||
return (
|
||||
bytes.fromhex(case["public"]),
|
||||
bytes.fromhex(case["alpha"]),
|
||||
bytes.fromhex(case["proof"]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", PROVABLE, ids=lambda c: c["id"])
|
||||
def test_prove_reproduces_the_vector_proof(case):
|
||||
secret = bytes.fromhex(case["secret"])
|
||||
public, alpha, proof = hexes(case)
|
||||
assert vrf.public_key(secret) == public == crypto.ed25519_public(secret)
|
||||
assert vrf.prove(secret, alpha) == proof
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", CASES, ids=lambda c: c["id"])
|
||||
def test_verify_returns_the_vector_output_or_nothing(case):
|
||||
public, alpha, proof = hexes(case)
|
||||
beta = vrf.verify(public, alpha, proof)
|
||||
assert (beta.hex() if beta else None) == case["beta"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", [c for c in CASES if c["beta"]], ids=lambda c: c["id"])
|
||||
def test_proof_to_hash_gives_the_vector_output(case):
|
||||
assert vrf.proof_to_hash(bytes.fromhex(case["proof"])).hex() == case["beta"]
|
||||
|
||||
|
||||
def test_rfc_9381_example_16_is_reproduced():
|
||||
case = next(c for c in CASES if c["id"] == "rfc9381-example-16")
|
||||
assert case["proof"].startswith("8657106690b5526245a92b003bb079cc")
|
||||
assert case["beta"].startswith("90cf1df3b703cce59e2a35b925d41116")
|
||||
|
||||
|
||||
def test_the_vectors_include_proofs_that_must_not_verify():
|
||||
assert {c["id"] for c in CASES if c["beta"] is None} == {
|
||||
"tampered-challenge",
|
||||
"other-input",
|
||||
}
|
||||
|
||||
|
||||
def test_verify_rejects_a_proof_for_another_input_or_key():
|
||||
case = PROVABLE[0]
|
||||
public, alpha, proof = hexes(case)
|
||||
assert vrf.verify(public, alpha + b"x", proof) is None
|
||||
assert vrf.verify(vrf.public_key(bytes(32)), alpha, proof) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("position", [0, 20, 31, 32, 47, 48, 70, 79])
|
||||
def test_verify_rejects_a_proof_with_any_byte_changed(position):
|
||||
public, alpha, proof = hexes(PROVABLE[0])
|
||||
bad = bytearray(proof)
|
||||
bad[position] ^= 0x01
|
||||
assert vrf.verify(public, alpha, bytes(bad)) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("length", [0, 79, 81])
|
||||
def test_verify_rejects_a_proof_of_the_wrong_length(length):
|
||||
public, alpha, proof = hexes(PROVABLE[0])
|
||||
assert vrf.verify(public, alpha, (proof + bytes(2))[:length]) is None
|
||||
assert vrf.proof_to_hash((proof + bytes(2))[:length]) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"public",
|
||||
[b"", bytes(31), b"\xff" * 32, IDENTITY_ENCODING, bytes(32)],
|
||||
ids=["empty", "short", "no-point", "identity", "order-four-point"],
|
||||
)
|
||||
def test_verify_rejects_a_public_key_that_is_no_usable_point(public):
|
||||
_, alpha, proof = hexes(PROVABLE[0])
|
||||
assert vrf.verify(public, alpha, proof) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"args",
|
||||
[(None, b"", bytes(80)), (bytes(32), None, bytes(80)), (bytes(32), b"", None)],
|
||||
)
|
||||
def test_verify_never_raises_on_wrong_types(args):
|
||||
assert vrf.verify(*args) is None
|
||||
|
||||
|
||||
def test_proof_whose_scalar_is_not_below_the_group_order_is_malformed():
|
||||
_, _, proof = hexes(PROVABLE[0])
|
||||
too_big = proof[:48] + vrf.Q.to_bytes(32, "little")
|
||||
assert vrf.proof_to_hash(too_big) is None
|
||||
assert vrf.decode_proof(too_big) is None
|
||||
|
||||
|
||||
def test_proof_whose_point_is_invalid_is_malformed():
|
||||
_, _, proof = hexes(PROVABLE[0])
|
||||
assert vrf.decode_proof(b"\xff" * 32 + proof[32:]) is None
|
||||
|
||||
|
||||
def test_decode_proof_splits_point_challenge_and_scalar():
|
||||
_, _, proof = hexes(PROVABLE[0])
|
||||
gamma, c, s = vrf.decode_proof(proof)
|
||||
assert vrf.point_to_string(gamma) == proof[:32]
|
||||
assert c == int.from_bytes(proof[32:48], "little")
|
||||
assert s == int.from_bytes(proof[48:], "little") < vrf.Q
|
||||
|
||||
|
||||
def test_the_base_point_encodes_as_in_rfc_8032():
|
||||
assert vrf.point_to_string(vrf.G) == BASE_POINT
|
||||
assert vrf.string_to_point(BASE_POINT) is not None
|
||||
assert vrf.public_key(bytes(32)) == crypto.ed25519_public(bytes(32))
|
||||
|
||||
|
||||
def test_string_to_point_refuses_wrong_lengths_and_noncanonical_coordinates():
|
||||
assert vrf.string_to_point(bytes(31)) is None
|
||||
assert vrf.string_to_point(bytes(33)) is None
|
||||
p_as_y = vrf.P.to_bytes(32, "little")
|
||||
assert vrf.string_to_point(p_as_y) is None
|
||||
|
||||
|
||||
def test_string_to_point_refuses_the_sign_bit_of_a_zero_coordinate():
|
||||
positive = vrf.string_to_point(IDENTITY_ENCODING)
|
||||
assert positive is not None
|
||||
assert positive[0] == 0
|
||||
negative = bytes([1]) + bytes(30) + bytes([0x80])
|
||||
assert vrf.string_to_point(negative) is None
|
||||
|
||||
|
||||
def test_about_half_of_all_y_coordinates_encode_no_point():
|
||||
found = [
|
||||
vrf.string_to_point(y.to_bytes(32, "little")) is not None for y in range(2, 66)
|
||||
]
|
||||
assert 16 < sum(found) < 48
|
||||
|
||||
|
||||
@given(st.binary(min_size=32, max_size=32))
|
||||
@PROPS
|
||||
def test_a_decoded_point_encodes_back_to_the_same_bytes(encoded):
|
||||
point = vrf.string_to_point(encoded)
|
||||
if point is not None:
|
||||
assert vrf.point_to_string(point) == encoded
|
||||
|
||||
|
||||
@given(st.binary(min_size=32, max_size=32), st.binary(max_size=40))
|
||||
@settings(PROPS, max_examples=12)
|
||||
def test_a_proof_verifies_to_its_hash_and_is_deterministic(secret, alpha):
|
||||
public = vrf.public_key(secret)
|
||||
proof = vrf.prove(secret, alpha)
|
||||
beta = vrf.verify(public, alpha, proof)
|
||||
assert len(proof) == 80
|
||||
assert beta is not None
|
||||
assert len(beta) == 64
|
||||
assert beta == vrf.proof_to_hash(proof)
|
||||
assert proof == vrf.prove(secret, alpha)
|
||||
assert vrf.verify(public, alpha + b"\x00", proof) is None
|
||||
|
||||
|
||||
def test_encode_to_curve_is_deterministic_and_depends_on_the_input():
|
||||
key = vrf.public_key(bytes(32))
|
||||
first = vrf.encode_to_curve(key, b"input")
|
||||
assert vrf.point_to_string(first) == vrf.point_to_string(
|
||||
vrf.encode_to_curve(key, b"input")
|
||||
)
|
||||
assert vrf.point_to_string(first) != vrf.point_to_string(
|
||||
vrf.encode_to_curve(key, b"other")
|
||||
)
|
||||
assert vrf.point_to_string(first) != vrf.point_to_string(
|
||||
vrf.encode_to_curve(vrf.public_key(bytes(31) + b"\x01"), b"input")
|
||||
)
|
||||
|
||||
|
||||
def test_encode_to_curve_gives_up_after_256_attempts(monkeypatch):
|
||||
monkeypatch.setattr(vrf, "string_to_point", lambda s: None)
|
||||
with pytest.raises(ValueError, match="no curve point found"):
|
||||
vrf.encode_to_curve(bytes(32), b"")
|
||||
|
||||
|
||||
def test_encode_to_curve_skips_points_the_cofactor_sends_to_the_identity(monkeypatch):
|
||||
monkeypatch.setattr(vrf, "string_to_point", lambda s: vrf.IDENTITY)
|
||||
with pytest.raises(ValueError, match="no curve point found"):
|
||||
vrf.encode_to_curve(bytes(32), b"")
|
||||
Reference in new issue
Block a user