Add the pytest suite and fix the bugs it found

The suite takes its data from the vectors and fixtures: 3635 tests, 97.9% branch
coverage, and paperspec check and paperspec stress give the same results as before.
Fixed: receipts.verify_signature raised KeyError for a record without a format;
traces.DOC_TAG matched a tag followed by a line feed; paperspec load and paperspec check
left SQLite connections open; the site builder copied the output folder's subfolders into
itself.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
This commit is contained in:
alexmandClaude committed 2026-10-19 04:32:44 -04:00
1 parent 30d310f64d
commit a8fc7237ae
76 files changed
+15564 -80

No files matched your search

+393
View File
@@ -0,0 +1,393 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Scienting
#
# Source Code: https://git.scient.ing/papers/paperspec
# See the LICENSE.md file for full license terms.
"""The schema store, the keyword-subset lint and validation (paperspec.schema)."""
import json
import sys
import pytest
from jsonschema import Draft202012Validator
from conftest import ROOT, read_jsonl
from helpers_lib2 import RECORDS, schema_store
from paperspec import schema
SCHEMA_FILES = sorted(p.name for p in (ROOT / "schemas").glob("*.schema.json"))
FORMATS = json.loads((ROOT / "registry" / "formats.json").read_text(encoding="utf-8"))[
"formats"
]
def write_schemas(folder, **schemas):
"""Write each keyword argument as `<name>.schema.json` under `folder`, and return the folder as text."""
for name, body in schemas.items():
full = {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": f"https://example.test/schemas/{name}.schema.json",
**body,
}
(folder / (name + ".schema.json")).write_text(
json.dumps(full), encoding="utf-8"
)
return str(folder)
def object_schema(**properties):
"""A closed object schema with the given properties."""
return {"type": "object", "properties": properties, "additionalProperties": False}
# --- the repository's schemas ---------------------------------------------------------------------------------------------
@pytest.mark.parametrize("name", SCHEMA_FILES)
def test_a_repository_schema_is_in_the_subset_and_valid_draft_2020_12(name):
body = schema_store().load(name)
assert body["$id"].endswith("/" + name)
Draft202012Validator.check_schema(body)
@pytest.mark.parametrize("entry", FORMATS, ids=lambda f: f["format"])
def test_every_format_names_a_schema_that_exists(entry):
assert (ROOT / "schemas" / entry["schema"]).exists() or (
ROOT / entry["schema"]
).exists()
def test_the_repository_has_a_schema_for_every_fixture_record_format():
by_format = {f["format"]: f["schema"] for f in FORMATS}
for path in sorted(RECORDS.glob("*.json")):
record = json.loads(path.read_text(encoding="utf-8"))
if "format" in record:
assert record["format"] in by_format, path.name
def fixture_schema(path, record):
"""The schema a record under `fixtures/records` is written for."""
if path.name == "view.vl.json":
return "view.schema.json"
if path.name == "layer.json":
return "layer.schema.json"
return {f["format"]: f["schema"] for f in FORMATS}[record["format"]]
FIXTURE_RECORDS = sorted(
p
for p in RECORDS.rglob("*.json")
if "log" not in p.relative_to(RECORDS).parts
and "datasets" not in p.relative_to(RECORDS).parts
)
@pytest.mark.parametrize(
"path", FIXTURE_RECORDS, ids=lambda p: p.relative_to(RECORDS).as_posix()
)
def test_a_fixture_record_is_valid_under_its_schema(path):
record = json.loads(path.read_text(encoding="utf-8"))
assert schema.validate(schema_store(), fixture_schema(path, record), record) == []
@pytest.mark.parametrize(
"path",
sorted(RECORDS.rglob("annotations.jsonl")),
ids=lambda p: p.parent.name,
)
def test_every_annotation_of_the_fixture_layers_is_valid(path):
for record in read_jsonl(path):
assert schema.validate(schema_store(), "annotation.schema.json", record) == []
# --- the lint -------------------------------------------------------------------------------------------------------------
@pytest.mark.parametrize(
"body",
[
{"type": "string", "pattern": "^[a-z]+$"},
{
"type": "object",
"properties": {"a": {"type": "integer", "minimum": 0}},
"required": ["a"],
},
{"items": {"type": "string"}, "minItems": 1, "uniqueItems": True},
{"additionalProperties": False},
{"$defs": {"x": {"enum": [1, 2]}}, "$ref": "#/$defs/x"},
{"oneOf": [{"const": 1}, {"const": 2}], "not": {"const": 3}},
{"if": {"type": "string"}, "then": {"minLength": 1}, "else": {"type": "null"}},
{"prefixItems": [{"type": "string"}], "contains": {"type": "integer"}},
{
"patternProperties": {"^x-": {"type": "string"}},
"propertyNames": {"maxLength": 4},
},
{"description": "Free text may say \\d and (?<name>) without being a pattern."},
{"type": "string", "format": "date-time", "default": "", "deprecated": True},
True,
["not", "a", "schema"],
],
ids=[
"pattern",
"properties",
"items",
"boolean-additional-properties",
"defs-and-ref",
"one-of-and-not",
"if-then-else",
"prefix-items-and-contains",
"pattern-properties-and-names",
"description-mentioning-syntax",
"format-default-deprecated",
"boolean-schema",
"not-a-mapping",
],
)
def test_lint_accepts_schemas_inside_the_subset(body):
assert schema.lint(body, "ok.schema.json") is None
@pytest.mark.parametrize(
("body", "where"),
[
({"unevaluatedProperties": False}, "w.json: keyword unevaluatedProperties"),
(
{"properties": {"a": {"dependentSchemas": {}}}},
"w.json/a: keyword dependentSchemas",
),
({"$defs": {"d": {"$anchor": "x"}}}, "w.json/d: keyword $anchor"),
(
{"items": {"contentEncoding": "base64"}},
"w.json/items: keyword contentEncoding",
),
(
{"additionalProperties": {"unevaluatedItems": True}},
"w.json/additionalProperties: keyword",
),
({"propertyNames": {"$dynamicRef": "#"}}, "w.json/propertyNames: keyword"),
({"not": {"multipleOf": 2}}, "w.json/not: keyword multipleOf"),
({"if": {"$vocabulary": {}}}, "w.json/if: keyword"),
({"then": {"contentMediaType": "x"}}, "w.json/then: keyword"),
({"else": {"readOnly": True}}, "w.json/else: keyword readOnly"),
({"contains": {"minContains": 1}}, "w.json/contains: keyword minContains"),
(
{"oneOf": [{"type": "null"}, {"title": "t", "writeOnly": True}]},
"w.json/oneOf/1: keyword writeOnly",
),
({"anyOf": [{"x": 1}]}, "w.json/anyOf/0: keyword x"),
({"allOf": [{}, {}, {"unevaluatedItems": 1}]}, "w.json/allOf/2: keyword"),
({"prefixItems": [{"dependentRequired2": 1}]}, "w.json/prefixItems/0: keyword"),
],
)
def test_lint_refuses_keywords_outside_the_subset_and_says_where(body, where):
with pytest.raises(schema.SchemaError) as excinfo:
schema.lint(body, "w.json")
assert where in str(excinfo.value)
assert "outside the subset" in str(excinfo.value)
@pytest.mark.parametrize("token", schema.FORBIDDEN_PATTERN_TOKENS)
def test_lint_refuses_regex_syntax_that_differs_between_dialects(token):
with pytest.raises(schema.SchemaError, match="dialect-dependent syntax"):
schema.lint({"pattern": "^a" + token + "z$"}, "w.json")
with pytest.raises(schema.SchemaError, match="dialect-dependent syntax"):
schema.lint({"patternProperties": {"^a" + token: {}}}, "w.json")
def test_lint_looks_into_nested_patterns():
body = {"properties": {"a": {"items": {"oneOf": [{"pattern": "\\d+"}]}}}}
with pytest.raises(schema.SchemaError, match=r"w.json/a/items/oneOf/0"):
schema.lint(body, "w.json")
def test_the_forbidden_tokens_are_the_digit_word_space_and_boundary_classes_and_named_groups():
assert set(schema.FORBIDDEN_PATTERN_TOKENS) == {
"\\d",
"\\w",
"\\s",
"\\b",
"\\D",
"\\W",
"\\S",
"\\B",
"(?<",
"(?P",
}
def test_the_keyword_subset_holds_the_keywords_the_chapter_allows():
assert {
"$ref",
"$defs",
"oneOf",
"patternProperties",
"dependentRequired",
} <= schema.KEYWORDS
assert (
not {"unevaluatedProperties", "$anchor", "dependentSchemas"} & schema.KEYWORDS
)
# --- the store ---------------------------------------------------------------------------------------------------------------
def test_load_reads_a_schema_once_and_lints_it(tmp_path):
folder = write_schemas(tmp_path, a=object_schema(x={"type": "string"}))
store = schema.Store(folder)
first = store.load("a.schema.json")
assert first["type"] == "object"
(tmp_path / "a.schema.json").write_text("{}", encoding="utf-8")
assert store.load("a.schema.json") is first
def test_load_refuses_a_schema_outside_the_subset(tmp_path):
folder = write_schemas(tmp_path, bad={"unevaluatedProperties": False})
with pytest.raises(schema.SchemaError, match=r"bad\.schema\.json"):
schema.Store(folder).load("bad.schema.json")
def test_load_of_a_missing_schema_fails(tmp_path):
with pytest.raises(FileNotFoundError):
schema.Store(str(tmp_path)).load("nothing.schema.json")
@pytest.fixture
def referencing_store(tmp_path):
common = {
"$defs": {
"name": {"type": "string", "minLength": 2},
"a/b": {"type": "integer"},
"t~k": {"type": "null"},
},
}
person = {
"type": "object",
"properties": {
"name": {"$ref": "common.schema.json#/$defs/name"},
"age": {"type": "integer"},
},
"required": ["name"],
"additionalProperties": False,
}
return schema.Store(write_schemas(tmp_path, common=common, person=person))
def test_deref_follows_a_pointer_into_the_same_or_another_schema(referencing_store):
name, node = referencing_store.deref(
"person.schema.json", "common.schema.json#/$defs/name"
)
assert (name, node) == ("common.schema.json", {"type": "string", "minLength": 2})
name, node = referencing_store.deref("person.schema.json", "#/properties/age")
assert (name, node) == ("person.schema.json", {"type": "integer"})
def test_deref_without_a_fragment_gives_the_whole_schema(referencing_store):
name, node = referencing_store.deref("person.schema.json", "common.schema.json")
assert name == "common.schema.json"
assert node is referencing_store.load("common.schema.json")
@pytest.mark.parametrize(
("pointer", "expected"),
[("#/$defs/a~1b", {"type": "integer"}), ("#/$defs/t~0k", {"type": "null"})],
)
def test_deref_unescapes_pointer_tokens(referencing_store, pointer, expected):
assert referencing_store.deref("common.schema.json", pointer)[1] == expected
def test_deref_of_a_missing_member_fails(referencing_store):
with pytest.raises(KeyError):
referencing_store.deref("common.schema.json", "#/$defs/nope")
def test_validate_follows_references_between_the_schemas_of_a_store(referencing_store):
assert (
schema.validate(
referencing_store, "person.schema.json", {"name": "Ana", "age": 3}
)
== []
)
got = schema.validate(referencing_store, "person.schema.json", {"name": "A"})
assert got == ["/name: 'A' is too short"]
def test_errors_are_reported_as_a_path_and_a_message_in_path_order(referencing_store):
got = schema.validate(
referencing_store, "person.schema.json", {"name": "A", "age": "x", "extra": 1}
)
assert got[0] == "/: Additional properties are not allowed ('extra' was unexpected)"
assert got[1:] == ["/age: 'x' is not of type 'integer'", "/name: 'A' is too short"]
def test_an_instance_of_the_wrong_shape_has_a_root_error(referencing_store):
assert schema.validate(referencing_store, "person.schema.json", [1]) == [
"/: [1] is not of type 'object'"
]
def test_array_indexes_appear_in_the_error_path(tmp_path):
folder = write_schemas(
tmp_path, list={"type": "array", "items": {"type": "integer"}}
)
got = schema.validate(schema.Store(folder), "list.schema.json", [1, "x", 3, "y"])
assert got == [
"/1: 'x' is not of type 'integer'",
"/3: 'y' is not of type 'integer'",
]
def test_registry_and_validators_are_built_once(referencing_store):
registry = referencing_store.registry()
assert referencing_store.registry() is registry
validator = referencing_store.validator("person.schema.json")
assert referencing_store.validator("person.schema.json") is validator
def test_the_registry_holds_every_schema_file_of_the_folder_by_its_id(
referencing_store,
):
registry = referencing_store.registry()
assert (
registry.get_or_retrieve(
"https://example.test/schemas/common.schema.json"
).value
is not None
)
assert len(registry) == 2
def test_an_annotation_of_the_fixture_layer_validates_and_a_broken_one_does_not():
annotation = read_jsonl(RECORDS / "layer" / "annotations.jsonl")[0]
store = schema_store()
assert schema.validate(store, "annotation.schema.json", annotation) == []
broken = {k: v for k, v in annotation.items() if k != "creator"}
errors = schema.validate(store, "annotation.schema.json", broken)
assert errors
assert any("creator" in e for e in errors)
assert all(e.startswith("/") for e in errors)
# --- the optional packages --------------------------------------------------------------------------------------------------
@pytest.mark.parametrize("missing", ["jsonschema", "referencing"])
def test_validation_without_the_packages_names_the_extra(
tmp_path, monkeypatch, missing
):
folder = write_schemas(tmp_path, a={"type": "string"})
monkeypatch.setitem(sys.modules, missing, None)
store = schema.Store(folder)
with pytest.raises(ImportError, match=r"paperspec\[schema\]"):
store.validator("a.schema.json")
with pytest.raises(ImportError, match=r"paperspec\[schema\]"):
store.registry()
def test_loading_and_linting_need_neither_package(tmp_path, monkeypatch):
folder = write_schemas(tmp_path, a={"type": "string"})
monkeypatch.setitem(sys.modules, "jsonschema", None)
monkeypatch.setitem(sys.modules, "referencing", None)
store = schema.Store(folder)
assert store.load("a.schema.json")["type"] == "string"
assert store.deref("a.schema.json", "#/type") == ("a.schema.json", "string")