Add the pytest suite and fix the bugs it found

The suite takes its data from the vectors and fixtures: 3635 tests, 97.9% branch
coverage, and paperspec check and paperspec stress give the same results as before.
Fixed: receipts.verify_signature raised KeyError for a record without a format;
traces.DOC_TAG matched a tag followed by a line feed; paperspec load and paperspec check
left SQLite connections open; the site builder copied the output folder's subfolders into
itself.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
This commit is contained in:
alexmandClaude committed 2026-10-19 04:32:44 -04:00
1 parent 30d310f64d
commit a8fc7237ae
76 files changed
+15564 -80

No files matched your search

+403
View File
@@ -0,0 +1,403 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Scienting
#
# Source Code: https://git.scient.ing/papers/paperspec
# See the LICENSE.md file for full license terms.
"""Traced citations: resolving a trace, the rules a trace breaks and disputes (paperspec.traces)."""
import copy
import json
import pytest
from conftest import ROOT, read_jsonl, vector_cases
from helpers_lib2 import DRAFT, HALE, RECORDS
from paperspec import anchors, corrections, package, traces
from paperspec.schema import Store
CASES = vector_cases("traces")
RESOLVES = [c for c in CASES if c["op"] == "resolve"]
PROBLEMS = [c for c in CASES if c["op"] == "problems"]
LAYER = read_jsonl(RECORDS / "traces" / "annotations.jsonl")
TRACES = [a for a in LAYER if a["body"]["kind"] == "trace"]
MODEL_TRACE = next(
a
for a in TRACES
if a["creator"]["type"] == "Software" and a["status"] == "confirmed"
)
PERSON_TRACE = next(a for a in TRACES if a["creator"]["type"] == "Person")
SAME_AS = ["doi:10.5555/example.hale.2026"]
@pytest.fixture(scope="module")
def schemas():
return Store(str(ROOT / "schemas"))
@pytest.fixture(scope="module")
def states():
"""The packages' states by path; tests copy a state before changing it."""
cache = {}
def get(path):
if path not in cache:
cache[path] = package.read_state(str(ROOT / path))
return cache[path]
return get
@pytest.fixture(scope="module")
def citing(states):
return states("fixtures/packages/" + DRAFT)
@pytest.fixture(scope="module")
def cited(states):
return states("fixtures/packages/" + HALE)
def evolve(state, steps, schemas):
"""The state after the corrections `steps`, applied through the gate."""
for step in steps:
state, _ = corrections.apply(state, step, store=schemas)
return state
def snapshot(state):
"""A comparable dump of the parts of a state that corrections change."""
return json.dumps(
[state.nodes, state.references, state.ids, state.corrections], sort_keys=True
)
def edited(annotation, **body):
"""A copy of `annotation` whose body has the given fields replaced; None removes a field."""
out = copy.deepcopy(annotation)
for key, value in body.items():
if value is None:
out["body"].pop(key, None)
else:
out["body"][key] = value
return out
# --- the vectors ---------------------------------------------------------------------------------------
@pytest.mark.parametrize("case", RESOLVES, ids=lambda c: c["id"])
def test_resolve_vectors(case, states, schemas):
start = states(case["citing"])
before = snapshot(start)
got = traces.resolve(
case["annotation"],
evolve(start, case["citing_corrections"], schemas),
evolve(states(case["cited"]), case["cited_corrections"], schemas),
case["same_as"],
)
assert got == case["expect"]
assert snapshot(start) == before
@pytest.mark.parametrize("case", RESOLVES, ids=lambda c: c["id"])
def test_a_resolution_is_shaped_as_the_result_says(case):
got = case["expect"]
assert got["result"] in traces.RESULTS
assert ("side" in got) == (got["result"] == "stale")
assert ("reason" in got) == (got["result"] != "current")
assert (got["open"] is None) == (got["cited"]["result"] == "stale")
assert got["cites"] == [] or got["open"]
@pytest.mark.parametrize("case", PROBLEMS, ids=lambda c: c["id"])
def test_problems_vectors(case, states):
got = traces.problems(
case["annotation"],
states(case["citing"]),
states(case["cited"]) if case["cited"] else None,
)
assert got == case["expect"]
def test_the_vectors_reach_every_outcome_and_both_rules():
assert {c["expect"]["result"] for c in RESOLVES} == set(traces.RESULTS)
broken = {x["rule"] for c in PROBLEMS for x in c["expect"]}
assert broken == {"layer.trace.source", "layer.trace.target"}
def test_one_vector_follows_a_citation_inside_the_passage():
followed = [c for c in RESOLVES if c["expect"]["cites"]]
assert [c["id"] for c in followed] == ["current-candidate-with-citation"]
item = followed[0]["expect"]["cites"][0]
assert set(item) == {"node", "run", "span", "text", "refs"}
assert item["refs"][0]["resolution"]["type"] == "doi"
# --- the fixture layer ------------------------------------------------------------------------------------
@pytest.mark.parametrize("annotation", TRACES, ids=lambda a: a["id"].rsplit("/", 1)[1])
def test_a_fixture_trace_breaks_no_rule_and_is_current(annotation, citing, cited):
assert traces.problems(annotation) == []
assert traces.problems(annotation, citing, cited) == []
got = traces.resolve(annotation, citing, cited, SAME_AS)
assert got["result"] == "current"
assert got["open"] is not None
@pytest.mark.parametrize("annotation", TRACES, ids=lambda a: a["id"].rsplit("/", 1)[1])
def test_a_fixture_traces_reference_resolves_to_its_source(annotation, citing):
source = annotation["body"]["source"].split("doc:", 1)[1]
record = next(
r for r in citing.references if r["node"] == annotation["body"]["reference"]
)
assert record["resolution"] == {"type": "document", "doc": source}
def test_the_layer_holds_three_traces_and_one_reply():
assert len(TRACES) == 3
assert len(LAYER) == 4
def test_a_rejected_candidate_resolves_like_any_other_trace(citing, cited):
rejected = next(a for a in TRACES if a["status"] == "rejected")
assert traces.resolve(rejected, citing, cited, SAME_AS)["result"] == "current"
# --- doc_tag and the helpers that follow replies --------------------------------------------------------------
def test_doc_tag_is_the_tag_a_source_writes():
assert (
traces.doc_tag("smithlab", HALE) == "tag:scient.ing,2026:doc:smithlab/" + HALE
)
assert traces.doc_tag("smithlab", HALE) == MODEL_TRACE["body"]["source"]
@pytest.mark.parametrize(
"tag",
[
"tag:scient.ing,2026:doc:smithlab/hale2026harmonic-damb",
"tag:scient.ing,2026:doc:a/hale2026harmonic-damb.v2",
"tag:scient.ing,2026:doc:a.b-c_d/smith1999x-j4qm.v3.final",
],
)
def test_a_document_tag_names_a_store_and_a_key(tag):
assert traces.DOC_TAG.match(tag)
@pytest.mark.parametrize(
"tag",
[
"hale2026harmonic-damb",
"tag:scient.ing,2026:doc:smithlab",
"tag:scient.ing,2026:doc:/hale2026harmonic-damb",
"tag:scient.ing,2026:doc:smithlab/Hale2026harmonic-damb",
"tag:scient.ing,2026:doc:smithlab/hale2026harmonic-damb/extra",
"tag:scient.ing,2026:annotation:smithlab/hale2026harmonic-damb",
],
)
def test_a_string_that_is_no_document_tag_does_not_match(tag):
assert traces.DOC_TAG.match(tag) is None
def test_a_document_tag_with_a_final_line_feed_does_not_match():
assert traces.DOC_TAG.match(MODEL_TRACE["body"]["source"] + "\n") is None
def test_only_the_model_trace_is_disputed():
ids = [a["id"] for a in TRACES if traces.disputed(a["id"], LAYER)]
assert ids == [MODEL_TRACE["id"]]
assert len(traces.disputes(MODEL_TRACE["id"], LAYER)) == 1
def test_a_dispute_is_an_assessing_reply_by_another_creator():
reply = traces.disputes(MODEL_TRACE["id"], LAYER)[0]
assert reply["motivation"] == "assessing"
assert reply["creator"] != MODEL_TRACE["creator"]
assert reply not in TRACES
@pytest.mark.parametrize(
"change",
[
{"status": "withdrawn"},
{"status": "rejected"},
{"motivation": "replying"},
{"motivation": None},
],
ids=["withdrawn", "rejected", "replying", "no-motivation"],
)
def test_a_reply_that_does_not_stand_or_does_not_assess_disputes_nothing(change):
reply = traces.disputes(MODEL_TRACE["id"], LAYER)[0]
assert not traces.disputed(MODEL_TRACE["id"], [{**reply, **change}])
@pytest.mark.parametrize("status", ["proposed", "confirmed"])
def test_a_reply_that_stands_disputes(status):
reply = traces.disputes(MODEL_TRACE["id"], LAYER)[0]
assert traces.disputed(MODEL_TRACE["id"], [{**reply, "status": status}])
def test_replies_are_the_annotations_that_target_an_annotation_without_a_selector():
reply = traces.disputes(MODEL_TRACE["id"], LAYER)[0]
assert traces.replies(MODEL_TRACE["id"], LAYER) == [reply]
assert traces.replies(reply["id"], LAYER) == []
assert traces.replies(PERSON_TRACE["id"], LAYER) == []
def test_replies_skip_annotations_with_odd_targets():
odd = [
{"target": "a string"},
{"target": None},
{},
{"target": {"source": "x", "selector": []}},
]
assert traces.replies("x", odd) == []
# --- calls with the wrong states or the wrong annotation -----------------------------------------------------------
def test_resolving_with_the_states_swapped_is_refused(citing, cited):
with pytest.raises(ValueError, match="citing_state is"):
traces.resolve(MODEL_TRACE, cited, citing)
def test_resolving_with_the_citing_state_twice_is_refused(citing):
with pytest.raises(ValueError, match="cited_state is"):
traces.resolve(MODEL_TRACE, citing, citing)
def test_problems_with_a_wrong_citing_state_is_refused(cited):
with pytest.raises(ValueError, match="citing_state is not the document"):
traces.problems(MODEL_TRACE, cited)
def test_problems_with_a_wrong_cited_state_is_refused(citing):
with pytest.raises(ValueError, match="cited_state is not the document"):
traces.problems(MODEL_TRACE, citing, citing)
def test_resolving_an_annotation_that_is_no_trace_is_refused(citing, cited):
layer = read_jsonl(RECORDS / "layer" / "annotations.jsonl")
with pytest.raises(ValueError, match="not a trace"):
traces.resolve(layer[0], citing, cited)
@pytest.mark.parametrize("which", ["target", "body"])
def test_resolving_a_trace_without_an_anchor_is_refused(citing, cited, which):
trace = copy.deepcopy(MODEL_TRACE)
holder = trace["target"] if which == "target" else trace["body"]
holder["selector"] = [
s for s in holder["selector"] if s["type"] != "AnchorSelector"
]
with pytest.raises(ValueError, match="needs an AnchorSelector"):
traces.resolve(trace, citing, cited)
def anchor_trace(annotation, state, node, on):
"""`annotation` with the anchor of its `on` (target or body) made over `node` of `state`."""
out = copy.deepcopy(annotation)
anchor = anchors.make_anchor(
state.doc(), state.index().values, node, state.rev, store="smithlab"
)
holder = out[on] if on == "target" else out["body"]
holder["selector"] = [{"anchor": anchor, "type": "AnchorSelector"}]
return out
def test_the_citations_of_a_passage_are_those_of_its_leaves_that_have_runs(
citing, cited
):
paragraph = "tgp1qey"
trace = anchor_trace(MODEL_TRACE, cited, paragraph, "body")
got = traces.resolve(trace, citing, cited, SAME_AS)
assert got["result"] == "current"
assert got["cited"]["node"] == paragraph
assert [c["node"] for c in got["cites"]] == ["kea5026", "hca6dc5"]
leaves = cited.doc().leaves(paragraph)
assert {c["node"] for c in got["cites"]} < set(leaves)
assert any(cited.doc().nodes[x]["kind"] == "equation" for x in leaves)
def test_a_trace_whose_target_is_a_paragraph_no_longer_cites_the_reference(
citing, cited
):
trace = anchor_trace(MODEL_TRACE, citing, "y4z5j53", "target")
got = traces.resolve(trace, citing, cited, SAME_AS)
assert got["citing"]["result"] == "current"
assert got["result"] == "unlinked"
assert rules_of(trace) == ["layer.trace.target"]
# --- the rules, one edit at a time ------------------------------------------------------------------------------------
def rules_of(annotation, *states):
"""The rules `annotation` breaks, checked against `states`."""
return [x["rule"] for x in traces.problems(annotation, *states)]
def test_a_source_that_is_not_a_tag_breaks_the_source_rule():
trace = edited(MODEL_TRACE, source="hale2026harmonic-damb")
assert rules_of(trace) == ["layer.trace.source"]
def test_a_missing_source_breaks_the_source_rule():
assert rules_of(edited(MODEL_TRACE, source=None)) == ["layer.trace.source"]
def test_an_anchor_in_another_store_than_the_source_breaks_the_source_rule():
trace = copy.deepcopy(MODEL_TRACE)
trace["body"]["selector"][0]["anchor"]["store"] = "elsewhere"
got = traces.problems(trace)
assert [x["rule"] for x in got] == ["layer.trace.source"]
assert "not in the source" in got[0]["message"]
def test_an_anchor_without_a_store_agrees_with_any_source_store():
trace = copy.deepcopy(MODEL_TRACE)
del trace["body"]["selector"][0]["anchor"]["store"]
assert rules_of(trace) == []
def test_a_quote_selector_that_differs_from_the_anchor_quote_breaks_the_source_rule():
trace = copy.deepcopy(MODEL_TRACE)
trace["body"]["selector"][1]["suffix"] = "something else"
assert rules_of(trace) == ["layer.trace.source"]
def test_a_quote_selector_beside_an_anchor_without_a_quote_breaks_the_source_rule():
trace = copy.deepcopy(MODEL_TRACE)
del trace["body"]["selector"][0]["anchor"]["quote"]
assert rules_of(trace) == ["layer.trace.source"]
def test_a_target_anchor_without_a_quote_breaks_the_target_rule():
trace = copy.deepcopy(MODEL_TRACE)
del trace["target"]["selector"][0]["anchor"]["quote"]
assert rules_of(trace) == ["layer.trace.target"]
def test_a_target_that_is_a_whole_document_breaks_the_target_rule():
trace = copy.deepcopy(MODEL_TRACE)
trace["target"] = {"source": trace["target"]["source"], "type": "SpecificResource"}
assert rules_of(trace) == ["layer.trace.target"]
def test_a_reference_that_is_not_in_the_citing_document_breaks_the_target_rule(citing):
trace = edited(MODEL_TRACE, reference="zzzzzzz")
assert rules_of(trace, citing) == ["layer.trace.target"]
def test_both_rules_are_reported_in_the_order_checked():
trace = edited(MODEL_TRACE, source="nowhere")
del trace["target"]["selector"][0]["anchor"]["quote"]
assert rules_of(trace) == ["layer.trace.source", "layer.trace.target"]
def test_the_trace_alone_is_checked_without_states():
assert traces.problems(MODEL_TRACE) == []
assert traces.problems(PERSON_TRACE) == []