Add the pytest suite and fix the bugs it found
The suite takes its data from the vectors and fixtures: 3635 tests, 97.9% branch coverage, and paperspec check and paperspec stress give the same results as before. Fixed: receipts.verify_signature raised KeyError for a record without a format; traces.DOC_TAG matched a tag followed by a line feed; paperspec load and paperspec check left SQLite connections open; the site builder copied the output folder's subfolders into itself. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
This commit is contained in:
76 files changed
+15564
-80
No files matched your search
@@ -0,0 +1,403 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Traced citations: resolving a trace, the rules a trace breaks and disputes (paperspec.traces)."""
|
||||
|
||||
import copy
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from conftest import ROOT, read_jsonl, vector_cases
|
||||
from helpers_lib2 import DRAFT, HALE, RECORDS
|
||||
from paperspec import anchors, corrections, package, traces
|
||||
from paperspec.schema import Store
|
||||
|
||||
CASES = vector_cases("traces")
|
||||
RESOLVES = [c for c in CASES if c["op"] == "resolve"]
|
||||
PROBLEMS = [c for c in CASES if c["op"] == "problems"]
|
||||
LAYER = read_jsonl(RECORDS / "traces" / "annotations.jsonl")
|
||||
TRACES = [a for a in LAYER if a["body"]["kind"] == "trace"]
|
||||
MODEL_TRACE = next(
|
||||
a
|
||||
for a in TRACES
|
||||
if a["creator"]["type"] == "Software" and a["status"] == "confirmed"
|
||||
)
|
||||
PERSON_TRACE = next(a for a in TRACES if a["creator"]["type"] == "Person")
|
||||
SAME_AS = ["doi:10.5555/example.hale.2026"]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def schemas():
|
||||
return Store(str(ROOT / "schemas"))
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def states():
|
||||
"""The packages' states by path; tests copy a state before changing it."""
|
||||
cache = {}
|
||||
|
||||
def get(path):
|
||||
if path not in cache:
|
||||
cache[path] = package.read_state(str(ROOT / path))
|
||||
return cache[path]
|
||||
|
||||
return get
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def citing(states):
|
||||
return states("fixtures/packages/" + DRAFT)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def cited(states):
|
||||
return states("fixtures/packages/" + HALE)
|
||||
|
||||
|
||||
def evolve(state, steps, schemas):
|
||||
"""The state after the corrections `steps`, applied through the gate."""
|
||||
for step in steps:
|
||||
state, _ = corrections.apply(state, step, store=schemas)
|
||||
return state
|
||||
|
||||
|
||||
def snapshot(state):
|
||||
"""A comparable dump of the parts of a state that corrections change."""
|
||||
return json.dumps(
|
||||
[state.nodes, state.references, state.ids, state.corrections], sort_keys=True
|
||||
)
|
||||
|
||||
|
||||
def edited(annotation, **body):
|
||||
"""A copy of `annotation` whose body has the given fields replaced; None removes a field."""
|
||||
out = copy.deepcopy(annotation)
|
||||
for key, value in body.items():
|
||||
if value is None:
|
||||
out["body"].pop(key, None)
|
||||
else:
|
||||
out["body"][key] = value
|
||||
return out
|
||||
|
||||
|
||||
# --- the vectors ---------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", RESOLVES, ids=lambda c: c["id"])
|
||||
def test_resolve_vectors(case, states, schemas):
|
||||
start = states(case["citing"])
|
||||
before = snapshot(start)
|
||||
got = traces.resolve(
|
||||
case["annotation"],
|
||||
evolve(start, case["citing_corrections"], schemas),
|
||||
evolve(states(case["cited"]), case["cited_corrections"], schemas),
|
||||
case["same_as"],
|
||||
)
|
||||
assert got == case["expect"]
|
||||
assert snapshot(start) == before
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", RESOLVES, ids=lambda c: c["id"])
|
||||
def test_a_resolution_is_shaped_as_the_result_says(case):
|
||||
got = case["expect"]
|
||||
assert got["result"] in traces.RESULTS
|
||||
assert ("side" in got) == (got["result"] == "stale")
|
||||
assert ("reason" in got) == (got["result"] != "current")
|
||||
assert (got["open"] is None) == (got["cited"]["result"] == "stale")
|
||||
assert got["cites"] == [] or got["open"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", PROBLEMS, ids=lambda c: c["id"])
|
||||
def test_problems_vectors(case, states):
|
||||
got = traces.problems(
|
||||
case["annotation"],
|
||||
states(case["citing"]),
|
||||
states(case["cited"]) if case["cited"] else None,
|
||||
)
|
||||
assert got == case["expect"]
|
||||
|
||||
|
||||
def test_the_vectors_reach_every_outcome_and_both_rules():
|
||||
assert {c["expect"]["result"] for c in RESOLVES} == set(traces.RESULTS)
|
||||
broken = {x["rule"] for c in PROBLEMS for x in c["expect"]}
|
||||
assert broken == {"layer.trace.source", "layer.trace.target"}
|
||||
|
||||
|
||||
def test_one_vector_follows_a_citation_inside_the_passage():
|
||||
followed = [c for c in RESOLVES if c["expect"]["cites"]]
|
||||
assert [c["id"] for c in followed] == ["current-candidate-with-citation"]
|
||||
item = followed[0]["expect"]["cites"][0]
|
||||
assert set(item) == {"node", "run", "span", "text", "refs"}
|
||||
assert item["refs"][0]["resolution"]["type"] == "doi"
|
||||
|
||||
|
||||
# --- the fixture layer ------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("annotation", TRACES, ids=lambda a: a["id"].rsplit("/", 1)[1])
|
||||
def test_a_fixture_trace_breaks_no_rule_and_is_current(annotation, citing, cited):
|
||||
assert traces.problems(annotation) == []
|
||||
assert traces.problems(annotation, citing, cited) == []
|
||||
got = traces.resolve(annotation, citing, cited, SAME_AS)
|
||||
assert got["result"] == "current"
|
||||
assert got["open"] is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("annotation", TRACES, ids=lambda a: a["id"].rsplit("/", 1)[1])
|
||||
def test_a_fixture_traces_reference_resolves_to_its_source(annotation, citing):
|
||||
source = annotation["body"]["source"].split("doc:", 1)[1]
|
||||
record = next(
|
||||
r for r in citing.references if r["node"] == annotation["body"]["reference"]
|
||||
)
|
||||
assert record["resolution"] == {"type": "document", "doc": source}
|
||||
|
||||
|
||||
def test_the_layer_holds_three_traces_and_one_reply():
|
||||
assert len(TRACES) == 3
|
||||
assert len(LAYER) == 4
|
||||
|
||||
|
||||
def test_a_rejected_candidate_resolves_like_any_other_trace(citing, cited):
|
||||
rejected = next(a for a in TRACES if a["status"] == "rejected")
|
||||
assert traces.resolve(rejected, citing, cited, SAME_AS)["result"] == "current"
|
||||
|
||||
|
||||
# --- doc_tag and the helpers that follow replies --------------------------------------------------------------
|
||||
|
||||
|
||||
def test_doc_tag_is_the_tag_a_source_writes():
|
||||
assert (
|
||||
traces.doc_tag("smithlab", HALE) == "tag:scient.ing,2026:doc:smithlab/" + HALE
|
||||
)
|
||||
assert traces.doc_tag("smithlab", HALE) == MODEL_TRACE["body"]["source"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tag",
|
||||
[
|
||||
"tag:scient.ing,2026:doc:smithlab/hale2026harmonic-damb",
|
||||
"tag:scient.ing,2026:doc:a/hale2026harmonic-damb.v2",
|
||||
"tag:scient.ing,2026:doc:a.b-c_d/smith1999x-j4qm.v3.final",
|
||||
],
|
||||
)
|
||||
def test_a_document_tag_names_a_store_and_a_key(tag):
|
||||
assert traces.DOC_TAG.match(tag)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tag",
|
||||
[
|
||||
"hale2026harmonic-damb",
|
||||
"tag:scient.ing,2026:doc:smithlab",
|
||||
"tag:scient.ing,2026:doc:/hale2026harmonic-damb",
|
||||
"tag:scient.ing,2026:doc:smithlab/Hale2026harmonic-damb",
|
||||
"tag:scient.ing,2026:doc:smithlab/hale2026harmonic-damb/extra",
|
||||
"tag:scient.ing,2026:annotation:smithlab/hale2026harmonic-damb",
|
||||
],
|
||||
)
|
||||
def test_a_string_that_is_no_document_tag_does_not_match(tag):
|
||||
assert traces.DOC_TAG.match(tag) is None
|
||||
|
||||
|
||||
def test_a_document_tag_with_a_final_line_feed_does_not_match():
|
||||
assert traces.DOC_TAG.match(MODEL_TRACE["body"]["source"] + "\n") is None
|
||||
|
||||
|
||||
def test_only_the_model_trace_is_disputed():
|
||||
ids = [a["id"] for a in TRACES if traces.disputed(a["id"], LAYER)]
|
||||
assert ids == [MODEL_TRACE["id"]]
|
||||
assert len(traces.disputes(MODEL_TRACE["id"], LAYER)) == 1
|
||||
|
||||
|
||||
def test_a_dispute_is_an_assessing_reply_by_another_creator():
|
||||
reply = traces.disputes(MODEL_TRACE["id"], LAYER)[0]
|
||||
assert reply["motivation"] == "assessing"
|
||||
assert reply["creator"] != MODEL_TRACE["creator"]
|
||||
assert reply not in TRACES
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"change",
|
||||
[
|
||||
{"status": "withdrawn"},
|
||||
{"status": "rejected"},
|
||||
{"motivation": "replying"},
|
||||
{"motivation": None},
|
||||
],
|
||||
ids=["withdrawn", "rejected", "replying", "no-motivation"],
|
||||
)
|
||||
def test_a_reply_that_does_not_stand_or_does_not_assess_disputes_nothing(change):
|
||||
reply = traces.disputes(MODEL_TRACE["id"], LAYER)[0]
|
||||
assert not traces.disputed(MODEL_TRACE["id"], [{**reply, **change}])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", ["proposed", "confirmed"])
|
||||
def test_a_reply_that_stands_disputes(status):
|
||||
reply = traces.disputes(MODEL_TRACE["id"], LAYER)[0]
|
||||
assert traces.disputed(MODEL_TRACE["id"], [{**reply, "status": status}])
|
||||
|
||||
|
||||
def test_replies_are_the_annotations_that_target_an_annotation_without_a_selector():
|
||||
reply = traces.disputes(MODEL_TRACE["id"], LAYER)[0]
|
||||
assert traces.replies(MODEL_TRACE["id"], LAYER) == [reply]
|
||||
assert traces.replies(reply["id"], LAYER) == []
|
||||
assert traces.replies(PERSON_TRACE["id"], LAYER) == []
|
||||
|
||||
|
||||
def test_replies_skip_annotations_with_odd_targets():
|
||||
odd = [
|
||||
{"target": "a string"},
|
||||
{"target": None},
|
||||
{},
|
||||
{"target": {"source": "x", "selector": []}},
|
||||
]
|
||||
assert traces.replies("x", odd) == []
|
||||
|
||||
|
||||
# --- calls with the wrong states or the wrong annotation -----------------------------------------------------------
|
||||
|
||||
|
||||
def test_resolving_with_the_states_swapped_is_refused(citing, cited):
|
||||
with pytest.raises(ValueError, match="citing_state is"):
|
||||
traces.resolve(MODEL_TRACE, cited, citing)
|
||||
|
||||
|
||||
def test_resolving_with_the_citing_state_twice_is_refused(citing):
|
||||
with pytest.raises(ValueError, match="cited_state is"):
|
||||
traces.resolve(MODEL_TRACE, citing, citing)
|
||||
|
||||
|
||||
def test_problems_with_a_wrong_citing_state_is_refused(cited):
|
||||
with pytest.raises(ValueError, match="citing_state is not the document"):
|
||||
traces.problems(MODEL_TRACE, cited)
|
||||
|
||||
|
||||
def test_problems_with_a_wrong_cited_state_is_refused(citing):
|
||||
with pytest.raises(ValueError, match="cited_state is not the document"):
|
||||
traces.problems(MODEL_TRACE, citing, citing)
|
||||
|
||||
|
||||
def test_resolving_an_annotation_that_is_no_trace_is_refused(citing, cited):
|
||||
layer = read_jsonl(RECORDS / "layer" / "annotations.jsonl")
|
||||
with pytest.raises(ValueError, match="not a trace"):
|
||||
traces.resolve(layer[0], citing, cited)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("which", ["target", "body"])
|
||||
def test_resolving_a_trace_without_an_anchor_is_refused(citing, cited, which):
|
||||
trace = copy.deepcopy(MODEL_TRACE)
|
||||
holder = trace["target"] if which == "target" else trace["body"]
|
||||
holder["selector"] = [
|
||||
s for s in holder["selector"] if s["type"] != "AnchorSelector"
|
||||
]
|
||||
with pytest.raises(ValueError, match="needs an AnchorSelector"):
|
||||
traces.resolve(trace, citing, cited)
|
||||
|
||||
|
||||
def anchor_trace(annotation, state, node, on):
|
||||
"""`annotation` with the anchor of its `on` (target or body) made over `node` of `state`."""
|
||||
out = copy.deepcopy(annotation)
|
||||
anchor = anchors.make_anchor(
|
||||
state.doc(), state.index().values, node, state.rev, store="smithlab"
|
||||
)
|
||||
holder = out[on] if on == "target" else out["body"]
|
||||
holder["selector"] = [{"anchor": anchor, "type": "AnchorSelector"}]
|
||||
return out
|
||||
|
||||
|
||||
def test_the_citations_of_a_passage_are_those_of_its_leaves_that_have_runs(
|
||||
citing, cited
|
||||
):
|
||||
paragraph = "tgp1qey"
|
||||
trace = anchor_trace(MODEL_TRACE, cited, paragraph, "body")
|
||||
got = traces.resolve(trace, citing, cited, SAME_AS)
|
||||
assert got["result"] == "current"
|
||||
assert got["cited"]["node"] == paragraph
|
||||
assert [c["node"] for c in got["cites"]] == ["kea5026", "hca6dc5"]
|
||||
leaves = cited.doc().leaves(paragraph)
|
||||
assert {c["node"] for c in got["cites"]} < set(leaves)
|
||||
assert any(cited.doc().nodes[x]["kind"] == "equation" for x in leaves)
|
||||
|
||||
|
||||
def test_a_trace_whose_target_is_a_paragraph_no_longer_cites_the_reference(
|
||||
citing, cited
|
||||
):
|
||||
trace = anchor_trace(MODEL_TRACE, citing, "y4z5j53", "target")
|
||||
got = traces.resolve(trace, citing, cited, SAME_AS)
|
||||
assert got["citing"]["result"] == "current"
|
||||
assert got["result"] == "unlinked"
|
||||
assert rules_of(trace) == ["layer.trace.target"]
|
||||
|
||||
|
||||
# --- the rules, one edit at a time ------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def rules_of(annotation, *states):
|
||||
"""The rules `annotation` breaks, checked against `states`."""
|
||||
return [x["rule"] for x in traces.problems(annotation, *states)]
|
||||
|
||||
|
||||
def test_a_source_that_is_not_a_tag_breaks_the_source_rule():
|
||||
trace = edited(MODEL_TRACE, source="hale2026harmonic-damb")
|
||||
assert rules_of(trace) == ["layer.trace.source"]
|
||||
|
||||
|
||||
def test_a_missing_source_breaks_the_source_rule():
|
||||
assert rules_of(edited(MODEL_TRACE, source=None)) == ["layer.trace.source"]
|
||||
|
||||
|
||||
def test_an_anchor_in_another_store_than_the_source_breaks_the_source_rule():
|
||||
trace = copy.deepcopy(MODEL_TRACE)
|
||||
trace["body"]["selector"][0]["anchor"]["store"] = "elsewhere"
|
||||
got = traces.problems(trace)
|
||||
assert [x["rule"] for x in got] == ["layer.trace.source"]
|
||||
assert "not in the source" in got[0]["message"]
|
||||
|
||||
|
||||
def test_an_anchor_without_a_store_agrees_with_any_source_store():
|
||||
trace = copy.deepcopy(MODEL_TRACE)
|
||||
del trace["body"]["selector"][0]["anchor"]["store"]
|
||||
assert rules_of(trace) == []
|
||||
|
||||
|
||||
def test_a_quote_selector_that_differs_from_the_anchor_quote_breaks_the_source_rule():
|
||||
trace = copy.deepcopy(MODEL_TRACE)
|
||||
trace["body"]["selector"][1]["suffix"] = "something else"
|
||||
assert rules_of(trace) == ["layer.trace.source"]
|
||||
|
||||
|
||||
def test_a_quote_selector_beside_an_anchor_without_a_quote_breaks_the_source_rule():
|
||||
trace = copy.deepcopy(MODEL_TRACE)
|
||||
del trace["body"]["selector"][0]["anchor"]["quote"]
|
||||
assert rules_of(trace) == ["layer.trace.source"]
|
||||
|
||||
|
||||
def test_a_target_anchor_without_a_quote_breaks_the_target_rule():
|
||||
trace = copy.deepcopy(MODEL_TRACE)
|
||||
del trace["target"]["selector"][0]["anchor"]["quote"]
|
||||
assert rules_of(trace) == ["layer.trace.target"]
|
||||
|
||||
|
||||
def test_a_target_that_is_a_whole_document_breaks_the_target_rule():
|
||||
trace = copy.deepcopy(MODEL_TRACE)
|
||||
trace["target"] = {"source": trace["target"]["source"], "type": "SpecificResource"}
|
||||
assert rules_of(trace) == ["layer.trace.target"]
|
||||
|
||||
|
||||
def test_a_reference_that_is_not_in_the_citing_document_breaks_the_target_rule(citing):
|
||||
trace = edited(MODEL_TRACE, reference="zzzzzzz")
|
||||
assert rules_of(trace, citing) == ["layer.trace.target"]
|
||||
|
||||
|
||||
def test_both_rules_are_reported_in_the_order_checked():
|
||||
trace = edited(MODEL_TRACE, source="nowhere")
|
||||
del trace["target"]["selector"][0]["anchor"]["quote"]
|
||||
assert rules_of(trace) == ["layer.trace.source", "layer.trace.target"]
|
||||
|
||||
|
||||
def test_the_trace_alone_is_checked_without_states():
|
||||
assert traces.problems(MODEL_TRACE) == []
|
||||
assert traces.problems(PERSON_TRACE) == []
|
||||
Reference in new issue
Block a user