Files
paperspec/tests/test_grade.py
T
alexmandClaude a8fc7237ae Add the pytest suite and fix the bugs it found
The suite takes its data from the vectors and fixtures: 3635 tests, 97.9% branch
coverage, and paperspec check and paperspec stress give the same results as before.
Fixed: receipts.verify_signature raised KeyError for a record without a format;
traces.DOC_TAG matched a tag followed by a line feed; paperspec load and paperspec check
left SQLite connections open; the site builder copied the output folder's subfolders into
itself.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
2026-10-19 04:32:44 -04:00

445 lines
16 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Scienting
#
# Source Code: https://git.scient.ing/papers/paperspec
# See the LICENSE.md file for full license terms.
"""Grading a submission against an exercise key (spec/19 §19.4)."""
from decimal import Decimal
from functools import cache
from typing import ClassVar
import pytest
from helpers_lib1 import HALE, package_state, vectors
from paperspec import anchors, grade as grading, hashing
from paperspec.tree import Refusal
@cache
def _sentences():
"""Plain text of three sentences of the example paper, and their IDs."""
doc = package_state(HALE).doc()
wanted = {
"terms": "Harmonic bond terms remain",
"water": "Water, H$_2$O",
"table": "Table 1 lists",
}
found = {}
for nid in doc.preorder():
if doc.nodes[nid]["kind"] != "sentence":
continue
text = hashing.plain(doc, nid)
for name, start in wanted.items():
if text.startswith(start):
found[name] = (nid, text)
return found
def anchor(name, span=None):
"""An anchor on a sentence of the example paper, current at its latest state."""
state = package_state(HALE)
doc = state.doc()
return anchors.make_anchor(
doc, hashing.compute(doc), _sentences()[name][0], state.rev, span, "smithlab"
)
def span_of(name, quote):
"""The span of `quote` within a sentence's plain text."""
text = _sentences()[name][1]
start = text.index(quote)
return [start, start + len(quote)]
def submission(answers, ident="s1"):
return {"id": ident, "exercise": "ex:1", "answers": answers}
def element(name, points="1", **feedback):
"""A key element anchored on a sentence."""
out = {"anchors": [anchor(name)], "points": points}
if feedback:
out["feedback"] = feedback
return out
def run(body, answers):
return grading.grade(body, submission(answers), package_state(HALE))
def scores(grade):
return [(i["element"], i["points"]) for i in grade["items"]]
@pytest.mark.parametrize("case", vectors("grading"))
def test_grading_vector(case):
state = package_state(case["package"])
assert grading.grade(case["exercise"], case["submission"], state) == case["expect"]
class TestDecimalStrings:
@pytest.mark.parametrize(
("value", "text"),
[
("1", "1"),
("0.50", "0.50"),
("1E+1", "10"),
(Decimal("2.50"), "2.50"),
("0.0", "0.0"),
("1e-3", "0.001"),
],
)
def test_dec_writes_the_decimal_as_a_plain_string(self, value, text):
assert grading.dec(value) == text
def test_a_missed_element_scores_zero_with_the_scale_of_its_points(self):
body = {"exercise": "anchor", "key": {"elements": [element("terms", "0.50")]}}
grade = run(body, {"anchors": [anchor("water")]})
assert grade["items"][0]["points"] == "0.00"
assert grade["items"][0]["max"] == "0.50"
assert grade["score"] == "0.00"
assert grade["max"] == "0.50"
def test_points_are_summed_exactly(self):
body = {
"exercise": "anchor",
"key": {
"elements": [
element("terms", "0.1"),
element("water", "0.2"),
element("table", "0.4"),
]
},
}
grade = run(body, {"anchors": [anchor("terms"), anchor("water")]})
assert grade["score"] == "0.3"
assert grade["max"] == "0.7"
assert scores(grade) == [
("element:0", "0.1"),
("element:1", "0.2"),
("element:2", "0.0"),
]
class TestAnchorExercise:
BODY: ClassVar[dict] = {
"exercise": "anchor",
"key": {
"elements": [
element("terms", "2", hit="Right.", miss="Wrong."),
element("water", "1"),
]
},
}
def test_the_grade_names_the_submission_and_its_exercise(self):
grade = run(self.BODY, {"anchors": []})
assert grade["format"] == "scienting-grade/1"
assert grade["submission"] == "s1"
assert grade["exercise"] == "ex:1"
def test_each_element_is_scored_by_whether_any_answer_hits_it(self):
grade = run(self.BODY, {"anchors": [anchor("water"), anchor("terms")]})
assert grade["score"] == "3"
assert grade["max"] == "3"
assert [i["feedback"] for i in grade["items"]] == ["Right.", ""]
def test_a_miss_gives_the_miss_feedback(self):
grade = run(self.BODY, {"anchors": [anchor("table")]})
assert grade["score"] == "0"
assert [i["feedback"] for i in grade["items"]] == ["Wrong.", ""]
def test_no_answer_scores_nothing(self):
assert run(self.BODY, {})["score"] == "0"
def test_one_answer_may_hit_several_elements(self):
body = {
"exercise": "anchor",
"key": {"elements": [element("terms"), element("terms", "3")]},
}
grade = run(body, {"anchors": [anchor("terms")]})
assert grade["score"] == "4"
def test_an_element_may_list_several_acceptable_anchors(self):
two = {"anchors": [anchor("terms"), anchor("water")], "points": "1"}
body = {"exercise": "anchor", "key": {"elements": [two]}}
assert run(body, {"anchors": [anchor("water")]})["score"] == "1"
assert run(body, {"anchors": [anchor("table")]})["score"] == "0"
def test_a_stale_answer_hits_nothing(self):
stale = dict(anchor("terms"), node="zzzzzzz")
assert run(self.BODY, {"anchors": [stale]})["score"] == "0"
def test_an_answer_whose_hash_moved_but_whose_quote_is_found_still_hits(self):
moved = dict(anchor("terms"), node_hash="0" * 64)
assert run(self.BODY, {"anchors": [moved]})["score"] == "2"
class TestMatchRules:
TEXT = "Harmonic bond terms remain the most common description of covalent"
def key_span(self):
return span_of("terms", "the most common description")
def body(self, rule=None):
key = {"anchors": [anchor("terms", self.key_span())], "points": "1"}
out = {"exercise": "anchor", "key": {"elements": [key]}}
if rule is not None:
out["rule"] = rule
return out
@pytest.mark.parametrize(
("quote", "points"),
[
("the most common description", "1"),
("most common description of covalent", "1"),
("the most common", "1"),
("the most", "0"),
("terms remain the", "0"),
("covalent stretching", "0"),
],
)
def test_overlap_needs_half_the_key_span_by_default(self, quote, points):
answer = anchor("terms", span_of("terms", quote))
body = self.body({"match": "overlap"})
assert run(body, {"anchors": [answer]})["score"] == points
def test_the_least_overlap_can_be_set(self):
answer = anchor("terms", span_of("terms", "the most"))
lenient = self.body({"match": "overlap", "min_overlap": "0.25"})
assert run(lenient, {"anchors": [answer]})["score"] == "1"
tight = self.body({"match": "overlap", "min_overlap": "1"})
assert run(tight, {"anchors": [answer]})["score"] == "0"
def test_an_answer_without_a_span_covers_the_whole_node(self):
body = self.body({"match": "overlap"})
assert run(body, {"anchors": [anchor("terms")]})["score"] == "1"
def test_an_overlap_rule_on_a_key_without_a_span_needs_only_the_node(self):
body = {
"exercise": "anchor",
"rule": {"match": "overlap"},
"key": {"elements": [element("terms")]},
}
answer = anchor("terms", span_of("terms", "the most"))
assert run(body, {"anchors": [answer]})["score"] == "1"
def test_same_node_ignores_spans(self):
answer = anchor("terms", span_of("terms", "covalent stretching"))
for rule in (None, {"match": "same-node"}, {}):
assert run(self.body(rule), {"anchors": [answer]})["score"] == "1"
def test_an_answer_on_another_node_never_matches(self):
answer = anchor("water", [0, 5])
assert (
run(self.body({"match": "overlap"}), {"anchors": [answer]})["score"] == "0"
)
def test_an_empty_key_span_cannot_be_overlapped(self):
key = {"anchors": [anchor("terms", [5, 5])], "points": "1"}
body = {
"exercise": "anchor",
"rule": {"match": "overlap"},
"key": {"elements": [key]},
}
answer = anchor("terms", [0, 20])
assert run(body, {"anchors": [answer]})["score"] == "0"
class TestCheckExercise:
BODY: ClassVar[dict] = {
"exercise": "check",
"key": {
"verdict": {
"value": "supports",
"points": "1",
"feedback": {"hit": "Yes.", "miss": "No."},
},
"elements": [element("terms")],
},
}
def test_the_verdict_comes_first_and_is_scored_on_its_own(self):
grade = run(self.BODY, {"verdict": "supports", "anchors": [anchor("terms")]})
assert scores(grade) == [("verdict", "1"), ("element:0", "1")]
assert grade["items"][0]["feedback"] == "Yes."
assert grade["score"] == "2"
def test_a_wrong_verdict_loses_only_its_points(self):
grade = run(self.BODY, {"verdict": "refutes", "anchors": [anchor("terms")]})
assert scores(grade) == [("verdict", "0"), ("element:0", "1")]
assert grade["items"][0]["feedback"] == "No."
def test_no_verdict_is_wrong(self):
assert run(self.BODY, {"anchors": [anchor("terms")]})["score"] == "1"
def test_a_verdict_without_feedback_has_empty_feedback(self):
body = {
"exercise": "check",
"key": {"verdict": {"value": "supports", "points": "1"}, "elements": []},
}
assert run(body, {"verdict": "supports"})["items"][0]["feedback"] == ""
class TestTraceExercise:
BODY: ClassVar[dict] = {
"exercise": "trace",
"key": {
"chain": [
{
"anchors": [anchor("terms")],
"points": "1",
"feedback": {"hit": "First."},
},
{"anchors": [anchor("water")], "points": "2"},
]
},
}
def test_each_link_is_matched_by_the_answers_at_its_position(self):
chain = [[anchor("terms")], [anchor("water")]]
grade = run(self.BODY, {"chain": chain})
assert scores(grade) == [("link:0", "1"), ("link:1", "2")]
assert grade["items"][0]["feedback"] == "First."
def test_an_answer_at_the_wrong_position_does_not_count(self):
grade = run(self.BODY, {"chain": [[anchor("water")], [anchor("terms")]]})
assert grade["score"] == "0"
def test_a_chain_that_stops_early_misses_the_rest(self):
grade = run(self.BODY, {"chain": [[anchor("terms")]]})
assert scores(grade) == [("link:0", "1"), ("link:1", "0")]
def test_a_link_may_be_answered_by_several_anchors(self):
grade = run(self.BODY, {"chain": [[anchor("table"), anchor("terms")], []]})
assert scores(grade) == [("link:0", "1"), ("link:1", "0")]
def test_a_missing_chain_scores_nothing(self):
assert run(self.BODY, {})["score"] == "0"
class TestReproduceExercise:
def body(self, value="3.14", tolerance="0.01", points="2"):
return {
"exercise": "reproduce",
"key": {
"value": value,
"tolerance": tolerance,
"points": points,
"feedback": {"hit": "Close.", "miss": "Off."},
},
}
@pytest.mark.parametrize(
("answer", "points"),
[
("3.14", "2"),
("3.15", "2"),
("3.13", "2"),
("3.1501", "0"),
("3.1299", "0"),
("-3.14", "0"),
("3.140000", "2"),
],
)
def test_the_value_must_be_within_the_tolerance_inclusive(self, answer, points):
grade = run(self.body(), {"value": answer})
assert scores(grade) == [("value", points)]
def test_values_are_compared_exactly_in_decimal(self):
assert run(self.body("0.3", "0"), {"value": "0.30"})["score"] == "2"
assert (
run(self.body("0.3", "0"), {"value": "0.30000000000000004"})["score"] == "0"
)
def test_feedback_follows_the_outcome(self):
assert run(self.body(), {"value": "3.14"})["items"][0]["feedback"] == "Close."
assert run(self.body(), {"value": "9"})["items"][0]["feedback"] == "Off."
class TestVerifyExercise:
BODY: ClassVar[dict] = {
"exercise": "verify",
"key": {
"errors": [
{
"anchors": [anchor("terms")],
"verdict": "wrong-number",
"points": "1",
"feedback": {"hit": "Found.", "miss": "Missed."},
},
{"anchors": [anchor("water")], "verdict": "wrong-unit", "points": "1"},
]
},
}
def flag(self, name, verdict):
return {"anchor": anchor(name), "verdict": verdict}
def test_an_error_is_found_by_a_flag_on_its_node_with_its_verdict(self):
flags = [self.flag("terms", "wrong-number"), self.flag("water", "wrong-unit")]
grade = run(self.BODY, {"flags": flags})
assert scores(grade) == [("error:0", "1"), ("error:1", "1")]
assert grade["items"][0]["feedback"] == "Found."
def test_the_wrong_verdict_on_the_right_node_misses(self):
grade = run(self.BODY, {"flags": [self.flag("terms", "wrong-unit")]})
assert scores(grade) == [("error:0", "0"), ("error:1", "0")]
assert grade["items"][0]["feedback"] == "Missed."
def test_any_one_of_several_flags_on_the_node_may_carry_the_verdict(self):
flags = [self.flag("terms", "other"), self.flag("terms", "wrong-number")]
assert run(self.BODY, {"flags": flags})["score"] == "1"
def test_a_flag_on_another_node_finds_nothing(self):
assert (
run(self.BODY, {"flags": [self.flag("table", "wrong-number")]})["score"]
== "0"
)
def test_no_flags_find_nothing(self):
assert run(self.BODY, {})["score"] == "0"
class TestRefusals:
def test_a_stale_anchor_in_the_key_refuses_the_grade(self):
stale = dict(anchor("terms"), node="zzzzzzz")
body = {
"exercise": "anchor",
"key": {"elements": [{"anchors": [stale], "points": "1"}]},
}
with pytest.raises(Refusal, match=r"grade\.key-stale") as excinfo:
run(body, {"anchors": []})
assert excinfo.value.rule == "grade.key-stale"
def test_every_part_of_the_key_is_checked_for_stale_anchors(self):
stale = dict(anchor("terms"), node="zzzzzzz")
for part, shape in (
("elements", {"verdict": {"value": "x", "points": "1"}}),
("chain", {}),
("errors", {}),
):
key = dict(shape, **{part: [{"anchors": [stale], "points": "1"}]})
with pytest.raises(Refusal, match="key-stale"):
run({"exercise": "check", "key": key}, {})
def test_an_unknown_kind_is_refused(self):
with pytest.raises(
Refusal, match=r"grade\.kind: unknown exercise kind essay"
) as excinfo:
run({"exercise": "essay", "key": {}}, {})
assert excinfo.value.rule == "grade.kind"
def test_key_anchors_lists_elements_then_chain_then_errors():
a, b, c = ({"node": x} for x in "abc")
key = {
"elements": [{"anchors": [a]}],
"chain": [{"anchors": [b]}],
"errors": [{"anchors": [c, a]}],
}
assert grading.key_anchors({"key": key}) == [a, b, c, a]
assert grading.key_anchors({"key": {}}) == []