The suite takes its data from the vectors and fixtures: 3635 tests, 97.9% branch coverage, and paperspec check and paperspec stress give the same results as before. Fixed: receipts.verify_signature raised KeyError for a record without a format; traces.DOC_TAG matched a tag followed by a line feed; paperspec load and paperspec check left SQLite connections open; the site builder copied the output folder's subfolders into itself. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
445 lines
16 KiB
Python
445 lines
16 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# Copyright 2026 Scienting
|
|
#
|
|
# Source Code: https://git.scient.ing/papers/paperspec
|
|
# See the LICENSE.md file for full license terms.
|
|
|
|
"""Grading a submission against an exercise key (spec/19 §19.4)."""
|
|
|
|
from decimal import Decimal
|
|
from functools import cache
|
|
from typing import ClassVar
|
|
|
|
import pytest
|
|
|
|
from helpers_lib1 import HALE, package_state, vectors
|
|
from paperspec import anchors, grade as grading, hashing
|
|
from paperspec.tree import Refusal
|
|
|
|
|
|
@cache
|
|
def _sentences():
|
|
"""Plain text of three sentences of the example paper, and their IDs."""
|
|
doc = package_state(HALE).doc()
|
|
wanted = {
|
|
"terms": "Harmonic bond terms remain",
|
|
"water": "Water, H$_2$O",
|
|
"table": "Table 1 lists",
|
|
}
|
|
found = {}
|
|
for nid in doc.preorder():
|
|
if doc.nodes[nid]["kind"] != "sentence":
|
|
continue
|
|
text = hashing.plain(doc, nid)
|
|
for name, start in wanted.items():
|
|
if text.startswith(start):
|
|
found[name] = (nid, text)
|
|
return found
|
|
|
|
|
|
def anchor(name, span=None):
|
|
"""An anchor on a sentence of the example paper, current at its latest state."""
|
|
state = package_state(HALE)
|
|
doc = state.doc()
|
|
return anchors.make_anchor(
|
|
doc, hashing.compute(doc), _sentences()[name][0], state.rev, span, "smithlab"
|
|
)
|
|
|
|
|
|
def span_of(name, quote):
|
|
"""The span of `quote` within a sentence's plain text."""
|
|
text = _sentences()[name][1]
|
|
start = text.index(quote)
|
|
return [start, start + len(quote)]
|
|
|
|
|
|
def submission(answers, ident="s1"):
|
|
return {"id": ident, "exercise": "ex:1", "answers": answers}
|
|
|
|
|
|
def element(name, points="1", **feedback):
|
|
"""A key element anchored on a sentence."""
|
|
out = {"anchors": [anchor(name)], "points": points}
|
|
if feedback:
|
|
out["feedback"] = feedback
|
|
return out
|
|
|
|
|
|
def run(body, answers):
|
|
return grading.grade(body, submission(answers), package_state(HALE))
|
|
|
|
|
|
def scores(grade):
|
|
return [(i["element"], i["points"]) for i in grade["items"]]
|
|
|
|
|
|
@pytest.mark.parametrize("case", vectors("grading"))
|
|
def test_grading_vector(case):
|
|
state = package_state(case["package"])
|
|
assert grading.grade(case["exercise"], case["submission"], state) == case["expect"]
|
|
|
|
|
|
class TestDecimalStrings:
|
|
@pytest.mark.parametrize(
|
|
("value", "text"),
|
|
[
|
|
("1", "1"),
|
|
("0.50", "0.50"),
|
|
("1E+1", "10"),
|
|
(Decimal("2.50"), "2.50"),
|
|
("0.0", "0.0"),
|
|
("1e-3", "0.001"),
|
|
],
|
|
)
|
|
def test_dec_writes_the_decimal_as_a_plain_string(self, value, text):
|
|
assert grading.dec(value) == text
|
|
|
|
def test_a_missed_element_scores_zero_with_the_scale_of_its_points(self):
|
|
body = {"exercise": "anchor", "key": {"elements": [element("terms", "0.50")]}}
|
|
grade = run(body, {"anchors": [anchor("water")]})
|
|
assert grade["items"][0]["points"] == "0.00"
|
|
assert grade["items"][0]["max"] == "0.50"
|
|
assert grade["score"] == "0.00"
|
|
assert grade["max"] == "0.50"
|
|
|
|
def test_points_are_summed_exactly(self):
|
|
body = {
|
|
"exercise": "anchor",
|
|
"key": {
|
|
"elements": [
|
|
element("terms", "0.1"),
|
|
element("water", "0.2"),
|
|
element("table", "0.4"),
|
|
]
|
|
},
|
|
}
|
|
grade = run(body, {"anchors": [anchor("terms"), anchor("water")]})
|
|
assert grade["score"] == "0.3"
|
|
assert grade["max"] == "0.7"
|
|
assert scores(grade) == [
|
|
("element:0", "0.1"),
|
|
("element:1", "0.2"),
|
|
("element:2", "0.0"),
|
|
]
|
|
|
|
|
|
class TestAnchorExercise:
|
|
BODY: ClassVar[dict] = {
|
|
"exercise": "anchor",
|
|
"key": {
|
|
"elements": [
|
|
element("terms", "2", hit="Right.", miss="Wrong."),
|
|
element("water", "1"),
|
|
]
|
|
},
|
|
}
|
|
|
|
def test_the_grade_names_the_submission_and_its_exercise(self):
|
|
grade = run(self.BODY, {"anchors": []})
|
|
assert grade["format"] == "scienting-grade/1"
|
|
assert grade["submission"] == "s1"
|
|
assert grade["exercise"] == "ex:1"
|
|
|
|
def test_each_element_is_scored_by_whether_any_answer_hits_it(self):
|
|
grade = run(self.BODY, {"anchors": [anchor("water"), anchor("terms")]})
|
|
assert grade["score"] == "3"
|
|
assert grade["max"] == "3"
|
|
assert [i["feedback"] for i in grade["items"]] == ["Right.", ""]
|
|
|
|
def test_a_miss_gives_the_miss_feedback(self):
|
|
grade = run(self.BODY, {"anchors": [anchor("table")]})
|
|
assert grade["score"] == "0"
|
|
assert [i["feedback"] for i in grade["items"]] == ["Wrong.", ""]
|
|
|
|
def test_no_answer_scores_nothing(self):
|
|
assert run(self.BODY, {})["score"] == "0"
|
|
|
|
def test_one_answer_may_hit_several_elements(self):
|
|
body = {
|
|
"exercise": "anchor",
|
|
"key": {"elements": [element("terms"), element("terms", "3")]},
|
|
}
|
|
grade = run(body, {"anchors": [anchor("terms")]})
|
|
assert grade["score"] == "4"
|
|
|
|
def test_an_element_may_list_several_acceptable_anchors(self):
|
|
two = {"anchors": [anchor("terms"), anchor("water")], "points": "1"}
|
|
body = {"exercise": "anchor", "key": {"elements": [two]}}
|
|
assert run(body, {"anchors": [anchor("water")]})["score"] == "1"
|
|
assert run(body, {"anchors": [anchor("table")]})["score"] == "0"
|
|
|
|
def test_a_stale_answer_hits_nothing(self):
|
|
stale = dict(anchor("terms"), node="zzzzzzz")
|
|
assert run(self.BODY, {"anchors": [stale]})["score"] == "0"
|
|
|
|
def test_an_answer_whose_hash_moved_but_whose_quote_is_found_still_hits(self):
|
|
moved = dict(anchor("terms"), node_hash="0" * 64)
|
|
assert run(self.BODY, {"anchors": [moved]})["score"] == "2"
|
|
|
|
|
|
class TestMatchRules:
|
|
TEXT = "Harmonic bond terms remain the most common description of covalent"
|
|
|
|
def key_span(self):
|
|
return span_of("terms", "the most common description")
|
|
|
|
def body(self, rule=None):
|
|
key = {"anchors": [anchor("terms", self.key_span())], "points": "1"}
|
|
out = {"exercise": "anchor", "key": {"elements": [key]}}
|
|
if rule is not None:
|
|
out["rule"] = rule
|
|
return out
|
|
|
|
@pytest.mark.parametrize(
|
|
("quote", "points"),
|
|
[
|
|
("the most common description", "1"),
|
|
("most common description of covalent", "1"),
|
|
("the most common", "1"),
|
|
("the most", "0"),
|
|
("terms remain the", "0"),
|
|
("covalent stretching", "0"),
|
|
],
|
|
)
|
|
def test_overlap_needs_half_the_key_span_by_default(self, quote, points):
|
|
answer = anchor("terms", span_of("terms", quote))
|
|
body = self.body({"match": "overlap"})
|
|
assert run(body, {"anchors": [answer]})["score"] == points
|
|
|
|
def test_the_least_overlap_can_be_set(self):
|
|
answer = anchor("terms", span_of("terms", "the most"))
|
|
lenient = self.body({"match": "overlap", "min_overlap": "0.25"})
|
|
assert run(lenient, {"anchors": [answer]})["score"] == "1"
|
|
tight = self.body({"match": "overlap", "min_overlap": "1"})
|
|
assert run(tight, {"anchors": [answer]})["score"] == "0"
|
|
|
|
def test_an_answer_without_a_span_covers_the_whole_node(self):
|
|
body = self.body({"match": "overlap"})
|
|
assert run(body, {"anchors": [anchor("terms")]})["score"] == "1"
|
|
|
|
def test_an_overlap_rule_on_a_key_without_a_span_needs_only_the_node(self):
|
|
body = {
|
|
"exercise": "anchor",
|
|
"rule": {"match": "overlap"},
|
|
"key": {"elements": [element("terms")]},
|
|
}
|
|
answer = anchor("terms", span_of("terms", "the most"))
|
|
assert run(body, {"anchors": [answer]})["score"] == "1"
|
|
|
|
def test_same_node_ignores_spans(self):
|
|
answer = anchor("terms", span_of("terms", "covalent stretching"))
|
|
for rule in (None, {"match": "same-node"}, {}):
|
|
assert run(self.body(rule), {"anchors": [answer]})["score"] == "1"
|
|
|
|
def test_an_answer_on_another_node_never_matches(self):
|
|
answer = anchor("water", [0, 5])
|
|
assert (
|
|
run(self.body({"match": "overlap"}), {"anchors": [answer]})["score"] == "0"
|
|
)
|
|
|
|
def test_an_empty_key_span_cannot_be_overlapped(self):
|
|
key = {"anchors": [anchor("terms", [5, 5])], "points": "1"}
|
|
body = {
|
|
"exercise": "anchor",
|
|
"rule": {"match": "overlap"},
|
|
"key": {"elements": [key]},
|
|
}
|
|
answer = anchor("terms", [0, 20])
|
|
assert run(body, {"anchors": [answer]})["score"] == "0"
|
|
|
|
|
|
class TestCheckExercise:
|
|
BODY: ClassVar[dict] = {
|
|
"exercise": "check",
|
|
"key": {
|
|
"verdict": {
|
|
"value": "supports",
|
|
"points": "1",
|
|
"feedback": {"hit": "Yes.", "miss": "No."},
|
|
},
|
|
"elements": [element("terms")],
|
|
},
|
|
}
|
|
|
|
def test_the_verdict_comes_first_and_is_scored_on_its_own(self):
|
|
grade = run(self.BODY, {"verdict": "supports", "anchors": [anchor("terms")]})
|
|
assert scores(grade) == [("verdict", "1"), ("element:0", "1")]
|
|
assert grade["items"][0]["feedback"] == "Yes."
|
|
assert grade["score"] == "2"
|
|
|
|
def test_a_wrong_verdict_loses_only_its_points(self):
|
|
grade = run(self.BODY, {"verdict": "refutes", "anchors": [anchor("terms")]})
|
|
assert scores(grade) == [("verdict", "0"), ("element:0", "1")]
|
|
assert grade["items"][0]["feedback"] == "No."
|
|
|
|
def test_no_verdict_is_wrong(self):
|
|
assert run(self.BODY, {"anchors": [anchor("terms")]})["score"] == "1"
|
|
|
|
def test_a_verdict_without_feedback_has_empty_feedback(self):
|
|
body = {
|
|
"exercise": "check",
|
|
"key": {"verdict": {"value": "supports", "points": "1"}, "elements": []},
|
|
}
|
|
assert run(body, {"verdict": "supports"})["items"][0]["feedback"] == ""
|
|
|
|
|
|
class TestTraceExercise:
|
|
BODY: ClassVar[dict] = {
|
|
"exercise": "trace",
|
|
"key": {
|
|
"chain": [
|
|
{
|
|
"anchors": [anchor("terms")],
|
|
"points": "1",
|
|
"feedback": {"hit": "First."},
|
|
},
|
|
{"anchors": [anchor("water")], "points": "2"},
|
|
]
|
|
},
|
|
}
|
|
|
|
def test_each_link_is_matched_by_the_answers_at_its_position(self):
|
|
chain = [[anchor("terms")], [anchor("water")]]
|
|
grade = run(self.BODY, {"chain": chain})
|
|
assert scores(grade) == [("link:0", "1"), ("link:1", "2")]
|
|
assert grade["items"][0]["feedback"] == "First."
|
|
|
|
def test_an_answer_at_the_wrong_position_does_not_count(self):
|
|
grade = run(self.BODY, {"chain": [[anchor("water")], [anchor("terms")]]})
|
|
assert grade["score"] == "0"
|
|
|
|
def test_a_chain_that_stops_early_misses_the_rest(self):
|
|
grade = run(self.BODY, {"chain": [[anchor("terms")]]})
|
|
assert scores(grade) == [("link:0", "1"), ("link:1", "0")]
|
|
|
|
def test_a_link_may_be_answered_by_several_anchors(self):
|
|
grade = run(self.BODY, {"chain": [[anchor("table"), anchor("terms")], []]})
|
|
assert scores(grade) == [("link:0", "1"), ("link:1", "0")]
|
|
|
|
def test_a_missing_chain_scores_nothing(self):
|
|
assert run(self.BODY, {})["score"] == "0"
|
|
|
|
|
|
class TestReproduceExercise:
|
|
def body(self, value="3.14", tolerance="0.01", points="2"):
|
|
return {
|
|
"exercise": "reproduce",
|
|
"key": {
|
|
"value": value,
|
|
"tolerance": tolerance,
|
|
"points": points,
|
|
"feedback": {"hit": "Close.", "miss": "Off."},
|
|
},
|
|
}
|
|
|
|
@pytest.mark.parametrize(
|
|
("answer", "points"),
|
|
[
|
|
("3.14", "2"),
|
|
("3.15", "2"),
|
|
("3.13", "2"),
|
|
("3.1501", "0"),
|
|
("3.1299", "0"),
|
|
("-3.14", "0"),
|
|
("3.140000", "2"),
|
|
],
|
|
)
|
|
def test_the_value_must_be_within_the_tolerance_inclusive(self, answer, points):
|
|
grade = run(self.body(), {"value": answer})
|
|
assert scores(grade) == [("value", points)]
|
|
|
|
def test_values_are_compared_exactly_in_decimal(self):
|
|
assert run(self.body("0.3", "0"), {"value": "0.30"})["score"] == "2"
|
|
assert (
|
|
run(self.body("0.3", "0"), {"value": "0.30000000000000004"})["score"] == "0"
|
|
)
|
|
|
|
def test_feedback_follows_the_outcome(self):
|
|
assert run(self.body(), {"value": "3.14"})["items"][0]["feedback"] == "Close."
|
|
assert run(self.body(), {"value": "9"})["items"][0]["feedback"] == "Off."
|
|
|
|
|
|
class TestVerifyExercise:
|
|
BODY: ClassVar[dict] = {
|
|
"exercise": "verify",
|
|
"key": {
|
|
"errors": [
|
|
{
|
|
"anchors": [anchor("terms")],
|
|
"verdict": "wrong-number",
|
|
"points": "1",
|
|
"feedback": {"hit": "Found.", "miss": "Missed."},
|
|
},
|
|
{"anchors": [anchor("water")], "verdict": "wrong-unit", "points": "1"},
|
|
]
|
|
},
|
|
}
|
|
|
|
def flag(self, name, verdict):
|
|
return {"anchor": anchor(name), "verdict": verdict}
|
|
|
|
def test_an_error_is_found_by_a_flag_on_its_node_with_its_verdict(self):
|
|
flags = [self.flag("terms", "wrong-number"), self.flag("water", "wrong-unit")]
|
|
grade = run(self.BODY, {"flags": flags})
|
|
assert scores(grade) == [("error:0", "1"), ("error:1", "1")]
|
|
assert grade["items"][0]["feedback"] == "Found."
|
|
|
|
def test_the_wrong_verdict_on_the_right_node_misses(self):
|
|
grade = run(self.BODY, {"flags": [self.flag("terms", "wrong-unit")]})
|
|
assert scores(grade) == [("error:0", "0"), ("error:1", "0")]
|
|
assert grade["items"][0]["feedback"] == "Missed."
|
|
|
|
def test_any_one_of_several_flags_on_the_node_may_carry_the_verdict(self):
|
|
flags = [self.flag("terms", "other"), self.flag("terms", "wrong-number")]
|
|
assert run(self.BODY, {"flags": flags})["score"] == "1"
|
|
|
|
def test_a_flag_on_another_node_finds_nothing(self):
|
|
assert (
|
|
run(self.BODY, {"flags": [self.flag("table", "wrong-number")]})["score"]
|
|
== "0"
|
|
)
|
|
|
|
def test_no_flags_find_nothing(self):
|
|
assert run(self.BODY, {})["score"] == "0"
|
|
|
|
|
|
class TestRefusals:
|
|
def test_a_stale_anchor_in_the_key_refuses_the_grade(self):
|
|
stale = dict(anchor("terms"), node="zzzzzzz")
|
|
body = {
|
|
"exercise": "anchor",
|
|
"key": {"elements": [{"anchors": [stale], "points": "1"}]},
|
|
}
|
|
with pytest.raises(Refusal, match=r"grade\.key-stale") as excinfo:
|
|
run(body, {"anchors": []})
|
|
assert excinfo.value.rule == "grade.key-stale"
|
|
|
|
def test_every_part_of_the_key_is_checked_for_stale_anchors(self):
|
|
stale = dict(anchor("terms"), node="zzzzzzz")
|
|
for part, shape in (
|
|
("elements", {"verdict": {"value": "x", "points": "1"}}),
|
|
("chain", {}),
|
|
("errors", {}),
|
|
):
|
|
key = dict(shape, **{part: [{"anchors": [stale], "points": "1"}]})
|
|
with pytest.raises(Refusal, match="key-stale"):
|
|
run({"exercise": "check", "key": key}, {})
|
|
|
|
def test_an_unknown_kind_is_refused(self):
|
|
with pytest.raises(
|
|
Refusal, match=r"grade\.kind: unknown exercise kind essay"
|
|
) as excinfo:
|
|
run({"exercise": "essay", "key": {}}, {})
|
|
assert excinfo.value.rule == "grade.kind"
|
|
|
|
|
|
def test_key_anchors_lists_elements_then_chain_then_errors():
|
|
a, b, c = ({"node": x} for x in "abc")
|
|
key = {
|
|
"elements": [{"anchors": [a]}],
|
|
"chain": [{"anchors": [b]}],
|
|
"errors": [{"anchors": [c, a]}],
|
|
}
|
|
assert grading.key_anchors({"key": key}) == [a, b, c, a]
|
|
assert grading.key_anchors({"key": {}}) == []
|