# SPDX-License-Identifier: Apache-2.0 # Copyright 2026 Scienting # # Source Code: https://git.scient.ing/papers/paperspec # See the LICENSE.md file for full license terms. """Grading a submission against an exercise key (spec/19 ยง19.4).""" from decimal import Decimal from functools import cache from typing import ClassVar import pytest from helpers_lib1 import HALE, package_state, vectors from paperspec import anchors, grade as grading, hashing from paperspec.tree import Refusal @cache def _sentences(): """Plain text of three sentences of the example paper, and their IDs.""" doc = package_state(HALE).doc() wanted = { "terms": "Harmonic bond terms remain", "water": "Water, H$_2$O", "table": "Table 1 lists", } found = {} for nid in doc.preorder(): if doc.nodes[nid]["kind"] != "sentence": continue text = hashing.plain(doc, nid) for name, start in wanted.items(): if text.startswith(start): found[name] = (nid, text) return found def anchor(name, span=None): """An anchor on a sentence of the example paper, current at its latest state.""" state = package_state(HALE) doc = state.doc() return anchors.make_anchor( doc, hashing.compute(doc), _sentences()[name][0], state.rev, span, "smithlab" ) def span_of(name, quote): """The span of `quote` within a sentence's plain text.""" text = _sentences()[name][1] start = text.index(quote) return [start, start + len(quote)] def submission(answers, ident="s1"): return {"id": ident, "exercise": "ex:1", "answers": answers} def element(name, points="1", **feedback): """A key element anchored on a sentence.""" out = {"anchors": [anchor(name)], "points": points} if feedback: out["feedback"] = feedback return out def run(body, answers): return grading.grade(body, submission(answers), package_state(HALE)) def scores(grade): return [(i["element"], i["points"]) for i in grade["items"]] @pytest.mark.parametrize("case", vectors("grading")) def test_grading_vector(case): state = package_state(case["package"]) assert grading.grade(case["exercise"], case["submission"], state) == case["expect"] class TestDecimalStrings: @pytest.mark.parametrize( ("value", "text"), [ ("1", "1"), ("0.50", "0.50"), ("1E+1", "10"), (Decimal("2.50"), "2.50"), ("0.0", "0.0"), ("1e-3", "0.001"), ], ) def test_dec_writes_the_decimal_as_a_plain_string(self, value, text): assert grading.dec(value) == text def test_a_missed_element_scores_zero_with_the_scale_of_its_points(self): body = {"exercise": "anchor", "key": {"elements": [element("terms", "0.50")]}} grade = run(body, {"anchors": [anchor("water")]}) assert grade["items"][0]["points"] == "0.00" assert grade["items"][0]["max"] == "0.50" assert grade["score"] == "0.00" assert grade["max"] == "0.50" def test_points_are_summed_exactly(self): body = { "exercise": "anchor", "key": { "elements": [ element("terms", "0.1"), element("water", "0.2"), element("table", "0.4"), ] }, } grade = run(body, {"anchors": [anchor("terms"), anchor("water")]}) assert grade["score"] == "0.3" assert grade["max"] == "0.7" assert scores(grade) == [ ("element:0", "0.1"), ("element:1", "0.2"), ("element:2", "0.0"), ] class TestAnchorExercise: BODY: ClassVar[dict] = { "exercise": "anchor", "key": { "elements": [ element("terms", "2", hit="Right.", miss="Wrong."), element("water", "1"), ] }, } def test_the_grade_names_the_submission_and_its_exercise(self): grade = run(self.BODY, {"anchors": []}) assert grade["format"] == "scienting-grade/1" assert grade["submission"] == "s1" assert grade["exercise"] == "ex:1" def test_each_element_is_scored_by_whether_any_answer_hits_it(self): grade = run(self.BODY, {"anchors": [anchor("water"), anchor("terms")]}) assert grade["score"] == "3" assert grade["max"] == "3" assert [i["feedback"] for i in grade["items"]] == ["Right.", ""] def test_a_miss_gives_the_miss_feedback(self): grade = run(self.BODY, {"anchors": [anchor("table")]}) assert grade["score"] == "0" assert [i["feedback"] for i in grade["items"]] == ["Wrong.", ""] def test_no_answer_scores_nothing(self): assert run(self.BODY, {})["score"] == "0" def test_one_answer_may_hit_several_elements(self): body = { "exercise": "anchor", "key": {"elements": [element("terms"), element("terms", "3")]}, } grade = run(body, {"anchors": [anchor("terms")]}) assert grade["score"] == "4" def test_an_element_may_list_several_acceptable_anchors(self): two = {"anchors": [anchor("terms"), anchor("water")], "points": "1"} body = {"exercise": "anchor", "key": {"elements": [two]}} assert run(body, {"anchors": [anchor("water")]})["score"] == "1" assert run(body, {"anchors": [anchor("table")]})["score"] == "0" def test_a_stale_answer_hits_nothing(self): stale = dict(anchor("terms"), node="zzzzzzz") assert run(self.BODY, {"anchors": [stale]})["score"] == "0" def test_an_answer_whose_hash_moved_but_whose_quote_is_found_still_hits(self): moved = dict(anchor("terms"), node_hash="0" * 64) assert run(self.BODY, {"anchors": [moved]})["score"] == "2" class TestMatchRules: TEXT = "Harmonic bond terms remain the most common description of covalent" def key_span(self): return span_of("terms", "the most common description") def body(self, rule=None): key = {"anchors": [anchor("terms", self.key_span())], "points": "1"} out = {"exercise": "anchor", "key": {"elements": [key]}} if rule is not None: out["rule"] = rule return out @pytest.mark.parametrize( ("quote", "points"), [ ("the most common description", "1"), ("most common description of covalent", "1"), ("the most common", "1"), ("the most", "0"), ("terms remain the", "0"), ("covalent stretching", "0"), ], ) def test_overlap_needs_half_the_key_span_by_default(self, quote, points): answer = anchor("terms", span_of("terms", quote)) body = self.body({"match": "overlap"}) assert run(body, {"anchors": [answer]})["score"] == points def test_the_least_overlap_can_be_set(self): answer = anchor("terms", span_of("terms", "the most")) lenient = self.body({"match": "overlap", "min_overlap": "0.25"}) assert run(lenient, {"anchors": [answer]})["score"] == "1" tight = self.body({"match": "overlap", "min_overlap": "1"}) assert run(tight, {"anchors": [answer]})["score"] == "0" def test_an_answer_without_a_span_covers_the_whole_node(self): body = self.body({"match": "overlap"}) assert run(body, {"anchors": [anchor("terms")]})["score"] == "1" def test_an_overlap_rule_on_a_key_without_a_span_needs_only_the_node(self): body = { "exercise": "anchor", "rule": {"match": "overlap"}, "key": {"elements": [element("terms")]}, } answer = anchor("terms", span_of("terms", "the most")) assert run(body, {"anchors": [answer]})["score"] == "1" def test_same_node_ignores_spans(self): answer = anchor("terms", span_of("terms", "covalent stretching")) for rule in (None, {"match": "same-node"}, {}): assert run(self.body(rule), {"anchors": [answer]})["score"] == "1" def test_an_answer_on_another_node_never_matches(self): answer = anchor("water", [0, 5]) assert ( run(self.body({"match": "overlap"}), {"anchors": [answer]})["score"] == "0" ) def test_an_empty_key_span_cannot_be_overlapped(self): key = {"anchors": [anchor("terms", [5, 5])], "points": "1"} body = { "exercise": "anchor", "rule": {"match": "overlap"}, "key": {"elements": [key]}, } answer = anchor("terms", [0, 20]) assert run(body, {"anchors": [answer]})["score"] == "0" class TestCheckExercise: BODY: ClassVar[dict] = { "exercise": "check", "key": { "verdict": { "value": "supports", "points": "1", "feedback": {"hit": "Yes.", "miss": "No."}, }, "elements": [element("terms")], }, } def test_the_verdict_comes_first_and_is_scored_on_its_own(self): grade = run(self.BODY, {"verdict": "supports", "anchors": [anchor("terms")]}) assert scores(grade) == [("verdict", "1"), ("element:0", "1")] assert grade["items"][0]["feedback"] == "Yes." assert grade["score"] == "2" def test_a_wrong_verdict_loses_only_its_points(self): grade = run(self.BODY, {"verdict": "refutes", "anchors": [anchor("terms")]}) assert scores(grade) == [("verdict", "0"), ("element:0", "1")] assert grade["items"][0]["feedback"] == "No." def test_no_verdict_is_wrong(self): assert run(self.BODY, {"anchors": [anchor("terms")]})["score"] == "1" def test_a_verdict_without_feedback_has_empty_feedback(self): body = { "exercise": "check", "key": {"verdict": {"value": "supports", "points": "1"}, "elements": []}, } assert run(body, {"verdict": "supports"})["items"][0]["feedback"] == "" class TestTraceExercise: BODY: ClassVar[dict] = { "exercise": "trace", "key": { "chain": [ { "anchors": [anchor("terms")], "points": "1", "feedback": {"hit": "First."}, }, {"anchors": [anchor("water")], "points": "2"}, ] }, } def test_each_link_is_matched_by_the_answers_at_its_position(self): chain = [[anchor("terms")], [anchor("water")]] grade = run(self.BODY, {"chain": chain}) assert scores(grade) == [("link:0", "1"), ("link:1", "2")] assert grade["items"][0]["feedback"] == "First." def test_an_answer_at_the_wrong_position_does_not_count(self): grade = run(self.BODY, {"chain": [[anchor("water")], [anchor("terms")]]}) assert grade["score"] == "0" def test_a_chain_that_stops_early_misses_the_rest(self): grade = run(self.BODY, {"chain": [[anchor("terms")]]}) assert scores(grade) == [("link:0", "1"), ("link:1", "0")] def test_a_link_may_be_answered_by_several_anchors(self): grade = run(self.BODY, {"chain": [[anchor("table"), anchor("terms")], []]}) assert scores(grade) == [("link:0", "1"), ("link:1", "0")] def test_a_missing_chain_scores_nothing(self): assert run(self.BODY, {})["score"] == "0" class TestReproduceExercise: def body(self, value="3.14", tolerance="0.01", points="2"): return { "exercise": "reproduce", "key": { "value": value, "tolerance": tolerance, "points": points, "feedback": {"hit": "Close.", "miss": "Off."}, }, } @pytest.mark.parametrize( ("answer", "points"), [ ("3.14", "2"), ("3.15", "2"), ("3.13", "2"), ("3.1501", "0"), ("3.1299", "0"), ("-3.14", "0"), ("3.140000", "2"), ], ) def test_the_value_must_be_within_the_tolerance_inclusive(self, answer, points): grade = run(self.body(), {"value": answer}) assert scores(grade) == [("value", points)] def test_values_are_compared_exactly_in_decimal(self): assert run(self.body("0.3", "0"), {"value": "0.30"})["score"] == "2" assert ( run(self.body("0.3", "0"), {"value": "0.30000000000000004"})["score"] == "0" ) def test_feedback_follows_the_outcome(self): assert run(self.body(), {"value": "3.14"})["items"][0]["feedback"] == "Close." assert run(self.body(), {"value": "9"})["items"][0]["feedback"] == "Off." class TestVerifyExercise: BODY: ClassVar[dict] = { "exercise": "verify", "key": { "errors": [ { "anchors": [anchor("terms")], "verdict": "wrong-number", "points": "1", "feedback": {"hit": "Found.", "miss": "Missed."}, }, {"anchors": [anchor("water")], "verdict": "wrong-unit", "points": "1"}, ] }, } def flag(self, name, verdict): return {"anchor": anchor(name), "verdict": verdict} def test_an_error_is_found_by_a_flag_on_its_node_with_its_verdict(self): flags = [self.flag("terms", "wrong-number"), self.flag("water", "wrong-unit")] grade = run(self.BODY, {"flags": flags}) assert scores(grade) == [("error:0", "1"), ("error:1", "1")] assert grade["items"][0]["feedback"] == "Found." def test_the_wrong_verdict_on_the_right_node_misses(self): grade = run(self.BODY, {"flags": [self.flag("terms", "wrong-unit")]}) assert scores(grade) == [("error:0", "0"), ("error:1", "0")] assert grade["items"][0]["feedback"] == "Missed." def test_any_one_of_several_flags_on_the_node_may_carry_the_verdict(self): flags = [self.flag("terms", "other"), self.flag("terms", "wrong-number")] assert run(self.BODY, {"flags": flags})["score"] == "1" def test_a_flag_on_another_node_finds_nothing(self): assert ( run(self.BODY, {"flags": [self.flag("table", "wrong-number")]})["score"] == "0" ) def test_no_flags_find_nothing(self): assert run(self.BODY, {})["score"] == "0" class TestRefusals: def test_a_stale_anchor_in_the_key_refuses_the_grade(self): stale = dict(anchor("terms"), node="zzzzzzz") body = { "exercise": "anchor", "key": {"elements": [{"anchors": [stale], "points": "1"}]}, } with pytest.raises(Refusal, match=r"grade\.key-stale") as excinfo: run(body, {"anchors": []}) assert excinfo.value.rule == "grade.key-stale" def test_every_part_of_the_key_is_checked_for_stale_anchors(self): stale = dict(anchor("terms"), node="zzzzzzz") for part, shape in ( ("elements", {"verdict": {"value": "x", "points": "1"}}), ("chain", {}), ("errors", {}), ): key = dict(shape, **{part: [{"anchors": [stale], "points": "1"}]}) with pytest.raises(Refusal, match="key-stale"): run({"exercise": "check", "key": key}, {}) def test_an_unknown_kind_is_refused(self): with pytest.raises( Refusal, match=r"grade\.kind: unknown exercise kind essay" ) as excinfo: run({"exercise": "essay", "key": {}}, {}) assert excinfo.value.rule == "grade.kind" def test_key_anchors_lists_elements_then_chain_then_errors(): a, b, c = ({"node": x} for x in "abc") key = { "elements": [{"anchors": [a]}], "chain": [{"anchors": [b]}], "errors": [{"anchors": [c, a]}], } assert grading.key_anchors({"key": key}) == [a, b, c, a] assert grading.key_anchors({"key": {}}) == []