91 lines
3.5 KiB
Python
91 lines
3.5 KiB
Python
"""Grading a scienting-submission/1 against a scienting-exercise/1 key.
|
|
|
|
Non-normative; see spec/19-exercises.md.
|
|
"""
|
|
|
|
from fractions import Fraction
|
|
|
|
from . import anchors as A
|
|
from .tree import Refusal
|
|
|
|
|
|
def dec(fr):
|
|
fr = Fraction(fr)
|
|
if fr.denominator == 1:
|
|
return str(fr.numerator)
|
|
s = "%.4f" % float(fr)
|
|
return s.rstrip("0").rstrip(".")
|
|
|
|
|
|
def _resolved(state, anchor):
|
|
r = A.resolve(state, anchor)
|
|
if r["result"] == "stale":
|
|
return None
|
|
return r["node"], r.get("span")
|
|
|
|
|
|
def _match(state, rule, student, key):
|
|
s, k = _resolved(state, student), _resolved(state, key)
|
|
if not s or not k or s[0] != k[0]:
|
|
return False
|
|
if rule.get("match") == "overlap" and k[1] is not None:
|
|
if s[1] is None:
|
|
return True
|
|
lo, hi = max(s[1][0], k[1][0]), min(s[1][1], k[1][1])
|
|
size = k[1][1] - k[1][0]
|
|
return size > 0 and Fraction(max(0, hi - lo), size) >= Fraction(rule.get("min_overlap", "0.5"))
|
|
return True
|
|
|
|
|
|
def _any(state, rule, students, keys):
|
|
return any(_match(state, rule, s, k) for s in students for k in keys)
|
|
|
|
|
|
def key_anchors(body):
|
|
key = body["key"]
|
|
out = []
|
|
for el in key.get("elements", []) + key.get("chain", []) + key.get("errors", []):
|
|
out += el["anchors"]
|
|
return out
|
|
|
|
|
|
def grade(body, submission, state):
|
|
for a in key_anchors(body):
|
|
if A.resolve(state, a)["result"] == "stale":
|
|
raise Refusal("grade.key-stale", "an anchor in the key is stale")
|
|
kind, key, rule, ans = body["exercise"], body["key"], body.get("rule", {}), submission["answers"]
|
|
items = []
|
|
|
|
def item(name, pts, got, fb):
|
|
items.append({"element": name, "points": dec(pts if got else 0), "max": dec(pts), "feedback": fb})
|
|
|
|
if kind in ("anchor", "check"):
|
|
if kind == "check":
|
|
v = key["verdict"]
|
|
ok = ans.get("verdict") == v["value"]
|
|
item("verdict", Fraction(v["points"]), ok, v.get("feedback", {}).get("hit" if ok else "miss", ""))
|
|
for i, el in enumerate(key["elements"]):
|
|
ok = _any(state, rule, ans.get("anchors", []), el["anchors"])
|
|
item("element:%d" % i, Fraction(el["points"]), ok, el.get("feedback", {}).get("hit" if ok else "miss", ""))
|
|
elif kind == "trace":
|
|
chain = ans.get("chain", [])
|
|
for i, link in enumerate(key["chain"]):
|
|
ok = i < len(chain) and _any(state, rule, chain[i], link["anchors"])
|
|
item("link:%d" % i, Fraction(link["points"]), ok, link.get("feedback", {}).get("hit" if ok else "miss", ""))
|
|
elif kind == "reproduce":
|
|
ok = abs(Fraction(ans["value"]) - Fraction(key["value"])) <= Fraction(key["tolerance"])
|
|
item("value", Fraction(key["points"]), ok, key.get("feedback", {}).get("hit" if ok else "miss", ""))
|
|
elif kind == "verify":
|
|
flags = ans.get("flags", [])
|
|
for i, err in enumerate(key["errors"]):
|
|
hit = [f for f in flags if _any(state, rule, [f["anchor"]], err["anchors"])]
|
|
ok = any(f["verdict"] == err["verdict"] for f in hit)
|
|
fb = err.get("feedback", {}).get("hit" if ok else "miss", "")
|
|
item("error:%d" % i, Fraction(err["points"]), ok, fb)
|
|
else:
|
|
raise Refusal("grade.kind", "unknown exercise kind " + kind)
|
|
score = sum(Fraction(x["points"]) for x in items)
|
|
total = sum(Fraction(x["max"]) for x in items)
|
|
return {"format": "scienting-grade/1", "submission": submission["id"], "exercise": submission["exercise"],
|
|
"score": dec(score), "max": dec(total), "items": items}
|