First implementation (#1)

Reviewed-on: #1

tools/oracle/grade.py

@@ -0,0 +1,93 @@
+"""Grading a scienting-submission/1 against a scienting-exercise/1 key.
+
+Non-normative; see spec/19-exercises.md.
+"""
+
+from decimal import Decimal
+from fractions import Fraction
+
+from . import anchors as A
+from .tree import Refusal
+
+
+def dec(d):
+    """A decimal string, exactly as the sum came out (spec/19 §19.4: points are decimal strings, summed exactly)."""
+    return format(Decimal(d), "f")
+
+
+def _zero_like(d):
+    """Zero with the scale of `d`, so a missed element worth "0.5" scores "0.0"."""
+    return Decimal(0).quantize(Decimal(d))
+
+
+def _resolved(state, anchor):
+    r = A.resolve(state, anchor)
+    if r["result"] == "stale":
+        return None
+    return r["node"], r.get("span")
+
+
+def _match(state, rule, student, key):
+    s, k = _resolved(state, student), _resolved(state, key)
+    if not s or not k or s[0] != k[0]:
+        return False
+    if rule.get("match") == "overlap" and k[1] is not None:
+        if s[1] is None:
+            return True
+        lo, hi = max(s[1][0], k[1][0]), min(s[1][1], k[1][1])
+        size = k[1][1] - k[1][0]
+        return size > 0 and Fraction(max(0, hi - lo), size) >= Fraction(rule.get("min_overlap", "0.5"))
+    return True
+
+
+def _any(state, rule, students, keys):
+    return any(_match(state, rule, s, k) for s in students for k in keys)
+
+
+def key_anchors(body):
+    key = body["key"]
+    out = []
+    for el in key.get("elements", []) + key.get("chain", []) + key.get("errors", []):
+        out += el["anchors"]
+    return out
+
+
+def grade(body, submission, state):
+    for a in key_anchors(body):
+        if A.resolve(state, a)["result"] == "stale":
+            raise Refusal("grade.key-stale", "an anchor in the key is stale")
+    kind, key, rule, ans = body["exercise"], body["key"], body.get("rule", {}), submission["answers"]
+    items = []
+
+    def item(name, pts, got, fb):
+        items.append({"element": name, "points": dec(pts if got else _zero_like(pts)), "max": dec(pts), "feedback": fb})
+
+    if kind in ("anchor", "check"):
+        if kind == "check":
+            v = key["verdict"]
+            ok = ans.get("verdict") == v["value"]
+            item("verdict", v["points"], ok, v.get("feedback", {}).get("hit" if ok else "miss", ""))
+        for i, el in enumerate(key["elements"]):
+            ok = _any(state, rule, ans.get("anchors", []), el["anchors"])
+            item("element:%d" % i, el["points"], ok, el.get("feedback", {}).get("hit" if ok else "miss", ""))
+    elif kind == "trace":
+        chain = ans.get("chain", [])
+        for i, link in enumerate(key["chain"]):
+            ok = i < len(chain) and _any(state, rule, chain[i], link["anchors"])
+            item("link:%d" % i, link["points"], ok, link.get("feedback", {}).get("hit" if ok else "miss", ""))
+    elif kind == "reproduce":
+        ok = abs(Decimal(ans["value"]) - Decimal(key["value"])) <= Decimal(key["tolerance"])
+        item("value", key["points"], ok, key.get("feedback", {}).get("hit" if ok else "miss", ""))
+    elif kind == "verify":
+        flags = ans.get("flags", [])
+        for i, err in enumerate(key["errors"]):
+            hit = [f for f in flags if _any(state, rule, [f["anchor"]], err["anchors"])]
+            ok = any(f["verdict"] == err["verdict"] for f in hit)
+            fb = err.get("feedback", {}).get("hit" if ok else "miss", "")
+            item("error:%d" % i, err["points"], ok, fb)
+    else:
+        raise Refusal("grade.kind", "unknown exercise kind " + kind)
+    score = sum((Decimal(x["points"]) for x in items), Decimal(0))
+    total = sum((Decimal(x["max"]) for x in items), Decimal(0))
+    return {"format": "scienting-grade/1", "submission": submission["id"], "exercise": submission["exercise"],
+            "score": dec(score), "max": dec(total), "items": items}