paperspec/tools/oracle/grade.py

"""Grading a scienting-submission/1 against a scienting-exercise/1 key.

Non-normative; see spec/19-exercises.md.
"""

from decimal import Decimal
from fractions import Fraction

from . import anchors as A
from .tree import Refusal


def dec(d):
    """A decimal string, exactly as the sum came out (spec/19 §19.4: points are decimal strings, summed exactly)."""
    return format(Decimal(d), "f")


def _zero_like(d):
    """Zero with the scale of `d`, so a missed element worth "0.5" scores "0.0"."""
    return Decimal(0).quantize(Decimal(d))


def _resolved(state, anchor):
    r = A.resolve(state, anchor)
    if r["result"] == "stale":
        return None
    return r["node"], r.get("span")


def _match(state, rule, student, key):
    s, k = _resolved(state, student), _resolved(state, key)
    if not s or not k or s[0] != k[0]:
        return False
    if rule.get("match") == "overlap" and k[1] is not None:
        if s[1] is None:
            return True
        lo, hi = max(s[1][0], k[1][0]), min(s[1][1], k[1][1])
        size = k[1][1] - k[1][0]
        return size > 0 and Fraction(max(0, hi - lo), size) >= Fraction(rule.get("min_overlap", "0.5"))
    return True


def _any(state, rule, students, keys):
    return any(_match(state, rule, s, k) for s in students for k in keys)


def key_anchors(body):
    key = body["key"]
    out = []
    for el in key.get("elements", []) + key.get("chain", []) + key.get("errors", []):
        out += el["anchors"]
    return out


def grade(body, submission, state):
    for a in key_anchors(body):
        if A.resolve(state, a)["result"] == "stale":
            raise Refusal("grade.key-stale", "an anchor in the key is stale")
    kind, key, rule, ans = body["exercise"], body["key"], body.get("rule", {}), submission["answers"]
    items = []

    def item(name, pts, got, fb):
        items.append({"element": name, "points": dec(pts if got else _zero_like(pts)), "max": dec(pts), "feedback": fb})

    if kind in ("anchor", "check"):
        if kind == "check":
            v = key["verdict"]
            ok = ans.get("verdict") == v["value"]
            item("verdict", v["points"], ok, v.get("feedback", {}).get("hit" if ok else "miss", ""))
        for i, el in enumerate(key["elements"]):
            ok = _any(state, rule, ans.get("anchors", []), el["anchors"])
            item("element:%d" % i, el["points"], ok, el.get("feedback", {}).get("hit" if ok else "miss", ""))
    elif kind == "trace":
        chain = ans.get("chain", [])
        for i, link in enumerate(key["chain"]):
            ok = i < len(chain) and _any(state, rule, chain[i], link["anchors"])
            item("link:%d" % i, link["points"], ok, link.get("feedback", {}).get("hit" if ok else "miss", ""))
    elif kind == "reproduce":
        ok = abs(Decimal(ans["value"]) - Decimal(key["value"])) <= Decimal(key["tolerance"])
        item("value", key["points"], ok, key.get("feedback", {}).get("hit" if ok else "miss", ""))
    elif kind == "verify":
        flags = ans.get("flags", [])
        for i, err in enumerate(key["errors"]):
            hit = [f for f in flags if _any(state, rule, [f["anchor"]], err["anchors"])]
            ok = any(f["verdict"] == err["verdict"] for f in hit)
            fb = err.get("feedback", {}).get("hit" if ok else "miss", "")
            item("error:%d" % i, err["points"], ok, fb)
    else:
        raise Refusal("grade.kind", "unknown exercise kind " + kind)
    score = sum((Decimal(x["points"]) for x in items), Decimal(0))
    total = sum((Decimal(x["max"]) for x in items), Decimal(0))
    return {"format": "scienting-grade/1", "submission": submission["id"], "exercise": submission["exercise"],
            "score": dec(score), "max": dec(total), "items": items}