feat: more comprehensive cohort report
Pipeline / check (pull_request) Failing after 35s
Pipeline / docs (pull_request) Skipped
Pipeline / nightly (pull_request) Skipped
Pipeline / release (pull_request) Skipped

This commit is contained in:
alexm committed 2026-09-22 01:19:35 -04:00
1 parent cea9769022
commit 220363d4d3
4 files changed
+418 -21

No files matched your search

+45
View File
@@ -620,6 +620,18 @@ pub fn cohort_value(diagnostic: &CohortDiagnostic, config: &RenderConfig) -> Val
.collect(),
),
);
// Separate from `questions` so no statistic can pick them up, and merged
// back in by the evidence section, which describes rather than measures.
out.insert(
"dropped-detail",
Value::Array(
diagnostic
.dropped_detail
.iter()
.map(|q| cohort_question_value(q, content))
.collect(),
),
);
out.insert(
"grades",
@@ -872,6 +884,18 @@ fn cohort_question_value(question: &CohortQuestionRow, content: bool) -> Value {
.collect(),
),
);
value.insert_some(
"stem",
question
.stem
.as_ref()
.map(|text| markup_value(text, content)),
);
value.insert("dropped", Value::Bool(question.dropped));
value.insert(
"dropped-full-credit",
Value::Bool(question.dropped_full_credit),
);
value.insert(
"taught-in",
Value::Array(question.taught_in.iter().map(Value::str).collect()),
@@ -902,6 +926,27 @@ fn cohort_question_value(question: &CohortQuestionRow, content: bool) -> Value {
.map(|option| {
let mut value = Value::dict();
value.insert("letter", Value::str(&option.letter));
value.insert_some(
"text",
option.text.as_ref().map(|text| markup_value(text, content)),
);
// The letter on each paper, so a statistic reported against
// the bank letter can be checked against a student's copy.
value.insert(
"printed",
Value::Array(
option
.printed
.iter()
.map(|printed| {
let mut pair = Value::dict();
pair.insert("form", Value::str(&printed.form));
pair.insert("letter", Value::str(&printed.letter));
pair
})
.collect(),
),
);
value.insert("count", Value::Int(option.count as i64));
value.insert("rate", Value::Float(option.rate));
value.insert("is-key", Value::Bool(option.is_key));
+143 -18
View File
@@ -121,11 +121,32 @@
discrimination: 0.10,
blank-rate: 0.0,
key: ("B",),
stem: [A sample question stem, shown so the option shares can be read against what was asked.],
options: (
(letter: "A", count: 9, rate: 0.375, is-key: false, point-biserial: 0.11, nonfunctioning: false),
(letter: "B", count: 10, rate: 0.417, is-key: true, point-biserial: 0.05, nonfunctioning: false),
(letter: "C", count: 5, rate: 0.208, is-key: false, point-biserial: -0.2, nonfunctioning: false),
(letter: "D", count: 0, rate: 0.0, is-key: false, nonfunctioning: true),
(
letter: "A",
text: [A sample distractor built on a real misconception.],
printed: ((form: "A", letter: "C"), (form: "B", letter: "A")),
count: 9, rate: 0.375, is-key: false, point-biserial: 0.11, nonfunctioning: false,
),
(
letter: "B",
text: [A sample key.],
printed: ((form: "A", letter: "D"), (form: "B", letter: "C")),
count: 10, rate: 0.417, is-key: true, point-biserial: 0.05, nonfunctioning: false,
),
(
letter: "C",
text: [A second sample distractor.],
printed: ((form: "A", letter: "A"), (form: "B", letter: "D")),
count: 5, rate: 0.208, is-key: false, point-biserial: -0.2, nonfunctioning: false,
),
(
letter: "D",
text: [A distractor nobody chose.],
printed: ((form: "A", letter: "B"), (form: "B", letter: "B")),
count: 0, rate: 0.0, is-key: false, nonfunctioning: true,
),
),
flags: ("ambiguous",),
notes: ([Distractor A drew as many strong students as the key.],),
@@ -168,6 +189,33 @@
biggest-surprise: (number: 14, expected: 0.45, observed: 0.86),
),
dropped-questions: (),
dropped-detail: (
(
number: 9,
item: "bank::q-sample-009",
level: 2,
targets: ("t-sample-gap",),
target-texts: ([A sample learning target.],),
dropped: true,
dropped-full-credit: true,
stem: [A sample question that was thrown out after the exam.],
taught-in: ("Entropy (L1.2)",),
lectures: ("L1.2",),
p: 0.18,
blank-rate: 0.0,
key: ("A",),
options: (
(letter: "A", text: [The keyed option, which few chose.], count: 4, rate: 0.18, is-key: true, nonfunctioning: false),
(letter: "B", text: [The option most students read as correct.], count: 15, rate: 0.68, is-key: false, nonfunctioning: false),
(letter: "C", text: [A third option.], count: 3, rate: 0.14, is-key: false, nonfunctioning: false),
),
flags: (),
notes: (),
prediction-notes: (),
calibrated: false,
by-form: (:),
),
),
revise: (),
forms: (
(id: "A", students: 12, mean: 73.5, sd: 10.2),
@@ -1077,28 +1125,45 @@
]
// ─────────────────────────────────────────────────────────────────────────────
// The revise queue, with option tables
// Every question, with option tables
// ─────────────────────────────────────────────────────────────────────────────
#let revise = cb-data.at("revise", default: ())
// Every scored question, not only the flagged ones. A question that raised no
// flag still has an option spread worth seeing: it is the reference for what a
// healthy item looks like on this exam, and the place to check a query about a
// question nothing was wrong with. The flagged ones are already called out by
// name in the triage lists above, so nothing is lost by putting them back among
// their neighbours here.
// Dropped items come from their own list, since they carry no statistics and
// belong in none of the tables above. They belong here: the option spread is
// the evidence that justified dropping them, and re-running the report after a
// drop should not erase it.
#let evidence = questions + cb-data.at("dropped-detail", default: ())
#if revise.len() > 0 [
#if evidence.len() > 0 [
#pagebreak(weak: true)
= The evidence, question by question
#explain[
The full breakdown for every flagged question: what the flags mean, and who
chose what. In question order, so you can find one by its number; the lists
above are the same questions sorted by what they ask of you. The r column
beside each option is the correlation between choosing that option and
scoring well on the rest of the exam, which is how a defensible distractor
announces itself.
The full breakdown for every question, flagged or not, including the ones
you dropped: what it asked, who chose what, and what any flags mean. A
dropped question is marked as such and carries no p, r, or D, because
dropping it overrode its credit. What students marked is untouched by the
drop, so its option spread still stands as the record of why it went. In question order, so you can find
one by its number; the lists above are the subset that needs a decision from
you, sorted by what they ask of you. The r column beside each option is the
correlation between choosing that option and scoring well on the rest of the
exam, which is how a defensible distractor announces itself.
Each option shows the bank letter first, then what it was lettered on each
form (#raw("A:D") means the bank's option was printed as D on form A), so a
row here can be read against the paper a student is holding.
]
// Question order here, not triage order. The lists above are for deciding
// what to do; this section is for looking one question up while you do it,
// and a reader with a number in hand should not have to scan every entry.
#for q in revise.sorted(key: q => q.number) [
#for q in evidence.sorted(key: q => q.number) [
#block(breakable: false, above: entry-gap, width: 100%)[
#{
let parts = (
@@ -1106,6 +1171,11 @@
let bits = (text(weight: "bold", fill: accent)[Question #q.number],)
let item = q.at("item", default: none)
if item != none { bits.push(text(size: size-meta, fill: luma(120))[#item]) }
if q.at("dropped", default: false) {
bits.push(text(size: size-micro, weight: "bold", fill: luma(110))[
DROPPED#if q.at("dropped-full-credit", default: false) [ · FULL CREDIT]
])
}
let flags = q.at("flags", default: ())
if flags.len() > 0 {
bits.push(stack(dir: ltr, spacing: 3pt, ..flags.map(flag-badge)))
@@ -1117,6 +1187,20 @@
let context_ = question-context(q)
if context_ != none { parts.push(context_) }
// The question itself. A row of option shares says a distractor drew
// 44% of the class; only the stem says whether that is a second
// defensible reading. Instructor copy only, which is the only place this
// section appears.
let stem = q.at("stem", default: none)
if stem != none {
parts.push(block(
width: 100%,
fill: luma(252),
stroke: (left: 2pt + luma(220)),
inset: (left: 8pt, rest: 6pt),
)[#text(size: size-small)[#markup(stem)]])
}
for note in q.at("notes", default: ()) {
parts.push(text(size: size-small)[— #markup(note)])
}
@@ -1141,19 +1225,50 @@
}
if show-options and q.at("options", default: ()).len() > 0 {
// The option column is two things: the bank letter every statistic is
// keyed by, and the letter the option actually carried on each paper.
// Without the second, a note about option D cannot be checked against
// the copy a student brings to office hours, since shuffling gives the
// same option a different letter on every form.
//
// The text goes last and takes the free column. It is the only cell
// whose length is not bounded, so anywhere else it either squeezes the
// numbers or wraps to three lines while they sit in a narrow gutter.
// Last, the numbers keep their natural widths and the text runs to the
// page edge.
parts.push(table(
columns: (auto, auto, auto, 2.6cm, auto),
columns: (auto, auto, auto, 2.6cm, auto, 1fr),
stroke: none,
align: (center + horizon, right + horizon, right + horizon, left + horizon, right + horizon),
align: (
center + horizon,
right + horizon,
right + horizon,
left + horizon,
right + horizon,
left + top,
),
inset: (x: 5pt, y: 3.5pt),
table.header(th[OPT], th[n], th[SHARE], th[], th[r]),
table.header(th[OPT], th[n], th[SHARE], th[], th[r], th[TEXT]),
..q
.options
.map(option => (
{
if option.at("is-key", default: false) {
let printed = option.at("printed", default: ())
let letter = if option.at("is-key", default: false) {
text(weight: "bold", fill: ok-color)[#option.letter]
} else { [#option.letter] }
if printed.len() == 0 {
letter
} else {
stack(
dir: ttb,
spacing: step * 0.25,
letter,
text(size: size-micro, fill: luma(125))[
#printed.map(p => p.form + ":" + p.letter).join(" ")
],
)
}
},
text(size: size-small)[#option.count],
text(size: size-small)[#pct(option.rate)],
@@ -1172,6 +1287,16 @@
text(size: size-small)[#signed(r)]
}
},
{
let body = option.at("text", default: none)
if body == none {
text(size: size-micro, fill: thin-color)[not in the bank]
} else if option.at("is-key", default: false) {
text(size: size-small, fill: ok-color.darken(25%))[#markup(body)]
} else {
text(size: size-small)[#markup(body)]
}
},
))
.flatten(),
))