feat: more comprehensive cohort report
This commit is contained in:
4 files changed
+418
-21
No files matched your search
@@ -620,6 +620,18 @@ pub fn cohort_value(diagnostic: &CohortDiagnostic, config: &RenderConfig) -> Val
|
||||
.collect(),
|
||||
),
|
||||
);
|
||||
// Separate from `questions` so no statistic can pick them up, and merged
|
||||
// back in by the evidence section, which describes rather than measures.
|
||||
out.insert(
|
||||
"dropped-detail",
|
||||
Value::Array(
|
||||
diagnostic
|
||||
.dropped_detail
|
||||
.iter()
|
||||
.map(|q| cohort_question_value(q, content))
|
||||
.collect(),
|
||||
),
|
||||
);
|
||||
|
||||
out.insert(
|
||||
"grades",
|
||||
@@ -872,6 +884,18 @@ fn cohort_question_value(question: &CohortQuestionRow, content: bool) -> Value {
|
||||
.collect(),
|
||||
),
|
||||
);
|
||||
value.insert_some(
|
||||
"stem",
|
||||
question
|
||||
.stem
|
||||
.as_ref()
|
||||
.map(|text| markup_value(text, content)),
|
||||
);
|
||||
value.insert("dropped", Value::Bool(question.dropped));
|
||||
value.insert(
|
||||
"dropped-full-credit",
|
||||
Value::Bool(question.dropped_full_credit),
|
||||
);
|
||||
value.insert(
|
||||
"taught-in",
|
||||
Value::Array(question.taught_in.iter().map(Value::str).collect()),
|
||||
@@ -902,6 +926,27 @@ fn cohort_question_value(question: &CohortQuestionRow, content: bool) -> Value {
|
||||
.map(|option| {
|
||||
let mut value = Value::dict();
|
||||
value.insert("letter", Value::str(&option.letter));
|
||||
value.insert_some(
|
||||
"text",
|
||||
option.text.as_ref().map(|text| markup_value(text, content)),
|
||||
);
|
||||
// The letter on each paper, so a statistic reported against
|
||||
// the bank letter can be checked against a student's copy.
|
||||
value.insert(
|
||||
"printed",
|
||||
Value::Array(
|
||||
option
|
||||
.printed
|
||||
.iter()
|
||||
.map(|printed| {
|
||||
let mut pair = Value::dict();
|
||||
pair.insert("form", Value::str(&printed.form));
|
||||
pair.insert("letter", Value::str(&printed.letter));
|
||||
pair
|
||||
})
|
||||
.collect(),
|
||||
),
|
||||
);
|
||||
value.insert("count", Value::Int(option.count as i64));
|
||||
value.insert("rate", Value::Float(option.rate));
|
||||
value.insert("is-key", Value::Bool(option.is_key));
|
||||
|
||||
@@ -121,11 +121,32 @@
|
||||
discrimination: 0.10,
|
||||
blank-rate: 0.0,
|
||||
key: ("B",),
|
||||
stem: [A sample question stem, shown so the option shares can be read against what was asked.],
|
||||
options: (
|
||||
(letter: "A", count: 9, rate: 0.375, is-key: false, point-biserial: 0.11, nonfunctioning: false),
|
||||
(letter: "B", count: 10, rate: 0.417, is-key: true, point-biserial: 0.05, nonfunctioning: false),
|
||||
(letter: "C", count: 5, rate: 0.208, is-key: false, point-biserial: -0.2, nonfunctioning: false),
|
||||
(letter: "D", count: 0, rate: 0.0, is-key: false, nonfunctioning: true),
|
||||
(
|
||||
letter: "A",
|
||||
text: [A sample distractor built on a real misconception.],
|
||||
printed: ((form: "A", letter: "C"), (form: "B", letter: "A")),
|
||||
count: 9, rate: 0.375, is-key: false, point-biserial: 0.11, nonfunctioning: false,
|
||||
),
|
||||
(
|
||||
letter: "B",
|
||||
text: [A sample key.],
|
||||
printed: ((form: "A", letter: "D"), (form: "B", letter: "C")),
|
||||
count: 10, rate: 0.417, is-key: true, point-biserial: 0.05, nonfunctioning: false,
|
||||
),
|
||||
(
|
||||
letter: "C",
|
||||
text: [A second sample distractor.],
|
||||
printed: ((form: "A", letter: "A"), (form: "B", letter: "D")),
|
||||
count: 5, rate: 0.208, is-key: false, point-biserial: -0.2, nonfunctioning: false,
|
||||
),
|
||||
(
|
||||
letter: "D",
|
||||
text: [A distractor nobody chose.],
|
||||
printed: ((form: "A", letter: "B"), (form: "B", letter: "B")),
|
||||
count: 0, rate: 0.0, is-key: false, nonfunctioning: true,
|
||||
),
|
||||
),
|
||||
flags: ("ambiguous",),
|
||||
notes: ([Distractor A drew as many strong students as the key.],),
|
||||
@@ -168,6 +189,33 @@
|
||||
biggest-surprise: (number: 14, expected: 0.45, observed: 0.86),
|
||||
),
|
||||
dropped-questions: (),
|
||||
dropped-detail: (
|
||||
(
|
||||
number: 9,
|
||||
item: "bank::q-sample-009",
|
||||
level: 2,
|
||||
targets: ("t-sample-gap",),
|
||||
target-texts: ([A sample learning target.],),
|
||||
dropped: true,
|
||||
dropped-full-credit: true,
|
||||
stem: [A sample question that was thrown out after the exam.],
|
||||
taught-in: ("Entropy (L1.2)",),
|
||||
lectures: ("L1.2",),
|
||||
p: 0.18,
|
||||
blank-rate: 0.0,
|
||||
key: ("A",),
|
||||
options: (
|
||||
(letter: "A", text: [The keyed option, which few chose.], count: 4, rate: 0.18, is-key: true, nonfunctioning: false),
|
||||
(letter: "B", text: [The option most students read as correct.], count: 15, rate: 0.68, is-key: false, nonfunctioning: false),
|
||||
(letter: "C", text: [A third option.], count: 3, rate: 0.14, is-key: false, nonfunctioning: false),
|
||||
),
|
||||
flags: (),
|
||||
notes: (),
|
||||
prediction-notes: (),
|
||||
calibrated: false,
|
||||
by-form: (:),
|
||||
),
|
||||
),
|
||||
revise: (),
|
||||
forms: (
|
||||
(id: "A", students: 12, mean: 73.5, sd: 10.2),
|
||||
@@ -1077,28 +1125,45 @@
|
||||
]
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// The revise queue, with option tables
|
||||
// Every question, with option tables
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
#let revise = cb-data.at("revise", default: ())
|
||||
// Every scored question, not only the flagged ones. A question that raised no
|
||||
// flag still has an option spread worth seeing: it is the reference for what a
|
||||
// healthy item looks like on this exam, and the place to check a query about a
|
||||
// question nothing was wrong with. The flagged ones are already called out by
|
||||
// name in the triage lists above, so nothing is lost by putting them back among
|
||||
// their neighbours here.
|
||||
// Dropped items come from their own list, since they carry no statistics and
|
||||
// belong in none of the tables above. They belong here: the option spread is
|
||||
// the evidence that justified dropping them, and re-running the report after a
|
||||
// drop should not erase it.
|
||||
#let evidence = questions + cb-data.at("dropped-detail", default: ())
|
||||
|
||||
#if revise.len() > 0 [
|
||||
#if evidence.len() > 0 [
|
||||
#pagebreak(weak: true)
|
||||
= The evidence, question by question
|
||||
|
||||
#explain[
|
||||
The full breakdown for every flagged question: what the flags mean, and who
|
||||
chose what. In question order, so you can find one by its number; the lists
|
||||
above are the same questions sorted by what they ask of you. The r column
|
||||
beside each option is the correlation between choosing that option and
|
||||
scoring well on the rest of the exam, which is how a defensible distractor
|
||||
announces itself.
|
||||
The full breakdown for every question, flagged or not, including the ones
|
||||
you dropped: what it asked, who chose what, and what any flags mean. A
|
||||
dropped question is marked as such and carries no p, r, or D, because
|
||||
dropping it overrode its credit. What students marked is untouched by the
|
||||
drop, so its option spread still stands as the record of why it went. In question order, so you can find
|
||||
one by its number; the lists above are the subset that needs a decision from
|
||||
you, sorted by what they ask of you. The r column beside each option is the
|
||||
correlation between choosing that option and scoring well on the rest of the
|
||||
exam, which is how a defensible distractor announces itself.
|
||||
|
||||
Each option shows the bank letter first, then what it was lettered on each
|
||||
form (#raw("A:D") means the bank's option was printed as D on form A), so a
|
||||
row here can be read against the paper a student is holding.
|
||||
]
|
||||
|
||||
// Question order here, not triage order. The lists above are for deciding
|
||||
// what to do; this section is for looking one question up while you do it,
|
||||
// and a reader with a number in hand should not have to scan every entry.
|
||||
#for q in revise.sorted(key: q => q.number) [
|
||||
#for q in evidence.sorted(key: q => q.number) [
|
||||
#block(breakable: false, above: entry-gap, width: 100%)[
|
||||
#{
|
||||
let parts = (
|
||||
@@ -1106,6 +1171,11 @@
|
||||
let bits = (text(weight: "bold", fill: accent)[Question #q.number],)
|
||||
let item = q.at("item", default: none)
|
||||
if item != none { bits.push(text(size: size-meta, fill: luma(120))[#item]) }
|
||||
if q.at("dropped", default: false) {
|
||||
bits.push(text(size: size-micro, weight: "bold", fill: luma(110))[
|
||||
DROPPED#if q.at("dropped-full-credit", default: false) [ · FULL CREDIT]
|
||||
])
|
||||
}
|
||||
let flags = q.at("flags", default: ())
|
||||
if flags.len() > 0 {
|
||||
bits.push(stack(dir: ltr, spacing: 3pt, ..flags.map(flag-badge)))
|
||||
@@ -1117,6 +1187,20 @@
|
||||
let context_ = question-context(q)
|
||||
if context_ != none { parts.push(context_) }
|
||||
|
||||
// The question itself. A row of option shares says a distractor drew
|
||||
// 44% of the class; only the stem says whether that is a second
|
||||
// defensible reading. Instructor copy only, which is the only place this
|
||||
// section appears.
|
||||
let stem = q.at("stem", default: none)
|
||||
if stem != none {
|
||||
parts.push(block(
|
||||
width: 100%,
|
||||
fill: luma(252),
|
||||
stroke: (left: 2pt + luma(220)),
|
||||
inset: (left: 8pt, rest: 6pt),
|
||||
)[#text(size: size-small)[#markup(stem)]])
|
||||
}
|
||||
|
||||
for note in q.at("notes", default: ()) {
|
||||
parts.push(text(size: size-small)[— #markup(note)])
|
||||
}
|
||||
@@ -1141,19 +1225,50 @@
|
||||
}
|
||||
|
||||
if show-options and q.at("options", default: ()).len() > 0 {
|
||||
// The option column is two things: the bank letter every statistic is
|
||||
// keyed by, and the letter the option actually carried on each paper.
|
||||
// Without the second, a note about option D cannot be checked against
|
||||
// the copy a student brings to office hours, since shuffling gives the
|
||||
// same option a different letter on every form.
|
||||
//
|
||||
// The text goes last and takes the free column. It is the only cell
|
||||
// whose length is not bounded, so anywhere else it either squeezes the
|
||||
// numbers or wraps to three lines while they sit in a narrow gutter.
|
||||
// Last, the numbers keep their natural widths and the text runs to the
|
||||
// page edge.
|
||||
parts.push(table(
|
||||
columns: (auto, auto, auto, 2.6cm, auto),
|
||||
columns: (auto, auto, auto, 2.6cm, auto, 1fr),
|
||||
stroke: none,
|
||||
align: (center + horizon, right + horizon, right + horizon, left + horizon, right + horizon),
|
||||
align: (
|
||||
center + horizon,
|
||||
right + horizon,
|
||||
right + horizon,
|
||||
left + horizon,
|
||||
right + horizon,
|
||||
left + top,
|
||||
),
|
||||
inset: (x: 5pt, y: 3.5pt),
|
||||
table.header(th[OPT], th[n], th[SHARE], th[], th[r]),
|
||||
table.header(th[OPT], th[n], th[SHARE], th[], th[r], th[TEXT]),
|
||||
..q
|
||||
.options
|
||||
.map(option => (
|
||||
{
|
||||
if option.at("is-key", default: false) {
|
||||
let printed = option.at("printed", default: ())
|
||||
let letter = if option.at("is-key", default: false) {
|
||||
text(weight: "bold", fill: ok-color)[#option.letter]
|
||||
} else { [#option.letter] }
|
||||
if printed.len() == 0 {
|
||||
letter
|
||||
} else {
|
||||
stack(
|
||||
dir: ttb,
|
||||
spacing: step * 0.25,
|
||||
letter,
|
||||
text(size: size-micro, fill: luma(125))[
|
||||
#printed.map(p => p.form + ":" + p.letter).join(" ")
|
||||
],
|
||||
)
|
||||
}
|
||||
},
|
||||
text(size: size-small)[#option.count],
|
||||
text(size: size-small)[#pct(option.rate)],
|
||||
@@ -1172,6 +1287,16 @@
|
||||
text(size: size-small)[#signed(r)]
|
||||
}
|
||||
},
|
||||
{
|
||||
let body = option.at("text", default: none)
|
||||
if body == none {
|
||||
text(size: size-micro, fill: thin-color)[not in the bank]
|
||||
} else if option.at("is-key", default: false) {
|
||||
text(size: size-small, fill: ok-color.darken(25%))[#markup(body)]
|
||||
} else {
|
||||
text(size: size-small)[#markup(body)]
|
||||
}
|
||||
},
|
||||
))
|
||||
.flatten(),
|
||||
))
|
||||
|
||||
Reference in new issue
Block a user