feat: more comprehensive cohort report

src/analysis/diagnostic.rs

@@ -1020,6 +1020,13 @@ pub struct CohortDiagnostic {
     pub predictions: PredictionSummary,
     /// Questions worth revisiting before reuse, worst first.
     pub revise: Vec<CohortQuestionRow>,
+    /// The dropped items, described but not measured.
+    ///
+    /// Kept out of [`CohortDiagnostic::questions`] so that no statistic above
+    /// silently includes an item that was thrown out, and reported alongside it
+    /// in the evidence section so that dropping a question does not erase the
+    /// evidence for having dropped it.
+    pub dropped_detail: Vec<CohortQuestionRow>,
     /// One row per form, when more than one was given.
     #[serde(skip_serializing_if = "Vec::is_empty")]
     pub forms: Vec<FormRow>,
@@ -1138,6 +1145,26 @@ pub struct CohortQuestionRow {
     /// What those targets ask, in the course's own words.
     #[serde(skip_serializing_if = "Vec::is_empty")]
     pub target_texts: Vec<String>,
+    /// Whether the item was dropped from scoring.
+    ///
+    /// A dropped item carries no p, r, or D, and appears in none of the
+    /// statistics above. It still appears in the evidence section, because the
+    /// option spread that justified dropping it is the record of why, and that
+    /// record should survive re-running the report afterwards.
+    #[serde(skip_serializing_if = "std::ops::Not::not")]
+    pub dropped: bool,
+    /// Whether the drop was applied as full credit to everyone.
+    #[serde(skip_serializing_if = "std::ops::Not::not")]
+    pub dropped_full_credit: bool,
+    /// The question as written.
+    ///
+    /// The instructor report reads better with it than without: a row of option
+    /// shares says a distractor drew 44% of the class, and only the stem says
+    /// whether that is a second defensible reading. Absent when the item has
+    /// left the bank, and withheld from any report that is not the instructor
+    /// copy.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub stem: Option<String>,
     /// Where the item was taught, as lecture titles and slide numbers.
     #[serde(skip_serializing_if = "Vec::is_empty")]
     pub taught_in: Vec<String>,
@@ -1185,8 +1212,25 @@ pub struct CohortQuestionRow {
 #[derive(Debug, Clone, Serialize)]
 #[serde(rename_all = "kebab-case")]
 pub struct OptionRow {
-    /// The bank letter.
+    /// The bank letter, which is the one every statistic is keyed by.
     pub letter: String,
+    /// The option as written, for a report that shows the question.
+    ///
+    /// Absent when the item is no longer in the bank, which is why this is an
+    /// option rather than an empty string: a missing option and an empty one
+    /// are different facts.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub text: Option<String>,
+    /// What this option was lettered on each printed form, worst case one entry
+    /// per form.
+    ///
+    /// Shuffling means the bank's option C is a different letter on every form,
+    /// so a statistic reported against C cannot be checked against a student's
+    /// paper without this map. It is the first thing anyone needs when a student
+    /// brings a paper to office hours, and working it out by hand from a seal is
+    /// the kind of task that gets done wrong once and then trusted.
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    pub printed: Vec<PrintedLetter>,
     /// How many chose it.
     pub count: usize,
     /// The share who chose it.
@@ -1200,6 +1244,16 @@ pub struct OptionRow {
     pub nonfunctioning: bool,
 }
 
+/// What one bank option was lettered on one form.
+#[derive(Debug, Clone, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct PrintedLetter {
+    /// The form id.
+    pub form: String,
+    /// The letter this option carried on that form's paper.
+    pub letter: String,
+}
+
 /// One form's summary.
 #[derive(Debug, Clone, Serialize)]
 #[serde(rename_all = "kebab-case")]
@@ -1502,11 +1556,17 @@ pub fn cohort(
         })
         .collect();
 
+    // The printed lettering per form, computed from the same two functions the
+    // exporter and the seal use, so the letters here are the letters on the
+    // paper rather than a second guess at them.
+    let forms: Vec<&crate::assessment::Form> = record.forms.iter().collect();
+
     let questions: Vec<CohortQuestionRow> = analysis
         .items
         .iter()
         .map(|item| {
             let meta = item_meta.get(&item.number);
+            let entry = item.item_ref.as_deref().and_then(|uid| catalog.get(uid));
             CohortQuestionRow {
                 number: item.number,
                 item: item.item_ref.clone(),
@@ -1533,6 +1593,9 @@ pub fn cohort(
                             .collect()
                     })
                     .unwrap_or_default(),
+                dropped: false,
+                dropped_full_credit: false,
+                stem: entry.map(|e| e.item.stem.clone()),
                 difficulty_band: difficulty_band(item.p_value).to_string(),
                 discrimination_band: discrimination_band(item.point_biserial).to_string(),
                 p_value: item.p_value,
@@ -1544,6 +1607,23 @@ pub fn cohort(
                     .options
                     .values()
                     .map(|option| OptionRow {
+                        text: entry.and_then(|e| {
+                            e.item
+                                .options
+                                .iter()
+                                .find(|o| o.id == option.letter)
+                                .map(|o| o.text.clone())
+                        }),
+                        printed: entry
+                            .map(|e| {
+                                printed_letters(
+                                    &forms,
+                                    &e.item,
+                                    item.item_ref.as_deref(),
+                                    &option.letter,
+                                )
+                            })
+                            .unwrap_or_default(),
                         letter: option.letter.clone(),
                         count: option.count,
                         rate: option.rate,
@@ -1582,6 +1662,106 @@ pub fn cohort(
         .collect();
     dropped_questions.sort_by_key(|d| d.number);
 
+    // The dropped items, described from the responses rather than from the
+    // scoring. Dropping an item overrides its credit, so p, r, and D are
+    // meaningless for it and are left out. What students actually marked is
+    // untouched by the drop, and that spread is the evidence that justified it.
+    let mut dropped_detail: Vec<CohortQuestionRow> = dropped_questions
+        .iter()
+        .map(|dropped| {
+            let placement = record.placement(dropped.number);
+            let uid = placement.map(|p| p.item.clone());
+            let entry = uid.as_deref().and_then(|uid| catalog.get(uid));
+            let responses = set.for_item(dropped.number);
+            let answered = responses
+                .iter()
+                .filter(|r| !r.chosen().is_empty())
+                .count()
+                .max(1);
+            let keyed: BTreeSet<String> = placement
+                .map(|p| p.key.iter().cloned().collect())
+                .filter(|k: &BTreeSet<String>| !k.is_empty())
+                .or_else(|| entry.map(|e| e.item.key_letters().into_iter().collect()))
+                .unwrap_or_default();
+
+            // One row per option the item declares, so an option nobody
+            // marked still shows as unchosen rather than vanishing.
+            let options: Vec<OptionRow> = entry
+                .map(|e| {
+                    e.item
+                        .options
+                        .iter()
+                        .map(|option| {
+                            let count = responses
+                                .iter()
+                                .filter(|r| r.chosen().iter().any(|l| *l == option.id))
+                                .count();
+                            OptionRow {
+                                text: Some(option.text.clone()),
+                                printed: printed_letters(
+                                    &forms,
+                                    &e.item,
+                                    uid.as_deref(),
+                                    &option.id,
+                                ),
+                                letter: option.id.clone(),
+                                count,
+                                rate: count as f64 / answered as f64,
+                                is_key: keyed.contains(&option.id),
+                                point_biserial: None,
+                                nonfunctioning: false,
+                            }
+                        })
+                        .collect()
+                })
+                .unwrap_or_default();
+
+            let targets = placement
+                .map(|p| p.learning_targets.clone())
+                .unwrap_or_default();
+            CohortQuestionRow {
+                number: dropped.number,
+                item: uid.clone(),
+                level: placement.and_then(|p| p.level).map(|l| l.code()),
+                target_texts: targets.iter().map(|id| course.text_for(id)).collect(),
+                targets,
+                dropped: true,
+                dropped_full_credit: dropped.full_credit,
+                stem: entry.map(|e| e.item.stem.clone()),
+                taught_in: uid
+                    .as_deref()
+                    .map(|uid| taught_in(catalog, uid))
+                    .unwrap_or_default(),
+                lectures: entry
+                    .map(|e| e.item.sources.iter().map(|s| s.lecture.clone()).collect())
+                    .unwrap_or_default(),
+                difficulty_band: String::new(),
+                discrimination_band: String::new(),
+                // The keyed share before the override, which is the closest
+                // honest reading of how the item performed. It is not a
+                // p-value: it counts marks, not credit.
+                p_value: options
+                    .iter()
+                    .filter(|o| o.is_key)
+                    .map(|o| o.rate)
+                    .sum::<f64>()
+                    .min(1.0),
+                point_biserial: None,
+                discrimination: None,
+                blank_rate: responses.iter().filter(|r| r.chosen().is_empty()).count() as f64
+                    / responses.len().max(1) as f64,
+                key: keyed.iter().cloned().collect(),
+                options,
+                flags: Vec::new(),
+                notes: Vec::new(),
+                prediction_notes: Vec::new(),
+                calibrated: false,
+                by_form: BTreeMap::new(),
+            }
+        })
+        .collect();
+    dropped_detail.sort_by_key(|q| q.number);
+
     let default_options = course.policy.options_per_item;
     let triage = triage(&questions, threshold, default_options);
     let predictions = prediction_summary(analysis);
@@ -1609,6 +1789,7 @@ pub fn cohort(
         gaps,
         questions,
         revise,
+        dropped_detail,
         forms: form_rows(set, cohort),
         blueprint: crate::select::check_blueprint(record, course),
         patterns: cohort
@@ -1669,6 +1850,53 @@ fn taught_in(catalog: &Catalog, uid: &str) -> Vec<String> {
         .collect()
 }
 
+/// Where one bank option landed on each printed form.
+///
+/// # Arguments
+///
+/// * `forms` - the record's forms, in declaration order.
+/// * `item` - the bank item, for its option count and lettering.
+/// * `uid` - the item's global id, which salts the permutation.
+/// * `letter` - the bank letter to locate.
+///
+/// # Returns
+///
+/// One entry per form that permutes its options. Forms printing the bank order
+/// unchanged are left out, since an entry saying C was printed as C is noise on
+/// every row.
+fn printed_letters(
+    forms: &[&crate::assessment::Form],
+    item: &crate::item::Item,
+    uid: Option<&str>,
+    letter: &str,
+) -> Vec<PrintedLetter> {
+    let Some(uid) = uid else {
+        return Vec::new();
+    };
+    let Some(source) = item.options.iter().position(|o| o.id == letter) else {
+        return Vec::new();
+    };
+    let n = item.options.len();
+    let mut out = Vec::new();
+    for form in forms {
+        if !form.shuffle_options {
+            continue;
+        }
+        // The same permutation the exporter and the seal use, so these are the
+        // letters on the paper rather than a second guess at them.
+        let order = crate::select::option_order(form, uid, n);
+        // `order[position] == source` means the option printed in that slot is
+        // the one being asked about.
+        if let Some(position) = order.iter().position(|index| *index == source) {
+            out.push(PrintedLetter {
+                form: form.id.clone(),
+                letter: crate::seal::printed_letter(position),
+            });
+        }
+    }
+    out
+}
+
 /// The difficulty band a p-value falls in.
 ///
 /// Three bands rather than five. The only distinction that changes what you do

src/data/decode.rs

@@ -651,8 +651,7 @@ pub fn apply(set: &mut ResponseSet, decoder: &FormDecoder, numbering: Numbering)
     if !colliding_positions.is_empty() {
         let list: Vec<String> = colliding_positions.iter().map(|n| n.to_string()).collect();
         let discarded = set.rows.len();
-        set.rows
-            .retain(|row| row.form.as_deref() != Some(UNMAPPED));
+        set.rows.retain(|row| row.form.as_deref() != Some(UNMAPPED));
         warnings.push(format!(
             "form {}: {} response(s) at position(s) {} could not be translated, and their raw \
              numbers are numbers this form does use. Keeping them would have given those \

src/export/typst/diagnostic.rs

@@ -620,6 +620,18 @@ pub fn cohort_value(diagnostic: &CohortDiagnostic, config: &RenderConfig) -> Val
                 .collect(),
         ),
     );
+    // Separate from `questions` so no statistic can pick them up, and merged
+    // back in by the evidence section, which describes rather than measures.
+    out.insert(
+        "dropped-detail",
+        Value::Array(
+            diagnostic
+                .dropped_detail
+                .iter()
+                .map(|q| cohort_question_value(q, content))
+                .collect(),
+        ),
+    );
 
     out.insert(
         "grades",
@@ -872,6 +884,18 @@ fn cohort_question_value(question: &CohortQuestionRow, content: bool) -> Value {
                 .collect(),
         ),
     );
+    value.insert_some(
+        "stem",
+        question
+            .stem
+            .as_ref()
+            .map(|text| markup_value(text, content)),
+    );
+    value.insert("dropped", Value::Bool(question.dropped));
+    value.insert(
+        "dropped-full-credit",
+        Value::Bool(question.dropped_full_credit),
+    );
     value.insert(
         "taught-in",
         Value::Array(question.taught_in.iter().map(Value::str).collect()),
@@ -902,6 +926,27 @@ fn cohort_question_value(question: &CohortQuestionRow, content: bool) -> Value {
                 .map(|option| {
                     let mut value = Value::dict();
                     value.insert("letter", Value::str(&option.letter));
+                    value.insert_some(
+                        "text",
+                        option.text.as_ref().map(|text| markup_value(text, content)),
+                    );
+                    // The letter on each paper, so a statistic reported against
+                    // the bank letter can be checked against a student's copy.
+                    value.insert(
+                        "printed",
+                        Value::Array(
+                            option
+                                .printed
+                                .iter()
+                                .map(|printed| {
+                                    let mut pair = Value::dict();
+                                    pair.insert("form", Value::str(&printed.form));
+                                    pair.insert("letter", Value::str(&printed.letter));
+                                    pair
+                                })
+                                .collect(),
+                        ),
+                    );
                     value.insert("count", Value::Int(option.count as i64));
                     value.insert("rate", Value::Float(option.rate));
                     value.insert("is-key", Value::Bool(option.is_key));

src/export/typst/templates/cohort-report.typ

@@ -121,11 +121,32 @@
       discrimination: 0.10,
       blank-rate: 0.0,
       key: ("B",),
+      stem: [A sample question stem, shown so the option shares can be read against what was asked.],
       options: (
-        (letter: "A", count: 9, rate: 0.375, is-key: false, point-biserial: 0.11, nonfunctioning: false),
-        (letter: "B", count: 10, rate: 0.417, is-key: true, point-biserial: 0.05, nonfunctioning: false),
-        (letter: "C", count: 5, rate: 0.208, is-key: false, point-biserial: -0.2, nonfunctioning: false),
-        (letter: "D", count: 0, rate: 0.0, is-key: false, nonfunctioning: true),
+        (
+          letter: "A",
+          text: [A sample distractor built on a real misconception.],
+          printed: ((form: "A", letter: "C"), (form: "B", letter: "A")),
+          count: 9, rate: 0.375, is-key: false, point-biserial: 0.11, nonfunctioning: false,
+        ),
+        (
+          letter: "B",
+          text: [A sample key.],
+          printed: ((form: "A", letter: "D"), (form: "B", letter: "C")),
+          count: 10, rate: 0.417, is-key: true, point-biserial: 0.05, nonfunctioning: false,
+        ),
+        (
+          letter: "C",
+          text: [A second sample distractor.],
+          printed: ((form: "A", letter: "A"), (form: "B", letter: "D")),
+          count: 5, rate: 0.208, is-key: false, point-biserial: -0.2, nonfunctioning: false,
+        ),
+        (
+          letter: "D",
+          text: [A distractor nobody chose.],
+          printed: ((form: "A", letter: "B"), (form: "B", letter: "B")),
+          count: 0, rate: 0.0, is-key: false, nonfunctioning: true,
+        ),
       ),
       flags: ("ambiguous",),
       notes: ([Distractor A drew as many strong students as the key.],),
@@ -168,6 +189,33 @@
     biggest-surprise: (number: 14, expected: 0.45, observed: 0.86),
   ),
   dropped-questions: (),
+  dropped-detail: (
+    (
+      number: 9,
+      item: "bank::q-sample-009",
+      level: 2,
+      targets: ("t-sample-gap",),
+      target-texts: ([A sample learning target.],),
+      dropped: true,
+      dropped-full-credit: true,
+      stem: [A sample question that was thrown out after the exam.],
+      taught-in: ("Entropy (L1.2)",),
+      lectures: ("L1.2",),
+      p: 0.18,
+      blank-rate: 0.0,
+      key: ("A",),
+      options: (
+        (letter: "A", text: [The keyed option, which few chose.], count: 4, rate: 0.18, is-key: true, nonfunctioning: false),
+        (letter: "B", text: [The option most students read as correct.], count: 15, rate: 0.68, is-key: false, nonfunctioning: false),
+        (letter: "C", text: [A third option.], count: 3, rate: 0.14, is-key: false, nonfunctioning: false),
+      ),
+      flags: (),
+      notes: (),
+      prediction-notes: (),
+      calibrated: false,
+      by-form: (:),
+    ),
+  ),
   revise: (),
   forms: (
     (id: "A", students: 12, mean: 73.5, sd: 10.2),
@@ -1077,28 +1125,45 @@
 ]
 
 // ─────────────────────────────────────────────────────────────────────────────
-// The revise queue, with option tables
+// Every question, with option tables
 // ─────────────────────────────────────────────────────────────────────────────
 
-#let revise = cb-data.at("revise", default: ())
+// Every scored question, not only the flagged ones. A question that raised no
+// flag still has an option spread worth seeing: it is the reference for what a
+// healthy item looks like on this exam, and the place to check a query about a
+// question nothing was wrong with. The flagged ones are already called out by
+// name in the triage lists above, so nothing is lost by putting them back among
+// their neighbours here.
+// Dropped items come from their own list, since they carry no statistics and
+// belong in none of the tables above. They belong here: the option spread is
+// the evidence that justified dropping them, and re-running the report after a
+// drop should not erase it.
+#let evidence = questions + cb-data.at("dropped-detail", default: ())
 
-#if revise.len() > 0 [
+#if evidence.len() > 0 [
   #pagebreak(weak: true)
   = The evidence, question by question
 
   #explain[
-    The full breakdown for every flagged question: what the flags mean, and who
-    chose what. In question order, so you can find one by its number; the lists
-    above are the same questions sorted by what they ask of you. The r column
-    beside each option is the correlation between choosing that option and
-    scoring well on the rest of the exam, which is how a defensible distractor
-    announces itself.
+    The full breakdown for every question, flagged or not, including the ones
+    you dropped: what it asked, who chose what, and what any flags mean. A
+    dropped question is marked as such and carries no p, r, or D, because
+    dropping it overrode its credit. What students marked is untouched by the
+    drop, so its option spread still stands as the record of why it went. In question order, so you can find
+    one by its number; the lists above are the subset that needs a decision from
+    you, sorted by what they ask of you. The r column beside each option is the
+    correlation between choosing that option and scoring well on the rest of the
+    exam, which is how a defensible distractor announces itself.
+
+    Each option shows the bank letter first, then what it was lettered on each
+    form (#raw("A:D") means the bank's option was printed as D on form A), so a
+    row here can be read against the paper a student is holding.
   ]
 
   // Question order here, not triage order. The lists above are for deciding
   // what to do; this section is for looking one question up while you do it,
   // and a reader with a number in hand should not have to scan every entry.
-  #for q in revise.sorted(key: q => q.number) [
+  #for q in evidence.sorted(key: q => q.number) [
     #block(breakable: false, above: entry-gap, width: 100%)[
       #{
         let parts = (
@@ -1106,6 +1171,11 @@
             let bits = (text(weight: "bold", fill: accent)[Question #q.number],)
             let item = q.at("item", default: none)
             if item != none { bits.push(text(size: size-meta, fill: luma(120))[#item]) }
+            if q.at("dropped", default: false) {
+              bits.push(text(size: size-micro, weight: "bold", fill: luma(110))[
+                DROPPED#if q.at("dropped-full-credit", default: false) [ · FULL CREDIT]
+              ])
+            }
             let flags = q.at("flags", default: ())
             if flags.len() > 0 {
               bits.push(stack(dir: ltr, spacing: 3pt, ..flags.map(flag-badge)))
@@ -1117,6 +1187,20 @@
         let context_ = question-context(q)
         if context_ != none { parts.push(context_) }
 
+        // The question itself. A row of option shares says a distractor drew
+        // 44% of the class; only the stem says whether that is a second
+        // defensible reading. Instructor copy only, which is the only place this
+        // section appears.
+        let stem = q.at("stem", default: none)
+        if stem != none {
+          parts.push(block(
+            width: 100%,
+            fill: luma(252),
+            stroke: (left: 2pt + luma(220)),
+            inset: (left: 8pt, rest: 6pt),
+          )[#text(size: size-small)[#markup(stem)]])
+        }
+
         for note in q.at("notes", default: ()) {
           parts.push(text(size: size-small)[— #markup(note)])
         }
@@ -1141,19 +1225,50 @@
         }
 
         if show-options and q.at("options", default: ()).len() > 0 {
+          // The option column is two things: the bank letter every statistic is
+          // keyed by, and the letter the option actually carried on each paper.
+          // Without the second, a note about option D cannot be checked against
+          // the copy a student brings to office hours, since shuffling gives the
+          // same option a different letter on every form.
+          //
+          // The text goes last and takes the free column. It is the only cell
+          // whose length is not bounded, so anywhere else it either squeezes the
+          // numbers or wraps to three lines while they sit in a narrow gutter.
+          // Last, the numbers keep their natural widths and the text runs to the
+          // page edge.
           parts.push(table(
-            columns: (auto, auto, auto, 2.6cm, auto),
+            columns: (auto, auto, auto, 2.6cm, auto, 1fr),
             stroke: none,
-            align: (center + horizon, right + horizon, right + horizon, left + horizon, right + horizon),
+            align: (
+              center + horizon,
+              right + horizon,
+              right + horizon,
+              left + horizon,
+              right + horizon,
+              left + top,
+            ),
             inset: (x: 5pt, y: 3.5pt),
-            table.header(th[OPT], th[n], th[SHARE], th[], th[r]),
+            table.header(th[OPT], th[n], th[SHARE], th[], th[r], th[TEXT]),
             ..q
               .options
               .map(option => (
                 {
-                  if option.at("is-key", default: false) {
+                  let printed = option.at("printed", default: ())
+                  let letter = if option.at("is-key", default: false) {
                     text(weight: "bold", fill: ok-color)[#option.letter]
                   } else { [#option.letter] }
+                  if printed.len() == 0 {
+                    letter
+                  } else {
+                    stack(
+                      dir: ttb,
+                      spacing: step * 0.25,
+                      letter,
+                      text(size: size-micro, fill: luma(125))[
+                        #printed.map(p => p.form + ":" + p.letter).join(" ")
+                      ],
+                    )
+                  }
                 },
                 text(size: size-small)[#option.count],
                 text(size: size-small)[#pct(option.rate)],
@@ -1172,6 +1287,16 @@
                     text(size: size-small)[#signed(r)]
                   }
                 },
+                {
+                  let body = option.at("text", default: none)
+                  if body == none {
+                    text(size: size-micro, fill: thin-color)[not in the bank]
+                  } else if option.at("is-key", default: false) {
+                    text(size: size-small, fill: ok-color.darken(25%))[#markup(body)]
+                  } else {
+                    text(size: size-small)[#markup(body)]
+                  }
+                },
               ))
               .flatten(),
           ))