feat: more comprehensive cohort report

src/analysis/diagnostic.rs

@@ -1020,6 +1020,13 @@ pub struct CohortDiagnostic {
     pub predictions: PredictionSummary,
     /// Questions worth revisiting before reuse, worst first.
     pub revise: Vec<CohortQuestionRow>,
+    /// The dropped items, described but not measured.
+    ///
+    /// Kept out of [`CohortDiagnostic::questions`] so that no statistic above
+    /// silently includes an item that was thrown out, and reported alongside it
+    /// in the evidence section so that dropping a question does not erase the
+    /// evidence for having dropped it.
+    pub dropped_detail: Vec<CohortQuestionRow>,
     /// One row per form, when more than one was given.
     #[serde(skip_serializing_if = "Vec::is_empty")]
     pub forms: Vec<FormRow>,
@@ -1138,6 +1145,26 @@ pub struct CohortQuestionRow {
     /// What those targets ask, in the course's own words.
     #[serde(skip_serializing_if = "Vec::is_empty")]
     pub target_texts: Vec<String>,
+    /// Whether the item was dropped from scoring.
+    ///
+    /// A dropped item carries no p, r, or D, and appears in none of the
+    /// statistics above. It still appears in the evidence section, because the
+    /// option spread that justified dropping it is the record of why, and that
+    /// record should survive re-running the report afterwards.
+    #[serde(skip_serializing_if = "std::ops::Not::not")]
+    pub dropped: bool,
+    /// Whether the drop was applied as full credit to everyone.
+    #[serde(skip_serializing_if = "std::ops::Not::not")]
+    pub dropped_full_credit: bool,
+    /// The question as written.
+    ///
+    /// The instructor report reads better with it than without: a row of option
+    /// shares says a distractor drew 44% of the class, and only the stem says
+    /// whether that is a second defensible reading. Absent when the item has
+    /// left the bank, and withheld from any report that is not the instructor
+    /// copy.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub stem: Option<String>,
     /// Where the item was taught, as lecture titles and slide numbers.
     #[serde(skip_serializing_if = "Vec::is_empty")]
     pub taught_in: Vec<String>,
@@ -1185,8 +1212,25 @@ pub struct CohortQuestionRow {
 #[derive(Debug, Clone, Serialize)]
 #[serde(rename_all = "kebab-case")]
 pub struct OptionRow {
-    /// The bank letter.
+    /// The bank letter, which is the one every statistic is keyed by.
     pub letter: String,
+    /// The option as written, for a report that shows the question.
+    ///
+    /// Absent when the item is no longer in the bank, which is why this is an
+    /// option rather than an empty string: a missing option and an empty one
+    /// are different facts.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub text: Option<String>,
+    /// What this option was lettered on each printed form, worst case one entry
+    /// per form.
+    ///
+    /// Shuffling means the bank's option C is a different letter on every form,
+    /// so a statistic reported against C cannot be checked against a student's
+    /// paper without this map. It is the first thing anyone needs when a student
+    /// brings a paper to office hours, and working it out by hand from a seal is
+    /// the kind of task that gets done wrong once and then trusted.
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    pub printed: Vec<PrintedLetter>,
     /// How many chose it.
     pub count: usize,
     /// The share who chose it.
@@ -1200,6 +1244,16 @@ pub struct OptionRow {
     pub nonfunctioning: bool,
 }
 
+/// What one bank option was lettered on one form.
+#[derive(Debug, Clone, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct PrintedLetter {
+    /// The form id.
+    pub form: String,
+    /// The letter this option carried on that form's paper.
+    pub letter: String,
+}
+
 /// One form's summary.
 #[derive(Debug, Clone, Serialize)]
 #[serde(rename_all = "kebab-case")]
@@ -1502,11 +1556,17 @@ pub fn cohort(
         })
         .collect();
 
+    // The printed lettering per form, computed from the same two functions the
+    // exporter and the seal use, so the letters here are the letters on the
+    // paper rather than a second guess at them.
+    let forms: Vec<&crate::assessment::Form> = record.forms.iter().collect();
+
     let questions: Vec<CohortQuestionRow> = analysis
         .items
         .iter()
         .map(|item| {
             let meta = item_meta.get(&item.number);
+            let entry = item.item_ref.as_deref().and_then(|uid| catalog.get(uid));
             CohortQuestionRow {
                 number: item.number,
                 item: item.item_ref.clone(),
@@ -1533,6 +1593,9 @@ pub fn cohort(
                             .collect()
                     })
                     .unwrap_or_default(),
+                dropped: false,
+                dropped_full_credit: false,
+                stem: entry.map(|e| e.item.stem.clone()),
                 difficulty_band: difficulty_band(item.p_value).to_string(),
                 discrimination_band: discrimination_band(item.point_biserial).to_string(),
                 p_value: item.p_value,
@@ -1544,6 +1607,23 @@ pub fn cohort(
                     .options
                     .values()
                     .map(|option| OptionRow {
+                        text: entry.and_then(|e| {
+                            e.item
+                                .options
+                                .iter()
+                                .find(|o| o.id == option.letter)
+                                .map(|o| o.text.clone())
+                        }),
+                        printed: entry
+                            .map(|e| {
+                                printed_letters(
+                                    &forms,
+                                    &e.item,
+                                    item.item_ref.as_deref(),
+                                    &option.letter,
+                                )
+                            })
+                            .unwrap_or_default(),
                         letter: option.letter.clone(),
                         count: option.count,
                         rate: option.rate,
@@ -1582,6 +1662,106 @@ pub fn cohort(
         .collect();
     dropped_questions.sort_by_key(|d| d.number);
 
+    // The dropped items, described from the responses rather than from the
+    // scoring. Dropping an item overrides its credit, so p, r, and D are
+    // meaningless for it and are left out. What students actually marked is
+    // untouched by the drop, and that spread is the evidence that justified it.
+    let mut dropped_detail: Vec<CohortQuestionRow> = dropped_questions
+        .iter()
+        .map(|dropped| {
+            let placement = record.placement(dropped.number);
+            let uid = placement.map(|p| p.item.clone());
+            let entry = uid.as_deref().and_then(|uid| catalog.get(uid));
+            let responses = set.for_item(dropped.number);
+            let answered = responses
+                .iter()
+                .filter(|r| !r.chosen().is_empty())
+                .count()
+                .max(1);
+            let keyed: BTreeSet<String> = placement
+                .map(|p| p.key.iter().cloned().collect())
+                .filter(|k: &BTreeSet<String>| !k.is_empty())
+                .or_else(|| entry.map(|e| e.item.key_letters().into_iter().collect()))
+                .unwrap_or_default();
+
+            // One row per option the item declares, so an option nobody
+            // marked still shows as unchosen rather than vanishing.
+            let options: Vec<OptionRow> = entry
+                .map(|e| {
+                    e.item
+                        .options
+                        .iter()
+                        .map(|option| {
+                            let count = responses
+                                .iter()
+                                .filter(|r| r.chosen().iter().any(|l| *l == option.id))
+                                .count();
+                            OptionRow {
+                                text: Some(option.text.clone()),
+                                printed: printed_letters(
+                                    &forms,
+                                    &e.item,
+                                    uid.as_deref(),
+                                    &option.id,
+                                ),
+                                letter: option.id.clone(),
+                                count,
+                                rate: count as f64 / answered as f64,
+                                is_key: keyed.contains(&option.id),
+                                point_biserial: None,
+                                nonfunctioning: false,
+                            }
+                        })
+                        .collect()
+                })
+                .unwrap_or_default();
+
+            let targets = placement
+                .map(|p| p.learning_targets.clone())
+                .unwrap_or_default();
+            CohortQuestionRow {
+                number: dropped.number,
+                item: uid.clone(),
+                level: placement.and_then(|p| p.level).map(|l| l.code()),
+                target_texts: targets.iter().map(|id| course.text_for(id)).collect(),
+                targets,
+                dropped: true,
+                dropped_full_credit: dropped.full_credit,
+                stem: entry.map(|e| e.item.stem.clone()),
+                taught_in: uid
+                    .as_deref()
+                    .map(|uid| taught_in(catalog, uid))
+                    .unwrap_or_default(),
+                lectures: entry
+                    .map(|e| e.item.sources.iter().map(|s| s.lecture.clone()).collect())
+                    .unwrap_or_default(),
+                difficulty_band: String::new(),
+                discrimination_band: String::new(),
+                // The keyed share before the override, which is the closest
+                // honest reading of how the item performed. It is not a
+                // p-value: it counts marks, not credit.
+                p_value: options
+                    .iter()
+                    .filter(|o| o.is_key)
+                    .map(|o| o.rate)
+                    .sum::<f64>()
+                    .min(1.0),
+                point_biserial: None,
+                discrimination: None,
+                blank_rate: responses.iter().filter(|r| r.chosen().is_empty()).count() as f64
+                    / responses.len().max(1) as f64,
+                key: keyed.iter().cloned().collect(),
+                options,
+                flags: Vec::new(),
+                notes: Vec::new(),
+                prediction_notes: Vec::new(),
+                calibrated: false,
+                by_form: BTreeMap::new(),
+            }
+        })
+        .collect();
+    dropped_detail.sort_by_key(|q| q.number);
+
     let default_options = course.policy.options_per_item;
     let triage = triage(&questions, threshold, default_options);
     let predictions = prediction_summary(analysis);
@@ -1609,6 +1789,7 @@ pub fn cohort(
         gaps,
         questions,
         revise,
+        dropped_detail,
         forms: form_rows(set, cohort),
         blueprint: crate::select::check_blueprint(record, course),
         patterns: cohort
@@ -1669,6 +1850,53 @@ fn taught_in(catalog: &Catalog, uid: &str) -> Vec<String> {
         .collect()
 }
 
+/// Where one bank option landed on each printed form.
+///
+/// # Arguments
+///
+/// * `forms` - the record's forms, in declaration order.
+/// * `item` - the bank item, for its option count and lettering.
+/// * `uid` - the item's global id, which salts the permutation.
+/// * `letter` - the bank letter to locate.
+///
+/// # Returns
+///
+/// One entry per form that permutes its options. Forms printing the bank order
+/// unchanged are left out, since an entry saying C was printed as C is noise on
+/// every row.
+fn printed_letters(
+    forms: &[&crate::assessment::Form],
+    item: &crate::item::Item,
+    uid: Option<&str>,
+    letter: &str,
+) -> Vec<PrintedLetter> {
+    let Some(uid) = uid else {
+        return Vec::new();
+    };
+    let Some(source) = item.options.iter().position(|o| o.id == letter) else {
+        return Vec::new();
+    };
+    let n = item.options.len();
+    let mut out = Vec::new();
+    for form in forms {
+        if !form.shuffle_options {
+            continue;
+        }
+        // The same permutation the exporter and the seal use, so these are the
+        // letters on the paper rather than a second guess at them.
+        let order = crate::select::option_order(form, uid, n);
+        // `order[position] == source` means the option printed in that slot is
+        // the one being asked about.
+        if let Some(position) = order.iter().position(|index| *index == source) {
+            out.push(PrintedLetter {
+                form: form.id.clone(),
+                letter: crate::seal::printed_letter(position),
+            });
+        }
+    }
+    out
+}
+
 /// The difficulty band a p-value falls in.
 ///
 /// Three bands rather than five. The only distinction that changes what you do

src/authoring/select.rs

@@ -479,6 +479,7 @@ pub fn to_record(
             credit_overrides: BTreeMap::new(),
             dropped: false,
             dropped_as: None,
+            dropped_before_printing: false,
         });
     }
 

src/data/decode.rs

@@ -81,6 +81,11 @@ pub enum Numbering {
     Recorded,
 }
 
+/// The form marker put on a row that could not be translated, so it can be
+/// removed after the borrow on `set.rows` ends. No real form id can collide
+/// with it: form ids come from the record and are short labels like `A`.
+const UNMAPPED: &str = "\u{1f}unmapped";
+
 /// One question's mapping on one form.
 #[derive(Debug, Clone)]
 pub struct QuestionMap {
@@ -354,6 +359,11 @@ impl FormDecoder {
         self.by_position.len()
     }
 
+    /// Every recorded question number this form carries.
+    pub fn numbers(&self) -> impl Iterator<Item = u32> + '_ {
+        self.by_position.values().map(|q| q.number)
+    }
+
     /// Whether the form prints nothing, which means the record is empty.
     pub fn is_empty(&self) -> bool {
         self.by_position.is_empty()
@@ -572,9 +582,17 @@ pub fn form_warning(claimed: &str, fits: &[FormFit]) -> Option<String> {
 pub fn apply(set: &mut ResponseSet, decoder: &FormDecoder, numbering: Numbering) -> Vec<String> {
     let mut warnings = Vec::new();
     let mut unmapped_positions: BTreeSet<u32> = BTreeSet::new();
+    let mut colliding_positions: BTreeSet<u32> = BTreeSet::new();
     let mut unmapped_letters: BTreeSet<String> = BTreeSet::new();
     let mut translated = 0usize;
 
+    // Numbers this form really uses. An untranslated row whose raw number is one
+    // of these would silently masquerade as that question, and two rows would
+    // then share a number: one the student's answer to it, one an answer to
+    // something else entirely. Nothing downstream can tell them apart, so the
+    // collision has to be caught here.
+    let recorded: BTreeSet<u32> = decoder.numbers().collect();
+
     for row in &mut set.rows {
         let belongs = row
             .form
@@ -587,6 +605,13 @@ pub fn apply(set: &mut ResponseSet, decoder: &FormDecoder, numbering: Numbering)
 
         let Some(map) = decoder.lookup(row.item_number, numbering) else {
             unmapped_positions.insert(row.item_number);
+            if recorded.contains(&row.item_number) {
+                colliding_positions.insert(row.item_number);
+                // Marked so the row can be discarded below. Attributing it to
+                // the question that legitimately holds this number would corrupt
+                // that question's statistics.
+                row.form = Some(UNMAPPED.to_string());
+            }
             continue;
         };
 
@@ -615,9 +640,27 @@ pub fn apply(set: &mut ResponseSet, decoder: &FormDecoder, numbering: Numbering)
     if !unmapped_positions.is_empty() {
         let list: Vec<String> = unmapped_positions.iter().map(|n| n.to_string()).collect();
         warnings.push(format!(
-            "form {}: question(s) {} are in the export but not on this form; they were left \
+            "form {}: question(s) {} are in the export but not on this form ({} printed). The \
-             untranslated",
+             export may have been taken before a question was dropped, or from a different \
+             form.",
             decoder.form,
+            list.join(", "),
+            decoder.len()
+        ));
+    }
+    if !colliding_positions.is_empty() {
+        let list: Vec<String> = colliding_positions.iter().map(|n| n.to_string()).collect();
+        let discarded = set.rows.len();
+        set.rows.retain(|row| row.form.as_deref() != Some(UNMAPPED));
+        warnings.push(format!(
+            "form {}: {} response(s) at position(s) {} could not be translated, and their raw \
+             numbers are numbers this form does use. Keeping them would have given those \
+             questions two different answers each, so they were discarded. This is the shape of \
+             an export made before a question was dropped: re-export the responses from the \
+             administration you sealed, or re-run with --recorded-numbers if the export already \
+             carries recorded numbers.",
+            decoder.form,
+            discarded - set.rows.len(),
             list.join(", ")
         ));
     }

src/export/practice.rs

@@ -223,7 +223,7 @@ pub fn render(catalog: &Catalog, record: &AssessmentFile, opts: &Options) -> Res
     let mut printed_stimulus: Option<String> = None;
 
     let layout = select::layout(record, &opts.form);
-    for (position, placement) in layout.iter().filter(|p| !p.dropped).enumerate() {
+    for (position, placement) in layout.iter().filter(|p| p.was_printed()).enumerate() {
         let entry = catalog.require(&placement.item)?;
         let item = &entry.item;
         let number = position + 1;
@@ -602,6 +602,7 @@ items:
                     credit_overrides: Default::default(),
                     dropped: false,
                     dropped_as: None,
+                    dropped_before_printing: false,
                 },
                 Placement {
                     number: 2,
@@ -616,6 +617,7 @@ items:
                     credit_overrides: Default::default(),
                     dropped: false,
                     dropped_as: None,
+                    dropped_before_printing: false,
                 },
             ],
         }

src/export/qti.rs

@@ -1398,6 +1398,7 @@ items:
                     credit_overrides: Default::default(),
                     dropped: false,
                     dropped_as: None,
+                    dropped_before_printing: false,
                 },
                 Placement {
                     number: 2,
@@ -1412,6 +1413,7 @@ items:
                     credit_overrides: Default::default(),
                     dropped: false,
                     dropped_as: None,
+                    dropped_before_printing: false,
                 },
             ],
         }

src/export/site.rs

@@ -867,6 +867,7 @@ items:
                     credit_overrides: Default::default(),
                     dropped: false,
                     dropped_as: None,
+                    dropped_before_printing: false,
                 },
                 Placement {
                     number: 2,
@@ -881,6 +882,7 @@ items:
                     credit_overrides: Default::default(),
                     dropped: false,
                     dropped_as: None,
+                    dropped_before_printing: false,
                 },
             ],
         }

src/export/typst.rs

@@ -440,6 +440,7 @@ mod tests {
                     credit_overrides: Default::default(),
                     dropped: true,
                     dropped_as: None,
+                    dropped_before_printing: false,
                 },
                 Placement {
                     number: 2,
@@ -454,12 +455,13 @@ mod tests {
                     credit_overrides: Default::default(),
                     dropped: false,
                     dropped_as: None,
+                    dropped_before_printing: false,
                 },
             ],
         };
         let printable: Vec<u32> = select::layout(&record, &Options::default().form)
             .into_iter()
-            .filter(|p| !p.dropped)
+            .filter(|p| p.was_printed())
             .map(|p| p.number)
             .collect();
         assert_eq!(printable, vec![2]);

src/export/typst/diagnostic.rs

@@ -620,6 +620,18 @@ pub fn cohort_value(diagnostic: &CohortDiagnostic, config: &RenderConfig) -> Val
                 .collect(),
         ),
     );
+    // Separate from `questions` so no statistic can pick them up, and merged
+    // back in by the evidence section, which describes rather than measures.
+    out.insert(
+        "dropped-detail",
+        Value::Array(
+            diagnostic
+                .dropped_detail
+                .iter()
+                .map(|q| cohort_question_value(q, content))
+                .collect(),
+        ),
+    );
 
     out.insert(
         "grades",
@@ -872,6 +884,18 @@ fn cohort_question_value(question: &CohortQuestionRow, content: bool) -> Value {
                 .collect(),
         ),
     );
+    value.insert_some(
+        "stem",
+        question
+            .stem
+            .as_ref()
+            .map(|text| markup_value(text, content)),
+    );
+    value.insert("dropped", Value::Bool(question.dropped));
+    value.insert(
+        "dropped-full-credit",
+        Value::Bool(question.dropped_full_credit),
+    );
     value.insert(
         "taught-in",
         Value::Array(question.taught_in.iter().map(Value::str).collect()),
@@ -902,6 +926,27 @@ fn cohort_question_value(question: &CohortQuestionRow, content: bool) -> Value {
                 .map(|option| {
                     let mut value = Value::dict();
                     value.insert("letter", Value::str(&option.letter));
+                    value.insert_some(
+                        "text",
+                        option.text.as_ref().map(|text| markup_value(text, content)),
+                    );
+                    // The letter on each paper, so a statistic reported against
+                    // the bank letter can be checked against a student's copy.
+                    value.insert(
+                        "printed",
+                        Value::Array(
+                            option
+                                .printed
+                                .iter()
+                                .map(|printed| {
+                                    let mut pair = Value::dict();
+                                    pair.insert("form", Value::str(&printed.form));
+                                    pair.insert("letter", Value::str(&printed.letter));
+                                    pair
+                                })
+                                .collect(),
+                        ),
+                    );
                     value.insert("count", Value::Int(option.count as i64));
                     value.insert("rate", Value::Float(option.rate));
                     value.insert("is-key", Value::Bool(option.is_key));

src/export/typst/templates/cohort-report.typ

@@ -121,11 +121,32 @@
       discrimination: 0.10,
       blank-rate: 0.0,
       key: ("B",),
+      stem: [A sample question stem, shown so the option shares can be read against what was asked.],
       options: (
-        (letter: "A", count: 9, rate: 0.375, is-key: false, point-biserial: 0.11, nonfunctioning: false),
+        (
-        (letter: "B", count: 10, rate: 0.417, is-key: true, point-biserial: 0.05, nonfunctioning: false),
+          letter: "A",
-        (letter: "C", count: 5, rate: 0.208, is-key: false, point-biserial: -0.2, nonfunctioning: false),
+          text: [A sample distractor built on a real misconception.],
-        (letter: "D", count: 0, rate: 0.0, is-key: false, nonfunctioning: true),
+          printed: ((form: "A", letter: "C"), (form: "B", letter: "A")),
+          count: 9, rate: 0.375, is-key: false, point-biserial: 0.11, nonfunctioning: false,
+        ),
+        (
+          letter: "B",
+          text: [A sample key.],
+          printed: ((form: "A", letter: "D"), (form: "B", letter: "C")),
+          count: 10, rate: 0.417, is-key: true, point-biserial: 0.05, nonfunctioning: false,
+        ),
+        (
+          letter: "C",
+          text: [A second sample distractor.],
+          printed: ((form: "A", letter: "A"), (form: "B", letter: "D")),
+          count: 5, rate: 0.208, is-key: false, point-biserial: -0.2, nonfunctioning: false,
+        ),
+        (
+          letter: "D",
+          text: [A distractor nobody chose.],
+          printed: ((form: "A", letter: "B"), (form: "B", letter: "B")),
+          count: 0, rate: 0.0, is-key: false, nonfunctioning: true,
+        ),
       ),
       flags: ("ambiguous",),
       notes: ([Distractor A drew as many strong students as the key.],),
@@ -168,6 +189,33 @@
     biggest-surprise: (number: 14, expected: 0.45, observed: 0.86),
   ),
   dropped-questions: (),
+  dropped-detail: (
+    (
+      number: 9,
+      item: "bank::q-sample-009",
+      level: 2,
+      targets: ("t-sample-gap",),
+      target-texts: ([A sample learning target.],),
+      dropped: true,
+      dropped-full-credit: true,
+      stem: [A sample question that was thrown out after the exam.],
+      taught-in: ("Entropy (L1.2)",),
+      lectures: ("L1.2",),
+      p: 0.18,
+      blank-rate: 0.0,
+      key: ("A",),
+      options: (
+        (letter: "A", text: [The keyed option, which few chose.], count: 4, rate: 0.18, is-key: true, nonfunctioning: false),
+        (letter: "B", text: [The option most students read as correct.], count: 15, rate: 0.68, is-key: false, nonfunctioning: false),
+        (letter: "C", text: [A third option.], count: 3, rate: 0.14, is-key: false, nonfunctioning: false),
+      ),
+      flags: (),
+      notes: (),
+      prediction-notes: (),
+      calibrated: false,
+      by-form: (:),
+    ),
+  ),
   revise: (),
   forms: (
     (id: "A", students: 12, mean: 73.5, sd: 10.2),
@@ -1077,23 +1125,45 @@
 ]
 
 // ─────────────────────────────────────────────────────────────────────────────
-// The revise queue, with option tables
+// Every question, with option tables
 // ─────────────────────────────────────────────────────────────────────────────
 
-#let revise = cb-data.at("revise", default: ())
+// Every scored question, not only the flagged ones. A question that raised no
+// flag still has an option spread worth seeing: it is the reference for what a
+// healthy item looks like on this exam, and the place to check a query about a
+// question nothing was wrong with. The flagged ones are already called out by
+// name in the triage lists above, so nothing is lost by putting them back among
+// their neighbours here.
+// Dropped items come from their own list, since they carry no statistics and
+// belong in none of the tables above. They belong here: the option spread is
+// the evidence that justified dropping them, and re-running the report after a
+// drop should not erase it.
+#let evidence = questions + cb-data.at("dropped-detail", default: ())
 
-#if revise.len() > 0 [
+#if evidence.len() > 0 [
   #pagebreak(weak: true)
   = The evidence, question by question
 
   #explain[
-    The full breakdown for every flagged question: what the flags mean, and who
+    The full breakdown for every question, flagged or not, including the ones
-    chose what. The r column beside each option is the correlation between
+    you dropped: what it asked, who chose what, and what any flags mean. A
-    choosing that option and scoring well on the rest of the exam, which is how a
+    dropped question is marked as such and carries no p, r, or D, because
-    defensible distractor announces itself.
+    dropping it overrode its credit. What students marked is untouched by the
+    drop, so its option spread still stands as the record of why it went. In question order, so you can find
+    one by its number; the lists above are the subset that needs a decision from
+    you, sorted by what they ask of you. The r column beside each option is the
+    correlation between choosing that option and scoring well on the rest of the
+    exam, which is how a defensible distractor announces itself.
+
+    Each option shows the bank letter first, then what it was lettered on each
+    form (#raw("A:D") means the bank's option was printed as D on form A), so a
+    row here can be read against the paper a student is holding.
   ]
 
-  #for q in revise [
+  // Question order here, not triage order. The lists above are for deciding
+  // what to do; this section is for looking one question up while you do it,
+  // and a reader with a number in hand should not have to scan every entry.
+  #for q in evidence.sorted(key: q => q.number) [
     #block(breakable: false, above: entry-gap, width: 100%)[
       #{
         let parts = (
@@ -1101,6 +1171,11 @@
             let bits = (text(weight: "bold", fill: accent)[Question #q.number],)
             let item = q.at("item", default: none)
             if item != none { bits.push(text(size: size-meta, fill: luma(120))[#item]) }
+            if q.at("dropped", default: false) {
+              bits.push(text(size: size-micro, weight: "bold", fill: luma(110))[
+                DROPPED#if q.at("dropped-full-credit", default: false) [ · FULL CREDIT]
+              ])
+            }
             let flags = q.at("flags", default: ())
             if flags.len() > 0 {
               bits.push(stack(dir: ltr, spacing: 3pt, ..flags.map(flag-badge)))
@@ -1112,6 +1187,20 @@
         let context_ = question-context(q)
         if context_ != none { parts.push(context_) }
 
+        // The question itself. A row of option shares says a distractor drew
+        // 44% of the class; only the stem says whether that is a second
+        // defensible reading. Instructor copy only, which is the only place this
+        // section appears.
+        let stem = q.at("stem", default: none)
+        if stem != none {
+          parts.push(block(
+            width: 100%,
+            fill: luma(252),
+            stroke: (left: 2pt + luma(220)),
+            inset: (left: 8pt, rest: 6pt),
+          )[#text(size: size-small)[#markup(stem)]])
+        }
+
         for note in q.at("notes", default: ()) {
           parts.push(text(size: size-small)[— #markup(note)])
         }
@@ -1136,19 +1225,50 @@
         }
 
         if show-options and q.at("options", default: ()).len() > 0 {
+          // The option column is two things: the bank letter every statistic is
+          // keyed by, and the letter the option actually carried on each paper.
+          // Without the second, a note about option D cannot be checked against
+          // the copy a student brings to office hours, since shuffling gives the
+          // same option a different letter on every form.
+          //
+          // The text goes last and takes the free column. It is the only cell
+          // whose length is not bounded, so anywhere else it either squeezes the
+          // numbers or wraps to three lines while they sit in a narrow gutter.
+          // Last, the numbers keep their natural widths and the text runs to the
+          // page edge.
           parts.push(table(
-            columns: (auto, auto, auto, 2.6cm, auto),
+            columns: (auto, auto, auto, 2.6cm, auto, 1fr),
             stroke: none,
-            align: (center + horizon, right + horizon, right + horizon, left + horizon, right + horizon),
+            align: (
+              center + horizon,
+              right + horizon,
+              right + horizon,
+              left + horizon,
+              right + horizon,
+              left + top,
+            ),
             inset: (x: 5pt, y: 3.5pt),
-            table.header(th[OPT], th[n], th[SHARE], th[], th[r]),
+            table.header(th[OPT], th[n], th[SHARE], th[], th[r], th[TEXT]),
             ..q
               .options
               .map(option => (
                 {
-                  if option.at("is-key", default: false) {
+                  let printed = option.at("printed", default: ())
+                  let letter = if option.at("is-key", default: false) {
                     text(weight: "bold", fill: ok-color)[#option.letter]
                   } else { [#option.letter] }
+                  if printed.len() == 0 {
+                    letter
+                  } else {
+                    stack(
+                      dir: ttb,
+                      spacing: step * 0.25,
+                      letter,
+                      text(size: size-micro, fill: luma(125))[
+                        #printed.map(p => p.form + ":" + p.letter).join(" ")
+                      ],
+                    )
+                  }
                 },
                 text(size: size-small)[#option.count],
                 text(size: size-small)[#pct(option.rate)],
@@ -1167,6 +1287,16 @@
                     text(size: size-small)[#signed(r)]
                   }
                 },
+                {
+                  let body = option.at("text", default: none)
+                  if body == none {
+                    text(size: size-micro, fill: thin-color)[not in the bank]
+                  } else if option.at("is-key", default: false) {
+                    text(size: size-small, fill: ok-color.darken(25%))[#markup(body)]
+                  } else {
+                    text(size: size-small)[#markup(body)]
+                  }
+                },
               ))
               .flatten(),
           ))

src/model/assessment.rs

@@ -305,6 +305,28 @@ pub struct Placement {
     /// contribute.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub dropped_as: Option<DropStyle>,
+    /// Set only when the item was pulled *before* the paper was printed.
+    ///
+    /// `dropped` on its own means what its own documentation says: the item was
+    /// printed, students answered it, and it was then taken out of scoring.
+    /// Such an item keeps its printed position, because it occupied one on the
+    /// page the students held, and the responses that come back are numbered
+    /// around it.
+    ///
+    /// An item pulled before printing never occupied a position, so every later
+    /// question moves up one. That case has to be distinguished, and it cannot
+    /// be inferred: both look identical in the record. Getting it wrong is not
+    /// a cosmetic error. Sealing a post-administration drop as if it had never
+    /// been printed renumbers every question after it, so each response is
+    /// attributed to the wrong item, the statistics for those items are
+    /// computed from answers to different questions, and nothing in the output
+    /// looks obviously wrong.
+    ///
+    /// Practically: leave this alone when you discover a bad question after the
+    /// exam, which is the common case. Set it when you cut a question from the
+    /// draft and reprinted.
+    #[serde(default, skip_serializing_if = "is_false")]
+    pub dropped_before_printing: bool,
 }
 
 /// How a dropped item was handled on the grading platform.
@@ -327,6 +349,19 @@ impl Placement {
     pub fn dropped_with_credit(&self) -> bool {
         self.dropped && self.dropped_as == Some(DropStyle::FullCredit)
     }
+
+    /// Whether this placement occupied a printed position on the paper.
+    ///
+    /// Everything that lays out a page or reads a page back goes through this,
+    /// so the printed form, the seal, and the decoder cannot disagree about
+    /// which question sat where.
+    ///
+    /// # Returns
+    ///
+    /// `true` unless the item was pulled before printing.
+    pub fn was_printed(&self) -> bool {
+        !(self.dropped && self.dropped_before_printing)
+    }
 }
 
 impl AssessmentFile {

src/model/seal.rs

@@ -518,12 +518,13 @@ fn sealed_item(
 fn sealed_form(catalog: &Catalog, record: &AssessmentFile, form: &Form) -> Result<SealedForm> {
     let mut questions = Vec::new();
 
-    // Dropped placements are not printed, so they take no printed position. A
+    // Only an item pulled before printing takes no printed position. An item
-    // drop recorded before sealing therefore shifts every later position, exactly
+    // dropped from scoring after the exam was on the page and keeps its place,
-    // as it shifts them on the page.
+    // so re-sealing after a drop describes the same paper the students held
+    // rather than renumbering everything after it.
     let printed: Vec<Placement> = select::layout(record, form)
         .into_iter()
-        .filter(|p| !p.dropped)
+        .filter(|p| p.was_printed())
         .collect();
 
     for (index, placement) in printed.iter().enumerate() {