feat: more comprehensive cohort report

src/analysis/diagnostic.rs

@@ -1020,6 +1020,13 @@ pub struct CohortDiagnostic {
     pub predictions: PredictionSummary,
     /// Questions worth revisiting before reuse, worst first.
     pub revise: Vec<CohortQuestionRow>,
+    /// The dropped items, described but not measured.
+    ///
+    /// Kept out of [`CohortDiagnostic::questions`] so that no statistic above
+    /// silently includes an item that was thrown out, and reported alongside it
+    /// in the evidence section so that dropping a question does not erase the
+    /// evidence for having dropped it.
+    pub dropped_detail: Vec<CohortQuestionRow>,
     /// One row per form, when more than one was given.
     #[serde(skip_serializing_if = "Vec::is_empty")]
     pub forms: Vec<FormRow>,
@@ -1138,6 +1145,26 @@ pub struct CohortQuestionRow {
     /// What those targets ask, in the course's own words.
     #[serde(skip_serializing_if = "Vec::is_empty")]
     pub target_texts: Vec<String>,
+    /// Whether the item was dropped from scoring.
+    ///
+    /// A dropped item carries no p, r, or D, and appears in none of the
+    /// statistics above. It still appears in the evidence section, because the
+    /// option spread that justified dropping it is the record of why, and that
+    /// record should survive re-running the report afterwards.
+    #[serde(skip_serializing_if = "std::ops::Not::not")]
+    pub dropped: bool,
+    /// Whether the drop was applied as full credit to everyone.
+    #[serde(skip_serializing_if = "std::ops::Not::not")]
+    pub dropped_full_credit: bool,
+    /// The question as written.
+    ///
+    /// The instructor report reads better with it than without: a row of option
+    /// shares says a distractor drew 44% of the class, and only the stem says
+    /// whether that is a second defensible reading. Absent when the item has
+    /// left the bank, and withheld from any report that is not the instructor
+    /// copy.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub stem: Option<String>,
     /// Where the item was taught, as lecture titles and slide numbers.
     #[serde(skip_serializing_if = "Vec::is_empty")]
     pub taught_in: Vec<String>,
@@ -1185,8 +1212,25 @@ pub struct CohortQuestionRow {
 #[derive(Debug, Clone, Serialize)]
 #[serde(rename_all = "kebab-case")]
 pub struct OptionRow {
-    /// The bank letter.
+    /// The bank letter, which is the one every statistic is keyed by.
     pub letter: String,
+    /// The option as written, for a report that shows the question.
+    ///
+    /// Absent when the item is no longer in the bank, which is why this is an
+    /// option rather than an empty string: a missing option and an empty one
+    /// are different facts.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub text: Option<String>,
+    /// What this option was lettered on each printed form, worst case one entry
+    /// per form.
+    ///
+    /// Shuffling means the bank's option C is a different letter on every form,
+    /// so a statistic reported against C cannot be checked against a student's
+    /// paper without this map. It is the first thing anyone needs when a student
+    /// brings a paper to office hours, and working it out by hand from a seal is
+    /// the kind of task that gets done wrong once and then trusted.
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    pub printed: Vec<PrintedLetter>,
     /// How many chose it.
     pub count: usize,
     /// The share who chose it.
@@ -1200,6 +1244,16 @@ pub struct OptionRow {
     pub nonfunctioning: bool,
 }
 
+/// What one bank option was lettered on one form.
+#[derive(Debug, Clone, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct PrintedLetter {
+    /// The form id.
+    pub form: String,
+    /// The letter this option carried on that form's paper.
+    pub letter: String,
+}
+
 /// One form's summary.
 #[derive(Debug, Clone, Serialize)]
 #[serde(rename_all = "kebab-case")]
@@ -1502,11 +1556,17 @@ pub fn cohort(
         })
         .collect();
 
+    // The printed lettering per form, computed from the same two functions the
+    // exporter and the seal use, so the letters here are the letters on the
+    // paper rather than a second guess at them.
+    let forms: Vec<&crate::assessment::Form> = record.forms.iter().collect();
+
     let questions: Vec<CohortQuestionRow> = analysis
         .items
         .iter()
         .map(|item| {
             let meta = item_meta.get(&item.number);
+            let entry = item.item_ref.as_deref().and_then(|uid| catalog.get(uid));
             CohortQuestionRow {
                 number: item.number,
                 item: item.item_ref.clone(),
@@ -1533,6 +1593,9 @@ pub fn cohort(
                             .collect()
                     })
                     .unwrap_or_default(),
+                dropped: false,
+                dropped_full_credit: false,
+                stem: entry.map(|e| e.item.stem.clone()),
                 difficulty_band: difficulty_band(item.p_value).to_string(),
                 discrimination_band: discrimination_band(item.point_biserial).to_string(),
                 p_value: item.p_value,
@@ -1544,6 +1607,23 @@ pub fn cohort(
                     .options
                     .values()
                     .map(|option| OptionRow {
+                        text: entry.and_then(|e| {
+                            e.item
+                                .options
+                                .iter()
+                                .find(|o| o.id == option.letter)
+                                .map(|o| o.text.clone())
+                        }),
+                        printed: entry
+                            .map(|e| {
+                                printed_letters(
+                                    &forms,
+                                    &e.item,
+                                    item.item_ref.as_deref(),
+                                    &option.letter,
+                                )
+                            })
+                            .unwrap_or_default(),
                         letter: option.letter.clone(),
                         count: option.count,
                         rate: option.rate,
@@ -1582,6 +1662,106 @@ pub fn cohort(
         .collect();
     dropped_questions.sort_by_key(|d| d.number);
 
+    // The dropped items, described from the responses rather than from the
+    // scoring. Dropping an item overrides its credit, so p, r, and D are
+    // meaningless for it and are left out. What students actually marked is
+    // untouched by the drop, and that spread is the evidence that justified it.
+    let mut dropped_detail: Vec<CohortQuestionRow> = dropped_questions
+        .iter()
+        .map(|dropped| {
+            let placement = record.placement(dropped.number);
+            let uid = placement.map(|p| p.item.clone());
+            let entry = uid.as_deref().and_then(|uid| catalog.get(uid));
+            let responses = set.for_item(dropped.number);
+            let answered = responses
+                .iter()
+                .filter(|r| !r.chosen().is_empty())
+                .count()
+                .max(1);
+            let keyed: BTreeSet<String> = placement
+                .map(|p| p.key.iter().cloned().collect())
+                .filter(|k: &BTreeSet<String>| !k.is_empty())
+                .or_else(|| entry.map(|e| e.item.key_letters().into_iter().collect()))
+                .unwrap_or_default();
+
+            // One row per option the item declares, so an option nobody
+            // marked still shows as unchosen rather than vanishing.
+            let options: Vec<OptionRow> = entry
+                .map(|e| {
+                    e.item
+                        .options
+                        .iter()
+                        .map(|option| {
+                            let count = responses
+                                .iter()
+                                .filter(|r| r.chosen().iter().any(|l| *l == option.id))
+                                .count();
+                            OptionRow {
+                                text: Some(option.text.clone()),
+                                printed: printed_letters(
+                                    &forms,
+                                    &e.item,
+                                    uid.as_deref(),
+                                    &option.id,
+                                ),
+                                letter: option.id.clone(),
+                                count,
+                                rate: count as f64 / answered as f64,
+                                is_key: keyed.contains(&option.id),
+                                point_biserial: None,
+                                nonfunctioning: false,
+                            }
+                        })
+                        .collect()
+                })
+                .unwrap_or_default();
+
+            let targets = placement
+                .map(|p| p.learning_targets.clone())
+                .unwrap_or_default();
+            CohortQuestionRow {
+                number: dropped.number,
+                item: uid.clone(),
+                level: placement.and_then(|p| p.level).map(|l| l.code()),
+                target_texts: targets.iter().map(|id| course.text_for(id)).collect(),
+                targets,
+                dropped: true,
+                dropped_full_credit: dropped.full_credit,
+                stem: entry.map(|e| e.item.stem.clone()),
+                taught_in: uid
+                    .as_deref()
+                    .map(|uid| taught_in(catalog, uid))
+                    .unwrap_or_default(),
+                lectures: entry
+                    .map(|e| e.item.sources.iter().map(|s| s.lecture.clone()).collect())
+                    .unwrap_or_default(),
+                difficulty_band: String::new(),
+                discrimination_band: String::new(),
+                // The keyed share before the override, which is the closest
+                // honest reading of how the item performed. It is not a
+                // p-value: it counts marks, not credit.
+                p_value: options
+                    .iter()
+                    .filter(|o| o.is_key)
+                    .map(|o| o.rate)
+                    .sum::<f64>()
+                    .min(1.0),
+                point_biserial: None,
+                discrimination: None,
+                blank_rate: responses.iter().filter(|r| r.chosen().is_empty()).count() as f64
+                    / responses.len().max(1) as f64,
+                key: keyed.iter().cloned().collect(),
+                options,
+                flags: Vec::new(),
+                notes: Vec::new(),
+                prediction_notes: Vec::new(),
+                calibrated: false,
+                by_form: BTreeMap::new(),
+            }
+        })
+        .collect();
+    dropped_detail.sort_by_key(|q| q.number);
+
     let default_options = course.policy.options_per_item;
     let triage = triage(&questions, threshold, default_options);
     let predictions = prediction_summary(analysis);
@@ -1609,6 +1789,7 @@ pub fn cohort(
         gaps,
         questions,
         revise,
+        dropped_detail,
         forms: form_rows(set, cohort),
         blueprint: crate::select::check_blueprint(record, course),
         patterns: cohort
@@ -1669,6 +1850,53 @@ fn taught_in(catalog: &Catalog, uid: &str) -> Vec<String> {
         .collect()
 }
 
+/// Where one bank option landed on each printed form.
+///
+/// # Arguments
+///
+/// * `forms` - the record's forms, in declaration order.
+/// * `item` - the bank item, for its option count and lettering.
+/// * `uid` - the item's global id, which salts the permutation.
+/// * `letter` - the bank letter to locate.
+///
+/// # Returns
+///
+/// One entry per form that permutes its options. Forms printing the bank order
+/// unchanged are left out, since an entry saying C was printed as C is noise on
+/// every row.
+fn printed_letters(
+    forms: &[&crate::assessment::Form],
+    item: &crate::item::Item,
+    uid: Option<&str>,
+    letter: &str,
+) -> Vec<PrintedLetter> {
+    let Some(uid) = uid else {
+        return Vec::new();
+    };
+    let Some(source) = item.options.iter().position(|o| o.id == letter) else {
+        return Vec::new();
+    };
+    let n = item.options.len();
+    let mut out = Vec::new();
+    for form in forms {
+        if !form.shuffle_options {
+            continue;
+        }
+        // The same permutation the exporter and the seal use, so these are the
+        // letters on the paper rather than a second guess at them.
+        let order = crate::select::option_order(form, uid, n);
+        // `order[position] == source` means the option printed in that slot is
+        // the one being asked about.
+        if let Some(position) = order.iter().position(|index| *index == source) {
+            out.push(PrintedLetter {
+                form: form.id.clone(),
+                letter: crate::seal::printed_letter(position),
+            });
+        }
+    }
+    out
+}
+
 /// The difficulty band a p-value falls in.
 ///
 /// Three bands rather than five. The only distinction that changes what you do