feat: improve dropped question support

src/analysis/classical.rs

@@ -396,6 +396,39 @@ pub fn analyze(
         ));
     }
 
+    // A drop changes every student's percentage, so the two ways to get it wrong
+    // are worth saying out loud. Both are silent otherwise: the numbers simply
+    // come out different from the platform's.
+    if let Some(record) = record {
+        for placement in record.items.iter().filter(|p| p.dropped) {
+            let rows = set.for_item(placement.number);
+            if rows.is_empty() {
+                continue;
+            }
+            let all_credited = rows.iter().all(|r| r.credit >= 0.999);
+            if placement.dropped_with_credit() && !all_credited {
+                let short = rows.iter().filter(|r| r.credit < 0.999).count();
+                warnings.push(format!(
+                    "question {} is marked `dropped_as: full_credit`, but {short} of {} responses \
+                     carry less than full credit. Either the platform was not regraded or the \
+                     export predates the regrade; until one of those is fixed this report's \
+                     percentages will sit below the grade of record",
+                    placement.number,
+                    rows.len()
+                ));
+            }
+            if !placement.dropped_with_credit() && all_credited {
+                warnings.push(format!(
+                    "question {} is dropped and every response carries full credit, which is what \
+                     crediting every option on the platform looks like. It is being removed from \
+                     the denominator here, so this report will read slightly lower than the \
+                     platform. Set `dropped_as: full_credit` if the platform kept the point",
+                    placement.number
+                ));
+            }
+        }
+    }
+
     let mut items = Vec::new();
     let mut p_values = Vec::new();
     let mut rpbs = Vec::new();
@@ -986,6 +1019,7 @@ mod tests {
             topics: vec![],
             bonus: false,
             dropped: false,
+            dropped_full_credit: false,
         }
     }
 

src/analysis/diagnostic.rs

@@ -172,12 +172,14 @@ pub struct StudentDiagnostic {
     /// One row per question, with no question in it.
     #[serde(skip_serializing_if = "Vec::is_empty")]
     pub questions: Vec<QuestionRow>,
-    /// Questions dropped from scoring, in number order.
+    /// Questions thrown out, in number order.
     ///
     /// Reported separately from the question rows so the document can say once,
-    /// in prose, why the denominator is smaller than the paper was.
+    /// in prose, what happened to them. The two kinds need different sentences:
+    /// a removed question is gone from the denominator, and a full-credit
+    /// question is still in it.
     #[serde(skip_serializing_if = "Vec::is_empty")]
-    pub dropped_questions: Vec<u32>,
+    pub dropped_questions: Vec<DroppedQuestion>,
     /// Which lectures to go back to, the one that would repay the most time
     /// first.
     #[serde(skip_serializing_if = "Vec::is_empty")]
@@ -360,6 +362,17 @@ pub struct QuestionRow {
     pub review: Vec<ItemReading>,
 }
 
+/// One question thrown out after the exam.
+#[derive(Debug, Clone, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct DroppedQuestion {
+    /// The recorded question number.
+    pub number: u32,
+    /// Whether the drop was applied by crediting every option, in which case the
+    /// question is still in the points of record.
+    pub full_credit: bool,
+}
+
 /// One citation to read again after missing a question.
 #[derive(Debug, Clone, Serialize)]
 #[serde(rename_all = "kebab-case")]
@@ -543,13 +556,16 @@ pub fn student(
         Vec::new()
     };
 
-    let mut dropped_questions: Vec<u32> = rows
+    let mut dropped_questions: Vec<DroppedQuestion> = rows
         .iter()
         .filter(|r| r.dropped)
-        .map(|r| r.item_number)
+        .map(|r| DroppedQuestion {
+            number: r.item_number,
+            full_credit: r.dropped_full_credit,
+        })
         .collect();
-    dropped_questions.sort_unstable();
-    dropped_questions.dedup();
+    dropped_questions.sort_by_key(|d| d.number);
+    dropped_questions.dedup_by_key(|d| d.number);
 
     // Built from the response rows rather than from `questions`, so a report with
     // `--no-questions` still says where to go back to; it just does not name the
@@ -952,11 +968,11 @@ pub struct CohortDiagnostic {
     pub lectures: Vec<CohortLectureRow>,
     /// Per-question statistics.
     pub questions: Vec<CohortQuestionRow>,
-    /// Questions dropped from scoring, which are therefore absent from every
-    /// table above. Recorded so the report says why rather than leaving a gap in
-    /// the numbering.
+    /// Questions thrown out, which are therefore absent from every table above.
+    /// Recorded so the report says why rather than leaving a gap in the
+    /// numbering.
     #[serde(skip_serializing_if = "Vec::is_empty")]
-    pub dropped_questions: Vec<u32>,
+    pub dropped_questions: Vec<DroppedQuestion>,
     /// What to do about each question that raised something.
     pub triage: Triage,
     /// How the authored expectations did.
@@ -1509,12 +1525,16 @@ pub fn cohort(
         .filter_map(|item| questions.iter().find(|q| q.number == item.number).cloned())
         .collect();
 
-    let mut dropped_questions: Vec<u32> = set
+    let mut dropped_questions: Vec<DroppedQuestion> = set
         .all_items()
         .into_iter()
         .filter(|n| set.item_dropped(*n))
+        .map(|number| DroppedQuestion {
+            number,
+            full_credit: set.for_item(number).iter().any(|r| r.dropped_full_credit),
+        })
         .collect();
-    dropped_questions.sort_unstable();
+    dropped_questions.sort_by_key(|d| d.number);
 
     let default_options = course.policy.options_per_item;
     let triage = triage(&questions, threshold, default_options);

src/analysis/students.rs

@@ -1112,6 +1112,7 @@ mod tests {
             topics: vec![],
             bonus: false,
             dropped: false,
+            dropped_full_credit: false,
         }
     }
 

src/authoring/select.rs

@@ -468,6 +468,7 @@ pub fn to_record(
             learning_objectives: e.item.learning_objectives.clone(),
             credit_overrides: BTreeMap::new(),
             dropped: false,
+            dropped_as: None,
         });
     }
 

src/data/canvas.rs

@@ -351,6 +351,7 @@ pub fn ingest(
                 topics: Vec::new(),
                 bonus: false,
                 dropped: false,
+                dropped_full_credit: false,
             });
         }
     }

src/data/decode.rs

@@ -709,6 +709,7 @@ mod tests {
             topics: Vec::new(),
             bonus: false,
             dropped: false,
+            dropped_full_credit: false,
         }
     }
 

src/data/gradescope.rs

@@ -659,6 +659,7 @@ pub fn to_responses(questions: &[Question], ctx: &Context) -> Import {
                 topics: Vec::new(),
                 bonus,
                 dropped: false,
+                dropped_full_credit: false,
             });
         }
     }

src/data/responses.rs

@@ -108,10 +108,26 @@ pub struct Response {
     pub bonus: bool,
     /// Whether the item was dropped after the fact.
     pub dropped: bool,
+    /// Whether that drop was applied by crediting every option, so the item is
+    /// still part of the points of record even though it is out of the
+    /// statistics.
+    ///
+    /// Written by [`ResponseSet::enrich`] from the placement's `dropped_as`.
+    #[serde(default)]
+    pub dropped_full_credit: bool,
 }
 
 impl Response {
-    /// Whether this row should count toward scored totals and item statistics.
+    /// Whether this row counts as evidence.
+    ///
+    /// Evidence means item statistics, objective mastery, level rates, and the
+    /// IRT fit. A dropped item is never evidence, however the drop was applied:
+    /// an item everyone was given has no variance to contribute and would only
+    /// flatter the objective it was written against.
+    ///
+    /// This is deliberately not the same question as [`Response::scores`]. The
+    /// two were one predicate until dropping a question stopped always meaning
+    /// removing it.
     ///
     /// # Returns
     ///
@@ -120,6 +136,21 @@ impl Response {
         !self.bonus && !self.dropped
     }
 
+    /// Whether this row counts toward the points of record.
+    ///
+    /// A question dropped by crediting every option still sits in the student's
+    /// total on the platform, and a report that disagreed with the platform
+    /// about a student's percentage would be worse than no report. So a
+    /// full-credit drop stays in both the numerator and the denominator here,
+    /// while a removed drop leaves both.
+    ///
+    /// # Returns
+    ///
+    /// `true` when the row belongs in the score.
+    pub fn scores(&self) -> bool {
+        !self.bonus && (!self.dropped || self.dropped_full_credit)
+    }
+
     /// The response coded for a dichotomous model.
     ///
     /// Partial credit is rounded toward the majority: a half-credit response is
@@ -188,7 +219,11 @@ impl ResponseSet {
         set.into_iter().map(|s| s.to_string()).collect()
     }
 
-    /// The distinct item numbers that count toward the scored total, sorted.
+    /// The distinct item numbers that count as evidence, sorted.
+    ///
+    /// Named for the scored total it once described; it is the analysis matrix's
+    /// item list, so it uses [`Response::counts`] and excludes every dropped
+    /// item. [`ResponseSet::points_available`] is the scoring denominator.
     pub fn scored_items(&self) -> Vec<u32> {
         let set: BTreeSet<u32> = self
             .rows
@@ -247,11 +282,12 @@ impl ResponseSet {
     ///
     /// # Returns
     ///
-    /// The sum of `score` over scored, undropped items.
+    /// The sum of `score` over the items that count toward the score, which
+    /// includes a question dropped by crediting every option.
     pub fn scored_total(&self, key: &str) -> f64 {
         self.rows
             .iter()
-            .filter(|r| r.student_key == key && r.counts())
+            .filter(|r| r.student_key == key && r.scores())
             .map(|r| r.score)
             .sum()
     }
@@ -269,7 +305,7 @@ impl ResponseSet {
     /// for each item so a student who skipped an item still has a denominator.
     pub fn points_available(&self) -> f64 {
         let mut per_item: BTreeMap<u32, f64> = BTreeMap::new();
-        for r in self.rows.iter().filter(|r| r.counts()) {
+        for r in self.rows.iter().filter(|r| r.scores()) {
             let e = per_item.entry(r.item_number).or_insert(0.0);
             if r.points_possible > *e {
                 *e = r.points_possible;
@@ -382,6 +418,7 @@ impl ResponseSet {
             r.item_version = p.version;
             r.bonus = r.bonus || p.bonus;
             r.dropped = r.dropped || p.dropped;
+            r.dropped_full_credit = r.dropped_full_credit || p.dropped_with_credit();
             if let Some(points) = p.points {
                 // The record is authoritative for points as administered; the
                 // export sometimes carries a stale maximum.
@@ -636,6 +673,9 @@ pub struct FlatResponse {
     /// Comma-joined eliminated letters in the bank's lettering.
     #[serde(default)]
     pub eliminated_source: String,
+    /// Whether a dropped item was dropped by crediting every option.
+    #[serde(default)]
+    pub dropped_full_credit: bool,
 }
 
 impl FlatResponse {
@@ -656,6 +696,7 @@ impl FlatResponse {
             assessment_id: r.assessment_id.clone(),
             date: r.date.map(|d| d.to_string()).unwrap_or_default(),
             form: r.form.clone().unwrap_or_default(),
+            dropped_full_credit: r.dropped_full_credit,
             form_position: r.form_position.unwrap_or(0),
             student_key: r.student_key.clone(),
             sid: r.sid.clone().unwrap_or_default(),
@@ -709,6 +750,7 @@ impl FlatResponse {
             assessment_id: self.assessment_id.clone(),
             date: self.date.parse().ok(),
             form: none_if_empty(&self.form),
+            dropped_full_credit: self.dropped_full_credit,
             form_position: if self.form_position == 0 {
                 None
             } else {
@@ -792,6 +834,7 @@ mod tests {
             topics: vec![],
             bonus: false,
             dropped: false,
+            dropped_full_credit: false,
         }
     }
 

src/data/store.rs

@@ -564,6 +564,7 @@ mod tests {
             topics: vec![],
             bonus: false,
             dropped: false,
+            dropped_full_credit: false,
         }
     }
 

src/data/store_parquet.rs

@@ -62,6 +62,7 @@ pub fn schema() -> Schema {
         Field::new("bonus", DataType::Boolean, false),
         Field::new("dropped", DataType::Boolean, false),
         Field::new("form_position", DataType::UInt32, false),
+        Field::new("dropped_full_credit", DataType::Boolean, false),
         Field::new("selected_source", DataType::Utf8, false),
         Field::new("eliminated_source", DataType::Utf8, false),
     ])
@@ -124,6 +125,7 @@ fn to_batch(rows: &[FlatResponse]) -> Result<RecordBatch> {
         boolc(|r| r.bonus),
         boolc(|r| r.dropped),
         u32c(|r| r.form_position),
+        boolc(|r| r.dropped_full_credit),
         s(|r| &r.selected_source),
         s(|r| &r.eliminated_source),
     ];
@@ -256,6 +258,11 @@ fn from_batch(batch: &RecordBatch, path: &Path) -> Result<Vec<FlatResponse>> {
             .column_by_name(name)
             .and_then(|c| c.as_any().downcast_ref::<UInt32Array>())
     };
+    let optional_bools = |name: &str| -> Option<&BooleanArray> {
+        batch
+            .column_by_name(name)
+            .and_then(|c| c.as_any().downcast_ref::<BooleanArray>())
+    };
 
     let administration_id = strings("administration_id")?;
     let course = strings("course")?;
@@ -284,6 +291,7 @@ fn from_batch(batch: &RecordBatch, path: &Path) -> Result<Vec<FlatResponse>> {
     let dropped = bools("dropped")?;
 
     let form_position = optional_uints("form_position");
+    let dropped_full_credit = optional_bools("dropped_full_credit");
     let selected_source = optional_strings("selected_source");
     let eliminated_source = optional_strings("eliminated_source");
 
@@ -322,6 +330,7 @@ fn from_batch(batch: &RecordBatch, path: &Path) -> Result<Vec<FlatResponse>> {
             topics: topics.value(i).to_string(),
             bonus: bonus.value(i),
             dropped: dropped.value(i),
+            dropped_full_credit: dropped_full_credit.map(|a| a.value(i)).unwrap_or(false),
         });
     }
     Ok(out)
@@ -367,6 +376,7 @@ mod tests {
             topics: "kinetics".into(),
             bonus: false,
             dropped: false,
+            dropped_full_credit: false,
             form_position: 0,
             selected_source: String::new(),
             eliminated_source: String::new(),

src/export/practice.rs

@@ -602,6 +602,7 @@ items:
                     learning_objectives: Vec::new(),
                     credit_overrides: Default::default(),
                     dropped: false,
+                    dropped_as: None,
                 },
                 Placement {
                     number: 2,
@@ -615,6 +616,7 @@ items:
                     learning_objectives: Vec::new(),
                     credit_overrides: Default::default(),
                     dropped: false,
+                    dropped_as: None,
                 },
             ],
         }

src/export/qti.rs

@@ -1377,6 +1377,7 @@ items:
                     learning_objectives: Vec::new(),
                     credit_overrides: Default::default(),
                     dropped: false,
+                    dropped_as: None,
                 },
                 Placement {
                     number: 2,
@@ -1390,6 +1391,7 @@ items:
                     learning_objectives: Vec::new(),
                     credit_overrides: Default::default(),
                     dropped: false,
+                    dropped_as: None,
                 },
             ],
         }

src/export/site.rs

@@ -866,6 +866,7 @@ items:
                     learning_objectives: Vec::new(),
                     credit_overrides: Default::default(),
                     dropped: false,
+                    dropped_as: None,
                 },
                 Placement {
                     number: 2,
@@ -879,6 +880,7 @@ items:
                     learning_objectives: Vec::new(),
                     credit_overrides: Default::default(),
                     dropped: false,
+                    dropped_as: None,
                 },
             ],
         }

src/export/typst.rs

@@ -427,6 +427,7 @@ mod tests {
                     learning_objectives: Vec::new(),
                     credit_overrides: Default::default(),
                     dropped: true,
+                    dropped_as: None,
                 },
                 Placement {
                     number: 2,
@@ -440,6 +441,7 @@ mod tests {
                     learning_objectives: Vec::new(),
                     credit_overrides: Default::default(),
                     dropped: false,
+                    dropped_as: None,
                 },
             ],
         };

src/export/typst/diagnostic.rs

@@ -355,7 +355,12 @@ pub fn student_value(diagnostic: &StudentDiagnostic, config: &RenderConfig) -> V
             diagnostic
                 .dropped_questions
                 .iter()
-                .map(|n| Value::Int(*n as i64))
+                .map(|dropped| {
+                    let mut value = Value::dict();
+                    value.insert("number", Value::Int(dropped.number as i64));
+                    value.insert("full-credit", Value::Bool(dropped.full_credit));
+                    value
+                })
                 .collect(),
         ),
     );
@@ -630,7 +635,12 @@ pub fn cohort_value(diagnostic: &CohortDiagnostic, config: &RenderConfig) -> Val
             diagnostic
                 .dropped_questions
                 .iter()
-                .map(|n| Value::Int(*n as i64))
+                .map(|dropped| {
+                    let mut value = Value::dict();
+                    value.insert("number", Value::Int(dropped.number as i64));
+                    value.insert("full-credit", Value::Bool(dropped.full_credit));
+                    value
+                })
                 .collect(),
         ),
     );

src/export/typst/templates/cohort-report.typ

@@ -994,11 +994,21 @@
       let dropped = cb-data.at("dropped-questions", default: ())
       if dropped.len() > 0 [
         #plural(dropped.len(), "Question", "Questions")
-        #dropped.map(str).join(", ")
+        #dropped.map(d => str(d.number)).join(", ")
         #plural(dropped.len(), "is", "are")
         marked dropped in the assessment record, so
         #plural(dropped.len(), "it is", "they are")
         absent from this table and from every statistic above.
+        #{
+          let credited = dropped.filter(d => d.at("full-credit", default: false))
+          if credited.len() > 0 [
+            #plural(credited.len(), "Question", "Questions")
+            #credited.map(d => str(d.number)).join(", ")
+            #plural(credited.len(), "was", "were") credited to every student, so
+            #plural(credited.len(), "it remains", "they remain") in the score
+            denominator and the means above match the grade of record.
+          ]
+        }
       ]
     }
   ]
@@ -1173,6 +1183,7 @@
 #let predictions = cb-data.at("predictions", default: (:))
 
 #if show-predictions and predictions.at("predicted", default: 0) > 0 [
+  #block(breakable: false)[
     = How your predictions did
 
     #let n = predictions.at("predicted", default: 0)
@@ -1236,6 +1247,7 @@
         ]
       }
     }
+  ]
 ]
 
 // ─────────────────────────────────────────────────────────────────────────────

src/export/typst/templates/student-report.typ

@@ -143,6 +143,7 @@
       review: ((citation: "KKW §6.2", title: "Molecules and Medicine", url: "https://example.edu/6/2"),),
     ),
   ),
+  dropped-questions: ((number: 35, full-credit: true),),
   review-lectures: (
     (
       lecture: "L1.1",
@@ -458,15 +459,30 @@
   #block(above: entry-gap)[
     #pad(right: prose-pad)[
       #text(size: size-lead)[
-        #plural(dropped-questions.len(), "Question", "Questions")
-        #dropped-questions.map(str).join(", ")
-        #plural(dropped-questions.len(), "was", "were")
-        dropped after the exam and #plural(dropped-questions.len(), "is", "are")
-        not part of anyone's score. Your percentage above is out of the
-        #plural(dropped-questions.len(), "remaining question", "remaining questions").
-        Nothing you wrote on
-        #plural(dropped-questions.len(), "it", "them")
-        counted for or against you.
+        #{
+          // Two kinds of drop, and they need different sentences. A credited
+          // question is still in the denominator, so telling a student it was
+          // removed would not match the arithmetic they can do themselves.
+          let credited = dropped-questions.filter(d => d.at("full-credit", default: false))
+          let removed = dropped-questions.filter(d => not d.at("full-credit", default: false))
+          let numbers = list => list.map(d => str(d.number)).join(", ")
+
+          if credited.len() > 0 [
+            #plural(credited.len(), "Question", "Questions") #numbers(credited)
+            #plural(credited.len(), "was", "were") thrown out after the exam.
+            Everyone received full credit for
+            #plural(credited.len(), "it", "them"), so
+            #plural(credited.len(), "it is", "they are") still counted in the
+            score above and whatever you chose made no difference.
+          ]
+          if removed.len() > 0 [
+            #plural(removed.len(), "Question", "Questions") #numbers(removed)
+            #plural(removed.len(), "was", "were") thrown out and removed from
+            scoring, so your percentage is out of the remaining questions.
+            Nothing you wrote on #plural(removed.len(), "it", "them") counted
+            either way.
+          ]
+        }
       ]
     ]
   ]
@@ -822,7 +838,8 @@
       #box(width: 0.7em, height: 0.7em, fill: ok-color.lighten(70%), radius: 2pt) right ·
       #box(width: 0.7em, height: 0.7em, fill: mid-color.lighten(70%), radius: 2pt) part marks ·
       #box(width: 0.7em, height: 0.7em, fill: bad-color.lighten(70%), radius: 2pt) not right ·
-      #box(width: 0.7em, height: 0.7em, fill: luma(210), radius: 2pt) left blank
+      #box(width: 0.7em, height: 0.7em, fill: luma(210), radius: 2pt) left blank ·
+      #box(width: 0.7em, height: 0.7em, fill: luma(150), radius: 2pt) dropped, not scored
     ]
   ]
 ]

src/model/assessment.rs

@@ -281,6 +281,48 @@ pub struct Placement {
     /// Set when an item was dropped from scoring after administration.
     #[serde(default, skip_serializing_if = "is_false")]
     pub dropped: bool,
+    /// How the drop was applied on the grading platform.
+    ///
+    /// Two ways to throw a question out, and they produce different
+    /// percentages. [`DropStyle::Removed`] takes the item out of the numerator
+    /// and the denominator: a student with 27 of 35 scores 77.1%.
+    /// [`DropStyle::FullCredit`] is what you do when the grade of record lives
+    /// somewhere else and the question cannot be removed from it: every option
+    /// is keyed, everyone earns the point, and the same student scores 28 of 36,
+    /// or 77.8%.
+    ///
+    /// Defaults to [`DropStyle::Removed`], which is what `dropped: true` meant
+    /// before this field existed. Set it to `full_credit` when you have credited
+    /// every option in the platform, so that the report agrees with the grade the
+    /// student can see.
+    ///
+    /// Either way the item is out of the item statistics, the objective
+    /// evidence, and the IRT fit: an item everyone got right has no variance to
+    /// contribute.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub dropped_as: Option<DropStyle>,
+}
+
+/// How a dropped item was handled on the grading platform.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum DropStyle {
+    /// Taken out of the numerator and the denominator.
+    Removed,
+    /// Every option credited, so the item stays in both.
+    FullCredit,
+}
+
+impl Placement {
+    /// Whether this placement was dropped by crediting every option.
+    ///
+    /// # Returns
+    ///
+    /// `true` only when the item is dropped *and* the drop was applied as full
+    /// credit, so the item still belongs in the points of record.
+    pub fn dropped_with_credit(&self) -> bool {
+        self.dropped && self.dropped_as == Some(DropStyle::FullCredit)
+    }
 }
 
 impl AssessmentFile {
@@ -451,6 +493,20 @@ impl AssessmentFile {
                     ));
                 }
             }
+
+            if p.dropped_as.is_some() && !p.dropped {
+                issues.push(format!(
+                    "question {}: `dropped_as` is set but `dropped` is not, so nothing is dropped",
+                    p.number
+                ));
+            }
+            if p.dropped && !p.credit_overrides.is_empty() {
+                issues.push(format!(
+                    "question {}: dropped and carrying credit overrides. Pick one: an override \
+                     rescores an option, a drop removes the question",
+                    p.number
+                ));
+            }
         }
 
         let mut form_ids: Vec<&str> = Vec::new();