fix: seals after changes

src/data/decode.rs

@@ -81,6 +81,11 @@ pub enum Numbering {
     Recorded,
 }
 
+/// The form marker put on a row that could not be translated, so it can be
+/// removed after the borrow on `set.rows` ends. No real form id can collide
+/// with it: form ids come from the record and are short labels like `A`.
+const UNMAPPED: &str = "\u{1f}unmapped";
+
 /// One question's mapping on one form.
 #[derive(Debug, Clone)]
 pub struct QuestionMap {
@@ -354,6 +359,11 @@ impl FormDecoder {
         self.by_position.len()
     }
 
+    /// Every recorded question number this form carries.
+    pub fn numbers(&self) -> impl Iterator<Item = u32> + '_ {
+        self.by_position.values().map(|q| q.number)
+    }
+
     /// Whether the form prints nothing, which means the record is empty.
     pub fn is_empty(&self) -> bool {
         self.by_position.is_empty()
@@ -572,9 +582,17 @@ pub fn form_warning(claimed: &str, fits: &[FormFit]) -> Option<String> {
 pub fn apply(set: &mut ResponseSet, decoder: &FormDecoder, numbering: Numbering) -> Vec<String> {
     let mut warnings = Vec::new();
     let mut unmapped_positions: BTreeSet<u32> = BTreeSet::new();
+    let mut colliding_positions: BTreeSet<u32> = BTreeSet::new();
     let mut unmapped_letters: BTreeSet<String> = BTreeSet::new();
     let mut translated = 0usize;
 
+    // Numbers this form really uses. An untranslated row whose raw number is one
+    // of these would silently masquerade as that question, and two rows would
+    // then share a number: one the student's answer to it, one an answer to
+    // something else entirely. Nothing downstream can tell them apart, so the
+    // collision has to be caught here.
+    let recorded: BTreeSet<u32> = decoder.numbers().collect();
+
     for row in &mut set.rows {
         let belongs = row
             .form
@@ -587,6 +605,13 @@ pub fn apply(set: &mut ResponseSet, decoder: &FormDecoder, numbering: Numbering)
 
         let Some(map) = decoder.lookup(row.item_number, numbering) else {
             unmapped_positions.insert(row.item_number);
+            if recorded.contains(&row.item_number) {
+                colliding_positions.insert(row.item_number);
+                // Marked so the row can be discarded below. Attributing it to
+                // the question that legitimately holds this number would corrupt
+                // that question's statistics.
+                row.form = Some(UNMAPPED.to_string());
+            }
             continue;
         };
 
@@ -615,9 +640,28 @@ pub fn apply(set: &mut ResponseSet, decoder: &FormDecoder, numbering: Numbering)
     if !unmapped_positions.is_empty() {
         let list: Vec<String> = unmapped_positions.iter().map(|n| n.to_string()).collect();
         warnings.push(format!(
-            "form {}: question(s) {} are in the export but not on this form; they were left \
-             untranslated",
+            "form {}: question(s) {} are in the export but not on this form ({} printed). The \
+             export may have been taken before a question was dropped, or from a different \
+             form.",
             decoder.form,
+            list.join(", "),
+            decoder.len()
+        ));
+    }
+    if !colliding_positions.is_empty() {
+        let list: Vec<String> = colliding_positions.iter().map(|n| n.to_string()).collect();
+        let discarded = set.rows.len();
+        set.rows
+            .retain(|row| row.form.as_deref() != Some(UNMAPPED));
+        warnings.push(format!(
+            "form {}: {} response(s) at position(s) {} could not be translated, and their raw \
+             numbers are numbers this form does use. Keeping them would have given those \
+             questions two different answers each, so they were discarded. This is the shape of \
+             an export made before a question was dropped: re-export the responses from the \
+             administration you sealed, or re-run with --recorded-numbers if the export already \
+             carries recorded numbers.",
+            decoder.form,
+            discarded - set.rows.len(),
             list.join(", ")
         ));
     }