// SPDX-License-Identifier: Prosperity-3.0.0 // Copyright Scientific Computing Studio // Source: https://git.scient.ing/education/coursebank //! Turning what a student marked into what a student chose. //! //! A grading export speaks in positions and printed letters. Question 14 is the //! fourteenth thing on the page; option C is the third bubble. An item bank speaks //! in ids and its own lettering. When forms shuffle, those two vocabularies //! disagree, and every analysis downstream of the disagreement is wrong in a way //! that looks right: //! //! * Pooled distractor statistics add form A's option C to form B's option C, //! which are different sentences. The resulting table is noise with the shape of //! data. //! * A student report looks up the misconception recorded on option C and shows it //! to a student who chose a different option. The feedback is confident, //! specific, and about the wrong thing. //! * Any `credit_overrides` written in the record's lettering are applied to //! whoever happened to mark that letter on their form. //! //! None of these fail loudly. That is the argument for doing the translation once, //! at ingest, and storing both sides of it. //! //! # What a decoder knows //! //! For one form: which recorded question number sits at each printed position, //! which bank letter each printed letter stands for, and which printed letters are //! keyed. It is built from a [`SealFile`] when one exists, and derived from the //! record and the form seed when one does not. Sealed is better, and not only //! because it is faster: a derived decoder describes the form the bank *would* //! print today, while a sealed one describes the form that was actually printed. //! //! # Catching a swapped directory //! //! Gradescope's point-value row reveals which printed letter earned full credit on //! every question. A decoder knows what that letter should be. Comparing them //! across a whole directory is close to a proof of which form the directory holds: //! agreement is near total for the right form and near chance for the wrong one. //! [`identify_form`] uses that to refuse an ingest that names form A over a //! directory of form B papers, which is otherwise a mistake nobody catches until //! the item statistics look strange three weeks later. use std::collections::{BTreeMap, BTreeSet}; use crate::assessment::{AssessmentFile, Form}; use crate::catalog::Catalog; use crate::error::{Error, Result}; use crate::gradescope::Question; use crate::responses::ResponseSet; use crate::seal::{SealFile, printed_letter}; use crate::select; /// Where a decoder's mapping came from. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Provenance { /// Read from a seal written before administration. Authoritative. Seal, /// Derived from the assessment record and the form seed, as of now. Derived, } impl Provenance { /// A short label for output. pub fn label(self) -> &'static str { match self { Provenance::Seal => "seal", Provenance::Derived => "derived from the record", } } } /// How a grading export numbers its questions. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Numbering { /// The export counts printed positions, which is what Gradescope's `N.csv` /// file names mean. Positions are translated to recorded numbers. Printed, /// The export already carries recorded question numbers, so numbers pass /// through untouched. Recorded, } /// The form marker put on a row that could not be translated, so it can be /// removed after the borrow on `set.rows` ends. No real form id can collide /// with it: form ids come from the record and are short labels like `A`. const UNMAPPED: &str = "\u{1f}unmapped"; /// One question's mapping on one form. #[derive(Debug, Clone)] pub struct QuestionMap { /// Printed position on the page, counting from 1. pub position: u32, /// The recorded question number, the join key to the record and the store. pub number: u32, /// The item's global id. pub item: String, /// Keyed letters as printed on this form. pub printed_key: Vec, /// Keyed letters in the bank's own lettering. pub canonical_key: Vec, /// Printed letter to bank letter. pub to_canonical: BTreeMap, /// Bank letter to printed letter. pub to_printed: BTreeMap, } impl QuestionMap { /// The bank letter a printed letter stands for. /// /// # Arguments /// /// * `printed` - the letter as the student saw it. /// /// # Returns /// /// The bank letter, or `None` when the printed letter is not one of this /// question's options. pub fn canonical(&self, printed: &str) -> Option<&str> { self.to_canonical .get(&printed.trim().to_ascii_uppercase()) .map(|s| s.as_str()) } /// How many options this question has. pub fn n_options(&self) -> usize { self.to_canonical.len() } /// Whether this question's options were actually permuted. pub fn is_permuted(&self) -> bool { self.to_canonical.iter().any(|(k, v)| k != v) } } /// One form's full mapping. #[derive(Debug, Clone)] pub struct FormDecoder { /// The form id. pub form: String, /// Where the mapping came from. pub provenance: Provenance, /// Questions by printed position. by_position: BTreeMap, /// Questions by recorded number. by_number: BTreeMap, } impl FormDecoder { /// Builds a decoder from the question maps. fn assemble(form: String, provenance: Provenance, maps: Vec) -> FormDecoder { let by_position = maps.iter().map(|m| (m.position, m.clone())).collect(); let by_number = maps.into_iter().map(|m| (m.number, m)).collect(); FormDecoder { form, provenance, by_position, by_number, } } /// Reads one form's mapping out of a seal. /// /// # Arguments /// /// * `seal` - the seal. /// * `form_id` - the form to decode, matched case-insensitively. /// /// # Returns /// /// The decoder. /// /// # Errors /// /// Returns [`Error::Usage`] when the seal does not cover that form. pub fn from_seal(seal: &SealFile, form_id: &str) -> Result { let form = seal.form(form_id).ok_or_else(|| { Error::usage(format!( "the seal for `{}` does not cover form `{form_id}`; it covers {}", seal.seal.assessment, seal.forms .iter() .map(|f| f.id.as_str()) .collect::>() .join(", ") )) })?; let maps = form .questions .iter() .map(|q| { let mut to_canonical = BTreeMap::new(); let mut to_printed = BTreeMap::new(); for map in &q.options { to_canonical.insert(map.printed.clone(), map.canonical.clone()); to_printed.insert(map.canonical.clone(), map.printed.clone()); } let canonical_key: Vec = q .printed_key .iter() .filter_map(|p| to_canonical.get(p).cloned()) .collect(); QuestionMap { position: q.position, number: q.number, item: q.item.clone(), printed_key: q.printed_key.clone(), canonical_key, to_canonical, to_printed, } }) .collect(); Ok(FormDecoder::assemble( form.id.clone(), Provenance::Seal, maps, )) } /// Derives one form's mapping from the record and the bank. /// /// Uses the same two functions every export calls, so a derived decoder and a /// freshly exported paper agree by construction. /// /// # Arguments /// /// * `catalog` - the loaded course. /// * `record` - the assessment record. /// * `form` - the form. /// /// # Returns /// /// The decoder. /// /// # Errors /// /// Returns [`Error::Unresolved`] when a placement references a missing item. pub fn derive(catalog: &Catalog, record: &AssessmentFile, form: &Form) -> Result { let printed: Vec<_> = select::layout(record, form) .into_iter() .filter(|p| !p.dropped) .collect(); let mut maps = Vec::with_capacity(printed.len()); for (index, placement) in printed.iter().enumerate() { let entry = catalog.require(&placement.item)?; let item = &entry.item; let shown = item.administered(&placement.key, &placement.distractors); let order = select::option_order(form, &placement.item, shown.len()); let canonical_key: BTreeSet = if placement.key.is_empty() { item.key_letters().into_iter().collect() } else { placement.key.iter().cloned().collect() }; let mut to_canonical = BTreeMap::new(); let mut to_printed = BTreeMap::new(); let mut printed_key = Vec::new(); for (position, source_index) in order.iter().enumerate() { let canonical = shown .get(*source_index) .map(|c| c.id.clone()) .unwrap_or_else(|| printed_letter(*source_index)); let label = printed_letter(position); if canonical_key.contains(&canonical) { printed_key.push(label.clone()); } to_canonical.insert(label.clone(), canonical.clone()); to_printed.insert(canonical, label); } maps.push(QuestionMap { position: index as u32 + 1, number: placement.number, item: placement.item.clone(), printed_key, canonical_key: canonical_key.into_iter().collect(), to_canonical, to_printed, }); } Ok(FormDecoder::assemble( form.id.clone(), Provenance::Derived, maps, )) } /// Builds a decoder, preferring the seal. /// /// # Arguments /// /// * `seal` - the seal, when one has been written. /// * `catalog` - the loaded course. /// * `record` - the assessment record. /// * `form` - the form. /// /// # Returns /// /// The decoder. /// /// # Errors /// /// As [`FormDecoder::from_seal`] and [`FormDecoder::derive`]. A seal that does /// not cover the requested form falls back to deriving rather than failing, /// since a form added after sealing is a real situation. pub fn resolve( seal: Option<&SealFile>, catalog: &Catalog, record: &AssessmentFile, form: &Form, ) -> Result { if let Some(seal) = seal { if seal.form(&form.id).is_some() { return FormDecoder::from_seal(seal, &form.id); } } FormDecoder::derive(catalog, record, form) } /// The question at a printed position. /// /// # Arguments /// /// * `position` - the printed position, counting from 1. pub fn at_position(&self, position: u32) -> Option<&QuestionMap> { self.by_position.get(&position) } /// The question with a recorded number. /// /// # Arguments /// /// * `number` - the recorded number. pub fn at_number(&self, number: u32) -> Option<&QuestionMap> { self.by_number.get(&number) } /// The question an export's numbering refers to. /// /// # Arguments /// /// * `n` - the number as the export gives it. /// * `numbering` - how the export numbers questions. pub fn lookup(&self, n: u32, numbering: Numbering) -> Option<&QuestionMap> { match numbering { Numbering::Printed => self.at_position(n), Numbering::Recorded => self.at_number(n), } } /// How many questions this form prints. pub fn len(&self) -> usize { self.by_position.len() } /// Every recorded question number this form carries. pub fn numbers(&self) -> impl Iterator + '_ { self.by_position.values().map(|q| q.number) } /// Whether the form prints nothing, which means the record is empty. pub fn is_empty(&self) -> bool { self.by_position.is_empty() } /// Whether any question on this form has permuted options. /// /// Used to decide whether to say anything about translation at all: on an /// unshuffled form the whole mechanism is an identity map and mentioning it /// is noise. pub fn is_permuted(&self) -> bool { self.by_position.values().any(|q| q.is_permuted()) } /// Whether printed positions and recorded numbers disagree anywhere. /// /// True when items were shuffled, and also when a bonus item sits mid-record, /// since the layout moves bonus items to the end of the paper. pub fn is_renumbered(&self) -> bool { self.by_position.values().any(|q| q.position != q.number) } } /// What a directory of graded questions says about which form it holds. #[derive(Debug, Clone)] pub struct FormFit { /// The form id. pub form: String, /// Questions whose graded key matched this form's printed key. pub matched: usize, /// Questions that could be compared at all. pub compared: usize, /// Question positions where the graded key disagreed. pub mismatches: Vec, } impl FormFit { /// The share of comparable questions that agreed. pub fn rate(&self) -> f64 { if self.compared == 0 { 0.0 } else { self.matched as f64 / self.compared as f64 } } /// Whether the fit is good enough to proceed without a warning. /// /// The threshold is high on purpose. A correctly matched directory agrees on /// every question; anything less than total agreement is either a regrade that /// moved a key or the wrong directory, and both are worth a sentence. pub fn is_convincing(&self) -> bool { self.compared > 0 && self.matched == self.compared } } /// Compares a parsed Gradescope directory against one form's expected keys. /// /// # Arguments /// /// * `questions` - the parsed question files. /// * `decoder` - the form to test against. /// * `numbering` - how the export numbers questions. /// /// # Returns /// /// The fit. pub fn fit_form(questions: &[Question], decoder: &FormDecoder, numbering: Numbering) -> FormFit { let mut matched = 0usize; let mut compared = 0usize; let mut mismatches = Vec::new(); for question in questions { let Some(map) = decoder.lookup(question.number, numbering) else { continue; }; let graded: BTreeSet = question.keyed().into_iter().collect(); if graded.is_empty() { continue; } let expected: BTreeSet = map.printed_key.iter().cloned().collect(); if expected.is_empty() { continue; } compared += 1; if graded == expected { matched += 1; } else { mismatches.push(question.number); } } FormFit { form: decoder.form.clone(), matched, compared, mismatches, } } /// Ranks every candidate form against a directory. /// /// # Arguments /// /// * `questions` - the parsed question files. /// * `decoders` - one decoder per declared form. /// * `numbering` - how the export numbers questions. /// /// # Returns /// /// The fits, best first. pub fn identify_form( questions: &[Question], decoders: &[FormDecoder], numbering: Numbering, ) -> Vec { let mut fits: Vec = decoders .iter() .map(|d| fit_form(questions, d, numbering)) .collect(); fits.sort_by(|a, b| { b.rate() .partial_cmp(&a.rate()) .unwrap_or(std::cmp::Ordering::Equal) .then_with(|| a.form.cmp(&b.form)) }); fits } /// Explains a fit in a sentence, or says nothing when the fit is perfect. /// /// # Arguments /// /// * `claimed` - the form the directory was ingested as. /// * `fits` - every form's fit, best first. /// /// # Returns /// /// A warning, or `None`. pub fn form_warning(claimed: &str, fits: &[FormFit]) -> Option { let mine = fits.iter().find(|f| f.form.eq_ignore_ascii_case(claimed))?; if mine.is_convincing() { return None; } if mine.compared == 0 { return Some(format!( "form {claimed}: the export carries no point values, so the graded keys could not be \ checked against the form. Nothing verified this directory is form {claimed}" )); } let better = fits .iter() .find(|f| !f.form.eq_ignore_ascii_case(claimed) && f.rate() > mine.rate()); let head = format!( "form {claimed}: the graded key matches this form on {} of {} question(s)", mine.matched, mine.compared ); let where_ = if mine.mismatches.is_empty() { String::new() } else { let list: Vec = mine .mismatches .iter() .take(8) .map(|n| n.to_string()) .collect(); format!( " (q{}{})", list.join(", q"), if mine.mismatches.len() > 8 { ", …" } else { "" } ) }; match better { Some(other) => Some(format!( "{head}{where_}, but matches form {} on {} of {}. This directory is almost certainly \ form {}, not form {claimed}", other.form, other.matched, other.compared, other.form )), None => Some(format!( "{head}{where_}. Either those questions were regraded after printing, or the form is \ not the one named" )), } } /// Translates a form's responses into the bank's vocabulary. /// /// Rewrites, for every row whose `form` matches this decoder: /// /// * `item_number`, from printed position to recorded number, when the export /// numbers by position; /// * `form_position`, recording where the question sat on the page; /// * `selected_source` and `eliminated_source`, the bank letters for what was /// marked. The printed letters stay in `selected` and `eliminated`, because what /// a student physically marked is the fact and the translation is the /// interpretation. /// /// Correctness and credit are untouched. Both come from the grading platform, /// which scored the paper the student actually held, and are already right. /// /// # Arguments /// /// * `set` - the responses to translate, in place. /// * `decoder` - the form's mapping. /// * `numbering` - how the export numbered questions. /// /// # Returns /// /// Warnings for anything that could not be translated. pub fn apply(set: &mut ResponseSet, decoder: &FormDecoder, numbering: Numbering) -> Vec { let mut warnings = Vec::new(); let mut unmapped_positions: BTreeSet = BTreeSet::new(); let mut colliding_positions: BTreeSet = BTreeSet::new(); let mut unmapped_letters: BTreeSet = BTreeSet::new(); let mut translated = 0usize; // Numbers this form really uses. An untranslated row whose raw number is one // of these would silently masquerade as that question, and two rows would // then share a number: one the student's answer to it, one an answer to // something else entirely. Nothing downstream can tell them apart, so the // collision has to be caught here. let recorded: BTreeSet = decoder.numbers().collect(); for row in &mut set.rows { let belongs = row .form .as_deref() .map(|f| f.eq_ignore_ascii_case(&decoder.form)) .unwrap_or(false); if !belongs { continue; } let Some(map) = decoder.lookup(row.item_number, numbering) else { unmapped_positions.insert(row.item_number); if recorded.contains(&row.item_number) { colliding_positions.insert(row.item_number); // Marked so the row can be discarded below. Attributing it to // the question that legitimately holds this number would corrupt // that question's statistics. row.form = Some(UNMAPPED.to_string()); } continue; }; row.form_position = Some(map.position); row.item_number = map.number; let mut convert = |letters: &[String]| -> Vec { let mut out = Vec::with_capacity(letters.len()); for letter in letters { match map.canonical(letter) { Some(canonical) => out.push(canonical.to_string()), None => { unmapped_letters.insert(format!("q{} {}", map.number, letter)); } } } out.sort(); out }; row.selected_source = convert(&row.selected); row.eliminated_source = convert(&row.eliminated); translated += 1; } if !unmapped_positions.is_empty() { let list: Vec = unmapped_positions.iter().map(|n| n.to_string()).collect(); warnings.push(format!( "form {}: question(s) {} are in the export but not on this form ({} printed). The \ export may have been taken before a question was dropped, or from a different \ form.", decoder.form, list.join(", "), decoder.len() )); } if !colliding_positions.is_empty() { let list: Vec = colliding_positions.iter().map(|n| n.to_string()).collect(); let discarded = set.rows.len(); set.rows.retain(|row| row.form.as_deref() != Some(UNMAPPED)); warnings.push(format!( "form {}: {} response(s) at position(s) {} could not be translated, and their raw \ numbers are numbers this form does use. Keeping them would have given those \ questions two different answers each, so they were discarded. This is the shape of \ an export made before a question was dropped: re-export the responses from the \ administration you sealed, or re-run with --recorded-numbers if the export already \ carries recorded numbers.", decoder.form, discarded - set.rows.len(), list.join(", ") )); } if !unmapped_letters.is_empty() { let list: Vec = unmapped_letters.iter().take(10).cloned().collect(); warnings.push(format!( "form {}: {} marked option(s) are not options on the printed form ({}), which usually \ means a rubric column was added by hand in Gradescope", decoder.form, unmapped_letters.len(), list.join(", ") )); } if translated == 0 { warnings.push(format!( "form {}: no response rows carried this form id, so nothing was translated", decoder.form )); } warnings } #[cfg(test)] mod tests { use super::*; use crate::responses::{Response, administration_id}; fn map(position: u32, number: u32, pairs: &[(&str, &str)], key: &str) -> QuestionMap { let mut to_canonical = BTreeMap::new(); let mut to_printed = BTreeMap::new(); for (printed, canonical) in pairs { to_canonical.insert(printed.to_string(), canonical.to_string()); to_printed.insert(canonical.to_string(), printed.to_string()); } let printed_key = vec![to_printed.get(key).cloned().unwrap_or_default()]; QuestionMap { position, number, item: format!("b::q-{number}"), printed_key, canonical_key: vec![key.to_string()], to_canonical, to_printed, } } fn decoder() -> FormDecoder { FormDecoder::assemble( "B".to_string(), Provenance::Seal, vec![ // Printed A..D show bank C, A, D, B. The key is bank A, printed B. map(1, 1, &[("A", "C"), ("B", "A"), ("C", "D"), ("D", "B")], "A"), // A bonus item recorded as 9 but printed last, at position 2. map(2, 9, &[("A", "B"), ("B", "A")], "B"), ], ) } fn row(number: u32, selected: &str) -> Response { Response { administration_id: administration_id("C", "2026f", "e1"), course: "C".into(), term: "2026f".into(), assessment_id: "e1".into(), date: None, form: Some("B".into()), student_key: "s1".into(), sid: None, name: None, email: None, section: None, item_number: number, form_position: None, item_ref: None, item_version: None, variant: None, selected: vec![selected.into()], eliminated: Vec::new(), selected_source: Vec::new(), eliminated_source: Vec::new(), correct: Some(false), credit: 0.0, points_possible: 1.0, score: 0.0, response_time_seconds: None, level: None, learning_targets: Vec::new(), topics: Vec::new(), bonus: false, dropped: false, dropped_full_credit: false, } } #[test] fn printed_letters_become_bank_letters() { let decoder = decoder(); let q = decoder.at_position(1).unwrap(); assert_eq!(q.canonical("A"), Some("C")); assert_eq!(q.canonical("D"), Some("B")); assert_eq!(q.canonical("E"), None); assert_eq!(q.printed_key, vec!["B".to_string()]); } #[test] fn positions_become_recorded_numbers() { let mut set = ResponseSet::new(); set.rows.push(row(2, "A")); let warnings = apply(&mut set, &decoder(), Numbering::Printed); assert!(warnings.is_empty(), "{warnings:?}"); assert_eq!(set.rows[0].item_number, 9, "position 2 is recorded as 9"); assert_eq!(set.rows[0].form_position, Some(2)); assert_eq!(set.rows[0].selected, vec!["A".to_string()], "printed kept"); assert_eq!(set.rows[0].selected_source, vec!["B".to_string()]); } #[test] fn rows_from_another_form_are_left_alone() { let mut set = ResponseSet::new(); let mut other = row(1, "A"); other.form = Some("A".into()); set.rows.push(other); apply(&mut set, &decoder(), Numbering::Printed); assert!(set.rows[0].selected_source.is_empty()); assert_eq!(set.rows[0].item_number, 1); } #[test] fn an_unknown_position_is_reported_not_guessed() { let mut set = ResponseSet::new(); set.rows.push(row(7, "A")); let warnings = apply(&mut set, &decoder(), Numbering::Printed); assert!( warnings.iter().any(|w| w.contains("not on this form")), "{warnings:?}" ); assert_eq!(set.rows[0].item_number, 7, "left as found"); } #[test] fn a_perfect_fit_says_nothing() { let fits = vec![FormFit { form: "A".into(), matched: 30, compared: 30, mismatches: Vec::new(), }]; assert!(form_warning("A", &fits).is_none()); } #[test] fn a_swapped_directory_names_the_form_it_really_is() { let fits = vec![ FormFit { form: "B".into(), matched: 30, compared: 30, mismatches: Vec::new(), }, FormFit { form: "A".into(), matched: 8, compared: 30, mismatches: (1..=22).collect(), }, ]; let warning = form_warning("A", &fits).expect("a mismatch this large must warn"); assert!(warning.contains("almost certainly form B"), "{warning}"); } #[test] fn a_single_regraded_key_warns_without_accusing_the_wrong_form() { let fits = vec![FormFit { form: "A".into(), matched: 29, compared: 30, mismatches: vec![14], }]; let warning = form_warning("A", &fits).unwrap(); assert!(warning.contains("q14"), "{warning}"); assert!(warning.contains("regraded"), "{warning}"); } }