// SPDX-License-Identifier: Prosperity-3.0.0 // Copyright Scienting // Source: https://git.scient.ing/education/coursebank //! Reading several forms of one exam back in at once. //! //! A two-form exam is two Gradescope assignments, each exporting its own //! directory of `1.csv` through `N.csv`, each numbered against its own paper. They //! are one administration: one set of students, one item pool, one set of //! statistics. Ingesting them one command at a time does not work, because the //! store keys on the administration and the second write replaces the first. //! //! So this module takes the whole set: //! //! ```text //! coursebank ingest gradescope A=exports/e1-a B=exports/e1-b --assessment e1 //! ``` //! //! and does four things the single-directory path cannot: //! //! *Checks each directory is the form it claims to be.* Every export carries the //! graded key in its point-value row; every form knows what its printed key should //! be. Comparing them catches a swapped pair of directories immediately rather //! than three weeks later, when the distractor table looks strange. See //! [`crate::decode::identify_form`]. //! //! *Translates each form into the bank's vocabulary before merging.* Otherwise //! form A's option C and form B's option C land in the same column of the same //! table while meaning different things. //! //! *Merges into one response set*, written once, so `analyze` and `report` see the //! whole class. //! //! *Reports what the merge revealed*: a student who appears on two forms, a form //! that is missing a question the other has, a form that ran materially harder //! than the other. use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; use crate::assessment::{AssessmentFile, Form}; use crate::catalog::Catalog; use crate::date::Date; use crate::decode::{self, FormDecoder, FormFit, Numbering}; use crate::error::{Error, Result}; use crate::gradescope::{self, Context, Question}; use crate::responses::ResponseSet; use crate::seal::SealFile; /// One directory of graded questions, and the form it holds. #[derive(Debug, Clone)] pub struct Source { /// The form id. pub form: String, /// The directory of per-question CSV exports. pub dir: PathBuf, } impl Source { /// Parses a `FORM=DIR` argument. /// /// A bare path is accepted and takes the fallback form, so the single-form /// case stays as short as it was. /// /// # Arguments /// /// * `text` - the argument, e.g. `A=exports/e1-a` or `exports/e1`. /// * `fallback` - the form to use when the argument names none. /// /// # Returns /// /// The source. /// /// # Errors /// /// Returns [`Error::Usage`] when the argument names no form and no fallback /// was given, or when the form label is empty. pub fn parse(text: &str, fallback: Option<&str>) -> Result { // Split on the first `=` only: a directory name may contain one, a form // label may not. if let Some((form, dir)) = text.split_once('=') { let form = form.trim(); if form.is_empty() { return Err(Error::usage(format!( "`{text}` has an empty form label; write it as FORM=DIR, e.g. A=exports/e1-a" ))); } if !dir.trim().is_empty() { return Ok(Source { form: form.to_string(), dir: PathBuf::from(dir.trim()), }); } } match fallback { Some(form) => Ok(Source { form: form.to_string(), dir: PathBuf::from(text.trim()), }), None => Err(Error::usage(format!( "`{text}` does not say which form it holds; write it as FORM=DIR (e.g. \ A=exports/e1-a) or pass --form" ))), } } } /// How to run an intake. #[derive(Debug, Clone)] pub struct Options { /// The administration date, overriding the record's. pub date: Option, /// How the exports number their questions. pub numbering: Numbering, /// Whether a form that fails its key check stops the ingest. /// /// On by default. A directory that does not match the form it was named as is /// the one ingest error that produces confident, wrong analysis rather than an /// obvious failure, so the default is to refuse and say so. pub strict: bool, } impl Default for Options { fn default() -> Options { Options { date: None, numbering: Numbering::Printed, strict: true, } } } /// What one form's directory contributed. #[derive(Debug, Clone)] pub struct FormIntake { /// The form id. pub form: String, /// The directory it came from. pub dir: PathBuf, /// Where its mapping came from. pub provenance: decode::Provenance, /// How many students it held. pub students: usize, /// How many questions it held. pub questions: usize, /// How the graded keys compared to every declared form, best first. pub fits: Vec, /// The parsed question files, kept so grading-time decisions stay available. pub parsed: Vec, } impl FormIntake { /// This form's own fit. pub fn own_fit(&self) -> Option<&FormFit> { self.fits .iter() .find(|f| f.form.eq_ignore_ascii_case(&self.form)) } } /// The result of reading every form of one administration. #[derive(Debug, Clone)] pub struct Intake { /// The merged, translated responses. pub responses: ResponseSet, /// Per-form detail. pub forms: Vec, /// Problems that did not stop the ingest. pub warnings: Vec, } impl Intake { /// Every parsed question file, across forms. /// /// # Returns /// /// Pairs of form id and question. pub fn questions(&self) -> Vec<(&str, &Question)> { self.forms .iter() .flat_map(|f| f.parsed.iter().map(move |q| (f.form.as_str(), q))) .collect() } } /// Reads every source into one response set. /// /// # Arguments /// /// * `catalog` - the loaded course. /// * `record` - the assessment record. /// * `seal` - the seal, when one was written. Strongly preferred: it describes the /// paper that was printed rather than the paper the bank would print today. /// * `sources` - the directories and the forms they hold. /// * `opts` - how to run. /// /// # Returns /// /// The merged intake. /// /// # Errors /// /// Returns [`Error::Usage`] when a source names a form the record does not /// declare, when two sources name the same form, or when a key check fails under /// `strict`. Propagates parse errors from the exports themselves. pub fn run( catalog: &Catalog, record: &AssessmentFile, seal: Option<&SealFile>, sources: &[Source], opts: &Options, ) -> Result { if sources.is_empty() { return Err(Error::usage( "no directories to ingest; pass one per form, e.g. A=exports/e1-a B=exports/e1-b" .to_string(), )); } let declared = declared_forms(record); let mut seen: BTreeSet = BTreeSet::new(); for source in sources { if !seen.insert(source.form.to_ascii_uppercase()) { return Err(Error::usage(format!( "form {} was given twice; each form is one directory", source.form ))); } if !declared .iter() .any(|f| f.id.eq_ignore_ascii_case(&source.form)) { return Err(Error::usage(format!( "the record for `{}` declares no form `{}`; it declares {}", record.assessment.id, source.form, declared .iter() .map(|f| f.id.as_str()) .collect::>() .join(", ") ))); } } // One decoder per declared form, not just per ingested form: identifying a // swapped directory means testing it against the forms it might be. let mut decoders: Vec = Vec::new(); for form in &declared { decoders.push(FormDecoder::resolve(seal, catalog, record, form)?); } let mut merged = ResponseSet::new(); let mut forms = Vec::new(); let mut warnings = Vec::new(); let mut blocking = Vec::new(); for source in sources { let decoder = decoders .iter() .find(|d| d.form.eq_ignore_ascii_case(&source.form)) .expect("every source's form was checked against the declared list"); let ctx = Context { course: catalog.course.course.code.clone(), term: record .assessment .term .clone() .unwrap_or_else(|| catalog.course.course.term.clone()), assessment_id: record.assessment.id.clone(), date: opts.date.or(record.assessment.date), form: Some(decoder.form.clone()), }; let import = gradescope::ingest_dir(&source.dir, &ctx)?; let mut set = import.responses; let fits = decode::identify_form(&import.questions, &decoders, opts.numbering); if let Some(problem) = decode::form_warning(&decoder.form, &fits) { let message = format!("{} [{}]", problem, source.dir.display()); let convincing_alternative = fits .iter() .any(|f| !f.form.eq_ignore_ascii_case(&decoder.form) && f.is_convincing()); if opts.strict && convincing_alternative { blocking.push(message); } else { warnings.push(message); } } warnings.extend(decode::apply(&mut set, decoder, opts.numbering)); forms.push(FormIntake { form: decoder.form.clone(), dir: source.dir.clone(), provenance: decoder.provenance, students: set.students().len(), questions: set.all_items().len(), fits, parsed: import.questions, }); merged.absorb(set); } if !blocking.is_empty() { blocking.push( "Nothing was written. Fix the form labels, or pass --allow-mismatch if the keys really \ did change after printing." .to_string(), ); return Err(Error::Invalid(blocking)); } warnings.extend(cross_form_checks(&merged, &forms, record)); merged.warnings.extend(warnings.clone()); Ok(Intake { responses: merged, forms, warnings, }) } /// The forms a record declares, with the implicit single form for a record that /// declares none. fn declared_forms(record: &AssessmentFile) -> Vec
{ if record.forms.is_empty() { vec![Form { id: "A".to_string(), seed: 0, shuffle_items: false, shuffle_options: false, }] } else { record.forms.clone() } } /// Checks that only merging several forms can make. fn cross_form_checks( merged: &ResponseSet, forms: &[FormIntake], record: &AssessmentFile, ) -> Vec { let mut out = Vec::new(); if forms.len() < 2 { return out; } // A student on two forms sat one exam and was graded twice, or two people // share an identifier. Either way the response set now double counts them. let mut by_student: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new(); for row in &merged.rows { if let Some(form) = row.form.as_deref() { by_student .entry(row.student_key.as_str()) .or_default() .insert(form); } } let doubled: Vec<&str> = by_student .iter() .filter(|(_, forms)| forms.len() > 1) .map(|(student, _)| *student) .collect(); if !doubled.is_empty() { out.push(format!( "{} student(s) appear on more than one form ({}); each is counted twice in every \ total until one submission is removed", doubled.len(), doubled .iter() .take(5) .copied() .collect::>() .join(", ") )); } // Every form is the same items in a different order, so a question present on // one and absent from another means a directory is short a file. let expected: BTreeSet = record .items .iter() .filter(|p| !p.dropped) .map(|p| p.number) .collect(); for form in forms { let present: BTreeSet = merged .rows .iter() .filter(|r| { r.form .as_deref() .map(|f| f.eq_ignore_ascii_case(&form.form)) .unwrap_or(false) }) .map(|r| r.item_number) .collect(); let missing: Vec = expected .difference(&present) .map(|n| n.to_string()) .collect(); if !missing.is_empty() { out.push(format!( "form {}: no responses for question(s) {}; the export directory is missing those \ files", form.form, missing.join(", ") )); } } // Forms are meant to be the same test. A large gap between their means is // either a permutation that made one form easier or an uneven split of the // class, and both are worth knowing before any grade is released. let means: Vec<(String, f64, usize)> = forms .iter() .map(|f| { let students: Vec<&str> = merged .rows .iter() .filter(|r| { r.form .as_deref() .map(|x| x.eq_ignore_ascii_case(&f.form)) .unwrap_or(false) }) .map(|r| r.student_key.as_str()) .collect::>() .into_iter() .collect(); let possible = merged.points_available(); let percents: Vec = students .iter() .map(|s| { if possible > 0.0 { 100.0 * merged.scored_total(s) / possible } else { 0.0 } }) .collect(); let mean = if percents.is_empty() { 0.0 } else { percents.iter().sum::() / percents.len() as f64 }; (f.form.clone(), mean, percents.len()) }) .collect(); if let (Some(low), Some(high)) = ( means .iter() .min_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal)), means .iter() .max_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal)), ) { let gap = high.1 - low.1; if gap >= 5.0 && low.2 >= 5 && high.2 >= 5 { out.push(format!( "form {} averaged {:.0}% and form {} averaged {:.0}%, a {:.0}-point gap. With {} \ and {} students that may be the split rather than the forms, but it is worth \ looking at before releasing grades", high.0, high.1, low.0, low.1, gap, high.2, low.2 )); } } out } /// A one-line summary of where a directory came from and what it held. /// /// # Arguments /// /// * `intake` - one form's intake. /// /// # Returns /// /// The line, without a trailing newline. pub fn describe(intake: &FormIntake) -> String { let fit = match intake.own_fit() { Some(fit) if fit.compared > 0 => { format!(", keys matched {}/{}", fit.matched, fit.compared) } _ => String::new(), }; format!( "form {}: {} student(s) x {} question(s) from {} (mapping {}{fit})", intake.form, intake.students, intake.questions, intake.dir.display(), intake.provenance.label(), ) } /// Whether a path looks like a Gradescope per-question export directory. /// /// Used to give a better error than "no numbered CSV files" when someone points /// at the zip they downloaded, or at the directory above the one they meant. /// /// # Arguments /// /// * `dir` - the candidate directory. /// /// # Returns /// /// `true` when it holds at least one file named like `1.csv` or `1_Title.csv`. pub fn looks_like_export(dir: &Path) -> bool { let Ok(entries) = std::fs::read_dir(dir) else { return false; }; entries .filter_map(|e| e.ok()) .any(|entry| gradescope::question_number(&entry.path()).is_some()) } #[cfg(test)] mod tests { use super::*; #[test] fn a_labelled_source_parses() { let source = Source::parse("B=exports/e1-b", None).unwrap(); assert_eq!(source.form, "B"); assert_eq!(source.dir, PathBuf::from("exports/e1-b")); } #[test] fn a_bare_path_needs_a_fallback_form() { let source = Source::parse("exports/e1", Some("A")).unwrap(); assert_eq!(source.form, "A"); assert_eq!(source.dir, PathBuf::from("exports/e1")); let err = Source::parse("exports/e1", None).unwrap_err(); assert!(err.to_string().contains("which form"), "{err}"); } #[test] fn a_windows_style_path_is_not_mistaken_for_a_label() { // The split is on the first `=`, and a drive letter has none, so this is // only a hazard for a path that genuinely contains one. let source = Source::parse("A=C:/exports/e1-a", None).unwrap(); assert_eq!(source.form, "A"); assert_eq!(source.dir, PathBuf::from("C:/exports/e1-a")); } #[test] fn an_empty_label_is_refused() { let err = Source::parse("=exports/e1", None).unwrap_err(); assert!(err.to_string().contains("empty form label"), "{err}"); } }