// SPDX-License-Identifier: Prosperity-3.0.0 // Copyright Scientific Computing Studio // Source: https://git.scient.ing/education/coursebank //! Reading Gradescope's per-question CSV exports. //! //! Gradescope exports one file per question, named `1.csv` through `N.csv`, in //! wide form. Every assumption encoded here was checked against a real 32-question, //! 24-student export rather than inferred from documentation, because several of //! them are not what the format suggests. //! //! The layout is: //! //! ```text //! Assignment Submission ID, Question Submission ID, First Name, Last Name, //! SID, Email, Sections, Score, Submission Time, , //! Adjustment, Comments, Grader, Tags //! ``` //! //! The rubric columns are the header slice strictly between `Submission Time` //! and `Adjustment`. Student cells in those columns are the literal strings //! `true` and `false`. //! //! Four things about real exports that a naive parser gets wrong: //! //! *Trailing rows are not aligned with the header.* After the student rows come //! `Point Values`, `Rubric Numbers`, and `Scoring Method` rows with a label, some //! empty cells, and then the numbers. The CSV reader must be in flexible mode or //! it errors on the whole file. //! //! *Rubric labels are not always bare option letters.* Real ones include //! `Selected C`, `Eliminated A`, and — this is the interesting case — a letter //! followed by an instructor's parenthetical, such as //! `B (Technically speaking, this answer describes HBD/HBA, but I can see how //! this distractor is poorly written.)`. Those columns carry partial credit. //! //! *Partial credit awarded to a distractor is evidence, not noise.* When you gave //! 1.0 of 1.5 points for choosing B, you decided at grading time that B was //! partly defensible. That is exactly the ambiguity signal item analysis is //! trying to detect, so it is captured as [`Flag::Ambiguous`] rather than //! rounded away. //! //! *Bonus questions have `max_points` of zero* while a rubric column still //! carries points. Key detection therefore uses the largest value across the //! maximum *and* every column, not the maximum alone. use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; use crate::date::Date; use crate::error::{Error, Result}; use crate::responses::{Response, ResponseSet, administration_id}; use crate::taxonomy::Flag; /// What a rubric column represents. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ColumnKind { /// The student selected this option. Select, /// The student eliminated this option, under elimination scoring. Eliminate, /// Something else: a catch-all rubric row such as /// `Incorrect selection with no eliminations`. Other, } /// One rubric column. #[derive(Debug, Clone)] pub struct RubricColumn { /// The header label, verbatim. pub label: String, /// What the column means. pub kind: ColumnKind, /// The option letter, when the label named one. pub letter: Option, /// The instructor's parenthetical annotation, when there was one. Worth /// keeping: it is usually the instructor explaining why an item is flawed. pub note: Option, /// Points this column awards, from the `Point Values` row. pub value: Option, } /// One student's row in a question file. #[derive(Debug, Clone)] pub struct StudentRow { /// The institutional student id. pub sid: Option, /// First and last name joined. pub name: Option, /// The email. pub email: Option, /// Section or lab. pub section: Option, /// Points awarded, authoritative. pub score: f64, /// The submission timestamp, verbatim. pub submission_time: Option, /// Indices of rubric columns marked `true`. pub marks: Vec, } /// One parsed question file. #[derive(Debug, Clone)] pub struct Question { /// The question number, from the file name. pub number: u32, /// Where it came from. pub path: PathBuf, /// The rubric columns, in header order. pub columns: Vec, /// The maximum points from the `Point Values` row. pub max_points: f64, /// The scoring method, when stated. pub scoring_method: Option, /// The student rows. pub rows: Vec, } impl Question { /// The largest point value anywhere in the question. /// /// Bonus questions record a maximum of zero while still awarding points, so /// the key has to be found by looking at the columns too. /// /// # Returns /// /// The largest value seen. pub fn top_value(&self) -> f64 { let mut top = self.max_points; for c in &self.columns { if let Some(v) = c.value { if v > top { top = v; } } } top } /// The option letters that earn full credit. /// /// # Returns /// /// The keyed letters, sorted. Empty when the point values were absent. pub fn keyed(&self) -> Vec { let top = self.top_value(); if top <= 0.0 { return Vec::new(); } let mut out: BTreeSet = BTreeSet::new(); for c in &self.columns { if c.kind == ColumnKind::Select { if let (Some(letter), Some(v)) = (&c.letter, c.value) { if (v - top).abs() < 1e-9 { out.insert(letter.clone()); } } } } out.into_iter().collect() } /// Distractors that were awarded partial credit at grading time. /// /// # Returns /// /// Letter, points awarded, and the instructor's note if any. pub fn partial_credit(&self) -> Vec<(String, f64, Option)> { let top = self.top_value(); let mut out = Vec::new(); for c in &self.columns { if c.kind != ColumnKind::Select { continue; } if let (Some(letter), Some(v)) = (&c.letter, c.value) { if v > 0.0 && v < top - 1e-9 { out.push((letter.clone(), v, c.note.clone())); } } } out } /// Whether this looks like a bonus question. /// /// # Returns /// /// `true` when the maximum is zero but a column still awards points. pub fn looks_like_bonus(&self) -> bool { self.max_points <= 0.0 && self.top_value() > 0.0 } /// Whether elimination scoring was in use. pub fn uses_elimination(&self) -> bool { self.columns.iter().any(|c| c.kind == ColumnKind::Eliminate) } /// Points the question was worth. /// /// # Returns /// /// The stated maximum, falling back to the largest column value for bonus /// questions so that credit is still a meaningful fraction. pub fn points_possible(&self) -> f64 { if self.max_points > 0.0 { self.max_points } else { self.top_value() } } /// Flags implied by how the question was graded. /// /// # Returns /// /// [`Flag::Ambiguous`] when a distractor received partial credit. pub fn implied_flags(&self) -> Vec { if self.partial_credit().is_empty() { Vec::new() } else { vec![Flag::Ambiguous] } } } /// What the ingest needs to know that the export does not say. #[derive(Debug, Clone)] pub struct Context { /// The course code. pub course: String, /// The term. pub term: String, /// The assessment id. pub assessment_id: String, /// The administration date. pub date: Option, /// The form, when forms were used. pub form: Option, } /// The result of reading a directory of question files. #[derive(Debug, Clone)] pub struct Import { /// The normalized responses. pub responses: ResponseSet, /// The parsed question files, kept so grading-time decisions can be folded /// into item calibration. pub questions: Vec, } /// Classifies a rubric column label. /// /// # Arguments /// /// * `label` - the header text. /// /// # Returns /// /// The kind, the option letter if the label named one, and any trailing /// annotation with its surrounding punctuation trimmed. pub fn classify(label: &str) -> (ColumnKind, Option, Option) { let trimmed = label.trim(); let lower = trimmed.to_ascii_lowercase(); let (kind, rest) = if let Some(r) = lower.strip_prefix("selected ") { ( ColumnKind::Select, trimmed[trimmed.len() - r.len()..].trim(), ) } else if let Some(r) = lower.strip_prefix("eliminated ") { ( ColumnKind::Eliminate, trimmed[trimmed.len() - r.len()..].trim(), ) } else { (ColumnKind::Select, trimmed) }; let mut chars = rest.chars(); let Some(first) = chars.next() else { return (ColumnKind::Other, None, Some(label.to_string())); }; let upper = first.to_ascii_uppercase(); let tail = chars.as_str(); // A single letter A-H, optionally followed by an annotation that does not // begin with another alphanumeric. `B (poorly written)` is option B; // `Both A and C are wrong` is not. let letter_like = upper.is_ascii_uppercase() && ('A'..='H').contains(&upper) && tail .chars() .next() .map(|c| !c.is_alphanumeric()) .unwrap_or(true); if !letter_like { return (ColumnKind::Other, None, Some(label.to_string())); } let note = tail.trim().trim_matches(|c: char| "().,;: ".contains(c)); let note = if note.is_empty() { None } else { Some(note.to_string()) }; (kind, Some(upper.to_string()), note) } /// Parses one question file. /// /// # Arguments /// /// * `path` - the CSV file. /// /// # Returns /// /// The parsed question. /// /// # Errors /// /// Returns [`Error::Csv`] on a malformed file and [`Error::Invalid`] when the /// header lacks the `Submission Time` column that delimits the rubric. pub fn parse_question(path: &Path) -> Result { let number = path .file_stem() .and_then(|s| s.to_str()) .and_then(|s| s.parse::().ok()) .ok_or_else(|| { Error::Invalid(vec![format!( "{} is not named like a Gradescope question export (expected `12.csv`)", path.display() )]) })?; // Flexible mode is required: the trailing `Point Values` rows are shorter // than the header. let mut reader = csv::ReaderBuilder::new() .flexible(true) .has_headers(false) .from_path(path) .map_err(|e| Error::Csv { path: path.to_path_buf(), source: e, })?; let mut records = reader.records(); let header = match records.next() { Some(r) => r.map_err(|e| Error::Csv { path: path.to_path_buf(), source: e, })?, None => return Err(Error::Invalid(vec![format!("{} is empty", path.display())])), }; let header: Vec = header.iter().map(|s| s.trim().to_string()).collect(); let index_of = |name: &str| header.iter().position(|h| h == name); let start = match index_of("Submission Time") { Some(i) => i + 1, None => { return Err(Error::Invalid(vec![format!( "{} has no `Submission Time` column, so the rubric columns cannot be located; \ is this a Gradescope per-question export?", path.display() )])); } }; let end = index_of("Adjustment").unwrap_or(header.len()); if end < start { return Err(Error::Invalid(vec![format!( "{} has `Adjustment` before `Submission Time`", path.display() )])); } let mut columns: Vec = header[start..end] .iter() .map(|label| { let (kind, letter, note) = classify(label); RubricColumn { label: label.clone(), kind, letter, note, value: None, } }) .collect(); let sid_col = index_of("SID"); let first_col = index_of("First Name"); let last_col = index_of("Last Name"); let email_col = index_of("Email"); let section_col = index_of("Sections"); let score_col = index_of("Score"); let time_col = index_of("Submission Time"); let mut rows = Vec::new(); let mut max_points = 0.0f64; let mut point_values: Vec = Vec::new(); let mut scoring_method = None; for rec in records { let rec = rec.map_err(|e| Error::Csv { path: path.to_path_buf(), source: e, })?; let cell = |i: Option| -> Option { i.and_then(|i| rec.get(i)) .map(|s| s.trim().to_string()) .filter(|s| !s.is_empty()) }; if rec.iter().all(|c| c.trim().is_empty()) { continue; } let label = rec.get(0).unwrap_or("").trim(); match label { "Point Values" => { // Label, then some empty cells, then the maximum, then one value // per rubric column. Alignment was verified across every file in // a real export, but collecting the non-empty numbers in order is // robust to a leading blank moving. let nums: Vec = rec .iter() .skip(1) .filter(|c| !c.trim().is_empty()) .filter_map(|c| c.trim().parse::().ok()) .collect(); if let Some((first, rest)) = nums.split_first() { max_points = *first; point_values = rest.to_vec(); } continue; } "Rubric Numbers" => continue, "Scoring Method" => { scoring_method = cell(Some(1)); continue; } _ => {} } // A student row must have a score cell that parses; anything else is a // trailing annotation row we have not seen before, and skipping it is // safer than failing the import. let score = match cell(score_col).and_then(|s| s.parse::().ok()) { Some(s) => s, None => { if cell(sid_col).is_none() && cell(email_col).is_none() { continue; } 0.0 } }; let marks: Vec = (0..columns.len()) .filter(|i| { rec.get(start + i) .map(|c| c.trim().eq_ignore_ascii_case("true")) .unwrap_or(false) }) .collect(); let name = match (cell(first_col), cell(last_col)) { (Some(f), Some(l)) => Some(format!("{f} {l}")), (Some(f), None) => Some(f), (None, Some(l)) => Some(l), (None, None) => None, }; rows.push(StudentRow { sid: cell(sid_col), name, email: cell(email_col), section: cell(section_col), score, submission_time: cell(time_col), marks, }); } // Attach point values when they line up; when they do not, leave them off // rather than misattribute credit to the wrong option. if point_values.len() == columns.len() { for (c, v) in columns.iter_mut().zip(point_values.iter()) { c.value = Some(*v); } } Ok(Question { number, path: path.to_path_buf(), columns, max_points, scoring_method, rows, }) } /// Reads every question file in a directory. /// /// # Arguments /// /// * `dir` - the directory of `N.csv` files. /// * `ctx` - identifying information the export does not carry. /// /// # Returns /// /// Normalized responses and the parsed questions. /// /// # Errors /// /// Returns [`Error::Io`] when the directory cannot be read and [`Error::Invalid`] /// when it holds no question files. pub fn ingest_dir(dir: &Path, ctx: &Context) -> Result { let mut paths: Vec = std::fs::read_dir(dir) .map_err(|e| Error::io(dir, e))? .filter_map(|e| e.ok()) .map(|e| e.path()) .filter(|p| { p.extension().and_then(|e| e.to_str()) == Some("csv") && p.file_stem() .and_then(|s| s.to_str()) .map(|s| s.chars().all(|c| c.is_ascii_digit())) .unwrap_or(false) }) .collect(); if paths.is_empty() { return Err(Error::Invalid(vec![format!( "{} holds no numbered CSV files; Gradescope exports one file per question, \ named `1.csv` through `N.csv`", dir.display() )])); } // Numeric order, so `10.csv` does not sort before `2.csv`. paths.sort_by_key(|p| { p.file_stem() .and_then(|s| s.to_str()) .and_then(|s| s.parse::().ok()) .unwrap_or(u32::MAX) }); let mut questions = Vec::new(); for p in &paths { questions.push(parse_question(p)?); } Ok(to_responses(&questions, ctx)) } /// Normalizes parsed questions into responses. /// /// # Arguments /// /// * `questions` - the parsed question files. /// * `ctx` - identifying information. /// /// # Returns /// /// The import, with warnings for anything that looked wrong but not fatal. pub fn to_responses(questions: &[Question], ctx: &Context) -> Import { let mut set = ResponseSet::new(); let admin = administration_id(&ctx.course, &ctx.term, &ctx.assessment_id); let mut counts: BTreeMap = BTreeMap::new(); for q in questions { let keyed: BTreeSet = q.keyed().into_iter().collect(); let points = q.points_possible(); let bonus = q.looks_like_bonus(); if keyed.is_empty() { set.warnings.push(format!( "question {}: no keyed option could be identified from the point values, so \ correctness is unknown; scores are still recorded", q.number )); } for (letter, value, note) in q.partial_credit() { set.warnings.push(format!( "question {}: option {letter} was awarded {value} of {points} points at grading \ time, which is recorded as an ambiguity signal{}", q.number, note.map(|n| format!(" ({n})")).unwrap_or_default() )); } for row in &q.rows { let mut selected = Vec::new(); let mut eliminated = Vec::new(); let mut other = Vec::new(); for &i in &row.marks { let c = &q.columns[i]; match (c.kind, &c.letter) { (ColumnKind::Select, Some(l)) => selected.push(l.clone()), (ColumnKind::Eliminate, Some(l)) => eliminated.push(l.clone()), _ => other.push(c.label.clone()), } } selected.sort(); eliminated.sort(); let credit = if points > 0.0 { row.score / points } else { 0.0 }; let correct = if keyed.is_empty() { None } else if selected.is_empty() && eliminated.is_empty() && other.is_empty() { // A wholly unmarked row is a blank response, not a wrong one. None } else { let chosen: BTreeSet = selected.iter().cloned().collect(); Some(chosen == keyed) }; let student_key = row .sid .clone() .or_else(|| row.email.clone()) .unwrap_or_else(|| format!("unknown-{}", counts.len())); *counts.entry(q.number).or_insert(0) += 1; set.rows.push(Response { administration_id: admin.clone(), course: ctx.course.clone(), term: ctx.term.clone(), assessment_id: ctx.assessment_id.clone(), date: ctx.date, form: ctx.form.clone(), student_key, sid: row.sid.clone(), name: row.name.clone(), email: row.email.clone(), section: row.section.clone(), item_number: q.number, item_ref: None, item_version: None, selected, eliminated, correct, credit, points_possible: points, score: row.score, response_time_seconds: None, level: None, learning_objectives: Vec::new(), topics: Vec::new(), bonus, dropped: false, }); } } // Every question should have the same number of submissions. A mismatch // usually means a student was excused from one question, which is worth // saying out loud because it changes per-item denominators. let sizes: BTreeSet = counts.values().copied().collect(); if sizes.len() > 1 { let mut odd: Vec = Vec::new(); let modal = *sizes.iter().next_back().unwrap_or(&0); for (n, c) in &counts { if *c != modal { odd.push(format!("q{n} has {c}")); } } set.warnings.push(format!( "submission counts differ across questions (most have {modal}): {}", odd.join(", ") )); } Import { responses: set, questions: questions.to_vec(), } } #[cfg(test)] mod tests { use super::*; #[test] fn classifies_bare_letters() { let (k, l, n) = classify("C"); assert_eq!(k, ColumnKind::Select); assert_eq!(l, Some("C".to_string())); assert_eq!(n, None); } #[test] fn classifies_selected_and_eliminated_prefixes() { assert_eq!(classify("Selected C").0, ColumnKind::Select); assert_eq!(classify("Selected C").1, Some("C".to_string())); assert_eq!(classify("Eliminated A").0, ColumnKind::Eliminate); assert_eq!(classify("Eliminated A").1, Some("A".to_string())); } #[test] fn keeps_the_instructors_annotation() { // Verbatim from a real export. let (k, l, n) = classify( "B (Technically speaking, this answer describes HBD/HBA, but I can see how this \ distractor is poorly written.)", ); assert_eq!(k, ColumnKind::Select); assert_eq!(l, Some("B".to_string())); assert!(n.unwrap().starts_with("Technically speaking")); let (_, l2, n2) = classify("B (preparation does not select the one most likely to bind)"); assert_eq!(l2, Some("B".to_string())); assert!(n2.is_some()); } #[test] fn rejects_prose_rubric_rows() { // Also verbatim: these are genuinely not option columns. let (k, l, _) = classify("Incorrect selection with no eliminations."); assert_eq!(k, ColumnKind::Other); assert_eq!(l, None); assert_eq!(classify("").0, ColumnKind::Other); } #[test] fn does_not_mistake_a_sentence_for_option_a() { // Starts with `A`, but the next character is alphanumeric. assert_eq!(classify("Answered in the margin").0, ColumnKind::Other); } fn q(max: f64, cols: &[(&str, f64)]) -> Question { Question { number: 1, path: PathBuf::from("1.csv"), columns: cols .iter() .map(|(label, v)| { let (kind, letter, note) = classify(label); RubricColumn { label: label.to_string(), kind, letter, note, value: Some(*v), } }) .collect(), max_points: max, scoring_method: None, rows: Vec::new(), } } #[test] fn finds_the_key_by_top_value() { let question = q(1.5, &[("A", 0.0), ("B", 0.0), ("C", 1.5), ("D", 0.0)]); assert_eq!(question.keyed(), vec!["C".to_string()]); assert!(question.partial_credit().is_empty()); } #[test] fn finds_the_key_on_a_bonus_question_with_zero_maximum() { // Real case: q31 and q32 of the sample exam. let question = q(0.0, &[("A", 0.0), ("B", 1.5), ("C", 0.0)]); assert!(question.looks_like_bonus()); assert_eq!(question.keyed(), vec!["B".to_string()]); assert_eq!(question.points_possible(), 1.5); } #[test] fn partial_credit_to_a_distractor_flags_ambiguity() { // Real case: q14, where B earned 1.49 of 1.5. let question = q(1.5, &[("A", 0.0), ("B (poorly written)", 1.49), ("C", 1.5)]); assert_eq!(question.keyed(), vec!["C".to_string()]); let partial = question.partial_credit(); assert_eq!(partial.len(), 1); assert_eq!(partial[0].0, "B"); assert!(partial[0].2.is_some(), "keeps the note"); assert_eq!(question.implied_flags(), vec![Flag::Ambiguous]); } #[test] fn detects_elimination_scoring() { let question = q( 0.9, &[ ("Selected C", 0.9), ("Eliminated A", 0.3), ("Eliminated B", 0.3), ], ); assert!(question.uses_elimination()); assert_eq!(question.keyed(), vec!["C".to_string()]); } #[test] fn parses_a_realistic_file() { let dir = std::env::temp_dir().join(format!("cb-gs-{}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); let path = dir.join("7.csv"); std::fs::write( &path, "Assignment Submission ID,Question Submission ID,First Name,Last Name,SID,Email,\ Sections,Score,Submission Time,A,B,C,D,Adjustment,Comments,Grader,Tags\n\ 1,11,Ada,Lovelace,1234567,ada@x.edu,L01,1.5,2026-04-01T10:00:00Z,false,false,true,\ false,,,,\n\ 2,12,Alan,Turing,7654321,alan@x.edu,L01,0.0,2026-04-01T10:05:00Z,true,false,false,\ false,,,,\n\ 3,13,Grace,Hopper,1111111,grace@x.edu,L02,0.0,2026-04-01T10:06:00Z,false,false,\ false,false,,,,\n\ Point Values,,,,,1.5,0,0,1.5,0\n\ Scoring Method,positive\n", ) .unwrap(); let question = parse_question(&path).unwrap(); assert_eq!(question.number, 7); assert_eq!(question.columns.len(), 4, "four rubric columns"); assert_eq!(question.max_points, 1.5); assert_eq!(question.keyed(), vec!["C".to_string()]); assert_eq!(question.rows.len(), 3, "trailing rows are not students"); assert_eq!(question.scoring_method.as_deref(), Some("positive")); let ctx = Context { course: "BIOSC1540".into(), term: "2026s".into(), assessment_id: "exam-4".into(), date: None, form: None, }; let import = to_responses(&[question], &ctx); let rows = &import.responses.rows; assert_eq!(rows.len(), 3); assert_eq!(rows[0].selected, vec!["C".to_string()]); assert_eq!(rows[0].correct, Some(true)); assert_eq!(rows[0].credit, 1.0); assert_eq!(rows[1].correct, Some(false)); // A student who marked nothing left it blank; that is not a wrong answer. assert_eq!(rows[2].correct, None); assert_eq!(rows[2].selected.len(), 0); std::fs::remove_dir_all(&dir).ok(); } #[test] fn rejects_a_file_without_the_rubric_delimiter() { let dir = std::env::temp_dir().join(format!("cb-gs-bad-{}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); let path = dir.join("1.csv"); std::fs::write(&path, "Name,Score\nAda,1\n").unwrap(); let err = parse_question(&path).unwrap_err(); assert!(err.to_string().contains("Submission Time")); std::fs::remove_dir_all(&dir).ok(); } }