Dev (#1)
Sync README to GitHub / sync (push) Successful in 12s
CI / check (push) Successful in 8m31s
Deploy docs / deploy (push) Successful in 6m44s
Nightly / nightly (push) Successful in 10m33s

Reviewed-on: #1
This commit was merged in pull request #1.
This commit is contained in:
alexm committed 2026-08-07 15:48:11 -04:00
1 parent 228a0da47f
commit abc0bdf621
79 files changed
+31370

No files matched your search

+622
View File
@@ -0,0 +1,622 @@
// SPDX-License-Identifier: Prosperity-3.0.0
// Copyright Scientific Computing Studio
// Source: https://git.scient.ing/education/coursebank
//! Reading Canvas's "Student Analysis" quiz export.
//!
//! Canvas exports one wide CSV for the whole quiz. After the student columns come
//! two columns per question: one headed `<question id>: <question text>` holding
//! the answer text the student chose, and one immediately after it holding the
//! points earned, headed with the points possible.
//!
//! The awkward part is that Canvas records answer text, not option letters. So
//! recovering "this student chose C" means matching the exported text back against
//! the item's options. That matching is done on normalized text because the text
//! makes a round trip through the QTI export and Canvas's own HTML sanitizer, and
//! comes back with different markup than it left with.
//!
//! When a match cannot be made, the row keeps its score and loses its option
//! letter, with a warning naming the question. Totals and per-objective mastery
//! still work, and only distractor analysis is affected.
//!
//! Questions are aligned to item numbers by matching stem text the same way, with
//! column order as the fallback. Order alone would be wrong for any quiz where
//! Canvas shuffled the questions.
use std::collections::BTreeMap;
use std::path::Path;
use crate::assessment::AssessmentFile;
use crate::catalog::Catalog;
use crate::date::Date;
use crate::error::{Error, Result};
use crate::responses::{Response, ResponseSet, administration_id};
/// What the ingest needs that the export does not carry.
pub type Context = crate::gradescope::Context;
/// One question column pair found in the header.
#[derive(Debug, Clone)]
struct QuestionColumn {
/// Index of the answer-text column.
text_index: usize,
/// Index of the points-earned column, when the header had one.
points_index: Option<usize>,
/// The Canvas question id, from the column header.
///
/// Kept for diagnostics rather than for matching: when a column cannot be
/// traced to the assessment record, or an answer cannot be traced to an option,
/// this id is what lets you find the question in Canvas and see what differs.
/// Matching itself goes by question text, because Canvas ids change when a quiz
/// is copied to a new term.
canvas_id: String,
/// The question text from the header.
question_text: String,
/// Points possible, parsed from the points column's header.
points_possible: f64,
/// The item number this maps to, once resolved.
item_number: Option<u32>,
}
/// Splits a question column header into its id and text.
///
/// # Arguments
///
/// * `header` - the column header.
///
/// # Returns
///
/// The id and text, or `None` when the header is not a question column.
fn split_question_header(header: &str) -> Option<(String, String)> {
let (id, rest) = header.split_once(':')?;
let id = id.trim();
if id.is_empty() || !id.chars().all(|c| c.is_ascii_digit()) {
return None;
}
Some((id.to_string(), rest.trim().to_string()))
}
/// Normalizes text for comparison.
///
/// Strips HTML tags, decodes the handful of entities that survive a QTI round
/// trip, drops punctuation, lowercases, and collapses whitespace.
///
/// # Arguments
///
/// * `s` - the text.
///
/// # Returns
///
/// The normalized form.
pub fn normalize(s: &str) -> String {
// Strip tags.
let mut stripped = String::with_capacity(s.len());
let mut in_tag = false;
for ch in s.chars() {
match ch {
'<' => in_tag = true,
'>' => {
in_tag = false;
stripped.push(' ');
}
_ if in_tag => {}
_ => stripped.push(ch),
}
}
// Decode the entities that actually appear.
let decoded = stripped
.replace("&nbsp;", " ")
.replace("&amp;", "&")
.replace("&lt;", "<")
.replace("&gt;", ">")
.replace("&quot;", "\"")
.replace("&apos;", "'")
.replace("&#39;", "'")
.replace("&rarr;", "->")
.replace("&harr;", "<->")
.replace("&minus;", "-");
let mut out = String::with_capacity(decoded.len());
let mut last_space = true;
for ch in decoded.chars() {
let c = ch.to_ascii_lowercase();
if c.is_alphanumeric() {
out.push(c);
last_space = false;
} else if (c.is_whitespace() || c == '-' || c == '_') && !last_space {
out.push(' ');
last_space = true;
}
// Everything else — punctuation, entity leftovers — is dropped.
}
out.trim().to_string()
}
/// Reads a Canvas Student Analysis export.
///
/// # Arguments
///
/// * `path` - the CSV file.
/// * `ctx` - identifying information.
/// * `record` - the assessment record, used to align questions to item numbers.
/// * `catalog` - the loaded course, used to recover option letters from answer
/// text. Without it, scores are still ingested but letters are not.
///
/// # Returns
///
/// The normalized responses.
///
/// # Errors
///
/// Returns [`Error::Csv`] on a malformed file and [`Error::Invalid`] when no
/// question columns can be found.
pub fn ingest(
path: &Path,
ctx: &Context,
record: Option<&AssessmentFile>,
catalog: Option<&Catalog>,
) -> Result<ResponseSet> {
let mut reader = csv::ReaderBuilder::new()
.flexible(true)
.has_headers(true)
.from_path(path)
.map_err(|e| Error::Csv {
path: path.to_path_buf(),
source: e,
})?;
let header = reader
.headers()
.map_err(|e| Error::Csv {
path: path.to_path_buf(),
source: e,
})?
.clone();
let header: Vec<String> = header.iter().map(|h| h.trim().to_string()).collect();
let find = |name: &str| header.iter().position(|h| h.eq_ignore_ascii_case(name));
let name_col = find("name");
let id_col = find("id");
let sis_col = find("sis_id").or_else(|| find("sis id"));
let section_col = find("section");
let submitted_col = find("submitted");
let attempt_col = find("attempt");
// Locate the question column pairs.
let mut questions: Vec<QuestionColumn> = Vec::new();
for (i, h) in header.iter().enumerate() {
if let Some((canvas_id, question_text)) = split_question_header(h) {
// The next column holds points earned; its header is the points
// possible. Canvas writes it as a bare number.
let (points_index, points_possible) = match header.get(i + 1) {
Some(next) => match next.trim().parse::<f64>() {
Ok(p) => (Some(i + 1), p),
Err(_) => (None, 0.0),
},
None => (None, 0.0),
};
questions.push(QuestionColumn {
text_index: i,
points_index,
canvas_id,
question_text,
points_possible,
item_number: None,
});
}
}
if questions.is_empty() {
return Err(Error::Invalid(vec![format!(
"{} has no question columns; a Canvas Student Analysis export heads each question \
`<id>: <question text>`. Did you export the Item Analysis instead?",
path.display()
)]));
}
let mut warnings = Vec::new();
align_questions(&mut questions, record, catalog, &mut warnings);
// Build the answer-text lookup once per question: normalized option text to
// option letter.
let mut answer_lookup: BTreeMap<usize, BTreeMap<String, String>> = BTreeMap::new();
let mut item_meta: BTreeMap<usize, (Option<String>, Vec<String>)> = BTreeMap::new();
if let (Some(cat), Some(rec)) = (catalog, record) {
for (qi, q) in questions.iter().enumerate() {
let Some(number) = q.item_number else {
continue;
};
let Some(placement) = rec.placement(number) else {
continue;
};
let Some(entry) = cat.get(&placement.item) else {
continue;
};
let mut map = BTreeMap::new();
for opt in &entry.item.options {
map.insert(normalize(&opt.text), opt.id.clone());
}
answer_lookup.insert(qi, map);
item_meta.insert(qi, (Some(placement.item.clone()), placement.key.clone()));
}
}
let admin = administration_id(&ctx.course, &ctx.term, &ctx.assessment_id);
let mut set = ResponseSet::new();
let mut unmatched_answers: BTreeMap<(u32, String), usize> = BTreeMap::new();
for rec in reader.records() {
let rec = rec.map_err(|e| Error::Csv {
path: path.to_path_buf(),
source: e,
})?;
let cell = |i: Option<usize>| -> Option<String> {
i.and_then(|i| rec.get(i))
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
};
let sid = cell(sis_col).or_else(|| cell(id_col));
let name = cell(name_col);
if sid.is_none() && name.is_none() {
continue;
}
// Canvas emits a "Test Student" row for anyone who previewed the quiz.
if name.as_deref() == Some("Test Student") {
continue;
}
let student_key = sid
.clone()
.or_else(|| name.clone())
.unwrap_or_else(|| "unknown".to_string());
let section = cell(section_col);
let submitted = cell(submitted_col);
let attempt = cell(attempt_col).and_then(|a| a.parse::<u32>().ok());
// Only the graded attempt is exported per row, but a student who never
// submitted still gets a row; those carry no responses.
if submitted.is_none() && attempt.is_none() {
continue;
}
for (qi, q) in questions.iter().enumerate() {
let Some(number) = q.item_number else {
continue;
};
let answer_text = rec.get(q.text_index).map(|s| s.trim()).unwrap_or("");
let score = q
.points_index
.and_then(|i| rec.get(i))
.and_then(|s| s.trim().parse::<f64>().ok())
.unwrap_or(0.0);
let points = q.points_possible;
let credit = if points > 0.0 { score / points } else { 0.0 };
let mut selected = Vec::new();
if !answer_text.is_empty() {
let normalized = normalize(answer_text);
match answer_lookup.get(&qi).and_then(|m| m.get(&normalized)) {
Some(letter) => selected.push(letter.clone()),
None => {
*unmatched_answers
.entry((number, q.canvas_id.clone()))
.or_insert(0) += 1;
}
}
}
let (item_ref, key) = item_meta.get(&qi).cloned().unwrap_or((None, Vec::new()));
let correct = if answer_text.is_empty() {
None
} else if points > 0.0 {
Some(credit >= 0.999)
} else if !key.is_empty() && !selected.is_empty() {
Some(selected == key)
} else {
None
};
set.rows.push(Response {
administration_id: admin.clone(),
course: ctx.course.clone(),
term: ctx.term.clone(),
assessment_id: ctx.assessment_id.clone(),
date: ctx.date,
form: ctx.form.clone(),
student_key: student_key.clone(),
sid: sid.clone(),
name: name.clone(),
email: None,
section: section.clone(),
item_number: number,
item_ref,
item_version: None,
selected,
eliminated: Vec::new(),
correct,
credit,
points_possible: points,
score,
response_time_seconds: None,
level: None,
learning_objectives: Vec::new(),
topics: Vec::new(),
bonus: false,
dropped: false,
});
}
}
for ((number, canvas_id), count) in unmatched_answers {
warnings.push(format!(
"question {number} (Canvas id {canvas_id}): {count} answer(s) did not match any option \
text, so those responses have a score but no option letter; distractor analysis for \
this item will be incomplete. The usual cause is the option text being edited in \
Canvas after import"
));
}
if set.rows.is_empty() {
warnings.push(
"no student rows were found; the export may contain only the header, or every row \
may be an unsubmitted attempt"
.to_string(),
);
}
set.warnings.extend(warnings);
Ok(set)
}
/// Assigns an item number to each question column.
///
/// Matching on stem text is preferred over column order because Canvas shuffles
/// questions when the quiz says to, and the export follows the shuffled order.
///
/// # Arguments
///
/// * `questions` - the columns, updated in place.
/// * `record` - the assessment record.
/// * `catalog` - the loaded course.
/// * `warnings` - collects anything ambiguous.
fn align_questions(
questions: &mut [QuestionColumn],
record: Option<&AssessmentFile>,
catalog: Option<&Catalog>,
warnings: &mut Vec<String>,
) {
let Some(rec) = record else {
// Without a record, the only sensible assumption is column order.
for (i, q) in questions.iter_mut().enumerate() {
q.item_number = Some(i as u32 + 1);
}
warnings.push(
"no assessment record was supplied, so questions were matched to item numbers by \
column order; pass --assessment to match on question text instead"
.to_string(),
);
return;
};
// Normalized stem to item number, when the catalog is available.
let mut by_stem: BTreeMap<String, Vec<u32>> = BTreeMap::new();
if let Some(cat) = catalog {
for p in &rec.items {
if let Some(entry) = cat.get(&p.item) {
by_stem
.entry(normalize(&entry.item.stem))
.or_default()
.push(p.number);
}
}
}
let mut used: Vec<u32> = Vec::new();
let mut unmatched: Vec<String> = Vec::new();
for (i, q) in questions.iter_mut().enumerate() {
let normalized = normalize(&q.question_text);
let candidates = by_stem.get(&normalized);
match candidates {
Some(numbers) => {
// Prefer a number not already claimed, so two items sharing a stem
// do not collapse onto one.
match numbers.iter().find(|n| !used.contains(n)) {
Some(n) => {
q.item_number = Some(*n);
used.push(*n);
}
None => {
q.item_number = Some(numbers[0]);
}
}
}
None => {
// Fall back to position, which is right for an unshuffled quiz.
let fallback = rec.items.get(i).map(|p| p.number).unwrap_or(i as u32 + 1);
if !used.contains(&fallback) {
used.push(fallback);
}
q.item_number = Some(fallback);
// Report the Canvas question id, not just a count. The id is what
// you can search for in Canvas to see which question this was, and
// an unmatched column usually means the text was edited there after
// import — so being able to find it is the whole remedy.
unmatched.push(format!("{} (assumed question {fallback})", q.canvas_id));
}
}
}
if !unmatched.is_empty() {
warnings.push(format!(
"{} of {} question column(s) could not be matched to the assessment record by question \
text and were matched by column order instead. Canvas question id(s): {}. Check that \
the record describes this quiz, and that the question text was not edited in Canvas \
after import",
unmatched.len(),
questions.len(),
unmatched.join(", ")
));
}
}
/// Guesses a date from a Canvas timestamp column.
///
/// # Arguments
///
/// * `s` - the timestamp, e.g. `2026-04-01 14:03:22 UTC`.
///
/// # Returns
///
/// The date, when the leading token parses as one.
pub fn date_from_timestamp(s: &str) -> Option<Date> {
s.split_whitespace().next()?.parse().ok()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn recognizes_question_headers() {
assert_eq!(
split_question_header("123456: Which enzyme catalyzes the step?"),
Some((
"123456".to_string(),
"Which enzyme catalyzes the step?".to_string()
))
);
assert_eq!(split_question_header("name"), None);
assert_eq!(split_question_header("n correct"), None);
// A colon in prose is not a question column.
assert_eq!(split_question_header("Note: read carefully"), None);
}
#[test]
fn normalization_survives_a_qti_round_trip() {
// Option text authored as `K#sub[m] increases` is exported as HTML and
// comes back from Canvas wrapped in tags. Both must normalize to the same
// thing as the HTML we sent, or letters cannot be recovered.
let sent = "K<sub>m</sub> increases";
let returned = "<p>K<sub>m</sub> increases</p>";
assert_eq!(normalize(sent), normalize(returned));
assert_eq!(normalize(returned), "k m increases");
assert_eq!(normalize("K&nbsp;m increases"), "k m increases");
}
#[test]
fn normalization_ignores_punctuation_and_case() {
assert_eq!(
normalize("The Rate, Increases!"),
normalize("the rate increases")
);
assert_eq!(normalize(" spaced out "), "spaced out");
assert_eq!(normalize("<b>bold</b> text"), "bold text");
}
#[test]
fn timestamps_yield_dates() {
assert_eq!(
date_from_timestamp("2026-04-01 14:03:22 UTC").map(|d| d.to_string()),
Some("2026-04-01".to_string())
);
assert_eq!(date_from_timestamp("not a date"), None);
}
#[test]
fn falls_back_to_column_order_without_a_record() {
let mut questions = vec![
QuestionColumn {
text_index: 8,
points_index: Some(9),
canvas_id: "1".into(),
question_text: "Q one".into(),
points_possible: 1.0,
item_number: None,
},
QuestionColumn {
text_index: 10,
points_index: Some(11),
canvas_id: "2".into(),
question_text: "Q two".into(),
points_possible: 1.0,
item_number: None,
},
];
let mut warnings = Vec::new();
align_questions(&mut questions, None, None, &mut warnings);
assert_eq!(questions[0].item_number, Some(1));
assert_eq!(questions[1].item_number, Some(2));
assert_eq!(warnings.len(), 1);
}
#[test]
fn parses_a_realistic_export() {
let dir = std::env::temp_dir().join(format!("cb-canvas-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("analysis.csv");
std::fs::write(
&path,
"name,id,sis_id,section,submitted,attempt,\
1001: Which enzyme?,1.0,1002: Which pathway?,1.0,n correct,n incorrect,score\n\
Ada Lovelace,9001,1234567,L01,2026-04-01 14:00:00 UTC,1,Hexokinase,1.0,Glycolysis,\
1.0,2,0,2.0\n\
Alan Turing,9002,7654321,L01,2026-04-01 14:05:00 UTC,1,Pyruvate kinase,0.0,\
Glycolysis,1.0,1,1,1.0\n\
Test Student,9999,,L01,2026-04-01 13:00:00 UTC,1,Hexokinase,1.0,Glycolysis,1.0,2,0,\
2.0\n",
)
.unwrap();
let ctx = Context {
course: "BIOSC1540".into(),
term: "2026s".into(),
assessment_id: "quiz-1".into(),
date: None,
form: None,
};
let set = ingest(&path, &ctx, None, None).unwrap();
// Two real students, two questions each. The preview row is discarded.
assert_eq!(set.rows.len(), 4, "{:?}", set.warnings);
assert!(
set.rows
.iter()
.all(|r| r.name.as_deref() != Some("Test Student"))
);
assert_eq!(set.scored_total("1234567"), 2.0);
assert_eq!(set.scored_total("7654321"), 1.0);
let alan_q1 = set
.rows
.iter()
.find(|r| r.student_key == "7654321" && r.item_number == 1)
.unwrap();
assert_eq!(alan_q1.correct, Some(false));
assert_eq!(alan_q1.credit, 0.0);
// No catalog was supplied, so no letter could be recovered.
assert!(alan_q1.selected.is_empty());
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn rejects_the_wrong_export_kind() {
let dir = std::env::temp_dir().join(format!("cb-canvas-bad-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("item.csv");
std::fs::write(&path, "question,discrimination index\nQ1,0.4\n").unwrap();
let ctx = Context {
course: "C".into(),
term: "T".into(),
assessment_id: "a".into(),
date: None,
form: None,
};
let err = ingest(&path, &ctx, None, None).unwrap_err();
assert!(err.to_string().contains("Student Analysis"));
std::fs::remove_dir_all(&dir).ok();
}
}
+859
View File
@@ -0,0 +1,859 @@
// SPDX-License-Identifier: Prosperity-3.0.0
// Copyright Scientific Computing Studio
// Source: https://git.scient.ing/education/coursebank
//! Reading Gradescope's per-question CSV exports.
//!
//! Gradescope exports one file per question, named `1.csv` through `N.csv`, in
//! wide form. Every assumption encoded here was checked against a real 32-question,
//! 24-student export rather than inferred from documentation, because several of
//! them are not what the format suggests.
//!
//! The layout is:
//!
//! ```text
//! Assignment Submission ID, Question Submission ID, First Name, Last Name,
//! SID, Email, Sections, Score, Submission Time, <rubric columns...>,
//! Adjustment, Comments, Grader, Tags
//! ```
//!
//! The rubric columns are the header slice strictly between `Submission Time`
//! and `Adjustment`. Student cells in those columns are the literal strings
//! `true` and `false`.
//!
//! Four things about real exports that a naive parser gets wrong:
//!
//! *Trailing rows are not aligned with the header.* After the student rows come
//! `Point Values`, `Rubric Numbers`, and `Scoring Method` rows with a label, some
//! empty cells, and then the numbers. The CSV reader must be in flexible mode or
//! it errors on the whole file.
//!
//! *Rubric labels are not always bare option letters.* Real ones include
//! `Selected C`, `Eliminated A`, and — this is the interesting case — a letter
//! followed by an instructor's parenthetical, such as
//! `B (Technically speaking, this answer describes HBD/HBA, but I can see how
//! this distractor is poorly written.)`. Those columns carry partial credit.
//!
//! *Partial credit awarded to a distractor is evidence, not noise.* When you gave
//! 1.0 of 1.5 points for choosing B, you decided at grading time that B was
//! partly defensible. That is exactly the ambiguity signal item analysis is
//! trying to detect, so it is captured as [`Flag::Ambiguous`] rather than
//! rounded away.
//!
//! *Bonus questions have `max_points` of zero* while a rubric column still
//! carries points. Key detection therefore uses the largest value across the
//! maximum *and* every column, not the maximum alone.
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use crate::date::Date;
use crate::error::{Error, Result};
use crate::responses::{Response, ResponseSet, administration_id};
use crate::taxonomy::Flag;
/// What a rubric column represents.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ColumnKind {
/// The student selected this option.
Select,
/// The student eliminated this option, under elimination scoring.
Eliminate,
/// Something else: a catch-all rubric row such as
/// `Incorrect selection with no eliminations`.
Other,
}
/// One rubric column.
#[derive(Debug, Clone)]
pub struct RubricColumn {
/// The header label, verbatim.
pub label: String,
/// What the column means.
pub kind: ColumnKind,
/// The option letter, when the label named one.
pub letter: Option<String>,
/// The instructor's parenthetical annotation, when there was one. Worth
/// keeping: it is usually the instructor explaining why an item is flawed.
pub note: Option<String>,
/// Points this column awards, from the `Point Values` row.
pub value: Option<f64>,
}
/// One student's row in a question file.
#[derive(Debug, Clone)]
pub struct StudentRow {
/// The institutional student id.
pub sid: Option<String>,
/// First and last name joined.
pub name: Option<String>,
/// The email.
pub email: Option<String>,
/// Section or lab.
pub section: Option<String>,
/// Points awarded, authoritative.
pub score: f64,
/// The submission timestamp, verbatim.
pub submission_time: Option<String>,
/// Indices of rubric columns marked `true`.
pub marks: Vec<usize>,
}
/// One parsed question file.
#[derive(Debug, Clone)]
pub struct Question {
/// The question number, from the file name.
pub number: u32,
/// Where it came from.
pub path: PathBuf,
/// The rubric columns, in header order.
pub columns: Vec<RubricColumn>,
/// The maximum points from the `Point Values` row.
pub max_points: f64,
/// The scoring method, when stated.
pub scoring_method: Option<String>,
/// The student rows.
pub rows: Vec<StudentRow>,
}
impl Question {
/// The largest point value anywhere in the question.
///
/// Bonus questions record a maximum of zero while still awarding points, so
/// the key has to be found by looking at the columns too.
///
/// # Returns
///
/// The largest value seen.
pub fn top_value(&self) -> f64 {
let mut top = self.max_points;
for c in &self.columns {
if let Some(v) = c.value {
if v > top {
top = v;
}
}
}
top
}
/// The option letters that earn full credit.
///
/// # Returns
///
/// The keyed letters, sorted. Empty when the point values were absent.
pub fn keyed(&self) -> Vec<String> {
let top = self.top_value();
if top <= 0.0 {
return Vec::new();
}
let mut out: BTreeSet<String> = BTreeSet::new();
for c in &self.columns {
if c.kind == ColumnKind::Select {
if let (Some(letter), Some(v)) = (&c.letter, c.value) {
if (v - top).abs() < 1e-9 {
out.insert(letter.clone());
}
}
}
}
out.into_iter().collect()
}
/// Distractors that were awarded partial credit at grading time.
///
/// # Returns
///
/// Letter, points awarded, and the instructor's note if any.
pub fn partial_credit(&self) -> Vec<(String, f64, Option<String>)> {
let top = self.top_value();
let mut out = Vec::new();
for c in &self.columns {
if c.kind != ColumnKind::Select {
continue;
}
if let (Some(letter), Some(v)) = (&c.letter, c.value) {
if v > 0.0 && v < top - 1e-9 {
out.push((letter.clone(), v, c.note.clone()));
}
}
}
out
}
/// Whether this looks like a bonus question.
///
/// # Returns
///
/// `true` when the maximum is zero but a column still awards points.
pub fn looks_like_bonus(&self) -> bool {
self.max_points <= 0.0 && self.top_value() > 0.0
}
/// Whether elimination scoring was in use.
pub fn uses_elimination(&self) -> bool {
self.columns.iter().any(|c| c.kind == ColumnKind::Eliminate)
}
/// Points the question was worth.
///
/// # Returns
///
/// The stated maximum, falling back to the largest column value for bonus
/// questions so that credit is still a meaningful fraction.
pub fn points_possible(&self) -> f64 {
if self.max_points > 0.0 {
self.max_points
} else {
self.top_value()
}
}
/// Flags implied by how the question was graded.
///
/// # Returns
///
/// [`Flag::Ambiguous`] when a distractor received partial credit.
pub fn implied_flags(&self) -> Vec<Flag> {
if self.partial_credit().is_empty() {
Vec::new()
} else {
vec![Flag::Ambiguous]
}
}
}
/// What the ingest needs to know that the export does not say.
#[derive(Debug, Clone)]
pub struct Context {
/// The course code.
pub course: String,
/// The term.
pub term: String,
/// The assessment id.
pub assessment_id: String,
/// The administration date.
pub date: Option<Date>,
/// The form, when forms were used.
pub form: Option<String>,
}
/// The result of reading a directory of question files.
#[derive(Debug, Clone)]
pub struct Import {
/// The normalized responses.
pub responses: ResponseSet,
/// The parsed question files, kept so grading-time decisions can be folded
/// into item calibration.
pub questions: Vec<Question>,
}
/// Classifies a rubric column label.
///
/// # Arguments
///
/// * `label` - the header text.
///
/// # Returns
///
/// The kind, the option letter if the label named one, and any trailing
/// annotation with its surrounding punctuation trimmed.
pub fn classify(label: &str) -> (ColumnKind, Option<String>, Option<String>) {
let trimmed = label.trim();
let lower = trimmed.to_ascii_lowercase();
let (kind, rest) = if let Some(r) = lower.strip_prefix("selected ") {
(
ColumnKind::Select,
trimmed[trimmed.len() - r.len()..].trim(),
)
} else if let Some(r) = lower.strip_prefix("eliminated ") {
(
ColumnKind::Eliminate,
trimmed[trimmed.len() - r.len()..].trim(),
)
} else {
(ColumnKind::Select, trimmed)
};
let mut chars = rest.chars();
let Some(first) = chars.next() else {
return (ColumnKind::Other, None, Some(label.to_string()));
};
let upper = first.to_ascii_uppercase();
let tail = chars.as_str();
// A single letter A-H, optionally followed by an annotation that does not
// begin with another alphanumeric. `B (poorly written)` is option B;
// `Both A and C are wrong` is not.
let letter_like = upper.is_ascii_uppercase()
&& ('A'..='H').contains(&upper)
&& tail
.chars()
.next()
.map(|c| !c.is_alphanumeric())
.unwrap_or(true);
if !letter_like {
return (ColumnKind::Other, None, Some(label.to_string()));
}
let note = tail.trim().trim_matches(|c: char| "().,;: ".contains(c));
let note = if note.is_empty() {
None
} else {
Some(note.to_string())
};
(kind, Some(upper.to_string()), note)
}
/// Parses one question file.
///
/// # Arguments
///
/// * `path` - the CSV file.
///
/// # Returns
///
/// The parsed question.
///
/// # Errors
///
/// Returns [`Error::Csv`] on a malformed file and [`Error::Invalid`] when the
/// header lacks the `Submission Time` column that delimits the rubric.
pub fn parse_question(path: &Path) -> Result<Question> {
let number = path
.file_stem()
.and_then(|s| s.to_str())
.and_then(|s| s.parse::<u32>().ok())
.ok_or_else(|| {
Error::Invalid(vec![format!(
"{} is not named like a Gradescope question export (expected `12.csv`)",
path.display()
)])
})?;
// Flexible mode is required: the trailing `Point Values` rows are shorter
// than the header.
let mut reader = csv::ReaderBuilder::new()
.flexible(true)
.has_headers(false)
.from_path(path)
.map_err(|e| Error::Csv {
path: path.to_path_buf(),
source: e,
})?;
let mut records = reader.records();
let header = match records.next() {
Some(r) => r.map_err(|e| Error::Csv {
path: path.to_path_buf(),
source: e,
})?,
None => return Err(Error::Invalid(vec![format!("{} is empty", path.display())])),
};
let header: Vec<String> = header.iter().map(|s| s.trim().to_string()).collect();
let index_of = |name: &str| header.iter().position(|h| h == name);
let start = match index_of("Submission Time") {
Some(i) => i + 1,
None => {
return Err(Error::Invalid(vec![format!(
"{} has no `Submission Time` column, so the rubric columns cannot be located; \
is this a Gradescope per-question export?",
path.display()
)]));
}
};
let end = index_of("Adjustment").unwrap_or(header.len());
if end < start {
return Err(Error::Invalid(vec![format!(
"{} has `Adjustment` before `Submission Time`",
path.display()
)]));
}
let mut columns: Vec<RubricColumn> = header[start..end]
.iter()
.map(|label| {
let (kind, letter, note) = classify(label);
RubricColumn {
label: label.clone(),
kind,
letter,
note,
value: None,
}
})
.collect();
let sid_col = index_of("SID");
let first_col = index_of("First Name");
let last_col = index_of("Last Name");
let email_col = index_of("Email");
let section_col = index_of("Sections");
let score_col = index_of("Score");
let time_col = index_of("Submission Time");
let mut rows = Vec::new();
let mut max_points = 0.0f64;
let mut point_values: Vec<f64> = Vec::new();
let mut scoring_method = None;
for rec in records {
let rec = rec.map_err(|e| Error::Csv {
path: path.to_path_buf(),
source: e,
})?;
let cell = |i: Option<usize>| -> Option<String> {
i.and_then(|i| rec.get(i))
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
};
if rec.iter().all(|c| c.trim().is_empty()) {
continue;
}
let label = rec.get(0).unwrap_or("").trim();
match label {
"Point Values" => {
// Label, then some empty cells, then the maximum, then one value
// per rubric column. Alignment was verified across every file in
// a real export, but collecting the non-empty numbers in order is
// robust to a leading blank moving.
let nums: Vec<f64> = rec
.iter()
.skip(1)
.filter(|c| !c.trim().is_empty())
.filter_map(|c| c.trim().parse::<f64>().ok())
.collect();
if let Some((first, rest)) = nums.split_first() {
max_points = *first;
point_values = rest.to_vec();
}
continue;
}
"Rubric Numbers" => continue,
"Scoring Method" => {
scoring_method = cell(Some(1));
continue;
}
_ => {}
}
// A student row must have a score cell that parses; anything else is a
// trailing annotation row we have not seen before, and skipping it is
// safer than failing the import.
let score = match cell(score_col).and_then(|s| s.parse::<f64>().ok()) {
Some(s) => s,
None => {
if cell(sid_col).is_none() && cell(email_col).is_none() {
continue;
}
0.0
}
};
let marks: Vec<usize> = (0..columns.len())
.filter(|i| {
rec.get(start + i)
.map(|c| c.trim().eq_ignore_ascii_case("true"))
.unwrap_or(false)
})
.collect();
let name = match (cell(first_col), cell(last_col)) {
(Some(f), Some(l)) => Some(format!("{f} {l}")),
(Some(f), None) => Some(f),
(None, Some(l)) => Some(l),
(None, None) => None,
};
rows.push(StudentRow {
sid: cell(sid_col),
name,
email: cell(email_col),
section: cell(section_col),
score,
submission_time: cell(time_col),
marks,
});
}
// Attach point values when they line up; when they do not, leave them off
// rather than misattribute credit to the wrong option.
if point_values.len() == columns.len() {
for (c, v) in columns.iter_mut().zip(point_values.iter()) {
c.value = Some(*v);
}
}
Ok(Question {
number,
path: path.to_path_buf(),
columns,
max_points,
scoring_method,
rows,
})
}
/// Reads every question file in a directory.
///
/// # Arguments
///
/// * `dir` - the directory of `N.csv` files.
/// * `ctx` - identifying information the export does not carry.
///
/// # Returns
///
/// Normalized responses and the parsed questions.
///
/// # Errors
///
/// Returns [`Error::Io`] when the directory cannot be read and [`Error::Invalid`]
/// when it holds no question files.
pub fn ingest_dir(dir: &Path, ctx: &Context) -> Result<Import> {
let mut paths: Vec<PathBuf> = std::fs::read_dir(dir)
.map_err(|e| Error::io(dir, e))?
.filter_map(|e| e.ok())
.map(|e| e.path())
.filter(|p| {
p.extension().and_then(|e| e.to_str()) == Some("csv")
&& p.file_stem()
.and_then(|s| s.to_str())
.map(|s| s.chars().all(|c| c.is_ascii_digit()))
.unwrap_or(false)
})
.collect();
if paths.is_empty() {
return Err(Error::Invalid(vec![format!(
"{} holds no numbered CSV files; Gradescope exports one file per question, \
named `1.csv` through `N.csv`",
dir.display()
)]));
}
// Numeric order, so `10.csv` does not sort before `2.csv`.
paths.sort_by_key(|p| {
p.file_stem()
.and_then(|s| s.to_str())
.and_then(|s| s.parse::<u32>().ok())
.unwrap_or(u32::MAX)
});
let mut questions = Vec::new();
for p in &paths {
questions.push(parse_question(p)?);
}
Ok(to_responses(&questions, ctx))
}
/// Normalizes parsed questions into responses.
///
/// # Arguments
///
/// * `questions` - the parsed question files.
/// * `ctx` - identifying information.
///
/// # Returns
///
/// The import, with warnings for anything that looked wrong but not fatal.
pub fn to_responses(questions: &[Question], ctx: &Context) -> Import {
let mut set = ResponseSet::new();
let admin = administration_id(&ctx.course, &ctx.term, &ctx.assessment_id);
let mut counts: BTreeMap<u32, usize> = BTreeMap::new();
for q in questions {
let keyed: BTreeSet<String> = q.keyed().into_iter().collect();
let points = q.points_possible();
let bonus = q.looks_like_bonus();
if keyed.is_empty() {
set.warnings.push(format!(
"question {}: no keyed option could be identified from the point values, so \
correctness is unknown; scores are still recorded",
q.number
));
}
for (letter, value, note) in q.partial_credit() {
set.warnings.push(format!(
"question {}: option {letter} was awarded {value} of {points} points at grading \
time, which is recorded as an ambiguity signal{}",
q.number,
note.map(|n| format!(" ({n})")).unwrap_or_default()
));
}
for row in &q.rows {
let mut selected = Vec::new();
let mut eliminated = Vec::new();
let mut other = Vec::new();
for &i in &row.marks {
let c = &q.columns[i];
match (c.kind, &c.letter) {
(ColumnKind::Select, Some(l)) => selected.push(l.clone()),
(ColumnKind::Eliminate, Some(l)) => eliminated.push(l.clone()),
_ => other.push(c.label.clone()),
}
}
selected.sort();
eliminated.sort();
let credit = if points > 0.0 {
row.score / points
} else {
0.0
};
let correct = if keyed.is_empty() {
None
} else if selected.is_empty() && eliminated.is_empty() && other.is_empty() {
// A wholly unmarked row is a blank response, not a wrong one.
None
} else {
let chosen: BTreeSet<String> = selected.iter().cloned().collect();
Some(chosen == keyed)
};
let student_key = row
.sid
.clone()
.or_else(|| row.email.clone())
.unwrap_or_else(|| format!("unknown-{}", counts.len()));
*counts.entry(q.number).or_insert(0) += 1;
set.rows.push(Response {
administration_id: admin.clone(),
course: ctx.course.clone(),
term: ctx.term.clone(),
assessment_id: ctx.assessment_id.clone(),
date: ctx.date,
form: ctx.form.clone(),
student_key,
sid: row.sid.clone(),
name: row.name.clone(),
email: row.email.clone(),
section: row.section.clone(),
item_number: q.number,
item_ref: None,
item_version: None,
selected,
eliminated,
correct,
credit,
points_possible: points,
score: row.score,
response_time_seconds: None,
level: None,
learning_objectives: Vec::new(),
topics: Vec::new(),
bonus,
dropped: false,
});
}
}
// Every question should have the same number of submissions. A mismatch
// usually means a student was excused from one question, which is worth
// saying out loud because it changes per-item denominators.
let sizes: BTreeSet<usize> = counts.values().copied().collect();
if sizes.len() > 1 {
let mut odd: Vec<String> = Vec::new();
let modal = *sizes.iter().next_back().unwrap_or(&0);
for (n, c) in &counts {
if *c != modal {
odd.push(format!("q{n} has {c}"));
}
}
set.warnings.push(format!(
"submission counts differ across questions (most have {modal}): {}",
odd.join(", ")
));
}
Import {
responses: set,
questions: questions.to_vec(),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn classifies_bare_letters() {
let (k, l, n) = classify("C");
assert_eq!(k, ColumnKind::Select);
assert_eq!(l, Some("C".to_string()));
assert_eq!(n, None);
}
#[test]
fn classifies_selected_and_eliminated_prefixes() {
assert_eq!(classify("Selected C").0, ColumnKind::Select);
assert_eq!(classify("Selected C").1, Some("C".to_string()));
assert_eq!(classify("Eliminated A").0, ColumnKind::Eliminate);
assert_eq!(classify("Eliminated A").1, Some("A".to_string()));
}
#[test]
fn keeps_the_instructors_annotation() {
// Verbatim from a real export.
let (k, l, n) = classify(
"B (Technically speaking, this answer describes HBD/HBA, but I can see how this \
distractor is poorly written.)",
);
assert_eq!(k, ColumnKind::Select);
assert_eq!(l, Some("B".to_string()));
assert!(n.unwrap().starts_with("Technically speaking"));
let (_, l2, n2) = classify("B (preparation does not select the one most likely to bind)");
assert_eq!(l2, Some("B".to_string()));
assert!(n2.is_some());
}
#[test]
fn rejects_prose_rubric_rows() {
// Also verbatim: these are genuinely not option columns.
let (k, l, _) = classify("Incorrect selection with no eliminations.");
assert_eq!(k, ColumnKind::Other);
assert_eq!(l, None);
assert_eq!(classify("").0, ColumnKind::Other);
}
#[test]
fn does_not_mistake_a_sentence_for_option_a() {
// Starts with `A`, but the next character is alphanumeric.
assert_eq!(classify("Answered in the margin").0, ColumnKind::Other);
}
fn q(max: f64, cols: &[(&str, f64)]) -> Question {
Question {
number: 1,
path: PathBuf::from("1.csv"),
columns: cols
.iter()
.map(|(label, v)| {
let (kind, letter, note) = classify(label);
RubricColumn {
label: label.to_string(),
kind,
letter,
note,
value: Some(*v),
}
})
.collect(),
max_points: max,
scoring_method: None,
rows: Vec::new(),
}
}
#[test]
fn finds_the_key_by_top_value() {
let question = q(1.5, &[("A", 0.0), ("B", 0.0), ("C", 1.5), ("D", 0.0)]);
assert_eq!(question.keyed(), vec!["C".to_string()]);
assert!(question.partial_credit().is_empty());
}
#[test]
fn finds_the_key_on_a_bonus_question_with_zero_maximum() {
// Real case: q31 and q32 of the sample exam.
let question = q(0.0, &[("A", 0.0), ("B", 1.5), ("C", 0.0)]);
assert!(question.looks_like_bonus());
assert_eq!(question.keyed(), vec!["B".to_string()]);
assert_eq!(question.points_possible(), 1.5);
}
#[test]
fn partial_credit_to_a_distractor_flags_ambiguity() {
// Real case: q14, where B earned 1.49 of 1.5.
let question = q(1.5, &[("A", 0.0), ("B (poorly written)", 1.49), ("C", 1.5)]);
assert_eq!(question.keyed(), vec!["C".to_string()]);
let partial = question.partial_credit();
assert_eq!(partial.len(), 1);
assert_eq!(partial[0].0, "B");
assert!(partial[0].2.is_some(), "keeps the note");
assert_eq!(question.implied_flags(), vec![Flag::Ambiguous]);
}
#[test]
fn detects_elimination_scoring() {
let question = q(
0.9,
&[
("Selected C", 0.9),
("Eliminated A", 0.3),
("Eliminated B", 0.3),
],
);
assert!(question.uses_elimination());
assert_eq!(question.keyed(), vec!["C".to_string()]);
}
#[test]
fn parses_a_realistic_file() {
let dir = std::env::temp_dir().join(format!("cb-gs-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("7.csv");
std::fs::write(
&path,
"Assignment Submission ID,Question Submission ID,First Name,Last Name,SID,Email,\
Sections,Score,Submission Time,A,B,C,D,Adjustment,Comments,Grader,Tags\n\
1,11,Ada,Lovelace,1234567,ada@x.edu,L01,1.5,2026-04-01T10:00:00Z,false,false,true,\
false,,,,\n\
2,12,Alan,Turing,7654321,alan@x.edu,L01,0.0,2026-04-01T10:05:00Z,true,false,false,\
false,,,,\n\
3,13,Grace,Hopper,1111111,grace@x.edu,L02,0.0,2026-04-01T10:06:00Z,false,false,\
false,false,,,,\n\
Point Values,,,,,1.5,0,0,1.5,0\n\
Scoring Method,positive\n",
)
.unwrap();
let question = parse_question(&path).unwrap();
assert_eq!(question.number, 7);
assert_eq!(question.columns.len(), 4, "four rubric columns");
assert_eq!(question.max_points, 1.5);
assert_eq!(question.keyed(), vec!["C".to_string()]);
assert_eq!(question.rows.len(), 3, "trailing rows are not students");
assert_eq!(question.scoring_method.as_deref(), Some("positive"));
let ctx = Context {
course: "BIOSC1540".into(),
term: "2026s".into(),
assessment_id: "exam-4".into(),
date: None,
form: None,
};
let import = to_responses(&[question], &ctx);
let rows = &import.responses.rows;
assert_eq!(rows.len(), 3);
assert_eq!(rows[0].selected, vec!["C".to_string()]);
assert_eq!(rows[0].correct, Some(true));
assert_eq!(rows[0].credit, 1.0);
assert_eq!(rows[1].correct, Some(false));
// A student who marked nothing left it blank; that is not a wrong answer.
assert_eq!(rows[2].correct, None);
assert_eq!(rows[2].selected.len(), 0);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn rejects_a_file_without_the_rubric_delimiter() {
let dir = std::env::temp_dir().join(format!("cb-gs-bad-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("1.csv");
std::fs::write(&path, "Name,Score\nAda,1\n").unwrap();
let err = parse_question(&path).unwrap_err();
assert!(err.to_string().contains("Submission Time"));
std::fs::remove_dir_all(&dir).ok();
}
}
+835
View File
@@ -0,0 +1,835 @@
// SPDX-License-Identifier: Prosperity-3.0.0
// Copyright Scientific Computing Studio
// Source: https://git.scient.ing/education/coursebank
//! The canonical response row.
//!
//! Every grading platform exports a different shape. Gradescope gives one file
//! per question, in wide form, with a column per rubric item. Canvas gives one
//! enormous file per quiz, in wide form, with two columns per question and answer
//! *text* instead of option letters. Neither shape is analyzable.
//!
//! So both are normalized into the long form defined here: one row per student
//! per item. Long form is what item analysis, IRT, and per-objective mastery all
//! want, it survives a question being added or dropped without changing the
//! schema, and it appends cleanly across terms — which is the whole point, since
//! item statistics only become trustworthy once several administrations are
//! pooled.
//!
//! Two fields deserve comment.
//!
//! `credit` is a *fraction* in `0.0..=1.0`, not points. Storing the fraction
//! keeps the response independent of the points an item happened to be worth on
//! one exam, so pooling across administrations that weighted an item differently
//! is still valid. `score` carries the points actually awarded.
//!
//! `student_key` is whatever identifier analysis should group by, and it may be a
//! pseudonym. The real SID lives in `sid`, which is dropped when pseudonymizing.
use std::collections::{BTreeMap, BTreeSet};
use serde::{Deserialize, Serialize};
use crate::assessment::AssessmentFile;
use crate::catalog::Catalog;
use crate::date::Date;
use crate::hash::pseudonym;
use crate::taxonomy::Level;
/// One student's response to one item on one administration.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Response {
/// Identifies this administration: course slug, term, and assessment id.
/// Rows from different administrations of the same exam differ here, which is
/// what makes pooled analysis separable again later.
pub administration_id: String,
/// The course code.
pub course: String,
/// The term, e.g. `2026s`.
pub term: String,
/// The assessment id.
pub assessment_id: String,
/// The administration date.
pub date: Option<Date>,
/// Which form the student took, when forms were used.
pub form: Option<String>,
/// The identifier analysis groups by. A pseudonym when pseudonymizing.
pub student_key: String,
/// The institutional student id, absent when pseudonymized.
pub sid: Option<String>,
/// The student's name, absent when pseudonymized.
pub name: Option<String>,
/// The student's email, absent when pseudonymized.
pub email: Option<String>,
/// Section or lab, kept because it is the grouping most likely to reveal a
/// delivery problem rather than a learning one.
pub section: Option<String>,
/// The question number on the form, which is the join key to the record.
pub item_number: u32,
/// The item's global id, once resolved against an assessment record.
pub item_ref: Option<String>,
/// The item version as administered.
pub item_version: Option<u32>,
/// Option letters the student chose.
pub selected: Vec<String>,
/// Option letters the student eliminated, for elimination-scored items.
pub eliminated: Vec<String>,
/// Whether the response earned full credit. `None` when it cannot be
/// determined, e.g. a blank response on an item with no recorded key.
pub correct: Option<bool>,
/// Fraction of the item's points earned, in `0.0..=1.0`. May exceed nothing
/// and may go negative on elimination scoring.
pub credit: f64,
/// Points the item was worth as administered.
pub points_possible: f64,
/// Points awarded, authoritative from the platform where available.
pub score: f64,
/// Seconds spent, when the platform reports it.
pub response_time_seconds: Option<f64>,
/// The item's level, denormalized so analysis need not carry the catalog.
pub level: Option<Level>,
/// The item's learning objectives, denormalized for per-objective mastery.
pub learning_objectives: Vec<String>,
/// The item's topics, denormalized.
pub topics: Vec<String>,
/// Whether the item was bonus, and so excluded from the scored total.
pub bonus: bool,
/// Whether the item was dropped after the fact.
pub dropped: bool,
}
impl Response {
/// Whether this row should count toward scored totals and item statistics.
///
/// # Returns
///
/// `true` for a scored, undropped item.
pub fn counts(&self) -> bool {
!self.bonus && !self.dropped
}
/// The response coded for a dichotomous model.
///
/// Partial credit is rounded toward the majority: a half-credit response is
/// coded incorrect. IRT here is dichotomous, and pretending otherwise would
/// misstate the model rather than the data.
///
/// # Returns
///
/// `Some(true)` for full credit, `Some(false)` for less, `None` when unknown.
pub fn dichotomous(&self) -> Option<bool> {
match self.correct {
Some(c) => Some(c),
None if self.credit > 0.0 => Some(self.credit >= 0.999),
None => None,
}
}
/// The selected options as a comma-joined string, for flat storage.
pub fn selected_joined(&self) -> String {
self.selected.join(",")
}
}
/// A set of responses plus anything worth telling the user about the ingest.
#[derive(Debug, Clone, Default)]
pub struct ResponseSet {
/// The rows, in ingest order.
pub rows: Vec<Response>,
/// Non-fatal problems: unmatched columns, students with no responses,
/// question numbers absent from the assessment record.
pub warnings: Vec<String>,
}
impl ResponseSet {
/// Creates an empty set.
pub fn new() -> ResponseSet {
ResponseSet::default()
}
/// Appends another set, keeping its warnings.
///
/// # Arguments
///
/// * `other` - the set to absorb.
pub fn absorb(&mut self, other: ResponseSet) {
self.rows.extend(other.rows);
self.warnings.extend(other.warnings);
}
/// The distinct student keys, in sorted order.
pub fn students(&self) -> Vec<String> {
let set: BTreeSet<&str> = self.rows.iter().map(|r| r.student_key.as_str()).collect();
set.into_iter().map(|s| s.to_string()).collect()
}
/// The distinct item numbers that count toward the scored total, sorted.
pub fn scored_items(&self) -> Vec<u32> {
let set: BTreeSet<u32> = self
.rows
.iter()
.filter(|r| r.counts())
.map(|r| r.item_number)
.collect();
set.into_iter().collect()
}
/// All distinct item numbers, sorted.
pub fn all_items(&self) -> Vec<u32> {
let set: BTreeSet<u32> = self.rows.iter().map(|r| r.item_number).collect();
set.into_iter().collect()
}
/// Every row for one item, in student order.
///
/// # Arguments
///
/// * `number` - the question number.
///
/// # Returns
///
/// The matching rows.
pub fn for_item(&self, number: u32) -> Vec<&Response> {
let mut v: Vec<&Response> = self
.rows
.iter()
.filter(|r| r.item_number == number)
.collect();
v.sort_by(|a, b| a.student_key.cmp(&b.student_key));
v
}
/// Every row for one student, in item order.
///
/// # Arguments
///
/// * `key` - the student key.
///
/// # Returns
///
/// The matching rows.
pub fn for_student(&self, key: &str) -> Vec<&Response> {
let mut v: Vec<&Response> = self.rows.iter().filter(|r| r.student_key == key).collect();
v.sort_by_key(|r| r.item_number);
v
}
/// Total points a student earned on scored items.
///
/// # Arguments
///
/// * `key` - the student key.
///
/// # Returns
///
/// The sum of `score` over scored, undropped items.
pub fn scored_total(&self, key: &str) -> f64 {
self.rows
.iter()
.filter(|r| r.student_key == key && r.counts())
.map(|r| r.score)
.sum()
}
/// Bonus points a student earned.
pub fn bonus_total(&self, key: &str) -> f64 {
self.rows
.iter()
.filter(|r| r.student_key == key && r.bonus && !r.dropped)
.map(|r| r.score)
.sum()
}
/// Points available on scored items, taken from the most generous row seen
/// for each item so a student who skipped an item still has a denominator.
pub fn points_available(&self) -> f64 {
let mut per_item: BTreeMap<u32, f64> = BTreeMap::new();
for r in self.rows.iter().filter(|r| r.counts()) {
let e = per_item.entry(r.item_number).or_insert(0.0);
if r.points_possible > *e {
*e = r.points_possible;
}
}
per_item.values().sum()
}
/// Builds the response matrix for psychometrics.
///
/// # Arguments
///
/// * `include_bonus` - whether bonus items belong in the matrix. They
/// normally do not: bonus items are usually hard and optional, so including
/// them inflates the appearance of a low-ability tail.
///
/// # Returns
///
/// The matrix, students by items.
pub fn matrix(&self, include_bonus: bool) -> Matrix {
let students = self.students();
let items: Vec<u32> = if include_bonus {
self.all_items()
.into_iter()
.filter(|n| !self.item_dropped(*n))
.collect()
} else {
self.scored_items()
};
let student_index: BTreeMap<&str, usize> = students
.iter()
.enumerate()
.map(|(i, s)| (s.as_str(), i))
.collect();
let item_index: BTreeMap<u32, usize> =
items.iter().enumerate().map(|(i, n)| (*n, i)).collect();
let mut credit = vec![vec![None; items.len()]; students.len()];
let mut coded = vec![vec![None; items.len()]; students.len()];
for r in &self.rows {
let Some(&si) = student_index.get(r.student_key.as_str()) else {
continue;
};
let Some(&ii) = item_index.get(&r.item_number) else {
continue;
};
credit[si][ii] = Some(r.credit);
coded[si][ii] = r.dichotomous().map(|c| if c { 1u8 } else { 0u8 });
}
Matrix {
students,
items,
credit,
coded,
}
}
/// Whether every row for an item is marked dropped.
///
/// # Arguments
///
/// * `number` - the question number.
///
/// # Returns
///
/// `true` when the item was dropped.
pub fn item_dropped(&self, number: u32) -> bool {
let mut any = false;
for r in self.rows.iter().filter(|r| r.item_number == number) {
any = true;
if !r.dropped {
return false;
}
}
any
}
/// Attaches item metadata from an assessment record and the catalog.
///
/// Ingest knows question numbers; only the record knows which item a number
/// referred to. Doing this as a separate pass means an export can be parsed
/// and inspected before the record is written, which is the order people
/// actually work in.
///
/// # Arguments
///
/// * `record` - the assessment record.
/// * `catalog` - the loaded course, for levels, objectives, and topics.
///
/// # Returns
///
/// Warnings for numbers absent from the record and for keys that disagree
/// with the record.
pub fn enrich(&mut self, record: &AssessmentFile, catalog: Option<&Catalog>) -> Vec<String> {
let mut warnings = Vec::new();
let mut unmatched: BTreeSet<u32> = BTreeSet::new();
let default_points = catalog
.map(|c| c.course.policy.points_per_item)
.unwrap_or(1.0);
for r in &mut self.rows {
let Some(p) = record.placement(r.item_number) else {
unmatched.insert(r.item_number);
continue;
};
r.item_ref = Some(p.item.clone());
r.item_version = p.version;
r.bonus = r.bonus || p.bonus;
r.dropped = r.dropped || p.dropped;
if let Some(points) = p.points {
// The record is authoritative for points as administered; the
// export sometimes carries a stale maximum.
if (points - r.points_possible).abs() > 1e-9 && r.points_possible > 0.0 {
let ratio = r.credit;
r.points_possible = points;
r.score = ratio * points;
}
if r.points_possible == 0.0 {
r.points_possible = points;
}
}
if let Some(cat) = catalog {
if let Some(entry) = cat.get(&p.item) {
r.level = Some(entry.item.level);
r.learning_objectives = if p.learning_objectives.is_empty() {
entry.item.learning_objectives.clone()
} else {
p.learning_objectives.clone()
};
r.topics = entry.item.topics.clone();
if r.points_possible == 0.0 && !p.bonus {
r.points_possible = entry.item.points(default_points);
}
}
} else {
r.level = p.level;
r.learning_objectives = p.learning_objectives.clone();
}
// Apply the record's credit overrides, which is how a decision to
// award partial credit after the fact becomes visible in analysis.
if !p.credit_overrides.is_empty() && r.selected.len() == 1 {
if let Some(over) = p.credit_overrides.get(&r.selected[0]) {
if (*over - r.credit).abs() > 1e-9 {
r.credit = *over;
r.score = *over * r.points_possible;
r.correct = Some(*over >= 0.999);
}
}
}
}
if !unmatched.is_empty() {
let list: Vec<String> = unmatched.iter().map(|n| n.to_string()).collect();
warnings.push(format!(
"question number(s) {} appear in the export but not in the assessment record; \
they will be analyzed without item metadata",
list.join(", ")
));
}
self.warnings.extend(warnings.clone());
warnings
}
/// Replaces identifiers with keyed pseudonyms.
///
/// The salt must be kept outside the course repository. Hashing a seven-digit
/// student id without a key is not de-identification: the whole space can be
/// enumerated in under a second, so anyone with the file recovers every id.
///
/// # Arguments
///
/// * `salt` - the HMAC key.
pub fn pseudonymize(&mut self, salt: &[u8]) {
for r in &mut self.rows {
let source = r
.sid
.clone()
.or_else(|| r.email.clone())
.unwrap_or_else(|| r.student_key.clone());
r.student_key = pseudonym(salt, &source, 12);
r.sid = None;
r.name = None;
r.email = None;
}
}
/// The administration ids present, sorted.
pub fn administrations(&self) -> Vec<String> {
let set: BTreeSet<&str> = self
.rows
.iter()
.map(|r| r.administration_id.as_str())
.collect();
set.into_iter().map(|s| s.to_string()).collect()
}
}
/// Builds an administration id.
///
/// # Arguments
///
/// * `course` - the course code.
/// * `term` - the term.
/// * `assessment` - the assessment id.
///
/// # Returns
///
/// A stable identifier such as `BIOSC1540/2026s/exam-4`.
pub fn administration_id(course: &str, term: &str, assessment: &str) -> String {
format!("{course}/{term}/{assessment}")
}
/// A response matrix, students by items.
#[derive(Debug, Clone)]
pub struct Matrix {
/// Student keys, one per row.
pub students: Vec<String>,
/// Question numbers, one per column.
pub items: Vec<u32>,
/// Credit fractions; `None` for a missing response.
pub credit: Vec<Vec<Option<f64>>>,
/// Dichotomous codes; `None` for a missing response.
pub coded: Vec<Vec<Option<u8>>>,
}
impl Matrix {
/// The number of examinees.
pub fn n_students(&self) -> usize {
self.students.len()
}
/// The number of items.
pub fn n_items(&self) -> usize {
self.items.len()
}
/// Per-student total of credit fractions, treating missing as zero.
///
/// # Returns
///
/// One total per student, in row order.
pub fn totals(&self) -> Vec<f64> {
self.credit
.iter()
.map(|row| row.iter().map(|c| c.unwrap_or(0.0)).sum())
.collect()
}
/// Per-student count of items answered correctly.
pub fn correct_counts(&self) -> Vec<f64> {
self.coded
.iter()
.map(|row| row.iter().map(|c| c.unwrap_or(0) as f64).sum())
.collect()
}
/// The column for one item, by index.
///
/// # Arguments
///
/// * `j` - the column index.
///
/// # Returns
///
/// The dichotomous codes down that column.
pub fn column(&self, j: usize) -> Vec<Option<u8>> {
self.coded.iter().map(|row| row[j]).collect()
}
/// Whether the matrix has enough data to analyze at all.
///
/// # Returns
///
/// `true` when there is at least one student and one item.
pub fn is_analyzable(&self) -> bool {
self.n_students() > 0 && self.n_items() > 0
}
}
/// A flat record for CSV and Parquet storage.
///
/// The nested vectors on [`Response`] do not survive a columnar format, so they
/// are joined here. This is the schema written to disk, and it is deliberately
/// wide and denormalized: it is a fact table, meant to be appended to and read
/// by other tools, not a normalized database.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FlatResponse {
/// Identifies the administration.
pub administration_id: String,
/// The course code.
pub course: String,
/// The term.
pub term: String,
/// The assessment id.
pub assessment_id: String,
/// The date as `YYYY-MM-DD`, empty when unknown.
pub date: String,
/// The form id, empty when there were no forms.
pub form: String,
/// The grouping key.
pub student_key: String,
/// The student id, empty when pseudonymized.
pub sid: String,
/// The student email, empty when pseudonymized.
pub email: String,
/// The section, empty when unknown.
pub section: String,
/// The question number.
pub item_number: u32,
/// The item's global id, empty when unresolved.
pub item_ref: String,
/// The item version, 0 when unknown.
pub item_version: u32,
/// Comma-joined selected letters.
pub selected: String,
/// Comma-joined eliminated letters.
pub eliminated: String,
/// `1`, `0`, or empty when unknown.
pub correct: String,
/// Credit fraction.
pub credit: f64,
/// Points possible.
pub points_possible: f64,
/// Points awarded.
pub score: f64,
/// Seconds spent, empty when unknown.
pub response_time_seconds: String,
/// The level code 1..5, 0 when unknown.
pub level: u8,
/// Comma-joined objective ids.
pub learning_objectives: String,
/// Comma-joined topics.
pub topics: String,
/// Whether the item was bonus.
pub bonus: bool,
/// Whether the item was dropped.
pub dropped: bool,
}
impl FlatResponse {
/// Flattens a response.
///
/// # Arguments
///
/// * `r` - the response.
///
/// # Returns
///
/// The flat record.
pub fn from_response(r: &Response) -> FlatResponse {
FlatResponse {
administration_id: r.administration_id.clone(),
course: r.course.clone(),
term: r.term.clone(),
assessment_id: r.assessment_id.clone(),
date: r.date.map(|d| d.to_string()).unwrap_or_default(),
form: r.form.clone().unwrap_or_default(),
student_key: r.student_key.clone(),
sid: r.sid.clone().unwrap_or_default(),
email: r.email.clone().unwrap_or_default(),
section: r.section.clone().unwrap_or_default(),
item_number: r.item_number,
item_ref: r.item_ref.clone().unwrap_or_default(),
item_version: r.item_version.unwrap_or(0),
selected: r.selected.join(","),
eliminated: r.eliminated.join(","),
correct: match r.correct {
Some(true) => "1".to_string(),
Some(false) => "0".to_string(),
None => String::new(),
},
credit: r.credit,
points_possible: r.points_possible,
score: r.score,
response_time_seconds: r
.response_time_seconds
.map(|s| format!("{s:.1}"))
.unwrap_or_default(),
level: r.level.map(|l| l.code()).unwrap_or(0),
learning_objectives: r.learning_objectives.join(","),
topics: r.topics.join(","),
bonus: r.bonus,
dropped: r.dropped,
}
}
/// Rebuilds a response from its flat form.
///
/// # Returns
///
/// The response. Unparseable optional fields become `None` rather than
/// failing the read, because a hand-edited CSV should still load.
pub fn to_response(&self) -> Response {
let split = |s: &str| -> Vec<String> {
s.split(',')
.map(|p| p.trim())
.filter(|p| !p.is_empty())
.map(|p| p.to_string())
.collect()
};
Response {
administration_id: self.administration_id.clone(),
course: self.course.clone(),
term: self.term.clone(),
assessment_id: self.assessment_id.clone(),
date: self.date.parse().ok(),
form: none_if_empty(&self.form),
student_key: self.student_key.clone(),
sid: none_if_empty(&self.sid),
name: None,
email: none_if_empty(&self.email),
section: none_if_empty(&self.section),
item_number: self.item_number,
item_ref: none_if_empty(&self.item_ref),
item_version: if self.item_version == 0 {
None
} else {
Some(self.item_version)
},
selected: split(&self.selected),
eliminated: split(&self.eliminated),
correct: match self.correct.as_str() {
"1" | "true" => Some(true),
"0" | "false" => Some(false),
_ => None,
},
credit: self.credit,
points_possible: self.points_possible,
score: self.score,
response_time_seconds: self.response_time_seconds.parse().ok(),
level: Level::from_code(self.level),
learning_objectives: split(&self.learning_objectives),
topics: split(&self.topics),
bonus: self.bonus,
dropped: self.dropped,
}
}
}
/// `None` for an empty string, `Some` otherwise.
fn none_if_empty(s: &str) -> Option<String> {
if s.is_empty() {
None
} else {
Some(s.to_string())
}
}
#[cfg(test)]
mod tests {
use super::*;
fn row(student: &str, number: u32, credit: f64) -> Response {
Response {
administration_id: "C/2026s/e1".into(),
course: "C".into(),
term: "2026s".into(),
assessment_id: "e1".into(),
date: None,
form: None,
student_key: student.into(),
sid: Some(format!("sid-{student}")),
name: None,
email: None,
section: None,
item_number: number,
item_ref: None,
item_version: None,
selected: vec!["A".into()],
eliminated: vec![],
correct: Some(credit >= 0.999),
credit,
points_possible: 2.0,
score: credit * 2.0,
response_time_seconds: None,
level: None,
learning_objectives: vec![],
topics: vec![],
bonus: false,
dropped: false,
}
}
#[test]
fn matrix_is_students_by_items() {
let mut set = ResponseSet::new();
set.rows.push(row("s1", 1, 1.0));
set.rows.push(row("s1", 2, 0.0));
set.rows.push(row("s2", 1, 1.0));
set.rows.push(row("s2", 2, 1.0));
let m = set.matrix(false);
assert_eq!(m.n_students(), 2);
assert_eq!(m.n_items(), 2);
assert_eq!(m.coded[0], vec![Some(1), Some(0)]);
assert_eq!(m.correct_counts(), vec![1.0, 2.0]);
}
#[test]
fn missing_responses_stay_missing() {
let mut set = ResponseSet::new();
set.rows.push(row("s1", 1, 1.0));
set.rows.push(row("s2", 2, 1.0));
let m = set.matrix(false);
// s1 never answered item 2, so that cell is absent rather than zero.
assert_eq!(m.coded[0][1], None);
assert_eq!(m.coded[1][0], None);
// Totals treat missing as zero, which is right for scoring.
assert_eq!(m.totals(), vec![1.0, 1.0]);
}
#[test]
fn bonus_items_are_excluded_by_default() {
let mut set = ResponseSet::new();
set.rows.push(row("s1", 1, 1.0));
let mut bonus = row("s1", 2, 1.0);
bonus.bonus = true;
set.rows.push(bonus);
assert_eq!(set.matrix(false).n_items(), 1);
assert_eq!(set.matrix(true).n_items(), 2);
assert_eq!(set.scored_total("s1"), 2.0);
assert_eq!(set.bonus_total("s1"), 2.0);
}
#[test]
fn dropped_items_leave_the_matrix() {
let mut set = ResponseSet::new();
let mut r = row("s1", 1, 0.0);
r.dropped = true;
set.rows.push(r);
set.rows.push(row("s1", 2, 1.0));
assert_eq!(set.matrix(false).items, vec![2]);
assert!(set.item_dropped(1));
assert!(!set.item_dropped(2));
}
#[test]
fn partial_credit_codes_as_incorrect_for_irt() {
let mut r = row("s1", 1, 0.5);
r.correct = None;
assert_eq!(r.dichotomous(), Some(false));
r.credit = 1.0;
assert_eq!(r.dichotomous(), Some(true));
}
#[test]
fn pseudonymizing_removes_identifiers() {
let mut set = ResponseSet::new();
set.rows.push(row("s1", 1, 1.0));
let before = set.rows[0].student_key.clone();
set.pseudonymize(b"secret-salt");
assert_ne!(set.rows[0].student_key, before);
assert!(set.rows[0].student_key.starts_with("s-"));
assert!(set.rows[0].sid.is_none());
assert!(set.rows[0].name.is_none());
}
#[test]
fn flattening_round_trips() {
let r = row("s1", 3, 0.5);
let flat = FlatResponse::from_response(&r);
let back = flat.to_response();
assert_eq!(back.student_key, "s1");
assert_eq!(back.item_number, 3);
assert_eq!(back.credit, 0.5);
assert_eq!(back.selected, vec!["A".to_string()]);
assert_eq!(back.correct, Some(false));
}
#[test]
fn points_available_uses_the_largest_seen_per_item() {
let mut set = ResponseSet::new();
set.rows.push(row("s1", 1, 1.0));
let mut r = row("s2", 1, 1.0);
r.points_possible = 3.0;
set.rows.push(r);
assert_eq!(set.points_available(), 3.0);
}
}
+669
View File
@@ -0,0 +1,669 @@
// SPDX-License-Identifier: Prosperity-3.0.0
// Copyright Scientific Computing Studio
// Source: https://git.scient.ing/education/coursebank
//! Where response data lives on disk.
//!
//! One file per administration, under `data/`, named after the administration id.
//! Not one big file, because an exam's responses are written once and then only
//! read: separate files mean re-ingesting Exam 4 cannot corrupt Exam 3, and a
//! term's data can be archived or excluded by moving files rather than filtering
//! rows.
//!
//! Parquet is the default format. It is columnar, typed, compressed, and readable
//! by pandas, polars, R, and DuckDB without an export step, which matters because
//! the point of storing this data is to still be able to analyze it in five years
//! with whatever tool exists then.
//!
//! CSV is the fallback, and it is a real fallback rather than a degraded mode:
//! `--no-default-features` builds the entire tool with CSV storage and loses
//! nothing but file size and read speed. Committing to a format you cannot open
//! without the right library version is how course data gets lost.
use std::path::{Path, PathBuf};
use crate::error::{Error, Result};
use crate::responses::{FlatResponse, Response, ResponseSet};
/// A storage format.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Format {
/// Apache Parquet, the default.
Parquet,
/// Comma-separated values.
Csv,
}
impl Format {
/// The file extension, without a dot.
pub fn extension(self) -> &'static str {
match self {
Format::Parquet => "parquet",
Format::Csv => "csv",
}
}
/// The format compiled in as the default.
///
/// # Returns
///
/// Parquet when the `parquet` feature is on, CSV otherwise.
pub fn preferred() -> Format {
#[cfg(feature = "parquet")]
{
Format::Parquet
}
#[cfg(not(feature = "parquet"))]
{
Format::Csv
}
}
/// Whether this format can be written by the current build.
pub fn is_available(self) -> bool {
match self {
Format::Csv => true,
Format::Parquet => cfg!(feature = "parquet"),
}
}
/// The format implied by a file extension.
///
/// # Arguments
///
/// * `path` - the file path.
///
/// # Returns
///
/// The format, or `None` when the extension is not one we write.
pub fn from_path(path: &Path) -> Option<Format> {
match path.extension().and_then(|e| e.to_str()) {
Some("parquet") => Some(Format::Parquet),
Some("csv") => Some(Format::Csv),
_ => None,
}
}
}
/// The response store rooted at a course's `data/` directory.
#[derive(Debug, Clone)]
pub struct Store {
/// The data directory.
pub dir: PathBuf,
/// The format to write.
pub format: Format,
}
impl Store {
/// Opens a store, creating the directory if needed.
///
/// # Arguments
///
/// * `dir` - the data directory.
///
/// # Returns
///
/// The store, writing the preferred available format.
///
/// # Errors
///
/// Returns [`Error::Io`] when the directory cannot be created.
pub fn open(dir: impl Into<PathBuf>) -> Result<Store> {
let dir = dir.into();
std::fs::create_dir_all(&dir).map_err(|e| Error::io(&dir, e))?;
Ok(Store {
dir,
format: Format::preferred(),
})
}
/// Sets the format to write.
///
/// # Arguments
///
/// * `format` - the format.
///
/// # Returns
///
/// The store, for chaining.
///
/// # Errors
///
/// Returns [`Error::FeatureDisabled`] when the format was compiled out.
pub fn with_format(mut self, format: Format) -> Result<Store> {
if !format.is_available() {
return Err(Error::FeatureDisabled("Parquet", "parquet"));
}
self.format = format;
Ok(self)
}
/// The path for an administration's responses.
///
/// # Arguments
///
/// * `administration_id` - the administration id.
///
/// # Returns
///
/// The path, with slashes in the id replaced so it is one file.
pub fn path_for(&self, administration_id: &str) -> PathBuf {
self.dir.join(format!(
"{}.{}",
sanitize(administration_id),
self.format.extension()
))
}
/// Writes one administration's responses, replacing any existing file.
///
/// Replacing rather than appending is deliberate. Re-ingesting after a
/// regrade should produce the corrected data, not two contradictory copies of
/// the same student's response, and there is no way to tell those apart later.
///
/// # Arguments
///
/// * `set` - the responses, which must all share one administration id.
///
/// # Returns
///
/// The paths written, one per administration found in the set.
///
/// # Errors
///
/// Returns [`Error::Io`] on a write failure and [`Error::FeatureDisabled`]
/// when writing Parquet without the feature.
pub fn write(&self, set: &ResponseSet) -> Result<Vec<PathBuf>> {
let mut written = Vec::new();
for admin in set.administrations() {
let rows: Vec<FlatResponse> = set
.rows
.iter()
.filter(|r| r.administration_id == admin)
.map(FlatResponse::from_response)
.collect();
let path = self.path_for(&admin);
match self.format {
Format::Csv => write_csv(&path, &rows)?,
Format::Parquet => write_parquet(&path, &rows)?,
}
written.push(path);
}
Ok(written)
}
/// Reads one administration's responses.
///
/// # Arguments
///
/// * `administration_id` - the administration id.
///
/// # Returns
///
/// The responses.
///
/// # Errors
///
/// Returns [`Error::Io`] when the file is missing.
pub fn read(&self, administration_id: &str) -> Result<ResponseSet> {
// Accept either format regardless of what this build prefers, so a repo
// written by a Parquet build is still readable by a lean one where the
// CSV happens to exist, and vice versa.
let stem = sanitize(administration_id);
for format in [self.format, Format::Parquet, Format::Csv] {
let path = self.dir.join(format!("{stem}.{}", format.extension()));
if path.exists() {
return read_path(&path);
}
}
Err(Error::Other(format!(
"no stored responses for `{administration_id}` in {}",
self.dir.display()
)))
}
/// Every stored data file, sorted.
///
/// # Returns
///
/// The paths.
///
/// # Errors
///
/// Returns [`Error::Io`] when the directory cannot be listed.
pub fn files(&self) -> Result<Vec<PathBuf>> {
if !self.dir.exists() {
return Ok(Vec::new());
}
let mut out: Vec<PathBuf> = std::fs::read_dir(&self.dir)
.map_err(|e| Error::io(&self.dir, e))?
.filter_map(|e| e.ok())
.map(|e| e.path())
.filter(|p| Format::from_path(p).is_some())
.collect();
out.sort();
Ok(out)
}
/// Reads every stored administration.
///
/// This is what pooled item statistics run on: several administrations of the
/// same item, which is the only way the numbers become trustworthy for a class
/// of twenty-five.
///
/// # Returns
///
/// All responses, with one file's failure recorded as a warning rather than
/// aborting the rest.
///
/// # Errors
///
/// Returns [`Error::Io`] when the directory cannot be listed.
pub fn read_all(&self) -> Result<ResponseSet> {
let mut set = ResponseSet::new();
for path in self.files()? {
match read_path(&path) {
Ok(part) => set.absorb(part),
Err(e) => set
.warnings
.push(format!("skipping {}: {e}", path.display())),
}
}
Ok(set)
}
/// Reads every administration of one assessment, across terms.
///
/// # Arguments
///
/// * `assessment_id` - the assessment id to match.
///
/// # Returns
///
/// The matching responses.
///
/// # Errors
///
/// Returns [`Error::Io`] when the directory cannot be listed.
pub fn read_assessment(&self, assessment_id: &str) -> Result<ResponseSet> {
let all = self.read_all()?;
let mut set = ResponseSet::new();
set.warnings = all.warnings;
set.rows = all
.rows
.into_iter()
.filter(|r| r.assessment_id == assessment_id)
.collect();
Ok(set)
}
}
/// Reads a data file, choosing the reader by extension.
///
/// # Arguments
///
/// * `path` - the file.
///
/// # Returns
///
/// The responses.
///
/// # Errors
///
/// Returns [`Error::Other`] for an unrecognized extension and
/// [`Error::FeatureDisabled`] for Parquet without the feature.
pub fn read_path(path: &Path) -> Result<ResponseSet> {
match Format::from_path(path) {
Some(Format::Csv) => read_csv(path),
Some(Format::Parquet) => read_parquet(path),
None => Err(Error::Other(format!(
"{} is not a response file; expected a .parquet or .csv",
path.display()
))),
}
}
/// Writes flat responses as CSV.
///
/// # Arguments
///
/// * `path` - the destination.
/// * `rows` - the rows.
///
/// # Errors
///
/// Returns [`Error::Csv`] on a serialization failure.
fn write_csv(path: &Path, rows: &[FlatResponse]) -> Result<()> {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent).map_err(|e| Error::io(parent, e))?;
}
let mut w = csv::Writer::from_path(path).map_err(|e| Error::Csv {
path: path.to_path_buf(),
source: e,
})?;
for r in rows {
w.serialize(r).map_err(|e| Error::Csv {
path: path.to_path_buf(),
source: e,
})?;
}
w.flush().map_err(|e| Error::io(path, e))?;
Ok(())
}
/// Reads flat responses from CSV.
///
/// # Arguments
///
/// * `path` - the file.
///
/// # Returns
///
/// The responses.
///
/// # Errors
///
/// Returns [`Error::Csv`] on a parse failure.
fn read_csv(path: &Path) -> Result<ResponseSet> {
let mut r = csv::Reader::from_path(path).map_err(|e| Error::Csv {
path: path.to_path_buf(),
source: e,
})?;
let mut set = ResponseSet::new();
for rec in r.deserialize::<FlatResponse>() {
let flat = rec.map_err(|e| Error::Csv {
path: path.to_path_buf(),
source: e,
})?;
set.rows.push(flat.to_response());
}
Ok(set)
}
/// Writes flat responses as Parquet.
///
/// # Arguments
///
/// * `path` - the destination.
/// * `rows` - the rows.
///
/// # Errors
///
/// Returns [`Error::FeatureDisabled`] when the feature is off.
#[cfg(feature = "parquet")]
fn write_parquet(path: &Path, rows: &[FlatResponse]) -> Result<()> {
crate::store_parquet::write(path, rows)
}
/// Stub for builds without Parquet support.
#[cfg(not(feature = "parquet"))]
fn write_parquet(_path: &Path, _rows: &[FlatResponse]) -> Result<()> {
Err(Error::FeatureDisabled("Parquet", "parquet"))
}
/// Reads flat responses from Parquet.
///
/// # Arguments
///
/// * `path` - the file.
///
/// # Returns
///
/// The responses.
///
/// # Errors
///
/// Returns [`Error::FeatureDisabled`] when the feature is off.
#[cfg(feature = "parquet")]
fn read_parquet(path: &Path) -> Result<ResponseSet> {
let rows = crate::store_parquet::read(path)?;
let mut set = ResponseSet::new();
set.rows = rows.iter().map(|f| f.to_response()).collect();
Ok(set)
}
/// Stub for builds without Parquet support.
#[cfg(not(feature = "parquet"))]
fn read_parquet(path: &Path) -> Result<ResponseSet> {
Err(Error::Other(format!(
"{} is a Parquet file, but this build has Parquet support compiled out; \
rebuild with `--features parquet`, or re-ingest with `--format csv`",
path.display()
)))
}
/// Makes an administration id usable as a file name.
///
/// # Arguments
///
/// * `s` - the id.
///
/// # Returns
///
/// The sanitized stem.
pub fn sanitize(s: &str) -> String {
let mut out = String::with_capacity(s.len());
let mut last_sep = false;
for ch in s.chars() {
if ch.is_ascii_alphanumeric() || ch == '-' || ch == '.' {
out.push(ch.to_ascii_lowercase());
last_sep = false;
} else if !last_sep {
out.push('_');
last_sep = true;
}
}
let trimmed = out.trim_matches('_').to_string();
if trimmed.is_empty() {
"responses".to_string()
} else {
trimmed
}
}
/// Exports responses to an arbitrary path, in the format its extension implies.
///
/// This exists so `export` is a separate verb from `ingest`: the store is the
/// system of record, and handing a colleague a CSV should not change it.
///
/// # Arguments
///
/// * `path` - the destination.
/// * `set` - the responses.
///
/// # Errors
///
/// Returns [`Error::Other`] for an unrecognized extension.
pub fn export(path: &Path, set: &ResponseSet) -> Result<()> {
let rows: Vec<FlatResponse> = set.rows.iter().map(FlatResponse::from_response).collect();
match Format::from_path(path) {
Some(Format::Csv) => write_csv(path, &rows),
Some(Format::Parquet) => write_parquet(path, &rows),
None => Err(Error::Other(format!(
"cannot tell what format {} should be; use a .csv or .parquet extension",
path.display()
))),
}
}
/// Summarizes what is in the store, for `coursebank data list`.
#[derive(Debug, Clone)]
pub struct StoredSummary {
/// The administration id.
pub administration_id: String,
/// The file.
pub path: PathBuf,
/// How many response rows.
pub rows: usize,
/// How many distinct students.
pub students: usize,
/// How many distinct items.
pub items: usize,
}
/// Summarizes every stored administration.
///
/// # Arguments
///
/// * `store` - the store.
///
/// # Returns
///
/// One summary per file, sorted by path.
///
/// # Errors
///
/// Returns [`Error::Io`] when the directory cannot be listed.
pub fn summarize(store: &Store) -> Result<Vec<StoredSummary>> {
let mut out = Vec::new();
for path in store.files()? {
let set = match read_path(&path) {
Ok(s) => s,
Err(_) => continue,
};
let admin = set.administrations().first().cloned().unwrap_or_else(|| {
path.file_stem()
.unwrap_or_default()
.to_string_lossy()
.into()
});
out.push(StoredSummary {
administration_id: admin,
path,
rows: set.rows.len(),
students: set.students().len(),
items: set.all_items().len(),
});
}
Ok(out)
}
/// Groups responses by administration.
///
/// # Arguments
///
/// * `set` - the responses.
///
/// # Returns
///
/// One set per administration, keyed by id.
pub fn split_by_administration(
set: &ResponseSet,
) -> std::collections::BTreeMap<String, Vec<&Response>> {
let mut out: std::collections::BTreeMap<String, Vec<&Response>> = Default::default();
for r in &set.rows {
out.entry(r.administration_id.clone()).or_default().push(r);
}
out
}
#[cfg(test)]
mod tests {
use super::*;
use crate::responses::Response;
fn row(student: &str, number: u32) -> Response {
Response {
administration_id: "BIOSC1540/2026s/exam-4".into(),
course: "BIOSC1540".into(),
term: "2026s".into(),
assessment_id: "exam-4".into(),
date: None,
form: None,
student_key: student.into(),
sid: None,
name: None,
email: None,
section: None,
item_number: number,
item_ref: Some("bank::q-x-001".into()),
item_version: Some(2),
selected: vec!["C".into()],
eliminated: vec![],
correct: Some(true),
credit: 1.0,
points_possible: 1.5,
score: 1.5,
response_time_seconds: None,
level: None,
learning_objectives: vec!["lo-a".into()],
topics: vec![],
bonus: false,
dropped: false,
}
}
#[test]
fn sanitizes_administration_ids() {
assert_eq!(sanitize("BIOSC1540/2026s/exam-4"), "biosc1540_2026s_exam-4");
assert_eq!(sanitize("///"), "responses");
// Runs of separators collapse rather than stacking underscores.
assert_eq!(sanitize("a // b"), "a_b");
}
#[test]
fn csv_round_trips_through_the_store() {
let dir = std::env::temp_dir().join(format!("cb-store-{}", std::process::id()));
std::fs::remove_dir_all(&dir).ok();
let store = Store::open(&dir).unwrap().with_format(Format::Csv).unwrap();
let mut set = ResponseSet::new();
set.rows.push(row("s1", 1));
set.rows.push(row("s2", 1));
let written = store.write(&set).unwrap();
assert_eq!(written.len(), 1);
assert!(written[0].exists());
let back = store.read("BIOSC1540/2026s/exam-4").unwrap();
assert_eq!(back.rows.len(), 2);
assert_eq!(back.rows[0].item_ref.as_deref(), Some("bank::q-x-001"));
assert_eq!(back.rows[0].selected, vec!["C".to_string()]);
assert_eq!(back.rows[0].learning_objectives, vec!["lo-a".to_string()]);
let all = store.read_all().unwrap();
assert_eq!(all.rows.len(), 2);
let summaries = summarize(&store).unwrap();
assert_eq!(summaries.len(), 1);
assert_eq!(summaries[0].students, 2);
assert_eq!(summaries[0].items, 1);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn rewriting_replaces_rather_than_duplicates() {
let dir = std::env::temp_dir().join(format!("cb-store-rw-{}", std::process::id()));
std::fs::remove_dir_all(&dir).ok();
let store = Store::open(&dir).unwrap().with_format(Format::Csv).unwrap();
let mut set = ResponseSet::new();
set.rows.push(row("s1", 1));
store.write(&set).unwrap();
store.write(&set).unwrap();
assert_eq!(store.read_all().unwrap().rows.len(), 1, "no duplicates");
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn missing_administrations_report_clearly() {
let dir = std::env::temp_dir().join(format!("cb-store-miss-{}", std::process::id()));
std::fs::remove_dir_all(&dir).ok();
let store = Store::open(&dir).unwrap().with_format(Format::Csv).unwrap();
let err = store.read("nope").unwrap_err();
assert!(err.to_string().contains("no stored responses"));
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn format_extensions_round_trip() {
assert_eq!(
Format::from_path(Path::new("a/b.parquet")),
Some(Format::Parquet)
);
assert_eq!(Format::from_path(Path::new("a/b.csv")), Some(Format::Csv));
assert_eq!(Format::from_path(Path::new("a/b.yaml")), None);
assert!(Format::Csv.is_available());
}
}
+391
View File
@@ -0,0 +1,391 @@
// SPDX-License-Identifier: Prosperity-3.0.0
// Copyright Scientific Computing Studio
// Source: https://git.scient.ing/education/coursebank
//! Parquet reading and writing, isolated so the rest of the crate never touches
//! Arrow types.
//!
//! Every use of `arrow` and `parquet` in this crate is in this file. That is
//! deliberate: those two crates move fast and release breaking versions together,
//! and confining them to one module means a version bump is a single-file edit
//! rather than a refactor. It also means `--no-default-features` compiles the
//! whole tool without them.
//!
//! The schema mirrors [`FlatResponse`] field for field. Optional values are
//! written with the same sentinels the CSV path uses — empty string, `0` for an
//! unknown level — rather than nulls, so that a Parquet file and a CSV file of
//! the same data load identically in pandas.
use std::path::Path;
use std::sync::Arc;
use arrow_array::{ArrayRef, BooleanArray, Float64Array, RecordBatch, StringArray, UInt32Array};
use arrow_schema::{DataType, Field, Schema};
use parquet::arrow::ArrowWriter;
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
use parquet::basic::Compression;
use parquet::file::properties::WriterProperties;
use crate::error::{Error, Result};
use crate::responses::FlatResponse;
/// The Arrow schema for stored responses.
///
/// # Returns
///
/// The schema, whose field order matches [`FlatResponse`].
pub fn schema() -> Schema {
Schema::new(vec![
Field::new("administration_id", DataType::Utf8, false),
Field::new("course", DataType::Utf8, false),
Field::new("term", DataType::Utf8, false),
Field::new("assessment_id", DataType::Utf8, false),
Field::new("date", DataType::Utf8, false),
Field::new("form", DataType::Utf8, false),
Field::new("student_key", DataType::Utf8, false),
Field::new("sid", DataType::Utf8, false),
Field::new("email", DataType::Utf8, false),
Field::new("section", DataType::Utf8, false),
Field::new("item_number", DataType::UInt32, false),
Field::new("item_ref", DataType::Utf8, false),
Field::new("item_version", DataType::UInt32, false),
Field::new("selected", DataType::Utf8, false),
Field::new("eliminated", DataType::Utf8, false),
Field::new("correct", DataType::Utf8, false),
Field::new("credit", DataType::Float64, false),
Field::new("points_possible", DataType::Float64, false),
Field::new("score", DataType::Float64, false),
Field::new("response_time_seconds", DataType::Utf8, false),
Field::new("level", DataType::UInt32, false),
Field::new("learning_objectives", DataType::Utf8, false),
Field::new("topics", DataType::Utf8, false),
Field::new("bonus", DataType::Boolean, false),
Field::new("dropped", DataType::Boolean, false),
])
}
/// Builds a record batch from flat responses.
///
/// # Arguments
///
/// * `rows` - the rows.
///
/// # Returns
///
/// The batch.
///
/// # Errors
///
/// Returns [`Error::Other`] when Arrow rejects the column set, which would mean
/// this function and [`schema`] have drifted apart.
fn to_batch(rows: &[FlatResponse]) -> Result<RecordBatch> {
let s = |f: fn(&FlatResponse) -> &str| -> ArrayRef {
Arc::new(StringArray::from(rows.iter().map(f).collect::<Vec<&str>>()))
};
let f64c = |f: fn(&FlatResponse) -> f64| -> ArrayRef {
Arc::new(Float64Array::from(rows.iter().map(f).collect::<Vec<f64>>()))
};
let u32c = |f: fn(&FlatResponse) -> u32| -> ArrayRef {
Arc::new(UInt32Array::from(rows.iter().map(f).collect::<Vec<u32>>()))
};
let boolc = |f: fn(&FlatResponse) -> bool| -> ArrayRef {
Arc::new(BooleanArray::from(
rows.iter().map(f).collect::<Vec<bool>>(),
))
};
let columns: Vec<ArrayRef> = vec![
s(|r| &r.administration_id),
s(|r| &r.course),
s(|r| &r.term),
s(|r| &r.assessment_id),
s(|r| &r.date),
s(|r| &r.form),
s(|r| &r.student_key),
s(|r| &r.sid),
s(|r| &r.email),
s(|r| &r.section),
u32c(|r| r.item_number),
s(|r| &r.item_ref),
u32c(|r| r.item_version),
s(|r| &r.selected),
s(|r| &r.eliminated),
s(|r| &r.correct),
f64c(|r| r.credit),
f64c(|r| r.points_possible),
f64c(|r| r.score),
s(|r| &r.response_time_seconds),
u32c(|r| r.level as u32),
s(|r| &r.learning_objectives),
s(|r| &r.topics),
boolc(|r| r.bonus),
boolc(|r| r.dropped),
];
RecordBatch::try_new(Arc::new(schema()), columns).map_err(|e| {
Error::Other(format!(
"internal error building a Parquet batch: {e}. This means the Arrow schema and the \
column builders in store_parquet.rs have drifted apart"
))
})
}
/// Writes flat responses to a Parquet file.
///
/// # Arguments
///
/// * `path` - the destination.
/// * `rows` - the rows.
///
/// # Errors
///
/// Returns [`Error::Io`] on a write failure and [`Error::Other`] when the
/// Parquet writer rejects the batch.
pub fn write(path: &Path, rows: &[FlatResponse]) -> Result<()> {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent).map_err(|e| Error::io(parent, e))?;
}
let batch = to_batch(rows)?;
let file = std::fs::File::create(path).map_err(|e| Error::io(path, e))?;
// Snappy rather than zstd: it is the format's most universally readable
// codec, and these files are small enough that the ratio difference is
// irrelevant next to being openable by an old pandas.
let props = WriterProperties::builder()
.set_compression(Compression::SNAPPY)
.build();
let mut writer = ArrowWriter::try_new(file, Arc::new(schema()), Some(props))
.map_err(|e| Error::Other(format!("cannot open {} for Parquet: {e}", path.display())))?;
writer
.write(&batch)
.map_err(|e| Error::Other(format!("cannot write {}: {e}", path.display())))?;
writer
.close()
.map_err(|e| Error::Other(format!("cannot finish {}: {e}", path.display())))?;
Ok(())
}
/// Reads flat responses from a Parquet file.
///
/// # Arguments
///
/// * `path` - the file.
///
/// # Returns
///
/// The rows.
///
/// # Errors
///
/// Returns [`Error::Io`] when the file cannot be opened and [`Error::Other`]
/// when its schema does not match what this crate writes.
pub fn read(path: &Path) -> Result<Vec<FlatResponse>> {
let file = std::fs::File::open(path).map_err(|e| Error::io(path, e))?;
let builder = ParquetRecordBatchReaderBuilder::try_new(file)
.map_err(|e| Error::Other(format!("cannot read {} as Parquet: {e}", path.display())))?;
let reader = builder
.build()
.map_err(|e| Error::Other(format!("cannot read {}: {e}", path.display())))?;
let mut out = Vec::new();
for batch in reader {
let batch =
batch.map_err(|e| Error::Other(format!("cannot read {}: {e}", path.display())))?;
out.extend(from_batch(&batch, path)?);
}
Ok(out)
}
/// Converts a record batch back into flat responses.
///
/// # Arguments
///
/// * `batch` - the batch.
/// * `path` - the source file, for error messages.
///
/// # Returns
///
/// The rows.
///
/// # Errors
///
/// Returns [`Error::Other`] when a column is missing or has the wrong type.
fn from_batch(batch: &RecordBatch, path: &Path) -> Result<Vec<FlatResponse>> {
let strings = |name: &str| -> Result<&StringArray> {
batch
.column_by_name(name)
.and_then(|c| c.as_any().downcast_ref::<StringArray>())
.ok_or_else(|| column_error(name, "string", path))
};
let floats = |name: &str| -> Result<&Float64Array> {
batch
.column_by_name(name)
.and_then(|c| c.as_any().downcast_ref::<Float64Array>())
.ok_or_else(|| column_error(name, "float64", path))
};
let uints = |name: &str| -> Result<&UInt32Array> {
batch
.column_by_name(name)
.and_then(|c| c.as_any().downcast_ref::<UInt32Array>())
.ok_or_else(|| column_error(name, "uint32", path))
};
let bools = |name: &str| -> Result<&BooleanArray> {
batch
.column_by_name(name)
.and_then(|c| c.as_any().downcast_ref::<BooleanArray>())
.ok_or_else(|| column_error(name, "boolean", path))
};
let administration_id = strings("administration_id")?;
let course = strings("course")?;
let term = strings("term")?;
let assessment_id = strings("assessment_id")?;
let date = strings("date")?;
let form = strings("form")?;
let student_key = strings("student_key")?;
let sid = strings("sid")?;
let email = strings("email")?;
let section = strings("section")?;
let item_number = uints("item_number")?;
let item_ref = strings("item_ref")?;
let item_version = uints("item_version")?;
let selected = strings("selected")?;
let eliminated = strings("eliminated")?;
let correct = strings("correct")?;
let credit = floats("credit")?;
let points_possible = floats("points_possible")?;
let score = floats("score")?;
let response_time_seconds = strings("response_time_seconds")?;
let level = uints("level")?;
let learning_objectives = strings("learning_objectives")?;
let topics = strings("topics")?;
let bonus = bools("bonus")?;
let dropped = bools("dropped")?;
let mut out = Vec::with_capacity(batch.num_rows());
for i in 0..batch.num_rows() {
out.push(FlatResponse {
administration_id: administration_id.value(i).to_string(),
course: course.value(i).to_string(),
term: term.value(i).to_string(),
assessment_id: assessment_id.value(i).to_string(),
date: date.value(i).to_string(),
form: form.value(i).to_string(),
student_key: student_key.value(i).to_string(),
sid: sid.value(i).to_string(),
email: email.value(i).to_string(),
section: section.value(i).to_string(),
item_number: item_number.value(i),
item_ref: item_ref.value(i).to_string(),
item_version: item_version.value(i),
selected: selected.value(i).to_string(),
eliminated: eliminated.value(i).to_string(),
correct: correct.value(i).to_string(),
credit: credit.value(i),
points_possible: points_possible.value(i),
score: score.value(i),
response_time_seconds: response_time_seconds.value(i).to_string(),
level: level.value(i) as u8,
learning_objectives: learning_objectives.value(i).to_string(),
topics: topics.value(i).to_string(),
bonus: bonus.value(i),
dropped: dropped.value(i),
});
}
Ok(out)
}
/// Builds the error for a missing or mistyped column.
fn column_error(name: &str, expected: &str, path: &Path) -> Error {
Error::Other(format!(
"{} is missing the `{name}` column or it is not {expected}; this file was probably not \
written by coursebank",
path.display()
))
}
#[cfg(test)]
mod tests {
use super::*;
fn flat(student: &str, number: u32) -> FlatResponse {
FlatResponse {
administration_id: "C/2026s/e1".into(),
course: "C".into(),
term: "2026s".into(),
assessment_id: "e1".into(),
date: "2026-04-01".into(),
form: String::new(),
student_key: student.into(),
sid: "1234567".into(),
email: String::new(),
section: "L01".into(),
item_number: number,
item_ref: "bank::q-a-001".into(),
item_version: 3,
selected: "C".into(),
eliminated: String::new(),
correct: "1".into(),
credit: 1.0,
points_possible: 1.5,
score: 1.5,
response_time_seconds: "42.0".into(),
level: 3,
learning_objectives: "lo-a,lo-b".into(),
topics: "kinetics".into(),
bonus: false,
dropped: false,
}
}
#[test]
fn schema_matches_the_column_builders() {
let rows = vec![flat("s1", 1)];
let batch = to_batch(&rows).expect("schema and builders agree");
assert_eq!(batch.num_columns(), schema().fields().len());
assert_eq!(batch.num_rows(), 1);
}
#[test]
fn round_trips_through_a_file() {
let dir = std::env::temp_dir().join(format!("cb-pq-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("e1.parquet");
let rows = vec![flat("s1", 1), flat("s2", 2)];
write(&path, &rows).unwrap();
let back = read(&path).unwrap();
assert_eq!(back.len(), 2);
assert_eq!(back[0].student_key, "s1");
assert_eq!(back[1].item_number, 2);
assert_eq!(back[0].learning_objectives, "lo-a,lo-b");
assert_eq!(back[0].credit, 1.0);
assert_eq!(back[0].level, 3);
assert!(!back[0].bonus);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn an_empty_batch_is_writable() {
let dir = std::env::temp_dir().join(format!("cb-pq-empty-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("empty.parquet");
write(&path, &[]).unwrap();
assert_eq!(read(&path).unwrap().len(), 0);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn a_foreign_file_is_rejected_clearly() {
let dir = std::env::temp_dir().join(format!("cb-pq-bad-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("not.parquet");
std::fs::write(&path, b"this is not parquet").unwrap();
let err = read(&path).unwrap_err();
assert!(err.to_string().contains("as Parquet"));
std::fs::remove_dir_all(&dir).ok();
}
}