feat: implement reports

This commit is contained in:
alexm committed 2026-09-19 20:06:41 -04:00
1 parent 6ae993d393
commit 994e9065e8
28 files changed
+7082 -63

No files matched your search

+6
View File
@@ -325,6 +325,10 @@ pub fn ingest(
assessment_id: ctx.assessment_id.clone(),
date: ctx.date,
form: ctx.form.clone(),
// Canvas numbers questions as the record does, and its exports
// carry no printed order, so there is no printed position to
// record and no letter map to decode against.
form_position: None,
student_key: student_key.clone(),
sid: sid.clone(),
name: name.clone(),
@@ -334,7 +338,9 @@ pub fn ingest(
item_ref,
item_version: None,
selected,
selected_source: Vec::new(),
eliminated: Vec::new(),
eliminated_source: Vec::new(),
correct,
credit,
points_possible: points,
+803
View File
@@ -0,0 +1,803 @@
// SPDX-License-Identifier: Prosperity-3.0.0
// Copyright Scientific Computing Studio
// Source: https://git.scient.ing/education/coursebank
//! Turning what a student marked into what a student chose.
//!
//! A grading export speaks in positions and printed letters. Question 14 is the
//! fourteenth thing on the page; option C is the third bubble. An item bank speaks
//! in ids and its own lettering. When forms shuffle, those two vocabularies
//! disagree, and every analysis downstream of the disagreement is wrong in a way
//! that looks right:
//!
//! * Pooled distractor statistics add form A's option C to form B's option C,
//! which are different sentences. The resulting table is noise with the shape of
//! data.
//! * A student report looks up the misconception recorded on option C and shows it
//! to a student who chose a different option. The feedback is confident,
//! specific, and about the wrong thing.
//! * Any `credit_overrides` written in the record's lettering are applied to
//! whoever happened to mark that letter on their form.
//!
//! None of these fail loudly. That is the argument for doing the translation once,
//! at ingest, and storing both sides of it.
//!
//! # What a decoder knows
//!
//! For one form: which recorded question number sits at each printed position,
//! which bank letter each printed letter stands for, and which printed letters are
//! keyed. It is built from a [`SealFile`] when one exists, and derived from the
//! record and the form seed when one does not. Sealed is better, and not only
//! because it is faster: a derived decoder describes the form the bank *would*
//! print today, while a sealed one describes the form that was actually printed.
//!
//! # Catching a swapped directory
//!
//! Gradescope's point-value row reveals which printed letter earned full credit on
//! every question. A decoder knows what that letter should be. Comparing them
//! across a whole directory is close to a proof of which form the directory holds:
//! agreement is near total for the right form and near chance for the wrong one.
//! [`identify_form`] uses that to refuse an ingest that names form A over a
//! directory of form B papers, which is otherwise a mistake nobody catches until
//! the item statistics look strange three weeks later.
use std::collections::{BTreeMap, BTreeSet};
use crate::assessment::{AssessmentFile, Form};
use crate::catalog::Catalog;
use crate::error::{Error, Result};
use crate::gradescope::Question;
use crate::responses::ResponseSet;
use crate::seal::{SealFile, printed_letter};
use crate::select;
/// Where a decoder's mapping came from.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Provenance {
/// Read from a seal written before administration. Authoritative.
Seal,
/// Derived from the assessment record and the form seed, as of now.
Derived,
}
impl Provenance {
/// A short label for output.
pub fn label(self) -> &'static str {
match self {
Provenance::Seal => "seal",
Provenance::Derived => "derived from the record",
}
}
}
/// How a grading export numbers its questions.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Numbering {
/// The export counts printed positions, which is what Gradescope's `N.csv`
/// file names mean. Positions are translated to recorded numbers.
Printed,
/// The export already carries recorded question numbers, so numbers pass
/// through untouched.
Recorded,
}
/// One question's mapping on one form.
#[derive(Debug, Clone)]
pub struct QuestionMap {
/// Printed position on the page, counting from 1.
pub position: u32,
/// The recorded question number, the join key to the record and the store.
pub number: u32,
/// The item's global id.
pub item: String,
/// Keyed letters as printed on this form.
pub printed_key: Vec<String>,
/// Keyed letters in the bank's own lettering.
pub canonical_key: Vec<String>,
/// Printed letter to bank letter.
pub to_canonical: BTreeMap<String, String>,
/// Bank letter to printed letter.
pub to_printed: BTreeMap<String, String>,
}
impl QuestionMap {
/// The bank letter a printed letter stands for.
///
/// # Arguments
///
/// * `printed` - the letter as the student saw it.
///
/// # Returns
///
/// The bank letter, or `None` when the printed letter is not one of this
/// question's options.
pub fn canonical(&self, printed: &str) -> Option<&str> {
self.to_canonical
.get(&printed.trim().to_ascii_uppercase())
.map(|s| s.as_str())
}
/// How many options this question has.
pub fn n_options(&self) -> usize {
self.to_canonical.len()
}
/// Whether this question's options were actually permuted.
pub fn is_permuted(&self) -> bool {
self.to_canonical.iter().any(|(k, v)| k != v)
}
}
/// One form's full mapping.
#[derive(Debug, Clone)]
pub struct FormDecoder {
/// The form id.
pub form: String,
/// Where the mapping came from.
pub provenance: Provenance,
/// Questions by printed position.
by_position: BTreeMap<u32, QuestionMap>,
/// Questions by recorded number.
by_number: BTreeMap<u32, QuestionMap>,
}
impl FormDecoder {
/// Builds a decoder from the question maps.
fn assemble(form: String, provenance: Provenance, maps: Vec<QuestionMap>) -> FormDecoder {
let by_position = maps.iter().map(|m| (m.position, m.clone())).collect();
let by_number = maps.into_iter().map(|m| (m.number, m)).collect();
FormDecoder {
form,
provenance,
by_position,
by_number,
}
}
/// Reads one form's mapping out of a seal.
///
/// # Arguments
///
/// * `seal` - the seal.
/// * `form_id` - the form to decode, matched case-insensitively.
///
/// # Returns
///
/// The decoder.
///
/// # Errors
///
/// Returns [`Error::Usage`] when the seal does not cover that form.
pub fn from_seal(seal: &SealFile, form_id: &str) -> Result<FormDecoder> {
let form = seal.form(form_id).ok_or_else(|| {
Error::usage(format!(
"the seal for `{}` does not cover form `{form_id}`; it covers {}",
seal.seal.assessment,
seal.forms
.iter()
.map(|f| f.id.as_str())
.collect::<Vec<_>>()
.join(", ")
))
})?;
let maps = form
.questions
.iter()
.map(|q| {
let mut to_canonical = BTreeMap::new();
let mut to_printed = BTreeMap::new();
for map in &q.options {
to_canonical.insert(map.printed.clone(), map.canonical.clone());
to_printed.insert(map.canonical.clone(), map.printed.clone());
}
let canonical_key: Vec<String> = q
.printed_key
.iter()
.filter_map(|p| to_canonical.get(p).cloned())
.collect();
QuestionMap {
position: q.position,
number: q.number,
item: q.item.clone(),
printed_key: q.printed_key.clone(),
canonical_key,
to_canonical,
to_printed,
}
})
.collect();
Ok(FormDecoder::assemble(
form.id.clone(),
Provenance::Seal,
maps,
))
}
/// Derives one form's mapping from the record and the bank.
///
/// Uses the same two functions every export calls, so a derived decoder and a
/// freshly exported paper agree by construction.
///
/// # Arguments
///
/// * `catalog` - the loaded course.
/// * `record` - the assessment record.
/// * `form` - the form.
///
/// # Returns
///
/// The decoder.
///
/// # Errors
///
/// Returns [`Error::Unresolved`] when a placement references a missing item.
pub fn derive(catalog: &Catalog, record: &AssessmentFile, form: &Form) -> Result<FormDecoder> {
let printed: Vec<_> = select::layout(record, form)
.into_iter()
.filter(|p| !p.dropped)
.collect();
let mut maps = Vec::with_capacity(printed.len());
for (index, placement) in printed.iter().enumerate() {
let entry = catalog.require(&placement.item)?;
let item = &entry.item;
let order = select::option_order(form, &placement.item, item.options.len());
let canonical_key: BTreeSet<String> = if placement.key.is_empty() {
item.key_letters().into_iter().collect()
} else {
placement.key.iter().cloned().collect()
};
let mut to_canonical = BTreeMap::new();
let mut to_printed = BTreeMap::new();
let mut printed_key = Vec::new();
for (position, source_index) in order.iter().enumerate() {
let canonical = item
.options
.get(*source_index)
.map(|c| c.id.clone())
.unwrap_or_else(|| printed_letter(*source_index));
let label = printed_letter(position);
if canonical_key.contains(&canonical) {
printed_key.push(label.clone());
}
to_canonical.insert(label.clone(), canonical.clone());
to_printed.insert(canonical, label);
}
maps.push(QuestionMap {
position: index as u32 + 1,
number: placement.number,
item: placement.item.clone(),
printed_key,
canonical_key: canonical_key.into_iter().collect(),
to_canonical,
to_printed,
});
}
Ok(FormDecoder::assemble(
form.id.clone(),
Provenance::Derived,
maps,
))
}
/// Builds a decoder, preferring the seal.
///
/// # Arguments
///
/// * `seal` - the seal, when one has been written.
/// * `catalog` - the loaded course.
/// * `record` - the assessment record.
/// * `form` - the form.
///
/// # Returns
///
/// The decoder.
///
/// # Errors
///
/// As [`FormDecoder::from_seal`] and [`FormDecoder::derive`]. A seal that does
/// not cover the requested form falls back to deriving rather than failing,
/// since a form added after sealing is a real situation.
pub fn resolve(
seal: Option<&SealFile>,
catalog: &Catalog,
record: &AssessmentFile,
form: &Form,
) -> Result<FormDecoder> {
if let Some(seal) = seal {
if seal.form(&form.id).is_some() {
return FormDecoder::from_seal(seal, &form.id);
}
}
FormDecoder::derive(catalog, record, form)
}
/// The question at a printed position.
///
/// # Arguments
///
/// * `position` - the printed position, counting from 1.
pub fn at_position(&self, position: u32) -> Option<&QuestionMap> {
self.by_position.get(&position)
}
/// The question with a recorded number.
///
/// # Arguments
///
/// * `number` - the recorded number.
pub fn at_number(&self, number: u32) -> Option<&QuestionMap> {
self.by_number.get(&number)
}
/// The question an export's numbering refers to.
///
/// # Arguments
///
/// * `n` - the number as the export gives it.
/// * `numbering` - how the export numbers questions.
pub fn lookup(&self, n: u32, numbering: Numbering) -> Option<&QuestionMap> {
match numbering {
Numbering::Printed => self.at_position(n),
Numbering::Recorded => self.at_number(n),
}
}
/// How many questions this form prints.
pub fn len(&self) -> usize {
self.by_position.len()
}
/// Whether the form prints nothing, which means the record is empty.
pub fn is_empty(&self) -> bool {
self.by_position.is_empty()
}
/// Whether any question on this form has permuted options.
///
/// Used to decide whether to say anything about translation at all: on an
/// unshuffled form the whole mechanism is an identity map and mentioning it
/// is noise.
pub fn is_permuted(&self) -> bool {
self.by_position.values().any(|q| q.is_permuted())
}
/// Whether printed positions and recorded numbers disagree anywhere.
///
/// True when items were shuffled, and also when a bonus item sits mid-record,
/// since the layout moves bonus items to the end of the paper.
pub fn is_renumbered(&self) -> bool {
self.by_position.values().any(|q| q.position != q.number)
}
}
/// What a directory of graded questions says about which form it holds.
#[derive(Debug, Clone)]
pub struct FormFit {
/// The form id.
pub form: String,
/// Questions whose graded key matched this form's printed key.
pub matched: usize,
/// Questions that could be compared at all.
pub compared: usize,
/// Question positions where the graded key disagreed.
pub mismatches: Vec<u32>,
}
impl FormFit {
/// The share of comparable questions that agreed.
pub fn rate(&self) -> f64 {
if self.compared == 0 {
0.0
} else {
self.matched as f64 / self.compared as f64
}
}
/// Whether the fit is good enough to proceed without a warning.
///
/// The threshold is high on purpose. A correctly matched directory agrees on
/// every question; anything less than total agreement is either a regrade that
/// moved a key or the wrong directory, and both are worth a sentence.
pub fn is_convincing(&self) -> bool {
self.compared > 0 && self.matched == self.compared
}
}
/// Compares a parsed Gradescope directory against one form's expected keys.
///
/// # Arguments
///
/// * `questions` - the parsed question files.
/// * `decoder` - the form to test against.
/// * `numbering` - how the export numbers questions.
///
/// # Returns
///
/// The fit.
pub fn fit_form(questions: &[Question], decoder: &FormDecoder, numbering: Numbering) -> FormFit {
let mut matched = 0usize;
let mut compared = 0usize;
let mut mismatches = Vec::new();
for question in questions {
let Some(map) = decoder.lookup(question.number, numbering) else {
continue;
};
let graded: BTreeSet<String> = question.keyed().into_iter().collect();
if graded.is_empty() {
continue;
}
let expected: BTreeSet<String> = map.printed_key.iter().cloned().collect();
if expected.is_empty() {
continue;
}
compared += 1;
if graded == expected {
matched += 1;
} else {
mismatches.push(question.number);
}
}
FormFit {
form: decoder.form.clone(),
matched,
compared,
mismatches,
}
}
/// Ranks every candidate form against a directory.
///
/// # Arguments
///
/// * `questions` - the parsed question files.
/// * `decoders` - one decoder per declared form.
/// * `numbering` - how the export numbers questions.
///
/// # Returns
///
/// The fits, best first.
pub fn identify_form(
questions: &[Question],
decoders: &[FormDecoder],
numbering: Numbering,
) -> Vec<FormFit> {
let mut fits: Vec<FormFit> = decoders
.iter()
.map(|d| fit_form(questions, d, numbering))
.collect();
fits.sort_by(|a, b| {
b.rate()
.partial_cmp(&a.rate())
.unwrap_or(std::cmp::Ordering::Equal)
.then_with(|| a.form.cmp(&b.form))
});
fits
}
/// Explains a fit in a sentence, or says nothing when the fit is perfect.
///
/// # Arguments
///
/// * `claimed` - the form the directory was ingested as.
/// * `fits` - every form's fit, best first.
///
/// # Returns
///
/// A warning, or `None`.
pub fn form_warning(claimed: &str, fits: &[FormFit]) -> Option<String> {
let mine = fits.iter().find(|f| f.form.eq_ignore_ascii_case(claimed))?;
if mine.is_convincing() {
return None;
}
if mine.compared == 0 {
return Some(format!(
"form {claimed}: the export carries no point values, so the graded keys could not be \
checked against the form. Nothing verified this directory is form {claimed}"
));
}
let better = fits
.iter()
.find(|f| !f.form.eq_ignore_ascii_case(claimed) && f.rate() > mine.rate());
let head = format!(
"form {claimed}: the graded key matches this form on {} of {} question(s)",
mine.matched, mine.compared
);
let where_ = if mine.mismatches.is_empty() {
String::new()
} else {
let list: Vec<String> = mine
.mismatches
.iter()
.take(8)
.map(|n| n.to_string())
.collect();
format!(
" (q{}{})",
list.join(", q"),
if mine.mismatches.len() > 8 {
", …"
} else {
""
}
)
};
match better {
Some(other) => Some(format!(
"{head}{where_}, but matches form {} on {} of {}. This directory is almost certainly \
form {}, not form {claimed}",
other.form, other.matched, other.compared, other.form
)),
None => Some(format!(
"{head}{where_}. Either those questions were regraded after printing, or the form is \
not the one named"
)),
}
}
/// Translates a form's responses into the bank's vocabulary.
///
/// Rewrites, for every row whose `form` matches this decoder:
///
/// * `item_number`, from printed position to recorded number, when the export
/// numbers by position;
/// * `form_position`, recording where the question sat on the page;
/// * `selected_source` and `eliminated_source`, the bank letters for what was
/// marked. The printed letters stay in `selected` and `eliminated`, because what
/// a student physically marked is the fact and the translation is the
/// interpretation.
///
/// Correctness and credit are untouched. Both come from the grading platform,
/// which scored the paper the student actually held, and are already right.
///
/// # Arguments
///
/// * `set` - the responses to translate, in place.
/// * `decoder` - the form's mapping.
/// * `numbering` - how the export numbered questions.
///
/// # Returns
///
/// Warnings for anything that could not be translated.
pub fn apply(set: &mut ResponseSet, decoder: &FormDecoder, numbering: Numbering) -> Vec<String> {
let mut warnings = Vec::new();
let mut unmapped_positions: BTreeSet<u32> = BTreeSet::new();
let mut unmapped_letters: BTreeSet<String> = BTreeSet::new();
let mut translated = 0usize;
for row in &mut set.rows {
let belongs = row
.form
.as_deref()
.map(|f| f.eq_ignore_ascii_case(&decoder.form))
.unwrap_or(false);
if !belongs {
continue;
}
let Some(map) = decoder.lookup(row.item_number, numbering) else {
unmapped_positions.insert(row.item_number);
continue;
};
row.form_position = Some(map.position);
row.item_number = map.number;
let mut convert = |letters: &[String]| -> Vec<String> {
let mut out = Vec::with_capacity(letters.len());
for letter in letters {
match map.canonical(letter) {
Some(canonical) => out.push(canonical.to_string()),
None => {
unmapped_letters.insert(format!("q{} {}", map.number, letter));
}
}
}
out.sort();
out
};
row.selected_source = convert(&row.selected);
row.eliminated_source = convert(&row.eliminated);
translated += 1;
}
if !unmapped_positions.is_empty() {
let list: Vec<String> = unmapped_positions.iter().map(|n| n.to_string()).collect();
warnings.push(format!(
"form {}: question(s) {} are in the export but not on this form; they were left \
untranslated",
decoder.form,
list.join(", ")
));
}
if !unmapped_letters.is_empty() {
let list: Vec<String> = unmapped_letters.iter().take(10).cloned().collect();
warnings.push(format!(
"form {}: {} marked option(s) are not options on the printed form ({}), which usually \
means a rubric column was added by hand in Gradescope",
decoder.form,
unmapped_letters.len(),
list.join(", ")
));
}
if translated == 0 {
warnings.push(format!(
"form {}: no response rows carried this form id, so nothing was translated",
decoder.form
));
}
warnings
}
#[cfg(test)]
mod tests {
use super::*;
use crate::responses::{Response, administration_id};
fn map(position: u32, number: u32, pairs: &[(&str, &str)], key: &str) -> QuestionMap {
let mut to_canonical = BTreeMap::new();
let mut to_printed = BTreeMap::new();
for (printed, canonical) in pairs {
to_canonical.insert(printed.to_string(), canonical.to_string());
to_printed.insert(canonical.to_string(), printed.to_string());
}
let printed_key = vec![to_printed.get(key).cloned().unwrap_or_default()];
QuestionMap {
position,
number,
item: format!("b::q-{number}"),
printed_key,
canonical_key: vec![key.to_string()],
to_canonical,
to_printed,
}
}
fn decoder() -> FormDecoder {
FormDecoder::assemble(
"B".to_string(),
Provenance::Seal,
vec![
// Printed A..D show bank C, A, D, B. The key is bank A, printed B.
map(1, 1, &[("A", "C"), ("B", "A"), ("C", "D"), ("D", "B")], "A"),
// A bonus item recorded as 9 but printed last, at position 2.
map(2, 9, &[("A", "B"), ("B", "A")], "B"),
],
)
}
fn row(number: u32, selected: &str) -> Response {
Response {
administration_id: administration_id("C", "2026f", "e1"),
course: "C".into(),
term: "2026f".into(),
assessment_id: "e1".into(),
date: None,
form: Some("B".into()),
student_key: "s1".into(),
sid: None,
name: None,
email: None,
section: None,
item_number: number,
form_position: None,
item_ref: None,
item_version: None,
selected: vec![selected.into()],
eliminated: Vec::new(),
selected_source: Vec::new(),
eliminated_source: Vec::new(),
correct: Some(false),
credit: 0.0,
points_possible: 1.0,
score: 0.0,
response_time_seconds: None,
level: None,
learning_objectives: Vec::new(),
topics: Vec::new(),
bonus: false,
dropped: false,
}
}
#[test]
fn printed_letters_become_bank_letters() {
let decoder = decoder();
let q = decoder.at_position(1).unwrap();
assert_eq!(q.canonical("A"), Some("C"));
assert_eq!(q.canonical("D"), Some("B"));
assert_eq!(q.canonical("E"), None);
assert_eq!(q.printed_key, vec!["B".to_string()]);
}
#[test]
fn positions_become_recorded_numbers() {
let mut set = ResponseSet::new();
set.rows.push(row(2, "A"));
let warnings = apply(&mut set, &decoder(), Numbering::Printed);
assert!(warnings.is_empty(), "{warnings:?}");
assert_eq!(set.rows[0].item_number, 9, "position 2 is recorded as 9");
assert_eq!(set.rows[0].form_position, Some(2));
assert_eq!(set.rows[0].selected, vec!["A".to_string()], "printed kept");
assert_eq!(set.rows[0].selected_source, vec!["B".to_string()]);
}
#[test]
fn rows_from_another_form_are_left_alone() {
let mut set = ResponseSet::new();
let mut other = row(1, "A");
other.form = Some("A".into());
set.rows.push(other);
apply(&mut set, &decoder(), Numbering::Printed);
assert!(set.rows[0].selected_source.is_empty());
assert_eq!(set.rows[0].item_number, 1);
}
#[test]
fn an_unknown_position_is_reported_not_guessed() {
let mut set = ResponseSet::new();
set.rows.push(row(7, "A"));
let warnings = apply(&mut set, &decoder(), Numbering::Printed);
assert!(
warnings.iter().any(|w| w.contains("not on this form")),
"{warnings:?}"
);
assert_eq!(set.rows[0].item_number, 7, "left as found");
}
#[test]
fn a_perfect_fit_says_nothing() {
let fits = vec![FormFit {
form: "A".into(),
matched: 30,
compared: 30,
mismatches: Vec::new(),
}];
assert!(form_warning("A", &fits).is_none());
}
#[test]
fn a_swapped_directory_names_the_form_it_really_is() {
let fits = vec![
FormFit {
form: "B".into(),
matched: 30,
compared: 30,
mismatches: Vec::new(),
},
FormFit {
form: "A".into(),
matched: 8,
compared: 30,
mismatches: (1..=22).collect(),
},
];
let warning = form_warning("A", &fits).expect("a mismatch this large must warn");
assert!(warning.contains("almost certainly form B"), "{warning}");
}
#[test]
fn a_single_regraded_key_warns_without_accusing_the_wrong_form() {
let fits = vec![FormFit {
form: "A".into(),
matched: 29,
compared: 30,
mismatches: vec![14],
}];
let warning = form_warning("A", &fits).unwrap();
assert!(warning.contains("q14"), "{warning}");
assert!(warning.contains("regraded"), "{warning}");
}
}
+6
View File
@@ -633,6 +633,10 @@ pub fn to_responses(questions: &[Question], ctx: &Context) -> Import {
assessment_id: ctx.assessment_id.clone(),
date: ctx.date,
form: ctx.form.clone(),
// The printed position and the bank's lettering are written
// later, by `decode::apply`, which is the only place that knows
// which form this directory holds.
form_position: None,
student_key,
sid: row.sid.clone(),
name: row.name.clone(),
@@ -642,7 +646,9 @@ pub fn to_responses(questions: &[Question], ctx: &Context) -> Import {
item_ref: None,
item_version: None,
selected,
selected_source: Vec::new(),
eliminated,
eliminated_source: Vec::new(),
correct,
credit,
points_possible: points,
+565
View File
@@ -0,0 +1,565 @@
// SPDX-License-Identifier: Prosperity-3.0.0
// Copyright Scientific Computing Studio
// Source: https://git.scient.ing/education/coursebank
//! Reading several forms of one exam back in at once.
//!
//! A two-form exam is two Gradescope assignments, each exporting its own
//! directory of `1.csv` through `N.csv`, each numbered against its own paper. They
//! are one administration: one set of students, one item pool, one set of
//! statistics. Ingesting them one command at a time does not work, because the
//! store keys on the administration and the second write replaces the first.
//!
//! So this module takes the whole set:
//!
//! ```text
//! coursebank ingest gradescope A=exports/e1-a B=exports/e1-b --assessment e1
//! ```
//!
//! and does four things the single-directory path cannot:
//!
//! *Checks each directory is the form it claims to be.* Every export carries the
//! graded key in its point-value row; every form knows what its printed key should
//! be. Comparing them catches a swapped pair of directories immediately rather
//! than three weeks later, when the distractor table looks strange. See
//! [`crate::decode::identify_form`].
//!
//! *Translates each form into the bank's vocabulary before merging.* Otherwise
//! form A's option C and form B's option C land in the same column of the same
//! table while meaning different things.
//!
//! *Merges into one response set*, written once, so `analyze` and `report` see the
//! whole class.
//!
//! *Reports what the merge revealed*: a student who appears on two forms, a form
//! that is missing a question the other has, a form that ran materially harder
//! than the other.
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use crate::assessment::{AssessmentFile, Form};
use crate::catalog::Catalog;
use crate::date::Date;
use crate::decode::{self, FormDecoder, FormFit, Numbering};
use crate::error::{Error, Result};
use crate::gradescope::{self, Context, Question};
use crate::responses::ResponseSet;
use crate::seal::SealFile;
/// One directory of graded questions, and the form it holds.
#[derive(Debug, Clone)]
pub struct Source {
/// The form id.
pub form: String,
/// The directory of per-question CSV exports.
pub dir: PathBuf,
}
impl Source {
/// Parses a `FORM=DIR` argument.
///
/// A bare path is accepted and takes the fallback form, so the single-form
/// case stays as short as it was.
///
/// # Arguments
///
/// * `text` - the argument, e.g. `A=exports/e1-a` or `exports/e1`.
/// * `fallback` - the form to use when the argument names none.
///
/// # Returns
///
/// The source.
///
/// # Errors
///
/// Returns [`Error::Usage`] when the argument names no form and no fallback
/// was given, or when the form label is empty.
pub fn parse(text: &str, fallback: Option<&str>) -> Result<Source> {
// Split on the first `=` only: a directory name may contain one, a form
// label may not.
if let Some((form, dir)) = text.split_once('=') {
let form = form.trim();
if form.is_empty() {
return Err(Error::usage(format!(
"`{text}` has an empty form label; write it as FORM=DIR, e.g. A=exports/e1-a"
)));
}
if !dir.trim().is_empty() {
return Ok(Source {
form: form.to_string(),
dir: PathBuf::from(dir.trim()),
});
}
}
match fallback {
Some(form) => Ok(Source {
form: form.to_string(),
dir: PathBuf::from(text.trim()),
}),
None => Err(Error::usage(format!(
"`{text}` does not say which form it holds; write it as FORM=DIR (e.g. \
A=exports/e1-a) or pass --form"
))),
}
}
}
/// How to run an intake.
#[derive(Debug, Clone)]
pub struct Options {
/// The administration date, overriding the record's.
pub date: Option<Date>,
/// How the exports number their questions.
pub numbering: Numbering,
/// Whether a form that fails its key check stops the ingest.
///
/// On by default. A directory that does not match the form it was named as is
/// the one ingest error that produces confident, wrong analysis rather than an
/// obvious failure, so the default is to refuse and say so.
pub strict: bool,
}
impl Default for Options {
fn default() -> Options {
Options {
date: None,
numbering: Numbering::Printed,
strict: true,
}
}
}
/// What one form's directory contributed.
#[derive(Debug, Clone)]
pub struct FormIntake {
/// The form id.
pub form: String,
/// The directory it came from.
pub dir: PathBuf,
/// Where its mapping came from.
pub provenance: decode::Provenance,
/// How many students it held.
pub students: usize,
/// How many questions it held.
pub questions: usize,
/// How the graded keys compared to every declared form, best first.
pub fits: Vec<FormFit>,
/// The parsed question files, kept so grading-time decisions stay available.
pub parsed: Vec<Question>,
}
impl FormIntake {
/// This form's own fit.
pub fn own_fit(&self) -> Option<&FormFit> {
self.fits
.iter()
.find(|f| f.form.eq_ignore_ascii_case(&self.form))
}
}
/// The result of reading every form of one administration.
#[derive(Debug, Clone)]
pub struct Intake {
/// The merged, translated responses.
pub responses: ResponseSet,
/// Per-form detail.
pub forms: Vec<FormIntake>,
/// Problems that did not stop the ingest.
pub warnings: Vec<String>,
}
impl Intake {
/// Every parsed question file, across forms.
///
/// # Returns
///
/// Pairs of form id and question.
pub fn questions(&self) -> Vec<(&str, &Question)> {
self.forms
.iter()
.flat_map(|f| f.parsed.iter().map(move |q| (f.form.as_str(), q)))
.collect()
}
}
/// Reads every source into one response set.
///
/// # Arguments
///
/// * `catalog` - the loaded course.
/// * `record` - the assessment record.
/// * `seal` - the seal, when one was written. Strongly preferred: it describes the
/// paper that was printed rather than the paper the bank would print today.
/// * `sources` - the directories and the forms they hold.
/// * `opts` - how to run.
///
/// # Returns
///
/// The merged intake.
///
/// # Errors
///
/// Returns [`Error::Usage`] when a source names a form the record does not
/// declare, when two sources name the same form, or when a key check fails under
/// `strict`. Propagates parse errors from the exports themselves.
pub fn run(
catalog: &Catalog,
record: &AssessmentFile,
seal: Option<&SealFile>,
sources: &[Source],
opts: &Options,
) -> Result<Intake> {
if sources.is_empty() {
return Err(Error::usage(
"no directories to ingest; pass one per form, e.g. A=exports/e1-a B=exports/e1-b"
.to_string(),
));
}
let declared = declared_forms(record);
let mut seen: BTreeSet<String> = BTreeSet::new();
for source in sources {
if !seen.insert(source.form.to_ascii_uppercase()) {
return Err(Error::usage(format!(
"form {} was given twice; each form is one directory",
source.form
)));
}
if !declared
.iter()
.any(|f| f.id.eq_ignore_ascii_case(&source.form))
{
return Err(Error::usage(format!(
"the record for `{}` declares no form `{}`; it declares {}",
record.assessment.id,
source.form,
declared
.iter()
.map(|f| f.id.as_str())
.collect::<Vec<_>>()
.join(", ")
)));
}
}
// One decoder per declared form, not just per ingested form: identifying a
// swapped directory means testing it against the forms it might be.
let mut decoders: Vec<FormDecoder> = Vec::new();
for form in &declared {
decoders.push(FormDecoder::resolve(seal, catalog, record, form)?);
}
let mut merged = ResponseSet::new();
let mut forms = Vec::new();
let mut warnings = Vec::new();
let mut blocking = Vec::new();
for source in sources {
let decoder = decoders
.iter()
.find(|d| d.form.eq_ignore_ascii_case(&source.form))
.expect("every source's form was checked against the declared list");
let ctx = Context {
course: catalog.course.course.code.clone(),
term: record
.assessment
.term
.clone()
.unwrap_or_else(|| catalog.course.course.term.clone()),
assessment_id: record.assessment.id.clone(),
date: opts.date.or(record.assessment.date),
form: Some(decoder.form.clone()),
};
let import = gradescope::ingest_dir(&source.dir, &ctx)?;
let mut set = import.responses;
let fits = decode::identify_form(&import.questions, &decoders, opts.numbering);
if let Some(problem) = decode::form_warning(&decoder.form, &fits) {
let message = format!("{} [{}]", problem, source.dir.display());
let convincing_alternative = fits
.iter()
.any(|f| !f.form.eq_ignore_ascii_case(&decoder.form) && f.is_convincing());
if opts.strict && convincing_alternative {
blocking.push(message);
} else {
warnings.push(message);
}
}
warnings.extend(decode::apply(&mut set, decoder, opts.numbering));
forms.push(FormIntake {
form: decoder.form.clone(),
dir: source.dir.clone(),
provenance: decoder.provenance,
students: set.students().len(),
questions: set.all_items().len(),
fits,
parsed: import.questions,
});
merged.absorb(set);
}
if !blocking.is_empty() {
blocking.push(
"Nothing was written. Fix the form labels, or pass --allow-mismatch if the keys really \
did change after printing."
.to_string(),
);
return Err(Error::Invalid(blocking));
}
warnings.extend(cross_form_checks(&merged, &forms, record));
merged.warnings.extend(warnings.clone());
Ok(Intake {
responses: merged,
forms,
warnings,
})
}
/// The forms a record declares, with the implicit single form for a record that
/// declares none.
fn declared_forms(record: &AssessmentFile) -> Vec<Form> {
if record.forms.is_empty() {
vec![Form {
id: "A".to_string(),
seed: 0,
shuffle_items: false,
shuffle_options: false,
}]
} else {
record.forms.clone()
}
}
/// Checks that only merging several forms can make.
fn cross_form_checks(
merged: &ResponseSet,
forms: &[FormIntake],
record: &AssessmentFile,
) -> Vec<String> {
let mut out = Vec::new();
if forms.len() < 2 {
return out;
}
// A student on two forms sat one exam and was graded twice, or two people
// share an identifier. Either way the response set now double counts them.
let mut by_student: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new();
for row in &merged.rows {
if let Some(form) = row.form.as_deref() {
by_student
.entry(row.student_key.as_str())
.or_default()
.insert(form);
}
}
let doubled: Vec<&str> = by_student
.iter()
.filter(|(_, forms)| forms.len() > 1)
.map(|(student, _)| *student)
.collect();
if !doubled.is_empty() {
out.push(format!(
"{} student(s) appear on more than one form ({}); each is counted twice in every \
total until one submission is removed",
doubled.len(),
doubled
.iter()
.take(5)
.copied()
.collect::<Vec<_>>()
.join(", ")
));
}
// Every form is the same items in a different order, so a question present on
// one and absent from another means a directory is short a file.
let expected: BTreeSet<u32> = record
.items
.iter()
.filter(|p| !p.dropped)
.map(|p| p.number)
.collect();
for form in forms {
let present: BTreeSet<u32> = merged
.rows
.iter()
.filter(|r| {
r.form
.as_deref()
.map(|f| f.eq_ignore_ascii_case(&form.form))
.unwrap_or(false)
})
.map(|r| r.item_number)
.collect();
let missing: Vec<String> = expected
.difference(&present)
.map(|n| n.to_string())
.collect();
if !missing.is_empty() {
out.push(format!(
"form {}: no responses for question(s) {}; the export directory is missing those \
files",
form.form,
missing.join(", ")
));
}
}
// Forms are meant to be the same test. A large gap between their means is
// either a permutation that made one form easier or an uneven split of the
// class, and both are worth knowing before any grade is released.
let means: Vec<(String, f64, usize)> = forms
.iter()
.map(|f| {
let students: Vec<&str> = merged
.rows
.iter()
.filter(|r| {
r.form
.as_deref()
.map(|x| x.eq_ignore_ascii_case(&f.form))
.unwrap_or(false)
})
.map(|r| r.student_key.as_str())
.collect::<BTreeSet<&str>>()
.into_iter()
.collect();
let possible = merged.points_available();
let percents: Vec<f64> = students
.iter()
.map(|s| {
if possible > 0.0 {
100.0 * merged.scored_total(s) / possible
} else {
0.0
}
})
.collect();
let mean = if percents.is_empty() {
0.0
} else {
percents.iter().sum::<f64>() / percents.len() as f64
};
(f.form.clone(), mean, percents.len())
})
.collect();
if let (Some(low), Some(high)) = (
means
.iter()
.min_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal)),
means
.iter()
.max_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal)),
) {
let gap = high.1 - low.1;
if gap >= 5.0 && low.2 >= 5 && high.2 >= 5 {
out.push(format!(
"form {} averaged {:.0}% and form {} averaged {:.0}%, a {:.0}-point gap. With {} \
and {} students that may be the split rather than the forms, but it is worth \
looking at before releasing grades",
high.0, high.1, low.0, low.1, gap, high.2, low.2
));
}
}
out
}
/// A one-line summary of where a directory came from and what it held.
///
/// # Arguments
///
/// * `intake` - one form's intake.
///
/// # Returns
///
/// The line, without a trailing newline.
pub fn describe(intake: &FormIntake) -> String {
let fit = match intake.own_fit() {
Some(fit) if fit.compared > 0 => {
format!(", keys matched {}/{}", fit.matched, fit.compared)
}
_ => String::new(),
};
format!(
"form {}: {} student(s) x {} question(s) from {} (mapping {}{fit})",
intake.form,
intake.students,
intake.questions,
intake.dir.display(),
intake.provenance.label(),
)
}
/// Whether a path looks like a Gradescope per-question export directory.
///
/// Used to give a better error than "no numbered CSV files" when someone points
/// at the zip they downloaded, or at the directory above the one they meant.
///
/// # Arguments
///
/// * `dir` - the candidate directory.
///
/// # Returns
///
/// `true` when it holds at least one file named like `1.csv`.
pub fn looks_like_export(dir: &Path) -> bool {
let Ok(entries) = std::fs::read_dir(dir) else {
return false;
};
entries.filter_map(|e| e.ok()).any(|entry| {
let path = entry.path();
path.extension().and_then(|e| e.to_str()) == Some("csv")
&& path
.file_stem()
.and_then(|s| s.to_str())
.map(|s| s.chars().all(|c| c.is_ascii_digit()))
.unwrap_or(false)
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_labelled_source_parses() {
let source = Source::parse("B=exports/e1-b", None).unwrap();
assert_eq!(source.form, "B");
assert_eq!(source.dir, PathBuf::from("exports/e1-b"));
}
#[test]
fn a_bare_path_needs_a_fallback_form() {
let source = Source::parse("exports/e1", Some("A")).unwrap();
assert_eq!(source.form, "A");
assert_eq!(source.dir, PathBuf::from("exports/e1"));
let err = Source::parse("exports/e1", None).unwrap_err();
assert!(err.to_string().contains("which form"), "{err}");
}
#[test]
fn a_windows_style_path_is_not_mistaken_for_a_label() {
// The split is on the first `=`, and a drive letter has none, so this is
// only a hazard for a path that genuinely contains one.
let source = Source::parse("A=C:/exports/e1-a", None).unwrap();
assert_eq!(source.form, "A");
assert_eq!(source.dir, PathBuf::from("C:/exports/e1-a"));
}
#[test]
fn an_empty_label_is_refused() {
let err = Source::parse("=exports/e1", None).unwrap_err();
assert!(err.to_string().contains("empty form label"), "{err}");
}
}
+65 -6
View File
@@ -52,6 +52,9 @@ pub struct Response {
pub date: Option<Date>,
/// Which form the student took, when forms were used.
pub form: Option<String>,
/// Where this question sat on the student's own form, when that differs from
/// the recorded number. Provenance, not a join key.
pub form_position: Option<u32>,
/// The identifier analysis groups by. A pseudonym when pseudonymizing.
pub student_key: String,
@@ -74,8 +77,14 @@ pub struct Response {
/// Option letters the student chose.
pub selected: Vec<String>,
/// The selected options in the bank's own lettering, written at ingest by
/// [`crate::decode::apply`]. Empty when the form was never decoded, which is
/// the case for data ingested before seals existed.
pub selected_source: Vec<String>,
/// Option letters the student eliminated, for elimination-scored items.
pub eliminated: Vec<String>,
/// The eliminated options in the bank's lettering.
pub eliminated_source: Vec<String>,
/// Whether the response earned full credit. `None` when it cannot be
/// determined, e.g. a blank response on an item with no recorded key.
pub correct: Option<bool>,
@@ -132,6 +141,19 @@ impl Response {
pub fn selected_joined(&self) -> String {
self.selected.join(",")
}
/// The choices, in the bank's lettering when it is known.
///
/// `selected` is what the student marked on the page they held; this is what
/// they chose. On an unshuffled form the two agree, which is why reading the
/// wrong one is a bug that only appears once you add a second form.
pub fn chosen(&self) -> &[String] {
if self.selected_source.is_empty() {
&self.selected
} else {
&self.selected_source
}
}
}
/// A set of responses plus anything worth telling the user about the ingest.
@@ -393,12 +415,23 @@ impl ResponseSet {
// Apply the record's credit overrides, which is how a decision to
// award partial credit after the fact becomes visible in analysis.
if !p.credit_overrides.is_empty() && r.selected.len() == 1 {
if let Some(over) = p.credit_overrides.get(&r.selected[0]) {
if (*over - r.credit).abs() > 1e-9 {
r.credit = *over;
r.score = *over * r.points_possible;
r.correct = Some(*over >= 0.999);
//
// The override map is keyed in the bank's letters, so the lookup has
// to use `chosen()` rather than `selected`. Cloning the letter first
// ends the borrow of `r` before the row is written to.
let chosen = if r.chosen().len() == 1 {
r.chosen().first().cloned()
} else {
None
};
if !p.credit_overrides.is_empty() {
if let Some(letter) = chosen {
if let Some(over) = p.credit_overrides.get(&letter) {
if (*over - r.credit).abs() > 1e-9 {
r.credit = *over;
r.score = *over * r.points_possible;
r.correct = Some(*over >= 0.999);
}
}
}
}
@@ -590,6 +623,19 @@ pub struct FlatResponse {
pub bonus: bool,
/// Whether the item was dropped.
pub dropped: bool,
// Appended rather than interleaved: the columns above are the order every
// CSV written so far uses, and `#[serde(default)]` is what keeps one written
// last term readable now that three more exist.
/// The printed position, 0 when unknown.
#[serde(default)]
pub form_position: u32,
/// Comma-joined selected letters in the bank's lettering.
#[serde(default)]
pub selected_source: String,
/// Comma-joined eliminated letters in the bank's lettering.
#[serde(default)]
pub eliminated_source: String,
}
impl FlatResponse {
@@ -610,6 +656,7 @@ impl FlatResponse {
assessment_id: r.assessment_id.clone(),
date: r.date.map(|d| d.to_string()).unwrap_or_default(),
form: r.form.clone().unwrap_or_default(),
form_position: r.form_position.unwrap_or(0),
student_key: r.student_key.clone(),
sid: r.sid.clone().unwrap_or_default(),
email: r.email.clone().unwrap_or_default(),
@@ -618,7 +665,9 @@ impl FlatResponse {
item_ref: r.item_ref.clone().unwrap_or_default(),
item_version: r.item_version.unwrap_or(0),
selected: r.selected.join(","),
selected_source: r.selected_source.join(","),
eliminated: r.eliminated.join(","),
eliminated_source: r.eliminated_source.join(","),
correct: match r.correct {
Some(true) => "1".to_string(),
Some(false) => "0".to_string(),
@@ -660,6 +709,11 @@ impl FlatResponse {
assessment_id: self.assessment_id.clone(),
date: self.date.parse().ok(),
form: none_if_empty(&self.form),
form_position: if self.form_position == 0 {
None
} else {
Some(self.form_position)
},
student_key: self.student_key.clone(),
sid: none_if_empty(&self.sid),
name: None,
@@ -673,7 +727,9 @@ impl FlatResponse {
Some(self.item_version)
},
selected: split(&self.selected),
selected_source: split(&self.selected_source),
eliminated: split(&self.eliminated),
eliminated_source: split(&self.eliminated_source),
correct: match self.correct.as_str() {
"1" | "true" => Some(true),
"0" | "false" => Some(false),
@@ -713,6 +769,7 @@ mod tests {
assessment_id: "e1".into(),
date: None,
form: None,
form_position: None,
student_key: student.into(),
sid: Some(format!("sid-{student}")),
name: None,
@@ -722,7 +779,9 @@ mod tests {
item_ref: None,
item_version: None,
selected: vec!["A".into()],
selected_source: vec![],
eliminated: vec![],
eliminated_source: vec![],
correct: Some(credit >= 0.999),
credit,
points_possible: 2.0,
+3
View File
@@ -541,6 +541,7 @@ mod tests {
assessment_id: "exam-4".into(),
date: None,
form: None,
form_position: None,
student_key: student.into(),
sid: None,
name: None,
@@ -550,7 +551,9 @@ mod tests {
item_ref: Some("bank::q-x-001".into()),
item_version: Some(2),
selected: vec!["C".into()],
selected_source: vec![],
eliminated: vec![],
eliminated_source: vec![],
correct: Some(true),
credit: 1.0,
points_possible: 1.5,
+34
View File
@@ -61,6 +61,9 @@ pub fn schema() -> Schema {
Field::new("topics", DataType::Utf8, false),
Field::new("bonus", DataType::Boolean, false),
Field::new("dropped", DataType::Boolean, false),
Field::new("form_position", DataType::UInt32, false),
Field::new("selected_source", DataType::Utf8, false),
Field::new("eliminated_source", DataType::Utf8, false),
])
}
@@ -120,6 +123,9 @@ fn to_batch(rows: &[FlatResponse]) -> Result<RecordBatch> {
s(|r| &r.topics),
boolc(|r| r.bonus),
boolc(|r| r.dropped),
u32c(|r| r.form_position),
s(|r| &r.selected_source),
s(|r| &r.eliminated_source),
];
RecordBatch::try_new(Arc::new(schema()), columns).map_err(|e| {
@@ -237,6 +243,20 @@ fn from_batch(batch: &RecordBatch, path: &Path) -> Result<Vec<FlatResponse>> {
.ok_or_else(|| column_error(name, "boolean", path))
};
// The three decode columns are read optionally rather than required, so a
// file written before seals existed still loads. A missing column is not a
// foreign file; it is last term's data.
let optional_strings = |name: &str| -> Option<&StringArray> {
batch
.column_by_name(name)
.and_then(|c| c.as_any().downcast_ref::<StringArray>())
};
let optional_uints = |name: &str| -> Option<&UInt32Array> {
batch
.column_by_name(name)
.and_then(|c| c.as_any().downcast_ref::<UInt32Array>())
};
let administration_id = strings("administration_id")?;
let course = strings("course")?;
let term = strings("term")?;
@@ -263,6 +283,10 @@ fn from_batch(batch: &RecordBatch, path: &Path) -> Result<Vec<FlatResponse>> {
let bonus = bools("bonus")?;
let dropped = bools("dropped")?;
let form_position = optional_uints("form_position");
let selected_source = optional_strings("selected_source");
let eliminated_source = optional_strings("eliminated_source");
let mut out = Vec::with_capacity(batch.num_rows());
for i in 0..batch.num_rows() {
out.push(FlatResponse {
@@ -272,6 +296,7 @@ fn from_batch(batch: &RecordBatch, path: &Path) -> Result<Vec<FlatResponse>> {
assessment_id: assessment_id.value(i).to_string(),
date: date.value(i).to_string(),
form: form.value(i).to_string(),
form_position: form_position.map(|a| a.value(i)).unwrap_or(0),
student_key: student_key.value(i).to_string(),
sid: sid.value(i).to_string(),
email: email.value(i).to_string(),
@@ -280,7 +305,13 @@ fn from_batch(batch: &RecordBatch, path: &Path) -> Result<Vec<FlatResponse>> {
item_ref: item_ref.value(i).to_string(),
item_version: item_version.value(i),
selected: selected.value(i).to_string(),
selected_source: selected_source
.map(|a| a.value(i).to_string())
.unwrap_or_default(),
eliminated: eliminated.value(i).to_string(),
eliminated_source: eliminated_source
.map(|a| a.value(i).to_string())
.unwrap_or_default(),
correct: correct.value(i).to_string(),
credit: credit.value(i),
points_possible: points_possible.value(i),
@@ -336,6 +367,9 @@ mod tests {
topics: "kinetics".into(),
bonus: false,
dropped: false,
form_position: 0,
selected_source: String::new(),
eliminated_source: String::new(),
}
}