education/coursebank
1
0
Fork 0
Code Issues Pull Requests 1 Actions Releases 1 Activity

feat: implement reports

Browse Source
This commit is contained in:
alexm
2026-09-19 20:06:41 -04:00
parent 6ae993d393
commit 994e9065e8
28 changed files with 7082 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/data/decode.rs
+803
View File
@@ -0,0 +1,803 @@
// SPDX-License-Identifier: Prosperity-3.0.0
// Copyright Scientific Computing Studio
// Source: https://git.scient.ing/education/coursebank
//! Turning what a student marked into what a student chose.
//!
//! A grading export speaks in positions and printed letters. Question 14 is the
//! fourteenth thing on the page; option C is the third bubble. An item bank speaks
//! in ids and its own lettering. When forms shuffle, those two vocabularies
//! disagree, and every analysis downstream of the disagreement is wrong in a way
//! that looks right:
//!
//! * Pooled distractor statistics add form A's option C to form B's option C,
//! which are different sentences. The resulting table is noise with the shape of
//! data.
//! * A student report looks up the misconception recorded on option C and shows it
//! to a student who chose a different option. The feedback is confident,
//! specific, and about the wrong thing.
//! * Any `credit_overrides` written in the record's lettering are applied to
//! whoever happened to mark that letter on their form.
//!
//! None of these fail loudly. That is the argument for doing the translation once,
//! at ingest, and storing both sides of it.
//!
//! # What a decoder knows
//!
//! For one form: which recorded question number sits at each printed position,
//! which bank letter each printed letter stands for, and which printed letters are
//! keyed. It is built from a [`SealFile`] when one exists, and derived from the
//! record and the form seed when one does not. Sealed is better, and not only
//! because it is faster: a derived decoder describes the form the bank *would*
//! print today, while a sealed one describes the form that was actually printed.
//!
//! # Catching a swapped directory
//!
//! Gradescope's point-value row reveals which printed letter earned full credit on
//! every question. A decoder knows what that letter should be. Comparing them
//! across a whole directory is close to a proof of which form the directory holds:
//! agreement is near total for the right form and near chance for the wrong one.
//! [`identify_form`] uses that to refuse an ingest that names form A over a
//! directory of form B papers, which is otherwise a mistake nobody catches until
//! the item statistics look strange three weeks later.
use std::collections::{BTreeMap, BTreeSet};
use crate::assessment::{AssessmentFile, Form};
use crate::catalog::Catalog;
use crate::error::{Error, Result};
use crate::gradescope::Question;
use crate::responses::ResponseSet;
use crate::seal::{SealFile, printed_letter};
use crate::select;
/// Where a decoder's mapping came from.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Provenance {
/// Read from a seal written before administration. Authoritative.
Seal,
/// Derived from the assessment record and the form seed, as of now.
Derived,
}
impl Provenance {
/// A short label for output.
pub fn label(self) -> &'static str {
match self {
Provenance::Seal => "seal",
Provenance::Derived => "derived from the record",
}
}
}
/// How a grading export numbers its questions.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Numbering {
/// The export counts printed positions, which is what Gradescope's `N.csv`
/// file names mean. Positions are translated to recorded numbers.
Printed,
/// The export already carries recorded question numbers, so numbers pass
/// through untouched.
Recorded,
}
/// One question's mapping on one form.
#[derive(Debug, Clone)]
pub struct QuestionMap {
/// Printed position on the page, counting from 1.
pub position: u32,
/// The recorded question number, the join key to the record and the store.
pub number: u32,
/// The item's global id.
pub item: String,
/// Keyed letters as printed on this form.
pub printed_key: Vec<String>,
/// Keyed letters in the bank's own lettering.
pub canonical_key: Vec<String>,
/// Printed letter to bank letter.
pub to_canonical: BTreeMap<String, String>,
/// Bank letter to printed letter.
pub to_printed: BTreeMap<String, String>,
}
impl QuestionMap {
/// The bank letter a printed letter stands for.
///
/// # Arguments
///
/// * `printed` - the letter as the student saw it.
///
/// # Returns
///
/// The bank letter, or `None` when the printed letter is not one of this
/// question's options.
pub fn canonical(&self, printed: &str) -> Option<&str> {
self.to_canonical
.get(&printed.trim().to_ascii_uppercase())
.map(|s| s.as_str())
}
/// How many options this question has.
pub fn n_options(&self) -> usize {
self.to_canonical.len()
}
/// Whether this question's options were actually permuted.
pub fn is_permuted(&self) -> bool {
self.to_canonical.iter().any(|(k, v)| k != v)
}
}
/// One form's full mapping.
#[derive(Debug, Clone)]
pub struct FormDecoder {
/// The form id.
pub form: String,
/// Where the mapping came from.
pub provenance: Provenance,
/// Questions by printed position.
by_position: BTreeMap<u32, QuestionMap>,
/// Questions by recorded number.
by_number: BTreeMap<u32, QuestionMap>,
}
impl FormDecoder {
/// Builds a decoder from the question maps.
fn assemble(form: String, provenance: Provenance, maps: Vec<QuestionMap>) -> FormDecoder {
let by_position = maps.iter().map(|m| (m.position, m.clone())).collect();
let by_number = maps.into_iter().map(|m| (m.number, m)).collect();
FormDecoder {
form,
provenance,
by_position,
by_number,
}
}
/// Reads one form's mapping out of a seal.
///
/// # Arguments
///
/// * `seal` - the seal.
/// * `form_id` - the form to decode, matched case-insensitively.
///
/// # Returns
///
/// The decoder.
///
/// # Errors
///
/// Returns [`Error::Usage`] when the seal does not cover that form.
pub fn from_seal(seal: &SealFile, form_id: &str) -> Result<FormDecoder> {
let form = seal.form(form_id).ok_or_else(|| {
Error::usage(format!(
"the seal for `{}` does not cover form `{form_id}`; it covers {}",
seal.seal.assessment,
seal.forms
.iter()
.map(|f| f.id.as_str())
.collect::<Vec<_>>()
.join(", ")
))
})?;
let maps = form
.questions
.iter()
.map(|q| {
let mut to_canonical = BTreeMap::new();
let mut to_printed = BTreeMap::new();
for map in &q.options {
to_canonical.insert(map.printed.clone(), map.canonical.clone());
to_printed.insert(map.canonical.clone(), map.printed.clone());
}
let canonical_key: Vec<String> = q
.printed_key
.iter()
.filter_map(|p| to_canonical.get(p).cloned())
.collect();
QuestionMap {
position: q.position,
number: q.number,
item: q.item.clone(),
printed_key: q.printed_key.clone(),
canonical_key,
to_canonical,
to_printed,
}
})
.collect();
Ok(FormDecoder::assemble(
form.id.clone(),
Provenance::Seal,
maps,
))
}
/// Derives one form's mapping from the record and the bank.
///
/// Uses the same two functions every export calls, so a derived decoder and a
/// freshly exported paper agree by construction.
///
/// # Arguments
///
/// * `catalog` - the loaded course.
/// * `record` - the assessment record.
/// * `form` - the form.
///
/// # Returns
///
/// The decoder.
///
/// # Errors
///
/// Returns [`Error::Unresolved`] when a placement references a missing item.
pub fn derive(catalog: &Catalog, record: &AssessmentFile, form: &Form) -> Result<FormDecoder> {
let printed: Vec<_> = select::layout(record, form)
.into_iter()
.filter(|p| !p.dropped)
.collect();
let mut maps = Vec::with_capacity(printed.len());
for (index, placement) in printed.iter().enumerate() {
let entry = catalog.require(&placement.item)?;
let item = &entry.item;
let order = select::option_order(form, &placement.item, item.options.len());
let canonical_key: BTreeSet<String> = if placement.key.is_empty() {
item.key_letters().into_iter().collect()
} else {
placement.key.iter().cloned().collect()
};
let mut to_canonical = BTreeMap::new();
let mut to_printed = BTreeMap::new();
let mut printed_key = Vec::new();
for (position, source_index) in order.iter().enumerate() {
let canonical = item
.options
.get(*source_index)
.map(|c| c.id.clone())
.unwrap_or_else(|| printed_letter(*source_index));
let label = printed_letter(position);
if canonical_key.contains(&canonical) {
printed_key.push(label.clone());
}
to_canonical.insert(label.clone(), canonical.clone());
to_printed.insert(canonical, label);
}
maps.push(QuestionMap {
position: index as u32 + 1,
number: placement.number,
item: placement.item.clone(),
printed_key,
canonical_key: canonical_key.into_iter().collect(),
to_canonical,
to_printed,
});
}
Ok(FormDecoder::assemble(
form.id.clone(),
Provenance::Derived,
maps,
))
}
/// Builds a decoder, preferring the seal.
///
/// # Arguments
///
/// * `seal` - the seal, when one has been written.
/// * `catalog` - the loaded course.
/// * `record` - the assessment record.
/// * `form` - the form.
///
/// # Returns
///
/// The decoder.
///
/// # Errors
///
/// As [`FormDecoder::from_seal`] and [`FormDecoder::derive`]. A seal that does
/// not cover the requested form falls back to deriving rather than failing,
/// since a form added after sealing is a real situation.
pub fn resolve(
seal: Option<&SealFile>,
catalog: &Catalog,
record: &AssessmentFile,
form: &Form,
) -> Result<FormDecoder> {
if let Some(seal) = seal {
if seal.form(&form.id).is_some() {
return FormDecoder::from_seal(seal, &form.id);
}
}
FormDecoder::derive(catalog, record, form)
}
/// The question at a printed position.
///
/// # Arguments
///
/// * `position` - the printed position, counting from 1.
pub fn at_position(&self, position: u32) -> Option<&QuestionMap> {
self.by_position.get(&position)
}
/// The question with a recorded number.
///
/// # Arguments
///
/// * `number` - the recorded number.
pub fn at_number(&self, number: u32) -> Option<&QuestionMap> {
self.by_number.get(&number)
}
/// The question an export's numbering refers to.
///
/// # Arguments
///
/// * `n` - the number as the export gives it.
/// * `numbering` - how the export numbers questions.
pub fn lookup(&self, n: u32, numbering: Numbering) -> Option<&QuestionMap> {
match numbering {
Numbering::Printed => self.at_position(n),
Numbering::Recorded => self.at_number(n),
}
}
/// How many questions this form prints.
pub fn len(&self) -> usize {
self.by_position.len()
}
/// Whether the form prints nothing, which means the record is empty.
pub fn is_empty(&self) -> bool {
self.by_position.is_empty()
}
/// Whether any question on this form has permuted options.
///
/// Used to decide whether to say anything about translation at all: on an
/// unshuffled form the whole mechanism is an identity map and mentioning it
/// is noise.
pub fn is_permuted(&self) -> bool {
self.by_position.values().any(|q| q.is_permuted())
}
/// Whether printed positions and recorded numbers disagree anywhere.
///
/// True when items were shuffled, and also when a bonus item sits mid-record,
/// since the layout moves bonus items to the end of the paper.
pub fn is_renumbered(&self) -> bool {
self.by_position.values().any(|q| q.position != q.number)
}
}
/// What a directory of graded questions says about which form it holds.
#[derive(Debug, Clone)]
pub struct FormFit {
/// The form id.
pub form: String,
/// Questions whose graded key matched this form's printed key.
pub matched: usize,
/// Questions that could be compared at all.
pub compared: usize,
/// Question positions where the graded key disagreed.
pub mismatches: Vec<u32>,
}
impl FormFit {
/// The share of comparable questions that agreed.
pub fn rate(&self) -> f64 {
if self.compared == 0 {
0.0
} else {
self.matched as f64 / self.compared as f64
}
}
/// Whether the fit is good enough to proceed without a warning.
///
/// The threshold is high on purpose. A correctly matched directory agrees on
/// every question; anything less than total agreement is either a regrade that
/// moved a key or the wrong directory, and both are worth a sentence.
pub fn is_convincing(&self) -> bool {
self.compared > 0 && self.matched == self.compared
}
}
/// Compares a parsed Gradescope directory against one form's expected keys.
///
/// # Arguments
///
/// * `questions` - the parsed question files.
/// * `decoder` - the form to test against.
/// * `numbering` - how the export numbers questions.
///
/// # Returns
///
/// The fit.
pub fn fit_form(questions: &[Question], decoder: &FormDecoder, numbering: Numbering) -> FormFit {
let mut matched = 0usize;
let mut compared = 0usize;
let mut mismatches = Vec::new();
for question in questions {
let Some(map) = decoder.lookup(question.number, numbering) else {
continue;
};
let graded: BTreeSet<String> = question.keyed().into_iter().collect();
if graded.is_empty() {
continue;
}
let expected: BTreeSet<String> = map.printed_key.iter().cloned().collect();
if expected.is_empty() {
continue;
}
compared += 1;
if graded == expected {
matched += 1;
} else {
mismatches.push(question.number);
}
}
FormFit {
form: decoder.form.clone(),
matched,
compared,
mismatches,
}
}
/// Ranks every candidate form against a directory.
///
/// # Arguments
///
/// * `questions` - the parsed question files.
/// * `decoders` - one decoder per declared form.
/// * `numbering` - how the export numbers questions.
///
/// # Returns
///
/// The fits, best first.
pub fn identify_form(
questions: &[Question],
decoders: &[FormDecoder],
numbering: Numbering,
) -> Vec<FormFit> {
let mut fits: Vec<FormFit> = decoders
.iter()
.map(|d| fit_form(questions, d, numbering))
.collect();
fits.sort_by(|a, b| {
b.rate()
.partial_cmp(&a.rate())
.unwrap_or(std::cmp::Ordering::Equal)
.then_with(|| a.form.cmp(&b.form))
});
fits
}
/// Explains a fit in a sentence, or says nothing when the fit is perfect.
///
/// # Arguments
///
/// * `claimed` - the form the directory was ingested as.
/// * `fits` - every form's fit, best first.
///
/// # Returns
///
/// A warning, or `None`.
pub fn form_warning(claimed: &str, fits: &[FormFit]) -> Option<String> {
let mine = fits.iter().find(|f| f.form.eq_ignore_ascii_case(claimed))?;
if mine.is_convincing() {
return None;
}
if mine.compared == 0 {
return Some(format!(
"form {claimed}: the export carries no point values, so the graded keys could not be \
checked against the form. Nothing verified this directory is form {claimed}"
));
}
let better = fits
.iter()
.find(|f| !f.form.eq_ignore_ascii_case(claimed) && f.rate() > mine.rate());
let head = format!(
"form {claimed}: the graded key matches this form on {} of {} question(s)",
mine.matched, mine.compared
);
let where_ = if mine.mismatches.is_empty() {
String::new()
} else {
let list: Vec<String> = mine
.mismatches
.iter()
.take(8)
.map(|n| n.to_string())
.collect();
format!(
" (q{}{})",
list.join(", q"),
if mine.mismatches.len() > 8 {
", …"
} else {
""
}
)
};
match better {
Some(other) => Some(format!(
"{head}{where_}, but matches form {} on {} of {}. This directory is almost certainly \
form {}, not form {claimed}",
other.form, other.matched, other.compared, other.form
)),
None => Some(format!(
"{head}{where_}. Either those questions were regraded after printing, or the form is \
not the one named"
)),
}
}
/// Translates a form's responses into the bank's vocabulary.
///
/// Rewrites, for every row whose `form` matches this decoder:
///
/// * `item_number`, from printed position to recorded number, when the export
/// numbers by position;
/// * `form_position`, recording where the question sat on the page;
/// * `selected_source` and `eliminated_source`, the bank letters for what was
/// marked. The printed letters stay in `selected` and `eliminated`, because what
/// a student physically marked is the fact and the translation is the
/// interpretation.
///
/// Correctness and credit are untouched. Both come from the grading platform,
/// which scored the paper the student actually held, and are already right.
///
/// # Arguments
///
/// * `set` - the responses to translate, in place.
/// * `decoder` - the form's mapping.
/// * `numbering` - how the export numbered questions.
///
/// # Returns
///
/// Warnings for anything that could not be translated.
pub fn apply(set: &mut ResponseSet, decoder: &FormDecoder, numbering: Numbering) -> Vec<String> {
let mut warnings = Vec::new();
let mut unmapped_positions: BTreeSet<u32> = BTreeSet::new();
let mut unmapped_letters: BTreeSet<String> = BTreeSet::new();
let mut translated = 0usize;
for row in &mut set.rows {
let belongs = row
.form
.as_deref()
.map(|f| f.eq_ignore_ascii_case(&decoder.form))
.unwrap_or(false);
if !belongs {
continue;
}
let Some(map) = decoder.lookup(row.item_number, numbering) else {
unmapped_positions.insert(row.item_number);
continue;
};
row.form_position = Some(map.position);
row.item_number = map.number;
let mut convert = |letters: &[String]| -> Vec<String> {
let mut out = Vec::with_capacity(letters.len());
for letter in letters {
match map.canonical(letter) {
Some(canonical) => out.push(canonical.to_string()),
None => {
unmapped_letters.insert(format!("q{} {}", map.number, letter));
}
}
}
out.sort();
out
};
row.selected_source = convert(&row.selected);
row.eliminated_source = convert(&row.eliminated);
translated += 1;
}
if !unmapped_positions.is_empty() {
let list: Vec<String> = unmapped_positions.iter().map(|n| n.to_string()).collect();
warnings.push(format!(
"form {}: question(s) {} are in the export but not on this form; they were left \
untranslated",
decoder.form,
list.join(", ")
));
}
if !unmapped_letters.is_empty() {
let list: Vec<String> = unmapped_letters.iter().take(10).cloned().collect();
warnings.push(format!(
"form {}: {} marked option(s) are not options on the printed form ({}), which usually \
means a rubric column was added by hand in Gradescope",
decoder.form,
unmapped_letters.len(),
list.join(", ")
));
}
if translated == 0 {
warnings.push(format!(
"form {}: no response rows carried this form id, so nothing was translated",
decoder.form
));
}
warnings
}
#[cfg(test)]
mod tests {
use super::*;
use crate::responses::{Response, administration_id};
fn map(position: u32, number: u32, pairs: &[(&str, &str)], key: &str) -> QuestionMap {
let mut to_canonical = BTreeMap::new();
let mut to_printed = BTreeMap::new();
for (printed, canonical) in pairs {
to_canonical.insert(printed.to_string(), canonical.to_string());
to_printed.insert(canonical.to_string(), printed.to_string());
}
let printed_key = vec![to_printed.get(key).cloned().unwrap_or_default()];
QuestionMap {
position,
number,
item: format!("b::q-{number}"),
printed_key,
canonical_key: vec![key.to_string()],
to_canonical,
to_printed,
}
}
fn decoder() -> FormDecoder {
FormDecoder::assemble(
"B".to_string(),
Provenance::Seal,
vec![
// Printed A..D show bank C, A, D, B. The key is bank A, printed B.
map(1, 1, &[("A", "C"), ("B", "A"), ("C", "D"), ("D", "B")], "A"),
// A bonus item recorded as 9 but printed last, at position 2.
map(2, 9, &[("A", "B"), ("B", "A")], "B"),
],
)
}
fn row(number: u32, selected: &str) -> Response {
Response {
administration_id: administration_id("C", "2026f", "e1"),
course: "C".into(),
term: "2026f".into(),
assessment_id: "e1".into(),
date: None,
form: Some("B".into()),
student_key: "s1".into(),
sid: None,
name: None,
email: None,
section: None,
item_number: number,
form_position: None,
item_ref: None,
item_version: None,
selected: vec![selected.into()],
eliminated: Vec::new(),
selected_source: Vec::new(),
eliminated_source: Vec::new(),
correct: Some(false),
credit: 0.0,
points_possible: 1.0,
score: 0.0,
response_time_seconds: None,
level: None,
learning_objectives: Vec::new(),
topics: Vec::new(),
bonus: false,
dropped: false,
}
}
#[test]
fn printed_letters_become_bank_letters() {
let decoder = decoder();
let q = decoder.at_position(1).unwrap();
assert_eq!(q.canonical("A"), Some("C"));
assert_eq!(q.canonical("D"), Some("B"));
assert_eq!(q.canonical("E"), None);
assert_eq!(q.printed_key, vec!["B".to_string()]);
}
#[test]
fn positions_become_recorded_numbers() {
let mut set = ResponseSet::new();
set.rows.push(row(2, "A"));
let warnings = apply(&mut set, &decoder(), Numbering::Printed);
assert!(warnings.is_empty(), "{warnings:?}");
assert_eq!(set.rows[0].item_number, 9, "position 2 is recorded as 9");
assert_eq!(set.rows[0].form_position, Some(2));
assert_eq!(set.rows[0].selected, vec!["A".to_string()], "printed kept");
assert_eq!(set.rows[0].selected_source, vec!["B".to_string()]);
}
#[test]
fn rows_from_another_form_are_left_alone() {
let mut set = ResponseSet::new();
let mut other = row(1, "A");
other.form = Some("A".into());
set.rows.push(other);
apply(&mut set, &decoder(), Numbering::Printed);
assert!(set.rows[0].selected_source.is_empty());
assert_eq!(set.rows[0].item_number, 1);
}
#[test]
fn an_unknown_position_is_reported_not_guessed() {
let mut set = ResponseSet::new();
set.rows.push(row(7, "A"));
let warnings = apply(&mut set, &decoder(), Numbering::Printed);
assert!(
warnings.iter().any(|w| w.contains("not on this form")),
"{warnings:?}"
);
assert_eq!(set.rows[0].item_number, 7, "left as found");
}
#[test]
fn a_perfect_fit_says_nothing() {
let fits = vec![FormFit {
form: "A".into(),
matched: 30,
compared: 30,
mismatches: Vec::new(),
}];
assert!(form_warning("A", &fits).is_none());
}
#[test]
fn a_swapped_directory_names_the_form_it_really_is() {
let fits = vec![
FormFit {
form: "B".into(),
matched: 30,
compared: 30,
mismatches: Vec::new(),
},
FormFit {
form: "A".into(),
matched: 8,
compared: 30,
mismatches: (1..=22).collect(),
},
];
let warning = form_warning("A", &fits).expect("a mismatch this large must warn");
assert!(warning.contains("almost certainly form B"), "{warning}");
}
#[test]
fn a_single_regraded_key_warns_without_accusing_the_wrong_form() {
let fits = vec![FormFit {
form: "A".into(),
matched: 29,
compared: 30,
mismatches: vec![14],
}];
let warning = form_warning("A", &fits).unwrap();
assert!(warning.contains("q14"), "{warning}");
assert!(warning.contains("regraded"), "{warning}");
}
}