feat: implement reports
This commit is contained in:
28 files changed
+7082
-63
No files matched your search
@@ -325,6 +325,10 @@ pub fn ingest(
|
||||
assessment_id: ctx.assessment_id.clone(),
|
||||
date: ctx.date,
|
||||
form: ctx.form.clone(),
|
||||
// Canvas numbers questions as the record does, and its exports
|
||||
// carry no printed order, so there is no printed position to
|
||||
// record and no letter map to decode against.
|
||||
form_position: None,
|
||||
student_key: student_key.clone(),
|
||||
sid: sid.clone(),
|
||||
name: name.clone(),
|
||||
@@ -334,7 +338,9 @@ pub fn ingest(
|
||||
item_ref,
|
||||
item_version: None,
|
||||
selected,
|
||||
selected_source: Vec::new(),
|
||||
eliminated: Vec::new(),
|
||||
eliminated_source: Vec::new(),
|
||||
correct,
|
||||
credit,
|
||||
points_possible: points,
|
||||
|
||||
@@ -0,0 +1,803 @@
|
||||
// SPDX-License-Identifier: Prosperity-3.0.0
|
||||
// Copyright Scientific Computing Studio
|
||||
// Source: https://git.scient.ing/education/coursebank
|
||||
|
||||
//! Turning what a student marked into what a student chose.
|
||||
//!
|
||||
//! A grading export speaks in positions and printed letters. Question 14 is the
|
||||
//! fourteenth thing on the page; option C is the third bubble. An item bank speaks
|
||||
//! in ids and its own lettering. When forms shuffle, those two vocabularies
|
||||
//! disagree, and every analysis downstream of the disagreement is wrong in a way
|
||||
//! that looks right:
|
||||
//!
|
||||
//! * Pooled distractor statistics add form A's option C to form B's option C,
|
||||
//! which are different sentences. The resulting table is noise with the shape of
|
||||
//! data.
|
||||
//! * A student report looks up the misconception recorded on option C and shows it
|
||||
//! to a student who chose a different option. The feedback is confident,
|
||||
//! specific, and about the wrong thing.
|
||||
//! * Any `credit_overrides` written in the record's lettering are applied to
|
||||
//! whoever happened to mark that letter on their form.
|
||||
//!
|
||||
//! None of these fail loudly. That is the argument for doing the translation once,
|
||||
//! at ingest, and storing both sides of it.
|
||||
//!
|
||||
//! # What a decoder knows
|
||||
//!
|
||||
//! For one form: which recorded question number sits at each printed position,
|
||||
//! which bank letter each printed letter stands for, and which printed letters are
|
||||
//! keyed. It is built from a [`SealFile`] when one exists, and derived from the
|
||||
//! record and the form seed when one does not. Sealed is better, and not only
|
||||
//! because it is faster: a derived decoder describes the form the bank *would*
|
||||
//! print today, while a sealed one describes the form that was actually printed.
|
||||
//!
|
||||
//! # Catching a swapped directory
|
||||
//!
|
||||
//! Gradescope's point-value row reveals which printed letter earned full credit on
|
||||
//! every question. A decoder knows what that letter should be. Comparing them
|
||||
//! across a whole directory is close to a proof of which form the directory holds:
|
||||
//! agreement is near total for the right form and near chance for the wrong one.
|
||||
//! [`identify_form`] uses that to refuse an ingest that names form A over a
|
||||
//! directory of form B papers, which is otherwise a mistake nobody catches until
|
||||
//! the item statistics look strange three weeks later.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
use crate::assessment::{AssessmentFile, Form};
|
||||
use crate::catalog::Catalog;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::gradescope::Question;
|
||||
use crate::responses::ResponseSet;
|
||||
use crate::seal::{SealFile, printed_letter};
|
||||
use crate::select;
|
||||
|
||||
/// Where a decoder's mapping came from.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Provenance {
|
||||
/// Read from a seal written before administration. Authoritative.
|
||||
Seal,
|
||||
/// Derived from the assessment record and the form seed, as of now.
|
||||
Derived,
|
||||
}
|
||||
|
||||
impl Provenance {
|
||||
/// A short label for output.
|
||||
pub fn label(self) -> &'static str {
|
||||
match self {
|
||||
Provenance::Seal => "seal",
|
||||
Provenance::Derived => "derived from the record",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// How a grading export numbers its questions.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Numbering {
|
||||
/// The export counts printed positions, which is what Gradescope's `N.csv`
|
||||
/// file names mean. Positions are translated to recorded numbers.
|
||||
Printed,
|
||||
/// The export already carries recorded question numbers, so numbers pass
|
||||
/// through untouched.
|
||||
Recorded,
|
||||
}
|
||||
|
||||
/// One question's mapping on one form.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct QuestionMap {
|
||||
/// Printed position on the page, counting from 1.
|
||||
pub position: u32,
|
||||
/// The recorded question number, the join key to the record and the store.
|
||||
pub number: u32,
|
||||
/// The item's global id.
|
||||
pub item: String,
|
||||
/// Keyed letters as printed on this form.
|
||||
pub printed_key: Vec<String>,
|
||||
/// Keyed letters in the bank's own lettering.
|
||||
pub canonical_key: Vec<String>,
|
||||
/// Printed letter to bank letter.
|
||||
pub to_canonical: BTreeMap<String, String>,
|
||||
/// Bank letter to printed letter.
|
||||
pub to_printed: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
impl QuestionMap {
|
||||
/// The bank letter a printed letter stands for.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `printed` - the letter as the student saw it.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The bank letter, or `None` when the printed letter is not one of this
|
||||
/// question's options.
|
||||
pub fn canonical(&self, printed: &str) -> Option<&str> {
|
||||
self.to_canonical
|
||||
.get(&printed.trim().to_ascii_uppercase())
|
||||
.map(|s| s.as_str())
|
||||
}
|
||||
|
||||
/// How many options this question has.
|
||||
pub fn n_options(&self) -> usize {
|
||||
self.to_canonical.len()
|
||||
}
|
||||
|
||||
/// Whether this question's options were actually permuted.
|
||||
pub fn is_permuted(&self) -> bool {
|
||||
self.to_canonical.iter().any(|(k, v)| k != v)
|
||||
}
|
||||
}
|
||||
|
||||
/// One form's full mapping.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FormDecoder {
|
||||
/// The form id.
|
||||
pub form: String,
|
||||
/// Where the mapping came from.
|
||||
pub provenance: Provenance,
|
||||
/// Questions by printed position.
|
||||
by_position: BTreeMap<u32, QuestionMap>,
|
||||
/// Questions by recorded number.
|
||||
by_number: BTreeMap<u32, QuestionMap>,
|
||||
}
|
||||
|
||||
impl FormDecoder {
|
||||
/// Builds a decoder from the question maps.
|
||||
fn assemble(form: String, provenance: Provenance, maps: Vec<QuestionMap>) -> FormDecoder {
|
||||
let by_position = maps.iter().map(|m| (m.position, m.clone())).collect();
|
||||
let by_number = maps.into_iter().map(|m| (m.number, m)).collect();
|
||||
FormDecoder {
|
||||
form,
|
||||
provenance,
|
||||
by_position,
|
||||
by_number,
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads one form's mapping out of a seal.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `seal` - the seal.
|
||||
/// * `form_id` - the form to decode, matched case-insensitively.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The decoder.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`Error::Usage`] when the seal does not cover that form.
|
||||
pub fn from_seal(seal: &SealFile, form_id: &str) -> Result<FormDecoder> {
|
||||
let form = seal.form(form_id).ok_or_else(|| {
|
||||
Error::usage(format!(
|
||||
"the seal for `{}` does not cover form `{form_id}`; it covers {}",
|
||||
seal.seal.assessment,
|
||||
seal.forms
|
||||
.iter()
|
||||
.map(|f| f.id.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
))
|
||||
})?;
|
||||
|
||||
let maps = form
|
||||
.questions
|
||||
.iter()
|
||||
.map(|q| {
|
||||
let mut to_canonical = BTreeMap::new();
|
||||
let mut to_printed = BTreeMap::new();
|
||||
for map in &q.options {
|
||||
to_canonical.insert(map.printed.clone(), map.canonical.clone());
|
||||
to_printed.insert(map.canonical.clone(), map.printed.clone());
|
||||
}
|
||||
let canonical_key: Vec<String> = q
|
||||
.printed_key
|
||||
.iter()
|
||||
.filter_map(|p| to_canonical.get(p).cloned())
|
||||
.collect();
|
||||
QuestionMap {
|
||||
position: q.position,
|
||||
number: q.number,
|
||||
item: q.item.clone(),
|
||||
printed_key: q.printed_key.clone(),
|
||||
canonical_key,
|
||||
to_canonical,
|
||||
to_printed,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(FormDecoder::assemble(
|
||||
form.id.clone(),
|
||||
Provenance::Seal,
|
||||
maps,
|
||||
))
|
||||
}
|
||||
|
||||
/// Derives one form's mapping from the record and the bank.
|
||||
///
|
||||
/// Uses the same two functions every export calls, so a derived decoder and a
|
||||
/// freshly exported paper agree by construction.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `catalog` - the loaded course.
|
||||
/// * `record` - the assessment record.
|
||||
/// * `form` - the form.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The decoder.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`Error::Unresolved`] when a placement references a missing item.
|
||||
pub fn derive(catalog: &Catalog, record: &AssessmentFile, form: &Form) -> Result<FormDecoder> {
|
||||
let printed: Vec<_> = select::layout(record, form)
|
||||
.into_iter()
|
||||
.filter(|p| !p.dropped)
|
||||
.collect();
|
||||
|
||||
let mut maps = Vec::with_capacity(printed.len());
|
||||
for (index, placement) in printed.iter().enumerate() {
|
||||
let entry = catalog.require(&placement.item)?;
|
||||
let item = &entry.item;
|
||||
let order = select::option_order(form, &placement.item, item.options.len());
|
||||
|
||||
let canonical_key: BTreeSet<String> = if placement.key.is_empty() {
|
||||
item.key_letters().into_iter().collect()
|
||||
} else {
|
||||
placement.key.iter().cloned().collect()
|
||||
};
|
||||
|
||||
let mut to_canonical = BTreeMap::new();
|
||||
let mut to_printed = BTreeMap::new();
|
||||
let mut printed_key = Vec::new();
|
||||
for (position, source_index) in order.iter().enumerate() {
|
||||
let canonical = item
|
||||
.options
|
||||
.get(*source_index)
|
||||
.map(|c| c.id.clone())
|
||||
.unwrap_or_else(|| printed_letter(*source_index));
|
||||
let label = printed_letter(position);
|
||||
if canonical_key.contains(&canonical) {
|
||||
printed_key.push(label.clone());
|
||||
}
|
||||
to_canonical.insert(label.clone(), canonical.clone());
|
||||
to_printed.insert(canonical, label);
|
||||
}
|
||||
|
||||
maps.push(QuestionMap {
|
||||
position: index as u32 + 1,
|
||||
number: placement.number,
|
||||
item: placement.item.clone(),
|
||||
printed_key,
|
||||
canonical_key: canonical_key.into_iter().collect(),
|
||||
to_canonical,
|
||||
to_printed,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(FormDecoder::assemble(
|
||||
form.id.clone(),
|
||||
Provenance::Derived,
|
||||
maps,
|
||||
))
|
||||
}
|
||||
|
||||
/// Builds a decoder, preferring the seal.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `seal` - the seal, when one has been written.
|
||||
/// * `catalog` - the loaded course.
|
||||
/// * `record` - the assessment record.
|
||||
/// * `form` - the form.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The decoder.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// As [`FormDecoder::from_seal`] and [`FormDecoder::derive`]. A seal that does
|
||||
/// not cover the requested form falls back to deriving rather than failing,
|
||||
/// since a form added after sealing is a real situation.
|
||||
pub fn resolve(
|
||||
seal: Option<&SealFile>,
|
||||
catalog: &Catalog,
|
||||
record: &AssessmentFile,
|
||||
form: &Form,
|
||||
) -> Result<FormDecoder> {
|
||||
if let Some(seal) = seal {
|
||||
if seal.form(&form.id).is_some() {
|
||||
return FormDecoder::from_seal(seal, &form.id);
|
||||
}
|
||||
}
|
||||
FormDecoder::derive(catalog, record, form)
|
||||
}
|
||||
|
||||
/// The question at a printed position.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `position` - the printed position, counting from 1.
|
||||
pub fn at_position(&self, position: u32) -> Option<&QuestionMap> {
|
||||
self.by_position.get(&position)
|
||||
}
|
||||
|
||||
/// The question with a recorded number.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `number` - the recorded number.
|
||||
pub fn at_number(&self, number: u32) -> Option<&QuestionMap> {
|
||||
self.by_number.get(&number)
|
||||
}
|
||||
|
||||
/// The question an export's numbering refers to.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `n` - the number as the export gives it.
|
||||
/// * `numbering` - how the export numbers questions.
|
||||
pub fn lookup(&self, n: u32, numbering: Numbering) -> Option<&QuestionMap> {
|
||||
match numbering {
|
||||
Numbering::Printed => self.at_position(n),
|
||||
Numbering::Recorded => self.at_number(n),
|
||||
}
|
||||
}
|
||||
|
||||
/// How many questions this form prints.
|
||||
pub fn len(&self) -> usize {
|
||||
self.by_position.len()
|
||||
}
|
||||
|
||||
/// Whether the form prints nothing, which means the record is empty.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.by_position.is_empty()
|
||||
}
|
||||
|
||||
/// Whether any question on this form has permuted options.
|
||||
///
|
||||
/// Used to decide whether to say anything about translation at all: on an
|
||||
/// unshuffled form the whole mechanism is an identity map and mentioning it
|
||||
/// is noise.
|
||||
pub fn is_permuted(&self) -> bool {
|
||||
self.by_position.values().any(|q| q.is_permuted())
|
||||
}
|
||||
|
||||
/// Whether printed positions and recorded numbers disagree anywhere.
|
||||
///
|
||||
/// True when items were shuffled, and also when a bonus item sits mid-record,
|
||||
/// since the layout moves bonus items to the end of the paper.
|
||||
pub fn is_renumbered(&self) -> bool {
|
||||
self.by_position.values().any(|q| q.position != q.number)
|
||||
}
|
||||
}
|
||||
|
||||
/// What a directory of graded questions says about which form it holds.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FormFit {
|
||||
/// The form id.
|
||||
pub form: String,
|
||||
/// Questions whose graded key matched this form's printed key.
|
||||
pub matched: usize,
|
||||
/// Questions that could be compared at all.
|
||||
pub compared: usize,
|
||||
/// Question positions where the graded key disagreed.
|
||||
pub mismatches: Vec<u32>,
|
||||
}
|
||||
|
||||
impl FormFit {
|
||||
/// The share of comparable questions that agreed.
|
||||
pub fn rate(&self) -> f64 {
|
||||
if self.compared == 0 {
|
||||
0.0
|
||||
} else {
|
||||
self.matched as f64 / self.compared as f64
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the fit is good enough to proceed without a warning.
|
||||
///
|
||||
/// The threshold is high on purpose. A correctly matched directory agrees on
|
||||
/// every question; anything less than total agreement is either a regrade that
|
||||
/// moved a key or the wrong directory, and both are worth a sentence.
|
||||
pub fn is_convincing(&self) -> bool {
|
||||
self.compared > 0 && self.matched == self.compared
|
||||
}
|
||||
}
|
||||
|
||||
/// Compares a parsed Gradescope directory against one form's expected keys.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `questions` - the parsed question files.
|
||||
/// * `decoder` - the form to test against.
|
||||
/// * `numbering` - how the export numbers questions.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The fit.
|
||||
pub fn fit_form(questions: &[Question], decoder: &FormDecoder, numbering: Numbering) -> FormFit {
|
||||
let mut matched = 0usize;
|
||||
let mut compared = 0usize;
|
||||
let mut mismatches = Vec::new();
|
||||
|
||||
for question in questions {
|
||||
let Some(map) = decoder.lookup(question.number, numbering) else {
|
||||
continue;
|
||||
};
|
||||
let graded: BTreeSet<String> = question.keyed().into_iter().collect();
|
||||
if graded.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let expected: BTreeSet<String> = map.printed_key.iter().cloned().collect();
|
||||
if expected.is_empty() {
|
||||
continue;
|
||||
}
|
||||
compared += 1;
|
||||
if graded == expected {
|
||||
matched += 1;
|
||||
} else {
|
||||
mismatches.push(question.number);
|
||||
}
|
||||
}
|
||||
|
||||
FormFit {
|
||||
form: decoder.form.clone(),
|
||||
matched,
|
||||
compared,
|
||||
mismatches,
|
||||
}
|
||||
}
|
||||
|
||||
/// Ranks every candidate form against a directory.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `questions` - the parsed question files.
|
||||
/// * `decoders` - one decoder per declared form.
|
||||
/// * `numbering` - how the export numbers questions.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The fits, best first.
|
||||
pub fn identify_form(
|
||||
questions: &[Question],
|
||||
decoders: &[FormDecoder],
|
||||
numbering: Numbering,
|
||||
) -> Vec<FormFit> {
|
||||
let mut fits: Vec<FormFit> = decoders
|
||||
.iter()
|
||||
.map(|d| fit_form(questions, d, numbering))
|
||||
.collect();
|
||||
fits.sort_by(|a, b| {
|
||||
b.rate()
|
||||
.partial_cmp(&a.rate())
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
.then_with(|| a.form.cmp(&b.form))
|
||||
});
|
||||
fits
|
||||
}
|
||||
|
||||
/// Explains a fit in a sentence, or says nothing when the fit is perfect.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `claimed` - the form the directory was ingested as.
|
||||
/// * `fits` - every form's fit, best first.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// A warning, or `None`.
|
||||
pub fn form_warning(claimed: &str, fits: &[FormFit]) -> Option<String> {
|
||||
let mine = fits.iter().find(|f| f.form.eq_ignore_ascii_case(claimed))?;
|
||||
if mine.is_convincing() {
|
||||
return None;
|
||||
}
|
||||
if mine.compared == 0 {
|
||||
return Some(format!(
|
||||
"form {claimed}: the export carries no point values, so the graded keys could not be \
|
||||
checked against the form. Nothing verified this directory is form {claimed}"
|
||||
));
|
||||
}
|
||||
|
||||
let better = fits
|
||||
.iter()
|
||||
.find(|f| !f.form.eq_ignore_ascii_case(claimed) && f.rate() > mine.rate());
|
||||
|
||||
let head = format!(
|
||||
"form {claimed}: the graded key matches this form on {} of {} question(s)",
|
||||
mine.matched, mine.compared
|
||||
);
|
||||
let where_ = if mine.mismatches.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
let list: Vec<String> = mine
|
||||
.mismatches
|
||||
.iter()
|
||||
.take(8)
|
||||
.map(|n| n.to_string())
|
||||
.collect();
|
||||
format!(
|
||||
" (q{}{})",
|
||||
list.join(", q"),
|
||||
if mine.mismatches.len() > 8 {
|
||||
", …"
|
||||
} else {
|
||||
""
|
||||
}
|
||||
)
|
||||
};
|
||||
match better {
|
||||
Some(other) => Some(format!(
|
||||
"{head}{where_}, but matches form {} on {} of {}. This directory is almost certainly \
|
||||
form {}, not form {claimed}",
|
||||
other.form, other.matched, other.compared, other.form
|
||||
)),
|
||||
None => Some(format!(
|
||||
"{head}{where_}. Either those questions were regraded after printing, or the form is \
|
||||
not the one named"
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Translates a form's responses into the bank's vocabulary.
|
||||
///
|
||||
/// Rewrites, for every row whose `form` matches this decoder:
|
||||
///
|
||||
/// * `item_number`, from printed position to recorded number, when the export
|
||||
/// numbers by position;
|
||||
/// * `form_position`, recording where the question sat on the page;
|
||||
/// * `selected_source` and `eliminated_source`, the bank letters for what was
|
||||
/// marked. The printed letters stay in `selected` and `eliminated`, because what
|
||||
/// a student physically marked is the fact and the translation is the
|
||||
/// interpretation.
|
||||
///
|
||||
/// Correctness and credit are untouched. Both come from the grading platform,
|
||||
/// which scored the paper the student actually held, and are already right.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `set` - the responses to translate, in place.
|
||||
/// * `decoder` - the form's mapping.
|
||||
/// * `numbering` - how the export numbered questions.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Warnings for anything that could not be translated.
|
||||
pub fn apply(set: &mut ResponseSet, decoder: &FormDecoder, numbering: Numbering) -> Vec<String> {
|
||||
let mut warnings = Vec::new();
|
||||
let mut unmapped_positions: BTreeSet<u32> = BTreeSet::new();
|
||||
let mut unmapped_letters: BTreeSet<String> = BTreeSet::new();
|
||||
let mut translated = 0usize;
|
||||
|
||||
for row in &mut set.rows {
|
||||
let belongs = row
|
||||
.form
|
||||
.as_deref()
|
||||
.map(|f| f.eq_ignore_ascii_case(&decoder.form))
|
||||
.unwrap_or(false);
|
||||
if !belongs {
|
||||
continue;
|
||||
}
|
||||
|
||||
let Some(map) = decoder.lookup(row.item_number, numbering) else {
|
||||
unmapped_positions.insert(row.item_number);
|
||||
continue;
|
||||
};
|
||||
|
||||
row.form_position = Some(map.position);
|
||||
row.item_number = map.number;
|
||||
|
||||
let mut convert = |letters: &[String]| -> Vec<String> {
|
||||
let mut out = Vec::with_capacity(letters.len());
|
||||
for letter in letters {
|
||||
match map.canonical(letter) {
|
||||
Some(canonical) => out.push(canonical.to_string()),
|
||||
None => {
|
||||
unmapped_letters.insert(format!("q{} {}", map.number, letter));
|
||||
}
|
||||
}
|
||||
}
|
||||
out.sort();
|
||||
out
|
||||
};
|
||||
|
||||
row.selected_source = convert(&row.selected);
|
||||
row.eliminated_source = convert(&row.eliminated);
|
||||
translated += 1;
|
||||
}
|
||||
|
||||
if !unmapped_positions.is_empty() {
|
||||
let list: Vec<String> = unmapped_positions.iter().map(|n| n.to_string()).collect();
|
||||
warnings.push(format!(
|
||||
"form {}: question(s) {} are in the export but not on this form; they were left \
|
||||
untranslated",
|
||||
decoder.form,
|
||||
list.join(", ")
|
||||
));
|
||||
}
|
||||
if !unmapped_letters.is_empty() {
|
||||
let list: Vec<String> = unmapped_letters.iter().take(10).cloned().collect();
|
||||
warnings.push(format!(
|
||||
"form {}: {} marked option(s) are not options on the printed form ({}), which usually \
|
||||
means a rubric column was added by hand in Gradescope",
|
||||
decoder.form,
|
||||
unmapped_letters.len(),
|
||||
list.join(", ")
|
||||
));
|
||||
}
|
||||
if translated == 0 {
|
||||
warnings.push(format!(
|
||||
"form {}: no response rows carried this form id, so nothing was translated",
|
||||
decoder.form
|
||||
));
|
||||
}
|
||||
|
||||
warnings
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::responses::{Response, administration_id};
|
||||
|
||||
fn map(position: u32, number: u32, pairs: &[(&str, &str)], key: &str) -> QuestionMap {
|
||||
let mut to_canonical = BTreeMap::new();
|
||||
let mut to_printed = BTreeMap::new();
|
||||
for (printed, canonical) in pairs {
|
||||
to_canonical.insert(printed.to_string(), canonical.to_string());
|
||||
to_printed.insert(canonical.to_string(), printed.to_string());
|
||||
}
|
||||
let printed_key = vec![to_printed.get(key).cloned().unwrap_or_default()];
|
||||
QuestionMap {
|
||||
position,
|
||||
number,
|
||||
item: format!("b::q-{number}"),
|
||||
printed_key,
|
||||
canonical_key: vec![key.to_string()],
|
||||
to_canonical,
|
||||
to_printed,
|
||||
}
|
||||
}
|
||||
|
||||
fn decoder() -> FormDecoder {
|
||||
FormDecoder::assemble(
|
||||
"B".to_string(),
|
||||
Provenance::Seal,
|
||||
vec![
|
||||
// Printed A..D show bank C, A, D, B. The key is bank A, printed B.
|
||||
map(1, 1, &[("A", "C"), ("B", "A"), ("C", "D"), ("D", "B")], "A"),
|
||||
// A bonus item recorded as 9 but printed last, at position 2.
|
||||
map(2, 9, &[("A", "B"), ("B", "A")], "B"),
|
||||
],
|
||||
)
|
||||
}
|
||||
|
||||
fn row(number: u32, selected: &str) -> Response {
|
||||
Response {
|
||||
administration_id: administration_id("C", "2026f", "e1"),
|
||||
course: "C".into(),
|
||||
term: "2026f".into(),
|
||||
assessment_id: "e1".into(),
|
||||
date: None,
|
||||
form: Some("B".into()),
|
||||
student_key: "s1".into(),
|
||||
sid: None,
|
||||
name: None,
|
||||
email: None,
|
||||
section: None,
|
||||
item_number: number,
|
||||
form_position: None,
|
||||
item_ref: None,
|
||||
item_version: None,
|
||||
selected: vec![selected.into()],
|
||||
eliminated: Vec::new(),
|
||||
selected_source: Vec::new(),
|
||||
eliminated_source: Vec::new(),
|
||||
correct: Some(false),
|
||||
credit: 0.0,
|
||||
points_possible: 1.0,
|
||||
score: 0.0,
|
||||
response_time_seconds: None,
|
||||
level: None,
|
||||
learning_objectives: Vec::new(),
|
||||
topics: Vec::new(),
|
||||
bonus: false,
|
||||
dropped: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn printed_letters_become_bank_letters() {
|
||||
let decoder = decoder();
|
||||
let q = decoder.at_position(1).unwrap();
|
||||
assert_eq!(q.canonical("A"), Some("C"));
|
||||
assert_eq!(q.canonical("D"), Some("B"));
|
||||
assert_eq!(q.canonical("E"), None);
|
||||
assert_eq!(q.printed_key, vec!["B".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn positions_become_recorded_numbers() {
|
||||
let mut set = ResponseSet::new();
|
||||
set.rows.push(row(2, "A"));
|
||||
let warnings = apply(&mut set, &decoder(), Numbering::Printed);
|
||||
assert!(warnings.is_empty(), "{warnings:?}");
|
||||
assert_eq!(set.rows[0].item_number, 9, "position 2 is recorded as 9");
|
||||
assert_eq!(set.rows[0].form_position, Some(2));
|
||||
assert_eq!(set.rows[0].selected, vec!["A".to_string()], "printed kept");
|
||||
assert_eq!(set.rows[0].selected_source, vec!["B".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rows_from_another_form_are_left_alone() {
|
||||
let mut set = ResponseSet::new();
|
||||
let mut other = row(1, "A");
|
||||
other.form = Some("A".into());
|
||||
set.rows.push(other);
|
||||
apply(&mut set, &decoder(), Numbering::Printed);
|
||||
assert!(set.rows[0].selected_source.is_empty());
|
||||
assert_eq!(set.rows[0].item_number, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unknown_position_is_reported_not_guessed() {
|
||||
let mut set = ResponseSet::new();
|
||||
set.rows.push(row(7, "A"));
|
||||
let warnings = apply(&mut set, &decoder(), Numbering::Printed);
|
||||
assert!(
|
||||
warnings.iter().any(|w| w.contains("not on this form")),
|
||||
"{warnings:?}"
|
||||
);
|
||||
assert_eq!(set.rows[0].item_number, 7, "left as found");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_perfect_fit_says_nothing() {
|
||||
let fits = vec![FormFit {
|
||||
form: "A".into(),
|
||||
matched: 30,
|
||||
compared: 30,
|
||||
mismatches: Vec::new(),
|
||||
}];
|
||||
assert!(form_warning("A", &fits).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_swapped_directory_names_the_form_it_really_is() {
|
||||
let fits = vec![
|
||||
FormFit {
|
||||
form: "B".into(),
|
||||
matched: 30,
|
||||
compared: 30,
|
||||
mismatches: Vec::new(),
|
||||
},
|
||||
FormFit {
|
||||
form: "A".into(),
|
||||
matched: 8,
|
||||
compared: 30,
|
||||
mismatches: (1..=22).collect(),
|
||||
},
|
||||
];
|
||||
let warning = form_warning("A", &fits).expect("a mismatch this large must warn");
|
||||
assert!(warning.contains("almost certainly form B"), "{warning}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_single_regraded_key_warns_without_accusing_the_wrong_form() {
|
||||
let fits = vec![FormFit {
|
||||
form: "A".into(),
|
||||
matched: 29,
|
||||
compared: 30,
|
||||
mismatches: vec![14],
|
||||
}];
|
||||
let warning = form_warning("A", &fits).unwrap();
|
||||
assert!(warning.contains("q14"), "{warning}");
|
||||
assert!(warning.contains("regraded"), "{warning}");
|
||||
}
|
||||
}
|
||||
@@ -633,6 +633,10 @@ pub fn to_responses(questions: &[Question], ctx: &Context) -> Import {
|
||||
assessment_id: ctx.assessment_id.clone(),
|
||||
date: ctx.date,
|
||||
form: ctx.form.clone(),
|
||||
// The printed position and the bank's lettering are written
|
||||
// later, by `decode::apply`, which is the only place that knows
|
||||
// which form this directory holds.
|
||||
form_position: None,
|
||||
student_key,
|
||||
sid: row.sid.clone(),
|
||||
name: row.name.clone(),
|
||||
@@ -642,7 +646,9 @@ pub fn to_responses(questions: &[Question], ctx: &Context) -> Import {
|
||||
item_ref: None,
|
||||
item_version: None,
|
||||
selected,
|
||||
selected_source: Vec::new(),
|
||||
eliminated,
|
||||
eliminated_source: Vec::new(),
|
||||
correct,
|
||||
credit,
|
||||
points_possible: points,
|
||||
|
||||
@@ -0,0 +1,565 @@
|
||||
// SPDX-License-Identifier: Prosperity-3.0.0
|
||||
// Copyright Scientific Computing Studio
|
||||
// Source: https://git.scient.ing/education/coursebank
|
||||
|
||||
//! Reading several forms of one exam back in at once.
|
||||
//!
|
||||
//! A two-form exam is two Gradescope assignments, each exporting its own
|
||||
//! directory of `1.csv` through `N.csv`, each numbered against its own paper. They
|
||||
//! are one administration: one set of students, one item pool, one set of
|
||||
//! statistics. Ingesting them one command at a time does not work, because the
|
||||
//! store keys on the administration and the second write replaces the first.
|
||||
//!
|
||||
//! So this module takes the whole set:
|
||||
//!
|
||||
//! ```text
|
||||
//! coursebank ingest gradescope A=exports/e1-a B=exports/e1-b --assessment e1
|
||||
//! ```
|
||||
//!
|
||||
//! and does four things the single-directory path cannot:
|
||||
//!
|
||||
//! *Checks each directory is the form it claims to be.* Every export carries the
|
||||
//! graded key in its point-value row; every form knows what its printed key should
|
||||
//! be. Comparing them catches a swapped pair of directories immediately rather
|
||||
//! than three weeks later, when the distractor table looks strange. See
|
||||
//! [`crate::decode::identify_form`].
|
||||
//!
|
||||
//! *Translates each form into the bank's vocabulary before merging.* Otherwise
|
||||
//! form A's option C and form B's option C land in the same column of the same
|
||||
//! table while meaning different things.
|
||||
//!
|
||||
//! *Merges into one response set*, written once, so `analyze` and `report` see the
|
||||
//! whole class.
|
||||
//!
|
||||
//! *Reports what the merge revealed*: a student who appears on two forms, a form
|
||||
//! that is missing a question the other has, a form that ran materially harder
|
||||
//! than the other.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::assessment::{AssessmentFile, Form};
|
||||
use crate::catalog::Catalog;
|
||||
use crate::date::Date;
|
||||
use crate::decode::{self, FormDecoder, FormFit, Numbering};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::gradescope::{self, Context, Question};
|
||||
use crate::responses::ResponseSet;
|
||||
use crate::seal::SealFile;
|
||||
|
||||
/// One directory of graded questions, and the form it holds.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Source {
|
||||
/// The form id.
|
||||
pub form: String,
|
||||
/// The directory of per-question CSV exports.
|
||||
pub dir: PathBuf,
|
||||
}
|
||||
|
||||
impl Source {
|
||||
/// Parses a `FORM=DIR` argument.
|
||||
///
|
||||
/// A bare path is accepted and takes the fallback form, so the single-form
|
||||
/// case stays as short as it was.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `text` - the argument, e.g. `A=exports/e1-a` or `exports/e1`.
|
||||
/// * `fallback` - the form to use when the argument names none.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The source.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`Error::Usage`] when the argument names no form and no fallback
|
||||
/// was given, or when the form label is empty.
|
||||
pub fn parse(text: &str, fallback: Option<&str>) -> Result<Source> {
|
||||
// Split on the first `=` only: a directory name may contain one, a form
|
||||
// label may not.
|
||||
if let Some((form, dir)) = text.split_once('=') {
|
||||
let form = form.trim();
|
||||
if form.is_empty() {
|
||||
return Err(Error::usage(format!(
|
||||
"`{text}` has an empty form label; write it as FORM=DIR, e.g. A=exports/e1-a"
|
||||
)));
|
||||
}
|
||||
if !dir.trim().is_empty() {
|
||||
return Ok(Source {
|
||||
form: form.to_string(),
|
||||
dir: PathBuf::from(dir.trim()),
|
||||
});
|
||||
}
|
||||
}
|
||||
match fallback {
|
||||
Some(form) => Ok(Source {
|
||||
form: form.to_string(),
|
||||
dir: PathBuf::from(text.trim()),
|
||||
}),
|
||||
None => Err(Error::usage(format!(
|
||||
"`{text}` does not say which form it holds; write it as FORM=DIR (e.g. \
|
||||
A=exports/e1-a) or pass --form"
|
||||
))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// How to run an intake.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Options {
|
||||
/// The administration date, overriding the record's.
|
||||
pub date: Option<Date>,
|
||||
/// How the exports number their questions.
|
||||
pub numbering: Numbering,
|
||||
/// Whether a form that fails its key check stops the ingest.
|
||||
///
|
||||
/// On by default. A directory that does not match the form it was named as is
|
||||
/// the one ingest error that produces confident, wrong analysis rather than an
|
||||
/// obvious failure, so the default is to refuse and say so.
|
||||
pub strict: bool,
|
||||
}
|
||||
|
||||
impl Default for Options {
|
||||
fn default() -> Options {
|
||||
Options {
|
||||
date: None,
|
||||
numbering: Numbering::Printed,
|
||||
strict: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// What one form's directory contributed.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FormIntake {
|
||||
/// The form id.
|
||||
pub form: String,
|
||||
/// The directory it came from.
|
||||
pub dir: PathBuf,
|
||||
/// Where its mapping came from.
|
||||
pub provenance: decode::Provenance,
|
||||
/// How many students it held.
|
||||
pub students: usize,
|
||||
/// How many questions it held.
|
||||
pub questions: usize,
|
||||
/// How the graded keys compared to every declared form, best first.
|
||||
pub fits: Vec<FormFit>,
|
||||
/// The parsed question files, kept so grading-time decisions stay available.
|
||||
pub parsed: Vec<Question>,
|
||||
}
|
||||
|
||||
impl FormIntake {
|
||||
/// This form's own fit.
|
||||
pub fn own_fit(&self) -> Option<&FormFit> {
|
||||
self.fits
|
||||
.iter()
|
||||
.find(|f| f.form.eq_ignore_ascii_case(&self.form))
|
||||
}
|
||||
}
|
||||
|
||||
/// The result of reading every form of one administration.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Intake {
|
||||
/// The merged, translated responses.
|
||||
pub responses: ResponseSet,
|
||||
/// Per-form detail.
|
||||
pub forms: Vec<FormIntake>,
|
||||
/// Problems that did not stop the ingest.
|
||||
pub warnings: Vec<String>,
|
||||
}
|
||||
|
||||
impl Intake {
|
||||
/// Every parsed question file, across forms.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Pairs of form id and question.
|
||||
pub fn questions(&self) -> Vec<(&str, &Question)> {
|
||||
self.forms
|
||||
.iter()
|
||||
.flat_map(|f| f.parsed.iter().map(move |q| (f.form.as_str(), q)))
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads every source into one response set.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `catalog` - the loaded course.
|
||||
/// * `record` - the assessment record.
|
||||
/// * `seal` - the seal, when one was written. Strongly preferred: it describes the
|
||||
/// paper that was printed rather than the paper the bank would print today.
|
||||
/// * `sources` - the directories and the forms they hold.
|
||||
/// * `opts` - how to run.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The merged intake.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`Error::Usage`] when a source names a form the record does not
|
||||
/// declare, when two sources name the same form, or when a key check fails under
|
||||
/// `strict`. Propagates parse errors from the exports themselves.
|
||||
pub fn run(
|
||||
catalog: &Catalog,
|
||||
record: &AssessmentFile,
|
||||
seal: Option<&SealFile>,
|
||||
sources: &[Source],
|
||||
opts: &Options,
|
||||
) -> Result<Intake> {
|
||||
if sources.is_empty() {
|
||||
return Err(Error::usage(
|
||||
"no directories to ingest; pass one per form, e.g. A=exports/e1-a B=exports/e1-b"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let declared = declared_forms(record);
|
||||
let mut seen: BTreeSet<String> = BTreeSet::new();
|
||||
for source in sources {
|
||||
if !seen.insert(source.form.to_ascii_uppercase()) {
|
||||
return Err(Error::usage(format!(
|
||||
"form {} was given twice; each form is one directory",
|
||||
source.form
|
||||
)));
|
||||
}
|
||||
if !declared
|
||||
.iter()
|
||||
.any(|f| f.id.eq_ignore_ascii_case(&source.form))
|
||||
{
|
||||
return Err(Error::usage(format!(
|
||||
"the record for `{}` declares no form `{}`; it declares {}",
|
||||
record.assessment.id,
|
||||
source.form,
|
||||
declared
|
||||
.iter()
|
||||
.map(|f| f.id.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
// One decoder per declared form, not just per ingested form: identifying a
|
||||
// swapped directory means testing it against the forms it might be.
|
||||
let mut decoders: Vec<FormDecoder> = Vec::new();
|
||||
for form in &declared {
|
||||
decoders.push(FormDecoder::resolve(seal, catalog, record, form)?);
|
||||
}
|
||||
|
||||
let mut merged = ResponseSet::new();
|
||||
let mut forms = Vec::new();
|
||||
let mut warnings = Vec::new();
|
||||
let mut blocking = Vec::new();
|
||||
|
||||
for source in sources {
|
||||
let decoder = decoders
|
||||
.iter()
|
||||
.find(|d| d.form.eq_ignore_ascii_case(&source.form))
|
||||
.expect("every source's form was checked against the declared list");
|
||||
|
||||
let ctx = Context {
|
||||
course: catalog.course.course.code.clone(),
|
||||
term: record
|
||||
.assessment
|
||||
.term
|
||||
.clone()
|
||||
.unwrap_or_else(|| catalog.course.course.term.clone()),
|
||||
assessment_id: record.assessment.id.clone(),
|
||||
date: opts.date.or(record.assessment.date),
|
||||
form: Some(decoder.form.clone()),
|
||||
};
|
||||
|
||||
let import = gradescope::ingest_dir(&source.dir, &ctx)?;
|
||||
let mut set = import.responses;
|
||||
|
||||
let fits = decode::identify_form(&import.questions, &decoders, opts.numbering);
|
||||
if let Some(problem) = decode::form_warning(&decoder.form, &fits) {
|
||||
let message = format!("{} [{}]", problem, source.dir.display());
|
||||
let convincing_alternative = fits
|
||||
.iter()
|
||||
.any(|f| !f.form.eq_ignore_ascii_case(&decoder.form) && f.is_convincing());
|
||||
if opts.strict && convincing_alternative {
|
||||
blocking.push(message);
|
||||
} else {
|
||||
warnings.push(message);
|
||||
}
|
||||
}
|
||||
|
||||
warnings.extend(decode::apply(&mut set, decoder, opts.numbering));
|
||||
|
||||
forms.push(FormIntake {
|
||||
form: decoder.form.clone(),
|
||||
dir: source.dir.clone(),
|
||||
provenance: decoder.provenance,
|
||||
students: set.students().len(),
|
||||
questions: set.all_items().len(),
|
||||
fits,
|
||||
parsed: import.questions,
|
||||
});
|
||||
|
||||
merged.absorb(set);
|
||||
}
|
||||
|
||||
if !blocking.is_empty() {
|
||||
blocking.push(
|
||||
"Nothing was written. Fix the form labels, or pass --allow-mismatch if the keys really \
|
||||
did change after printing."
|
||||
.to_string(),
|
||||
);
|
||||
return Err(Error::Invalid(blocking));
|
||||
}
|
||||
|
||||
warnings.extend(cross_form_checks(&merged, &forms, record));
|
||||
merged.warnings.extend(warnings.clone());
|
||||
|
||||
Ok(Intake {
|
||||
responses: merged,
|
||||
forms,
|
||||
warnings,
|
||||
})
|
||||
}
|
||||
|
||||
/// The forms a record declares, with the implicit single form for a record that
|
||||
/// declares none.
|
||||
fn declared_forms(record: &AssessmentFile) -> Vec<Form> {
|
||||
if record.forms.is_empty() {
|
||||
vec![Form {
|
||||
id: "A".to_string(),
|
||||
seed: 0,
|
||||
shuffle_items: false,
|
||||
shuffle_options: false,
|
||||
}]
|
||||
} else {
|
||||
record.forms.clone()
|
||||
}
|
||||
}
|
||||
|
||||
/// Checks that only merging several forms can make.
|
||||
fn cross_form_checks(
|
||||
merged: &ResponseSet,
|
||||
forms: &[FormIntake],
|
||||
record: &AssessmentFile,
|
||||
) -> Vec<String> {
|
||||
let mut out = Vec::new();
|
||||
if forms.len() < 2 {
|
||||
return out;
|
||||
}
|
||||
|
||||
// A student on two forms sat one exam and was graded twice, or two people
|
||||
// share an identifier. Either way the response set now double counts them.
|
||||
let mut by_student: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new();
|
||||
for row in &merged.rows {
|
||||
if let Some(form) = row.form.as_deref() {
|
||||
by_student
|
||||
.entry(row.student_key.as_str())
|
||||
.or_default()
|
||||
.insert(form);
|
||||
}
|
||||
}
|
||||
let doubled: Vec<&str> = by_student
|
||||
.iter()
|
||||
.filter(|(_, forms)| forms.len() > 1)
|
||||
.map(|(student, _)| *student)
|
||||
.collect();
|
||||
if !doubled.is_empty() {
|
||||
out.push(format!(
|
||||
"{} student(s) appear on more than one form ({}); each is counted twice in every \
|
||||
total until one submission is removed",
|
||||
doubled.len(),
|
||||
doubled
|
||||
.iter()
|
||||
.take(5)
|
||||
.copied()
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
));
|
||||
}
|
||||
|
||||
// Every form is the same items in a different order, so a question present on
|
||||
// one and absent from another means a directory is short a file.
|
||||
let expected: BTreeSet<u32> = record
|
||||
.items
|
||||
.iter()
|
||||
.filter(|p| !p.dropped)
|
||||
.map(|p| p.number)
|
||||
.collect();
|
||||
for form in forms {
|
||||
let present: BTreeSet<u32> = merged
|
||||
.rows
|
||||
.iter()
|
||||
.filter(|r| {
|
||||
r.form
|
||||
.as_deref()
|
||||
.map(|f| f.eq_ignore_ascii_case(&form.form))
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.map(|r| r.item_number)
|
||||
.collect();
|
||||
let missing: Vec<String> = expected
|
||||
.difference(&present)
|
||||
.map(|n| n.to_string())
|
||||
.collect();
|
||||
if !missing.is_empty() {
|
||||
out.push(format!(
|
||||
"form {}: no responses for question(s) {}; the export directory is missing those \
|
||||
files",
|
||||
form.form,
|
||||
missing.join(", ")
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// Forms are meant to be the same test. A large gap between their means is
|
||||
// either a permutation that made one form easier or an uneven split of the
|
||||
// class, and both are worth knowing before any grade is released.
|
||||
let means: Vec<(String, f64, usize)> = forms
|
||||
.iter()
|
||||
.map(|f| {
|
||||
let students: Vec<&str> = merged
|
||||
.rows
|
||||
.iter()
|
||||
.filter(|r| {
|
||||
r.form
|
||||
.as_deref()
|
||||
.map(|x| x.eq_ignore_ascii_case(&f.form))
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.map(|r| r.student_key.as_str())
|
||||
.collect::<BTreeSet<&str>>()
|
||||
.into_iter()
|
||||
.collect();
|
||||
let possible = merged.points_available();
|
||||
let percents: Vec<f64> = students
|
||||
.iter()
|
||||
.map(|s| {
|
||||
if possible > 0.0 {
|
||||
100.0 * merged.scored_total(s) / possible
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let mean = if percents.is_empty() {
|
||||
0.0
|
||||
} else {
|
||||
percents.iter().sum::<f64>() / percents.len() as f64
|
||||
};
|
||||
(f.form.clone(), mean, percents.len())
|
||||
})
|
||||
.collect();
|
||||
|
||||
if let (Some(low), Some(high)) = (
|
||||
means
|
||||
.iter()
|
||||
.min_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal)),
|
||||
means
|
||||
.iter()
|
||||
.max_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal)),
|
||||
) {
|
||||
let gap = high.1 - low.1;
|
||||
if gap >= 5.0 && low.2 >= 5 && high.2 >= 5 {
|
||||
out.push(format!(
|
||||
"form {} averaged {:.0}% and form {} averaged {:.0}%, a {:.0}-point gap. With {} \
|
||||
and {} students that may be the split rather than the forms, but it is worth \
|
||||
looking at before releasing grades",
|
||||
high.0, high.1, low.0, low.1, gap, high.2, low.2
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
/// A one-line summary of where a directory came from and what it held.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `intake` - one form's intake.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The line, without a trailing newline.
|
||||
pub fn describe(intake: &FormIntake) -> String {
|
||||
let fit = match intake.own_fit() {
|
||||
Some(fit) if fit.compared > 0 => {
|
||||
format!(", keys matched {}/{}", fit.matched, fit.compared)
|
||||
}
|
||||
_ => String::new(),
|
||||
};
|
||||
format!(
|
||||
"form {}: {} student(s) x {} question(s) from {} (mapping {}{fit})",
|
||||
intake.form,
|
||||
intake.students,
|
||||
intake.questions,
|
||||
intake.dir.display(),
|
||||
intake.provenance.label(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Whether a path looks like a Gradescope per-question export directory.
|
||||
///
|
||||
/// Used to give a better error than "no numbered CSV files" when someone points
|
||||
/// at the zip they downloaded, or at the directory above the one they meant.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `dir` - the candidate directory.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// `true` when it holds at least one file named like `1.csv`.
|
||||
pub fn looks_like_export(dir: &Path) -> bool {
|
||||
let Ok(entries) = std::fs::read_dir(dir) else {
|
||||
return false;
|
||||
};
|
||||
entries.filter_map(|e| e.ok()).any(|entry| {
|
||||
let path = entry.path();
|
||||
path.extension().and_then(|e| e.to_str()) == Some("csv")
|
||||
&& path
|
||||
.file_stem()
|
||||
.and_then(|s| s.to_str())
|
||||
.map(|s| s.chars().all(|c| c.is_ascii_digit()))
|
||||
.unwrap_or(false)
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_labelled_source_parses() {
|
||||
let source = Source::parse("B=exports/e1-b", None).unwrap();
|
||||
assert_eq!(source.form, "B");
|
||||
assert_eq!(source.dir, PathBuf::from("exports/e1-b"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_bare_path_needs_a_fallback_form() {
|
||||
let source = Source::parse("exports/e1", Some("A")).unwrap();
|
||||
assert_eq!(source.form, "A");
|
||||
assert_eq!(source.dir, PathBuf::from("exports/e1"));
|
||||
|
||||
let err = Source::parse("exports/e1", None).unwrap_err();
|
||||
assert!(err.to_string().contains("which form"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_windows_style_path_is_not_mistaken_for_a_label() {
|
||||
// The split is on the first `=`, and a drive letter has none, so this is
|
||||
// only a hazard for a path that genuinely contains one.
|
||||
let source = Source::parse("A=C:/exports/e1-a", None).unwrap();
|
||||
assert_eq!(source.form, "A");
|
||||
assert_eq!(source.dir, PathBuf::from("C:/exports/e1-a"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_label_is_refused() {
|
||||
let err = Source::parse("=exports/e1", None).unwrap_err();
|
||||
assert!(err.to_string().contains("empty form label"), "{err}");
|
||||
}
|
||||
}
|
||||
+65
-6
@@ -52,6 +52,9 @@ pub struct Response {
|
||||
pub date: Option<Date>,
|
||||
/// Which form the student took, when forms were used.
|
||||
pub form: Option<String>,
|
||||
/// Where this question sat on the student's own form, when that differs from
|
||||
/// the recorded number. Provenance, not a join key.
|
||||
pub form_position: Option<u32>,
|
||||
|
||||
/// The identifier analysis groups by. A pseudonym when pseudonymizing.
|
||||
pub student_key: String,
|
||||
@@ -74,8 +77,14 @@ pub struct Response {
|
||||
|
||||
/// Option letters the student chose.
|
||||
pub selected: Vec<String>,
|
||||
/// The selected options in the bank's own lettering, written at ingest by
|
||||
/// [`crate::decode::apply`]. Empty when the form was never decoded, which is
|
||||
/// the case for data ingested before seals existed.
|
||||
pub selected_source: Vec<String>,
|
||||
/// Option letters the student eliminated, for elimination-scored items.
|
||||
pub eliminated: Vec<String>,
|
||||
/// The eliminated options in the bank's lettering.
|
||||
pub eliminated_source: Vec<String>,
|
||||
/// Whether the response earned full credit. `None` when it cannot be
|
||||
/// determined, e.g. a blank response on an item with no recorded key.
|
||||
pub correct: Option<bool>,
|
||||
@@ -132,6 +141,19 @@ impl Response {
|
||||
pub fn selected_joined(&self) -> String {
|
||||
self.selected.join(",")
|
||||
}
|
||||
|
||||
/// The choices, in the bank's lettering when it is known.
|
||||
///
|
||||
/// `selected` is what the student marked on the page they held; this is what
|
||||
/// they chose. On an unshuffled form the two agree, which is why reading the
|
||||
/// wrong one is a bug that only appears once you add a second form.
|
||||
pub fn chosen(&self) -> &[String] {
|
||||
if self.selected_source.is_empty() {
|
||||
&self.selected
|
||||
} else {
|
||||
&self.selected_source
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A set of responses plus anything worth telling the user about the ingest.
|
||||
@@ -393,12 +415,23 @@ impl ResponseSet {
|
||||
|
||||
// Apply the record's credit overrides, which is how a decision to
|
||||
// award partial credit after the fact becomes visible in analysis.
|
||||
if !p.credit_overrides.is_empty() && r.selected.len() == 1 {
|
||||
if let Some(over) = p.credit_overrides.get(&r.selected[0]) {
|
||||
if (*over - r.credit).abs() > 1e-9 {
|
||||
r.credit = *over;
|
||||
r.score = *over * r.points_possible;
|
||||
r.correct = Some(*over >= 0.999);
|
||||
//
|
||||
// The override map is keyed in the bank's letters, so the lookup has
|
||||
// to use `chosen()` rather than `selected`. Cloning the letter first
|
||||
// ends the borrow of `r` before the row is written to.
|
||||
let chosen = if r.chosen().len() == 1 {
|
||||
r.chosen().first().cloned()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if !p.credit_overrides.is_empty() {
|
||||
if let Some(letter) = chosen {
|
||||
if let Some(over) = p.credit_overrides.get(&letter) {
|
||||
if (*over - r.credit).abs() > 1e-9 {
|
||||
r.credit = *over;
|
||||
r.score = *over * r.points_possible;
|
||||
r.correct = Some(*over >= 0.999);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -590,6 +623,19 @@ pub struct FlatResponse {
|
||||
pub bonus: bool,
|
||||
/// Whether the item was dropped.
|
||||
pub dropped: bool,
|
||||
|
||||
// Appended rather than interleaved: the columns above are the order every
|
||||
// CSV written so far uses, and `#[serde(default)]` is what keeps one written
|
||||
// last term readable now that three more exist.
|
||||
/// The printed position, 0 when unknown.
|
||||
#[serde(default)]
|
||||
pub form_position: u32,
|
||||
/// Comma-joined selected letters in the bank's lettering.
|
||||
#[serde(default)]
|
||||
pub selected_source: String,
|
||||
/// Comma-joined eliminated letters in the bank's lettering.
|
||||
#[serde(default)]
|
||||
pub eliminated_source: String,
|
||||
}
|
||||
|
||||
impl FlatResponse {
|
||||
@@ -610,6 +656,7 @@ impl FlatResponse {
|
||||
assessment_id: r.assessment_id.clone(),
|
||||
date: r.date.map(|d| d.to_string()).unwrap_or_default(),
|
||||
form: r.form.clone().unwrap_or_default(),
|
||||
form_position: r.form_position.unwrap_or(0),
|
||||
student_key: r.student_key.clone(),
|
||||
sid: r.sid.clone().unwrap_or_default(),
|
||||
email: r.email.clone().unwrap_or_default(),
|
||||
@@ -618,7 +665,9 @@ impl FlatResponse {
|
||||
item_ref: r.item_ref.clone().unwrap_or_default(),
|
||||
item_version: r.item_version.unwrap_or(0),
|
||||
selected: r.selected.join(","),
|
||||
selected_source: r.selected_source.join(","),
|
||||
eliminated: r.eliminated.join(","),
|
||||
eliminated_source: r.eliminated_source.join(","),
|
||||
correct: match r.correct {
|
||||
Some(true) => "1".to_string(),
|
||||
Some(false) => "0".to_string(),
|
||||
@@ -660,6 +709,11 @@ impl FlatResponse {
|
||||
assessment_id: self.assessment_id.clone(),
|
||||
date: self.date.parse().ok(),
|
||||
form: none_if_empty(&self.form),
|
||||
form_position: if self.form_position == 0 {
|
||||
None
|
||||
} else {
|
||||
Some(self.form_position)
|
||||
},
|
||||
student_key: self.student_key.clone(),
|
||||
sid: none_if_empty(&self.sid),
|
||||
name: None,
|
||||
@@ -673,7 +727,9 @@ impl FlatResponse {
|
||||
Some(self.item_version)
|
||||
},
|
||||
selected: split(&self.selected),
|
||||
selected_source: split(&self.selected_source),
|
||||
eliminated: split(&self.eliminated),
|
||||
eliminated_source: split(&self.eliminated_source),
|
||||
correct: match self.correct.as_str() {
|
||||
"1" | "true" => Some(true),
|
||||
"0" | "false" => Some(false),
|
||||
@@ -713,6 +769,7 @@ mod tests {
|
||||
assessment_id: "e1".into(),
|
||||
date: None,
|
||||
form: None,
|
||||
form_position: None,
|
||||
student_key: student.into(),
|
||||
sid: Some(format!("sid-{student}")),
|
||||
name: None,
|
||||
@@ -722,7 +779,9 @@ mod tests {
|
||||
item_ref: None,
|
||||
item_version: None,
|
||||
selected: vec!["A".into()],
|
||||
selected_source: vec![],
|
||||
eliminated: vec![],
|
||||
eliminated_source: vec![],
|
||||
correct: Some(credit >= 0.999),
|
||||
credit,
|
||||
points_possible: 2.0,
|
||||
|
||||
@@ -541,6 +541,7 @@ mod tests {
|
||||
assessment_id: "exam-4".into(),
|
||||
date: None,
|
||||
form: None,
|
||||
form_position: None,
|
||||
student_key: student.into(),
|
||||
sid: None,
|
||||
name: None,
|
||||
@@ -550,7 +551,9 @@ mod tests {
|
||||
item_ref: Some("bank::q-x-001".into()),
|
||||
item_version: Some(2),
|
||||
selected: vec!["C".into()],
|
||||
selected_source: vec![],
|
||||
eliminated: vec![],
|
||||
eliminated_source: vec![],
|
||||
correct: Some(true),
|
||||
credit: 1.0,
|
||||
points_possible: 1.5,
|
||||
|
||||
@@ -61,6 +61,9 @@ pub fn schema() -> Schema {
|
||||
Field::new("topics", DataType::Utf8, false),
|
||||
Field::new("bonus", DataType::Boolean, false),
|
||||
Field::new("dropped", DataType::Boolean, false),
|
||||
Field::new("form_position", DataType::UInt32, false),
|
||||
Field::new("selected_source", DataType::Utf8, false),
|
||||
Field::new("eliminated_source", DataType::Utf8, false),
|
||||
])
|
||||
}
|
||||
|
||||
@@ -120,6 +123,9 @@ fn to_batch(rows: &[FlatResponse]) -> Result<RecordBatch> {
|
||||
s(|r| &r.topics),
|
||||
boolc(|r| r.bonus),
|
||||
boolc(|r| r.dropped),
|
||||
u32c(|r| r.form_position),
|
||||
s(|r| &r.selected_source),
|
||||
s(|r| &r.eliminated_source),
|
||||
];
|
||||
|
||||
RecordBatch::try_new(Arc::new(schema()), columns).map_err(|e| {
|
||||
@@ -237,6 +243,20 @@ fn from_batch(batch: &RecordBatch, path: &Path) -> Result<Vec<FlatResponse>> {
|
||||
.ok_or_else(|| column_error(name, "boolean", path))
|
||||
};
|
||||
|
||||
// The three decode columns are read optionally rather than required, so a
|
||||
// file written before seals existed still loads. A missing column is not a
|
||||
// foreign file; it is last term's data.
|
||||
let optional_strings = |name: &str| -> Option<&StringArray> {
|
||||
batch
|
||||
.column_by_name(name)
|
||||
.and_then(|c| c.as_any().downcast_ref::<StringArray>())
|
||||
};
|
||||
let optional_uints = |name: &str| -> Option<&UInt32Array> {
|
||||
batch
|
||||
.column_by_name(name)
|
||||
.and_then(|c| c.as_any().downcast_ref::<UInt32Array>())
|
||||
};
|
||||
|
||||
let administration_id = strings("administration_id")?;
|
||||
let course = strings("course")?;
|
||||
let term = strings("term")?;
|
||||
@@ -263,6 +283,10 @@ fn from_batch(batch: &RecordBatch, path: &Path) -> Result<Vec<FlatResponse>> {
|
||||
let bonus = bools("bonus")?;
|
||||
let dropped = bools("dropped")?;
|
||||
|
||||
let form_position = optional_uints("form_position");
|
||||
let selected_source = optional_strings("selected_source");
|
||||
let eliminated_source = optional_strings("eliminated_source");
|
||||
|
||||
let mut out = Vec::with_capacity(batch.num_rows());
|
||||
for i in 0..batch.num_rows() {
|
||||
out.push(FlatResponse {
|
||||
@@ -272,6 +296,7 @@ fn from_batch(batch: &RecordBatch, path: &Path) -> Result<Vec<FlatResponse>> {
|
||||
assessment_id: assessment_id.value(i).to_string(),
|
||||
date: date.value(i).to_string(),
|
||||
form: form.value(i).to_string(),
|
||||
form_position: form_position.map(|a| a.value(i)).unwrap_or(0),
|
||||
student_key: student_key.value(i).to_string(),
|
||||
sid: sid.value(i).to_string(),
|
||||
email: email.value(i).to_string(),
|
||||
@@ -280,7 +305,13 @@ fn from_batch(batch: &RecordBatch, path: &Path) -> Result<Vec<FlatResponse>> {
|
||||
item_ref: item_ref.value(i).to_string(),
|
||||
item_version: item_version.value(i),
|
||||
selected: selected.value(i).to_string(),
|
||||
selected_source: selected_source
|
||||
.map(|a| a.value(i).to_string())
|
||||
.unwrap_or_default(),
|
||||
eliminated: eliminated.value(i).to_string(),
|
||||
eliminated_source: eliminated_source
|
||||
.map(|a| a.value(i).to_string())
|
||||
.unwrap_or_default(),
|
||||
correct: correct.value(i).to_string(),
|
||||
credit: credit.value(i),
|
||||
points_possible: points_possible.value(i),
|
||||
@@ -336,6 +367,9 @@ mod tests {
|
||||
topics: "kinetics".into(),
|
||||
bonus: false,
|
||||
dropped: false,
|
||||
form_position: 0,
|
||||
selected_source: String::new(),
|
||||
eliminated_source: String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user