849 lines
29 KiB
Rust
849 lines
29 KiB
Rust
// SPDX-License-Identifier: Prosperity-3.0.0
|
|
// Copyright Scientific Computing Studio
|
|
// Source: https://git.scient.ing/education/coursebank
|
|
|
|
//! Turning what a student marked into what a student chose.
|
|
//!
|
|
//! A grading export speaks in positions and printed letters. Question 14 is the
|
|
//! fourteenth thing on the page; option C is the third bubble. An item bank speaks
|
|
//! in ids and its own lettering. When forms shuffle, those two vocabularies
|
|
//! disagree, and every analysis downstream of the disagreement is wrong in a way
|
|
//! that looks right:
|
|
//!
|
|
//! * Pooled distractor statistics add form A's option C to form B's option C,
|
|
//! which are different sentences. The resulting table is noise with the shape of
|
|
//! data.
|
|
//! * A student report looks up the misconception recorded on option C and shows it
|
|
//! to a student who chose a different option. The feedback is confident,
|
|
//! specific, and about the wrong thing.
|
|
//! * Any `credit_overrides` written in the record's lettering are applied to
|
|
//! whoever happened to mark that letter on their form.
|
|
//!
|
|
//! None of these fail loudly. That is the argument for doing the translation once,
|
|
//! at ingest, and storing both sides of it.
|
|
//!
|
|
//! # What a decoder knows
|
|
//!
|
|
//! For one form: which recorded question number sits at each printed position,
|
|
//! which bank letter each printed letter stands for, and which printed letters are
|
|
//! keyed. It is built from a [`SealFile`] when one exists, and derived from the
|
|
//! record and the form seed when one does not. Sealed is better, and not only
|
|
//! because it is faster: a derived decoder describes the form the bank *would*
|
|
//! print today, while a sealed one describes the form that was actually printed.
|
|
//!
|
|
//! # Catching a swapped directory
|
|
//!
|
|
//! Gradescope's point-value row reveals which printed letter earned full credit on
|
|
//! every question. A decoder knows what that letter should be. Comparing them
|
|
//! across a whole directory is close to a proof of which form the directory holds:
|
|
//! agreement is near total for the right form and near chance for the wrong one.
|
|
//! [`identify_form`] uses that to refuse an ingest that names form A over a
|
|
//! directory of form B papers, which is otherwise a mistake nobody catches until
|
|
//! the item statistics look strange three weeks later.
|
|
|
|
use std::collections::{BTreeMap, BTreeSet};
|
|
|
|
use crate::assessment::{AssessmentFile, Form};
|
|
use crate::catalog::Catalog;
|
|
use crate::error::{Error, Result};
|
|
use crate::gradescope::Question;
|
|
use crate::responses::ResponseSet;
|
|
use crate::seal::{SealFile, printed_letter};
|
|
use crate::select;
|
|
|
|
/// Where a decoder's mapping came from.
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub enum Provenance {
|
|
/// Read from a seal written before administration. Authoritative.
|
|
Seal,
|
|
/// Derived from the assessment record and the form seed, as of now.
|
|
Derived,
|
|
}
|
|
|
|
impl Provenance {
|
|
/// A short label for output.
|
|
pub fn label(self) -> &'static str {
|
|
match self {
|
|
Provenance::Seal => "seal",
|
|
Provenance::Derived => "derived from the record",
|
|
}
|
|
}
|
|
}
|
|
|
|
/// How a grading export numbers its questions.
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub enum Numbering {
|
|
/// The export counts printed positions, which is what Gradescope's `N.csv`
|
|
/// file names mean. Positions are translated to recorded numbers.
|
|
Printed,
|
|
/// The export already carries recorded question numbers, so numbers pass
|
|
/// through untouched.
|
|
Recorded,
|
|
}
|
|
|
|
/// The form marker put on a row that could not be translated, so it can be
|
|
/// removed after the borrow on `set.rows` ends. No real form id can collide
|
|
/// with it: form ids come from the record and are short labels like `A`.
|
|
const UNMAPPED: &str = "\u{1f}unmapped";
|
|
|
|
/// One question's mapping on one form.
|
|
#[derive(Debug, Clone)]
|
|
pub struct QuestionMap {
|
|
/// Printed position on the page, counting from 1.
|
|
pub position: u32,
|
|
/// The recorded question number, the join key to the record and the store.
|
|
pub number: u32,
|
|
/// The item's global id.
|
|
pub item: String,
|
|
/// Keyed letters as printed on this form.
|
|
pub printed_key: Vec<String>,
|
|
/// Keyed letters in the bank's own lettering.
|
|
pub canonical_key: Vec<String>,
|
|
/// Printed letter to bank letter.
|
|
pub to_canonical: BTreeMap<String, String>,
|
|
/// Bank letter to printed letter.
|
|
pub to_printed: BTreeMap<String, String>,
|
|
}
|
|
|
|
impl QuestionMap {
|
|
/// The bank letter a printed letter stands for.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `printed` - the letter as the student saw it.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The bank letter, or `None` when the printed letter is not one of this
|
|
/// question's options.
|
|
pub fn canonical(&self, printed: &str) -> Option<&str> {
|
|
self.to_canonical
|
|
.get(&printed.trim().to_ascii_uppercase())
|
|
.map(|s| s.as_str())
|
|
}
|
|
|
|
/// How many options this question has.
|
|
pub fn n_options(&self) -> usize {
|
|
self.to_canonical.len()
|
|
}
|
|
|
|
/// Whether this question's options were actually permuted.
|
|
pub fn is_permuted(&self) -> bool {
|
|
self.to_canonical.iter().any(|(k, v)| k != v)
|
|
}
|
|
}
|
|
|
|
/// One form's full mapping.
|
|
#[derive(Debug, Clone)]
|
|
pub struct FormDecoder {
|
|
/// The form id.
|
|
pub form: String,
|
|
/// Where the mapping came from.
|
|
pub provenance: Provenance,
|
|
/// Questions by printed position.
|
|
by_position: BTreeMap<u32, QuestionMap>,
|
|
/// Questions by recorded number.
|
|
by_number: BTreeMap<u32, QuestionMap>,
|
|
}
|
|
|
|
impl FormDecoder {
|
|
/// Builds a decoder from the question maps.
|
|
fn assemble(form: String, provenance: Provenance, maps: Vec<QuestionMap>) -> FormDecoder {
|
|
let by_position = maps.iter().map(|m| (m.position, m.clone())).collect();
|
|
let by_number = maps.into_iter().map(|m| (m.number, m)).collect();
|
|
FormDecoder {
|
|
form,
|
|
provenance,
|
|
by_position,
|
|
by_number,
|
|
}
|
|
}
|
|
|
|
/// Reads one form's mapping out of a seal.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `seal` - the seal.
|
|
/// * `form_id` - the form to decode, matched case-insensitively.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The decoder.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns [`Error::Usage`] when the seal does not cover that form.
|
|
pub fn from_seal(seal: &SealFile, form_id: &str) -> Result<FormDecoder> {
|
|
let form = seal.form(form_id).ok_or_else(|| {
|
|
Error::usage(format!(
|
|
"the seal for `{}` does not cover form `{form_id}`; it covers {}",
|
|
seal.seal.assessment,
|
|
seal.forms
|
|
.iter()
|
|
.map(|f| f.id.as_str())
|
|
.collect::<Vec<_>>()
|
|
.join(", ")
|
|
))
|
|
})?;
|
|
|
|
let maps = form
|
|
.questions
|
|
.iter()
|
|
.map(|q| {
|
|
let mut to_canonical = BTreeMap::new();
|
|
let mut to_printed = BTreeMap::new();
|
|
for map in &q.options {
|
|
to_canonical.insert(map.printed.clone(), map.canonical.clone());
|
|
to_printed.insert(map.canonical.clone(), map.printed.clone());
|
|
}
|
|
let canonical_key: Vec<String> = q
|
|
.printed_key
|
|
.iter()
|
|
.filter_map(|p| to_canonical.get(p).cloned())
|
|
.collect();
|
|
QuestionMap {
|
|
position: q.position,
|
|
number: q.number,
|
|
item: q.item.clone(),
|
|
printed_key: q.printed_key.clone(),
|
|
canonical_key,
|
|
to_canonical,
|
|
to_printed,
|
|
}
|
|
})
|
|
.collect();
|
|
|
|
Ok(FormDecoder::assemble(
|
|
form.id.clone(),
|
|
Provenance::Seal,
|
|
maps,
|
|
))
|
|
}
|
|
|
|
/// Derives one form's mapping from the record and the bank.
|
|
///
|
|
/// Uses the same two functions every export calls, so a derived decoder and a
|
|
/// freshly exported paper agree by construction.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `catalog` - the loaded course.
|
|
/// * `record` - the assessment record.
|
|
/// * `form` - the form.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The decoder.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns [`Error::Unresolved`] when a placement references a missing item.
|
|
pub fn derive(catalog: &Catalog, record: &AssessmentFile, form: &Form) -> Result<FormDecoder> {
|
|
let printed: Vec<_> = select::layout(record, form)
|
|
.into_iter()
|
|
.filter(|p| !p.dropped)
|
|
.collect();
|
|
|
|
let mut maps = Vec::with_capacity(printed.len());
|
|
for (index, placement) in printed.iter().enumerate() {
|
|
let entry = catalog.require(&placement.item)?;
|
|
let item = &entry.item;
|
|
let shown = item.administered(&placement.key, &placement.distractors);
|
|
let order = select::option_order(form, &placement.item, shown.len());
|
|
|
|
let canonical_key: BTreeSet<String> = if placement.key.is_empty() {
|
|
item.key_letters().into_iter().collect()
|
|
} else {
|
|
placement.key.iter().cloned().collect()
|
|
};
|
|
|
|
let mut to_canonical = BTreeMap::new();
|
|
let mut to_printed = BTreeMap::new();
|
|
let mut printed_key = Vec::new();
|
|
for (position, source_index) in order.iter().enumerate() {
|
|
let canonical = shown
|
|
.get(*source_index)
|
|
.map(|c| c.id.clone())
|
|
.unwrap_or_else(|| printed_letter(*source_index));
|
|
let label = printed_letter(position);
|
|
if canonical_key.contains(&canonical) {
|
|
printed_key.push(label.clone());
|
|
}
|
|
to_canonical.insert(label.clone(), canonical.clone());
|
|
to_printed.insert(canonical, label);
|
|
}
|
|
|
|
maps.push(QuestionMap {
|
|
position: index as u32 + 1,
|
|
number: placement.number,
|
|
item: placement.item.clone(),
|
|
printed_key,
|
|
canonical_key: canonical_key.into_iter().collect(),
|
|
to_canonical,
|
|
to_printed,
|
|
});
|
|
}
|
|
|
|
Ok(FormDecoder::assemble(
|
|
form.id.clone(),
|
|
Provenance::Derived,
|
|
maps,
|
|
))
|
|
}
|
|
|
|
/// Builds a decoder, preferring the seal.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `seal` - the seal, when one has been written.
|
|
/// * `catalog` - the loaded course.
|
|
/// * `record` - the assessment record.
|
|
/// * `form` - the form.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The decoder.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// As [`FormDecoder::from_seal`] and [`FormDecoder::derive`]. A seal that does
|
|
/// not cover the requested form falls back to deriving rather than failing,
|
|
/// since a form added after sealing is a real situation.
|
|
pub fn resolve(
|
|
seal: Option<&SealFile>,
|
|
catalog: &Catalog,
|
|
record: &AssessmentFile,
|
|
form: &Form,
|
|
) -> Result<FormDecoder> {
|
|
if let Some(seal) = seal {
|
|
if seal.form(&form.id).is_some() {
|
|
return FormDecoder::from_seal(seal, &form.id);
|
|
}
|
|
}
|
|
FormDecoder::derive(catalog, record, form)
|
|
}
|
|
|
|
/// The question at a printed position.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `position` - the printed position, counting from 1.
|
|
pub fn at_position(&self, position: u32) -> Option<&QuestionMap> {
|
|
self.by_position.get(&position)
|
|
}
|
|
|
|
/// The question with a recorded number.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `number` - the recorded number.
|
|
pub fn at_number(&self, number: u32) -> Option<&QuestionMap> {
|
|
self.by_number.get(&number)
|
|
}
|
|
|
|
/// The question an export's numbering refers to.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `n` - the number as the export gives it.
|
|
/// * `numbering` - how the export numbers questions.
|
|
pub fn lookup(&self, n: u32, numbering: Numbering) -> Option<&QuestionMap> {
|
|
match numbering {
|
|
Numbering::Printed => self.at_position(n),
|
|
Numbering::Recorded => self.at_number(n),
|
|
}
|
|
}
|
|
|
|
/// How many questions this form prints.
|
|
pub fn len(&self) -> usize {
|
|
self.by_position.len()
|
|
}
|
|
|
|
/// Every recorded question number this form carries.
|
|
pub fn numbers(&self) -> impl Iterator<Item = u32> + '_ {
|
|
self.by_position.values().map(|q| q.number)
|
|
}
|
|
|
|
/// Whether the form prints nothing, which means the record is empty.
|
|
pub fn is_empty(&self) -> bool {
|
|
self.by_position.is_empty()
|
|
}
|
|
|
|
/// Whether any question on this form has permuted options.
|
|
///
|
|
/// Used to decide whether to say anything about translation at all: on an
|
|
/// unshuffled form the whole mechanism is an identity map and mentioning it
|
|
/// is noise.
|
|
pub fn is_permuted(&self) -> bool {
|
|
self.by_position.values().any(|q| q.is_permuted())
|
|
}
|
|
|
|
/// Whether printed positions and recorded numbers disagree anywhere.
|
|
///
|
|
/// True when items were shuffled, and also when a bonus item sits mid-record,
|
|
/// since the layout moves bonus items to the end of the paper.
|
|
pub fn is_renumbered(&self) -> bool {
|
|
self.by_position.values().any(|q| q.position != q.number)
|
|
}
|
|
}
|
|
|
|
/// What a directory of graded questions says about which form it holds.
|
|
#[derive(Debug, Clone)]
|
|
pub struct FormFit {
|
|
/// The form id.
|
|
pub form: String,
|
|
/// Questions whose graded key matched this form's printed key.
|
|
pub matched: usize,
|
|
/// Questions that could be compared at all.
|
|
pub compared: usize,
|
|
/// Question positions where the graded key disagreed.
|
|
pub mismatches: Vec<u32>,
|
|
}
|
|
|
|
impl FormFit {
|
|
/// The share of comparable questions that agreed.
|
|
pub fn rate(&self) -> f64 {
|
|
if self.compared == 0 {
|
|
0.0
|
|
} else {
|
|
self.matched as f64 / self.compared as f64
|
|
}
|
|
}
|
|
|
|
/// Whether the fit is good enough to proceed without a warning.
|
|
///
|
|
/// The threshold is high on purpose. A correctly matched directory agrees on
|
|
/// every question; anything less than total agreement is either a regrade that
|
|
/// moved a key or the wrong directory, and both are worth a sentence.
|
|
pub fn is_convincing(&self) -> bool {
|
|
self.compared > 0 && self.matched == self.compared
|
|
}
|
|
}
|
|
|
|
/// Compares a parsed Gradescope directory against one form's expected keys.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `questions` - the parsed question files.
|
|
/// * `decoder` - the form to test against.
|
|
/// * `numbering` - how the export numbers questions.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The fit.
|
|
pub fn fit_form(questions: &[Question], decoder: &FormDecoder, numbering: Numbering) -> FormFit {
|
|
let mut matched = 0usize;
|
|
let mut compared = 0usize;
|
|
let mut mismatches = Vec::new();
|
|
|
|
for question in questions {
|
|
let Some(map) = decoder.lookup(question.number, numbering) else {
|
|
continue;
|
|
};
|
|
let graded: BTreeSet<String> = question.keyed().into_iter().collect();
|
|
if graded.is_empty() {
|
|
continue;
|
|
}
|
|
let expected: BTreeSet<String> = map.printed_key.iter().cloned().collect();
|
|
if expected.is_empty() {
|
|
continue;
|
|
}
|
|
compared += 1;
|
|
if graded == expected {
|
|
matched += 1;
|
|
} else {
|
|
mismatches.push(question.number);
|
|
}
|
|
}
|
|
|
|
FormFit {
|
|
form: decoder.form.clone(),
|
|
matched,
|
|
compared,
|
|
mismatches,
|
|
}
|
|
}
|
|
|
|
/// Ranks every candidate form against a directory.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `questions` - the parsed question files.
|
|
/// * `decoders` - one decoder per declared form.
|
|
/// * `numbering` - how the export numbers questions.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The fits, best first.
|
|
pub fn identify_form(
|
|
questions: &[Question],
|
|
decoders: &[FormDecoder],
|
|
numbering: Numbering,
|
|
) -> Vec<FormFit> {
|
|
let mut fits: Vec<FormFit> = decoders
|
|
.iter()
|
|
.map(|d| fit_form(questions, d, numbering))
|
|
.collect();
|
|
fits.sort_by(|a, b| {
|
|
b.rate()
|
|
.partial_cmp(&a.rate())
|
|
.unwrap_or(std::cmp::Ordering::Equal)
|
|
.then_with(|| a.form.cmp(&b.form))
|
|
});
|
|
fits
|
|
}
|
|
|
|
/// Explains a fit in a sentence, or says nothing when the fit is perfect.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `claimed` - the form the directory was ingested as.
|
|
/// * `fits` - every form's fit, best first.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// A warning, or `None`.
|
|
pub fn form_warning(claimed: &str, fits: &[FormFit]) -> Option<String> {
|
|
let mine = fits.iter().find(|f| f.form.eq_ignore_ascii_case(claimed))?;
|
|
if mine.is_convincing() {
|
|
return None;
|
|
}
|
|
if mine.compared == 0 {
|
|
return Some(format!(
|
|
"form {claimed}: the export carries no point values, so the graded keys could not be \
|
|
checked against the form. Nothing verified this directory is form {claimed}"
|
|
));
|
|
}
|
|
|
|
let better = fits
|
|
.iter()
|
|
.find(|f| !f.form.eq_ignore_ascii_case(claimed) && f.rate() > mine.rate());
|
|
|
|
let head = format!(
|
|
"form {claimed}: the graded key matches this form on {} of {} question(s)",
|
|
mine.matched, mine.compared
|
|
);
|
|
let where_ = if mine.mismatches.is_empty() {
|
|
String::new()
|
|
} else {
|
|
let list: Vec<String> = mine
|
|
.mismatches
|
|
.iter()
|
|
.take(8)
|
|
.map(|n| n.to_string())
|
|
.collect();
|
|
format!(
|
|
" (q{}{})",
|
|
list.join(", q"),
|
|
if mine.mismatches.len() > 8 {
|
|
", …"
|
|
} else {
|
|
""
|
|
}
|
|
)
|
|
};
|
|
match better {
|
|
Some(other) => Some(format!(
|
|
"{head}{where_}, but matches form {} on {} of {}. This directory is almost certainly \
|
|
form {}, not form {claimed}",
|
|
other.form, other.matched, other.compared, other.form
|
|
)),
|
|
None => Some(format!(
|
|
"{head}{where_}. Either those questions were regraded after printing, or the form is \
|
|
not the one named"
|
|
)),
|
|
}
|
|
}
|
|
|
|
/// Translates a form's responses into the bank's vocabulary.
|
|
///
|
|
/// Rewrites, for every row whose `form` matches this decoder:
|
|
///
|
|
/// * `item_number`, from printed position to recorded number, when the export
|
|
/// numbers by position;
|
|
/// * `form_position`, recording where the question sat on the page;
|
|
/// * `selected_source` and `eliminated_source`, the bank letters for what was
|
|
/// marked. The printed letters stay in `selected` and `eliminated`, because what
|
|
/// a student physically marked is the fact and the translation is the
|
|
/// interpretation.
|
|
///
|
|
/// Correctness and credit are untouched. Both come from the grading platform,
|
|
/// which scored the paper the student actually held, and are already right.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `set` - the responses to translate, in place.
|
|
/// * `decoder` - the form's mapping.
|
|
/// * `numbering` - how the export numbered questions.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// Warnings for anything that could not be translated.
|
|
pub fn apply(set: &mut ResponseSet, decoder: &FormDecoder, numbering: Numbering) -> Vec<String> {
|
|
let mut warnings = Vec::new();
|
|
let mut unmapped_positions: BTreeSet<u32> = BTreeSet::new();
|
|
let mut colliding_positions: BTreeSet<u32> = BTreeSet::new();
|
|
let mut unmapped_letters: BTreeSet<String> = BTreeSet::new();
|
|
let mut translated = 0usize;
|
|
|
|
// Numbers this form really uses. An untranslated row whose raw number is one
|
|
// of these would silently masquerade as that question, and two rows would
|
|
// then share a number: one the student's answer to it, one an answer to
|
|
// something else entirely. Nothing downstream can tell them apart, so the
|
|
// collision has to be caught here.
|
|
let recorded: BTreeSet<u32> = decoder.numbers().collect();
|
|
|
|
for row in &mut set.rows {
|
|
let belongs = row
|
|
.form
|
|
.as_deref()
|
|
.map(|f| f.eq_ignore_ascii_case(&decoder.form))
|
|
.unwrap_or(false);
|
|
if !belongs {
|
|
continue;
|
|
}
|
|
|
|
let Some(map) = decoder.lookup(row.item_number, numbering) else {
|
|
unmapped_positions.insert(row.item_number);
|
|
if recorded.contains(&row.item_number) {
|
|
colliding_positions.insert(row.item_number);
|
|
// Marked so the row can be discarded below. Attributing it to
|
|
// the question that legitimately holds this number would corrupt
|
|
// that question's statistics.
|
|
row.form = Some(UNMAPPED.to_string());
|
|
}
|
|
continue;
|
|
};
|
|
|
|
row.form_position = Some(map.position);
|
|
row.item_number = map.number;
|
|
|
|
let mut convert = |letters: &[String]| -> Vec<String> {
|
|
let mut out = Vec::with_capacity(letters.len());
|
|
for letter in letters {
|
|
match map.canonical(letter) {
|
|
Some(canonical) => out.push(canonical.to_string()),
|
|
None => {
|
|
unmapped_letters.insert(format!("q{} {}", map.number, letter));
|
|
}
|
|
}
|
|
}
|
|
out.sort();
|
|
out
|
|
};
|
|
|
|
row.selected_source = convert(&row.selected);
|
|
row.eliminated_source = convert(&row.eliminated);
|
|
translated += 1;
|
|
}
|
|
|
|
if !unmapped_positions.is_empty() {
|
|
let list: Vec<String> = unmapped_positions.iter().map(|n| n.to_string()).collect();
|
|
warnings.push(format!(
|
|
"form {}: question(s) {} are in the export but not on this form ({} printed). The \
|
|
export may have been taken before a question was dropped, or from a different \
|
|
form.",
|
|
decoder.form,
|
|
list.join(", "),
|
|
decoder.len()
|
|
));
|
|
}
|
|
if !colliding_positions.is_empty() {
|
|
let list: Vec<String> = colliding_positions.iter().map(|n| n.to_string()).collect();
|
|
let discarded = set.rows.len();
|
|
set.rows.retain(|row| row.form.as_deref() != Some(UNMAPPED));
|
|
warnings.push(format!(
|
|
"form {}: {} response(s) at position(s) {} could not be translated, and their raw \
|
|
numbers are numbers this form does use. Keeping them would have given those \
|
|
questions two different answers each, so they were discarded. This is the shape of \
|
|
an export made before a question was dropped: re-export the responses from the \
|
|
administration you sealed, or re-run with --recorded-numbers if the export already \
|
|
carries recorded numbers.",
|
|
decoder.form,
|
|
discarded - set.rows.len(),
|
|
list.join(", ")
|
|
));
|
|
}
|
|
if !unmapped_letters.is_empty() {
|
|
let list: Vec<String> = unmapped_letters.iter().take(10).cloned().collect();
|
|
warnings.push(format!(
|
|
"form {}: {} marked option(s) are not options on the printed form ({}), which usually \
|
|
means a rubric column was added by hand in Gradescope",
|
|
decoder.form,
|
|
unmapped_letters.len(),
|
|
list.join(", ")
|
|
));
|
|
}
|
|
if translated == 0 {
|
|
warnings.push(format!(
|
|
"form {}: no response rows carried this form id, so nothing was translated",
|
|
decoder.form
|
|
));
|
|
}
|
|
|
|
warnings
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::responses::{Response, administration_id};
|
|
|
|
fn map(position: u32, number: u32, pairs: &[(&str, &str)], key: &str) -> QuestionMap {
|
|
let mut to_canonical = BTreeMap::new();
|
|
let mut to_printed = BTreeMap::new();
|
|
for (printed, canonical) in pairs {
|
|
to_canonical.insert(printed.to_string(), canonical.to_string());
|
|
to_printed.insert(canonical.to_string(), printed.to_string());
|
|
}
|
|
let printed_key = vec![to_printed.get(key).cloned().unwrap_or_default()];
|
|
QuestionMap {
|
|
position,
|
|
number,
|
|
item: format!("b::q-{number}"),
|
|
printed_key,
|
|
canonical_key: vec![key.to_string()],
|
|
to_canonical,
|
|
to_printed,
|
|
}
|
|
}
|
|
|
|
fn decoder() -> FormDecoder {
|
|
FormDecoder::assemble(
|
|
"B".to_string(),
|
|
Provenance::Seal,
|
|
vec![
|
|
// Printed A..D show bank C, A, D, B. The key is bank A, printed B.
|
|
map(1, 1, &[("A", "C"), ("B", "A"), ("C", "D"), ("D", "B")], "A"),
|
|
// A bonus item recorded as 9 but printed last, at position 2.
|
|
map(2, 9, &[("A", "B"), ("B", "A")], "B"),
|
|
],
|
|
)
|
|
}
|
|
|
|
fn row(number: u32, selected: &str) -> Response {
|
|
Response {
|
|
administration_id: administration_id("C", "2026f", "e1"),
|
|
course: "C".into(),
|
|
term: "2026f".into(),
|
|
assessment_id: "e1".into(),
|
|
date: None,
|
|
form: Some("B".into()),
|
|
student_key: "s1".into(),
|
|
sid: None,
|
|
name: None,
|
|
email: None,
|
|
section: None,
|
|
item_number: number,
|
|
form_position: None,
|
|
item_ref: None,
|
|
item_version: None,
|
|
variant: None,
|
|
selected: vec![selected.into()],
|
|
eliminated: Vec::new(),
|
|
selected_source: Vec::new(),
|
|
eliminated_source: Vec::new(),
|
|
correct: Some(false),
|
|
credit: 0.0,
|
|
points_possible: 1.0,
|
|
score: 0.0,
|
|
response_time_seconds: None,
|
|
level: None,
|
|
learning_targets: Vec::new(),
|
|
topics: Vec::new(),
|
|
bonus: false,
|
|
dropped: false,
|
|
dropped_full_credit: false,
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn printed_letters_become_bank_letters() {
|
|
let decoder = decoder();
|
|
let q = decoder.at_position(1).unwrap();
|
|
assert_eq!(q.canonical("A"), Some("C"));
|
|
assert_eq!(q.canonical("D"), Some("B"));
|
|
assert_eq!(q.canonical("E"), None);
|
|
assert_eq!(q.printed_key, vec!["B".to_string()]);
|
|
}
|
|
|
|
#[test]
|
|
fn positions_become_recorded_numbers() {
|
|
let mut set = ResponseSet::new();
|
|
set.rows.push(row(2, "A"));
|
|
let warnings = apply(&mut set, &decoder(), Numbering::Printed);
|
|
assert!(warnings.is_empty(), "{warnings:?}");
|
|
assert_eq!(set.rows[0].item_number, 9, "position 2 is recorded as 9");
|
|
assert_eq!(set.rows[0].form_position, Some(2));
|
|
assert_eq!(set.rows[0].selected, vec!["A".to_string()], "printed kept");
|
|
assert_eq!(set.rows[0].selected_source, vec!["B".to_string()]);
|
|
}
|
|
|
|
#[test]
|
|
fn rows_from_another_form_are_left_alone() {
|
|
let mut set = ResponseSet::new();
|
|
let mut other = row(1, "A");
|
|
other.form = Some("A".into());
|
|
set.rows.push(other);
|
|
apply(&mut set, &decoder(), Numbering::Printed);
|
|
assert!(set.rows[0].selected_source.is_empty());
|
|
assert_eq!(set.rows[0].item_number, 1);
|
|
}
|
|
|
|
#[test]
|
|
fn an_unknown_position_is_reported_not_guessed() {
|
|
let mut set = ResponseSet::new();
|
|
set.rows.push(row(7, "A"));
|
|
let warnings = apply(&mut set, &decoder(), Numbering::Printed);
|
|
assert!(
|
|
warnings.iter().any(|w| w.contains("not on this form")),
|
|
"{warnings:?}"
|
|
);
|
|
assert_eq!(set.rows[0].item_number, 7, "left as found");
|
|
}
|
|
|
|
#[test]
|
|
fn a_perfect_fit_says_nothing() {
|
|
let fits = vec![FormFit {
|
|
form: "A".into(),
|
|
matched: 30,
|
|
compared: 30,
|
|
mismatches: Vec::new(),
|
|
}];
|
|
assert!(form_warning("A", &fits).is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn a_swapped_directory_names_the_form_it_really_is() {
|
|
let fits = vec![
|
|
FormFit {
|
|
form: "B".into(),
|
|
matched: 30,
|
|
compared: 30,
|
|
mismatches: Vec::new(),
|
|
},
|
|
FormFit {
|
|
form: "A".into(),
|
|
matched: 8,
|
|
compared: 30,
|
|
mismatches: (1..=22).collect(),
|
|
},
|
|
];
|
|
let warning = form_warning("A", &fits).expect("a mismatch this large must warn");
|
|
assert!(warning.contains("almost certainly form B"), "{warning}");
|
|
}
|
|
|
|
#[test]
|
|
fn a_single_regraded_key_warns_without_accusing_the_wrong_form() {
|
|
let fits = vec![FormFit {
|
|
form: "A".into(),
|
|
matched: 29,
|
|
compared: 30,
|
|
mismatches: vec![14],
|
|
}];
|
|
let warning = form_warning("A", &fits).unwrap();
|
|
assert!(warning.contains("q14"), "{warning}");
|
|
assert!(warning.contains("regraded"), "{warning}");
|
|
}
|
|
}
|