Files
coursebank/src/data/intake.rs
T
alexm 15175f64fd
Pipeline / check (pull_request) Successful in 12m11s
Pipeline / docs (pull_request) Skipped
Pipeline / nightly (pull_request) Skipped
Pipeline / release (pull_request) Skipped
feat: include bonus
2026-11-02 08:22:07 -05:00

560 lines
18 KiB
Rust

// SPDX-License-Identifier: Prosperity-3.0.0
// Copyright Scienting
// Source: https://git.scient.ing/education/coursebank
//! Reading several forms of one exam back in at once.
//!
//! A two-form exam is two Gradescope assignments, each exporting its own
//! directory of `1.csv` through `N.csv`, each numbered against its own paper. They
//! are one administration: one set of students, one item pool, one set of
//! statistics. Ingesting them one command at a time does not work, because the
//! store keys on the administration and the second write replaces the first.
//!
//! So this module takes the whole set:
//!
//! ```text
//! coursebank ingest gradescope A=exports/e1-a B=exports/e1-b --assessment e1
//! ```
//!
//! and does four things the single-directory path cannot:
//!
//! *Checks each directory is the form it claims to be.* Every export carries the
//! graded key in its point-value row; every form knows what its printed key should
//! be. Comparing them catches a swapped pair of directories immediately rather
//! than three weeks later, when the distractor table looks strange. See
//! [`crate::decode::identify_form`].
//!
//! *Translates each form into the bank's vocabulary before merging.* Otherwise
//! form A's option C and form B's option C land in the same column of the same
//! table while meaning different things.
//!
//! *Merges into one response set*, written once, so `analyze` and `report` see the
//! whole class.
//!
//! *Reports what the merge revealed*: a student who appears on two forms, a form
//! that is missing a question the other has, a form that ran materially harder
//! than the other.
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use crate::assessment::{AssessmentFile, Form};
use crate::catalog::Catalog;
use crate::date::Date;
use crate::decode::{self, FormDecoder, FormFit, Numbering};
use crate::error::{Error, Result};
use crate::gradescope::{self, Context, Question};
use crate::responses::ResponseSet;
use crate::seal::SealFile;
/// One directory of graded questions, and the form it holds.
#[derive(Debug, Clone)]
pub struct Source {
/// The form id.
pub form: String,
/// The directory of per-question CSV exports.
pub dir: PathBuf,
}
impl Source {
/// Parses a `FORM=DIR` argument.
///
/// A bare path is accepted and takes the fallback form, so the single-form
/// case stays as short as it was.
///
/// # Arguments
///
/// * `text` - the argument, e.g. `A=exports/e1-a` or `exports/e1`.
/// * `fallback` - the form to use when the argument names none.
///
/// # Returns
///
/// The source.
///
/// # Errors
///
/// Returns [`Error::Usage`] when the argument names no form and no fallback
/// was given, or when the form label is empty.
pub fn parse(text: &str, fallback: Option<&str>) -> Result<Source> {
// Split on the first `=` only: a directory name may contain one, a form
// label may not.
if let Some((form, dir)) = text.split_once('=') {
let form = form.trim();
if form.is_empty() {
return Err(Error::usage(format!(
"`{text}` has an empty form label; write it as FORM=DIR, e.g. A=exports/e1-a"
)));
}
if !dir.trim().is_empty() {
return Ok(Source {
form: form.to_string(),
dir: PathBuf::from(dir.trim()),
});
}
}
match fallback {
Some(form) => Ok(Source {
form: form.to_string(),
dir: PathBuf::from(text.trim()),
}),
None => Err(Error::usage(format!(
"`{text}` does not say which form it holds; write it as FORM=DIR (e.g. \
A=exports/e1-a) or pass --form"
))),
}
}
}
/// How to run an intake.
#[derive(Debug, Clone)]
pub struct Options {
/// The administration date, overriding the record's.
pub date: Option<Date>,
/// How the exports number their questions.
pub numbering: Numbering,
/// Whether a form that fails its key check stops the ingest.
///
/// On by default. A directory that does not match the form it was named as is
/// the one ingest error that produces confident, wrong analysis rather than an
/// obvious failure, so the default is to refuse and say so.
pub strict: bool,
}
impl Default for Options {
fn default() -> Options {
Options {
date: None,
numbering: Numbering::Printed,
strict: true,
}
}
}
/// What one form's directory contributed.
#[derive(Debug, Clone)]
pub struct FormIntake {
/// The form id.
pub form: String,
/// The directory it came from.
pub dir: PathBuf,
/// Where its mapping came from.
pub provenance: decode::Provenance,
/// How many students it held.
pub students: usize,
/// How many questions it held.
pub questions: usize,
/// How the graded keys compared to every declared form, best first.
pub fits: Vec<FormFit>,
/// The parsed question files, kept so grading-time decisions stay available.
pub parsed: Vec<Question>,
}
impl FormIntake {
/// This form's own fit.
pub fn own_fit(&self) -> Option<&FormFit> {
self.fits
.iter()
.find(|f| f.form.eq_ignore_ascii_case(&self.form))
}
}
/// The result of reading every form of one administration.
#[derive(Debug, Clone)]
pub struct Intake {
/// The merged, translated responses.
pub responses: ResponseSet,
/// Per-form detail.
pub forms: Vec<FormIntake>,
/// Problems that did not stop the ingest.
pub warnings: Vec<String>,
}
impl Intake {
/// Every parsed question file, across forms.
///
/// # Returns
///
/// Pairs of form id and question.
pub fn questions(&self) -> Vec<(&str, &Question)> {
self.forms
.iter()
.flat_map(|f| f.parsed.iter().map(move |q| (f.form.as_str(), q)))
.collect()
}
}
/// Reads every source into one response set.
///
/// # Arguments
///
/// * `catalog` - the loaded course.
/// * `record` - the assessment record.
/// * `seal` - the seal, when one was written. Strongly preferred: it describes the
/// paper that was printed rather than the paper the bank would print today.
/// * `sources` - the directories and the forms they hold.
/// * `opts` - how to run.
///
/// # Returns
///
/// The merged intake.
///
/// # Errors
///
/// Returns [`Error::Usage`] when a source names a form the record does not
/// declare, when two sources name the same form, or when a key check fails under
/// `strict`. Propagates parse errors from the exports themselves.
pub fn run(
catalog: &Catalog,
record: &AssessmentFile,
seal: Option<&SealFile>,
sources: &[Source],
opts: &Options,
) -> Result<Intake> {
if sources.is_empty() {
return Err(Error::usage(
"no directories to ingest; pass one per form, e.g. A=exports/e1-a B=exports/e1-b"
.to_string(),
));
}
let declared = declared_forms(record);
let mut seen: BTreeSet<String> = BTreeSet::new();
for source in sources {
if !seen.insert(source.form.to_ascii_uppercase()) {
return Err(Error::usage(format!(
"form {} was given twice; each form is one directory",
source.form
)));
}
if !declared
.iter()
.any(|f| f.id.eq_ignore_ascii_case(&source.form))
{
return Err(Error::usage(format!(
"the record for `{}` declares no form `{}`; it declares {}",
record.assessment.id,
source.form,
declared
.iter()
.map(|f| f.id.as_str())
.collect::<Vec<_>>()
.join(", ")
)));
}
}
// One decoder per declared form, not just per ingested form: identifying a
// swapped directory means testing it against the forms it might be.
let mut decoders: Vec<FormDecoder> = Vec::new();
for form in &declared {
decoders.push(FormDecoder::resolve(seal, catalog, record, form)?);
}
let mut merged = ResponseSet::new();
let mut forms = Vec::new();
let mut warnings = Vec::new();
let mut blocking = Vec::new();
for source in sources {
let decoder = decoders
.iter()
.find(|d| d.form.eq_ignore_ascii_case(&source.form))
.expect("every source's form was checked against the declared list");
let ctx = Context {
course: catalog.course.course.code.clone(),
term: record
.assessment
.term
.clone()
.unwrap_or_else(|| catalog.course.course.term.clone()),
assessment_id: record.assessment.id.clone(),
date: opts.date.or(record.assessment.date),
form: Some(decoder.form.clone()),
};
let import = gradescope::ingest_dir(&source.dir, &ctx)?;
let mut set = import.responses;
let fits = decode::identify_form(&import.questions, &decoders, opts.numbering);
if let Some(problem) = decode::form_warning(&decoder.form, &fits) {
let message = format!("{} [{}]", problem, source.dir.display());
let convincing_alternative = fits
.iter()
.any(|f| !f.form.eq_ignore_ascii_case(&decoder.form) && f.is_convincing());
if opts.strict && convincing_alternative {
blocking.push(message);
} else {
warnings.push(message);
}
}
warnings.extend(decode::apply(&mut set, decoder, opts.numbering));
forms.push(FormIntake {
form: decoder.form.clone(),
dir: source.dir.clone(),
provenance: decoder.provenance,
students: set.students().len(),
questions: set.all_items().len(),
fits,
parsed: import.questions,
});
merged.absorb(set);
}
if !blocking.is_empty() {
blocking.push(
"Nothing was written. Fix the form labels, or pass --allow-mismatch if the keys really \
did change after printing."
.to_string(),
);
return Err(Error::Invalid(blocking));
}
warnings.extend(cross_form_checks(&merged, &forms, record));
merged.warnings.extend(warnings.clone());
Ok(Intake {
responses: merged,
forms,
warnings,
})
}
/// The forms a record declares, with the implicit single form for a record that
/// declares none.
fn declared_forms(record: &AssessmentFile) -> Vec<Form> {
if record.forms.is_empty() {
vec![Form {
id: "A".to_string(),
seed: 0,
shuffle_items: false,
shuffle_options: false,
}]
} else {
record.forms.clone()
}
}
/// Checks that only merging several forms can make.
fn cross_form_checks(
merged: &ResponseSet,
forms: &[FormIntake],
record: &AssessmentFile,
) -> Vec<String> {
let mut out = Vec::new();
if forms.len() < 2 {
return out;
}
// A student on two forms sat one exam and was graded twice, or two people
// share an identifier. Either way the response set now double counts them.
let mut by_student: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new();
for row in &merged.rows {
if let Some(form) = row.form.as_deref() {
by_student
.entry(row.student_key.as_str())
.or_default()
.insert(form);
}
}
let doubled: Vec<&str> = by_student
.iter()
.filter(|(_, forms)| forms.len() > 1)
.map(|(student, _)| *student)
.collect();
if !doubled.is_empty() {
out.push(format!(
"{} student(s) appear on more than one form ({}); each is counted twice in every \
total until one submission is removed",
doubled.len(),
doubled
.iter()
.take(5)
.copied()
.collect::<Vec<_>>()
.join(", ")
));
}
// Every form is the same items in a different order, so a question present on
// one and absent from another means a directory is short a file.
let expected: BTreeSet<u32> = record
.items
.iter()
.filter(|p| !p.dropped)
.map(|p| p.number)
.collect();
for form in forms {
let present: BTreeSet<u32> = merged
.rows
.iter()
.filter(|r| {
r.form
.as_deref()
.map(|f| f.eq_ignore_ascii_case(&form.form))
.unwrap_or(false)
})
.map(|r| r.item_number)
.collect();
let missing: Vec<String> = expected
.difference(&present)
.map(|n| n.to_string())
.collect();
if !missing.is_empty() {
out.push(format!(
"form {}: no responses for question(s) {}; the export directory is missing those \
files",
form.form,
missing.join(", ")
));
}
}
// Forms are meant to be the same test. A large gap between their means is
// either a permutation that made one form easier or an uneven split of the
// class, and both are worth knowing before any grade is released.
let means: Vec<(String, f64, usize)> = forms
.iter()
.map(|f| {
let students: Vec<&str> = merged
.rows
.iter()
.filter(|r| {
r.form
.as_deref()
.map(|x| x.eq_ignore_ascii_case(&f.form))
.unwrap_or(false)
})
.map(|r| r.student_key.as_str())
.collect::<BTreeSet<&str>>()
.into_iter()
.collect();
let possible = merged.points_available();
let percents: Vec<f64> = students
.iter()
.map(|s| {
if possible > 0.0 {
100.0 * merged.scored_total(s) / possible
} else {
0.0
}
})
.collect();
let mean = if percents.is_empty() {
0.0
} else {
percents.iter().sum::<f64>() / percents.len() as f64
};
(f.form.clone(), mean, percents.len())
})
.collect();
if let (Some(low), Some(high)) = (
means
.iter()
.min_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal)),
means
.iter()
.max_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal)),
) {
let gap = high.1 - low.1;
if gap >= 5.0 && low.2 >= 5 && high.2 >= 5 {
out.push(format!(
"form {} averaged {:.0}% and form {} averaged {:.0}%, a {:.0}-point gap. With {} \
and {} students that may be the split rather than the forms, but it is worth \
looking at before releasing grades",
high.0, high.1, low.0, low.1, gap, high.2, low.2
));
}
}
out
}
/// A one-line summary of where a directory came from and what it held.
///
/// # Arguments
///
/// * `intake` - one form's intake.
///
/// # Returns
///
/// The line, without a trailing newline.
pub fn describe(intake: &FormIntake) -> String {
let fit = match intake.own_fit() {
Some(fit) if fit.compared > 0 => {
format!(", keys matched {}/{}", fit.matched, fit.compared)
}
_ => String::new(),
};
format!(
"form {}: {} student(s) x {} question(s) from {} (mapping {}{fit})",
intake.form,
intake.students,
intake.questions,
intake.dir.display(),
intake.provenance.label(),
)
}
/// Whether a path looks like a Gradescope per-question export directory.
///
/// Used to give a better error than "no numbered CSV files" when someone points
/// at the zip they downloaded, or at the directory above the one they meant.
///
/// # Arguments
///
/// * `dir` - the candidate directory.
///
/// # Returns
///
/// `true` when it holds at least one file named like `1.csv` or `1_Title.csv`.
pub fn looks_like_export(dir: &Path) -> bool {
let Ok(entries) = std::fs::read_dir(dir) else {
return false;
};
entries
.filter_map(|e| e.ok())
.any(|entry| gradescope::question_number(&entry.path()).is_some())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_labelled_source_parses() {
let source = Source::parse("B=exports/e1-b", None).unwrap();
assert_eq!(source.form, "B");
assert_eq!(source.dir, PathBuf::from("exports/e1-b"));
}
#[test]
fn a_bare_path_needs_a_fallback_form() {
let source = Source::parse("exports/e1", Some("A")).unwrap();
assert_eq!(source.form, "A");
assert_eq!(source.dir, PathBuf::from("exports/e1"));
let err = Source::parse("exports/e1", None).unwrap_err();
assert!(err.to_string().contains("which form"), "{err}");
}
#[test]
fn a_windows_style_path_is_not_mistaken_for_a_label() {
// The split is on the first `=`, and a drive letter has none, so this is
// only a hazard for a path that genuinely contains one.
let source = Source::parse("A=C:/exports/e1-a", None).unwrap();
assert_eq!(source.form, "A");
assert_eq!(source.dir, PathBuf::from("C:/exports/e1-a"));
}
#[test]
fn an_empty_label_is_refused() {
let err = Source::parse("=exports/e1", None).unwrap_err();
assert!(err.to_string().contains("empty form label"), "{err}");
}
}