560 lines
18 KiB
Rust
560 lines
18 KiB
Rust
// SPDX-License-Identifier: Prosperity-3.0.0
|
|
// Copyright Scienting
|
|
// Source: https://git.scient.ing/education/coursebank
|
|
|
|
//! Reading several forms of one exam back in at once.
|
|
//!
|
|
//! A two-form exam is two Gradescope assignments, each exporting its own
|
|
//! directory of `1.csv` through `N.csv`, each numbered against its own paper. They
|
|
//! are one administration: one set of students, one item pool, one set of
|
|
//! statistics. Ingesting them one command at a time does not work, because the
|
|
//! store keys on the administration and the second write replaces the first.
|
|
//!
|
|
//! So this module takes the whole set:
|
|
//!
|
|
//! ```text
|
|
//! coursebank ingest gradescope A=exports/e1-a B=exports/e1-b --assessment e1
|
|
//! ```
|
|
//!
|
|
//! and does four things the single-directory path cannot:
|
|
//!
|
|
//! *Checks each directory is the form it claims to be.* Every export carries the
|
|
//! graded key in its point-value row; every form knows what its printed key should
|
|
//! be. Comparing them catches a swapped pair of directories immediately rather
|
|
//! than three weeks later, when the distractor table looks strange. See
|
|
//! [`crate::decode::identify_form`].
|
|
//!
|
|
//! *Translates each form into the bank's vocabulary before merging.* Otherwise
|
|
//! form A's option C and form B's option C land in the same column of the same
|
|
//! table while meaning different things.
|
|
//!
|
|
//! *Merges into one response set*, written once, so `analyze` and `report` see the
|
|
//! whole class.
|
|
//!
|
|
//! *Reports what the merge revealed*: a student who appears on two forms, a form
|
|
//! that is missing a question the other has, a form that ran materially harder
|
|
//! than the other.
|
|
|
|
use std::collections::{BTreeMap, BTreeSet};
|
|
use std::path::{Path, PathBuf};
|
|
|
|
use crate::assessment::{AssessmentFile, Form};
|
|
use crate::catalog::Catalog;
|
|
use crate::date::Date;
|
|
use crate::decode::{self, FormDecoder, FormFit, Numbering};
|
|
use crate::error::{Error, Result};
|
|
use crate::gradescope::{self, Context, Question};
|
|
use crate::responses::ResponseSet;
|
|
use crate::seal::SealFile;
|
|
|
|
/// One directory of graded questions, and the form it holds.
|
|
#[derive(Debug, Clone)]
|
|
pub struct Source {
|
|
/// The form id.
|
|
pub form: String,
|
|
/// The directory of per-question CSV exports.
|
|
pub dir: PathBuf,
|
|
}
|
|
|
|
impl Source {
|
|
/// Parses a `FORM=DIR` argument.
|
|
///
|
|
/// A bare path is accepted and takes the fallback form, so the single-form
|
|
/// case stays as short as it was.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `text` - the argument, e.g. `A=exports/e1-a` or `exports/e1`.
|
|
/// * `fallback` - the form to use when the argument names none.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The source.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns [`Error::Usage`] when the argument names no form and no fallback
|
|
/// was given, or when the form label is empty.
|
|
pub fn parse(text: &str, fallback: Option<&str>) -> Result<Source> {
|
|
// Split on the first `=` only: a directory name may contain one, a form
|
|
// label may not.
|
|
if let Some((form, dir)) = text.split_once('=') {
|
|
let form = form.trim();
|
|
if form.is_empty() {
|
|
return Err(Error::usage(format!(
|
|
"`{text}` has an empty form label; write it as FORM=DIR, e.g. A=exports/e1-a"
|
|
)));
|
|
}
|
|
if !dir.trim().is_empty() {
|
|
return Ok(Source {
|
|
form: form.to_string(),
|
|
dir: PathBuf::from(dir.trim()),
|
|
});
|
|
}
|
|
}
|
|
match fallback {
|
|
Some(form) => Ok(Source {
|
|
form: form.to_string(),
|
|
dir: PathBuf::from(text.trim()),
|
|
}),
|
|
None => Err(Error::usage(format!(
|
|
"`{text}` does not say which form it holds; write it as FORM=DIR (e.g. \
|
|
A=exports/e1-a) or pass --form"
|
|
))),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// How to run an intake.
|
|
#[derive(Debug, Clone)]
|
|
pub struct Options {
|
|
/// The administration date, overriding the record's.
|
|
pub date: Option<Date>,
|
|
/// How the exports number their questions.
|
|
pub numbering: Numbering,
|
|
/// Whether a form that fails its key check stops the ingest.
|
|
///
|
|
/// On by default. A directory that does not match the form it was named as is
|
|
/// the one ingest error that produces confident, wrong analysis rather than an
|
|
/// obvious failure, so the default is to refuse and say so.
|
|
pub strict: bool,
|
|
}
|
|
|
|
impl Default for Options {
|
|
fn default() -> Options {
|
|
Options {
|
|
date: None,
|
|
numbering: Numbering::Printed,
|
|
strict: true,
|
|
}
|
|
}
|
|
}
|
|
|
|
/// What one form's directory contributed.
|
|
#[derive(Debug, Clone)]
|
|
pub struct FormIntake {
|
|
/// The form id.
|
|
pub form: String,
|
|
/// The directory it came from.
|
|
pub dir: PathBuf,
|
|
/// Where its mapping came from.
|
|
pub provenance: decode::Provenance,
|
|
/// How many students it held.
|
|
pub students: usize,
|
|
/// How many questions it held.
|
|
pub questions: usize,
|
|
/// How the graded keys compared to every declared form, best first.
|
|
pub fits: Vec<FormFit>,
|
|
/// The parsed question files, kept so grading-time decisions stay available.
|
|
pub parsed: Vec<Question>,
|
|
}
|
|
|
|
impl FormIntake {
|
|
/// This form's own fit.
|
|
pub fn own_fit(&self) -> Option<&FormFit> {
|
|
self.fits
|
|
.iter()
|
|
.find(|f| f.form.eq_ignore_ascii_case(&self.form))
|
|
}
|
|
}
|
|
|
|
/// The result of reading every form of one administration.
|
|
#[derive(Debug, Clone)]
|
|
pub struct Intake {
|
|
/// The merged, translated responses.
|
|
pub responses: ResponseSet,
|
|
/// Per-form detail.
|
|
pub forms: Vec<FormIntake>,
|
|
/// Problems that did not stop the ingest.
|
|
pub warnings: Vec<String>,
|
|
}
|
|
|
|
impl Intake {
|
|
/// Every parsed question file, across forms.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// Pairs of form id and question.
|
|
pub fn questions(&self) -> Vec<(&str, &Question)> {
|
|
self.forms
|
|
.iter()
|
|
.flat_map(|f| f.parsed.iter().map(move |q| (f.form.as_str(), q)))
|
|
.collect()
|
|
}
|
|
}
|
|
|
|
/// Reads every source into one response set.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `catalog` - the loaded course.
|
|
/// * `record` - the assessment record.
|
|
/// * `seal` - the seal, when one was written. Strongly preferred: it describes the
|
|
/// paper that was printed rather than the paper the bank would print today.
|
|
/// * `sources` - the directories and the forms they hold.
|
|
/// * `opts` - how to run.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The merged intake.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns [`Error::Usage`] when a source names a form the record does not
|
|
/// declare, when two sources name the same form, or when a key check fails under
|
|
/// `strict`. Propagates parse errors from the exports themselves.
|
|
pub fn run(
|
|
catalog: &Catalog,
|
|
record: &AssessmentFile,
|
|
seal: Option<&SealFile>,
|
|
sources: &[Source],
|
|
opts: &Options,
|
|
) -> Result<Intake> {
|
|
if sources.is_empty() {
|
|
return Err(Error::usage(
|
|
"no directories to ingest; pass one per form, e.g. A=exports/e1-a B=exports/e1-b"
|
|
.to_string(),
|
|
));
|
|
}
|
|
|
|
let declared = declared_forms(record);
|
|
let mut seen: BTreeSet<String> = BTreeSet::new();
|
|
for source in sources {
|
|
if !seen.insert(source.form.to_ascii_uppercase()) {
|
|
return Err(Error::usage(format!(
|
|
"form {} was given twice; each form is one directory",
|
|
source.form
|
|
)));
|
|
}
|
|
if !declared
|
|
.iter()
|
|
.any(|f| f.id.eq_ignore_ascii_case(&source.form))
|
|
{
|
|
return Err(Error::usage(format!(
|
|
"the record for `{}` declares no form `{}`; it declares {}",
|
|
record.assessment.id,
|
|
source.form,
|
|
declared
|
|
.iter()
|
|
.map(|f| f.id.as_str())
|
|
.collect::<Vec<_>>()
|
|
.join(", ")
|
|
)));
|
|
}
|
|
}
|
|
|
|
// One decoder per declared form, not just per ingested form: identifying a
|
|
// swapped directory means testing it against the forms it might be.
|
|
let mut decoders: Vec<FormDecoder> = Vec::new();
|
|
for form in &declared {
|
|
decoders.push(FormDecoder::resolve(seal, catalog, record, form)?);
|
|
}
|
|
|
|
let mut merged = ResponseSet::new();
|
|
let mut forms = Vec::new();
|
|
let mut warnings = Vec::new();
|
|
let mut blocking = Vec::new();
|
|
|
|
for source in sources {
|
|
let decoder = decoders
|
|
.iter()
|
|
.find(|d| d.form.eq_ignore_ascii_case(&source.form))
|
|
.expect("every source's form was checked against the declared list");
|
|
|
|
let ctx = Context {
|
|
course: catalog.course.course.code.clone(),
|
|
term: record
|
|
.assessment
|
|
.term
|
|
.clone()
|
|
.unwrap_or_else(|| catalog.course.course.term.clone()),
|
|
assessment_id: record.assessment.id.clone(),
|
|
date: opts.date.or(record.assessment.date),
|
|
form: Some(decoder.form.clone()),
|
|
};
|
|
|
|
let import = gradescope::ingest_dir(&source.dir, &ctx)?;
|
|
let mut set = import.responses;
|
|
|
|
let fits = decode::identify_form(&import.questions, &decoders, opts.numbering);
|
|
if let Some(problem) = decode::form_warning(&decoder.form, &fits) {
|
|
let message = format!("{} [{}]", problem, source.dir.display());
|
|
let convincing_alternative = fits
|
|
.iter()
|
|
.any(|f| !f.form.eq_ignore_ascii_case(&decoder.form) && f.is_convincing());
|
|
if opts.strict && convincing_alternative {
|
|
blocking.push(message);
|
|
} else {
|
|
warnings.push(message);
|
|
}
|
|
}
|
|
|
|
warnings.extend(decode::apply(&mut set, decoder, opts.numbering));
|
|
|
|
forms.push(FormIntake {
|
|
form: decoder.form.clone(),
|
|
dir: source.dir.clone(),
|
|
provenance: decoder.provenance,
|
|
students: set.students().len(),
|
|
questions: set.all_items().len(),
|
|
fits,
|
|
parsed: import.questions,
|
|
});
|
|
|
|
merged.absorb(set);
|
|
}
|
|
|
|
if !blocking.is_empty() {
|
|
blocking.push(
|
|
"Nothing was written. Fix the form labels, or pass --allow-mismatch if the keys really \
|
|
did change after printing."
|
|
.to_string(),
|
|
);
|
|
return Err(Error::Invalid(blocking));
|
|
}
|
|
|
|
warnings.extend(cross_form_checks(&merged, &forms, record));
|
|
merged.warnings.extend(warnings.clone());
|
|
|
|
Ok(Intake {
|
|
responses: merged,
|
|
forms,
|
|
warnings,
|
|
})
|
|
}
|
|
|
|
/// The forms a record declares, with the implicit single form for a record that
|
|
/// declares none.
|
|
fn declared_forms(record: &AssessmentFile) -> Vec<Form> {
|
|
if record.forms.is_empty() {
|
|
vec![Form {
|
|
id: "A".to_string(),
|
|
seed: 0,
|
|
shuffle_items: false,
|
|
shuffle_options: false,
|
|
}]
|
|
} else {
|
|
record.forms.clone()
|
|
}
|
|
}
|
|
|
|
/// Checks that only merging several forms can make.
|
|
fn cross_form_checks(
|
|
merged: &ResponseSet,
|
|
forms: &[FormIntake],
|
|
record: &AssessmentFile,
|
|
) -> Vec<String> {
|
|
let mut out = Vec::new();
|
|
if forms.len() < 2 {
|
|
return out;
|
|
}
|
|
|
|
// A student on two forms sat one exam and was graded twice, or two people
|
|
// share an identifier. Either way the response set now double counts them.
|
|
let mut by_student: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new();
|
|
for row in &merged.rows {
|
|
if let Some(form) = row.form.as_deref() {
|
|
by_student
|
|
.entry(row.student_key.as_str())
|
|
.or_default()
|
|
.insert(form);
|
|
}
|
|
}
|
|
let doubled: Vec<&str> = by_student
|
|
.iter()
|
|
.filter(|(_, forms)| forms.len() > 1)
|
|
.map(|(student, _)| *student)
|
|
.collect();
|
|
if !doubled.is_empty() {
|
|
out.push(format!(
|
|
"{} student(s) appear on more than one form ({}); each is counted twice in every \
|
|
total until one submission is removed",
|
|
doubled.len(),
|
|
doubled
|
|
.iter()
|
|
.take(5)
|
|
.copied()
|
|
.collect::<Vec<_>>()
|
|
.join(", ")
|
|
));
|
|
}
|
|
|
|
// Every form is the same items in a different order, so a question present on
|
|
// one and absent from another means a directory is short a file.
|
|
let expected: BTreeSet<u32> = record
|
|
.items
|
|
.iter()
|
|
.filter(|p| !p.dropped)
|
|
.map(|p| p.number)
|
|
.collect();
|
|
for form in forms {
|
|
let present: BTreeSet<u32> = merged
|
|
.rows
|
|
.iter()
|
|
.filter(|r| {
|
|
r.form
|
|
.as_deref()
|
|
.map(|f| f.eq_ignore_ascii_case(&form.form))
|
|
.unwrap_or(false)
|
|
})
|
|
.map(|r| r.item_number)
|
|
.collect();
|
|
let missing: Vec<String> = expected
|
|
.difference(&present)
|
|
.map(|n| n.to_string())
|
|
.collect();
|
|
if !missing.is_empty() {
|
|
out.push(format!(
|
|
"form {}: no responses for question(s) {}; the export directory is missing those \
|
|
files",
|
|
form.form,
|
|
missing.join(", ")
|
|
));
|
|
}
|
|
}
|
|
|
|
// Forms are meant to be the same test. A large gap between their means is
|
|
// either a permutation that made one form easier or an uneven split of the
|
|
// class, and both are worth knowing before any grade is released.
|
|
let means: Vec<(String, f64, usize)> = forms
|
|
.iter()
|
|
.map(|f| {
|
|
let students: Vec<&str> = merged
|
|
.rows
|
|
.iter()
|
|
.filter(|r| {
|
|
r.form
|
|
.as_deref()
|
|
.map(|x| x.eq_ignore_ascii_case(&f.form))
|
|
.unwrap_or(false)
|
|
})
|
|
.map(|r| r.student_key.as_str())
|
|
.collect::<BTreeSet<&str>>()
|
|
.into_iter()
|
|
.collect();
|
|
let possible = merged.points_available();
|
|
let percents: Vec<f64> = students
|
|
.iter()
|
|
.map(|s| {
|
|
if possible > 0.0 {
|
|
100.0 * merged.scored_total(s) / possible
|
|
} else {
|
|
0.0
|
|
}
|
|
})
|
|
.collect();
|
|
let mean = if percents.is_empty() {
|
|
0.0
|
|
} else {
|
|
percents.iter().sum::<f64>() / percents.len() as f64
|
|
};
|
|
(f.form.clone(), mean, percents.len())
|
|
})
|
|
.collect();
|
|
|
|
if let (Some(low), Some(high)) = (
|
|
means
|
|
.iter()
|
|
.min_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal)),
|
|
means
|
|
.iter()
|
|
.max_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal)),
|
|
) {
|
|
let gap = high.1 - low.1;
|
|
if gap >= 5.0 && low.2 >= 5 && high.2 >= 5 {
|
|
out.push(format!(
|
|
"form {} averaged {:.0}% and form {} averaged {:.0}%, a {:.0}-point gap. With {} \
|
|
and {} students that may be the split rather than the forms, but it is worth \
|
|
looking at before releasing grades",
|
|
high.0, high.1, low.0, low.1, gap, high.2, low.2
|
|
));
|
|
}
|
|
}
|
|
|
|
out
|
|
}
|
|
|
|
/// A one-line summary of where a directory came from and what it held.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `intake` - one form's intake.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The line, without a trailing newline.
|
|
pub fn describe(intake: &FormIntake) -> String {
|
|
let fit = match intake.own_fit() {
|
|
Some(fit) if fit.compared > 0 => {
|
|
format!(", keys matched {}/{}", fit.matched, fit.compared)
|
|
}
|
|
_ => String::new(),
|
|
};
|
|
format!(
|
|
"form {}: {} student(s) x {} question(s) from {} (mapping {}{fit})",
|
|
intake.form,
|
|
intake.students,
|
|
intake.questions,
|
|
intake.dir.display(),
|
|
intake.provenance.label(),
|
|
)
|
|
}
|
|
|
|
/// Whether a path looks like a Gradescope per-question export directory.
|
|
///
|
|
/// Used to give a better error than "no numbered CSV files" when someone points
|
|
/// at the zip they downloaded, or at the directory above the one they meant.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `dir` - the candidate directory.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// `true` when it holds at least one file named like `1.csv` or `1_Title.csv`.
|
|
pub fn looks_like_export(dir: &Path) -> bool {
|
|
let Ok(entries) = std::fs::read_dir(dir) else {
|
|
return false;
|
|
};
|
|
entries
|
|
.filter_map(|e| e.ok())
|
|
.any(|entry| gradescope::question_number(&entry.path()).is_some())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn a_labelled_source_parses() {
|
|
let source = Source::parse("B=exports/e1-b", None).unwrap();
|
|
assert_eq!(source.form, "B");
|
|
assert_eq!(source.dir, PathBuf::from("exports/e1-b"));
|
|
}
|
|
|
|
#[test]
|
|
fn a_bare_path_needs_a_fallback_form() {
|
|
let source = Source::parse("exports/e1", Some("A")).unwrap();
|
|
assert_eq!(source.form, "A");
|
|
assert_eq!(source.dir, PathBuf::from("exports/e1"));
|
|
|
|
let err = Source::parse("exports/e1", None).unwrap_err();
|
|
assert!(err.to_string().contains("which form"), "{err}");
|
|
}
|
|
|
|
#[test]
|
|
fn a_windows_style_path_is_not_mistaken_for_a_label() {
|
|
// The split is on the first `=`, and a drive letter has none, so this is
|
|
// only a hazard for a path that genuinely contains one.
|
|
let source = Source::parse("A=C:/exports/e1-a", None).unwrap();
|
|
assert_eq!(source.form, "A");
|
|
assert_eq!(source.dir, PathBuf::from("C:/exports/e1-a"));
|
|
}
|
|
|
|
#[test]
|
|
fn an_empty_label_is_refused() {
|
|
let err = Source::parse("=exports/e1", None).unwrap_err();
|
|
assert!(err.to_string().contains("empty form label"), "{err}");
|
|
}
|
|
}
|