feat: implement reports
This commit is contained in:
28 files changed
+7082
-63
No files matched your search
@@ -0,0 +1,565 @@
|
||||
// SPDX-License-Identifier: Prosperity-3.0.0
|
||||
// Copyright Scientific Computing Studio
|
||||
// Source: https://git.scient.ing/education/coursebank
|
||||
|
||||
//! Reading several forms of one exam back in at once.
|
||||
//!
|
||||
//! A two-form exam is two Gradescope assignments, each exporting its own
|
||||
//! directory of `1.csv` through `N.csv`, each numbered against its own paper. They
|
||||
//! are one administration: one set of students, one item pool, one set of
|
||||
//! statistics. Ingesting them one command at a time does not work, because the
|
||||
//! store keys on the administration and the second write replaces the first.
|
||||
//!
|
||||
//! So this module takes the whole set:
|
||||
//!
|
||||
//! ```text
|
||||
//! coursebank ingest gradescope A=exports/e1-a B=exports/e1-b --assessment e1
|
||||
//! ```
|
||||
//!
|
||||
//! and does four things the single-directory path cannot:
|
||||
//!
|
||||
//! *Checks each directory is the form it claims to be.* Every export carries the
|
||||
//! graded key in its point-value row; every form knows what its printed key should
|
||||
//! be. Comparing them catches a swapped pair of directories immediately rather
|
||||
//! than three weeks later, when the distractor table looks strange. See
|
||||
//! [`crate::decode::identify_form`].
|
||||
//!
|
||||
//! *Translates each form into the bank's vocabulary before merging.* Otherwise
|
||||
//! form A's option C and form B's option C land in the same column of the same
|
||||
//! table while meaning different things.
|
||||
//!
|
||||
//! *Merges into one response set*, written once, so `analyze` and `report` see the
|
||||
//! whole class.
|
||||
//!
|
||||
//! *Reports what the merge revealed*: a student who appears on two forms, a form
|
||||
//! that is missing a question the other has, a form that ran materially harder
|
||||
//! than the other.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::assessment::{AssessmentFile, Form};
|
||||
use crate::catalog::Catalog;
|
||||
use crate::date::Date;
|
||||
use crate::decode::{self, FormDecoder, FormFit, Numbering};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::gradescope::{self, Context, Question};
|
||||
use crate::responses::ResponseSet;
|
||||
use crate::seal::SealFile;
|
||||
|
||||
/// One directory of graded questions, and the form it holds.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Source {
|
||||
/// The form id.
|
||||
pub form: String,
|
||||
/// The directory of per-question CSV exports.
|
||||
pub dir: PathBuf,
|
||||
}
|
||||
|
||||
impl Source {
|
||||
/// Parses a `FORM=DIR` argument.
|
||||
///
|
||||
/// A bare path is accepted and takes the fallback form, so the single-form
|
||||
/// case stays as short as it was.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `text` - the argument, e.g. `A=exports/e1-a` or `exports/e1`.
|
||||
/// * `fallback` - the form to use when the argument names none.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The source.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`Error::Usage`] when the argument names no form and no fallback
|
||||
/// was given, or when the form label is empty.
|
||||
pub fn parse(text: &str, fallback: Option<&str>) -> Result<Source> {
|
||||
// Split on the first `=` only: a directory name may contain one, a form
|
||||
// label may not.
|
||||
if let Some((form, dir)) = text.split_once('=') {
|
||||
let form = form.trim();
|
||||
if form.is_empty() {
|
||||
return Err(Error::usage(format!(
|
||||
"`{text}` has an empty form label; write it as FORM=DIR, e.g. A=exports/e1-a"
|
||||
)));
|
||||
}
|
||||
if !dir.trim().is_empty() {
|
||||
return Ok(Source {
|
||||
form: form.to_string(),
|
||||
dir: PathBuf::from(dir.trim()),
|
||||
});
|
||||
}
|
||||
}
|
||||
match fallback {
|
||||
Some(form) => Ok(Source {
|
||||
form: form.to_string(),
|
||||
dir: PathBuf::from(text.trim()),
|
||||
}),
|
||||
None => Err(Error::usage(format!(
|
||||
"`{text}` does not say which form it holds; write it as FORM=DIR (e.g. \
|
||||
A=exports/e1-a) or pass --form"
|
||||
))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// How to run an intake.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Options {
|
||||
/// The administration date, overriding the record's.
|
||||
pub date: Option<Date>,
|
||||
/// How the exports number their questions.
|
||||
pub numbering: Numbering,
|
||||
/// Whether a form that fails its key check stops the ingest.
|
||||
///
|
||||
/// On by default. A directory that does not match the form it was named as is
|
||||
/// the one ingest error that produces confident, wrong analysis rather than an
|
||||
/// obvious failure, so the default is to refuse and say so.
|
||||
pub strict: bool,
|
||||
}
|
||||
|
||||
impl Default for Options {
|
||||
fn default() -> Options {
|
||||
Options {
|
||||
date: None,
|
||||
numbering: Numbering::Printed,
|
||||
strict: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// What one form's directory contributed.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FormIntake {
|
||||
/// The form id.
|
||||
pub form: String,
|
||||
/// The directory it came from.
|
||||
pub dir: PathBuf,
|
||||
/// Where its mapping came from.
|
||||
pub provenance: decode::Provenance,
|
||||
/// How many students it held.
|
||||
pub students: usize,
|
||||
/// How many questions it held.
|
||||
pub questions: usize,
|
||||
/// How the graded keys compared to every declared form, best first.
|
||||
pub fits: Vec<FormFit>,
|
||||
/// The parsed question files, kept so grading-time decisions stay available.
|
||||
pub parsed: Vec<Question>,
|
||||
}
|
||||
|
||||
impl FormIntake {
|
||||
/// This form's own fit.
|
||||
pub fn own_fit(&self) -> Option<&FormFit> {
|
||||
self.fits
|
||||
.iter()
|
||||
.find(|f| f.form.eq_ignore_ascii_case(&self.form))
|
||||
}
|
||||
}
|
||||
|
||||
/// The result of reading every form of one administration.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Intake {
|
||||
/// The merged, translated responses.
|
||||
pub responses: ResponseSet,
|
||||
/// Per-form detail.
|
||||
pub forms: Vec<FormIntake>,
|
||||
/// Problems that did not stop the ingest.
|
||||
pub warnings: Vec<String>,
|
||||
}
|
||||
|
||||
impl Intake {
|
||||
/// Every parsed question file, across forms.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Pairs of form id and question.
|
||||
pub fn questions(&self) -> Vec<(&str, &Question)> {
|
||||
self.forms
|
||||
.iter()
|
||||
.flat_map(|f| f.parsed.iter().map(move |q| (f.form.as_str(), q)))
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads every source into one response set.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `catalog` - the loaded course.
|
||||
/// * `record` - the assessment record.
|
||||
/// * `seal` - the seal, when one was written. Strongly preferred: it describes the
|
||||
/// paper that was printed rather than the paper the bank would print today.
|
||||
/// * `sources` - the directories and the forms they hold.
|
||||
/// * `opts` - how to run.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The merged intake.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`Error::Usage`] when a source names a form the record does not
|
||||
/// declare, when two sources name the same form, or when a key check fails under
|
||||
/// `strict`. Propagates parse errors from the exports themselves.
|
||||
pub fn run(
|
||||
catalog: &Catalog,
|
||||
record: &AssessmentFile,
|
||||
seal: Option<&SealFile>,
|
||||
sources: &[Source],
|
||||
opts: &Options,
|
||||
) -> Result<Intake> {
|
||||
if sources.is_empty() {
|
||||
return Err(Error::usage(
|
||||
"no directories to ingest; pass one per form, e.g. A=exports/e1-a B=exports/e1-b"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let declared = declared_forms(record);
|
||||
let mut seen: BTreeSet<String> = BTreeSet::new();
|
||||
for source in sources {
|
||||
if !seen.insert(source.form.to_ascii_uppercase()) {
|
||||
return Err(Error::usage(format!(
|
||||
"form {} was given twice; each form is one directory",
|
||||
source.form
|
||||
)));
|
||||
}
|
||||
if !declared
|
||||
.iter()
|
||||
.any(|f| f.id.eq_ignore_ascii_case(&source.form))
|
||||
{
|
||||
return Err(Error::usage(format!(
|
||||
"the record for `{}` declares no form `{}`; it declares {}",
|
||||
record.assessment.id,
|
||||
source.form,
|
||||
declared
|
||||
.iter()
|
||||
.map(|f| f.id.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
// One decoder per declared form, not just per ingested form: identifying a
|
||||
// swapped directory means testing it against the forms it might be.
|
||||
let mut decoders: Vec<FormDecoder> = Vec::new();
|
||||
for form in &declared {
|
||||
decoders.push(FormDecoder::resolve(seal, catalog, record, form)?);
|
||||
}
|
||||
|
||||
let mut merged = ResponseSet::new();
|
||||
let mut forms = Vec::new();
|
||||
let mut warnings = Vec::new();
|
||||
let mut blocking = Vec::new();
|
||||
|
||||
for source in sources {
|
||||
let decoder = decoders
|
||||
.iter()
|
||||
.find(|d| d.form.eq_ignore_ascii_case(&source.form))
|
||||
.expect("every source's form was checked against the declared list");
|
||||
|
||||
let ctx = Context {
|
||||
course: catalog.course.course.code.clone(),
|
||||
term: record
|
||||
.assessment
|
||||
.term
|
||||
.clone()
|
||||
.unwrap_or_else(|| catalog.course.course.term.clone()),
|
||||
assessment_id: record.assessment.id.clone(),
|
||||
date: opts.date.or(record.assessment.date),
|
||||
form: Some(decoder.form.clone()),
|
||||
};
|
||||
|
||||
let import = gradescope::ingest_dir(&source.dir, &ctx)?;
|
||||
let mut set = import.responses;
|
||||
|
||||
let fits = decode::identify_form(&import.questions, &decoders, opts.numbering);
|
||||
if let Some(problem) = decode::form_warning(&decoder.form, &fits) {
|
||||
let message = format!("{} [{}]", problem, source.dir.display());
|
||||
let convincing_alternative = fits
|
||||
.iter()
|
||||
.any(|f| !f.form.eq_ignore_ascii_case(&decoder.form) && f.is_convincing());
|
||||
if opts.strict && convincing_alternative {
|
||||
blocking.push(message);
|
||||
} else {
|
||||
warnings.push(message);
|
||||
}
|
||||
}
|
||||
|
||||
warnings.extend(decode::apply(&mut set, decoder, opts.numbering));
|
||||
|
||||
forms.push(FormIntake {
|
||||
form: decoder.form.clone(),
|
||||
dir: source.dir.clone(),
|
||||
provenance: decoder.provenance,
|
||||
students: set.students().len(),
|
||||
questions: set.all_items().len(),
|
||||
fits,
|
||||
parsed: import.questions,
|
||||
});
|
||||
|
||||
merged.absorb(set);
|
||||
}
|
||||
|
||||
if !blocking.is_empty() {
|
||||
blocking.push(
|
||||
"Nothing was written. Fix the form labels, or pass --allow-mismatch if the keys really \
|
||||
did change after printing."
|
||||
.to_string(),
|
||||
);
|
||||
return Err(Error::Invalid(blocking));
|
||||
}
|
||||
|
||||
warnings.extend(cross_form_checks(&merged, &forms, record));
|
||||
merged.warnings.extend(warnings.clone());
|
||||
|
||||
Ok(Intake {
|
||||
responses: merged,
|
||||
forms,
|
||||
warnings,
|
||||
})
|
||||
}
|
||||
|
||||
/// The forms a record declares, with the implicit single form for a record that
|
||||
/// declares none.
|
||||
fn declared_forms(record: &AssessmentFile) -> Vec<Form> {
|
||||
if record.forms.is_empty() {
|
||||
vec![Form {
|
||||
id: "A".to_string(),
|
||||
seed: 0,
|
||||
shuffle_items: false,
|
||||
shuffle_options: false,
|
||||
}]
|
||||
} else {
|
||||
record.forms.clone()
|
||||
}
|
||||
}
|
||||
|
||||
/// Checks that only merging several forms can make.
|
||||
fn cross_form_checks(
|
||||
merged: &ResponseSet,
|
||||
forms: &[FormIntake],
|
||||
record: &AssessmentFile,
|
||||
) -> Vec<String> {
|
||||
let mut out = Vec::new();
|
||||
if forms.len() < 2 {
|
||||
return out;
|
||||
}
|
||||
|
||||
// A student on two forms sat one exam and was graded twice, or two people
|
||||
// share an identifier. Either way the response set now double counts them.
|
||||
let mut by_student: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new();
|
||||
for row in &merged.rows {
|
||||
if let Some(form) = row.form.as_deref() {
|
||||
by_student
|
||||
.entry(row.student_key.as_str())
|
||||
.or_default()
|
||||
.insert(form);
|
||||
}
|
||||
}
|
||||
let doubled: Vec<&str> = by_student
|
||||
.iter()
|
||||
.filter(|(_, forms)| forms.len() > 1)
|
||||
.map(|(student, _)| *student)
|
||||
.collect();
|
||||
if !doubled.is_empty() {
|
||||
out.push(format!(
|
||||
"{} student(s) appear on more than one form ({}); each is counted twice in every \
|
||||
total until one submission is removed",
|
||||
doubled.len(),
|
||||
doubled
|
||||
.iter()
|
||||
.take(5)
|
||||
.copied()
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
));
|
||||
}
|
||||
|
||||
// Every form is the same items in a different order, so a question present on
|
||||
// one and absent from another means a directory is short a file.
|
||||
let expected: BTreeSet<u32> = record
|
||||
.items
|
||||
.iter()
|
||||
.filter(|p| !p.dropped)
|
||||
.map(|p| p.number)
|
||||
.collect();
|
||||
for form in forms {
|
||||
let present: BTreeSet<u32> = merged
|
||||
.rows
|
||||
.iter()
|
||||
.filter(|r| {
|
||||
r.form
|
||||
.as_deref()
|
||||
.map(|f| f.eq_ignore_ascii_case(&form.form))
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.map(|r| r.item_number)
|
||||
.collect();
|
||||
let missing: Vec<String> = expected
|
||||
.difference(&present)
|
||||
.map(|n| n.to_string())
|
||||
.collect();
|
||||
if !missing.is_empty() {
|
||||
out.push(format!(
|
||||
"form {}: no responses for question(s) {}; the export directory is missing those \
|
||||
files",
|
||||
form.form,
|
||||
missing.join(", ")
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// Forms are meant to be the same test. A large gap between their means is
|
||||
// either a permutation that made one form easier or an uneven split of the
|
||||
// class, and both are worth knowing before any grade is released.
|
||||
let means: Vec<(String, f64, usize)> = forms
|
||||
.iter()
|
||||
.map(|f| {
|
||||
let students: Vec<&str> = merged
|
||||
.rows
|
||||
.iter()
|
||||
.filter(|r| {
|
||||
r.form
|
||||
.as_deref()
|
||||
.map(|x| x.eq_ignore_ascii_case(&f.form))
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.map(|r| r.student_key.as_str())
|
||||
.collect::<BTreeSet<&str>>()
|
||||
.into_iter()
|
||||
.collect();
|
||||
let possible = merged.points_available();
|
||||
let percents: Vec<f64> = students
|
||||
.iter()
|
||||
.map(|s| {
|
||||
if possible > 0.0 {
|
||||
100.0 * merged.scored_total(s) / possible
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let mean = if percents.is_empty() {
|
||||
0.0
|
||||
} else {
|
||||
percents.iter().sum::<f64>() / percents.len() as f64
|
||||
};
|
||||
(f.form.clone(), mean, percents.len())
|
||||
})
|
||||
.collect();
|
||||
|
||||
if let (Some(low), Some(high)) = (
|
||||
means
|
||||
.iter()
|
||||
.min_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal)),
|
||||
means
|
||||
.iter()
|
||||
.max_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal)),
|
||||
) {
|
||||
let gap = high.1 - low.1;
|
||||
if gap >= 5.0 && low.2 >= 5 && high.2 >= 5 {
|
||||
out.push(format!(
|
||||
"form {} averaged {:.0}% and form {} averaged {:.0}%, a {:.0}-point gap. With {} \
|
||||
and {} students that may be the split rather than the forms, but it is worth \
|
||||
looking at before releasing grades",
|
||||
high.0, high.1, low.0, low.1, gap, high.2, low.2
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
/// A one-line summary of where a directory came from and what it held.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `intake` - one form's intake.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The line, without a trailing newline.
|
||||
pub fn describe(intake: &FormIntake) -> String {
|
||||
let fit = match intake.own_fit() {
|
||||
Some(fit) if fit.compared > 0 => {
|
||||
format!(", keys matched {}/{}", fit.matched, fit.compared)
|
||||
}
|
||||
_ => String::new(),
|
||||
};
|
||||
format!(
|
||||
"form {}: {} student(s) x {} question(s) from {} (mapping {}{fit})",
|
||||
intake.form,
|
||||
intake.students,
|
||||
intake.questions,
|
||||
intake.dir.display(),
|
||||
intake.provenance.label(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Whether a path looks like a Gradescope per-question export directory.
|
||||
///
|
||||
/// Used to give a better error than "no numbered CSV files" when someone points
|
||||
/// at the zip they downloaded, or at the directory above the one they meant.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `dir` - the candidate directory.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// `true` when it holds at least one file named like `1.csv`.
|
||||
pub fn looks_like_export(dir: &Path) -> bool {
|
||||
let Ok(entries) = std::fs::read_dir(dir) else {
|
||||
return false;
|
||||
};
|
||||
entries.filter_map(|e| e.ok()).any(|entry| {
|
||||
let path = entry.path();
|
||||
path.extension().and_then(|e| e.to_str()) == Some("csv")
|
||||
&& path
|
||||
.file_stem()
|
||||
.and_then(|s| s.to_str())
|
||||
.map(|s| s.chars().all(|c| c.is_ascii_digit()))
|
||||
.unwrap_or(false)
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_labelled_source_parses() {
|
||||
let source = Source::parse("B=exports/e1-b", None).unwrap();
|
||||
assert_eq!(source.form, "B");
|
||||
assert_eq!(source.dir, PathBuf::from("exports/e1-b"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_bare_path_needs_a_fallback_form() {
|
||||
let source = Source::parse("exports/e1", Some("A")).unwrap();
|
||||
assert_eq!(source.form, "A");
|
||||
assert_eq!(source.dir, PathBuf::from("exports/e1"));
|
||||
|
||||
let err = Source::parse("exports/e1", None).unwrap_err();
|
||||
assert!(err.to_string().contains("which form"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_windows_style_path_is_not_mistaken_for_a_label() {
|
||||
// The split is on the first `=`, and a drive letter has none, so this is
|
||||
// only a hazard for a path that genuinely contains one.
|
||||
let source = Source::parse("A=C:/exports/e1-a", None).unwrap();
|
||||
assert_eq!(source.form, "A");
|
||||
assert_eq!(source.dir, PathBuf::from("C:/exports/e1-a"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_label_is_refused() {
|
||||
let err = Source::parse("=exports/e1", None).unwrap_err();
|
||||
assert!(err.to_string().contains("empty form label"), "{err}");
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user