From 09c3222f8d9664672e192f56d906882893b6b1a7 Mon Sep 17 00:00:00 2001 From: Alex Maldonado Date: Wed, 5 Aug 2026 23:18:16 -0400 Subject: [PATCH] feat: initial package draft --- src/analysis.rs | 35 + src/analysis/calibrate.rs | 857 +++++++++++++++ src/analysis/classical.rs | 1037 ++++++++++++++++++ src/analysis/irt.rs | 1171 ++++++++++++++++++++ src/analysis/students.rs | 1142 ++++++++++++++++++++ src/authoring.rs | 23 + src/authoring/jsonschema.rs | 1019 +++++++++++++++++ src/authoring/lint.rs | 1461 +++++++++++++++++++++++++ src/authoring/select.rs | 717 ++++++++++++ src/data.rs | 29 + src/data/canvas.rs | 619 +++++++++++ src/data/gradescope.rs | 855 +++++++++++++++ src/data/responses.rs | 831 ++++++++++++++ src/data/store.rs | 665 ++++++++++++ src/data/store_parquet.rs | 387 +++++++ src/error.rs | 129 +++ src/export.rs | 21 + src/export/qti.rs | 686 ++++++++++++ src/export/report.rs | 1084 +++++++++++++++++++ src/export/typst.rs | 518 +++++++++ src/lib.rs | 106 ++ src/main.rs | 2043 +++++++++++++++++++++++++++++++++++ src/model.rs | 28 + src/model/assessment.rs | 824 ++++++++++++++ src/model/bank.rs | 906 ++++++++++++++++ src/model/catalog.rs | 640 +++++++++++ src/model/course.rs | 785 ++++++++++++++ src/model/item.rs | 820 ++++++++++++++ src/model/taxonomy.rs | 639 +++++++++++ src/util.rs | 13 + src/util/date.rs | 248 +++++ src/util/hash.rs | 253 +++++ src/util/markup.rs | 378 +++++++ src/util/rng.rs | 169 +++ src/util/yaml.rs | 224 ++++ src/util/zipfile.rs | 242 +++++ 36 files changed, 21604 insertions(+) create mode 100644 src/analysis.rs create mode 100644 src/analysis/calibrate.rs create mode 100644 src/analysis/classical.rs create mode 100644 src/analysis/irt.rs create mode 100644 src/analysis/students.rs create mode 100644 src/authoring.rs create mode 100644 src/authoring/jsonschema.rs create mode 100644 src/authoring/lint.rs create mode 100644 src/authoring/select.rs create mode 100644 src/data.rs create mode 100644 src/data/canvas.rs create mode 100644 src/data/gradescope.rs create mode 100644 src/data/responses.rs create mode 100644 src/data/store.rs create mode 100644 src/data/store_parquet.rs create mode 100644 src/error.rs create mode 100644 src/export.rs create mode 100644 src/export/qti.rs create mode 100644 src/export/report.rs create mode 100644 src/export/typst.rs create mode 100644 src/model.rs create mode 100644 src/model/assessment.rs create mode 100644 src/model/bank.rs create mode 100644 src/model/catalog.rs create mode 100644 src/model/course.rs create mode 100644 src/model/item.rs create mode 100644 src/model/taxonomy.rs create mode 100644 src/util.rs create mode 100644 src/util/date.rs create mode 100644 src/util/hash.rs create mode 100644 src/util/markup.rs create mode 100644 src/util/rng.rs create mode 100644 src/util/yaml.rs create mode 100644 src/util/zipfile.rs diff --git a/src/analysis.rs b/src/analysis.rs new file mode 100644 index 0000000..12e148c --- /dev/null +++ b/src/analysis.rs @@ -0,0 +1,35 @@ +//! Psychometrics: what the responses say about the items and about the students. +//! +//! ```text +//! responses ──▶ classical ──┬──▶ students ──▶ reports +//! └──▶ irt ───────┤ +//! └──▶ calibrate ──▶ back onto the items +//! ``` +//! +//! [`classical`] is the one that changes what you do next. The corrected +//! point-biserial answers "did the students who knew the material get this right?", +//! and a negative one almost always means a keying error or a second defensible +//! reading of the stem. +//! +//! [`irt`] adds difficulty-aware ability estimates. Its priors are on by default +//! and that is not a stylistic choice: unpenalized maximum likelihood has no finite +//! solution for an item everyone answered correctly, and real classroom exams +//! contain those routinely. +//! +//! [`students`] turns item statistics into per-objective standing, and is careful +//! about what three questions can honestly support — it classifies on the observed +//! rate and reports confidence from the Wilson interval separately. +//! +//! [`calibrate`] is the arrow back to authoring, and the reason the system +//! compounds: statistics written onto an item are there the next time you consider +//! using it. It writes to [`crate::model::bank`] files, which is the one place this +//! module reaches upward, and it always shows a diff first. +//! +//! Everything here labels its own uncertainty. A point-biserial from twenty-four +//! examinees has a standard error near 0.2, and a tool that reports it to three +//! decimals without saying so is lying by omission. + +pub mod calibrate; +pub mod classical; +pub mod irt; +pub mod students; diff --git a/src/analysis/calibrate.rs b/src/analysis/calibrate.rs new file mode 100644 index 0000000..4daf17c --- /dev/null +++ b/src/analysis/calibrate.rs @@ -0,0 +1,857 @@ +//! Writing statistics back onto the items. +//! +//! This is the step that makes the whole system compound. Analysis that lives only +//! in a report gets read once; analysis written back onto the item is there the +//! next time you consider using it, and the linter can refuse to reuse a question +//! that behaved badly. +//! +//! Calibration accumulates. An item's `calibration` block is not overwritten +//! with the latest administration's numbers; the raw responses from every +//! administration are pooled and the statistics recomputed. +//! +//! Rewording resets it. Every calibration records the fingerprint of the item +//! text it was computed from. Change the stem or an option and the fingerprint +//! changes, the calibration is stale, and the linter says so. Retag the metadata +//! and nothing changes, because the fingerprint covers only what a student saw. +//! Without that rule, pooled statistics quietly become a mixture of two different +//! questions. +//! +//! Nothing is written without being shown. Calibration produces a diff you +//! read before it touches the file. The numbers here decide whether a question gets +//! used again, and a silent automated rewrite of a reviewed bank is not something +//! you want in a course repository. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::PathBuf; + +use crate::assessment::AssessmentFile; +use crate::bank::BankFile; +use crate::catalog::Catalog; +use crate::classical::{self, Analysis, ItemAnalysis, Thresholds}; +use crate::date::Date; +use crate::error::{Error, Result}; +use crate::irt::{self, Fit}; +use crate::item::{Calibration, IrtParams, OptionStat}; +use crate::responses::ResponseSet; +use crate::store::Store; +use crate::taxonomy::Flag; +use crate::yaml; + +/// What calibration would change about one item. +#[derive(Debug, Clone)] +pub struct Change { + /// The item's global id. + pub uid: String, + /// The bank file it lives in. + pub path: PathBuf, + /// The calibration that would be written. + pub calibration: Calibration, + /// Human-readable before-and-after lines. + pub diff: Vec, + /// Whether the item previously had no calibration at all. + pub is_new: bool, + /// Whether the previous calibration was computed from different item text. + pub was_stale: bool, +} + +/// The full plan, across items. +#[derive(Debug, Clone)] +pub struct Plan { + /// One entry per item whose calibration would change. + pub changes: Vec, + /// Items that were analyzed but could not be matched to a bank item. + pub unmatched: Vec, + /// Cautions worth printing before the diff. + pub warnings: Vec, + /// How many administrations were pooled. + pub administrations: Vec, +} + +impl Plan { + /// Whether anything would change. + pub fn is_empty(&self) -> bool { + self.changes.is_empty() + } + + /// Renders the plan for a terminal. + /// + /// # Returns + /// + /// A human-readable diff. + pub fn render(&self) -> String { + let mut out = String::new(); + if !self.administrations.is_empty() { + out.push_str(&format!( + "Pooling {} administration(s): {}\n\n", + self.administrations.len(), + self.administrations.join(", ") + )); + } + for w in &self.warnings { + out.push_str(&format!("! {w}\n")); + } + if !self.warnings.is_empty() { + out.push('\n'); + } + + if self.changes.is_empty() { + out.push_str("No calibration changes.\n"); + } + for c in &self.changes { + let tag = if c.is_new { + " (new)" + } else if c.was_stale { + " (previous calibration was computed from different text)" + } else { + "" + }; + out.push_str(&format!("{}{tag}\n", c.uid)); + for line in &c.diff { + out.push_str(&format!(" {line}\n")); + } + out.push('\n'); + } + + if !self.unmatched.is_empty() { + out.push_str(&format!( + "{} analyzed item(s) could not be traced to a bank item and were skipped: {}\n", + self.unmatched.len(), + self.unmatched.join(", ") + )); + } + out + } +} + +/// Options for building a plan. +#[derive(Debug, Clone)] +pub struct Options { + /// Classical thresholds. + pub thresholds: Thresholds, + /// Whether to fit IRT and record the parameters. + pub irt: bool, + /// IRT settings. + pub irt_options: irt::Options, + /// Whether to include practice assessments. Off by default: practice + /// conditions differ enough that pooling them contaminates the statistics. + pub include_practice: bool, + /// Minimum pooled examinees before writing anything at all. + pub minimum_n: usize, +} + +impl Default for Options { + fn default() -> Options { + Options { + thresholds: Thresholds::default(), + irt: true, + irt_options: irt::Options::default(), + include_practice: false, + minimum_n: 10, + } + } +} + +/// Builds a calibration plan by pooling every stored administration. +/// +/// # Arguments +/// +/// * `catalog` - the loaded course. +/// * `store` - the response store. +/// * `opts` - calibration options. +/// +/// # Returns +/// +/// The plan, which changes nothing until [`apply`] is called. +/// +/// # Errors +/// +/// Returns [`Error::Io`] when the store cannot be read. +pub fn plan(catalog: &Catalog, store: &Store, opts: &Options) -> Result { + let records = AssessmentFile::load_all(&catalog.layout.assessments())?; + let by_id: BTreeMap<&str, &AssessmentFile> = records + .iter() + .map(|r| (r.assessment.id.as_str(), r)) + .collect(); + + let stored = store.read_all()?; + let mut warnings = stored.warnings.clone(); + + // Analysis has to happen per administration, not per item. A corrected + // point-biserial is a correlation against the rest of that test, so it can + // only be computed while the whole administration is in hand. Pooling happens + // afterward, on the resulting numbers. + let mut per_admin: BTreeMap = BTreeMap::new(); + let mut administrations: BTreeSet = BTreeSet::new(); + + for admin in stored.administrations() { + let rows: Vec = stored + .rows + .iter() + .filter(|r| r.administration_id == admin) + .cloned() + .collect(); + let Some(first) = rows.first() else { continue }; + let record = by_id.get(first.assessment_id.as_str()).copied(); + + if let Some(rec) = record { + if !opts.include_practice && !rec.assessment.kind.counts_for_calibration() { + continue; + } + } + + let mut set = ResponseSet::new(); + set.rows = rows; + let analysis = classical::analyze(&set, &opts.thresholds, record, Some(catalog)); + administrations.insert(admin.clone()); + per_admin.insert(admin, analysis); + } + + // Now group the per-item results by the bank item they refer to. The same item + // may have been question 7 one term and question 12 the next. + let mut by_item: BTreeMap> = BTreeMap::new(); + let mut unmatched: BTreeSet = BTreeSet::new(); + + for (admin, analysis) in &per_admin { + for item in &analysis.items { + match &item.item_ref { + Some(uid) if catalog.get(uid).is_some() => { + by_item + .entry(uid.clone()) + .or_default() + .push((admin.clone(), item.clone())); + } + Some(uid) => { + unmatched.insert(uid.clone()); + } + None => { + unmatched.insert(format!("{admin}#{}", item.number)); + } + } + } + } + + if by_item.is_empty() { + warnings.push( + "no stored responses could be traced to bank items; check that `ingest` was run with \ + an assessment record so item references were recorded" + .to_string(), + ); + } + + // The IRT fit uses whichever administration has the most complete matrix, + // rather than a pooled matrix. Pooling responses across forms into one matrix + // would treat students who never saw an item as having missed it in a way the + // likelihood cannot distinguish from a linked design, and honest linking is a + // bigger problem than this tool should pretend to solve. + let irt_fit = if opts.irt { + best_fit(&stored, opts) + } else { + None + }; + if opts.irt && irt_fit.is_none() { + warnings.push( + "IRT was requested but no single administration had enough data to fit; classical \ + statistics will still be written" + .to_string(), + ); + } + if let Some((admin, fit)) = &irt_fit { + warnings.extend(fit.warnings.clone()); + warnings.push(format!( + "IRT parameters come from a single administration ({admin}) rather than from the \ + pooled data, because linking across forms is not attempted" + )); + } + + let mut changes = Vec::new(); + + for (uid, appearances) in &by_item { + let entry = match catalog.get(uid) { + Some(e) => e, + None => continue, + }; + + let pooled = pool(appearances); + if pooled.n < opts.minimum_n { + continue; + } + + // The IRT parameters come from whichever single administration was fitted, + // matched by the question number this item held there. + let irt_params = match &irt_fit { + Some((fitted_admin, fit)) => appearances + .iter() + .find(|(admin, _)| admin == fitted_admin) + .and_then(|(_, item)| fit.items.iter().find(|i| i.number == item.number)) + .map(|i| i.to_params()), + None => None, + }; + + let calibration = Calibration { + administrations: appearances.iter().map(|(a, _)| a.clone()).collect(), + updated: Some(Date::today()), + fingerprint: Some(entry.item.fingerprint()), + n_examinees: Some(pooled.n), + p_value: Some(round4(pooled.p_value)), + point_biserial: pooled.point_biserial.map(round4), + discrimination_index: pooled.discrimination_index.map(round4), + mean_response_time_seconds: None, + rapid_guess_rate: None, + option_stats: pooled.option_stats.clone(), + irt: irt_params, + flags: pooled.flags.clone(), + }; + + let previous = entry.item.calibration.as_ref(); + let diff = diff_calibration(previous, &calibration); + if diff.is_empty() { + continue; + } + + changes.push(Change { + uid: uid.clone(), + path: entry.path.clone(), + calibration, + diff, + is_new: previous.is_none(), + was_stale: previous + .map(|p| p.fingerprint.as_deref() != Some(entry.item.fingerprint().as_str())) + .unwrap_or(false), + }); + } + + Ok(Plan { + changes, + unmatched: unmatched.into_iter().collect(), + warnings, + administrations: administrations.into_iter().collect(), + }) +} + +/// Pooled statistics for one item. +struct Pooled { + /// Total examinees across administrations. + n: usize, + /// Examinee-weighted difficulty. + p_value: f64, + /// Examinee-weighted point-biserial. + point_biserial: Option, + /// Examinee-weighted discrimination index. + discrimination_index: Option, + /// Pooled per-option statistics. + option_stats: BTreeMap, + /// The union of flags raised in any administration. + flags: Vec, +} + +/// Pools one item's statistics across the administrations it appeared in. +/// +/// Difficulty pools by weighted average, since a proportion correct is comparable +/// across administrations of the same text. Discrimination is also averaged rather +/// than recomputed, and that is the important subtlety: a corrected point-biserial +/// is a correlation against the rest of that test, so the only meaningful pooled +/// value is a weighted average of the within-administration correlations. Merging +/// response matrices from different exams and correlating across the whole thing +/// would produce a number that looks more precise and means less. +/// +/// # Arguments +/// +/// * `appearances` - the administration id and item analysis for each appearance. +/// +/// # Returns +/// +/// The pooled statistics. +fn pool(appearances: &[(String, ItemAnalysis)]) -> Pooled { + let mut total_n = 0usize; + let mut p_weighted = 0.0; + let mut rpb_weighted = 0.0; + let mut rpb_weight = 0.0; + let mut d_weighted = 0.0; + let mut d_weight = 0.0; + let mut flags: BTreeSet = BTreeSet::new(); + + // Per-option accumulators, since option letters are stable across forms even + // when the printed order is not. + let mut rate_weighted: BTreeMap = BTreeMap::new(); + let mut option_rpb: BTreeMap = BTreeMap::new(); + let mut upper: BTreeMap = BTreeMap::new(); + let mut lower: BTreeMap = BTreeMap::new(); + + for (_, item) in appearances { + let n = item.n as f64; + total_n += item.n; + p_weighted += item.p_value * n; + + if let Some(r) = item.point_biserial { + rpb_weighted += r * n; + rpb_weight += n; + } + if let Some(d) = item.discrimination_index { + d_weighted += d * n; + d_weight += n; + } + for f in &item.flags { + flags.insert(*f); + } + for (letter, o) in &item.options { + *rate_weighted.entry(letter.clone()).or_insert(0.0) += o.rate * n; + if let Some(r) = o.point_biserial { + let e = option_rpb.entry(letter.clone()).or_insert((0.0, 0.0)); + e.0 += r * n; + e.1 += n; + } + if let Some(r) = o.upper_rate { + let e = upper.entry(letter.clone()).or_insert((0.0, 0.0)); + e.0 += r * n; + e.1 += n; + } + if let Some(r) = o.lower_rate { + let e = lower.entry(letter.clone()).or_insert((0.0, 0.0)); + e.0 += r * n; + e.1 += n; + } + } + } + + let denominator = total_n.max(1) as f64; + let average = |m: &BTreeMap, letter: &str| -> Option { + m.get(letter) + .filter(|(_, w)| *w > 0.0) + .map(|(sum, w)| round4(sum / w)) + }; + + let option_stats: BTreeMap = rate_weighted + .keys() + .map(|letter| { + ( + letter.clone(), + OptionStat { + selection_rate: Some(round4(rate_weighted[letter] / denominator)), + point_biserial: average(&option_rpb, letter), + upper_group_rate: average(&upper, letter), + lower_group_rate: average(&lower, letter), + }, + ) + }) + .collect(); + + Pooled { + n: total_n, + p_value: p_weighted / denominator, + point_biserial: if rpb_weight > 0.0 { + Some(rpb_weighted / rpb_weight) + } else { + None + }, + discrimination_index: if d_weight > 0.0 { + Some(d_weighted / d_weight) + } else { + None + }, + option_stats, + flags: flags.into_iter().collect(), + } +} + +/// Picks the administration with the most complete matrix and fits IRT to it. +/// +/// # Arguments +/// +/// * `stored` - every stored response. +/// * `opts` - calibration options. +/// +/// # Returns +/// +/// The administration id and its fit, or `None` when none is large enough. +fn best_fit(stored: &ResponseSet, opts: &Options) -> Option<(String, Fit)> { + let mut best: Option<(String, usize, usize)> = None; + for admin in stored.administrations() { + let mut set = ResponseSet::new(); + set.rows = stored + .rows + .iter() + .filter(|r| r.administration_id == admin) + .cloned() + .collect(); + let m = set.matrix(false); + let cells = m.n_students() * m.n_items(); + if m.n_students() < opts.minimum_n || m.n_items() < 5 { + continue; + } + if best.as_ref().map(|(_, c, _)| cells > *c).unwrap_or(true) { + best = Some((admin, cells, m.n_items())); + } + } + + let (admin, _, _) = best?; + let mut set = ResponseSet::new(); + set.rows = stored + .rows + .iter() + .filter(|r| r.administration_id == admin) + .cloned() + .collect(); + let fit = irt::fit(&set.matrix(false), &opts.irt_options); + Some((admin, fit)) +} + +/// Describes the difference between two calibrations. +/// +/// # Arguments +/// +/// * `previous` - the existing calibration, if any. +/// * `next` - the computed calibration. +/// +/// # Returns +/// +/// One line per changed field; empty when nothing meaningful changed. +fn diff_calibration(previous: Option<&Calibration>, next: &Calibration) -> Vec { + let mut out = Vec::new(); + let show = |label: &str, before: Option, after: Option, out: &mut Vec| match ( + before, after, + ) { + (Some(b), Some(a)) if (b - a).abs() > 5e-4 => { + out.push(format!("{label}: {b:.3} -> {a:.3}")); + } + (None, Some(a)) => out.push(format!("{label}: (none) -> {a:.3}")), + _ => {} + }; + + let p = previous; + show("p-value", p.and_then(|c| c.p_value), next.p_value, &mut out); + show( + "point-biserial", + p.and_then(|c| c.point_biserial), + next.point_biserial, + &mut out, + ); + show( + "discrimination index", + p.and_then(|c| c.discrimination_index), + next.discrimination_index, + &mut out, + ); + + let before_n = p.and_then(|c| c.n_examinees).unwrap_or(0); + if let Some(n) = next.n_examinees { + if n != before_n { + out.push(format!("examinees: {before_n} -> {n}")); + } + } + + let before_flags: BTreeSet = p + .map(|c| c.flags.iter().copied().collect()) + .unwrap_or_default(); + let after_flags: BTreeSet = next.flags.iter().copied().collect(); + let added: Vec<&str> = after_flags + .difference(&before_flags) + .map(|f| f.as_str()) + .collect(); + let removed: Vec<&str> = before_flags + .difference(&after_flags) + .map(|f| f.as_str()) + .collect(); + if !added.is_empty() { + out.push(format!("flags added: {}", added.join(", "))); + } + if !removed.is_empty() { + out.push(format!("flags cleared: {}", removed.join(", "))); + } + + match (p.and_then(|c| c.irt.as_ref()), next.irt.as_ref()) { + (Some(b), Some(a)) if (b.a - a.a).abs() > 5e-3 || (b.b - a.b).abs() > 5e-3 => { + out.push(format!( + "IRT: a {:.2} -> {:.2}, b {:+.2} -> {:+.2}", + b.a, a.a, b.b, a.b + )); + } + (None, Some(a)) => out.push(format!("IRT: (none) -> a {:.2}, b {:+.2}", a.a, a.b)), + _ => {} + } + + if p.map(|c| c.fingerprint.as_deref()) != Some(next.fingerprint.as_deref()) { + out.push("fingerprint updated to the current item text".to_string()); + } + + out +} + +/// Applies a plan, rewriting the affected bank files. +/// +/// Files are rewritten one at a time and each is re-read before editing, so a plan +/// built against a bank that has since changed on disk fails loudly rather than +/// clobbering the newer version. +/// +/// # Arguments +/// +/// * `plan` - the plan to apply. +/// +/// # Returns +/// +/// The bank files rewritten. +/// +/// # Errors +/// +/// Returns [`Error::Unresolved`] when an item in the plan is no longer in its bank, +/// and [`Error::Io`] on a write failure. +pub fn apply(plan: &Plan) -> Result> { + // Group by file so each is read and written once. + let mut by_file: BTreeMap<&PathBuf, Vec<&Change>> = BTreeMap::new(); + for change in &plan.changes { + by_file.entry(&change.path).or_default().push(change); + } + + let mut written = Vec::new(); + for (path, changes) in by_file { + let mut bank: BankFile = yaml::read(path)?; + for change in changes { + // The uid is `bank::item`; match on the item part. + let item_id = change + .uid + .split_once("::") + .map(|(_, id)| id) + .unwrap_or(&change.uid); + let target = bank.items.iter_mut().find(|i| i.id == item_id); + match target { + Some(item) => item.calibration = Some(change.calibration.clone()), + None => { + return Err(Error::Unresolved { + kind: "item", + id: change.uid.clone(), + context: Some(format!( + "{} — the bank changed since the plan was built; re-run calibration", + path.display() + )), + }) + } + } + } + yaml::write(path, &bank)?; + written.push(path.clone()); + } + Ok(written) +} + +/// Builds a plan for a single administration, from an in-memory analysis. +/// +/// Useful right after an exam, before deciding whether to drop a question. +/// +/// # Arguments +/// +/// * `catalog` - the loaded course. +/// * `record` - the assessment record. +/// * `analysis` - the analysis of that administration. +/// * `fit` - an optional IRT fit. +/// +/// # Returns +/// +/// The plan. +pub fn plan_from_analysis( + catalog: &Catalog, + record: &AssessmentFile, + analysis: &Analysis, + fit: Option<&Fit>, +) -> Plan { + let admin = crate::responses::administration_id( + &catalog.course.course.code, + record + .assessment + .term + .as_deref() + .unwrap_or(&catalog.course.course.term), + &record.assessment.id, + ); + + let mut changes = Vec::new(); + let mut unmatched = Vec::new(); + + for item in &analysis.items { + let Some(uid) = item.item_ref.clone() else { + unmatched.push(format!("question {}", item.number)); + continue; + }; + let Some(entry) = catalog.get(&uid) else { + unmatched.push(uid); + continue; + }; + + let irt_params: Option = fit.and_then(|f| { + f.items + .iter() + .find(|i| i.number == item.number) + .map(|i| i.to_params()) + }); + + let calibration = Calibration { + administrations: vec![admin.clone()], + updated: Some(Date::today()), + fingerprint: Some(entry.item.fingerprint()), + n_examinees: Some(item.n), + p_value: Some(round4(item.p_value)), + point_biserial: item.point_biserial.map(round4), + discrimination_index: item.discrimination_index.map(round4), + mean_response_time_seconds: None, + rapid_guess_rate: None, + option_stats: item + .options + .iter() + .map(|(letter, o)| (letter.clone(), o.to_option_stat())) + .collect(), + irt: irt_params, + flags: item.flags.clone(), + }; + + let previous = entry.item.calibration.as_ref(); + let diff = diff_calibration(previous, &calibration); + if diff.is_empty() { + continue; + } + changes.push(Change { + uid, + path: entry.path.clone(), + calibration, + diff, + is_new: previous.is_none(), + was_stale: previous + .map(|p| p.fingerprint.as_deref() != Some(entry.item.fingerprint().as_str())) + .unwrap_or(false), + }); + } + + Plan { + changes, + unmatched, + warnings: analysis.warnings.clone(), + administrations: vec![admin], + } +} + +/// Rounds to four decimals. +fn round4(x: f64) -> f64 { + (x * 1e4).round() / 1e4 +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::item::IrtModel; + + fn calibration(p: f64, rpb: Option, n: usize, fingerprint: &str) -> Calibration { + Calibration { + administrations: vec!["C/2026S/e1".into()], + updated: None, + fingerprint: Some(fingerprint.to_string()), + n_examinees: Some(n), + p_value: Some(p), + point_biserial: rpb, + discrimination_index: None, + mean_response_time_seconds: None, + rapid_guess_rate: None, + option_stats: BTreeMap::new(), + irt: None, + flags: Vec::new(), + } + } + + #[test] + fn a_first_calibration_is_all_new() { + let next = calibration(0.7, Some(0.3), 24, "abc"); + let diff = diff_calibration(None, &next); + assert!(diff.iter().any(|d| d.contains("p-value: (none)"))); + assert!(diff.iter().any(|d| d.contains("examinees: 0 -> 24"))); + } + + #[test] + fn identical_calibrations_produce_no_diff() { + let previous = calibration(0.7, Some(0.3), 24, "abc"); + let next = calibration(0.7, Some(0.3), 24, "abc"); + assert!(diff_calibration(Some(&previous), &next).is_empty()); + } + + #[test] + fn tiny_changes_are_not_reported() { + let previous = calibration(0.7000, Some(0.30), 24, "abc"); + let next = calibration(0.7001, Some(0.30), 24, "abc"); + assert!( + diff_calibration(Some(&previous), &next).is_empty(), + "a change below the display precision is noise" + ); + } + + #[test] + fn a_changed_fingerprint_is_called_out() { + let previous = calibration(0.7, Some(0.3), 24, "old-text"); + let next = calibration(0.7, Some(0.3), 24, "new-text"); + let diff = diff_calibration(Some(&previous), &next); + assert!(diff.iter().any(|d| d.contains("fingerprint"))); + } + + #[test] + fn flag_changes_are_reported_in_both_directions() { + let mut previous = calibration(0.7, Some(0.3), 24, "abc"); + previous.flags = vec![Flag::TooEasy]; + let mut next = calibration(0.7, Some(0.3), 24, "abc"); + next.flags = vec![Flag::NegativeDiscrimination]; + + let diff = diff_calibration(Some(&previous), &next); + assert!(diff + .iter() + .any(|d| d.contains("flags added") && d.contains("negative_discrimination"))); + assert!(diff + .iter() + .any(|d| d.contains("flags cleared") && d.contains("too_easy"))); + } + + #[test] + fn irt_changes_are_reported() { + let previous = calibration(0.7, None, 24, "abc"); + let mut next = calibration(0.7, None, 24, "abc"); + next.irt = Some(IrtParams { + model: IrtModel::TwoPl, + a: 1.2, + b: -0.4, + c: None, + se_a: None, + se_b: None, + n: Some(24), + bayesian: true, + }); + let diff = diff_calibration(Some(&previous), &next); + assert!(diff.iter().any(|d| d.contains("IRT: (none)"))); + } + + #[test] + fn an_empty_plan_renders_readably() { + let plan = Plan { + changes: Vec::new(), + unmatched: Vec::new(), + warnings: Vec::new(), + administrations: Vec::new(), + }; + assert!(plan.is_empty()); + assert!(plan.render().contains("No calibration changes")); + } + + #[test] + fn a_plan_renders_its_diff_and_warnings() { + let plan = Plan { + changes: vec![Change { + uid: "bank::q-a-001".into(), + path: PathBuf::from("banks/bank.yaml"), + calibration: calibration(0.7, Some(0.3), 24, "abc"), + diff: vec!["p-value: (none) -> 0.700".into()], + is_new: true, + was_stale: false, + }], + unmatched: vec!["question 9".into()], + warnings: vec!["only 24 examinees".into()], + administrations: vec!["C/2026S/e1".into()], + }; + let text = plan.render(); + assert!(text.contains("bank::q-a-001 (new)")); + assert!(text.contains("p-value")); + assert!(text.contains("only 24 examinees")); + assert!(text.contains("question 9")); + assert!(text.contains("Pooling 1 administration")); + } +} diff --git a/src/analysis/classical.rs b/src/analysis/classical.rs new file mode 100644 index 0000000..ce74c3a --- /dev/null +++ b/src/analysis/classical.rs @@ -0,0 +1,1037 @@ +//! Classical item analysis. +//! +//! The corrected point-biserial is the one to look at first. It correlates +//! each student's response with their total score on the other items, which is +//! the only version that answers the question you care about: did students who +//! knew the material get this right? The uncorrected version includes the item in +//! its own criterion and is therefore biased upward, which flatters bad items. +//! A negative corrected point-biserial almost always means the key is wrong or +//! the stem has a second defensible reading, and it is the single most reliable +//! signal in the whole tool. +//! +//! The p-value is difficulty, and on its own it means very little. An item +//! everyone answered correctly is not a bad item; it may be a deliberate anchor. +//! An item everyone missed is only a problem if it also failed to discriminate. +//! +//! Distractor analysis is where poorly worded questions reveal themselves. If +//! the strongest quarter of the class picked distractor B at a higher rate than +//! the key, B is either the better answer or the stem is ambiguous. That is a +//! different diagnosis from "hard," and it calls for rewriting rather than +//! reteaching. +//! +//! One convention worth stating: a blank response counts as incorrect for +//! difficulty and correlations, because on a scored exam it is worth zero and +//! excluding it would make an item that students skipped look easier than it was. +//! The blank rate is reported separately so a widely skipped item is still visible +//! as such. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::assessment::AssessmentFile; +use crate::catalog::Catalog; +use crate::item::{Design, OptionStat}; +use crate::responses::ResponseSet; +use crate::taxonomy::Flag; + +/// Cut points for flagging items. +#[derive(Debug, Clone)] +pub struct Thresholds { + /// A p-value above this is "too easy". + pub too_easy: f64, + /// A p-value below this is "too hard", but only when discrimination is also + /// poor: a hard item that separates students is doing its job. + pub too_hard: f64, + /// A point-biserial below this is weak discrimination. + pub low_discrimination: f64, + /// A point-biserial below this is treated as genuinely *negative*, which is + /// the blocking "check your key" finding. + /// + /// This is not zero, and the gap matters. In a class of twenty-four the + /// standard error of a point-biserial is around 0.2, so an observed -0.004 is + /// indistinguishable from no relationship at all. Alarming on it would send + /// you hunting for a keying error that is not there, and after a few false + /// alarms the flag stops being read. Values in the dead zone are reported as + /// low discrimination instead, which is what they actually are. + pub negative_discrimination: f64, + /// A selection rate at or below this makes a distractor nonfunctioning. + pub nonfunctioning: f64, + /// How far observed difficulty may drift from the authored expectation. + pub design_tolerance: f64, + /// Fraction of the class in the upper and lower comparison groups. Kelley's + /// 0.27 maximizes the difference between the groups for a normal + /// distribution, and it remains the convention. + pub group_fraction: f64, + /// Below this many examinees, statistics are reported with a caution. + pub small_sample: usize, +} + +impl Default for Thresholds { + fn default() -> Thresholds { + Thresholds { + too_easy: 0.95, + too_hard: 0.25, + low_discrimination: 0.15, + negative_discrimination: -0.05, + nonfunctioning: 0.05, + design_tolerance: 0.25, + group_fraction: 0.27, + small_sample: 100, + } + } +} + +/// Statistics for one option. +#[derive(Debug, Clone)] +pub struct OptionAnalysis { + /// The option letter. + pub letter: String, + /// How many students chose it. + pub count: usize, + /// Fraction of responses that chose it. + pub rate: f64, + /// Correlation between choosing this option and total score on other items. + /// Should be strongly positive for the key and negative for distractors. + pub point_biserial: Option, + /// Selection rate in the upper group. + pub upper_rate: Option, + /// Selection rate in the lower group. + pub lower_rate: Option, + /// Whether this option is keyed. + pub is_key: bool, + /// Mean credit awarded to students who chose it, which exposes partial credit + /// granted during grading. + pub mean_credit: f64, +} + +impl OptionAnalysis { + /// Converts to the form stored on an item's calibration block. + pub fn to_option_stat(&self) -> OptionStat { + OptionStat { + selection_rate: Some(self.rate), + point_biserial: self.point_biserial, + upper_group_rate: self.upper_rate, + lower_group_rate: self.lower_rate, + } + } +} + +/// Statistics for one item. +#[derive(Debug, Clone)] +pub struct ItemAnalysis { + /// The question number on the form. + pub number: u32, + /// The item's global id, when known. + pub item_ref: Option, + /// How many students the item was administered to. + pub n: usize, + /// How many gave a non-blank response. + pub n_answered: usize, + /// Fraction blank. + pub blank_rate: f64, + /// Proportion correct, counting blanks as incorrect. + pub p_value: f64, + /// Mean credit earned, which differs from `p_value` when partial credit was + /// awarded. + pub mean_credit: f64, + /// Corrected item-total point-biserial. `None` when the item has no variance, + /// which happens whenever everyone answered the same way. + pub point_biserial: Option, + /// Upper-group minus lower-group proportion correct. + pub discrimination_index: Option, + /// Proportion correct in the upper group. + pub upper_rate: Option, + /// Proportion correct in the lower group. + pub lower_rate: Option, + /// The keyed letters. + pub key: Vec, + /// Per-option statistics, by letter. + pub options: BTreeMap, + /// Machine-detected problems. + pub flags: Vec, + /// Human-readable explanations tied to the flags. + pub notes: Vec, +} + +impl ItemAnalysis { + /// Whether the item needs attention before reuse. + pub fn needs_revision(&self) -> bool { + self.flags.iter().any(|f| f.is_blocking()) + } + + /// The item's most serious flag, for sorting a work queue. + pub fn worst_flag(&self) -> Option { + self.flags + .iter() + .copied() + .find(|f| f.is_blocking()) + .or_else(|| self.flags.first().copied()) + } +} + +/// Whole-test reliability. +#[derive(Debug, Clone)] +pub struct Reliability { + /// Number of scored items. + pub n_items: usize, + /// Number of examinees. + pub n_students: usize, + /// Mean total score, in items correct. + pub mean: f64, + /// Standard deviation of total score. + pub sd: f64, + /// KR-20, which is Cronbach's alpha for dichotomous items. `None` when there + /// is too little variance to compute it. + pub alpha: Option, + /// Standard error of measurement, in the same units as the total score. + pub sem: Option, + /// Mean p-value across items. + pub mean_p: f64, + /// Mean point-biserial across items that had one. + pub mean_point_biserial: Option, +} + +impl Reliability { + /// A plain-language reading of the alpha value. + /// + /// Interpretation bands are worth printing because alpha is routinely + /// over-read: it depends on test length as much as item quality, so a + /// thirty-item classroom exam at 0.6 is unremarkable, not broken. + /// + /// # Returns + /// + /// A sentence about what the value means for this test. + pub fn interpretation(&self) -> String { + let Some(alpha) = self.alpha else { + return "Reliability could not be computed: there is not enough variation in total \ + scores." + .to_string(); + }; + let band = if alpha >= 0.9 { + "very high, typical of a long standardized test" + } else if alpha >= 0.8 { + "high" + } else if alpha >= 0.7 { + "acceptable for a classroom exam" + } else if alpha >= 0.6 { + "modest, which is common for a short exam in a small class" + } else if alpha >= 0.5 { + "low; treat individual student scores as rough" + } else { + "very low; the total score is not measuring one thing consistently" + }; + let mut s = format!("KR-20 is {alpha:.2} ({band})."); + if let Some(sem) = self.sem { + s.push_str(&format!( + " The standard error of measurement is {sem:.1} items, so a student's true score \ + is roughly within ±{:.0} items of what they scored.", + sem * 1.96 + )); + } + if self.n_items < 20 { + s.push_str( + " Alpha rises with test length, so a short test understates how well its items \ + work.", + ); + } + s + } +} + +/// The result of analyzing one administration. +#[derive(Debug, Clone)] +pub struct Analysis { + /// Per-item statistics, in question order. + pub items: Vec, + /// Whole-test reliability. + pub reliability: Reliability, + /// Cautions about the analysis itself. + pub warnings: Vec, +} + +impl Analysis { + /// Items that need revision, worst first. + /// + /// # Returns + /// + /// References to the flagged items, blocking flags before advisory ones. + pub fn revise_queue(&self) -> Vec<&ItemAnalysis> { + let mut out: Vec<&ItemAnalysis> = + self.items.iter().filter(|i| !i.flags.is_empty()).collect(); + out.sort_by(|a, b| { + b.needs_revision() + .cmp(&a.needs_revision()) + .then_with(|| { + a.point_biserial + .unwrap_or(1.0) + .partial_cmp(&b.point_biserial.unwrap_or(1.0)) + .unwrap_or(std::cmp::Ordering::Equal) + }) + .then_with(|| a.number.cmp(&b.number)) + }); + out + } +} + +/// Runs item analysis over one administration. +/// +/// # Arguments +/// +/// * `set` - the responses. Only scored, undropped items are analyzed. +/// * `t` - flag thresholds. +/// * `record` - the assessment record, for keys and item references. +/// * `catalog` - the loaded course, for authored expectations. +/// +/// # Returns +/// +/// The analysis, including cautions when the sample is too small to trust. +pub fn analyze( + set: &ResponseSet, + t: &Thresholds, + record: Option<&AssessmentFile>, + catalog: Option<&Catalog>, +) -> Analysis { + let matrix = set.matrix(false); + let mut warnings = Vec::new(); + + if !matrix.is_analyzable() { + return Analysis { + items: Vec::new(), + reliability: Reliability { + n_items: 0, + n_students: 0, + mean: 0.0, + sd: 0.0, + alpha: None, + sem: None, + mean_p: 0.0, + mean_point_biserial: None, + }, + warnings: vec![ + "there are no scored responses to analyze; check that ingest matched the \ + assessment record" + .to_string(), + ], + }; + } + + let n_students = matrix.n_students(); + if n_students < t.small_sample { + warnings.push(format!( + "these statistics come from {n_students} examinees. Point-biserials from a class this \ + size have a standard error near {:.2}, so treat anything between -0.2 and 0.2 as \ + indistinguishable from zero and pool several administrations before retiring an item", + 1.0 / ((n_students as f64 - 1.0).max(1.0)).sqrt() + )); + } + + // Blanks count as incorrect for scoring purposes. + let coded: Vec> = matrix + .coded + .iter() + .map(|row| row.iter().map(|c| c.unwrap_or(0) as f64).collect()) + .collect(); + let totals: Vec = coded.iter().map(|row| row.iter().sum()).collect(); + + // Upper and lower groups by total score, Kelley's fraction. + let group_size = ((n_students as f64 * t.group_fraction).round() as usize).max(1); + let mut order: Vec = (0..n_students).collect(); + order.sort_by(|a, b| { + totals[*b] + .partial_cmp(&totals[*a]) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| matrix.students[*a].cmp(&matrix.students[*b])) + }); + let upper: BTreeSet = order.iter().take(group_size).copied().collect(); + let lower: BTreeSet = order.iter().rev().take(group_size).copied().collect(); + let groups_usable = n_students >= 6 && upper.is_disjoint(&lower); + if !groups_usable && n_students > 0 { + warnings.push(format!( + "with {n_students} examinees the upper and lower comparison groups would overlap, so \ + the discrimination index is omitted; the point-biserial uses the whole class and is \ + reported instead" + )); + } + + let mut items = Vec::new(); + let mut p_values = Vec::new(); + let mut rpbs = Vec::new(); + + for (j, number) in matrix.items.iter().enumerate() { + let rows = set.for_item(*number); + let x: Vec = coded.iter().map(|r| r[j]).collect(); + let rest: Vec = totals + .iter() + .zip(x.iter()) + .map(|(total, xi)| total - xi) + .collect(); + + let n = matrix.n_students(); + let n_answered = matrix.coded.iter().filter(|r| r[j].is_some()).count(); + let p_value = mean(&x); + let rpb = correlation(&x, &rest); + + let (upper_rate, lower_rate, discrimination_index) = if groups_usable { + let u = mean_of(&x, &upper); + let l = mean_of(&x, &lower); + (Some(u), Some(l), Some(u - l)) + } else { + (None, None, None) + }; + + // Keys: prefer the record, fall back to what the data says earned credit. + let key: Vec = record + .and_then(|r| r.placement(*number)) + .map(|p| p.key.clone()) + .filter(|k| !k.is_empty()) + .unwrap_or_else(|| infer_key(&rows)); + + let mean_credit = if rows.is_empty() { + 0.0 + } else { + rows.iter().map(|r| r.credit).sum::() / rows.len() as f64 + }; + + // Per-option statistics. + let student_index: BTreeMap<&str, usize> = matrix + .students + .iter() + .enumerate() + .map(|(i, s)| (s.as_str(), i)) + .collect(); + let mut chose: BTreeMap> = BTreeMap::new(); + let mut credits: BTreeMap> = BTreeMap::new(); + let mut blank = 0usize; + for r in &rows { + if r.selected.is_empty() { + blank += 1; + continue; + } + // A multiple-response item is credited to the joined set, so that + // "chose A and C" is one response pattern rather than two options. + let label = r.selected.join("+"); + if let Some(&si) = student_index.get(r.student_key.as_str()) { + chose.entry(label.clone()).or_default().push(si); + } + credits.entry(label).or_default().push(r.credit); + } + + // Every declared option appears, even one nobody chose: a rate of zero is + // the finding. + let mut letters: BTreeSet = chose.keys().cloned().collect(); + if let (Some(rec), Some(cat)) = (record, catalog) { + if let Some(p) = rec.placement(*number) { + if let Some(entry) = cat.get(&p.item) { + for o in &entry.item.options { + letters.insert(o.id.clone()); + } + } + } + } + + let responded = rows.len().max(1); + let mut options = BTreeMap::new(); + for letter in letters { + let indices = chose.get(&letter).cloned().unwrap_or_default(); + let count = indices.len(); + let indicator: Vec = (0..n) + .map(|i| if indices.contains(&i) { 1.0 } else { 0.0 }) + .collect(); + let set_indices: BTreeSet = indices.iter().copied().collect(); + let cr = credits.get(&letter).cloned().unwrap_or_default(); + options.insert( + letter.clone(), + OptionAnalysis { + letter: letter.clone(), + count, + rate: count as f64 / responded as f64, + point_biserial: correlation(&indicator, &rest), + upper_rate: if groups_usable { + Some(mean_of(&indicator, &upper)) + } else { + None + }, + lower_rate: if groups_usable { + Some(mean_of(&indicator, &lower)) + } else { + None + }, + is_key: key.contains(&letter) || (key.len() > 1 && letter == key.join("+")), + mean_credit: if cr.is_empty() { + 0.0 + } else { + cr.iter().sum::() / cr.len() as f64 + }, + }, + ); + let _ = set_indices; + } + + let design = record + .and_then(|r| r.placement(*number)) + .and_then(|p| catalog.and_then(|c| c.get(&p.item))) + .and_then(|e| e.item.design.clone()); + + let mut analysis = ItemAnalysis { + number: *number, + item_ref: record + .and_then(|r| r.placement(*number)) + .map(|p| p.item.clone()), + n, + n_answered, + blank_rate: blank as f64 / responded as f64, + p_value, + mean_credit, + point_biserial: rpb, + discrimination_index, + upper_rate, + lower_rate, + key, + options, + flags: Vec::new(), + notes: Vec::new(), + }; + + flag_item(&mut analysis, t, design.as_ref(), &rows); + + p_values.push(p_value); + if let Some(r) = rpb { + rpbs.push(r); + } + items.push(analysis); + } + + let reliability = reliability(&coded, &totals, &p_values, &rpbs); + + Analysis { + items, + reliability, + warnings, + } +} + +/// Applies the flag rules to one item. +/// +/// # Arguments +/// +/// * `a` - the item analysis, updated in place. +/// * `t` - the thresholds. +/// * `design` - the authored expectation, when available. +/// * `rows` - the raw responses, for partial-credit detection. +fn flag_item( + a: &mut ItemAnalysis, + t: &Thresholds, + design: Option<&Design>, + rows: &[&crate::responses::Response], +) { + // Discrimination first: it is the finding that changes what you do. + match a.point_biserial { + Some(r) if r < t.negative_discrimination => { + a.flags.push(Flag::NegativeDiscrimination); + a.notes.push(format!( + "students who did better overall did worse on this item (r = {r:.2}). Check the \ + key before anything else." + )); + } + Some(r) if r < t.low_discrimination => { + a.flags.push(Flag::LowDiscrimination); + a.notes.push(if r < 0.0 { + format!( + "this item did not separate stronger from weaker students (r = {r:.2}, which \ + is indistinguishable from zero at this sample size)." + ) + } else { + format!("this item barely separates stronger from weaker students (r = {r:.2}).") + }); + } + None => { + a.notes.push( + "every student responded the same way, so this item has no variance and no \ + correlation can be computed." + .to_string(), + ); + } + _ => {} + } + + if a.p_value > t.too_easy { + a.flags.push(Flag::TooEasy); + a.notes.push(format!( + "{:.0}% answered correctly. Fine as an opening anchor, but it carries little \ + information about who knows what.", + a.p_value * 100.0 + )); + } + if a.p_value < t.too_hard { + // Hard and discriminating is a good item, not a broken one. + let discriminates = a + .point_biserial + .map(|r| r >= t.low_discrimination) + .unwrap_or(false); + if !discriminates { + a.flags.push(Flag::TooHard); + a.notes.push(format!( + "only {:.0}% answered correctly, and the item did not separate students. That \ + pattern usually means the stem is unclear or a prerequisite is missing, not that \ + the content is hard.", + a.p_value * 100.0 + )); + } + } + + // Distractor analysis: the real source of "poorly worded question" findings. + let key_rpb = a + .options + .values() + .filter(|o| o.is_key) + .filter_map(|o| o.point_biserial) + .fold(f64::NEG_INFINITY, f64::max); + + for o in a.options.values() { + if o.is_key { + continue; + } + if let Some(r) = o.point_biserial { + if key_rpb.is_finite() && r > key_rpb && o.rate >= 0.1 { + a.flags.push(Flag::DistractorOutperformsKey); + a.notes.push(format!( + "option {} correlates with overall performance better than the key does \ + (r = {r:.2} against {key_rpb:.2}), and {:.0}% chose it. Either it is the \ + better answer or the stem admits it.", + o.letter, + o.rate * 100.0 + )); + } + } + if let (Some(upper), Some(key_upper)) = ( + o.upper_rate, + a.options + .values() + .filter(|k| k.is_key) + .filter_map(|k| k.upper_rate) + .fold(None, |acc: Option, v| { + Some(acc.map_or(v, |a| a.max(v))) + }), + ) { + if upper > key_upper && upper >= 0.25 { + a.flags.push(Flag::KeyUnderperforms); + a.notes.push(format!( + "the strongest students chose option {} more often than the key ({:.0}% \ + against {:.0}%). That split is the signature of two defensible readings.", + o.letter, + upper * 100.0, + key_upper * 100.0 + )); + } + } + if o.rate <= t.nonfunctioning { + a.flags.push(Flag::NonfunctioningDistractor); + a.notes.push(format!( + "option {} was chosen by {:.0}% of students, so it is not doing any work. \ + Replace it with a plausible error students actually make.", + o.letter, + o.rate * 100.0 + )); + } + } + + // Partial credit awarded to a non-key option is a grading-time admission of + // ambiguity, and it is the strongest such signal available. + let ambiguous = a + .options + .values() + .any(|o| !o.is_key && o.mean_credit > 0.0 && o.count > 0); + if ambiguous { + a.flags.push(Flag::Ambiguous); + let letters: Vec = a + .options + .values() + .filter(|o| !o.is_key && o.mean_credit > 0.0 && o.count > 0) + .map(|o| format!("{} ({:.0}%)", o.letter, o.mean_credit * 100.0)) + .collect(); + a.notes.push(format!( + "partial credit was awarded at grading time to {}, which records a decision that the \ + item admitted more than one reading. Rewrite the stem rather than re-deciding this \ + every term.", + letters.join(", ") + )); + } + + // Rapid guessing, when the platform reported response times. + let times: Vec = rows + .iter() + .filter_map(|r| r.response_time_seconds) + .collect(); + if times.len() >= 5 { + let rapid = times.iter().filter(|t| **t < 5.0).count() as f64 / times.len() as f64; + if rapid > 0.15 { + a.flags.push(Flag::HighRapidGuess); + a.notes.push(format!( + "{:.0}% of responses arrived in under five seconds, which is faster than the stem \ + can be read. That is usually about the item's position on the form or time \ + pressure, not the item.", + rapid * 100.0 + )); + } + } + + // Did the item behave as authored? + if let Some(d) = design { + if let Some(expected) = d.expected_difficulty { + if (expected - a.p_value).abs() > t.design_tolerance { + a.flags.push(Flag::DesignMismatch); + a.notes.push(format!( + "you expected about {:.0}% correct and observed {:.0}%. Worth knowing whether \ + your model of the students or the item is off.", + expected * 100.0, + a.p_value * 100.0 + )); + } + } + if let (Some(band), Some(r)) = (d.expected_discrimination, a.point_biserial) { + let (low, high) = band.expected_band(); + if r < low || r > high { + if !a.flags.contains(&Flag::DesignMismatch) { + a.flags.push(Flag::DesignMismatch); + } + a.notes.push(format!( + "you expected {} discrimination ({low:.2} to {high:.2}) and observed {r:.2}.", + format!("{band:?}").to_lowercase() + )); + } + } + } + + a.flags.sort(); + a.flags.dedup(); +} + +/// Computes whole-test reliability. +/// +/// # Arguments +/// +/// * `coded` - the 0/1 response matrix. +/// * `totals` - per-student totals. +/// * `p_values` - per-item p-values. +/// * `rpbs` - per-item point-biserials that could be computed. +/// +/// # Returns +/// +/// The reliability summary. +fn reliability(coded: &[Vec], totals: &[f64], p_values: &[f64], rpbs: &[f64]) -> Reliability { + let n_students = coded.len(); + let n_items = coded.first().map(|r| r.len()).unwrap_or(0); + // Named distinctly from the `mean` and `sd` helpers: binding `let mean = + // mean(totals)` shadows the function for the rest of the scope, which then + // makes the later `mean(p_values)` a call on an f64. + let mean_total = mean(totals); + let sd_total = sd(totals); + + // KR-20. The variance terms use the population form, which is the convention + // for this coefficient and matches what other packages report. + let alpha = if n_items > 1 && sd_total > 0.0 { + let sum_pq: f64 = p_values.iter().map(|p| p * (1.0 - p)).sum(); + let k = n_items as f64; + let variance = sd_total * sd_total; + Some((k / (k - 1.0)) * (1.0 - sum_pq / variance)) + } else { + None + }; + + let sem = alpha.map(|a| sd_total * (1.0 - a).max(0.0).sqrt()); + + Reliability { + n_items, + n_students, + mean: mean_total, + sd: sd_total, + alpha, + sem, + mean_p: mean(p_values), + mean_point_biserial: if rpbs.is_empty() { + None + } else { + Some(mean(rpbs)) + }, + } +} + +/// Infers the key from which options earned full credit. +/// +/// Used when no assessment record is available, so that an export can be analyzed +/// before its record is written. +/// +/// # Arguments +/// +/// * `rows` - the responses for one item. +/// +/// # Returns +/// +/// The letters that appear on full-credit responses. +fn infer_key(rows: &[&crate::responses::Response]) -> Vec { + let mut out: BTreeSet = BTreeSet::new(); + for r in rows { + if r.credit >= 0.999 { + for letter in &r.selected { + out.insert(letter.clone()); + } + } + } + out.into_iter().collect() +} + +/// The arithmetic mean, zero for an empty slice. +fn mean(v: &[f64]) -> f64 { + if v.is_empty() { + 0.0 + } else { + v.iter().sum::() / v.len() as f64 + } +} + +/// The population standard deviation. +fn sd(v: &[f64]) -> f64 { + if v.len() < 2 { + return 0.0; + } + let m = mean(v); + (v.iter().map(|x| (x - m) * (x - m)).sum::() / v.len() as f64).sqrt() +} + +/// The mean of the entries at the given indices. +fn mean_of(v: &[f64], indices: &BTreeSet) -> f64 { + if indices.is_empty() { + return 0.0; + } + indices.iter().map(|i| v[*i]).sum::() / indices.len() as f64 +} + +/// The Pearson correlation of two equal-length vectors. +/// +/// # Arguments +/// +/// * `x` - the first vector. +/// * `y` - the second vector. +/// +/// # Returns +/// +/// The correlation, or `None` when either vector has no variance. Returning +/// `None` rather than a NaN matters: an item everyone answered correctly has no +/// correlation, and that is a meaningful result to report rather than a number to +/// propagate. +pub fn correlation(x: &[f64], y: &[f64]) -> Option { + if x.len() != y.len() || x.len() < 2 { + return None; + } + let mx = mean(x); + let my = mean(y); + let mut sxy = 0.0; + let mut sxx = 0.0; + let mut syy = 0.0; + for (xi, yi) in x.iter().zip(y.iter()) { + let dx = xi - mx; + let dy = yi - my; + sxy += dx * dy; + sxx += dx * dx; + syy += dy * dy; + } + if sxx <= f64::EPSILON || syy <= f64::EPSILON { + return None; + } + Some(sxy / (sxx * syy).sqrt()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::responses::Response; + + fn resp(student: &str, number: u32, letter: &str, credit: f64) -> Response { + Response { + administration_id: "C/T/a".into(), + course: "C".into(), + term: "T".into(), + assessment_id: "a".into(), + date: None, + form: None, + student_key: student.into(), + sid: None, + name: None, + email: None, + section: None, + item_number: number, + item_ref: None, + item_version: None, + selected: if letter.is_empty() { + vec![] + } else { + vec![letter.to_string()] + }, + eliminated: vec![], + correct: Some(credit >= 0.999), + credit, + points_possible: 1.0, + score: credit, + response_time_seconds: None, + level: None, + learning_objectives: vec![], + topics: vec![], + bonus: false, + dropped: false, + } + } + + /// Twelve students; item 1 discriminates, item 2 is keyed backwards. + fn sample() -> ResponseSet { + let mut set = ResponseSet::new(); + for i in 0..12 { + let strong = i < 6; + let s = format!("s{i:02}"); + // Item 1: strong students right, weak wrong. + set.rows.push(resp( + &s, + 1, + if strong { "A" } else { "B" }, + if strong { 1.0 } else { 0.0 }, + )); + // Item 2: reversed, which is what a keying error looks like. + set.rows.push(resp( + &s, + 2, + if strong { "C" } else { "D" }, + if strong { 0.0 } else { 1.0 }, + )); + // Item 3: everyone correct. + set.rows.push(resp(&s, 3, "A", 1.0)); + // Item 4: gives the totals some spread. + set.rows.push(resp( + &s, + 4, + if i % 2 == 0 { "A" } else { "B" }, + if i % 2 == 0 { 1.0 } else { 0.0 }, + )); + } + set + } + + #[test] + fn correlation_returns_none_without_variance() { + assert_eq!(correlation(&[1.0, 1.0, 1.0], &[1.0, 2.0, 3.0]), None); + assert_eq!(correlation(&[1.0], &[1.0]), None); + let r = correlation(&[1.0, 2.0, 3.0], &[2.0, 4.0, 6.0]).unwrap(); + assert!((r - 1.0).abs() < 1e-12, "perfect correlation is 1, got {r}"); + let r = correlation(&[1.0, 2.0, 3.0], &[3.0, 2.0, 1.0]).unwrap(); + assert!((r + 1.0).abs() < 1e-12); + } + + #[test] + fn flags_a_reversed_key_as_negative_discrimination() { + let set = sample(); + let a = analyze(&set, &Thresholds::default(), None, None); + let item2 = a.items.iter().find(|i| i.number == 2).unwrap(); + assert!( + item2.point_biserial.unwrap() < 0.0, + "got {:?}", + item2.point_biserial + ); + assert!(item2.flags.contains(&Flag::NegativeDiscrimination)); + assert!(item2.needs_revision()); + } + + #[test] + fn a_good_item_is_not_flagged_for_discrimination() { + let set = sample(); + let a = analyze(&set, &Thresholds::default(), None, None); + let item1 = a.items.iter().find(|i| i.number == 1).unwrap(); + assert!(item1.point_biserial.unwrap() > 0.5); + assert!(!item1.flags.contains(&Flag::NegativeDiscrimination)); + assert!(!item1.flags.contains(&Flag::LowDiscrimination)); + } + + #[test] + fn a_unanimous_item_has_no_correlation_and_is_flagged_easy() { + let set = sample(); + let a = analyze(&set, &Thresholds::default(), None, None); + let item3 = a.items.iter().find(|i| i.number == 3).unwrap(); + assert_eq!(item3.p_value, 1.0); + assert_eq!(item3.point_biserial, None, "no variance, so no correlation"); + assert!(item3.flags.contains(&Flag::TooEasy)); + } + + #[test] + fn blank_responses_count_as_incorrect_but_are_reported() { + let mut set = ResponseSet::new(); + for i in 0..10 { + let s = format!("s{i}"); + let letter = if i < 5 { "A" } else { "" }; + set.rows + .push(resp(&s, 1, letter, if i < 5 { 1.0 } else { 0.0 })); + set.rows + .push(resp(&s, 2, "A", if i < 7 { 1.0 } else { 0.0 })); + } + let a = analyze(&set, &Thresholds::default(), None, None); + let item1 = a.items.iter().find(|i| i.number == 1).unwrap(); + assert_eq!(item1.p_value, 0.5); + assert_eq!(item1.blank_rate, 0.5); + assert_eq!( + item1.n_answered, 10, + "a blank is still an administered item" + ); + } + + #[test] + fn partial_credit_to_a_distractor_flags_ambiguity() { + let mut set = ResponseSet::new(); + for i in 0..10 { + let s = format!("s{i}"); + if i < 5 { + set.rows.push(resp(&s, 1, "A", 1.0)); + } else { + // B earned two thirds of the points at grading time. + set.rows.push(resp(&s, 1, "B", 0.667)); + } + set.rows + .push(resp(&s, 2, "A", if i % 2 == 0 { 1.0 } else { 0.0 })); + } + let a = analyze(&set, &Thresholds::default(), None, None); + let item1 = a.items.iter().find(|i| i.number == 1).unwrap(); + assert!(item1.flags.contains(&Flag::Ambiguous), "{:?}", item1.flags); + assert!(item1.notes.iter().any(|n| n.contains("partial credit"))); + } + + #[test] + fn reliability_is_computed_and_explained() { + let set = sample(); + let a = analyze(&set, &Thresholds::default(), None, None); + assert_eq!(a.reliability.n_items, 4); + assert_eq!(a.reliability.n_students, 12); + assert!(a.reliability.alpha.is_some()); + let text = a.reliability.interpretation(); + assert!(text.contains("KR-20")); + // A four-item test must carry the length caveat. + assert!(text.contains("test length")); + } + + #[test] + fn small_samples_get_a_caution() { + let set = sample(); + let a = analyze(&set, &Thresholds::default(), None, None); + assert!(a.warnings.iter().any(|w| w.contains("examinees"))); + } + + #[test] + fn empty_input_does_not_panic() { + let a = analyze(&ResponseSet::new(), &Thresholds::default(), None, None); + assert!(a.items.is_empty()); + assert_eq!(a.reliability.alpha, None); + assert!(!a.warnings.is_empty()); + } + + #[test] + fn the_revise_queue_puts_blocking_flags_first() { + let set = sample(); + let a = analyze(&set, &Thresholds::default(), None, None); + let queue = a.revise_queue(); + assert!(!queue.is_empty()); + assert_eq!(queue[0].number, 2, "the reversed key comes first"); + } +} diff --git a/src/analysis/irt.rs b/src/analysis/irt.rs new file mode 100644 index 0000000..dc7d394 --- /dev/null +++ b/src/analysis/irt.rs @@ -0,0 +1,1171 @@ +//! Item response theory: marginal maximum likelihood estimation by EM. +//! +//! The estimator here was prototyped and validated against a real 24-examinee, +//! 30-item exam before being written in Rust. On that data it converges in 31 EM +//! iterations, and the resulting abilities correlate 0.994 with total score — +//! which is the sanity check that matters, because if IRT abilities did *not* +//! track total score on a short unidimensional test, something would be wrong. +//! +//! # Why priors are on by default +//! +//! Classroom exams break unpenalized maximum likelihood routinely. Any item that +//! everyone answered correctly has no finite maximum: the likelihood increases +//! without bound as difficulty goes to negative infinity, and the optimizer walks +//! off to whatever bound you set. The real exam this was built against had two +//! such items. +//! +//! A weakly informative prior fixes this properly rather than by clamping. With +//! `a ~ lognormal(0, 0.5)` and `b ~ Normal(0, 2)`, those two items settled at +//! a = 1.26, b = −3.08: very easy, moderately discriminating, which is an honest +//! description of an item everyone got right. The priors are weak enough that +//! items with real information in them are essentially unaffected. +//! +//! Priors can be turned off, and doing so is legitimate for a large pooled +//! dataset. On a single small class it produces divergence, not objectivity. +//! +//! # What sample size buys you +//! +//! Rasch needs the fewest examinees, since it estimates one parameter per item. +//! 2PL wants a few hundred to pin discrimination down; 3PL wants a thousand and +//! is not honestly estimable from one section of twenty-five, which is why asking +//! for it produces a warning rather than a refusal. The parameters are still +//! useful for ranking items and for computing abilities that respect item +//! difficulty. They are not useful as absolute values to publish. + +use std::collections::BTreeMap; + +use crate::item::{IrtModel, IrtParams}; +use crate::responses::Matrix; + +/// The number of quadrature points. +/// +/// Forty-one points over ±4 is far more than a short test needs; the cost is +/// trivial at this scale and it removes quadrature coarseness as a possible +/// explanation for anything surprising in the output. +const QUAD_POINTS: usize = 41; +/// The lower bound of the ability grid. +const QUAD_LO: f64 = -4.0; +/// The upper bound of the ability grid. +const QUAD_HI: f64 = 4.0; + +/// Smallest allowed discrimination, to keep the Newton step in a sane region. +const A_MIN: f64 = 0.02; +/// Largest allowed discrimination. +const A_MAX: f64 = 4.0; +/// Largest allowed absolute difficulty. +const B_MAX: f64 = 8.0; + +/// Weakly informative priors on the item parameters. +#[derive(Debug, Clone)] +pub struct Priors { + /// Whether to use priors at all. + pub enabled: bool, + /// Mean of `log(a)`. + pub mu_log_a: f64, + /// Standard deviation of `log(a)`. + pub sd_log_a: f64, + /// Mean of `b`. + pub mu_b: f64, + /// Standard deviation of `b`. + pub sd_b: f64, +} + +impl Default for Priors { + fn default() -> Priors { + Priors { + enabled: true, + mu_log_a: 0.0, + sd_log_a: 0.5, + mu_b: 0.0, + sd_b: 2.0, + } + } +} + +/// Estimation settings. +#[derive(Debug, Clone)] +pub struct Options { + /// Which model to fit. + pub model: IrtModel, + /// The priors. + pub priors: Priors, + /// Maximum EM iterations. + pub max_iterations: usize, + /// Convergence tolerance on the largest parameter change. + pub tolerance: f64, + /// Guessing values to search over for the 3PL, as a lower asymptote. + pub guess_grid: Vec, +} + +impl Default for Options { + fn default() -> Options { + Options { + model: IrtModel::TwoPl, + priors: Priors::default(), + max_iterations: 200, + tolerance: 1e-5, + // A five-option item has a chance floor near 0.2; above 0.4 the + // parameter stops being "guessing" and starts absorbing real + // misfit. + guess_grid: vec![0.0, 0.05, 0.10, 0.15, 0.20, 0.25, 0.30, 0.35, 0.40], + } + } +} + +/// One item's estimated parameters. +#[derive(Debug, Clone)] +pub struct ItemFit { + /// The question number. + pub number: u32, + /// Discrimination. + pub a: f64, + /// Difficulty, on the ability scale. + pub b: f64, + /// Lower asymptote, for the 3PL. + pub c: Option, + /// Standard error of `a`, when the information matrix was invertible. + pub se_a: Option, + /// Standard error of `b`. + pub se_b: Option, + /// How many examinees contributed. + pub n: usize, + /// Whether a prior was applied. + pub bayesian: bool, + /// The model fitted. + pub model: IrtModel, + /// Notes about this item's estimation, such as a parameter pinned by its + /// prior because the data alone did not identify it. + pub notes: Vec, +} + +impl ItemFit { + /// Converts to the form stored on an item's calibration block. + /// + /// # Arguments + /// + /// * `n` - the examinee count to record. + /// + /// # Returns + /// + /// The parameters. + pub fn to_params(&self) -> IrtParams { + IrtParams { + model: self.model, + a: round6(self.a), + b: round6(self.b), + c: self.c.map(round6), + se_a: self.se_a.map(round6), + se_b: self.se_b.map(round6), + n: Some(self.n), + bayesian: self.bayesian, + } + } + + /// The probability a student of a given ability answers correctly. + /// + /// # Arguments + /// + /// * `theta` - the ability. + /// + /// # Returns + /// + /// The probability. + pub fn probability(&self, theta: f64) -> f64 { + let c = self.c.unwrap_or(0.0); + c + (1.0 - c) * logistic(self.a * (theta - self.b)) + } + + /// Fisher information at a given ability. + /// + /// Where an item is informative is the practical question when choosing items: + /// an item is most useful for distinguishing students whose ability is near + /// its difficulty. + /// + /// # Arguments + /// + /// * `theta` - the ability. + /// + /// # Returns + /// + /// The information. + pub fn information(&self, theta: f64) -> f64 { + let p = self.probability(theta); + let c = self.c.unwrap_or(0.0); + if p <= 0.0 || p >= 1.0 || c >= 1.0 { + return 0.0; + } + // For the 3PL this reduces to the 2PL form when c = 0. + let num = self.a * self.a * (p - c) * (p - c) * (1.0 - p); + let den = (1.0 - c) * (1.0 - c) * p; + if den <= 0.0 { + 0.0 + } else { + num / den + } + } +} + +/// One examinee's estimated ability. +#[derive(Debug, Clone)] +pub struct Ability { + /// The student key. + pub student_key: String, + /// Expected a posteriori ability estimate. + pub theta: f64, + /// Posterior standard deviation, which is the standard error of the estimate. + pub se: f64, + /// Number of items answered. + pub n_items: usize, +} + +impl Ability { + /// A plain-language band for the estimate. + /// + /// # Returns + /// + /// A short description, deliberately coarse because the standard error on a + /// thirty-item test is around half a logit and finer distinctions would be + /// noise. + pub fn band(&self) -> &'static str { + if self.theta >= 1.0 { + "well above the class average" + } else if self.theta >= 0.35 { + "above the class average" + } else if self.theta > -0.35 { + "near the class average" + } else if self.theta > -1.0 { + "below the class average" + } else { + "well below the class average" + } + } +} + +/// The result of an estimation run. +#[derive(Debug, Clone)] +pub struct Fit { + /// Per-item parameters, in question order. + pub items: Vec, + /// Per-examinee abilities, in student order. + pub abilities: Vec, + /// EM iterations used. + pub iterations: usize, + /// Whether the tolerance was reached. + pub converged: bool, + /// Marginal log-likelihood at the solution. + pub log_likelihood: f64, + /// Cautions about the fit. + pub warnings: Vec, +} + +impl Fit { + /// Test information at a given ability, summed over items. + /// + /// # Arguments + /// + /// * `theta` - the ability. + /// + /// # Returns + /// + /// The total information, whose reciprocal square root is the standard error + /// of measurement at that ability. + pub fn test_information(&self, theta: f64) -> f64 { + self.items.iter().map(|i| i.information(theta)).sum() + } + + /// Standard error of measurement at a given ability. + /// + /// # Arguments + /// + /// * `theta` - the ability. + /// + /// # Returns + /// + /// The standard error, or `None` where the test carries no information. + pub fn standard_error(&self, theta: f64) -> Option { + let info = self.test_information(theta); + if info <= 0.0 { + None + } else { + Some(1.0 / info.sqrt()) + } + } + + /// Where the test measures best, on a coarse grid. + /// + /// Worth reporting because a test can be reliable overall while measuring + /// almost nothing about the students you most need to distinguish. + /// + /// # Returns + /// + /// The ability with the most information. + pub fn peak_information(&self) -> f64 { + let mut best = (-4.0, f64::NEG_INFINITY); + let mut theta = -4.0; + while theta <= 4.0 { + let info = self.test_information(theta); + if info > best.1 { + best = (theta, info); + } + theta += 0.1; + } + best.0 + } + + /// Abilities keyed by student. + pub fn ability_map(&self) -> BTreeMap { + self.abilities + .iter() + .map(|a| (a.student_key.clone(), a.theta)) + .collect() + } +} + +/// The logistic function, guarded against overflow. +/// +/// # Arguments +/// +/// * `z` - the linear predictor. +/// +/// # Returns +/// +/// The probability. +fn logistic(z: f64) -> f64 { + if z < -40.0 { + 0.0 + } else if z > 40.0 { + 1.0 + } else { + 1.0 / (1.0 + (-z).exp()) + } +} + +/// Rounds to six decimals, so YAML output does not carry meaningless precision. +fn round6(x: f64) -> f64 { + (x * 1e6).round() / 1e6 +} + +/// The quadrature grid: equally spaced points with standard normal weights. +/// +/// # Returns +/// +/// The points and their normalized weights. +fn quadrature() -> (Vec, Vec) { + let mut theta = Vec::with_capacity(QUAD_POINTS); + let mut weight = Vec::with_capacity(QUAD_POINTS); + for k in 0..QUAD_POINTS { + let t = QUAD_LO + (QUAD_HI - QUAD_LO) * k as f64 / (QUAD_POINTS - 1) as f64; + theta.push(t); + weight.push((-0.5 * t * t).exp()); + } + let total: f64 = weight.iter().sum(); + for w in weight.iter_mut() { + *w /= total; + } + (theta, weight) +} + +/// Fits an IRT model to a response matrix. +/// +/// # Arguments +/// +/// * `matrix` - the dichotomous response matrix. Missing responses are treated as +/// not administered rather than incorrect, which is the correct handling for a +/// likelihood: a student who never saw an item tells you nothing about it. +/// * `opts` - estimation settings. +/// +/// # Returns +/// +/// The fit, with warnings about sample size and any item whose parameters were +/// determined by its prior rather than by the data. +pub fn fit(matrix: &Matrix, opts: &Options) -> Fit { + let n = matrix.n_students(); + let j_count = matrix.n_items(); + let mut warnings = Vec::new(); + + if n == 0 || j_count == 0 { + return Fit { + items: Vec::new(), + abilities: Vec::new(), + iterations: 0, + converged: false, + log_likelihood: f64::NAN, + warnings: vec!["there is nothing to fit: the response matrix is empty".to_string()], + }; + } + + let needed = match opts.model { + IrtModel::Rasch => 100, + IrtModel::TwoPl => 200, + IrtModel::ThreePl => 1000, + }; + if n < needed { + warnings.push(format!( + "fitting {} to {n} examinees. This model usually wants {needed} or more, so treat the \ + parameters as a ranking of items rather than as absolute values, and pool several \ + administrations before acting on any single number", + model_name(opts.model) + )); + } + if !opts.priors.enabled { + warnings.push( + "priors are disabled. Any item answered the same way by every examinee has no finite \ + maximum likelihood estimate, and its difficulty will be pinned at the bound instead \ + of estimated" + .to_string(), + ); + } + + let (grid, base_weight) = quadrature(); + let n_quad = grid.len(); + + // Starting values. Difficulty from the p-value is a much better start than + // zero and saves several EM iterations. + let mut a = vec![1.0f64; j_count]; + let mut b = vec![0.0f64; j_count]; + let mut c = vec![0.0f64; j_count]; + for j in 0..j_count { + let column = matrix.column(j); + let answered: Vec = column.into_iter().flatten().collect(); + if answered.is_empty() { + continue; + } + let p = answered.iter().map(|v| *v as f64).sum::() / answered.len() as f64; + let clamped = p.clamp(0.03, 0.97); + // Inverse logistic of the p-value, which is the difficulty a Rasch model + // implies when discrimination is one. + b[j] = -(clamped / (1.0 - clamped)).ln(); + } + + let mut iterations = 0usize; + let mut converged = false; + let mut notes: Vec> = vec![Vec::new(); j_count]; + + for iteration in 0..opts.max_iterations { + iterations = iteration + 1; + + // ---- E step: expected counts at each quadrature point ---- + // Counts are accumulated per item rather than globally, so an item + // administered to only some examinees is not charged for the others. + let mut n_kj = vec![vec![0.0f64; j_count]; n_quad]; + let mut r_k = vec![vec![0.0f64; j_count]; n_quad]; + + // Response probabilities on the grid, computed once per iteration. + let mut p_grid = vec![vec![0.0f64; j_count]; n_quad]; + for k in 0..n_quad { + for j in 0..j_count { + p_grid[k][j] = c[j] + (1.0 - c[j]) * logistic(a[j] * (grid[k] - b[j])); + } + } + + for i in 0..n { + let posterior = posterior_for(&matrix.coded[i], &p_grid, &base_weight); + for j in 0..j_count { + let Some(u) = matrix.coded[i][j] else { + continue; + }; + for k in 0..n_quad { + n_kj[k][j] += posterior[k]; + if u == 1 { + r_k[k][j] += posterior[k]; + } + } + } + } + + // ---- M step: one two-parameter Newton solve per item ---- + let mut delta = 0.0f64; + for j in 0..j_count { + let counts: Vec<(f64, f64)> = (0..n_quad).map(|k| (n_kj[k][j], r_k[k][j])).collect(); + + let (new_a, new_b, new_c) = match opts.model { + IrtModel::Rasch => { + let nb = newton_rasch(&grid, &counts, b[j], &opts.priors); + (1.0, nb, 0.0) + } + IrtModel::TwoPl => { + let (na, nb) = newton_2pl(&grid, &counts, a[j], b[j], 0.0, &opts.priors); + (na, nb, 0.0) + } + IrtModel::ThreePl => { + // A profile search over the lower asymptote: for each + // candidate, fit a and b, and keep whichever candidate gives + // the best expected complete-data log-likelihood. Estimating + // c jointly with two other parameters from a small sample is + // where 3PL fits go unstable. + let mut best = (a[j], b[j], 0.0, f64::NEG_INFINITY); + for cand in &opts.guess_grid { + let (na, nb) = newton_2pl(&grid, &counts, a[j], b[j], *cand, &opts.priors); + let ll = expected_ll(&grid, &counts, na, nb, *cand); + if ll > best.3 { + best = (na, nb, *cand, ll); + } + } + (best.0, best.1, best.2) + } + }; + + delta = delta + .max((new_a - a[j]).abs()) + .max((new_b - b[j]).abs()) + .max((new_c - c[j]).abs()); + a[j] = new_a; + b[j] = new_b; + c[j] = new_c; + } + + if delta < opts.tolerance { + converged = true; + break; + } + } + + if !converged { + warnings.push(format!( + "estimation stopped at the iteration limit ({}) without meeting the tolerance of \ + {:.0e}. The parameters are usable but not fully settled; a near-degenerate item is \ + the usual cause", + opts.max_iterations, opts.tolerance + )); + } + + // ---- Standard errors and per-item notes ---- + let (grid_final, weight_final) = (grid.clone(), base_weight.clone()); + let mut p_grid = vec![vec![0.0f64; j_count]; n_quad]; + for k in 0..n_quad { + for j in 0..j_count { + p_grid[k][j] = c[j] + (1.0 - c[j]) * logistic(a[j] * (grid_final[k] - b[j])); + } + } + let mut n_kj = vec![vec![0.0f64; j_count]; n_quad]; + let mut r_k = vec![vec![0.0f64; j_count]; n_quad]; + for i in 0..n { + let posterior = posterior_for(&matrix.coded[i], &p_grid, &weight_final); + for j in 0..j_count { + if matrix.coded[i][j].is_some() { + for k in 0..n_quad { + n_kj[k][j] += posterior[k]; + if matrix.coded[i][j] == Some(1) { + r_k[k][j] += posterior[k]; + } + } + } + } + } + + let mut items = Vec::with_capacity(j_count); + for j in 0..j_count { + let counts: Vec<(f64, f64)> = (0..n_quad).map(|k| (n_kj[k][j], r_k[k][j])).collect(); + let (se_a, se_b) = standard_errors(&grid_final, &counts, a[j], b[j], c[j], &opts.priors); + + let column = matrix.column(j); + let answered: Vec = column.into_iter().flatten().collect(); + let n_item = answered.len(); + let all_same = !answered.is_empty() && answered.iter().all(|v| *v == answered[0]); + if all_same { + notes[j].push(format!( + "every examinee answered this item the same way, so its parameters come from the \ + prior rather than from the data; b = {:.2} means only \"outside the range this \ + class could resolve\"", + b[j] + )); + } + if a[j] <= A_MIN + 1e-9 { + notes[j].push( + "discrimination hit its lower bound, which means the responses carry no \ + information about ability ordering" + .to_string(), + ); + } + if b[j].abs() >= B_MAX - 1e-9 { + notes[j].push( + "difficulty hit its bound and should be read as \"off the scale\"".to_string(), + ); + } + + items.push(ItemFit { + number: matrix.items[j], + a: a[j], + b: b[j], + c: if opts.model == IrtModel::ThreePl { + Some(c[j]) + } else { + None + }, + se_a, + se_b, + n: n_item, + bayesian: opts.priors.enabled, + model: opts.model, + notes: notes[j].clone(), + }); + } + + // ---- Abilities, expected a posteriori ---- + let mut abilities = Vec::with_capacity(n); + let mut log_likelihood = 0.0f64; + for i in 0..n { + let (posterior, marginal) = + posterior_and_marginal(&matrix.coded[i], &p_grid, &weight_final); + log_likelihood += marginal; + let theta: f64 = (0..n_quad).map(|k| posterior[k] * grid_final[k]).sum(); + let variance: f64 = (0..n_quad) + .map(|k| posterior[k] * (grid_final[k] - theta) * (grid_final[k] - theta)) + .sum(); + abilities.push(Ability { + student_key: matrix.students[i].clone(), + theta: round6(theta), + se: round6(variance.max(0.0).sqrt()), + n_items: matrix.coded[i].iter().filter(|v| v.is_some()).count(), + }); + } + + Fit { + items, + abilities, + iterations, + converged, + log_likelihood, + warnings, + } +} + +/// The posterior distribution over the ability grid for one examinee. +/// +/// # Arguments +/// +/// * `responses` - the examinee's coded responses; `None` entries are skipped. +/// * `p_grid` - response probabilities by quadrature point and item. +/// * `weight` - the prior weights. +/// +/// # Returns +/// +/// The normalized posterior. +fn posterior_for(responses: &[Option], p_grid: &[Vec], weight: &[f64]) -> Vec { + posterior_and_marginal(responses, p_grid, weight).0 +} + +/// The posterior and the marginal log-likelihood for one examinee. +/// +/// Working in logs and subtracting the maximum before exponentiating is what keeps +/// this stable: with thirty items the raw likelihood at an implausible ability +/// underflows to zero in double precision, and the posterior becomes all NaN. +/// +/// # Arguments +/// +/// * `responses` - the coded responses. +/// * `p_grid` - response probabilities by quadrature point and item. +/// * `weight` - the prior weights. +/// +/// # Returns +/// +/// The normalized posterior and the examinee's marginal log-likelihood. +fn posterior_and_marginal( + responses: &[Option], + p_grid: &[Vec], + weight: &[f64], +) -> (Vec, f64) { + let n_quad = weight.len(); + let mut log_like = vec![0.0f64; n_quad]; + for k in 0..n_quad { + let mut total = 0.0; + for (j, response) in responses.iter().enumerate() { + let Some(u) = response else { continue }; + let p = p_grid[k][j].clamp(1e-12, 1.0 - 1e-12); + total += if *u == 1 { p.ln() } else { (1.0 - p).ln() }; + } + log_like[k] = total; + } + + let max = log_like.iter().copied().fold(f64::NEG_INFINITY, f64::max); + let mut posterior = vec![0.0f64; n_quad]; + let mut sum = 0.0; + for k in 0..n_quad { + posterior[k] = weight[k] * (log_like[k] - max).exp(); + sum += posterior[k]; + } + if sum <= 0.0 || !sum.is_finite() { + // Degenerate: fall back to the prior rather than emitting NaN. + return (weight.to_vec(), f64::NAN); + } + for p in posterior.iter_mut() { + *p /= sum; + } + (posterior, max + sum.ln()) +} + +/// One item's expected complete-data log-likelihood, used to choose `c`. +/// +/// # Arguments +/// +/// * `grid` - the ability grid. +/// * `counts` - expected `(administered, correct)` counts per grid point. +/// * `a` - discrimination. +/// * `b` - difficulty. +/// * `c` - lower asymptote. +/// +/// # Returns +/// +/// The expected log-likelihood. +fn expected_ll(grid: &[f64], counts: &[(f64, f64)], a: f64, b: f64, c: f64) -> f64 { + let mut total = 0.0; + for (k, (n_k, r_k)) in counts.iter().enumerate() { + let p = (c + (1.0 - c) * logistic(a * (grid[k] - b))).clamp(1e-12, 1.0 - 1e-12); + total += r_k * p.ln() + (n_k - r_k) * (1.0 - p).ln(); + } + total +} + +/// The 2PL M-step: a damped two-parameter Newton solve. +/// +/// The gradient and Hessian are analytic. With `W = n·P·(1−P)`, `e = r − n·P`, and +/// `u = θ − b`: +/// +/// ```text +/// ∂L/∂a = Σ e·u ∂L/∂b = −a·Σ e +/// H = [ −Σ W·u² −Σ e + a·Σ W·u ] +/// [ −Σ e + a·Σ W·u −a²·Σ W ] +/// ``` +/// +/// # Arguments +/// +/// * `grid` - the ability grid. +/// * `counts` - expected `(administered, correct)` counts per grid point. +/// * `a0` - starting discrimination. +/// * `b0` - starting difficulty. +/// * `c` - the fixed lower asymptote. +/// * `priors` - the priors to add to the gradient and Hessian. +/// +/// # Returns +/// +/// The updated `(a, b)`. +fn newton_2pl( + grid: &[f64], + counts: &[(f64, f64)], + a0: f64, + b0: f64, + c: f64, + priors: &Priors, +) -> (f64, f64) { + let mut a = a0; + let mut b = b0; + + for _ in 0..30 { + let mut sum_e = 0.0; + let mut sum_w = 0.0; + let mut sum_wu = 0.0; + let mut sum_wu2 = 0.0; + let mut g_a = 0.0; + + for (k, (n_k, r_k)) in counts.iter().enumerate() { + if *n_k <= 0.0 { + continue; + } + let theta = grid[k]; + let u = theta - b; + let p_star = logistic(a * u); + let p = c + (1.0 - c) * p_star; + let p = p.clamp(1e-12, 1.0 - 1e-12); + // With a lower asymptote, the derivative of P with respect to the + // linear predictor carries a (1−c) factor and the residual is scaled + // by (P*−0)/(P−c); for c = 0 this reduces to the plain 2PL form. + let scale = if c > 0.0 { + (1.0 - c) * (p_star * (1.0 - p_star)) / (p * (1.0 - p)) + } else { + 1.0 + }; + let e = (r_k - n_k * p) * scale; + // The information weight is n·(∂P/∂η)²/(P(1−P)) where η is the linear + // predictor. Written as n·scale²·P(1−P) it is exact for any c, and + // reduces to the familiar n·P(1−P) when c = 0. + let w = n_k * scale * scale * p * (1.0 - p); + + sum_e += e; + sum_w += w; + sum_wu += w * u; + sum_wu2 += w * u * u; + g_a += e * u; + } + + let mut g1 = g_a; + let mut g2 = -a * sum_e; + let mut h11 = -sum_wu2; + // Not `mut`: the priors on a and b are independent, so neither contributes a + // cross-derivative term to h12. + let h12 = -sum_e + a * sum_wu; + let mut h22 = -a * a * sum_w; + + if priors.enabled { + // log-normal on a: the log-density in a is + // −log(a) − (log a − μ)² / (2σ²), differentiated twice in a. + let la = a.ln(); + let s2 = priors.sd_log_a * priors.sd_log_a; + g1 += -1.0 / a - (la - priors.mu_log_a) / (s2 * a); + h11 += 1.0 / (a * a) + (la - priors.mu_log_a) / (s2 * a * a) - 1.0 / (s2 * a * a); + // Normal on b. + let t2 = priors.sd_b * priors.sd_b; + g2 += -(b - priors.mu_b) / t2; + h22 += -1.0 / t2; + } + + let det = h11 * h22 - h12 * h12; + if det.abs() < 1e-12 { + break; + } + let da = -(h22 * g1 - h12 * g2) / det; + let db = -(-h12 * g1 + h11 * g2) / det; + + // Damping: halve the step until it stays inside the admissible region. + // Without this a single wild Newton step can throw a near-degenerate item + // to a bound it never recovers from. + let mut step = 1.0; + while step > 1e-3 && (a + step * da <= A_MIN || (b + step * db).abs() > B_MAX) { + step *= 0.5; + } + let new_a = (a + step * da).clamp(A_MIN, A_MAX); + let new_b = (b + step * db).clamp(-B_MAX, B_MAX); + let moved = (new_a - a).abs().max((new_b - b).abs()); + a = new_a; + b = new_b; + if moved < 1e-8 { + break; + } + } + + (a, b) +} + +/// The Rasch M-step: a one-parameter Newton solve with discrimination fixed at 1. +/// +/// # Arguments +/// +/// * `grid` - the ability grid. +/// * `counts` - expected `(administered, correct)` counts per grid point. +/// * `b0` - starting difficulty. +/// * `priors` - the priors. +/// +/// # Returns +/// +/// The updated difficulty. +fn newton_rasch(grid: &[f64], counts: &[(f64, f64)], b0: f64, priors: &Priors) -> f64 { + let mut b = b0; + for _ in 0..30 { + let mut g = 0.0; + let mut h = 0.0; + for (k, (n_k, r_k)) in counts.iter().enumerate() { + if *n_k <= 0.0 { + continue; + } + let p = logistic(grid[k] - b); + g += -(r_k - n_k * p); + h += -n_k * p * (1.0 - p); + } + if priors.enabled { + let t2 = priors.sd_b * priors.sd_b; + g += -(b - priors.mu_b) / t2; + h += -1.0 / t2; + } + if h.abs() < 1e-12 { + break; + } + let step = -g / h; + let new_b = (b + step).clamp(-B_MAX, B_MAX); + let moved = (new_b - b).abs(); + b = new_b; + if moved < 1e-8 { + break; + } + } + b +} + +/// Standard errors from the observed information at the solution. +/// +/// The information matrix is the negative Hessian; inverting it gives the +/// asymptotic covariance. When the determinant is not positive the parameters are +/// not locally identified, and `None` is returned rather than a fabricated number. +/// +/// # Arguments +/// +/// * `grid` - the ability grid. +/// * `counts` - expected counts. +/// * `a` - discrimination at the solution. +/// * `b` - difficulty at the solution. +/// * `c` - lower asymptote. +/// * `priors` - the priors, which contribute to the information. +/// +/// # Returns +/// +/// The standard errors of `a` and `b`. +fn standard_errors( + grid: &[f64], + counts: &[(f64, f64)], + a: f64, + b: f64, + c: f64, + priors: &Priors, +) -> (Option, Option) { + let mut sum_e = 0.0; + let mut sum_w = 0.0; + let mut sum_wu = 0.0; + let mut sum_wu2 = 0.0; + + for (k, (n_k, r_k)) in counts.iter().enumerate() { + if *n_k <= 0.0 { + continue; + } + let u = grid[k] - b; + let p_star = logistic(a * u); + let p = (c + (1.0 - c) * p_star).clamp(1e-12, 1.0 - 1e-12); + let w = n_k * p_star * (1.0 - p_star); + sum_e += r_k - n_k * p; + sum_w += w; + sum_wu += w * u; + sum_wu2 += w * u * u; + } + + let mut h11 = -sum_wu2; + // Not `mut`: see newton_2pl — the priors add no cross term. + let h12 = -sum_e + a * sum_wu; + let mut h22 = -a * a * sum_w; + if priors.enabled { + let la = a.ln(); + let s2 = priors.sd_log_a * priors.sd_log_a; + h11 += 1.0 / (a * a) + (la - priors.mu_log_a) / (s2 * a * a) - 1.0 / (s2 * a * a); + h22 += -1.0 / (priors.sd_b * priors.sd_b); + } + + let det = h11 * h22 - h12 * h12; + if det <= 0.0 || !det.is_finite() { + return (None, None); + } + // Inverse of the negative Hessian, diagonal entries. + let var_a = -h22 / det; + let var_b = -h11 / det; + ( + if var_a > 0.0 { + Some(var_a.sqrt()) + } else { + None + }, + if var_b > 0.0 { + Some(var_b.sqrt()) + } else { + None + }, + ) +} + +/// The printable name of a model. +fn model_name(model: IrtModel) -> &'static str { + match model { + IrtModel::Rasch => "the Rasch model", + IrtModel::TwoPl => "a 2PL model", + IrtModel::ThreePl => "a 3PL model", + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::responses::Matrix; + + /// Builds a matrix from rows of 0/1 characters. + fn matrix_from(rows: &[&str]) -> Matrix { + let n_items = rows[0].len(); + let students: Vec = (0..rows.len()).map(|i| format!("s{i:02}")).collect(); + let items: Vec = (1..=n_items as u32).collect(); + let coded: Vec>> = rows + .iter() + .map(|r| { + r.chars() + .map(|ch| match ch { + '1' => Some(1), + '0' => Some(0), + _ => None, + }) + .collect() + }) + .collect(); + let credit: Vec>> = coded + .iter() + .map(|row| row.iter().map(|c| c.map(|v| v as f64)).collect()) + .collect(); + Matrix { + students, + items, + credit, + coded, + } + } + + /// A Guttman-like pattern: ability and difficulty both ordered. + fn ordered_matrix() -> Matrix { + matrix_from(&[ + "111111", "111110", "111100", "111000", "110000", "100000", "111111", "111110", + "111100", "111000", "110000", "100000", + ]) + } + + #[test] + fn logistic_is_stable_at_the_extremes() { + assert_eq!(logistic(-100.0), 0.0); + assert_eq!(logistic(100.0), 1.0); + assert!((logistic(0.0) - 0.5).abs() < 1e-12); + } + + #[test] + fn quadrature_weights_sum_to_one() { + let (theta, weight) = quadrature(); + assert_eq!(theta.len(), QUAD_POINTS); + let total: f64 = weight.iter().sum(); + assert!((total - 1.0).abs() < 1e-12, "got {total}"); + // Symmetric about zero. + assert!((weight[0] - weight[QUAD_POINTS - 1]).abs() < 1e-15); + } + + #[test] + fn difficulty_orders_with_the_p_value() { + let m = ordered_matrix(); + let f = fit(&m, &Options::default()); + assert_eq!(f.items.len(), 6); + // Item 1 was answered correctly by everyone, item 6 by almost nobody, so + // difficulty must increase across the form. + let b: Vec = f.items.iter().map(|i| i.b).collect(); + for w in b.windows(2) { + assert!(w[0] < w[1], "difficulty must be monotone, got {b:?}"); + } + } + + #[test] + fn abilities_track_total_score() { + let m = ordered_matrix(); + let f = fit(&m, &Options::default()); + let totals = m.correct_counts(); + let thetas: Vec = f.abilities.iter().map(|a| a.theta).collect(); + let r = crate::classical::correlation(&thetas, &totals).unwrap(); + // This is the property that matters: if abilities did not track total + // score on a short unidimensional test, the estimator would be wrong. + assert!(r > 0.95, "theta must track total score, got r = {r}"); + } + + #[test] + fn converges_on_well_behaved_data() { + let f = fit(&ordered_matrix(), &Options::default()); + assert!( + f.converged, + "should converge in {} iterations", + f.iterations + ); + assert!(f.iterations < 200); + assert!(f.log_likelihood.is_finite()); + } + + #[test] + fn a_unanimous_item_is_pinned_by_the_prior_and_says_so() { + // Every examinee correct on item 1. Unpenalized ML has no finite maximum + // here; the prior must keep it in range and the note must explain it. + let m = matrix_from(&["1101", "1110", "1100", "1010", "1111", "1000"]); + let f = fit(&m, &Options::default()); + let item1 = &f.items[0]; + assert!( + item1.b < -1.0, + "must be estimated as very easy, got {}", + item1.b + ); + assert!(item1.b > -B_MAX, "must not run off to the bound"); + assert!( + item1.notes.iter().any(|n| n.contains("prior")), + "must explain itself: {:?}", + item1.notes + ); + assert!(item1.bayesian); + } + + #[test] + fn small_samples_get_a_warning() { + let f = fit(&ordered_matrix(), &Options::default()); + assert!( + f.warnings.iter().any(|w| w.contains("examinees")), + "{:?}", + f.warnings + ); + } + + #[test] + fn disabling_priors_is_announced() { + let mut opts = Options::default(); + opts.priors.enabled = false; + let f = fit(&ordered_matrix(), &opts); + assert!(f.warnings.iter().any(|w| w.contains("priors are disabled"))); + } + + #[test] + fn rasch_fixes_discrimination_at_one() { + let mut opts = Options::default(); + opts.model = IrtModel::Rasch; + let f = fit(&ordered_matrix(), &opts); + assert!(f.items.iter().all(|i| (i.a - 1.0).abs() < 1e-12)); + assert!(f.items.iter().all(|i| i.c.is_none())); + } + + #[test] + fn three_pl_reports_a_lower_asymptote_and_warns() { + let mut opts = Options::default(); + opts.model = IrtModel::ThreePl; + let f = fit(&ordered_matrix(), &opts); + assert!(f.items.iter().all(|i| i.c.is_some())); + assert!(f + .items + .iter() + .all(|i| i.c.unwrap() >= 0.0 && i.c.unwrap() <= 0.4)); + assert!(f.warnings.iter().any(|w| w.contains("1000"))); + } + + #[test] + fn missing_responses_do_not_count_as_wrong() { + // Item 4 was administered to only the first three examinees. If missing + // were treated as incorrect it would look far harder than it is. + let with_missing = matrix_from(&["111.", "1110", "1101", "1..1", "110.", "100."]); + let f = fit(&with_missing, &Options::default()); + let item4 = &f.items[3]; + assert!( + item4.n < 6, + "only the administered responses count, got {}", + item4.n + ); + } + + #[test] + fn information_peaks_near_difficulty() { + let item = ItemFit { + number: 1, + a: 1.5, + b: 0.5, + c: None, + se_a: None, + se_b: None, + n: 30, + bayesian: true, + model: IrtModel::TwoPl, + notes: Vec::new(), + }; + // An item is most informative about students whose ability matches it. + assert!(item.information(0.5) > item.information(-1.5)); + assert!(item.information(0.5) > item.information(2.5)); + assert!((item.probability(0.5) - 0.5).abs() < 1e-12); + } + + #[test] + fn empty_input_does_not_panic() { + let m = Matrix { + students: Vec::new(), + items: Vec::new(), + credit: Vec::new(), + coded: Vec::new(), + }; + let f = fit(&m, &Options::default()); + assert!(f.items.is_empty()); + assert!(!f.warnings.is_empty()); + } +} diff --git a/src/analysis/students.rs b/src/analysis/students.rs new file mode 100644 index 0000000..1f3c49d --- /dev/null +++ b/src/analysis/students.rs @@ -0,0 +1,1142 @@ +//! Turning responses into something you can say to a student. +//! +//! Item analysis tells you about items. This module tells you about people: which +//! objectives a student has actually met, which ones they are close on, and what +//! specifically to do next. +//! +//! # On declaring mastery from three questions +//! +//! The honest answer is that you often cannot. Two items on an objective give a +//! proportion with an enormous confidence interval — two out of two correct is +//! consistent with a true rate anywhere from about 0.55 upward. So this module +//! does three things instead of pretending otherwise. +//! +//! It refuses to classify at all below `min_items_for_mastery`, reporting "not +//! enough evidence", which is a finding about your blueprint rather than about the +//! student. It reports the Wilson score interval alongside every rate, because +//! Wilson behaves sensibly at the boundaries where the normal approximation +//! produces intervals extending past 1.0. And it separates the *classification* +//! (which uses the observed rate, so it is usable) from the *confidence* (which +//! uses the interval, so it is honest). A student can be "meeting" an objective +//! provisionally, and the report says so. +//! +//! # Comparison to the cohort +//! +//! Per-level performance is reported against the class rather than in absolute +//! terms, because "you got 60% of the Analyze items" means nothing to a student +//! without knowing that the class average was 55%. The comparison is descriptive, +//! not a curve. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::course::{CourseFile, Policy}; +use crate::responses::{Response, ResponseSet}; +use crate::rng::Rng; +use crate::taxonomy::Level; + +/// How well a student has met one objective. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Mastery { + /// Met the threshold. + Meeting, + /// Partway there. + Developing, + /// Not yet. + NotYet, + /// Too few items on this objective to say anything. This is a gap in the + /// assessment, not a judgment about the student. + NotEnoughEvidence, +} + +impl Mastery { + /// A label for reports. + pub fn label(self) -> &'static str { + match self { + Mastery::Meeting => "meeting", + Mastery::Developing => "developing", + Mastery::NotYet => "not yet", + Mastery::NotEnoughEvidence => "not enough evidence", + } + } + + /// A short symbol for compact tables. + pub fn symbol(self) -> &'static str { + match self { + Mastery::Meeting => "✓", + Mastery::Developing => "~", + Mastery::NotYet => "✗", + Mastery::NotEnoughEvidence => "?", + } + } +} + +/// One student's standing on one objective. +#[derive(Debug, Clone)] +pub struct ObjectiveMastery { + /// The objective id. + pub objective: String, + /// The objective text, for reports. + pub text: String, + /// How many items on this objective the student saw. + pub n_items: usize, + /// How many they got right, counting partial credit. + pub credit: f64, + /// Observed rate, `credit / n_items`. + pub rate: f64, + /// Lower end of the Wilson interval. + pub wilson_lower: f64, + /// Upper end of the Wilson interval. + pub wilson_upper: f64, + /// The class's rate on the same objective. + pub cohort_rate: f64, + /// The classification. + pub status: Mastery, + /// Whether the interval, not just the point estimate, clears the threshold. + pub confident: bool, + /// Which levels the objective was assessed at, since meeting an objective at + /// Remember is a different claim from meeting it at Analyze. + pub levels: Vec, +} + +/// One student's performance at one cognitive level. +#[derive(Debug, Clone)] +pub struct LevelProfile { + /// The level. + pub level: Level, + /// How many items at this level. + pub n_items: usize, + /// The student's rate. + pub rate: f64, + /// The class's rate. + pub cohort_rate: f64, + /// Difference from the class, in class standard deviations. `None` when the + /// class had no spread at this level. + pub z: Option, +} + +impl LevelProfile { + /// A plain-language comparison to the class. + pub fn comparison(&self) -> &'static str { + match self.z { + Some(z) if z >= 1.0 => "well above the class", + Some(z) if z >= 0.4 => "above the class", + Some(z) if z > -0.4 => "about the same as the class", + Some(z) if z > -1.0 => "below the class", + Some(_) => "well below the class", + None => "the class did not vary here", + } + } +} + +/// An item the student got wrong, with what to do about it. +#[derive(Debug, Clone)] +pub struct MissedItem { + /// The question number. + pub number: u32, + /// The item's global id. + pub item_ref: Option, + /// What the student chose. + pub selected: Vec, + /// Credit earned, since a partially credited response is not a clean miss. + pub credit: f64, + /// The level. + pub level: Option, + /// The objectives involved. + pub learning_objectives: Vec, + /// The misconception the chosen distractor was written to detect. + pub misconception: Option, + /// Feedback written for a student who chose that option. + pub feedback: Option, + /// Where to go back to: lecture titles and slide numbers. + pub study: Vec, +} + +/// Everything needed to write one student's report. +#[derive(Debug, Clone)] +pub struct StudentSummary { + /// The grouping key. + pub student_key: String, + /// The name, when not pseudonymized. + pub name: Option, + /// The student id, when not pseudonymized. + pub sid: Option, + /// Points earned on scored items. + pub points: f64, + /// Points available on scored items. + pub points_possible: f64, + /// Percentage on scored items. + pub percent: f64, + /// Bonus points earned. + pub bonus_points: f64, + /// Items answered correctly. + pub correct: usize, + /// Items administered. + pub n_items: usize, + /// Where the score falls relative to the class, as a coarse band. + pub band: String, + /// IRT ability, when an IRT fit was supplied. + pub theta: Option, + /// Standard error of the ability estimate. + pub theta_se: Option, + /// Per-objective standing, in the course's objective order. + pub objectives: Vec, + /// Per-level standing. + pub levels: Vec, + /// Objectives the student is clearly meeting. + pub strengths: Vec, + /// Objectives to work on, worst first. + pub focus: Vec, + /// Missed items with targeted guidance. + pub missed: Vec, +} + +impl StudentSummary { + /// The display name, falling back to the key. + pub fn display_name(&self) -> String { + self.name + .clone() + .unwrap_or_else(|| self.student_key.clone()) + } +} + +/// Class-level context. +#[derive(Debug, Clone)] +pub struct Cohort { + /// Per-student summaries, sorted by key. + pub students: Vec, + /// Class rate per objective. + pub objective_rates: BTreeMap, + /// Class rate per level. + pub level_rates: BTreeMap, + /// Mean percentage. + pub mean_percent: f64, + /// Standard deviation of percentage. + pub sd_percent: f64, + /// Objectives the class as a whole did not meet, worst first. This is the + /// list that should change what you reteach. + pub class_gaps: Vec<(String, f64)>, + /// Optional grouping of students by response profile. + pub archetypes: Vec, +} + +/// A cluster of students with a similar profile across levels. +#[derive(Debug, Clone)] +pub struct Archetype { + /// A label describing the pattern. + pub label: String, + /// The student keys in this cluster. + pub members: Vec, + /// Mean rate at each level for this cluster. + pub level_means: BTreeMap, +} + +/// The Wilson score interval for a binomial proportion. +/// +/// Preferred over the normal approximation because it stays inside `[0, 1]` and +/// behaves at the boundaries, which is exactly where classroom data lives: a +/// student who got three out of three needs an interval, and the textbook formula +/// gives width zero there. +/// +/// # Arguments +/// +/// * `successes` - the number of successes, which may be fractional when partial +/// credit is involved. +/// * `n` - the number of trials. +/// * `z` - the standard normal quantile; 1.96 for a two-sided 95% interval. +/// +/// # Returns +/// +/// The lower and upper bounds, or `(0.0, 1.0)` when there are no trials. +pub fn wilson(successes: f64, n: usize, z: f64) -> (f64, f64) { + if n == 0 { + return (0.0, 1.0); + } + let n = n as f64; + let p = (successes / n).clamp(0.0, 1.0); + let z2 = z * z; + let denominator = 1.0 + z2 / n; + let center = p + z2 / (2.0 * n); + let spread = z * ((p * (1.0 - p) / n) + z2 / (4.0 * n * n)).sqrt(); + ( + ((center - spread) / denominator).clamp(0.0, 1.0), + ((center + spread) / denominator).clamp(0.0, 1.0), + ) +} + +/// Builds per-student summaries for one administration. +/// +/// # Arguments +/// +/// * `set` - the responses, already enriched with item metadata. +/// * `course` - the course, for objective text, order, and policy. +/// * `catalog` - the loaded course, for misconception feedback on missed items. +/// * `fit` - an optional IRT fit, whose abilities are attached when present. +/// +/// # Returns +/// +/// The cohort. +pub fn summarize( + set: &ResponseSet, + course: &CourseFile, + catalog: Option<&crate::catalog::Catalog>, + fit: Option<&crate::irt::Fit>, +) -> Cohort { + let policy = &course.policy; + let students = set.students(); + + // Class rates first: every student's report is relative to these. + let objective_rates = rates_by_objective(&set.rows.iter().collect::>()); + let level_rates = rates_by_level(&set.rows.iter().collect::>()); + + // Per-level spread across students, for the z comparisons. + let mut level_values: BTreeMap> = BTreeMap::new(); + for key in &students { + let rows = set.for_student(key); + for (level, rate) in rates_by_level(&rows) { + level_values.entry(level).or_default().push(rate); + } + } + let level_sd: BTreeMap = level_values + .iter() + .map(|(level, values)| (*level, sd(values))) + .collect(); + + let percents: Vec = students + .iter() + .map(|key| { + let earned = set.scored_total(key); + let possible = set.points_available(); + if possible > 0.0 { + 100.0 * earned / possible + } else { + 0.0 + } + }) + .collect(); + let mean_percent = mean(&percents); + let sd_percent = sd(&percents); + + let ability = fit.map(|f| f.ability_map()).unwrap_or_default(); + let ability_se: BTreeMap = fit + .map(|f| { + f.abilities + .iter() + .map(|a| (a.student_key.clone(), a.se)) + .collect() + }) + .unwrap_or_default(); + + let order = course.objectives_in_order(); + let mut summaries = Vec::with_capacity(students.len()); + + for (index, key) in students.iter().enumerate() { + let rows = set.for_student(key); + let points = set.scored_total(key); + let possible = set.points_available(); + let percent = percents[index]; + + let correct = rows + .iter() + .filter(|r| r.counts() && r.correct == Some(true)) + .count(); + let n_items = rows.iter().filter(|r| r.counts()).count(); + + // Objectives, in the course's declared order so reports read the way the + // course is taught rather than alphabetically. + let per_objective = rates_by_objective(&rows); + let counts = counts_by_objective(&rows); + let mut objectives = Vec::new(); + let mut seen: BTreeSet<&String> = BTreeSet::new(); + for id in order.iter().chain(per_objective.keys()) { + if !seen.insert(id) { + continue; + } + let Some((n, credit)) = counts.get(id).copied() else { + continue; + }; + objectives.push(objective_mastery( + id, + course, + n, + credit, + objective_rates.get(id).copied().unwrap_or(0.0), + &rows, + policy, + )); + } + + // Levels. + let student_levels = rates_by_level(&rows); + let level_counts = counts_by_level(&rows); + let levels: Vec = Level::ALL + .iter() + .filter_map(|level| { + let (n, _) = level_counts.get(level).copied()?; + if n == 0 { + return None; + } + let rate = student_levels.get(level).copied().unwrap_or(0.0); + let cohort_rate = level_rates.get(level).copied().unwrap_or(0.0); + let spread = level_sd.get(level).copied().unwrap_or(0.0); + Some(LevelProfile { + level: *level, + n_items: n, + rate, + cohort_rate, + z: if spread > 1e-9 { + Some((rate - cohort_rate) / spread) + } else { + None + }, + }) + }) + .collect(); + + // Strengths and focus areas. Strengths need confidence, focus areas do + // not: telling a student to review something they may already know costs + // them an hour, while telling them they have mastered something they have + // not costs them the next exam. + let strengths: Vec = objectives + .iter() + .filter(|o| o.status == Mastery::Meeting && o.confident) + .map(|o| o.objective.clone()) + .collect(); + let mut focus_pairs: Vec<(&ObjectiveMastery, f64)> = objectives + .iter() + .filter(|o| matches!(o.status, Mastery::NotYet | Mastery::Developing)) + .map(|o| (o, o.rate)) + .collect(); + focus_pairs.sort_by(|a, b| { + a.1.partial_cmp(&b.1) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.0.objective.cmp(&b.0.objective)) + }); + let focus: Vec = focus_pairs + .iter() + .map(|(o, _)| o.objective.clone()) + .collect(); + + let missed = missed_items(&rows, catalog, course); + + summaries.push(StudentSummary { + student_key: key.clone(), + name: rows.first().and_then(|r| r.name.clone()), + sid: rows.first().and_then(|r| r.sid.clone()), + points, + points_possible: possible, + percent, + bonus_points: set.bonus_total(key), + correct, + n_items, + band: band_for(percent, &percents), + theta: ability.get(key).copied(), + theta_se: ability_se.get(key).copied(), + objectives, + levels, + strengths, + focus, + missed, + }); + } + + // Class gaps: objectives where the whole class fell short. These are the ones + // to reteach rather than to send individual students away to review. + let mut class_gaps: Vec<(String, f64)> = objective_rates + .iter() + .filter(|(_, rate)| **rate < policy.mastery_threshold) + .map(|(id, rate)| (id.clone(), *rate)) + .collect(); + class_gaps.sort_by(|a, b| { + a.1.partial_cmp(&b.1) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.0.cmp(&b.0)) + }); + + let archetypes = cluster(&summaries, 3); + + Cohort { + students: summaries, + objective_rates, + level_rates, + mean_percent, + sd_percent, + class_gaps, + archetypes, + } +} + +/// Builds one objective's mastery record. +/// +/// # Arguments +/// +/// * `id` - the objective id. +/// * `course` - the course, for text and policy. +/// * `n` - items on this objective. +/// * `credit` - total credit earned. +/// * `cohort_rate` - the class rate. +/// * `rows` - the student's responses, for the level list. +/// * `policy` - the course policy. +/// +/// # Returns +/// +/// The record. +fn objective_mastery( + id: &str, + course: &CourseFile, + n: usize, + credit: f64, + cohort_rate: f64, + rows: &[&Response], + policy: &Policy, +) -> ObjectiveMastery { + let rate = if n > 0 { credit / n as f64 } else { 0.0 }; + let (lower, upper) = wilson(credit, n, 1.96); + + let status = if n < policy.min_items_for_mastery.max(1) { + Mastery::NotEnoughEvidence + } else if rate >= policy.mastery_threshold { + Mastery::Meeting + } else if rate >= policy.mastery_threshold * 0.6 { + Mastery::Developing + } else { + Mastery::NotYet + }; + + let levels: Vec = rows + .iter() + .filter(|r| r.learning_objectives.iter().any(|o| o == id)) + .filter_map(|r| r.level) + .collect::>() + .into_iter() + .collect(); + + ObjectiveMastery { + objective: id.to_string(), + text: course.objective_text(id), + n_items: n, + credit, + rate, + wilson_lower: lower, + wilson_upper: upper, + cohort_rate, + status, + confident: lower >= policy.mastery_threshold, + levels, + } +} + +/// Collects missed items with targeted guidance. +/// +/// The guidance comes from the item's own authoring: the `misconception` recorded +/// on the distractor the student actually chose, and the lecture and slides the +/// item was written from. This is why authoring distractors deliberately pays off +/// twice — once when writing the item, and again in every report afterward. +/// +/// # Arguments +/// +/// * `rows` - the student's responses. +/// * `catalog` - the loaded course. +/// * `course` - the course, for lecture titles. +/// +/// # Returns +/// +/// The missed items, in question order. +fn missed_items( + rows: &[&Response], + catalog: Option<&crate::catalog::Catalog>, + course: &CourseFile, +) -> Vec { + let mut out = Vec::new(); + for r in rows { + if !r.counts() || r.credit >= 0.999 { + continue; + } + let mut misconception = None; + let mut feedback = None; + let mut study = Vec::new(); + + if let (Some(cat), Some(uid)) = (catalog, r.item_ref.as_deref()) { + if let Some(entry) = cat.get(uid) { + // Feedback for the specific option chosen, which is the whole + // point of recording per-distractor misconceptions. + if let Some(letter) = r.selected.first() { + if let Some(choice) = entry.item.option(letter) { + misconception = choice.misconception.clone(); + feedback = choice.student_text().map(|s| s.to_string()); + } + } + for source in &entry.item.sources { + let title = course + .lectures + .get(&source.lecture) + .map(|l| l.title.clone()) + .unwrap_or_else(|| source.lecture.clone()); + if source.slides.is_empty() { + study.push(title); + } else { + let slides: Vec = + source.slides.iter().map(|s| s.to_string()).collect(); + study.push(format!("{title}, slides {}", slides.join(", "))); + } + for reading in &source.readings { + study.push(reading.clone()); + } + } + } + } + + out.push(MissedItem { + number: r.item_number, + item_ref: r.item_ref.clone(), + selected: r.selected.clone(), + credit: r.credit, + level: r.level, + learning_objectives: r.learning_objectives.clone(), + misconception, + feedback, + study, + }); + } + out +} + +/// Credit rate per objective over a set of responses. +/// +/// # Arguments +/// +/// * `rows` - the responses. +/// +/// # Returns +/// +/// The rate for each objective mentioned. +pub fn rates_by_objective(rows: &[&Response]) -> BTreeMap { + counts_by_objective(rows) + .into_iter() + .map(|(id, (n, credit))| { + let rate = if n > 0 { credit / n as f64 } else { 0.0 }; + (id, rate) + }) + .collect() +} + +/// Item counts and credit per objective. +/// +/// An item tagged with two objectives counts toward both. That double counting is +/// intentional: the question "how is this student doing on kinetics" should use +/// every item that measured kinetics. +/// +/// # Arguments +/// +/// * `rows` - the responses. +/// +/// # Returns +/// +/// `(item count, total credit)` per objective. +pub fn counts_by_objective(rows: &[&Response]) -> BTreeMap { + let mut out: BTreeMap = BTreeMap::new(); + for r in rows { + if !r.counts() { + continue; + } + for objective in &r.learning_objectives { + let e = out.entry(objective.clone()).or_insert((0, 0.0)); + e.0 += 1; + e.1 += r.credit.clamp(0.0, 1.0); + } + } + out +} + +/// Credit rate per level. +/// +/// # Arguments +/// +/// * `rows` - the responses. +/// +/// # Returns +/// +/// The rate for each level present. +pub fn rates_by_level(rows: &[&Response]) -> BTreeMap { + counts_by_level(rows) + .into_iter() + .map(|(level, (n, credit))| { + let rate = if n > 0 { credit / n as f64 } else { 0.0 }; + (level, rate) + }) + .collect() +} + +/// Item counts and credit per level. +/// +/// # Arguments +/// +/// * `rows` - the responses. +/// +/// # Returns +/// +/// `(item count, total credit)` per level. +pub fn counts_by_level(rows: &[&Response]) -> BTreeMap { + let mut out: BTreeMap = BTreeMap::new(); + for r in rows { + if !r.counts() { + continue; + } + if let Some(level) = r.level { + let e = out.entry(level).or_insert((0, 0.0)); + e.0 += 1; + e.1 += r.credit.clamp(0.0, 1.0); + } + } + out +} + +/// A coarse band for a score within a class. +/// +/// Quartile bands rather than an exact percentile, because a percentile computed +/// from twenty-four students implies a precision it does not have, and because +/// telling a student they are "37th percentile" invites comparison in a way that +/// "middle half of the class" does not. +/// +/// # Arguments +/// +/// * `percent` - the student's percentage. +/// * `all` - every student's percentage. +/// +/// # Returns +/// +/// The band label. +fn band_for(percent: f64, all: &[f64]) -> String { + if all.len() < 4 { + return "the class is too small to place this meaningfully".to_string(); + } + let mut sorted = all.to_vec(); + sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let below = sorted.iter().filter(|x| **x < percent).count() as f64; + let fraction = below / all.len() as f64; + if fraction >= 0.75 { + "top quarter of the class".to_string() + } else if fraction >= 0.5 { + "upper middle of the class".to_string() + } else if fraction >= 0.25 { + "lower middle of the class".to_string() + } else { + "bottom quarter of the class".to_string() + } +} + +/// Groups students by their profile across levels. +/// +/// This is descriptive, not diagnostic. It answers "are there recognizable +/// patterns in how this class is struggling" — for instance a group that handles +/// recall fine and falls apart on application, which calls for different +/// instruction than a group that is uniformly behind. +/// +/// k-means with a seeded, deterministic initialization, so the same data always +/// produces the same groups. +/// +/// # Arguments +/// +/// * `students` - the summaries. +/// * `k` - how many clusters to look for. +/// +/// # Returns +/// +/// The clusters, largest first. Empty when there are too few students to bother. +pub fn cluster(students: &[StudentSummary], k: usize) -> Vec { + // Below about three students per cluster the groups are noise. + if students.len() < k * 3 || k == 0 { + return Vec::new(); + } + + // Feature vector: rate at each level that anyone was assessed on. + let levels: Vec = students + .iter() + .flat_map(|s| s.levels.iter().map(|l| l.level)) + .collect::>() + .into_iter() + .collect(); + if levels.len() < 2 { + return Vec::new(); + } + + let points: Vec> = students + .iter() + .map(|s| { + levels + .iter() + .map(|level| { + s.levels + .iter() + .find(|l| l.level == *level) + .map(|l| l.rate) + .unwrap_or(0.0) + }) + .collect() + }) + .collect(); + + // Standardize each dimension so a level everyone did well on does not + // dominate the distance. + let mut standardized = points.clone(); + for d in 0..levels.len() { + let column: Vec = points.iter().map(|p| p[d]).collect(); + let m = mean(&column); + let s = sd(&column); + for (i, point) in standardized.iter_mut().enumerate() { + point[d] = if s > 1e-9 { + (points[i][d] - m) / s + } else { + 0.0 + }; + } + } + + // Seeded k-means++ initialization. + let mut rng = Rng::from_label("coursebank/archetypes"); + let mut centers: Vec> = + vec![standardized[rng.below(standardized.len() as u64) as usize].clone()]; + while centers.len() < k { + let distances: Vec = standardized + .iter() + .map(|p| { + centers + .iter() + .map(|c| squared_distance(p, c)) + .fold(f64::INFINITY, f64::min) + }) + .collect(); + let total: f64 = distances.iter().sum(); + if total <= 0.0 { + break; + } + let mut target = rng.unit() * total; + let mut chosen = standardized.len() - 1; + for (i, d) in distances.iter().enumerate() { + target -= d; + if target <= 0.0 { + chosen = i; + break; + } + } + centers.push(standardized[chosen].clone()); + } + + let mut assignment = vec![0usize; standardized.len()]; + for _ in 0..50 { + let mut changed = false; + for (i, p) in standardized.iter().enumerate() { + let mut best = (0usize, f64::INFINITY); + for (c, center) in centers.iter().enumerate() { + let d = squared_distance(p, center); + if d < best.1 { + best = (c, d); + } + } + if assignment[i] != best.0 { + assignment[i] = best.0; + changed = true; + } + } + for (c, center) in centers.iter_mut().enumerate() { + let members: Vec<&Vec> = standardized + .iter() + .enumerate() + .filter(|(i, _)| assignment[*i] == c) + .map(|(_, p)| p) + .collect(); + if members.is_empty() { + continue; + } + for d in 0..levels.len() { + center[d] = members.iter().map(|p| p[d]).sum::() / members.len() as f64; + } + } + if !changed { + break; + } + } + + let mut out = Vec::new(); + for c in 0..centers.len() { + let members: Vec = students + .iter() + .enumerate() + .filter(|(i, _)| assignment[*i] == c) + .map(|(_, s)| s.student_key.clone()) + .collect(); + if members.is_empty() { + continue; + } + let mut level_means = BTreeMap::new(); + for (d, level) in levels.iter().enumerate() { + let values: Vec = students + .iter() + .enumerate() + .filter(|(i, _)| assignment[*i] == c) + .map(|(i, _)| points[i][d]) + .collect(); + level_means.insert(*level, mean(&values)); + } + out.push(Archetype { + label: label_for(&level_means), + members, + level_means, + }); + } + out.sort_by(|a, b| b.members.len().cmp(&a.members.len())); + out +} + +/// Names a cluster from its level profile. +/// +/// # Arguments +/// +/// * `means` - mean rate at each level. +/// +/// # Returns +/// +/// A descriptive label. +fn label_for(means: &BTreeMap) -> String { + let values: Vec = means.values().copied().collect(); + let overall = mean(&values); + + // Is the profile flat, or does it fall off with cognitive demand? + let low: Vec = means + .iter() + .filter(|(l, _)| l.code() <= 2) + .map(|(_, v)| *v) + .collect(); + let high: Vec = means + .iter() + .filter(|(l, _)| l.code() >= 3) + .map(|(_, v)| *v) + .collect(); + + if !low.is_empty() && !high.is_empty() { + let drop = mean(&low) - mean(&high); + if drop > 0.25 { + return "knows the material, struggles to apply it".to_string(); + } + if drop < -0.15 { + return "reasons well, gaps in recall".to_string(); + } + } + + if overall >= 0.85 { + "consistently strong".to_string() + } else if overall >= 0.65 { + "solid with scattered gaps".to_string() + } else { + "behind across the board".to_string() + } +} + +/// Squared Euclidean distance. +fn squared_distance(a: &[f64], b: &[f64]) -> f64 { + a.iter().zip(b.iter()).map(|(x, y)| (x - y) * (x - y)).sum() +} + +/// The arithmetic mean, zero for an empty slice. +fn mean(v: &[f64]) -> f64 { + if v.is_empty() { + 0.0 + } else { + v.iter().sum::() / v.len() as f64 + } +} + +/// The population standard deviation. +fn sd(v: &[f64]) -> f64 { + if v.len() < 2 { + return 0.0; + } + let m = mean(v); + (v.iter().map(|x| (x - m) * (x - m)).sum::() / v.len() as f64).sqrt() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn wilson_stays_inside_zero_and_one() { + // Three out of three: the naive interval has zero width, Wilson does not. + let (lo, hi) = wilson(3.0, 3, 1.96); + assert!(lo > 0.0 && lo < 1.0, "lower bound {lo}"); + assert_eq!(hi, 1.0); + assert!(lo < 0.5, "three items cannot establish a high rate: {lo}"); + + // Zero out of four. + let (lo, hi) = wilson(0.0, 4, 1.96); + assert_eq!(lo, 0.0); + assert!(hi > 0.0 && hi < 1.0); + + // No data at all. + assert_eq!(wilson(0.0, 0, 1.96), (0.0, 1.0)); + } + + #[test] + fn wilson_narrows_as_n_grows() { + let (lo_small, hi_small) = wilson(8.0, 10, 1.96); + let (lo_big, hi_big) = wilson(80.0, 100, 1.96); + assert!( + (hi_big - lo_big) < (hi_small - lo_small), + "more data must give a tighter interval" + ); + } + + #[test] + fn two_items_never_claim_confident_mastery() { + // The classification may say "meeting", but confidence must not, because + // two items cannot establish a rate of 0.75. + let (lower, _) = wilson(2.0, 2, 1.96); + assert!(lower < 0.75, "got {lower}"); + } + + #[test] + fn bands_describe_position_coarsely() { + let all = vec![50.0, 60.0, 70.0, 80.0, 90.0, 95.0, 40.0, 30.0]; + assert!(band_for(95.0, &all).contains("top")); + assert!(band_for(30.0, &all).contains("bottom")); + // Too few students to place anyone. + assert!(band_for(50.0, &[50.0, 60.0]).contains("too small")); + } + + #[test] + fn objective_counts_credit_every_tagged_item() { + let rows = vec![ + make("s1", 1, 1.0, &["lo-a", "lo-b"], Some(Level::Remember)), + make("s1", 2, 0.0, &["lo-a"], Some(Level::Apply)), + ]; + let refs: Vec<&Response> = rows.iter().collect(); + let counts = counts_by_objective(&refs); + // lo-a saw both items; lo-b only the first. + assert_eq!(counts["lo-a"], (2, 1.0)); + assert_eq!(counts["lo-b"], (1, 1.0)); + let rates = rates_by_objective(&refs); + assert_eq!(rates["lo-a"], 0.5); + assert_eq!(rates["lo-b"], 1.0); + } + + #[test] + fn level_rates_ignore_untagged_items() { + let rows = vec![ + make("s1", 1, 1.0, &[], Some(Level::Remember)), + make("s1", 2, 0.0, &[], None), + ]; + let refs: Vec<&Response> = rows.iter().collect(); + let counts = counts_by_level(&refs); + assert_eq!(counts.len(), 1); + assert_eq!(counts[&Level::Remember], (1, 1.0)); + } + + #[test] + fn mastery_labels_are_stable() { + assert_eq!(Mastery::Meeting.label(), "meeting"); + assert_eq!(Mastery::NotEnoughEvidence.label(), "not enough evidence"); + } + + #[test] + fn clustering_needs_enough_students() { + assert!(cluster(&[], 3).is_empty()); + let few: Vec = (0..4).map(|i| summary(&format!("s{i}"))).collect(); + assert!(cluster(&few, 3).is_empty(), "four students, three clusters"); + } + + #[test] + fn clustering_is_deterministic_and_separates_profiles() { + // Half the class is strong on recall and weak on application; half is + // uniformly strong. Those are different problems. + let mut students = Vec::new(); + for i in 0..12 { + let mut s = summary(&format!("s{i:02}")); + let (recall, apply) = if i < 6 { (0.95, 0.35) } else { (0.9, 0.85) }; + s.levels = vec![ + profile(Level::Remember, recall), + profile(Level::Apply, apply), + ]; + students.push(s); + } + let first = cluster(&students, 2); + let second = cluster(&students, 2); + assert_eq!(first.len(), 2); + assert_eq!( + first.iter().map(|a| a.members.clone()).collect::>(), + second.iter().map(|a| a.members.clone()).collect::>(), + "clustering must be reproducible" + ); + // The two groups must not be mixed together. + let sizes: Vec = first.iter().map(|a| a.members.len()).collect(); + assert_eq!(sizes, vec![6, 6], "got {sizes:?}"); + assert!(first.iter().any(|a| a.label.contains("struggles to apply"))); + } + + fn make( + student: &str, + number: u32, + credit: f64, + objectives: &[&str], + level: Option, + ) -> Response { + Response { + administration_id: "C/T/a".into(), + course: "C".into(), + term: "T".into(), + assessment_id: "a".into(), + date: None, + form: None, + student_key: student.into(), + sid: None, + name: None, + email: None, + section: None, + item_number: number, + item_ref: None, + item_version: None, + selected: vec!["A".into()], + eliminated: vec![], + correct: Some(credit >= 0.999), + credit, + points_possible: 1.0, + score: credit, + response_time_seconds: None, + level, + learning_objectives: objectives.iter().map(|s| s.to_string()).collect(), + topics: vec![], + bonus: false, + dropped: false, + } + } + + fn summary(key: &str) -> StudentSummary { + StudentSummary { + student_key: key.to_string(), + name: None, + sid: None, + points: 0.0, + points_possible: 0.0, + percent: 0.0, + bonus_points: 0.0, + correct: 0, + n_items: 0, + band: String::new(), + theta: None, + theta_se: None, + objectives: Vec::new(), + levels: Vec::new(), + strengths: Vec::new(), + focus: Vec::new(), + missed: Vec::new(), + } + } + + fn profile(level: Level, rate: f64) -> LevelProfile { + LevelProfile { + level, + n_items: 4, + rate, + cohort_rate: rate, + z: None, + } + } +} diff --git a/src/authoring.rs b/src/authoring.rs new file mode 100644 index 0000000..bab926a --- /dev/null +++ b/src/authoring.rs @@ -0,0 +1,23 @@ +//! Support for writing items and building assessments from them. +//! +//! The three modules here are what you interact with before an exam exists, and +//! they divide by how much authority each one has. +//! +//! [`lint`] has none. Its 26 rules are advice with stable codes, every one +//! silenceable with `--ignore`. It is deliberately separate from the validation in +//! [`crate::model::bank`], which enforces what must be true and fails. Conflating +//! the two produces a tool that either blocks you on style or lets real errors +//! through. +//! +//! [`select`] draws an assessment from a blueprint. It places objective minimums +//! before level quotas, because a coverage requirement is the constraint most +//! likely to become unsatisfiable, and it prefers least-recently-used items so a +//! bank rotates rather than converging on your favourites. +//! +//! [`jsonschema`] emits JSON Schema so an editor autocompletes the YAML. That is +//! a better authoring experience than any validator: catching `cognitve_process` +//! as you type beats reading it in a list afterward. + +pub mod jsonschema; +pub mod lint; +pub mod select; diff --git a/src/authoring/jsonschema.rs b/src/authoring/jsonschema.rs new file mode 100644 index 0000000..302d4ee --- /dev/null +++ b/src/authoring/jsonschema.rs @@ -0,0 +1,1019 @@ +//! Emitting JSON Schema for the YAML formats. +//! +//! The point of this module is autocomplete. Every editor with a YAML language +//! server reads a `# yaml-language-server: $schema=...` modeline, and once it does, +//! writing an item becomes a matter of tabbing through valid `cognitive_process` +//! values instead of looking them up. That is a much better authoring experience +//! than running a validator afterward and reading a list of typos. +//! +//! The schemas are written by hand rather than derived from the Rust types. That is +//! a real cost — two definitions to keep in step — bought for two reasons: the +//! schema can carry prose descriptions aimed at whoever is writing the item, which +//! is what shows up in editor tooltips, and it can encode `enum` value lists that a +//! generic derivation would emit as bare strings. The crate's own validation +//! remains authoritative; the schema is for the editor. + +use serde_json::{json, Value}; + +use crate::course::SCHEMA_VERSION; +use crate::error::Result; +use crate::taxonomy::{CognitiveProcess, ErrorType, Flag, Level}; + +/// The base URL schemas refer to each other by. +const BASE: &str = "https://coursebank.dev/schema"; + +/// The three schema kinds. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Kind { + /// `course.yaml`. + Course, + /// `banks/*.yaml`. + Bank, + /// `assessments/*.yaml`. + Assessment, +} + +impl Kind { + /// All three kinds. + pub const ALL: [Kind; 3] = [Kind::Course, Kind::Bank, Kind::Assessment]; + + /// The file name a schema is written to. + pub fn filename(self) -> &'static str { + match self { + Kind::Course => "course.schema.json", + Kind::Bank => "bank.schema.json", + Kind::Assessment => "assessment.schema.json", + } + } + + /// The modeline that points an editor at this schema. + /// + /// # Arguments + /// + /// * `relative_path` - the path from the YAML file to the schema directory. + /// + /// # Returns + /// + /// A comment line to place at the top of the YAML file. + pub fn modeline(self, relative_path: &str) -> String { + format!( + "# yaml-language-server: $schema={}/{}", + relative_path.trim_end_matches('/'), + self.filename() + ) + } +} + +/// Builds a schema. +/// +/// # Arguments +/// +/// * `kind` - which schema to build. +/// +/// # Returns +/// +/// The schema as JSON. +pub fn schema(kind: Kind) -> Value { + match kind { + Kind::Course => course_schema(), + Kind::Bank => bank_schema(), + Kind::Assessment => assessment_schema(), + } +} + +/// Writes all three schemas to a directory. +/// +/// # Arguments +/// +/// * `dir` - the destination directory. +/// +/// # Returns +/// +/// The paths written. +/// +/// # Errors +/// +/// Returns [`crate::error::Error::Io`] on a write failure. +pub fn write_all(dir: &std::path::Path) -> Result> { + std::fs::create_dir_all(dir).map_err(|e| crate::error::Error::io(dir, e))?; + let mut written = Vec::new(); + for kind in Kind::ALL { + let path = dir.join(kind.filename()); + crate::yaml::write_json(&path, &schema(kind))?; + written.push(path); + } + Ok(written) +} + +/// The string values of an enum, for a schema `enum` list. +fn strings>(values: &[T]) -> Value { + Value::Array( + values + .iter() + .map(|v| Value::String(v.as_ref().to_string())) + .collect(), + ) +} + +/// A schema fragment for a required non-empty string. +fn text(description: &str) -> Value { + json!({ "type": "string", "minLength": 1, "description": description }) +} + +/// A schema fragment for an array of strings. +fn string_array(description: &str) -> Value { + json!({ + "type": "array", + "items": { "type": "string" }, + "description": description + }) +} + +/// A schema fragment for a proportion in `[0, 1]`. +fn proportion(description: &str) -> Value { + json!({ + "type": "number", + "minimum": 0.0, + "maximum": 1.0, + "description": description + }) +} + +/// A schema fragment for a `YYYY-MM-DD` date. +fn date(description: &str) -> Value { + json!({ + "type": "string", + "pattern": "^\\d{4}-\\d{2}-\\d{2}$", + "description": description + }) +} + +/// The level enum, with the taxonomy spelled out in the description so it appears +/// in editor tooltips. +fn level() -> Value { + let descriptions: Vec = Level::ALL + .iter() + .map(|l| format!("{} = {} ({})", l.code(), l.name(), l.blurb())) + .collect(); + json!({ + "type": "integer", + "minimum": 1, + "maximum": 5, + "description": format!("Cognitive level. {}", descriptions.join("; ")) + }) +} + +/// The cognitive process enum, grouped by level in the description. +fn cognitive_process() -> Value { + let all: Vec<&str> = CognitiveProcess::ALL.iter().map(|p| p.as_str()).collect(); + let by_level: Vec = Level::ALL + .iter() + .map(|l| { + let names: Vec<&str> = l.processes().iter().map(|p| p.as_str()).collect(); + format!("level {}: {}", l.code(), names.join(", ")) + }) + .collect(); + json!({ + "type": "string", + "enum": strings(&all), + "description": format!( + "The specific cognitive operation. Must belong to the item's level — {}.", + by_level.join("; ") + ) + }) +} + +/// The distractor error-type enum, with each gloss in the description. +fn error_type() -> Value { + let all: Vec<&str> = ErrorType::ALL.iter().map(|e| e.as_str()).collect(); + let glosses: Vec = ErrorType::ALL + .iter() + .map(|e| format!("{} — {}", e.as_str(), e.gloss())) + .collect(); + json!({ + "type": "string", + "enum": strings(&all), + "description": format!( + "What kind of mistake this distractor is designed to catch. {}", + glosses.join("; ") + ) + }) +} + +/// The schema for the `course` identity block. +fn course_identity_schema() -> Value { + json!({ + "type": "object", + "required": ["code", "title", "term"], + "additionalProperties": false, + "properties": { + "code": text("Course code, e.g. BIOSC 1540."), + "title": text("Course title."), + "term": text("Term, e.g. 2026S."), + "institution": { "type": "string" }, + "instructors": string_array("Instructor names."), + "slug": { + "type": "string", + "description": "Short identifier used in file names and administration ids. \ + Derived from the code if omitted." + } + } + }) +} + +/// The schema for the course-wide policy block. +fn policy_schema() -> Value { + json!({ + "type": "object", + "additionalProperties": false, + "description": "Course-wide defaults and rules that validation enforces.", + "properties": { + "points_per_item": { + "type": "number", + "exclusiveMinimum": 0.0, + "description": "Default points for an item that does not set its own." + }, + "options_per_item": { + "type": "integer", + "minimum": 2, + "description": "Expected option count; the linter flags items that differ." + }, + "bonus_levels": { + "type": "array", + "items": level(), + "description": "Levels a bonus item may be drawn from." + }, + "allow_partial_credit": { + "type": "boolean", + "description": "Whether any option may carry partial credit. When false, \ + validation rejects items that do." + }, + "partial_credit_floor_level": level(), + "mastery_threshold": proportion( + "Rate at which an objective counts as met. 0.75 is a common choice." + ), + "min_items_for_mastery": { + "type": "integer", + "minimum": 1, + "description": "Below this many items on an objective, reports say 'not enough \ + evidence' rather than classifying." + } + } + }) +} + +/// The schema for one unit. +fn unit_schema() -> Value { + json!({ + "type": "object", + "required": ["id", "title"], + "additionalProperties": false, + "properties": { + "id": text("Unit id."), + "title": text("Unit title."), + "description": { "type": "string" } + } + }) +} + +/// The schema for one lecture. +fn lecture_schema() -> Value { + json!({ + "type": "object", + "required": ["title"], + "additionalProperties": false, + "properties": { + "title": text("Lecture title."), + "date": date("Date delivered."), + "unit": { "type": "string", "description": "Unit id." }, + "slides_url": { "type": "string" }, + "readings": string_array("Readings assigned with this lecture.") + } + }) +} + +/// The schema for one learning objective. +fn objective_schema() -> Value { + json!({ + "type": "object", + "required": ["text"], + "additionalProperties": false, + "properties": { + "text": text("The objective as a student would read it. Start with a verb."), + "unit": { "type": "string" }, + "lectures": string_array("Lecture ids that cover this."), + "level_ceiling": level(), + "prerequisites": string_array( + "Objective ids that must come first. Cycles are rejected." + ), + "tags": string_array("Free-form tags."), + "assessed": { + "type": "boolean", + "description": "Set false for an objective you teach but do not test; coverage \ + reporting will stop flagging it as a gap." + } + } + }) +} + +/// The schema for one shared stimulus. +fn stimulus_schema() -> Value { + json!({ + "type": "object", + "required": ["body"], + "additionalProperties": false, + "properties": { + "body": text("The stimulus text, in coursebank markup."), + "asset": { "type": "string", "description": "Path to an image." }, + "caption": { "type": "string" } + } + }) +} + +/// The course schema. +fn course_schema() -> Value { + json!({ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": format!("{BASE}/course.schema.json"), + "title": "coursebank course file", + "description": "Course identity, policy, and the registries that item and assessment \ + files reference by id.", + "type": "object", + "required": ["course"], + "additionalProperties": false, + "properties": { + "schema_version": { + "type": ["string", "number"], + "description": format!("Format version; currently {SCHEMA_VERSION}.") + }, + "course": course_identity_schema(), + "policy": policy_schema(), + "units": { + "type": "array", + "description": "Course units, in teaching order. That order drives report layout.", + "items": unit_schema() + }, + "lectures": { + "type": "object", + "description": "Lectures by id. Items cite these so reports can tell a student \ + where to go back to.", + "additionalProperties": lecture_schema() + }, + "learning_objectives": { + "type": "object", + "description": "Objectives by id. Everything downstream — coverage, mastery, \ + student reports — keys off these.", + "additionalProperties": objective_schema() + }, + "stimuli": { + "type": "object", + "description": "Shared passages, figures, or data that several items refer to.", + "additionalProperties": stimulus_schema() + } + } + }) +} + +/// One option's schema. +/// +/// Split out from [`item_schema`] rather than inlined, because `serde_json`'s +/// `json!` macro recurses once per nesting level *and* once per key-value pair. A +/// single literal describing the whole item exceeded the default recursion limit of +/// 128, so each subtree gets its own shallow invocation. Keeping them small also +/// means adding a field later cannot silently reintroduce the problem. +fn option_schema() -> Value { + json!({ + "type": "object", + "required": ["id", "text"], + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "pattern": "^[A-H]$", + "description": "Option letter. Identity, not print position — shuffled forms \ + relabel on the way out." + }, + "text": text("The option as a student reads it."), + "correct": { "type": "boolean" }, + "credit": proportion("Partial credit. Requires defensible: true and a defense."), + "explanation": { + "type": "string", + "description": "Why this option is right or wrong. For you, not the student." + }, + "hint": { "type": "string" }, + "misconception": { + "type": "string", + "description": "The specific wrong belief that leads here. This text is what \ + student reports use, so write it as a completion of 'students \ + pick this when ...'." + }, + "error_type": error_type(), + "defensible": { + "type": "boolean", + "description": "This option has a reading under which it is arguably correct. \ + Required before granting partial credit." + }, + "defense": { + "type": "string", + "description": "The argument for that reading. Required when defensible is true." + }, + "feedback_student": { + "type": "string", + "description": "Shown to a student who chose this, on Canvas and in reports." + }, + "selection_rate_expected": proportion( + "How often you expect this to be chosen. Compared against reality." + ) + } + }) +} + +/// The schema for where an item's material was taught. +fn source_schema() -> Value { + json!({ + "type": "object", + "required": ["lecture"], + "additionalProperties": false, + "properties": { + "lecture": text("Lecture id from course.yaml."), + "slides": { "type": "array", "items": { "type": "integer", "minimum": 1 } }, + "readings": string_array("Specific readings."), + "recording_seconds": { "type": "integer", "minimum": 0 } + } + }) +} + +/// The schema for an attached figure. +fn asset_schema() -> Value { + json!({ + "type": "object", + "required": ["path"], + "additionalProperties": false, + "properties": { + "path": text("Path to the image, relative to the course root."), + "alt": { + "type": "string", + "description": "Alt text. Required in practice: the linter flags an asset without \ + it, because an exam question a screen reader cannot convey is not \ + answerable." + }, + "caption": { "type": "string" } + } + }) +} + +/// The schema for authored design intent. +fn design_schema() -> Value { + json!({ + "type": "object", + "additionalProperties": false, + "description": "What you expected before giving the item. Kept separate from observed \ + statistics so the two can be compared.", + "properties": { + "expected_difficulty": proportion("Fraction you expect to answer correctly."), + "expected_discrimination": { + "type": "string", + "enum": ["low", "moderate", "high"] + }, + "expected_time_seconds": { "type": "number", "exclusiveMinimum": 0.0 }, + "rationale": { + "type": "string", + "description": "Why this item exists and what it is meant to catch." + } + } + }) +} + +/// The schema for one option's observed statistics. +fn option_stat_schema() -> Value { + json!({ + "type": "object", + "additionalProperties": false, + "properties": { + "selection_rate": proportion("How often chosen."), + "point_biserial": { "type": "number", "minimum": -1.0, "maximum": 1.0 }, + "upper_group_rate": proportion("Rate in the top 27%."), + "lower_group_rate": proportion("Rate in the bottom 27%.") + } + }) +} + +/// The schema for fitted IRT parameters. +fn irt_schema() -> Value { + json!({ + "type": "object", + "required": ["a", "b"], + "additionalProperties": false, + "properties": { + "model": { "type": "string", "enum": ["rasch", "2pl", "3pl"] }, + "a": { "type": "number", "description": "Discrimination." }, + "b": { "type": "number", "description": "Difficulty." }, + "c": proportion("Lower asymptote, 3PL only."), + "se_a": { "type": "number", "minimum": 0.0 }, + "se_b": { "type": "number", "minimum": 0.0 }, + "n": { "type": "integer", "minimum": 0 }, + "bayesian": { + "type": "boolean", + "description": "Whether priors were used. On a class-sized sample they should be." + } + } + }) +} + +/// The schema for pooled observed statistics. +fn calibration_schema() -> Value { + let flags: Vec<&str> = Flag::ALL.iter().map(|f| f.as_str()).collect(); + json!({ + "type": "object", + "additionalProperties": false, + "description": "Written by `coursebank calibrate`, not by hand. Pooled across \ + administrations.", + "properties": { + "administrations": string_array("Administration ids pooled here."), + "updated": date("When calibration last ran."), + "fingerprint": { + "type": "string", + "description": "Hash of what a student saw. When it stops matching the item, these \ + statistics describe a different question." + }, + "n_examinees": { "type": "integer", "minimum": 0 }, + "p_value": proportion("Observed proportion correct."), + "point_biserial": { "type": "number", "minimum": -1.0, "maximum": 1.0 }, + "discrimination_index": { "type": "number", "minimum": -1.0, "maximum": 1.0 }, + "mean_response_time_seconds": { "type": "number", "minimum": 0.0 }, + "rapid_guess_rate": proportion("Fraction answered faster than readable."), + "option_stats": { + "type": "object", + "additionalProperties": option_stat_schema() + }, + "irt": irt_schema(), + "flags": { + "type": "array", + "items": { "type": "string", "enum": strings(&flags) } + } + } + }) +} + +/// The schema for a recorded review decision. +fn review_schema() -> Value { + json!({ + "type": "object", + "additionalProperties": false, + "properties": { + "reviewed_by": { "type": "string" }, + "reviewed_on": date("Review date."), + "action": { + "type": "string", + "enum": ["keep", "revise", "award_partial_credit", "correct_key", "retire", + "monitor"] + }, + "notes": { "type": "string" } + } + }) +} + +/// The schema for one revision-history entry. +fn history_schema() -> Value { + json!({ + "type": "object", + "required": ["version", "date", "change"], + "additionalProperties": false, + "properties": { + "version": { "type": "integer", "minimum": 1 }, + "date": date("When the change was made."), + "author": { "type": "string" }, + "change": text("What changed and why.") + } + }) +} + +/// The schema for a retirement record. +fn retirement_schema() -> Value { + json!({ + "type": "object", + "required": ["on", "reason"], + "additionalProperties": false, + "properties": { + "on": date("Retirement date."), + "reason": text("Why it was retired."), + "replaced_by": { "type": "string", "description": "Successor item id." } + } + }) +} + +/// The identity and classification half of the item schema. +/// +/// Split from [`item_content_properties`] purely to keep each `json!` invocation +/// short; the two are merged into one `properties` object by [`item_schema`]. +fn item_identity_properties() -> Value { + json!({ + "id": { + "type": "string", + "pattern": "^q-[a-z0-9]+(-[a-z0-9]+)*-[0-9]{3}$", + "description": "Item id, e.g. q-glycolysis-014. Stable forever: assessment records \ + and stored responses refer to it." + }, + "version": { + "type": "integer", + "minimum": 1, + "description": "Bump when you change what a student sees. Recorded on every \ + administration so drift is detectable." + }, + "status": { + "type": "string", + "enum": ["draft", "in_review", "needs_revision", "approved", "retired"], + "description": "Only approved items can be drawn into an assessment." + }, + "level": level(), + "cognitive_process": cognitive_process(), + "format": { + "type": "string", + "enum": ["single_best_answer", "multiple_response", "true_false"], + "description": "single_best_answer requires exactly one keyed option; \ + multiple_response requires at least two." + }, + "bonus": { "type": "boolean" }, + "points": { "type": "number", "exclusiveMinimum": 0.0 }, + "author": { "type": "string" }, + "notes_private": { + "type": "string", + "description": "Never exported anywhere a student can see." + } + }) +} + +/// The content and evidence half of the item schema. +fn item_content_properties() -> Value { + json!({ + "title": { + "type": "string", + "description": "Short internal label. Never shown to students." + }, + "stimulus": { "type": "string", "description": "Stimulus id from course.yaml." }, + "stem": text( + "The question. Ask something specific; the linter flags stems with no task in them." + ), + "options": { + "type": "array", + "minItems": 2, + "maxItems": 8, + "items": option_schema() + }, + "learning_objectives": string_array( + "Objective ids this item measures. Reports aggregate on these, so an item with none \ + contributes to nothing." + ), + "sources": { + "type": "array", + "description": "Where the material was taught. Drives the 'review this' lines in \ + student reports.", + "items": source_schema() + }, + "topics": string_array("Free-form topics, used for blueprint filtering."), + "prerequisites": string_array("Objective ids a student needs before this item."), + "assets": { "type": "array", "items": asset_schema() }, + "design": design_schema(), + "calibration": calibration_schema(), + "review": review_schema(), + "history": { + "type": "array", + "description": "One entry per version. Versions must increase.", + "items": history_schema() + }, + "retired": retirement_schema() + }) +} + +/// The item schema fragment, shared by the bank schema. +/// +/// Assembled from the helpers above rather than written as one literal. See +/// [`option_schema`] for why. +fn item_schema() -> Value { + let mut properties = serde_json::Map::new(); + for half in [item_identity_properties(), item_content_properties()] { + if let Value::Object(map) = half { + properties.extend(map); + } + } + json!({ + "type": "object", + "required": ["id", "level", "stem", "options"], + "additionalProperties": false, + "properties": Value::Object(properties) + }) +} + +/// The schema for a bank's `bank` metadata block. +fn bank_meta_schema() -> Value { + json!({ + "type": "object", + "required": ["id", "title"], + "additionalProperties": false, + "properties": { + "id": text("Bank id, unique within the course."), + "title": text("Human-readable title."), + "description": { "type": "string" }, + "scope": bank_scope_schema() + } + }) +} + +/// The schema for what a bank is meant to cover. +fn bank_scope_schema() -> Value { + json!({ + "type": "object", + "additionalProperties": false, + "description": "What this bank is meant to cover. Validation warns when an item strays \ + outside it.", + "properties": { + "lectures": string_array("Lecture ids."), + "learning_objectives": string_array("Objective ids."), + "units": string_array("Unit ids."), + "topics": string_array("Topics.") + } + }) +} + +/// The schema for per-file item defaults. +fn bank_defaults_schema() -> Value { + json!({ + "type": "object", + "additionalProperties": false, + "description": "Applied to every item in the file that does not set the field. Saves \ + repeating yourself; the item always wins.", + "properties": { + "author": { "type": "string" }, + "points": { "type": "number", "exclusiveMinimum": 0.0 }, + "options_per_item": { "type": "integer", "minimum": 2 }, + "topics": string_array("Topics added to every item."), + "lectures": string_array("Lecture ids for items with no sources of their own.") + } + }) +} + +/// The bank schema. +fn bank_schema() -> Value { + json!({ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": format!("{BASE}/bank.schema.json"), + "title": "coursebank item bank", + "description": "A collection of items. Split banks by unit or topic; ids must be unique \ + across the whole course, not just within a file.", + "type": "object", + "required": ["bank", "items"], + "additionalProperties": false, + "properties": { + "schema_version": { "type": ["string", "number"] }, + "bank": bank_meta_schema(), + "defaults": bank_defaults_schema(), + "items": { "type": "array", "items": item_schema() } + } + }) +} + +/// The schema for the `assessment` metadata block. +fn assessment_meta_schema() -> Value { + json!({ + "type": "object", + "required": ["id", "title"], + "additionalProperties": false, + "properties": { + "id": text("Assessment id, e.g. exam-4. Used in administration ids."), + "title": text("Printed title."), + "term": { "type": "string", "description": "Defaults to the course term." }, + "kind": { + "type": "string", + "enum": ["exam", "quiz", "homework", "practice", "final"], + "description": "practice is excluded from calibration by default, since \ + conditions differ too much to pool." + }, + "date": date("Administration date. Drives reuse cooldowns."), + "platform": { "type": "string", "enum": ["paper", "canvas", "other"] }, + "minutes_allowed": { "type": "number", "exclusiveMinimum": 0.0 }, + "attempts": { + "type": "integer", + "description": "Canvas attempt limit; -1 for unlimited." + }, + "shuffle": { "type": "boolean" }, + "scoring_policy": { "type": "string", "enum": ["keep_highest", "keep_latest"] }, + "instructions": { "type": "string" }, + "notes": { "type": "string" } + } + }) +} + +/// The schema for the blueprint an assessment was drawn to. +fn blueprint_schema() -> Value { + json!({ + "type": "object", + "additionalProperties": false, + "description": "The design the form was drawn to satisfy. Kept so the form can be checked \ + against the intent afterward.", + "properties": { + "level_counts": { + "type": "object", + "description": "How many scored items at each level, keyed by level number.", + "additionalProperties": { "type": "integer", "minimum": 0 } + }, + "bonus_counts": { + "type": "object", + "additionalProperties": { "type": "integer", "minimum": 0 } + }, + "objective_minimums": { + "type": "object", + "description": "Minimum items per objective. Placed before level quotas, because \ + a coverage requirement is the constraint most likely to become \ + unsatisfiable.", + "additionalProperties": { "type": "integer", "minimum": 0 } + }, + "lectures": string_array("Restrict the draw to these lectures."), + "topics": string_array("Restrict the draw to these topics."), + "banks": string_array("Restrict the draw to these banks."), + "max_per_bank": { "type": "integer", "minimum": 1 }, + "cooldown_days": { + "type": "integer", + "minimum": 0, + "description": "Avoid items used within this many days. Relaxed with a warning \ + rather than failing the draw." + }, + "seed": { + "type": "integer", + "minimum": 0, + "description": "Makes the draw reproducible." + } + } + }) +} + +/// The schema for one alternate form. +fn form_schema() -> Value { + json!({ + "type": "object", + "required": ["id", "seed"], + "additionalProperties": false, + "properties": { + "id": text("Form label, e.g. A."), + "seed": { "type": "integer", "minimum": 0 }, + "shuffle_items": { "type": "boolean" }, + "shuffle_options": { "type": "boolean" } + } + }) +} + +/// The schema for one question placement. +fn placement_schema() -> Value { + json!({ + "type": "object", + "required": ["number", "item"], + "additionalProperties": false, + "properties": { + "number": { + "type": "integer", + "minimum": 1, + "description": "The question number as administered. This is the join key to \ + Gradescope and Canvas exports, so it must not change after the \ + fact." + }, + "item": text("Item reference, `bank::item-id` or a bare item id."), + "version": { "type": "integer", "minimum": 1 }, + "fingerprint": { + "type": "string", + "description": "What the item looked like when given. Validation warns if the item \ + has since changed." + }, + "points": { "type": "number", "minimum": 0.0 }, + "bonus": { "type": "boolean" }, + "key": string_array("Keyed option letters as administered."), + "level": level(), + "learning_objectives": string_array("Objectives as administered."), + "credit_overrides": { + "type": "object", + "description": "Partial credit decided after the fact, by option letter. Recording \ + it here keeps the rescoring decision with the administration it \ + applies to.", + "additionalProperties": { "type": "number", "minimum": 0.0, "maximum": 1.0 } + }, + "dropped": { + "type": "boolean", + "description": "Excluded from scoring and from statistics." + } + } + }) +} + +/// The assessment schema. +fn assessment_schema() -> Value { + json!({ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": format!("{BASE}/assessment.schema.json"), + "title": "coursebank assessment record", + "description": "A record of what was given, to whom, and when. This is the join between \ + item banks and grading data, and it is the source of truth for reuse \ + history — there is no separate ledger to drift out of step.", + "type": "object", + "required": ["assessment"], + "additionalProperties": false, + "properties": { + "schema_version": { "type": ["string", "number"] }, + "assessment": assessment_meta_schema(), + "blueprint": blueprint_schema(), + "forms": { + "type": "array", + "description": "Alternate forms. Option order is derived from the seed rather than \ + stored, so every export of a form agrees.", + "items": form_schema() + }, + "items": { + "type": "array", + "description": "One entry per question, in number order.", + "items": placement_schema() + } + } + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_schema_is_well_formed() { + for kind in Kind::ALL { + let s = schema(kind); + assert!(s["$schema"].is_string(), "{kind:?} needs a $schema"); + assert!(s["$id"].is_string(), "{kind:?} needs an $id"); + assert_eq!(s["type"], "object"); + assert!( + s["additionalProperties"] == false, + "{kind:?} must reject unknown keys, matching the Rust deserializer" + ); + // Round-trips as JSON. + let text = serde_json::to_string(&s).unwrap(); + let back: Value = serde_json::from_str(&text).unwrap(); + assert_eq!(back, s); + } + } + + #[test] + fn the_level_enum_lists_all_five() { + let l = level(); + assert_eq!(l["minimum"], 1); + assert_eq!(l["maximum"], 5); + let description = l["description"].as_str().unwrap(); + for level in Level::ALL { + assert!( + description.contains(level.name()), + "missing {}", + level.name() + ); + } + } + + #[test] + fn the_process_enum_matches_the_taxonomy() { + let p = cognitive_process(); + let listed = p["enum"].as_array().unwrap(); + assert_eq!(listed.len(), CognitiveProcess::ALL.len()); + assert!(listed.contains(&Value::String("differentiate".into()))); + } + + #[test] + fn the_item_schema_constrains_ids_and_options() { + let item = item_schema(); + let props = &item["properties"]; + assert!(props["id"]["pattern"].as_str().unwrap().starts_with("^q-")); + assert_eq!(props["options"]["minItems"], 2); + assert_eq!(props["options"]["maxItems"], 8); + assert_eq!( + props["options"]["items"]["properties"]["id"]["pattern"], + "^[A-H]$" + ); + } + + #[test] + fn modelines_point_at_the_right_file() { + assert_eq!( + Kind::Bank.modeline("../.coursebank/schema"), + "# yaml-language-server: $schema=../.coursebank/schema/bank.schema.json" + ); + // A trailing slash must not double up. + assert!(Kind::Course + .modeline("schema/") + .ends_with("schema/course.schema.json")); + } + + #[test] + fn schemas_write_to_disk() { + let dir = std::env::temp_dir().join(format!("cb-schema-{}", std::process::id())); + std::fs::remove_dir_all(&dir).ok(); + let written = write_all(&dir).unwrap(); + assert_eq!(written.len(), 3); + for path in &written { + assert!(path.exists()); + let text = std::fs::read_to_string(path).unwrap(); + let _: Value = serde_json::from_str(&text).expect("valid JSON on disk"); + } + std::fs::remove_dir_all(&dir).ok(); + } +} diff --git a/src/authoring/lint.rs b/src/authoring/lint.rs new file mode 100644 index 0000000..ed71eae --- /dev/null +++ b/src/authoring/lint.rs @@ -0,0 +1,1461 @@ +//! Authoring-quality checks: is this item *well written*? +//! +//! Validation asks whether a file is usable. Linting asks the harder question, +//! and the answers are advisory: a lint finding is a prompt to look, not a +//! verdict. Some rules will fire on items you meant to write that way, which is +//! why every rule has a code you can silence. +//! +//! The rules fall into four families, and it is worth knowing which is which +//! because they have different reliability. +//! +//! *Cueing* rules are the most valuable, because they catch items that measure +//! test-taking rather than learning. If the key is reliably the longest option, a +//! student who knows nothing can beat the item. These rules are mechanical and +//! trustworthy. +//! +//! *Clarity* rules look for stems that do not pose a definite task, unemphasized +//! negation, and prose that reads well above the level of the course. They catch +//! real problems and also produce the most false positives. +//! +//! *Completeness* rules check that distractors are designed rather than filler, +//! and that there is something to say to a student who picks one. They are what +//! make the reporting features possible at all. +//! +//! *Evidence* rules compare what you predicted against what happened, and flag +//! statistics that describe a version of the item you have since edited. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::catalog::{Catalog, Entry, Severity}; +use crate::course::CourseFile; +use crate::item::Item; +use crate::taxonomy::{Discrimination, Format, Level, Status}; + +/// A lint rule. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Rule { + // --- cueing --- + /// The keyed option is conspicuously longer than the distractors. + KeyIsLongest, + /// Option lengths vary so much that length itself is informative. + UnevenOptionLength, + /// A distinctive word appears in the stem and only in the key. + WordRepeatCue, + /// An "all of the above" or "none of the above" option. + AllOrNoneOfTheAbove, + /// A distractor stated in absolute terms, which trained students discard. + AbsoluteDistractor, + /// One option's text is contained in another's. + OverlappingOptions, + /// Two options say close to the same thing. + NearDuplicateOptions, + /// The keyed letter is imbalanced across the bank. + KeyPositionImbalance, + /// An item's option count differs from its neighbors'. + InconsistentOptionCount, + + // --- clarity --- + /// The stem does not pose a definite question or completion. + StemHasNoTask, + /// The stem is long enough that reading load competes with the construct. + StemTooLong, + /// Negation is present but not emphasized. + UnemphasizedNegation, + /// A vague quantifier that makes more than one option defensible. + VagueQualifier, + /// The prose reads well above the expected level. + HighReadingLoad, + /// A stem that asks which statement is "true" without a focus. + UnfocusedStem, + + // --- completeness --- + /// A distractor with no named misconception or error type. + UndesignedDistractor, + /// A distractor with no explanation. + UnexplainedDistractor, + /// Nothing to show a student who chose this option. + NoStudentFeedback, + /// A figure with no alt text. + AssetWithoutAltText, + /// True/False used at an analytic level, where it carries little signal. + WeakFormatForLevel, + /// A level-5 item scored in the graded total against course policy. + ScoredBonusLevel, + /// An expectation of low discrimination on a higher-level item. + ContradictoryDesign, + + // --- evidence --- + /// Statistics describe an older version of the item. + StaleCalibration, + /// Observed difficulty was far from predicted difficulty. + DifficultyMissed, + /// Observed discrimination contradicted the prediction. + DiscriminationMissed, + /// Two items in the course have nearly the same stem. + DuplicateStem, +} + +impl Rule { + /// Every rule, grouped as the families appear above. + /// + /// Used by `--list-rules`, and by the test that keeps this list in step with + /// the enum. + pub const ALL: [Rule; 26] = [ + Rule::KeyIsLongest, + Rule::UnevenOptionLength, + Rule::WordRepeatCue, + Rule::AllOrNoneOfTheAbove, + Rule::AbsoluteDistractor, + Rule::OverlappingOptions, + Rule::NearDuplicateOptions, + Rule::KeyPositionImbalance, + Rule::InconsistentOptionCount, + Rule::StemHasNoTask, + Rule::StemTooLong, + Rule::UnemphasizedNegation, + Rule::VagueQualifier, + Rule::HighReadingLoad, + Rule::UnfocusedStem, + Rule::UndesignedDistractor, + Rule::UnexplainedDistractor, + Rule::NoStudentFeedback, + Rule::AssetWithoutAltText, + Rule::WeakFormatForLevel, + Rule::ScoredBonusLevel, + Rule::ContradictoryDesign, + Rule::StaleCalibration, + Rule::DifficultyMissed, + Rule::DiscriminationMissed, + Rule::DuplicateStem, + ]; + + /// The rule's nominal severity, for `--list-rules` and `--min-severity`. + /// + /// A finding may carry a different severity than its rule's nominal one when + /// the specific case is worse than usual — an unevenly sized option set is + /// low by default but medium when the spread is extreme. This value is the + /// rule's typical weight, which is what a listing should show. + /// + /// The three cueing rules that rise to high are the ones a test-wise student + /// can exploit without knowing the material, which makes them scoring bugs + /// rather than style notes. + pub fn severity(self) -> Severity { + use Rule as R; + match self { + // A student who can find the key without the content is not being + // measured on the content. + R::KeyIsLongest | R::WordRepeatCue => Severity::High, + // Statistics attached to text that has since changed are actively + // misleading, which is worse than absent. + R::StaleCalibration => Severity::High, + // An unanswerable question for a screen-reader user. + R::AssetWithoutAltText => Severity::High, + + R::AllOrNoneOfTheAbove + | R::OverlappingOptions + | R::NearDuplicateOptions + | R::KeyPositionImbalance + | R::StemHasNoTask + | R::UnemphasizedNegation + | R::VagueQualifier + | R::UnfocusedStem + | R::UndesignedDistractor + | R::WeakFormatForLevel + | R::ScoredBonusLevel + | R::ContradictoryDesign + | R::DuplicateStem => Severity::Medium, + + R::UnevenOptionLength + | R::AbsoluteDistractor + | R::InconsistentOptionCount + | R::StemTooLong + | R::HighReadingLoad + | R::UnexplainedDistractor + | R::NoStudentFeedback + | R::DifficultyMissed + | R::DiscriminationMissed => Severity::Low, + } + } + + /// The stable code used to silence the rule from the command line. + pub fn code(self) -> &'static str { + match self { + Rule::KeyIsLongest => "cue-key-longest", + Rule::UnevenOptionLength => "cue-uneven-length", + Rule::WordRepeatCue => "cue-word-repeat", + Rule::AllOrNoneOfTheAbove => "cue-all-of-the-above", + Rule::AbsoluteDistractor => "cue-absolute", + Rule::OverlappingOptions => "cue-overlap", + Rule::NearDuplicateOptions => "cue-near-duplicate", + Rule::KeyPositionImbalance => "cue-key-position", + Rule::InconsistentOptionCount => "cue-option-count", + Rule::StemHasNoTask => "clarity-no-task", + Rule::StemTooLong => "clarity-stem-length", + Rule::UnemphasizedNegation => "clarity-negation", + Rule::VagueQualifier => "clarity-vague", + Rule::HighReadingLoad => "clarity-reading-load", + Rule::UnfocusedStem => "clarity-unfocused", + Rule::UndesignedDistractor => "complete-distractor-design", + Rule::UnexplainedDistractor => "complete-distractor-explanation", + Rule::NoStudentFeedback => "complete-student-feedback", + Rule::AssetWithoutAltText => "complete-alt-text", + Rule::WeakFormatForLevel => "complete-format-level", + Rule::ScoredBonusLevel => "complete-bonus-policy", + Rule::ContradictoryDesign => "complete-design-conflict", + Rule::StaleCalibration => "evidence-stale", + Rule::DifficultyMissed => "evidence-difficulty", + Rule::DiscriminationMissed => "evidence-discrimination", + Rule::DuplicateStem => "evidence-duplicate-stem", + } + } + + /// Which family the rule belongs to. + pub fn family(self) -> &'static str { + match self { + Rule::KeyIsLongest + | Rule::UnevenOptionLength + | Rule::WordRepeatCue + | Rule::AllOrNoneOfTheAbove + | Rule::AbsoluteDistractor + | Rule::OverlappingOptions + | Rule::NearDuplicateOptions + | Rule::KeyPositionImbalance + | Rule::InconsistentOptionCount => "cueing", + Rule::StemHasNoTask + | Rule::StemTooLong + | Rule::UnemphasizedNegation + | Rule::VagueQualifier + | Rule::HighReadingLoad + | Rule::UnfocusedStem => "clarity", + Rule::UndesignedDistractor + | Rule::UnexplainedDistractor + | Rule::NoStudentFeedback + | Rule::AssetWithoutAltText + | Rule::WeakFormatForLevel + | Rule::ScoredBonusLevel + | Rule::ContradictoryDesign => "completeness", + Rule::StaleCalibration + | Rule::DifficultyMissed + | Rule::DiscriminationMissed + | Rule::DuplicateStem => "evidence", + } + } + + /// Every rule, for `coursebank lint --list-rules`. + pub fn all() -> Vec { + vec![ + Rule::KeyIsLongest, + Rule::UnevenOptionLength, + Rule::WordRepeatCue, + Rule::AllOrNoneOfTheAbove, + Rule::AbsoluteDistractor, + Rule::OverlappingOptions, + Rule::NearDuplicateOptions, + Rule::KeyPositionImbalance, + Rule::InconsistentOptionCount, + Rule::StemHasNoTask, + Rule::StemTooLong, + Rule::UnemphasizedNegation, + Rule::VagueQualifier, + Rule::HighReadingLoad, + Rule::UnfocusedStem, + Rule::UndesignedDistractor, + Rule::UnexplainedDistractor, + Rule::NoStudentFeedback, + Rule::AssetWithoutAltText, + Rule::WeakFormatForLevel, + Rule::ScoredBonusLevel, + Rule::ContradictoryDesign, + Rule::StaleCalibration, + Rule::DifficultyMissed, + Rule::DiscriminationMissed, + Rule::DuplicateStem, + ] + } + + /// What the rule is looking for, one line. + pub fn description(self) -> &'static str { + match self { + Rule::KeyIsLongest => "the key is much longer than every distractor", + Rule::UnevenOptionLength => "option lengths differ enough to be a cue", + Rule::WordRepeatCue => "a distinctive stem word appears only in the key", + Rule::AllOrNoneOfTheAbove => "an all-of-the-above or none-of-the-above option", + Rule::AbsoluteDistractor => "a distractor phrased in absolutes", + Rule::OverlappingOptions => "one option contains another", + Rule::NearDuplicateOptions => "two options are near duplicates", + Rule::KeyPositionImbalance => "keyed letters are unevenly distributed in the bank", + Rule::InconsistentOptionCount => "option count differs from the bank default", + Rule::StemHasNoTask => "the stem poses no definite task", + Rule::StemTooLong => "the stem is very long", + Rule::UnemphasizedNegation => "negation is not emphasized", + Rule::VagueQualifier => "a vague quantifier makes several options defensible", + Rule::HighReadingLoad => "reading level is well above the course", + Rule::UnfocusedStem => "the stem asks which statement is true, without a focus", + Rule::UndesignedDistractor => "a distractor names no misconception or error type", + Rule::UnexplainedDistractor => "a distractor has no explanation", + Rule::NoStudentFeedback => "nothing to show a student who chose this option", + Rule::AssetWithoutAltText => "a figure has no alt text", + Rule::WeakFormatForLevel => "true/false at an analytic level", + Rule::ScoredBonusLevel => "a level the policy reserves for bonus is scored", + Rule::ContradictoryDesign => "low expected discrimination on a higher-level item", + Rule::StaleCalibration => "statistics describe an older version of the item", + Rule::DifficultyMissed => "observed difficulty was far from predicted", + Rule::DiscriminationMissed => "observed discrimination contradicted the prediction", + Rule::DuplicateStem => "two items have nearly the same stem", + } + } +} + +/// One lint finding. +#[derive(Debug, Clone)] +pub struct Finding { + /// The item's global id, or a bank id for bank-wide findings. + pub subject: String, + /// The rule that fired. + pub rule: Rule, + /// How much attention it deserves. + pub severity: Severity, + /// What was found, and where. + pub message: String, +} + +/// Thresholds, so the judgment calls are visible and adjustable rather than +/// buried as literals in the middle of a function. +#[derive(Debug, Clone)] +pub struct Thresholds { + /// Stem word count above which the stem is called long. + pub stem_words: usize, + /// Ratio of longest to mean distractor length that counts as a length cue. + pub key_length_ratio: f64, + /// Ratio of longest to shortest option that counts as uneven. + pub option_spread_ratio: f64, + /// Jaccard similarity above which two options are near duplicates. + pub option_similarity: f64, + /// Jaccard similarity above which two stems are near duplicates. + pub stem_similarity: f64, + /// Flesch-Kincaid grade above which reading load is flagged. + pub reading_grade: f64, + /// The proportion of a bank's keys any one letter may hold. + pub key_share: f64, + /// The fewest items in a bank before key position is worth testing. + pub key_position_min_items: usize, + /// Absolute difference between predicted and observed difficulty that counts + /// as a missed prediction. + pub difficulty_tolerance: f64, +} + +impl Default for Thresholds { + fn default() -> Thresholds { + Thresholds { + stem_words: 70, + key_length_ratio: 1.5, + option_spread_ratio: 3.0, + option_similarity: 0.8, + stem_similarity: 0.85, + reading_grade: 16.0, + key_share: 0.4, + key_position_min_items: 10, + difficulty_tolerance: 0.25, + } + } +} + +/// Lints every item in a course, plus the bank-wide and course-wide rules. +/// +/// # Arguments +/// +/// * `catalog` - the loaded course. +/// * `t` - thresholds. +/// +/// # Returns +/// +/// Findings, ordered by severity then subject so the important ones come first. +pub fn lint_catalog(catalog: &Catalog, t: &Thresholds) -> Vec { + let mut out = Vec::new(); + + for entry in &catalog.entries { + if entry.item.status == Status::Retired { + continue; + } + out.extend(lint_item(entry, &catalog.course, t)); + } + + out.extend(lint_key_positions(catalog, t)); + out.extend(lint_option_counts(catalog)); + out.extend(lint_duplicate_stems(catalog, t)); + + out.sort_by(|a, b| { + b.severity + .cmp(&a.severity) + .then(a.subject.cmp(&b.subject)) + .then(a.rule.cmp(&b.rule)) + }); + out +} + +/// Lints one item. +/// +/// # Arguments +/// +/// * `entry` - the catalog entry. +/// * `course` - the course, for policy and expected reading level. +/// * `t` - thresholds. +/// +/// # Returns +/// +/// Findings for this item. +pub fn lint_item(entry: &Entry, course: &CourseFile, t: &Thresholds) -> Vec { + let it = &entry.item; + let uid = entry.uid.clone(); + let mut f = Vec::new(); + let mut push = |rule: Rule, severity: Severity, message: String| { + f.push(Finding { + subject: uid.clone(), + rule, + severity, + message, + }); + }; + + // ---------------------------------------------------------------- clarity + let stem = it.stem.trim(); + let stem_lower = stem.to_lowercase(); + let words: Vec<&str> = stem.split_whitespace().collect(); + + if !stem.contains('?') && !stem.ends_with(':') && !stem.ends_with("___") { + push( + Rule::StemHasNoTask, + Severity::Medium, + "the stem neither asks a question nor sets up a completion; a student \ + has to infer what is being asked" + .to_string(), + ); + } + if words.len() > t.stem_words { + push( + Rule::StemTooLong, + Severity::Low, + format!( + "the stem runs {} words; consider moving background into a shared stimulus", + words.len() + ), + ); + } + + // Negation is legitimate but must be visible. Emphasis means the word is + // uppercase or wrapped in markup. + for neg in ["not", "except", "least", "never", "incorrect", "false"] { + if contains_word(&stem_lower, neg) && !negation_is_emphasized(stem, neg) { + push( + Rule::UnemphasizedNegation, + Severity::Medium, + format!( + "the stem turns on `{neg}` without emphasis; students skim past it. \ + Write it as `{}` or bold it.", + neg.to_uppercase() + ), + ); + break; + } + } + + for vague in [ + "often", + "usually", + "generally", + "sometimes", + "may", + "might", + "several", + "many", + "frequently", + "typically", + ] { + if contains_word(&stem_lower, vague) { + push( + Rule::VagueQualifier, + Severity::Low, + format!( + "`{vague}` in the stem can make more than one option defensible; \ + pin the condition down" + ), + ); + break; + } + } + + if stem_lower.contains("which of the following is true") + || stem_lower.contains("which statement is true") + || stem_lower.contains("which of the following statements is correct") + { + push( + Rule::UnfocusedStem, + Severity::Medium, + "an unfocused `which is true` stem tests scanning rather than a single idea; \ + name the concept the item is about" + .to_string(), + ); + } + + let grade = flesch_kincaid_grade(stem); + if grade > t.reading_grade { + push( + Rule::HighReadingLoad, + Severity::Low, + format!( + "stem reads at about grade {grade:.0}; long sentences add reading load \ + that is not part of what you are measuring" + ), + ); + } + + // ----------------------------------------------------------------- cueing + let keys: Vec<&crate::item::Choice> = it.options.iter().filter(|o| o.correct).collect(); + let distractors: Vec<&crate::item::Choice> = it.options.iter().filter(|o| !o.correct).collect(); + + if !keys.is_empty() && !distractors.is_empty() { + let key_len = keys + .iter() + .map(|o| o.text.trim().chars().count()) + .max() + .unwrap_or(0) as f64; + let mean_distractor = distractors + .iter() + .map(|o| o.text.trim().chars().count() as f64) + .sum::() + / distractors.len() as f64; + if mean_distractor > 0.0 && key_len / mean_distractor >= t.key_length_ratio { + push( + Rule::KeyIsLongest, + Severity::High, + format!( + "the key is {:.1}x the average distractor length ({key_len:.0} vs \ + {mean_distractor:.0} characters); a test-wise student can pick it \ + without knowing the content", + key_len / mean_distractor + ), + ); + } + } + + let lens: Vec = it + .options + .iter() + .map(|o| o.text.trim().chars().count().max(1)) + .collect(); + if let (Some(&mx), Some(&mn)) = (lens.iter().max(), lens.iter().min()) { + if mn > 0 && (mx as f64) / (mn as f64) >= t.option_spread_ratio { + push( + Rule::UnevenOptionLength, + Severity::Low, + format!("option lengths run {mn} to {mx} characters; even them out"), + ); + } + } + + // A distinctive word shared by the stem and the key alone is a clang cue. + if let Some(word) = repeated_cue_word(it) { + push( + Rule::WordRepeatCue, + Severity::Medium, + format!( + "`{word}` appears in the stem and in the key but in no distractor; \ + the echo points at the answer" + ), + ); + } + + for o in &it.options { + let low = o.text.trim().to_lowercase(); + if low.starts_with("all of the above") + || low.starts_with("none of the above") + || low.starts_with("both a and b") + || low == "a and b" + || low == "all of these" + || low == "none of these" + { + push( + Rule::AllOrNoneOfTheAbove, + Severity::Medium, + format!( + "option {} is `{}`; partial knowledge answers it and shuffling \ + answers in Canvas breaks it", + o.id, + o.text.trim() + ), + ); + } + if !o.correct { + for abs in [ + "always", + "never", + "all ", + "none ", + "every ", + "no exceptions", + ] { + if low.contains(abs) { + push( + Rule::AbsoluteDistractor, + Severity::Low, + format!( + "distractor {} is phrased absolutely (`{}`); students discard \ + absolutes on principle", + o.id, + abs.trim() + ), + ); + break; + } + } + } + } + + for i in 0..it.options.len() { + for j in (i + 1)..it.options.len() { + let a = it.options[i].text.trim().to_lowercase(); + let b = it.options[j].text.trim().to_lowercase(); + if a.is_empty() || b.is_empty() { + continue; + } + if a != b && (a.contains(&b) || b.contains(&a)) { + push( + Rule::OverlappingOptions, + Severity::Medium, + format!( + "option {} contains option {}; one cannot be right without the other", + if a.contains(&b) { + &it.options[i].id + } else { + &it.options[j].id + }, + if a.contains(&b) { + &it.options[j].id + } else { + &it.options[i].id + } + ), + ); + } else { + let sim = jaccard(&tokens(&a), &tokens(&b)); + if sim >= t.option_similarity { + push( + Rule::NearDuplicateOptions, + Severity::Medium, + format!( + "options {} and {} are {:.0}% the same; they are not two \ + distinct ideas", + it.options[i].id, + it.options[j].id, + sim * 100.0 + ), + ); + } + } + } + } + + // ----------------------------------------------------------- completeness + // These are only worth insisting on once an item is meant to be used. + let is_ready = matches!(it.status, Status::Approved | Status::InReview); + if is_ready { + for o in &it.options { + if !o.correct { + if o.misconception.is_none() && o.error_type.is_none() { + push( + Rule::UndesignedDistractor, + Severity::Medium, + format!( + "distractor {} names no misconception or error_type; if you \ + cannot say what mistake it captures, it is filler and its \ + selection rate will tell you nothing", + o.id + ), + ); + } + if o.explanation.is_none() { + push( + Rule::UnexplainedDistractor, + Severity::Low, + format!("distractor {} has no explanation", o.id), + ); + } + } + if o.student_text().is_none() { + push( + Rule::NoStudentFeedback, + Severity::Low, + format!( + "option {} carries no text a post-exam report could show a \ + student who chose it", + o.id + ), + ); + } + } + } + + for a in &it.assets { + if a.alt + .as_deref() + .map(|s| s.trim().is_empty()) + .unwrap_or(true) + { + push( + Rule::AssetWithoutAltText, + Severity::Medium, + format!("asset `{}` has no alt text", a.path), + ); + } + } + + if it.format == Format::TrueFalse && it.level >= Level::Analyze { + push( + Rule::WeakFormatForLevel, + Severity::Low, + format!( + "true/false at level {} gives a 50% floor and little diagnostic signal", + it.level.code() + ), + ); + } + + if course.policy.bonus_levels.contains(&it.level) && !it.bonus { + push( + Rule::ScoredBonusLevel, + Severity::Medium, + format!( + "the course policy reserves level {} for bonus items, but this one is scored", + it.level.code() + ), + ); + } + + if let Some(d) = &it.design { + if d.expected_discrimination == Some(Discrimination::Low) && it.level >= Level::Apply { + push( + Rule::ContradictoryDesign, + Severity::Low, + format!( + "a level {} item expected to discriminate poorly is doing the work of \ + a level 1 anchor; check the level or the expectation", + it.level.code() + ), + ); + } + } + + // --------------------------------------------------------------- evidence + if !it.calibration_is_current() { + push( + Rule::StaleCalibration, + Severity::High, + "the item was edited after it was calibrated, so its statistics and IRT \ + parameters describe a question you no longer ask" + .to_string(), + ); + } + + if let (Some(design), Some(cal)) = (&it.design, &it.calibration) { + if let (Some(expected), Some(observed)) = (design.expected_difficulty, cal.p_value) { + let gap = (expected - observed).abs(); + if gap >= t.difficulty_tolerance { + push( + Rule::DifficultyMissed, + Severity::Medium, + format!( + "you predicted {:.0}% correct and observed {:.0}%; either the \ + cohort or the item is not what you thought", + expected * 100.0, + observed * 100.0 + ), + ); + } + } + if let (Some(expected), Some(observed)) = + (design.expected_discrimination, cal.point_biserial) + { + let (lo, hi) = expected.expected_band(); + if observed < lo || observed > hi { + push( + Rule::DiscriminationMissed, + Severity::Low, + format!( + "expected {:?} discrimination (roughly {lo:.2} to {hi:.2}) but \ + observed a point-biserial of {observed:.2}", + expected + ), + ); + } + } + } + + f +} + +/// Flags banks whose keyed letters cluster on one position. +/// +/// Instructors reach for C. Students know it. This is a bank-level rule because +/// a single item cannot be imbalanced. +/// +/// # Arguments +/// +/// * `catalog` - the loaded course. +/// * `t` - thresholds. +/// +/// # Returns +/// +/// One finding per offending bank. +fn lint_key_positions(catalog: &Catalog, t: &Thresholds) -> Vec { + let mut by_bank: BTreeMap<&str, Vec<&Entry>> = BTreeMap::new(); + for e in &catalog.entries { + if e.item.status == Status::Retired || e.item.is_multi_key() { + continue; + } + by_bank.entry(e.bank.as_str()).or_default().push(e); + } + + let mut out = Vec::new(); + for (bank, entries) in by_bank { + if entries.len() < t.key_position_min_items { + continue; + } + let mut counts: BTreeMap = BTreeMap::new(); + for e in &entries { + for letter in e.item.key_letters() { + *counts.entry(letter).or_insert(0) += 1; + } + } + let total: usize = counts.values().sum(); + if total == 0 { + continue; + } + for (letter, n) in &counts { + let share = *n as f64 / total as f64; + if share > t.key_share { + out.push(Finding { + subject: bank.to_string(), + rule: Rule::KeyPositionImbalance, + severity: Severity::Medium, + message: format!( + "{:.0}% of keys in this bank are `{letter}` ({n} of {total}); \ + shuffle at export time or rebalance while authoring", + share * 100.0 + ), + }); + } + } + } + out +} + +/// Flags items whose option count differs from the mode of their bank. +/// +/// # Arguments +/// +/// * `catalog` - the loaded course. +/// +/// # Returns +/// +/// One finding per odd item. +fn lint_option_counts(catalog: &Catalog) -> Vec { + let mut by_bank: BTreeMap<&str, Vec<&Entry>> = BTreeMap::new(); + for e in &catalog.entries { + if e.item.status == Status::Retired { + continue; + } + by_bank.entry(e.bank.as_str()).or_default().push(e); + } + + let mut out = Vec::new(); + for (_, entries) in by_bank { + if entries.len() < 4 { + continue; + } + let mut counts: BTreeMap = BTreeMap::new(); + for e in &entries { + *counts.entry(e.item.options.len()).or_insert(0) += 1; + } + let modal = counts + .iter() + .max_by_key(|(_, n)| **n) + .map(|(k, _)| *k) + .unwrap_or(0); + // Only complain when there is a clear norm to deviate from. + if counts.get(&modal).copied().unwrap_or(0) * 2 <= entries.len() { + continue; + } + for e in &entries { + if e.item.options.len() != modal { + out.push(Finding { + subject: e.uid.clone(), + rule: Rule::InconsistentOptionCount, + severity: Severity::Low, + message: format!( + "has {} options where the rest of the bank has {modal}; an odd \ + count is itself a cue on a printed form", + e.item.options.len() + ), + }); + } + } + } + out +} + +/// Flags pairs of items with nearly the same stem. +/// +/// Two near-identical items are usually an accident of copying, and putting both +/// on one form double-counts a single piece of knowledge. +/// +/// # Arguments +/// +/// * `catalog` - the loaded course. +/// * `t` - thresholds. +/// +/// # Returns +/// +/// One finding per duplicate pair. +fn lint_duplicate_stems(catalog: &Catalog, t: &Thresholds) -> Vec { + let live: Vec<&Entry> = catalog + .entries + .iter() + .filter(|e| e.item.status != Status::Retired) + .collect(); + + let toks: Vec> = live.iter().map(|e| tokens(&e.item.stem)).collect(); + let mut out = Vec::new(); + for i in 0..live.len() { + for j in (i + 1)..live.len() { + if toks[i].len() < 4 || toks[j].len() < 4 { + continue; + } + let sim = jaccard(&toks[i], &toks[j]); + if sim >= t.stem_similarity { + out.push(Finding { + subject: live[i].uid.clone(), + rule: Rule::DuplicateStem, + severity: Severity::Medium, + message: format!( + "stem is {:.0}% the same as `{}`; putting both on one form \ + double-counts one idea", + sim * 100.0, + live[j].uid + ), + }); + } + } + } + out +} + +/// A word that appears in the stem and in the key but in no distractor. +/// +/// Short and common words are ignored, since "the" appearing in both proves +/// nothing. +/// +/// # Arguments +/// +/// * `it` - the item. +/// +/// # Returns +/// +/// The cue word, when one exists. +fn repeated_cue_word(it: &Item) -> Option { + let stem_words = tokens(&it.stem); + let key_words: BTreeSet = it + .options + .iter() + .filter(|o| o.correct) + .flat_map(|o| tokens(&o.text)) + .collect(); + let distractor_words: BTreeSet = it + .options + .iter() + .filter(|o| !o.correct) + .flat_map(|o| tokens(&o.text)) + .collect(); + + for w in stem_words.intersection(&key_words) { + // A technical term is long; a function word is not. + if w.chars().count() >= 7 && !distractor_words.contains(w) { + return Some(w.clone()); + } + } + None +} + +/// Whether a negation word is emphasized in the stem. +/// +/// # Arguments +/// +/// * `stem` - the stem as written. +/// * `neg` - the lowercase negation word. +/// +/// # Returns +/// +/// `true` when the word appears uppercased or inside emphasis markup. +fn negation_is_emphasized(stem: &str, neg: &str) -> bool { + let upper = neg.to_uppercase(); + if stem.contains(&upper) { + return true; + } + // `*not*`, `**not**`, `_not_` all count as emphasis. + for wrap in ['*', '_'] { + let pattern = format!("{wrap}{neg}{wrap}"); + if stem.to_lowercase().contains(&pattern) { + return true; + } + } + false +} + +/// Whether a lowercase haystack contains a word as a whole word. +/// +/// # Arguments +/// +/// * `haystack` - lowercase text. +/// * `needle` - the lowercase word. +/// +/// # Returns +/// +/// `true` on a whole-word match. +fn contains_word(haystack: &str, needle: &str) -> bool { + haystack + .split(|c: char| !c.is_alphanumeric() && c != '\'') + .any(|w| w == needle) +} + +/// Lowercased alphabetic tokens of length 3 or more. +/// +/// # Arguments +/// +/// * `text` - the text to tokenize. +/// +/// # Returns +/// +/// The token set. +fn tokens(text: &str) -> BTreeSet { + text.split(|c: char| !c.is_alphanumeric()) + .map(|w| w.to_lowercase()) + .filter(|w| w.chars().count() >= 3) + .collect() +} + +/// Jaccard similarity of two token sets. +/// +/// # Arguments +/// +/// * `a` - the first set. +/// * `b` - the second set. +/// +/// # Returns +/// +/// Similarity in `[0, 1]`; 0 when both are empty. +fn jaccard(a: &BTreeSet, b: &BTreeSet) -> f64 { + if a.is_empty() && b.is_empty() { + return 0.0; + } + let inter = a.intersection(b).count() as f64; + let union = a.union(b).count() as f64; + if union == 0.0 { + 0.0 + } else { + inter / union + } +} + +/// The Flesch-Kincaid grade level of a passage. +/// +/// This is a crude instrument, and it is used only to notice prose that is +/// *unusually* dense, never to prescribe a target. +/// +/// # Arguments +/// +/// * `text` - the passage. +/// +/// # Returns +/// +/// The estimated US grade level, 0 for empty input. +pub fn flesch_kincaid_grade(text: &str) -> f64 { + let words: Vec<&str> = text.split_whitespace().collect(); + if words.is_empty() { + return 0.0; + } + let sentences = text + .chars() + .filter(|c| matches!(c, '.' | '?' | '!' | ';')) + .count() + .max(1); + let syllables: usize = words.iter().map(|w| syllables(w)).sum(); + let w = words.len() as f64; + let s = sentences as f64; + let y = syllables as f64; + 0.39 * (w / s) + 11.8 * (y / w) - 15.59 +} + +/// Estimates the syllable count of a word. +/// +/// Counts vowel groups, drops a silent trailing `e`, and never returns 0. +/// +/// # Arguments +/// +/// * `word` - the word. +/// +/// # Returns +/// +/// The estimated syllable count, at least 1. +fn syllables(word: &str) -> usize { + let w: String = word + .chars() + .filter(|c| c.is_alphabetic()) + .map(|c| c.to_ascii_lowercase()) + .collect(); + if w.is_empty() { + return 0; + } + let is_vowel = |c: char| matches!(c, 'a' | 'e' | 'i' | 'o' | 'u' | 'y'); + let mut count = 0; + let mut prev_vowel = false; + for c in w.chars() { + let v = is_vowel(c); + if v && !prev_vowel { + count += 1; + } + prev_vowel = v; + } + if w.ends_with('e') && count > 1 { + count -= 1; + } + count.max(1) +} + +/// Groups findings by rule for a summary table. +/// +/// # Arguments +/// +/// * `findings` - the findings to summarize. +/// +/// # Returns +/// +/// A map from rule to count. +pub fn summarize(findings: &[Finding]) -> BTreeMap { + let mut out: BTreeMap = BTreeMap::new(); + for f in findings { + *out.entry(f.rule).or_insert(0) += 1; + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_rule_appears_in_all_exactly_once() { + let mut seen = Rule::ALL.to_vec(); + let before = seen.len(); + seen.sort_by_key(|r| r.code()); + seen.dedup_by_key(|r| r.code()); + assert_eq!(seen.len(), before, "Rule::ALL has a duplicate"); + } + + #[test] + fn every_rule_has_a_unique_nonempty_code_and_description() { + let mut codes: Vec<&str> = Rule::ALL.iter().map(|r| r.code()).collect(); + codes.sort_unstable(); + let count = codes.len(); + codes.dedup(); + assert_eq!(codes.len(), count, "rule codes must be unique"); + for rule in Rule::ALL { + assert!(!rule.code().is_empty()); + assert!(!rule.description().is_empty(), "{:?}", rule); + // Codes are what users type into --ignore, so they must be shell-safe. + assert!( + rule.code() + .chars() + .all(|c| c.is_ascii_lowercase() || c == '-'), + "{} is not a plain lowercase code", + rule.code() + ); + } + } + use crate::catalog::Entry; + use std::path::PathBuf; + + fn course() -> CourseFile { + serde_yaml_ng::from_str("course: { code: X, title: Y, term: Z }").unwrap() + } + + fn entry(yaml: &str) -> Entry { + let item: Item = serde_yaml_ng::from_str(yaml).expect("item parses"); + Entry { + uid: format!("b::{}", item.id), + bank: "b".into(), + path: PathBuf::from("b.yaml"), + index: 0, + item, + } + } + + fn codes(yaml: &str) -> Vec<&'static str> { + let e = entry(yaml); + let mut c: Vec<&'static str> = lint_item(&e, &course(), &Thresholds::default()) + .iter() + .map(|f| f.rule.code()) + .collect(); + c.sort_unstable(); + c.dedup(); + c + } + + #[test] + fn clean_item_passes() { + let c = codes( + r#" +id: q-a-001 +status: draft +level: 2 +stem: Which mechanism best explains the sigmoidal binding curve? +options: + - { id: A, text: Ligand binding shifts the tetramer to a higher-affinity state, correct: true } + - { id: B, text: Each subunit binds with the same fixed affinity throughout } + - { id: C, text: Ligand is consumed as it binds, depleting the available pool } + - { id: D, text: The heme iron changes oxidation state upon binding } +"#, + ); + assert!(c.is_empty(), "expected no findings, got {c:?}"); + } + + #[test] + fn detects_the_length_cue() { + let c = codes( + r#" +id: q-a-001 +status: draft +level: 1 +stem: What is the hydrophobic effect? +options: + - { id: A, text: "The tendency of nonpolar groups to associate in water, driven mainly by the resulting increase in the entropy of the surrounding solvent shell", correct: true } + - { id: B, text: Van der Waals attraction } + - { id: C, text: Hydrogen bonding } + - { id: D, text: Heat release } +"#, + ); + assert!(c.contains(&"cue-key-longest"), "{c:?}"); + } + + #[test] + fn detects_unemphasized_negation_and_accepts_emphasis() { + let bare = codes( + r#" +id: q-a-001 +status: draft +level: 1 +stem: Which of these is not a product of the reaction? +options: + - { id: A, text: aaaa, correct: true } + - { id: B, text: bbbb } + - { id: C, text: cccc } +"#, + ); + assert!(bare.contains(&"clarity-negation"), "{bare:?}"); + + let emphasized = codes( + r#" +id: q-a-001 +status: draft +level: 1 +stem: Which of these is NOT a product of the reaction? +options: + - { id: A, text: aaaa, correct: true } + - { id: B, text: bbbb } + - { id: C, text: cccc } +"#, + ); + assert!(!emphasized.contains(&"clarity-negation"), "{emphasized:?}"); + } + + #[test] + fn detects_all_of_the_above_and_overlap() { + let c = codes( + r#" +id: q-a-001 +status: draft +level: 1 +stem: Which applies? +options: + - { id: A, text: The enzyme is inhibited } + - { id: B, text: The enzyme is inhibited competitively } + - { id: C, text: All of the above, correct: true } +"#, + ); + assert!(c.contains(&"cue-all-of-the-above"), "{c:?}"); + assert!(c.contains(&"cue-overlap"), "{c:?}"); + } + + #[test] + fn detects_the_word_repeat_cue() { + let c = codes( + r#" +id: q-a-001 +status: draft +level: 2 +stem: Which process explains cooperativity in this system? +options: + - { id: A, text: Conformational coupling produces cooperativity, correct: true } + - { id: B, text: Independent binding at each site } + - { id: C, text: Substrate depletion during the assay } +"#, + ); + assert!(c.contains(&"cue-word-repeat"), "{c:?}"); + } + + #[test] + fn stem_without_a_task_is_flagged_but_completions_are_not() { + assert!(codes( + r#" +id: q-a-001 +status: draft +level: 1 +stem: The hydrophobic effect. +options: + - { id: A, text: aaaa, correct: true } + - { id: B, text: bbbb } +"# + ) + .contains(&"clarity-no-task")); + + assert!(!codes( + r#" +id: q-a-001 +status: draft +level: 1 +stem: "The initial velocity will most nearly:" +options: + - { id: A, text: aaaa, correct: true } + - { id: B, text: bbbb } +"# + ) + .contains(&"clarity-no-task")); + } + + #[test] + fn completeness_rules_only_apply_once_ready() { + let draft = codes( + r#" +id: q-a-001 +status: draft +level: 1 +stem: What is x? +options: + - { id: A, text: right, correct: true } + - { id: B, text: wrong } +"#, + ); + assert!(!draft.contains(&"complete-distractor-design")); + + let ready = codes( + r#" +id: q-a-001 +status: in_review +level: 1 +stem: What is x? +options: + - { id: A, text: right, correct: true } + - { id: B, text: wrong } +"#, + ); + assert!(ready.contains(&"complete-distractor-design"), "{ready:?}"); + } + + #[test] + fn stale_calibration_is_high_severity() { + let e = entry( + r#" +id: q-a-001 +status: approved +level: 1 +stem: What is x? +options: + - { id: A, text: right, correct: true } + - { id: B, text: wrong } +calibration: + fingerprint: "0000000000000000" + p_value: 0.8 +"#, + ); + let f = lint_item(&e, &course(), &Thresholds::default()); + let stale = f + .iter() + .find(|f| f.rule == Rule::StaleCalibration) + .expect("stale calibration detected"); + assert_eq!(stale.severity, Severity::High); + } + + #[test] + fn missed_predictions_are_reported() { + let e = entry( + r#" +id: q-a-001 +status: approved +level: 3 +stem: What is x? +options: + - { id: A, text: right, correct: true } + - { id: B, text: wrong } +design: + expected_difficulty: 0.85 + expected_discrimination: high +calibration: + p_value: 0.30 + point_biserial: 0.05 +"#, + ); + let mut c: Vec<&str> = lint_item(&e, &course(), &Thresholds::default()) + .iter() + .map(|f| f.rule.code()) + .collect(); + c.sort_unstable(); + assert!(c.contains(&"evidence-difficulty"), "{c:?}"); + assert!(c.contains(&"evidence-discrimination"), "{c:?}"); + } + + #[test] + fn reading_grade_is_sane() { + let simple = flesch_kincaid_grade("The cat sat on the mat. It was a warm day."); + let dense = flesch_kincaid_grade( + "Notwithstanding the aforementioned considerations regarding \ + thermodynamic favorability, the conformational equilibrium demonstrates \ + substantial entropic contributions attributable to solvent reorganization.", + ); + assert!(simple < 6.0, "simple prose scored {simple}"); + assert!(dense > 16.0, "dense prose scored {dense}"); + } + + #[test] + fn syllable_estimates_are_close_enough() { + assert_eq!(syllables("cat"), 1); + assert_eq!(syllables("water"), 2); + assert_eq!(syllables("enzyme"), 2); + assert_eq!(syllables("a"), 1); + assert_eq!(syllables(""), 0); + } + + #[test] + fn jaccard_and_tokens_behave() { + assert_eq!(jaccard(&tokens("the enzyme"), &tokens("the enzyme")), 1.0); + assert_eq!(jaccard(&tokens("alpha"), &tokens("beta")), 0.0); + // Short words are dropped, so "a b c" has no tokens. + assert!(tokens("a b c").is_empty()); + } + + #[test] + fn every_rule_has_a_unique_code() { + let mut codes: Vec<&str> = Rule::all().iter().map(|r| r.code()).collect(); + let n = codes.len(); + codes.sort_unstable(); + codes.dedup(); + assert_eq!(codes.len(), n, "rule codes must be unique"); + } +} diff --git a/src/authoring/select.rs b/src/authoring/select.rs new file mode 100644 index 0000000..6e4be18 --- /dev/null +++ b/src/authoring/select.rs @@ -0,0 +1,717 @@ +//! Drawing an assessment from the item pool. +//! +//! Assembly is a constrained draw, not a random sample, and the constraints are +//! the point. A blueprint says how many items at each level; the pool says which +//! items are eligible; usage history says which ones students have seen recently. +//! What comes out is a form that matches the design you intended rather than +//! whichever questions happened to be at the top of the file. +//! +//! Two ordering rules do most of the work. +//! +//! *Objective minimums come first.* If a blueprint requires two items on +//! `lo-mm-kinetics`, those are placed before the level quotas are filled by +//! anything else, because a level quota can always be filled and a coverage +//! requirement often cannot. Filling in the other order strands the requirement. +//! +//! *Within a level, prefer the least recently used item.* Never-used items go +//! first, then the oldest, then the least often used. This spreads exposure +//! across the bank instead of wearing out your favorite twelve questions, and it +//! is the mechanism that makes writing new items pay off. +//! +//! Every tie is broken by a seeded shuffle, so a draw is reproducible from the +//! seed recorded in the assessment file. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::assessment::{ + Assessment, AssessmentFile, Blueprint, Form, History, Kind, Placement, Platform, +}; +use crate::catalog::Catalog; +use crate::course::SCHEMA_VERSION; +use crate::date::Date; +use crate::error::{Error, Result}; +use crate::rng::Rng; +use crate::taxonomy::Level; + +/// The result of a draw. +#[derive(Debug, Clone)] +pub struct Selection { + /// Scored item ids, grouped and ordered by level. + pub scored: Vec, + /// Bonus item ids. + pub bonus: Vec, + /// Things the caller should know: quotas filled by relaxing a constraint, + /// levels that came up short, cooldowns that had to be ignored. + pub notes: Vec, +} + +impl Selection { + /// Every selected id, scored then bonus. + pub fn all(&self) -> Vec { + let mut v = self.scored.clone(); + v.extend(self.bonus.clone()); + v + } +} + +/// Draws an assessment from the pool according to a blueprint. +/// +/// # Arguments +/// +/// * `catalog` - the loaded course. +/// * `blueprint` - the design to satisfy. +/// * `history` - usage history, for the least-recently-used preference. +/// * `as_of` - the date of the assessment, against which cooldowns are measured. +/// +/// # Returns +/// +/// The selection, together with notes about any constraint that had to bend. +/// +/// # Errors +/// +/// Returns [`Error::Infeasible`] when a level quota cannot be met even after +/// relaxing the reuse cooldown, with a message saying how many items were +/// available and how many were asked for. +pub fn select( + catalog: &Catalog, + blueprint: &Blueprint, + history: &History, + as_of: Date, +) -> Result { + let seed = blueprint.seed.unwrap_or(0); + let mut notes = Vec::new(); + + // ------------------------------------------------------------------ pool + let eligible: Vec<&crate::catalog::Entry> = catalog + .assemblable() + .into_iter() + .filter(|e| passes_filters(e, blueprint)) + .collect(); + + if eligible.is_empty() { + return Err(Error::Infeasible( + "no approved items match the blueprint's bank, lecture, and topic filters".into(), + )); + } + + let cooldown = blueprint.cooldown_days.unwrap_or(0); + let mut chosen: Vec = Vec::new(); + let mut per_bank: BTreeMap = BTreeMap::new(); + + // ---------------------------------------------------- objective minimums + // Placed first, because a coverage requirement is the constraint most likely + // to become unsatisfiable once the level quotas are full. + for (objective, needed) in &blueprint.objective_minimums { + let mut have = 0; + let mut candidates: Vec<&crate::catalog::Entry> = eligible + .iter() + .copied() + .filter(|e| e.item.learning_objectives.iter().any(|o| o == objective)) + .filter(|e| !e.item.bonus) + .collect(); + rank(&mut candidates, history, seed, "objective"); + + for e in candidates { + if have >= *needed { + break; + } + if chosen.contains(&e.uid) { + have += 1; + continue; + } + if !bank_has_room(&per_bank, &e.bank, blueprint) { + continue; + } + if cooldown > 0 && history.in_cooldown(&e.uid, cooldown, as_of) { + continue; + } + chosen.push(e.uid.clone()); + *per_bank.entry(e.bank.clone()).or_insert(0) += 1; + have += 1; + } + if have < *needed { + notes.push(format!( + "objective `{objective}` requires {needed} item(s) but only {have} could be \ + placed; write more items on it or lower the requirement" + )); + } + } + + // ------------------------------------------------------- level quotas + let mut scored: Vec = Vec::new(); + for (level, want) in &blueprint.level_counts { + if *want == 0 { + continue; + } + let (picked, level_notes) = fill_level( + &eligible, + *level, + *want, + false, + history, + seed, + cooldown, + as_of, + &mut chosen, + &mut per_bank, + blueprint, + )?; + scored.extend(picked); + notes.extend(level_notes); + } + + // Items placed to satisfy an objective minimum are scored items too, and + // they must appear exactly once. + for uid in &chosen { + if !scored.contains(uid) { + if let Some(e) = catalog.get(uid) { + if !e.item.bonus { + scored.push(uid.clone()); + } + } + } + } + + // ------------------------------------------------------------ bonus items + let mut bonus: Vec = Vec::new(); + for (level, want) in &blueprint.bonus_counts { + if *want == 0 { + continue; + } + let (picked, level_notes) = fill_level( + &eligible, + *level, + *want, + true, + history, + seed, + cooldown, + as_of, + &mut chosen, + &mut per_bank, + blueprint, + )?; + bonus.extend(picked); + notes.extend(level_notes); + } + + // Order the scored items by level so the form ramps in difficulty. Students + // meet the recall items first, which is both kinder and better measurement: + // an early hard item costs time that later easy items cannot recover. + scored.sort_by_key(|uid| { + let e = catalog.get(uid); + ( + e.map(|e| e.item.level.code()).unwrap_or(9), + e.map(|e| e.uid.clone()).unwrap_or_default(), + ) + }); + + Ok(Selection { + scored, + bonus, + notes, + }) +} + +/// Fills one level's quota. +/// +/// Cooldowns are relaxed rather than allowed to fail the draw, because an exam +/// that must be given on Thursday is better assembled from a recently used item +/// with a warning than not assembled at all. +/// +/// # Arguments +/// +/// * `eligible` - the filtered pool. +/// * `level` - the level to fill. +/// * `want` - how many items are needed. +/// * `bonus` - whether to draw bonus items. +/// * `history` - usage history. +/// * `seed` - the tie-breaking seed. +/// * `cooldown` - the reuse cooldown in days, 0 to disable. +/// * `as_of` - the assessment date. +/// * `chosen` - ids already taken, updated in place. +/// * `per_bank` - per-bank counts, updated in place. +/// * `blueprint` - for the per-bank cap. +/// +/// # Returns +/// +/// The ids picked and any notes. +/// +/// # Errors +/// +/// Returns [`Error::Infeasible`] when the level cannot be filled at all. +#[allow(clippy::too_many_arguments)] +fn fill_level( + eligible: &[&crate::catalog::Entry], + level: Level, + want: usize, + bonus: bool, + history: &History, + seed: u64, + cooldown: i64, + as_of: Date, + chosen: &mut Vec, + per_bank: &mut BTreeMap, + blueprint: &Blueprint, +) -> Result<(Vec, Vec)> { + let mut notes = Vec::new(); + let mut candidates: Vec<&crate::catalog::Entry> = eligible + .iter() + .copied() + .filter(|e| e.item.level == level && e.item.bonus == bonus) + .collect(); + + let pool_size = candidates.len(); + if pool_size < want { + return Err(Error::Infeasible(format!( + "level {} needs {want} {}item(s) but only {pool_size} approved item(s) are \ + available; write more or lower the quota", + level.code(), + if bonus { "bonus " } else { "" } + ))); + } + + rank( + &mut candidates, + history, + seed, + &format!("L{}", level.code()), + ); + + let mut picked = Vec::new(); + let mut skipped_for_cooldown = 0usize; + let mut skipped_for_bank = 0usize; + + // Two passes: honor every constraint, then relax the cooldown if short. + for relax in [false, true] { + for e in &candidates { + if picked.len() >= want { + break; + } + if chosen.contains(&e.uid) { + continue; + } + if !bank_has_room(per_bank, &e.bank, blueprint) { + if !relax { + skipped_for_bank += 1; + } + continue; + } + if !relax && cooldown > 0 && history.in_cooldown(&e.uid, cooldown, as_of) { + skipped_for_cooldown += 1; + continue; + } + if relax && cooldown > 0 && history.in_cooldown(&e.uid, cooldown, as_of) { + notes.push(format!( + "level {}: reused `{}` inside the {cooldown}-day cooldown (last used {})", + level.code(), + e.uid, + history + .last_used(&e.uid) + .map(|d| d.to_string()) + .unwrap_or_else(|| "unknown".into()) + )); + } + picked.push(e.uid.clone()); + chosen.push(e.uid.clone()); + *per_bank.entry(e.bank.clone()).or_insert(0) += 1; + } + if picked.len() >= want { + break; + } + } + + if picked.len() < want { + return Err(Error::Infeasible(format!( + "level {} needs {want} item(s); {pool_size} exist but only {} could be placed \ + ({skipped_for_cooldown} blocked by the reuse cooldown, {skipped_for_bank} by the \ + per-bank cap)", + level.code(), + picked.len() + ))); + } + + Ok((picked, notes)) +} + +/// Whether an entry passes the blueprint's inclusion filters. +/// +/// # Arguments +/// +/// * `e` - the entry. +/// * `b` - the blueprint. +/// +/// # Returns +/// +/// `true` when the item is eligible. +fn passes_filters(e: &crate::catalog::Entry, b: &Blueprint) -> bool { + if !b.banks.is_empty() && !b.banks.contains(&e.bank) { + return false; + } + if !b.lectures.is_empty() + && !e + .item + .sources + .iter() + .any(|s| b.lectures.contains(&s.lecture)) + { + return false; + } + if !b.topics.is_empty() && !e.item.topics.iter().any(|t| b.topics.contains(t)) { + return false; + } + true +} + +/// Whether a bank may contribute another item. +/// +/// # Arguments +/// +/// * `per_bank` - counts so far. +/// * `bank` - the bank in question. +/// * `b` - the blueprint, for the cap. +/// +/// # Returns +/// +/// `true` when there is room. +fn bank_has_room(per_bank: &BTreeMap, bank: &str, b: &Blueprint) -> bool { + match b.max_per_bank { + Some(cap) => per_bank.get(bank).copied().unwrap_or(0) < cap, + None => true, + } +} + +/// Orders candidates least-recently-used first, with a seeded tie-break. +/// +/// # Arguments +/// +/// * `candidates` - the candidates to order, sorted in place. +/// * `history` - usage history. +/// * `seed` - the tie-breaking seed. +/// * `salt` - distinguishes the shuffles used for different levels, so two +/// levels drawing from overlapping pools do not tie-break identically. +fn rank(candidates: &mut Vec<&crate::catalog::Entry>, history: &History, seed: u64, salt: &str) { + // Shuffle first so the sort's stability turns into a random tie-break. + let mut rng = Rng::from_label(&format!("{seed}/{salt}")); + rng.shuffle(candidates); + + candidates.sort_by_key(|e| { + let last = history.last_used(&e.uid); + ( + // Never used sorts before ever used. + if last.is_some() { 1 } else { 0 }, + last.map(|d| d.days_since_epoch()).unwrap_or(i64::MIN), + history.use_count(&e.uid), + ) + }); +} + +/// Turns a selection into an assessment record ready to write. +/// +/// The record captures the key and fingerprint of every item *as of now*, which +/// is what makes later analysis honest about drift. +/// +/// # Arguments +/// +/// * `catalog` - the loaded course. +/// * `selection` - the draw. +/// * `id` - the assessment id. +/// * `title` - the printed title. +/// * `kind` - the kind of assessment. +/// * `date` - the administration date. +/// * `platform` - where it will be administered. +/// * `blueprint` - the blueprint used, recorded for later comparison. +/// * `forms` - how many alternate forms to declare. +/// +/// # Returns +/// +/// The assessment record. +/// +/// # Errors +/// +/// Returns [`Error::Unresolved`] if a selected id has vanished from the catalog. +#[allow(clippy::too_many_arguments)] +pub fn to_record( + catalog: &Catalog, + selection: &Selection, + id: &str, + title: &str, + kind: Kind, + date: Date, + platform: Platform, + blueprint: &Blueprint, + forms: usize, +) -> Result { + let default_points = catalog.course.policy.points_per_item; + let mut items = Vec::new(); + let mut number = 1u32; + + for (uid, is_bonus) in selection + .scored + .iter() + .map(|u| (u, false)) + .chain(selection.bonus.iter().map(|u| (u, true))) + { + let e = catalog.require(uid)?; + items.push(Placement { + number, + item: uid.clone(), + version: Some(e.item.version), + fingerprint: Some(e.item.fingerprint()), + points: Some(e.item.points(default_points)), + bonus: is_bonus || e.item.bonus, + key: e.item.key_letters(), + level: Some(e.item.level), + learning_objectives: e.item.learning_objectives.clone(), + credit_overrides: BTreeMap::new(), + dropped: false, + }); + number += 1; + } + + let form_list: Vec
= (0..forms) + .map(|i| { + let label = form_label(i); + Form { + seed: Rng::from_label(&format!("{id}/form-{label}")).next_u64(), + id: label, + shuffle_items: false, + shuffle_options: true, + } + }) + .collect(); + + Ok(AssessmentFile { + schema_version: SCHEMA_VERSION.to_string(), + assessment: Assessment { + id: id.to_string(), + title: title.to_string(), + term: Some(catalog.course.course.term.clone()), + kind, + date: Some(date), + platform, + minutes_allowed: None, + attempts: None, + shuffle: None, + scoring_policy: None, + instructions: None, + notes: None, + }, + blueprint: Some(blueprint.clone()), + forms: form_list, + items, + }) +} + +/// The label for the nth form: A, B, ... Z, AA, AB, ... +/// +/// # Arguments +/// +/// * `i` - the zero-based form index. +/// +/// # Returns +/// +/// The label. +fn form_label(i: usize) -> String { + let mut n = i; + let mut out = String::new(); + loop { + out.insert(0, (b'A' + (n % 26) as u8) as char); + if n < 26 { + break; + } + n = n / 26 - 1; + } + out +} + +/// The order items appear in on one form. +/// +/// Permuting a form is a display concern, so it is computed on demand from the +/// recorded seed rather than stored. That keeps the record small and guarantees +/// every export of form B agrees. +/// +/// # Arguments +/// +/// * `record` - the assessment record. +/// * `form` - the form to lay out. +/// +/// # Returns +/// +/// Placements in printed order for this form. The `number` field is left as +/// recorded, since it is the join key to grading data and must not change +/// between forms; use the position in the returned vector for what to print. +pub fn layout(record: &AssessmentFile, form: &Form) -> Vec { + // Bonus items always come last, whatever the shuffle says: they are outside + // the scored total, and burying one mid-form invites students to spend time + // there that the graded questions needed. + let mut scored: Vec = record.items.iter().filter(|p| !p.bonus).cloned().collect(); + let bonus: Vec = record.items.iter().filter(|p| p.bonus).cloned().collect(); + + if form.shuffle_items { + let mut rng = Rng::new(form.seed); + rng.shuffle(&mut scored); + } + + scored.into_iter().chain(bonus.into_iter()).collect() +} + +/// The option order for one item on one form. +/// +/// # Arguments +/// +/// * `form` - the form. +/// * `uid` - the item's global id, which salts the permutation so two items on +/// the same form do not permute identically. +/// * `n` - the number of options. +/// +/// # Returns +/// +/// A permutation of `0..n`. +pub fn option_order(form: &Form, uid: &str, n: usize) -> Vec { + let mut order: Vec = (0..n).collect(); + if form.shuffle_options && n > 1 { + let mut rng = Rng::from_label(&format!("{}/{}/{}", form.seed, form.id, uid)); + rng.shuffle(&mut order); + } + order +} + +/// Compares a record against its blueprint. +/// +/// # Arguments +/// +/// * `record` - the assessment record. +/// +/// # Returns +/// +/// One message per discrepancy, empty when the form matches the design. +pub fn check_blueprint(record: &AssessmentFile) -> Vec { + let Some(bp) = &record.blueprint else { + return vec!["the record carries no blueprint to check against".into()]; + }; + let actual = record.level_counts(); + let mut out = Vec::new(); + for level in Level::ALL { + let want = bp.level_counts.get(&level).copied().unwrap_or(0); + let got = actual.get(&level).copied().unwrap_or(0); + if want != got { + out.push(format!( + "level {}: blueprint asks for {want}, the form has {got}", + level.code() + )); + } + } + for (objective, needed) in &bp.objective_minimums { + let got = record + .items + .iter() + .filter(|p| p.learning_objectives.iter().any(|o| o == objective)) + .count(); + if got < *needed { + out.push(format!( + "objective `{objective}`: blueprint asks for {needed} item(s), the form has {got}" + )); + } + } + out +} + +/// The set of objectives an assessment covers. +/// +/// # Arguments +/// +/// * `record` - the assessment record. +/// +/// # Returns +/// +/// The objective ids, deduplicated. +pub fn covered_objectives(record: &AssessmentFile) -> BTreeSet { + record + .items + .iter() + .flat_map(|p| p.learning_objectives.iter().cloned()) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn form_labels_extend_past_z() { + assert_eq!(form_label(0), "A"); + assert_eq!(form_label(1), "B"); + assert_eq!(form_label(25), "Z"); + assert_eq!(form_label(26), "AA"); + assert_eq!(form_label(27), "AB"); + } + + #[test] + fn option_order_is_a_reproducible_permutation() { + let form = Form { + id: "A".into(), + seed: 12345, + shuffle_items: false, + shuffle_options: true, + }; + let a = option_order(&form, "b::q-1", 5); + let b = option_order(&form, "b::q-1", 5); + assert_eq!(a, b, "same inputs give the same order"); + + let other = option_order(&form, "b::q-2", 5); + assert_ne!(a, other, "different items permute differently"); + + let mut sorted = a.clone(); + sorted.sort_unstable(); + assert_eq!(sorted, vec![0, 1, 2, 3, 4]); + } + + #[test] + fn option_order_is_identity_when_shuffling_is_off() { + let form = Form { + id: "A".into(), + seed: 1, + shuffle_items: false, + shuffle_options: false, + }; + assert_eq!(option_order(&form, "b::q-1", 4), vec![0, 1, 2, 3]); + } + + #[test] + fn blueprint_check_reports_shortfalls() { + let record: AssessmentFile = serde_yaml_ng::from_str( + r#" +assessment: { id: x, title: X } +blueprint: + level_counts: { 1: 2, 3: 1 } + objective_minimums: { lo-key: 2 } +items: + - { number: 1, item: "b::q-1", level: 1, learning_objectives: [lo-key] } + - { number: 2, item: "b::q-2", level: 1 } +"#, + ) + .unwrap(); + let issues = check_blueprint(&record); + assert!(issues.iter().any(|i| i.contains("level 3")), "{issues:?}"); + assert!(issues.iter().any(|i| i.contains("lo-key")), "{issues:?}"); + // Level 1 matches, so it must not be reported. + assert!(!issues.iter().any(|i| i.contains("level 1"))); + } + + #[test] + fn covered_objectives_deduplicates() { + let record: AssessmentFile = serde_yaml_ng::from_str( + r#" +assessment: { id: x, title: X } +items: + - { number: 1, item: "b::q-1", learning_objectives: [lo-a, lo-b] } + - { number: 2, item: "b::q-2", learning_objectives: [lo-a] } +"#, + ) + .unwrap(); + let set = covered_objectives(&record); + assert_eq!(set.len(), 2); + assert!(set.contains("lo-a")); + } +} diff --git a/src/data.rs b/src/data.rs new file mode 100644 index 0000000..e61bd7a --- /dev/null +++ b/src/data.rs @@ -0,0 +1,29 @@ +//! Getting grading data in, and keeping it. +//! +//! Every platform exports a different shape and none of them is analyzable, so +//! this module's job is to turn all of them into one long-format row per student +//! per item — see [`responses::Response`]. +//! +//! ```text +//! gradescope ──┐ +//! ├──▶ responses (canonical long form) ──▶ store ──▶ data/*.parquet +//! canvas ──────┘ +//! ``` +//! +//! [`gradescope`] and [`canvas`] are both written against real exports rather than +//! documentation, because the formats do things the documentation does not mention: +//! bonus questions carrying a maximum of zero, trailing rows shorter than the +//! header, rubric labels with the instructor's parenthetical attached, and answer +//! *text* where you wanted an option letter. +//! +//! [`store`] keeps one file per administration rather than one big table, so +//! re-ingesting one exam cannot corrupt another. [`store_parquet`] contains every +//! use of `arrow` and `parquet` in the entire crate, which is what makes +//! `--no-default-features` a one-file change rather than a refactor. + +pub mod canvas; +pub mod gradescope; +pub mod responses; +pub mod store; +#[cfg(feature = "parquet")] +pub mod store_parquet; diff --git a/src/data/canvas.rs b/src/data/canvas.rs new file mode 100644 index 0000000..9310532 --- /dev/null +++ b/src/data/canvas.rs @@ -0,0 +1,619 @@ +//! Reading Canvas's "Student Analysis" quiz export. +//! +//! Canvas exports one wide CSV for the whole quiz. After the student columns come +//! two columns per question: one headed `: ` holding +//! the answer text the student chose, and one immediately after it holding the +//! points earned, headed with the points possible. +//! +//! The awkward part is that Canvas records answer text, not option letters. So +//! recovering "this student chose C" means matching the exported text back against +//! the item's options. That matching is done on normalized text because the text +//! makes a round trip through the QTI export and Canvas's own HTML sanitizer, and +//! comes back with different markup than it left with. +//! +//! When a match cannot be made, the row keeps its score and loses its option +//! letter, with a warning naming the question. Totals and per-objective mastery +//! still work, and only distractor analysis is affected. +//! +//! Questions are aligned to item numbers by matching stem text the same way, with +//! column order as the fallback. Order alone would be wrong for any quiz where +//! Canvas shuffled the questions. + +use std::collections::BTreeMap; +use std::path::Path; + +use crate::assessment::AssessmentFile; +use crate::catalog::Catalog; +use crate::date::Date; +use crate::error::{Error, Result}; +use crate::responses::{administration_id, Response, ResponseSet}; + +/// What the ingest needs that the export does not carry. +pub type Context = crate::gradescope::Context; + +/// One question column pair found in the header. +#[derive(Debug, Clone)] +struct QuestionColumn { + /// Index of the answer-text column. + text_index: usize, + /// Index of the points-earned column, when the header had one. + points_index: Option, + /// The Canvas question id, from the column header. + /// + /// Kept for diagnostics rather than for matching: when a column cannot be + /// traced to the assessment record, or an answer cannot be traced to an option, + /// this id is what lets you find the question in Canvas and see what differs. + /// Matching itself goes by question text, because Canvas ids change when a quiz + /// is copied to a new term. + canvas_id: String, + /// The question text from the header. + question_text: String, + /// Points possible, parsed from the points column's header. + points_possible: f64, + /// The item number this maps to, once resolved. + item_number: Option, +} + +/// Splits a question column header into its id and text. +/// +/// # Arguments +/// +/// * `header` - the column header. +/// +/// # Returns +/// +/// The id and text, or `None` when the header is not a question column. +fn split_question_header(header: &str) -> Option<(String, String)> { + let (id, rest) = header.split_once(':')?; + let id = id.trim(); + if id.is_empty() || !id.chars().all(|c| c.is_ascii_digit()) { + return None; + } + Some((id.to_string(), rest.trim().to_string())) +} + +/// Normalizes text for comparison. +/// +/// Strips HTML tags, decodes the handful of entities that survive a QTI round +/// trip, drops punctuation, lowercases, and collapses whitespace. +/// +/// # Arguments +/// +/// * `s` - the text. +/// +/// # Returns +/// +/// The normalized form. +pub fn normalize(s: &str) -> String { + // Strip tags. + let mut stripped = String::with_capacity(s.len()); + let mut in_tag = false; + for ch in s.chars() { + match ch { + '<' => in_tag = true, + '>' => { + in_tag = false; + stripped.push(' '); + } + _ if in_tag => {} + _ => stripped.push(ch), + } + } + + // Decode the entities that actually appear. + let decoded = stripped + .replace(" ", " ") + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace(""", "\"") + .replace("'", "'") + .replace("'", "'") + .replace("→", "->") + .replace("↔", "<->") + .replace("−", "-"); + + let mut out = String::with_capacity(decoded.len()); + let mut last_space = true; + for ch in decoded.chars() { + let c = ch.to_ascii_lowercase(); + if c.is_alphanumeric() { + out.push(c); + last_space = false; + } else if c.is_whitespace() || c == '-' || c == '_' { + if !last_space { + out.push(' '); + last_space = true; + } + } + // Everything else — punctuation, entity leftovers — is dropped. + } + out.trim().to_string() +} + +/// Reads a Canvas Student Analysis export. +/// +/// # Arguments +/// +/// * `path` - the CSV file. +/// * `ctx` - identifying information. +/// * `record` - the assessment record, used to align questions to item numbers. +/// * `catalog` - the loaded course, used to recover option letters from answer +/// text. Without it, scores are still ingested but letters are not. +/// +/// # Returns +/// +/// The normalized responses. +/// +/// # Errors +/// +/// Returns [`Error::Csv`] on a malformed file and [`Error::Invalid`] when no +/// question columns can be found. +pub fn ingest( + path: &Path, + ctx: &Context, + record: Option<&AssessmentFile>, + catalog: Option<&Catalog>, +) -> Result { + let mut reader = csv::ReaderBuilder::new() + .flexible(true) + .has_headers(true) + .from_path(path) + .map_err(|e| Error::Csv { + path: path.to_path_buf(), + source: e, + })?; + + let header = reader + .headers() + .map_err(|e| Error::Csv { + path: path.to_path_buf(), + source: e, + })? + .clone(); + let header: Vec = header.iter().map(|h| h.trim().to_string()).collect(); + + let find = |name: &str| header.iter().position(|h| h.eq_ignore_ascii_case(name)); + let name_col = find("name"); + let id_col = find("id"); + let sis_col = find("sis_id").or_else(|| find("sis id")); + let section_col = find("section"); + let submitted_col = find("submitted"); + let attempt_col = find("attempt"); + + // Locate the question column pairs. + let mut questions: Vec = Vec::new(); + for (i, h) in header.iter().enumerate() { + if let Some((canvas_id, question_text)) = split_question_header(h) { + // The next column holds points earned; its header is the points + // possible. Canvas writes it as a bare number. + let (points_index, points_possible) = match header.get(i + 1) { + Some(next) => match next.trim().parse::() { + Ok(p) => (Some(i + 1), p), + Err(_) => (None, 0.0), + }, + None => (None, 0.0), + }; + questions.push(QuestionColumn { + text_index: i, + points_index, + canvas_id, + question_text, + points_possible, + item_number: None, + }); + } + } + + if questions.is_empty() { + return Err(Error::Invalid(vec![format!( + "{} has no question columns; a Canvas Student Analysis export heads each question \ + `: `. Did you export the Item Analysis instead?", + path.display() + )])); + } + + let mut warnings = Vec::new(); + align_questions(&mut questions, record, catalog, &mut warnings); + + // Build the answer-text lookup once per question: normalized option text to + // option letter. + let mut answer_lookup: BTreeMap> = BTreeMap::new(); + let mut item_meta: BTreeMap, Vec)> = BTreeMap::new(); + if let (Some(cat), Some(rec)) = (catalog, record) { + for (qi, q) in questions.iter().enumerate() { + let Some(number) = q.item_number else { + continue; + }; + let Some(placement) = rec.placement(number) else { + continue; + }; + let Some(entry) = cat.get(&placement.item) else { + continue; + }; + let mut map = BTreeMap::new(); + for opt in &entry.item.options { + map.insert(normalize(&opt.text), opt.id.clone()); + } + answer_lookup.insert(qi, map); + item_meta.insert(qi, (Some(placement.item.clone()), placement.key.clone())); + } + } + + let admin = administration_id(&ctx.course, &ctx.term, &ctx.assessment_id); + let mut set = ResponseSet::new(); + let mut unmatched_answers: BTreeMap<(u32, String), usize> = BTreeMap::new(); + + for rec in reader.records() { + let rec = rec.map_err(|e| Error::Csv { + path: path.to_path_buf(), + source: e, + })?; + let cell = |i: Option| -> Option { + i.and_then(|i| rec.get(i)) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + }; + + let sid = cell(sis_col).or_else(|| cell(id_col)); + let name = cell(name_col); + if sid.is_none() && name.is_none() { + continue; + } + // Canvas emits a "Test Student" row for anyone who previewed the quiz. + if name.as_deref() == Some("Test Student") { + continue; + } + let student_key = sid + .clone() + .or_else(|| name.clone()) + .unwrap_or_else(|| "unknown".to_string()); + let section = cell(section_col); + let submitted = cell(submitted_col); + let attempt = cell(attempt_col).and_then(|a| a.parse::().ok()); + // Only the graded attempt is exported per row, but a student who never + // submitted still gets a row; those carry no responses. + if submitted.is_none() && attempt.is_none() { + continue; + } + + for (qi, q) in questions.iter().enumerate() { + let Some(number) = q.item_number else { + continue; + }; + let answer_text = rec.get(q.text_index).map(|s| s.trim()).unwrap_or(""); + let score = q + .points_index + .and_then(|i| rec.get(i)) + .and_then(|s| s.trim().parse::().ok()) + .unwrap_or(0.0); + + let points = q.points_possible; + let credit = if points > 0.0 { score / points } else { 0.0 }; + + let mut selected = Vec::new(); + if !answer_text.is_empty() { + let normalized = normalize(answer_text); + match answer_lookup.get(&qi).and_then(|m| m.get(&normalized)) { + Some(letter) => selected.push(letter.clone()), + None => { + *unmatched_answers + .entry((number, q.canvas_id.clone())) + .or_insert(0) += 1; + } + } + } + + let (item_ref, key) = item_meta.get(&qi).cloned().unwrap_or((None, Vec::new())); + + let correct = if answer_text.is_empty() { + None + } else if points > 0.0 { + Some(credit >= 0.999) + } else if !key.is_empty() && !selected.is_empty() { + Some(selected == key) + } else { + None + }; + + set.rows.push(Response { + administration_id: admin.clone(), + course: ctx.course.clone(), + term: ctx.term.clone(), + assessment_id: ctx.assessment_id.clone(), + date: ctx.date, + form: ctx.form.clone(), + student_key: student_key.clone(), + sid: sid.clone(), + name: name.clone(), + email: None, + section: section.clone(), + item_number: number, + item_ref, + item_version: None, + selected, + eliminated: Vec::new(), + correct, + credit, + points_possible: points, + score, + response_time_seconds: None, + level: None, + learning_objectives: Vec::new(), + topics: Vec::new(), + bonus: false, + dropped: false, + }); + } + } + + for ((number, canvas_id), count) in unmatched_answers { + warnings.push(format!( + "question {number} (Canvas id {canvas_id}): {count} answer(s) did not match any option \ + text, so those responses have a score but no option letter; distractor analysis for \ + this item will be incomplete. The usual cause is the option text being edited in \ + Canvas after import" + )); + } + + if set.rows.is_empty() { + warnings.push( + "no student rows were found; the export may contain only the header, or every row \ + may be an unsubmitted attempt" + .to_string(), + ); + } + + set.warnings.extend(warnings); + Ok(set) +} + +/// Assigns an item number to each question column. +/// +/// Matching on stem text is preferred over column order because Canvas shuffles +/// questions when the quiz says to, and the export follows the shuffled order. +/// +/// # Arguments +/// +/// * `questions` - the columns, updated in place. +/// * `record` - the assessment record. +/// * `catalog` - the loaded course. +/// * `warnings` - collects anything ambiguous. +fn align_questions( + questions: &mut [QuestionColumn], + record: Option<&AssessmentFile>, + catalog: Option<&Catalog>, + warnings: &mut Vec, +) { + let Some(rec) = record else { + // Without a record, the only sensible assumption is column order. + for (i, q) in questions.iter_mut().enumerate() { + q.item_number = Some(i as u32 + 1); + } + warnings.push( + "no assessment record was supplied, so questions were matched to item numbers by \ + column order; pass --assessment to match on question text instead" + .to_string(), + ); + return; + }; + + // Normalized stem to item number, when the catalog is available. + let mut by_stem: BTreeMap> = BTreeMap::new(); + if let Some(cat) = catalog { + for p in &rec.items { + if let Some(entry) = cat.get(&p.item) { + by_stem + .entry(normalize(&entry.item.stem)) + .or_default() + .push(p.number); + } + } + } + + let mut used: Vec = Vec::new(); + let mut unmatched: Vec = Vec::new(); + + for (i, q) in questions.iter_mut().enumerate() { + let normalized = normalize(&q.question_text); + let candidates = by_stem.get(&normalized); + match candidates { + Some(numbers) => { + // Prefer a number not already claimed, so two items sharing a stem + // do not collapse onto one. + match numbers.iter().find(|n| !used.contains(n)) { + Some(n) => { + q.item_number = Some(*n); + used.push(*n); + } + None => { + q.item_number = Some(numbers[0]); + } + } + } + None => { + // Fall back to position, which is right for an unshuffled quiz. + let fallback = rec.items.get(i).map(|p| p.number).unwrap_or(i as u32 + 1); + if !used.contains(&fallback) { + used.push(fallback); + } + q.item_number = Some(fallback); + // Report the Canvas question id, not just a count. The id is what + // you can search for in Canvas to see which question this was, and + // an unmatched column usually means the text was edited there after + // import — so being able to find it is the whole remedy. + unmatched.push(format!("{} (assumed question {fallback})", q.canvas_id)); + } + } + } + + if !unmatched.is_empty() { + warnings.push(format!( + "{} of {} question column(s) could not be matched to the assessment record by question \ + text and were matched by column order instead. Canvas question id(s): {}. Check that \ + the record describes this quiz, and that the question text was not edited in Canvas \ + after import", + unmatched.len(), + questions.len(), + unmatched.join(", ") + )); + } +} + +/// Guesses a date from a Canvas timestamp column. +/// +/// # Arguments +/// +/// * `s` - the timestamp, e.g. `2026-04-01 14:03:22 UTC`. +/// +/// # Returns +/// +/// The date, when the leading token parses as one. +pub fn date_from_timestamp(s: &str) -> Option { + s.split_whitespace().next()?.parse().ok() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn recognizes_question_headers() { + assert_eq!( + split_question_header("123456: Which enzyme catalyzes the step?"), + Some(( + "123456".to_string(), + "Which enzyme catalyzes the step?".to_string() + )) + ); + assert_eq!(split_question_header("name"), None); + assert_eq!(split_question_header("n correct"), None); + // A colon in prose is not a question column. + assert_eq!(split_question_header("Note: read carefully"), None); + } + + #[test] + fn normalization_survives_a_qti_round_trip() { + // Option text authored as `K#sub[m] increases` is exported as HTML and + // comes back from Canvas wrapped in tags. Both must normalize to the same + // thing as the HTML we sent, or letters cannot be recovered. + let sent = "Km increases"; + let returned = "

Km increases

"; + assert_eq!(normalize(sent), normalize(returned)); + assert_eq!(normalize(returned), "k m increases"); + assert_eq!(normalize("K m increases"), "k m increases"); + } + + #[test] + fn normalization_ignores_punctuation_and_case() { + assert_eq!( + normalize("The Rate, Increases!"), + normalize("the rate increases") + ); + assert_eq!(normalize(" spaced out "), "spaced out"); + assert_eq!(normalize("bold text"), "bold text"); + } + + #[test] + fn timestamps_yield_dates() { + assert_eq!( + date_from_timestamp("2026-04-01 14:03:22 UTC").map(|d| d.to_string()), + Some("2026-04-01".to_string()) + ); + assert_eq!(date_from_timestamp("not a date"), None); + } + + #[test] + fn falls_back_to_column_order_without_a_record() { + let mut questions = vec![ + QuestionColumn { + text_index: 8, + points_index: Some(9), + canvas_id: "1".into(), + question_text: "Q one".into(), + points_possible: 1.0, + item_number: None, + }, + QuestionColumn { + text_index: 10, + points_index: Some(11), + canvas_id: "2".into(), + question_text: "Q two".into(), + points_possible: 1.0, + item_number: None, + }, + ]; + let mut warnings = Vec::new(); + align_questions(&mut questions, None, None, &mut warnings); + assert_eq!(questions[0].item_number, Some(1)); + assert_eq!(questions[1].item_number, Some(2)); + assert_eq!(warnings.len(), 1); + } + + #[test] + fn parses_a_realistic_export() { + let dir = std::env::temp_dir().join(format!("cb-canvas-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("analysis.csv"); + std::fs::write( + &path, + "name,id,sis_id,section,submitted,attempt,\ + 1001: Which enzyme?,1.0,1002: Which pathway?,1.0,n correct,n incorrect,score\n\ + Ada Lovelace,9001,1234567,L01,2026-04-01 14:00:00 UTC,1,Hexokinase,1.0,Glycolysis,\ + 1.0,2,0,2.0\n\ + Alan Turing,9002,7654321,L01,2026-04-01 14:05:00 UTC,1,Pyruvate kinase,0.0,\ + Glycolysis,1.0,1,1,1.0\n\ + Test Student,9999,,L01,2026-04-01 13:00:00 UTC,1,Hexokinase,1.0,Glycolysis,1.0,2,0,\ + 2.0\n", + ) + .unwrap(); + + let ctx = Context { + course: "BIOSC1540".into(), + term: "2026S".into(), + assessment_id: "quiz-1".into(), + date: None, + form: None, + }; + let set = ingest(&path, &ctx, None, None).unwrap(); + + // Two real students, two questions each. The preview row is discarded. + assert_eq!(set.rows.len(), 4, "{:?}", set.warnings); + assert!(set + .rows + .iter() + .all(|r| r.name.as_deref() != Some("Test Student"))); + assert_eq!(set.scored_total("1234567"), 2.0); + assert_eq!(set.scored_total("7654321"), 1.0); + + let alan_q1 = set + .rows + .iter() + .find(|r| r.student_key == "7654321" && r.item_number == 1) + .unwrap(); + assert_eq!(alan_q1.correct, Some(false)); + assert_eq!(alan_q1.credit, 0.0); + // No catalog was supplied, so no letter could be recovered. + assert!(alan_q1.selected.is_empty()); + + std::fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn rejects_the_wrong_export_kind() { + let dir = std::env::temp_dir().join(format!("cb-canvas-bad-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("item.csv"); + std::fs::write(&path, "question,discrimination index\nQ1,0.4\n").unwrap(); + let ctx = Context { + course: "C".into(), + term: "T".into(), + assessment_id: "a".into(), + date: None, + form: None, + }; + let err = ingest(&path, &ctx, None, None).unwrap_err(); + assert!(err.to_string().contains("Student Analysis")); + std::fs::remove_dir_all(&dir).ok(); + } +} diff --git a/src/data/gradescope.rs b/src/data/gradescope.rs new file mode 100644 index 0000000..21dad37 --- /dev/null +++ b/src/data/gradescope.rs @@ -0,0 +1,855 @@ +//! Reading Gradescope's per-question CSV exports. +//! +//! Gradescope exports one file per question, named `1.csv` through `N.csv`, in +//! wide form. Every assumption encoded here was checked against a real 32-question, +//! 24-student export rather than inferred from documentation, because several of +//! them are not what the format suggests. +//! +//! The layout is: +//! +//! ```text +//! Assignment Submission ID, Question Submission ID, First Name, Last Name, +//! SID, Email, Sections, Score, Submission Time, , +//! Adjustment, Comments, Grader, Tags +//! ``` +//! +//! The rubric columns are the header slice strictly between `Submission Time` +//! and `Adjustment`. Student cells in those columns are the literal strings +//! `true` and `false`. +//! +//! Four things about real exports that a naive parser gets wrong: +//! +//! *Trailing rows are not aligned with the header.* After the student rows come +//! `Point Values`, `Rubric Numbers`, and `Scoring Method` rows with a label, some +//! empty cells, and then the numbers. The CSV reader must be in flexible mode or +//! it errors on the whole file. +//! +//! *Rubric labels are not always bare option letters.* Real ones include +//! `Selected C`, `Eliminated A`, and — this is the interesting case — a letter +//! followed by an instructor's parenthetical, such as +//! `B (Technically speaking, this answer describes HBD/HBA, but I can see how +//! this distractor is poorly written.)`. Those columns carry partial credit. +//! +//! *Partial credit awarded to a distractor is evidence, not noise.* When you gave +//! 1.0 of 1.5 points for choosing B, you decided at grading time that B was +//! partly defensible. That is exactly the ambiguity signal item analysis is +//! trying to detect, so it is captured as [`Flag::Ambiguous`] rather than +//! rounded away. +//! +//! *Bonus questions have `max_points` of zero* while a rubric column still +//! carries points. Key detection therefore uses the largest value across the +//! maximum *and* every column, not the maximum alone. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +use crate::date::Date; +use crate::error::{Error, Result}; +use crate::responses::{administration_id, Response, ResponseSet}; +use crate::taxonomy::Flag; + +/// What a rubric column represents. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ColumnKind { + /// The student selected this option. + Select, + /// The student eliminated this option, under elimination scoring. + Eliminate, + /// Something else: a catch-all rubric row such as + /// `Incorrect selection with no eliminations`. + Other, +} + +/// One rubric column. +#[derive(Debug, Clone)] +pub struct RubricColumn { + /// The header label, verbatim. + pub label: String, + /// What the column means. + pub kind: ColumnKind, + /// The option letter, when the label named one. + pub letter: Option, + /// The instructor's parenthetical annotation, when there was one. Worth + /// keeping: it is usually the instructor explaining why an item is flawed. + pub note: Option, + /// Points this column awards, from the `Point Values` row. + pub value: Option, +} + +/// One student's row in a question file. +#[derive(Debug, Clone)] +pub struct StudentRow { + /// The institutional student id. + pub sid: Option, + /// First and last name joined. + pub name: Option, + /// The email. + pub email: Option, + /// Section or lab. + pub section: Option, + /// Points awarded, authoritative. + pub score: f64, + /// The submission timestamp, verbatim. + pub submission_time: Option, + /// Indices of rubric columns marked `true`. + pub marks: Vec, +} + +/// One parsed question file. +#[derive(Debug, Clone)] +pub struct Question { + /// The question number, from the file name. + pub number: u32, + /// Where it came from. + pub path: PathBuf, + /// The rubric columns, in header order. + pub columns: Vec, + /// The maximum points from the `Point Values` row. + pub max_points: f64, + /// The scoring method, when stated. + pub scoring_method: Option, + /// The student rows. + pub rows: Vec, +} + +impl Question { + /// The largest point value anywhere in the question. + /// + /// Bonus questions record a maximum of zero while still awarding points, so + /// the key has to be found by looking at the columns too. + /// + /// # Returns + /// + /// The largest value seen. + pub fn top_value(&self) -> f64 { + let mut top = self.max_points; + for c in &self.columns { + if let Some(v) = c.value { + if v > top { + top = v; + } + } + } + top + } + + /// The option letters that earn full credit. + /// + /// # Returns + /// + /// The keyed letters, sorted. Empty when the point values were absent. + pub fn keyed(&self) -> Vec { + let top = self.top_value(); + if top <= 0.0 { + return Vec::new(); + } + let mut out: BTreeSet = BTreeSet::new(); + for c in &self.columns { + if c.kind == ColumnKind::Select { + if let (Some(letter), Some(v)) = (&c.letter, c.value) { + if (v - top).abs() < 1e-9 { + out.insert(letter.clone()); + } + } + } + } + out.into_iter().collect() + } + + /// Distractors that were awarded partial credit at grading time. + /// + /// # Returns + /// + /// Letter, points awarded, and the instructor's note if any. + pub fn partial_credit(&self) -> Vec<(String, f64, Option)> { + let top = self.top_value(); + let mut out = Vec::new(); + for c in &self.columns { + if c.kind != ColumnKind::Select { + continue; + } + if let (Some(letter), Some(v)) = (&c.letter, c.value) { + if v > 0.0 && v < top - 1e-9 { + out.push((letter.clone(), v, c.note.clone())); + } + } + } + out + } + + /// Whether this looks like a bonus question. + /// + /// # Returns + /// + /// `true` when the maximum is zero but a column still awards points. + pub fn looks_like_bonus(&self) -> bool { + self.max_points <= 0.0 && self.top_value() > 0.0 + } + + /// Whether elimination scoring was in use. + pub fn uses_elimination(&self) -> bool { + self.columns.iter().any(|c| c.kind == ColumnKind::Eliminate) + } + + /// Points the question was worth. + /// + /// # Returns + /// + /// The stated maximum, falling back to the largest column value for bonus + /// questions so that credit is still a meaningful fraction. + pub fn points_possible(&self) -> f64 { + if self.max_points > 0.0 { + self.max_points + } else { + self.top_value() + } + } + + /// Flags implied by how the question was graded. + /// + /// # Returns + /// + /// [`Flag::Ambiguous`] when a distractor received partial credit. + pub fn implied_flags(&self) -> Vec { + if self.partial_credit().is_empty() { + Vec::new() + } else { + vec![Flag::Ambiguous] + } + } +} + +/// What the ingest needs to know that the export does not say. +#[derive(Debug, Clone)] +pub struct Context { + /// The course code. + pub course: String, + /// The term. + pub term: String, + /// The assessment id. + pub assessment_id: String, + /// The administration date. + pub date: Option, + /// The form, when forms were used. + pub form: Option, +} + +/// The result of reading a directory of question files. +#[derive(Debug, Clone)] +pub struct Import { + /// The normalized responses. + pub responses: ResponseSet, + /// The parsed question files, kept so grading-time decisions can be folded + /// into item calibration. + pub questions: Vec, +} + +/// Classifies a rubric column label. +/// +/// # Arguments +/// +/// * `label` - the header text. +/// +/// # Returns +/// +/// The kind, the option letter if the label named one, and any trailing +/// annotation with its surrounding punctuation trimmed. +pub fn classify(label: &str) -> (ColumnKind, Option, Option) { + let trimmed = label.trim(); + let lower = trimmed.to_ascii_lowercase(); + + let (kind, rest) = if let Some(r) = lower.strip_prefix("selected ") { + ( + ColumnKind::Select, + trimmed[trimmed.len() - r.len()..].trim(), + ) + } else if let Some(r) = lower.strip_prefix("eliminated ") { + ( + ColumnKind::Eliminate, + trimmed[trimmed.len() - r.len()..].trim(), + ) + } else { + (ColumnKind::Select, trimmed) + }; + + let mut chars = rest.chars(); + let Some(first) = chars.next() else { + return (ColumnKind::Other, None, Some(label.to_string())); + }; + let upper = first.to_ascii_uppercase(); + let tail = chars.as_str(); + + // A single letter A-H, optionally followed by an annotation that does not + // begin with another alphanumeric. `B (poorly written)` is option B; + // `Both A and C are wrong` is not. + let letter_like = upper.is_ascii_uppercase() + && ('A'..='H').contains(&upper) + && tail + .chars() + .next() + .map(|c| !c.is_alphanumeric()) + .unwrap_or(true); + + if !letter_like { + return (ColumnKind::Other, None, Some(label.to_string())); + } + + let note = tail.trim().trim_matches(|c: char| "().,;: ".contains(c)); + let note = if note.is_empty() { + None + } else { + Some(note.to_string()) + }; + (kind, Some(upper.to_string()), note) +} + +/// Parses one question file. +/// +/// # Arguments +/// +/// * `path` - the CSV file. +/// +/// # Returns +/// +/// The parsed question. +/// +/// # Errors +/// +/// Returns [`Error::Csv`] on a malformed file and [`Error::Invalid`] when the +/// header lacks the `Submission Time` column that delimits the rubric. +pub fn parse_question(path: &Path) -> Result { + let number = path + .file_stem() + .and_then(|s| s.to_str()) + .and_then(|s| s.parse::().ok()) + .ok_or_else(|| { + Error::Invalid(vec![format!( + "{} is not named like a Gradescope question export (expected `12.csv`)", + path.display() + )]) + })?; + + // Flexible mode is required: the trailing `Point Values` rows are shorter + // than the header. + let mut reader = csv::ReaderBuilder::new() + .flexible(true) + .has_headers(false) + .from_path(path) + .map_err(|e| Error::Csv { + path: path.to_path_buf(), + source: e, + })?; + + let mut records = reader.records(); + let header = match records.next() { + Some(r) => r.map_err(|e| Error::Csv { + path: path.to_path_buf(), + source: e, + })?, + None => return Err(Error::Invalid(vec![format!("{} is empty", path.display())])), + }; + let header: Vec = header.iter().map(|s| s.trim().to_string()).collect(); + + let index_of = |name: &str| header.iter().position(|h| h == name); + let start = match index_of("Submission Time") { + Some(i) => i + 1, + None => { + return Err(Error::Invalid(vec![format!( + "{} has no `Submission Time` column, so the rubric columns cannot be located; \ + is this a Gradescope per-question export?", + path.display() + )])) + } + }; + let end = index_of("Adjustment").unwrap_or(header.len()); + if end < start { + return Err(Error::Invalid(vec![format!( + "{} has `Adjustment` before `Submission Time`", + path.display() + )])); + } + + let mut columns: Vec = header[start..end] + .iter() + .map(|label| { + let (kind, letter, note) = classify(label); + RubricColumn { + label: label.clone(), + kind, + letter, + note, + value: None, + } + }) + .collect(); + + let sid_col = index_of("SID"); + let first_col = index_of("First Name"); + let last_col = index_of("Last Name"); + let email_col = index_of("Email"); + let section_col = index_of("Sections"); + let score_col = index_of("Score"); + let time_col = index_of("Submission Time"); + + let mut rows = Vec::new(); + let mut max_points = 0.0f64; + let mut point_values: Vec = Vec::new(); + let mut scoring_method = None; + + for rec in records { + let rec = rec.map_err(|e| Error::Csv { + path: path.to_path_buf(), + source: e, + })?; + let cell = |i: Option| -> Option { + i.and_then(|i| rec.get(i)) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + }; + + if rec.iter().all(|c| c.trim().is_empty()) { + continue; + } + let label = rec.get(0).unwrap_or("").trim(); + + match label { + "Point Values" => { + // Label, then some empty cells, then the maximum, then one value + // per rubric column. Alignment was verified across every file in + // a real export, but collecting the non-empty numbers in order is + // robust to a leading blank moving. + let nums: Vec = rec + .iter() + .skip(1) + .filter(|c| !c.trim().is_empty()) + .filter_map(|c| c.trim().parse::().ok()) + .collect(); + if let Some((first, rest)) = nums.split_first() { + max_points = *first; + point_values = rest.to_vec(); + } + continue; + } + "Rubric Numbers" => continue, + "Scoring Method" => { + scoring_method = cell(Some(1)); + continue; + } + _ => {} + } + + // A student row must have a score cell that parses; anything else is a + // trailing annotation row we have not seen before, and skipping it is + // safer than failing the import. + let score = match cell(score_col).and_then(|s| s.parse::().ok()) { + Some(s) => s, + None => { + if cell(sid_col).is_none() && cell(email_col).is_none() { + continue; + } + 0.0 + } + }; + + let marks: Vec = (0..columns.len()) + .filter(|i| { + rec.get(start + i) + .map(|c| c.trim().eq_ignore_ascii_case("true")) + .unwrap_or(false) + }) + .collect(); + + let name = match (cell(first_col), cell(last_col)) { + (Some(f), Some(l)) => Some(format!("{f} {l}")), + (Some(f), None) => Some(f), + (None, Some(l)) => Some(l), + (None, None) => None, + }; + + rows.push(StudentRow { + sid: cell(sid_col), + name, + email: cell(email_col), + section: cell(section_col), + score, + submission_time: cell(time_col), + marks, + }); + } + + // Attach point values when they line up; when they do not, leave them off + // rather than misattribute credit to the wrong option. + if point_values.len() == columns.len() { + for (c, v) in columns.iter_mut().zip(point_values.iter()) { + c.value = Some(*v); + } + } + + Ok(Question { + number, + path: path.to_path_buf(), + columns, + max_points, + scoring_method, + rows, + }) +} + +/// Reads every question file in a directory. +/// +/// # Arguments +/// +/// * `dir` - the directory of `N.csv` files. +/// * `ctx` - identifying information the export does not carry. +/// +/// # Returns +/// +/// Normalized responses and the parsed questions. +/// +/// # Errors +/// +/// Returns [`Error::Io`] when the directory cannot be read and [`Error::Invalid`] +/// when it holds no question files. +pub fn ingest_dir(dir: &Path, ctx: &Context) -> Result { + let mut paths: Vec = std::fs::read_dir(dir) + .map_err(|e| Error::io(dir, e))? + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .filter(|p| { + p.extension().and_then(|e| e.to_str()) == Some("csv") + && p.file_stem() + .and_then(|s| s.to_str()) + .map(|s| s.chars().all(|c| c.is_ascii_digit())) + .unwrap_or(false) + }) + .collect(); + + if paths.is_empty() { + return Err(Error::Invalid(vec![format!( + "{} holds no numbered CSV files; Gradescope exports one file per question, \ + named `1.csv` through `N.csv`", + dir.display() + )])); + } + + // Numeric order, so `10.csv` does not sort before `2.csv`. + paths.sort_by_key(|p| { + p.file_stem() + .and_then(|s| s.to_str()) + .and_then(|s| s.parse::().ok()) + .unwrap_or(u32::MAX) + }); + + let mut questions = Vec::new(); + for p in &paths { + questions.push(parse_question(p)?); + } + + Ok(to_responses(&questions, ctx)) +} + +/// Normalizes parsed questions into responses. +/// +/// # Arguments +/// +/// * `questions` - the parsed question files. +/// * `ctx` - identifying information. +/// +/// # Returns +/// +/// The import, with warnings for anything that looked wrong but not fatal. +pub fn to_responses(questions: &[Question], ctx: &Context) -> Import { + let mut set = ResponseSet::new(); + let admin = administration_id(&ctx.course, &ctx.term, &ctx.assessment_id); + let mut counts: BTreeMap = BTreeMap::new(); + + for q in questions { + let keyed: BTreeSet = q.keyed().into_iter().collect(); + let points = q.points_possible(); + let bonus = q.looks_like_bonus(); + + if keyed.is_empty() { + set.warnings.push(format!( + "question {}: no keyed option could be identified from the point values, so \ + correctness is unknown; scores are still recorded", + q.number + )); + } + for (letter, value, note) in q.partial_credit() { + set.warnings.push(format!( + "question {}: option {letter} was awarded {value} of {points} points at grading \ + time, which is recorded as an ambiguity signal{}", + q.number, + note.map(|n| format!(" ({n})")).unwrap_or_default() + )); + } + + for row in &q.rows { + let mut selected = Vec::new(); + let mut eliminated = Vec::new(); + let mut other = Vec::new(); + for &i in &row.marks { + let c = &q.columns[i]; + match (c.kind, &c.letter) { + (ColumnKind::Select, Some(l)) => selected.push(l.clone()), + (ColumnKind::Eliminate, Some(l)) => eliminated.push(l.clone()), + _ => other.push(c.label.clone()), + } + } + selected.sort(); + eliminated.sort(); + + let credit = if points > 0.0 { + row.score / points + } else { + 0.0 + }; + let correct = if keyed.is_empty() { + None + } else if selected.is_empty() && eliminated.is_empty() && other.is_empty() { + // A wholly unmarked row is a blank response, not a wrong one. + None + } else { + let chosen: BTreeSet = selected.iter().cloned().collect(); + Some(chosen == keyed) + }; + + let student_key = row + .sid + .clone() + .or_else(|| row.email.clone()) + .unwrap_or_else(|| format!("unknown-{}", counts.len())); + + *counts.entry(q.number).or_insert(0) += 1; + + set.rows.push(Response { + administration_id: admin.clone(), + course: ctx.course.clone(), + term: ctx.term.clone(), + assessment_id: ctx.assessment_id.clone(), + date: ctx.date, + form: ctx.form.clone(), + student_key, + sid: row.sid.clone(), + name: row.name.clone(), + email: row.email.clone(), + section: row.section.clone(), + item_number: q.number, + item_ref: None, + item_version: None, + selected, + eliminated, + correct, + credit, + points_possible: points, + score: row.score, + response_time_seconds: None, + level: None, + learning_objectives: Vec::new(), + topics: Vec::new(), + bonus, + dropped: false, + }); + } + } + + // Every question should have the same number of submissions. A mismatch + // usually means a student was excused from one question, which is worth + // saying out loud because it changes per-item denominators. + let sizes: BTreeSet = counts.values().copied().collect(); + if sizes.len() > 1 { + let mut odd: Vec = Vec::new(); + let modal = *sizes.iter().next_back().unwrap_or(&0); + for (n, c) in &counts { + if *c != modal { + odd.push(format!("q{n} has {c}")); + } + } + set.warnings.push(format!( + "submission counts differ across questions (most have {modal}): {}", + odd.join(", ") + )); + } + + Import { + responses: set, + questions: questions.to_vec(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn classifies_bare_letters() { + let (k, l, n) = classify("C"); + assert_eq!(k, ColumnKind::Select); + assert_eq!(l, Some("C".to_string())); + assert_eq!(n, None); + } + + #[test] + fn classifies_selected_and_eliminated_prefixes() { + assert_eq!(classify("Selected C").0, ColumnKind::Select); + assert_eq!(classify("Selected C").1, Some("C".to_string())); + assert_eq!(classify("Eliminated A").0, ColumnKind::Eliminate); + assert_eq!(classify("Eliminated A").1, Some("A".to_string())); + } + + #[test] + fn keeps_the_instructors_annotation() { + // Verbatim from a real export. + let (k, l, n) = classify( + "B (Technically speaking, this answer describes HBD/HBA, but I can see how this \ + distractor is poorly written.)", + ); + assert_eq!(k, ColumnKind::Select); + assert_eq!(l, Some("B".to_string())); + assert!(n.unwrap().starts_with("Technically speaking")); + + let (_, l2, n2) = classify("B (preparation does not select the one most likely to bind)"); + assert_eq!(l2, Some("B".to_string())); + assert!(n2.is_some()); + } + + #[test] + fn rejects_prose_rubric_rows() { + // Also verbatim: these are genuinely not option columns. + let (k, l, _) = classify("Incorrect selection with no eliminations."); + assert_eq!(k, ColumnKind::Other); + assert_eq!(l, None); + assert_eq!(classify("").0, ColumnKind::Other); + } + + #[test] + fn does_not_mistake_a_sentence_for_option_a() { + // Starts with `A`, but the next character is alphanumeric. + assert_eq!(classify("Answered in the margin").0, ColumnKind::Other); + } + + fn q(max: f64, cols: &[(&str, f64)]) -> Question { + Question { + number: 1, + path: PathBuf::from("1.csv"), + columns: cols + .iter() + .map(|(label, v)| { + let (kind, letter, note) = classify(label); + RubricColumn { + label: label.to_string(), + kind, + letter, + note, + value: Some(*v), + } + }) + .collect(), + max_points: max, + scoring_method: None, + rows: Vec::new(), + } + } + + #[test] + fn finds_the_key_by_top_value() { + let question = q(1.5, &[("A", 0.0), ("B", 0.0), ("C", 1.5), ("D", 0.0)]); + assert_eq!(question.keyed(), vec!["C".to_string()]); + assert!(question.partial_credit().is_empty()); + } + + #[test] + fn finds_the_key_on_a_bonus_question_with_zero_maximum() { + // Real case: q31 and q32 of the sample exam. + let question = q(0.0, &[("A", 0.0), ("B", 1.5), ("C", 0.0)]); + assert!(question.looks_like_bonus()); + assert_eq!(question.keyed(), vec!["B".to_string()]); + assert_eq!(question.points_possible(), 1.5); + } + + #[test] + fn partial_credit_to_a_distractor_flags_ambiguity() { + // Real case: q14, where B earned 1.49 of 1.5. + let question = q(1.5, &[("A", 0.0), ("B (poorly written)", 1.49), ("C", 1.5)]); + assert_eq!(question.keyed(), vec!["C".to_string()]); + let partial = question.partial_credit(); + assert_eq!(partial.len(), 1); + assert_eq!(partial[0].0, "B"); + assert!(partial[0].2.is_some(), "keeps the note"); + assert_eq!(question.implied_flags(), vec![Flag::Ambiguous]); + } + + #[test] + fn detects_elimination_scoring() { + let question = q( + 0.9, + &[ + ("Selected C", 0.9), + ("Eliminated A", 0.3), + ("Eliminated B", 0.3), + ], + ); + assert!(question.uses_elimination()); + assert_eq!(question.keyed(), vec!["C".to_string()]); + } + + #[test] + fn parses_a_realistic_file() { + let dir = std::env::temp_dir().join(format!("cb-gs-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("7.csv"); + std::fs::write( + &path, + "Assignment Submission ID,Question Submission ID,First Name,Last Name,SID,Email,\ + Sections,Score,Submission Time,A,B,C,D,Adjustment,Comments,Grader,Tags\n\ + 1,11,Ada,Lovelace,1234567,ada@x.edu,L01,1.5,2026-04-01T10:00:00Z,false,false,true,\ + false,,,,\n\ + 2,12,Alan,Turing,7654321,alan@x.edu,L01,0.0,2026-04-01T10:05:00Z,true,false,false,\ + false,,,,\n\ + 3,13,Grace,Hopper,1111111,grace@x.edu,L02,0.0,2026-04-01T10:06:00Z,false,false,\ + false,false,,,,\n\ + Point Values,,,,,1.5,0,0,1.5,0\n\ + Scoring Method,positive\n", + ) + .unwrap(); + + let question = parse_question(&path).unwrap(); + assert_eq!(question.number, 7); + assert_eq!(question.columns.len(), 4, "four rubric columns"); + assert_eq!(question.max_points, 1.5); + assert_eq!(question.keyed(), vec!["C".to_string()]); + assert_eq!(question.rows.len(), 3, "trailing rows are not students"); + assert_eq!(question.scoring_method.as_deref(), Some("positive")); + + let ctx = Context { + course: "BIOSC1540".into(), + term: "2026S".into(), + assessment_id: "exam-4".into(), + date: None, + form: None, + }; + let import = to_responses(&[question], &ctx); + let rows = &import.responses.rows; + assert_eq!(rows.len(), 3); + assert_eq!(rows[0].selected, vec!["C".to_string()]); + assert_eq!(rows[0].correct, Some(true)); + assert_eq!(rows[0].credit, 1.0); + assert_eq!(rows[1].correct, Some(false)); + // A student who marked nothing left it blank; that is not a wrong answer. + assert_eq!(rows[2].correct, None); + assert_eq!(rows[2].selected.len(), 0); + + std::fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn rejects_a_file_without_the_rubric_delimiter() { + let dir = std::env::temp_dir().join(format!("cb-gs-bad-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("1.csv"); + std::fs::write(&path, "Name,Score\nAda,1\n").unwrap(); + let err = parse_question(&path).unwrap_err(); + assert!(err.to_string().contains("Submission Time")); + std::fs::remove_dir_all(&dir).ok(); + } +} diff --git a/src/data/responses.rs b/src/data/responses.rs new file mode 100644 index 0000000..cd17671 --- /dev/null +++ b/src/data/responses.rs @@ -0,0 +1,831 @@ +//! The canonical response row. +//! +//! Every grading platform exports a different shape. Gradescope gives one file +//! per question, in wide form, with a column per rubric item. Canvas gives one +//! enormous file per quiz, in wide form, with two columns per question and answer +//! *text* instead of option letters. Neither shape is analyzable. +//! +//! So both are normalized into the long form defined here: one row per student +//! per item. Long form is what item analysis, IRT, and per-objective mastery all +//! want, it survives a question being added or dropped without changing the +//! schema, and it appends cleanly across terms — which is the whole point, since +//! item statistics only become trustworthy once several administrations are +//! pooled. +//! +//! Two fields deserve comment. +//! +//! `credit` is a *fraction* in `0.0..=1.0`, not points. Storing the fraction +//! keeps the response independent of the points an item happened to be worth on +//! one exam, so pooling across administrations that weighted an item differently +//! is still valid. `score` carries the points actually awarded. +//! +//! `student_key` is whatever identifier analysis should group by, and it may be a +//! pseudonym. The real SID lives in `sid`, which is dropped when pseudonymizing. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde::{Deserialize, Serialize}; + +use crate::assessment::AssessmentFile; +use crate::catalog::Catalog; +use crate::date::Date; +use crate::hash::pseudonym; +use crate::taxonomy::Level; +/// One student's response to one item on one administration. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Response { + /// Identifies this administration: course slug, term, and assessment id. + /// Rows from different administrations of the same exam differ here, which is + /// what makes pooled analysis separable again later. + pub administration_id: String, + /// The course code. + pub course: String, + /// The term, e.g. `2026S`. + pub term: String, + /// The assessment id. + pub assessment_id: String, + /// The administration date. + pub date: Option, + /// Which form the student took, when forms were used. + pub form: Option, + + /// The identifier analysis groups by. A pseudonym when pseudonymizing. + pub student_key: String, + /// The institutional student id, absent when pseudonymized. + pub sid: Option, + /// The student's name, absent when pseudonymized. + pub name: Option, + /// The student's email, absent when pseudonymized. + pub email: Option, + /// Section or lab, kept because it is the grouping most likely to reveal a + /// delivery problem rather than a learning one. + pub section: Option, + + /// The question number on the form, which is the join key to the record. + pub item_number: u32, + /// The item's global id, once resolved against an assessment record. + pub item_ref: Option, + /// The item version as administered. + pub item_version: Option, + + /// Option letters the student chose. + pub selected: Vec, + /// Option letters the student eliminated, for elimination-scored items. + pub eliminated: Vec, + /// Whether the response earned full credit. `None` when it cannot be + /// determined, e.g. a blank response on an item with no recorded key. + pub correct: Option, + /// Fraction of the item's points earned, in `0.0..=1.0`. May exceed nothing + /// and may go negative on elimination scoring. + pub credit: f64, + /// Points the item was worth as administered. + pub points_possible: f64, + /// Points awarded, authoritative from the platform where available. + pub score: f64, + /// Seconds spent, when the platform reports it. + pub response_time_seconds: Option, + + /// The item's level, denormalized so analysis need not carry the catalog. + pub level: Option, + /// The item's learning objectives, denormalized for per-objective mastery. + pub learning_objectives: Vec, + /// The item's topics, denormalized. + pub topics: Vec, + /// Whether the item was bonus, and so excluded from the scored total. + pub bonus: bool, + /// Whether the item was dropped after the fact. + pub dropped: bool, +} + +impl Response { + /// Whether this row should count toward scored totals and item statistics. + /// + /// # Returns + /// + /// `true` for a scored, undropped item. + pub fn counts(&self) -> bool { + !self.bonus && !self.dropped + } + + /// The response coded for a dichotomous model. + /// + /// Partial credit is rounded toward the majority: a half-credit response is + /// coded incorrect. IRT here is dichotomous, and pretending otherwise would + /// misstate the model rather than the data. + /// + /// # Returns + /// + /// `Some(true)` for full credit, `Some(false)` for less, `None` when unknown. + pub fn dichotomous(&self) -> Option { + match self.correct { + Some(c) => Some(c), + None if self.credit > 0.0 => Some(self.credit >= 0.999), + None => None, + } + } + + /// The selected options as a comma-joined string, for flat storage. + pub fn selected_joined(&self) -> String { + self.selected.join(",") + } +} + +/// A set of responses plus anything worth telling the user about the ingest. +#[derive(Debug, Clone, Default)] +pub struct ResponseSet { + /// The rows, in ingest order. + pub rows: Vec, + /// Non-fatal problems: unmatched columns, students with no responses, + /// question numbers absent from the assessment record. + pub warnings: Vec, +} + +impl ResponseSet { + /// Creates an empty set. + pub fn new() -> ResponseSet { + ResponseSet::default() + } + + /// Appends another set, keeping its warnings. + /// + /// # Arguments + /// + /// * `other` - the set to absorb. + pub fn absorb(&mut self, other: ResponseSet) { + self.rows.extend(other.rows); + self.warnings.extend(other.warnings); + } + + /// The distinct student keys, in sorted order. + pub fn students(&self) -> Vec { + let set: BTreeSet<&str> = self.rows.iter().map(|r| r.student_key.as_str()).collect(); + set.into_iter().map(|s| s.to_string()).collect() + } + + /// The distinct item numbers that count toward the scored total, sorted. + pub fn scored_items(&self) -> Vec { + let set: BTreeSet = self + .rows + .iter() + .filter(|r| r.counts()) + .map(|r| r.item_number) + .collect(); + set.into_iter().collect() + } + + /// All distinct item numbers, sorted. + pub fn all_items(&self) -> Vec { + let set: BTreeSet = self.rows.iter().map(|r| r.item_number).collect(); + set.into_iter().collect() + } + + /// Every row for one item, in student order. + /// + /// # Arguments + /// + /// * `number` - the question number. + /// + /// # Returns + /// + /// The matching rows. + pub fn for_item(&self, number: u32) -> Vec<&Response> { + let mut v: Vec<&Response> = self + .rows + .iter() + .filter(|r| r.item_number == number) + .collect(); + v.sort_by(|a, b| a.student_key.cmp(&b.student_key)); + v + } + + /// Every row for one student, in item order. + /// + /// # Arguments + /// + /// * `key` - the student key. + /// + /// # Returns + /// + /// The matching rows. + pub fn for_student(&self, key: &str) -> Vec<&Response> { + let mut v: Vec<&Response> = self.rows.iter().filter(|r| r.student_key == key).collect(); + v.sort_by_key(|r| r.item_number); + v + } + + /// Total points a student earned on scored items. + /// + /// # Arguments + /// + /// * `key` - the student key. + /// + /// # Returns + /// + /// The sum of `score` over scored, undropped items. + pub fn scored_total(&self, key: &str) -> f64 { + self.rows + .iter() + .filter(|r| r.student_key == key && r.counts()) + .map(|r| r.score) + .sum() + } + + /// Bonus points a student earned. + pub fn bonus_total(&self, key: &str) -> f64 { + self.rows + .iter() + .filter(|r| r.student_key == key && r.bonus && !r.dropped) + .map(|r| r.score) + .sum() + } + + /// Points available on scored items, taken from the most generous row seen + /// for each item so a student who skipped an item still has a denominator. + pub fn points_available(&self) -> f64 { + let mut per_item: BTreeMap = BTreeMap::new(); + for r in self.rows.iter().filter(|r| r.counts()) { + let e = per_item.entry(r.item_number).or_insert(0.0); + if r.points_possible > *e { + *e = r.points_possible; + } + } + per_item.values().sum() + } + + /// Builds the response matrix for psychometrics. + /// + /// # Arguments + /// + /// * `include_bonus` - whether bonus items belong in the matrix. They + /// normally do not: bonus items are usually hard and optional, so including + /// them inflates the appearance of a low-ability tail. + /// + /// # Returns + /// + /// The matrix, students by items. + pub fn matrix(&self, include_bonus: bool) -> Matrix { + let students = self.students(); + let items: Vec = if include_bonus { + self.all_items() + .into_iter() + .filter(|n| !self.item_dropped(*n)) + .collect() + } else { + self.scored_items() + }; + + let student_index: BTreeMap<&str, usize> = students + .iter() + .enumerate() + .map(|(i, s)| (s.as_str(), i)) + .collect(); + let item_index: BTreeMap = + items.iter().enumerate().map(|(i, n)| (*n, i)).collect(); + + let mut credit = vec![vec![None; items.len()]; students.len()]; + let mut coded = vec![vec![None; items.len()]; students.len()]; + + for r in &self.rows { + let Some(&si) = student_index.get(r.student_key.as_str()) else { + continue; + }; + let Some(&ii) = item_index.get(&r.item_number) else { + continue; + }; + credit[si][ii] = Some(r.credit); + coded[si][ii] = r.dichotomous().map(|c| if c { 1u8 } else { 0u8 }); + } + + Matrix { + students, + items, + credit, + coded, + } + } + + /// Whether every row for an item is marked dropped. + /// + /// # Arguments + /// + /// * `number` - the question number. + /// + /// # Returns + /// + /// `true` when the item was dropped. + pub fn item_dropped(&self, number: u32) -> bool { + let mut any = false; + for r in self.rows.iter().filter(|r| r.item_number == number) { + any = true; + if !r.dropped { + return false; + } + } + any + } + + /// Attaches item metadata from an assessment record and the catalog. + /// + /// Ingest knows question numbers; only the record knows which item a number + /// referred to. Doing this as a separate pass means an export can be parsed + /// and inspected before the record is written, which is the order people + /// actually work in. + /// + /// # Arguments + /// + /// * `record` - the assessment record. + /// * `catalog` - the loaded course, for levels, objectives, and topics. + /// + /// # Returns + /// + /// Warnings for numbers absent from the record and for keys that disagree + /// with the record. + pub fn enrich(&mut self, record: &AssessmentFile, catalog: Option<&Catalog>) -> Vec { + let mut warnings = Vec::new(); + let mut unmatched: BTreeSet = BTreeSet::new(); + let default_points = catalog + .map(|c| c.course.policy.points_per_item) + .unwrap_or(1.0); + + for r in &mut self.rows { + let Some(p) = record.placement(r.item_number) else { + unmatched.insert(r.item_number); + continue; + }; + r.item_ref = Some(p.item.clone()); + r.item_version = p.version; + r.bonus = r.bonus || p.bonus; + r.dropped = r.dropped || p.dropped; + if let Some(points) = p.points { + // The record is authoritative for points as administered; the + // export sometimes carries a stale maximum. + if (points - r.points_possible).abs() > 1e-9 && r.points_possible > 0.0 { + let ratio = r.credit; + r.points_possible = points; + r.score = ratio * points; + } + if r.points_possible == 0.0 { + r.points_possible = points; + } + } + + if let Some(cat) = catalog { + if let Some(entry) = cat.get(&p.item) { + r.level = Some(entry.item.level); + r.learning_objectives = if p.learning_objectives.is_empty() { + entry.item.learning_objectives.clone() + } else { + p.learning_objectives.clone() + }; + r.topics = entry.item.topics.clone(); + if r.points_possible == 0.0 && !p.bonus { + r.points_possible = entry.item.points(default_points); + } + } + } else { + r.level = p.level; + r.learning_objectives = p.learning_objectives.clone(); + } + + // Apply the record's credit overrides, which is how a decision to + // award partial credit after the fact becomes visible in analysis. + if !p.credit_overrides.is_empty() && r.selected.len() == 1 { + if let Some(over) = p.credit_overrides.get(&r.selected[0]) { + if (*over - r.credit).abs() > 1e-9 { + r.credit = *over; + r.score = *over * r.points_possible; + r.correct = Some(*over >= 0.999); + } + } + } + } + + if !unmatched.is_empty() { + let list: Vec = unmatched.iter().map(|n| n.to_string()).collect(); + warnings.push(format!( + "question number(s) {} appear in the export but not in the assessment record; \ + they will be analyzed without item metadata", + list.join(", ") + )); + } + self.warnings.extend(warnings.clone()); + warnings + } + + /// Replaces identifiers with keyed pseudonyms. + /// + /// The salt must be kept outside the course repository. Hashing a seven-digit + /// student id without a key is not de-identification: the whole space can be + /// enumerated in under a second, so anyone with the file recovers every id. + /// + /// # Arguments + /// + /// * `salt` - the HMAC key. + pub fn pseudonymize(&mut self, salt: &[u8]) { + for r in &mut self.rows { + let source = r + .sid + .clone() + .or_else(|| r.email.clone()) + .unwrap_or_else(|| r.student_key.clone()); + r.student_key = pseudonym(salt, &source, 12); + r.sid = None; + r.name = None; + r.email = None; + } + } + + /// The administration ids present, sorted. + pub fn administrations(&self) -> Vec { + let set: BTreeSet<&str> = self + .rows + .iter() + .map(|r| r.administration_id.as_str()) + .collect(); + set.into_iter().map(|s| s.to_string()).collect() + } +} + +/// Builds an administration id. +/// +/// # Arguments +/// +/// * `course` - the course code. +/// * `term` - the term. +/// * `assessment` - the assessment id. +/// +/// # Returns +/// +/// A stable identifier such as `BIOSC1540/2026S/exam-4`. +pub fn administration_id(course: &str, term: &str, assessment: &str) -> String { + format!("{course}/{term}/{assessment}") +} + +/// A response matrix, students by items. +#[derive(Debug, Clone)] +pub struct Matrix { + /// Student keys, one per row. + pub students: Vec, + /// Question numbers, one per column. + pub items: Vec, + /// Credit fractions; `None` for a missing response. + pub credit: Vec>>, + /// Dichotomous codes; `None` for a missing response. + pub coded: Vec>>, +} + +impl Matrix { + /// The number of examinees. + pub fn n_students(&self) -> usize { + self.students.len() + } + + /// The number of items. + pub fn n_items(&self) -> usize { + self.items.len() + } + + /// Per-student total of credit fractions, treating missing as zero. + /// + /// # Returns + /// + /// One total per student, in row order. + pub fn totals(&self) -> Vec { + self.credit + .iter() + .map(|row| row.iter().map(|c| c.unwrap_or(0.0)).sum()) + .collect() + } + + /// Per-student count of items answered correctly. + pub fn correct_counts(&self) -> Vec { + self.coded + .iter() + .map(|row| row.iter().map(|c| c.unwrap_or(0) as f64).sum()) + .collect() + } + + /// The column for one item, by index. + /// + /// # Arguments + /// + /// * `j` - the column index. + /// + /// # Returns + /// + /// The dichotomous codes down that column. + pub fn column(&self, j: usize) -> Vec> { + self.coded.iter().map(|row| row[j]).collect() + } + + /// Whether the matrix has enough data to analyze at all. + /// + /// # Returns + /// + /// `true` when there is at least one student and one item. + pub fn is_analyzable(&self) -> bool { + self.n_students() > 0 && self.n_items() > 0 + } +} + +/// A flat record for CSV and Parquet storage. +/// +/// The nested vectors on [`Response`] do not survive a columnar format, so they +/// are joined here. This is the schema written to disk, and it is deliberately +/// wide and denormalized: it is a fact table, meant to be appended to and read +/// by other tools, not a normalized database. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FlatResponse { + /// Identifies the administration. + pub administration_id: String, + /// The course code. + pub course: String, + /// The term. + pub term: String, + /// The assessment id. + pub assessment_id: String, + /// The date as `YYYY-MM-DD`, empty when unknown. + pub date: String, + /// The form id, empty when there were no forms. + pub form: String, + /// The grouping key. + pub student_key: String, + /// The student id, empty when pseudonymized. + pub sid: String, + /// The student email, empty when pseudonymized. + pub email: String, + /// The section, empty when unknown. + pub section: String, + /// The question number. + pub item_number: u32, + /// The item's global id, empty when unresolved. + pub item_ref: String, + /// The item version, 0 when unknown. + pub item_version: u32, + /// Comma-joined selected letters. + pub selected: String, + /// Comma-joined eliminated letters. + pub eliminated: String, + /// `1`, `0`, or empty when unknown. + pub correct: String, + /// Credit fraction. + pub credit: f64, + /// Points possible. + pub points_possible: f64, + /// Points awarded. + pub score: f64, + /// Seconds spent, empty when unknown. + pub response_time_seconds: String, + /// The level code 1..5, 0 when unknown. + pub level: u8, + /// Comma-joined objective ids. + pub learning_objectives: String, + /// Comma-joined topics. + pub topics: String, + /// Whether the item was bonus. + pub bonus: bool, + /// Whether the item was dropped. + pub dropped: bool, +} + +impl FlatResponse { + /// Flattens a response. + /// + /// # Arguments + /// + /// * `r` - the response. + /// + /// # Returns + /// + /// The flat record. + pub fn from_response(r: &Response) -> FlatResponse { + FlatResponse { + administration_id: r.administration_id.clone(), + course: r.course.clone(), + term: r.term.clone(), + assessment_id: r.assessment_id.clone(), + date: r.date.map(|d| d.to_string()).unwrap_or_default(), + form: r.form.clone().unwrap_or_default(), + student_key: r.student_key.clone(), + sid: r.sid.clone().unwrap_or_default(), + email: r.email.clone().unwrap_or_default(), + section: r.section.clone().unwrap_or_default(), + item_number: r.item_number, + item_ref: r.item_ref.clone().unwrap_or_default(), + item_version: r.item_version.unwrap_or(0), + selected: r.selected.join(","), + eliminated: r.eliminated.join(","), + correct: match r.correct { + Some(true) => "1".to_string(), + Some(false) => "0".to_string(), + None => String::new(), + }, + credit: r.credit, + points_possible: r.points_possible, + score: r.score, + response_time_seconds: r + .response_time_seconds + .map(|s| format!("{s:.1}")) + .unwrap_or_default(), + level: r.level.map(|l| l.code()).unwrap_or(0), + learning_objectives: r.learning_objectives.join(","), + topics: r.topics.join(","), + bonus: r.bonus, + dropped: r.dropped, + } + } + + /// Rebuilds a response from its flat form. + /// + /// # Returns + /// + /// The response. Unparseable optional fields become `None` rather than + /// failing the read, because a hand-edited CSV should still load. + pub fn to_response(&self) -> Response { + let split = |s: &str| -> Vec { + s.split(',') + .map(|p| p.trim()) + .filter(|p| !p.is_empty()) + .map(|p| p.to_string()) + .collect() + }; + Response { + administration_id: self.administration_id.clone(), + course: self.course.clone(), + term: self.term.clone(), + assessment_id: self.assessment_id.clone(), + date: self.date.parse().ok(), + form: none_if_empty(&self.form), + student_key: self.student_key.clone(), + sid: none_if_empty(&self.sid), + name: None, + email: none_if_empty(&self.email), + section: none_if_empty(&self.section), + item_number: self.item_number, + item_ref: none_if_empty(&self.item_ref), + item_version: if self.item_version == 0 { + None + } else { + Some(self.item_version) + }, + selected: split(&self.selected), + eliminated: split(&self.eliminated), + correct: match self.correct.as_str() { + "1" | "true" => Some(true), + "0" | "false" => Some(false), + _ => None, + }, + credit: self.credit, + points_possible: self.points_possible, + score: self.score, + response_time_seconds: self.response_time_seconds.parse().ok(), + level: Level::from_code(self.level), + learning_objectives: split(&self.learning_objectives), + topics: split(&self.topics), + bonus: self.bonus, + dropped: self.dropped, + } + } +} + +/// `None` for an empty string, `Some` otherwise. +fn none_if_empty(s: &str) -> Option { + if s.is_empty() { + None + } else { + Some(s.to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn row(student: &str, number: u32, credit: f64) -> Response { + Response { + administration_id: "C/2026S/e1".into(), + course: "C".into(), + term: "2026S".into(), + assessment_id: "e1".into(), + date: None, + form: None, + student_key: student.into(), + sid: Some(format!("sid-{student}")), + name: None, + email: None, + section: None, + item_number: number, + item_ref: None, + item_version: None, + selected: vec!["A".into()], + eliminated: vec![], + correct: Some(credit >= 0.999), + credit, + points_possible: 2.0, + score: credit * 2.0, + response_time_seconds: None, + level: None, + learning_objectives: vec![], + topics: vec![], + bonus: false, + dropped: false, + } + } + + #[test] + fn matrix_is_students_by_items() { + let mut set = ResponseSet::new(); + set.rows.push(row("s1", 1, 1.0)); + set.rows.push(row("s1", 2, 0.0)); + set.rows.push(row("s2", 1, 1.0)); + set.rows.push(row("s2", 2, 1.0)); + + let m = set.matrix(false); + assert_eq!(m.n_students(), 2); + assert_eq!(m.n_items(), 2); + assert_eq!(m.coded[0], vec![Some(1), Some(0)]); + assert_eq!(m.correct_counts(), vec![1.0, 2.0]); + } + + #[test] + fn missing_responses_stay_missing() { + let mut set = ResponseSet::new(); + set.rows.push(row("s1", 1, 1.0)); + set.rows.push(row("s2", 2, 1.0)); + let m = set.matrix(false); + // s1 never answered item 2, so that cell is absent rather than zero. + assert_eq!(m.coded[0][1], None); + assert_eq!(m.coded[1][0], None); + // Totals treat missing as zero, which is right for scoring. + assert_eq!(m.totals(), vec![1.0, 1.0]); + } + + #[test] + fn bonus_items_are_excluded_by_default() { + let mut set = ResponseSet::new(); + set.rows.push(row("s1", 1, 1.0)); + let mut bonus = row("s1", 2, 1.0); + bonus.bonus = true; + set.rows.push(bonus); + + assert_eq!(set.matrix(false).n_items(), 1); + assert_eq!(set.matrix(true).n_items(), 2); + assert_eq!(set.scored_total("s1"), 2.0); + assert_eq!(set.bonus_total("s1"), 2.0); + } + + #[test] + fn dropped_items_leave_the_matrix() { + let mut set = ResponseSet::new(); + let mut r = row("s1", 1, 0.0); + r.dropped = true; + set.rows.push(r); + set.rows.push(row("s1", 2, 1.0)); + assert_eq!(set.matrix(false).items, vec![2]); + assert!(set.item_dropped(1)); + assert!(!set.item_dropped(2)); + } + + #[test] + fn partial_credit_codes_as_incorrect_for_irt() { + let mut r = row("s1", 1, 0.5); + r.correct = None; + assert_eq!(r.dichotomous(), Some(false)); + r.credit = 1.0; + assert_eq!(r.dichotomous(), Some(true)); + } + + #[test] + fn pseudonymizing_removes_identifiers() { + let mut set = ResponseSet::new(); + set.rows.push(row("s1", 1, 1.0)); + let before = set.rows[0].student_key.clone(); + set.pseudonymize(b"secret-salt"); + assert_ne!(set.rows[0].student_key, before); + assert!(set.rows[0].student_key.starts_with("s-")); + assert!(set.rows[0].sid.is_none()); + assert!(set.rows[0].name.is_none()); + } + + #[test] + fn flattening_round_trips() { + let r = row("s1", 3, 0.5); + let flat = FlatResponse::from_response(&r); + let back = flat.to_response(); + assert_eq!(back.student_key, "s1"); + assert_eq!(back.item_number, 3); + assert_eq!(back.credit, 0.5); + assert_eq!(back.selected, vec!["A".to_string()]); + assert_eq!(back.correct, Some(false)); + } + + #[test] + fn points_available_uses_the_largest_seen_per_item() { + let mut set = ResponseSet::new(); + set.rows.push(row("s1", 1, 1.0)); + let mut r = row("s2", 1, 1.0); + r.points_possible = 3.0; + set.rows.push(r); + assert_eq!(set.points_available(), 3.0); + } +} diff --git a/src/data/store.rs b/src/data/store.rs new file mode 100644 index 0000000..d6ef024 --- /dev/null +++ b/src/data/store.rs @@ -0,0 +1,665 @@ +//! Where response data lives on disk. +//! +//! One file per administration, under `data/`, named after the administration id. +//! Not one big file, because an exam's responses are written once and then only +//! read: separate files mean re-ingesting Exam 4 cannot corrupt Exam 3, and a +//! term's data can be archived or excluded by moving files rather than filtering +//! rows. +//! +//! Parquet is the default format. It is columnar, typed, compressed, and readable +//! by pandas, polars, R, and DuckDB without an export step, which matters because +//! the point of storing this data is to still be able to analyze it in five years +//! with whatever tool exists then. +//! +//! CSV is the fallback, and it is a real fallback rather than a degraded mode: +//! `--no-default-features` builds the entire tool with CSV storage and loses +//! nothing but file size and read speed. Committing to a format you cannot open +//! without the right library version is how course data gets lost. + +use std::path::{Path, PathBuf}; + +use crate::error::{Error, Result}; +use crate::responses::{FlatResponse, Response, ResponseSet}; + +/// A storage format. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Format { + /// Apache Parquet, the default. + Parquet, + /// Comma-separated values. + Csv, +} + +impl Format { + /// The file extension, without a dot. + pub fn extension(self) -> &'static str { + match self { + Format::Parquet => "parquet", + Format::Csv => "csv", + } + } + + /// The format compiled in as the default. + /// + /// # Returns + /// + /// Parquet when the `parquet` feature is on, CSV otherwise. + pub fn preferred() -> Format { + #[cfg(feature = "parquet")] + { + Format::Parquet + } + #[cfg(not(feature = "parquet"))] + { + Format::Csv + } + } + + /// Whether this format can be written by the current build. + pub fn is_available(self) -> bool { + match self { + Format::Csv => true, + Format::Parquet => cfg!(feature = "parquet"), + } + } + + /// The format implied by a file extension. + /// + /// # Arguments + /// + /// * `path` - the file path. + /// + /// # Returns + /// + /// The format, or `None` when the extension is not one we write. + pub fn from_path(path: &Path) -> Option { + match path.extension().and_then(|e| e.to_str()) { + Some("parquet") => Some(Format::Parquet), + Some("csv") => Some(Format::Csv), + _ => None, + } + } +} + +/// The response store rooted at a course's `data/` directory. +#[derive(Debug, Clone)] +pub struct Store { + /// The data directory. + pub dir: PathBuf, + /// The format to write. + pub format: Format, +} + +impl Store { + /// Opens a store, creating the directory if needed. + /// + /// # Arguments + /// + /// * `dir` - the data directory. + /// + /// # Returns + /// + /// The store, writing the preferred available format. + /// + /// # Errors + /// + /// Returns [`Error::Io`] when the directory cannot be created. + pub fn open(dir: impl Into) -> Result { + let dir = dir.into(); + std::fs::create_dir_all(&dir).map_err(|e| Error::io(&dir, e))?; + Ok(Store { + dir, + format: Format::preferred(), + }) + } + + /// Sets the format to write. + /// + /// # Arguments + /// + /// * `format` - the format. + /// + /// # Returns + /// + /// The store, for chaining. + /// + /// # Errors + /// + /// Returns [`Error::FeatureDisabled`] when the format was compiled out. + pub fn with_format(mut self, format: Format) -> Result { + if !format.is_available() { + return Err(Error::FeatureDisabled("Parquet", "parquet")); + } + self.format = format; + Ok(self) + } + + /// The path for an administration's responses. + /// + /// # Arguments + /// + /// * `administration_id` - the administration id. + /// + /// # Returns + /// + /// The path, with slashes in the id replaced so it is one file. + pub fn path_for(&self, administration_id: &str) -> PathBuf { + self.dir.join(format!( + "{}.{}", + sanitize(administration_id), + self.format.extension() + )) + } + + /// Writes one administration's responses, replacing any existing file. + /// + /// Replacing rather than appending is deliberate. Re-ingesting after a + /// regrade should produce the corrected data, not two contradictory copies of + /// the same student's response, and there is no way to tell those apart later. + /// + /// # Arguments + /// + /// * `set` - the responses, which must all share one administration id. + /// + /// # Returns + /// + /// The paths written, one per administration found in the set. + /// + /// # Errors + /// + /// Returns [`Error::Io`] on a write failure and [`Error::FeatureDisabled`] + /// when writing Parquet without the feature. + pub fn write(&self, set: &ResponseSet) -> Result> { + let mut written = Vec::new(); + for admin in set.administrations() { + let rows: Vec = set + .rows + .iter() + .filter(|r| r.administration_id == admin) + .map(FlatResponse::from_response) + .collect(); + let path = self.path_for(&admin); + match self.format { + Format::Csv => write_csv(&path, &rows)?, + Format::Parquet => write_parquet(&path, &rows)?, + } + written.push(path); + } + Ok(written) + } + + /// Reads one administration's responses. + /// + /// # Arguments + /// + /// * `administration_id` - the administration id. + /// + /// # Returns + /// + /// The responses. + /// + /// # Errors + /// + /// Returns [`Error::Io`] when the file is missing. + pub fn read(&self, administration_id: &str) -> Result { + // Accept either format regardless of what this build prefers, so a repo + // written by a Parquet build is still readable by a lean one where the + // CSV happens to exist, and vice versa. + let stem = sanitize(administration_id); + for format in [self.format, Format::Parquet, Format::Csv] { + let path = self.dir.join(format!("{stem}.{}", format.extension())); + if path.exists() { + return read_path(&path); + } + } + Err(Error::Other(format!( + "no stored responses for `{administration_id}` in {}", + self.dir.display() + ))) + } + + /// Every stored data file, sorted. + /// + /// # Returns + /// + /// The paths. + /// + /// # Errors + /// + /// Returns [`Error::Io`] when the directory cannot be listed. + pub fn files(&self) -> Result> { + if !self.dir.exists() { + return Ok(Vec::new()); + } + let mut out: Vec = std::fs::read_dir(&self.dir) + .map_err(|e| Error::io(&self.dir, e))? + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .filter(|p| Format::from_path(p).is_some()) + .collect(); + out.sort(); + Ok(out) + } + + /// Reads every stored administration. + /// + /// This is what pooled item statistics run on: several administrations of the + /// same item, which is the only way the numbers become trustworthy for a class + /// of twenty-five. + /// + /// # Returns + /// + /// All responses, with one file's failure recorded as a warning rather than + /// aborting the rest. + /// + /// # Errors + /// + /// Returns [`Error::Io`] when the directory cannot be listed. + pub fn read_all(&self) -> Result { + let mut set = ResponseSet::new(); + for path in self.files()? { + match read_path(&path) { + Ok(part) => set.absorb(part), + Err(e) => set + .warnings + .push(format!("skipping {}: {e}", path.display())), + } + } + Ok(set) + } + + /// Reads every administration of one assessment, across terms. + /// + /// # Arguments + /// + /// * `assessment_id` - the assessment id to match. + /// + /// # Returns + /// + /// The matching responses. + /// + /// # Errors + /// + /// Returns [`Error::Io`] when the directory cannot be listed. + pub fn read_assessment(&self, assessment_id: &str) -> Result { + let all = self.read_all()?; + let mut set = ResponseSet::new(); + set.warnings = all.warnings; + set.rows = all + .rows + .into_iter() + .filter(|r| r.assessment_id == assessment_id) + .collect(); + Ok(set) + } +} + +/// Reads a data file, choosing the reader by extension. +/// +/// # Arguments +/// +/// * `path` - the file. +/// +/// # Returns +/// +/// The responses. +/// +/// # Errors +/// +/// Returns [`Error::Other`] for an unrecognized extension and +/// [`Error::FeatureDisabled`] for Parquet without the feature. +pub fn read_path(path: &Path) -> Result { + match Format::from_path(path) { + Some(Format::Csv) => read_csv(path), + Some(Format::Parquet) => read_parquet(path), + None => Err(Error::Other(format!( + "{} is not a response file; expected a .parquet or .csv", + path.display() + ))), + } +} + +/// Writes flat responses as CSV. +/// +/// # Arguments +/// +/// * `path` - the destination. +/// * `rows` - the rows. +/// +/// # Errors +/// +/// Returns [`Error::Csv`] on a serialization failure. +fn write_csv(path: &Path, rows: &[FlatResponse]) -> Result<()> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|e| Error::io(parent, e))?; + } + let mut w = csv::Writer::from_path(path).map_err(|e| Error::Csv { + path: path.to_path_buf(), + source: e, + })?; + for r in rows { + w.serialize(r).map_err(|e| Error::Csv { + path: path.to_path_buf(), + source: e, + })?; + } + w.flush().map_err(|e| Error::io(path, e))?; + Ok(()) +} + +/// Reads flat responses from CSV. +/// +/// # Arguments +/// +/// * `path` - the file. +/// +/// # Returns +/// +/// The responses. +/// +/// # Errors +/// +/// Returns [`Error::Csv`] on a parse failure. +fn read_csv(path: &Path) -> Result { + let mut r = csv::Reader::from_path(path).map_err(|e| Error::Csv { + path: path.to_path_buf(), + source: e, + })?; + let mut set = ResponseSet::new(); + for rec in r.deserialize::() { + let flat = rec.map_err(|e| Error::Csv { + path: path.to_path_buf(), + source: e, + })?; + set.rows.push(flat.to_response()); + } + Ok(set) +} + +/// Writes flat responses as Parquet. +/// +/// # Arguments +/// +/// * `path` - the destination. +/// * `rows` - the rows. +/// +/// # Errors +/// +/// Returns [`Error::FeatureDisabled`] when the feature is off. +#[cfg(feature = "parquet")] +fn write_parquet(path: &Path, rows: &[FlatResponse]) -> Result<()> { + crate::store_parquet::write(path, rows) +} + +/// Stub for builds without Parquet support. +#[cfg(not(feature = "parquet"))] +fn write_parquet(_path: &Path, _rows: &[FlatResponse]) -> Result<()> { + Err(Error::FeatureDisabled("Parquet", "parquet")) +} + +/// Reads flat responses from Parquet. +/// +/// # Arguments +/// +/// * `path` - the file. +/// +/// # Returns +/// +/// The responses. +/// +/// # Errors +/// +/// Returns [`Error::FeatureDisabled`] when the feature is off. +#[cfg(feature = "parquet")] +fn read_parquet(path: &Path) -> Result { + let rows = crate::store_parquet::read(path)?; + let mut set = ResponseSet::new(); + set.rows = rows.iter().map(|f| f.to_response()).collect(); + Ok(set) +} + +/// Stub for builds without Parquet support. +#[cfg(not(feature = "parquet"))] +fn read_parquet(path: &Path) -> Result { + Err(Error::Other(format!( + "{} is a Parquet file, but this build has Parquet support compiled out; \ + rebuild with `--features parquet`, or re-ingest with `--format csv`", + path.display() + ))) +} + +/// Makes an administration id usable as a file name. +/// +/// # Arguments +/// +/// * `s` - the id. +/// +/// # Returns +/// +/// The sanitized stem. +pub fn sanitize(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut last_sep = false; + for ch in s.chars() { + if ch.is_ascii_alphanumeric() || ch == '-' || ch == '.' { + out.push(ch.to_ascii_lowercase()); + last_sep = false; + } else if !last_sep { + out.push('_'); + last_sep = true; + } + } + let trimmed = out.trim_matches('_').to_string(); + if trimmed.is_empty() { + "responses".to_string() + } else { + trimmed + } +} + +/// Exports responses to an arbitrary path, in the format its extension implies. +/// +/// This exists so `export` is a separate verb from `ingest`: the store is the +/// system of record, and handing a colleague a CSV should not change it. +/// +/// # Arguments +/// +/// * `path` - the destination. +/// * `set` - the responses. +/// +/// # Errors +/// +/// Returns [`Error::Other`] for an unrecognized extension. +pub fn export(path: &Path, set: &ResponseSet) -> Result<()> { + let rows: Vec = set.rows.iter().map(FlatResponse::from_response).collect(); + match Format::from_path(path) { + Some(Format::Csv) => write_csv(path, &rows), + Some(Format::Parquet) => write_parquet(path, &rows), + None => Err(Error::Other(format!( + "cannot tell what format {} should be; use a .csv or .parquet extension", + path.display() + ))), + } +} + +/// Summarizes what is in the store, for `coursebank data list`. +#[derive(Debug, Clone)] +pub struct StoredSummary { + /// The administration id. + pub administration_id: String, + /// The file. + pub path: PathBuf, + /// How many response rows. + pub rows: usize, + /// How many distinct students. + pub students: usize, + /// How many distinct items. + pub items: usize, +} + +/// Summarizes every stored administration. +/// +/// # Arguments +/// +/// * `store` - the store. +/// +/// # Returns +/// +/// One summary per file, sorted by path. +/// +/// # Errors +/// +/// Returns [`Error::Io`] when the directory cannot be listed. +pub fn summarize(store: &Store) -> Result> { + let mut out = Vec::new(); + for path in store.files()? { + let set = match read_path(&path) { + Ok(s) => s, + Err(_) => continue, + }; + let admin = set.administrations().first().cloned().unwrap_or_else(|| { + path.file_stem() + .unwrap_or_default() + .to_string_lossy() + .into() + }); + out.push(StoredSummary { + administration_id: admin, + path, + rows: set.rows.len(), + students: set.students().len(), + items: set.all_items().len(), + }); + } + Ok(out) +} + +/// Groups responses by administration. +/// +/// # Arguments +/// +/// * `set` - the responses. +/// +/// # Returns +/// +/// One set per administration, keyed by id. +pub fn split_by_administration( + set: &ResponseSet, +) -> std::collections::BTreeMap> { + let mut out: std::collections::BTreeMap> = Default::default(); + for r in &set.rows { + out.entry(r.administration_id.clone()).or_default().push(r); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::responses::Response; + + fn row(student: &str, number: u32) -> Response { + Response { + administration_id: "BIOSC1540/2026S/exam-4".into(), + course: "BIOSC1540".into(), + term: "2026S".into(), + assessment_id: "exam-4".into(), + date: None, + form: None, + student_key: student.into(), + sid: None, + name: None, + email: None, + section: None, + item_number: number, + item_ref: Some("bank::q-x-001".into()), + item_version: Some(2), + selected: vec!["C".into()], + eliminated: vec![], + correct: Some(true), + credit: 1.0, + points_possible: 1.5, + score: 1.5, + response_time_seconds: None, + level: None, + learning_objectives: vec!["lo-a".into()], + topics: vec![], + bonus: false, + dropped: false, + } + } + + #[test] + fn sanitizes_administration_ids() { + assert_eq!(sanitize("BIOSC1540/2026S/exam-4"), "biosc1540_2026s_exam-4"); + assert_eq!(sanitize("///"), "responses"); + // Runs of separators collapse rather than stacking underscores. + assert_eq!(sanitize("a // b"), "a_b"); + } + + #[test] + fn csv_round_trips_through_the_store() { + let dir = std::env::temp_dir().join(format!("cb-store-{}", std::process::id())); + std::fs::remove_dir_all(&dir).ok(); + let store = Store::open(&dir).unwrap().with_format(Format::Csv).unwrap(); + + let mut set = ResponseSet::new(); + set.rows.push(row("s1", 1)); + set.rows.push(row("s2", 1)); + let written = store.write(&set).unwrap(); + assert_eq!(written.len(), 1); + assert!(written[0].exists()); + + let back = store.read("BIOSC1540/2026S/exam-4").unwrap(); + assert_eq!(back.rows.len(), 2); + assert_eq!(back.rows[0].item_ref.as_deref(), Some("bank::q-x-001")); + assert_eq!(back.rows[0].selected, vec!["C".to_string()]); + assert_eq!(back.rows[0].learning_objectives, vec!["lo-a".to_string()]); + + let all = store.read_all().unwrap(); + assert_eq!(all.rows.len(), 2); + + let summaries = summarize(&store).unwrap(); + assert_eq!(summaries.len(), 1); + assert_eq!(summaries[0].students, 2); + assert_eq!(summaries[0].items, 1); + + std::fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn rewriting_replaces_rather_than_duplicates() { + let dir = std::env::temp_dir().join(format!("cb-store-rw-{}", std::process::id())); + std::fs::remove_dir_all(&dir).ok(); + let store = Store::open(&dir).unwrap().with_format(Format::Csv).unwrap(); + + let mut set = ResponseSet::new(); + set.rows.push(row("s1", 1)); + store.write(&set).unwrap(); + store.write(&set).unwrap(); + + assert_eq!(store.read_all().unwrap().rows.len(), 1, "no duplicates"); + std::fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn missing_administrations_report_clearly() { + let dir = std::env::temp_dir().join(format!("cb-store-miss-{}", std::process::id())); + std::fs::remove_dir_all(&dir).ok(); + let store = Store::open(&dir).unwrap().with_format(Format::Csv).unwrap(); + let err = store.read("nope").unwrap_err(); + assert!(err.to_string().contains("no stored responses")); + std::fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn format_extensions_round_trip() { + assert_eq!( + Format::from_path(Path::new("a/b.parquet")), + Some(Format::Parquet) + ); + assert_eq!(Format::from_path(Path::new("a/b.csv")), Some(Format::Csv)); + assert_eq!(Format::from_path(Path::new("a/b.yaml")), None); + assert!(Format::Csv.is_available()); + } +} diff --git a/src/data/store_parquet.rs b/src/data/store_parquet.rs new file mode 100644 index 0000000..6109d33 --- /dev/null +++ b/src/data/store_parquet.rs @@ -0,0 +1,387 @@ +//! Parquet reading and writing, isolated so the rest of the crate never touches +//! Arrow types. +//! +//! Every use of `arrow` and `parquet` in this crate is in this file. That is +//! deliberate: those two crates move fast and release breaking versions together, +//! and confining them to one module means a version bump is a single-file edit +//! rather than a refactor. It also means `--no-default-features` compiles the +//! whole tool without them. +//! +//! The schema mirrors [`FlatResponse`] field for field. Optional values are +//! written with the same sentinels the CSV path uses — empty string, `0` for an +//! unknown level — rather than nulls, so that a Parquet file and a CSV file of +//! the same data load identically in pandas. + +use std::path::Path; +use std::sync::Arc; + +use arrow_array::{ArrayRef, BooleanArray, Float64Array, RecordBatch, StringArray, UInt32Array}; +use arrow_schema::{DataType, Field, Schema}; +use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; +use parquet::arrow::ArrowWriter; +use parquet::basic::Compression; +use parquet::file::properties::WriterProperties; + +use crate::error::{Error, Result}; +use crate::responses::FlatResponse; + +/// The Arrow schema for stored responses. +/// +/// # Returns +/// +/// The schema, whose field order matches [`FlatResponse`]. +pub fn schema() -> Schema { + Schema::new(vec![ + Field::new("administration_id", DataType::Utf8, false), + Field::new("course", DataType::Utf8, false), + Field::new("term", DataType::Utf8, false), + Field::new("assessment_id", DataType::Utf8, false), + Field::new("date", DataType::Utf8, false), + Field::new("form", DataType::Utf8, false), + Field::new("student_key", DataType::Utf8, false), + Field::new("sid", DataType::Utf8, false), + Field::new("email", DataType::Utf8, false), + Field::new("section", DataType::Utf8, false), + Field::new("item_number", DataType::UInt32, false), + Field::new("item_ref", DataType::Utf8, false), + Field::new("item_version", DataType::UInt32, false), + Field::new("selected", DataType::Utf8, false), + Field::new("eliminated", DataType::Utf8, false), + Field::new("correct", DataType::Utf8, false), + Field::new("credit", DataType::Float64, false), + Field::new("points_possible", DataType::Float64, false), + Field::new("score", DataType::Float64, false), + Field::new("response_time_seconds", DataType::Utf8, false), + Field::new("level", DataType::UInt32, false), + Field::new("learning_objectives", DataType::Utf8, false), + Field::new("topics", DataType::Utf8, false), + Field::new("bonus", DataType::Boolean, false), + Field::new("dropped", DataType::Boolean, false), + ]) +} + +/// Builds a record batch from flat responses. +/// +/// # Arguments +/// +/// * `rows` - the rows. +/// +/// # Returns +/// +/// The batch. +/// +/// # Errors +/// +/// Returns [`Error::Other`] when Arrow rejects the column set, which would mean +/// this function and [`schema`] have drifted apart. +fn to_batch(rows: &[FlatResponse]) -> Result { + let s = |f: fn(&FlatResponse) -> &str| -> ArrayRef { + Arc::new(StringArray::from(rows.iter().map(f).collect::>())) + }; + let f64c = |f: fn(&FlatResponse) -> f64| -> ArrayRef { + Arc::new(Float64Array::from(rows.iter().map(f).collect::>())) + }; + let u32c = |f: fn(&FlatResponse) -> u32| -> ArrayRef { + Arc::new(UInt32Array::from(rows.iter().map(f).collect::>())) + }; + let boolc = |f: fn(&FlatResponse) -> bool| -> ArrayRef { + Arc::new(BooleanArray::from( + rows.iter().map(f).collect::>(), + )) + }; + + let columns: Vec = vec![ + s(|r| &r.administration_id), + s(|r| &r.course), + s(|r| &r.term), + s(|r| &r.assessment_id), + s(|r| &r.date), + s(|r| &r.form), + s(|r| &r.student_key), + s(|r| &r.sid), + s(|r| &r.email), + s(|r| &r.section), + u32c(|r| r.item_number), + s(|r| &r.item_ref), + u32c(|r| r.item_version), + s(|r| &r.selected), + s(|r| &r.eliminated), + s(|r| &r.correct), + f64c(|r| r.credit), + f64c(|r| r.points_possible), + f64c(|r| r.score), + s(|r| &r.response_time_seconds), + u32c(|r| r.level as u32), + s(|r| &r.learning_objectives), + s(|r| &r.topics), + boolc(|r| r.bonus), + boolc(|r| r.dropped), + ]; + + RecordBatch::try_new(Arc::new(schema()), columns).map_err(|e| { + Error::Other(format!( + "internal error building a Parquet batch: {e}. This means the Arrow schema and the \ + column builders in store_parquet.rs have drifted apart" + )) + }) +} + +/// Writes flat responses to a Parquet file. +/// +/// # Arguments +/// +/// * `path` - the destination. +/// * `rows` - the rows. +/// +/// # Errors +/// +/// Returns [`Error::Io`] on a write failure and [`Error::Other`] when the +/// Parquet writer rejects the batch. +pub fn write(path: &Path, rows: &[FlatResponse]) -> Result<()> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|e| Error::io(parent, e))?; + } + let batch = to_batch(rows)?; + let file = std::fs::File::create(path).map_err(|e| Error::io(path, e))?; + + // Snappy rather than zstd: it is the format's most universally readable + // codec, and these files are small enough that the ratio difference is + // irrelevant next to being openable by an old pandas. + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + + let mut writer = ArrowWriter::try_new(file, Arc::new(schema()), Some(props)) + .map_err(|e| Error::Other(format!("cannot open {} for Parquet: {e}", path.display())))?; + writer + .write(&batch) + .map_err(|e| Error::Other(format!("cannot write {}: {e}", path.display())))?; + writer + .close() + .map_err(|e| Error::Other(format!("cannot finish {}: {e}", path.display())))?; + Ok(()) +} + +/// Reads flat responses from a Parquet file. +/// +/// # Arguments +/// +/// * `path` - the file. +/// +/// # Returns +/// +/// The rows. +/// +/// # Errors +/// +/// Returns [`Error::Io`] when the file cannot be opened and [`Error::Other`] +/// when its schema does not match what this crate writes. +pub fn read(path: &Path) -> Result> { + let file = std::fs::File::open(path).map_err(|e| Error::io(path, e))?; + let builder = ParquetRecordBatchReaderBuilder::try_new(file) + .map_err(|e| Error::Other(format!("cannot read {} as Parquet: {e}", path.display())))?; + let reader = builder + .build() + .map_err(|e| Error::Other(format!("cannot read {}: {e}", path.display())))?; + + let mut out = Vec::new(); + for batch in reader { + let batch = + batch.map_err(|e| Error::Other(format!("cannot read {}: {e}", path.display())))?; + out.extend(from_batch(&batch, path)?); + } + Ok(out) +} + +/// Converts a record batch back into flat responses. +/// +/// # Arguments +/// +/// * `batch` - the batch. +/// * `path` - the source file, for error messages. +/// +/// # Returns +/// +/// The rows. +/// +/// # Errors +/// +/// Returns [`Error::Other`] when a column is missing or has the wrong type. +fn from_batch(batch: &RecordBatch, path: &Path) -> Result> { + let strings = |name: &str| -> Result<&StringArray> { + batch + .column_by_name(name) + .and_then(|c| c.as_any().downcast_ref::()) + .ok_or_else(|| column_error(name, "string", path)) + }; + let floats = |name: &str| -> Result<&Float64Array> { + batch + .column_by_name(name) + .and_then(|c| c.as_any().downcast_ref::()) + .ok_or_else(|| column_error(name, "float64", path)) + }; + let uints = |name: &str| -> Result<&UInt32Array> { + batch + .column_by_name(name) + .and_then(|c| c.as_any().downcast_ref::()) + .ok_or_else(|| column_error(name, "uint32", path)) + }; + let bools = |name: &str| -> Result<&BooleanArray> { + batch + .column_by_name(name) + .and_then(|c| c.as_any().downcast_ref::()) + .ok_or_else(|| column_error(name, "boolean", path)) + }; + + let administration_id = strings("administration_id")?; + let course = strings("course")?; + let term = strings("term")?; + let assessment_id = strings("assessment_id")?; + let date = strings("date")?; + let form = strings("form")?; + let student_key = strings("student_key")?; + let sid = strings("sid")?; + let email = strings("email")?; + let section = strings("section")?; + let item_number = uints("item_number")?; + let item_ref = strings("item_ref")?; + let item_version = uints("item_version")?; + let selected = strings("selected")?; + let eliminated = strings("eliminated")?; + let correct = strings("correct")?; + let credit = floats("credit")?; + let points_possible = floats("points_possible")?; + let score = floats("score")?; + let response_time_seconds = strings("response_time_seconds")?; + let level = uints("level")?; + let learning_objectives = strings("learning_objectives")?; + let topics = strings("topics")?; + let bonus = bools("bonus")?; + let dropped = bools("dropped")?; + + let mut out = Vec::with_capacity(batch.num_rows()); + for i in 0..batch.num_rows() { + out.push(FlatResponse { + administration_id: administration_id.value(i).to_string(), + course: course.value(i).to_string(), + term: term.value(i).to_string(), + assessment_id: assessment_id.value(i).to_string(), + date: date.value(i).to_string(), + form: form.value(i).to_string(), + student_key: student_key.value(i).to_string(), + sid: sid.value(i).to_string(), + email: email.value(i).to_string(), + section: section.value(i).to_string(), + item_number: item_number.value(i), + item_ref: item_ref.value(i).to_string(), + item_version: item_version.value(i), + selected: selected.value(i).to_string(), + eliminated: eliminated.value(i).to_string(), + correct: correct.value(i).to_string(), + credit: credit.value(i), + points_possible: points_possible.value(i), + score: score.value(i), + response_time_seconds: response_time_seconds.value(i).to_string(), + level: level.value(i) as u8, + learning_objectives: learning_objectives.value(i).to_string(), + topics: topics.value(i).to_string(), + bonus: bonus.value(i), + dropped: dropped.value(i), + }); + } + Ok(out) +} + +/// Builds the error for a missing or mistyped column. +fn column_error(name: &str, expected: &str, path: &Path) -> Error { + Error::Other(format!( + "{} is missing the `{name}` column or it is not {expected}; this file was probably not \ + written by coursebank", + path.display() + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn flat(student: &str, number: u32) -> FlatResponse { + FlatResponse { + administration_id: "C/2026S/e1".into(), + course: "C".into(), + term: "2026S".into(), + assessment_id: "e1".into(), + date: "2026-04-01".into(), + form: String::new(), + student_key: student.into(), + sid: "1234567".into(), + email: String::new(), + section: "L01".into(), + item_number: number, + item_ref: "bank::q-a-001".into(), + item_version: 3, + selected: "C".into(), + eliminated: String::new(), + correct: "1".into(), + credit: 1.0, + points_possible: 1.5, + score: 1.5, + response_time_seconds: "42.0".into(), + level: 3, + learning_objectives: "lo-a,lo-b".into(), + topics: "kinetics".into(), + bonus: false, + dropped: false, + } + } + + #[test] + fn schema_matches_the_column_builders() { + let rows = vec![flat("s1", 1)]; + let batch = to_batch(&rows).expect("schema and builders agree"); + assert_eq!(batch.num_columns(), schema().fields().len()); + assert_eq!(batch.num_rows(), 1); + } + + #[test] + fn round_trips_through_a_file() { + let dir = std::env::temp_dir().join(format!("cb-pq-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("e1.parquet"); + + let rows = vec![flat("s1", 1), flat("s2", 2)]; + write(&path, &rows).unwrap(); + let back = read(&path).unwrap(); + + assert_eq!(back.len(), 2); + assert_eq!(back[0].student_key, "s1"); + assert_eq!(back[1].item_number, 2); + assert_eq!(back[0].learning_objectives, "lo-a,lo-b"); + assert_eq!(back[0].credit, 1.0); + assert_eq!(back[0].level, 3); + assert!(!back[0].bonus); + + std::fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn an_empty_batch_is_writable() { + let dir = std::env::temp_dir().join(format!("cb-pq-empty-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("empty.parquet"); + write(&path, &[]).unwrap(); + assert_eq!(read(&path).unwrap().len(), 0); + std::fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn a_foreign_file_is_rejected_clearly() { + let dir = std::env::temp_dir().join(format!("cb-pq-bad-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("not.parquet"); + std::fs::write(&path, b"this is not parquet").unwrap(); + let err = read(&path).unwrap_err(); + assert!(err.to_string().contains("as Parquet")); + std::fs::remove_dir_all(&dir).ok(); + } +} diff --git a/src/error.rs b/src/error.rs new file mode 100644 index 0000000..e4b8545 --- /dev/null +++ b/src/error.rs @@ -0,0 +1,129 @@ +//! One error type for the whole crate. +//! +//! Every fallible operation returns [`Error`]. Variants carry the path or +//! identifier at fault so a message is actionable without a backtrace. The +//! [`Error::Invalid`] variant carries a list of problems rather than one, +//! because validation is meant to report everything wrong with a file in a +//! single pass instead of making the author fix one issue per run. + +use std::path::PathBuf; + +/// The crate result alias. +pub type Result = std::result::Result; + +/// Anything that can go wrong loading, validating, or transforming course data. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// A file could not be read or written. + #[error("cannot read or write {path}: {source}")] + Io { + /// The offending path. + path: PathBuf, + /// The underlying I/O failure. + source: std::io::Error, + }, + + /// An I/O failure with no natural path to attach. + #[error("i/o error: {0}")] + BareIo(#[from] std::io::Error), + + /// A YAML file did not match the schema. + #[error("{path} is not valid coursebank YAML: {source}")] + Yaml { + /// The offending path. + path: PathBuf, + /// The parse failure, which includes a line and column. + source: serde_yaml_ng::Error, + }, + + /// A JSON file did not parse. + #[error("{path} is not valid JSON: {source}")] + Json { + /// The offending path. + path: PathBuf, + /// The parse failure. + source: serde_json::Error, + }, + + /// A CSV file did not parse. + #[error("{path} is not valid CSV: {source}")] + Csv { + /// The offending path. + path: PathBuf, + /// The parse failure. + source: csv::Error, + }, + + /// A file was structurally fine but semantically wrong. Holds every problem + /// found so one run fixes one file. + #[error("{} problem(s) found:\n{}", .0.len(), format_issues(.0))] + Invalid(Vec), + + /// A reference did not resolve: an unknown objective, lecture, item, or bank. + #[error("unknown {kind} `{id}`{}", context_suffix(.context))] + Unresolved { + /// What kind of thing was referenced, e.g. `"learning objective"`. + kind: &'static str, + /// The identifier that did not resolve. + id: String, + /// Where the dangling reference was found, if known. + context: Option, + }, + + /// The requested selection could not be satisfied from the available items. + #[error("cannot satisfy the blueprint: {0}")] + Infeasible(String), + + /// A date string was not `YYYY-MM-DD`, or was not a real calendar date. + #[error("`{0}` is not a date in YYYY-MM-DD form")] + BadDate(String), + + /// A command-line argument was well-formed but unusable. + #[error("{0}")] + Usage(String), + + /// A capability was compiled out. + #[error("{0} support was not compiled in; rebuild with `--features {1}`")] + FeatureDisabled(&'static str, &'static str), + + /// Something went wrong that does not deserve its own variant. + #[error("{0}")] + Other(String), +} + +impl Error { + /// Wraps an I/O failure with the path that caused it. + pub fn io(path: impl Into, source: std::io::Error) -> Error { + Error::Io { + path: path.into(), + source, + } + } + + /// Builds an [`Error::Other`] from anything displayable. + pub fn other(msg: impl std::fmt::Display) -> Error { + Error::Other(msg.to_string()) + } + + /// Builds an [`Error::Usage`] from anything displayable. + pub fn usage(msg: impl std::fmt::Display) -> Error { + Error::Usage(msg.to_string()) + } +} + +/// Renders an issue list as an indented bullet list. +fn format_issues(issues: &[String]) -> String { + issues + .iter() + .map(|i| format!(" - {i}")) + .collect::>() + .join("\n") +} + +/// Renders the optional context of an unresolved reference. +fn context_suffix(context: &Option) -> String { + match context { + Some(c) => format!(" (referenced by {c})"), + None => String::new(), + } +} diff --git a/src/export.rs b/src/export.rs new file mode 100644 index 0000000..d6007ca --- /dev/null +++ b/src/export.rs @@ -0,0 +1,21 @@ +//! Turning course data into documents. +//! +//! | Module | Produces | For | +//! |:--|:--|:--| +//! | [`qti`] | a QTI 1.2 zip | importing into Canvas | +//! | [`typst`] | `.typ` source | a printed exam, answer key, and bubble sheet | +//! | [`report`] | Markdown and HTML | students, and yourself | +//! +//! [`qti`] and [`typst`] share one rule that is easy to get wrong: a form's answer +//! key must be generated from the same permutation that produced its question +//! paper. Both derive option order from the form's recorded seed rather than +//! storing it, so every export of form B agrees with every other. +//! +//! [`report`] writes two documents with different content, not different tones. The +//! student report answers "what should I do next?" and deliberately omits correct +//! answers, other students' data, and any numeric rank. +//! The instructor report answers "what should I fix?" and holds the item statistics. + +pub mod qti; +pub mod report; +pub mod typst; diff --git a/src/export/qti.rs b/src/export/qti.rs new file mode 100644 index 0000000..a40800e --- /dev/null +++ b/src/export/qti.rs @@ -0,0 +1,686 @@ +//! Exporting an assessment as a Canvas-importable QTI 1.2 package. +//! +//! QTI 1.2 is a fussy, half-abandoned standard, and Canvas reads a particular +//! dialect of it. Two details are worth recording because they are easy to get +//! wrong and produce silent misbehavior rather than an import error. +//! +//! First, question HTML lives inside `` as +//! character data, which means it is escaped once on the way in and unescaped +//! once by Canvas. So `→` is written as `&rarr;`. Skipping that step +//! produces XML that parses and renders as literal `→` to students. +//! +//! Second, identifiers must be stable. Canvas keys re-imports and question banks +//! off them, so a package regenerated after a typo fix should carry the same ids +//! as the original. Every id here is derived by hashing the assessment id +//! together with the item's global id, never from a clock or a counter. +//! +//! Option order comes from the form, so exporting form A and form B of the same +//! assessment gives two packages that ask the same questions in different orders, +//! and the answer keys are guaranteed to agree with what was printed. + +use crate::assessment::{AssessmentFile, Form, ScoringPolicy}; +use crate::catalog::Catalog; +use crate::error::{Error, Result}; +use crate::hash::{hex, sha256}; +use crate::item::Item; +use crate::markup; +use crate::select; +use crate::taxonomy::Format; +use crate::zipfile::ZipBuilder; + +/// The QTI 1.2 namespace. +const QTI_NS: &str = "http://www.imsglobal.org/xsd/ims_qtiasiv1p2"; +/// The schema location Canvas expects alongside it. +const QTI_SCHEMA: &str = "http://www.imsglobal.org/xsd/ims_qtiasiv1p2 \ + http://www.imsglobal.org/xsd/ims_qtiasiv1p2p1.xsd"; +/// The XML Schema instance namespace. +const XSI_NS: &str = "http://www.w3.org/2001/XMLSchema-instance"; +/// The IMS content packaging namespace. +const IMSCP_NS: &str = "http://www.imsglobal.org/xsd/imscp_v1p1"; +/// The IMS metadata namespace. +const IMSMD_NS: &str = "http://www.imsglobal.org/xsd/imsmd_v1p2"; +/// The content packaging schema location. +const IMSCP_SCHEMA: &str = "http://www.imsglobal.org/xsd/imscp_v1p1 imscp_v1p1.xsd \ + http://www.imsglobal.org/xsd/imsmd_v1p2 imsmd_v1p2p2.xsd"; + +// --------------------------------------------------------------------------- +// A very small XML tree +// --------------------------------------------------------------------------- + +/// One XML element. +#[derive(Debug, Clone)] +struct Node { + tag: String, + attrs: Vec<(String, String)>, + text: Option, + children: Vec, +} + +impl Node { + /// Creates an element with no attributes, text, or children. + fn new(tag: &str) -> Node { + Node { + tag: tag.to_string(), + attrs: Vec::new(), + text: None, + children: Vec::new(), + } + } + + /// Adds an attribute, returning self for chaining. + fn attr(mut self, k: &str, v: impl Into) -> Node { + self.attrs.push((k.to_string(), v.into())); + self + } + + /// Sets the element text, returning self for chaining. + fn text(mut self, t: impl Into) -> Node { + self.text = Some(t.into()); + self + } + + /// Appends a child, returning self for chaining. + fn child(mut self, c: Node) -> Node { + self.children.push(c); + self + } + + /// Appends several children, returning self for chaining. + fn children(mut self, cs: Vec) -> Node { + self.children.extend(cs); + self + } + + /// Renders the element and its subtree. + /// + /// # Arguments + /// + /// * `depth` - the indentation level. + /// + /// # Returns + /// + /// Indented XML, newline-terminated. + fn render(&self, depth: usize) -> String { + let pad = " ".repeat(depth); + let mut attrs = String::new(); + for (k, v) in &self.attrs { + attrs.push_str(&format!(" {k}=\"{}\"", escape_attr(v))); + } + + if self.children.is_empty() { + match &self.text { + None => format!("{pad}<{}{attrs}/>\n", self.tag), + Some(t) => format!( + "{pad}<{}{attrs}>{}\n", + self.tag, + escape_text(t), + self.tag + ), + } + } else { + let mut out = format!("{pad}<{}{attrs}>\n", self.tag); + if let Some(t) = &self.text { + out.push_str(&format!("{pad} {}\n", escape_text(t))); + } + for c in &self.children { + out.push_str(&c.render(depth + 1)); + } + out.push_str(&format!("{pad}\n", self.tag)); + out + } + } + + /// Renders a complete document with an XML declaration. + fn document(&self) -> String { + format!( + "\n{}", + self.render(0) + ) + } +} + +/// Escapes text content. +fn escape_text(s: &str) -> String { + s.replace('&', "&") + .replace('<', "<") + .replace('>', ">") +} + +/// Escapes an attribute value. +fn escape_attr(s: &str) -> String { + escape_text(s).replace('"', """) +} + +// --------------------------------------------------------------------------- +// Package construction +// --------------------------------------------------------------------------- + +/// Options for a QTI export. +#[derive(Debug, Clone)] +pub struct QtiOptions { + /// Which form's option order to use. + pub form: Form, + /// Whether to include per-option feedback. Turn it off for a practice quiz + /// you intend to reuse as a graded one, since Canvas shows this feedback + /// immediately. + pub include_feedback: bool, + /// Whether to let Canvas shuffle answers on top of the form's own order. + pub shuffle_in_canvas: bool, + /// Maximum attempts; `-1` for unlimited. + pub attempts: i64, + /// How repeated attempts are scored. + pub scoring_policy: ScoringPolicy, +} + +impl Default for QtiOptions { + fn default() -> QtiOptions { + QtiOptions { + form: Form { + id: "A".to_string(), + seed: 0, + shuffle_items: false, + shuffle_options: false, + }, + include_feedback: true, + shuffle_in_canvas: false, + attempts: 1, + scoring_policy: ScoringPolicy::KeepHighest, + } + } +} + +/// A rendered QTI package, ready to write. +#[derive(Debug, Clone)] +pub struct Package { + /// The name of the quiz XML file inside the archive. + pub quiz_filename: String, + /// The quiz XML. + pub quiz_xml: String, + /// The manifest XML. + pub manifest_xml: String, +} + +impl Package { + /// Writes the package as a Canvas-importable zip. + /// + /// # Arguments + /// + /// * `path` - the destination `.zip` path. + /// + /// # Errors + /// + /// Returns [`Error::Io`] on a write failure. + pub fn write_zip(&self, path: &std::path::Path) -> Result<()> { + let mut z = ZipBuilder::new(); + // Canvas only recognizes the archive as QTI when the manifest sits at the + // root rather than inside a directory. + z.add_text("imsmanifest.xml", &self.manifest_xml); + z.add_text(&self.quiz_filename, &self.quiz_xml); + z.write_to(path) + } +} + +/// Builds a QTI package for an assessment. +/// +/// # Arguments +/// +/// * `catalog` - the loaded course, for resolving items. +/// * `record` - the assessment record. +/// * `opts` - export options. +/// +/// # Returns +/// +/// The rendered package. +/// +/// # Errors +/// +/// Returns [`Error::Unresolved`] when a placement references a missing item, and +/// [`Error::Invalid`] when an item cannot be represented in QTI, such as one with +/// no keyed option. +pub fn build(catalog: &Catalog, record: &AssessmentFile, opts: &QtiOptions) -> Result { + let assessment_id = qti_id(&format!("{}/assessment", record.assessment.id)); + let default_points = catalog.course.policy.points_per_item; + + let mut problems = Vec::new(); + let mut items = Vec::new(); + + for placement in select::layout(record, &opts.form) { + let entry = catalog.require(&placement.item)?; + let item = &entry.item; + if item.key_indices().is_empty() { + problems.push(format!( + "question {} ({}) has no keyed option, so Canvas cannot score it", + placement.number, placement.item + )); + continue; + } + let points = placement + .points + .unwrap_or_else(|| item.points(default_points)); + items.push(build_item( + &record.assessment.id, + &placement.item, + item, + points, + opts, + )); + } + + if !problems.is_empty() { + return Err(Error::Invalid(problems)); + } + + let metadata = vec![ + metadata_field("cc_maxattempts", &opts.attempts.to_string()), + metadata_field("cc_quiz_scoring_policy", opts.scoring_policy.as_str()), + metadata_field( + "cc_shuffle_answers", + if opts.shuffle_in_canvas { + "true" + } else { + "false" + }, + ), + ]; + + let title = if record.forms.len() > 1 { + format!("{} (form {})", record.assessment.title, opts.form.id) + } else { + record.assessment.title.clone() + }; + + let assessment = Node::new("assessment") + .attr("ident", assessment_id.clone()) + .attr("title", title.clone()) + .child(Node::new("qtimetadata").children(metadata)) + .child( + Node::new("section") + .attr("ident", "root_section") + .children(items), + ); + + let root = Node::new("questestinterop") + .attr("xmlns", QTI_NS) + .attr("xmlns:xsi", XSI_NS) + .attr("xsi:schemaLocation", QTI_SCHEMA) + .child(assessment); + + let quiz_filename = format!("{}.xml", slug_filename(&record.assessment.id)); + let manifest = build_manifest( + &quiz_filename, + &assessment_id, + &title, + &record.assessment.id, + ); + + Ok(Package { + quiz_filename, + quiz_xml: root.document(), + manifest_xml: manifest.document(), + }) +} + +/// Builds one `` element. +/// +/// # Arguments +/// +/// * `assessment_id` - salts the generated ids. +/// * `uid` - the item's global id. +/// * `item` - the item. +/// * `points` - points as administered. +/// * `opts` - export options. +/// +/// # Returns +/// +/// The element. +fn build_item(assessment_id: &str, uid: &str, item: &Item, points: f64, opts: &QtiOptions) -> Node { + let order = select::option_order(&opts.form, uid, item.options.len()); + let ordered: Vec<&crate::item::Choice> = order.iter().map(|i| &item.options[*i]).collect(); + + // Option identifiers are numeric, mirroring Canvas's own exports, and are + // derived from the item id so they survive regeneration. + let opt_ids: Vec = ordered + .iter() + .map(|o| short_id(&format!("{assessment_id}/{uid}/{}", o.id))) + .collect(); + + let item_meta = Node::new("itemmetadata").child(Node::new("qtimetadata").children(vec![ + metadata_field("question_type", item.format.qti_type()), + metadata_field("points_possible", &format!("{points:.2}")), + metadata_field("original_answer_ids", &opt_ids.join(",")), + metadata_field("assessment_question_identifierref", &qti_id(uid)), + ])); + + let cardinality = if item.format == Format::MultipleResponse { + "Multiple" + } else { + "Single" + }; + + let labels: Vec = ordered + .iter() + .zip(opt_ids.iter()) + .map(|(o, id)| { + Node::new("response_label") + .attr("ident", id.clone()) + .child(mattext(&markup::to_html(&o.text))) + }) + .collect(); + + let presentation = Node::new("presentation") + .child(mattext(&format!( + "
{}
", + markup::to_html(&item.stem) + ))) + .child( + Node::new("response_lid") + .attr("ident", "response1") + .attr("rcardinality", cardinality) + .child(Node::new("render_choice").children(labels)), + ); + + // --- response processing --- + let mut resprocessing = Node::new("resprocessing").child( + Node::new("outcomes").child( + Node::new("decvar") + .attr("maxvalue", "100") + .attr("minvalue", "0") + .attr("varname", "SCORE") + .attr("vartype", "Decimal"), + ), + ); + + // A pass-through condition per option, so choosing anything triggers its + // feedback. `continue="Yes"` is what allows scoring to be evaluated after. + if opts.include_feedback { + for (o, id) in ordered.iter().zip(opt_ids.iter()) { + if o.student_text().is_none() { + continue; + } + resprocessing = resprocessing.child( + Node::new("respcondition") + .attr("continue", "Yes") + .child( + Node::new("conditionvar").child( + Node::new("varequal") + .attr("respident", "response1") + .text(id.clone()), + ), + ) + .child( + Node::new("displayfeedback") + .attr("feedbacktype", "Response") + .attr("linkrefid", format!("{id}_fb")), + ), + ); + } + } + + let correct: Vec<&String> = ordered + .iter() + .zip(opt_ids.iter()) + .filter(|(o, _)| o.correct) + .map(|(_, id)| id) + .collect(); + let incorrect: Vec<&String> = ordered + .iter() + .zip(opt_ids.iter()) + .filter(|(o, _)| !o.correct) + .map(|(_, id)| id) + .collect(); + + let condition = if item.format == Format::MultipleResponse { + // Full credit only for the exact set: every keyed option chosen and no + // unkeyed one. Without the negations, checking every box scores 100. + let mut and = Node::new("and"); + for id in &correct { + and = and.child( + Node::new("varequal") + .attr("respident", "response1") + .text((*id).clone()), + ); + } + for id in &incorrect { + and = and.child( + Node::new("not").child( + Node::new("varequal") + .attr("respident", "response1") + .text((*id).clone()), + ), + ); + } + Node::new("conditionvar").child(and) + } else { + Node::new("conditionvar").child( + Node::new("varequal") + .attr("respident", "response1") + .text(correct.first().map(|s| (*s).clone()).unwrap_or_default()), + ) + }; + + resprocessing = resprocessing.child( + Node::new("respcondition") + .attr("continue", "No") + .child(condition) + .child( + Node::new("setvar") + .attr("action", "Set") + .attr("varname", "SCORE") + .text("100"), + ), + ); + + let mut node = Node::new("item") + .attr("ident", qti_id(&format!("{assessment_id}/{uid}"))) + .attr("title", item.display_title()) + .child(item_meta) + .child(presentation) + .child(resprocessing); + + if opts.include_feedback { + for (o, id) in ordered.iter().zip(opt_ids.iter()) { + if let Some(text) = o.student_text() { + node = node.child( + Node::new("itemfeedback") + .attr("ident", format!("{id}_fb")) + .child( + Node::new("flow_mat") + .child(mattext(&format!("
{}
", markup::to_html(text)))), + ), + ); + } + } + } + + node +} + +/// A `` pair. +/// +/// # Arguments +/// +/// * `html` - the HTML fragment, which is escaped on the way in. +/// +/// # Returns +/// +/// The element. +fn mattext(html: &str) -> Node { + Node::new("material").child( + Node::new("mattext") + .attr("texttype", "text/html") + .text(html), + ) +} + +/// A `` pair. +fn metadata_field(label: &str, entry: &str) -> Node { + Node::new("qtimetadatafield") + .child(Node::new("fieldlabel").text(label)) + .child(Node::new("fieldentry").text(entry)) +} + +/// Builds the IMS content package manifest. +/// +/// # Arguments +/// +/// * `quiz_filename` - the quiz XML file name. +/// * `assessment_id` - the assessment identifier, reused as the resource id. +/// * `title` - the human title. +/// * `salt` - salts the manifest identifier. +/// +/// # Returns +/// +/// The manifest element. +fn build_manifest(quiz_filename: &str, assessment_id: &str, title: &str, salt: &str) -> Node { + let lom = Node::new("imsmd:lom").child( + Node::new("imsmd:general").child( + Node::new("imsmd:title").child( + Node::new("imsmd:langstring") + .attr("xml:lang", "en-US") + .text(title), + ), + ), + ); + + Node::new("manifest") + .attr("identifier", format!("man{}", qti_id(salt))) + .attr("xmlns", IMSCP_NS) + .attr("xmlns:imsmd", IMSMD_NS) + .attr("xmlns:xsi", XSI_NS) + .attr("xsi:schemaLocation", IMSCP_SCHEMA) + .child( + Node::new("metadata") + .child(Node::new("schema").text("IMS Content")) + .child(Node::new("schemaversion").text("1.1.3")) + .child(lom), + ) + .child( + Node::new("organizations").attr("default", "root").child( + Node::new("organization") + .attr("identifier", "root") + .attr("structure", "rooted"), + ), + ) + .child( + Node::new("resources").child( + Node::new("resource") + .attr("identifier", assessment_id) + .attr("type", "imsqti_xmlv1p2") + .attr("href", quiz_filename) + .child(Node::new("file").attr("href", quiz_filename)), + ), + ) +} + +/// A deterministic QTI identifier: `g` followed by 32 hex characters. +/// +/// # Arguments +/// +/// * `key` - the stable string to derive from. +/// +/// # Returns +/// +/// The identifier. +fn qti_id(key: &str) -> String { + let digest = hex(&sha256(key.as_bytes())); + format!("g{}", &digest[..32]) +} + +/// A deterministic short numeric identifier, as Canvas uses for answers. +/// +/// # Arguments +/// +/// * `key` - the stable string to derive from. +/// +/// # Returns +/// +/// A four-digit numeric string. +fn short_id(key: &str) -> String { + let d = sha256(key.as_bytes()); + let n = u32::from_be_bytes([d[0], d[1], d[2], d[3]]); + // 1000..9999 keeps the width fixed, which some Canvas importers prefer. + format!("{}", 1000 + (n % 9000)) +} + +/// Makes a file-name-safe slug. +fn slug_filename(s: &str) -> String { + let mut out = String::new(); + for ch in s.chars() { + if ch.is_ascii_alphanumeric() || ch == '-' || ch == '_' { + out.push(ch); + } else { + out.push('_'); + } + } + if out.is_empty() { + "quiz".to_string() + } else { + out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn escapes_html_once_inside_mattext() { + let n = mattext("

a → b

"); + let xml = n.render(0); + // The HTML is character data, so its markup is escaped. + assert!(xml.contains("<p>a &rarr; b</p>"), "{xml}"); + assert!(!xml.contains("

")); + } + + #[test] + fn attributes_are_escaped() { + let xml = Node::new("item") + .attr("title", "a \"quoted\" & ") + .render(0); + assert!(xml.contains(""quoted"")); + assert!(xml.contains("&")); + assert!(!xml.contains("")); + } + + #[test] + fn ids_are_deterministic_and_well_formed() { + assert_eq!(qti_id("a"), qti_id("a")); + assert_ne!(qti_id("a"), qti_id("b")); + let id = qti_id("exam-4/assessment"); + assert_eq!(id.len(), 33); + assert!(id.starts_with('g')); + assert!(id[1..].chars().all(|c| c.is_ascii_hexdigit())); + + let s = short_id("x"); + assert_eq!(s.len(), 4); + assert!(s.parse::().unwrap() >= 1000); + } + + #[test] + fn empty_element_renders_self_closing() { + assert_eq!( + Node::new("file").attr("href", "q.xml").render(0), + "\n" + ); + } + + #[test] + fn document_has_a_declaration() { + let doc = Node::new("root").document(); + assert!(doc.starts_with("\n")); + } + + #[test] + fn manifest_points_at_the_quiz_file() { + let m = build_manifest("exam-4.xml", "gabc", "Exam 4", "exam-4").render(0); + assert!(m.contains("imsqti_xmlv1p2")); + assert!(m.contains("href=\"exam-4.xml\"")); + assert!(m.contains("Exam 4")); + } + + #[test] + fn slug_filename_is_safe() { + assert_eq!(slug_filename("exam-4 2026s"), "exam-4_2026s"); + assert_eq!(slug_filename(""), "quiz"); + } +} diff --git a/src/export/report.rs b/src/export/report.rs new file mode 100644 index 0000000..4b859e0 --- /dev/null +++ b/src/export/report.rs @@ -0,0 +1,1084 @@ +//! Writing reports for students and for yourself. +//! +//! Two audiences, two documents, and the difference between them is not tone but +//! content. +//! +//! The **student report** answers "what should I do next?". It gives a score, a +//! coarse position in the class, per-objective standing, and — the part that +//! actually helps — for each missed question, the misconception that the specific +//! distractor they chose was written to detect, plus where in the course to go +//! back to. It never prints a correct answer, never names another student, and +//! never reports a rank. Those omissions are deliberate: a report that reveals +//! keys cannot be sent out before a makeup exam, and a report that gives a rank +//! invites the student to read it as a verdict rather than as instructions. +//! +//! The **instructor report** answers "what should I fix?". Item statistics, the +//! revision queue, distractor tables, reliability, blueprint coverage, and the +//! class-level objectives that nobody met. That last section is the one that +//! should change your teaching rather than any individual student's studying. +//! +//! Output is Markdown. It is readable as-is in a terminal or a text editor, it +//! diffs cleanly, and [`to_html`] converts it for emailing or posting. The HTML +//! converter handles exactly the Markdown this module emits — headings, tables, +//! lists, bold, italic, code, blockquotes, rules — and nothing more; it is not a +//! general Markdown implementation and does not pretend to be. + +use std::collections::BTreeMap; + +use crate::assessment::AssessmentFile; +use crate::catalog::Catalog; +use crate::classical::Analysis; +use crate::course::CourseFile; +use crate::date::Date; +use crate::irt::Fit; +use crate::students::{Cohort, Mastery, StudentSummary}; +use crate::taxonomy::Level; + +/// What to include in a student report. +#[derive(Debug, Clone)] +pub struct StudentOptions { + /// Whether to include the per-objective table. + pub objectives: bool, + /// Whether to include the per-level comparison to the class. + pub levels: bool, + /// Whether to include per-question guidance on missed items. + pub missed: bool, + /// Whether to show the class mean and the student's band. Turn this off for a + /// course where any comparison is unwelcome. + pub comparison: bool, + /// Whether to include the IRT ability estimate. Off by default: it is not + /// meaningful to most students and invites misreading. + pub ability: bool, + /// A closing note appended verbatim, e.g. office-hours information. + pub closing: Option, +} + +impl Default for StudentOptions { + fn default() -> StudentOptions { + StudentOptions { + objectives: true, + levels: true, + missed: true, + comparison: true, + ability: false, + closing: None, + } + } +} + +/// Writes one student's report. +/// +/// # Arguments +/// +/// * `summary` - the student's summary. +/// * `cohort` - the class context, for means. +/// * `course` - the course, for titles. +/// * `record` - the assessment record, for the title and date. +/// * `opts` - what to include. +/// +/// # Returns +/// +/// A Markdown document. +pub fn student( + summary: &StudentSummary, + cohort: &Cohort, + course: &CourseFile, + record: &AssessmentFile, + opts: &StudentOptions, +) -> String { + let mut out = String::new(); + + out.push_str(&format!( + "# {} — {}\n\n", + record.assessment.title, course.course.code + )); + out.push_str(&format!("**{}**\n\n", summary.display_name())); + + // ---------------------------------------------------------------- score + out.push_str(&format!( + "You scored **{:.1} of {:.1} points ({:.0}%)**", + summary.points, summary.points_possible, summary.percent + )); + if summary.bonus_points > 0.0 { + out.push_str(&format!( + ", plus {:.1} bonus point{}", + summary.bonus_points, + if (summary.bonus_points - 1.0).abs() < 1e-9 { + "" + } else { + "s" + } + )); + } + out.push_str(&format!( + ", answering {} of {} questions correctly.\n\n", + summary.correct, summary.n_items + )); + + if opts.comparison { + out.push_str(&format!( + "The class averaged {:.0}%. Your score is in the {}.\n\n", + cohort.mean_percent, summary.band + )); + } + + if opts.ability { + if let (Some(theta), Some(se)) = (summary.theta, summary.theta_se) { + out.push_str(&format!( + "Adjusting for how difficult each question turned out to be, your estimated \ + standing is {} (θ = {theta:+.2}, ± {:.2}). The margin is wide because a single \ + exam is a small amount of evidence.\n\n", + crate::irt::Ability { + student_key: summary.student_key.clone(), + theta, + se, + n_items: summary.n_items, + } + .band(), + se * 1.96 + )); + } + } + + // ----------------------------------------------------------- objectives + if opts.objectives && !summary.objectives.is_empty() { + out.push_str("## What this exam says about each learning objective\n\n"); + out.push_str("| | Objective | You | Class | Items |\n|:--|:--|--:|--:|--:|\n"); + for o in &summary.objectives { + let you = if o.status == Mastery::NotEnoughEvidence { + format!("{:.0}%", o.rate * 100.0) + } else { + format!("{:.0}%", o.rate * 100.0) + }; + out.push_str(&format!( + "| {} | {} | {} | {:.0}% | {} |\n", + o.status.symbol(), + escape_pipes(&o.text), + you, + o.cohort_rate * 100.0, + o.n_items + )); + } + out.push('\n'); + out.push_str("✓ meeting · ~ developing · ✗ not yet · ? too few questions to tell\n\n"); + + // The "too few questions" cases are an honest caveat about the exam, and + // saying so protects the student from over-reading a single data point. + let thin: Vec<&str> = summary + .objectives + .iter() + .filter(|o| o.status == Mastery::NotEnoughEvidence) + .map(|o| o.text.as_str()) + .collect(); + if !thin.is_empty() { + out.push_str(&format!( + "This exam had too few questions on {} to say anything reliable about {}. Treat \ + those rows as information about the exam, not about you.\n\n", + list(&thin), + if thin.len() == 1 { "it" } else { "them" } + )); + } + } + + // --------------------------------------------------------------- levels + if opts.levels && summary.levels.len() > 1 { + out.push_str("## Kinds of thinking\n\n"); + out.push_str( + "Questions on this exam asked for different kinds of thinking. Comparing your rate \ + across them often shows more than the total score does.\n\n", + ); + out.push_str("| Kind of question | You | Class | |\n|:--|--:|--:|:--|\n"); + for l in &summary.levels { + out.push_str(&format!( + "| {} — {} | {:.0}% | {:.0}% | {} |\n", + l.level.name(), + l.level.blurb(), + l.rate * 100.0, + l.cohort_rate * 100.0, + bar(l.rate) + )); + } + out.push('\n'); + + // The interesting pattern: fine on recall, falling apart on application. + let recall: Vec = summary + .levels + .iter() + .filter(|l| l.level.code() <= 2) + .map(|l| l.rate) + .collect(); + let applied: Vec = summary + .levels + .iter() + .filter(|l| l.level.code() >= 3) + .map(|l| l.rate) + .collect(); + if !recall.is_empty() && !applied.is_empty() { + let drop = average(&recall) - average(&applied); + if drop > 0.2 { + out.push_str( + "You are recalling the material but losing ground when you have to use it. \ + That usually means more practice working problems rather than more rereading \ + — rereading feels productive and mostly rebuilds recognition.\n\n", + ); + } else if drop < -0.2 { + out.push_str( + "You reason well with the material when it is in front of you, but specific \ + facts and terms are costing you points. That is the more tractable of the two \ + problems: targeted memorization of the terms below will help.\n\n", + ); + } + } + } + + // ---------------------------------------------------------- what to do + if !summary.focus.is_empty() { + out.push_str("## Where to put your time\n\n"); + out.push_str("In this order:\n\n"); + for (i, id) in summary.focus.iter().take(4).enumerate() { + let text = course.objective_text(id); + out.push_str(&format!("{}. {}\n", i + 1, text)); + } + out.push('\n'); + + // Distinguish "you missed this" from "the class missed this", because the + // second is going to be retaught and does not need solo review. + let class_gaps: Vec<&str> = cohort + .class_gaps + .iter() + .filter(|(id, _)| summary.focus.contains(id)) + .map(|(id, _)| id.as_str()) + .collect(); + if !class_gaps.is_empty() { + let texts: Vec = class_gaps + .iter() + .map(|id| course.objective_text(id)) + .collect(); + let refs: Vec<&str> = texts.iter().map(|s| s.as_str()).collect(); + out.push_str(&format!( + "Most of the class also struggled with {}, so expect it to come back in class. \ + Prioritize the other items above for solo review.\n\n", + list(&refs) + )); + } + } + + if !summary.strengths.is_empty() { + let texts: Vec = summary + .strengths + .iter() + .take(4) + .map(|id| course.objective_text(id)) + .collect(); + let refs: Vec<&str> = texts.iter().map(|s| s.as_str()).collect(); + out.push_str(&format!("You have clearly got {}.\n\n", list(&refs))); + } + + // --------------------------------------------------------- missed items + if opts.missed && !summary.missed.is_empty() { + out.push_str("## Question by question\n\n"); + out.push_str( + "For each question you missed, here is what the answer you chose usually indicates, \ + and where to go back to. Correct answers are not listed here.\n\n", + ); + for m in &summary.missed { + let partial = if m.credit > 0.0 { + format!(" (partial credit: {:.0}%)", m.credit * 100.0) + } else { + String::new() + }; + out.push_str(&format!("**Question {}**{partial}\n\n", m.number)); + if let Some(text) = &m.feedback { + out.push_str(&format!("{text}\n\n")); + } else if let Some(misconception) = &m.misconception { + out.push_str(&format!( + "The option you chose is the one students pick when {misconception}\n\n" + )); + } + if !m.study.is_empty() { + out.push_str(&format!("Review: {}\n\n", m.study.join("; "))); + } + } + } + + if let Some(closing) = &opts.closing { + out.push_str("---\n\n"); + out.push_str(closing); + out.push_str("\n\n"); + } + + out.push_str(&format!( + "---\n\n*Generated {} for {}. Percentages on individual objectives come from a handful of \ + questions each and carry real uncertainty; read them as directions, not measurements.*\n", + Date::today(), + summary.display_name() + )); + + out +} + +/// Writes the instructor's report on one administration. +/// +/// # Arguments +/// +/// * `analysis` - classical item analysis. +/// * `cohort` - per-student summaries and class rates. +/// * `catalog` - the loaded course. +/// * `record` - the assessment record. +/// * `fit` - an optional IRT fit. +/// +/// # Returns +/// +/// A Markdown document. +pub fn cohort( + analysis: &Analysis, + cohort: &Cohort, + catalog: &Catalog, + record: &AssessmentFile, + fit: Option<&Fit>, +) -> String { + let mut out = String::new(); + let course = &catalog.course; + + out.push_str(&format!( + "# {} — item analysis\n\n{} · {} · {}\n\n", + record.assessment.title, + course.course.code, + record + .assessment + .term + .clone() + .unwrap_or_else(|| course.course.term.clone()), + record + .assessment + .date + .map(|d| d.to_string()) + .unwrap_or_else(|| "date not recorded".into()) + )); + + // ------------------------------------------------------------- summary + let r = &analysis.reliability; + out.push_str("## Summary\n\n"); + out.push_str(&format!( + "- {} examinees, {} scored items\n- Mean score {:.1} of {} ({:.0}%), SD {:.2}\n- \ + Mean p-value {:.2}, mean point-biserial {}\n", + r.n_students, + r.n_items, + r.mean, + r.n_items, + if r.n_items > 0 { + 100.0 * r.mean / r.n_items as f64 + } else { + 0.0 + }, + r.sd, + r.mean_p, + r.mean_point_biserial + .map(|v| format!("{v:+.2}")) + .unwrap_or_else(|| "n/a".into()) + )); + out.push('\n'); + out.push_str(&r.interpretation()); + out.push_str("\n\n"); + + if let Some(f) = fit { + let peak = f.peak_information(); + out.push_str(&format!( + "The IRT fit ({} model, {} iterations{}) measures most precisely around θ = {peak:+.1}", + match f.items.first().map(|i| i.model) { + Some(m) => m.as_str(), + None => "?", + }, + f.iterations, + if f.converged { + "" + } else { + ", did not converge" + } + )); + match f.standard_error(peak) { + Some(se) => out.push_str(&format!( + ", where the standard error is {se:.2} logits.\n\n" + )), + None => out.push_str(".\n\n"), + } + } + + for w in &analysis.warnings { + out.push_str(&format!("> {w}\n\n")); + } + + // ------------------------------------------------------- revise queue + let queue = analysis.revise_queue(); + out.push_str("## What to revise\n\n"); + if queue.is_empty() { + out.push_str( + "Nothing was flagged. Unusual, and worth a skeptical glance at whether the \ + key and the record actually matched the exam.\n\n", + ); + } else { + let blocking = queue.iter().filter(|i| i.needs_revision()).count(); + out.push_str(&format!( + "{} item(s) flagged; {blocking} need attention before being used again.\n\n", + queue.len() + )); + for item in queue { + let label = item + .item_ref + .clone() + .unwrap_or_else(|| format!("question {}", item.number)); + out.push_str(&format!( + "### Q{} — {}{}\n\n", + item.number, + label, + if item.needs_revision() { " ⚠" } else { "" } + )); + out.push_str(&format!( + "p = {:.2} · r = {} · flags: {}\n\n", + item.p_value, + item.point_biserial + .map(|v| format!("{v:+.2}")) + .unwrap_or_else(|| "n/a".into()), + item.flags + .iter() + .map(|f| f.as_str()) + .collect::>() + .join(", ") + )); + for note in &item.notes { + out.push_str(&format!("- {note}\n")); + } + out.push('\n'); + + // The distractor table is where a poorly worded item shows itself. + if item.options.len() > 1 { + out.push_str( + "| Option | Chose | r | Upper | Lower | |\n|:--|--:|--:|--:|--:|:--|\n", + ); + for o in item.options.values() { + out.push_str(&format!( + "| {} | {:.0}% | {} | {} | {} | {} |\n", + o.letter, + o.rate * 100.0, + o.point_biserial + .map(|v| format!("{v:+.2}")) + .unwrap_or_else(|| "n/a".into()), + o.upper_rate + .map(|v| format!("{:.0}%", v * 100.0)) + .unwrap_or_else(|| "-".into()), + o.lower_rate + .map(|v| format!("{:.0}%", v * 100.0)) + .unwrap_or_else(|| "-".into()), + if o.is_key { "**key**" } else { "" } + )); + } + out.push('\n'); + } + } + } + + // ---------------------------------------------------------- item table + out.push_str("## Every item\n\n"); + out.push_str( + "| Q | Item | Lv | p | r | D | Blank | Flags |\n|--:|:--|--:|--:|--:|--:|--:|:--|\n", + ); + for item in &analysis.items { + let level = record + .placement(item.number) + .and_then(|p| p.level) + .map(|l| l.code().to_string()) + .unwrap_or_else(|| "-".into()); + out.push_str(&format!( + "| {} | {} | {} | {:.2} | {} | {} | {:.0}% | {} |\n", + item.number, + item.item_ref.clone().unwrap_or_default(), + level, + item.p_value, + item.point_biserial + .map(|v| format!("{v:+.2}")) + .unwrap_or_else(|| "n/a".into()), + item.discrimination_index + .map(|v| format!("{v:+.2}")) + .unwrap_or_else(|| "-".into()), + item.blank_rate * 100.0, + item.flags + .iter() + .map(|f| f.as_str()) + .collect::>() + .join(" ") + )); + } + out.push('\n'); + + if let Some(f) = fit { + out.push_str("## IRT parameters\n\n"); + out.push_str("| Q | a | b | SE(a) | SE(b) | n | Notes |\n|--:|--:|--:|--:|--:|--:|:--|\n"); + for item in &f.items { + out.push_str(&format!( + "| {} | {:.2} | {:+.2} | {} | {} | {} | {} |\n", + item.number, + item.a, + item.b, + item.se_a + .map(|v| format!("{v:.2}")) + .unwrap_or_else(|| "-".into()), + item.se_b + .map(|v| format!("{v:.2}")) + .unwrap_or_else(|| "-".into()), + item.n, + item.notes.join(" ") + )); + } + out.push('\n'); + for w in &f.warnings { + out.push_str(&format!("> {w}\n\n")); + } + } + + // -------------------------------------------------------- class gaps + out.push_str("## Objectives the class did not meet\n\n"); + if cohort.class_gaps.is_empty() { + out.push_str("Every assessed objective cleared the mastery threshold.\n\n"); + } else { + out.push_str(&format!( + "Below the {:.0}% threshold. These are the candidates for reteaching rather than for \ + individual review.\n\n", + course.policy.mastery_threshold * 100.0 + )); + out.push_str("| Objective | Class rate |\n|:--|--:|\n"); + for (id, rate) in &cohort.class_gaps { + out.push_str(&format!( + "| {} | {:.0}% |\n", + escape_pipes(&course.objective_text(id)), + rate * 100.0 + )); + } + out.push('\n'); + } + + // ------------------------------------------------------ level coverage + out.push_str("## Coverage and class performance by level\n\n"); + let counts = record.level_counts(); + out.push_str("| Level | Items | Class rate |\n|:--|--:|--:|\n"); + for level in Level::ALL { + let n = counts.get(&level).copied().unwrap_or(0); + if n == 0 { + continue; + } + out.push_str(&format!( + "| {} {} | {} | {:.0}% |\n", + level.code(), + level.name(), + n, + cohort.level_rates.get(&level).copied().unwrap_or(0.0) * 100.0 + )); + } + out.push('\n'); + + if let Some(bp) = &record.blueprint { + let drift = crate::select::check_blueprint(record); + if !drift.is_empty() { + out.push_str("Blueprint drift:\n\n"); + for d in &drift { + out.push_str(&format!("- {d}\n")); + } + out.push('\n'); + } + let _ = bp; + } + + // -------------------------------------------------------- archetypes + if !cohort.archetypes.is_empty() { + out.push_str("## Patterns across students\n\n"); + out.push_str( + "Descriptive grouping by performance profile across levels, not a diagnosis. Useful \ + for deciding whether a review session should target one gap or several.\n\n", + ); + for a in &cohort.archetypes { + let means: Vec = a + .level_means + .iter() + .map(|(l, v)| format!("L{}: {:.0}%", l.code(), v * 100.0)) + .collect(); + out.push_str(&format!( + "- **{}** — {} student(s); {}\n", + a.label, + a.members.len(), + means.join(", ") + )); + } + out.push('\n'); + } + + out.push_str(&format!("---\n\n*Generated {}.*\n", Date::today())); + out +} + +/// Writes a one-line-per-student roster of scores. +/// +/// # Arguments +/// +/// * `cohort` - the class. +/// +/// # Returns +/// +/// A Markdown table. +pub fn roster(cohort: &Cohort) -> String { + let mut out = + String::from("| Student | Points | % | Correct | Focus |\n|:--|--:|--:|--:|:--|\n"); + let mut sorted: Vec<&StudentSummary> = cohort.students.iter().collect(); + sorted.sort_by(|a, b| { + b.percent + .partial_cmp(&a.percent) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.student_key.cmp(&b.student_key)) + }); + for s in sorted { + out.push_str(&format!( + "| {} | {:.1} | {:.0}% | {}/{} | {} |\n", + s.display_name(), + s.points, + s.percent, + s.correct, + s.n_items, + s.focus + .iter() + .take(2) + .cloned() + .collect::>() + .join(", ") + )); + } + out +} + +/// A small unicode bar for a rate in `0.0..=1.0`. +/// +/// # Arguments +/// +/// * `rate` - the value. +/// +/// # Returns +/// +/// A ten-cell bar. +fn bar(rate: f64) -> String { + let filled = (rate.clamp(0.0, 1.0) * 10.0).round() as usize; + format!("{}{}", "█".repeat(filled), "░".repeat(10 - filled)) +} + +/// The mean of a slice, zero when empty. +fn average(v: &[f64]) -> f64 { + if v.is_empty() { + 0.0 + } else { + v.iter().sum::() / v.len() as f64 + } +} + +/// Joins items into an English list. +/// +/// # Arguments +/// +/// * `items` - the items. +/// +/// # Returns +/// +/// `"a"`, `"a and b"`, or `"a, b, and c"`. +fn list(items: &[&str]) -> String { + match items.len() { + 0 => String::new(), + 1 => items[0].to_string(), + 2 => format!("{} and {}", items[0], items[1]), + _ => { + let head = items[..items.len() - 1].join(", "); + format!("{head}, and {}", items[items.len() - 1]) + } + } +} + +/// Escapes pipes so objective text cannot break a Markdown table. +fn escape_pipes(s: &str) -> String { + s.replace('|', "\\|") +} + +/// Converts the Markdown this module emits into a standalone HTML document. +/// +/// Handles headings, paragraphs, unordered and ordered lists, tables, +/// blockquotes, horizontal rules, and inline bold, italic, and code. This is not a +/// general Markdown implementation: it covers the constructs the generators above +/// produce, and unrecognized syntax passes through as escaped text rather than +/// being silently mangled. +/// +/// # Arguments +/// +/// * `markdown` - the document. +/// * `title` - the HTML title. +/// +/// # Returns +/// +/// A complete HTML document with embedded styles, so it can be emailed or opened +/// with no other files. +pub fn to_html(markdown: &str, title: &str) -> String { + let mut body = String::new(); + let mut lines = markdown.lines().peekable(); + let mut list_kind: Option<&str> = None; + + // Closes an open list, if any. + fn close_list(body: &mut String, list_kind: &mut Option<&str>) { + if let Some(tag) = list_kind.take() { + body.push_str(&format!("\n")); + } + } + + while let Some(line) = lines.next() { + let trimmed = line.trim(); + + if trimmed.is_empty() { + close_list(&mut body, &mut list_kind); + continue; + } + + if trimmed.starts_with("---") && trimmed.chars().all(|c| c == '-') { + close_list(&mut body, &mut list_kind); + body.push_str("

\n"); + continue; + } + + if let Some(rest) = trimmed.strip_prefix("> ") { + close_list(&mut body, &mut list_kind); + body.push_str(&format!("
{}
\n", inline(rest))); + continue; + } + + // Headings. + let hashes = trimmed.chars().take_while(|c| *c == '#').count(); + if hashes > 0 && hashes <= 6 && trimmed.chars().nth(hashes) == Some(' ') { + close_list(&mut body, &mut list_kind); + let text = trimmed[hashes + 1..].trim(); + body.push_str(&format!("{}\n", inline(text))); + continue; + } + + // Tables: a header row followed by an alignment row. + if trimmed.starts_with('|') { + let is_separator = |s: &str| { + s.trim().starts_with('|') + && s.chars().all(|c| matches!(c, '|' | '-' | ':' | ' ' | '\t')) + && s.contains('-') + }; + if lines.peek().map(|n| is_separator(n)).unwrap_or(false) { + close_list(&mut body, &mut list_kind); + lines.next(); + body.push_str("\n"); + for cell in split_row(trimmed) { + body.push_str(&format!("", inline(&cell))); + } + body.push_str("\n\n"); + while let Some(next) = lines.peek() { + if !next.trim().starts_with('|') { + break; + } + // `peek` just succeeded, so `next` cannot be `None`; a + // `let else` says that without a panic in the path. + let Some(row) = lines.next() else { break }; + body.push_str(""); + for cell in split_row(row.trim()) { + body.push_str(&format!("", inline(&cell))); + } + body.push_str("\n"); + } + body.push_str("\n
{}
{}
\n"); + continue; + } + } + + // Lists. + if let Some(rest) = trimmed.strip_prefix("- ") { + if list_kind != Some("ul") { + close_list(&mut body, &mut list_kind); + body.push_str("
    \n"); + list_kind = Some("ul"); + } + body.push_str(&format!("
  • {}
    • \n", inline(rest))); + continue; + } + if let Some((prefix, rest)) = trimmed.split_once(". ") { + if !prefix.is_empty() && prefix.chars().all(|c| c.is_ascii_digit()) { + if list_kind != Some("ol") { + close_list(&mut body, &mut list_kind); + body.push_str("
      \n"); + list_kind = Some("ol"); + } + body.push_str(&format!("
    1. {}
      2. \n", inline(rest))); + continue; + } + } + + close_list(&mut body, &mut list_kind); + body.push_str(&format!("

      {}

      \n", inline(trimmed))); + } + close_list(&mut body, &mut list_kind); + + format!( + "\n\n\n\n\ + \n\ + {}\n\n\n\n{}\n\n", + crate::markup::escape_html(title), + STYLE, + body + ) +} + +/// Splits a Markdown table row into cells. +fn split_row(row: &str) -> Vec { + let inner = row.trim().trim_start_matches('|').trim_end_matches('|'); + inner + .split('|') + .map(|c| c.trim().replace("\\|", "|")) + .collect() +} + +/// Converts inline Markdown to HTML. +/// +/// Escapes first, then applies emphasis, so text containing angle brackets cannot +/// become markup. +fn inline(s: &str) -> String { + let escaped = crate::markup::escape_html(s); + let mut out = escaped; + out = wrap(&out, "**", "", ""); + out = wrap(&out, "`", "", ""); + out = wrap(&out, "*", "", ""); + out +} + +/// Replaces paired delimiters, leaving unpaired ones literal. +fn wrap(s: &str, delim: &str, open: &str, close: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut rest = s; + loop { + let Some(i) = rest.find(delim) else { + out.push_str(rest); + return out; + }; + let after = &rest[i + delim.len()..]; + let Some(j) = after.find(delim) else { + out.push_str(rest); + return out; + }; + if j == 0 { + out.push_str(&rest[..i + delim.len()]); + rest = after; + continue; + } + out.push_str(&rest[..i]); + out.push_str(open); + out.push_str(&after[..j]); + out.push_str(close); + rest = &after[j + delim.len()..]; + } +} + +/// Embedded stylesheet for HTML reports. +const STYLE: &str = "\ +:root { color-scheme: light dark; } +body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; + max-width: 46rem; margin: 2.5rem auto; padding: 0 1.25rem; line-height: 1.55; + color: #1a1a1a; background: #fff; } +h1 { font-size: 1.6rem; margin-bottom: 0.2rem; } +h2 { font-size: 1.15rem; margin-top: 2rem; border-bottom: 1px solid #e5e5e5; + padding-bottom: 0.3rem; } +h3 { font-size: 1rem; margin-top: 1.5rem; } +table { border-collapse: collapse; width: 100%; margin: 1rem 0; font-size: 0.92rem; } +th, td { border-bottom: 1px solid #e5e5e5; padding: 0.4rem 0.6rem; text-align: left; } +th { background: #fafafa; font-weight: 600; } +td:nth-child(n+3), th:nth-child(n+3) { text-align: right; } +code { background: #f5f5f5; padding: 0.1rem 0.3rem; border-radius: 3px; + font-size: 0.9em; } +blockquote { border-left: 3px solid #d0d0d0; margin: 1rem 0; padding: 0.3rem 0 0.3rem 1rem; + color: #555; font-size: 0.95rem; } +hr { border: none; border-top: 1px solid #e5e5e5; margin: 2rem 0; } +em { color: #555; } +ul, ol { padding-left: 1.4rem; } +@media (prefers-color-scheme: dark) { + body { color: #e8e8e8; background: #1a1a1a; } + th { background: #262626; } + th, td { border-bottom-color: #333; } + code { background: #2a2a2a; } + h2 { border-bottom-color: #333; } + blockquote { border-left-color: #444; color: #aaa; } +} +"; + +/// Writes every student's report to a directory. +/// +/// # Arguments +/// +/// * `dir` - the destination directory. +/// * `cohort` - the class. +/// * `course` - the course. +/// * `record` - the assessment record. +/// * `opts` - report options. +/// * `html` - whether to write HTML alongside Markdown. +/// +/// # Returns +/// +/// The paths written. +/// +/// # Errors +/// +/// Returns [`crate::error::Error::Io`] on a write failure. +pub fn write_all_students( + dir: &std::path::Path, + cohort: &Cohort, + course: &CourseFile, + record: &AssessmentFile, + opts: &StudentOptions, + html: bool, +) -> crate::error::Result> { + std::fs::create_dir_all(dir).map_err(|e| crate::error::Error::io(dir, e))?; + let mut written = Vec::new(); + for s in &cohort.students { + let stem = crate::store::sanitize(&s.student_key); + let markdown = student(s, cohort, course, record, opts); + let md_path = dir.join(format!("{stem}.md")); + crate::yaml::write_text(&md_path, &markdown)?; + written.push(md_path); + if html { + let title = format!("{} — {}", record.assessment.title, s.display_name()); + let html_path = dir.join(format!("{stem}.html")); + crate::yaml::write_text(&html_path, &to_html(&markdown, &title))?; + written.push(html_path); + } + } + Ok(written) +} + +/// Per-objective class rates as a compact table, for pasting into a syllabus +/// review or a curriculum committee document. +/// +/// # Arguments +/// +/// * `cohort` - the class. +/// * `course` - the course, for objective text. +/// +/// # Returns +/// +/// A Markdown table. +pub fn objective_summary(cohort: &Cohort, course: &CourseFile) -> String { + let mut out = String::from("| Objective | Class rate |\n|:--|--:|\n"); + let mut rows: Vec<(&String, &f64)> = cohort.objective_rates.iter().collect(); + rows.sort_by(|a, b| { + a.1.partial_cmp(b.1) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.0.cmp(b.0)) + }); + for (id, rate) in rows { + out.push_str(&format!( + "| {} | {:.0}% |\n", + escape_pipes(&course.objective_text(id)), + rate * 100.0 + )); + } + out +} + +/// Counts flags across an analysis, for a headline figure. +/// +/// # Arguments +/// +/// * `analysis` - the analysis. +/// +/// # Returns +/// +/// How many items carry each flag. +pub fn flag_counts(analysis: &Analysis) -> BTreeMap<&'static str, usize> { + let mut out = BTreeMap::new(); + for item in &analysis.items { + for flag in &item.flags { + *out.entry(flag.as_str()).or_insert(0) += 1; + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bars_are_ten_cells_wide() { + assert_eq!(bar(0.0).chars().count(), 10); + assert_eq!(bar(1.0).chars().count(), 10); + assert_eq!(bar(0.5).chars().count(), 10); + assert!(bar(1.0).starts_with('█')); + assert!(bar(0.0).starts_with('░')); + // Out-of-range input must not panic or overflow. + assert_eq!(bar(2.0).chars().count(), 10); + assert_eq!(bar(-1.0).chars().count(), 10); + } + + #[test] + fn english_lists_read_correctly() { + assert_eq!(list(&[]), ""); + assert_eq!(list(&["a"]), "a"); + assert_eq!(list(&["a", "b"]), "a and b"); + assert_eq!(list(&["a", "b", "c"]), "a, b, and c"); + } + + #[test] + fn pipes_in_objective_text_do_not_break_tables() { + assert_eq!(escape_pipes("a | b"), "a \\| b"); + } + + #[test] + fn html_conversion_handles_headings_and_paragraphs() { + let html = to_html("# Title\n\nSome text.\n", "T"); + assert!(html.contains("

      Title

      ")); + assert!(html.contains("

      Some text.

      ")); + assert!(html.starts_with("")); + assert!(html.contains("T")); + } + + #[test] + fn html_conversion_builds_tables() { + let md = "| A | B |\n|:--|--:|\n| 1 | 2 |\n"; + let html = to_html(md, "T"); + assert!(html.contains("")); + assert!(html.contains("")); + assert!(html.contains("")); + assert!(html.contains("")); + } + + #[test] + fn html_conversion_handles_both_list_kinds() { + let html = to_html("- one\n- two\n\n1. first\n2. second\n", "T"); + assert!(html.contains("
        ")); + assert!(html.contains("
      • one
        • ")); + assert!(html.contains("
          ")); + assert!(html.contains("
        1. first
          2. ")); + // Lists must be closed, not left dangling. + assert_eq!(html.matches("
            ").count(), html.matches("
          ").count()); + assert_eq!(html.matches("
            ").count(), html.matches("
          ").count()); + } + + #[test] + fn html_conversion_escapes_before_emphasis() { + let html = to_html("**bold** and \n", "T"); + assert!(html.contains("bold")); + assert!(!html.contains(""); + assert!(!out.contains("
      A2