387 lines
14 KiB
Rust
387 lines
14 KiB
Rust
// SPDX-License-Identifier: Prosperity-3.0.0
|
|
// Copyright Scientific Computing Studio
|
|
// Source: https://git.scient.ing/education/coursebank
|
|
|
|
//! The responses-to-report pipeline.
|
|
//!
|
|
//! Once an assessment has been given, responses come back through [`ingest`],
|
|
//! statistics come out of [`analyze`] (classical, IRT, or per-student), [`calibrate`]
|
|
//! writes those statistics back onto the items, and [`report`] produces the
|
|
//! student and cohort documents. [`data`] lists what the response store holds.
|
|
|
|
use coursebank::calibrate;
|
|
use coursebank::canvas;
|
|
use coursebank::classical::{self, Thresholds};
|
|
use coursebank::error::Result;
|
|
use coursebank::gradescope;
|
|
use coursebank::irt;
|
|
use coursebank::layout::Layout;
|
|
use coursebank::report;
|
|
use coursebank::store::{self, Store};
|
|
use coursebank::students;
|
|
use coursebank::yaml;
|
|
|
|
use crate::cli::{AnalyzeCommand, CalibrateArgs, Cli, IngestCommand, ReportCommand};
|
|
use crate::commands::Outcome;
|
|
use crate::helpers::{context, load, load_record, read_salt, responses_for, truncate};
|
|
|
|
/// `ingest`: read a Gradescope directory or a Canvas CSV into the response store.
|
|
///
|
|
/// Enriches the parsed responses against the record, optionally pseudonymizes the
|
|
/// identifiers, and — unless `--dry-run` — writes them in the chosen format.
|
|
pub(crate) fn ingest(cli: &Cli, sub: &IngestCommand) -> Result<Outcome> {
|
|
let catalog = load(cli)?;
|
|
|
|
let (common, mut set) = match sub {
|
|
IngestCommand::Gradescope { dir, common } => {
|
|
let record = load_record(&catalog, &common.assessment)?;
|
|
let ctx = context(&catalog, &record, common)?;
|
|
let import = gradescope::ingest_dir(dir, &ctx)?;
|
|
|
|
// Grading-time partial credit is an ambiguity signal worth surfacing
|
|
// right here, while the exam is fresh.
|
|
for question in &import.questions {
|
|
for (letter, value, note) in question.partial_credit() {
|
|
println!(
|
|
"! q{}: option {letter} earned {value} of {} points at grading time{}",
|
|
question.number,
|
|
question.points_possible(),
|
|
note.map(|n| format!(" — {n}")).unwrap_or_default()
|
|
);
|
|
}
|
|
}
|
|
(common, import.responses)
|
|
}
|
|
IngestCommand::Canvas { file, common } => {
|
|
let record = load_record(&catalog, &common.assessment)?;
|
|
let ctx = context(&catalog, &record, common)?;
|
|
let set = canvas::ingest(file, &ctx, Some(&record), Some(&catalog))?;
|
|
(common, set)
|
|
}
|
|
};
|
|
|
|
let record = load_record(&catalog, &common.assessment)?;
|
|
set.enrich(&record, Some(&catalog));
|
|
|
|
if common.pseudonymize {
|
|
let salt = read_salt(common.salt_file.as_deref())?;
|
|
set.pseudonymize(&salt);
|
|
println!("identifiers replaced with keyed pseudonyms");
|
|
}
|
|
|
|
for warning in &set.warnings {
|
|
println!("! {warning}");
|
|
}
|
|
|
|
println!(
|
|
"\n{} response(s): {} student(s) x {} item(s)",
|
|
set.rows.len(),
|
|
set.students().len(),
|
|
set.all_items().len()
|
|
);
|
|
|
|
if common.dry_run {
|
|
println!("(dry run, nothing written)");
|
|
return Ok(Outcome::Ok);
|
|
}
|
|
|
|
let mut store = Store::open(catalog.layout.data())?;
|
|
if let Some(format) = common.format {
|
|
store = store.with_format(format.as_format())?;
|
|
}
|
|
for path in store.write(&set)? {
|
|
println!("wrote {}", path.display());
|
|
}
|
|
println!(
|
|
"\nNext: coursebank analyze items {}\n coursebank report cohort {}",
|
|
common.assessment, common.assessment
|
|
);
|
|
Ok(Outcome::Ok)
|
|
}
|
|
|
|
/// `analyze`: classical item analysis, an IRT fit, or a per-student summary.
|
|
///
|
|
/// `items` returns [`Outcome::Findings`] when there is a revise queue.
|
|
pub(crate) fn analyze(cli: &Cli, sub: &AnalyzeCommand) -> Result<Outcome> {
|
|
let catalog = load(cli)?;
|
|
let store = Store::open(catalog.layout.data())?;
|
|
|
|
match sub {
|
|
AnalyzeCommand::Items { id, pooled } => {
|
|
let record = load_record(&catalog, id)?;
|
|
let set = responses_for(&store, &catalog, &record, *pooled)?;
|
|
let analysis =
|
|
classical::analyze(&set, &Thresholds::default(), Some(&record), Some(&catalog));
|
|
|
|
for w in &analysis.warnings {
|
|
println!("! {w}\n");
|
|
}
|
|
println!("{}\n", analysis.reliability.interpretation());
|
|
println!(
|
|
"{:>3} {:>5} {:>6} {:>6} {:>6} FLAGS",
|
|
"Q", "p", "r", "D", "blank"
|
|
);
|
|
for item in &analysis.items {
|
|
println!(
|
|
"{:>3} {:>5.2} {:>6} {:>6} {:>5.0}% {}",
|
|
item.number,
|
|
item.p_value,
|
|
item.point_biserial
|
|
.map(|v| format!("{v:+.2}"))
|
|
.unwrap_or_else(|| "n/a".into()),
|
|
item.discrimination_index
|
|
.map(|v| format!("{v:+.2}"))
|
|
.unwrap_or_else(|| "-".into()),
|
|
item.blank_rate * 100.0,
|
|
item.flags
|
|
.iter()
|
|
.map(|f| f.as_str())
|
|
.collect::<Vec<_>>()
|
|
.join(" ")
|
|
);
|
|
}
|
|
|
|
let queue = analysis.revise_queue();
|
|
if !queue.is_empty() {
|
|
println!("\n{} item(s) to look at, worst first:", queue.len());
|
|
for item in queue.iter().take(10) {
|
|
println!(
|
|
" q{:<3} {}",
|
|
item.number,
|
|
item.notes.first().map(|s| s.as_str()).unwrap_or("")
|
|
);
|
|
}
|
|
return Ok(Outcome::Findings);
|
|
}
|
|
Ok(Outcome::Ok)
|
|
}
|
|
AnalyzeCommand::Irt {
|
|
id,
|
|
model,
|
|
no_priors,
|
|
} => {
|
|
let record = load_record(&catalog, id)?;
|
|
let set = responses_for(&store, &catalog, &record, false)?;
|
|
let mut opts = irt::Options {
|
|
model: model.as_model(),
|
|
..irt::Options::default()
|
|
};
|
|
opts.priors.enabled = !no_priors;
|
|
|
|
let fit = irt::fit(&set.matrix(false), &opts);
|
|
for w in &fit.warnings {
|
|
println!("! {w}\n");
|
|
}
|
|
println!(
|
|
"{} model, {} iteration(s), {}",
|
|
model.as_model().as_str(),
|
|
fit.iterations,
|
|
if fit.converged {
|
|
"converged"
|
|
} else {
|
|
"did NOT converge"
|
|
}
|
|
);
|
|
println!(
|
|
"measures most precisely near θ = {:+.1}\n",
|
|
fit.peak_information()
|
|
);
|
|
|
|
println!(
|
|
"{:>3} {:>6} {:>7} {:>7} {:>7} NOTES",
|
|
"Q", "a", "b", "SE(a)", "SE(b)"
|
|
);
|
|
for item in &fit.items {
|
|
println!(
|
|
"{:>3} {:>6.2} {:>+7.2} {:>7} {:>7} {}",
|
|
item.number,
|
|
item.a,
|
|
item.b,
|
|
item.se_a
|
|
.map(|v| format!("{v:.2}"))
|
|
.unwrap_or_else(|| "-".into()),
|
|
item.se_b
|
|
.map(|v| format!("{v:.2}"))
|
|
.unwrap_or_else(|| "-".into()),
|
|
item.notes.first().map(|s| s.as_str()).unwrap_or("")
|
|
);
|
|
}
|
|
|
|
let mut abilities = fit.abilities.clone();
|
|
abilities.sort_by(|a, b| {
|
|
b.theta
|
|
.partial_cmp(&a.theta)
|
|
.unwrap_or(std::cmp::Ordering::Equal)
|
|
});
|
|
println!(
|
|
"\nability range {:+.2} to {:+.2}",
|
|
abilities.last().map(|a| a.theta).unwrap_or(0.0),
|
|
abilities.first().map(|a| a.theta).unwrap_or(0.0)
|
|
);
|
|
Ok(Outcome::Ok)
|
|
}
|
|
AnalyzeCommand::Students { id } => {
|
|
let record = load_record(&catalog, id)?;
|
|
let set = responses_for(&store, &catalog, &record, false)?;
|
|
let cohort = students::summarize(&set, &catalog.course, Some(&catalog), None);
|
|
|
|
println!(
|
|
"{} student(s), mean {:.0}% (SD {:.1})\n",
|
|
cohort.students.len(),
|
|
cohort.mean_percent,
|
|
cohort.sd_percent
|
|
);
|
|
print!("{}", report::roster(&cohort));
|
|
|
|
if !cohort.class_gaps.is_empty() {
|
|
println!("\nObjectives the class did not meet:");
|
|
for (objective, rate) in &cohort.class_gaps {
|
|
println!(
|
|
" {:>4.0}% {}",
|
|
rate * 100.0,
|
|
catalog.course.objective_text(objective)
|
|
);
|
|
}
|
|
}
|
|
if !cohort.archetypes.is_empty() {
|
|
println!("\nPatterns:");
|
|
for a in &cohort.archetypes {
|
|
println!(" {:<40} {} student(s)", a.label, a.members.len());
|
|
}
|
|
}
|
|
Ok(Outcome::Ok)
|
|
}
|
|
}
|
|
}
|
|
|
|
/// `calibrate`: fold stored statistics back onto the items.
|
|
///
|
|
/// Prints the plan and stops unless `--apply` is given; refuses to write until at
|
|
/// least `--min-n` examinees are pooled.
|
|
pub(crate) fn calibrate(cli: &Cli, args: &CalibrateArgs) -> Result<Outcome> {
|
|
let catalog = load(cli)?;
|
|
let store = Store::open(catalog.layout.data())?;
|
|
let opts = calibrate::Options {
|
|
irt: !args.no_irt,
|
|
include_practice: args.include_practice,
|
|
minimum_n: args.min_n,
|
|
..calibrate::Options::default()
|
|
};
|
|
|
|
let plan = calibrate::plan(&catalog, &store, &opts)?;
|
|
print!("{}", plan.render());
|
|
|
|
if plan.is_empty() {
|
|
return Ok(Outcome::Ok);
|
|
}
|
|
if !args.apply {
|
|
println!(
|
|
"Nothing written. Re-run with --apply to write these {} change(s) into the bank \
|
|
files, then review the git diff.",
|
|
plan.changes.len()
|
|
);
|
|
return Ok(Outcome::Ok);
|
|
}
|
|
|
|
for path in calibrate::apply(&plan)? {
|
|
println!("updated {}", path.display());
|
|
}
|
|
println!("\nReview the diff before committing: git diff banks/");
|
|
Ok(Outcome::Ok)
|
|
}
|
|
|
|
/// `report`: write per-student reports or the instructor's cohort item analysis.
|
|
pub(crate) fn report(cli: &Cli, sub: &ReportCommand) -> Result<Outcome> {
|
|
let catalog = load(cli)?;
|
|
let store = Store::open(catalog.layout.data())?;
|
|
|
|
match sub {
|
|
ReportCommand::Students {
|
|
id,
|
|
html,
|
|
out,
|
|
ability,
|
|
no_comparison,
|
|
} => {
|
|
let record = load_record(&catalog, id)?;
|
|
let set = responses_for(&store, &catalog, &record, false)?;
|
|
let fit = if *ability {
|
|
Some(irt::fit(&set.matrix(false), &irt::Options::default()))
|
|
} else {
|
|
None
|
|
};
|
|
let cohort = students::summarize(&set, &catalog.course, Some(&catalog), fit.as_ref());
|
|
|
|
let opts = report::StudentOptions {
|
|
ability: *ability,
|
|
comparison: !no_comparison,
|
|
..report::StudentOptions::default()
|
|
};
|
|
let dir = out
|
|
.clone()
|
|
.unwrap_or_else(|| catalog.layout.reports().join(id));
|
|
let written =
|
|
report::write_all_students(&dir, &cohort, &catalog.course, &record, &opts, *html)?;
|
|
println!(
|
|
"wrote {} file(s) for {} student(s) in {}",
|
|
written.len(),
|
|
cohort.students.len(),
|
|
dir.display()
|
|
);
|
|
Ok(Outcome::Ok)
|
|
}
|
|
ReportCommand::Cohort { id, html, out } => {
|
|
let record = load_record(&catalog, id)?;
|
|
let set = responses_for(&store, &catalog, &record, false)?;
|
|
let analysis =
|
|
classical::analyze(&set, &Thresholds::default(), Some(&record), Some(&catalog));
|
|
let fit = irt::fit(&set.matrix(false), &irt::Options::default());
|
|
let cohort = students::summarize(&set, &catalog.course, Some(&catalog), Some(&fit));
|
|
|
|
let markdown = report::cohort(&analysis, &cohort, &catalog, &record, Some(&fit));
|
|
let path = out
|
|
.clone()
|
|
.unwrap_or_else(|| catalog.layout.reports().join(format!("{id}-cohort.md")));
|
|
yaml::write_text(&path, &markdown)?;
|
|
println!("wrote {}", path.display());
|
|
|
|
if *html {
|
|
let html_path = path.with_extension("html");
|
|
let title = format!("{} — item analysis", record.assessment.title);
|
|
yaml::write_text(&html_path, &report::to_html(&markdown, &title))?;
|
|
println!("wrote {}", html_path.display());
|
|
}
|
|
Ok(Outcome::Ok)
|
|
}
|
|
}
|
|
}
|
|
|
|
/// `data`: list every administration held in the response store.
|
|
pub(crate) fn data(cli: &Cli) -> Result<Outcome> {
|
|
let layout = Layout::new(&cli.course);
|
|
let store = Store::open(layout.data())?;
|
|
let summaries = store::summarize(&store)?;
|
|
if summaries.is_empty() {
|
|
println!("no stored responses in {}", store.dir.display());
|
|
return Ok(Outcome::Ok);
|
|
}
|
|
println!(
|
|
"{:<40} {:>7} {:>9} {:>7} FILE",
|
|
"ADMINISTRATION", "ROWS", "STUDENTS", "ITEMS"
|
|
);
|
|
for s in &summaries {
|
|
println!(
|
|
"{:<40} {:>7} {:>9} {:>7} {}",
|
|
truncate(&s.administration_id, 40),
|
|
s.rows,
|
|
s.students,
|
|
s.items,
|
|
s.path
|
|
.file_name()
|
|
.map(|f| f.to_string_lossy().to_string())
|
|
.unwrap_or_default()
|
|
);
|
|
}
|
|
Ok(Outcome::Ok)
|
|
}
|