@@ -0,0 +1,386 @@
|
||||
// SPDX-License-Identifier: Prosperity-3.0.0
|
||||
// Copyright Scientific Computing Studio
|
||||
// Source: https://git.scient.ing/education/coursebank
|
||||
|
||||
//! The responses-to-report pipeline.
|
||||
//!
|
||||
//! Once an assessment has been given, responses come back through [`ingest`],
|
||||
//! statistics come out of [`analyze`] (classical, IRT, or per-student), [`calibrate`]
|
||||
//! writes those statistics back onto the items, and [`report`] produces the
|
||||
//! student and cohort documents. [`data`] lists what the response store holds.
|
||||
|
||||
use coursebank::calibrate;
|
||||
use coursebank::canvas;
|
||||
use coursebank::classical::{self, Thresholds};
|
||||
use coursebank::error::Result;
|
||||
use coursebank::gradescope;
|
||||
use coursebank::irt;
|
||||
use coursebank::layout::Layout;
|
||||
use coursebank::report;
|
||||
use coursebank::store::{self, Store};
|
||||
use coursebank::students;
|
||||
use coursebank::yaml;
|
||||
|
||||
use crate::cli::{AnalyzeCommand, CalibrateArgs, Cli, IngestCommand, ReportCommand};
|
||||
use crate::commands::Outcome;
|
||||
use crate::helpers::{context, load, load_record, read_salt, responses_for, truncate};
|
||||
|
||||
/// `ingest`: read a Gradescope directory or a Canvas CSV into the response store.
|
||||
///
|
||||
/// Enriches the parsed responses against the record, optionally pseudonymizes the
|
||||
/// identifiers, and — unless `--dry-run` — writes them in the chosen format.
|
||||
pub(crate) fn ingest(cli: &Cli, sub: &IngestCommand) -> Result<Outcome> {
|
||||
let catalog = load(cli)?;
|
||||
|
||||
let (common, mut set) = match sub {
|
||||
IngestCommand::Gradescope { dir, common } => {
|
||||
let record = load_record(&catalog, &common.assessment)?;
|
||||
let ctx = context(&catalog, &record, common)?;
|
||||
let import = gradescope::ingest_dir(dir, &ctx)?;
|
||||
|
||||
// Grading-time partial credit is an ambiguity signal worth surfacing
|
||||
// right here, while the exam is fresh.
|
||||
for question in &import.questions {
|
||||
for (letter, value, note) in question.partial_credit() {
|
||||
println!(
|
||||
"! q{}: option {letter} earned {value} of {} points at grading time{}",
|
||||
question.number,
|
||||
question.points_possible(),
|
||||
note.map(|n| format!(" — {n}")).unwrap_or_default()
|
||||
);
|
||||
}
|
||||
}
|
||||
(common, import.responses)
|
||||
}
|
||||
IngestCommand::Canvas { file, common } => {
|
||||
let record = load_record(&catalog, &common.assessment)?;
|
||||
let ctx = context(&catalog, &record, common)?;
|
||||
let set = canvas::ingest(file, &ctx, Some(&record), Some(&catalog))?;
|
||||
(common, set)
|
||||
}
|
||||
};
|
||||
|
||||
let record = load_record(&catalog, &common.assessment)?;
|
||||
set.enrich(&record, Some(&catalog));
|
||||
|
||||
if common.pseudonymize {
|
||||
let salt = read_salt(common.salt_file.as_deref())?;
|
||||
set.pseudonymize(&salt);
|
||||
println!("identifiers replaced with keyed pseudonyms");
|
||||
}
|
||||
|
||||
for warning in &set.warnings {
|
||||
println!("! {warning}");
|
||||
}
|
||||
|
||||
println!(
|
||||
"\n{} response(s): {} student(s) x {} item(s)",
|
||||
set.rows.len(),
|
||||
set.students().len(),
|
||||
set.all_items().len()
|
||||
);
|
||||
|
||||
if common.dry_run {
|
||||
println!("(dry run, nothing written)");
|
||||
return Ok(Outcome::Ok);
|
||||
}
|
||||
|
||||
let mut store = Store::open(catalog.layout.data())?;
|
||||
if let Some(format) = common.format {
|
||||
store = store.with_format(format.as_format())?;
|
||||
}
|
||||
for path in store.write(&set)? {
|
||||
println!("wrote {}", path.display());
|
||||
}
|
||||
println!(
|
||||
"\nNext: coursebank analyze items {}\n coursebank report cohort {}",
|
||||
common.assessment, common.assessment
|
||||
);
|
||||
Ok(Outcome::Ok)
|
||||
}
|
||||
|
||||
/// `analyze`: classical item analysis, an IRT fit, or a per-student summary.
|
||||
///
|
||||
/// `items` returns [`Outcome::Findings`] when there is a revise queue.
|
||||
pub(crate) fn analyze(cli: &Cli, sub: &AnalyzeCommand) -> Result<Outcome> {
|
||||
let catalog = load(cli)?;
|
||||
let store = Store::open(catalog.layout.data())?;
|
||||
|
||||
match sub {
|
||||
AnalyzeCommand::Items { id, pooled } => {
|
||||
let record = load_record(&catalog, id)?;
|
||||
let set = responses_for(&store, &catalog, &record, *pooled)?;
|
||||
let analysis =
|
||||
classical::analyze(&set, &Thresholds::default(), Some(&record), Some(&catalog));
|
||||
|
||||
for w in &analysis.warnings {
|
||||
println!("! {w}\n");
|
||||
}
|
||||
println!("{}\n", analysis.reliability.interpretation());
|
||||
println!(
|
||||
"{:>3} {:>5} {:>6} {:>6} {:>6} FLAGS",
|
||||
"Q", "p", "r", "D", "blank"
|
||||
);
|
||||
for item in &analysis.items {
|
||||
println!(
|
||||
"{:>3} {:>5.2} {:>6} {:>6} {:>5.0}% {}",
|
||||
item.number,
|
||||
item.p_value,
|
||||
item.point_biserial
|
||||
.map(|v| format!("{v:+.2}"))
|
||||
.unwrap_or_else(|| "n/a".into()),
|
||||
item.discrimination_index
|
||||
.map(|v| format!("{v:+.2}"))
|
||||
.unwrap_or_else(|| "-".into()),
|
||||
item.blank_rate * 100.0,
|
||||
item.flags
|
||||
.iter()
|
||||
.map(|f| f.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
);
|
||||
}
|
||||
|
||||
let queue = analysis.revise_queue();
|
||||
if !queue.is_empty() {
|
||||
println!("\n{} item(s) to look at, worst first:", queue.len());
|
||||
for item in queue.iter().take(10) {
|
||||
println!(
|
||||
" q{:<3} {}",
|
||||
item.number,
|
||||
item.notes.first().map(|s| s.as_str()).unwrap_or("")
|
||||
);
|
||||
}
|
||||
return Ok(Outcome::Findings);
|
||||
}
|
||||
Ok(Outcome::Ok)
|
||||
}
|
||||
AnalyzeCommand::Irt {
|
||||
id,
|
||||
model,
|
||||
no_priors,
|
||||
} => {
|
||||
let record = load_record(&catalog, id)?;
|
||||
let set = responses_for(&store, &catalog, &record, false)?;
|
||||
let mut opts = irt::Options {
|
||||
model: model.as_model(),
|
||||
..irt::Options::default()
|
||||
};
|
||||
opts.priors.enabled = !no_priors;
|
||||
|
||||
let fit = irt::fit(&set.matrix(false), &opts);
|
||||
for w in &fit.warnings {
|
||||
println!("! {w}\n");
|
||||
}
|
||||
println!(
|
||||
"{} model, {} iteration(s), {}",
|
||||
model.as_model().as_str(),
|
||||
fit.iterations,
|
||||
if fit.converged {
|
||||
"converged"
|
||||
} else {
|
||||
"did NOT converge"
|
||||
}
|
||||
);
|
||||
println!(
|
||||
"measures most precisely near θ = {:+.1}\n",
|
||||
fit.peak_information()
|
||||
);
|
||||
|
||||
println!(
|
||||
"{:>3} {:>6} {:>7} {:>7} {:>7} NOTES",
|
||||
"Q", "a", "b", "SE(a)", "SE(b)"
|
||||
);
|
||||
for item in &fit.items {
|
||||
println!(
|
||||
"{:>3} {:>6.2} {:>+7.2} {:>7} {:>7} {}",
|
||||
item.number,
|
||||
item.a,
|
||||
item.b,
|
||||
item.se_a
|
||||
.map(|v| format!("{v:.2}"))
|
||||
.unwrap_or_else(|| "-".into()),
|
||||
item.se_b
|
||||
.map(|v| format!("{v:.2}"))
|
||||
.unwrap_or_else(|| "-".into()),
|
||||
item.notes.first().map(|s| s.as_str()).unwrap_or("")
|
||||
);
|
||||
}
|
||||
|
||||
let mut abilities = fit.abilities.clone();
|
||||
abilities.sort_by(|a, b| {
|
||||
b.theta
|
||||
.partial_cmp(&a.theta)
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
});
|
||||
println!(
|
||||
"\nability range {:+.2} to {:+.2}",
|
||||
abilities.last().map(|a| a.theta).unwrap_or(0.0),
|
||||
abilities.first().map(|a| a.theta).unwrap_or(0.0)
|
||||
);
|
||||
Ok(Outcome::Ok)
|
||||
}
|
||||
AnalyzeCommand::Students { id } => {
|
||||
let record = load_record(&catalog, id)?;
|
||||
let set = responses_for(&store, &catalog, &record, false)?;
|
||||
let cohort = students::summarize(&set, &catalog.course, Some(&catalog), None);
|
||||
|
||||
println!(
|
||||
"{} student(s), mean {:.0}% (SD {:.1})\n",
|
||||
cohort.students.len(),
|
||||
cohort.mean_percent,
|
||||
cohort.sd_percent
|
||||
);
|
||||
print!("{}", report::roster(&cohort));
|
||||
|
||||
if !cohort.class_gaps.is_empty() {
|
||||
println!("\nObjectives the class did not meet:");
|
||||
for (objective, rate) in &cohort.class_gaps {
|
||||
println!(
|
||||
" {:>4.0}% {}",
|
||||
rate * 100.0,
|
||||
catalog.course.objective_text(objective)
|
||||
);
|
||||
}
|
||||
}
|
||||
if !cohort.archetypes.is_empty() {
|
||||
println!("\nPatterns:");
|
||||
for a in &cohort.archetypes {
|
||||
println!(" {:<40} {} student(s)", a.label, a.members.len());
|
||||
}
|
||||
}
|
||||
Ok(Outcome::Ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `calibrate`: fold stored statistics back onto the items.
|
||||
///
|
||||
/// Prints the plan and stops unless `--apply` is given; refuses to write until at
|
||||
/// least `--min-n` examinees are pooled.
|
||||
pub(crate) fn calibrate(cli: &Cli, args: &CalibrateArgs) -> Result<Outcome> {
|
||||
let catalog = load(cli)?;
|
||||
let store = Store::open(catalog.layout.data())?;
|
||||
let opts = calibrate::Options {
|
||||
irt: !args.no_irt,
|
||||
include_practice: args.include_practice,
|
||||
minimum_n: args.min_n,
|
||||
..calibrate::Options::default()
|
||||
};
|
||||
|
||||
let plan = calibrate::plan(&catalog, &store, &opts)?;
|
||||
print!("{}", plan.render());
|
||||
|
||||
if plan.is_empty() {
|
||||
return Ok(Outcome::Ok);
|
||||
}
|
||||
if !args.apply {
|
||||
println!(
|
||||
"Nothing written. Re-run with --apply to write these {} change(s) into the bank \
|
||||
files, then review the git diff.",
|
||||
plan.changes.len()
|
||||
);
|
||||
return Ok(Outcome::Ok);
|
||||
}
|
||||
|
||||
for path in calibrate::apply(&plan)? {
|
||||
println!("updated {}", path.display());
|
||||
}
|
||||
println!("\nReview the diff before committing: git diff banks/");
|
||||
Ok(Outcome::Ok)
|
||||
}
|
||||
|
||||
/// `report`: write per-student reports or the instructor's cohort item analysis.
|
||||
pub(crate) fn report(cli: &Cli, sub: &ReportCommand) -> Result<Outcome> {
|
||||
let catalog = load(cli)?;
|
||||
let store = Store::open(catalog.layout.data())?;
|
||||
|
||||
match sub {
|
||||
ReportCommand::Students {
|
||||
id,
|
||||
html,
|
||||
out,
|
||||
ability,
|
||||
no_comparison,
|
||||
} => {
|
||||
let record = load_record(&catalog, id)?;
|
||||
let set = responses_for(&store, &catalog, &record, false)?;
|
||||
let fit = if *ability {
|
||||
Some(irt::fit(&set.matrix(false), &irt::Options::default()))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let cohort = students::summarize(&set, &catalog.course, Some(&catalog), fit.as_ref());
|
||||
|
||||
let opts = report::StudentOptions {
|
||||
ability: *ability,
|
||||
comparison: !no_comparison,
|
||||
..report::StudentOptions::default()
|
||||
};
|
||||
let dir = out
|
||||
.clone()
|
||||
.unwrap_or_else(|| catalog.layout.reports().join(id));
|
||||
let written =
|
||||
report::write_all_students(&dir, &cohort, &catalog.course, &record, &opts, *html)?;
|
||||
println!(
|
||||
"wrote {} file(s) for {} student(s) in {}",
|
||||
written.len(),
|
||||
cohort.students.len(),
|
||||
dir.display()
|
||||
);
|
||||
Ok(Outcome::Ok)
|
||||
}
|
||||
ReportCommand::Cohort { id, html, out } => {
|
||||
let record = load_record(&catalog, id)?;
|
||||
let set = responses_for(&store, &catalog, &record, false)?;
|
||||
let analysis =
|
||||
classical::analyze(&set, &Thresholds::default(), Some(&record), Some(&catalog));
|
||||
let fit = irt::fit(&set.matrix(false), &irt::Options::default());
|
||||
let cohort = students::summarize(&set, &catalog.course, Some(&catalog), Some(&fit));
|
||||
|
||||
let markdown = report::cohort(&analysis, &cohort, &catalog, &record, Some(&fit));
|
||||
let path = out
|
||||
.clone()
|
||||
.unwrap_or_else(|| catalog.layout.reports().join(format!("{id}-cohort.md")));
|
||||
yaml::write_text(&path, &markdown)?;
|
||||
println!("wrote {}", path.display());
|
||||
|
||||
if *html {
|
||||
let html_path = path.with_extension("html");
|
||||
let title = format!("{} — item analysis", record.assessment.title);
|
||||
yaml::write_text(&html_path, &report::to_html(&markdown, &title))?;
|
||||
println!("wrote {}", html_path.display());
|
||||
}
|
||||
Ok(Outcome::Ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `data`: list every administration held in the response store.
|
||||
pub(crate) fn data(cli: &Cli) -> Result<Outcome> {
|
||||
let layout = Layout::new(&cli.course);
|
||||
let store = Store::open(layout.data())?;
|
||||
let summaries = store::summarize(&store)?;
|
||||
if summaries.is_empty() {
|
||||
println!("no stored responses in {}", store.dir.display());
|
||||
return Ok(Outcome::Ok);
|
||||
}
|
||||
println!(
|
||||
"{:<40} {:>7} {:>9} {:>7} FILE",
|
||||
"ADMINISTRATION", "ROWS", "STUDENTS", "ITEMS"
|
||||
);
|
||||
for s in &summaries {
|
||||
println!(
|
||||
"{:<40} {:>7} {:>9} {:>7} {}",
|
||||
truncate(&s.administration_id, 40),
|
||||
s.rows,
|
||||
s.students,
|
||||
s.items,
|
||||
s.path
|
||||
.file_name()
|
||||
.map(|f| f.to_string_lossy().to_string())
|
||||
.unwrap_or_default()
|
||||
);
|
||||
}
|
||||
Ok(Outcome::Ok)
|
||||
}
|
||||
Reference in New Issue
Block a user