feat: improvement
Pipeline / check (pull_request) Successful in 5m0s
Pipeline / docs (pull_request) Skipped
Pipeline / nightly (pull_request) Skipped
Pipeline / release (pull_request) Skipped

This commit is contained in:
alexm committed 2026-09-27 01:11:09 -04:00
1 parent 5ac1e317c0
commit c6d6ee10b6
18 files changed
+2538 -108

No files matched your search

+6 -32
View File
@@ -530,38 +530,12 @@ fn validate_item(
}
// --- calibration plausibility ------
if let Some(c) = &it.calibration {
if let Some(p) = c.p_value {
if !(0.0..=1.0).contains(&p) {
issues.push(format!(
"calibration.p_value must be between 0 and 1, got {p}"
));
}
}
if let Some(r) = c.point_biserial {
if !(-1.0..=1.0).contains(&r) {
issues.push(format!(
"calibration.point_biserial must be between -1 and 1, got {r}"
));
}
}
for option in c.option_stats.keys() {
if it.option(option).is_none() {
issues.push(format!(
"calibration.option_stats has `{option}`, which is not an option of this item"
));
}
}
if let Some(irt) = &c.irt {
if irt.a <= 0.0 {
issues.push(format!("calibration.irt.a must be positive, got {}", irt.a));
}
if let Some(cp) = irt.c {
if !(0.0..1.0).contains(&cp) {
issues.push(format!("calibration.irt.c must be in [0, 1), got {cp}"));
}
}
}
if it.calibration.is_some() {
issues.push(
"has a `calibration:` block, but statistics live in analysis/calibration.yaml since \
2.0. A bank's diff should be a change of intent, not the output of a grading run."
.to_string(),
);
}
// --- retirement -----
+838
View File
@@ -0,0 +1,838 @@
// SPDX-License-Identifier: Prosperity-3.0.0
// Copyright Scientific Computing Studio
// Source: https://git.scient.ing/education/coursebank
//! Where the statistics live, which is not in the bank.
//!
//! A bank file is a reviewed artifact: someone wrote the question, someone
//! argued about the distractors, and the diff on it should be a change of
//! intent. Statistics are neither reviewed nor intended — they are what
//! happened — and writing them back into the bank means every grading run
//! produces a diff on a file whose history is supposed to be about wording.
//!
//! So the evidence lives in `analysis/`, in two kinds of file:
//!
//! | File | Format | Rewritten? | Holds |
//! |:--|:--|:--|:--|
//! | `analysis/administrations/<id>-items.csv` | CSV | never | one row per question |
//! | `analysis/administrations/<id>-options.csv` | CSV | never | one row per question and option |
//! | `analysis/calibration.yaml` | YAML | by `calibrate` | the pooled per-item view |
//!
//! The format follows the shape. An administration record is a table — fixed
//! columns, one row per question, machine-written, never hand-edited — so it is
//! CSV: one line per item rather than fifteen, which diffs better, and it loads
//! straight into pandas or DuckDB, which is much of the point of committing it.
//! The pooled view is not a table. It is three levels deep, item to variant to
//! option history, with fitted parameters and variable-length lists, and as CSV
//! that would be three files joined by keys — a relational schema for the one
//! file a person actually reads in a pull request. That stays YAML.
//!
//! Neither is Parquet, and the reason is the review workflow: a binary file
//! shows nothing in a diff and cannot be merged. Parquet is right for `data/`
//! precisely because that is bulk, ignored, and never reviewed.
//!
//! An administration file is written once and not touched again, for the same
//! reason a seal is not: it is a record of a thing that happened on a day. The
//! calibration file is the accepted rollup — what `lint` compares your
//! predictions against, and what a report reads — and `calibrate` proposes
//! changes to it as a diff you review before committing.
//!
//! Both are meant to be committed. Neither can carry student data, and that is
//! a property of the types rather than a promise: there is no field for a
//! student key, an identifier, a section, or an ability estimate, and the
//! structures reject unknown keys, so a file carrying one fails to load rather
//! than being quietly accepted. Everything per-person stays in `data/`, which
//! is what your `.gitignore` is for.
//!
//! # Linking back to the bank
//!
//! By item id, which since 2.0 names the item course-wide and has no file name
//! in it, and by variant digest, which says which option set the numbers
//! describe. A record also carries the stem digest it was measured against, so
//! [`CalibrationFile::validate`] can say that an item has been reworded since —
//! the statistics then describe a question that no longer exists under that id.
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
use crate::course::SCHEMA_VERSION;
use crate::date::Date;
use crate::error::{Error, Result};
use crate::item::{Calibration, IrtModel, IrtParams, OptionStat};
use crate::taxonomy::Flag;
use crate::yaml;
/// The file name of the pooled calibration store, under `analysis/`.
pub const CALIBRATION_FILE: &str = "calibration.yaml";
/// The pooled per-item statistics: `analysis/calibration.yaml`.
///
/// Keyed by item id. This is the file `calibrate` rewrites and the one
/// everything else reads; [`crate::catalog::Catalog::load`] fills each item's
/// in-memory calibration from it, so nothing downstream has to know where the
/// numbers came from.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct CalibrationFile {
/// Schema version.
#[serde(
default = "default_version",
deserialize_with = "yaml::flexible_string"
)]
pub schema_version: String,
/// One entry per calibrated item, by item id.
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub items: BTreeMap<String, Calibration>,
}
impl CalibrationFile {
/// Loads the store, or an empty one when the file does not exist yet.
///
/// Absence is not an error: a course that has not graded anything has no
/// statistics, and every command that reads them has to work anyway.
///
/// # Arguments
///
/// * `path` - the file, usually `analysis/calibration.yaml`.
///
/// # Returns
///
/// The store.
///
/// # Errors
///
/// Returns [`Error::Yaml`] when the file exists and does not parse.
pub fn load(path: &Path) -> Result<CalibrationFile> {
if !path.is_file() {
return Ok(CalibrationFile::default());
}
yaml::read(path)
}
/// Writes the store.
///
/// # Arguments
///
/// * `path` - the destination.
///
/// # Errors
///
/// Returns [`Error::Io`] on a write failure.
pub fn save(&self, path: &Path) -> Result<()> {
yaml::write(path, self)
}
/// The calibration recorded for one item.
///
/// # Arguments
///
/// * `item` - the item id.
///
/// # Returns
///
/// The record, or `None` when the item has never been calibrated.
pub fn get(&self, item: &str) -> Option<&Calibration> {
self.items.get(item)
}
/// Checks the store against the bank it describes.
///
/// The checks that matter for a file kept apart from what it refers to: a
/// record for an item that no longer exists, an option id the item does not
/// have, and — the one worth having — statistics measured against a stem
/// that has since been reworded, which since 2.0 means they describe a
/// different question wearing the same id.
///
/// # Arguments
///
/// * `catalog` - the loaded course.
///
/// # Returns
///
/// One message per problem, empty when the store agrees with the bank.
pub fn validate(&self, catalog: &crate::catalog::Catalog) -> Vec<String> {
let mut issues = Vec::new();
for (id, calibration) in &self.items {
let Some(entry) = catalog.get(id) else {
issues.push(format!(
"calibration for `{id}`: no such item. Statistics outlive an item only if \
it is retired, not deleted — a retired item keeps its id so its numbers \
still mean something."
));
continue;
};
let item = &entry.item;
for variant in &calibration.variants {
for option in variant.option_stats.keys() {
if item.option(option).is_none() {
issues.push(format!(
"calibration for `{id}`: variant `{}` has statistics for `{option}`, \
which is not an option of this item",
short(&variant.variant)
));
}
}
if !variant.key.is_empty()
&& variant.variant != item.variant_digest(&variant.key, &variant.distractors)
{
issues.push(format!(
"calibration for `{id}`: variant `{}` was measured against an option set \
that has since been reworded, so its numbers describe wording no \
student now sees",
short(&variant.variant)
));
}
}
for option in calibration.options.keys() {
if item.option(option).is_none() {
issues.push(format!(
"calibration for `{id}`: an option history names `{option}`, which is \
not an option of this item"
));
}
}
}
issues
}
}
/// What one administration measured: `analysis/administrations/<id>.yaml`.
///
/// Written once, when the exam is analyzed, and never rewritten. It is the
/// audit trail under [`CalibrationFile`]: the pooled numbers say an item sits
/// at 0.63, and these say which exams that came from and what each one saw.
/// Keeping them also means the history survives losing `data/`, which is
/// ignored by git and rotates.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct MeasurementFile {
/// Schema version.
#[serde(
default = "default_version",
deserialize_with = "yaml::flexible_string"
)]
pub schema_version: String,
/// What was administered, and how it was analyzed.
pub administration: MeasurementMeta,
/// One entry per question, in the order it was printed.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub items: Vec<Measurement>,
}
/// What an administration was, for a reader two years later.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct MeasurementMeta {
/// The administration id these numbers came from.
pub id: String,
/// The assessment that was administered.
pub assessment: String,
/// The term.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub term: Option<String>,
/// The date it was given.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub date: Option<Date>,
/// The forms in play.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub forms: Vec<String>,
/// How many examinees the numbers pool over.
///
/// The one number to read before any of the others. A point-biserial on
/// twenty-seven students is a different kind of claim than one on three
/// hundred, and nothing below records how thin it is.
pub n_examinees: usize,
/// The item response model fitted, when one was.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub model: Option<IrtModel>,
/// When the analysis was run.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub generated: Option<Date>,
/// The version of the tool that ran it, since the numbers depend on it.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub coursebank: Option<String>,
}
/// One question's statistics from one administration.
///
/// Cohort aggregates only. There is deliberately no per-section or per-form
/// breakdown: those get small, and a small cell crossed with anything else is
/// how an aggregate stops being one.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Measurement {
/// The item id.
pub item: String,
/// The question number it was printed as.
pub number: u32,
/// The option set administered. See [`crate::item::Item::variant_digest`].
#[serde(default, skip_serializing_if = "Option::is_none")]
pub variant: Option<String>,
/// The stem as administered. See [`crate::item::Item::stem_digest`].
#[serde(default, skip_serializing_if = "Option::is_none")]
pub stem_digest: Option<String>,
/// Examinees who saw it.
pub n: usize,
/// Proportion correct.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub p_value: Option<f64>,
/// Corrected item-total point-biserial correlation.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub point_biserial: Option<f64>,
/// Upper-minus-lower-group discrimination index.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub discrimination_index: Option<f64>,
/// The option ids keyed correct, so a row in the options file says whether
/// it describes the answer or a distractor without a join.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub key: Vec<String>,
/// Per-option behaviour, by option id.
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub option_stats: BTreeMap<String, OptionStat>,
/// Fitted parameters, when the sample supported a fit.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub irt: Option<IrtParams>,
/// Machine-detected problems with this question on this administration.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub flags: Vec<Flag>,
}
/// The columns of an items file, and the only ones accepted on read.
///
/// An allowlist rather than a type-level guarantee. In YAML the structures
/// reject unknown keys, so a file carrying a student column could not be
/// loaded; CSV readers are tolerant of extra columns, so the same assurance has
/// to be an explicit check. This is it, and [`MeasurementFile::read_csv`]
/// refuses any header not named here.
pub const ITEM_COLUMNS: [&str; 16] = [
"administration_id",
"assessment",
"term",
"date",
"forms",
"n_examinees",
"coursebank",
"generated",
"item",
"number",
"variant",
"stem_digest",
"n",
"p_value",
"point_biserial",
"discrimination_index",
];
/// The columns of an options file, and the only ones accepted on read.
pub const OPTION_COLUMNS: [&str; 9] = [
"administration_id",
"item",
"number",
"option",
"keyed",
"selection_rate",
"point_biserial",
"upper_group_rate",
"lower_group_rate",
];
/// One row of an items file.
#[derive(Debug, Clone, Serialize, Deserialize)]
struct ItemRow {
administration_id: String,
assessment: String,
term: String,
date: String,
forms: String,
n_examinees: usize,
coursebank: String,
generated: String,
item: String,
number: u32,
variant: String,
stem_digest: String,
n: usize,
p_value: String,
point_biserial: String,
discrimination_index: String,
}
/// One row of an options file.
#[derive(Debug, Clone, Serialize, Deserialize)]
struct OptionRow {
administration_id: String,
item: String,
number: u32,
option: String,
keyed: bool,
selection_rate: String,
point_biserial: String,
upper_group_rate: String,
lower_group_rate: String,
}
impl MeasurementFile {
/// The two file names this administration writes, items first.
///
/// # Arguments
///
/// * `dir` - usually `analysis/administrations`.
///
/// # Returns
///
/// The items path and the options path.
pub fn paths(&self, dir: &Path) -> (PathBuf, PathBuf) {
let stem = crate::course::slugify(&self.administration.id);
(
dir.join(format!("{stem}-items.csv")),
dir.join(format!("{stem}-options.csv")),
)
}
/// Writes the two files, refusing to overwrite either.
///
/// An administration is a thing that happened once, so replacing its record
/// is a deliberate act: delete the files first if you mean to re-analyze.
///
/// Two files rather than one because the data is two shapes — one row per
/// question, one row per question and option — and a single sparse table
/// serves neither. The administration's metadata repeats on every row,
/// which is what makes each file independently loadable and is the same
/// convention the response store already uses.
///
/// # Arguments
///
/// * `dir` - the directory to write into.
///
/// # Returns
///
/// The paths written.
///
/// # Errors
///
/// Returns [`Error::Usage`] when either file exists, and [`Error::Io`] or
/// [`Error::Csv`] on a write failure.
pub fn write_csv(&self, dir: &Path) -> Result<Vec<PathBuf>> {
let (items_path, options_path) = self.paths(dir);
for path in [&items_path, &options_path] {
if path.exists() {
return Err(Error::usage(format!(
"{} already records this administration. It happened once, so replacing it \
is a deliberate act: delete it first if you mean to re-analyze.",
path.display()
)));
}
}
std::fs::create_dir_all(dir).map_err(|e| Error::io(dir, e))?;
let meta = &self.administration;
let mut items = csv::Writer::from_path(&items_path).map_err(|e| Error::Csv {
path: items_path.clone(),
source: e,
})?;
let mut options = csv::Writer::from_path(&options_path).map_err(|e| Error::Csv {
path: options_path.clone(),
source: e,
})?;
for measurement in &self.items {
items
.serialize(ItemRow {
administration_id: meta.id.clone(),
assessment: meta.assessment.clone(),
term: meta.term.clone().unwrap_or_default(),
date: meta.date.map(|d| d.to_string()).unwrap_or_default(),
forms: meta.forms.join(";"),
n_examinees: meta.n_examinees,
coursebank: meta.coursebank.clone().unwrap_or_default(),
generated: meta.generated.map(|d| d.to_string()).unwrap_or_default(),
item: measurement.item.clone(),
number: measurement.number,
variant: measurement.variant.clone().unwrap_or_default(),
stem_digest: measurement.stem_digest.clone().unwrap_or_default(),
n: measurement.n,
p_value: number(measurement.p_value),
point_biserial: number(measurement.point_biserial),
discrimination_index: number(measurement.discrimination_index),
})
.map_err(|e| Error::Csv {
path: items_path.clone(),
source: e,
})?;
for (option, stat) in &measurement.option_stats {
options
.serialize(OptionRow {
administration_id: meta.id.clone(),
item: measurement.item.clone(),
number: measurement.number,
option: option.clone(),
keyed: measurement.key.iter().any(|k| k == option),
selection_rate: number(stat.selection_rate),
point_biserial: number(stat.point_biserial),
upper_group_rate: number(stat.upper_group_rate),
lower_group_rate: number(stat.lower_group_rate),
})
.map_err(|e| Error::Csv {
path: options_path.clone(),
source: e,
})?;
}
}
items.flush().map_err(|e| Error::io(&items_path, e))?;
options.flush().map_err(|e| Error::io(&options_path, e))?;
Ok(vec![items_path, options_path])
}
/// Reads one administration back from its two files.
///
/// # Arguments
///
/// * `items_path` - the items file. The options file is found beside it.
///
/// # Returns
///
/// The administration, with its per-option statistics reattached.
///
/// # Errors
///
/// Returns [`Error::Csv`] on a parse failure and [`Error::Invalid`] when a
/// file carries a column that is not in [`ITEM_COLUMNS`] or
/// [`OPTION_COLUMNS`] — which is how a student column is caught.
pub fn read_csv(items_path: &Path) -> Result<MeasurementFile> {
let options_path = PathBuf::from(
items_path
.to_string_lossy()
.replace("-items.csv", "-options.csv"),
);
let mut reader = open_csv(items_path, &ITEM_COLUMNS)?;
let mut meta = MeasurementMeta::default();
let mut items: Vec<Measurement> = Vec::new();
for row in reader.deserialize::<ItemRow>() {
let row = row.map_err(|e| Error::Csv {
path: items_path.to_path_buf(),
source: e,
})?;
meta = MeasurementMeta {
id: row.administration_id.clone(),
assessment: row.assessment.clone(),
term: some(&row.term),
date: parse_date(&row.date),
forms: row
.forms
.split(';')
.filter(|f| !f.is_empty())
.map(str::to_string)
.collect(),
n_examinees: row.n_examinees,
model: meta.model,
generated: parse_date(&row.generated),
coursebank: some(&row.coursebank),
};
items.push(Measurement {
item: row.item,
number: row.number,
variant: some(&row.variant),
stem_digest: some(&row.stem_digest),
n: row.n,
p_value: parse(&row.p_value),
point_biserial: parse(&row.point_biserial),
discrimination_index: parse(&row.discrimination_index),
..Measurement::default()
});
}
if options_path.is_file() {
let mut reader = open_csv(&options_path, &OPTION_COLUMNS)?;
for row in reader.deserialize::<OptionRow>() {
let row = row.map_err(|e| Error::Csv {
path: options_path.clone(),
source: e,
})?;
let Some(target) = items.iter_mut().find(|i| i.number == row.number) else {
continue;
};
if row.keyed && !target.key.contains(&row.option) {
target.key.push(row.option.clone());
}
target.option_stats.insert(
row.option,
OptionStat {
selection_rate: parse(&row.selection_rate),
point_biserial: parse(&row.point_biserial),
upper_group_rate: parse(&row.upper_group_rate),
lower_group_rate: parse(&row.lower_group_rate),
},
);
}
}
Ok(MeasurementFile {
schema_version: default_version(),
administration: meta,
items,
})
}
/// Reads every administration under a directory, oldest first.
///
/// # Arguments
///
/// * `dir` - usually `analysis/administrations`.
///
/// # Returns
///
/// The administrations, empty when the directory does not exist.
///
/// # Errors
///
/// Propagates read failures.
pub fn load_all(dir: &Path) -> Result<Vec<MeasurementFile>> {
if !dir.is_dir() {
return Ok(Vec::new());
}
let mut paths: Vec<PathBuf> = std::fs::read_dir(dir)
.map_err(|e| Error::io(dir, e))?
.filter_map(|e| e.ok().map(|e| e.path()))
.filter(|p| p.to_string_lossy().ends_with("-items.csv"))
.collect();
paths.sort();
let mut out = Vec::new();
for path in &paths {
out.push(MeasurementFile::read_csv(path)?);
}
out.sort_by(|a, b| {
a.administration
.date
.cmp(&b.administration.date)
.then(a.administration.id.cmp(&b.administration.id))
});
Ok(out)
}
}
/// Opens a CSV and refuses any column that is not on the allowlist.
fn open_csv(path: &Path, allowed: &[&str]) -> Result<csv::Reader<std::fs::File>> {
let mut reader = csv::Reader::from_path(path).map_err(|e| Error::Csv {
path: path.to_path_buf(),
source: e,
})?;
let headers = reader
.headers()
.map_err(|e| Error::Csv {
path: path.to_path_buf(),
source: e,
})?
.clone();
let unexpected: Vec<&str> = headers.iter().filter(|h| !allowed.contains(h)).collect();
if !unexpected.is_empty() {
return Err(Error::Invalid(vec![format!(
"{}: unexpected column(s) {}. These files are committed, so they hold cohort \
aggregates and nothing else — anything per-student belongs in data/, which is \
ignored.",
path.display(),
unexpected.join(", ")
)]));
}
Ok(reader)
}
/// A float as a CSV cell, empty when absent.
fn number(value: Option<f64>) -> String {
value.map(|v| format!("{v}")).unwrap_or_default()
}
/// A CSV cell as a float, absent when empty or unparseable.
fn parse(cell: &str) -> Option<f64> {
cell.trim().parse().ok()
}
/// A CSV cell as a string, absent when empty.
fn some(cell: &str) -> Option<String> {
(!cell.trim().is_empty()).then(|| cell.trim().to_string())
}
/// A CSV cell as a date, absent when empty or unparseable.
fn parse_date(cell: &str) -> Option<Date> {
some(cell).and_then(|c| c.parse().ok())
}
/// The schema version new files are written with.
fn default_version() -> String {
SCHEMA_VERSION.to_string()
}
/// A digest shortened for a message.
fn short(digest: &str) -> String {
digest.chars().take(8).collect()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::item::VariantCalibration;
fn tmp(tag: &str) -> std::path::PathBuf {
let p = std::env::temp_dir().join(format!("coursebank-cal-{tag}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&p);
std::fs::create_dir_all(&p).unwrap();
p
}
#[test]
fn an_absent_store_is_empty_rather_than_an_error() {
let dir = tmp("absent");
let store = CalibrationFile::load(&dir.join(CALIBRATION_FILE)).unwrap();
assert!(store.items.is_empty());
assert!(store.get("q-x").is_none());
}
#[test]
fn the_store_round_trips() {
let dir = tmp("round");
let path = dir.join(CALIBRATION_FILE);
let mut store = CalibrationFile::default();
store.items.insert(
"q-x".to_string(),
Calibration {
n_examinees: Some(27),
p_value: Some(0.63),
variants: vec![VariantCalibration {
variant: "4c81fa".into(),
n_examinees: Some(27),
..VariantCalibration::default()
}],
..Calibration::default()
},
);
store.save(&path).unwrap();
let back = CalibrationFile::load(&path).unwrap();
assert_eq!(back.get("q-x").unwrap().p_value, Some(0.63));
assert_eq!(back.get("q-x").unwrap().variants.len(), 1);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn an_administration_round_trips_through_two_csvs() {
let dir = tmp("csv");
let file = MeasurementFile {
schema_version: default_version(),
administration: MeasurementMeta {
id: "e1-2026f".into(),
assessment: "e1".into(),
term: Some("2026f".into()),
date: Some(Date::new(2026, 9, 15).unwrap()),
forms: vec!["A".into(), "B".into()],
n_examinees: 27,
model: None,
generated: Some(Date::new(2026, 9, 27).unwrap()),
coursebank: Some("0.0.0".into()),
},
items: vec![Measurement {
item: "q-fastq-quality-length-match".into(),
number: 1,
variant: Some("237d62f9af222f78".into()),
stem_digest: Some("8b22e0".into()),
n: 27,
p_value: Some(0.5926),
point_biserial: Some(0.31),
discrimination_index: None,
key: vec!["o-fourth-line".into()],
option_stats: [
(
"o-fourth-line".to_string(),
OptionStat {
selection_rate: Some(0.5926),
point_biserial: Some(0.31),
upper_group_rate: None,
lower_group_rate: None,
},
),
(
"o-third-line".to_string(),
OptionStat {
selection_rate: Some(0.1852),
point_biserial: Some(-0.18),
upper_group_rate: None,
lower_group_rate: None,
},
),
]
.into_iter()
.collect(),
irt: None,
flags: Vec::new(),
}],
};
let written = file.write_csv(&dir).unwrap();
assert_eq!(written.len(), 2, "one table per shape");
assert!(written[0].ends_with("e1-2026f-items.csv"));
assert!(written[1].ends_with("e1-2026f-options.csv"));
// One line per question, loadable on its own: the administration's
// metadata repeats on the row, as the response store already does.
let items = std::fs::read_to_string(&written[0]).unwrap();
assert!(
items.starts_with("administration_id,assessment,term,date,forms"),
"{items}"
);
assert!(
items.contains("e1-2026f,e1,2026f,2026-09-15,A;B,27"),
"{items}"
);
let options = std::fs::read_to_string(&written[1]).unwrap();
assert!(options.contains("o-fourth-line,true"), "{options}");
assert!(options.contains("o-third-line,false"), "{options}");
let back = MeasurementFile::read_csv(&written[0]).unwrap();
assert_eq!(back.administration.id, "e1-2026f");
assert_eq!(back.administration.n_examinees, 27);
assert_eq!(back.administration.forms, vec!["A", "B"]);
assert_eq!(back.items.len(), 1);
assert_eq!(back.items[0].p_value, Some(0.5926));
assert_eq!(back.items[0].key, vec!["o-fourth-line"]);
assert_eq!(back.items[0].option_stats.len(), 2);
assert_eq!(back.items[0].discrimination_index, None);
assert_eq!(MeasurementFile::load_all(&dir).unwrap().len(), 1);
// It happened once, so the record is not replaced by accident.
let err = file.write_csv(&dir).unwrap_err().to_string();
assert!(err.contains("deliberate"), "{err}");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn a_student_column_is_refused_on_read() {
let dir = tmp("identifiers");
let path = dir.join("e1-2026f-items.csv");
std::fs::write(
&path,
"administration_id,assessment,item,number,n,student_key\n e1-2026f,e1,q-x,1,27,abc123\n",
)
.unwrap();
// In YAML this fell out of the type, which rejects unknown keys. A CSV
// reader tolerates extra columns, so the same assurance has to be an
// explicit allowlist — and this is the test that it is one.
let err = MeasurementFile::read_csv(&path).unwrap_err().to_string();
assert!(err.contains("student_key"), "{err}");
assert!(err.contains("cohort aggregates"), "{err}");
let _ = std::fs::remove_dir_all(&dir);
}
}
+18
View File
@@ -75,6 +75,12 @@ pub struct Catalog {
pub entries: Vec<Entry>,
/// Bank metadata by bank id.
pub banks: BTreeMap<String, crate::bank::BankMeta>,
/// The statistics, loaded from `analysis/` rather than from the banks.
///
/// Each entry's [`crate::item::Item::calibration`] is filled from this at
/// load, so everything downstream reads one item and does not have to know
/// that the numbers and the wording come from different files.
pub calibration: crate::calibration::CalibrationFile,
/// Map from global id to index into `entries`.
index: BTreeMap<String, usize>,
}
@@ -104,7 +110,10 @@ impl Catalog {
entries: Vec::new(),
banks: BTreeMap::new(),
index: BTreeMap::new(),
calibration: crate::calibration::CalibrationFile::default(),
};
catalog.calibration =
crate::calibration::CalibrationFile::load(&catalog.layout.calibration_file())?;
let mut problems = Vec::new();
let mut files = yaml::list_yaml(&catalog.layout.banks())?;
@@ -145,6 +154,15 @@ impl Catalog {
if !problems.is_empty() {
return Err(Error::Invalid(problems));
}
// Statistics are attached here rather than parsed from the bank, which
// is why an item can be read as one thing while its wording and its
// evidence live in files with different review cycles.
for entry in &mut catalog.entries {
if let Some(calibration) = catalog.calibration.items.get(&entry.uid) {
entry.item.calibration = Some(calibration.clone());
}
}
Ok(catalog)
}
+69 -9
View File
@@ -800,15 +800,25 @@ pub struct Objective {
/// The lectures that develop it.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub lectures: Vec<String>,
/// Position in teaching order, low first.
/// Position in teaching order, low first. Derived; authoring it is
/// deprecated.
///
/// The registry is a map, so declaration order is lost on load, and sorting
/// The registry is a map, so declaration order is lost on load and sorting
/// by id would put `lo-enthalpy` before `lo-first-law` when the second is a
/// prerequisite of the first. Anything that prints objectives in the order
/// you teach them, a lecture page above all, needs this. Objectives without
/// it sort last, by id.
/// prerequisite of the first. Something has to supply the order.
///
/// Since 2.0 that something is [`Lecture::teaches`], which is a sequence:
/// the position of an objective in the list of what a lecture covers, and
/// the position of the lecture in the course, together say when it is
/// taught. [`fragment::assemble`] fills this in from those two, so a course
/// that used to carry thirty-nine hand-kept integers now carries none, and
/// inserting an objective is a one-line edit rather than a renumber.
///
/// An authored value still wins, so a 1.0 course loads unchanged.
/// `coursebank migrate order` removes them.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub order: Option<u32>,
/// The highest level you intend to assess this objective at. Assembling an
/// item above the ceiling is a warning: either the item overreaches or the
/// ceiling needs raising.
@@ -865,8 +875,20 @@ pub struct Target {
///
/// Ordered within its objective rather than across the course, so two
/// targets under different objectives never compete for a position and
/// inserting one renumbers nothing outside its own group.
#[serde(default, skip_serializing_if = "Option::is_none")]
/// Retained only so a pre-2.0 course still loads. Ignored.
///
/// Targets do not have an order. An objective's targets are a set of
/// question templates, not steps in a sequence: they are not taught in
/// order, an exam samples from them rather than working through them, and
/// the study workflow reads the list as a checklist and counts what it can
/// do cold. A position would assert a sequence that does not exist.
///
/// Where a list has to be printed, [`CourseFile::targets`] orders it by
/// ceiling and then by id. Where one target genuinely depends on another,
/// that is [`Target::prerequisites`], which says so directly.
///
/// `coursebank migrate order` removes it.
#[serde(default, skip_serializing)]
pub order: Option<u32>,
/// The highest level you intend to assess this target at. Omit it to inherit
/// the objective's.
@@ -1183,6 +1205,33 @@ impl CourseFile {
));
}
}
// A manuscript with no journal is a citation nobody can print. The
// fields exist; a note that carries them instead is data the reading
// list cannot link and the bibliography exporters cannot use.
if matches!(
reference.kind,
ReferenceKind::Article | ReferenceKind::Preprint
) && reference.container.is_none()
{
issues.push(format!(
"reference `{key}`: an {} needs a `container` — the journal, preprint \
server, or proceedings it appeared in. Run `coursebank migrate references` \
if it is sitting in the `note`.",
match reference.kind {
ReferenceKind::Preprint => "preprint",
_ => "article",
}
));
}
if let Some(note) = &reference.note {
if crate::citation::looks_like_a_citation(note) {
issues.push(format!(
"reference `{key}`: the note still carries a citation. Volume, pages, \
and DOI have their own fields, and a DOI in a note is a link nobody \
can follow. `coursebank migrate references` takes it apart."
));
}
}
if let Some(pmid) = &reference.pmid {
if !pmid.trim().chars().all(|c| c.is_ascii_digit()) {
issues.push(format!(
@@ -1646,9 +1695,20 @@ impl CourseFile {
.filter(|(_, target)| target.objective == objective)
.map(|(id, _)| id)
.collect();
// By ceiling, then by id. A target is a question template rather than a
// step in a sequence — an objective's targets are not taught in an
// order, and an exam samples from them — so there is no teaching order
// to print. What there is is depth, and grouping by it puts the
// checklist in the order the study methods apply: recall for a Level 1
// target, explanation for Level 2, variations and written solutions
// above that. Ties break by id so the list is stable.
ids.sort_by_key(|id| {
let target = &self.learning_targets[*id];
(target.order.unwrap_or(u32::MAX), (*id).clone())
(
self.effective_level_ceiling(id)
.map(|l| l.code())
.unwrap_or(0),
(*id).clone(),
)
});
ids.into_iter().map(String::as_str).collect()
}
+108
View File
@@ -535,6 +535,27 @@ impl Merge {
}
}
// Teaching order, from the two sequences that already declare it: the
// lectures in course order, and each lecture's `teaches` list. This is
// what lets an objective stop carrying a hand-kept integer.
let mut position = 0u32;
let ordered: Vec<String> = self.lectures.keys().cloned().collect();
for lecture_id in ordered {
let teaches = self
.lectures
.get(&lecture_id)
.map(|l| l.teaches.clone())
.unwrap_or_default();
for objective_id in teaches {
position += 1;
if let Some(objective) = self.objectives.get_mut(&objective_id) {
if objective.order.is_none() {
objective.order = Some(position);
}
}
}
}
// A target with no lecture of its own is taught wherever its objective
// is. Collected first: the read of `objectives` and the write to
// `targets` cannot overlap in one pass.
@@ -800,6 +821,93 @@ units:
assert!(course.origin("nothing-like-this").is_none());
}
#[test]
fn teaching_order_is_derived_from_the_two_sequences_that_declare_it() {
let root = tmp("order");
write(&root, "course.yaml", ROOT);
write(
&root,
"lectures/l-1-2.yaml",
"lectures:\n L1.2:\n title: One\n teaches: [lo-second, lo-first]\n",
);
write(
&root,
"lectures/l-1-3.yaml",
"lectures:\n L1.3:\n title: Two\n teaches: [lo-third]\n",
);
write(
&root,
"objectives/lo-first.yaml",
r#"learning_objectives:
lo-first:
text: A.
lo-second:
text: B.
lo-third:
text: C.
"#,
);
let course = assemble(&root).unwrap();
// Position in `teaches`, not id order: the lecture lists `lo-second`
// first and that is what teaching it first means.
assert_eq!(course.learning_objectives["lo-second"].order, Some(1));
assert_eq!(course.learning_objectives["lo-first"].order, Some(2));
assert_eq!(course.learning_objectives["lo-third"].order, Some(3));
assert_eq!(
course.lecture_objectives("L1.2"),
vec!["lo-second", "lo-first"]
);
}
#[test]
fn targets_print_by_ceiling_rather_than_in_a_sequence() {
let root = tmp("target-order");
write(&root, "course.yaml", ROOT);
write(
&root,
"lectures/l-1-2.yaml",
"lectures:\n L1.2:\n title: One\n teaches: [lo-x]\n",
);
write(
&root,
"objectives/lo-x.yaml",
r#"learning_objectives:
lo-x:
text: A.
level_ceiling: 3
learning_targets:
t-predict:
text: Predict the effect.
objective: lo-x
level_ceiling: 3
t-define:
text: Define the term.
objective: lo-x
level_ceiling: 1
t-explain:
text: Explain the mechanism.
objective: lo-x
level_ceiling: 2
"#,
);
let course = assemble(&root).unwrap();
assert!(course.validate().is_empty(), "{:?}", course.validate());
// Shallowest first, which is the order the study methods apply in:
// recall, then explanation, then variations. Not id order, which would
// put `t-define` after `t-predict` for no reason at all.
assert_eq!(
course.targets("lo-x"),
vec!["t-define", "t-explain", "t-predict"]
);
// A target with no ceiling of its own inherits the objective's, so it
// sorts where that puts it.
assert_eq!(course.learning_targets["t-define"].order, None);
}
#[test]
fn teaching_the_same_objective_from_two_lectures_unions() {
let root = tmp("union");
+8 -1
View File
@@ -157,7 +157,14 @@ pub struct Item {
pub design: Option<Design>,
/// What the evidence says, accumulated across administrations.
#[serde(default, skip_serializing_if = "Option::is_none")]
/// What the statistics say, filled in from `analysis/` at load.
///
/// Read from the store and never written back: `skip_serializing` means a
/// bank file cannot acquire a `calibration:` block by being round-tripped
/// through this type. A bank is a reviewed artifact whose diff should be a
/// change of intent, and every grading run would otherwise produce a diff
/// on it. See [`crate::calibration`].
#[serde(default, skip_serializing)]
pub calibration: Option<Calibration>,
/// The last review decision recorded for this item.
+20
View File
@@ -64,6 +64,24 @@ impl Layout {
self.root.join("banks")
}
/// Directory holding the statistics, which are kept out of the banks.
///
/// Committed, unlike [`Layout::data`]: everything under here is a cohort
/// aggregate with no student in it. See [`crate::calibration`].
pub fn analysis(&self) -> PathBuf {
self.root.join("analysis")
}
/// The pooled per-item calibration store.
pub fn calibration_file(&self) -> PathBuf {
self.analysis().join(crate::calibration::CALIBRATION_FILE)
}
/// Directory holding one immutable record per administration.
pub fn measurements(&self) -> PathBuf {
self.analysis().join("administrations")
}
/// Directory holding assessment records.
pub fn assessments(&self) -> PathBuf {
self.root.join("assessments")
@@ -117,6 +135,8 @@ impl Layout {
pub fn create_all(&self) -> Result<()> {
for dir in [
self.root.clone(),
self.analysis(),
self.measurements(),
self.lectures(),
self.objectives(),
self.banks(),
+80
View File
@@ -124,6 +124,26 @@ pub struct SealMeta {
/// from the file's own contents; [`SealFile::verify`] rebuilds a seal from the
/// live course and compares this against it.
pub digest: String,
/// The digests this file had before it was rewritten, oldest first.
///
/// A seal is meant to be tamper-evident, which puts a migration that
/// renames an id inside it in an awkward position: the id is part of
/// [`SealFile::digest_input`], so rewriting the file invalidates the digest
/// that vouched for it, and recomputing it quietly would produce a record
/// that looks untouched and is not.
///
/// So a migration does both — recomputes the digest and leaves the old one
/// here. What the seal then says is the honest thing: these are the
/// contents, this is what they hash to, and here is what the file hashed to
/// before each rewrite. Anyone holding an earlier copy, a backup or a git
/// revision, can check it against the right entry.
///
/// Deliberately outside [`SealFile::digest_input`]: a field that recorded
/// past digests and was itself covered by the current one could not be
/// appended to without invalidating what it describes.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub superseded_digests: Vec<String>,
}
/// What wrote a seal.
@@ -458,6 +478,7 @@ pub fn build(catalog: &Catalog, record: &AssessmentFile, opts: &Options) -> Resu
content: opts.content,
dropped,
digest: String::new(),
superseded_digests: Vec::new(),
},
items,
forms: sealed_forms,
@@ -787,6 +808,39 @@ impl SealFile {
buf
}
/// Renames the items this seal froze, keeping the digest honest.
///
/// The only sanctioned way to rewrite a seal. It recomputes the digest,
/// because the ids are part of what the digest covers, and records the
/// previous one in [`SealMeta::superseded_digests`], because a recomputed
/// digest with no trace of the recompute is a record that claims never to
/// have been touched.
///
/// # Arguments
///
/// * `rename` - a map from old item id to new.
///
/// # Returns
///
/// How many placements were renamed. Zero leaves the file alone, digest
/// included.
pub fn rename_items(&mut self, rename: &BTreeMap<String, String>) -> usize {
let mut renamed = 0;
for item in &mut self.items {
if let Some(new) = rename.get(&item.item) {
item.item = new.clone();
renamed += 1;
}
}
if renamed == 0 {
return 0;
}
let previous = std::mem::take(&mut self.seal.digest);
self.seal.superseded_digests.push(previous);
self.seal.digest = self.recompute_digest();
renamed
}
/// Recomputes this file's digest from its own contents.
///
/// # Returns
@@ -1195,6 +1249,7 @@ mod tests {
content: true,
dropped: Vec::new(),
digest: String::new(),
superseded_digests: Vec::new(),
},
items: vec![SealedItem {
number: 1,
@@ -1343,6 +1398,31 @@ mod tests {
assert_eq!(short("sha256:0123456789abcdef"), "sha256:01234567");
}
#[test]
fn renaming_an_item_recomputes_the_digest_and_says_it_did() {
let mut seal = sample();
seal.seal.digest = seal.recompute_digest();
let original = seal.seal.digest.clone();
assert!(seal.check_self().is_empty());
let mut rename = BTreeMap::new();
rename.insert(seal.items[0].item.clone(), "q-renamed".to_string());
assert_eq!(seal.rename_items(&rename), 1);
// The digest covers the ids, so it has to move — and the file has to
// admit that it moved rather than looking untouched.
assert_eq!(seal.items[0].item, "q-renamed");
assert_ne!(seal.seal.digest, original);
assert_eq!(seal.seal.superseded_digests, vec![original]);
assert!(seal.check_self().is_empty(), "{:?}", seal.check_self());
// A rename that matches nothing leaves the file entirely alone.
let steady = seal.seal.digest.clone();
assert_eq!(seal.rename_items(&BTreeMap::new()), 0);
assert_eq!(seal.seal.digest, steady);
assert_eq!(seal.seal.superseded_digests.len(), 1);
}
#[test]
fn round_trips_through_yaml() {
let file = sample();