feat: improvement
This commit is contained in:
18 files changed
+2538
-108
No files matched your search
+6
-32
@@ -530,38 +530,12 @@ fn validate_item(
|
||||
}
|
||||
|
||||
// --- calibration plausibility ------
|
||||
if let Some(c) = &it.calibration {
|
||||
if let Some(p) = c.p_value {
|
||||
if !(0.0..=1.0).contains(&p) {
|
||||
issues.push(format!(
|
||||
"calibration.p_value must be between 0 and 1, got {p}"
|
||||
));
|
||||
}
|
||||
}
|
||||
if let Some(r) = c.point_biserial {
|
||||
if !(-1.0..=1.0).contains(&r) {
|
||||
issues.push(format!(
|
||||
"calibration.point_biserial must be between -1 and 1, got {r}"
|
||||
));
|
||||
}
|
||||
}
|
||||
for option in c.option_stats.keys() {
|
||||
if it.option(option).is_none() {
|
||||
issues.push(format!(
|
||||
"calibration.option_stats has `{option}`, which is not an option of this item"
|
||||
));
|
||||
}
|
||||
}
|
||||
if let Some(irt) = &c.irt {
|
||||
if irt.a <= 0.0 {
|
||||
issues.push(format!("calibration.irt.a must be positive, got {}", irt.a));
|
||||
}
|
||||
if let Some(cp) = irt.c {
|
||||
if !(0.0..1.0).contains(&cp) {
|
||||
issues.push(format!("calibration.irt.c must be in [0, 1), got {cp}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
if it.calibration.is_some() {
|
||||
issues.push(
|
||||
"has a `calibration:` block, but statistics live in analysis/calibration.yaml since \
|
||||
2.0. A bank's diff should be a change of intent, not the output of a grading run."
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
// --- retirement -----
|
||||
|
||||
@@ -0,0 +1,838 @@
|
||||
// SPDX-License-Identifier: Prosperity-3.0.0
|
||||
// Copyright Scientific Computing Studio
|
||||
// Source: https://git.scient.ing/education/coursebank
|
||||
|
||||
//! Where the statistics live, which is not in the bank.
|
||||
//!
|
||||
//! A bank file is a reviewed artifact: someone wrote the question, someone
|
||||
//! argued about the distractors, and the diff on it should be a change of
|
||||
//! intent. Statistics are neither reviewed nor intended — they are what
|
||||
//! happened — and writing them back into the bank means every grading run
|
||||
//! produces a diff on a file whose history is supposed to be about wording.
|
||||
//!
|
||||
//! So the evidence lives in `analysis/`, in two kinds of file:
|
||||
//!
|
||||
//! | File | Format | Rewritten? | Holds |
|
||||
//! |:--|:--|:--|:--|
|
||||
//! | `analysis/administrations/<id>-items.csv` | CSV | never | one row per question |
|
||||
//! | `analysis/administrations/<id>-options.csv` | CSV | never | one row per question and option |
|
||||
//! | `analysis/calibration.yaml` | YAML | by `calibrate` | the pooled per-item view |
|
||||
//!
|
||||
//! The format follows the shape. An administration record is a table — fixed
|
||||
//! columns, one row per question, machine-written, never hand-edited — so it is
|
||||
//! CSV: one line per item rather than fifteen, which diffs better, and it loads
|
||||
//! straight into pandas or DuckDB, which is much of the point of committing it.
|
||||
//! The pooled view is not a table. It is three levels deep, item to variant to
|
||||
//! option history, with fitted parameters and variable-length lists, and as CSV
|
||||
//! that would be three files joined by keys — a relational schema for the one
|
||||
//! file a person actually reads in a pull request. That stays YAML.
|
||||
//!
|
||||
//! Neither is Parquet, and the reason is the review workflow: a binary file
|
||||
//! shows nothing in a diff and cannot be merged. Parquet is right for `data/`
|
||||
//! precisely because that is bulk, ignored, and never reviewed.
|
||||
//!
|
||||
//! An administration file is written once and not touched again, for the same
|
||||
//! reason a seal is not: it is a record of a thing that happened on a day. The
|
||||
//! calibration file is the accepted rollup — what `lint` compares your
|
||||
//! predictions against, and what a report reads — and `calibrate` proposes
|
||||
//! changes to it as a diff you review before committing.
|
||||
//!
|
||||
//! Both are meant to be committed. Neither can carry student data, and that is
|
||||
//! a property of the types rather than a promise: there is no field for a
|
||||
//! student key, an identifier, a section, or an ability estimate, and the
|
||||
//! structures reject unknown keys, so a file carrying one fails to load rather
|
||||
//! than being quietly accepted. Everything per-person stays in `data/`, which
|
||||
//! is what your `.gitignore` is for.
|
||||
//!
|
||||
//! # Linking back to the bank
|
||||
//!
|
||||
//! By item id, which since 2.0 names the item course-wide and has no file name
|
||||
//! in it, and by variant digest, which says which option set the numbers
|
||||
//! describe. A record also carries the stem digest it was measured against, so
|
||||
//! [`CalibrationFile::validate`] can say that an item has been reworded since —
|
||||
//! the statistics then describe a question that no longer exists under that id.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::course::SCHEMA_VERSION;
|
||||
use crate::date::Date;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::item::{Calibration, IrtModel, IrtParams, OptionStat};
|
||||
use crate::taxonomy::Flag;
|
||||
use crate::yaml;
|
||||
|
||||
/// The file name of the pooled calibration store, under `analysis/`.
|
||||
pub const CALIBRATION_FILE: &str = "calibration.yaml";
|
||||
|
||||
/// The pooled per-item statistics: `analysis/calibration.yaml`.
|
||||
///
|
||||
/// Keyed by item id. This is the file `calibrate` rewrites and the one
|
||||
/// everything else reads; [`crate::catalog::Catalog::load`] fills each item's
|
||||
/// in-memory calibration from it, so nothing downstream has to know where the
|
||||
/// numbers came from.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct CalibrationFile {
|
||||
/// Schema version.
|
||||
#[serde(
|
||||
default = "default_version",
|
||||
deserialize_with = "yaml::flexible_string"
|
||||
)]
|
||||
pub schema_version: String,
|
||||
|
||||
/// One entry per calibrated item, by item id.
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub items: BTreeMap<String, Calibration>,
|
||||
}
|
||||
|
||||
impl CalibrationFile {
|
||||
/// Loads the store, or an empty one when the file does not exist yet.
|
||||
///
|
||||
/// Absence is not an error: a course that has not graded anything has no
|
||||
/// statistics, and every command that reads them has to work anyway.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `path` - the file, usually `analysis/calibration.yaml`.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The store.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`Error::Yaml`] when the file exists and does not parse.
|
||||
pub fn load(path: &Path) -> Result<CalibrationFile> {
|
||||
if !path.is_file() {
|
||||
return Ok(CalibrationFile::default());
|
||||
}
|
||||
yaml::read(path)
|
||||
}
|
||||
|
||||
/// Writes the store.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `path` - the destination.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`Error::Io`] on a write failure.
|
||||
pub fn save(&self, path: &Path) -> Result<()> {
|
||||
yaml::write(path, self)
|
||||
}
|
||||
|
||||
/// The calibration recorded for one item.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `item` - the item id.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The record, or `None` when the item has never been calibrated.
|
||||
pub fn get(&self, item: &str) -> Option<&Calibration> {
|
||||
self.items.get(item)
|
||||
}
|
||||
|
||||
/// Checks the store against the bank it describes.
|
||||
///
|
||||
/// The checks that matter for a file kept apart from what it refers to: a
|
||||
/// record for an item that no longer exists, an option id the item does not
|
||||
/// have, and — the one worth having — statistics measured against a stem
|
||||
/// that has since been reworded, which since 2.0 means they describe a
|
||||
/// different question wearing the same id.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `catalog` - the loaded course.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// One message per problem, empty when the store agrees with the bank.
|
||||
pub fn validate(&self, catalog: &crate::catalog::Catalog) -> Vec<String> {
|
||||
let mut issues = Vec::new();
|
||||
|
||||
for (id, calibration) in &self.items {
|
||||
let Some(entry) = catalog.get(id) else {
|
||||
issues.push(format!(
|
||||
"calibration for `{id}`: no such item. Statistics outlive an item only if \
|
||||
it is retired, not deleted — a retired item keeps its id so its numbers \
|
||||
still mean something."
|
||||
));
|
||||
continue;
|
||||
};
|
||||
let item = &entry.item;
|
||||
|
||||
for variant in &calibration.variants {
|
||||
for option in variant.option_stats.keys() {
|
||||
if item.option(option).is_none() {
|
||||
issues.push(format!(
|
||||
"calibration for `{id}`: variant `{}` has statistics for `{option}`, \
|
||||
which is not an option of this item",
|
||||
short(&variant.variant)
|
||||
));
|
||||
}
|
||||
}
|
||||
if !variant.key.is_empty()
|
||||
&& variant.variant != item.variant_digest(&variant.key, &variant.distractors)
|
||||
{
|
||||
issues.push(format!(
|
||||
"calibration for `{id}`: variant `{}` was measured against an option set \
|
||||
that has since been reworded, so its numbers describe wording no \
|
||||
student now sees",
|
||||
short(&variant.variant)
|
||||
));
|
||||
}
|
||||
}
|
||||
for option in calibration.options.keys() {
|
||||
if item.option(option).is_none() {
|
||||
issues.push(format!(
|
||||
"calibration for `{id}`: an option history names `{option}`, which is \
|
||||
not an option of this item"
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
issues
|
||||
}
|
||||
}
|
||||
|
||||
/// What one administration measured: `analysis/administrations/<id>.yaml`.
|
||||
///
|
||||
/// Written once, when the exam is analyzed, and never rewritten. It is the
|
||||
/// audit trail under [`CalibrationFile`]: the pooled numbers say an item sits
|
||||
/// at 0.63, and these say which exams that came from and what each one saw.
|
||||
/// Keeping them also means the history survives losing `data/`, which is
|
||||
/// ignored by git and rotates.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MeasurementFile {
|
||||
/// Schema version.
|
||||
#[serde(
|
||||
default = "default_version",
|
||||
deserialize_with = "yaml::flexible_string"
|
||||
)]
|
||||
pub schema_version: String,
|
||||
|
||||
/// What was administered, and how it was analyzed.
|
||||
pub administration: MeasurementMeta,
|
||||
|
||||
/// One entry per question, in the order it was printed.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub items: Vec<Measurement>,
|
||||
}
|
||||
|
||||
/// What an administration was, for a reader two years later.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MeasurementMeta {
|
||||
/// The administration id these numbers came from.
|
||||
pub id: String,
|
||||
/// The assessment that was administered.
|
||||
pub assessment: String,
|
||||
/// The term.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub term: Option<String>,
|
||||
/// The date it was given.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub date: Option<Date>,
|
||||
/// The forms in play.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub forms: Vec<String>,
|
||||
/// How many examinees the numbers pool over.
|
||||
///
|
||||
/// The one number to read before any of the others. A point-biserial on
|
||||
/// twenty-seven students is a different kind of claim than one on three
|
||||
/// hundred, and nothing below records how thin it is.
|
||||
pub n_examinees: usize,
|
||||
/// The item response model fitted, when one was.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub model: Option<IrtModel>,
|
||||
/// When the analysis was run.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub generated: Option<Date>,
|
||||
/// The version of the tool that ran it, since the numbers depend on it.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub coursebank: Option<String>,
|
||||
}
|
||||
|
||||
/// One question's statistics from one administration.
|
||||
///
|
||||
/// Cohort aggregates only. There is deliberately no per-section or per-form
|
||||
/// breakdown: those get small, and a small cell crossed with anything else is
|
||||
/// how an aggregate stops being one.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Measurement {
|
||||
/// The item id.
|
||||
pub item: String,
|
||||
/// The question number it was printed as.
|
||||
pub number: u32,
|
||||
/// The option set administered. See [`crate::item::Item::variant_digest`].
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub variant: Option<String>,
|
||||
/// The stem as administered. See [`crate::item::Item::stem_digest`].
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub stem_digest: Option<String>,
|
||||
/// Examinees who saw it.
|
||||
pub n: usize,
|
||||
/// Proportion correct.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub p_value: Option<f64>,
|
||||
/// Corrected item-total point-biserial correlation.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub point_biserial: Option<f64>,
|
||||
/// Upper-minus-lower-group discrimination index.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub discrimination_index: Option<f64>,
|
||||
/// The option ids keyed correct, so a row in the options file says whether
|
||||
/// it describes the answer or a distractor without a join.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub key: Vec<String>,
|
||||
/// Per-option behaviour, by option id.
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub option_stats: BTreeMap<String, OptionStat>,
|
||||
/// Fitted parameters, when the sample supported a fit.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub irt: Option<IrtParams>,
|
||||
/// Machine-detected problems with this question on this administration.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub flags: Vec<Flag>,
|
||||
}
|
||||
|
||||
/// The columns of an items file, and the only ones accepted on read.
|
||||
///
|
||||
/// An allowlist rather than a type-level guarantee. In YAML the structures
|
||||
/// reject unknown keys, so a file carrying a student column could not be
|
||||
/// loaded; CSV readers are tolerant of extra columns, so the same assurance has
|
||||
/// to be an explicit check. This is it, and [`MeasurementFile::read_csv`]
|
||||
/// refuses any header not named here.
|
||||
pub const ITEM_COLUMNS: [&str; 16] = [
|
||||
"administration_id",
|
||||
"assessment",
|
||||
"term",
|
||||
"date",
|
||||
"forms",
|
||||
"n_examinees",
|
||||
"coursebank",
|
||||
"generated",
|
||||
"item",
|
||||
"number",
|
||||
"variant",
|
||||
"stem_digest",
|
||||
"n",
|
||||
"p_value",
|
||||
"point_biserial",
|
||||
"discrimination_index",
|
||||
];
|
||||
|
||||
/// The columns of an options file, and the only ones accepted on read.
|
||||
pub const OPTION_COLUMNS: [&str; 9] = [
|
||||
"administration_id",
|
||||
"item",
|
||||
"number",
|
||||
"option",
|
||||
"keyed",
|
||||
"selection_rate",
|
||||
"point_biserial",
|
||||
"upper_group_rate",
|
||||
"lower_group_rate",
|
||||
];
|
||||
|
||||
/// One row of an items file.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
struct ItemRow {
|
||||
administration_id: String,
|
||||
assessment: String,
|
||||
term: String,
|
||||
date: String,
|
||||
forms: String,
|
||||
n_examinees: usize,
|
||||
coursebank: String,
|
||||
generated: String,
|
||||
item: String,
|
||||
number: u32,
|
||||
variant: String,
|
||||
stem_digest: String,
|
||||
n: usize,
|
||||
p_value: String,
|
||||
point_biserial: String,
|
||||
discrimination_index: String,
|
||||
}
|
||||
|
||||
/// One row of an options file.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
struct OptionRow {
|
||||
administration_id: String,
|
||||
item: String,
|
||||
number: u32,
|
||||
option: String,
|
||||
keyed: bool,
|
||||
selection_rate: String,
|
||||
point_biserial: String,
|
||||
upper_group_rate: String,
|
||||
lower_group_rate: String,
|
||||
}
|
||||
|
||||
impl MeasurementFile {
|
||||
/// The two file names this administration writes, items first.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `dir` - usually `analysis/administrations`.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The items path and the options path.
|
||||
pub fn paths(&self, dir: &Path) -> (PathBuf, PathBuf) {
|
||||
let stem = crate::course::slugify(&self.administration.id);
|
||||
(
|
||||
dir.join(format!("{stem}-items.csv")),
|
||||
dir.join(format!("{stem}-options.csv")),
|
||||
)
|
||||
}
|
||||
|
||||
/// Writes the two files, refusing to overwrite either.
|
||||
///
|
||||
/// An administration is a thing that happened once, so replacing its record
|
||||
/// is a deliberate act: delete the files first if you mean to re-analyze.
|
||||
///
|
||||
/// Two files rather than one because the data is two shapes — one row per
|
||||
/// question, one row per question and option — and a single sparse table
|
||||
/// serves neither. The administration's metadata repeats on every row,
|
||||
/// which is what makes each file independently loadable and is the same
|
||||
/// convention the response store already uses.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `dir` - the directory to write into.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The paths written.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`Error::Usage`] when either file exists, and [`Error::Io`] or
|
||||
/// [`Error::Csv`] on a write failure.
|
||||
pub fn write_csv(&self, dir: &Path) -> Result<Vec<PathBuf>> {
|
||||
let (items_path, options_path) = self.paths(dir);
|
||||
for path in [&items_path, &options_path] {
|
||||
if path.exists() {
|
||||
return Err(Error::usage(format!(
|
||||
"{} already records this administration. It happened once, so replacing it \
|
||||
is a deliberate act: delete it first if you mean to re-analyze.",
|
||||
path.display()
|
||||
)));
|
||||
}
|
||||
}
|
||||
std::fs::create_dir_all(dir).map_err(|e| Error::io(dir, e))?;
|
||||
|
||||
let meta = &self.administration;
|
||||
let mut items = csv::Writer::from_path(&items_path).map_err(|e| Error::Csv {
|
||||
path: items_path.clone(),
|
||||
source: e,
|
||||
})?;
|
||||
let mut options = csv::Writer::from_path(&options_path).map_err(|e| Error::Csv {
|
||||
path: options_path.clone(),
|
||||
source: e,
|
||||
})?;
|
||||
|
||||
for measurement in &self.items {
|
||||
items
|
||||
.serialize(ItemRow {
|
||||
administration_id: meta.id.clone(),
|
||||
assessment: meta.assessment.clone(),
|
||||
term: meta.term.clone().unwrap_or_default(),
|
||||
date: meta.date.map(|d| d.to_string()).unwrap_or_default(),
|
||||
forms: meta.forms.join(";"),
|
||||
n_examinees: meta.n_examinees,
|
||||
coursebank: meta.coursebank.clone().unwrap_or_default(),
|
||||
generated: meta.generated.map(|d| d.to_string()).unwrap_or_default(),
|
||||
item: measurement.item.clone(),
|
||||
number: measurement.number,
|
||||
variant: measurement.variant.clone().unwrap_or_default(),
|
||||
stem_digest: measurement.stem_digest.clone().unwrap_or_default(),
|
||||
n: measurement.n,
|
||||
p_value: number(measurement.p_value),
|
||||
point_biserial: number(measurement.point_biserial),
|
||||
discrimination_index: number(measurement.discrimination_index),
|
||||
})
|
||||
.map_err(|e| Error::Csv {
|
||||
path: items_path.clone(),
|
||||
source: e,
|
||||
})?;
|
||||
|
||||
for (option, stat) in &measurement.option_stats {
|
||||
options
|
||||
.serialize(OptionRow {
|
||||
administration_id: meta.id.clone(),
|
||||
item: measurement.item.clone(),
|
||||
number: measurement.number,
|
||||
option: option.clone(),
|
||||
keyed: measurement.key.iter().any(|k| k == option),
|
||||
selection_rate: number(stat.selection_rate),
|
||||
point_biserial: number(stat.point_biserial),
|
||||
upper_group_rate: number(stat.upper_group_rate),
|
||||
lower_group_rate: number(stat.lower_group_rate),
|
||||
})
|
||||
.map_err(|e| Error::Csv {
|
||||
path: options_path.clone(),
|
||||
source: e,
|
||||
})?;
|
||||
}
|
||||
}
|
||||
|
||||
items.flush().map_err(|e| Error::io(&items_path, e))?;
|
||||
options.flush().map_err(|e| Error::io(&options_path, e))?;
|
||||
Ok(vec![items_path, options_path])
|
||||
}
|
||||
|
||||
/// Reads one administration back from its two files.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `items_path` - the items file. The options file is found beside it.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The administration, with its per-option statistics reattached.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`Error::Csv`] on a parse failure and [`Error::Invalid`] when a
|
||||
/// file carries a column that is not in [`ITEM_COLUMNS`] or
|
||||
/// [`OPTION_COLUMNS`] — which is how a student column is caught.
|
||||
pub fn read_csv(items_path: &Path) -> Result<MeasurementFile> {
|
||||
let options_path = PathBuf::from(
|
||||
items_path
|
||||
.to_string_lossy()
|
||||
.replace("-items.csv", "-options.csv"),
|
||||
);
|
||||
|
||||
let mut reader = open_csv(items_path, &ITEM_COLUMNS)?;
|
||||
let mut meta = MeasurementMeta::default();
|
||||
let mut items: Vec<Measurement> = Vec::new();
|
||||
for row in reader.deserialize::<ItemRow>() {
|
||||
let row = row.map_err(|e| Error::Csv {
|
||||
path: items_path.to_path_buf(),
|
||||
source: e,
|
||||
})?;
|
||||
meta = MeasurementMeta {
|
||||
id: row.administration_id.clone(),
|
||||
assessment: row.assessment.clone(),
|
||||
term: some(&row.term),
|
||||
date: parse_date(&row.date),
|
||||
forms: row
|
||||
.forms
|
||||
.split(';')
|
||||
.filter(|f| !f.is_empty())
|
||||
.map(str::to_string)
|
||||
.collect(),
|
||||
n_examinees: row.n_examinees,
|
||||
model: meta.model,
|
||||
generated: parse_date(&row.generated),
|
||||
coursebank: some(&row.coursebank),
|
||||
};
|
||||
items.push(Measurement {
|
||||
item: row.item,
|
||||
number: row.number,
|
||||
variant: some(&row.variant),
|
||||
stem_digest: some(&row.stem_digest),
|
||||
n: row.n,
|
||||
p_value: parse(&row.p_value),
|
||||
point_biserial: parse(&row.point_biserial),
|
||||
discrimination_index: parse(&row.discrimination_index),
|
||||
..Measurement::default()
|
||||
});
|
||||
}
|
||||
|
||||
if options_path.is_file() {
|
||||
let mut reader = open_csv(&options_path, &OPTION_COLUMNS)?;
|
||||
for row in reader.deserialize::<OptionRow>() {
|
||||
let row = row.map_err(|e| Error::Csv {
|
||||
path: options_path.clone(),
|
||||
source: e,
|
||||
})?;
|
||||
let Some(target) = items.iter_mut().find(|i| i.number == row.number) else {
|
||||
continue;
|
||||
};
|
||||
if row.keyed && !target.key.contains(&row.option) {
|
||||
target.key.push(row.option.clone());
|
||||
}
|
||||
target.option_stats.insert(
|
||||
row.option,
|
||||
OptionStat {
|
||||
selection_rate: parse(&row.selection_rate),
|
||||
point_biserial: parse(&row.point_biserial),
|
||||
upper_group_rate: parse(&row.upper_group_rate),
|
||||
lower_group_rate: parse(&row.lower_group_rate),
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(MeasurementFile {
|
||||
schema_version: default_version(),
|
||||
administration: meta,
|
||||
items,
|
||||
})
|
||||
}
|
||||
|
||||
/// Reads every administration under a directory, oldest first.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `dir` - usually `analysis/administrations`.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The administrations, empty when the directory does not exist.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Propagates read failures.
|
||||
pub fn load_all(dir: &Path) -> Result<Vec<MeasurementFile>> {
|
||||
if !dir.is_dir() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let mut paths: Vec<PathBuf> = std::fs::read_dir(dir)
|
||||
.map_err(|e| Error::io(dir, e))?
|
||||
.filter_map(|e| e.ok().map(|e| e.path()))
|
||||
.filter(|p| p.to_string_lossy().ends_with("-items.csv"))
|
||||
.collect();
|
||||
paths.sort();
|
||||
|
||||
let mut out = Vec::new();
|
||||
for path in &paths {
|
||||
out.push(MeasurementFile::read_csv(path)?);
|
||||
}
|
||||
out.sort_by(|a, b| {
|
||||
a.administration
|
||||
.date
|
||||
.cmp(&b.administration.date)
|
||||
.then(a.administration.id.cmp(&b.administration.id))
|
||||
});
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
/// Opens a CSV and refuses any column that is not on the allowlist.
|
||||
fn open_csv(path: &Path, allowed: &[&str]) -> Result<csv::Reader<std::fs::File>> {
|
||||
let mut reader = csv::Reader::from_path(path).map_err(|e| Error::Csv {
|
||||
path: path.to_path_buf(),
|
||||
source: e,
|
||||
})?;
|
||||
let headers = reader
|
||||
.headers()
|
||||
.map_err(|e| Error::Csv {
|
||||
path: path.to_path_buf(),
|
||||
source: e,
|
||||
})?
|
||||
.clone();
|
||||
|
||||
let unexpected: Vec<&str> = headers.iter().filter(|h| !allowed.contains(h)).collect();
|
||||
if !unexpected.is_empty() {
|
||||
return Err(Error::Invalid(vec![format!(
|
||||
"{}: unexpected column(s) {}. These files are committed, so they hold cohort \
|
||||
aggregates and nothing else — anything per-student belongs in data/, which is \
|
||||
ignored.",
|
||||
path.display(),
|
||||
unexpected.join(", ")
|
||||
)]));
|
||||
}
|
||||
Ok(reader)
|
||||
}
|
||||
|
||||
/// A float as a CSV cell, empty when absent.
|
||||
fn number(value: Option<f64>) -> String {
|
||||
value.map(|v| format!("{v}")).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// A CSV cell as a float, absent when empty or unparseable.
|
||||
fn parse(cell: &str) -> Option<f64> {
|
||||
cell.trim().parse().ok()
|
||||
}
|
||||
|
||||
/// A CSV cell as a string, absent when empty.
|
||||
fn some(cell: &str) -> Option<String> {
|
||||
(!cell.trim().is_empty()).then(|| cell.trim().to_string())
|
||||
}
|
||||
|
||||
/// A CSV cell as a date, absent when empty or unparseable.
|
||||
fn parse_date(cell: &str) -> Option<Date> {
|
||||
some(cell).and_then(|c| c.parse().ok())
|
||||
}
|
||||
|
||||
/// The schema version new files are written with.
|
||||
fn default_version() -> String {
|
||||
SCHEMA_VERSION.to_string()
|
||||
}
|
||||
|
||||
/// A digest shortened for a message.
|
||||
fn short(digest: &str) -> String {
|
||||
digest.chars().take(8).collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::item::VariantCalibration;
|
||||
|
||||
fn tmp(tag: &str) -> std::path::PathBuf {
|
||||
let p = std::env::temp_dir().join(format!("coursebank-cal-{tag}-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&p);
|
||||
std::fs::create_dir_all(&p).unwrap();
|
||||
p
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_absent_store_is_empty_rather_than_an_error() {
|
||||
let dir = tmp("absent");
|
||||
let store = CalibrationFile::load(&dir.join(CALIBRATION_FILE)).unwrap();
|
||||
assert!(store.items.is_empty());
|
||||
assert!(store.get("q-x").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_store_round_trips() {
|
||||
let dir = tmp("round");
|
||||
let path = dir.join(CALIBRATION_FILE);
|
||||
|
||||
let mut store = CalibrationFile::default();
|
||||
store.items.insert(
|
||||
"q-x".to_string(),
|
||||
Calibration {
|
||||
n_examinees: Some(27),
|
||||
p_value: Some(0.63),
|
||||
variants: vec![VariantCalibration {
|
||||
variant: "4c81fa".into(),
|
||||
n_examinees: Some(27),
|
||||
..VariantCalibration::default()
|
||||
}],
|
||||
..Calibration::default()
|
||||
},
|
||||
);
|
||||
store.save(&path).unwrap();
|
||||
|
||||
let back = CalibrationFile::load(&path).unwrap();
|
||||
assert_eq!(back.get("q-x").unwrap().p_value, Some(0.63));
|
||||
assert_eq!(back.get("q-x").unwrap().variants.len(), 1);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_administration_round_trips_through_two_csvs() {
|
||||
let dir = tmp("csv");
|
||||
let file = MeasurementFile {
|
||||
schema_version: default_version(),
|
||||
administration: MeasurementMeta {
|
||||
id: "e1-2026f".into(),
|
||||
assessment: "e1".into(),
|
||||
term: Some("2026f".into()),
|
||||
date: Some(Date::new(2026, 9, 15).unwrap()),
|
||||
forms: vec!["A".into(), "B".into()],
|
||||
n_examinees: 27,
|
||||
model: None,
|
||||
generated: Some(Date::new(2026, 9, 27).unwrap()),
|
||||
coursebank: Some("0.0.0".into()),
|
||||
},
|
||||
items: vec![Measurement {
|
||||
item: "q-fastq-quality-length-match".into(),
|
||||
number: 1,
|
||||
variant: Some("237d62f9af222f78".into()),
|
||||
stem_digest: Some("8b22e0".into()),
|
||||
n: 27,
|
||||
p_value: Some(0.5926),
|
||||
point_biserial: Some(0.31),
|
||||
discrimination_index: None,
|
||||
key: vec!["o-fourth-line".into()],
|
||||
option_stats: [
|
||||
(
|
||||
"o-fourth-line".to_string(),
|
||||
OptionStat {
|
||||
selection_rate: Some(0.5926),
|
||||
point_biserial: Some(0.31),
|
||||
upper_group_rate: None,
|
||||
lower_group_rate: None,
|
||||
},
|
||||
),
|
||||
(
|
||||
"o-third-line".to_string(),
|
||||
OptionStat {
|
||||
selection_rate: Some(0.1852),
|
||||
point_biserial: Some(-0.18),
|
||||
upper_group_rate: None,
|
||||
lower_group_rate: None,
|
||||
},
|
||||
),
|
||||
]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
irt: None,
|
||||
flags: Vec::new(),
|
||||
}],
|
||||
};
|
||||
|
||||
let written = file.write_csv(&dir).unwrap();
|
||||
assert_eq!(written.len(), 2, "one table per shape");
|
||||
assert!(written[0].ends_with("e1-2026f-items.csv"));
|
||||
assert!(written[1].ends_with("e1-2026f-options.csv"));
|
||||
|
||||
// One line per question, loadable on its own: the administration's
|
||||
// metadata repeats on the row, as the response store already does.
|
||||
let items = std::fs::read_to_string(&written[0]).unwrap();
|
||||
assert!(
|
||||
items.starts_with("administration_id,assessment,term,date,forms"),
|
||||
"{items}"
|
||||
);
|
||||
assert!(
|
||||
items.contains("e1-2026f,e1,2026f,2026-09-15,A;B,27"),
|
||||
"{items}"
|
||||
);
|
||||
|
||||
let options = std::fs::read_to_string(&written[1]).unwrap();
|
||||
assert!(options.contains("o-fourth-line,true"), "{options}");
|
||||
assert!(options.contains("o-third-line,false"), "{options}");
|
||||
|
||||
let back = MeasurementFile::read_csv(&written[0]).unwrap();
|
||||
assert_eq!(back.administration.id, "e1-2026f");
|
||||
assert_eq!(back.administration.n_examinees, 27);
|
||||
assert_eq!(back.administration.forms, vec!["A", "B"]);
|
||||
assert_eq!(back.items.len(), 1);
|
||||
assert_eq!(back.items[0].p_value, Some(0.5926));
|
||||
assert_eq!(back.items[0].key, vec!["o-fourth-line"]);
|
||||
assert_eq!(back.items[0].option_stats.len(), 2);
|
||||
assert_eq!(back.items[0].discrimination_index, None);
|
||||
|
||||
assert_eq!(MeasurementFile::load_all(&dir).unwrap().len(), 1);
|
||||
|
||||
// It happened once, so the record is not replaced by accident.
|
||||
let err = file.write_csv(&dir).unwrap_err().to_string();
|
||||
assert!(err.contains("deliberate"), "{err}");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_student_column_is_refused_on_read() {
|
||||
let dir = tmp("identifiers");
|
||||
let path = dir.join("e1-2026f-items.csv");
|
||||
std::fs::write(
|
||||
&path,
|
||||
"administration_id,assessment,item,number,n,student_key\n e1-2026f,e1,q-x,1,27,abc123\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// In YAML this fell out of the type, which rejects unknown keys. A CSV
|
||||
// reader tolerates extra columns, so the same assurance has to be an
|
||||
// explicit allowlist — and this is the test that it is one.
|
||||
let err = MeasurementFile::read_csv(&path).unwrap_err().to_string();
|
||||
assert!(err.contains("student_key"), "{err}");
|
||||
assert!(err.contains("cohort aggregates"), "{err}");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
}
|
||||
@@ -75,6 +75,12 @@ pub struct Catalog {
|
||||
pub entries: Vec<Entry>,
|
||||
/// Bank metadata by bank id.
|
||||
pub banks: BTreeMap<String, crate::bank::BankMeta>,
|
||||
/// The statistics, loaded from `analysis/` rather than from the banks.
|
||||
///
|
||||
/// Each entry's [`crate::item::Item::calibration`] is filled from this at
|
||||
/// load, so everything downstream reads one item and does not have to know
|
||||
/// that the numbers and the wording come from different files.
|
||||
pub calibration: crate::calibration::CalibrationFile,
|
||||
/// Map from global id to index into `entries`.
|
||||
index: BTreeMap<String, usize>,
|
||||
}
|
||||
@@ -104,7 +110,10 @@ impl Catalog {
|
||||
entries: Vec::new(),
|
||||
banks: BTreeMap::new(),
|
||||
index: BTreeMap::new(),
|
||||
calibration: crate::calibration::CalibrationFile::default(),
|
||||
};
|
||||
catalog.calibration =
|
||||
crate::calibration::CalibrationFile::load(&catalog.layout.calibration_file())?;
|
||||
|
||||
let mut problems = Vec::new();
|
||||
let mut files = yaml::list_yaml(&catalog.layout.banks())?;
|
||||
@@ -145,6 +154,15 @@ impl Catalog {
|
||||
if !problems.is_empty() {
|
||||
return Err(Error::Invalid(problems));
|
||||
}
|
||||
// Statistics are attached here rather than parsed from the bank, which
|
||||
// is why an item can be read as one thing while its wording and its
|
||||
// evidence live in files with different review cycles.
|
||||
for entry in &mut catalog.entries {
|
||||
if let Some(calibration) = catalog.calibration.items.get(&entry.uid) {
|
||||
entry.item.calibration = Some(calibration.clone());
|
||||
}
|
||||
}
|
||||
|
||||
Ok(catalog)
|
||||
}
|
||||
|
||||
|
||||
+69
-9
@@ -800,15 +800,25 @@ pub struct Objective {
|
||||
/// The lectures that develop it.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub lectures: Vec<String>,
|
||||
/// Position in teaching order, low first.
|
||||
/// Position in teaching order, low first. Derived; authoring it is
|
||||
/// deprecated.
|
||||
///
|
||||
/// The registry is a map, so declaration order is lost on load, and sorting
|
||||
/// The registry is a map, so declaration order is lost on load and sorting
|
||||
/// by id would put `lo-enthalpy` before `lo-first-law` when the second is a
|
||||
/// prerequisite of the first. Anything that prints objectives in the order
|
||||
/// you teach them, a lecture page above all, needs this. Objectives without
|
||||
/// it sort last, by id.
|
||||
/// prerequisite of the first. Something has to supply the order.
|
||||
///
|
||||
/// Since 2.0 that something is [`Lecture::teaches`], which is a sequence:
|
||||
/// the position of an objective in the list of what a lecture covers, and
|
||||
/// the position of the lecture in the course, together say when it is
|
||||
/// taught. [`fragment::assemble`] fills this in from those two, so a course
|
||||
/// that used to carry thirty-nine hand-kept integers now carries none, and
|
||||
/// inserting an objective is a one-line edit rather than a renumber.
|
||||
///
|
||||
/// An authored value still wins, so a 1.0 course loads unchanged.
|
||||
/// `coursebank migrate order` removes them.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub order: Option<u32>,
|
||||
|
||||
/// The highest level you intend to assess this objective at. Assembling an
|
||||
/// item above the ceiling is a warning: either the item overreaches or the
|
||||
/// ceiling needs raising.
|
||||
@@ -865,8 +875,20 @@ pub struct Target {
|
||||
///
|
||||
/// Ordered within its objective rather than across the course, so two
|
||||
/// targets under different objectives never compete for a position and
|
||||
/// inserting one renumbers nothing outside its own group.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
/// Retained only so a pre-2.0 course still loads. Ignored.
|
||||
///
|
||||
/// Targets do not have an order. An objective's targets are a set of
|
||||
/// question templates, not steps in a sequence: they are not taught in
|
||||
/// order, an exam samples from them rather than working through them, and
|
||||
/// the study workflow reads the list as a checklist and counts what it can
|
||||
/// do cold. A position would assert a sequence that does not exist.
|
||||
///
|
||||
/// Where a list has to be printed, [`CourseFile::targets`] orders it by
|
||||
/// ceiling and then by id. Where one target genuinely depends on another,
|
||||
/// that is [`Target::prerequisites`], which says so directly.
|
||||
///
|
||||
/// `coursebank migrate order` removes it.
|
||||
#[serde(default, skip_serializing)]
|
||||
pub order: Option<u32>,
|
||||
/// The highest level you intend to assess this target at. Omit it to inherit
|
||||
/// the objective's.
|
||||
@@ -1183,6 +1205,33 @@ impl CourseFile {
|
||||
));
|
||||
}
|
||||
}
|
||||
// A manuscript with no journal is a citation nobody can print. The
|
||||
// fields exist; a note that carries them instead is data the reading
|
||||
// list cannot link and the bibliography exporters cannot use.
|
||||
if matches!(
|
||||
reference.kind,
|
||||
ReferenceKind::Article | ReferenceKind::Preprint
|
||||
) && reference.container.is_none()
|
||||
{
|
||||
issues.push(format!(
|
||||
"reference `{key}`: an {} needs a `container` — the journal, preprint \
|
||||
server, or proceedings it appeared in. Run `coursebank migrate references` \
|
||||
if it is sitting in the `note`.",
|
||||
match reference.kind {
|
||||
ReferenceKind::Preprint => "preprint",
|
||||
_ => "article",
|
||||
}
|
||||
));
|
||||
}
|
||||
if let Some(note) = &reference.note {
|
||||
if crate::citation::looks_like_a_citation(note) {
|
||||
issues.push(format!(
|
||||
"reference `{key}`: the note still carries a citation. Volume, pages, \
|
||||
and DOI have their own fields, and a DOI in a note is a link nobody \
|
||||
can follow. `coursebank migrate references` takes it apart."
|
||||
));
|
||||
}
|
||||
}
|
||||
if let Some(pmid) = &reference.pmid {
|
||||
if !pmid.trim().chars().all(|c| c.is_ascii_digit()) {
|
||||
issues.push(format!(
|
||||
@@ -1646,9 +1695,20 @@ impl CourseFile {
|
||||
.filter(|(_, target)| target.objective == objective)
|
||||
.map(|(id, _)| id)
|
||||
.collect();
|
||||
// By ceiling, then by id. A target is a question template rather than a
|
||||
// step in a sequence — an objective's targets are not taught in an
|
||||
// order, and an exam samples from them — so there is no teaching order
|
||||
// to print. What there is is depth, and grouping by it puts the
|
||||
// checklist in the order the study methods apply: recall for a Level 1
|
||||
// target, explanation for Level 2, variations and written solutions
|
||||
// above that. Ties break by id so the list is stable.
|
||||
ids.sort_by_key(|id| {
|
||||
let target = &self.learning_targets[*id];
|
||||
(target.order.unwrap_or(u32::MAX), (*id).clone())
|
||||
(
|
||||
self.effective_level_ceiling(id)
|
||||
.map(|l| l.code())
|
||||
.unwrap_or(0),
|
||||
(*id).clone(),
|
||||
)
|
||||
});
|
||||
ids.into_iter().map(String::as_str).collect()
|
||||
}
|
||||
|
||||
@@ -535,6 +535,27 @@ impl Merge {
|
||||
}
|
||||
}
|
||||
|
||||
// Teaching order, from the two sequences that already declare it: the
|
||||
// lectures in course order, and each lecture's `teaches` list. This is
|
||||
// what lets an objective stop carrying a hand-kept integer.
|
||||
let mut position = 0u32;
|
||||
let ordered: Vec<String> = self.lectures.keys().cloned().collect();
|
||||
for lecture_id in ordered {
|
||||
let teaches = self
|
||||
.lectures
|
||||
.get(&lecture_id)
|
||||
.map(|l| l.teaches.clone())
|
||||
.unwrap_or_default();
|
||||
for objective_id in teaches {
|
||||
position += 1;
|
||||
if let Some(objective) = self.objectives.get_mut(&objective_id) {
|
||||
if objective.order.is_none() {
|
||||
objective.order = Some(position);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A target with no lecture of its own is taught wherever its objective
|
||||
// is. Collected first: the read of `objectives` and the write to
|
||||
// `targets` cannot overlap in one pass.
|
||||
@@ -800,6 +821,93 @@ units:
|
||||
assert!(course.origin("nothing-like-this").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn teaching_order_is_derived_from_the_two_sequences_that_declare_it() {
|
||||
let root = tmp("order");
|
||||
write(&root, "course.yaml", ROOT);
|
||||
write(
|
||||
&root,
|
||||
"lectures/l-1-2.yaml",
|
||||
"lectures:\n L1.2:\n title: One\n teaches: [lo-second, lo-first]\n",
|
||||
);
|
||||
write(
|
||||
&root,
|
||||
"lectures/l-1-3.yaml",
|
||||
"lectures:\n L1.3:\n title: Two\n teaches: [lo-third]\n",
|
||||
);
|
||||
write(
|
||||
&root,
|
||||
"objectives/lo-first.yaml",
|
||||
r#"learning_objectives:
|
||||
lo-first:
|
||||
text: A.
|
||||
lo-second:
|
||||
text: B.
|
||||
lo-third:
|
||||
text: C.
|
||||
"#,
|
||||
);
|
||||
|
||||
let course = assemble(&root).unwrap();
|
||||
// Position in `teaches`, not id order: the lecture lists `lo-second`
|
||||
// first and that is what teaching it first means.
|
||||
assert_eq!(course.learning_objectives["lo-second"].order, Some(1));
|
||||
assert_eq!(course.learning_objectives["lo-first"].order, Some(2));
|
||||
assert_eq!(course.learning_objectives["lo-third"].order, Some(3));
|
||||
assert_eq!(
|
||||
course.lecture_objectives("L1.2"),
|
||||
vec!["lo-second", "lo-first"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn targets_print_by_ceiling_rather_than_in_a_sequence() {
|
||||
let root = tmp("target-order");
|
||||
write(&root, "course.yaml", ROOT);
|
||||
write(
|
||||
&root,
|
||||
"lectures/l-1-2.yaml",
|
||||
"lectures:\n L1.2:\n title: One\n teaches: [lo-x]\n",
|
||||
);
|
||||
write(
|
||||
&root,
|
||||
"objectives/lo-x.yaml",
|
||||
r#"learning_objectives:
|
||||
lo-x:
|
||||
text: A.
|
||||
level_ceiling: 3
|
||||
learning_targets:
|
||||
t-predict:
|
||||
text: Predict the effect.
|
||||
objective: lo-x
|
||||
level_ceiling: 3
|
||||
t-define:
|
||||
text: Define the term.
|
||||
objective: lo-x
|
||||
level_ceiling: 1
|
||||
t-explain:
|
||||
text: Explain the mechanism.
|
||||
objective: lo-x
|
||||
level_ceiling: 2
|
||||
"#,
|
||||
);
|
||||
|
||||
let course = assemble(&root).unwrap();
|
||||
assert!(course.validate().is_empty(), "{:?}", course.validate());
|
||||
|
||||
// Shallowest first, which is the order the study methods apply in:
|
||||
// recall, then explanation, then variations. Not id order, which would
|
||||
// put `t-define` after `t-predict` for no reason at all.
|
||||
assert_eq!(
|
||||
course.targets("lo-x"),
|
||||
vec!["t-define", "t-explain", "t-predict"]
|
||||
);
|
||||
|
||||
// A target with no ceiling of its own inherits the objective's, so it
|
||||
// sorts where that puts it.
|
||||
assert_eq!(course.learning_targets["t-define"].order, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn teaching_the_same_objective_from_two_lectures_unions() {
|
||||
let root = tmp("union");
|
||||
|
||||
+8
-1
@@ -157,7 +157,14 @@ pub struct Item {
|
||||
pub design: Option<Design>,
|
||||
|
||||
/// What the evidence says, accumulated across administrations.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
/// What the statistics say, filled in from `analysis/` at load.
|
||||
///
|
||||
/// Read from the store and never written back: `skip_serializing` means a
|
||||
/// bank file cannot acquire a `calibration:` block by being round-tripped
|
||||
/// through this type. A bank is a reviewed artifact whose diff should be a
|
||||
/// change of intent, and every grading run would otherwise produce a diff
|
||||
/// on it. See [`crate::calibration`].
|
||||
#[serde(default, skip_serializing)]
|
||||
pub calibration: Option<Calibration>,
|
||||
|
||||
/// The last review decision recorded for this item.
|
||||
|
||||
@@ -64,6 +64,24 @@ impl Layout {
|
||||
self.root.join("banks")
|
||||
}
|
||||
|
||||
/// Directory holding the statistics, which are kept out of the banks.
|
||||
///
|
||||
/// Committed, unlike [`Layout::data`]: everything under here is a cohort
|
||||
/// aggregate with no student in it. See [`crate::calibration`].
|
||||
pub fn analysis(&self) -> PathBuf {
|
||||
self.root.join("analysis")
|
||||
}
|
||||
|
||||
/// The pooled per-item calibration store.
|
||||
pub fn calibration_file(&self) -> PathBuf {
|
||||
self.analysis().join(crate::calibration::CALIBRATION_FILE)
|
||||
}
|
||||
|
||||
/// Directory holding one immutable record per administration.
|
||||
pub fn measurements(&self) -> PathBuf {
|
||||
self.analysis().join("administrations")
|
||||
}
|
||||
|
||||
/// Directory holding assessment records.
|
||||
pub fn assessments(&self) -> PathBuf {
|
||||
self.root.join("assessments")
|
||||
@@ -117,6 +135,8 @@ impl Layout {
|
||||
pub fn create_all(&self) -> Result<()> {
|
||||
for dir in [
|
||||
self.root.clone(),
|
||||
self.analysis(),
|
||||
self.measurements(),
|
||||
self.lectures(),
|
||||
self.objectives(),
|
||||
self.banks(),
|
||||
|
||||
@@ -124,6 +124,26 @@ pub struct SealMeta {
|
||||
/// from the file's own contents; [`SealFile::verify`] rebuilds a seal from the
|
||||
/// live course and compares this against it.
|
||||
pub digest: String,
|
||||
|
||||
/// The digests this file had before it was rewritten, oldest first.
|
||||
///
|
||||
/// A seal is meant to be tamper-evident, which puts a migration that
|
||||
/// renames an id inside it in an awkward position: the id is part of
|
||||
/// [`SealFile::digest_input`], so rewriting the file invalidates the digest
|
||||
/// that vouched for it, and recomputing it quietly would produce a record
|
||||
/// that looks untouched and is not.
|
||||
///
|
||||
/// So a migration does both — recomputes the digest and leaves the old one
|
||||
/// here. What the seal then says is the honest thing: these are the
|
||||
/// contents, this is what they hash to, and here is what the file hashed to
|
||||
/// before each rewrite. Anyone holding an earlier copy, a backup or a git
|
||||
/// revision, can check it against the right entry.
|
||||
///
|
||||
/// Deliberately outside [`SealFile::digest_input`]: a field that recorded
|
||||
/// past digests and was itself covered by the current one could not be
|
||||
/// appended to without invalidating what it describes.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub superseded_digests: Vec<String>,
|
||||
}
|
||||
|
||||
/// What wrote a seal.
|
||||
@@ -458,6 +478,7 @@ pub fn build(catalog: &Catalog, record: &AssessmentFile, opts: &Options) -> Resu
|
||||
content: opts.content,
|
||||
dropped,
|
||||
digest: String::new(),
|
||||
superseded_digests: Vec::new(),
|
||||
},
|
||||
items,
|
||||
forms: sealed_forms,
|
||||
@@ -787,6 +808,39 @@ impl SealFile {
|
||||
buf
|
||||
}
|
||||
|
||||
/// Renames the items this seal froze, keeping the digest honest.
|
||||
///
|
||||
/// The only sanctioned way to rewrite a seal. It recomputes the digest,
|
||||
/// because the ids are part of what the digest covers, and records the
|
||||
/// previous one in [`SealMeta::superseded_digests`], because a recomputed
|
||||
/// digest with no trace of the recompute is a record that claims never to
|
||||
/// have been touched.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `rename` - a map from old item id to new.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// How many placements were renamed. Zero leaves the file alone, digest
|
||||
/// included.
|
||||
pub fn rename_items(&mut self, rename: &BTreeMap<String, String>) -> usize {
|
||||
let mut renamed = 0;
|
||||
for item in &mut self.items {
|
||||
if let Some(new) = rename.get(&item.item) {
|
||||
item.item = new.clone();
|
||||
renamed += 1;
|
||||
}
|
||||
}
|
||||
if renamed == 0 {
|
||||
return 0;
|
||||
}
|
||||
let previous = std::mem::take(&mut self.seal.digest);
|
||||
self.seal.superseded_digests.push(previous);
|
||||
self.seal.digest = self.recompute_digest();
|
||||
renamed
|
||||
}
|
||||
|
||||
/// Recomputes this file's digest from its own contents.
|
||||
///
|
||||
/// # Returns
|
||||
@@ -1195,6 +1249,7 @@ mod tests {
|
||||
content: true,
|
||||
dropped: Vec::new(),
|
||||
digest: String::new(),
|
||||
superseded_digests: Vec::new(),
|
||||
},
|
||||
items: vec![SealedItem {
|
||||
number: 1,
|
||||
@@ -1343,6 +1398,31 @@ mod tests {
|
||||
assert_eq!(short("sha256:0123456789abcdef"), "sha256:01234567");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renaming_an_item_recomputes_the_digest_and_says_it_did() {
|
||||
let mut seal = sample();
|
||||
seal.seal.digest = seal.recompute_digest();
|
||||
let original = seal.seal.digest.clone();
|
||||
assert!(seal.check_self().is_empty());
|
||||
|
||||
let mut rename = BTreeMap::new();
|
||||
rename.insert(seal.items[0].item.clone(), "q-renamed".to_string());
|
||||
assert_eq!(seal.rename_items(&rename), 1);
|
||||
|
||||
// The digest covers the ids, so it has to move — and the file has to
|
||||
// admit that it moved rather than looking untouched.
|
||||
assert_eq!(seal.items[0].item, "q-renamed");
|
||||
assert_ne!(seal.seal.digest, original);
|
||||
assert_eq!(seal.seal.superseded_digests, vec![original]);
|
||||
assert!(seal.check_self().is_empty(), "{:?}", seal.check_self());
|
||||
|
||||
// A rename that matches nothing leaves the file entirely alone.
|
||||
let steady = seal.seal.digest.clone();
|
||||
assert_eq!(seal.rename_items(&BTreeMap::new()), 0);
|
||||
assert_eq!(seal.seal.digest, steady);
|
||||
assert_eq!(seal.seal.superseded_digests.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn round_trips_through_yaml() {
|
||||
let file = sample();
|
||||
|
||||
Reference in new issue
Block a user