feat: cooked up something fierce
Pipeline / check (pull_request) Successful in 3m9s
Pipeline / docs (pull_request) Skipped
Pipeline / nightly (pull_request) Skipped
Pipeline / release (pull_request) Skipped

This commit is contained in:
alexm committed 2026-09-26 18:22:25 -04:00
1 parent eabc98ad31
commit 5ac1e317c0
26 files changed
+1485 -44

No files matched your search

+41
View File
@@ -282,6 +282,26 @@ pub struct Placement {
/// The keyed letters as administered.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub key: Vec<String>,
/// The option ids offered alongside the key.
///
/// Resolved when the assessment is assembled and written out explicitly,
/// never sampled at export time. A blueprint may ask for a draw; the record
/// holds what was drawn. Otherwise a bank edit between assembling and
/// printing silently changes the paper, and the key printed on Tuesday
/// disagrees with the one printed on Wednesday.
///
/// Empty means the whole pool, which is what every pre-2.0 record meant.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub distractors: Vec<String>,
/// The digest of the item as this administration showed it.
///
/// See [`crate::item::Item::variant_digest`]. The key statistics pool on:
/// two administrations of one stem with different distractors are two
/// items, and averaging them is averaging different questions.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub variant: Option<String>,
/// The level as administered, denormalized so a record reads standalone.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub level: Option<Level>,
@@ -357,6 +377,27 @@ pub enum DropStyle {
}
impl Placement {
/// The variant this placement administered.
///
/// Recorded when the assessment was assembled; derived from the option set
/// otherwise, which is what makes every record written before 2.0 poolable
/// without being rewritten. A pre-2.0 placement names its key and no
/// distractors, which means the whole pool — a well-defined option set, and
/// so a well-defined variant.
///
/// # Arguments
///
/// * `item` - the item this placement names.
///
/// # Returns
///
/// The digest.
pub fn variant_of(&self, item: &crate::item::Item) -> String {
self.variant
.clone()
.unwrap_or_else(|| item.variant_digest(&self.key, &self.distractors))
}
/// Whether this placement was dropped by crediting every option.
///
/// # Returns
+55 -10
View File
@@ -403,6 +403,15 @@ fn validate_item(
if seen.contains(&o.id.as_str()) {
issues.push(format!("option {pos}: duplicate option id `{}`", o.id));
}
if let Some(retirement) = &o.retired {
if retirement.reason.trim().is_empty() {
issues.push(format!(
"option {pos}: retired without a reason. The reason is the finding — what \
the option did or failed to do — and it is the only part of a retirement \
that is worth anything later."
));
}
}
seen.push(&o.id);
let credit = o.credit();
@@ -442,14 +451,27 @@ fn validate_item(
}
// --- key ---
// Counted over the pool that can still be drawn: a retired option is a
// record, not an offer.
let (live_keys, live_distractors) = it.pool();
let keys = it.key_indices();
match it.format {
Format::SingleBestAnswer => {
if keys.len() != 1 {
issues.push(format!(
"single_best_answer needs exactly one keyed option, has {}",
keys.len()
));
// Several defensible keys is a pool, not a bug — it is what lets you
// test whether "fourth" or "last of the four" is doing the work.
// Exactly one of them reaches a student, and that is the
// placement's business: see
// [`crate::catalog::Catalog::validate_record`].
if live_keys.is_empty() {
issues.push("single_best_answer needs at least one keyed option".into());
}
if live_distractors.is_empty() {
let retired = it.options.iter().any(|o| o.retired.is_some());
issues.push(if retired {
"every distractor is retired, so nothing can be drawn against the key".into()
} else {
"has no option that is not keyed correct, so it asks nothing".to_string()
});
}
}
Format::MultipleResponse => {
@@ -780,20 +802,43 @@ mod tests {
options:
- { id: A, text: a, correct: true }
- { id: B, text: b, correct: true }
- id: q-a-003
status: draft
level: 1
format: single_best_answer
stem: s
options:
- { id: o-key-one, text: a, correct: true }
- { id: o-key-two, text: b, correct: true }
- { id: o-wrong-one, text: c }
- { id: o-wrong-two, text: d }
"#,
);
let issues = b.validate(None);
// No key at all is still a bank problem: nothing can be drawn from it.
assert!(
issues
.iter()
.any(|i| i.contains("exactly one keyed option"))
.any(|i| i.starts_with("q-a-001") && i.contains("at least one keyed option")),
"{issues:?}"
);
assert_eq!(
// Keying every option is still a bank problem, for the older reason: a
// question with nothing to choose against asks nothing.
assert!(
issues
.iter()
.filter(|i| i.contains("exactly one keyed option"))
.count(),
2
.any(|i| i.starts_with("q-a-002") && i.contains("asks nothing")),
"{issues:?}"
);
// Two defensible keys alongside real distractors is not a problem. It
// is the pool doing its job — it is what lets you test whether the
// wording of the key is what students are answering. Exactly one of
// them reaches a student, and that is checked against the placement
// that administers it, in `Catalog::validate_placement`.
assert!(
!issues.iter().any(|i| i.starts_with("q-a-003")
&& (i.contains("keyed option") || i.contains("asks nothing"))),
"{issues:?}"
);
}
+112 -7
View File
@@ -20,13 +20,13 @@
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use crate::assessment::AssessmentFile;
use crate::assessment::{AssessmentFile, Placement};
use crate::bank::BankFile;
use crate::course::CourseFile;
use crate::error::{Error, Result};
use crate::item::Item;
use crate::layout::Layout;
use crate::taxonomy::{Level, Status, Tier};
use crate::taxonomy::{Format, Level, Status, Tier};
use crate::yaml;
/// One item plus everything needed to locate it again.
@@ -261,6 +261,115 @@ impl Catalog {
Ok(issues)
}
/// Checks one placement's option set against the item's pool.
///
/// The checks that moved here from the bank when options became a pool. A
/// bank holding two defensible keys and six distractors is sound; what has
/// to hold for a *form* is that exactly one key reached the student, that
/// none of the distractors was true, and that the count matches policy.
/// None of that can be decided by looking at the item alone.
///
/// # Arguments
///
/// * `p` - the placement.
/// * `item` - the item it names.
///
/// # Returns
///
/// One message per problem.
fn validate_placement(&self, p: &Placement, item: &Item) -> Vec<String> {
let mut issues = Vec::new();
if !item.format.has_options() {
return issues;
}
let at = |number: u32| format!("question {number} ({})", p.item);
for id in p.key.iter().chain(p.distractors.iter()) {
if item.option(id).is_none() {
issues.push(format!(
"{}: `{id}` is not an option of this item",
at(p.number)
));
}
}
for id in &p.distractors {
if p.key.iter().any(|k| k == id) {
issues.push(format!(
"{}: `{id}` is listed as both the key and a distractor",
at(p.number)
));
}
if item.option(id).is_some_and(|o| o.correct) {
issues.push(format!(
"{}: `{id}` is offered as a distractor but the bank keys it correct",
at(p.number)
));
}
}
for id in &p.key {
if item.option(id).is_some_and(|o| !o.correct) {
issues.push(format!(
"{}: `{id}` is keyed correct here but the bank does not key it. An option is \
true or it is not; which true option a form uses is this record's choice, \
but not whether it is true.",
at(p.number)
));
}
}
let single = item.format == Format::SingleBestAnswer;
if single && p.key.len() > 1 {
issues.push(format!(
"{}: single_best_answer administers exactly one key, this names {}",
at(p.number),
p.key.len()
));
}
// No distractor list means the whole pool, which is what a pre-2.0
// record means and what an item whose pool is its form still means.
// The record still has to say which key, when the pool offers a choice.
if p.distractors.is_empty() {
let (keys, _) = item.pool();
if p.key.is_empty() && keys.len() > 1 && single {
issues.push(format!(
"{}: the item offers {} defensible keys, so the record has to say which one \
this assessment used",
at(p.number),
keys.len()
));
}
} else {
let shown = item.administered(&p.key, &p.distractors);
let expected = self.course.policy.options_per_item;
if shown.len() != expected {
issues.push(format!(
"{}: administers {} option(s), but course policy is {expected} per item",
at(p.number),
shown.len()
));
}
if single && p.key.is_empty() {
issues.push(format!(
"{}: names its distractors but not its key, so what was marked correct is \
left to whatever the bank says today",
at(p.number)
));
}
}
if let Some(recorded) = &p.variant {
if *recorded != item.variant_digest(&p.key, &p.distractors) {
issues.push(format!(
"{}: an administered option has been reworded since this assessment. \
Statistics pooled under this variant describe the older wording.",
at(p.number)
));
}
}
issues
}
/// Checks every sealed administration's stems against the bank.
///
/// The seal is the authority on what was administered, so this is the
@@ -334,11 +443,7 @@ impl Catalog {
));
}
}
if !p.key.is_empty() && p.key != entry.item.key_letters() {
issues.push(format!(
"question {} ({}): the recorded key {:?} differs from the item's current key {:?}",
p.number, p.item, p.key, entry.item.key_letters()));
}
issues.extend(self.validate_placement(p, &entry.item));
}
}
}
+319
View File
@@ -257,6 +257,17 @@ pub struct Choice {
/// Your a priori guess at how often this option is chosen.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selection_rate_expected: Option<f64>,
/// Why this option is no longer drawn, when it is not.
///
/// A retired option stays in the file forever. It has to: a seal and four
/// terms of response rows refer to it by id, and deleting it would turn
/// every one of those references into a dangling one. What retirement does
/// is take it out of the pool an assessment draws from, with the reason
/// attached — "selected by 1 of 96 across two administrations" is a finding
/// about the option, and the place for it is next to the option.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub retired: Option<Retirement>,
}
impl Choice {
@@ -605,6 +616,89 @@ pub struct Calibration {
/// Machine-detected problems.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub flags: Vec<Flag>,
/// One record per option set ever administered.
///
/// What the flat fields above cannot express once options are a pool. A
/// stem shown with distractors `{third, second, first}` is a measurably
/// easier item than the same stem with `{third, plus-line, line-two}`, so a
/// p-value pooled across both is the average of two different questions.
/// Statistics are computed and compared per variant; the flat fields remain
/// as the pre-2.0 summary, and for an item whose pool is its form the two
/// agree.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub variants: Vec<VariantCalibration>,
/// One record per option, pooled across every set it appeared in.
///
/// The capability the pool is worth the trouble for. Selection rates are
/// shares of a fixed set, so they are only comparable *within* a variant —
/// which means this view supports exactly one kind of claim, and it is the
/// useful one: this option draws nobody, anywhere. That is the evidence
/// that retires a distractor, and one administration cannot supply it.
#[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
pub options: std::collections::BTreeMap<String, OptionHistory>,
}
/// Statistics for one option set, as administered.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct VariantCalibration {
/// The digest this record describes. See [`Item::variant_digest`].
pub variant: String,
/// The option ids keyed correct.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub key: Vec<String>,
/// The option ids offered alongside them.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub distractors: Vec<String>,
/// The administrations pooled into these numbers.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub administrations: Vec<String>,
/// Examinees pooled.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub n_examinees: Option<usize>,
/// Proportion correct.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub p_value: Option<f64>,
/// Corrected item-total point-biserial correlation.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub point_biserial: Option<f64>,
/// Upper-minus-lower-group discrimination index.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub discrimination_index: Option<f64>,
/// Per-option behaviour within this set, keyed by option id.
#[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
pub option_stats: std::collections::BTreeMap<String, OptionStat>,
/// Fitted item response theory parameters for this set.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub irt: Option<IrtParams>,
/// Machine-detected problems with this set.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub flags: Vec<Flag>,
}
/// What one option has done across every set it has appeared in.
///
/// Deliberately coarse. Averaging selection rates across variants is not
/// meaningful — each is a share of a different set — so `mean_selection_rate`
/// is a summary for reading, not a statistic to act on. `never_chosen` is the
/// one field that carries weight, and it needs several administrations to earn.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct OptionHistory {
/// How many distinct variants this option has appeared in.
#[serde(default)]
pub appearances: usize,
/// Examinees who saw it, summed across those variants.
#[serde(default)]
pub n_examinees: usize,
/// Mean of its within-variant selection rates. For reading only.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub mean_selection_rate: Option<f64>,
/// Whether it has never been chosen, anywhere.
#[serde(default, skip_serializing_if = "is_false")]
pub never_chosen: bool,
}
/// How one option behaved.
@@ -932,6 +1026,98 @@ impl Item {
fingerprint(parts.iter().map(|s| s.as_str()))
}
/// The options an assessment administers, in the order the bank declares
/// them.
///
/// Since 2.0 `options` is a *pool*: it may hold several defensible keys and
/// more distractors than any one form shows, and which of them a student
/// saw is a property of the placement rather than of the item. Everything
/// that renders, seals, decodes, or scores an administration has to work
/// from this rather than from `options`, or the paper and the key disagree.
///
/// Bank order, not administered order: the per-form permutation is
/// [`crate::select::option_order`]'s business, and keeping the two separate
/// is what lets one item appear on three forms with one set of statistics.
///
/// # Arguments
///
/// * `key` - the option ids keyed correct for this administration.
/// * `distractors` - the option ids offered alongside them.
///
/// # Returns
///
/// The named options, or the whole live pool when `distractors` is empty.
///
/// `distractors` is what says the set was chosen, not `key`. A pre-2.0
/// record names its key and nothing else — `key: [D]` with no distractor
/// list — and it means "all of them, and D is the right one". Reading that
/// as "administer D alone" would print a one-option paper for every
/// assessment ever recorded.
pub fn administered(&self, key: &[String], distractors: &[String]) -> Vec<&Choice> {
if distractors.is_empty() {
return self
.options
.iter()
.filter(|o| o.retired.is_none())
.collect();
}
self.options
.iter()
.filter(|o| key.contains(&o.id) || distractors.contains(&o.id))
.collect()
}
/// The options that may still be drawn.
///
/// # Returns
///
/// Every option not retired, split into candidate keys and distractors.
pub fn pool(&self) -> (Vec<&Choice>, Vec<&Choice>) {
let live = || self.options.iter().filter(|o| o.retired.is_none());
(
live().filter(|o| o.correct).collect(),
live().filter(|o| !o.correct).collect(),
)
}
/// A digest of the item as one administration showed it.
///
/// The pooling key for statistics, and the reason
/// [`Item::fingerprint`] cannot be. A stem with distractors
/// `{third, second, first}` is a measurably easier item than the same stem
/// with `{third, plus-line, line-two}`, so pooling a p-value across both is
/// averaging two different questions. Covers the stem, the administered
/// options, and which of them was keyed — the last because the same option
/// set with a different key is again a different item.
///
/// # Arguments
///
/// * `key` - the option ids keyed correct for this administration.
/// * `distractors` - the option ids offered alongside them.
///
/// # Returns
///
/// The digest as hex.
pub fn variant_digest(&self, key: &[String], distractors: &[String]) -> String {
let mut parts = vec![self.stem_digest()];
let mut shown: Vec<&Choice> = self.administered(key, distractors);
shown.sort_by(|a, b| a.id.cmp(&b.id));
for option in shown {
let keyed = if key.is_empty() {
option.correct
} else {
key.contains(&option.id)
};
parts.push(format!(
"{}|{}|{}",
option.id,
if keyed { "1" } else { "0" },
option.text.trim()
));
}
fingerprint(parts.iter().map(|s| s.as_str()))
}
/// A digest of what the item asks, without its options.
///
/// The identity check. [`Item::fingerprint`] covers the options too, which
@@ -955,6 +1141,44 @@ impl Item {
fingerprint(parts.iter().map(|s| s.as_str()))
}
/// The calibration recorded for one option set.
///
/// # Arguments
///
/// * `variant` - the digest from [`Item::variant_digest`].
///
/// # Returns
///
/// The record, or `None` when this set has not been calibrated.
pub fn calibration_for(&self, variant: &str) -> Option<&VariantCalibration> {
self.calibration
.as_ref()?
.variants
.iter()
.find(|v| v.variant == variant)
}
/// Whether a variant's recorded statistics still describe it.
///
/// Staleness gets *narrower* with a pool rather than wider: rewording one
/// distractor used to invalidate the item's whole calibration, and now it
/// invalidates only the sets that distractor appeared in.
///
/// # Arguments
///
/// * `variant` - the digest to check.
///
/// # Returns
///
/// `false` only when a record exists for that digest and the digest no
/// longer matches what the option ids now say.
pub fn variant_is_current(&self, variant: &str) -> bool {
match self.calibration_for(variant) {
Some(record) => variant == self.variant_digest(&record.key, &record.distractors),
None => true,
}
}
/// Whether the recorded calibration matches the current content.
///
/// # Returns
@@ -1131,6 +1355,101 @@ options:
assert_eq!(a.fingerprint(), b.fingerprint());
}
#[test]
fn a_pool_administers_a_subset_and_defaults_to_everything() {
let mut it = item(MINIMAL);
let all: Vec<String> = it.options.iter().map(|o| o.id.clone()).collect();
// Unstated means the whole pool, which is what a pre-2.0 record meant.
assert_eq!(it.administered(&[], &[]).len(), all.len());
// A retired option leaves the pool but not the file.
it.options[1].retired = Some(Retirement {
on: Date::new(2026, 9, 20).unwrap(),
reason: "chosen by 1 of 96 across two administrations".into(),
replaced_by: None,
});
let shown = it.administered(&[], &[]);
assert_eq!(shown.len(), all.len() - 1);
assert!(!shown.iter().any(|o| o.id == all[1]));
// Still resolvable: a seal and four terms of rows refer to it.
assert!(it.option(&all[1]).is_some());
// A key with no distractor list is a pre-2.0 record, and it means all
// of them. Reading it as "administer the key alone" would print a
// one-option paper for every assessment already recorded.
assert_eq!(it.administered(&[all[2].clone()], &[]).len(), all.len() - 1);
// Named explicitly, bank order is kept whatever order the lists are in.
let shown = it.administered(&[all[2].clone()], &[all[0].clone()]);
assert_eq!(
shown.iter().map(|o| o.id.clone()).collect::<Vec<_>>(),
vec![all[0].clone(), all[2].clone()]
);
}
#[test]
fn the_variant_digest_tracks_the_option_set_and_the_stem_does_not() {
let it = item(MINIMAL);
let ids: Vec<String> = it.options.iter().map(|o| o.id.clone()).collect();
let one = it.variant_digest(&[ids[0].clone()], &[ids[1].clone()]);
let two = it.variant_digest(&[ids[0].clone()], &[ids[2].clone()]);
// A different distractor is a different item: same stem, different
// difficulty, so pooling a p-value across both would average two
// questions.
assert_ne!(one, two, "a swapped distractor is a new variant");
// The stem is unmoved by any of it.
assert_eq!(it.stem_digest(), item(MINIMAL).stem_digest());
// Order of the lists is not part of the identity.
assert_eq!(
it.variant_digest(&[ids[0].clone()], &[ids[2].clone(), ids[1].clone()]),
it.variant_digest(&[ids[0].clone()], &[ids[1].clone(), ids[2].clone()])
);
}
#[test]
fn a_variant_goes_stale_alone_rather_than_taking_the_item_with_it() {
let mut it = item(MINIMAL);
let ids: Vec<String> = it.options.iter().map(|o| o.id.clone()).collect();
let one = it.variant_digest(&[ids[0].clone()], &[ids[1].clone()]);
let two = it.variant_digest(&[ids[0].clone()], &[ids[2].clone()]);
it.calibration = Some(Calibration {
variants: vec![
VariantCalibration {
variant: one.clone(),
key: vec![ids[0].clone()],
distractors: vec![ids[1].clone()],
n_examinees: Some(96),
p_value: Some(0.84),
..VariantCalibration::default()
},
VariantCalibration {
variant: two.clone(),
key: vec![ids[0].clone()],
distractors: vec![ids[2].clone()],
n_examinees: Some(32),
..VariantCalibration::default()
},
],
..Calibration::default()
});
assert_eq!(it.calibration_for(&one).unwrap().n_examinees, Some(96));
assert!(it.calibration_for("nothing-like-this").is_none());
assert!(it.variant_is_current(&one));
assert!(it.variant_is_current(&two));
// Rewording the option that only the second set used leaves the first
// set's numbers standing. Before the pool, one distractor edit
// invalidated every statistic the item had.
it.options[2].text = "a different distractor".into();
assert!(it.variant_is_current(&one), "the first set never showed it");
assert!(!it.variant_is_current(&two));
}
#[test]
fn stale_calibration_is_detectable() {
let mut it = item(MINIMAL);
+4 -3
View File
@@ -539,7 +539,9 @@ fn sealed_form(catalog: &Catalog, record: &AssessmentFile, form: &Form) -> Resul
for (index, placement) in printed.iter().enumerate() {
let entry = catalog.require(&placement.item)?;
let item = &entry.item;
let n = item.options.len();
// The pool is not the paper: seal what this placement administered.
let shown = item.administered(&placement.key, &placement.distractors);
let n = shown.len();
let order = select::option_order(form, &placement.item, n);
let canonical_key: BTreeSet<String> = if placement.key.is_empty() {
@@ -551,8 +553,7 @@ fn sealed_form(catalog: &Catalog, record: &AssessmentFile, form: &Form) -> Resul
let mut options = Vec::with_capacity(n);
let mut printed_key = Vec::new();
for (position, source_index) in order.iter().enumerate() {
let canonical = item
.options
let canonical = shown
.get(*source_index)
.map(|c| c.id.clone())
.unwrap_or_else(|| printed_letter(*source_index));