feat: cooked up something fierce
This commit is contained in:
26 files changed
+1485
-44
No files matched your search
@@ -282,6 +282,26 @@ pub struct Placement {
|
||||
/// The keyed letters as administered.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub key: Vec<String>,
|
||||
|
||||
/// The option ids offered alongside the key.
|
||||
///
|
||||
/// Resolved when the assessment is assembled and written out explicitly,
|
||||
/// never sampled at export time. A blueprint may ask for a draw; the record
|
||||
/// holds what was drawn. Otherwise a bank edit between assembling and
|
||||
/// printing silently changes the paper, and the key printed on Tuesday
|
||||
/// disagrees with the one printed on Wednesday.
|
||||
///
|
||||
/// Empty means the whole pool, which is what every pre-2.0 record meant.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub distractors: Vec<String>,
|
||||
|
||||
/// The digest of the item as this administration showed it.
|
||||
///
|
||||
/// See [`crate::item::Item::variant_digest`]. The key statistics pool on:
|
||||
/// two administrations of one stem with different distractors are two
|
||||
/// items, and averaging them is averaging different questions.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub variant: Option<String>,
|
||||
/// The level as administered, denormalized so a record reads standalone.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub level: Option<Level>,
|
||||
@@ -357,6 +377,27 @@ pub enum DropStyle {
|
||||
}
|
||||
|
||||
impl Placement {
|
||||
/// The variant this placement administered.
|
||||
///
|
||||
/// Recorded when the assessment was assembled; derived from the option set
|
||||
/// otherwise, which is what makes every record written before 2.0 poolable
|
||||
/// without being rewritten. A pre-2.0 placement names its key and no
|
||||
/// distractors, which means the whole pool — a well-defined option set, and
|
||||
/// so a well-defined variant.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `item` - the item this placement names.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The digest.
|
||||
pub fn variant_of(&self, item: &crate::item::Item) -> String {
|
||||
self.variant
|
||||
.clone()
|
||||
.unwrap_or_else(|| item.variant_digest(&self.key, &self.distractors))
|
||||
}
|
||||
|
||||
/// Whether this placement was dropped by crediting every option.
|
||||
///
|
||||
/// # Returns
|
||||
|
||||
+55
-10
@@ -403,6 +403,15 @@ fn validate_item(
|
||||
if seen.contains(&o.id.as_str()) {
|
||||
issues.push(format!("option {pos}: duplicate option id `{}`", o.id));
|
||||
}
|
||||
if let Some(retirement) = &o.retired {
|
||||
if retirement.reason.trim().is_empty() {
|
||||
issues.push(format!(
|
||||
"option {pos}: retired without a reason. The reason is the finding — what \
|
||||
the option did or failed to do — and it is the only part of a retirement \
|
||||
that is worth anything later."
|
||||
));
|
||||
}
|
||||
}
|
||||
seen.push(&o.id);
|
||||
|
||||
let credit = o.credit();
|
||||
@@ -442,14 +451,27 @@ fn validate_item(
|
||||
}
|
||||
|
||||
// --- key ---
|
||||
// Counted over the pool that can still be drawn: a retired option is a
|
||||
// record, not an offer.
|
||||
let (live_keys, live_distractors) = it.pool();
|
||||
let keys = it.key_indices();
|
||||
match it.format {
|
||||
Format::SingleBestAnswer => {
|
||||
if keys.len() != 1 {
|
||||
issues.push(format!(
|
||||
"single_best_answer needs exactly one keyed option, has {}",
|
||||
keys.len()
|
||||
));
|
||||
// Several defensible keys is a pool, not a bug — it is what lets you
|
||||
// test whether "fourth" or "last of the four" is doing the work.
|
||||
// Exactly one of them reaches a student, and that is the
|
||||
// placement's business: see
|
||||
// [`crate::catalog::Catalog::validate_record`].
|
||||
if live_keys.is_empty() {
|
||||
issues.push("single_best_answer needs at least one keyed option".into());
|
||||
}
|
||||
if live_distractors.is_empty() {
|
||||
let retired = it.options.iter().any(|o| o.retired.is_some());
|
||||
issues.push(if retired {
|
||||
"every distractor is retired, so nothing can be drawn against the key".into()
|
||||
} else {
|
||||
"has no option that is not keyed correct, so it asks nothing".to_string()
|
||||
});
|
||||
}
|
||||
}
|
||||
Format::MultipleResponse => {
|
||||
@@ -780,20 +802,43 @@ mod tests {
|
||||
options:
|
||||
- { id: A, text: a, correct: true }
|
||||
- { id: B, text: b, correct: true }
|
||||
- id: q-a-003
|
||||
status: draft
|
||||
level: 1
|
||||
format: single_best_answer
|
||||
stem: s
|
||||
options:
|
||||
- { id: o-key-one, text: a, correct: true }
|
||||
- { id: o-key-two, text: b, correct: true }
|
||||
- { id: o-wrong-one, text: c }
|
||||
- { id: o-wrong-two, text: d }
|
||||
"#,
|
||||
);
|
||||
let issues = b.validate(None);
|
||||
// No key at all is still a bank problem: nothing can be drawn from it.
|
||||
assert!(
|
||||
issues
|
||||
.iter()
|
||||
.any(|i| i.contains("exactly one keyed option"))
|
||||
.any(|i| i.starts_with("q-a-001") && i.contains("at least one keyed option")),
|
||||
"{issues:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
// Keying every option is still a bank problem, for the older reason: a
|
||||
// question with nothing to choose against asks nothing.
|
||||
assert!(
|
||||
issues
|
||||
.iter()
|
||||
.filter(|i| i.contains("exactly one keyed option"))
|
||||
.count(),
|
||||
2
|
||||
.any(|i| i.starts_with("q-a-002") && i.contains("asks nothing")),
|
||||
"{issues:?}"
|
||||
);
|
||||
// Two defensible keys alongside real distractors is not a problem. It
|
||||
// is the pool doing its job — it is what lets you test whether the
|
||||
// wording of the key is what students are answering. Exactly one of
|
||||
// them reaches a student, and that is checked against the placement
|
||||
// that administers it, in `Catalog::validate_placement`.
|
||||
assert!(
|
||||
!issues.iter().any(|i| i.starts_with("q-a-003")
|
||||
&& (i.contains("keyed option") || i.contains("asks nothing"))),
|
||||
"{issues:?}"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
+112
-7
@@ -20,13 +20,13 @@
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::assessment::AssessmentFile;
|
||||
use crate::assessment::{AssessmentFile, Placement};
|
||||
use crate::bank::BankFile;
|
||||
use crate::course::CourseFile;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::item::Item;
|
||||
use crate::layout::Layout;
|
||||
use crate::taxonomy::{Level, Status, Tier};
|
||||
use crate::taxonomy::{Format, Level, Status, Tier};
|
||||
use crate::yaml;
|
||||
|
||||
/// One item plus everything needed to locate it again.
|
||||
@@ -261,6 +261,115 @@ impl Catalog {
|
||||
Ok(issues)
|
||||
}
|
||||
|
||||
/// Checks one placement's option set against the item's pool.
|
||||
///
|
||||
/// The checks that moved here from the bank when options became a pool. A
|
||||
/// bank holding two defensible keys and six distractors is sound; what has
|
||||
/// to hold for a *form* is that exactly one key reached the student, that
|
||||
/// none of the distractors was true, and that the count matches policy.
|
||||
/// None of that can be decided by looking at the item alone.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `p` - the placement.
|
||||
/// * `item` - the item it names.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// One message per problem.
|
||||
fn validate_placement(&self, p: &Placement, item: &Item) -> Vec<String> {
|
||||
let mut issues = Vec::new();
|
||||
if !item.format.has_options() {
|
||||
return issues;
|
||||
}
|
||||
let at = |number: u32| format!("question {number} ({})", p.item);
|
||||
|
||||
for id in p.key.iter().chain(p.distractors.iter()) {
|
||||
if item.option(id).is_none() {
|
||||
issues.push(format!(
|
||||
"{}: `{id}` is not an option of this item",
|
||||
at(p.number)
|
||||
));
|
||||
}
|
||||
}
|
||||
for id in &p.distractors {
|
||||
if p.key.iter().any(|k| k == id) {
|
||||
issues.push(format!(
|
||||
"{}: `{id}` is listed as both the key and a distractor",
|
||||
at(p.number)
|
||||
));
|
||||
}
|
||||
if item.option(id).is_some_and(|o| o.correct) {
|
||||
issues.push(format!(
|
||||
"{}: `{id}` is offered as a distractor but the bank keys it correct",
|
||||
at(p.number)
|
||||
));
|
||||
}
|
||||
}
|
||||
for id in &p.key {
|
||||
if item.option(id).is_some_and(|o| !o.correct) {
|
||||
issues.push(format!(
|
||||
"{}: `{id}` is keyed correct here but the bank does not key it. An option is \
|
||||
true or it is not; which true option a form uses is this record's choice, \
|
||||
but not whether it is true.",
|
||||
at(p.number)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let single = item.format == Format::SingleBestAnswer;
|
||||
if single && p.key.len() > 1 {
|
||||
issues.push(format!(
|
||||
"{}: single_best_answer administers exactly one key, this names {}",
|
||||
at(p.number),
|
||||
p.key.len()
|
||||
));
|
||||
}
|
||||
|
||||
// No distractor list means the whole pool, which is what a pre-2.0
|
||||
// record means and what an item whose pool is its form still means.
|
||||
// The record still has to say which key, when the pool offers a choice.
|
||||
if p.distractors.is_empty() {
|
||||
let (keys, _) = item.pool();
|
||||
if p.key.is_empty() && keys.len() > 1 && single {
|
||||
issues.push(format!(
|
||||
"{}: the item offers {} defensible keys, so the record has to say which one \
|
||||
this assessment used",
|
||||
at(p.number),
|
||||
keys.len()
|
||||
));
|
||||
}
|
||||
} else {
|
||||
let shown = item.administered(&p.key, &p.distractors);
|
||||
let expected = self.course.policy.options_per_item;
|
||||
if shown.len() != expected {
|
||||
issues.push(format!(
|
||||
"{}: administers {} option(s), but course policy is {expected} per item",
|
||||
at(p.number),
|
||||
shown.len()
|
||||
));
|
||||
}
|
||||
if single && p.key.is_empty() {
|
||||
issues.push(format!(
|
||||
"{}: names its distractors but not its key, so what was marked correct is \
|
||||
left to whatever the bank says today",
|
||||
at(p.number)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(recorded) = &p.variant {
|
||||
if *recorded != item.variant_digest(&p.key, &p.distractors) {
|
||||
issues.push(format!(
|
||||
"{}: an administered option has been reworded since this assessment. \
|
||||
Statistics pooled under this variant describe the older wording.",
|
||||
at(p.number)
|
||||
));
|
||||
}
|
||||
}
|
||||
issues
|
||||
}
|
||||
|
||||
/// Checks every sealed administration's stems against the bank.
|
||||
///
|
||||
/// The seal is the authority on what was administered, so this is the
|
||||
@@ -334,11 +443,7 @@ impl Catalog {
|
||||
));
|
||||
}
|
||||
}
|
||||
if !p.key.is_empty() && p.key != entry.item.key_letters() {
|
||||
issues.push(format!(
|
||||
"question {} ({}): the recorded key {:?} differs from the item's current key {:?}",
|
||||
p.number, p.item, p.key, entry.item.key_letters()));
|
||||
}
|
||||
issues.extend(self.validate_placement(p, &entry.item));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -257,6 +257,17 @@ pub struct Choice {
|
||||
/// Your a priori guess at how often this option is chosen.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub selection_rate_expected: Option<f64>,
|
||||
|
||||
/// Why this option is no longer drawn, when it is not.
|
||||
///
|
||||
/// A retired option stays in the file forever. It has to: a seal and four
|
||||
/// terms of response rows refer to it by id, and deleting it would turn
|
||||
/// every one of those references into a dangling one. What retirement does
|
||||
/// is take it out of the pool an assessment draws from, with the reason
|
||||
/// attached — "selected by 1 of 96 across two administrations" is a finding
|
||||
/// about the option, and the place for it is next to the option.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub retired: Option<Retirement>,
|
||||
}
|
||||
|
||||
impl Choice {
|
||||
@@ -605,6 +616,89 @@ pub struct Calibration {
|
||||
/// Machine-detected problems.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub flags: Vec<Flag>,
|
||||
|
||||
/// One record per option set ever administered.
|
||||
///
|
||||
/// What the flat fields above cannot express once options are a pool. A
|
||||
/// stem shown with distractors `{third, second, first}` is a measurably
|
||||
/// easier item than the same stem with `{third, plus-line, line-two}`, so a
|
||||
/// p-value pooled across both is the average of two different questions.
|
||||
/// Statistics are computed and compared per variant; the flat fields remain
|
||||
/// as the pre-2.0 summary, and for an item whose pool is its form the two
|
||||
/// agree.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub variants: Vec<VariantCalibration>,
|
||||
|
||||
/// One record per option, pooled across every set it appeared in.
|
||||
///
|
||||
/// The capability the pool is worth the trouble for. Selection rates are
|
||||
/// shares of a fixed set, so they are only comparable *within* a variant —
|
||||
/// which means this view supports exactly one kind of claim, and it is the
|
||||
/// useful one: this option draws nobody, anywhere. That is the evidence
|
||||
/// that retires a distractor, and one administration cannot supply it.
|
||||
#[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
|
||||
pub options: std::collections::BTreeMap<String, OptionHistory>,
|
||||
}
|
||||
|
||||
/// Statistics for one option set, as administered.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct VariantCalibration {
|
||||
/// The digest this record describes. See [`Item::variant_digest`].
|
||||
pub variant: String,
|
||||
/// The option ids keyed correct.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub key: Vec<String>,
|
||||
/// The option ids offered alongside them.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub distractors: Vec<String>,
|
||||
/// The administrations pooled into these numbers.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub administrations: Vec<String>,
|
||||
/// Examinees pooled.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub n_examinees: Option<usize>,
|
||||
/// Proportion correct.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub p_value: Option<f64>,
|
||||
/// Corrected item-total point-biserial correlation.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub point_biserial: Option<f64>,
|
||||
/// Upper-minus-lower-group discrimination index.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub discrimination_index: Option<f64>,
|
||||
/// Per-option behaviour within this set, keyed by option id.
|
||||
#[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
|
||||
pub option_stats: std::collections::BTreeMap<String, OptionStat>,
|
||||
/// Fitted item response theory parameters for this set.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub irt: Option<IrtParams>,
|
||||
/// Machine-detected problems with this set.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub flags: Vec<Flag>,
|
||||
}
|
||||
|
||||
/// What one option has done across every set it has appeared in.
|
||||
///
|
||||
/// Deliberately coarse. Averaging selection rates across variants is not
|
||||
/// meaningful — each is a share of a different set — so `mean_selection_rate`
|
||||
/// is a summary for reading, not a statistic to act on. `never_chosen` is the
|
||||
/// one field that carries weight, and it needs several administrations to earn.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct OptionHistory {
|
||||
/// How many distinct variants this option has appeared in.
|
||||
#[serde(default)]
|
||||
pub appearances: usize,
|
||||
/// Examinees who saw it, summed across those variants.
|
||||
#[serde(default)]
|
||||
pub n_examinees: usize,
|
||||
/// Mean of its within-variant selection rates. For reading only.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub mean_selection_rate: Option<f64>,
|
||||
/// Whether it has never been chosen, anywhere.
|
||||
#[serde(default, skip_serializing_if = "is_false")]
|
||||
pub never_chosen: bool,
|
||||
}
|
||||
|
||||
/// How one option behaved.
|
||||
@@ -932,6 +1026,98 @@ impl Item {
|
||||
fingerprint(parts.iter().map(|s| s.as_str()))
|
||||
}
|
||||
|
||||
/// The options an assessment administers, in the order the bank declares
|
||||
/// them.
|
||||
///
|
||||
/// Since 2.0 `options` is a *pool*: it may hold several defensible keys and
|
||||
/// more distractors than any one form shows, and which of them a student
|
||||
/// saw is a property of the placement rather than of the item. Everything
|
||||
/// that renders, seals, decodes, or scores an administration has to work
|
||||
/// from this rather than from `options`, or the paper and the key disagree.
|
||||
///
|
||||
/// Bank order, not administered order: the per-form permutation is
|
||||
/// [`crate::select::option_order`]'s business, and keeping the two separate
|
||||
/// is what lets one item appear on three forms with one set of statistics.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `key` - the option ids keyed correct for this administration.
|
||||
/// * `distractors` - the option ids offered alongside them.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The named options, or the whole live pool when `distractors` is empty.
|
||||
///
|
||||
/// `distractors` is what says the set was chosen, not `key`. A pre-2.0
|
||||
/// record names its key and nothing else — `key: [D]` with no distractor
|
||||
/// list — and it means "all of them, and D is the right one". Reading that
|
||||
/// as "administer D alone" would print a one-option paper for every
|
||||
/// assessment ever recorded.
|
||||
pub fn administered(&self, key: &[String], distractors: &[String]) -> Vec<&Choice> {
|
||||
if distractors.is_empty() {
|
||||
return self
|
||||
.options
|
||||
.iter()
|
||||
.filter(|o| o.retired.is_none())
|
||||
.collect();
|
||||
}
|
||||
self.options
|
||||
.iter()
|
||||
.filter(|o| key.contains(&o.id) || distractors.contains(&o.id))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The options that may still be drawn.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Every option not retired, split into candidate keys and distractors.
|
||||
pub fn pool(&self) -> (Vec<&Choice>, Vec<&Choice>) {
|
||||
let live = || self.options.iter().filter(|o| o.retired.is_none());
|
||||
(
|
||||
live().filter(|o| o.correct).collect(),
|
||||
live().filter(|o| !o.correct).collect(),
|
||||
)
|
||||
}
|
||||
|
||||
/// A digest of the item as one administration showed it.
|
||||
///
|
||||
/// The pooling key for statistics, and the reason
|
||||
/// [`Item::fingerprint`] cannot be. A stem with distractors
|
||||
/// `{third, second, first}` is a measurably easier item than the same stem
|
||||
/// with `{third, plus-line, line-two}`, so pooling a p-value across both is
|
||||
/// averaging two different questions. Covers the stem, the administered
|
||||
/// options, and which of them was keyed — the last because the same option
|
||||
/// set with a different key is again a different item.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `key` - the option ids keyed correct for this administration.
|
||||
/// * `distractors` - the option ids offered alongside them.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The digest as hex.
|
||||
pub fn variant_digest(&self, key: &[String], distractors: &[String]) -> String {
|
||||
let mut parts = vec![self.stem_digest()];
|
||||
let mut shown: Vec<&Choice> = self.administered(key, distractors);
|
||||
shown.sort_by(|a, b| a.id.cmp(&b.id));
|
||||
for option in shown {
|
||||
let keyed = if key.is_empty() {
|
||||
option.correct
|
||||
} else {
|
||||
key.contains(&option.id)
|
||||
};
|
||||
parts.push(format!(
|
||||
"{}|{}|{}",
|
||||
option.id,
|
||||
if keyed { "1" } else { "0" },
|
||||
option.text.trim()
|
||||
));
|
||||
}
|
||||
fingerprint(parts.iter().map(|s| s.as_str()))
|
||||
}
|
||||
|
||||
/// A digest of what the item asks, without its options.
|
||||
///
|
||||
/// The identity check. [`Item::fingerprint`] covers the options too, which
|
||||
@@ -955,6 +1141,44 @@ impl Item {
|
||||
fingerprint(parts.iter().map(|s| s.as_str()))
|
||||
}
|
||||
|
||||
/// The calibration recorded for one option set.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `variant` - the digest from [`Item::variant_digest`].
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The record, or `None` when this set has not been calibrated.
|
||||
pub fn calibration_for(&self, variant: &str) -> Option<&VariantCalibration> {
|
||||
self.calibration
|
||||
.as_ref()?
|
||||
.variants
|
||||
.iter()
|
||||
.find(|v| v.variant == variant)
|
||||
}
|
||||
|
||||
/// Whether a variant's recorded statistics still describe it.
|
||||
///
|
||||
/// Staleness gets *narrower* with a pool rather than wider: rewording one
|
||||
/// distractor used to invalidate the item's whole calibration, and now it
|
||||
/// invalidates only the sets that distractor appeared in.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `variant` - the digest to check.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// `false` only when a record exists for that digest and the digest no
|
||||
/// longer matches what the option ids now say.
|
||||
pub fn variant_is_current(&self, variant: &str) -> bool {
|
||||
match self.calibration_for(variant) {
|
||||
Some(record) => variant == self.variant_digest(&record.key, &record.distractors),
|
||||
None => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the recorded calibration matches the current content.
|
||||
///
|
||||
/// # Returns
|
||||
@@ -1131,6 +1355,101 @@ options:
|
||||
assert_eq!(a.fingerprint(), b.fingerprint());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_pool_administers_a_subset_and_defaults_to_everything() {
|
||||
let mut it = item(MINIMAL);
|
||||
let all: Vec<String> = it.options.iter().map(|o| o.id.clone()).collect();
|
||||
|
||||
// Unstated means the whole pool, which is what a pre-2.0 record meant.
|
||||
assert_eq!(it.administered(&[], &[]).len(), all.len());
|
||||
|
||||
// A retired option leaves the pool but not the file.
|
||||
it.options[1].retired = Some(Retirement {
|
||||
on: Date::new(2026, 9, 20).unwrap(),
|
||||
reason: "chosen by 1 of 96 across two administrations".into(),
|
||||
replaced_by: None,
|
||||
});
|
||||
let shown = it.administered(&[], &[]);
|
||||
assert_eq!(shown.len(), all.len() - 1);
|
||||
assert!(!shown.iter().any(|o| o.id == all[1]));
|
||||
// Still resolvable: a seal and four terms of rows refer to it.
|
||||
assert!(it.option(&all[1]).is_some());
|
||||
|
||||
// A key with no distractor list is a pre-2.0 record, and it means all
|
||||
// of them. Reading it as "administer the key alone" would print a
|
||||
// one-option paper for every assessment already recorded.
|
||||
assert_eq!(it.administered(&[all[2].clone()], &[]).len(), all.len() - 1);
|
||||
|
||||
// Named explicitly, bank order is kept whatever order the lists are in.
|
||||
let shown = it.administered(&[all[2].clone()], &[all[0].clone()]);
|
||||
assert_eq!(
|
||||
shown.iter().map(|o| o.id.clone()).collect::<Vec<_>>(),
|
||||
vec![all[0].clone(), all[2].clone()]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_variant_digest_tracks_the_option_set_and_the_stem_does_not() {
|
||||
let it = item(MINIMAL);
|
||||
let ids: Vec<String> = it.options.iter().map(|o| o.id.clone()).collect();
|
||||
|
||||
let one = it.variant_digest(&[ids[0].clone()], &[ids[1].clone()]);
|
||||
let two = it.variant_digest(&[ids[0].clone()], &[ids[2].clone()]);
|
||||
// A different distractor is a different item: same stem, different
|
||||
// difficulty, so pooling a p-value across both would average two
|
||||
// questions.
|
||||
assert_ne!(one, two, "a swapped distractor is a new variant");
|
||||
// The stem is unmoved by any of it.
|
||||
assert_eq!(it.stem_digest(), item(MINIMAL).stem_digest());
|
||||
|
||||
// Order of the lists is not part of the identity.
|
||||
assert_eq!(
|
||||
it.variant_digest(&[ids[0].clone()], &[ids[2].clone(), ids[1].clone()]),
|
||||
it.variant_digest(&[ids[0].clone()], &[ids[1].clone(), ids[2].clone()])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_variant_goes_stale_alone_rather_than_taking_the_item_with_it() {
|
||||
let mut it = item(MINIMAL);
|
||||
let ids: Vec<String> = it.options.iter().map(|o| o.id.clone()).collect();
|
||||
let one = it.variant_digest(&[ids[0].clone()], &[ids[1].clone()]);
|
||||
let two = it.variant_digest(&[ids[0].clone()], &[ids[2].clone()]);
|
||||
|
||||
it.calibration = Some(Calibration {
|
||||
variants: vec![
|
||||
VariantCalibration {
|
||||
variant: one.clone(),
|
||||
key: vec![ids[0].clone()],
|
||||
distractors: vec![ids[1].clone()],
|
||||
n_examinees: Some(96),
|
||||
p_value: Some(0.84),
|
||||
..VariantCalibration::default()
|
||||
},
|
||||
VariantCalibration {
|
||||
variant: two.clone(),
|
||||
key: vec![ids[0].clone()],
|
||||
distractors: vec![ids[2].clone()],
|
||||
n_examinees: Some(32),
|
||||
..VariantCalibration::default()
|
||||
},
|
||||
],
|
||||
..Calibration::default()
|
||||
});
|
||||
|
||||
assert_eq!(it.calibration_for(&one).unwrap().n_examinees, Some(96));
|
||||
assert!(it.calibration_for("nothing-like-this").is_none());
|
||||
assert!(it.variant_is_current(&one));
|
||||
assert!(it.variant_is_current(&two));
|
||||
|
||||
// Rewording the option that only the second set used leaves the first
|
||||
// set's numbers standing. Before the pool, one distractor edit
|
||||
// invalidated every statistic the item had.
|
||||
it.options[2].text = "a different distractor".into();
|
||||
assert!(it.variant_is_current(&one), "the first set never showed it");
|
||||
assert!(!it.variant_is_current(&two));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_calibration_is_detectable() {
|
||||
let mut it = item(MINIMAL);
|
||||
|
||||
+4
-3
@@ -539,7 +539,9 @@ fn sealed_form(catalog: &Catalog, record: &AssessmentFile, form: &Form) -> Resul
|
||||
for (index, placement) in printed.iter().enumerate() {
|
||||
let entry = catalog.require(&placement.item)?;
|
||||
let item = &entry.item;
|
||||
let n = item.options.len();
|
||||
// The pool is not the paper: seal what this placement administered.
|
||||
let shown = item.administered(&placement.key, &placement.distractors);
|
||||
let n = shown.len();
|
||||
let order = select::option_order(form, &placement.item, n);
|
||||
|
||||
let canonical_key: BTreeSet<String> = if placement.key.is_empty() {
|
||||
@@ -551,8 +553,7 @@ fn sealed_form(catalog: &Catalog, record: &AssessmentFile, form: &Form) -> Resul
|
||||
let mut options = Vec::with_capacity(n);
|
||||
let mut printed_key = Vec::new();
|
||||
for (position, source_index) in order.iter().enumerate() {
|
||||
let canonical = item
|
||||
.options
|
||||
let canonical = shown
|
||||
.get(*source_index)
|
||||
.map(|c| c.id.clone())
|
||||
.unwrap_or_else(|| printed_letter(*source_index));
|
||||
|
||||
Reference in new issue
Block a user