// SPDX-License-Identifier: Prosperity-3.0.0 // Copyright Scientific Computing Studio // Source: https://git.scient.ing/education/coursebank //! The item: one assessable question, with both the intent it was written with //! and the evidence it has produced. //! //! The organizing idea is that those two things belong in one versioned place. //! [`Design`] is written before an item is ever used and says what you predict: //! how hard, how discriminating, how long, and what the item is meant to reveal. //! [`Calibration`] is written by the tool afterwards and says what happened. //! Keeping them adjacent is what turns a question bank into an instrument you //! can improve, because every administration produces a checkable prediction. //! //! One deliberate departure from a naive design: [`Calibration`] is *cumulative* //! rather than per-administration. Raw per-response data belongs in the Parquet //! tables under `data/`, which are far better at holding it, and an item's YAML //! holds the rolled-up estimate plus a list of which administrations went into //! it. That keeps bank files readable and reviewable in a pull request while //! still letting statistics accumulate across terms. use serde::{Deserialize, Serialize}; use crate::date::Date; use crate::hash::fingerprint; use crate::taxonomy::{ CognitiveProcess, Discrimination, ErrorType, Flag, Format, Level, ReviewAction, Status, }; /// One assessable question. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Item { /// Stable id, unique within its bank, conventionally `q--NNN`. /// /// Ids are never reused and never renumbered: the id is the join key that /// ties an item to every assessment it has appeared on and every response /// row ever recorded for it. pub id: String, /// Revision counter, bumped whenever the content changes in a way that /// invalidates pooled statistics. #[serde(default = "one_u32")] pub version: u32, /// Workflow state; only [`Status::Approved`] items may be assembled. pub status: Status, /// Cognitive demand, 1 through 5. pub level: Level, /// The specific process the item elicits. Required for approval, and checked /// against `level`. #[serde(default, skip_serializing_if = "Option::is_none")] pub cognitive_process: Option, /// Response format. #[serde(default = "default_format")] pub format: Format, /// A bonus item, scored outside the graded total. #[serde(default, skip_serializing_if = "is_false")] pub bonus: bool, /// Point value; falls back to the course policy when absent. #[serde(default, skip_serializing_if = "Option::is_none")] pub points: Option, /// A short human title, used in tables and Canvas question names. #[serde(default, skip_serializing_if = "Option::is_none")] pub title: Option, /// Id of a shared stimulus in the course registry, for case-based testlets. #[serde(default, skip_serializing_if = "Option::is_none")] pub stimulus: Option, /// The prompt. pub stem: String, /// The answer options in canonical order. Shuffling happens at export time /// per form, never here, so the bank stays diffable. pub options: Vec, /// Objectives this item measures, as ids into the course registry. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub learning_objectives: Vec, /// Where the material was taught. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub sources: Vec, /// Free-form topic tags, for slicing a bank by subject rather than lecture. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub topics: Vec, /// Item ids or objective ids a student needs before this is fair. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub prerequisites: Vec, /// Figures or data files reproduced with the item. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub assets: Vec, /// What you predicted before using it. #[serde(default, skip_serializing_if = "Option::is_none")] pub design: Option, /// What the evidence says, accumulated across administrations. #[serde(default, skip_serializing_if = "Option::is_none")] pub calibration: Option, /// The last review decision recorded for this item. #[serde(default, skip_serializing_if = "Option::is_none")] pub review: Option, /// Append-only change log. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub history: Vec, /// The author of record. #[serde(default, skip_serializing_if = "Option::is_none")] pub author: Option, /// Notes that must never reach a student. #[serde(default, skip_serializing_if = "Option::is_none")] pub notes_private: Option, /// Set when the item was retired, with the reason. #[serde(default, skip_serializing_if = "Option::is_none")] pub retired: Option, } /// One answer option. /// /// The optional fields are what separate a designed distractor from filler. An /// option that names the [`ErrorType`] it targets and the misconception behind it /// is one you can report on: when a third of the cohort picks it, you know what /// they were thinking, and the student report can say so. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Choice { /// Option letter, `A` through `H`. pub id: String, /// The option text. pub text: String, /// Whether this option is keyed correct. #[serde(default)] pub correct: bool, /// Credit awarded, from 0 to 1. Absent means 1.0 when correct, else 0.0. #[serde(default, skip_serializing_if = "Option::is_none")] pub credit: Option, /// Instructor-facing rationale for why this option is right or wrong. #[serde(default, skip_serializing_if = "Option::is_none")] pub explanation: Option, /// The nudge you would give a student reconsidering this option. #[serde(default, skip_serializing_if = "Option::is_none")] pub hint: Option, /// The specific wrong idea this distractor is built to capture. #[serde(default, skip_serializing_if = "Option::is_none")] pub misconception: Option, /// The category of that error. #[serde(default, skip_serializing_if = "Option::is_none")] pub error_type: Option, /// Whether a wrong option is defensible enough to earn partial credit. #[serde(default, skip_serializing_if = "is_false")] pub defensible: bool, /// The argument for why it is defensible. Required whenever credit is /// awarded to a wrong option, so partial credit is always justified in /// writing rather than by memory. #[serde(default, skip_serializing_if = "Option::is_none")] pub defense: Option, /// Text released to students after the assessment. This is what a student /// report shows them when they chose this option. #[serde(default, skip_serializing_if = "Option::is_none")] pub feedback_student: Option, /// Your a priori guess at how often this option is chosen. #[serde(default, skip_serializing_if = "Option::is_none")] pub selection_rate_expected: Option, } impl Choice { /// The credit this option earns, resolving the default from `correct`. /// /// # Returns /// /// Credit in `[0, 1]`. pub fn credit(&self) -> f64 { self.credit.unwrap_or(if self.correct { 1.0 } else { 0.0 }) } /// Whether this option awards credit without being keyed correct. pub fn is_partial(&self) -> bool { !self.correct && self.credit() > 0.0 } /// The best available student-facing explanation of this option. /// /// Prefers explicit student feedback, then the misconception, then the /// instructor explanation, so a report degrades gracefully as authoring /// completeness varies. /// /// # Returns /// /// The text, or `None` when the option carries no rationale at all. pub fn student_text(&self) -> Option<&str> { self.feedback_student .as_deref() .or(self.misconception.as_deref()) .or(self.explanation.as_deref()) } } /// Where the assessed material was taught. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Source { /// Lecture id in the course registry. pub lecture: String, /// Slide numbers, so a student report can point at a page rather than a /// whole lecture. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub slides: Vec, /// Readings, cited however you cite them. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub readings: Vec, /// A timestamp into a recording, in seconds. #[serde(default, skip_serializing_if = "Option::is_none")] pub recording_seconds: Option, } /// A figure or data file reproduced with an item. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Asset { /// Path relative to the course root. pub path: String, /// Alt text. Required in practice for accessibility; the linter says so. #[serde(default, skip_serializing_if = "Option::is_none")] pub alt: Option, /// A caption printed below the figure. #[serde(default, skip_serializing_if = "Option::is_none")] pub caption: Option, } /// The a priori design of an item: your predictions, written down. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Design { /// The proportion of the target cohort you expect to answer correctly. #[serde(default, skip_serializing_if = "Option::is_none")] pub expected_difficulty: Option, /// How sharply you expect it to separate students. #[serde(default, skip_serializing_if = "Option::is_none")] pub expected_discrimination: Option, /// How long you expect it to take, in seconds. Summed over a form, this is /// how you check that an exam fits the period. #[serde(default, skip_serializing_if = "Option::is_none")] pub expected_time_seconds: Option, /// What the item is meant to reveal, and why it sits at its level. #[serde(default, skip_serializing_if = "Option::is_none")] pub rationale: Option, } /// Accumulated evidence about an item's behavior. #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Calibration { /// The administrations pooled into these numbers. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub administrations: Vec, /// When the calibration was last recomputed. #[serde(default, skip_serializing_if = "Option::is_none")] pub updated: Option, /// The content fingerprint these statistics describe. If it differs from the /// item's current fingerprint, the item was edited after calibration and the /// numbers are stale; the linter says so. #[serde(default, skip_serializing_if = "Option::is_none")] pub fingerprint: Option, /// Total examinees pooled. #[serde(default, skip_serializing_if = "Option::is_none")] pub n_examinees: Option, /// Proportion correct. #[serde(default, skip_serializing_if = "Option::is_none")] pub p_value: Option, /// Corrected item-total point-biserial correlation. #[serde(default, skip_serializing_if = "Option::is_none")] pub point_biserial: Option, /// Upper-minus-lower-group discrimination index. #[serde(default, skip_serializing_if = "Option::is_none")] pub discrimination_index: Option, /// Mean response time, when the platform reports it. #[serde(default, skip_serializing_if = "Option::is_none")] pub mean_response_time_seconds: Option, /// Proportion of responses faster than plausible reading time. #[serde(default, skip_serializing_if = "Option::is_none")] pub rapid_guess_rate: Option, /// Per-option behavior, keyed by option letter. #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")] pub option_stats: std::collections::BTreeMap, /// Fitted item response theory parameters. #[serde(default, skip_serializing_if = "Option::is_none")] pub irt: Option, /// Machine-detected problems. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub flags: Vec, } /// How one option behaved. #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct OptionStat { /// Proportion of examinees who chose it. #[serde(default, skip_serializing_if = "Option::is_none")] pub selection_rate: Option, /// Correlation between choosing it and total score. Negative for a working /// distractor; positive on a distractor is a warning. #[serde(default, skip_serializing_if = "Option::is_none")] pub point_biserial: Option, /// Selection rate among the top scoring group. #[serde(default, skip_serializing_if = "Option::is_none")] pub upper_group_rate: Option, /// Selection rate among the bottom scoring group. #[serde(default, skip_serializing_if = "Option::is_none")] pub lower_group_rate: Option, } /// Fitted item response theory parameters. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct IrtParams { /// Which model was fitted. pub model: IrtModel, /// Discrimination. pub a: f64, /// Difficulty, on the same scale as ability. pub b: f64, /// Lower asymptote, the guessing parameter. #[serde(default, skip_serializing_if = "Option::is_none")] pub c: Option, /// Standard error of `a`. #[serde(default, skip_serializing_if = "Option::is_none")] pub se_a: Option, /// Standard error of `b`. #[serde(default, skip_serializing_if = "Option::is_none")] pub se_b: Option, /// Examinees the fit was based on. Small samples give unstable parameters, /// so this travels with them rather than being looked up later. #[serde(default, skip_serializing_if = "Option::is_none")] pub n: Option, /// Whether priors were used, which matters when interpreting `a`. #[serde(default, skip_serializing_if = "is_false")] pub bayesian: bool, } /// The item response theory model family. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum IrtModel { /// One parameter: difficulty only, discrimination fixed at 1. Rasch, /// Two parameters: discrimination and difficulty. #[serde(rename = "2pl")] TwoPl, /// Three parameters, adding a lower asymptote for guessing. #[serde(rename = "3pl")] ThreePl, } impl IrtModel { /// The token used in YAML. pub fn as_str(self) -> &'static str { match self { IrtModel::Rasch => "rasch", IrtModel::TwoPl => "2pl", IrtModel::ThreePl => "3pl", } } } /// A recorded review decision. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Review { /// Who reviewed it. #[serde(default, skip_serializing_if = "Option::is_none")] pub reviewed_by: Option, /// When. #[serde(default, skip_serializing_if = "Option::is_none")] pub reviewed_on: Option, /// What was decided. pub action: ReviewAction, /// Why. #[serde(default, skip_serializing_if = "Option::is_none")] pub notes: Option, } /// Why and when an item left service. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Retirement { /// When it was retired. pub on: Date, /// Why. pub reason: String, /// A replacement item id, when one exists. #[serde(default, skip_serializing_if = "Option::is_none")] pub replaced_by: Option, } /// One entry in an item's change log. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct HistoryEntry { /// The version this change produced. pub version: u32, /// When it was made. pub date: Date, /// Who made it. #[serde(default, skip_serializing_if = "Option::is_none")] pub author: Option, /// What changed. pub change: String, } impl Item { /// Builds a draft item with everything optional left empty. /// /// A constructor rather than a `Default` implementation, because there is no /// sensible default id, stem, or option set: an item missing any of those is /// not a lesser item, it is not an item. Requiring them at construction means /// the only way to get a half-built `Item` is deliberately. /// /// The result is `Status::Draft` and deliberately will not pass /// [`Item::is_assemblable`] — it still needs learning objectives, sources, and /// a cognitive process before it can be drawn onto an assessment. /// /// # Arguments /// /// * `id` - the item id. /// * `level` - the cognitive level. /// * `stem` - the question. /// * `options` - the answer options. /// /// # Returns /// /// The draft item. pub fn draft(id: &str, level: Level, stem: &str, options: Vec) -> Item { Item { id: id.to_string(), version: 1, status: Status::Draft, level, cognitive_process: None, format: if options.iter().filter(|o| o.correct).count() > 1 { Format::MultipleResponse } else { Format::SingleBestAnswer }, bonus: false, points: None, title: None, stimulus: None, stem: stem.to_string(), options, learning_objectives: Vec::new(), sources: Vec::new(), topics: Vec::new(), prerequisites: Vec::new(), assets: Vec::new(), design: None, calibration: None, review: None, history: Vec::new(), author: None, notes_private: None, retired: None, } } /// Indices of the keyed-correct options. /// /// # Returns /// /// Zero-based indices into `options`. pub fn key_indices(&self) -> Vec { self.options .iter() .enumerate() .filter(|(_, o)| o.correct) .map(|(i, _)| i) .collect() } /// The keyed-correct option letters, sorted. /// /// # Returns /// /// Letters such as `["C"]` or `["A", "B"]`. pub fn key_letters(&self) -> Vec { let mut out: Vec = self .options .iter() .filter(|o| o.correct) .map(|o| o.id.clone()) .collect(); out.sort(); out } /// Looks up an option by letter. /// /// # Arguments /// /// * `letter` - the option id, case insensitive. /// /// # Returns /// /// The option, or `None`. pub fn option(&self, letter: &str) -> Option<&Choice> { self.options .iter() .find(|o| o.id.eq_ignore_ascii_case(letter)) } /// Whether the item keys more than one option. pub fn is_multi_key(&self) -> bool { self.key_indices().len() > 1 } /// The display title, falling back to a truncated stem. /// /// # Returns /// /// A short label suitable for a table or a Canvas question name. pub fn display_title(&self) -> String { if let Some(t) = &self.title { if !t.trim().is_empty() { return t.trim().to_string(); } } let flat = self.stem.split_whitespace().collect::>().join(" "); if flat.chars().count() <= 60 { flat } else { let head: String = flat.chars().take(57).collect(); format!("{head}...") } } /// A content fingerprint over everything that affects what a student sees. /// /// Metadata deliberately does not contribute: retagging an objective must not /// invalidate pooled statistics, but rewording an option must. /// /// # Returns /// /// The fingerprint as hex. pub fn fingerprint(&self) -> String { let mut parts: Vec = vec![self.stem.trim().to_string()]; // Canonicalize by option letter so reordering the YAML block, which does // not change the item, does not change the fingerprint. let mut opts: Vec<&Choice> = self.options.iter().collect(); opts.sort_by(|a, b| a.id.cmp(&b.id)); for o in opts { parts.push(format!( "{}|{}|{}", o.id, if o.correct { "1" } else { "0" }, o.text.trim() )); } if let Some(s) = &self.stimulus { parts.push(format!("stimulus:{s}")); } fingerprint(parts.iter().map(|s| s.as_str())) } /// Whether the recorded calibration matches the current content. /// /// # Returns /// /// `false` when the item was edited after it was calibrated. pub fn calibration_is_current(&self) -> bool { match self .calibration .as_ref() .and_then(|c| c.fingerprint.as_ref()) { Some(fp) => *fp == self.fingerprint(), None => true, } } /// The point value, resolving against a course default. /// /// # Arguments /// /// * `default_points` - the course policy value. /// /// # Returns /// /// The point value to use. pub fn points(&self, default_points: f64) -> f64 { self.points.unwrap_or(default_points) } /// Whether this item may be placed on a graded assessment. pub fn is_assemblable(&self) -> bool { self.status.is_usable() && self.retired.is_none() } /// The expected time in seconds, falling back to a level-based estimate. /// /// The fallbacks are rough but useful: without them, a form's total time /// estimate silently drops every item that has no `design` block. /// /// # Returns /// /// Seconds. pub fn expected_seconds(&self) -> f64 { if let Some(t) = self.design.as_ref().and_then(|d| d.expected_time_seconds) { return t; } match self.level { Level::Remember => 35.0, Level::Understand => 65.0, Level::Apply => 95.0, Level::Analyze => 130.0, Level::Create => 165.0, } } /// Appends a change-log entry and bumps the version. /// /// # Arguments /// /// * `change` - a description of what changed. /// * `author` - who made the change. pub fn record_change(&mut self, change: &str, author: Option<&str>) { self.version += 1; self.history.push(HistoryEntry { version: self.version, date: Date::today(), author: author.map(|a| a.to_string()), change: change.to_string(), }); } } fn one_u32() -> u32 { 1 } fn default_format() -> Format { Format::SingleBestAnswer } fn is_false(b: &bool) -> bool { !*b } #[cfg(test)] mod tests { use super::*; fn item(src: &str) -> Item { serde_yaml_ng::from_str(src).expect("item parses") } const MINIMAL: &str = r#" id: q-demo-001 status: draft level: 2 stem: Which statement best explains the effect? options: - { id: A, text: Right, correct: true } - { id: B, text: Wrong } - { id: C, text: Also wrong } "#; #[test] fn minimal_item_parses_with_defaults() { let it = item(MINIMAL); assert_eq!(it.version, 1); assert_eq!(it.format, Format::SingleBestAnswer); assert!(!it.bonus); assert_eq!(it.key_letters(), vec!["A"]); assert!(!it.is_multi_key()); assert!(!it.is_assemblable(), "drafts are not assemblable"); } #[test] fn credit_defaults_from_correctness() { let it = item(MINIMAL); assert_eq!(it.option("A").unwrap().credit(), 1.0); assert_eq!(it.option("B").unwrap().credit(), 0.0); assert!(!it.option("B").unwrap().is_partial()); let with_partial = item( r#" id: q-demo-002 status: draft level: 5 stem: s options: - { id: A, text: Right, correct: true } - { id: B, text: Defensible, credit: 0.5, defensible: true, defense: because } - { id: C, text: Wrong } "#, ); assert!(with_partial.option("B").unwrap().is_partial()); assert_eq!(with_partial.option("B").unwrap().credit(), 0.5); } #[test] fn fingerprint_tracks_content_not_metadata() { let base = item(MINIMAL); let mut retagged = base.clone(); retagged.topics = vec!["kinetics".into()]; retagged.learning_objectives = vec!["lo-a".into()]; retagged.author = Some("someone".into()); assert_eq!( base.fingerprint(), retagged.fingerprint(), "metadata must not invalidate pooled statistics" ); let mut reworded = base.clone(); reworded.options[1].text = "Wrong, but differently".into(); assert_ne!(base.fingerprint(), reworded.fingerprint()); let mut rekeyed = base.clone(); rekeyed.options[0].correct = false; rekeyed.options[1].correct = true; assert_ne!(base.fingerprint(), rekeyed.fingerprint()); } #[test] fn fingerprint_ignores_yaml_option_order() { let a = item(MINIMAL); let b = item( r#" id: q-demo-001 status: draft level: 2 stem: Which statement best explains the effect? options: - { id: C, text: Also wrong } - { id: A, text: Right, correct: true } - { id: B, text: Wrong } "#, ); assert_eq!(a.fingerprint(), b.fingerprint()); } #[test] fn stale_calibration_is_detectable() { let mut it = item(MINIMAL); assert!(it.calibration_is_current(), "no calibration is not stale"); let fp = it.fingerprint(); it.calibration = Some(Calibration { fingerprint: Some(fp), ..Calibration::default() }); assert!(it.calibration_is_current()); it.stem = "A different question entirely?".into(); assert!(!it.calibration_is_current()); } #[test] fn display_title_truncates_long_stems() { let mut it = item(MINIMAL); it.title = None; it.stem = "word ".repeat(40); let t = it.display_title(); assert!(t.ends_with("...")); assert_eq!(t.chars().count(), 60); } #[test] fn unknown_item_keys_are_rejected() { let bad = serde_yaml_ng::from_str::( r#" id: q-demo-001 status: draft level: 2 stem: s steam: oops options: - { id: A, text: a, correct: true } "#, ); assert!(bad.is_err()); } #[test] fn record_change_bumps_version_and_logs() { let mut it = item(MINIMAL); it.record_change("clarified the stem", Some("Alex")); assert_eq!(it.version, 2); assert_eq!(it.history.len(), 1); assert_eq!(it.history[0].version, 2); } #[test] fn irt_model_tokens_round_trip() { assert_eq!(serde_json::to_string(&IrtModel::TwoPl).unwrap(), "\"2pl\""); assert_eq!( serde_json::from_str::("\"3pl\"").unwrap(), IrtModel::ThreePl ); } }