// SPDX-License-Identifier: Prosperity-3.0.0 // Copyright Scientific Computing Studio // Source: https://git.scient.ing/education/coursebank //! The bank file: a collection of items scoped to a topic or a lecture. //! //! One bank per topic (or per lecture, if that suits how you teach) is the unit //! of authoring. Banks are small enough to review in a pull request, they let //! two people write questions without colliding, and `bank.scope` records what //! the file is *for* so `coursebank catalog` can tell you that you have eleven //! items on enzyme kinetics and none on regulation. //! //! Validation here is split in two on purpose. [`BankFile::validate`] checks what //! must be true for the file to be usable at all: ids are unique, a keyed answer //! exists, an approved item is fully specified, a level and its cognitive process //! agree. The softer question of whether an item is *well written* lives in //! [`crate::lint`], because those checks are advisory and you should be able to //! ship a file that trips a few of them. use std::collections::BTreeMap; use std::path::Path; use serde::{Deserialize, Serialize}; use crate::course::{CourseFile, SCHEMA_VERSION}; use crate::date::Date; use crate::error::Result; use crate::item::Item; use crate::taxonomy::{Format, Level, Status}; use crate::yaml; /// A whole bank file. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct BankFile { /// Schema version this file targets. #[serde( default = "default_version", deserialize_with = "yaml::flexible_string" )] pub schema_version: String, /// Bank identity and scope. pub bank: BankMeta, /// Values applied to every item in the file that does not set its own. #[serde(default)] pub defaults: BankDefaults, /// The items. #[serde(default)] pub items: Vec, } /// Bank identity and scope. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct BankMeta { /// Stable id, unique across the course. Item ids are namespaced by it. pub id: String, /// Human title. pub title: String, /// What this bank covers. #[serde(default, skip_serializing_if = "Option::is_none")] pub description: Option, /// What the bank is scoped to, so coverage can be reported against it. #[serde(default)] pub scope: Scope, /// Who maintains it. #[serde(default, skip_serializing_if = "Option::is_none")] pub maintainer: Option, /// When it was created. #[serde(default, skip_serializing_if = "Option::is_none")] pub created: Option, /// When it was last touched. #[serde(default, skip_serializing_if = "Option::is_none")] pub updated: Option, } /// What a bank is scoped to. /// /// A bank may be scoped by lecture, by objective, by topic, or by none of them. /// Declaring the scope is what lets the catalog report *gaps*: it can only tell /// you that lecture 12 has no Apply-level items if it knows lecture 12 is /// supposed to be covered here. #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Scope { /// Lectures this bank draws from. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub lectures: Vec, /// Objectives this bank is responsible for covering. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub learning_objectives: Vec, /// Units this bank belongs to. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub units: Vec, /// Topics this bank is about. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub topics: Vec, } /// Per-file defaults, so common metadata is written once. #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct BankDefaults { /// Default author for items in this file. #[serde(default, skip_serializing_if = "Option::is_none")] pub author: Option, /// Default point value. #[serde(default, skip_serializing_if = "Option::is_none")] pub points: Option, /// Expected option count; the linter flags items that differ, since an /// inconsistent option count across a form is itself a cue to students. #[serde(default, skip_serializing_if = "Option::is_none")] pub options_per_item: Option, /// Topics added to every item in the file. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub topics: Vec, /// Sources applied to items that declare none. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub lectures: Vec, } impl BankFile { /// Loads a bank file from disk. /// /// # Arguments /// /// * `path` - the YAML file. /// /// # Returns /// /// The parsed bank. /// /// # Errors /// /// Returns [`crate::error::Error::Io`] or [`crate::error::Error::Yaml`]. pub fn load(path: &Path) -> Result { yaml::read(path) } /// Writes a bank file back out as YAML. /// /// Round-tripping loses comments, which is why calibration is written by an /// explicit `coursebank calibrate` step rather than as a side effect of /// anything else: you should be able to see the diff it produces. /// /// # Arguments /// /// * `path` - the destination. /// /// # Errors /// /// Returns [`crate::error::Error::Io`] on a write failure. pub fn save(&self, path: &Path) -> Result<()> { yaml::write(path, self) } /// Applies file defaults to items that omit the corresponding field. /// /// Called after loading so the rest of the crate never has to think about /// defaults again. pub fn apply_defaults(&mut self) { let d = self.defaults.clone(); for item in &mut self.items { if item.author.is_none() { item.author = d.author.clone(); } if item.points.is_none() { item.points = d.points; } for t in &d.topics { if !item.topics.contains(t) { item.topics.push(t.clone()); } } if item.sources.is_empty() { for lec in &d.lectures { item.sources.push(crate::item::Source { lecture: lec.clone(), slides: Vec::new(), readings: Vec::new(), recording_seconds: None, }); } } } } /// Loads a bank and applies its defaults. /// /// # Arguments /// /// * `path` - the YAML file. /// /// # Returns /// /// The parsed bank with defaults resolved. /// /// # Errors /// /// Propagates load errors. pub fn load_resolved(path: &Path) -> Result { let mut b = BankFile::load(path)?; b.apply_defaults(); Ok(b) } /// Checks every invariant that must hold for the file to be usable. /// /// Returns all problems rather than the first, so one run fixes one file. /// When a course file is supplied, cross-file references are checked too. /// /// # Arguments /// /// * `course` - the course registry, for resolving objective and lecture /// references. Pass `None` to check only what is local to the file. /// /// # Returns /// /// Every problem found, empty when the file is sound. pub fn validate(&self, course: Option<&CourseFile>) -> Vec { let mut issues = Vec::new(); if self.bank.id.trim().is_empty() { issues.push("bank.id is empty".into()); } if self.bank.title.trim().is_empty() { issues.push("bank.title is empty".into()); } if let (Some(created), Some(updated)) = (self.bank.created, self.bank.updated) { if updated < created { issues.push(format!( "bank.updated ({updated}) is before bank.created ({created})" )); } } // Duplicate item ids inside the file. let mut counts: BTreeMap<&str, usize> = BTreeMap::new(); for it in &self.items { *counts.entry(it.id.as_str()).or_insert(0) += 1; } for (id, n) in &counts { if *n > 1 { issues.push(format!("duplicate item id `{id}` appears {n} times")); } } if let Some(c) = course { for lec in &self.bank.scope.lectures { if !c.lectures.contains_key(lec) { issues.push(format!("bank.scope: unknown lecture `{lec}`")); } } for lo in &self.bank.scope.learning_objectives { if !c.learning_objectives.contains_key(lo) { issues.push(format!("bank.scope: unknown learning objective `{lo}`")); } } } for it in &self.items { issues.extend( validate_item(it, course, self.defaults.options_per_item) .into_iter() .map(|m| format!("{}: {m}", it.id)), ); } issues } /// Items that may be placed on a graded assessment. /// /// # Returns /// /// References to approved, unretired items. pub fn assemblable(&self) -> Vec<&Item> { self.items.iter().filter(|i| i.is_assemblable()).collect() } /// Counts of assemblable, non-bonus items by level. /// /// This is the five-tuple you check a blueprint against. /// /// # Returns /// /// A map from level to count. pub fn level_counts(&self) -> BTreeMap { let mut out: BTreeMap = Level::ALL.iter().map(|l| (*l, 0)).collect(); for it in self.items.iter().filter(|i| i.is_assemblable() && !i.bonus) { *out.entry(it.level).or_insert(0) += 1; } out } /// A skeleton bank file for `coursebank bank new`. /// /// # Arguments /// /// * `id` - the bank id. /// * `title` - the bank title. /// /// # Returns /// /// A bank with no items. pub fn skeleton(id: &str, title: &str) -> BankFile { BankFile { schema_version: SCHEMA_VERSION.to_string(), bank: BankMeta { id: id.to_string(), title: title.to_string(), description: None, scope: Scope::default(), maintainer: None, created: Some(Date::today()), updated: Some(Date::today()), }, defaults: BankDefaults::default(), items: Vec::new(), } } } /// Validates one item. /// /// # Arguments /// /// * `it` - the item. /// * `course` - the course registry, when available. /// * `expected_options` - the file's declared option count, when set. /// /// # Returns /// /// Problems found, without the item id prefix. fn validate_item( it: &Item, course: Option<&CourseFile>, expected_options: Option, ) -> Vec { let mut issues = Vec::new(); if it.id.trim().is_empty() { issues.push("empty id".into()); } if it.stem.trim().is_empty() { issues.push("empty stem".into()); } if it.version == 0 { issues.push("version must be at least 1".into()); } // --- options ---- // An open-response item takes no options; its answer lives in `solution`. // Every other format needs at least two things to choose between. if it.format.has_options() { if it.options.len() < 2 { issues.push(format!( "needs at least 2 options, has {}", it.options.len() )); } } else if !it.options.is_empty() { issues.push(format!( "{} items take no options, but {} were given; put the answer in `solution`", it.format.as_str(), it.options.len() )); } let mut seen: Vec<&str> = Vec::new(); for (i, o) in it.options.iter().enumerate() { let pos = i + 1; if o.text.trim().is_empty() { issues.push(format!("option {pos}: empty text")); } let letter_ok = o.id.len() == 1 && o.id .chars() .next() .map(|c| c.is_ascii_uppercase() && c <= 'H') .unwrap_or(false); if !letter_ok { issues.push(format!( "option {pos}: id `{}` must be a single letter A through H", o.id )); } if seen.contains(&o.id.as_str()) { issues.push(format!("option {pos}: duplicate option id `{}`", o.id)); } seen.push(&o.id); let credit = o.credit(); if !(0.0..=1.0).contains(&credit) { issues.push(format!( "option {}: credit must be between 0 and 1, got {credit}", o.id )); } // Partial credit must be argued for in writing, not remembered. if o.is_partial() && o.defense.is_none() { issues.push(format!( "option {}: awards credit {credit} but gives no `defense`", o.id )); } if o.is_partial() && !o.defensible { issues.push(format!( "option {}: awards credit {credit} but is not marked `defensible: true`", o.id )); } if o.correct && credit == 0.0 { issues.push(format!( "option {}: keyed correct but earns no credit", o.id )); } } if let Some(n) = expected_options { if it.options.len() != n && !it.options.is_empty() { issues.push(format!( "has {} options but the bank declares {n} per item", it.options.len() )); } } // --- key --- let keys = it.key_indices(); match it.format { Format::SingleBestAnswer => { if keys.len() != 1 { issues.push(format!( "single_best_answer needs exactly one keyed option, has {}", keys.len() )); } } Format::MultipleResponse => { if keys.is_empty() { issues.push("multiple_response needs at least one keyed option".into()); } if keys.len() == it.options.len() { issues.push("multiple_response keys every option, so it asks nothing".into()); } } Format::TrueFalse => { if it.options.len() != 2 { issues.push(format!( "true_false needs exactly 2 options, has {}", it.options.len() )); } if keys.len() != 1 { issues.push("true_false needs exactly one keyed option".into()); } } Format::OpenResponse => { if !keys.is_empty() { issues.push("open_response items have no keyed option".into()); } } } // --- level and process must agree ------- if let Some(p) = it.cognitive_process { if !it.level.allows(p) { issues.push(format!( "cognitive_process `{p}` belongs to level {} but the item is level {}", p.level().code(), it.level.code() )); } } // --- design plausibility ------ if let Some(d) = &it.design { if let Some(x) = d.expected_difficulty { if !(0.0..=1.0).contains(&x) { issues.push(format!( "design.expected_difficulty must be between 0 and 1, got {x}" )); } } if let Some(t) = d.expected_time_seconds { if t <= 0.0 { issues.push(format!( "design.expected_time_seconds must be positive, got {t}" )); } } } // --- calibration plausibility ------ if let Some(c) = &it.calibration { if let Some(p) = c.p_value { if !(0.0..=1.0).contains(&p) { issues.push(format!( "calibration.p_value must be between 0 and 1, got {p}" )); } } if let Some(r) = c.point_biserial { if !(-1.0..=1.0).contains(&r) { issues.push(format!( "calibration.point_biserial must be between -1 and 1, got {r}" )); } } for letter in c.option_stats.keys() { if it.option(letter).is_none() { issues.push(format!( "calibration.option_stats has `{letter}`, which is not an option of this item" )); } } if let Some(irt) = &c.irt { if irt.a <= 0.0 { issues.push(format!("calibration.irt.a must be positive, got {}", irt.a)); } if let Some(cp) = irt.c { if !(0.0..1.0).contains(&cp) { issues.push(format!("calibration.irt.c must be in [0, 1), got {cp}")); } } } } // --- history must be coherent ------ let mut last_version = 0u32; for (i, h) in it.history.iter().enumerate() { if h.version <= last_version { issues.push(format!( "history entry {} has version {} which does not increase", i + 1, h.version )); } last_version = h.version; } if !it.history.is_empty() && last_version > it.version { issues.push(format!( "history records version {last_version} but the item says version {}", it.version )); } // --- retirement ----- if it.retired.is_some() && it.status != Status::Retired { issues.push(format!( "has a `retired` block but status is `{}`", it.status )); } // --- approval gate ------- // Approval is what permits an item onto a graded assessment, so it is the // right place to require that the item is fully sourced and designed. if it.status == Status::Approved { if it.cognitive_process.is_none() { issues.push("approved items must declare a cognitive_process".into()); } if it.learning_objectives.is_empty() { issues.push("approved items must reference at least one learning objective".into()); } if it.sources.is_empty() { issues.push("approved items must cite at least one source".into()); } if it.design.is_none() { issues.push("approved items must carry a design block".into()); } // An open-response item is graded from its solution, so approving one with // neither a model answer nor a rubric would leave nothing to mark it by. if !it.format.has_options() { let gradeable = it .solution .as_ref() .is_some_and(|s| s.model_answer.is_some() || !s.rubric.is_empty()); if !gradeable { issues.push( "approved open_response items need a solution with a model_answer or a rubric" .into(), ); } } } // --- cross-file references ---- if let Some(c) = course { for lo in &it.learning_objectives { match c.learning_objectives.get(lo) { None => issues.push(format!("unknown learning objective `{lo}`")), Some(obj) => { if let Some(ceiling) = obj.level_ceiling { if it.level > ceiling { issues.push(format!( "level {} exceeds the ceiling {} declared for objective `{lo}`", it.level.code(), ceiling.code() )); } } if !obj.assessed { issues.push(format!( "objective `{lo}` is marked `assessed: false` but this item measures it" )); } } } } for s in &it.sources { if !c.lectures.contains_key(&s.lecture) { issues.push(format!("unknown lecture `{}`", s.lecture)); } } if let Some(st) = &it.stimulus { if !c.stimuli.contains_key(st) { issues.push(format!("unknown stimulus `{st}`")); } } // A citation that names a reference key must name a real one, so a review // pointer in the solutions document never resolves to nothing. for citation in it.solution.iter().flat_map(|s| &s.review) { if let Some(key) = &citation.reference { if !c.references.contains_key(key) { issues.push(format!("solution.review cites unknown reference `{key}`")); } } } if let Some(floor) = c.policy.partial_credit_floor_level { for o in &it.options { if o.is_partial() && it.level < floor { issues.push(format!( "option {} awards partial credit at level {}, below the course floor of {}", o.id, it.level.code(), floor.code() )); } } } if !c.policy.allow_partial_credit && it.options.iter().any(|o| o.is_partial()) { issues.push("awards partial credit, which the course policy disallows".into()); } } issues } fn default_version() -> String { SCHEMA_VERSION.to_string() } #[cfg(test)] mod tests { use super::*; fn bank(items_yaml: &str) -> BankFile { let src = format!("bank:\n id: b\n title: Bank\ndefaults: {{}}\nitems:\n{items_yaml}"); let mut b: BankFile = serde_yaml_ng::from_str(&src).expect("bank parses"); b.apply_defaults(); b } #[test] fn sound_bank_validates_clean() { let b = bank( r#" - id: q-a-001 status: draft level: 1 stem: What is x? options: - { id: A, text: right, correct: true } - { id: B, text: wrong } - { id: C, text: wrong too } "#, ); assert!(b.validate(None).is_empty(), "{:?}", b.validate(None)); } #[test] fn open_response_validates_without_options_and_rejects_them() { // No options is fine, and no key is required. let ok = bank( r#" - id: q-a-op-001 status: draft level: 2 format: open_response stem: Explain the first law. solution: model_answer: Energy is conserved. "#, ); assert!(ok.validate(None).is_empty(), "{:?}", ok.validate(None)); // Giving an open-response item options is the mistake, and so is approving // one with nothing to grade it by. let bad = bank( r#" - id: q-a-op-002 status: approved level: 2 format: open_response cognitive_process: explain learning_objectives: [lo-x] sources: [{ lecture: L1.1 }] design: { rationale: r } stem: Explain the first law. options: - { id: A, text: a, correct: true } - { id: B, text: b } "#, ); let issues = bad.validate(None); assert!( issues.iter().any(|i| i.contains("take no options")), "{issues:?}" ); assert!( issues .iter() .any(|i| i.contains("model_answer or a rubric")), "{issues:?}" ); } #[test] fn catches_missing_and_multiple_keys() { let b = bank( r#" - id: q-a-001 status: draft level: 1 stem: s options: - { id: A, text: a } - { id: B, text: b } - id: q-a-002 status: draft level: 1 format: single_best_answer stem: s options: - { id: A, text: a, correct: true } - { id: B, text: b, correct: true } "#, ); let issues = b.validate(None); assert!( issues .iter() .any(|i| i.contains("exactly one keyed option")) ); assert_eq!( issues .iter() .filter(|i| i.contains("exactly one keyed option")) .count(), 2 ); } #[test] fn catches_duplicate_ids_and_letters() { let b = bank( r#" - id: q-a-001 status: draft level: 1 stem: s options: - { id: A, text: a, correct: true } - { id: A, text: b } - id: q-a-001 status: draft level: 1 stem: s options: - { id: A, text: a, correct: true } - { id: B, text: b } "#, ); let issues = b.validate(None); assert!(issues.iter().any(|i| i.contains("duplicate item id"))); assert!(issues.iter().any(|i| i.contains("duplicate option id"))); } #[test] fn level_and_process_must_agree() { let b = bank( r#" - id: q-a-001 status: draft level: 3 cognitive_process: recall stem: s options: - { id: A, text: a, correct: true } - { id: B, text: b } "#, ); let issues = b.validate(None); assert!( issues.iter().any(|i| i.contains("belongs to level 1")), "{issues:?}" ); } #[test] fn approval_requires_full_specification() { let b = bank( r#" - id: q-a-001 status: approved level: 1 stem: s options: - { id: A, text: a, correct: true } - { id: B, text: b } "#, ); let issues = b.validate(None); for want in [ "cognitive_process", "learning objective", "source", "design block", ] { assert!( issues.iter().any(|i| i.contains(want)), "expected a complaint about {want}, got {issues:?}" ); } } #[test] fn partial_credit_needs_a_written_defense() { let b = bank( r#" - id: q-a-001 status: draft level: 5 stem: s options: - { id: A, text: a, correct: true } - { id: B, text: b, credit: 0.5 } "#, ); let issues = b.validate(None); assert!(issues.iter().any(|i| i.contains("no `defense`"))); assert!(issues.iter().any(|i| i.contains("defensible: true"))); } #[test] fn defaults_fill_in_items() { let src = r#" bank: { id: b, title: Bank } defaults: author: Alex points: 1.5 topics: [kinetics] lectures: [L11] items: - id: q-a-001 status: draft level: 1 stem: s options: - { id: A, text: a, correct: true } - { id: B, text: b } - id: q-a-002 status: draft level: 1 stem: s author: Someone Else topics: [kinetics] sources: [{ lecture: L12 }] options: - { id: A, text: a, correct: true } - { id: B, text: b } "#; let mut b: BankFile = serde_yaml_ng::from_str(src).unwrap(); b.apply_defaults(); assert_eq!(b.items[0].author.as_deref(), Some("Alex")); assert_eq!(b.items[0].points, Some(1.5)); assert_eq!(b.items[0].topics, vec!["kinetics"]); assert_eq!(b.items[0].sources[0].lecture, "L11"); // Explicit values win, and topics are not duplicated. assert_eq!(b.items[1].author.as_deref(), Some("Someone Else")); assert_eq!(b.items[1].topics, vec!["kinetics"]); assert_eq!(b.items[1].sources[0].lecture, "L12"); } #[test] fn cross_file_references_are_checked_against_the_course() { let course: CourseFile = serde_yaml_ng::from_str( r#" course: { code: X, title: Y, term: Z } lectures: L11: { title: Kinetics } learning_objectives: lo-known: { text: Do the thing, level_ceiling: 2 } "#, ) .unwrap(); let b = bank( r#" - id: q-a-001 status: draft level: 4 stem: s learning_objectives: [lo-known, lo-unknown] sources: [{ lecture: L99 }] options: - { id: A, text: a, correct: true } - { id: B, text: b } "#, ); let issues = b.validate(Some(&course)); assert!( issues .iter() .any(|i| i.contains("unknown learning objective `lo-unknown`")) ); assert!(issues.iter().any(|i| i.contains("unknown lecture `L99`"))); assert!( issues.iter().any(|i| i.contains("exceeds the ceiling")), "{issues:?}" ); } #[test] fn history_versions_must_increase() { let b = bank( r#" - id: q-a-001 version: 2 status: draft level: 1 stem: s options: - { id: A, text: a, correct: true } - { id: B, text: b } history: - { version: 2, date: 2026-01-01, change: second } - { version: 1, date: 2026-01-02, change: first } "#, ); let issues = b.validate(None); assert!(issues.iter().any(|i| i.contains("does not increase"))); } #[test] fn level_counts_exclude_drafts_and_bonuses() { let b = bank( r#" - id: q-a-001 status: approved level: 1 cognitive_process: recall stem: s learning_objectives: [lo] sources: [{ lecture: L1 }] design: { expected_difficulty: 0.8 } options: - { id: A, text: a, correct: true } - { id: B, text: b } - id: q-a-002 status: draft level: 1 stem: s options: - { id: A, text: a, correct: true } - { id: B, text: b } - id: q-a-003 status: approved level: 5 cognitive_process: generate bonus: true stem: s learning_objectives: [lo] sources: [{ lecture: L1 }] design: { expected_difficulty: 0.3 } options: - { id: A, text: a, correct: true } - { id: B, text: b } "#, ); let counts = b.level_counts(); assert_eq!(counts[&Level::Remember], 1); assert_eq!(counts[&Level::Create], 0, "bonus items are not scored"); assert_eq!(b.assemblable().len(), 2); } }