// SPDX-License-Identifier: Prosperity-3.0.0 // Copyright Scienting // Source: https://git.scient.ing/education/coursebank //! Loading a whole course at once, and reporting on what it contains. //! //! A [`Catalog`] is every bank in a course, indexed so that an item can be found //! by its id, and so that questions like "how many Apply //! level items do I have on lecture 12" have a cheap answer. //! //! The global id is the join key for everything downstream: assessment records //! reference it, response tables carry it, and usage history is keyed by it. Bank //! ids therefore have to be unique across a course, which the catalog enforces //! at load time rather than letting a silent collision merge two items. //! //! The coverage report is the part that changes how you write. It is easy to //! accumulate forty recall items and believe a topic is covered; a table showing //! that eleven objectives have no item above level 1 is harder to ignore. use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; use crate::assessment::{AssessmentFile, Placement}; use crate::bank::BankFile; use crate::course::CourseFile; use crate::error::{Error, Result}; use crate::item::Item; use crate::layout::Layout; use crate::taxonomy::{Format, Level, Status, Tier}; use crate::yaml; /// One item plus everything needed to locate it again. #[derive(Debug, Clone)] pub struct Entry { /// The item's id, which names it course-wide. /// /// The bank is in [`Entry::bank`] and the file in [`Entry::path`], neither /// of which is part of the identity: this is the join key that response /// data carries, and a join key with a file name in it renames itself every /// time the files are reorganized. pub uid: String, /// The bank id. pub bank: String, /// The file the item came from. pub path: PathBuf, /// Position within the bank file, for stable ordering. pub index: usize, /// The item itself. pub item: Item, } impl Entry { /// The item's point value, resolving against the course policy. /// /// # Arguments /// /// * `course` - the course whose policy supplies the default. /// /// # Returns /// /// The point value. pub fn points(&self, course: &CourseFile) -> f64 { self.item.points(course.policy.points_per_item) } } /// Every bank in a course, indexed. #[derive(Debug, Clone)] pub struct Catalog { /// The course registry. pub course: CourseFile, /// The resolved directory layout. pub layout: Layout, /// Every item, in stable order: by bank id, then by position in the file. pub entries: Vec, /// Bank metadata by bank id. pub banks: BTreeMap, /// The statistics, loaded from `analysis/` rather than from the banks. /// /// Each entry's [`crate::item::Item::calibration`] is filled from this at /// load, so everything downstream reads one item and does not have to know /// that the numbers and the wording come from different files. pub calibration: crate::calibration::CalibrationFile, /// Map from global id to index into `entries`. index: BTreeMap, } impl Catalog { /// Loads a whole course from its directory. /// /// # Arguments /// /// * `root` - the course directory containing `course.yaml` and `banks/`. /// /// # Returns /// /// The catalog. /// /// # Errors /// /// Returns a load error for the course file or any bank, and /// [`Error::Invalid`] when two banks share an id or two items share a global /// id, since either makes the join key ambiguous. pub fn load(root: &Path) -> Result { let layout = Layout::new(root); let course = CourseFile::load_dir(root)?; let mut catalog = Catalog { course, layout, entries: Vec::new(), banks: BTreeMap::new(), index: BTreeMap::new(), calibration: crate::calibration::CalibrationFile::default(), }; catalog.calibration = crate::calibration::CalibrationFile::load(&catalog.layout.calibration_file())?; let mut problems = Vec::new(); let mut files = yaml::list_yaml(&catalog.layout.banks())?; files.sort(); for path in files { let bank = BankFile::load_resolved(&path)?; let bank_id = bank.bank.id.clone(); if let Some(existing) = catalog.banks.get(&bank_id) { problems.push(format!( "bank id `{bank_id}` is used by two files (`{}` and `{}`)", existing.title, bank.bank.title )); continue; } catalog.banks.insert(bank_id.clone(), bank.bank.clone()); for (i, item) in bank.items.into_iter().enumerate() { let uid = item.id.clone(); if let Some(first) = catalog.index.get(&uid) { problems.push(format!( "item id `{uid}` is used twice: in bank `{}` and in bank `{bank_id}`. An \ id names one question course-wide, because response data joins on it.", catalog.entries[*first].bank )); continue; } catalog.index.insert(uid.clone(), catalog.entries.len()); catalog.entries.push(Entry { uid, bank: bank_id.clone(), path: path.clone(), index: i, item, }); } } if !problems.is_empty() { return Err(Error::Invalid(problems)); } // Statistics are attached here rather than parsed from the bank, which // is why an item can be read as one thing while its wording and its // evidence live in files with different review cycles. for entry in &mut catalog.entries { if let Some(calibration) = catalog.calibration.items.get(&entry.uid) { entry.item.calibration = Some(calibration.clone()); } } Ok(catalog) } /// Looks up an item by global id. /// /// A pre-2.0 `bank::item` id resolves to the item it used to name, so an /// assessment record or a parquet file written before the change still /// joins. See [`crate::item::canonical_id`]. /// /// # Arguments /// /// * `uid` - the item id, in either form. /// /// # Returns /// /// The entry, or `None`. pub fn get(&self, uid: &str) -> Option<&Entry> { self.index .get(uid) .or_else(|| self.index.get(crate::item::canonical_id(uid))) .map(|i| &self.entries[*i]) } /// Looks up an item by global id, erroring when absent. /// /// # Arguments /// /// * `uid` - the global id. /// /// # Returns /// /// The entry. /// /// # Errors /// /// Returns [`Error::Unresolved`] when no such item exists. pub fn require(&self, uid: &str) -> Result<&Entry> { self.get(uid).ok_or_else(|| Error::Unresolved { kind: "item", id: uid.to_string(), context: None, }) } /// Resolves an id written in either form to the canonical one. /// /// Since 2.0 an item id is already course-wide, so this is the identity for /// anything current. What it is still for is the old `bank::item` form, /// which appears in assessment records, seals, and response files written /// before the change, and which a person may well still type. /// /// # Arguments /// /// * `id` - an item id, in either form. /// /// # Returns /// /// The canonical id. /// /// # Errors /// /// Returns [`Error::Unresolved`] when nothing matches. pub fn resolve(&self, id: &str) -> Result { if self.index.contains_key(id) { return Ok(id.to_string()); } let canonical = crate::item::canonical_id(id); if self.index.contains_key(canonical) { return Ok(canonical.to_string()); } Err(Error::Unresolved { kind: "item", id: id.to_string(), context: None, }) } /// Every item that may be placed on a graded assessment. /// /// # Returns /// /// References to approved, unretired entries. pub fn assemblable(&self) -> Vec<&Entry> { self.entries .iter() .filter(|e| e.item.is_assemblable()) .collect() } /// Validates every bank against the course registry. /// /// # Returns /// /// Problems, prefixed with the file they came from. pub fn validate(&self) -> Result> { // Not prefixed here: `CourseFile::validate` attributes each message to // the fragment that defined the id, which for an unsplit course is // `course.yaml` and for a split one is the file worth opening. let mut issues: Vec = self.course.validate(); for path in yaml::list_yaml(&self.layout.banks())? { let bank = BankFile::load_resolved(&path)?; let name = path .file_name() .and_then(|n| n.to_str()) .unwrap_or("?") .to_string(); issues.extend( bank.validate(Some(&self.course)) .into_iter() .map(|m| format!("{name}: {m}")), ); } Ok(issues) } /// Checks one placement's option set against the item's pool. /// /// The checks that moved here from the bank when options became a pool. A /// bank holding two defensible keys and six distractors is sound; what has /// to hold for a *form* is that exactly one key reached the student, that /// none of the distractors was true, and that the count matches policy. /// None of that can be decided by looking at the item alone. /// /// # Arguments /// /// * `p` - the placement. /// * `item` - the item it names. /// /// # Returns /// /// One message per problem. fn validate_placement(&self, p: &Placement, item: &Item) -> Vec { let mut issues = Vec::new(); if !item.format.has_options() { return issues; } let at = |number: u32| format!("question {number} ({})", p.item); for id in p.key.iter().chain(p.distractors.iter()) { if item.option(id).is_none() { issues.push(format!( "{}: `{id}` is not an option of this item", at(p.number) )); } } for id in &p.distractors { if p.key.iter().any(|k| k == id) { issues.push(format!( "{}: `{id}` is listed as both the key and a distractor", at(p.number) )); } if item.option(id).is_some_and(|o| o.correct) { issues.push(format!( "{}: `{id}` is offered as a distractor but the bank keys it correct", at(p.number) )); } } for id in &p.key { if item.option(id).is_some_and(|o| !o.correct) { issues.push(format!( "{}: `{id}` is keyed correct here but the bank does not key it. An option is \ true or it is not; which true option a form uses is this record's choice, \ but not whether it is true.", at(p.number) )); } } let single = item.format == Format::SingleBestAnswer; if single && p.key.len() > 1 { issues.push(format!( "{}: single_best_answer administers exactly one key, this names {}", at(p.number), p.key.len() )); } // No distractor list means the whole pool, which is what a pre-2.0 // record means and what an item whose pool is its form still means. // The record still has to say which key, when the pool offers a choice. if p.distractors.is_empty() { let (keys, _) = item.pool(); if p.key.is_empty() && keys.len() > 1 && single { issues.push(format!( "{}: the item offers {} defensible keys, so the record has to say which one \ this assessment used", at(p.number), keys.len() )); } } else { let shown = item.administered(&p.key, &p.distractors); let expected = self.course.policy.options_per_item; if shown.len() != expected { issues.push(format!( "{}: administers {} option(s), but course policy is {expected} per item", at(p.number), shown.len() )); } if single && p.key.is_empty() { issues.push(format!( "{}: names its distractors but not its key, so what was marked correct is \ left to whatever the bank says today", at(p.number) )); } } if let Some(recorded) = &p.variant { if *recorded != item.variant_digest(&p.key, &p.distractors) { issues.push(format!( "{}: an administered option has been reworded since this assessment. \ Statistics pooled under this variant describe the older wording.", at(p.number) )); } } issues } /// Checks every sealed administration's stems against the bank. /// /// The seal is the authority on what was administered, so this is the /// comparison that matters: a record can be edited, but a seal is written /// before the exam is printed and digested against tampering. A stem that /// no longer matches the one a cohort answered means the id now names a /// different question, and every statistic pooled under it is describing /// two things at once. /// /// # Arguments /// /// * `seals` - the sealed administrations to check. /// /// # Returns /// /// One message per stem that has moved out from under its seal. pub fn validate_seals(&self, seals: &[crate::seal::SealFile]) -> Vec { let mut issues = Vec::new(); for seal in seals { for item in &seal.items { let Some(digest) = &item.stem_digest else { continue; }; let Some(entry) = self.get(&item.item) else { continue; }; if *digest != entry.item.stem_digest() { issues.push(format!( "{}: question {} (`{}`) was administered with a different stem than the \ bank now holds. Statistics from that administration describe the older \ wording; give the new wording its own id.", seal.seal.assessment, item.number, item.item )); } } } issues } /// Validates a record's internal invariants, then its references against this /// catalog: unknown items, keys that drifted, and fingerprints showing the /// item was reworded since it was administered. pub fn validate_record(&self, record: &AssessmentFile) -> Vec { let mut issues = record.validate(); for p in &record.items { match self.get(&p.item) { None => issues.push(format!("question {}: unknown item `{}`", p.number, p.item)), Some(entry) => { // The rule the stem digest exists to enforce. A changed // fingerprint is a note: the statistics describe an older // wording. A changed stem is an error: whatever was // administered is not the question the bank now holds, so // the id is being reused for two different questions. if let Some(digest) = &p.stem_digest { if *digest != entry.item.stem_digest() { issues.push(format!( "question {} ({}): the stem has been reworded since this \ assessment. A reworded stem is a new question: give the new \ wording a new id with `supersedes: {}`, and leave this one as \ it was administered.", p.number, p.item, p.item )); } } if let Some(fp) = &p.fingerprint { if *fp != entry.item.fingerprint() { issues.push(format!( "question {} ({}): the item has been edited since this assessment; \ statistics from this administration describe the older wording", p.number, p.item )); } } issues.extend(self.validate_placement(p, &entry.item)); } } } issues } /// Estimated working time for a record, in minutes. pub fn estimated_minutes(&self, record: &AssessmentFile) -> f64 { let seconds: f64 = record .items .iter() .filter_map(|p| self.get(&p.item)) .map(|e| e.item.expected_seconds()) .sum(); seconds / 60.0 } /// Counts of assemblable, non-bonus items by level. /// /// # Returns /// /// A map from level to count, with every level present. pub fn level_counts(&self) -> BTreeMap { let mut out: BTreeMap = Level::ALL.iter().map(|l| (*l, 0)).collect(); for e in self.assemblable().iter().filter(|e| !e.item.bonus) { *out.entry(e.item.level).or_insert(0) += 1; } out } /// Counts of items by workflow status, over the whole course. /// /// # Returns /// /// A map from status to count. pub fn status_counts(&self) -> BTreeMap { let mut out: BTreeMap = BTreeMap::new(); for e in &self.entries { *out.entry(e.item.status).or_insert(0) += 1; } out } /// Items tagged with a given learning target. /// /// # Arguments /// /// * `target` - the target id. /// /// # Returns /// /// Matching entries. pub fn by_target(&self, target: &str) -> Vec<&Entry> { self.entries .iter() .filter(|e| e.item.learning_targets.iter().any(|t| t == target)) .collect() } /// Items sourced to a given lecture. /// /// # Arguments /// /// * `lecture` - the lecture id. /// /// # Returns /// /// Matching entries. pub fn by_lecture(&self, lecture: &str) -> Vec<&Entry> { self.entries .iter() .filter(|e| e.item.sources.iter().any(|s| s.lecture == lecture)) .collect() } /// Items carrying a given topic tag. /// /// # Arguments /// /// * `topic` - the topic tag. /// /// # Returns /// /// Matching entries. pub fn by_topic(&self, topic: &str) -> Vec<&Entry> { self.entries .iter() .filter(|e| e.item.topics.iter().any(|t| t == topic)) .collect() } /// Every topic tag in use, with counts. /// /// # Returns /// /// A map from topic to item count. pub fn topics(&self) -> BTreeMap { let mut out: BTreeMap = BTreeMap::new(); for e in &self.entries { for t in &e.item.topics { *out.entry(t.clone()).or_insert(0) += 1; } } out } /// Items that measure a given objective, through any of its targets. /// /// # Arguments /// /// * `objective` - the objective id. /// /// # Returns /// /// Entries tagged with the objective itself or with any of its targets, /// each appearing once even when it is tagged with two of them. pub fn by_objective(&self, objective: &str) -> Vec<&Entry> { self.entries .iter() .filter(|e| { e.item .learning_targets .iter() .any(|t| t == objective || self.course.objective_for(t) == objective) }) .collect() } /// Builds the coverage report. /// /// Rows cover both tiers, because the two answer different questions. An /// objective row answers "can I build an exam that reports on this /// objective", and aggregates every item under it. A target row answers /// "which specific things have I written items for", which is the /// authoring queue, and a target with no items is the most common and least /// visible hole in a bank: the objective looks well covered while a third of /// what it claims has never been asked. /// /// # Returns /// /// One row per assessed entry plus a list of course-wide gaps. pub fn coverage(&self) -> Coverage { let mut rows = Vec::new(); for id in self.course.registry_in_order() { if !self.course.is_objective(&id) && !self.course.is_target(&id) { continue; } if !self.course.is_assessed(&id) { continue; } let targets = self.course.targets(&id); let tier = if self.course.is_objective(&id) { Tier::Objective } else { Tier::Target }; // An objective's pool is everything under it; a target's is what is // tagged to it directly. An objective with no targets is its own // target, so the two agree there. let items = if targets.is_empty() { self.by_target(&id) } else { self.by_objective(&id) }; let usable: Vec<&&Entry> = items.iter().filter(|e| e.item.is_assemblable()).collect(); let mut levels: BTreeSet = BTreeSet::new(); for e in &usable { levels.insert(e.item.level); } let targets_covered = targets .iter() .filter(|target| { self.by_target(target) .iter() .any(|e| e.item.is_assemblable()) }) .count(); rows.push(CoverageRow { id: id.clone(), text: self.course.text_for(&id), tier, objective: self .course .is_target(&id) .then(|| self.course.objective_for(&id).to_string()), unit: self.course.objective_unit(&id).map(str::to_string), targets: targets.len(), targets_covered, total: items.len(), assemblable: usable.len(), max_level: levels.iter().next_back().copied(), levels: levels.into_iter().collect(), ceiling: self.course.effective_level_ceiling(&id), }); } let mut gaps = Vec::new(); for row in &rows { // Gaps are raised against the tier that can be acted on. "No items // at all" is worth saying about a target, because writing one is the // fix. "Resting on a single item" is worth saying about an // objective, because a target resting on one item is the normal and // intended case, and flagging two hundred of them would bury the // rows that matter. if row.total == 0 { gaps.push(Gap::Uncovered(row.id.clone())); } else if row.assemblable == 0 { gaps.push(Gap::NoApprovedItems(row.id.clone())); } else if row.assemblable == 1 && row.tier == Tier::Objective { gaps.push(Gap::SingleItem(row.id.clone())); } // An objective assessed only at the recall level is the most common // and most consequential blind spot: it looks covered in a count and // is not covered in fact. if row.tier == Tier::Objective && row.assemblable > 0 && row.max_level == Some(Level::Remember) { if let Some(ceiling) = row.ceiling { if ceiling > Level::Remember { gaps.push(Gap::RecallOnly(row.id.clone())); } } else { gaps.push(Gap::RecallOnly(row.id.clone())); } } } for lec_id in self.course.lectures.keys() { if self .by_lecture(lec_id) .iter() .filter(|e| e.item.is_assemblable()) .count() == 0 { gaps.push(Gap::LectureUnassessed(lec_id.clone())); } } // Items with no objective at all cannot appear in any student report. for e in &self.entries { if e.item.learning_targets.is_empty() && e.item.is_assemblable() { gaps.push(Gap::ItemWithoutObjective(e.uid.clone())); } } Coverage { rows, gaps } } } /// The coverage report. #[derive(Debug, Clone)] pub struct Coverage { /// One row per assessed registry entry, objectives first with their targets /// following each. pub rows: Vec, /// Course-wide gaps worth acting on. pub gaps: Vec, } /// Coverage of one registry entry, at either tier. #[derive(Debug, Clone)] pub struct CoverageRow { /// The registry id. pub id: String, /// Its text. pub text: String, /// Whether this is an objective, whose counts aggregate every item under it, /// or a target, whose counts are its own items. pub tier: Tier, /// The objective this row sits under, for a target. pub objective: Option, /// The unit it belongs to, inherited from the objective when not declared. pub unit: Option, /// Targets in the registry, for an objective row. pub targets: usize, /// Targets with at least one usable item. /// /// The number to look at when an objective looks well covered: twenty items /// spread over four of its nine targets is a different bank from twenty /// items spread over all nine. pub targets_covered: usize, /// Items referencing it, at any status. pub total: usize, /// Items that could actually be used. pub assemblable: usize, /// The highest level assessed. pub max_level: Option, /// Every level assessed. pub levels: Vec, /// The ceiling in force, inherited from the objective when not declared. pub ceiling: Option, } /// A specific, actionable hole in the item pool. #[derive(Debug, Clone, PartialEq, Eq)] pub enum Gap { /// An assessed objective or target with no items at all. Uncovered(String), /// An entry whose items are all drafts or retired. NoApprovedItems(String), /// An objective resting on a single item, so one bad item hides it entirely. SingleItem(String), /// An objective assessed only at the recall level despite a higher ceiling. RecallOnly(String), /// A lecture with no usable items. LectureUnassessed(String), /// A usable item that references no objective, so it can never appear in a /// student report. ItemWithoutObjective(String), } impl Gap { /// A one-line description. pub fn message(&self) -> String { match self { Gap::Uncovered(id) => format!("objective `{id}` has no items"), Gap::NoApprovedItems(id) => { format!("objective `{id}` has items but none are approved") } Gap::SingleItem(id) => { format!("objective `{id}` rests on a single item; one bad item hides it") } Gap::RecallOnly(id) => { format!("objective `{id}` is assessed only at level 1") } Gap::LectureUnassessed(id) => format!("lecture `{id}` has no usable items"), Gap::ItemWithoutObjective(uid) => { format!("item `{uid}` has no learning objective, so it cannot appear in a report") } } } /// How much attention the gap deserves. pub fn severity(&self) -> Severity { match self { Gap::Uncovered(_) | Gap::NoApprovedItems(_) => Severity::High, Gap::RecallOnly(_) | Gap::ItemWithoutObjective(_) => Severity::Medium, Gap::SingleItem(_) | Gap::LectureUnassessed(_) => Severity::Low, } } } /// How much attention a finding deserves. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum Severity { /// Information, not a problem: a count of entries waiting for approval, or /// a convention that is reported but not enforced. `coursebank lint` /// prints notes and does not fail on them. Note, /// Worth knowing. Low, /// Worth scheduling. Medium, /// Worth fixing before the next assessment. High, } impl Severity { /// A short label for report output. pub fn label(self) -> &'static str { match self { Severity::Note => "note", Severity::Low => "low", Severity::Medium => "medium", Severity::High => "high", } } } #[cfg(test)] mod tests { use super::*; fn write_course(dir: &Path, extra_lo: &str) { std::fs::create_dir_all(dir.join("banks")).unwrap(); std::fs::write( dir.join("course.yaml"), format!( r#" course: {{ code: TEST 101, title: Testing, term: Fall 2026 }} lectures: L01: {{ title: One }} L02: {{ title: Two }} learning_objectives: lo-covered: {{ text: Covered objective, lectures: [L01] }} lo-bare: {{ text: Uncovered objective, lectures: [L02] }} {extra_lo} "# ), ) .unwrap(); } fn write_bank(dir: &Path, name: &str, body: &str) { std::fs::write(dir.join("banks").join(name), body).unwrap(); } fn tmp(tag: &str) -> PathBuf { let p = std::env::temp_dir().join(format!("coursebank-test-{tag}-{}", std::process::id())); let _ = std::fs::remove_dir_all(&p); std::fs::create_dir_all(&p).unwrap(); p } const APPROVED: &str = r#" bank: id: b1 title: First bank items: - id: q-x-001 status: approved level: 1 cognitive_process: recall stem: What is x? learning_targets: [lo-covered] sources: [{ lecture: L01 }] design: { expected_difficulty: 0.8 } options: - { id: A, text: right, correct: true } - { id: B, text: wrong } "#; #[test] fn loads_and_indexes_items() { let dir = tmp("load"); write_course(&dir, ""); write_bank(&dir, "b1.yaml", APPROVED); let cat = Catalog::load(&dir).expect("catalog loads"); assert_eq!(cat.entries.len(), 1); // The id names the item course-wide; the bank is where it is kept. assert_eq!(cat.entries[0].uid, "q-x-001"); assert_eq!(cat.entries[0].bank, "b1"); assert!(cat.get("q-x-001").is_some()); assert_eq!(cat.resolve("q-x-001").unwrap(), "q-x-001"); // A record or a parquet file written before 2.0 still joins. assert!(cat.get("b1::q-x-001").is_some()); assert_eq!(cat.resolve("b1::q-x-001").unwrap(), "q-x-001"); assert!(cat.get("b1::q-nonexistent").is_none()); assert!(cat.validate().unwrap().is_empty()); let _ = std::fs::remove_dir_all(&dir); } #[test] fn one_item_id_in_two_banks_is_fatal() { let dir = tmp("dupitem"); write_course(&dir, ""); write_bank(&dir, "b1.yaml", APPROVED); write_bank(&dir, "b2.yaml", &APPROVED.replace("id: b1", "id: b2")); let message = Catalog::load(&dir).unwrap_err().to_string(); assert!(message.contains("`q-x-001` is used twice"), "{message}"); assert!(message.contains("course-wide"), "{message}"); let _ = std::fs::remove_dir_all(&dir); } #[test] fn duplicate_bank_ids_are_fatal() { let dir = tmp("dupbank"); write_course(&dir, ""); write_bank(&dir, "a.yaml", APPROVED); write_bank(&dir, "b.yaml", APPROVED); let err = Catalog::load(&dir).expect_err("must reject colliding bank ids"); assert!(format!("{err}").contains("two files")); let _ = std::fs::remove_dir_all(&dir); } #[test] fn coverage_finds_real_gaps() { let dir = tmp("coverage"); write_course(&dir, ""); write_bank(&dir, "b1.yaml", APPROVED); let cat = Catalog::load(&dir).unwrap(); let cov = cat.coverage(); assert_eq!(cov.rows.len(), 2); assert!(cov.gaps.contains(&Gap::Uncovered("lo-bare".into()))); assert!(cov.gaps.contains(&Gap::SingleItem("lo-covered".into()))); assert!(cov.gaps.contains(&Gap::RecallOnly("lo-covered".into()))); assert!(cov.gaps.contains(&Gap::LectureUnassessed("L02".into()))); assert_eq!(Gap::Uncovered("x".into()).severity(), Severity::High); let _ = std::fs::remove_dir_all(&dir); } #[test] fn coverage_aggregates_targets_and_names_the_untested_ones() { let dir = tmp("tiered-coverage"); std::fs::create_dir_all(dir.join("banks")).unwrap(); std::fs::write( dir.join("course.yaml"), r#" course: { code: TEST 101, title: Testing, term: Fall 2026 } lectures: L01: { title: One } learning_objectives: lo-binding: { text: Quantify binding., lectures: [L01], order: 1, level_ceiling: 3 } learning_targets: t-kd: { text: Write the expression., objective: lo-binding, order: 1 } t-plot: { text: Read a plot., objective: lo-binding, order: 2 } "#, ) .unwrap(); // Two items, both on the same target. let bank = APPROVED.replace("lo-covered", "t-kd").replace( " - { id: B, text: wrong }\n", " - { id: B, text: wrong }\n - id: q-x-002\n status: approved\n level: 3\n \ cognitive_process: implement\n stem: And again?\n learning_targets: [t-kd]\n \ sources: [{ lecture: L01 }]\n design: { expected_difficulty: 0.5 }\n options:\n \ - { id: A, text: right, correct: true }\n - { id: B, text: wrong }\n", ); write_bank(&dir, "b1.yaml", &bank); let cat = Catalog::load(&dir).unwrap(); let cov = cat.coverage(); let objective = cov .rows .iter() .find(|r| r.id == "lo-binding") .expect("the objective row"); assert_eq!(objective.tier, Tier::Objective); assert_eq!( objective.assemblable, 2, "the objective aggregates its targets" ); assert_eq!( (objective.targets_covered, objective.targets), (1, 2), "two items, but only one of the two targets has been asked about" ); // Resting on one item is the normal case for a target, so it is not a // gap; never having been asked about at all is. assert!(!cov.gaps.contains(&Gap::SingleItem("t-kd".into()))); assert!(cov.gaps.contains(&Gap::Uncovered("t-plot".into()))); let _ = std::fs::remove_dir_all(&dir); } #[test] fn recall_only_respects_a_recall_ceiling() { let dir = tmp("ceiling"); // An objective you only ever intend to test at level 1 is not a gap. write_course(&dir, " lo-ceil: { text: Just recall, level_ceiling: 1 }"); write_bank(&dir, "b1.yaml", &APPROVED.replace("lo-covered", "lo-ceil")); let cat = Catalog::load(&dir).unwrap(); let cov = cat.coverage(); assert!(!cov.gaps.contains(&Gap::RecallOnly("lo-ceil".into()))); let _ = std::fs::remove_dir_all(&dir); } }