Files
coursebank/src/model/catalog.rs
T
alexmandClaude b8fdf4d6a8 feat(lint): glossary rules for terms, equations, values, misconceptions
Fourteen rules in a new `glossary` family (src/authoring/lint/glossary.rs):
unused entries, renamed or coined terms with no textbook name, coined
terms with no exam wording, circular definitions, coined terms in items,
unsourced values, value variants in prose or items, policy-forbidden
target wording, explanation terms a target does not list, recall values
printed in items, deep targets on equations with no implications, and
unresolved {lo-}/{t-} references in 2.0 prose fields. Two of them are
notes: entries defined away from the objective that first needs them,
and the count of proposed entries.

Adds Severity::Note. `coursebank lint` prints notes by default and does
not fail on them, so a course with proposed entries still passes a lint
gate. A distractor that names a registry misconception counts as designed
and as having student feedback.

On the example the glossary rules report exactly glossary-unused on
val-per-decade and glossary-proposed for two entries.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 12:02:50 -05:00

1015 lines
36 KiB
Rust

// SPDX-License-Identifier: Prosperity-3.0.0
// Copyright Scienting
// Source: https://git.scient.ing/education/coursebank
//! Loading a whole course at once, and reporting on what it contains.
//!
//! A [`Catalog`] is every bank in a course, indexed so that an item can be found
//! by its id, and so that questions like "how many Apply
//! level items do I have on lecture 12" have a cheap answer.
//!
//! The global id is the join key for everything downstream: assessment records
//! reference it, response tables carry it, and usage history is keyed by it. Bank
//! ids therefore have to be unique across a course, which the catalog enforces
//! at load time rather than letting a silent collision merge two items.
//!
//! The coverage report is the part that changes how you write. It is easy to
//! accumulate forty recall items and believe a topic is covered; a table showing
//! that eleven objectives have no item above level 1 is harder to ignore.
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use crate::assessment::{AssessmentFile, Placement};
use crate::bank::BankFile;
use crate::course::CourseFile;
use crate::error::{Error, Result};
use crate::item::Item;
use crate::layout::Layout;
use crate::taxonomy::{Format, Level, Status, Tier};
use crate::yaml;
/// One item plus everything needed to locate it again.
#[derive(Debug, Clone)]
pub struct Entry {
/// The item's id, which names it course-wide.
///
/// The bank is in [`Entry::bank`] and the file in [`Entry::path`], neither
/// of which is part of the identity: this is the join key that response
/// data carries, and a join key with a file name in it renames itself every
/// time the files are reorganized.
pub uid: String,
/// The bank id.
pub bank: String,
/// The file the item came from.
pub path: PathBuf,
/// Position within the bank file, for stable ordering.
pub index: usize,
/// The item itself.
pub item: Item,
}
impl Entry {
/// The item's point value, resolving against the course policy.
///
/// # Arguments
///
/// * `course` - the course whose policy supplies the default.
///
/// # Returns
///
/// The point value.
pub fn points(&self, course: &CourseFile) -> f64 {
self.item.points(course.policy.points_per_item)
}
}
/// Every bank in a course, indexed.
#[derive(Debug, Clone)]
pub struct Catalog {
/// The course registry.
pub course: CourseFile,
/// The resolved directory layout.
pub layout: Layout,
/// Every item, in stable order: by bank id, then by position in the file.
pub entries: Vec<Entry>,
/// Bank metadata by bank id.
pub banks: BTreeMap<String, crate::bank::BankMeta>,
/// The statistics, loaded from `analysis/` rather than from the banks.
///
/// Each entry's [`crate::item::Item::calibration`] is filled from this at
/// load, so everything downstream reads one item and does not have to know
/// that the numbers and the wording come from different files.
pub calibration: crate::calibration::CalibrationFile,
/// Map from global id to index into `entries`.
index: BTreeMap<String, usize>,
}
impl Catalog {
/// Loads a whole course from its directory.
///
/// # Arguments
///
/// * `root` - the course directory containing `course.yaml` and `banks/`.
///
/// # Returns
///
/// The catalog.
///
/// # Errors
///
/// Returns a load error for the course file or any bank, and
/// [`Error::Invalid`] when two banks share an id or two items share a global
/// id, since either makes the join key ambiguous.
pub fn load(root: &Path) -> Result<Catalog> {
let layout = Layout::new(root);
let course = CourseFile::load_dir(root)?;
let mut catalog = Catalog {
course,
layout,
entries: Vec::new(),
banks: BTreeMap::new(),
index: BTreeMap::new(),
calibration: crate::calibration::CalibrationFile::default(),
};
catalog.calibration =
crate::calibration::CalibrationFile::load(&catalog.layout.calibration_file())?;
let mut problems = Vec::new();
let mut files = yaml::list_yaml(&catalog.layout.banks())?;
files.sort();
for path in files {
let bank = BankFile::load_resolved(&path)?;
let bank_id = bank.bank.id.clone();
if let Some(existing) = catalog.banks.get(&bank_id) {
problems.push(format!(
"bank id `{bank_id}` is used by two files (`{}` and `{}`)",
existing.title, bank.bank.title
));
continue;
}
catalog.banks.insert(bank_id.clone(), bank.bank.clone());
for (i, item) in bank.items.into_iter().enumerate() {
let uid = item.id.clone();
if let Some(first) = catalog.index.get(&uid) {
problems.push(format!(
"item id `{uid}` is used twice: in bank `{}` and in bank `{bank_id}`. An \
id names one question course-wide, because response data joins on it.",
catalog.entries[*first].bank
));
continue;
}
catalog.index.insert(uid.clone(), catalog.entries.len());
catalog.entries.push(Entry {
uid,
bank: bank_id.clone(),
path: path.clone(),
index: i,
item,
});
}
}
if !problems.is_empty() {
return Err(Error::Invalid(problems));
}
// Statistics are attached here rather than parsed from the bank, which
// is why an item can be read as one thing while its wording and its
// evidence live in files with different review cycles.
for entry in &mut catalog.entries {
if let Some(calibration) = catalog.calibration.items.get(&entry.uid) {
entry.item.calibration = Some(calibration.clone());
}
}
Ok(catalog)
}
/// Looks up an item by global id.
///
/// A pre-2.0 `bank::item` id resolves to the item it used to name, so an
/// assessment record or a parquet file written before the change still
/// joins. See [`crate::item::canonical_id`].
///
/// # Arguments
///
/// * `uid` - the item id, in either form.
///
/// # Returns
///
/// The entry, or `None`.
pub fn get(&self, uid: &str) -> Option<&Entry> {
self.index
.get(uid)
.or_else(|| self.index.get(crate::item::canonical_id(uid)))
.map(|i| &self.entries[*i])
}
/// Looks up an item by global id, erroring when absent.
///
/// # Arguments
///
/// * `uid` - the global id.
///
/// # Returns
///
/// The entry.
///
/// # Errors
///
/// Returns [`Error::Unresolved`] when no such item exists.
pub fn require(&self, uid: &str) -> Result<&Entry> {
self.get(uid).ok_or_else(|| Error::Unresolved {
kind: "item",
id: uid.to_string(),
context: None,
})
}
/// Resolves an id written in either form to the canonical one.
///
/// Since 2.0 an item id is already course-wide, so this is the identity for
/// anything current. What it is still for is the old `bank::item` form,
/// which appears in assessment records, seals, and response files written
/// before the change, and which a person may well still type.
///
/// # Arguments
///
/// * `id` - an item id, in either form.
///
/// # Returns
///
/// The canonical id.
///
/// # Errors
///
/// Returns [`Error::Unresolved`] when nothing matches.
pub fn resolve(&self, id: &str) -> Result<String> {
if self.index.contains_key(id) {
return Ok(id.to_string());
}
let canonical = crate::item::canonical_id(id);
if self.index.contains_key(canonical) {
return Ok(canonical.to_string());
}
Err(Error::Unresolved {
kind: "item",
id: id.to_string(),
context: None,
})
}
/// Every item that may be placed on a graded assessment.
///
/// # Returns
///
/// References to approved, unretired entries.
pub fn assemblable(&self) -> Vec<&Entry> {
self.entries
.iter()
.filter(|e| e.item.is_assemblable())
.collect()
}
/// Validates every bank against the course registry.
///
/// # Returns
///
/// Problems, prefixed with the file they came from.
pub fn validate(&self) -> Result<Vec<String>> {
// Not prefixed here: `CourseFile::validate` attributes each message to
// the fragment that defined the id, which for an unsplit course is
// `course.yaml` and for a split one is the file worth opening.
let mut issues: Vec<String> = self.course.validate();
for path in yaml::list_yaml(&self.layout.banks())? {
let bank = BankFile::load_resolved(&path)?;
let name = path
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("?")
.to_string();
issues.extend(
bank.validate(Some(&self.course))
.into_iter()
.map(|m| format!("{name}: {m}")),
);
}
Ok(issues)
}
/// Checks one placement's option set against the item's pool.
///
/// The checks that moved here from the bank when options became a pool. A
/// bank holding two defensible keys and six distractors is sound; what has
/// to hold for a *form* is that exactly one key reached the student, that
/// none of the distractors was true, and that the count matches policy.
/// None of that can be decided by looking at the item alone.
///
/// # Arguments
///
/// * `p` - the placement.
/// * `item` - the item it names.
///
/// # Returns
///
/// One message per problem.
fn validate_placement(&self, p: &Placement, item: &Item) -> Vec<String> {
let mut issues = Vec::new();
if !item.format.has_options() {
return issues;
}
let at = |number: u32| format!("question {number} ({})", p.item);
for id in p.key.iter().chain(p.distractors.iter()) {
if item.option(id).is_none() {
issues.push(format!(
"{}: `{id}` is not an option of this item",
at(p.number)
));
}
}
for id in &p.distractors {
if p.key.iter().any(|k| k == id) {
issues.push(format!(
"{}: `{id}` is listed as both the key and a distractor",
at(p.number)
));
}
if item.option(id).is_some_and(|o| o.correct) {
issues.push(format!(
"{}: `{id}` is offered as a distractor but the bank keys it correct",
at(p.number)
));
}
}
for id in &p.key {
if item.option(id).is_some_and(|o| !o.correct) {
issues.push(format!(
"{}: `{id}` is keyed correct here but the bank does not key it. An option is \
true or it is not; which true option a form uses is this record's choice, \
but not whether it is true.",
at(p.number)
));
}
}
let single = item.format == Format::SingleBestAnswer;
if single && p.key.len() > 1 {
issues.push(format!(
"{}: single_best_answer administers exactly one key, this names {}",
at(p.number),
p.key.len()
));
}
// No distractor list means the whole pool, which is what a pre-2.0
// record means and what an item whose pool is its form still means.
// The record still has to say which key, when the pool offers a choice.
if p.distractors.is_empty() {
let (keys, _) = item.pool();
if p.key.is_empty() && keys.len() > 1 && single {
issues.push(format!(
"{}: the item offers {} defensible keys, so the record has to say which one \
this assessment used",
at(p.number),
keys.len()
));
}
} else {
let shown = item.administered(&p.key, &p.distractors);
let expected = self.course.policy.options_per_item;
if shown.len() != expected {
issues.push(format!(
"{}: administers {} option(s), but course policy is {expected} per item",
at(p.number),
shown.len()
));
}
if single && p.key.is_empty() {
issues.push(format!(
"{}: names its distractors but not its key, so what was marked correct is \
left to whatever the bank says today",
at(p.number)
));
}
}
if let Some(recorded) = &p.variant {
if *recorded != item.variant_digest(&p.key, &p.distractors) {
issues.push(format!(
"{}: an administered option has been reworded since this assessment. \
Statistics pooled under this variant describe the older wording.",
at(p.number)
));
}
}
issues
}
/// Checks every sealed administration's stems against the bank.
///
/// The seal is the authority on what was administered, so this is the
/// comparison that matters: a record can be edited, but a seal is written
/// before the exam is printed and digested against tampering. A stem that
/// no longer matches the one a cohort answered means the id now names a
/// different question, and every statistic pooled under it is describing
/// two things at once.
///
/// # Arguments
///
/// * `seals` - the sealed administrations to check.
///
/// # Returns
///
/// One message per stem that has moved out from under its seal.
pub fn validate_seals(&self, seals: &[crate::seal::SealFile]) -> Vec<String> {
let mut issues = Vec::new();
for seal in seals {
for item in &seal.items {
let Some(digest) = &item.stem_digest else {
continue;
};
let Some(entry) = self.get(&item.item) else {
continue;
};
if *digest != entry.item.stem_digest() {
issues.push(format!(
"{}: question {} (`{}`) was administered with a different stem than the \
bank now holds. Statistics from that administration describe the older \
wording; give the new wording its own id.",
seal.seal.assessment, item.number, item.item
));
}
}
}
issues
}
/// Validates a record's internal invariants, then its references against this
/// catalog: unknown items, keys that drifted, and fingerprints showing the
/// item was reworded since it was administered.
pub fn validate_record(&self, record: &AssessmentFile) -> Vec<String> {
let mut issues = record.validate();
for p in &record.items {
match self.get(&p.item) {
None => issues.push(format!("question {}: unknown item `{}`", p.number, p.item)),
Some(entry) => {
// The rule the stem digest exists to enforce. A changed
// fingerprint is a note: the statistics describe an older
// wording. A changed stem is an error: whatever was
// administered is not the question the bank now holds, so
// the id is being reused for two different questions.
if let Some(digest) = &p.stem_digest {
if *digest != entry.item.stem_digest() {
issues.push(format!(
"question {} ({}): the stem has been reworded since this \
assessment. A reworded stem is a new question: give the new \
wording a new id with `supersedes: {}`, and leave this one as \
it was administered.",
p.number, p.item, p.item
));
}
}
if let Some(fp) = &p.fingerprint {
if *fp != entry.item.fingerprint() {
issues.push(format!(
"question {} ({}): the item has been edited since this assessment; \
statistics from this administration describe the older wording",
p.number, p.item
));
}
}
issues.extend(self.validate_placement(p, &entry.item));
}
}
}
issues
}
/// Estimated working time for a record, in minutes.
pub fn estimated_minutes(&self, record: &AssessmentFile) -> f64 {
let seconds: f64 = record
.items
.iter()
.filter_map(|p| self.get(&p.item))
.map(|e| e.item.expected_seconds())
.sum();
seconds / 60.0
}
/// Counts of assemblable, non-bonus items by level.
///
/// # Returns
///
/// A map from level to count, with every level present.
pub fn level_counts(&self) -> BTreeMap<Level, usize> {
let mut out: BTreeMap<Level, usize> = Level::ALL.iter().map(|l| (*l, 0)).collect();
for e in self.assemblable().iter().filter(|e| !e.item.bonus) {
*out.entry(e.item.level).or_insert(0) += 1;
}
out
}
/// Counts of items by workflow status, over the whole course.
///
/// # Returns
///
/// A map from status to count.
pub fn status_counts(&self) -> BTreeMap<Status, usize> {
let mut out: BTreeMap<Status, usize> = BTreeMap::new();
for e in &self.entries {
*out.entry(e.item.status).or_insert(0) += 1;
}
out
}
/// Items tagged with a given learning target.
///
/// # Arguments
///
/// * `target` - the target id.
///
/// # Returns
///
/// Matching entries.
pub fn by_target(&self, target: &str) -> Vec<&Entry> {
self.entries
.iter()
.filter(|e| e.item.learning_targets.iter().any(|t| t == target))
.collect()
}
/// Items sourced to a given lecture.
///
/// # Arguments
///
/// * `lecture` - the lecture id.
///
/// # Returns
///
/// Matching entries.
pub fn by_lecture(&self, lecture: &str) -> Vec<&Entry> {
self.entries
.iter()
.filter(|e| e.item.sources.iter().any(|s| s.lecture == lecture))
.collect()
}
/// Items carrying a given topic tag.
///
/// # Arguments
///
/// * `topic` - the topic tag.
///
/// # Returns
///
/// Matching entries.
pub fn by_topic(&self, topic: &str) -> Vec<&Entry> {
self.entries
.iter()
.filter(|e| e.item.topics.iter().any(|t| t == topic))
.collect()
}
/// Every topic tag in use, with counts.
///
/// # Returns
///
/// A map from topic to item count.
pub fn topics(&self) -> BTreeMap<String, usize> {
let mut out: BTreeMap<String, usize> = BTreeMap::new();
for e in &self.entries {
for t in &e.item.topics {
*out.entry(t.clone()).or_insert(0) += 1;
}
}
out
}
/// Items that measure a given objective, through any of its targets.
///
/// # Arguments
///
/// * `objective` - the objective id.
///
/// # Returns
///
/// Entries tagged with the objective itself or with any of its targets,
/// each appearing once even when it is tagged with two of them.
pub fn by_objective(&self, objective: &str) -> Vec<&Entry> {
self.entries
.iter()
.filter(|e| {
e.item
.learning_targets
.iter()
.any(|t| t == objective || self.course.objective_for(t) == objective)
})
.collect()
}
/// Builds the coverage report.
///
/// Rows cover both tiers, because the two answer different questions. An
/// objective row answers "can I build an exam that reports on this
/// objective", and aggregates every item under it. A target row answers
/// "which specific things have I written items for", which is the
/// authoring queue, and a target with no items is the most common and least
/// visible hole in a bank: the objective looks well covered while a third of
/// what it claims has never been asked.
///
/// # Returns
///
/// One row per assessed entry plus a list of course-wide gaps.
pub fn coverage(&self) -> Coverage {
let mut rows = Vec::new();
for id in self.course.registry_in_order() {
if !self.course.is_objective(&id) && !self.course.is_target(&id) {
continue;
}
if !self.course.is_assessed(&id) {
continue;
}
let targets = self.course.targets(&id);
let tier = if self.course.is_objective(&id) {
Tier::Objective
} else {
Tier::Target
};
// An objective's pool is everything under it; a target's is what is
// tagged to it directly. An objective with no targets is its own
// target, so the two agree there.
let items = if targets.is_empty() {
self.by_target(&id)
} else {
self.by_objective(&id)
};
let usable: Vec<&&Entry> = items.iter().filter(|e| e.item.is_assemblable()).collect();
let mut levels: BTreeSet<Level> = BTreeSet::new();
for e in &usable {
levels.insert(e.item.level);
}
let targets_covered = targets
.iter()
.filter(|target| {
self.by_target(target)
.iter()
.any(|e| e.item.is_assemblable())
})
.count();
rows.push(CoverageRow {
id: id.clone(),
text: self.course.text_for(&id),
tier,
objective: self
.course
.is_target(&id)
.then(|| self.course.objective_for(&id).to_string()),
unit: self.course.objective_unit(&id).map(str::to_string),
targets: targets.len(),
targets_covered,
total: items.len(),
assemblable: usable.len(),
max_level: levels.iter().next_back().copied(),
levels: levels.into_iter().collect(),
ceiling: self.course.effective_level_ceiling(&id),
});
}
let mut gaps = Vec::new();
for row in &rows {
// Gaps are raised against the tier that can be acted on. "No items
// at all" is worth saying about a target, because writing one is the
// fix. "Resting on a single item" is worth saying about an
// objective, because a target resting on one item is the normal and
// intended case, and flagging two hundred of them would bury the
// rows that matter.
if row.total == 0 {
gaps.push(Gap::Uncovered(row.id.clone()));
} else if row.assemblable == 0 {
gaps.push(Gap::NoApprovedItems(row.id.clone()));
} else if row.assemblable == 1 && row.tier == Tier::Objective {
gaps.push(Gap::SingleItem(row.id.clone()));
}
// An objective assessed only at the recall level is the most common
// and most consequential blind spot: it looks covered in a count and
// is not covered in fact.
if row.tier == Tier::Objective
&& row.assemblable > 0
&& row.max_level == Some(Level::Remember)
{
if let Some(ceiling) = row.ceiling {
if ceiling > Level::Remember {
gaps.push(Gap::RecallOnly(row.id.clone()));
}
} else {
gaps.push(Gap::RecallOnly(row.id.clone()));
}
}
}
for lec_id in self.course.lectures.keys() {
if self
.by_lecture(lec_id)
.iter()
.filter(|e| e.item.is_assemblable())
.count()
== 0
{
gaps.push(Gap::LectureUnassessed(lec_id.clone()));
}
}
// Items with no objective at all cannot appear in any student report.
for e in &self.entries {
if e.item.learning_targets.is_empty() && e.item.is_assemblable() {
gaps.push(Gap::ItemWithoutObjective(e.uid.clone()));
}
}
Coverage { rows, gaps }
}
}
/// The coverage report.
#[derive(Debug, Clone)]
pub struct Coverage {
/// One row per assessed registry entry, objectives first with their targets
/// following each.
pub rows: Vec<CoverageRow>,
/// Course-wide gaps worth acting on.
pub gaps: Vec<Gap>,
}
/// Coverage of one registry entry, at either tier.
#[derive(Debug, Clone)]
pub struct CoverageRow {
/// The registry id.
pub id: String,
/// Its text.
pub text: String,
/// Whether this is an objective, whose counts aggregate every item under it,
/// or a target, whose counts are its own items.
pub tier: Tier,
/// The objective this row sits under, for a target.
pub objective: Option<String>,
/// The unit it belongs to, inherited from the objective when not declared.
pub unit: Option<String>,
/// Targets in the registry, for an objective row.
pub targets: usize,
/// Targets with at least one usable item.
///
/// The number to look at when an objective looks well covered: twenty items
/// spread over four of its nine targets is a different bank from twenty
/// items spread over all nine.
pub targets_covered: usize,
/// Items referencing it, at any status.
pub total: usize,
/// Items that could actually be used.
pub assemblable: usize,
/// The highest level assessed.
pub max_level: Option<Level>,
/// Every level assessed.
pub levels: Vec<Level>,
/// The ceiling in force, inherited from the objective when not declared.
pub ceiling: Option<Level>,
}
/// A specific, actionable hole in the item pool.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Gap {
/// An assessed objective or target with no items at all.
Uncovered(String),
/// An entry whose items are all drafts or retired.
NoApprovedItems(String),
/// An objective resting on a single item, so one bad item hides it entirely.
SingleItem(String),
/// An objective assessed only at the recall level despite a higher ceiling.
RecallOnly(String),
/// A lecture with no usable items.
LectureUnassessed(String),
/// A usable item that references no objective, so it can never appear in a
/// student report.
ItemWithoutObjective(String),
}
impl Gap {
/// A one-line description.
pub fn message(&self) -> String {
match self {
Gap::Uncovered(id) => format!("objective `{id}` has no items"),
Gap::NoApprovedItems(id) => {
format!("objective `{id}` has items but none are approved")
}
Gap::SingleItem(id) => {
format!("objective `{id}` rests on a single item; one bad item hides it")
}
Gap::RecallOnly(id) => {
format!("objective `{id}` is assessed only at level 1")
}
Gap::LectureUnassessed(id) => format!("lecture `{id}` has no usable items"),
Gap::ItemWithoutObjective(uid) => {
format!("item `{uid}` has no learning objective, so it cannot appear in a report")
}
}
}
/// How much attention the gap deserves.
pub fn severity(&self) -> Severity {
match self {
Gap::Uncovered(_) | Gap::NoApprovedItems(_) => Severity::High,
Gap::RecallOnly(_) | Gap::ItemWithoutObjective(_) => Severity::Medium,
Gap::SingleItem(_) | Gap::LectureUnassessed(_) => Severity::Low,
}
}
}
/// How much attention a finding deserves.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum Severity {
/// Information, not a problem: a count of entries waiting for approval, or
/// a convention that is reported but not enforced. `coursebank lint`
/// prints notes and does not fail on them.
Note,
/// Worth knowing.
Low,
/// Worth scheduling.
Medium,
/// Worth fixing before the next assessment.
High,
}
impl Severity {
/// A short label for report output.
pub fn label(self) -> &'static str {
match self {
Severity::Note => "note",
Severity::Low => "low",
Severity::Medium => "medium",
Severity::High => "high",
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn write_course(dir: &Path, extra_lo: &str) {
std::fs::create_dir_all(dir.join("banks")).unwrap();
std::fs::write(
dir.join("course.yaml"),
format!(
r#"
course: {{ code: TEST 101, title: Testing, term: Fall 2026 }}
lectures:
L01: {{ title: One }}
L02: {{ title: Two }}
learning_objectives:
lo-covered: {{ text: Covered objective, lectures: [L01] }}
lo-bare: {{ text: Uncovered objective, lectures: [L02] }}
{extra_lo}
"#
),
)
.unwrap();
}
fn write_bank(dir: &Path, name: &str, body: &str) {
std::fs::write(dir.join("banks").join(name), body).unwrap();
}
fn tmp(tag: &str) -> PathBuf {
let p = std::env::temp_dir().join(format!("coursebank-test-{tag}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&p);
std::fs::create_dir_all(&p).unwrap();
p
}
const APPROVED: &str = r#"
bank:
id: b1
title: First bank
items:
- id: q-x-001
status: approved
level: 1
cognitive_process: recall
stem: What is x?
learning_targets: [lo-covered]
sources: [{ lecture: L01 }]
design: { expected_difficulty: 0.8 }
options:
- { id: A, text: right, correct: true }
- { id: B, text: wrong }
"#;
#[test]
fn loads_and_indexes_items() {
let dir = tmp("load");
write_course(&dir, "");
write_bank(&dir, "b1.yaml", APPROVED);
let cat = Catalog::load(&dir).expect("catalog loads");
assert_eq!(cat.entries.len(), 1);
// The id names the item course-wide; the bank is where it is kept.
assert_eq!(cat.entries[0].uid, "q-x-001");
assert_eq!(cat.entries[0].bank, "b1");
assert!(cat.get("q-x-001").is_some());
assert_eq!(cat.resolve("q-x-001").unwrap(), "q-x-001");
// A record or a parquet file written before 2.0 still joins.
assert!(cat.get("b1::q-x-001").is_some());
assert_eq!(cat.resolve("b1::q-x-001").unwrap(), "q-x-001");
assert!(cat.get("b1::q-nonexistent").is_none());
assert!(cat.validate().unwrap().is_empty());
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn one_item_id_in_two_banks_is_fatal() {
let dir = tmp("dupitem");
write_course(&dir, "");
write_bank(&dir, "b1.yaml", APPROVED);
write_bank(&dir, "b2.yaml", &APPROVED.replace("id: b1", "id: b2"));
let message = Catalog::load(&dir).unwrap_err().to_string();
assert!(message.contains("`q-x-001` is used twice"), "{message}");
assert!(message.contains("course-wide"), "{message}");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn duplicate_bank_ids_are_fatal() {
let dir = tmp("dupbank");
write_course(&dir, "");
write_bank(&dir, "a.yaml", APPROVED);
write_bank(&dir, "b.yaml", APPROVED);
let err = Catalog::load(&dir).expect_err("must reject colliding bank ids");
assert!(format!("{err}").contains("two files"));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn coverage_finds_real_gaps() {
let dir = tmp("coverage");
write_course(&dir, "");
write_bank(&dir, "b1.yaml", APPROVED);
let cat = Catalog::load(&dir).unwrap();
let cov = cat.coverage();
assert_eq!(cov.rows.len(), 2);
assert!(cov.gaps.contains(&Gap::Uncovered("lo-bare".into())));
assert!(cov.gaps.contains(&Gap::SingleItem("lo-covered".into())));
assert!(cov.gaps.contains(&Gap::RecallOnly("lo-covered".into())));
assert!(cov.gaps.contains(&Gap::LectureUnassessed("L02".into())));
assert_eq!(Gap::Uncovered("x".into()).severity(), Severity::High);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn coverage_aggregates_targets_and_names_the_untested_ones() {
let dir = tmp("tiered-coverage");
std::fs::create_dir_all(dir.join("banks")).unwrap();
std::fs::write(
dir.join("course.yaml"),
r#"
course: { code: TEST 101, title: Testing, term: Fall 2026 }
lectures:
L01: { title: One }
learning_objectives:
lo-binding: { text: Quantify binding., lectures: [L01], order: 1, level_ceiling: 3 }
learning_targets:
t-kd: { text: Write the expression., objective: lo-binding, order: 1 }
t-plot: { text: Read a plot., objective: lo-binding, order: 2 }
"#,
)
.unwrap();
// Two items, both on the same target.
let bank = APPROVED.replace("lo-covered", "t-kd").replace(
" - { id: B, text: wrong }\n",
" - { id: B, text: wrong }\n - id: q-x-002\n status: approved\n level: 3\n \
cognitive_process: implement\n stem: And again?\n learning_targets: [t-kd]\n \
sources: [{ lecture: L01 }]\n design: { expected_difficulty: 0.5 }\n options:\n \
- { id: A, text: right, correct: true }\n - { id: B, text: wrong }\n",
);
write_bank(&dir, "b1.yaml", &bank);
let cat = Catalog::load(&dir).unwrap();
let cov = cat.coverage();
let objective = cov
.rows
.iter()
.find(|r| r.id == "lo-binding")
.expect("the objective row");
assert_eq!(objective.tier, Tier::Objective);
assert_eq!(
objective.assemblable, 2,
"the objective aggregates its targets"
);
assert_eq!(
(objective.targets_covered, objective.targets),
(1, 2),
"two items, but only one of the two targets has been asked about"
);
// Resting on one item is the normal case for a target, so it is not a
// gap; never having been asked about at all is.
assert!(!cov.gaps.contains(&Gap::SingleItem("t-kd".into())));
assert!(cov.gaps.contains(&Gap::Uncovered("t-plot".into())));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn recall_only_respects_a_recall_ceiling() {
let dir = tmp("ceiling");
// An objective you only ever intend to test at level 1 is not a gap.
write_course(&dir, " lo-ceil: { text: Just recall, level_ceiling: 1 }");
write_bank(&dir, "b1.yaml", &APPROVED.replace("lo-covered", "lo-ceil"));
let cat = Catalog::load(&dir).unwrap();
let cov = cat.coverage();
assert!(!cov.gaps.contains(&Gap::RecallOnly("lo-ceil".into())));
let _ = std::fs::remove_dir_all(&dir);
}
}