Dev (#1)
Sync README to GitHub / sync (push) Successful in 12s
CI / check (push) Successful in 8m31s
Deploy docs / deploy (push) Successful in 6m44s
Nightly / nightly (push) Successful in 10m33s

Reviewed-on: #1
This commit was merged in pull request #1.
This commit is contained in:
alexm committed 2026-08-07 15:48:11 -04:00
1 parent 228a0da47f
commit abc0bdf621
79 files changed
+31370

No files matched your search

+686
View File
@@ -0,0 +1,686 @@
// SPDX-License-Identifier: Prosperity-3.0.0
// Copyright Scientific Computing Studio
// Source: https://git.scient.ing/education/coursebank
//! Loading a whole course at once, and reporting on what it contains.
//!
//! A [`Catalog`] is every bank in a course, indexed so that an item can be found
//! by its global id (`bank::item`), and so that questions like "how many Apply
//! level items do I have on lecture 12" have a cheap answer.
//!
//! The global id is the join key for everything downstream: assessment records
//! reference it, response tables carry it, and usage history is keyed by it. Bank
//! ids therefore have to be unique across a course, which the catalog enforces
//! at load time rather than letting a silent collision merge two items.
//!
//! The coverage report is the part that changes how you write. It is easy to
//! accumulate forty recall items and believe a topic is covered; a table showing
//! that eleven objectives have no item above level 1 is harder to ignore.
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use crate::assessment::AssessmentFile;
use crate::bank::BankFile;
use crate::course::CourseFile;
use crate::error::{Error, Result};
use crate::item::Item;
use crate::layout::Layout;
use crate::taxonomy::{Level, Status};
use crate::yaml;
/// One item plus everything needed to locate it again.
#[derive(Debug, Clone)]
pub struct Entry {
/// The globally unique id, `bank::item`.
pub uid: String,
/// The bank id.
pub bank: String,
/// The file the item came from.
pub path: PathBuf,
/// Position within the bank file, for stable ordering.
pub index: usize,
/// The item itself.
pub item: Item,
}
impl Entry {
/// The item's point value, resolving against the course policy.
///
/// # Arguments
///
/// * `course` - the course whose policy supplies the default.
///
/// # Returns
///
/// The point value.
pub fn points(&self, course: &CourseFile) -> f64 {
self.item.points(course.policy.points_per_item)
}
}
/// Every bank in a course, indexed.
#[derive(Debug, Clone)]
pub struct Catalog {
/// The course registry.
pub course: CourseFile,
/// The resolved directory layout.
pub layout: Layout,
/// Every item, in stable order: by bank id, then by position in the file.
pub entries: Vec<Entry>,
/// Bank metadata by bank id.
pub banks: BTreeMap<String, crate::bank::BankMeta>,
/// Map from global id to index into `entries`.
index: BTreeMap<String, usize>,
}
impl Catalog {
/// Loads a whole course from its directory.
///
/// # Arguments
///
/// * `root` - the course directory containing `course.yaml` and `banks/`.
///
/// # Returns
///
/// The catalog.
///
/// # Errors
///
/// Returns a load error for the course file or any bank, and
/// [`Error::Invalid`] when two banks share an id or two items share a global
/// id, since either makes the join key ambiguous.
pub fn load(root: &Path) -> Result<Catalog> {
let layout = Layout::new(root);
let course = CourseFile::load(&layout.course_file())?;
let mut catalog = Catalog {
course,
layout,
entries: Vec::new(),
banks: BTreeMap::new(),
index: BTreeMap::new(),
};
let mut problems = Vec::new();
let mut files = yaml::list_yaml(&catalog.layout.banks())?;
files.sort();
for path in files {
let bank = BankFile::load_resolved(&path)?;
let bank_id = bank.bank.id.clone();
if let Some(existing) = catalog.banks.get(&bank_id) {
problems.push(format!(
"bank id `{bank_id}` is used by two files (`{}` and `{}`)",
existing.title, bank.bank.title
));
continue;
}
catalog.banks.insert(bank_id.clone(), bank.bank.clone());
for (i, item) in bank.items.into_iter().enumerate() {
let uid = format!("{bank_id}::{}", item.id);
if catalog.index.contains_key(&uid) {
problems.push(format!("duplicate global item id `{uid}`"));
continue;
}
catalog.index.insert(uid.clone(), catalog.entries.len());
catalog.entries.push(Entry {
uid,
bank: bank_id.clone(),
path: path.clone(),
index: i,
item,
});
}
}
if !problems.is_empty() {
return Err(Error::Invalid(problems));
}
Ok(catalog)
}
/// Looks up an item by global id.
///
/// # Arguments
///
/// * `uid` - the global id, `bank::item`.
///
/// # Returns
///
/// The entry, or `None`.
pub fn get(&self, uid: &str) -> Option<&Entry> {
self.index.get(uid).map(|i| &self.entries[*i])
}
/// Looks up an item by global id, erroring when absent.
///
/// # Arguments
///
/// * `uid` - the global id.
///
/// # Returns
///
/// The entry.
///
/// # Errors
///
/// Returns [`Error::Unresolved`] when no such item exists.
pub fn require(&self, uid: &str) -> Result<&Entry> {
self.get(uid).ok_or_else(|| Error::Unresolved {
kind: "item",
id: uid.to_string(),
context: None,
})
}
/// Resolves a possibly-unqualified id to a global id.
///
/// Typing `q-mm-kinetics-001` on the command line should work when that id is
/// unambiguous across the course, because remembering which bank a question
/// lives in is exactly the sort of bookkeeping this tool exists to remove.
///
/// # Arguments
///
/// * `id` - a global id, or a bare item id.
///
/// # Returns
///
/// The global id.
///
/// # Errors
///
/// Returns [`Error::Unresolved`] when nothing matches, or [`Error::Usage`]
/// when a bare id matches items in more than one bank.
pub fn resolve(&self, id: &str) -> Result<String> {
if self.index.contains_key(id) {
return Ok(id.to_string());
}
let matches: Vec<&Entry> = self.entries.iter().filter(|e| e.item.id == id).collect();
match matches.len() {
0 => Err(Error::Unresolved {
kind: "item",
id: id.to_string(),
context: None,
}),
1 => Ok(matches[0].uid.clone()),
_ => Err(Error::usage(format!(
"`{id}` is ambiguous; it exists in {}. Use the full `bank::item` form.",
matches
.iter()
.map(|e| e.bank.as_str())
.collect::<Vec<_>>()
.join(", ")
))),
}
}
/// Every item that may be placed on a graded assessment.
///
/// # Returns
///
/// References to approved, unretired entries.
pub fn assemblable(&self) -> Vec<&Entry> {
self.entries
.iter()
.filter(|e| e.item.is_assemblable())
.collect()
}
/// Validates every bank against the course registry.
///
/// # Returns
///
/// Problems, prefixed with the file they came from.
pub fn validate(&self) -> Result<Vec<String>> {
let mut issues: Vec<String> = self
.course
.validate()
.into_iter()
.map(|m| format!("course.yaml: {m}"))
.collect();
for path in yaml::list_yaml(&self.layout.banks())? {
let bank = BankFile::load_resolved(&path)?;
let name = path
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("?")
.to_string();
issues.extend(
bank.validate(Some(&self.course))
.into_iter()
.map(|m| format!("{name}: {m}")),
);
}
Ok(issues)
}
/// Validates a record's internal invariants, then its references against this
/// catalog: unknown items, keys that drifted, and fingerprints showing the
/// item was reworded since it was administered.
pub fn validate_record(&self, record: &AssessmentFile) -> Vec<String> {
let mut issues = record.validate();
for p in &record.items {
match self.get(&p.item) {
None => issues.push(format!("question {}: unknown item `{}`", p.number, p.item)),
Some(entry) => {
if let Some(fp) = &p.fingerprint {
if *fp != entry.item.fingerprint() {
issues.push(format!(
"question {} ({}): the item has been edited since this assessment; \
statistics from this administration describe the older wording",
p.number, p.item
));
}
}
if !p.key.is_empty() && p.key != entry.item.key_letters() {
issues.push(format!(
"question {} ({}): the recorded key {:?} differs from the item's current key {:?}",
p.number, p.item, p.key, entry.item.key_letters()));
}
}
}
}
issues
}
/// Estimated working time for a record, in minutes.
pub fn estimated_minutes(&self, record: &AssessmentFile) -> f64 {
let seconds: f64 = record
.items
.iter()
.filter_map(|p| self.get(&p.item))
.map(|e| e.item.expected_seconds())
.sum();
seconds / 60.0
}
/// Counts of assemblable, non-bonus items by level.
///
/// # Returns
///
/// A map from level to count, with every level present.
pub fn level_counts(&self) -> BTreeMap<Level, usize> {
let mut out: BTreeMap<Level, usize> = Level::ALL.iter().map(|l| (*l, 0)).collect();
for e in self.assemblable().iter().filter(|e| !e.item.bonus) {
*out.entry(e.item.level).or_insert(0) += 1;
}
out
}
/// Counts of items by workflow status, over the whole course.
///
/// # Returns
///
/// A map from status to count.
pub fn status_counts(&self) -> BTreeMap<Status, usize> {
let mut out: BTreeMap<Status, usize> = BTreeMap::new();
for e in &self.entries {
*out.entry(e.item.status).or_insert(0) += 1;
}
out
}
/// Items that measure a given objective.
///
/// # Arguments
///
/// * `objective` - the objective id.
///
/// # Returns
///
/// Matching entries.
pub fn by_objective(&self, objective: &str) -> Vec<&Entry> {
self.entries
.iter()
.filter(|e| e.item.learning_objectives.iter().any(|o| o == objective))
.collect()
}
/// Items sourced to a given lecture.
///
/// # Arguments
///
/// * `lecture` - the lecture id.
///
/// # Returns
///
/// Matching entries.
pub fn by_lecture(&self, lecture: &str) -> Vec<&Entry> {
self.entries
.iter()
.filter(|e| e.item.sources.iter().any(|s| s.lecture == lecture))
.collect()
}
/// Items carrying a given topic tag.
///
/// # Arguments
///
/// * `topic` - the topic tag.
///
/// # Returns
///
/// Matching entries.
pub fn by_topic(&self, topic: &str) -> Vec<&Entry> {
self.entries
.iter()
.filter(|e| e.item.topics.iter().any(|t| t == topic))
.collect()
}
/// Every topic tag in use, with counts.
///
/// # Returns
///
/// A map from topic to item count.
pub fn topics(&self) -> BTreeMap<String, usize> {
let mut out: BTreeMap<String, usize> = BTreeMap::new();
for e in &self.entries {
for t in &e.item.topics {
*out.entry(t.clone()).or_insert(0) += 1;
}
}
out
}
/// Builds the coverage report.
///
/// # Returns
///
/// One row per assessed objective plus a list of course-wide gaps.
pub fn coverage(&self) -> Coverage {
let mut rows = Vec::new();
for id in self.course.objectives_in_order() {
let obj = &self.course.learning_objectives[&id];
if !obj.assessed {
continue;
}
let items = self.by_objective(&id);
let usable: Vec<&&Entry> = items.iter().filter(|e| e.item.is_assemblable()).collect();
let mut levels: BTreeSet<Level> = BTreeSet::new();
for e in &usable {
levels.insert(e.item.level);
}
rows.push(CoverageRow {
objective: id.clone(),
text: obj.text.clone(),
unit: obj.unit.clone(),
total: items.len(),
assemblable: usable.len(),
max_level: levels.iter().next_back().copied(),
levels: levels.into_iter().collect(),
ceiling: obj.level_ceiling,
});
}
let mut gaps = Vec::new();
for row in &rows {
if row.total == 0 {
gaps.push(Gap::Uncovered(row.objective.clone()));
} else if row.assemblable == 0 {
gaps.push(Gap::NoApprovedItems(row.objective.clone()));
} else if row.assemblable == 1 {
gaps.push(Gap::SingleItem(row.objective.clone()));
}
// An objective assessed only at the recall level is the most common
// and most consequential blind spot: it looks covered in a count and
// is not covered in fact.
if row.assemblable > 0 && row.max_level == Some(Level::Remember) {
if let Some(ceiling) = row.ceiling {
if ceiling > Level::Remember {
gaps.push(Gap::RecallOnly(row.objective.clone()));
}
} else {
gaps.push(Gap::RecallOnly(row.objective.clone()));
}
}
}
for lec_id in self.course.lectures.keys() {
if self
.by_lecture(lec_id)
.iter()
.filter(|e| e.item.is_assemblable())
.count()
== 0
{
gaps.push(Gap::LectureUnassessed(lec_id.clone()));
}
}
// Items with no objective at all cannot appear in any student report.
for e in &self.entries {
if e.item.learning_objectives.is_empty() && e.item.is_assemblable() {
gaps.push(Gap::ItemWithoutObjective(e.uid.clone()));
}
}
Coverage { rows, gaps }
}
}
/// The coverage report.
#[derive(Debug, Clone)]
pub struct Coverage {
/// One row per assessed objective.
pub rows: Vec<CoverageRow>,
/// Course-wide gaps worth acting on.
pub gaps: Vec<Gap>,
}
/// Coverage of one objective.
#[derive(Debug, Clone)]
pub struct CoverageRow {
/// The objective id.
pub objective: String,
/// The objective text.
pub text: String,
/// The unit it belongs to.
pub unit: Option<String>,
/// Items referencing it, at any status.
pub total: usize,
/// Items that could actually be used.
pub assemblable: usize,
/// The highest level assessed.
pub max_level: Option<Level>,
/// Every level assessed.
pub levels: Vec<Level>,
/// The declared ceiling, when set.
pub ceiling: Option<Level>,
}
/// A specific, actionable hole in the item pool.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Gap {
/// An assessed objective with no items at all.
Uncovered(String),
/// An objective whose items are all drafts or retired.
NoApprovedItems(String),
/// An objective resting on a single item, so one bad item hides it entirely.
SingleItem(String),
/// An objective assessed only at the recall level despite a higher ceiling.
RecallOnly(String),
/// A lecture with no usable items.
LectureUnassessed(String),
/// A usable item that references no objective, so it can never appear in a
/// student report.
ItemWithoutObjective(String),
}
impl Gap {
/// A one-line description.
pub fn message(&self) -> String {
match self {
Gap::Uncovered(id) => format!("objective `{id}` has no items"),
Gap::NoApprovedItems(id) => {
format!("objective `{id}` has items but none are approved")
}
Gap::SingleItem(id) => {
format!("objective `{id}` rests on a single item; one bad item hides it")
}
Gap::RecallOnly(id) => {
format!("objective `{id}` is assessed only at level 1")
}
Gap::LectureUnassessed(id) => format!("lecture `{id}` has no usable items"),
Gap::ItemWithoutObjective(uid) => {
format!("item `{uid}` has no learning objective, so it cannot appear in a report")
}
}
}
/// How much attention the gap deserves.
pub fn severity(&self) -> Severity {
match self {
Gap::Uncovered(_) | Gap::NoApprovedItems(_) => Severity::High,
Gap::RecallOnly(_) | Gap::ItemWithoutObjective(_) => Severity::Medium,
Gap::SingleItem(_) | Gap::LectureUnassessed(_) => Severity::Low,
}
}
}
/// How much attention a finding deserves.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum Severity {
/// Worth knowing.
Low,
/// Worth scheduling.
Medium,
/// Worth fixing before the next assessment.
High,
}
impl Severity {
/// A short label for report output.
pub fn label(self) -> &'static str {
match self {
Severity::Low => "low",
Severity::Medium => "medium",
Severity::High => "high",
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn write_course(dir: &Path, extra_lo: &str) {
std::fs::create_dir_all(dir.join("banks")).unwrap();
std::fs::write(
dir.join("course.yaml"),
format!(
r#"
course: {{ code: TEST 101, title: Testing, term: Fall 2026 }}
lectures:
L01: {{ title: One }}
L02: {{ title: Two }}
learning_objectives:
lo-covered: {{ text: Covered objective, lectures: [L01] }}
lo-bare: {{ text: Uncovered objective, lectures: [L02] }}
{extra_lo}
"#
),
)
.unwrap();
}
fn write_bank(dir: &Path, name: &str, body: &str) {
std::fs::write(dir.join("banks").join(name), body).unwrap();
}
fn tmp(tag: &str) -> PathBuf {
let p = std::env::temp_dir().join(format!("coursebank-test-{tag}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&p);
std::fs::create_dir_all(&p).unwrap();
p
}
const APPROVED: &str = r#"
bank:
id: b1
title: First bank
items:
- id: q-x-001
status: approved
level: 1
cognitive_process: recall
stem: What is x?
learning_objectives: [lo-covered]
sources: [{ lecture: L01 }]
design: { expected_difficulty: 0.8 }
options:
- { id: A, text: right, correct: true }
- { id: B, text: wrong }
"#;
#[test]
fn loads_and_indexes_items() {
let dir = tmp("load");
write_course(&dir, "");
write_bank(&dir, "b1.yaml", APPROVED);
let cat = Catalog::load(&dir).expect("catalog loads");
assert_eq!(cat.entries.len(), 1);
assert_eq!(cat.entries[0].uid, "b1::q-x-001");
assert!(cat.get("b1::q-x-001").is_some());
assert_eq!(cat.resolve("q-x-001").unwrap(), "b1::q-x-001");
assert!(cat.validate().unwrap().is_empty());
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn duplicate_bank_ids_are_fatal() {
let dir = tmp("dupbank");
write_course(&dir, "");
write_bank(&dir, "a.yaml", APPROVED);
write_bank(&dir, "b.yaml", APPROVED);
let err = Catalog::load(&dir).expect_err("must reject colliding bank ids");
assert!(format!("{err}").contains("two files"));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn ambiguous_bare_ids_are_rejected() {
let dir = tmp("ambig");
write_course(&dir, "");
write_bank(&dir, "a.yaml", APPROVED);
write_bank(&dir, "b.yaml", &APPROVED.replace("id: b1", "id: b2"));
let cat = Catalog::load(&dir).expect("distinct banks load");
assert_eq!(cat.entries.len(), 2);
let err = cat.resolve("q-x-001").expect_err("bare id is ambiguous");
assert!(format!("{err}").contains("ambiguous"));
// The fully qualified form still works.
assert_eq!(cat.resolve("b2::q-x-001").unwrap(), "b2::q-x-001");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn coverage_finds_real_gaps() {
let dir = tmp("coverage");
write_course(&dir, "");
write_bank(&dir, "b1.yaml", APPROVED);
let cat = Catalog::load(&dir).unwrap();
let cov = cat.coverage();
assert_eq!(cov.rows.len(), 2);
assert!(cov.gaps.contains(&Gap::Uncovered("lo-bare".into())));
assert!(cov.gaps.contains(&Gap::SingleItem("lo-covered".into())));
assert!(cov.gaps.contains(&Gap::RecallOnly("lo-covered".into())));
assert!(cov.gaps.contains(&Gap::LectureUnassessed("L02".into())));
assert_eq!(Gap::Uncovered("x".into()).severity(), Severity::High);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn recall_only_respects_a_recall_ceiling() {
let dir = tmp("ceiling");
// An objective you only ever intend to test at level 1 is not a gap.
write_course(&dir, " lo-ceil: { text: Just recall, level_ceiling: 1 }");
write_bank(&dir, "b1.yaml", &APPROVED.replace("lo-covered", "lo-ceil"));
let cat = Catalog::load(&dir).unwrap();
let cov = cat.coverage();
assert!(!cov.gaps.contains(&Gap::RecallOnly("lo-ceil".into())));
let _ = std::fs::remove_dir_all(&dir);
}
}