Dev (#1)
Sync README to GitHub / sync (push) Successful in 12s
CI / check (push) Successful in 8m31s
Deploy docs / deploy (push) Successful in 6m44s
Nightly / nightly (push) Successful in 10m33s

Reviewed-on: #1
This commit was merged in pull request #1.
This commit is contained in:
alexm committed 2026-08-07 15:48:11 -04:00
1 parent 228a0da47f
commit abc0bdf621
79 files changed
+31370

No files matched your search

+627
View File
@@ -0,0 +1,627 @@
// SPDX-License-Identifier: Prosperity-3.0.0
// Copyright Scientific Computing Studio
// Source: https://git.scient.ing/education/coursebank
//! The assessment record: what you actually asked, in what order, on what date.
//!
//! This file is the hinge of the whole tool, and it is worth being explicit about
//! why it exists as a separate artifact.
//!
//! Grading exports do not know about your item bank. Gradescope gives you
//! `14.csv`; Canvas gives you a column header. Both identify questions by
//! *position on a form*. An item bank identifies questions by stable id. The
//! assessment record is the only place those two namespaces meet, and without it
//! there is no way to say that question 14 of Exam 4 was
//! `docking::q-scoring-003` at version 2.
//!
//! Recording it also makes usage history free. Rather than maintaining a separate
//! ledger that can drift out of sync with reality, `coursebank usage` scans the
//! assessment records: an item was used exactly when it appears on a record. The
//! records are the source of truth, and they are small, readable, and diffable.
//!
//! Each placement stores the resolved key and content fingerprint *as used*. A
//! year later, when the item has been reworded twice, you can still see what the
//! students in front of you were asked.
use std::collections::BTreeMap;
use std::path::Path;
use serde::{Deserialize, Serialize};
use crate::course::SCHEMA_VERSION;
use crate::date::Date;
use crate::error::Result;
use crate::taxonomy::Level;
use crate::yaml;
/// A whole assessment record file.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct AssessmentFile {
/// Schema version this file targets.
#[serde(
default = "default_version",
deserialize_with = "yaml::flexible_string"
)]
pub schema_version: String,
/// Identity and administration details.
pub assessment: Assessment,
/// The blueprint this was assembled against, when it was assembled by the
/// tool. Keeping it lets you check the form you shipped against the design
/// you intended.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub blueprint: Option<Blueprint>,
/// Alternate forms, each a permutation of the same items.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub forms: Vec<Form>,
/// The items, in printed order.
#[serde(default)]
pub items: Vec<Placement>,
}
/// Assessment identity and administration.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Assessment {
/// Stable id, e.g. `exam-4-2026s`. Response tables carry this.
pub id: String,
/// Human title as printed, e.g. `Exam 4`.
pub title: String,
/// The term this administration belongs to. Items outlive terms, so the term
/// lives here rather than on the item.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub term: Option<String>,
/// What kind of assessment this is.
#[serde(default = "default_kind")]
pub kind: Kind,
/// When it was administered.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub date: Option<Date>,
/// Where it was administered.
#[serde(default = "default_platform")]
pub platform: Platform,
/// Time allowed, in minutes. Compared against the summed expected time.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub minutes_allowed: Option<f64>,
/// Attempts allowed; `-1` means unlimited. Only meaningful online.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub attempts: Option<i64>,
/// Whether the platform should shuffle options.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub shuffle: Option<bool>,
/// How repeated attempts are scored.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub scoring_policy: Option<ScoringPolicy>,
/// Instructions printed at the top of a paper form.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub instructions: Option<String>,
/// Notes to yourself about this administration.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub notes: Option<String>,
}
/// What kind of assessment a record describes.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Kind {
/// A summative in-term exam.
Exam,
/// A short graded check.
Quiz,
/// Graded work done outside class.
Homework,
/// Ungraded practice.
Practice,
/// A cumulative final.
Final,
}
impl Kind {
/// The token used in YAML and in file names.
pub fn as_str(self) -> &'static str {
match self {
Kind::Exam => "exam",
Kind::Quiz => "quiz",
Kind::Homework => "homework",
Kind::Practice => "practice",
Kind::Final => "final",
}
}
/// Whether results from this kind should feed item calibration.
///
/// Practice work is excluded by default: it is usually untimed, open book,
/// and attempted more than once, so pooling it with exam data would bias
/// every difficulty estimate downward.
pub fn counts_for_calibration(self) -> bool {
!matches!(self, Kind::Practice)
}
}
/// Where an assessment was administered, which determines the export format and
/// the shape of the grading data that comes back.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Platform {
/// Printed and scanned, graded in Gradescope.
Paper,
/// A Canvas quiz.
Canvas,
/// Delivered on paper but graded elsewhere.
Other,
}
/// How repeated attempts are scored.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ScoringPolicy {
/// Keep the best attempt.
KeepHighest,
/// Keep the most recent attempt.
KeepLatest,
}
impl ScoringPolicy {
/// The token Canvas expects in a QTI package.
pub fn as_str(self) -> &'static str {
match self {
ScoringPolicy::KeepHighest => "keep_highest",
ScoringPolicy::KeepLatest => "keep_latest",
}
}
}
/// The design an assessment was assembled against.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Blueprint {
/// How many scored items to draw at each level.
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub level_counts: BTreeMap<Level, usize>,
/// How many bonus items to draw at each level.
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub bonus_counts: BTreeMap<Level, usize>,
/// Objectives that must appear, with a minimum item count each.
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub objective_minimums: BTreeMap<String, usize>,
/// Restrict the draw to these lectures.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub lectures: Vec<String>,
/// Restrict the draw to these topics.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub topics: Vec<String>,
/// Restrict the draw to these banks.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub banks: Vec<String>,
/// The most items any one bank may contribute, so a form is not dominated by
/// whichever topic you happened to write the most questions about.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max_per_bank: Option<usize>,
/// Do not reuse an item used within this many days.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cooldown_days: Option<i64>,
/// The seed used, so the draw can be reproduced exactly.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub seed: Option<u64>,
}
impl Blueprint {
/// Total scored items requested.
pub fn scored_total(&self) -> usize {
self.level_counts.values().sum()
}
/// Total bonus items requested.
pub fn bonus_total(&self) -> usize {
self.bonus_counts.values().sum()
}
}
/// One alternate form of the same assessment.
///
/// A form does not change which items appear, only their order and the order of
/// their options, so every form measures the same thing and one answer key
/// generator serves them all.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Form {
/// Form label, e.g. `A`.
pub id: String,
/// Seed for the permutation. Reproducing a printed form requires it.
pub seed: u64,
/// Whether item order is permuted.
#[serde(default = "yes")]
pub shuffle_items: bool,
/// Whether option order is permuted within each item.
#[serde(default = "yes")]
pub shuffle_options: bool,
}
/// One item as placed on the form.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Placement {
/// Printed question number. This is the join key to grading exports, which
/// is the entire reason this record exists.
pub number: u32,
/// The item's global id, `bank::item`.
pub item: String,
/// The item version used.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub version: Option<u32>,
/// The content fingerprint as used, so later edits are detectable.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub fingerprint: Option<String>,
/// Points as administered.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub points: Option<f64>,
/// Whether it was scored as bonus here.
#[serde(default, skip_serializing_if = "is_false")]
pub bonus: bool,
/// The keyed letters as administered.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub key: Vec<String>,
/// The level as administered, denormalized so a record reads standalone.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub level: Option<Level>,
/// Objectives as administered, denormalized for the same reason.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub learning_objectives: Vec<String>,
/// Credit awarded to non-keyed options after the fact, keyed by letter.
///
/// When item analysis or a student challenge leads you to credit a
/// distractor, recording it here keeps the rescoring decision with the
/// administration it applies to instead of quietly editing the item.
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub credit_overrides: BTreeMap<String, f64>,
/// Set when an item was dropped from scoring after administration.
#[serde(default, skip_serializing_if = "is_false")]
pub dropped: bool,
}
impl AssessmentFile {
/// Loads an assessment record.
///
/// # Arguments
///
/// * `path` - the YAML file.
///
/// # Returns
///
/// The parsed record.
///
/// # Errors
///
/// Returns a load error.
pub fn load(path: &Path) -> Result<AssessmentFile> {
yaml::read(path)
}
/// Writes the record back out.
///
/// # Arguments
///
/// * `path` - the destination.
///
/// # Errors
///
/// Returns [`crate::error::Error::Io`] on a write failure.
pub fn save(&self, path: &Path) -> Result<()> {
yaml::write(path, self)
}
/// Loads every assessment record in a directory, sorted by date then id.
///
/// # Arguments
///
/// * `dir` - the assessments directory.
///
/// # Returns
///
/// The records, empty when the directory does not exist.
///
/// # Errors
///
/// Propagates load errors.
pub fn load_all(dir: &Path) -> Result<Vec<AssessmentFile>> {
let mut out = Vec::new();
for path in yaml::list_yaml(dir)? {
out.push(AssessmentFile::load(&path)?);
}
out.sort_by(|a, b| {
a.assessment
.date
.cmp(&b.assessment.date)
.then(a.assessment.id.cmp(&b.assessment.id))
});
Ok(out)
}
/// The placement at a printed question number.
///
/// # Arguments
///
/// * `number` - the printed number.
///
/// # Returns
///
/// The placement, or `None`.
pub fn placement(&self, number: u32) -> Option<&Placement> {
self.items.iter().find(|p| p.number == number)
}
/// Total scored points, excluding bonus and dropped items.
///
/// # Arguments
///
/// * `default_points` - the course policy default.
///
/// # Returns
///
/// The total.
pub fn total_points(&self, default_points: f64) -> f64 {
self.items
.iter()
.filter(|p| !p.bonus && !p.dropped)
.map(|p| p.points.unwrap_or(default_points))
.sum()
}
/// Counts of scored placements by level.
///
/// # Returns
///
/// A map from level to count.
pub fn level_counts(&self) -> BTreeMap<Level, usize> {
let mut out: BTreeMap<Level, usize> = BTreeMap::new();
for p in self.items.iter().filter(|p| !p.bonus) {
if let Some(l) = p.level {
*out.entry(l).or_insert(0) += 1;
}
}
out
}
/// Checks the record for internal problems and for drift from the bank.
///
/// The drift check is the valuable part: if an item was reworded after this
/// assessment was given, the fingerprints disagree, and any analysis that
/// pools this administration with a later one is comparing two questions.
///
/// # Arguments
///
/// * `catalog` - the loaded course, for resolving item references.
///
/// # Returns
///
/// Every problem found.
pub fn validate(&self) -> Vec<String> {
let mut issues = Vec::new();
if self.assessment.id.trim().is_empty() {
issues.push("assessment.id is empty".into());
}
if self.items.is_empty() {
issues.push("the record lists no items".into());
}
let mut numbers: BTreeMap<u32, usize> = BTreeMap::new();
for p in &self.items {
*numbers.entry(p.number).or_insert(0) += 1;
}
for (n, count) in &numbers {
if *count > 1 {
issues.push(format!("question number {n} is used {count} times"));
}
}
// Gaps are legal but almost always a mistake, since grading exports are
// numbered contiguously.
let mut sorted: Vec<u32> = numbers.keys().copied().collect();
sorted.sort_unstable();
for (i, n) in sorted.iter().enumerate() {
let expected = i as u32 + 1;
if *n != expected {
issues.push(format!(
"question numbers are not contiguous from 1; expected {expected}, found {n}"
));
break;
}
}
let mut seen_items: BTreeMap<&str, u32> = BTreeMap::new();
for p in &self.items {
if let Some(first) = seen_items.get(p.item.as_str()) {
issues.push(format!(
"item `{}` appears twice, at questions {first} and {}",
p.item, p.number
));
} else {
seen_items.insert(p.item.as_str(), p.number);
}
for (letter, credit) in &p.credit_overrides {
if !(0.0..=1.0).contains(credit) {
issues.push(format!(
"question {}: credit override for `{letter}` must be in [0, 1], got {credit}",
p.number
));
}
}
}
let mut form_ids: Vec<&str> = Vec::new();
for form in &self.forms {
if form_ids.contains(&form.id.as_str()) {
issues.push(format!("duplicate form id `{}`", form.id));
}
form_ids.push(&form.id);
}
issues
}
/// A skeleton record for `coursebank assessment new`.
///
/// # Arguments
///
/// * `id` - the assessment id.
/// * `title` - the printed title.
/// * `kind` - the kind of assessment.
///
/// # Returns
///
/// A record with no items.
pub fn skeleton(id: &str, title: &str, kind: Kind) -> AssessmentFile {
AssessmentFile {
schema_version: SCHEMA_VERSION.to_string(),
assessment: Assessment {
id: id.to_string(),
title: title.to_string(),
term: None,
kind,
date: Some(Date::today()),
platform: Platform::Paper,
minutes_allowed: None,
attempts: None,
shuffle: None,
scoring_policy: None,
instructions: None,
notes: None,
},
blueprint: None,
forms: Vec::new(),
items: Vec::new(),
}
}
}
/// One recorded appearance of an item, derived from the assessment records.
#[derive(Debug, Clone)]
pub struct Usage {
/// The item's global id.
pub item: String,
/// The assessment id it appeared on.
pub assessment: String,
/// The assessment title.
pub title: String,
/// The kind of assessment.
pub kind: Kind,
/// When it was administered, when recorded.
pub date: Option<Date>,
/// The printed question number.
pub number: u32,
/// The fingerprint as used.
pub fingerprint: Option<String>,
}
fn default_version() -> String {
SCHEMA_VERSION.to_string()
}
fn default_kind() -> Kind {
Kind::Exam
}
fn default_platform() -> Platform {
Platform::Paper
}
fn yes() -> bool {
true
}
fn is_false(b: &bool) -> bool {
!*b
}
#[cfg(test)]
mod tests {
use super::*;
fn record(src: &str) -> AssessmentFile {
serde_yaml_ng::from_str(src).expect("assessment parses")
}
const SIMPLE: &str = r#"
assessment:
id: exam-4-2026s
title: Exam 4
kind: exam
date: 2026-04-23
platform: paper
items:
- { number: 1, item: "b1::q-a-001", points: 1.5, key: [B], level: 1 }
- { number: 2, item: "b1::q-a-002", points: 1.5, key: [C], level: 3 }
- { number: 3, item: "b1::q-a-003", points: 1.5, bonus: true, key: [D], level: 5 }
"#;
#[test]
fn parses_and_totals_only_scored_points() {
let a = record(SIMPLE);
assert_eq!(a.items.len(), 3);
assert_eq!(a.total_points(1.0), 3.0, "bonus is excluded");
assert_eq!(a.placement(2).unwrap().item, "b1::q-a-002");
assert!(a.validate().is_empty(), "{:?}", a.validate());
}
#[test]
fn level_counts_skip_bonus() {
let counts = record(SIMPLE).level_counts();
assert_eq!(counts.get(&Level::Remember), Some(&1));
assert_eq!(counts.get(&Level::Apply), Some(&1));
assert_eq!(counts.get(&Level::Create), None);
}
#[test]
fn catches_duplicate_numbers_and_repeated_items() {
let a = record(
r#"
assessment: { id: x, title: X }
items:
- { number: 1, item: "b::q-1" }
- { number: 1, item: "b::q-1" }
"#,
);
let issues = a.validate();
assert!(issues.iter().any(|i| i.contains("used 2 times")));
assert!(issues.iter().any(|i| i.contains("appears twice")));
}
#[test]
fn catches_non_contiguous_numbering() {
let a = record(
r#"
assessment: { id: x, title: X }
items:
- { number: 1, item: "b::q-1" }
- { number: 3, item: "b::q-2" }
"#,
);
assert!(a.validate().iter().any(|i| i.contains("not contiguous")));
}
#[test]
fn rejects_out_of_range_credit_overrides() {
let a = record(
r#"
assessment: { id: x, title: X }
items:
- { number: 1, item: "b::q-1", credit_overrides: { B: 1.5 } }
"#,
);
assert!(a.validate().iter().any(|i| i.contains("must be in [0, 1]")));
}
#[test]
fn blueprint_totals() {
let b: Blueprint = serde_yaml_ng::from_str(
r#"
level_counts: { 1: 6, 2: 10, 3: 12, 4: 8 }
bonus_counts: { 5: 2 }
"#,
)
.unwrap();
assert_eq!(b.scored_total(), 36);
assert_eq!(b.bonus_total(), 2);
}
}
+914
View File
@@ -0,0 +1,914 @@
// SPDX-License-Identifier: Prosperity-3.0.0
// Copyright Scientific Computing Studio
// Source: https://git.scient.ing/education/coursebank
//! The bank file: a collection of items scoped to a topic or a lecture.
//!
//! One bank per topic (or per lecture, if that suits how you teach) is the unit
//! of authoring. Banks are small enough to review in a pull request, they let
//! two people write questions without colliding, and `bank.scope` records what
//! the file is *for* so `coursebank catalog` can tell you that you have eleven
//! items on enzyme kinetics and none on regulation.
//!
//! Validation here is split in two on purpose. [`BankFile::validate`] checks what
//! must be true for the file to be usable at all: ids are unique, a keyed answer
//! exists, an approved item is fully specified, a level and its cognitive process
//! agree. The softer question of whether an item is *well written* lives in
//! [`crate::lint`], because those checks are advisory and you should be able to
//! ship a file that trips a few of them.
use std::collections::BTreeMap;
use std::path::Path;
use serde::{Deserialize, Serialize};
use crate::course::{CourseFile, SCHEMA_VERSION};
use crate::date::Date;
use crate::error::Result;
use crate::item::Item;
use crate::taxonomy::{Format, Level, Status};
use crate::yaml;
/// A whole bank file.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct BankFile {
/// Schema version this file targets.
#[serde(
default = "default_version",
deserialize_with = "yaml::flexible_string"
)]
pub schema_version: String,
/// Bank identity and scope.
pub bank: BankMeta,
/// Values applied to every item in the file that does not set its own.
#[serde(default)]
pub defaults: BankDefaults,
/// The items.
#[serde(default)]
pub items: Vec<Item>,
}
/// Bank identity and scope.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct BankMeta {
/// Stable id, unique across the course. Item ids are namespaced by it.
pub id: String,
/// Human title.
pub title: String,
/// What this bank covers.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub description: Option<String>,
/// What the bank is scoped to, so coverage can be reported against it.
#[serde(default)]
pub scope: Scope,
/// Who maintains it.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub maintainer: Option<String>,
/// When it was created.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub created: Option<Date>,
/// When it was last touched.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub updated: Option<Date>,
}
/// What a bank is scoped to.
///
/// A bank may be scoped by lecture, by objective, by topic, or by none of them.
/// Declaring the scope is what lets the catalog report *gaps*: it can only tell
/// you that lecture 12 has no Apply-level items if it knows lecture 12 is
/// supposed to be covered here.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Scope {
/// Lectures this bank draws from.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub lectures: Vec<String>,
/// Objectives this bank is responsible for covering.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub learning_objectives: Vec<String>,
/// Units this bank belongs to.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub units: Vec<String>,
/// Topics this bank is about.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub topics: Vec<String>,
}
/// Per-file defaults, so common metadata is written once.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct BankDefaults {
/// Default author for items in this file.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub author: Option<String>,
/// Default point value.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub points: Option<f64>,
/// Expected option count; the linter flags items that differ, since an
/// inconsistent option count across a form is itself a cue to students.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub options_per_item: Option<usize>,
/// Topics added to every item in the file.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub topics: Vec<String>,
/// Sources applied to items that declare none.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub lectures: Vec<String>,
}
impl BankFile {
/// Loads a bank file from disk.
///
/// # Arguments
///
/// * `path` - the YAML file.
///
/// # Returns
///
/// The parsed bank.
///
/// # Errors
///
/// Returns [`crate::error::Error::Io`] or [`crate::error::Error::Yaml`].
pub fn load(path: &Path) -> Result<BankFile> {
yaml::read(path)
}
/// Writes a bank file back out as YAML.
///
/// Round-tripping loses comments, which is why calibration is written by an
/// explicit `coursebank calibrate` step rather than as a side effect of
/// anything else: you should be able to see the diff it produces.
///
/// # Arguments
///
/// * `path` - the destination.
///
/// # Errors
///
/// Returns [`crate::error::Error::Io`] on a write failure.
pub fn save(&self, path: &Path) -> Result<()> {
yaml::write(path, self)
}
/// Applies file defaults to items that omit the corresponding field.
///
/// Called after loading so the rest of the crate never has to think about
/// defaults again.
pub fn apply_defaults(&mut self) {
let d = self.defaults.clone();
for item in &mut self.items {
if item.author.is_none() {
item.author = d.author.clone();
}
if item.points.is_none() {
item.points = d.points;
}
for t in &d.topics {
if !item.topics.contains(t) {
item.topics.push(t.clone());
}
}
if item.sources.is_empty() {
for lec in &d.lectures {
item.sources.push(crate::item::Source {
lecture: lec.clone(),
slides: Vec::new(),
readings: Vec::new(),
recording_seconds: None,
});
}
}
}
}
/// Loads a bank and applies its defaults.
///
/// # Arguments
///
/// * `path` - the YAML file.
///
/// # Returns
///
/// The parsed bank with defaults resolved.
///
/// # Errors
///
/// Propagates load errors.
pub fn load_resolved(path: &Path) -> Result<BankFile> {
let mut b = BankFile::load(path)?;
b.apply_defaults();
Ok(b)
}
/// Checks every invariant that must hold for the file to be usable.
///
/// Returns all problems rather than the first, so one run fixes one file.
/// When a course file is supplied, cross-file references are checked too.
///
/// # Arguments
///
/// * `course` - the course registry, for resolving objective and lecture
/// references. Pass `None` to check only what is local to the file.
///
/// # Returns
///
/// Every problem found, empty when the file is sound.
pub fn validate(&self, course: Option<&CourseFile>) -> Vec<String> {
let mut issues = Vec::new();
if self.bank.id.trim().is_empty() {
issues.push("bank.id is empty".into());
}
if self.bank.title.trim().is_empty() {
issues.push("bank.title is empty".into());
}
if let (Some(created), Some(updated)) = (self.bank.created, self.bank.updated) {
if updated < created {
issues.push(format!(
"bank.updated ({updated}) is before bank.created ({created})"
));
}
}
// Duplicate item ids inside the file.
let mut counts: BTreeMap<&str, usize> = BTreeMap::new();
for it in &self.items {
*counts.entry(it.id.as_str()).or_insert(0) += 1;
}
for (id, n) in &counts {
if *n > 1 {
issues.push(format!("duplicate item id `{id}` appears {n} times"));
}
}
if let Some(c) = course {
for lec in &self.bank.scope.lectures {
if !c.lectures.contains_key(lec) {
issues.push(format!("bank.scope: unknown lecture `{lec}`"));
}
}
for lo in &self.bank.scope.learning_objectives {
if !c.learning_objectives.contains_key(lo) {
issues.push(format!("bank.scope: unknown learning objective `{lo}`"));
}
}
}
for it in &self.items {
issues.extend(
validate_item(it, course, self.defaults.options_per_item)
.into_iter()
.map(|m| format!("{}: {m}", it.id)),
);
}
issues
}
/// Items that may be placed on a graded assessment.
///
/// # Returns
///
/// References to approved, unretired items.
pub fn assemblable(&self) -> Vec<&Item> {
self.items.iter().filter(|i| i.is_assemblable()).collect()
}
/// Counts of assemblable, non-bonus items by level.
///
/// This is the five-tuple you check a blueprint against.
///
/// # Returns
///
/// A map from level to count.
pub fn level_counts(&self) -> BTreeMap<Level, usize> {
let mut out: BTreeMap<Level, usize> = Level::ALL.iter().map(|l| (*l, 0)).collect();
for it in self.items.iter().filter(|i| i.is_assemblable() && !i.bonus) {
*out.entry(it.level).or_insert(0) += 1;
}
out
}
/// A skeleton bank file for `coursebank bank new`.
///
/// # Arguments
///
/// * `id` - the bank id.
/// * `title` - the bank title.
///
/// # Returns
///
/// A bank with no items.
pub fn skeleton(id: &str, title: &str) -> BankFile {
BankFile {
schema_version: SCHEMA_VERSION.to_string(),
bank: BankMeta {
id: id.to_string(),
title: title.to_string(),
description: None,
scope: Scope::default(),
maintainer: None,
created: Some(Date::today()),
updated: Some(Date::today()),
},
defaults: BankDefaults::default(),
items: Vec::new(),
}
}
}
/// Validates one item.
///
/// # Arguments
///
/// * `it` - the item.
/// * `course` - the course registry, when available.
/// * `expected_options` - the file's declared option count, when set.
///
/// # Returns
///
/// Problems found, without the item id prefix.
fn validate_item(
it: &Item,
course: Option<&CourseFile>,
expected_options: Option<usize>,
) -> Vec<String> {
let mut issues = Vec::new();
if it.id.trim().is_empty() {
issues.push("empty id".into());
}
if it.stem.trim().is_empty() {
issues.push("empty stem".into());
}
if it.version == 0 {
issues.push("version must be at least 1".into());
}
// --- options -----------------------------------------------------------
if it.options.len() < 2 {
issues.push(format!(
"needs at least 2 options, has {}",
it.options.len()
));
}
let mut seen: Vec<&str> = Vec::new();
for (i, o) in it.options.iter().enumerate() {
let pos = i + 1;
if o.text.trim().is_empty() {
issues.push(format!("option {pos}: empty text"));
}
let letter_ok = o.id.len() == 1
&& o.id
.chars()
.next()
.map(|c| c.is_ascii_uppercase() && c <= 'H')
.unwrap_or(false);
if !letter_ok {
issues.push(format!(
"option {pos}: id `{}` must be a single letter A through H",
o.id
));
}
if seen.contains(&o.id.as_str()) {
issues.push(format!("option {pos}: duplicate option id `{}`", o.id));
}
seen.push(&o.id);
let credit = o.credit();
if !(0.0..=1.0).contains(&credit) {
issues.push(format!(
"option {}: credit must be between 0 and 1, got {credit}",
o.id
));
}
// Partial credit must be argued for in writing, not remembered.
if o.is_partial() && o.defense.is_none() {
issues.push(format!(
"option {}: awards credit {credit} but gives no `defense`",
o.id
));
}
if o.is_partial() && !o.defensible {
issues.push(format!(
"option {}: awards credit {credit} but is not marked `defensible: true`",
o.id
));
}
if o.correct && credit == 0.0 {
issues.push(format!(
"option {}: keyed correct but earns no credit",
o.id
));
}
}
if let Some(n) = expected_options {
if it.options.len() != n && !it.options.is_empty() {
issues.push(format!(
"has {} options but the bank declares {n} per item",
it.options.len()
));
}
}
// --- key ---------------------------------------------------------------
let keys = it.key_indices();
match it.format {
Format::SingleBestAnswer => {
if keys.len() != 1 {
issues.push(format!(
"single_best_answer needs exactly one keyed option, has {}",
keys.len()
));
}
}
Format::MultipleResponse => {
if keys.is_empty() {
issues.push("multiple_response needs at least one keyed option".into());
}
if keys.len() == it.options.len() {
issues.push("multiple_response keys every option, so it asks nothing".into());
}
}
Format::TrueFalse => {
if it.options.len() != 2 {
issues.push(format!(
"true_false needs exactly 2 options, has {}",
it.options.len()
));
}
if keys.len() != 1 {
issues.push("true_false needs exactly one keyed option".into());
}
}
}
// --- level and process must agree -------------------------------------
if let Some(p) = it.cognitive_process {
if !it.level.allows(p) {
issues.push(format!(
"cognitive_process `{p}` belongs to level {} but the item is level {}",
p.level().code(),
it.level.code()
));
}
}
// --- design plausibility ----------------------------------------------
if let Some(d) = &it.design {
if let Some(x) = d.expected_difficulty {
if !(0.0..=1.0).contains(&x) {
issues.push(format!(
"design.expected_difficulty must be between 0 and 1, got {x}"
));
}
}
if let Some(t) = d.expected_time_seconds {
if t <= 0.0 {
issues.push(format!(
"design.expected_time_seconds must be positive, got {t}"
));
}
}
}
// --- calibration plausibility -----------------------------------------
if let Some(c) = &it.calibration {
if let Some(p) = c.p_value {
if !(0.0..=1.0).contains(&p) {
issues.push(format!(
"calibration.p_value must be between 0 and 1, got {p}"
));
}
}
if let Some(r) = c.point_biserial {
if !(-1.0..=1.0).contains(&r) {
issues.push(format!(
"calibration.point_biserial must be between -1 and 1, got {r}"
));
}
}
for letter in c.option_stats.keys() {
if it.option(letter).is_none() {
issues.push(format!(
"calibration.option_stats has `{letter}`, which is not an option of this item"
));
}
}
if let Some(irt) = &c.irt {
if irt.a <= 0.0 {
issues.push(format!("calibration.irt.a must be positive, got {}", irt.a));
}
if let Some(cp) = irt.c {
if !(0.0..1.0).contains(&cp) {
issues.push(format!("calibration.irt.c must be in [0, 1), got {cp}"));
}
}
}
}
// --- history must be coherent -----------------------------------------
let mut last_version = 0u32;
for (i, h) in it.history.iter().enumerate() {
if h.version <= last_version {
issues.push(format!(
"history entry {} has version {} which does not increase",
i + 1,
h.version
));
}
last_version = h.version;
}
if !it.history.is_empty() && last_version > it.version {
issues.push(format!(
"history records version {last_version} but the item says version {}",
it.version
));
}
// --- retirement -------------------------------------------------------
if it.retired.is_some() && it.status != Status::Retired {
issues.push(format!(
"has a `retired` block but status is `{}`",
it.status
));
}
// --- approval gate ----------------------------------------------------
// Approval is what permits an item onto a graded assessment, so it is the
// right place to require that the item is fully sourced and designed.
if it.status == Status::Approved {
if it.cognitive_process.is_none() {
issues.push("approved items must declare a cognitive_process".into());
}
if it.learning_objectives.is_empty() {
issues.push("approved items must reference at least one learning objective".into());
}
if it.sources.is_empty() {
issues.push("approved items must cite at least one source".into());
}
if it.design.is_none() {
issues.push("approved items must carry a design block".into());
}
}
// --- cross-file references --------------------------------------------
if let Some(c) = course {
for lo in &it.learning_objectives {
match c.learning_objectives.get(lo) {
None => issues.push(format!("unknown learning objective `{lo}`")),
Some(obj) => {
if let Some(ceiling) = obj.level_ceiling {
if it.level > ceiling {
issues.push(format!(
"level {} exceeds the ceiling {} declared for objective `{lo}`",
it.level.code(),
ceiling.code()
));
}
}
if !obj.assessed {
issues.push(format!(
"objective `{lo}` is marked `assessed: false` but this item measures it"
));
}
}
}
}
for s in &it.sources {
if !c.lectures.contains_key(&s.lecture) {
issues.push(format!("unknown lecture `{}`", s.lecture));
}
}
if let Some(st) = &it.stimulus {
if !c.stimuli.contains_key(st) {
issues.push(format!("unknown stimulus `{st}`"));
}
}
if let Some(floor) = c.policy.partial_credit_floor_level {
for o in &it.options {
if o.is_partial() && it.level < floor {
issues.push(format!(
"option {} awards partial credit at level {}, below the course floor of {}",
o.id,
it.level.code(),
floor.code()
));
}
}
}
if !c.policy.allow_partial_credit && it.options.iter().any(|o| o.is_partial()) {
issues.push("awards partial credit, which the course policy disallows".into());
}
}
issues
}
fn default_version() -> String {
SCHEMA_VERSION.to_string()
}
#[cfg(test)]
mod tests {
use super::*;
fn bank(items_yaml: &str) -> BankFile {
let src = format!("bank:\n id: b\n title: Bank\ndefaults: {{}}\nitems:\n{items_yaml}");
let mut b: BankFile = serde_yaml_ng::from_str(&src).expect("bank parses");
b.apply_defaults();
b
}
#[test]
fn sound_bank_validates_clean() {
let b = bank(
r#"
- id: q-a-001
status: draft
level: 1
stem: What is x?
options:
- { id: A, text: right, correct: true }
- { id: B, text: wrong }
- { id: C, text: wrong too }
"#,
);
assert!(b.validate(None).is_empty(), "{:?}", b.validate(None));
}
#[test]
fn catches_missing_and_multiple_keys() {
let b = bank(
r#"
- id: q-a-001
status: draft
level: 1
stem: s
options:
- { id: A, text: a }
- { id: B, text: b }
- id: q-a-002
status: draft
level: 1
format: single_best_answer
stem: s
options:
- { id: A, text: a, correct: true }
- { id: B, text: b, correct: true }
"#,
);
let issues = b.validate(None);
assert!(
issues
.iter()
.any(|i| i.contains("exactly one keyed option"))
);
assert_eq!(
issues
.iter()
.filter(|i| i.contains("exactly one keyed option"))
.count(),
2
);
}
#[test]
fn catches_duplicate_ids_and_letters() {
let b = bank(
r#"
- id: q-a-001
status: draft
level: 1
stem: s
options:
- { id: A, text: a, correct: true }
- { id: A, text: b }
- id: q-a-001
status: draft
level: 1
stem: s
options:
- { id: A, text: a, correct: true }
- { id: B, text: b }
"#,
);
let issues = b.validate(None);
assert!(issues.iter().any(|i| i.contains("duplicate item id")));
assert!(issues.iter().any(|i| i.contains("duplicate option id")));
}
#[test]
fn level_and_process_must_agree() {
let b = bank(
r#"
- id: q-a-001
status: draft
level: 3
cognitive_process: recall
stem: s
options:
- { id: A, text: a, correct: true }
- { id: B, text: b }
"#,
);
let issues = b.validate(None);
assert!(
issues.iter().any(|i| i.contains("belongs to level 1")),
"{issues:?}"
);
}
#[test]
fn approval_requires_full_specification() {
let b = bank(
r#"
- id: q-a-001
status: approved
level: 1
stem: s
options:
- { id: A, text: a, correct: true }
- { id: B, text: b }
"#,
);
let issues = b.validate(None);
for want in [
"cognitive_process",
"learning objective",
"source",
"design block",
] {
assert!(
issues.iter().any(|i| i.contains(want)),
"expected a complaint about {want}, got {issues:?}"
);
}
}
#[test]
fn partial_credit_needs_a_written_defense() {
let b = bank(
r#"
- id: q-a-001
status: draft
level: 5
stem: s
options:
- { id: A, text: a, correct: true }
- { id: B, text: b, credit: 0.5 }
"#,
);
let issues = b.validate(None);
assert!(issues.iter().any(|i| i.contains("no `defense`")));
assert!(issues.iter().any(|i| i.contains("defensible: true")));
}
#[test]
fn defaults_fill_in_items() {
let src = r#"
bank: { id: b, title: Bank }
defaults:
author: Alex
points: 1.5
topics: [kinetics]
lectures: [L11]
items:
- id: q-a-001
status: draft
level: 1
stem: s
options:
- { id: A, text: a, correct: true }
- { id: B, text: b }
- id: q-a-002
status: draft
level: 1
stem: s
author: Someone Else
topics: [kinetics]
sources: [{ lecture: L12 }]
options:
- { id: A, text: a, correct: true }
- { id: B, text: b }
"#;
let mut b: BankFile = serde_yaml_ng::from_str(src).unwrap();
b.apply_defaults();
assert_eq!(b.items[0].author.as_deref(), Some("Alex"));
assert_eq!(b.items[0].points, Some(1.5));
assert_eq!(b.items[0].topics, vec!["kinetics"]);
assert_eq!(b.items[0].sources[0].lecture, "L11");
// Explicit values win, and topics are not duplicated.
assert_eq!(b.items[1].author.as_deref(), Some("Someone Else"));
assert_eq!(b.items[1].topics, vec!["kinetics"]);
assert_eq!(b.items[1].sources[0].lecture, "L12");
}
#[test]
fn cross_file_references_are_checked_against_the_course() {
let course: CourseFile = serde_yaml_ng::from_str(
r#"
course: { code: X, title: Y, term: Z }
lectures:
L11: { title: Kinetics }
learning_objectives:
lo-known: { text: Do the thing, level_ceiling: 2 }
"#,
)
.unwrap();
let b = bank(
r#"
- id: q-a-001
status: draft
level: 4
stem: s
learning_objectives: [lo-known, lo-unknown]
sources: [{ lecture: L99 }]
options:
- { id: A, text: a, correct: true }
- { id: B, text: b }
"#,
);
let issues = b.validate(Some(&course));
assert!(
issues
.iter()
.any(|i| i.contains("unknown learning objective `lo-unknown`"))
);
assert!(issues.iter().any(|i| i.contains("unknown lecture `L99`")));
assert!(
issues.iter().any(|i| i.contains("exceeds the ceiling")),
"{issues:?}"
);
}
#[test]
fn history_versions_must_increase() {
let b = bank(
r#"
- id: q-a-001
version: 2
status: draft
level: 1
stem: s
options:
- { id: A, text: a, correct: true }
- { id: B, text: b }
history:
- { version: 2, date: 2026-01-01, change: second }
- { version: 1, date: 2026-01-02, change: first }
"#,
);
let issues = b.validate(None);
assert!(issues.iter().any(|i| i.contains("does not increase")));
}
#[test]
fn level_counts_exclude_drafts_and_bonuses() {
let b = bank(
r#"
- id: q-a-001
status: approved
level: 1
cognitive_process: recall
stem: s
learning_objectives: [lo]
sources: [{ lecture: L1 }]
design: { expected_difficulty: 0.8 }
options:
- { id: A, text: a, correct: true }
- { id: B, text: b }
- id: q-a-002
status: draft
level: 1
stem: s
options:
- { id: A, text: a, correct: true }
- { id: B, text: b }
- id: q-a-003
status: approved
level: 5
cognitive_process: generate
bonus: true
stem: s
learning_objectives: [lo]
sources: [{ lecture: L1 }]
design: { expected_difficulty: 0.3 }
options:
- { id: A, text: a, correct: true }
- { id: B, text: b }
"#,
);
let counts = b.level_counts();
assert_eq!(counts[&Level::Remember], 1);
assert_eq!(counts[&Level::Create], 0, "bonus items are not scored");
assert_eq!(b.assemblable().len(), 2);
}
}
+686
View File
@@ -0,0 +1,686 @@
// SPDX-License-Identifier: Prosperity-3.0.0
// Copyright Scientific Computing Studio
// Source: https://git.scient.ing/education/coursebank
//! Loading a whole course at once, and reporting on what it contains.
//!
//! A [`Catalog`] is every bank in a course, indexed so that an item can be found
//! by its global id (`bank::item`), and so that questions like "how many Apply
//! level items do I have on lecture 12" have a cheap answer.
//!
//! The global id is the join key for everything downstream: assessment records
//! reference it, response tables carry it, and usage history is keyed by it. Bank
//! ids therefore have to be unique across a course, which the catalog enforces
//! at load time rather than letting a silent collision merge two items.
//!
//! The coverage report is the part that changes how you write. It is easy to
//! accumulate forty recall items and believe a topic is covered; a table showing
//! that eleven objectives have no item above level 1 is harder to ignore.
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use crate::assessment::AssessmentFile;
use crate::bank::BankFile;
use crate::course::CourseFile;
use crate::error::{Error, Result};
use crate::item::Item;
use crate::layout::Layout;
use crate::taxonomy::{Level, Status};
use crate::yaml;
/// One item plus everything needed to locate it again.
#[derive(Debug, Clone)]
pub struct Entry {
/// The globally unique id, `bank::item`.
pub uid: String,
/// The bank id.
pub bank: String,
/// The file the item came from.
pub path: PathBuf,
/// Position within the bank file, for stable ordering.
pub index: usize,
/// The item itself.
pub item: Item,
}
impl Entry {
/// The item's point value, resolving against the course policy.
///
/// # Arguments
///
/// * `course` - the course whose policy supplies the default.
///
/// # Returns
///
/// The point value.
pub fn points(&self, course: &CourseFile) -> f64 {
self.item.points(course.policy.points_per_item)
}
}
/// Every bank in a course, indexed.
#[derive(Debug, Clone)]
pub struct Catalog {
/// The course registry.
pub course: CourseFile,
/// The resolved directory layout.
pub layout: Layout,
/// Every item, in stable order: by bank id, then by position in the file.
pub entries: Vec<Entry>,
/// Bank metadata by bank id.
pub banks: BTreeMap<String, crate::bank::BankMeta>,
/// Map from global id to index into `entries`.
index: BTreeMap<String, usize>,
}
impl Catalog {
/// Loads a whole course from its directory.
///
/// # Arguments
///
/// * `root` - the course directory containing `course.yaml` and `banks/`.
///
/// # Returns
///
/// The catalog.
///
/// # Errors
///
/// Returns a load error for the course file or any bank, and
/// [`Error::Invalid`] when two banks share an id or two items share a global
/// id, since either makes the join key ambiguous.
pub fn load(root: &Path) -> Result<Catalog> {
let layout = Layout::new(root);
let course = CourseFile::load(&layout.course_file())?;
let mut catalog = Catalog {
course,
layout,
entries: Vec::new(),
banks: BTreeMap::new(),
index: BTreeMap::new(),
};
let mut problems = Vec::new();
let mut files = yaml::list_yaml(&catalog.layout.banks())?;
files.sort();
for path in files {
let bank = BankFile::load_resolved(&path)?;
let bank_id = bank.bank.id.clone();
if let Some(existing) = catalog.banks.get(&bank_id) {
problems.push(format!(
"bank id `{bank_id}` is used by two files (`{}` and `{}`)",
existing.title, bank.bank.title
));
continue;
}
catalog.banks.insert(bank_id.clone(), bank.bank.clone());
for (i, item) in bank.items.into_iter().enumerate() {
let uid = format!("{bank_id}::{}", item.id);
if catalog.index.contains_key(&uid) {
problems.push(format!("duplicate global item id `{uid}`"));
continue;
}
catalog.index.insert(uid.clone(), catalog.entries.len());
catalog.entries.push(Entry {
uid,
bank: bank_id.clone(),
path: path.clone(),
index: i,
item,
});
}
}
if !problems.is_empty() {
return Err(Error::Invalid(problems));
}
Ok(catalog)
}
/// Looks up an item by global id.
///
/// # Arguments
///
/// * `uid` - the global id, `bank::item`.
///
/// # Returns
///
/// The entry, or `None`.
pub fn get(&self, uid: &str) -> Option<&Entry> {
self.index.get(uid).map(|i| &self.entries[*i])
}
/// Looks up an item by global id, erroring when absent.
///
/// # Arguments
///
/// * `uid` - the global id.
///
/// # Returns
///
/// The entry.
///
/// # Errors
///
/// Returns [`Error::Unresolved`] when no such item exists.
pub fn require(&self, uid: &str) -> Result<&Entry> {
self.get(uid).ok_or_else(|| Error::Unresolved {
kind: "item",
id: uid.to_string(),
context: None,
})
}
/// Resolves a possibly-unqualified id to a global id.
///
/// Typing `q-mm-kinetics-001` on the command line should work when that id is
/// unambiguous across the course, because remembering which bank a question
/// lives in is exactly the sort of bookkeeping this tool exists to remove.
///
/// # Arguments
///
/// * `id` - a global id, or a bare item id.
///
/// # Returns
///
/// The global id.
///
/// # Errors
///
/// Returns [`Error::Unresolved`] when nothing matches, or [`Error::Usage`]
/// when a bare id matches items in more than one bank.
pub fn resolve(&self, id: &str) -> Result<String> {
if self.index.contains_key(id) {
return Ok(id.to_string());
}
let matches: Vec<&Entry> = self.entries.iter().filter(|e| e.item.id == id).collect();
match matches.len() {
0 => Err(Error::Unresolved {
kind: "item",
id: id.to_string(),
context: None,
}),
1 => Ok(matches[0].uid.clone()),
_ => Err(Error::usage(format!(
"`{id}` is ambiguous; it exists in {}. Use the full `bank::item` form.",
matches
.iter()
.map(|e| e.bank.as_str())
.collect::<Vec<_>>()
.join(", ")
))),
}
}
/// Every item that may be placed on a graded assessment.
///
/// # Returns
///
/// References to approved, unretired entries.
pub fn assemblable(&self) -> Vec<&Entry> {
self.entries
.iter()
.filter(|e| e.item.is_assemblable())
.collect()
}
/// Validates every bank against the course registry.
///
/// # Returns
///
/// Problems, prefixed with the file they came from.
pub fn validate(&self) -> Result<Vec<String>> {
let mut issues: Vec<String> = self
.course
.validate()
.into_iter()
.map(|m| format!("course.yaml: {m}"))
.collect();
for path in yaml::list_yaml(&self.layout.banks())? {
let bank = BankFile::load_resolved(&path)?;
let name = path
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("?")
.to_string();
issues.extend(
bank.validate(Some(&self.course))
.into_iter()
.map(|m| format!("{name}: {m}")),
);
}
Ok(issues)
}
/// Validates a record's internal invariants, then its references against this
/// catalog: unknown items, keys that drifted, and fingerprints showing the
/// item was reworded since it was administered.
pub fn validate_record(&self, record: &AssessmentFile) -> Vec<String> {
let mut issues = record.validate();
for p in &record.items {
match self.get(&p.item) {
None => issues.push(format!("question {}: unknown item `{}`", p.number, p.item)),
Some(entry) => {
if let Some(fp) = &p.fingerprint {
if *fp != entry.item.fingerprint() {
issues.push(format!(
"question {} ({}): the item has been edited since this assessment; \
statistics from this administration describe the older wording",
p.number, p.item
));
}
}
if !p.key.is_empty() && p.key != entry.item.key_letters() {
issues.push(format!(
"question {} ({}): the recorded key {:?} differs from the item's current key {:?}",
p.number, p.item, p.key, entry.item.key_letters()));
}
}
}
}
issues
}
/// Estimated working time for a record, in minutes.
pub fn estimated_minutes(&self, record: &AssessmentFile) -> f64 {
let seconds: f64 = record
.items
.iter()
.filter_map(|p| self.get(&p.item))
.map(|e| e.item.expected_seconds())
.sum();
seconds / 60.0
}
/// Counts of assemblable, non-bonus items by level.
///
/// # Returns
///
/// A map from level to count, with every level present.
pub fn level_counts(&self) -> BTreeMap<Level, usize> {
let mut out: BTreeMap<Level, usize> = Level::ALL.iter().map(|l| (*l, 0)).collect();
for e in self.assemblable().iter().filter(|e| !e.item.bonus) {
*out.entry(e.item.level).or_insert(0) += 1;
}
out
}
/// Counts of items by workflow status, over the whole course.
///
/// # Returns
///
/// A map from status to count.
pub fn status_counts(&self) -> BTreeMap<Status, usize> {
let mut out: BTreeMap<Status, usize> = BTreeMap::new();
for e in &self.entries {
*out.entry(e.item.status).or_insert(0) += 1;
}
out
}
/// Items that measure a given objective.
///
/// # Arguments
///
/// * `objective` - the objective id.
///
/// # Returns
///
/// Matching entries.
pub fn by_objective(&self, objective: &str) -> Vec<&Entry> {
self.entries
.iter()
.filter(|e| e.item.learning_objectives.iter().any(|o| o == objective))
.collect()
}
/// Items sourced to a given lecture.
///
/// # Arguments
///
/// * `lecture` - the lecture id.
///
/// # Returns
///
/// Matching entries.
pub fn by_lecture(&self, lecture: &str) -> Vec<&Entry> {
self.entries
.iter()
.filter(|e| e.item.sources.iter().any(|s| s.lecture == lecture))
.collect()
}
/// Items carrying a given topic tag.
///
/// # Arguments
///
/// * `topic` - the topic tag.
///
/// # Returns
///
/// Matching entries.
pub fn by_topic(&self, topic: &str) -> Vec<&Entry> {
self.entries
.iter()
.filter(|e| e.item.topics.iter().any(|t| t == topic))
.collect()
}
/// Every topic tag in use, with counts.
///
/// # Returns
///
/// A map from topic to item count.
pub fn topics(&self) -> BTreeMap<String, usize> {
let mut out: BTreeMap<String, usize> = BTreeMap::new();
for e in &self.entries {
for t in &e.item.topics {
*out.entry(t.clone()).or_insert(0) += 1;
}
}
out
}
/// Builds the coverage report.
///
/// # Returns
///
/// One row per assessed objective plus a list of course-wide gaps.
pub fn coverage(&self) -> Coverage {
let mut rows = Vec::new();
for id in self.course.objectives_in_order() {
let obj = &self.course.learning_objectives[&id];
if !obj.assessed {
continue;
}
let items = self.by_objective(&id);
let usable: Vec<&&Entry> = items.iter().filter(|e| e.item.is_assemblable()).collect();
let mut levels: BTreeSet<Level> = BTreeSet::new();
for e in &usable {
levels.insert(e.item.level);
}
rows.push(CoverageRow {
objective: id.clone(),
text: obj.text.clone(),
unit: obj.unit.clone(),
total: items.len(),
assemblable: usable.len(),
max_level: levels.iter().next_back().copied(),
levels: levels.into_iter().collect(),
ceiling: obj.level_ceiling,
});
}
let mut gaps = Vec::new();
for row in &rows {
if row.total == 0 {
gaps.push(Gap::Uncovered(row.objective.clone()));
} else if row.assemblable == 0 {
gaps.push(Gap::NoApprovedItems(row.objective.clone()));
} else if row.assemblable == 1 {
gaps.push(Gap::SingleItem(row.objective.clone()));
}
// An objective assessed only at the recall level is the most common
// and most consequential blind spot: it looks covered in a count and
// is not covered in fact.
if row.assemblable > 0 && row.max_level == Some(Level::Remember) {
if let Some(ceiling) = row.ceiling {
if ceiling > Level::Remember {
gaps.push(Gap::RecallOnly(row.objective.clone()));
}
} else {
gaps.push(Gap::RecallOnly(row.objective.clone()));
}
}
}
for lec_id in self.course.lectures.keys() {
if self
.by_lecture(lec_id)
.iter()
.filter(|e| e.item.is_assemblable())
.count()
== 0
{
gaps.push(Gap::LectureUnassessed(lec_id.clone()));
}
}
// Items with no objective at all cannot appear in any student report.
for e in &self.entries {
if e.item.learning_objectives.is_empty() && e.item.is_assemblable() {
gaps.push(Gap::ItemWithoutObjective(e.uid.clone()));
}
}
Coverage { rows, gaps }
}
}
/// The coverage report.
#[derive(Debug, Clone)]
pub struct Coverage {
/// One row per assessed objective.
pub rows: Vec<CoverageRow>,
/// Course-wide gaps worth acting on.
pub gaps: Vec<Gap>,
}
/// Coverage of one objective.
#[derive(Debug, Clone)]
pub struct CoverageRow {
/// The objective id.
pub objective: String,
/// The objective text.
pub text: String,
/// The unit it belongs to.
pub unit: Option<String>,
/// Items referencing it, at any status.
pub total: usize,
/// Items that could actually be used.
pub assemblable: usize,
/// The highest level assessed.
pub max_level: Option<Level>,
/// Every level assessed.
pub levels: Vec<Level>,
/// The declared ceiling, when set.
pub ceiling: Option<Level>,
}
/// A specific, actionable hole in the item pool.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Gap {
/// An assessed objective with no items at all.
Uncovered(String),
/// An objective whose items are all drafts or retired.
NoApprovedItems(String),
/// An objective resting on a single item, so one bad item hides it entirely.
SingleItem(String),
/// An objective assessed only at the recall level despite a higher ceiling.
RecallOnly(String),
/// A lecture with no usable items.
LectureUnassessed(String),
/// A usable item that references no objective, so it can never appear in a
/// student report.
ItemWithoutObjective(String),
}
impl Gap {
/// A one-line description.
pub fn message(&self) -> String {
match self {
Gap::Uncovered(id) => format!("objective `{id}` has no items"),
Gap::NoApprovedItems(id) => {
format!("objective `{id}` has items but none are approved")
}
Gap::SingleItem(id) => {
format!("objective `{id}` rests on a single item; one bad item hides it")
}
Gap::RecallOnly(id) => {
format!("objective `{id}` is assessed only at level 1")
}
Gap::LectureUnassessed(id) => format!("lecture `{id}` has no usable items"),
Gap::ItemWithoutObjective(uid) => {
format!("item `{uid}` has no learning objective, so it cannot appear in a report")
}
}
}
/// How much attention the gap deserves.
pub fn severity(&self) -> Severity {
match self {
Gap::Uncovered(_) | Gap::NoApprovedItems(_) => Severity::High,
Gap::RecallOnly(_) | Gap::ItemWithoutObjective(_) => Severity::Medium,
Gap::SingleItem(_) | Gap::LectureUnassessed(_) => Severity::Low,
}
}
}
/// How much attention a finding deserves.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum Severity {
/// Worth knowing.
Low,
/// Worth scheduling.
Medium,
/// Worth fixing before the next assessment.
High,
}
impl Severity {
/// A short label for report output.
pub fn label(self) -> &'static str {
match self {
Severity::Low => "low",
Severity::Medium => "medium",
Severity::High => "high",
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn write_course(dir: &Path, extra_lo: &str) {
std::fs::create_dir_all(dir.join("banks")).unwrap();
std::fs::write(
dir.join("course.yaml"),
format!(
r#"
course: {{ code: TEST 101, title: Testing, term: Fall 2026 }}
lectures:
L01: {{ title: One }}
L02: {{ title: Two }}
learning_objectives:
lo-covered: {{ text: Covered objective, lectures: [L01] }}
lo-bare: {{ text: Uncovered objective, lectures: [L02] }}
{extra_lo}
"#
),
)
.unwrap();
}
fn write_bank(dir: &Path, name: &str, body: &str) {
std::fs::write(dir.join("banks").join(name), body).unwrap();
}
fn tmp(tag: &str) -> PathBuf {
let p = std::env::temp_dir().join(format!("coursebank-test-{tag}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&p);
std::fs::create_dir_all(&p).unwrap();
p
}
const APPROVED: &str = r#"
bank:
id: b1
title: First bank
items:
- id: q-x-001
status: approved
level: 1
cognitive_process: recall
stem: What is x?
learning_objectives: [lo-covered]
sources: [{ lecture: L01 }]
design: { expected_difficulty: 0.8 }
options:
- { id: A, text: right, correct: true }
- { id: B, text: wrong }
"#;
#[test]
fn loads_and_indexes_items() {
let dir = tmp("load");
write_course(&dir, "");
write_bank(&dir, "b1.yaml", APPROVED);
let cat = Catalog::load(&dir).expect("catalog loads");
assert_eq!(cat.entries.len(), 1);
assert_eq!(cat.entries[0].uid, "b1::q-x-001");
assert!(cat.get("b1::q-x-001").is_some());
assert_eq!(cat.resolve("q-x-001").unwrap(), "b1::q-x-001");
assert!(cat.validate().unwrap().is_empty());
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn duplicate_bank_ids_are_fatal() {
let dir = tmp("dupbank");
write_course(&dir, "");
write_bank(&dir, "a.yaml", APPROVED);
write_bank(&dir, "b.yaml", APPROVED);
let err = Catalog::load(&dir).expect_err("must reject colliding bank ids");
assert!(format!("{err}").contains("two files"));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn ambiguous_bare_ids_are_rejected() {
let dir = tmp("ambig");
write_course(&dir, "");
write_bank(&dir, "a.yaml", APPROVED);
write_bank(&dir, "b.yaml", &APPROVED.replace("id: b1", "id: b2"));
let cat = Catalog::load(&dir).expect("distinct banks load");
assert_eq!(cat.entries.len(), 2);
let err = cat.resolve("q-x-001").expect_err("bare id is ambiguous");
assert!(format!("{err}").contains("ambiguous"));
// The fully qualified form still works.
assert_eq!(cat.resolve("b2::q-x-001").unwrap(), "b2::q-x-001");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn coverage_finds_real_gaps() {
let dir = tmp("coverage");
write_course(&dir, "");
write_bank(&dir, "b1.yaml", APPROVED);
let cat = Catalog::load(&dir).unwrap();
let cov = cat.coverage();
assert_eq!(cov.rows.len(), 2);
assert!(cov.gaps.contains(&Gap::Uncovered("lo-bare".into())));
assert!(cov.gaps.contains(&Gap::SingleItem("lo-covered".into())));
assert!(cov.gaps.contains(&Gap::RecallOnly("lo-covered".into())));
assert!(cov.gaps.contains(&Gap::LectureUnassessed("L02".into())));
assert_eq!(Gap::Uncovered("x".into()).severity(), Severity::High);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn recall_only_respects_a_recall_ceiling() {
let dir = tmp("ceiling");
// An objective you only ever intend to test at level 1 is not a gap.
write_course(&dir, " lo-ceil: { text: Just recall, level_ceiling: 1 }");
write_bank(&dir, "b1.yaml", &APPROVED.replace("lo-covered", "lo-ceil"));
let cat = Catalog::load(&dir).unwrap();
let cov = cat.coverage();
assert!(!cov.gaps.contains(&Gap::RecallOnly("lo-ceil".into())));
let _ = std::fs::remove_dir_all(&dir);
}
}
+709
View File
@@ -0,0 +1,709 @@
// SPDX-License-Identifier: Prosperity-3.0.0
// Copyright Scientific Computing Studio
// Source: https://git.scient.ing/education/coursebank
//! The course file: identity plus the registries every bank references.
//!
//! Learning objectives and lectures are declared once, in `course.yaml`, and
//! referenced by id from items. That is the single most load-bearing decision in
//! the schema. It means an objective's wording lives in exactly one place, so
//! rewording it updates every report; it means a report can name what a student
//! missed by objective rather than by question number; and it means a dangling
//! reference is a hard error instead of a silently misspelled string that splits
//! your coverage table into two near-identical rows.
//!
//! The course file also declares the term. Items live across terms, so the term
//! belongs to the course and the administration, never to the item.
use std::collections::BTreeMap;
use std::path::Path;
use serde::{Deserialize, Serialize};
use crate::date::Date;
use crate::error::{Error, Result};
use crate::taxonomy::Level;
use crate::yaml;
/// The schema version this build of the tool writes.
pub const SCHEMA_VERSION: &str = "1.0";
/// The canonical file name inside a course directory.
pub const COURSE_FILE: &str = "course.yaml";
/// A whole course file.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct CourseFile {
/// Schema version this file targets.
#[serde(
default = "default_version",
deserialize_with = "yaml::flexible_string"
)]
pub schema_version: String,
/// Course identity.
pub course: Course,
/// Grading and assembly conventions that apply course-wide.
#[serde(default)]
pub policy: Policy,
/// Units or modules, ordered as taught.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub units: Vec<Unit>,
/// Lectures or sessions, keyed by id such as `L11`.
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub lectures: BTreeMap<String, Lecture>,
/// Learning objectives, keyed by id such as `lo-mm-kinetics`.
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub learning_objectives: BTreeMap<String, Objective>,
/// Shared stimuli for case-based testlets, keyed by id.
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub stimuli: BTreeMap<String, Stimulus>,
}
/// Course identity.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Course {
/// Catalog code, e.g. `BIOSC 1540`.
pub code: String,
/// Full title.
pub title: String,
/// The term this instance runs in, e.g. `Spring 2026`.
pub term: String,
/// Granting institution.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub institution: Option<String>,
/// Instructors of record.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub instructors: Vec<String>,
/// A short slug used in generated file names; derived from `code` if absent.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub slug: Option<String>,
}
impl Course {
/// A filesystem-safe short name for this course.
///
/// # Returns
///
/// The declared slug, or one derived from the course code.
pub fn slug(&self) -> String {
match &self.slug {
Some(s) => s.clone(),
None => slugify(&self.code),
}
}
}
/// Course-wide conventions, so they are stated once rather than per assessment.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Policy {
/// Default points per scored item.
#[serde(default = "one")]
pub points_per_item: f64,
/// Default number of options an item should offer.
#[serde(default = "four")]
pub options_per_item: usize,
/// Levels that may carry bonus rather than scored items.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub bonus_levels: Vec<Level>,
/// Whether partial credit for defensible distractors is permitted at all.
#[serde(default = "yes")]
pub allow_partial_credit: bool,
/// The lowest level at which a defensible wrong answer is considered
/// plausible. Awarding credit below this is flagged, because at the recall
/// level a "reasonable wrong answer" usually means the item is unclear.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub partial_credit_floor_level: Option<Level>,
/// Mastery threshold for objective-level reporting, as a proportion.
#[serde(default = "mastery_default")]
pub mastery_threshold: f64,
/// The fewest items on an objective before a report will call it mastered.
#[serde(default = "two_usize")]
pub min_items_for_mastery: usize,
}
impl Default for Policy {
fn default() -> Policy {
Policy {
points_per_item: 1.0,
options_per_item: 4,
bonus_levels: Vec::new(),
allow_partial_credit: true,
partial_credit_floor_level: None,
mastery_threshold: mastery_default(),
min_items_for_mastery: 2,
}
}
}
/// A unit or module of the course.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Unit {
/// Stable id, referenced by lectures and objectives.
pub id: String,
/// Human title.
pub title: String,
/// Optional longer description.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub description: Option<String>,
}
/// A lecture or class session.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Lecture {
/// Human title.
pub title: String,
/// The date it ran, when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub date: Option<Date>,
/// The unit it belongs to.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub unit: Option<String>,
/// Where the slides live, for study guidance in student reports.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub slides_url: Option<String>,
/// Assigned readings for the session.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub readings: Vec<String>,
}
/// A learning objective.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Objective {
/// The objective as you would state it to students. Reports quote this
/// verbatim, so write it in the second person and start with a verb.
pub text: String,
/// The unit it belongs to.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub unit: Option<String>,
/// The lectures that develop it.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub lectures: Vec<String>,
/// The highest level you intend to assess this objective at. Assembling an
/// item above the ceiling is a warning: either the item overreaches or the
/// ceiling needs raising.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub level_ceiling: Option<Level>,
/// Objectives that must be secure before this one is reachable. Student
/// reports walk this backwards to suggest where to start reviewing.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub prerequisites: Vec<String>,
/// Free-form tags.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub tags: Vec<String>,
/// Whether this objective is assessed at all, or is aspirational.
#[serde(default = "yes")]
pub assessed: bool,
}
/// A shared stem or vignette used by several items.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Stimulus {
/// The vignette body, in the markup described in `docs/AUTHORING.md`.
pub body: String,
/// An image or data file to reproduce alongside it.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub asset: Option<String>,
/// A caption for the asset.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub caption: Option<String>,
}
impl CourseFile {
/// Loads a course file from disk without validating cross-references.
///
/// # Arguments
///
/// * `path` - path to `course.yaml`.
///
/// # Returns
///
/// The parsed course file.
///
/// # Errors
///
/// Returns [`Error::Io`] if unreadable and [`Error::Yaml`] if it does not
/// match the schema. Unknown keys are errors, so a misspelled field is
/// caught rather than ignored.
pub fn load(path: &Path) -> Result<CourseFile> {
yaml::read(path)
}
/// Finds and loads the course file for a course directory.
///
/// # Arguments
///
/// * `dir` - the course directory.
///
/// # Returns
///
/// The parsed course file.
///
/// # Errors
///
/// Propagates load errors, including absence of `course.yaml`.
pub fn load_dir(dir: &Path) -> Result<CourseFile> {
CourseFile::load(&dir.join(COURSE_FILE))
}
/// Writes the course file back out as YAML.
///
/// # Arguments
///
/// * `path` - destination path.
///
/// # Errors
///
/// Returns [`Error::Io`] on a write failure.
pub fn save(&self, path: &Path) -> Result<()> {
yaml::write(path, self)
}
/// Checks internal consistency of the registries.
///
/// # Returns
///
/// Every problem found, empty when the file is sound.
pub fn validate(&self) -> Vec<String> {
let mut issues = Vec::new();
if self.course.code.trim().is_empty() {
issues.push("course.code is empty".into());
}
if self.course.title.trim().is_empty() {
issues.push("course.title is empty".into());
}
if self.course.term.trim().is_empty() {
issues.push("course.term is empty".into());
}
if !(0.0..=1.0).contains(&self.policy.mastery_threshold) {
issues.push(format!(
"policy.mastery_threshold must be between 0 and 1, got {}",
self.policy.mastery_threshold
));
}
let unit_ids: Vec<&String> = self.units.iter().map(|u| &u.id).collect();
let mut unit_counts: BTreeMap<&str, usize> = BTreeMap::new();
for u in &self.units {
*unit_counts.entry(u.id.as_str()).or_insert(0) += 1;
}
for (id, n) in &unit_counts {
if *n > 1 {
issues.push(format!("units: duplicate id `{id}` declared {n} times"));
}
}
for (id, lec) in &self.lectures {
if lec.title.trim().is_empty() {
issues.push(format!("lecture `{id}`: empty title"));
}
if let Some(u) = &lec.unit {
if !unit_ids.contains(&u) {
issues.push(format!("lecture `{id}`: unknown unit `{u}`"));
}
}
}
for (id, lo) in &self.learning_objectives {
if lo.text.trim().is_empty() {
issues.push(format!("objective `{id}`: empty text"));
}
if let Some(u) = &lo.unit {
if !unit_ids.contains(&u) {
issues.push(format!("objective `{id}`: unknown unit `{u}`"));
}
}
for lec in &lo.lectures {
if !self.lectures.contains_key(lec) {
issues.push(format!("objective `{id}`: unknown lecture `{lec}`"));
}
}
for pre in &lo.prerequisites {
if !self.learning_objectives.contains_key(pre) {
issues.push(format!(
"objective `{id}`: unknown prerequisite objective `{pre}`"
));
}
if pre == id {
issues.push(format!("objective `{id}`: lists itself as a prerequisite"));
}
}
}
issues.extend(self.prerequisite_cycles());
issues
}
/// Detects cycles in the objective prerequisite graph.
///
/// A cycle would make a study-order suggestion loop forever, so it is worth
/// catching at validation time rather than at report time.
///
/// # Returns
///
/// One message per objective that participates in a cycle.
fn prerequisite_cycles(&self) -> Vec<String> {
// Iterative depth-first search with three colors.
#[derive(PartialEq, Clone, Copy)]
enum Mark {
White,
Gray,
Black,
}
let mut color: BTreeMap<&String, Mark> = self
.learning_objectives
.keys()
.map(|k| (k, Mark::White))
.collect();
let mut issues = Vec::new();
for root in self.learning_objectives.keys() {
if color[root] != Mark::White {
continue;
}
let mut stack: Vec<(&String, usize)> = vec![(root, 0)];
color.insert(root, Mark::Gray);
while let Some((node, idx)) = stack.pop() {
let empty: Vec<String> = Vec::new();
let pres = self
.learning_objectives
.get(node)
.map(|o| &o.prerequisites)
.unwrap_or(&empty);
if idx < pres.len() {
stack.push((node, idx + 1));
if let Some((key, _)) = self.learning_objectives.get_key_value(&pres[idx]) {
match color[key] {
Mark::Gray => issues.push(format!(
"objective `{node}`: prerequisite cycle through `{key}`"
)),
Mark::White => {
color.insert(key, Mark::Gray);
stack.push((key, 0));
}
Mark::Black => {}
}
}
} else {
color.insert(node, Mark::Black);
}
}
}
issues
}
/// Looks up an objective, erroring on a dangling reference.
///
/// # Arguments
///
/// * `id` - the objective id.
/// * `context` - what referenced it, for the error message.
///
/// # Returns
///
/// The objective.
///
/// # Errors
///
/// Returns [`Error::Unresolved`] when the id is not registered.
pub fn objective(&self, id: &str, context: &str) -> Result<&Objective> {
self.learning_objectives
.get(id)
.ok_or_else(|| Error::Unresolved {
kind: "learning objective",
id: id.to_string(),
context: Some(context.to_string()),
})
}
/// Looks up a lecture, erroring on a dangling reference.
///
/// # Arguments
///
/// * `id` - the lecture id.
/// * `context` - what referenced it, for the error message.
///
/// # Returns
///
/// The lecture.
///
/// # Errors
///
/// Returns [`Error::Unresolved`] when the id is not registered.
pub fn lecture(&self, id: &str, context: &str) -> Result<&Lecture> {
self.lectures.get(id).ok_or_else(|| Error::Unresolved {
kind: "lecture",
id: id.to_string(),
context: Some(context.to_string()),
})
}
/// The objective text, or the bare id when unregistered.
///
/// Report rendering uses this so a missing objective degrades to a readable
/// label instead of failing a whole report.
///
/// # Arguments
///
/// * `id` - the objective id.
///
/// # Returns
///
/// The display text.
pub fn objective_text(&self, id: &str) -> String {
self.learning_objectives
.get(id)
.map(|o| o.text.clone())
.unwrap_or_else(|| id.to_string())
}
/// Objectives in a stable teaching order: by unit as declared, then by id.
///
/// # Returns
///
/// Objective ids in report order.
pub fn objectives_in_order(&self) -> Vec<String> {
let unit_rank: BTreeMap<&str, usize> = self
.units
.iter()
.enumerate()
.map(|(i, u)| (u.id.as_str(), i))
.collect();
let mut ids: Vec<&String> = self.learning_objectives.keys().collect();
ids.sort_by_key(|id| {
let lo = &self.learning_objectives[*id];
let rank = lo
.unit
.as_deref()
.and_then(|u| unit_rank.get(u).copied())
.unwrap_or(usize::MAX);
(rank, (*id).clone())
});
ids.into_iter().cloned().collect()
}
/// A skeleton course file for `coursebank init`.
///
/// # Arguments
///
/// * `code` - the course code.
/// * `title` - the course title.
/// * `term` - the term.
///
/// # Returns
///
/// A minimal but valid course file.
pub fn skeleton(code: &str, title: &str, term: &str) -> CourseFile {
let mut lectures = BTreeMap::new();
lectures.insert(
"L01".to_string(),
Lecture {
title: "Course introduction".to_string(),
date: None,
unit: Some("u-intro".to_string()),
slides_url: None,
readings: Vec::new(),
},
);
let mut los = BTreeMap::new();
los.insert(
"lo-example".to_string(),
Objective {
text: "Replace this with an objective stated as a student action.".to_string(),
unit: Some("u-intro".to_string()),
lectures: vec!["L01".to_string()],
level_ceiling: Some(Level::Understand),
prerequisites: Vec::new(),
tags: Vec::new(),
assessed: true,
},
);
CourseFile {
schema_version: SCHEMA_VERSION.to_string(),
course: Course {
code: code.to_string(),
title: title.to_string(),
term: term.to_string(),
institution: None,
instructors: Vec::new(),
slug: None,
},
policy: Policy::default(),
units: vec![Unit {
id: "u-intro".to_string(),
title: "Introduction".to_string(),
description: None,
}],
lectures,
learning_objectives: los,
stimuli: BTreeMap::new(),
}
}
}
/// Lowercases and hyphenates a string for use in file names and ids.
///
/// # Arguments
///
/// * `s` - the text to convert.
///
/// # Returns
///
/// A slug of lowercase alphanumerics separated by single hyphens.
pub fn slugify(s: &str) -> String {
let mut out = String::new();
let mut prev_dash = true;
for ch in s.chars() {
if ch.is_ascii_alphanumeric() {
out.push(ch.to_ascii_lowercase());
prev_dash = false;
} else if !prev_dash {
out.push('-');
prev_dash = true;
}
}
while out.ends_with('-') {
out.pop();
}
out
}
fn default_version() -> String {
SCHEMA_VERSION.to_string()
}
fn one() -> f64 {
1.0
}
fn four() -> usize {
4
}
fn two_usize() -> usize {
2
}
fn yes() -> bool {
true
}
fn mastery_default() -> f64 {
0.75
}
#[cfg(test)]
mod tests {
use super::*;
fn parse(src: &str) -> CourseFile {
serde_yaml_ng::from_str(src).expect("course parses")
}
#[test]
fn minimal_course_parses_with_defaults() {
let c = parse(
r#"
course:
code: BIOSC 1540
title: Computational Biology
term: Spring 2026
"#,
);
assert_eq!(c.schema_version, "1.0");
assert_eq!(c.policy.options_per_item, 4);
assert_eq!(c.course.slug(), "biosc-1540");
assert!(c.validate().is_empty());
}
#[test]
fn unquoted_schema_version_still_parses() {
// YAML reads 1.0 as a float; authors should not have to remember quotes.
let c = parse(
r#"
schema_version: 1.0
course: { code: X, title: Y, term: Z }
"#,
);
assert_eq!(c.schema_version, "1.0");
}
#[test]
fn unknown_keys_are_rejected() {
let e = serde_yaml_ng::from_str::<CourseFile>(
r#"
course: { code: X, title: Y, term: Z }
lecutres: {}
"#,
);
assert!(e.is_err(), "a misspelled top-level key must not be ignored");
}
#[test]
fn dangling_references_are_reported() {
let c = parse(
r#"
course: { code: X, title: Y, term: Z }
units:
- { id: u-a, title: A }
lectures:
L01: { title: One, unit: u-nope }
learning_objectives:
lo-a: { text: Do a thing, unit: u-a, lectures: [L99], prerequisites: [lo-missing] }
"#,
);
let issues = c.validate();
assert!(issues.iter().any(|i| i.contains("unknown unit `u-nope`")));
assert!(issues.iter().any(|i| i.contains("unknown lecture `L99`")));
assert!(issues.iter().any(|i| i.contains("lo-missing")));
}
#[test]
fn prerequisite_cycles_are_detected() {
let c = parse(
r#"
course: { code: X, title: Y, term: Z }
learning_objectives:
lo-a: { text: A, prerequisites: [lo-b] }
lo-b: { text: B, prerequisites: [lo-a] }
"#,
);
let issues = c.validate();
assert!(
issues.iter().any(|i| i.contains("cycle")),
"expected a cycle report, got {issues:?}"
);
}
#[test]
fn objectives_sort_by_declared_unit_order() {
let c = parse(
r#"
course: { code: X, title: Y, term: Z }
units:
- { id: u-second, title: Second }
- { id: u-first, title: First }
learning_objectives:
lo-z: { text: Z, unit: u-second }
lo-a: { text: A, unit: u-first }
"#,
);
// Declared order wins over alphabetical order of unit ids.
assert_eq!(c.objectives_in_order(), vec!["lo-z", "lo-a"]);
}
#[test]
fn slugify_collapses_punctuation() {
assert_eq!(slugify("BIOSC 1540"), "biosc-1540");
assert_eq!(slugify("Exam 4 -- Final!"), "exam-4-final");
assert_eq!(slugify(" "), "");
}
}
+183
View File
@@ -0,0 +1,183 @@
// SPDX-License-Identifier: Prosperity-3.0.0
// Copyright Scientific Computing Studio
// Source: https://git.scient.ing/education/coursebank
//! The usage history of a course, derived by scanning assessment records.
//!
//! There is deliberately no separate ledger file. A ledger duplicates
//! information the records already hold and then drifts away from it; this
//! derives the same answers from the one artifact that has to be right anyway.
//! An item was used exactly when it appears on a record.
//!
//! This is an aggregate over the records, the same kind of thing as
//! [`crate::catalog::Catalog`], which is why it sits beside the schema rather
//! than inside [`crate::assessment`].
use std::path::Path;
use crate::assessment::{AssessmentFile, Kind};
use crate::date::Date;
use crate::error::Result;
/// One recorded appearance of an item, derived from the assessment records.
#[derive(Debug, Clone)]
pub struct Usage {
/// The item's global id.
pub item: String,
/// The assessment id it appeared on.
pub assessment: String,
/// The assessment title.
pub title: String,
/// The kind of assessment.
pub kind: Kind,
/// When it was administered, when recorded.
pub date: Option<Date>,
/// The printed question number.
pub number: u32,
/// The fingerprint as used.
pub fingerprint: Option<String>,
}
/// The usage history of a course, built by scanning assessment records.
#[derive(Debug, Clone, Default)]
pub struct History {
/// Every recorded appearance, newest last.
pub usages: Vec<Usage>,
}
impl History {
/// Builds the history from a directory of assessment records.
///
/// # Arguments
///
/// * `dir` - the assessments directory.
///
/// # Returns
///
/// The history.
///
/// # Errors
///
/// Propagates load errors.
pub fn load(dir: &Path) -> Result<History> {
let mut usages = Vec::new();
for file in AssessmentFile::load_all(dir)? {
for p in &file.items {
usages.push(Usage {
item: p.item.clone(),
assessment: file.assessment.id.clone(),
title: file.assessment.title.clone(),
kind: file.assessment.kind,
date: file.assessment.date,
number: p.number,
fingerprint: p.fingerprint.clone(),
});
}
}
usages.sort_by(|a, b| {
a.date
.cmp(&b.date)
.then(a.assessment.cmp(&b.assessment))
.then(a.number.cmp(&b.number))
});
Ok(History { usages })
}
/// Every appearance of one item, oldest first.
///
/// # Arguments
///
/// * `uid` - the item's global id.
///
/// # Returns
///
/// Matching usages.
pub fn for_item(&self, uid: &str) -> Vec<&Usage> {
self.usages.iter().filter(|u| u.item == uid).collect()
}
/// The most recent date an item was used.
///
/// # Arguments
///
/// * `uid` - the item's global id.
///
/// # Returns
///
/// The date, or `None` when never used or never dated.
pub fn last_used(&self, uid: &str) -> Option<Date> {
self.for_item(uid).iter().filter_map(|u| u.date).max()
}
/// How many times an item has been used.
///
/// # Arguments
///
/// * `uid` - the item's global id.
///
/// # Returns
///
/// The count.
pub fn use_count(&self, uid: &str) -> usize {
self.for_item(uid).len()
}
/// Whether an item is still inside its reuse cooldown.
///
/// # Arguments
///
/// * `uid` - the item's global id.
/// * `cooldown_days` - the minimum gap between uses.
/// * `as_of` - the date of the assessment being assembled.
///
/// # Returns
///
/// `true` when the item was used too recently to reuse.
pub fn in_cooldown(&self, uid: &str, cooldown_days: i64, as_of: Date) -> bool {
match self.last_used(uid) {
Some(last) => last.days_until(as_of) < cooldown_days,
None => false,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn history_tracks_last_use_and_cooldown() {
let h = History {
usages: vec![
Usage {
item: "b::q-1".into(),
assessment: "exam-1".into(),
title: "Exam 1".into(),
kind: Kind::Exam,
date: Some("2025-09-15".parse().unwrap()),
number: 4,
fingerprint: None,
},
Usage {
item: "b::q-1".into(),
assessment: "exam-3".into(),
title: "Exam 3".into(),
kind: Kind::Exam,
date: Some("2026-02-10".parse().unwrap()),
number: 7,
fingerprint: None,
},
],
};
assert_eq!(h.use_count("b::q-1"), 2);
assert_eq!(h.use_count("b::q-2"), 0);
assert_eq!(h.last_used("b::q-1").unwrap().to_string(), "2026-02-10");
let exam_date: Date = "2026-04-23".parse().unwrap();
// 72 days elapsed, so a 90-day cooldown still blocks it and a 60-day one
// does not.
assert!(h.in_cooldown("b::q-1", 90, exam_date));
assert!(!h.in_cooldown("b::q-1", 60, exam_date));
assert!(!h.in_cooldown("b::never-used", 3650, exam_date));
}
}
+824
View File
@@ -0,0 +1,824 @@
// SPDX-License-Identifier: Prosperity-3.0.0
// Copyright Scientific Computing Studio
// Source: https://git.scient.ing/education/coursebank
//! The item: one assessable question, with both the intent it was written with
//! and the evidence it has produced.
//!
//! The organizing idea is that those two things belong in one versioned place.
//! [`Design`] is written before an item is ever used and says what you predict:
//! how hard, how discriminating, how long, and what the item is meant to reveal.
//! [`Calibration`] is written by the tool afterwards and says what happened.
//! Keeping them adjacent is what turns a question bank into an instrument you
//! can improve, because every administration produces a checkable prediction.
//!
//! One deliberate departure from a naive design: [`Calibration`] is *cumulative*
//! rather than per-administration. Raw per-response data belongs in the Parquet
//! tables under `data/`, which are far better at holding it, and an item's YAML
//! holds the rolled-up estimate plus a list of which administrations went into
//! it. That keeps bank files readable and reviewable in a pull request while
//! still letting statistics accumulate across terms.
use serde::{Deserialize, Serialize};
use crate::date::Date;
use crate::hash::fingerprint;
use crate::taxonomy::{
CognitiveProcess, Discrimination, ErrorType, Flag, Format, Level, ReviewAction, Status,
};
/// One assessable question.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Item {
/// Stable id, unique within its bank, conventionally `q-<slug>-NNN`.
///
/// Ids are never reused and never renumbered: the id is the join key that
/// ties an item to every assessment it has appeared on and every response
/// row ever recorded for it.
pub id: String,
/// Revision counter, bumped whenever the content changes in a way that
/// invalidates pooled statistics.
#[serde(default = "one_u32")]
pub version: u32,
/// Workflow state; only [`Status::Approved`] items may be assembled.
pub status: Status,
/// Cognitive demand, 1 through 5.
pub level: Level,
/// The specific process the item elicits. Required for approval, and checked
/// against `level`.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cognitive_process: Option<CognitiveProcess>,
/// Response format.
#[serde(default = "default_format")]
pub format: Format,
/// A bonus item, scored outside the graded total.
#[serde(default, skip_serializing_if = "is_false")]
pub bonus: bool,
/// Point value; falls back to the course policy when absent.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub points: Option<f64>,
/// A short human title, used in tables and Canvas question names.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub title: Option<String>,
/// Id of a shared stimulus in the course registry, for case-based testlets.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub stimulus: Option<String>,
/// The prompt.
pub stem: String,
/// The answer options in canonical order. Shuffling happens at export time
/// per form, never here, so the bank stays diffable.
pub options: Vec<Choice>,
/// Objectives this item measures, as ids into the course registry.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub learning_objectives: Vec<String>,
/// Where the material was taught.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub sources: Vec<Source>,
/// Free-form topic tags, for slicing a bank by subject rather than lecture.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub topics: Vec<String>,
/// Item ids or objective ids a student needs before this is fair.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub prerequisites: Vec<String>,
/// Figures or data files reproduced with the item.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub assets: Vec<Asset>,
/// What you predicted before using it.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub design: Option<Design>,
/// What the evidence says, accumulated across administrations.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub calibration: Option<Calibration>,
/// The last review decision recorded for this item.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub review: Option<Review>,
/// Append-only change log.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub history: Vec<HistoryEntry>,
/// The author of record.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub author: Option<String>,
/// Notes that must never reach a student.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub notes_private: Option<String>,
/// Set when the item was retired, with the reason.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub retired: Option<Retirement>,
}
/// One answer option.
///
/// The optional fields are what separate a designed distractor from filler. An
/// option that names the [`ErrorType`] it targets and the misconception behind it
/// is one you can report on: when a third of the cohort picks it, you know what
/// they were thinking, and the student report can say so.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Choice {
/// Option letter, `A` through `H`.
pub id: String,
/// The option text.
pub text: String,
/// Whether this option is keyed correct.
#[serde(default)]
pub correct: bool,
/// Credit awarded, from 0 to 1. Absent means 1.0 when correct, else 0.0.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub credit: Option<f64>,
/// Instructor-facing rationale for why this option is right or wrong.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub explanation: Option<String>,
/// The nudge you would give a student reconsidering this option.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub hint: Option<String>,
/// The specific wrong idea this distractor is built to capture.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub misconception: Option<String>,
/// The category of that error.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub error_type: Option<ErrorType>,
/// Whether a wrong option is defensible enough to earn partial credit.
#[serde(default, skip_serializing_if = "is_false")]
pub defensible: bool,
/// The argument for why it is defensible. Required whenever credit is
/// awarded to a wrong option, so partial credit is always justified in
/// writing rather than by memory.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub defense: Option<String>,
/// Text released to students after the assessment. This is what a student
/// report shows them when they chose this option.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub feedback_student: Option<String>,
/// Your a priori guess at how often this option is chosen.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selection_rate_expected: Option<f64>,
}
impl Choice {
/// The credit this option earns, resolving the default from `correct`.
///
/// # Returns
///
/// Credit in `[0, 1]`.
pub fn credit(&self) -> f64 {
self.credit.unwrap_or(if self.correct { 1.0 } else { 0.0 })
}
/// Whether this option awards credit without being keyed correct.
pub fn is_partial(&self) -> bool {
!self.correct && self.credit() > 0.0
}
/// The best available student-facing explanation of this option.
///
/// Prefers explicit student feedback, then the misconception, then the
/// instructor explanation, so a report degrades gracefully as authoring
/// completeness varies.
///
/// # Returns
///
/// The text, or `None` when the option carries no rationale at all.
pub fn student_text(&self) -> Option<&str> {
self.feedback_student
.as_deref()
.or(self.misconception.as_deref())
.or(self.explanation.as_deref())
}
}
/// Where the assessed material was taught.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Source {
/// Lecture id in the course registry.
pub lecture: String,
/// Slide numbers, so a student report can point at a page rather than a
/// whole lecture.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub slides: Vec<u32>,
/// Readings, cited however you cite them.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub readings: Vec<String>,
/// A timestamp into a recording, in seconds.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub recording_seconds: Option<u32>,
}
/// A figure or data file reproduced with an item.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Asset {
/// Path relative to the course root.
pub path: String,
/// Alt text. Required in practice for accessibility; the linter says so.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub alt: Option<String>,
/// A caption printed below the figure.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub caption: Option<String>,
}
/// The a priori design of an item: your predictions, written down.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Design {
/// The proportion of the target cohort you expect to answer correctly.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub expected_difficulty: Option<f64>,
/// How sharply you expect it to separate students.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub expected_discrimination: Option<Discrimination>,
/// How long you expect it to take, in seconds. Summed over a form, this is
/// how you check that an exam fits the period.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub expected_time_seconds: Option<f64>,
/// What the item is meant to reveal, and why it sits at its level.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub rationale: Option<String>,
}
/// Accumulated evidence about an item's behavior.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Calibration {
/// The administrations pooled into these numbers.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub administrations: Vec<String>,
/// When the calibration was last recomputed.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub updated: Option<Date>,
/// The content fingerprint these statistics describe. If it differs from the
/// item's current fingerprint, the item was edited after calibration and the
/// numbers are stale; the linter says so.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub fingerprint: Option<String>,
/// Total examinees pooled.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub n_examinees: Option<usize>,
/// Proportion correct.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub p_value: Option<f64>,
/// Corrected item-total point-biserial correlation.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub point_biserial: Option<f64>,
/// Upper-minus-lower-group discrimination index.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub discrimination_index: Option<f64>,
/// Mean response time, when the platform reports it.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub mean_response_time_seconds: Option<f64>,
/// Proportion of responses faster than plausible reading time.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub rapid_guess_rate: Option<f64>,
/// Per-option behavior, keyed by option letter.
#[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
pub option_stats: std::collections::BTreeMap<String, OptionStat>,
/// Fitted item response theory parameters.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub irt: Option<IrtParams>,
/// Machine-detected problems.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub flags: Vec<Flag>,
}
/// How one option behaved.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct OptionStat {
/// Proportion of examinees who chose it.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selection_rate: Option<f64>,
/// Correlation between choosing it and total score. Negative for a working
/// distractor; positive on a distractor is a warning.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub point_biserial: Option<f64>,
/// Selection rate among the top scoring group.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub upper_group_rate: Option<f64>,
/// Selection rate among the bottom scoring group.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub lower_group_rate: Option<f64>,
}
/// Fitted item response theory parameters.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct IrtParams {
/// Which model was fitted.
pub model: IrtModel,
/// Discrimination.
pub a: f64,
/// Difficulty, on the same scale as ability.
pub b: f64,
/// Lower asymptote, the guessing parameter.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub c: Option<f64>,
/// Standard error of `a`.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub se_a: Option<f64>,
/// Standard error of `b`.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub se_b: Option<f64>,
/// Examinees the fit was based on. Small samples give unstable parameters,
/// so this travels with them rather than being looked up later.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub n: Option<usize>,
/// Whether priors were used, which matters when interpreting `a`.
#[serde(default, skip_serializing_if = "is_false")]
pub bayesian: bool,
}
/// The item response theory model family.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum IrtModel {
/// One parameter: difficulty only, discrimination fixed at 1.
Rasch,
/// Two parameters: discrimination and difficulty.
#[serde(rename = "2pl")]
TwoPl,
/// Three parameters, adding a lower asymptote for guessing.
#[serde(rename = "3pl")]
ThreePl,
}
impl IrtModel {
/// The token used in YAML.
pub fn as_str(self) -> &'static str {
match self {
IrtModel::Rasch => "rasch",
IrtModel::TwoPl => "2pl",
IrtModel::ThreePl => "3pl",
}
}
}
/// A recorded review decision.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Review {
/// Who reviewed it.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub reviewed_by: Option<String>,
/// When.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub reviewed_on: Option<Date>,
/// What was decided.
pub action: ReviewAction,
/// Why.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub notes: Option<String>,
}
/// Why and when an item left service.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Retirement {
/// When it was retired.
pub on: Date,
/// Why.
pub reason: String,
/// A replacement item id, when one exists.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub replaced_by: Option<String>,
}
/// One entry in an item's change log.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct HistoryEntry {
/// The version this change produced.
pub version: u32,
/// When it was made.
pub date: Date,
/// Who made it.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub author: Option<String>,
/// What changed.
pub change: String,
}
impl Item {
/// Builds a draft item with everything optional left empty.
///
/// A constructor rather than a `Default` implementation, because there is no
/// sensible default id, stem, or option set: an item missing any of those is
/// not a lesser item, it is not an item. Requiring them at construction means
/// the only way to get a half-built `Item` is deliberately.
///
/// The result is `Status::Draft` and deliberately will not pass
/// [`Item::is_assemblable`] — it still needs learning objectives, sources, and
/// a cognitive process before it can be drawn onto an assessment.
///
/// # Arguments
///
/// * `id` - the item id.
/// * `level` - the cognitive level.
/// * `stem` - the question.
/// * `options` - the answer options.
///
/// # Returns
///
/// The draft item.
pub fn draft(id: &str, level: Level, stem: &str, options: Vec<Choice>) -> Item {
Item {
id: id.to_string(),
version: 1,
status: Status::Draft,
level,
cognitive_process: None,
format: if options.iter().filter(|o| o.correct).count() > 1 {
Format::MultipleResponse
} else {
Format::SingleBestAnswer
},
bonus: false,
points: None,
title: None,
stimulus: None,
stem: stem.to_string(),
options,
learning_objectives: Vec::new(),
sources: Vec::new(),
topics: Vec::new(),
prerequisites: Vec::new(),
assets: Vec::new(),
design: None,
calibration: None,
review: None,
history: Vec::new(),
author: None,
notes_private: None,
retired: None,
}
}
/// Indices of the keyed-correct options.
///
/// # Returns
///
/// Zero-based indices into `options`.
pub fn key_indices(&self) -> Vec<usize> {
self.options
.iter()
.enumerate()
.filter(|(_, o)| o.correct)
.map(|(i, _)| i)
.collect()
}
/// The keyed-correct option letters, sorted.
///
/// # Returns
///
/// Letters such as `["C"]` or `["A", "B"]`.
pub fn key_letters(&self) -> Vec<String> {
let mut out: Vec<String> = self
.options
.iter()
.filter(|o| o.correct)
.map(|o| o.id.clone())
.collect();
out.sort();
out
}
/// Looks up an option by letter.
///
/// # Arguments
///
/// * `letter` - the option id, case insensitive.
///
/// # Returns
///
/// The option, or `None`.
pub fn option(&self, letter: &str) -> Option<&Choice> {
self.options
.iter()
.find(|o| o.id.eq_ignore_ascii_case(letter))
}
/// Whether the item keys more than one option.
pub fn is_multi_key(&self) -> bool {
self.key_indices().len() > 1
}
/// The display title, falling back to a truncated stem.
///
/// # Returns
///
/// A short label suitable for a table or a Canvas question name.
pub fn display_title(&self) -> String {
if let Some(t) = &self.title {
if !t.trim().is_empty() {
return t.trim().to_string();
}
}
let flat = self.stem.split_whitespace().collect::<Vec<_>>().join(" ");
if flat.chars().count() <= 60 {
flat
} else {
let head: String = flat.chars().take(57).collect();
format!("{head}...")
}
}
/// A content fingerprint over everything that affects what a student sees.
///
/// Metadata deliberately does not contribute: retagging an objective must not
/// invalidate pooled statistics, but rewording an option must.
///
/// # Returns
///
/// The fingerprint as hex.
pub fn fingerprint(&self) -> String {
let mut parts: Vec<String> = vec![self.stem.trim().to_string()];
// Canonicalize by option letter so reordering the YAML block, which does
// not change the item, does not change the fingerprint.
let mut opts: Vec<&Choice> = self.options.iter().collect();
opts.sort_by(|a, b| a.id.cmp(&b.id));
for o in opts {
parts.push(format!(
"{}|{}|{}",
o.id,
if o.correct { "1" } else { "0" },
o.text.trim()
));
}
if let Some(s) = &self.stimulus {
parts.push(format!("stimulus:{s}"));
}
fingerprint(parts.iter().map(|s| s.as_str()))
}
/// Whether the recorded calibration matches the current content.
///
/// # Returns
///
/// `false` when the item was edited after it was calibrated.
pub fn calibration_is_current(&self) -> bool {
match self
.calibration
.as_ref()
.and_then(|c| c.fingerprint.as_ref())
{
Some(fp) => *fp == self.fingerprint(),
None => true,
}
}
/// The point value, resolving against a course default.
///
/// # Arguments
///
/// * `default_points` - the course policy value.
///
/// # Returns
///
/// The point value to use.
pub fn points(&self, default_points: f64) -> f64 {
self.points.unwrap_or(default_points)
}
/// Whether this item may be placed on a graded assessment.
pub fn is_assemblable(&self) -> bool {
self.status.is_usable() && self.retired.is_none()
}
/// The expected time in seconds, falling back to a level-based estimate.
///
/// The fallbacks are rough but useful: without them, a form's total time
/// estimate silently drops every item that has no `design` block.
///
/// # Returns
///
/// Seconds.
pub fn expected_seconds(&self) -> f64 {
if let Some(t) = self.design.as_ref().and_then(|d| d.expected_time_seconds) {
return t;
}
match self.level {
Level::Remember => 35.0,
Level::Understand => 65.0,
Level::Apply => 95.0,
Level::Analyze => 130.0,
Level::Create => 165.0,
}
}
/// Appends a change-log entry and bumps the version.
///
/// # Arguments
///
/// * `change` - a description of what changed.
/// * `author` - who made the change.
pub fn record_change(&mut self, change: &str, author: Option<&str>) {
self.version += 1;
self.history.push(HistoryEntry {
version: self.version,
date: Date::today(),
author: author.map(|a| a.to_string()),
change: change.to_string(),
});
}
}
fn one_u32() -> u32 {
1
}
fn default_format() -> Format {
Format::SingleBestAnswer
}
fn is_false(b: &bool) -> bool {
!*b
}
#[cfg(test)]
mod tests {
use super::*;
fn item(src: &str) -> Item {
serde_yaml_ng::from_str(src).expect("item parses")
}
const MINIMAL: &str = r#"
id: q-demo-001
status: draft
level: 2
stem: Which statement best explains the effect?
options:
- { id: A, text: Right, correct: true }
- { id: B, text: Wrong }
- { id: C, text: Also wrong }
"#;
#[test]
fn minimal_item_parses_with_defaults() {
let it = item(MINIMAL);
assert_eq!(it.version, 1);
assert_eq!(it.format, Format::SingleBestAnswer);
assert!(!it.bonus);
assert_eq!(it.key_letters(), vec!["A"]);
assert!(!it.is_multi_key());
assert!(!it.is_assemblable(), "drafts are not assemblable");
}
#[test]
fn credit_defaults_from_correctness() {
let it = item(MINIMAL);
assert_eq!(it.option("A").unwrap().credit(), 1.0);
assert_eq!(it.option("B").unwrap().credit(), 0.0);
assert!(!it.option("B").unwrap().is_partial());
let with_partial = item(
r#"
id: q-demo-002
status: draft
level: 5
stem: s
options:
- { id: A, text: Right, correct: true }
- { id: B, text: Defensible, credit: 0.5, defensible: true, defense: because }
- { id: C, text: Wrong }
"#,
);
assert!(with_partial.option("B").unwrap().is_partial());
assert_eq!(with_partial.option("B").unwrap().credit(), 0.5);
}
#[test]
fn fingerprint_tracks_content_not_metadata() {
let base = item(MINIMAL);
let mut retagged = base.clone();
retagged.topics = vec!["kinetics".into()];
retagged.learning_objectives = vec!["lo-a".into()];
retagged.author = Some("someone".into());
assert_eq!(
base.fingerprint(),
retagged.fingerprint(),
"metadata must not invalidate pooled statistics"
);
let mut reworded = base.clone();
reworded.options[1].text = "Wrong, but differently".into();
assert_ne!(base.fingerprint(), reworded.fingerprint());
let mut rekeyed = base.clone();
rekeyed.options[0].correct = false;
rekeyed.options[1].correct = true;
assert_ne!(base.fingerprint(), rekeyed.fingerprint());
}
#[test]
fn fingerprint_ignores_yaml_option_order() {
let a = item(MINIMAL);
let b = item(
r#"
id: q-demo-001
status: draft
level: 2
stem: Which statement best explains the effect?
options:
- { id: C, text: Also wrong }
- { id: A, text: Right, correct: true }
- { id: B, text: Wrong }
"#,
);
assert_eq!(a.fingerprint(), b.fingerprint());
}
#[test]
fn stale_calibration_is_detectable() {
let mut it = item(MINIMAL);
assert!(it.calibration_is_current(), "no calibration is not stale");
let fp = it.fingerprint();
it.calibration = Some(Calibration {
fingerprint: Some(fp),
..Calibration::default()
});
assert!(it.calibration_is_current());
it.stem = "A different question entirely?".into();
assert!(!it.calibration_is_current());
}
#[test]
fn display_title_truncates_long_stems() {
let mut it = item(MINIMAL);
it.title = None;
it.stem = "word ".repeat(40);
let t = it.display_title();
assert!(t.ends_with("..."));
assert_eq!(t.chars().count(), 60);
}
#[test]
fn unknown_item_keys_are_rejected() {
let bad = serde_yaml_ng::from_str::<Item>(
r#"
id: q-demo-001
status: draft
level: 2
stem: s
steam: oops
options:
- { id: A, text: a, correct: true }
"#,
);
assert!(bad.is_err());
}
#[test]
fn record_change_bumps_version_and_logs() {
let mut it = item(MINIMAL);
it.record_change("clarified the stem", Some("Alex"));
assert_eq!(it.version, 2);
assert_eq!(it.history.len(), 1);
assert_eq!(it.history[0].version, 2);
}
#[test]
fn irt_model_tokens_round_trip() {
assert_eq!(serde_json::to_string(&IrtModel::TwoPl).unwrap(), "\"2pl\"");
assert_eq!(
serde_json::from_str::<IrtModel>("\"3pl\"").unwrap(),
IrtModel::ThreePl
);
}
}
+98
View File
@@ -0,0 +1,98 @@
// SPDX-License-Identifier: Prosperity-3.0.0
// Copyright Scientific Computing Studio
// Source: https://git.scient.ing/education/coursebank
//! The on-disk layout of a course directory.
use std::path::PathBuf;
use crate::course::COURSE_FILE;
use crate::error::{Error, Result};
/// The standard directory layout of a course, resolved from a root.
#[derive(Debug, Clone)]
pub struct Layout {
/// The course root.
pub root: PathBuf,
}
impl Layout {
/// Builds a layout from a course root directory.
///
/// # Arguments
///
/// * `root` - the course directory.
///
/// # Returns
///
/// The layout.
pub fn new(root: impl Into<PathBuf>) -> Layout {
Layout { root: root.into() }
}
/// Path to `course.yaml`.
pub fn course_file(&self) -> PathBuf {
self.root.join(COURSE_FILE)
}
/// Directory holding item bank YAML files.
pub fn banks(&self) -> PathBuf {
self.root.join("banks")
}
/// Directory holding assessment records.
pub fn assessments(&self) -> PathBuf {
self.root.join("assessments")
}
/// Directory holding response tables and derived statistics.
pub fn data(&self) -> PathBuf {
self.root.join("data")
}
/// Directory holding generated reports.
pub fn reports(&self) -> PathBuf {
self.root.join("reports")
}
/// Directory holding generated exports such as QTI packages.
pub fn build(&self) -> PathBuf {
self.root.join("build")
}
/// Directory holding emitted JSON Schema files for editor validation.
pub fn schema(&self) -> PathBuf {
self.root.join("schema")
}
/// Directory holding Typst export templates and their configuration.
///
/// Unlike the other directories, this one is *not* created by
/// [`Layout::create_all`]. Its absence is meaningful: a course with no
/// `templates/` directory uses the templates compiled into the binary, and
/// creating an empty one on `init` would suggest a customization step is
/// required when it is not. `coursebank template dump` creates it on demand.
pub fn templates(&self) -> PathBuf {
self.root.join("templates")
}
/// Creates every directory in the layout.
///
/// # Errors
///
/// Returns [`Error::Io`] if a directory cannot be created.
pub fn create_all(&self) -> Result<()> {
for dir in [
self.root.clone(),
self.banks(),
self.assessments(),
self.data(),
self.reports(),
self.build(),
self.schema(),
] {
std::fs::create_dir_all(&dir).map_err(|e| Error::io(&dir, e))?;
}
Ok(())
}
}
+636
View File
@@ -0,0 +1,636 @@
// SPDX-License-Identifier: Prosperity-3.0.0
// Copyright Scientific Computing Studio
// Source: https://git.scient.ing/education/coursebank
//! The pedagogical vocabulary: levels, cognitive processes, error types, and
//! workflow states.
use std::fmt;
use serde::{Deserialize, Serialize};
/// Cognitive demand, following the revised Bloom taxonomy.
///
/// Serialized as the integers 1 through 5 so YAML reads `level: 3`. The derived
/// [`Ord`] follows declaration order, which is ascending demand.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(try_from = "u8", into = "u8")]
pub enum Level {
/// Retrieve a fact or definition.
Remember,
/// Construct meaning; explain, compare, classify.
Understand,
/// Carry out a procedure in a given situation.
Apply,
/// Break material apart and relate the pieces.
Analyze,
/// Judge against criteria, or assemble something new.
Create,
}
impl Level {
/// Every level in ascending order, for iteration in reports.
pub const ALL: [Level; 5] = [
Level::Remember,
Level::Understand,
Level::Apply,
Level::Analyze,
Level::Create,
];
/// The numeric code used in YAML and on printed badges.
pub fn code(self) -> u8 {
match self {
Level::Remember => 1,
Level::Understand => 2,
Level::Apply => 3,
Level::Analyze => 4,
Level::Create => 5,
}
}
/// The level for a numeric code, if it is one.
///
/// Flat storage writes `0` for an unknown level rather than an empty cell,
/// because a numeric column with holes in it is awkward in every columnar
/// format. This turns that convention back into an `Option`.
///
/// # Arguments
///
/// * `code` - the numeric code, where anything outside 1..=5 means unknown.
///
/// # Returns
///
/// The level, or `None`.
pub fn from_code(code: u8) -> Option<Level> {
match code {
1 => Some(Level::Remember),
2 => Some(Level::Understand),
3 => Some(Level::Apply),
4 => Some(Level::Analyze),
5 => Some(Level::Create),
_ => None,
}
}
/// The category name, e.g. `"Apply"`.
pub fn name(self) -> &'static str {
match self {
Level::Remember => "Remember",
Level::Understand => "Understand",
Level::Apply => "Apply",
Level::Analyze => "Analyze",
Level::Create => "Evaluate/Create",
}
}
/// A one-line description suitable for a student-facing report.
pub fn blurb(self) -> &'static str {
match self {
Level::Remember => "recalling terms, facts, and definitions",
Level::Understand => "explaining ideas in your own words",
Level::Apply => "using a procedure in a new situation",
Level::Analyze => "taking a situation apart and relating the pieces",
Level::Create => "judging alternatives or building something new",
}
}
/// The cognitive processes that belong to this level.
pub fn processes(self) -> &'static [CognitiveProcess] {
use CognitiveProcess as P;
match self {
Level::Remember => &[P::Recognize, P::Recall],
Level::Understand => &[
P::Interpret,
P::Exemplify,
P::Classify,
P::Summarize,
P::Infer,
P::Compare,
P::Explain,
],
Level::Apply => &[P::Execute, P::Implement],
Level::Analyze => &[P::Differentiate, P::Organize, P::Attribute],
Level::Create => &[P::Check, P::Critique, P::Generate, P::Plan, P::Produce],
}
}
/// Whether a process is consistent with this level.
///
/// # Arguments
///
/// * `process` - the process to check.
///
/// # Returns
///
/// `true` when the pairing is coherent.
pub fn allows(self, process: CognitiveProcess) -> bool {
self.processes().contains(&process)
}
}
impl TryFrom<u8> for Level {
type Error = String;
fn try_from(v: u8) -> Result<Level, String> {
match v {
1 => Ok(Level::Remember),
2 => Ok(Level::Understand),
3 => Ok(Level::Apply),
4 => Ok(Level::Analyze),
5 => Ok(Level::Create),
other => Err(format!("level must be 1 through 5, got {other}")),
}
}
}
impl From<Level> for u8 {
fn from(l: Level) -> u8 {
l.code()
}
}
impl fmt::Display for Level {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "L{}", self.code())
}
}
/// The specific cognitive process an item is designed to elicit.
///
/// Naming the process, not just the level, is what makes the level claim
/// checkable: it forces you to say which of the several things "Understand"
/// could mean you actually wrote.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum CognitiveProcess {
/// Identify a previously encountered item.
Recognize,
/// Retrieve from long-term memory unaided.
Recall,
/// Restate in another representation.
Interpret,
/// Give an instance of a category.
Exemplify,
/// Assign an instance to a category.
Classify,
/// Abstract a general theme.
Summarize,
/// Draw a logical conclusion from given information.
Infer,
/// Detect correspondences between two things.
Compare,
/// Construct a cause-and-effect account.
Explain,
/// Apply a procedure to a familiar task.
Execute,
/// Apply a procedure to an unfamiliar task.
Implement,
/// Distinguish relevant from irrelevant parts.
Differentiate,
/// Determine how elements fit a structure.
Organize,
/// Determine a point of view or intent.
Attribute,
/// Test for internal consistency.
Check,
/// Judge against external criteria.
Critique,
/// Propose alternative hypotheses.
Generate,
/// Devise a procedure.
Plan,
/// Construct a product.
Produce,
}
impl CognitiveProcess {
/// The level this process belongs to.
pub fn level(self) -> Level {
for level in Level::ALL {
if level.allows(self) {
return level;
}
}
// Unreachable: every variant appears in exactly one level's list.
Level::Remember
}
/// Every process, in level order.
pub const ALL: [CognitiveProcess; 19] = [
CognitiveProcess::Recognize,
CognitiveProcess::Recall,
CognitiveProcess::Interpret,
CognitiveProcess::Exemplify,
CognitiveProcess::Classify,
CognitiveProcess::Summarize,
CognitiveProcess::Infer,
CognitiveProcess::Compare,
CognitiveProcess::Explain,
CognitiveProcess::Execute,
CognitiveProcess::Implement,
CognitiveProcess::Differentiate,
CognitiveProcess::Organize,
CognitiveProcess::Attribute,
CognitiveProcess::Check,
CognitiveProcess::Critique,
CognitiveProcess::Generate,
CognitiveProcess::Plan,
CognitiveProcess::Produce,
];
/// The snake_case token used in YAML.
pub fn as_str(self) -> &'static str {
use CognitiveProcess as P;
match self {
P::Recognize => "recognize",
P::Recall => "recall",
P::Interpret => "interpret",
P::Exemplify => "exemplify",
P::Classify => "classify",
P::Summarize => "summarize",
P::Infer => "infer",
P::Compare => "compare",
P::Explain => "explain",
P::Execute => "execute",
P::Implement => "implement",
P::Differentiate => "differentiate",
P::Organize => "organize",
P::Attribute => "attribute",
P::Check => "check",
P::Critique => "critique",
P::Generate => "generate",
P::Plan => "plan",
P::Produce => "produce",
}
}
}
impl fmt::Display for CognitiveProcess {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
/// The category of mistake a distractor is built to capture.
///
/// This does double duty. It disciplines authoring, because a distractor you
/// cannot name an error for is probably filler. And it makes item analysis
/// legible afterwards: a high selection rate on a `DroppedStep` option tells you
/// where in a procedure students slip, which a bare letter never would.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ErrorType {
/// Confused one remembered fact for a neighboring one.
RecallConfusion,
/// Swapped two terms that sound or look alike.
TerminologySwap,
/// Holds a specific, nameable wrong model.
Misconception,
/// Knows part of the idea but not all of it.
IncompleteUnderstanding,
/// Applied a valid rule outside its scope.
Overgeneralization,
/// True but irrelevant to the question asked.
PlausibleIrrelevant,
/// Omitted a step in a procedure.
DroppedStep,
/// Inverted the direction of a relationship.
ReversedRelationship,
/// Used a procedure that does not apply here.
WrongProcedure,
/// Right method, wrong sign or order of magnitude.
SignOrMagnitudeError,
/// Ignored a condition that interacts with the answer.
IgnoresInteractingCondition,
/// Correct as far as it goes, but not the best answer.
CorrectButIncomplete,
/// Took a heuristic shortcut that usually works.
CommonShortcut,
}
impl ErrorType {
/// Every error type.
pub const ALL: [ErrorType; 13] = [
ErrorType::RecallConfusion,
ErrorType::TerminologySwap,
ErrorType::Misconception,
ErrorType::IncompleteUnderstanding,
ErrorType::Overgeneralization,
ErrorType::PlausibleIrrelevant,
ErrorType::DroppedStep,
ErrorType::ReversedRelationship,
ErrorType::WrongProcedure,
ErrorType::SignOrMagnitudeError,
ErrorType::IgnoresInteractingCondition,
ErrorType::CorrectButIncomplete,
ErrorType::CommonShortcut,
];
/// The snake_case token used in YAML.
pub fn as_str(self) -> &'static str {
use ErrorType as E;
match self {
E::RecallConfusion => "recall_confusion",
E::TerminologySwap => "terminology_swap",
E::Misconception => "misconception",
E::IncompleteUnderstanding => "incomplete_understanding",
E::Overgeneralization => "overgeneralization",
E::PlausibleIrrelevant => "plausible_irrelevant",
E::DroppedStep => "dropped_step",
E::ReversedRelationship => "reversed_relationship",
E::WrongProcedure => "wrong_procedure",
E::SignOrMagnitudeError => "sign_or_magnitude_error",
E::IgnoresInteractingCondition => "ignores_interacting_condition",
E::CorrectButIncomplete => "correct_but_incomplete",
E::CommonShortcut => "common_shortcut",
}
}
/// A short instructor-facing gloss.
pub fn gloss(self) -> &'static str {
use ErrorType as E;
match self {
E::RecallConfusion => "confused with a neighboring fact",
E::TerminologySwap => "swapped similar terms",
E::Misconception => "specific wrong model",
E::IncompleteUnderstanding => "partial grasp of the idea",
E::Overgeneralization => "applied a rule outside its scope",
E::PlausibleIrrelevant => "true but not what was asked",
E::DroppedStep => "skipped a step",
E::ReversedRelationship => "reversed the direction",
E::WrongProcedure => "used the wrong procedure",
E::SignOrMagnitudeError => "sign or magnitude slip",
E::IgnoresInteractingCondition => "ignored an interacting condition",
E::CorrectButIncomplete => "correct but not best",
E::CommonShortcut => "took a familiar shortcut",
}
}
}
/// Where an item sits in the authoring workflow.
///
/// The state gates what the validator requires. A draft may be a bare idea; an
/// approved item must be fully sourced and designed, because approval is what
/// permits it onto a graded assessment. Retired items are kept forever so the
/// bank is an append-only record of what you have asked students.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Status {
/// Captured but not yet worked out.
Draft,
/// Written and awaiting review.
InReview,
/// Reviewed and sent back for changes.
NeedsRevision,
/// Cleared for use on a graded assessment.
Approved,
/// Withdrawn from use but retained for the record.
Retired,
}
impl Status {
/// Whether an item in this state may appear on a graded assessment.
pub fn is_usable(self) -> bool {
matches!(self, Status::Approved)
}
/// The snake_case token used in YAML.
pub fn as_str(self) -> &'static str {
match self {
Status::Draft => "draft",
Status::InReview => "in_review",
Status::NeedsRevision => "needs_revision",
Status::Approved => "approved",
Status::Retired => "retired",
}
}
}
impl fmt::Display for Status {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
/// The response format of an item.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Format {
/// Exactly one keyed option.
SingleBestAnswer,
/// One or more keyed options; the student must find every one.
MultipleResponse,
/// Two options, True and False.
TrueFalse,
}
impl Format {
/// The QTI question type Canvas expects for this format.
pub fn qti_type(self) -> &'static str {
match self {
Format::SingleBestAnswer => "multiple_choice_question",
Format::MultipleResponse => "multiple_answers_question",
Format::TrueFalse => "true_false_question",
}
}
}
/// How strongly an item is expected to separate strong from weak students.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Discrimination {
/// Most prepared students get it; it anchors rather than separates.
Low,
/// Separates somewhat.
Moderate,
/// Expected to separate sharply.
High,
}
impl Discrimination {
/// The point-biserial band this expectation implies, as `(low, high)`.
///
/// Used to check an a priori expectation against the observed statistic.
pub fn expected_band(self) -> (f64, f64) {
match self {
Discrimination::Low => (-1.0, 0.20),
Discrimination::Moderate => (0.15, 0.40),
Discrimination::High => (0.30, 1.0),
}
}
}
/// What was done about an item after reviewing its statistics.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ReviewAction {
/// Behaved as intended; leave it alone.
Keep,
/// Rewrite before reusing.
Revise,
/// Credit a defensible distractor for this administration.
AwardPartialCredit,
/// The key was wrong; fix it and rescore.
CorrectKey,
/// Withdraw from use.
Retire,
/// Keep but watch on the next administration.
Monitor,
}
/// A machine-detected problem with an item's observed behavior.
///
/// These are written by analysis, not by hand, and they are the queue you work
/// through when deciding what to revise.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Flag {
/// Weaker students outperformed stronger ones. Almost always a keying error
/// or a genuinely ambiguous stem.
NegativeDiscrimination,
/// Barely separates students.
LowDiscrimination,
/// A distractor correlates with total score better than the key does.
DistractorOutperformsKey,
/// Strong students chose one particular distractor at a high rate, which is
/// the signature of a second defensible reading.
KeyUnderperforms,
/// Nearly everyone answered correctly; carries little information.
TooEasy,
/// Nearly everyone answered incorrectly and it did not discriminate.
TooHard,
/// Many responses were faster than plausible reading time.
HighRapidGuess,
/// A distractor almost nobody chose; it is doing no work.
NonfunctioningDistractor,
/// Performance differed across groups after conditioning on ability.
DifFlagged,
/// Marked ambiguous by hand or inferred from partial credit awarded to a
/// distractor during grading.
Ambiguous,
/// The observed difficulty was far from the difficulty you predicted.
DesignMismatch,
}
impl Flag {
/// Every flag.
pub const ALL: [Flag; 11] = [
Flag::NegativeDiscrimination,
Flag::LowDiscrimination,
Flag::DistractorOutperformsKey,
Flag::KeyUnderperforms,
Flag::TooEasy,
Flag::TooHard,
Flag::HighRapidGuess,
Flag::NonfunctioningDistractor,
Flag::DifFlagged,
Flag::Ambiguous,
Flag::DesignMismatch,
];
/// The flag's stable code, matching its YAML spelling.
pub fn as_str(self) -> &'static str {
match self {
Flag::NegativeDiscrimination => "negative_discrimination",
Flag::LowDiscrimination => "low_discrimination",
Flag::DistractorOutperformsKey => "distractor_outperforms_key",
Flag::KeyUnderperforms => "key_underperforms",
Flag::TooEasy => "too_easy",
Flag::TooHard => "too_hard",
Flag::HighRapidGuess => "high_rapid_guess",
Flag::NonfunctioningDistractor => "nonfunctioning_distractor",
Flag::DifFlagged => "dif_flagged",
Flag::Ambiguous => "ambiguous",
Flag::DesignMismatch => "design_mismatch",
}
}
/// Whether the flag should stop an item from being reused as written.
///
/// Distinguishing blocking from advisory flags is what turns analysis into a
/// workflow: a negative discrimination is a keying bug to fix before the item
/// is ever given again, while an easy item is merely uninformative.
///
/// # Returns
///
/// `true` for flags that demand a revision.
pub fn is_blocking(self) -> bool {
matches!(
self,
Flag::NegativeDiscrimination
| Flag::DistractorOutperformsKey
| Flag::Ambiguous
| Flag::KeyUnderperforms
)
}
/// A short explanation of what the flag means and what to do about it.
pub fn advice(self) -> &'static str {
match self {
Flag::NegativeDiscrimination => {
"check the key first, then the stem for a second valid reading"
}
Flag::LowDiscrimination => "expected for anchors; investigate if the level is 3+",
Flag::DistractorOutperformsKey => "the distractor may be the better answer",
Flag::KeyUnderperforms => "strong students split; look for an ambiguity",
Flag::TooEasy => "fine as an anchor, wasteful if you meant it to discriminate",
Flag::TooHard => "check for a missing prerequisite or an unclear stem",
Flag::HighRapidGuess => "position on the form or time pressure, not the item",
Flag::NonfunctioningDistractor => "replace it with a plausible error",
Flag::DifFlagged => "inspect wording for content unrelated to the objective",
Flag::Ambiguous => "rewrite the stem to exclude the second reading",
Flag::DesignMismatch => "update your expectation or revise the item",
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn levels_order_by_demand() {
assert!(Level::Remember < Level::Create);
assert_eq!(Level::Apply.code(), 3);
}
#[test]
fn every_process_belongs_to_exactly_one_level() {
let mut seen = Vec::new();
for level in Level::ALL {
for p in level.processes() {
assert!(!seen.contains(p), "{p} appears under two levels");
seen.push(*p);
assert_eq!(p.level(), level);
}
}
assert_eq!(seen.len(), 19, "all processes are assigned");
}
#[test]
fn level_process_pairing_is_checked() {
assert!(Level::Apply.allows(CognitiveProcess::Implement));
assert!(!Level::Apply.allows(CognitiveProcess::Recall));
}
#[test]
fn level_serializes_as_an_integer() {
assert_eq!(serde_json::to_string(&Level::Apply).unwrap(), "3");
assert_eq!(serde_json::from_str::<Level>("4").unwrap(), Level::Analyze);
assert!(serde_json::from_str::<Level>("6").is_err());
assert!(serde_json::from_str::<Level>("0").is_err());
}
#[test]
fn processes_use_snake_case() {
assert_eq!(
serde_json::to_string(&CognitiveProcess::Implement).unwrap(),
"\"implement\""
);
assert_eq!(
serde_json::from_str::<Status>("\"in_review\"").unwrap(),
Status::InReview
);
}
}