1 parent
228a0da47f
commit
abc0bdf621
79 files changed
+31370
No files matched your search
@@ -0,0 +1,627 @@
|
||||
// SPDX-License-Identifier: Prosperity-3.0.0
|
||||
// Copyright Scientific Computing Studio
|
||||
// Source: https://git.scient.ing/education/coursebank
|
||||
|
||||
//! The assessment record: what you actually asked, in what order, on what date.
|
||||
//!
|
||||
//! This file is the hinge of the whole tool, and it is worth being explicit about
|
||||
//! why it exists as a separate artifact.
|
||||
//!
|
||||
//! Grading exports do not know about your item bank. Gradescope gives you
|
||||
//! `14.csv`; Canvas gives you a column header. Both identify questions by
|
||||
//! *position on a form*. An item bank identifies questions by stable id. The
|
||||
//! assessment record is the only place those two namespaces meet, and without it
|
||||
//! there is no way to say that question 14 of Exam 4 was
|
||||
//! `docking::q-scoring-003` at version 2.
|
||||
//!
|
||||
//! Recording it also makes usage history free. Rather than maintaining a separate
|
||||
//! ledger that can drift out of sync with reality, `coursebank usage` scans the
|
||||
//! assessment records: an item was used exactly when it appears on a record. The
|
||||
//! records are the source of truth, and they are small, readable, and diffable.
|
||||
//!
|
||||
//! Each placement stores the resolved key and content fingerprint *as used*. A
|
||||
//! year later, when the item has been reworded twice, you can still see what the
|
||||
//! students in front of you were asked.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::Path;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::course::SCHEMA_VERSION;
|
||||
use crate::date::Date;
|
||||
use crate::error::Result;
|
||||
use crate::taxonomy::Level;
|
||||
use crate::yaml;
|
||||
|
||||
/// A whole assessment record file.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct AssessmentFile {
|
||||
/// Schema version this file targets.
|
||||
#[serde(
|
||||
default = "default_version",
|
||||
deserialize_with = "yaml::flexible_string"
|
||||
)]
|
||||
pub schema_version: String,
|
||||
|
||||
/// Identity and administration details.
|
||||
pub assessment: Assessment,
|
||||
|
||||
/// The blueprint this was assembled against, when it was assembled by the
|
||||
/// tool. Keeping it lets you check the form you shipped against the design
|
||||
/// you intended.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub blueprint: Option<Blueprint>,
|
||||
|
||||
/// Alternate forms, each a permutation of the same items.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub forms: Vec<Form>,
|
||||
|
||||
/// The items, in printed order.
|
||||
#[serde(default)]
|
||||
pub items: Vec<Placement>,
|
||||
}
|
||||
|
||||
/// Assessment identity and administration.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Assessment {
|
||||
/// Stable id, e.g. `exam-4-2026s`. Response tables carry this.
|
||||
pub id: String,
|
||||
/// Human title as printed, e.g. `Exam 4`.
|
||||
pub title: String,
|
||||
/// The term this administration belongs to. Items outlive terms, so the term
|
||||
/// lives here rather than on the item.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub term: Option<String>,
|
||||
/// What kind of assessment this is.
|
||||
#[serde(default = "default_kind")]
|
||||
pub kind: Kind,
|
||||
/// When it was administered.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub date: Option<Date>,
|
||||
/// Where it was administered.
|
||||
#[serde(default = "default_platform")]
|
||||
pub platform: Platform,
|
||||
/// Time allowed, in minutes. Compared against the summed expected time.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub minutes_allowed: Option<f64>,
|
||||
/// Attempts allowed; `-1` means unlimited. Only meaningful online.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub attempts: Option<i64>,
|
||||
/// Whether the platform should shuffle options.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub shuffle: Option<bool>,
|
||||
/// How repeated attempts are scored.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub scoring_policy: Option<ScoringPolicy>,
|
||||
/// Instructions printed at the top of a paper form.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub instructions: Option<String>,
|
||||
/// Notes to yourself about this administration.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub notes: Option<String>,
|
||||
}
|
||||
|
||||
/// What kind of assessment a record describes.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Kind {
|
||||
/// A summative in-term exam.
|
||||
Exam,
|
||||
/// A short graded check.
|
||||
Quiz,
|
||||
/// Graded work done outside class.
|
||||
Homework,
|
||||
/// Ungraded practice.
|
||||
Practice,
|
||||
/// A cumulative final.
|
||||
Final,
|
||||
}
|
||||
|
||||
impl Kind {
|
||||
/// The token used in YAML and in file names.
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Kind::Exam => "exam",
|
||||
Kind::Quiz => "quiz",
|
||||
Kind::Homework => "homework",
|
||||
Kind::Practice => "practice",
|
||||
Kind::Final => "final",
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether results from this kind should feed item calibration.
|
||||
///
|
||||
/// Practice work is excluded by default: it is usually untimed, open book,
|
||||
/// and attempted more than once, so pooling it with exam data would bias
|
||||
/// every difficulty estimate downward.
|
||||
pub fn counts_for_calibration(self) -> bool {
|
||||
!matches!(self, Kind::Practice)
|
||||
}
|
||||
}
|
||||
|
||||
/// Where an assessment was administered, which determines the export format and
|
||||
/// the shape of the grading data that comes back.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Platform {
|
||||
/// Printed and scanned, graded in Gradescope.
|
||||
Paper,
|
||||
/// A Canvas quiz.
|
||||
Canvas,
|
||||
/// Delivered on paper but graded elsewhere.
|
||||
Other,
|
||||
}
|
||||
|
||||
/// How repeated attempts are scored.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ScoringPolicy {
|
||||
/// Keep the best attempt.
|
||||
KeepHighest,
|
||||
/// Keep the most recent attempt.
|
||||
KeepLatest,
|
||||
}
|
||||
|
||||
impl ScoringPolicy {
|
||||
/// The token Canvas expects in a QTI package.
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
ScoringPolicy::KeepHighest => "keep_highest",
|
||||
ScoringPolicy::KeepLatest => "keep_latest",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The design an assessment was assembled against.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Blueprint {
|
||||
/// How many scored items to draw at each level.
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub level_counts: BTreeMap<Level, usize>,
|
||||
/// How many bonus items to draw at each level.
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub bonus_counts: BTreeMap<Level, usize>,
|
||||
/// Objectives that must appear, with a minimum item count each.
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub objective_minimums: BTreeMap<String, usize>,
|
||||
/// Restrict the draw to these lectures.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub lectures: Vec<String>,
|
||||
/// Restrict the draw to these topics.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub topics: Vec<String>,
|
||||
/// Restrict the draw to these banks.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub banks: Vec<String>,
|
||||
/// The most items any one bank may contribute, so a form is not dominated by
|
||||
/// whichever topic you happened to write the most questions about.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max_per_bank: Option<usize>,
|
||||
/// Do not reuse an item used within this many days.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cooldown_days: Option<i64>,
|
||||
/// The seed used, so the draw can be reproduced exactly.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub seed: Option<u64>,
|
||||
}
|
||||
|
||||
impl Blueprint {
|
||||
/// Total scored items requested.
|
||||
pub fn scored_total(&self) -> usize {
|
||||
self.level_counts.values().sum()
|
||||
}
|
||||
|
||||
/// Total bonus items requested.
|
||||
pub fn bonus_total(&self) -> usize {
|
||||
self.bonus_counts.values().sum()
|
||||
}
|
||||
}
|
||||
|
||||
/// One alternate form of the same assessment.
|
||||
///
|
||||
/// A form does not change which items appear, only their order and the order of
|
||||
/// their options, so every form measures the same thing and one answer key
|
||||
/// generator serves them all.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Form {
|
||||
/// Form label, e.g. `A`.
|
||||
pub id: String,
|
||||
/// Seed for the permutation. Reproducing a printed form requires it.
|
||||
pub seed: u64,
|
||||
/// Whether item order is permuted.
|
||||
#[serde(default = "yes")]
|
||||
pub shuffle_items: bool,
|
||||
/// Whether option order is permuted within each item.
|
||||
#[serde(default = "yes")]
|
||||
pub shuffle_options: bool,
|
||||
}
|
||||
|
||||
/// One item as placed on the form.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Placement {
|
||||
/// Printed question number. This is the join key to grading exports, which
|
||||
/// is the entire reason this record exists.
|
||||
pub number: u32,
|
||||
/// The item's global id, `bank::item`.
|
||||
pub item: String,
|
||||
/// The item version used.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub version: Option<u32>,
|
||||
/// The content fingerprint as used, so later edits are detectable.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub fingerprint: Option<String>,
|
||||
/// Points as administered.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub points: Option<f64>,
|
||||
/// Whether it was scored as bonus here.
|
||||
#[serde(default, skip_serializing_if = "is_false")]
|
||||
pub bonus: bool,
|
||||
/// The keyed letters as administered.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub key: Vec<String>,
|
||||
/// The level as administered, denormalized so a record reads standalone.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub level: Option<Level>,
|
||||
/// Objectives as administered, denormalized for the same reason.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub learning_objectives: Vec<String>,
|
||||
/// Credit awarded to non-keyed options after the fact, keyed by letter.
|
||||
///
|
||||
/// When item analysis or a student challenge leads you to credit a
|
||||
/// distractor, recording it here keeps the rescoring decision with the
|
||||
/// administration it applies to instead of quietly editing the item.
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub credit_overrides: BTreeMap<String, f64>,
|
||||
/// Set when an item was dropped from scoring after administration.
|
||||
#[serde(default, skip_serializing_if = "is_false")]
|
||||
pub dropped: bool,
|
||||
}
|
||||
|
||||
impl AssessmentFile {
|
||||
/// Loads an assessment record.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `path` - the YAML file.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The parsed record.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns a load error.
|
||||
pub fn load(path: &Path) -> Result<AssessmentFile> {
|
||||
yaml::read(path)
|
||||
}
|
||||
|
||||
/// Writes the record back out.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `path` - the destination.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`crate::error::Error::Io`] on a write failure.
|
||||
pub fn save(&self, path: &Path) -> Result<()> {
|
||||
yaml::write(path, self)
|
||||
}
|
||||
|
||||
/// Loads every assessment record in a directory, sorted by date then id.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `dir` - the assessments directory.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The records, empty when the directory does not exist.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Propagates load errors.
|
||||
pub fn load_all(dir: &Path) -> Result<Vec<AssessmentFile>> {
|
||||
let mut out = Vec::new();
|
||||
for path in yaml::list_yaml(dir)? {
|
||||
out.push(AssessmentFile::load(&path)?);
|
||||
}
|
||||
out.sort_by(|a, b| {
|
||||
a.assessment
|
||||
.date
|
||||
.cmp(&b.assessment.date)
|
||||
.then(a.assessment.id.cmp(&b.assessment.id))
|
||||
});
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// The placement at a printed question number.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `number` - the printed number.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The placement, or `None`.
|
||||
pub fn placement(&self, number: u32) -> Option<&Placement> {
|
||||
self.items.iter().find(|p| p.number == number)
|
||||
}
|
||||
|
||||
/// Total scored points, excluding bonus and dropped items.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `default_points` - the course policy default.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The total.
|
||||
pub fn total_points(&self, default_points: f64) -> f64 {
|
||||
self.items
|
||||
.iter()
|
||||
.filter(|p| !p.bonus && !p.dropped)
|
||||
.map(|p| p.points.unwrap_or(default_points))
|
||||
.sum()
|
||||
}
|
||||
|
||||
/// Counts of scored placements by level.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// A map from level to count.
|
||||
pub fn level_counts(&self) -> BTreeMap<Level, usize> {
|
||||
let mut out: BTreeMap<Level, usize> = BTreeMap::new();
|
||||
for p in self.items.iter().filter(|p| !p.bonus) {
|
||||
if let Some(l) = p.level {
|
||||
*out.entry(l).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Checks the record for internal problems and for drift from the bank.
|
||||
///
|
||||
/// The drift check is the valuable part: if an item was reworded after this
|
||||
/// assessment was given, the fingerprints disagree, and any analysis that
|
||||
/// pools this administration with a later one is comparing two questions.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `catalog` - the loaded course, for resolving item references.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Every problem found.
|
||||
pub fn validate(&self) -> Vec<String> {
|
||||
let mut issues = Vec::new();
|
||||
|
||||
if self.assessment.id.trim().is_empty() {
|
||||
issues.push("assessment.id is empty".into());
|
||||
}
|
||||
if self.items.is_empty() {
|
||||
issues.push("the record lists no items".into());
|
||||
}
|
||||
|
||||
let mut numbers: BTreeMap<u32, usize> = BTreeMap::new();
|
||||
for p in &self.items {
|
||||
*numbers.entry(p.number).or_insert(0) += 1;
|
||||
}
|
||||
for (n, count) in &numbers {
|
||||
if *count > 1 {
|
||||
issues.push(format!("question number {n} is used {count} times"));
|
||||
}
|
||||
}
|
||||
// Gaps are legal but almost always a mistake, since grading exports are
|
||||
// numbered contiguously.
|
||||
let mut sorted: Vec<u32> = numbers.keys().copied().collect();
|
||||
sorted.sort_unstable();
|
||||
for (i, n) in sorted.iter().enumerate() {
|
||||
let expected = i as u32 + 1;
|
||||
if *n != expected {
|
||||
issues.push(format!(
|
||||
"question numbers are not contiguous from 1; expected {expected}, found {n}"
|
||||
));
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let mut seen_items: BTreeMap<&str, u32> = BTreeMap::new();
|
||||
for p in &self.items {
|
||||
if let Some(first) = seen_items.get(p.item.as_str()) {
|
||||
issues.push(format!(
|
||||
"item `{}` appears twice, at questions {first} and {}",
|
||||
p.item, p.number
|
||||
));
|
||||
} else {
|
||||
seen_items.insert(p.item.as_str(), p.number);
|
||||
}
|
||||
|
||||
for (letter, credit) in &p.credit_overrides {
|
||||
if !(0.0..=1.0).contains(credit) {
|
||||
issues.push(format!(
|
||||
"question {}: credit override for `{letter}` must be in [0, 1], got {credit}",
|
||||
p.number
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut form_ids: Vec<&str> = Vec::new();
|
||||
for form in &self.forms {
|
||||
if form_ids.contains(&form.id.as_str()) {
|
||||
issues.push(format!("duplicate form id `{}`", form.id));
|
||||
}
|
||||
form_ids.push(&form.id);
|
||||
}
|
||||
|
||||
issues
|
||||
}
|
||||
|
||||
/// A skeleton record for `coursebank assessment new`.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `id` - the assessment id.
|
||||
/// * `title` - the printed title.
|
||||
/// * `kind` - the kind of assessment.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// A record with no items.
|
||||
pub fn skeleton(id: &str, title: &str, kind: Kind) -> AssessmentFile {
|
||||
AssessmentFile {
|
||||
schema_version: SCHEMA_VERSION.to_string(),
|
||||
assessment: Assessment {
|
||||
id: id.to_string(),
|
||||
title: title.to_string(),
|
||||
term: None,
|
||||
kind,
|
||||
date: Some(Date::today()),
|
||||
platform: Platform::Paper,
|
||||
minutes_allowed: None,
|
||||
attempts: None,
|
||||
shuffle: None,
|
||||
scoring_policy: None,
|
||||
instructions: None,
|
||||
notes: None,
|
||||
},
|
||||
blueprint: None,
|
||||
forms: Vec::new(),
|
||||
items: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One recorded appearance of an item, derived from the assessment records.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Usage {
|
||||
/// The item's global id.
|
||||
pub item: String,
|
||||
/// The assessment id it appeared on.
|
||||
pub assessment: String,
|
||||
/// The assessment title.
|
||||
pub title: String,
|
||||
/// The kind of assessment.
|
||||
pub kind: Kind,
|
||||
/// When it was administered, when recorded.
|
||||
pub date: Option<Date>,
|
||||
/// The printed question number.
|
||||
pub number: u32,
|
||||
/// The fingerprint as used.
|
||||
pub fingerprint: Option<String>,
|
||||
}
|
||||
|
||||
fn default_version() -> String {
|
||||
SCHEMA_VERSION.to_string()
|
||||
}
|
||||
fn default_kind() -> Kind {
|
||||
Kind::Exam
|
||||
}
|
||||
fn default_platform() -> Platform {
|
||||
Platform::Paper
|
||||
}
|
||||
fn yes() -> bool {
|
||||
true
|
||||
}
|
||||
fn is_false(b: &bool) -> bool {
|
||||
!*b
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn record(src: &str) -> AssessmentFile {
|
||||
serde_yaml_ng::from_str(src).expect("assessment parses")
|
||||
}
|
||||
|
||||
const SIMPLE: &str = r#"
|
||||
assessment:
|
||||
id: exam-4-2026s
|
||||
title: Exam 4
|
||||
kind: exam
|
||||
date: 2026-04-23
|
||||
platform: paper
|
||||
items:
|
||||
- { number: 1, item: "b1::q-a-001", points: 1.5, key: [B], level: 1 }
|
||||
- { number: 2, item: "b1::q-a-002", points: 1.5, key: [C], level: 3 }
|
||||
- { number: 3, item: "b1::q-a-003", points: 1.5, bonus: true, key: [D], level: 5 }
|
||||
"#;
|
||||
|
||||
#[test]
|
||||
fn parses_and_totals_only_scored_points() {
|
||||
let a = record(SIMPLE);
|
||||
assert_eq!(a.items.len(), 3);
|
||||
assert_eq!(a.total_points(1.0), 3.0, "bonus is excluded");
|
||||
assert_eq!(a.placement(2).unwrap().item, "b1::q-a-002");
|
||||
assert!(a.validate().is_empty(), "{:?}", a.validate());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn level_counts_skip_bonus() {
|
||||
let counts = record(SIMPLE).level_counts();
|
||||
assert_eq!(counts.get(&Level::Remember), Some(&1));
|
||||
assert_eq!(counts.get(&Level::Apply), Some(&1));
|
||||
assert_eq!(counts.get(&Level::Create), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn catches_duplicate_numbers_and_repeated_items() {
|
||||
let a = record(
|
||||
r#"
|
||||
assessment: { id: x, title: X }
|
||||
items:
|
||||
- { number: 1, item: "b::q-1" }
|
||||
- { number: 1, item: "b::q-1" }
|
||||
"#,
|
||||
);
|
||||
let issues = a.validate();
|
||||
assert!(issues.iter().any(|i| i.contains("used 2 times")));
|
||||
assert!(issues.iter().any(|i| i.contains("appears twice")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn catches_non_contiguous_numbering() {
|
||||
let a = record(
|
||||
r#"
|
||||
assessment: { id: x, title: X }
|
||||
items:
|
||||
- { number: 1, item: "b::q-1" }
|
||||
- { number: 3, item: "b::q-2" }
|
||||
"#,
|
||||
);
|
||||
assert!(a.validate().iter().any(|i| i.contains("not contiguous")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_out_of_range_credit_overrides() {
|
||||
let a = record(
|
||||
r#"
|
||||
assessment: { id: x, title: X }
|
||||
items:
|
||||
- { number: 1, item: "b::q-1", credit_overrides: { B: 1.5 } }
|
||||
"#,
|
||||
);
|
||||
assert!(a.validate().iter().any(|i| i.contains("must be in [0, 1]")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blueprint_totals() {
|
||||
let b: Blueprint = serde_yaml_ng::from_str(
|
||||
r#"
|
||||
level_counts: { 1: 6, 2: 10, 3: 12, 4: 8 }
|
||||
bonus_counts: { 5: 2 }
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(b.scored_total(), 36);
|
||||
assert_eq!(b.bonus_total(), 2);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,914 @@
|
||||
// SPDX-License-Identifier: Prosperity-3.0.0
|
||||
// Copyright Scientific Computing Studio
|
||||
// Source: https://git.scient.ing/education/coursebank
|
||||
|
||||
//! The bank file: a collection of items scoped to a topic or a lecture.
|
||||
//!
|
||||
//! One bank per topic (or per lecture, if that suits how you teach) is the unit
|
||||
//! of authoring. Banks are small enough to review in a pull request, they let
|
||||
//! two people write questions without colliding, and `bank.scope` records what
|
||||
//! the file is *for* so `coursebank catalog` can tell you that you have eleven
|
||||
//! items on enzyme kinetics and none on regulation.
|
||||
//!
|
||||
//! Validation here is split in two on purpose. [`BankFile::validate`] checks what
|
||||
//! must be true for the file to be usable at all: ids are unique, a keyed answer
|
||||
//! exists, an approved item is fully specified, a level and its cognitive process
|
||||
//! agree. The softer question of whether an item is *well written* lives in
|
||||
//! [`crate::lint`], because those checks are advisory and you should be able to
|
||||
//! ship a file that trips a few of them.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::Path;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::course::{CourseFile, SCHEMA_VERSION};
|
||||
use crate::date::Date;
|
||||
use crate::error::Result;
|
||||
use crate::item::Item;
|
||||
use crate::taxonomy::{Format, Level, Status};
|
||||
use crate::yaml;
|
||||
|
||||
/// A whole bank file.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BankFile {
|
||||
/// Schema version this file targets.
|
||||
#[serde(
|
||||
default = "default_version",
|
||||
deserialize_with = "yaml::flexible_string"
|
||||
)]
|
||||
pub schema_version: String,
|
||||
|
||||
/// Bank identity and scope.
|
||||
pub bank: BankMeta,
|
||||
|
||||
/// Values applied to every item in the file that does not set its own.
|
||||
#[serde(default)]
|
||||
pub defaults: BankDefaults,
|
||||
|
||||
/// The items.
|
||||
#[serde(default)]
|
||||
pub items: Vec<Item>,
|
||||
}
|
||||
|
||||
/// Bank identity and scope.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BankMeta {
|
||||
/// Stable id, unique across the course. Item ids are namespaced by it.
|
||||
pub id: String,
|
||||
/// Human title.
|
||||
pub title: String,
|
||||
/// What this bank covers.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub description: Option<String>,
|
||||
/// What the bank is scoped to, so coverage can be reported against it.
|
||||
#[serde(default)]
|
||||
pub scope: Scope,
|
||||
/// Who maintains it.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub maintainer: Option<String>,
|
||||
/// When it was created.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub created: Option<Date>,
|
||||
/// When it was last touched.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub updated: Option<Date>,
|
||||
}
|
||||
|
||||
/// What a bank is scoped to.
|
||||
///
|
||||
/// A bank may be scoped by lecture, by objective, by topic, or by none of them.
|
||||
/// Declaring the scope is what lets the catalog report *gaps*: it can only tell
|
||||
/// you that lecture 12 has no Apply-level items if it knows lecture 12 is
|
||||
/// supposed to be covered here.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Scope {
|
||||
/// Lectures this bank draws from.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub lectures: Vec<String>,
|
||||
/// Objectives this bank is responsible for covering.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub learning_objectives: Vec<String>,
|
||||
/// Units this bank belongs to.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub units: Vec<String>,
|
||||
/// Topics this bank is about.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub topics: Vec<String>,
|
||||
}
|
||||
|
||||
/// Per-file defaults, so common metadata is written once.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BankDefaults {
|
||||
/// Default author for items in this file.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub author: Option<String>,
|
||||
/// Default point value.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub points: Option<f64>,
|
||||
/// Expected option count; the linter flags items that differ, since an
|
||||
/// inconsistent option count across a form is itself a cue to students.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub options_per_item: Option<usize>,
|
||||
/// Topics added to every item in the file.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub topics: Vec<String>,
|
||||
/// Sources applied to items that declare none.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub lectures: Vec<String>,
|
||||
}
|
||||
|
||||
impl BankFile {
|
||||
/// Loads a bank file from disk.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `path` - the YAML file.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The parsed bank.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`crate::error::Error::Io`] or [`crate::error::Error::Yaml`].
|
||||
pub fn load(path: &Path) -> Result<BankFile> {
|
||||
yaml::read(path)
|
||||
}
|
||||
|
||||
/// Writes a bank file back out as YAML.
|
||||
///
|
||||
/// Round-tripping loses comments, which is why calibration is written by an
|
||||
/// explicit `coursebank calibrate` step rather than as a side effect of
|
||||
/// anything else: you should be able to see the diff it produces.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `path` - the destination.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`crate::error::Error::Io`] on a write failure.
|
||||
pub fn save(&self, path: &Path) -> Result<()> {
|
||||
yaml::write(path, self)
|
||||
}
|
||||
|
||||
/// Applies file defaults to items that omit the corresponding field.
|
||||
///
|
||||
/// Called after loading so the rest of the crate never has to think about
|
||||
/// defaults again.
|
||||
pub fn apply_defaults(&mut self) {
|
||||
let d = self.defaults.clone();
|
||||
for item in &mut self.items {
|
||||
if item.author.is_none() {
|
||||
item.author = d.author.clone();
|
||||
}
|
||||
if item.points.is_none() {
|
||||
item.points = d.points;
|
||||
}
|
||||
for t in &d.topics {
|
||||
if !item.topics.contains(t) {
|
||||
item.topics.push(t.clone());
|
||||
}
|
||||
}
|
||||
if item.sources.is_empty() {
|
||||
for lec in &d.lectures {
|
||||
item.sources.push(crate::item::Source {
|
||||
lecture: lec.clone(),
|
||||
slides: Vec::new(),
|
||||
readings: Vec::new(),
|
||||
recording_seconds: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Loads a bank and applies its defaults.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `path` - the YAML file.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The parsed bank with defaults resolved.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Propagates load errors.
|
||||
pub fn load_resolved(path: &Path) -> Result<BankFile> {
|
||||
let mut b = BankFile::load(path)?;
|
||||
b.apply_defaults();
|
||||
Ok(b)
|
||||
}
|
||||
|
||||
/// Checks every invariant that must hold for the file to be usable.
|
||||
///
|
||||
/// Returns all problems rather than the first, so one run fixes one file.
|
||||
/// When a course file is supplied, cross-file references are checked too.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `course` - the course registry, for resolving objective and lecture
|
||||
/// references. Pass `None` to check only what is local to the file.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Every problem found, empty when the file is sound.
|
||||
pub fn validate(&self, course: Option<&CourseFile>) -> Vec<String> {
|
||||
let mut issues = Vec::new();
|
||||
|
||||
if self.bank.id.trim().is_empty() {
|
||||
issues.push("bank.id is empty".into());
|
||||
}
|
||||
if self.bank.title.trim().is_empty() {
|
||||
issues.push("bank.title is empty".into());
|
||||
}
|
||||
if let (Some(created), Some(updated)) = (self.bank.created, self.bank.updated) {
|
||||
if updated < created {
|
||||
issues.push(format!(
|
||||
"bank.updated ({updated}) is before bank.created ({created})"
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// Duplicate item ids inside the file.
|
||||
let mut counts: BTreeMap<&str, usize> = BTreeMap::new();
|
||||
for it in &self.items {
|
||||
*counts.entry(it.id.as_str()).or_insert(0) += 1;
|
||||
}
|
||||
for (id, n) in &counts {
|
||||
if *n > 1 {
|
||||
issues.push(format!("duplicate item id `{id}` appears {n} times"));
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(c) = course {
|
||||
for lec in &self.bank.scope.lectures {
|
||||
if !c.lectures.contains_key(lec) {
|
||||
issues.push(format!("bank.scope: unknown lecture `{lec}`"));
|
||||
}
|
||||
}
|
||||
for lo in &self.bank.scope.learning_objectives {
|
||||
if !c.learning_objectives.contains_key(lo) {
|
||||
issues.push(format!("bank.scope: unknown learning objective `{lo}`"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for it in &self.items {
|
||||
issues.extend(
|
||||
validate_item(it, course, self.defaults.options_per_item)
|
||||
.into_iter()
|
||||
.map(|m| format!("{}: {m}", it.id)),
|
||||
);
|
||||
}
|
||||
|
||||
issues
|
||||
}
|
||||
|
||||
/// Items that may be placed on a graded assessment.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// References to approved, unretired items.
|
||||
pub fn assemblable(&self) -> Vec<&Item> {
|
||||
self.items.iter().filter(|i| i.is_assemblable()).collect()
|
||||
}
|
||||
|
||||
/// Counts of assemblable, non-bonus items by level.
|
||||
///
|
||||
/// This is the five-tuple you check a blueprint against.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// A map from level to count.
|
||||
pub fn level_counts(&self) -> BTreeMap<Level, usize> {
|
||||
let mut out: BTreeMap<Level, usize> = Level::ALL.iter().map(|l| (*l, 0)).collect();
|
||||
for it in self.items.iter().filter(|i| i.is_assemblable() && !i.bonus) {
|
||||
*out.entry(it.level).or_insert(0) += 1;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// A skeleton bank file for `coursebank bank new`.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `id` - the bank id.
|
||||
/// * `title` - the bank title.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// A bank with no items.
|
||||
pub fn skeleton(id: &str, title: &str) -> BankFile {
|
||||
BankFile {
|
||||
schema_version: SCHEMA_VERSION.to_string(),
|
||||
bank: BankMeta {
|
||||
id: id.to_string(),
|
||||
title: title.to_string(),
|
||||
description: None,
|
||||
scope: Scope::default(),
|
||||
maintainer: None,
|
||||
created: Some(Date::today()),
|
||||
updated: Some(Date::today()),
|
||||
},
|
||||
defaults: BankDefaults::default(),
|
||||
items: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Validates one item.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `it` - the item.
|
||||
/// * `course` - the course registry, when available.
|
||||
/// * `expected_options` - the file's declared option count, when set.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Problems found, without the item id prefix.
|
||||
fn validate_item(
|
||||
it: &Item,
|
||||
course: Option<&CourseFile>,
|
||||
expected_options: Option<usize>,
|
||||
) -> Vec<String> {
|
||||
let mut issues = Vec::new();
|
||||
|
||||
if it.id.trim().is_empty() {
|
||||
issues.push("empty id".into());
|
||||
}
|
||||
if it.stem.trim().is_empty() {
|
||||
issues.push("empty stem".into());
|
||||
}
|
||||
if it.version == 0 {
|
||||
issues.push("version must be at least 1".into());
|
||||
}
|
||||
|
||||
// --- options -----------------------------------------------------------
|
||||
if it.options.len() < 2 {
|
||||
issues.push(format!(
|
||||
"needs at least 2 options, has {}",
|
||||
it.options.len()
|
||||
));
|
||||
}
|
||||
let mut seen: Vec<&str> = Vec::new();
|
||||
for (i, o) in it.options.iter().enumerate() {
|
||||
let pos = i + 1;
|
||||
if o.text.trim().is_empty() {
|
||||
issues.push(format!("option {pos}: empty text"));
|
||||
}
|
||||
let letter_ok = o.id.len() == 1
|
||||
&& o.id
|
||||
.chars()
|
||||
.next()
|
||||
.map(|c| c.is_ascii_uppercase() && c <= 'H')
|
||||
.unwrap_or(false);
|
||||
if !letter_ok {
|
||||
issues.push(format!(
|
||||
"option {pos}: id `{}` must be a single letter A through H",
|
||||
o.id
|
||||
));
|
||||
}
|
||||
if seen.contains(&o.id.as_str()) {
|
||||
issues.push(format!("option {pos}: duplicate option id `{}`", o.id));
|
||||
}
|
||||
seen.push(&o.id);
|
||||
|
||||
let credit = o.credit();
|
||||
if !(0.0..=1.0).contains(&credit) {
|
||||
issues.push(format!(
|
||||
"option {}: credit must be between 0 and 1, got {credit}",
|
||||
o.id
|
||||
));
|
||||
}
|
||||
// Partial credit must be argued for in writing, not remembered.
|
||||
if o.is_partial() && o.defense.is_none() {
|
||||
issues.push(format!(
|
||||
"option {}: awards credit {credit} but gives no `defense`",
|
||||
o.id
|
||||
));
|
||||
}
|
||||
if o.is_partial() && !o.defensible {
|
||||
issues.push(format!(
|
||||
"option {}: awards credit {credit} but is not marked `defensible: true`",
|
||||
o.id
|
||||
));
|
||||
}
|
||||
if o.correct && credit == 0.0 {
|
||||
issues.push(format!(
|
||||
"option {}: keyed correct but earns no credit",
|
||||
o.id
|
||||
));
|
||||
}
|
||||
}
|
||||
if let Some(n) = expected_options {
|
||||
if it.options.len() != n && !it.options.is_empty() {
|
||||
issues.push(format!(
|
||||
"has {} options but the bank declares {n} per item",
|
||||
it.options.len()
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// --- key ---------------------------------------------------------------
|
||||
let keys = it.key_indices();
|
||||
match it.format {
|
||||
Format::SingleBestAnswer => {
|
||||
if keys.len() != 1 {
|
||||
issues.push(format!(
|
||||
"single_best_answer needs exactly one keyed option, has {}",
|
||||
keys.len()
|
||||
));
|
||||
}
|
||||
}
|
||||
Format::MultipleResponse => {
|
||||
if keys.is_empty() {
|
||||
issues.push("multiple_response needs at least one keyed option".into());
|
||||
}
|
||||
if keys.len() == it.options.len() {
|
||||
issues.push("multiple_response keys every option, so it asks nothing".into());
|
||||
}
|
||||
}
|
||||
Format::TrueFalse => {
|
||||
if it.options.len() != 2 {
|
||||
issues.push(format!(
|
||||
"true_false needs exactly 2 options, has {}",
|
||||
it.options.len()
|
||||
));
|
||||
}
|
||||
if keys.len() != 1 {
|
||||
issues.push("true_false needs exactly one keyed option".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- level and process must agree -------------------------------------
|
||||
if let Some(p) = it.cognitive_process {
|
||||
if !it.level.allows(p) {
|
||||
issues.push(format!(
|
||||
"cognitive_process `{p}` belongs to level {} but the item is level {}",
|
||||
p.level().code(),
|
||||
it.level.code()
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// --- design plausibility ----------------------------------------------
|
||||
if let Some(d) = &it.design {
|
||||
if let Some(x) = d.expected_difficulty {
|
||||
if !(0.0..=1.0).contains(&x) {
|
||||
issues.push(format!(
|
||||
"design.expected_difficulty must be between 0 and 1, got {x}"
|
||||
));
|
||||
}
|
||||
}
|
||||
if let Some(t) = d.expected_time_seconds {
|
||||
if t <= 0.0 {
|
||||
issues.push(format!(
|
||||
"design.expected_time_seconds must be positive, got {t}"
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- calibration plausibility -----------------------------------------
|
||||
if let Some(c) = &it.calibration {
|
||||
if let Some(p) = c.p_value {
|
||||
if !(0.0..=1.0).contains(&p) {
|
||||
issues.push(format!(
|
||||
"calibration.p_value must be between 0 and 1, got {p}"
|
||||
));
|
||||
}
|
||||
}
|
||||
if let Some(r) = c.point_biserial {
|
||||
if !(-1.0..=1.0).contains(&r) {
|
||||
issues.push(format!(
|
||||
"calibration.point_biserial must be between -1 and 1, got {r}"
|
||||
));
|
||||
}
|
||||
}
|
||||
for letter in c.option_stats.keys() {
|
||||
if it.option(letter).is_none() {
|
||||
issues.push(format!(
|
||||
"calibration.option_stats has `{letter}`, which is not an option of this item"
|
||||
));
|
||||
}
|
||||
}
|
||||
if let Some(irt) = &c.irt {
|
||||
if irt.a <= 0.0 {
|
||||
issues.push(format!("calibration.irt.a must be positive, got {}", irt.a));
|
||||
}
|
||||
if let Some(cp) = irt.c {
|
||||
if !(0.0..1.0).contains(&cp) {
|
||||
issues.push(format!("calibration.irt.c must be in [0, 1), got {cp}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- history must be coherent -----------------------------------------
|
||||
let mut last_version = 0u32;
|
||||
for (i, h) in it.history.iter().enumerate() {
|
||||
if h.version <= last_version {
|
||||
issues.push(format!(
|
||||
"history entry {} has version {} which does not increase",
|
||||
i + 1,
|
||||
h.version
|
||||
));
|
||||
}
|
||||
last_version = h.version;
|
||||
}
|
||||
if !it.history.is_empty() && last_version > it.version {
|
||||
issues.push(format!(
|
||||
"history records version {last_version} but the item says version {}",
|
||||
it.version
|
||||
));
|
||||
}
|
||||
|
||||
// --- retirement -------------------------------------------------------
|
||||
if it.retired.is_some() && it.status != Status::Retired {
|
||||
issues.push(format!(
|
||||
"has a `retired` block but status is `{}`",
|
||||
it.status
|
||||
));
|
||||
}
|
||||
|
||||
// --- approval gate ----------------------------------------------------
|
||||
// Approval is what permits an item onto a graded assessment, so it is the
|
||||
// right place to require that the item is fully sourced and designed.
|
||||
if it.status == Status::Approved {
|
||||
if it.cognitive_process.is_none() {
|
||||
issues.push("approved items must declare a cognitive_process".into());
|
||||
}
|
||||
if it.learning_objectives.is_empty() {
|
||||
issues.push("approved items must reference at least one learning objective".into());
|
||||
}
|
||||
if it.sources.is_empty() {
|
||||
issues.push("approved items must cite at least one source".into());
|
||||
}
|
||||
if it.design.is_none() {
|
||||
issues.push("approved items must carry a design block".into());
|
||||
}
|
||||
}
|
||||
|
||||
// --- cross-file references --------------------------------------------
|
||||
if let Some(c) = course {
|
||||
for lo in &it.learning_objectives {
|
||||
match c.learning_objectives.get(lo) {
|
||||
None => issues.push(format!("unknown learning objective `{lo}`")),
|
||||
Some(obj) => {
|
||||
if let Some(ceiling) = obj.level_ceiling {
|
||||
if it.level > ceiling {
|
||||
issues.push(format!(
|
||||
"level {} exceeds the ceiling {} declared for objective `{lo}`",
|
||||
it.level.code(),
|
||||
ceiling.code()
|
||||
));
|
||||
}
|
||||
}
|
||||
if !obj.assessed {
|
||||
issues.push(format!(
|
||||
"objective `{lo}` is marked `assessed: false` but this item measures it"
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for s in &it.sources {
|
||||
if !c.lectures.contains_key(&s.lecture) {
|
||||
issues.push(format!("unknown lecture `{}`", s.lecture));
|
||||
}
|
||||
}
|
||||
if let Some(st) = &it.stimulus {
|
||||
if !c.stimuli.contains_key(st) {
|
||||
issues.push(format!("unknown stimulus `{st}`"));
|
||||
}
|
||||
}
|
||||
if let Some(floor) = c.policy.partial_credit_floor_level {
|
||||
for o in &it.options {
|
||||
if o.is_partial() && it.level < floor {
|
||||
issues.push(format!(
|
||||
"option {} awards partial credit at level {}, below the course floor of {}",
|
||||
o.id,
|
||||
it.level.code(),
|
||||
floor.code()
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
if !c.policy.allow_partial_credit && it.options.iter().any(|o| o.is_partial()) {
|
||||
issues.push("awards partial credit, which the course policy disallows".into());
|
||||
}
|
||||
}
|
||||
|
||||
issues
|
||||
}
|
||||
|
||||
fn default_version() -> String {
|
||||
SCHEMA_VERSION.to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn bank(items_yaml: &str) -> BankFile {
|
||||
let src = format!("bank:\n id: b\n title: Bank\ndefaults: {{}}\nitems:\n{items_yaml}");
|
||||
let mut b: BankFile = serde_yaml_ng::from_str(&src).expect("bank parses");
|
||||
b.apply_defaults();
|
||||
b
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sound_bank_validates_clean() {
|
||||
let b = bank(
|
||||
r#"
|
||||
- id: q-a-001
|
||||
status: draft
|
||||
level: 1
|
||||
stem: What is x?
|
||||
options:
|
||||
- { id: A, text: right, correct: true }
|
||||
- { id: B, text: wrong }
|
||||
- { id: C, text: wrong too }
|
||||
"#,
|
||||
);
|
||||
assert!(b.validate(None).is_empty(), "{:?}", b.validate(None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn catches_missing_and_multiple_keys() {
|
||||
let b = bank(
|
||||
r#"
|
||||
- id: q-a-001
|
||||
status: draft
|
||||
level: 1
|
||||
stem: s
|
||||
options:
|
||||
- { id: A, text: a }
|
||||
- { id: B, text: b }
|
||||
- id: q-a-002
|
||||
status: draft
|
||||
level: 1
|
||||
format: single_best_answer
|
||||
stem: s
|
||||
options:
|
||||
- { id: A, text: a, correct: true }
|
||||
- { id: B, text: b, correct: true }
|
||||
"#,
|
||||
);
|
||||
let issues = b.validate(None);
|
||||
assert!(
|
||||
issues
|
||||
.iter()
|
||||
.any(|i| i.contains("exactly one keyed option"))
|
||||
);
|
||||
assert_eq!(
|
||||
issues
|
||||
.iter()
|
||||
.filter(|i| i.contains("exactly one keyed option"))
|
||||
.count(),
|
||||
2
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn catches_duplicate_ids_and_letters() {
|
||||
let b = bank(
|
||||
r#"
|
||||
- id: q-a-001
|
||||
status: draft
|
||||
level: 1
|
||||
stem: s
|
||||
options:
|
||||
- { id: A, text: a, correct: true }
|
||||
- { id: A, text: b }
|
||||
- id: q-a-001
|
||||
status: draft
|
||||
level: 1
|
||||
stem: s
|
||||
options:
|
||||
- { id: A, text: a, correct: true }
|
||||
- { id: B, text: b }
|
||||
"#,
|
||||
);
|
||||
let issues = b.validate(None);
|
||||
assert!(issues.iter().any(|i| i.contains("duplicate item id")));
|
||||
assert!(issues.iter().any(|i| i.contains("duplicate option id")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn level_and_process_must_agree() {
|
||||
let b = bank(
|
||||
r#"
|
||||
- id: q-a-001
|
||||
status: draft
|
||||
level: 3
|
||||
cognitive_process: recall
|
||||
stem: s
|
||||
options:
|
||||
- { id: A, text: a, correct: true }
|
||||
- { id: B, text: b }
|
||||
"#,
|
||||
);
|
||||
let issues = b.validate(None);
|
||||
assert!(
|
||||
issues.iter().any(|i| i.contains("belongs to level 1")),
|
||||
"{issues:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn approval_requires_full_specification() {
|
||||
let b = bank(
|
||||
r#"
|
||||
- id: q-a-001
|
||||
status: approved
|
||||
level: 1
|
||||
stem: s
|
||||
options:
|
||||
- { id: A, text: a, correct: true }
|
||||
- { id: B, text: b }
|
||||
"#,
|
||||
);
|
||||
let issues = b.validate(None);
|
||||
for want in [
|
||||
"cognitive_process",
|
||||
"learning objective",
|
||||
"source",
|
||||
"design block",
|
||||
] {
|
||||
assert!(
|
||||
issues.iter().any(|i| i.contains(want)),
|
||||
"expected a complaint about {want}, got {issues:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_credit_needs_a_written_defense() {
|
||||
let b = bank(
|
||||
r#"
|
||||
- id: q-a-001
|
||||
status: draft
|
||||
level: 5
|
||||
stem: s
|
||||
options:
|
||||
- { id: A, text: a, correct: true }
|
||||
- { id: B, text: b, credit: 0.5 }
|
||||
"#,
|
||||
);
|
||||
let issues = b.validate(None);
|
||||
assert!(issues.iter().any(|i| i.contains("no `defense`")));
|
||||
assert!(issues.iter().any(|i| i.contains("defensible: true")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn defaults_fill_in_items() {
|
||||
let src = r#"
|
||||
bank: { id: b, title: Bank }
|
||||
defaults:
|
||||
author: Alex
|
||||
points: 1.5
|
||||
topics: [kinetics]
|
||||
lectures: [L11]
|
||||
items:
|
||||
- id: q-a-001
|
||||
status: draft
|
||||
level: 1
|
||||
stem: s
|
||||
options:
|
||||
- { id: A, text: a, correct: true }
|
||||
- { id: B, text: b }
|
||||
- id: q-a-002
|
||||
status: draft
|
||||
level: 1
|
||||
stem: s
|
||||
author: Someone Else
|
||||
topics: [kinetics]
|
||||
sources: [{ lecture: L12 }]
|
||||
options:
|
||||
- { id: A, text: a, correct: true }
|
||||
- { id: B, text: b }
|
||||
"#;
|
||||
let mut b: BankFile = serde_yaml_ng::from_str(src).unwrap();
|
||||
b.apply_defaults();
|
||||
assert_eq!(b.items[0].author.as_deref(), Some("Alex"));
|
||||
assert_eq!(b.items[0].points, Some(1.5));
|
||||
assert_eq!(b.items[0].topics, vec!["kinetics"]);
|
||||
assert_eq!(b.items[0].sources[0].lecture, "L11");
|
||||
// Explicit values win, and topics are not duplicated.
|
||||
assert_eq!(b.items[1].author.as_deref(), Some("Someone Else"));
|
||||
assert_eq!(b.items[1].topics, vec!["kinetics"]);
|
||||
assert_eq!(b.items[1].sources[0].lecture, "L12");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cross_file_references_are_checked_against_the_course() {
|
||||
let course: CourseFile = serde_yaml_ng::from_str(
|
||||
r#"
|
||||
course: { code: X, title: Y, term: Z }
|
||||
lectures:
|
||||
L11: { title: Kinetics }
|
||||
learning_objectives:
|
||||
lo-known: { text: Do the thing, level_ceiling: 2 }
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
let b = bank(
|
||||
r#"
|
||||
- id: q-a-001
|
||||
status: draft
|
||||
level: 4
|
||||
stem: s
|
||||
learning_objectives: [lo-known, lo-unknown]
|
||||
sources: [{ lecture: L99 }]
|
||||
options:
|
||||
- { id: A, text: a, correct: true }
|
||||
- { id: B, text: b }
|
||||
"#,
|
||||
);
|
||||
let issues = b.validate(Some(&course));
|
||||
assert!(
|
||||
issues
|
||||
.iter()
|
||||
.any(|i| i.contains("unknown learning objective `lo-unknown`"))
|
||||
);
|
||||
assert!(issues.iter().any(|i| i.contains("unknown lecture `L99`")));
|
||||
assert!(
|
||||
issues.iter().any(|i| i.contains("exceeds the ceiling")),
|
||||
"{issues:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn history_versions_must_increase() {
|
||||
let b = bank(
|
||||
r#"
|
||||
- id: q-a-001
|
||||
version: 2
|
||||
status: draft
|
||||
level: 1
|
||||
stem: s
|
||||
options:
|
||||
- { id: A, text: a, correct: true }
|
||||
- { id: B, text: b }
|
||||
history:
|
||||
- { version: 2, date: 2026-01-01, change: second }
|
||||
- { version: 1, date: 2026-01-02, change: first }
|
||||
"#,
|
||||
);
|
||||
let issues = b.validate(None);
|
||||
assert!(issues.iter().any(|i| i.contains("does not increase")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn level_counts_exclude_drafts_and_bonuses() {
|
||||
let b = bank(
|
||||
r#"
|
||||
- id: q-a-001
|
||||
status: approved
|
||||
level: 1
|
||||
cognitive_process: recall
|
||||
stem: s
|
||||
learning_objectives: [lo]
|
||||
sources: [{ lecture: L1 }]
|
||||
design: { expected_difficulty: 0.8 }
|
||||
options:
|
||||
- { id: A, text: a, correct: true }
|
||||
- { id: B, text: b }
|
||||
- id: q-a-002
|
||||
status: draft
|
||||
level: 1
|
||||
stem: s
|
||||
options:
|
||||
- { id: A, text: a, correct: true }
|
||||
- { id: B, text: b }
|
||||
- id: q-a-003
|
||||
status: approved
|
||||
level: 5
|
||||
cognitive_process: generate
|
||||
bonus: true
|
||||
stem: s
|
||||
learning_objectives: [lo]
|
||||
sources: [{ lecture: L1 }]
|
||||
design: { expected_difficulty: 0.3 }
|
||||
options:
|
||||
- { id: A, text: a, correct: true }
|
||||
- { id: B, text: b }
|
||||
"#,
|
||||
);
|
||||
let counts = b.level_counts();
|
||||
assert_eq!(counts[&Level::Remember], 1);
|
||||
assert_eq!(counts[&Level::Create], 0, "bonus items are not scored");
|
||||
assert_eq!(b.assemblable().len(), 2);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,686 @@
|
||||
// SPDX-License-Identifier: Prosperity-3.0.0
|
||||
// Copyright Scientific Computing Studio
|
||||
// Source: https://git.scient.ing/education/coursebank
|
||||
|
||||
//! Loading a whole course at once, and reporting on what it contains.
|
||||
//!
|
||||
//! A [`Catalog`] is every bank in a course, indexed so that an item can be found
|
||||
//! by its global id (`bank::item`), and so that questions like "how many Apply
|
||||
//! level items do I have on lecture 12" have a cheap answer.
|
||||
//!
|
||||
//! The global id is the join key for everything downstream: assessment records
|
||||
//! reference it, response tables carry it, and usage history is keyed by it. Bank
|
||||
//! ids therefore have to be unique across a course, which the catalog enforces
|
||||
//! at load time rather than letting a silent collision merge two items.
|
||||
//!
|
||||
//! The coverage report is the part that changes how you write. It is easy to
|
||||
//! accumulate forty recall items and believe a topic is covered; a table showing
|
||||
//! that eleven objectives have no item above level 1 is harder to ignore.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::assessment::AssessmentFile;
|
||||
use crate::bank::BankFile;
|
||||
use crate::course::CourseFile;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::item::Item;
|
||||
use crate::layout::Layout;
|
||||
use crate::taxonomy::{Level, Status};
|
||||
use crate::yaml;
|
||||
|
||||
/// One item plus everything needed to locate it again.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Entry {
|
||||
/// The globally unique id, `bank::item`.
|
||||
pub uid: String,
|
||||
/// The bank id.
|
||||
pub bank: String,
|
||||
/// The file the item came from.
|
||||
pub path: PathBuf,
|
||||
/// Position within the bank file, for stable ordering.
|
||||
pub index: usize,
|
||||
/// The item itself.
|
||||
pub item: Item,
|
||||
}
|
||||
|
||||
impl Entry {
|
||||
/// The item's point value, resolving against the course policy.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `course` - the course whose policy supplies the default.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The point value.
|
||||
pub fn points(&self, course: &CourseFile) -> f64 {
|
||||
self.item.points(course.policy.points_per_item)
|
||||
}
|
||||
}
|
||||
|
||||
/// Every bank in a course, indexed.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Catalog {
|
||||
/// The course registry.
|
||||
pub course: CourseFile,
|
||||
/// The resolved directory layout.
|
||||
pub layout: Layout,
|
||||
/// Every item, in stable order: by bank id, then by position in the file.
|
||||
pub entries: Vec<Entry>,
|
||||
/// Bank metadata by bank id.
|
||||
pub banks: BTreeMap<String, crate::bank::BankMeta>,
|
||||
/// Map from global id to index into `entries`.
|
||||
index: BTreeMap<String, usize>,
|
||||
}
|
||||
|
||||
impl Catalog {
|
||||
/// Loads a whole course from its directory.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `root` - the course directory containing `course.yaml` and `banks/`.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The catalog.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns a load error for the course file or any bank, and
|
||||
/// [`Error::Invalid`] when two banks share an id or two items share a global
|
||||
/// id, since either makes the join key ambiguous.
|
||||
pub fn load(root: &Path) -> Result<Catalog> {
|
||||
let layout = Layout::new(root);
|
||||
let course = CourseFile::load(&layout.course_file())?;
|
||||
let mut catalog = Catalog {
|
||||
course,
|
||||
layout,
|
||||
entries: Vec::new(),
|
||||
banks: BTreeMap::new(),
|
||||
index: BTreeMap::new(),
|
||||
};
|
||||
|
||||
let mut problems = Vec::new();
|
||||
let mut files = yaml::list_yaml(&catalog.layout.banks())?;
|
||||
files.sort();
|
||||
|
||||
for path in files {
|
||||
let bank = BankFile::load_resolved(&path)?;
|
||||
let bank_id = bank.bank.id.clone();
|
||||
if let Some(existing) = catalog.banks.get(&bank_id) {
|
||||
problems.push(format!(
|
||||
"bank id `{bank_id}` is used by two files (`{}` and `{}`)",
|
||||
existing.title, bank.bank.title
|
||||
));
|
||||
continue;
|
||||
}
|
||||
catalog.banks.insert(bank_id.clone(), bank.bank.clone());
|
||||
for (i, item) in bank.items.into_iter().enumerate() {
|
||||
let uid = format!("{bank_id}::{}", item.id);
|
||||
if catalog.index.contains_key(&uid) {
|
||||
problems.push(format!("duplicate global item id `{uid}`"));
|
||||
continue;
|
||||
}
|
||||
catalog.index.insert(uid.clone(), catalog.entries.len());
|
||||
catalog.entries.push(Entry {
|
||||
uid,
|
||||
bank: bank_id.clone(),
|
||||
path: path.clone(),
|
||||
index: i,
|
||||
item,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if !problems.is_empty() {
|
||||
return Err(Error::Invalid(problems));
|
||||
}
|
||||
Ok(catalog)
|
||||
}
|
||||
|
||||
/// Looks up an item by global id.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `uid` - the global id, `bank::item`.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The entry, or `None`.
|
||||
pub fn get(&self, uid: &str) -> Option<&Entry> {
|
||||
self.index.get(uid).map(|i| &self.entries[*i])
|
||||
}
|
||||
|
||||
/// Looks up an item by global id, erroring when absent.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `uid` - the global id.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The entry.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`Error::Unresolved`] when no such item exists.
|
||||
pub fn require(&self, uid: &str) -> Result<&Entry> {
|
||||
self.get(uid).ok_or_else(|| Error::Unresolved {
|
||||
kind: "item",
|
||||
id: uid.to_string(),
|
||||
context: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Resolves a possibly-unqualified id to a global id.
|
||||
///
|
||||
/// Typing `q-mm-kinetics-001` on the command line should work when that id is
|
||||
/// unambiguous across the course, because remembering which bank a question
|
||||
/// lives in is exactly the sort of bookkeeping this tool exists to remove.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `id` - a global id, or a bare item id.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The global id.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`Error::Unresolved`] when nothing matches, or [`Error::Usage`]
|
||||
/// when a bare id matches items in more than one bank.
|
||||
pub fn resolve(&self, id: &str) -> Result<String> {
|
||||
if self.index.contains_key(id) {
|
||||
return Ok(id.to_string());
|
||||
}
|
||||
let matches: Vec<&Entry> = self.entries.iter().filter(|e| e.item.id == id).collect();
|
||||
match matches.len() {
|
||||
0 => Err(Error::Unresolved {
|
||||
kind: "item",
|
||||
id: id.to_string(),
|
||||
context: None,
|
||||
}),
|
||||
1 => Ok(matches[0].uid.clone()),
|
||||
_ => Err(Error::usage(format!(
|
||||
"`{id}` is ambiguous; it exists in {}. Use the full `bank::item` form.",
|
||||
matches
|
||||
.iter()
|
||||
.map(|e| e.bank.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
/// Every item that may be placed on a graded assessment.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// References to approved, unretired entries.
|
||||
pub fn assemblable(&self) -> Vec<&Entry> {
|
||||
self.entries
|
||||
.iter()
|
||||
.filter(|e| e.item.is_assemblable())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Validates every bank against the course registry.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Problems, prefixed with the file they came from.
|
||||
pub fn validate(&self) -> Result<Vec<String>> {
|
||||
let mut issues: Vec<String> = self
|
||||
.course
|
||||
.validate()
|
||||
.into_iter()
|
||||
.map(|m| format!("course.yaml: {m}"))
|
||||
.collect();
|
||||
|
||||
for path in yaml::list_yaml(&self.layout.banks())? {
|
||||
let bank = BankFile::load_resolved(&path)?;
|
||||
let name = path
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or("?")
|
||||
.to_string();
|
||||
issues.extend(
|
||||
bank.validate(Some(&self.course))
|
||||
.into_iter()
|
||||
.map(|m| format!("{name}: {m}")),
|
||||
);
|
||||
}
|
||||
Ok(issues)
|
||||
}
|
||||
|
||||
/// Validates a record's internal invariants, then its references against this
|
||||
/// catalog: unknown items, keys that drifted, and fingerprints showing the
|
||||
/// item was reworded since it was administered.
|
||||
pub fn validate_record(&self, record: &AssessmentFile) -> Vec<String> {
|
||||
let mut issues = record.validate();
|
||||
for p in &record.items {
|
||||
match self.get(&p.item) {
|
||||
None => issues.push(format!("question {}: unknown item `{}`", p.number, p.item)),
|
||||
Some(entry) => {
|
||||
if let Some(fp) = &p.fingerprint {
|
||||
if *fp != entry.item.fingerprint() {
|
||||
issues.push(format!(
|
||||
"question {} ({}): the item has been edited since this assessment; \
|
||||
statistics from this administration describe the older wording",
|
||||
p.number, p.item
|
||||
));
|
||||
}
|
||||
}
|
||||
if !p.key.is_empty() && p.key != entry.item.key_letters() {
|
||||
issues.push(format!(
|
||||
"question {} ({}): the recorded key {:?} differs from the item's current key {:?}",
|
||||
p.number, p.item, p.key, entry.item.key_letters()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
issues
|
||||
}
|
||||
|
||||
/// Estimated working time for a record, in minutes.
|
||||
pub fn estimated_minutes(&self, record: &AssessmentFile) -> f64 {
|
||||
let seconds: f64 = record
|
||||
.items
|
||||
.iter()
|
||||
.filter_map(|p| self.get(&p.item))
|
||||
.map(|e| e.item.expected_seconds())
|
||||
.sum();
|
||||
seconds / 60.0
|
||||
}
|
||||
|
||||
/// Counts of assemblable, non-bonus items by level.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// A map from level to count, with every level present.
|
||||
pub fn level_counts(&self) -> BTreeMap<Level, usize> {
|
||||
let mut out: BTreeMap<Level, usize> = Level::ALL.iter().map(|l| (*l, 0)).collect();
|
||||
for e in self.assemblable().iter().filter(|e| !e.item.bonus) {
|
||||
*out.entry(e.item.level).or_insert(0) += 1;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Counts of items by workflow status, over the whole course.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// A map from status to count.
|
||||
pub fn status_counts(&self) -> BTreeMap<Status, usize> {
|
||||
let mut out: BTreeMap<Status, usize> = BTreeMap::new();
|
||||
for e in &self.entries {
|
||||
*out.entry(e.item.status).or_insert(0) += 1;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Items that measure a given objective.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `objective` - the objective id.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Matching entries.
|
||||
pub fn by_objective(&self, objective: &str) -> Vec<&Entry> {
|
||||
self.entries
|
||||
.iter()
|
||||
.filter(|e| e.item.learning_objectives.iter().any(|o| o == objective))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Items sourced to a given lecture.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `lecture` - the lecture id.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Matching entries.
|
||||
pub fn by_lecture(&self, lecture: &str) -> Vec<&Entry> {
|
||||
self.entries
|
||||
.iter()
|
||||
.filter(|e| e.item.sources.iter().any(|s| s.lecture == lecture))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Items carrying a given topic tag.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `topic` - the topic tag.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Matching entries.
|
||||
pub fn by_topic(&self, topic: &str) -> Vec<&Entry> {
|
||||
self.entries
|
||||
.iter()
|
||||
.filter(|e| e.item.topics.iter().any(|t| t == topic))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Every topic tag in use, with counts.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// A map from topic to item count.
|
||||
pub fn topics(&self) -> BTreeMap<String, usize> {
|
||||
let mut out: BTreeMap<String, usize> = BTreeMap::new();
|
||||
for e in &self.entries {
|
||||
for t in &e.item.topics {
|
||||
*out.entry(t.clone()).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Builds the coverage report.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// One row per assessed objective plus a list of course-wide gaps.
|
||||
pub fn coverage(&self) -> Coverage {
|
||||
let mut rows = Vec::new();
|
||||
for id in self.course.objectives_in_order() {
|
||||
let obj = &self.course.learning_objectives[&id];
|
||||
if !obj.assessed {
|
||||
continue;
|
||||
}
|
||||
let items = self.by_objective(&id);
|
||||
let usable: Vec<&&Entry> = items.iter().filter(|e| e.item.is_assemblable()).collect();
|
||||
let mut levels: BTreeSet<Level> = BTreeSet::new();
|
||||
for e in &usable {
|
||||
levels.insert(e.item.level);
|
||||
}
|
||||
rows.push(CoverageRow {
|
||||
objective: id.clone(),
|
||||
text: obj.text.clone(),
|
||||
unit: obj.unit.clone(),
|
||||
total: items.len(),
|
||||
assemblable: usable.len(),
|
||||
max_level: levels.iter().next_back().copied(),
|
||||
levels: levels.into_iter().collect(),
|
||||
ceiling: obj.level_ceiling,
|
||||
});
|
||||
}
|
||||
|
||||
let mut gaps = Vec::new();
|
||||
for row in &rows {
|
||||
if row.total == 0 {
|
||||
gaps.push(Gap::Uncovered(row.objective.clone()));
|
||||
} else if row.assemblable == 0 {
|
||||
gaps.push(Gap::NoApprovedItems(row.objective.clone()));
|
||||
} else if row.assemblable == 1 {
|
||||
gaps.push(Gap::SingleItem(row.objective.clone()));
|
||||
}
|
||||
// An objective assessed only at the recall level is the most common
|
||||
// and most consequential blind spot: it looks covered in a count and
|
||||
// is not covered in fact.
|
||||
if row.assemblable > 0 && row.max_level == Some(Level::Remember) {
|
||||
if let Some(ceiling) = row.ceiling {
|
||||
if ceiling > Level::Remember {
|
||||
gaps.push(Gap::RecallOnly(row.objective.clone()));
|
||||
}
|
||||
} else {
|
||||
gaps.push(Gap::RecallOnly(row.objective.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for lec_id in self.course.lectures.keys() {
|
||||
if self
|
||||
.by_lecture(lec_id)
|
||||
.iter()
|
||||
.filter(|e| e.item.is_assemblable())
|
||||
.count()
|
||||
== 0
|
||||
{
|
||||
gaps.push(Gap::LectureUnassessed(lec_id.clone()));
|
||||
}
|
||||
}
|
||||
|
||||
// Items with no objective at all cannot appear in any student report.
|
||||
for e in &self.entries {
|
||||
if e.item.learning_objectives.is_empty() && e.item.is_assemblable() {
|
||||
gaps.push(Gap::ItemWithoutObjective(e.uid.clone()));
|
||||
}
|
||||
}
|
||||
|
||||
Coverage { rows, gaps }
|
||||
}
|
||||
}
|
||||
|
||||
/// The coverage report.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Coverage {
|
||||
/// One row per assessed objective.
|
||||
pub rows: Vec<CoverageRow>,
|
||||
/// Course-wide gaps worth acting on.
|
||||
pub gaps: Vec<Gap>,
|
||||
}
|
||||
|
||||
/// Coverage of one objective.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CoverageRow {
|
||||
/// The objective id.
|
||||
pub objective: String,
|
||||
/// The objective text.
|
||||
pub text: String,
|
||||
/// The unit it belongs to.
|
||||
pub unit: Option<String>,
|
||||
/// Items referencing it, at any status.
|
||||
pub total: usize,
|
||||
/// Items that could actually be used.
|
||||
pub assemblable: usize,
|
||||
/// The highest level assessed.
|
||||
pub max_level: Option<Level>,
|
||||
/// Every level assessed.
|
||||
pub levels: Vec<Level>,
|
||||
/// The declared ceiling, when set.
|
||||
pub ceiling: Option<Level>,
|
||||
}
|
||||
|
||||
/// A specific, actionable hole in the item pool.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Gap {
|
||||
/// An assessed objective with no items at all.
|
||||
Uncovered(String),
|
||||
/// An objective whose items are all drafts or retired.
|
||||
NoApprovedItems(String),
|
||||
/// An objective resting on a single item, so one bad item hides it entirely.
|
||||
SingleItem(String),
|
||||
/// An objective assessed only at the recall level despite a higher ceiling.
|
||||
RecallOnly(String),
|
||||
/// A lecture with no usable items.
|
||||
LectureUnassessed(String),
|
||||
/// A usable item that references no objective, so it can never appear in a
|
||||
/// student report.
|
||||
ItemWithoutObjective(String),
|
||||
}
|
||||
|
||||
impl Gap {
|
||||
/// A one-line description.
|
||||
pub fn message(&self) -> String {
|
||||
match self {
|
||||
Gap::Uncovered(id) => format!("objective `{id}` has no items"),
|
||||
Gap::NoApprovedItems(id) => {
|
||||
format!("objective `{id}` has items but none are approved")
|
||||
}
|
||||
Gap::SingleItem(id) => {
|
||||
format!("objective `{id}` rests on a single item; one bad item hides it")
|
||||
}
|
||||
Gap::RecallOnly(id) => {
|
||||
format!("objective `{id}` is assessed only at level 1")
|
||||
}
|
||||
Gap::LectureUnassessed(id) => format!("lecture `{id}` has no usable items"),
|
||||
Gap::ItemWithoutObjective(uid) => {
|
||||
format!("item `{uid}` has no learning objective, so it cannot appear in a report")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// How much attention the gap deserves.
|
||||
pub fn severity(&self) -> Severity {
|
||||
match self {
|
||||
Gap::Uncovered(_) | Gap::NoApprovedItems(_) => Severity::High,
|
||||
Gap::RecallOnly(_) | Gap::ItemWithoutObjective(_) => Severity::Medium,
|
||||
Gap::SingleItem(_) | Gap::LectureUnassessed(_) => Severity::Low,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// How much attention a finding deserves.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub enum Severity {
|
||||
/// Worth knowing.
|
||||
Low,
|
||||
/// Worth scheduling.
|
||||
Medium,
|
||||
/// Worth fixing before the next assessment.
|
||||
High,
|
||||
}
|
||||
|
||||
impl Severity {
|
||||
/// A short label for report output.
|
||||
pub fn label(self) -> &'static str {
|
||||
match self {
|
||||
Severity::Low => "low",
|
||||
Severity::Medium => "medium",
|
||||
Severity::High => "high",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn write_course(dir: &Path, extra_lo: &str) {
|
||||
std::fs::create_dir_all(dir.join("banks")).unwrap();
|
||||
std::fs::write(
|
||||
dir.join("course.yaml"),
|
||||
format!(
|
||||
r#"
|
||||
course: {{ code: TEST 101, title: Testing, term: Fall 2026 }}
|
||||
lectures:
|
||||
L01: {{ title: One }}
|
||||
L02: {{ title: Two }}
|
||||
learning_objectives:
|
||||
lo-covered: {{ text: Covered objective, lectures: [L01] }}
|
||||
lo-bare: {{ text: Uncovered objective, lectures: [L02] }}
|
||||
{extra_lo}
|
||||
"#
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn write_bank(dir: &Path, name: &str, body: &str) {
|
||||
std::fs::write(dir.join("banks").join(name), body).unwrap();
|
||||
}
|
||||
|
||||
fn tmp(tag: &str) -> PathBuf {
|
||||
let p = std::env::temp_dir().join(format!("coursebank-test-{tag}-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&p);
|
||||
std::fs::create_dir_all(&p).unwrap();
|
||||
p
|
||||
}
|
||||
|
||||
const APPROVED: &str = r#"
|
||||
bank:
|
||||
id: b1
|
||||
title: First bank
|
||||
items:
|
||||
- id: q-x-001
|
||||
status: approved
|
||||
level: 1
|
||||
cognitive_process: recall
|
||||
stem: What is x?
|
||||
learning_objectives: [lo-covered]
|
||||
sources: [{ lecture: L01 }]
|
||||
design: { expected_difficulty: 0.8 }
|
||||
options:
|
||||
- { id: A, text: right, correct: true }
|
||||
- { id: B, text: wrong }
|
||||
"#;
|
||||
|
||||
#[test]
|
||||
fn loads_and_indexes_items() {
|
||||
let dir = tmp("load");
|
||||
write_course(&dir, "");
|
||||
write_bank(&dir, "b1.yaml", APPROVED);
|
||||
|
||||
let cat = Catalog::load(&dir).expect("catalog loads");
|
||||
assert_eq!(cat.entries.len(), 1);
|
||||
assert_eq!(cat.entries[0].uid, "b1::q-x-001");
|
||||
assert!(cat.get("b1::q-x-001").is_some());
|
||||
assert_eq!(cat.resolve("q-x-001").unwrap(), "b1::q-x-001");
|
||||
assert!(cat.validate().unwrap().is_empty());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn duplicate_bank_ids_are_fatal() {
|
||||
let dir = tmp("dupbank");
|
||||
write_course(&dir, "");
|
||||
write_bank(&dir, "a.yaml", APPROVED);
|
||||
write_bank(&dir, "b.yaml", APPROVED);
|
||||
let err = Catalog::load(&dir).expect_err("must reject colliding bank ids");
|
||||
assert!(format!("{err}").contains("two files"));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ambiguous_bare_ids_are_rejected() {
|
||||
let dir = tmp("ambig");
|
||||
write_course(&dir, "");
|
||||
write_bank(&dir, "a.yaml", APPROVED);
|
||||
write_bank(&dir, "b.yaml", &APPROVED.replace("id: b1", "id: b2"));
|
||||
let cat = Catalog::load(&dir).expect("distinct banks load");
|
||||
assert_eq!(cat.entries.len(), 2);
|
||||
let err = cat.resolve("q-x-001").expect_err("bare id is ambiguous");
|
||||
assert!(format!("{err}").contains("ambiguous"));
|
||||
// The fully qualified form still works.
|
||||
assert_eq!(cat.resolve("b2::q-x-001").unwrap(), "b2::q-x-001");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn coverage_finds_real_gaps() {
|
||||
let dir = tmp("coverage");
|
||||
write_course(&dir, "");
|
||||
write_bank(&dir, "b1.yaml", APPROVED);
|
||||
let cat = Catalog::load(&dir).unwrap();
|
||||
let cov = cat.coverage();
|
||||
|
||||
assert_eq!(cov.rows.len(), 2);
|
||||
assert!(cov.gaps.contains(&Gap::Uncovered("lo-bare".into())));
|
||||
assert!(cov.gaps.contains(&Gap::SingleItem("lo-covered".into())));
|
||||
assert!(cov.gaps.contains(&Gap::RecallOnly("lo-covered".into())));
|
||||
assert!(cov.gaps.contains(&Gap::LectureUnassessed("L02".into())));
|
||||
assert_eq!(Gap::Uncovered("x".into()).severity(), Severity::High);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recall_only_respects_a_recall_ceiling() {
|
||||
let dir = tmp("ceiling");
|
||||
// An objective you only ever intend to test at level 1 is not a gap.
|
||||
write_course(&dir, " lo-ceil: { text: Just recall, level_ceiling: 1 }");
|
||||
write_bank(&dir, "b1.yaml", &APPROVED.replace("lo-covered", "lo-ceil"));
|
||||
let cat = Catalog::load(&dir).unwrap();
|
||||
let cov = cat.coverage();
|
||||
assert!(!cov.gaps.contains(&Gap::RecallOnly("lo-ceil".into())));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,709 @@
|
||||
// SPDX-License-Identifier: Prosperity-3.0.0
|
||||
// Copyright Scientific Computing Studio
|
||||
// Source: https://git.scient.ing/education/coursebank
|
||||
|
||||
//! The course file: identity plus the registries every bank references.
|
||||
//!
|
||||
//! Learning objectives and lectures are declared once, in `course.yaml`, and
|
||||
//! referenced by id from items. That is the single most load-bearing decision in
|
||||
//! the schema. It means an objective's wording lives in exactly one place, so
|
||||
//! rewording it updates every report; it means a report can name what a student
|
||||
//! missed by objective rather than by question number; and it means a dangling
|
||||
//! reference is a hard error instead of a silently misspelled string that splits
|
||||
//! your coverage table into two near-identical rows.
|
||||
//!
|
||||
//! The course file also declares the term. Items live across terms, so the term
|
||||
//! belongs to the course and the administration, never to the item.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::Path;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::date::Date;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::taxonomy::Level;
|
||||
use crate::yaml;
|
||||
|
||||
/// The schema version this build of the tool writes.
|
||||
pub const SCHEMA_VERSION: &str = "1.0";
|
||||
|
||||
/// The canonical file name inside a course directory.
|
||||
pub const COURSE_FILE: &str = "course.yaml";
|
||||
|
||||
/// A whole course file.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct CourseFile {
|
||||
/// Schema version this file targets.
|
||||
#[serde(
|
||||
default = "default_version",
|
||||
deserialize_with = "yaml::flexible_string"
|
||||
)]
|
||||
pub schema_version: String,
|
||||
|
||||
/// Course identity.
|
||||
pub course: Course,
|
||||
|
||||
/// Grading and assembly conventions that apply course-wide.
|
||||
#[serde(default)]
|
||||
pub policy: Policy,
|
||||
|
||||
/// Units or modules, ordered as taught.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub units: Vec<Unit>,
|
||||
|
||||
/// Lectures or sessions, keyed by id such as `L11`.
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub lectures: BTreeMap<String, Lecture>,
|
||||
|
||||
/// Learning objectives, keyed by id such as `lo-mm-kinetics`.
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub learning_objectives: BTreeMap<String, Objective>,
|
||||
|
||||
/// Shared stimuli for case-based testlets, keyed by id.
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub stimuli: BTreeMap<String, Stimulus>,
|
||||
}
|
||||
|
||||
/// Course identity.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Course {
|
||||
/// Catalog code, e.g. `BIOSC 1540`.
|
||||
pub code: String,
|
||||
/// Full title.
|
||||
pub title: String,
|
||||
/// The term this instance runs in, e.g. `Spring 2026`.
|
||||
pub term: String,
|
||||
/// Granting institution.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub institution: Option<String>,
|
||||
/// Instructors of record.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub instructors: Vec<String>,
|
||||
/// A short slug used in generated file names; derived from `code` if absent.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub slug: Option<String>,
|
||||
}
|
||||
|
||||
impl Course {
|
||||
/// A filesystem-safe short name for this course.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The declared slug, or one derived from the course code.
|
||||
pub fn slug(&self) -> String {
|
||||
match &self.slug {
|
||||
Some(s) => s.clone(),
|
||||
None => slugify(&self.code),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Course-wide conventions, so they are stated once rather than per assessment.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Policy {
|
||||
/// Default points per scored item.
|
||||
#[serde(default = "one")]
|
||||
pub points_per_item: f64,
|
||||
/// Default number of options an item should offer.
|
||||
#[serde(default = "four")]
|
||||
pub options_per_item: usize,
|
||||
/// Levels that may carry bonus rather than scored items.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub bonus_levels: Vec<Level>,
|
||||
/// Whether partial credit for defensible distractors is permitted at all.
|
||||
#[serde(default = "yes")]
|
||||
pub allow_partial_credit: bool,
|
||||
/// The lowest level at which a defensible wrong answer is considered
|
||||
/// plausible. Awarding credit below this is flagged, because at the recall
|
||||
/// level a "reasonable wrong answer" usually means the item is unclear.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub partial_credit_floor_level: Option<Level>,
|
||||
/// Mastery threshold for objective-level reporting, as a proportion.
|
||||
#[serde(default = "mastery_default")]
|
||||
pub mastery_threshold: f64,
|
||||
/// The fewest items on an objective before a report will call it mastered.
|
||||
#[serde(default = "two_usize")]
|
||||
pub min_items_for_mastery: usize,
|
||||
}
|
||||
|
||||
impl Default for Policy {
|
||||
fn default() -> Policy {
|
||||
Policy {
|
||||
points_per_item: 1.0,
|
||||
options_per_item: 4,
|
||||
bonus_levels: Vec::new(),
|
||||
allow_partial_credit: true,
|
||||
partial_credit_floor_level: None,
|
||||
mastery_threshold: mastery_default(),
|
||||
min_items_for_mastery: 2,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A unit or module of the course.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Unit {
|
||||
/// Stable id, referenced by lectures and objectives.
|
||||
pub id: String,
|
||||
/// Human title.
|
||||
pub title: String,
|
||||
/// Optional longer description.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub description: Option<String>,
|
||||
}
|
||||
|
||||
/// A lecture or class session.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Lecture {
|
||||
/// Human title.
|
||||
pub title: String,
|
||||
/// The date it ran, when known.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub date: Option<Date>,
|
||||
/// The unit it belongs to.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub unit: Option<String>,
|
||||
/// Where the slides live, for study guidance in student reports.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub slides_url: Option<String>,
|
||||
/// Assigned readings for the session.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub readings: Vec<String>,
|
||||
}
|
||||
|
||||
/// A learning objective.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Objective {
|
||||
/// The objective as you would state it to students. Reports quote this
|
||||
/// verbatim, so write it in the second person and start with a verb.
|
||||
pub text: String,
|
||||
/// The unit it belongs to.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub unit: Option<String>,
|
||||
/// The lectures that develop it.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub lectures: Vec<String>,
|
||||
/// The highest level you intend to assess this objective at. Assembling an
|
||||
/// item above the ceiling is a warning: either the item overreaches or the
|
||||
/// ceiling needs raising.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub level_ceiling: Option<Level>,
|
||||
/// Objectives that must be secure before this one is reachable. Student
|
||||
/// reports walk this backwards to suggest where to start reviewing.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub prerequisites: Vec<String>,
|
||||
/// Free-form tags.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub tags: Vec<String>,
|
||||
/// Whether this objective is assessed at all, or is aspirational.
|
||||
#[serde(default = "yes")]
|
||||
pub assessed: bool,
|
||||
}
|
||||
|
||||
/// A shared stem or vignette used by several items.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Stimulus {
|
||||
/// The vignette body, in the markup described in `docs/AUTHORING.md`.
|
||||
pub body: String,
|
||||
/// An image or data file to reproduce alongside it.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub asset: Option<String>,
|
||||
/// A caption for the asset.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub caption: Option<String>,
|
||||
}
|
||||
|
||||
impl CourseFile {
|
||||
/// Loads a course file from disk without validating cross-references.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `path` - path to `course.yaml`.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The parsed course file.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`Error::Io`] if unreadable and [`Error::Yaml`] if it does not
|
||||
/// match the schema. Unknown keys are errors, so a misspelled field is
|
||||
/// caught rather than ignored.
|
||||
pub fn load(path: &Path) -> Result<CourseFile> {
|
||||
yaml::read(path)
|
||||
}
|
||||
|
||||
/// Finds and loads the course file for a course directory.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `dir` - the course directory.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The parsed course file.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Propagates load errors, including absence of `course.yaml`.
|
||||
pub fn load_dir(dir: &Path) -> Result<CourseFile> {
|
||||
CourseFile::load(&dir.join(COURSE_FILE))
|
||||
}
|
||||
|
||||
/// Writes the course file back out as YAML.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `path` - destination path.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`Error::Io`] on a write failure.
|
||||
pub fn save(&self, path: &Path) -> Result<()> {
|
||||
yaml::write(path, self)
|
||||
}
|
||||
|
||||
/// Checks internal consistency of the registries.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Every problem found, empty when the file is sound.
|
||||
pub fn validate(&self) -> Vec<String> {
|
||||
let mut issues = Vec::new();
|
||||
|
||||
if self.course.code.trim().is_empty() {
|
||||
issues.push("course.code is empty".into());
|
||||
}
|
||||
if self.course.title.trim().is_empty() {
|
||||
issues.push("course.title is empty".into());
|
||||
}
|
||||
if self.course.term.trim().is_empty() {
|
||||
issues.push("course.term is empty".into());
|
||||
}
|
||||
if !(0.0..=1.0).contains(&self.policy.mastery_threshold) {
|
||||
issues.push(format!(
|
||||
"policy.mastery_threshold must be between 0 and 1, got {}",
|
||||
self.policy.mastery_threshold
|
||||
));
|
||||
}
|
||||
|
||||
let unit_ids: Vec<&String> = self.units.iter().map(|u| &u.id).collect();
|
||||
let mut unit_counts: BTreeMap<&str, usize> = BTreeMap::new();
|
||||
for u in &self.units {
|
||||
*unit_counts.entry(u.id.as_str()).or_insert(0) += 1;
|
||||
}
|
||||
for (id, n) in &unit_counts {
|
||||
if *n > 1 {
|
||||
issues.push(format!("units: duplicate id `{id}` declared {n} times"));
|
||||
}
|
||||
}
|
||||
|
||||
for (id, lec) in &self.lectures {
|
||||
if lec.title.trim().is_empty() {
|
||||
issues.push(format!("lecture `{id}`: empty title"));
|
||||
}
|
||||
if let Some(u) = &lec.unit {
|
||||
if !unit_ids.contains(&u) {
|
||||
issues.push(format!("lecture `{id}`: unknown unit `{u}`"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (id, lo) in &self.learning_objectives {
|
||||
if lo.text.trim().is_empty() {
|
||||
issues.push(format!("objective `{id}`: empty text"));
|
||||
}
|
||||
if let Some(u) = &lo.unit {
|
||||
if !unit_ids.contains(&u) {
|
||||
issues.push(format!("objective `{id}`: unknown unit `{u}`"));
|
||||
}
|
||||
}
|
||||
for lec in &lo.lectures {
|
||||
if !self.lectures.contains_key(lec) {
|
||||
issues.push(format!("objective `{id}`: unknown lecture `{lec}`"));
|
||||
}
|
||||
}
|
||||
for pre in &lo.prerequisites {
|
||||
if !self.learning_objectives.contains_key(pre) {
|
||||
issues.push(format!(
|
||||
"objective `{id}`: unknown prerequisite objective `{pre}`"
|
||||
));
|
||||
}
|
||||
if pre == id {
|
||||
issues.push(format!("objective `{id}`: lists itself as a prerequisite"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
issues.extend(self.prerequisite_cycles());
|
||||
issues
|
||||
}
|
||||
|
||||
/// Detects cycles in the objective prerequisite graph.
|
||||
///
|
||||
/// A cycle would make a study-order suggestion loop forever, so it is worth
|
||||
/// catching at validation time rather than at report time.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// One message per objective that participates in a cycle.
|
||||
fn prerequisite_cycles(&self) -> Vec<String> {
|
||||
// Iterative depth-first search with three colors.
|
||||
#[derive(PartialEq, Clone, Copy)]
|
||||
enum Mark {
|
||||
White,
|
||||
Gray,
|
||||
Black,
|
||||
}
|
||||
let mut color: BTreeMap<&String, Mark> = self
|
||||
.learning_objectives
|
||||
.keys()
|
||||
.map(|k| (k, Mark::White))
|
||||
.collect();
|
||||
let mut issues = Vec::new();
|
||||
|
||||
for root in self.learning_objectives.keys() {
|
||||
if color[root] != Mark::White {
|
||||
continue;
|
||||
}
|
||||
let mut stack: Vec<(&String, usize)> = vec![(root, 0)];
|
||||
color.insert(root, Mark::Gray);
|
||||
while let Some((node, idx)) = stack.pop() {
|
||||
let empty: Vec<String> = Vec::new();
|
||||
let pres = self
|
||||
.learning_objectives
|
||||
.get(node)
|
||||
.map(|o| &o.prerequisites)
|
||||
.unwrap_or(&empty);
|
||||
if idx < pres.len() {
|
||||
stack.push((node, idx + 1));
|
||||
if let Some((key, _)) = self.learning_objectives.get_key_value(&pres[idx]) {
|
||||
match color[key] {
|
||||
Mark::Gray => issues.push(format!(
|
||||
"objective `{node}`: prerequisite cycle through `{key}`"
|
||||
)),
|
||||
Mark::White => {
|
||||
color.insert(key, Mark::Gray);
|
||||
stack.push((key, 0));
|
||||
}
|
||||
Mark::Black => {}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
color.insert(node, Mark::Black);
|
||||
}
|
||||
}
|
||||
}
|
||||
issues
|
||||
}
|
||||
|
||||
/// Looks up an objective, erroring on a dangling reference.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `id` - the objective id.
|
||||
/// * `context` - what referenced it, for the error message.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The objective.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`Error::Unresolved`] when the id is not registered.
|
||||
pub fn objective(&self, id: &str, context: &str) -> Result<&Objective> {
|
||||
self.learning_objectives
|
||||
.get(id)
|
||||
.ok_or_else(|| Error::Unresolved {
|
||||
kind: "learning objective",
|
||||
id: id.to_string(),
|
||||
context: Some(context.to_string()),
|
||||
})
|
||||
}
|
||||
|
||||
/// Looks up a lecture, erroring on a dangling reference.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `id` - the lecture id.
|
||||
/// * `context` - what referenced it, for the error message.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The lecture.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`Error::Unresolved`] when the id is not registered.
|
||||
pub fn lecture(&self, id: &str, context: &str) -> Result<&Lecture> {
|
||||
self.lectures.get(id).ok_or_else(|| Error::Unresolved {
|
||||
kind: "lecture",
|
||||
id: id.to_string(),
|
||||
context: Some(context.to_string()),
|
||||
})
|
||||
}
|
||||
|
||||
/// The objective text, or the bare id when unregistered.
|
||||
///
|
||||
/// Report rendering uses this so a missing objective degrades to a readable
|
||||
/// label instead of failing a whole report.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `id` - the objective id.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The display text.
|
||||
pub fn objective_text(&self, id: &str) -> String {
|
||||
self.learning_objectives
|
||||
.get(id)
|
||||
.map(|o| o.text.clone())
|
||||
.unwrap_or_else(|| id.to_string())
|
||||
}
|
||||
|
||||
/// Objectives in a stable teaching order: by unit as declared, then by id.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Objective ids in report order.
|
||||
pub fn objectives_in_order(&self) -> Vec<String> {
|
||||
let unit_rank: BTreeMap<&str, usize> = self
|
||||
.units
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, u)| (u.id.as_str(), i))
|
||||
.collect();
|
||||
let mut ids: Vec<&String> = self.learning_objectives.keys().collect();
|
||||
ids.sort_by_key(|id| {
|
||||
let lo = &self.learning_objectives[*id];
|
||||
let rank = lo
|
||||
.unit
|
||||
.as_deref()
|
||||
.and_then(|u| unit_rank.get(u).copied())
|
||||
.unwrap_or(usize::MAX);
|
||||
(rank, (*id).clone())
|
||||
});
|
||||
ids.into_iter().cloned().collect()
|
||||
}
|
||||
|
||||
/// A skeleton course file for `coursebank init`.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `code` - the course code.
|
||||
/// * `title` - the course title.
|
||||
/// * `term` - the term.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// A minimal but valid course file.
|
||||
pub fn skeleton(code: &str, title: &str, term: &str) -> CourseFile {
|
||||
let mut lectures = BTreeMap::new();
|
||||
lectures.insert(
|
||||
"L01".to_string(),
|
||||
Lecture {
|
||||
title: "Course introduction".to_string(),
|
||||
date: None,
|
||||
unit: Some("u-intro".to_string()),
|
||||
slides_url: None,
|
||||
readings: Vec::new(),
|
||||
},
|
||||
);
|
||||
let mut los = BTreeMap::new();
|
||||
los.insert(
|
||||
"lo-example".to_string(),
|
||||
Objective {
|
||||
text: "Replace this with an objective stated as a student action.".to_string(),
|
||||
unit: Some("u-intro".to_string()),
|
||||
lectures: vec!["L01".to_string()],
|
||||
level_ceiling: Some(Level::Understand),
|
||||
prerequisites: Vec::new(),
|
||||
tags: Vec::new(),
|
||||
assessed: true,
|
||||
},
|
||||
);
|
||||
CourseFile {
|
||||
schema_version: SCHEMA_VERSION.to_string(),
|
||||
course: Course {
|
||||
code: code.to_string(),
|
||||
title: title.to_string(),
|
||||
term: term.to_string(),
|
||||
institution: None,
|
||||
instructors: Vec::new(),
|
||||
slug: None,
|
||||
},
|
||||
policy: Policy::default(),
|
||||
units: vec![Unit {
|
||||
id: "u-intro".to_string(),
|
||||
title: "Introduction".to_string(),
|
||||
description: None,
|
||||
}],
|
||||
lectures,
|
||||
learning_objectives: los,
|
||||
stimuli: BTreeMap::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Lowercases and hyphenates a string for use in file names and ids.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `s` - the text to convert.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// A slug of lowercase alphanumerics separated by single hyphens.
|
||||
pub fn slugify(s: &str) -> String {
|
||||
let mut out = String::new();
|
||||
let mut prev_dash = true;
|
||||
for ch in s.chars() {
|
||||
if ch.is_ascii_alphanumeric() {
|
||||
out.push(ch.to_ascii_lowercase());
|
||||
prev_dash = false;
|
||||
} else if !prev_dash {
|
||||
out.push('-');
|
||||
prev_dash = true;
|
||||
}
|
||||
}
|
||||
while out.ends_with('-') {
|
||||
out.pop();
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn default_version() -> String {
|
||||
SCHEMA_VERSION.to_string()
|
||||
}
|
||||
fn one() -> f64 {
|
||||
1.0
|
||||
}
|
||||
fn four() -> usize {
|
||||
4
|
||||
}
|
||||
fn two_usize() -> usize {
|
||||
2
|
||||
}
|
||||
fn yes() -> bool {
|
||||
true
|
||||
}
|
||||
fn mastery_default() -> f64 {
|
||||
0.75
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn parse(src: &str) -> CourseFile {
|
||||
serde_yaml_ng::from_str(src).expect("course parses")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn minimal_course_parses_with_defaults() {
|
||||
let c = parse(
|
||||
r#"
|
||||
course:
|
||||
code: BIOSC 1540
|
||||
title: Computational Biology
|
||||
term: Spring 2026
|
||||
"#,
|
||||
);
|
||||
assert_eq!(c.schema_version, "1.0");
|
||||
assert_eq!(c.policy.options_per_item, 4);
|
||||
assert_eq!(c.course.slug(), "biosc-1540");
|
||||
assert!(c.validate().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unquoted_schema_version_still_parses() {
|
||||
// YAML reads 1.0 as a float; authors should not have to remember quotes.
|
||||
let c = parse(
|
||||
r#"
|
||||
schema_version: 1.0
|
||||
course: { code: X, title: Y, term: Z }
|
||||
"#,
|
||||
);
|
||||
assert_eq!(c.schema_version, "1.0");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_keys_are_rejected() {
|
||||
let e = serde_yaml_ng::from_str::<CourseFile>(
|
||||
r#"
|
||||
course: { code: X, title: Y, term: Z }
|
||||
lecutres: {}
|
||||
"#,
|
||||
);
|
||||
assert!(e.is_err(), "a misspelled top-level key must not be ignored");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dangling_references_are_reported() {
|
||||
let c = parse(
|
||||
r#"
|
||||
course: { code: X, title: Y, term: Z }
|
||||
units:
|
||||
- { id: u-a, title: A }
|
||||
lectures:
|
||||
L01: { title: One, unit: u-nope }
|
||||
learning_objectives:
|
||||
lo-a: { text: Do a thing, unit: u-a, lectures: [L99], prerequisites: [lo-missing] }
|
||||
"#,
|
||||
);
|
||||
let issues = c.validate();
|
||||
assert!(issues.iter().any(|i| i.contains("unknown unit `u-nope`")));
|
||||
assert!(issues.iter().any(|i| i.contains("unknown lecture `L99`")));
|
||||
assert!(issues.iter().any(|i| i.contains("lo-missing")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prerequisite_cycles_are_detected() {
|
||||
let c = parse(
|
||||
r#"
|
||||
course: { code: X, title: Y, term: Z }
|
||||
learning_objectives:
|
||||
lo-a: { text: A, prerequisites: [lo-b] }
|
||||
lo-b: { text: B, prerequisites: [lo-a] }
|
||||
"#,
|
||||
);
|
||||
let issues = c.validate();
|
||||
assert!(
|
||||
issues.iter().any(|i| i.contains("cycle")),
|
||||
"expected a cycle report, got {issues:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn objectives_sort_by_declared_unit_order() {
|
||||
let c = parse(
|
||||
r#"
|
||||
course: { code: X, title: Y, term: Z }
|
||||
units:
|
||||
- { id: u-second, title: Second }
|
||||
- { id: u-first, title: First }
|
||||
learning_objectives:
|
||||
lo-z: { text: Z, unit: u-second }
|
||||
lo-a: { text: A, unit: u-first }
|
||||
"#,
|
||||
);
|
||||
// Declared order wins over alphabetical order of unit ids.
|
||||
assert_eq!(c.objectives_in_order(), vec!["lo-z", "lo-a"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn slugify_collapses_punctuation() {
|
||||
assert_eq!(slugify("BIOSC 1540"), "biosc-1540");
|
||||
assert_eq!(slugify("Exam 4 -- Final!"), "exam-4-final");
|
||||
assert_eq!(slugify(" "), "");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
// SPDX-License-Identifier: Prosperity-3.0.0
|
||||
// Copyright Scientific Computing Studio
|
||||
// Source: https://git.scient.ing/education/coursebank
|
||||
|
||||
//! The usage history of a course, derived by scanning assessment records.
|
||||
//!
|
||||
//! There is deliberately no separate ledger file. A ledger duplicates
|
||||
//! information the records already hold and then drifts away from it; this
|
||||
//! derives the same answers from the one artifact that has to be right anyway.
|
||||
//! An item was used exactly when it appears on a record.
|
||||
//!
|
||||
//! This is an aggregate over the records, the same kind of thing as
|
||||
//! [`crate::catalog::Catalog`], which is why it sits beside the schema rather
|
||||
//! than inside [`crate::assessment`].
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use crate::assessment::{AssessmentFile, Kind};
|
||||
use crate::date::Date;
|
||||
use crate::error::Result;
|
||||
|
||||
/// One recorded appearance of an item, derived from the assessment records.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Usage {
|
||||
/// The item's global id.
|
||||
pub item: String,
|
||||
/// The assessment id it appeared on.
|
||||
pub assessment: String,
|
||||
/// The assessment title.
|
||||
pub title: String,
|
||||
/// The kind of assessment.
|
||||
pub kind: Kind,
|
||||
/// When it was administered, when recorded.
|
||||
pub date: Option<Date>,
|
||||
/// The printed question number.
|
||||
pub number: u32,
|
||||
/// The fingerprint as used.
|
||||
pub fingerprint: Option<String>,
|
||||
}
|
||||
|
||||
/// The usage history of a course, built by scanning assessment records.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct History {
|
||||
/// Every recorded appearance, newest last.
|
||||
pub usages: Vec<Usage>,
|
||||
}
|
||||
|
||||
impl History {
|
||||
/// Builds the history from a directory of assessment records.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `dir` - the assessments directory.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The history.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Propagates load errors.
|
||||
pub fn load(dir: &Path) -> Result<History> {
|
||||
let mut usages = Vec::new();
|
||||
for file in AssessmentFile::load_all(dir)? {
|
||||
for p in &file.items {
|
||||
usages.push(Usage {
|
||||
item: p.item.clone(),
|
||||
assessment: file.assessment.id.clone(),
|
||||
title: file.assessment.title.clone(),
|
||||
kind: file.assessment.kind,
|
||||
date: file.assessment.date,
|
||||
number: p.number,
|
||||
fingerprint: p.fingerprint.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
usages.sort_by(|a, b| {
|
||||
a.date
|
||||
.cmp(&b.date)
|
||||
.then(a.assessment.cmp(&b.assessment))
|
||||
.then(a.number.cmp(&b.number))
|
||||
});
|
||||
Ok(History { usages })
|
||||
}
|
||||
|
||||
/// Every appearance of one item, oldest first.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `uid` - the item's global id.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Matching usages.
|
||||
pub fn for_item(&self, uid: &str) -> Vec<&Usage> {
|
||||
self.usages.iter().filter(|u| u.item == uid).collect()
|
||||
}
|
||||
|
||||
/// The most recent date an item was used.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `uid` - the item's global id.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The date, or `None` when never used or never dated.
|
||||
pub fn last_used(&self, uid: &str) -> Option<Date> {
|
||||
self.for_item(uid).iter().filter_map(|u| u.date).max()
|
||||
}
|
||||
|
||||
/// How many times an item has been used.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `uid` - the item's global id.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The count.
|
||||
pub fn use_count(&self, uid: &str) -> usize {
|
||||
self.for_item(uid).len()
|
||||
}
|
||||
|
||||
/// Whether an item is still inside its reuse cooldown.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `uid` - the item's global id.
|
||||
/// * `cooldown_days` - the minimum gap between uses.
|
||||
/// * `as_of` - the date of the assessment being assembled.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// `true` when the item was used too recently to reuse.
|
||||
pub fn in_cooldown(&self, uid: &str, cooldown_days: i64, as_of: Date) -> bool {
|
||||
match self.last_used(uid) {
|
||||
Some(last) => last.days_until(as_of) < cooldown_days,
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn history_tracks_last_use_and_cooldown() {
|
||||
let h = History {
|
||||
usages: vec![
|
||||
Usage {
|
||||
item: "b::q-1".into(),
|
||||
assessment: "exam-1".into(),
|
||||
title: "Exam 1".into(),
|
||||
kind: Kind::Exam,
|
||||
date: Some("2025-09-15".parse().unwrap()),
|
||||
number: 4,
|
||||
fingerprint: None,
|
||||
},
|
||||
Usage {
|
||||
item: "b::q-1".into(),
|
||||
assessment: "exam-3".into(),
|
||||
title: "Exam 3".into(),
|
||||
kind: Kind::Exam,
|
||||
date: Some("2026-02-10".parse().unwrap()),
|
||||
number: 7,
|
||||
fingerprint: None,
|
||||
},
|
||||
],
|
||||
};
|
||||
assert_eq!(h.use_count("b::q-1"), 2);
|
||||
assert_eq!(h.use_count("b::q-2"), 0);
|
||||
assert_eq!(h.last_used("b::q-1").unwrap().to_string(), "2026-02-10");
|
||||
|
||||
let exam_date: Date = "2026-04-23".parse().unwrap();
|
||||
// 72 days elapsed, so a 90-day cooldown still blocks it and a 60-day one
|
||||
// does not.
|
||||
assert!(h.in_cooldown("b::q-1", 90, exam_date));
|
||||
assert!(!h.in_cooldown("b::q-1", 60, exam_date));
|
||||
assert!(!h.in_cooldown("b::never-used", 3650, exam_date));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,824 @@
|
||||
// SPDX-License-Identifier: Prosperity-3.0.0
|
||||
// Copyright Scientific Computing Studio
|
||||
// Source: https://git.scient.ing/education/coursebank
|
||||
|
||||
//! The item: one assessable question, with both the intent it was written with
|
||||
//! and the evidence it has produced.
|
||||
//!
|
||||
//! The organizing idea is that those two things belong in one versioned place.
|
||||
//! [`Design`] is written before an item is ever used and says what you predict:
|
||||
//! how hard, how discriminating, how long, and what the item is meant to reveal.
|
||||
//! [`Calibration`] is written by the tool afterwards and says what happened.
|
||||
//! Keeping them adjacent is what turns a question bank into an instrument you
|
||||
//! can improve, because every administration produces a checkable prediction.
|
||||
//!
|
||||
//! One deliberate departure from a naive design: [`Calibration`] is *cumulative*
|
||||
//! rather than per-administration. Raw per-response data belongs in the Parquet
|
||||
//! tables under `data/`, which are far better at holding it, and an item's YAML
|
||||
//! holds the rolled-up estimate plus a list of which administrations went into
|
||||
//! it. That keeps bank files readable and reviewable in a pull request while
|
||||
//! still letting statistics accumulate across terms.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::date::Date;
|
||||
use crate::hash::fingerprint;
|
||||
use crate::taxonomy::{
|
||||
CognitiveProcess, Discrimination, ErrorType, Flag, Format, Level, ReviewAction, Status,
|
||||
};
|
||||
|
||||
/// One assessable question.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Item {
|
||||
/// Stable id, unique within its bank, conventionally `q-<slug>-NNN`.
|
||||
///
|
||||
/// Ids are never reused and never renumbered: the id is the join key that
|
||||
/// ties an item to every assessment it has appeared on and every response
|
||||
/// row ever recorded for it.
|
||||
pub id: String,
|
||||
|
||||
/// Revision counter, bumped whenever the content changes in a way that
|
||||
/// invalidates pooled statistics.
|
||||
#[serde(default = "one_u32")]
|
||||
pub version: u32,
|
||||
|
||||
/// Workflow state; only [`Status::Approved`] items may be assembled.
|
||||
pub status: Status,
|
||||
|
||||
/// Cognitive demand, 1 through 5.
|
||||
pub level: Level,
|
||||
|
||||
/// The specific process the item elicits. Required for approval, and checked
|
||||
/// against `level`.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cognitive_process: Option<CognitiveProcess>,
|
||||
|
||||
/// Response format.
|
||||
#[serde(default = "default_format")]
|
||||
pub format: Format,
|
||||
|
||||
/// A bonus item, scored outside the graded total.
|
||||
#[serde(default, skip_serializing_if = "is_false")]
|
||||
pub bonus: bool,
|
||||
|
||||
/// Point value; falls back to the course policy when absent.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub points: Option<f64>,
|
||||
|
||||
/// A short human title, used in tables and Canvas question names.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub title: Option<String>,
|
||||
|
||||
/// Id of a shared stimulus in the course registry, for case-based testlets.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub stimulus: Option<String>,
|
||||
|
||||
/// The prompt.
|
||||
pub stem: String,
|
||||
|
||||
/// The answer options in canonical order. Shuffling happens at export time
|
||||
/// per form, never here, so the bank stays diffable.
|
||||
pub options: Vec<Choice>,
|
||||
|
||||
/// Objectives this item measures, as ids into the course registry.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub learning_objectives: Vec<String>,
|
||||
|
||||
/// Where the material was taught.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub sources: Vec<Source>,
|
||||
|
||||
/// Free-form topic tags, for slicing a bank by subject rather than lecture.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub topics: Vec<String>,
|
||||
|
||||
/// Item ids or objective ids a student needs before this is fair.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub prerequisites: Vec<String>,
|
||||
|
||||
/// Figures or data files reproduced with the item.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub assets: Vec<Asset>,
|
||||
|
||||
/// What you predicted before using it.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub design: Option<Design>,
|
||||
|
||||
/// What the evidence says, accumulated across administrations.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub calibration: Option<Calibration>,
|
||||
|
||||
/// The last review decision recorded for this item.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub review: Option<Review>,
|
||||
|
||||
/// Append-only change log.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub history: Vec<HistoryEntry>,
|
||||
|
||||
/// The author of record.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub author: Option<String>,
|
||||
|
||||
/// Notes that must never reach a student.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub notes_private: Option<String>,
|
||||
|
||||
/// Set when the item was retired, with the reason.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub retired: Option<Retirement>,
|
||||
}
|
||||
|
||||
/// One answer option.
|
||||
///
|
||||
/// The optional fields are what separate a designed distractor from filler. An
|
||||
/// option that names the [`ErrorType`] it targets and the misconception behind it
|
||||
/// is one you can report on: when a third of the cohort picks it, you know what
|
||||
/// they were thinking, and the student report can say so.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Choice {
|
||||
/// Option letter, `A` through `H`.
|
||||
pub id: String,
|
||||
|
||||
/// The option text.
|
||||
pub text: String,
|
||||
|
||||
/// Whether this option is keyed correct.
|
||||
#[serde(default)]
|
||||
pub correct: bool,
|
||||
|
||||
/// Credit awarded, from 0 to 1. Absent means 1.0 when correct, else 0.0.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub credit: Option<f64>,
|
||||
|
||||
/// Instructor-facing rationale for why this option is right or wrong.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub explanation: Option<String>,
|
||||
|
||||
/// The nudge you would give a student reconsidering this option.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub hint: Option<String>,
|
||||
|
||||
/// The specific wrong idea this distractor is built to capture.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub misconception: Option<String>,
|
||||
|
||||
/// The category of that error.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub error_type: Option<ErrorType>,
|
||||
|
||||
/// Whether a wrong option is defensible enough to earn partial credit.
|
||||
#[serde(default, skip_serializing_if = "is_false")]
|
||||
pub defensible: bool,
|
||||
|
||||
/// The argument for why it is defensible. Required whenever credit is
|
||||
/// awarded to a wrong option, so partial credit is always justified in
|
||||
/// writing rather than by memory.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub defense: Option<String>,
|
||||
|
||||
/// Text released to students after the assessment. This is what a student
|
||||
/// report shows them when they chose this option.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub feedback_student: Option<String>,
|
||||
|
||||
/// Your a priori guess at how often this option is chosen.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub selection_rate_expected: Option<f64>,
|
||||
}
|
||||
|
||||
impl Choice {
|
||||
/// The credit this option earns, resolving the default from `correct`.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Credit in `[0, 1]`.
|
||||
pub fn credit(&self) -> f64 {
|
||||
self.credit.unwrap_or(if self.correct { 1.0 } else { 0.0 })
|
||||
}
|
||||
|
||||
/// Whether this option awards credit without being keyed correct.
|
||||
pub fn is_partial(&self) -> bool {
|
||||
!self.correct && self.credit() > 0.0
|
||||
}
|
||||
|
||||
/// The best available student-facing explanation of this option.
|
||||
///
|
||||
/// Prefers explicit student feedback, then the misconception, then the
|
||||
/// instructor explanation, so a report degrades gracefully as authoring
|
||||
/// completeness varies.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The text, or `None` when the option carries no rationale at all.
|
||||
pub fn student_text(&self) -> Option<&str> {
|
||||
self.feedback_student
|
||||
.as_deref()
|
||||
.or(self.misconception.as_deref())
|
||||
.or(self.explanation.as_deref())
|
||||
}
|
||||
}
|
||||
|
||||
/// Where the assessed material was taught.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Source {
|
||||
/// Lecture id in the course registry.
|
||||
pub lecture: String,
|
||||
/// Slide numbers, so a student report can point at a page rather than a
|
||||
/// whole lecture.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub slides: Vec<u32>,
|
||||
/// Readings, cited however you cite them.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub readings: Vec<String>,
|
||||
/// A timestamp into a recording, in seconds.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub recording_seconds: Option<u32>,
|
||||
}
|
||||
|
||||
/// A figure or data file reproduced with an item.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Asset {
|
||||
/// Path relative to the course root.
|
||||
pub path: String,
|
||||
/// Alt text. Required in practice for accessibility; the linter says so.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub alt: Option<String>,
|
||||
/// A caption printed below the figure.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub caption: Option<String>,
|
||||
}
|
||||
|
||||
/// The a priori design of an item: your predictions, written down.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Design {
|
||||
/// The proportion of the target cohort you expect to answer correctly.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub expected_difficulty: Option<f64>,
|
||||
/// How sharply you expect it to separate students.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub expected_discrimination: Option<Discrimination>,
|
||||
/// How long you expect it to take, in seconds. Summed over a form, this is
|
||||
/// how you check that an exam fits the period.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub expected_time_seconds: Option<f64>,
|
||||
/// What the item is meant to reveal, and why it sits at its level.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub rationale: Option<String>,
|
||||
}
|
||||
|
||||
/// Accumulated evidence about an item's behavior.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Calibration {
|
||||
/// The administrations pooled into these numbers.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub administrations: Vec<String>,
|
||||
/// When the calibration was last recomputed.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub updated: Option<Date>,
|
||||
/// The content fingerprint these statistics describe. If it differs from the
|
||||
/// item's current fingerprint, the item was edited after calibration and the
|
||||
/// numbers are stale; the linter says so.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub fingerprint: Option<String>,
|
||||
/// Total examinees pooled.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub n_examinees: Option<usize>,
|
||||
/// Proportion correct.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub p_value: Option<f64>,
|
||||
/// Corrected item-total point-biserial correlation.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub point_biserial: Option<f64>,
|
||||
/// Upper-minus-lower-group discrimination index.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub discrimination_index: Option<f64>,
|
||||
/// Mean response time, when the platform reports it.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub mean_response_time_seconds: Option<f64>,
|
||||
/// Proportion of responses faster than plausible reading time.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub rapid_guess_rate: Option<f64>,
|
||||
/// Per-option behavior, keyed by option letter.
|
||||
#[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
|
||||
pub option_stats: std::collections::BTreeMap<String, OptionStat>,
|
||||
/// Fitted item response theory parameters.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub irt: Option<IrtParams>,
|
||||
/// Machine-detected problems.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub flags: Vec<Flag>,
|
||||
}
|
||||
|
||||
/// How one option behaved.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct OptionStat {
|
||||
/// Proportion of examinees who chose it.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub selection_rate: Option<f64>,
|
||||
/// Correlation between choosing it and total score. Negative for a working
|
||||
/// distractor; positive on a distractor is a warning.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub point_biserial: Option<f64>,
|
||||
/// Selection rate among the top scoring group.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub upper_group_rate: Option<f64>,
|
||||
/// Selection rate among the bottom scoring group.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub lower_group_rate: Option<f64>,
|
||||
}
|
||||
|
||||
/// Fitted item response theory parameters.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct IrtParams {
|
||||
/// Which model was fitted.
|
||||
pub model: IrtModel,
|
||||
/// Discrimination.
|
||||
pub a: f64,
|
||||
/// Difficulty, on the same scale as ability.
|
||||
pub b: f64,
|
||||
/// Lower asymptote, the guessing parameter.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub c: Option<f64>,
|
||||
/// Standard error of `a`.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub se_a: Option<f64>,
|
||||
/// Standard error of `b`.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub se_b: Option<f64>,
|
||||
/// Examinees the fit was based on. Small samples give unstable parameters,
|
||||
/// so this travels with them rather than being looked up later.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub n: Option<usize>,
|
||||
/// Whether priors were used, which matters when interpreting `a`.
|
||||
#[serde(default, skip_serializing_if = "is_false")]
|
||||
pub bayesian: bool,
|
||||
}
|
||||
|
||||
/// The item response theory model family.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum IrtModel {
|
||||
/// One parameter: difficulty only, discrimination fixed at 1.
|
||||
Rasch,
|
||||
/// Two parameters: discrimination and difficulty.
|
||||
#[serde(rename = "2pl")]
|
||||
TwoPl,
|
||||
/// Three parameters, adding a lower asymptote for guessing.
|
||||
#[serde(rename = "3pl")]
|
||||
ThreePl,
|
||||
}
|
||||
|
||||
impl IrtModel {
|
||||
/// The token used in YAML.
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
IrtModel::Rasch => "rasch",
|
||||
IrtModel::TwoPl => "2pl",
|
||||
IrtModel::ThreePl => "3pl",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A recorded review decision.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Review {
|
||||
/// Who reviewed it.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub reviewed_by: Option<String>,
|
||||
/// When.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub reviewed_on: Option<Date>,
|
||||
/// What was decided.
|
||||
pub action: ReviewAction,
|
||||
/// Why.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub notes: Option<String>,
|
||||
}
|
||||
|
||||
/// Why and when an item left service.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Retirement {
|
||||
/// When it was retired.
|
||||
pub on: Date,
|
||||
/// Why.
|
||||
pub reason: String,
|
||||
/// A replacement item id, when one exists.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub replaced_by: Option<String>,
|
||||
}
|
||||
|
||||
/// One entry in an item's change log.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct HistoryEntry {
|
||||
/// The version this change produced.
|
||||
pub version: u32,
|
||||
/// When it was made.
|
||||
pub date: Date,
|
||||
/// Who made it.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub author: Option<String>,
|
||||
/// What changed.
|
||||
pub change: String,
|
||||
}
|
||||
|
||||
impl Item {
|
||||
/// Builds a draft item with everything optional left empty.
|
||||
///
|
||||
/// A constructor rather than a `Default` implementation, because there is no
|
||||
/// sensible default id, stem, or option set: an item missing any of those is
|
||||
/// not a lesser item, it is not an item. Requiring them at construction means
|
||||
/// the only way to get a half-built `Item` is deliberately.
|
||||
///
|
||||
/// The result is `Status::Draft` and deliberately will not pass
|
||||
/// [`Item::is_assemblable`] — it still needs learning objectives, sources, and
|
||||
/// a cognitive process before it can be drawn onto an assessment.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `id` - the item id.
|
||||
/// * `level` - the cognitive level.
|
||||
/// * `stem` - the question.
|
||||
/// * `options` - the answer options.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The draft item.
|
||||
pub fn draft(id: &str, level: Level, stem: &str, options: Vec<Choice>) -> Item {
|
||||
Item {
|
||||
id: id.to_string(),
|
||||
version: 1,
|
||||
status: Status::Draft,
|
||||
level,
|
||||
cognitive_process: None,
|
||||
format: if options.iter().filter(|o| o.correct).count() > 1 {
|
||||
Format::MultipleResponse
|
||||
} else {
|
||||
Format::SingleBestAnswer
|
||||
},
|
||||
bonus: false,
|
||||
points: None,
|
||||
title: None,
|
||||
stimulus: None,
|
||||
stem: stem.to_string(),
|
||||
options,
|
||||
learning_objectives: Vec::new(),
|
||||
sources: Vec::new(),
|
||||
topics: Vec::new(),
|
||||
prerequisites: Vec::new(),
|
||||
assets: Vec::new(),
|
||||
design: None,
|
||||
calibration: None,
|
||||
review: None,
|
||||
history: Vec::new(),
|
||||
author: None,
|
||||
notes_private: None,
|
||||
retired: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Indices of the keyed-correct options.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Zero-based indices into `options`.
|
||||
pub fn key_indices(&self) -> Vec<usize> {
|
||||
self.options
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, o)| o.correct)
|
||||
.map(|(i, _)| i)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The keyed-correct option letters, sorted.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Letters such as `["C"]` or `["A", "B"]`.
|
||||
pub fn key_letters(&self) -> Vec<String> {
|
||||
let mut out: Vec<String> = self
|
||||
.options
|
||||
.iter()
|
||||
.filter(|o| o.correct)
|
||||
.map(|o| o.id.clone())
|
||||
.collect();
|
||||
out.sort();
|
||||
out
|
||||
}
|
||||
|
||||
/// Looks up an option by letter.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `letter` - the option id, case insensitive.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The option, or `None`.
|
||||
pub fn option(&self, letter: &str) -> Option<&Choice> {
|
||||
self.options
|
||||
.iter()
|
||||
.find(|o| o.id.eq_ignore_ascii_case(letter))
|
||||
}
|
||||
|
||||
/// Whether the item keys more than one option.
|
||||
pub fn is_multi_key(&self) -> bool {
|
||||
self.key_indices().len() > 1
|
||||
}
|
||||
|
||||
/// The display title, falling back to a truncated stem.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// A short label suitable for a table or a Canvas question name.
|
||||
pub fn display_title(&self) -> String {
|
||||
if let Some(t) = &self.title {
|
||||
if !t.trim().is_empty() {
|
||||
return t.trim().to_string();
|
||||
}
|
||||
}
|
||||
let flat = self.stem.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
if flat.chars().count() <= 60 {
|
||||
flat
|
||||
} else {
|
||||
let head: String = flat.chars().take(57).collect();
|
||||
format!("{head}...")
|
||||
}
|
||||
}
|
||||
|
||||
/// A content fingerprint over everything that affects what a student sees.
|
||||
///
|
||||
/// Metadata deliberately does not contribute: retagging an objective must not
|
||||
/// invalidate pooled statistics, but rewording an option must.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The fingerprint as hex.
|
||||
pub fn fingerprint(&self) -> String {
|
||||
let mut parts: Vec<String> = vec![self.stem.trim().to_string()];
|
||||
// Canonicalize by option letter so reordering the YAML block, which does
|
||||
// not change the item, does not change the fingerprint.
|
||||
let mut opts: Vec<&Choice> = self.options.iter().collect();
|
||||
opts.sort_by(|a, b| a.id.cmp(&b.id));
|
||||
for o in opts {
|
||||
parts.push(format!(
|
||||
"{}|{}|{}",
|
||||
o.id,
|
||||
if o.correct { "1" } else { "0" },
|
||||
o.text.trim()
|
||||
));
|
||||
}
|
||||
if let Some(s) = &self.stimulus {
|
||||
parts.push(format!("stimulus:{s}"));
|
||||
}
|
||||
fingerprint(parts.iter().map(|s| s.as_str()))
|
||||
}
|
||||
|
||||
/// Whether the recorded calibration matches the current content.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// `false` when the item was edited after it was calibrated.
|
||||
pub fn calibration_is_current(&self) -> bool {
|
||||
match self
|
||||
.calibration
|
||||
.as_ref()
|
||||
.and_then(|c| c.fingerprint.as_ref())
|
||||
{
|
||||
Some(fp) => *fp == self.fingerprint(),
|
||||
None => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// The point value, resolving against a course default.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `default_points` - the course policy value.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The point value to use.
|
||||
pub fn points(&self, default_points: f64) -> f64 {
|
||||
self.points.unwrap_or(default_points)
|
||||
}
|
||||
|
||||
/// Whether this item may be placed on a graded assessment.
|
||||
pub fn is_assemblable(&self) -> bool {
|
||||
self.status.is_usable() && self.retired.is_none()
|
||||
}
|
||||
|
||||
/// The expected time in seconds, falling back to a level-based estimate.
|
||||
///
|
||||
/// The fallbacks are rough but useful: without them, a form's total time
|
||||
/// estimate silently drops every item that has no `design` block.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Seconds.
|
||||
pub fn expected_seconds(&self) -> f64 {
|
||||
if let Some(t) = self.design.as_ref().and_then(|d| d.expected_time_seconds) {
|
||||
return t;
|
||||
}
|
||||
match self.level {
|
||||
Level::Remember => 35.0,
|
||||
Level::Understand => 65.0,
|
||||
Level::Apply => 95.0,
|
||||
Level::Analyze => 130.0,
|
||||
Level::Create => 165.0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Appends a change-log entry and bumps the version.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `change` - a description of what changed.
|
||||
/// * `author` - who made the change.
|
||||
pub fn record_change(&mut self, change: &str, author: Option<&str>) {
|
||||
self.version += 1;
|
||||
self.history.push(HistoryEntry {
|
||||
version: self.version,
|
||||
date: Date::today(),
|
||||
author: author.map(|a| a.to_string()),
|
||||
change: change.to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
fn one_u32() -> u32 {
|
||||
1
|
||||
}
|
||||
fn default_format() -> Format {
|
||||
Format::SingleBestAnswer
|
||||
}
|
||||
fn is_false(b: &bool) -> bool {
|
||||
!*b
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn item(src: &str) -> Item {
|
||||
serde_yaml_ng::from_str(src).expect("item parses")
|
||||
}
|
||||
|
||||
const MINIMAL: &str = r#"
|
||||
id: q-demo-001
|
||||
status: draft
|
||||
level: 2
|
||||
stem: Which statement best explains the effect?
|
||||
options:
|
||||
- { id: A, text: Right, correct: true }
|
||||
- { id: B, text: Wrong }
|
||||
- { id: C, text: Also wrong }
|
||||
"#;
|
||||
|
||||
#[test]
|
||||
fn minimal_item_parses_with_defaults() {
|
||||
let it = item(MINIMAL);
|
||||
assert_eq!(it.version, 1);
|
||||
assert_eq!(it.format, Format::SingleBestAnswer);
|
||||
assert!(!it.bonus);
|
||||
assert_eq!(it.key_letters(), vec!["A"]);
|
||||
assert!(!it.is_multi_key());
|
||||
assert!(!it.is_assemblable(), "drafts are not assemblable");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn credit_defaults_from_correctness() {
|
||||
let it = item(MINIMAL);
|
||||
assert_eq!(it.option("A").unwrap().credit(), 1.0);
|
||||
assert_eq!(it.option("B").unwrap().credit(), 0.0);
|
||||
assert!(!it.option("B").unwrap().is_partial());
|
||||
|
||||
let with_partial = item(
|
||||
r#"
|
||||
id: q-demo-002
|
||||
status: draft
|
||||
level: 5
|
||||
stem: s
|
||||
options:
|
||||
- { id: A, text: Right, correct: true }
|
||||
- { id: B, text: Defensible, credit: 0.5, defensible: true, defense: because }
|
||||
- { id: C, text: Wrong }
|
||||
"#,
|
||||
);
|
||||
assert!(with_partial.option("B").unwrap().is_partial());
|
||||
assert_eq!(with_partial.option("B").unwrap().credit(), 0.5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fingerprint_tracks_content_not_metadata() {
|
||||
let base = item(MINIMAL);
|
||||
let mut retagged = base.clone();
|
||||
retagged.topics = vec!["kinetics".into()];
|
||||
retagged.learning_objectives = vec!["lo-a".into()];
|
||||
retagged.author = Some("someone".into());
|
||||
assert_eq!(
|
||||
base.fingerprint(),
|
||||
retagged.fingerprint(),
|
||||
"metadata must not invalidate pooled statistics"
|
||||
);
|
||||
|
||||
let mut reworded = base.clone();
|
||||
reworded.options[1].text = "Wrong, but differently".into();
|
||||
assert_ne!(base.fingerprint(), reworded.fingerprint());
|
||||
|
||||
let mut rekeyed = base.clone();
|
||||
rekeyed.options[0].correct = false;
|
||||
rekeyed.options[1].correct = true;
|
||||
assert_ne!(base.fingerprint(), rekeyed.fingerprint());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fingerprint_ignores_yaml_option_order() {
|
||||
let a = item(MINIMAL);
|
||||
let b = item(
|
||||
r#"
|
||||
id: q-demo-001
|
||||
status: draft
|
||||
level: 2
|
||||
stem: Which statement best explains the effect?
|
||||
options:
|
||||
- { id: C, text: Also wrong }
|
||||
- { id: A, text: Right, correct: true }
|
||||
- { id: B, text: Wrong }
|
||||
"#,
|
||||
);
|
||||
assert_eq!(a.fingerprint(), b.fingerprint());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_calibration_is_detectable() {
|
||||
let mut it = item(MINIMAL);
|
||||
assert!(it.calibration_is_current(), "no calibration is not stale");
|
||||
|
||||
let fp = it.fingerprint();
|
||||
it.calibration = Some(Calibration {
|
||||
fingerprint: Some(fp),
|
||||
..Calibration::default()
|
||||
});
|
||||
assert!(it.calibration_is_current());
|
||||
|
||||
it.stem = "A different question entirely?".into();
|
||||
assert!(!it.calibration_is_current());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn display_title_truncates_long_stems() {
|
||||
let mut it = item(MINIMAL);
|
||||
it.title = None;
|
||||
it.stem = "word ".repeat(40);
|
||||
let t = it.display_title();
|
||||
assert!(t.ends_with("..."));
|
||||
assert_eq!(t.chars().count(), 60);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_item_keys_are_rejected() {
|
||||
let bad = serde_yaml_ng::from_str::<Item>(
|
||||
r#"
|
||||
id: q-demo-001
|
||||
status: draft
|
||||
level: 2
|
||||
stem: s
|
||||
steam: oops
|
||||
options:
|
||||
- { id: A, text: a, correct: true }
|
||||
"#,
|
||||
);
|
||||
assert!(bad.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_change_bumps_version_and_logs() {
|
||||
let mut it = item(MINIMAL);
|
||||
it.record_change("clarified the stem", Some("Alex"));
|
||||
assert_eq!(it.version, 2);
|
||||
assert_eq!(it.history.len(), 1);
|
||||
assert_eq!(it.history[0].version, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn irt_model_tokens_round_trip() {
|
||||
assert_eq!(serde_json::to_string(&IrtModel::TwoPl).unwrap(), "\"2pl\"");
|
||||
assert_eq!(
|
||||
serde_json::from_str::<IrtModel>("\"3pl\"").unwrap(),
|
||||
IrtModel::ThreePl
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
// SPDX-License-Identifier: Prosperity-3.0.0
|
||||
// Copyright Scientific Computing Studio
|
||||
// Source: https://git.scient.ing/education/coursebank
|
||||
|
||||
//! The on-disk layout of a course directory.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use crate::course::COURSE_FILE;
|
||||
use crate::error::{Error, Result};
|
||||
|
||||
/// The standard directory layout of a course, resolved from a root.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Layout {
|
||||
/// The course root.
|
||||
pub root: PathBuf,
|
||||
}
|
||||
|
||||
impl Layout {
|
||||
/// Builds a layout from a course root directory.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `root` - the course directory.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The layout.
|
||||
pub fn new(root: impl Into<PathBuf>) -> Layout {
|
||||
Layout { root: root.into() }
|
||||
}
|
||||
|
||||
/// Path to `course.yaml`.
|
||||
pub fn course_file(&self) -> PathBuf {
|
||||
self.root.join(COURSE_FILE)
|
||||
}
|
||||
|
||||
/// Directory holding item bank YAML files.
|
||||
pub fn banks(&self) -> PathBuf {
|
||||
self.root.join("banks")
|
||||
}
|
||||
|
||||
/// Directory holding assessment records.
|
||||
pub fn assessments(&self) -> PathBuf {
|
||||
self.root.join("assessments")
|
||||
}
|
||||
|
||||
/// Directory holding response tables and derived statistics.
|
||||
pub fn data(&self) -> PathBuf {
|
||||
self.root.join("data")
|
||||
}
|
||||
|
||||
/// Directory holding generated reports.
|
||||
pub fn reports(&self) -> PathBuf {
|
||||
self.root.join("reports")
|
||||
}
|
||||
|
||||
/// Directory holding generated exports such as QTI packages.
|
||||
pub fn build(&self) -> PathBuf {
|
||||
self.root.join("build")
|
||||
}
|
||||
|
||||
/// Directory holding emitted JSON Schema files for editor validation.
|
||||
pub fn schema(&self) -> PathBuf {
|
||||
self.root.join("schema")
|
||||
}
|
||||
|
||||
/// Directory holding Typst export templates and their configuration.
|
||||
///
|
||||
/// Unlike the other directories, this one is *not* created by
|
||||
/// [`Layout::create_all`]. Its absence is meaningful: a course with no
|
||||
/// `templates/` directory uses the templates compiled into the binary, and
|
||||
/// creating an empty one on `init` would suggest a customization step is
|
||||
/// required when it is not. `coursebank template dump` creates it on demand.
|
||||
pub fn templates(&self) -> PathBuf {
|
||||
self.root.join("templates")
|
||||
}
|
||||
|
||||
/// Creates every directory in the layout.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`Error::Io`] if a directory cannot be created.
|
||||
pub fn create_all(&self) -> Result<()> {
|
||||
for dir in [
|
||||
self.root.clone(),
|
||||
self.banks(),
|
||||
self.assessments(),
|
||||
self.data(),
|
||||
self.reports(),
|
||||
self.build(),
|
||||
self.schema(),
|
||||
] {
|
||||
std::fs::create_dir_all(&dir).map_err(|e| Error::io(&dir, e))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,636 @@
|
||||
// SPDX-License-Identifier: Prosperity-3.0.0
|
||||
// Copyright Scientific Computing Studio
|
||||
// Source: https://git.scient.ing/education/coursebank
|
||||
|
||||
//! The pedagogical vocabulary: levels, cognitive processes, error types, and
|
||||
//! workflow states.
|
||||
|
||||
use std::fmt;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Cognitive demand, following the revised Bloom taxonomy.
|
||||
///
|
||||
/// Serialized as the integers 1 through 5 so YAML reads `level: 3`. The derived
|
||||
/// [`Ord`] follows declaration order, which is ascending demand.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||
#[serde(try_from = "u8", into = "u8")]
|
||||
pub enum Level {
|
||||
/// Retrieve a fact or definition.
|
||||
Remember,
|
||||
/// Construct meaning; explain, compare, classify.
|
||||
Understand,
|
||||
/// Carry out a procedure in a given situation.
|
||||
Apply,
|
||||
/// Break material apart and relate the pieces.
|
||||
Analyze,
|
||||
/// Judge against criteria, or assemble something new.
|
||||
Create,
|
||||
}
|
||||
|
||||
impl Level {
|
||||
/// Every level in ascending order, for iteration in reports.
|
||||
pub const ALL: [Level; 5] = [
|
||||
Level::Remember,
|
||||
Level::Understand,
|
||||
Level::Apply,
|
||||
Level::Analyze,
|
||||
Level::Create,
|
||||
];
|
||||
|
||||
/// The numeric code used in YAML and on printed badges.
|
||||
pub fn code(self) -> u8 {
|
||||
match self {
|
||||
Level::Remember => 1,
|
||||
Level::Understand => 2,
|
||||
Level::Apply => 3,
|
||||
Level::Analyze => 4,
|
||||
Level::Create => 5,
|
||||
}
|
||||
}
|
||||
|
||||
/// The level for a numeric code, if it is one.
|
||||
///
|
||||
/// Flat storage writes `0` for an unknown level rather than an empty cell,
|
||||
/// because a numeric column with holes in it is awkward in every columnar
|
||||
/// format. This turns that convention back into an `Option`.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `code` - the numeric code, where anything outside 1..=5 means unknown.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The level, or `None`.
|
||||
pub fn from_code(code: u8) -> Option<Level> {
|
||||
match code {
|
||||
1 => Some(Level::Remember),
|
||||
2 => Some(Level::Understand),
|
||||
3 => Some(Level::Apply),
|
||||
4 => Some(Level::Analyze),
|
||||
5 => Some(Level::Create),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The category name, e.g. `"Apply"`.
|
||||
pub fn name(self) -> &'static str {
|
||||
match self {
|
||||
Level::Remember => "Remember",
|
||||
Level::Understand => "Understand",
|
||||
Level::Apply => "Apply",
|
||||
Level::Analyze => "Analyze",
|
||||
Level::Create => "Evaluate/Create",
|
||||
}
|
||||
}
|
||||
|
||||
/// A one-line description suitable for a student-facing report.
|
||||
pub fn blurb(self) -> &'static str {
|
||||
match self {
|
||||
Level::Remember => "recalling terms, facts, and definitions",
|
||||
Level::Understand => "explaining ideas in your own words",
|
||||
Level::Apply => "using a procedure in a new situation",
|
||||
Level::Analyze => "taking a situation apart and relating the pieces",
|
||||
Level::Create => "judging alternatives or building something new",
|
||||
}
|
||||
}
|
||||
|
||||
/// The cognitive processes that belong to this level.
|
||||
pub fn processes(self) -> &'static [CognitiveProcess] {
|
||||
use CognitiveProcess as P;
|
||||
match self {
|
||||
Level::Remember => &[P::Recognize, P::Recall],
|
||||
Level::Understand => &[
|
||||
P::Interpret,
|
||||
P::Exemplify,
|
||||
P::Classify,
|
||||
P::Summarize,
|
||||
P::Infer,
|
||||
P::Compare,
|
||||
P::Explain,
|
||||
],
|
||||
Level::Apply => &[P::Execute, P::Implement],
|
||||
Level::Analyze => &[P::Differentiate, P::Organize, P::Attribute],
|
||||
Level::Create => &[P::Check, P::Critique, P::Generate, P::Plan, P::Produce],
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a process is consistent with this level.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `process` - the process to check.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// `true` when the pairing is coherent.
|
||||
pub fn allows(self, process: CognitiveProcess) -> bool {
|
||||
self.processes().contains(&process)
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<u8> for Level {
|
||||
type Error = String;
|
||||
|
||||
fn try_from(v: u8) -> Result<Level, String> {
|
||||
match v {
|
||||
1 => Ok(Level::Remember),
|
||||
2 => Ok(Level::Understand),
|
||||
3 => Ok(Level::Apply),
|
||||
4 => Ok(Level::Analyze),
|
||||
5 => Ok(Level::Create),
|
||||
other => Err(format!("level must be 1 through 5, got {other}")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Level> for u8 {
|
||||
fn from(l: Level) -> u8 {
|
||||
l.code()
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for Level {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "L{}", self.code())
|
||||
}
|
||||
}
|
||||
|
||||
/// The specific cognitive process an item is designed to elicit.
|
||||
///
|
||||
/// Naming the process, not just the level, is what makes the level claim
|
||||
/// checkable: it forces you to say which of the several things "Understand"
|
||||
/// could mean you actually wrote.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum CognitiveProcess {
|
||||
/// Identify a previously encountered item.
|
||||
Recognize,
|
||||
/// Retrieve from long-term memory unaided.
|
||||
Recall,
|
||||
/// Restate in another representation.
|
||||
Interpret,
|
||||
/// Give an instance of a category.
|
||||
Exemplify,
|
||||
/// Assign an instance to a category.
|
||||
Classify,
|
||||
/// Abstract a general theme.
|
||||
Summarize,
|
||||
/// Draw a logical conclusion from given information.
|
||||
Infer,
|
||||
/// Detect correspondences between two things.
|
||||
Compare,
|
||||
/// Construct a cause-and-effect account.
|
||||
Explain,
|
||||
/// Apply a procedure to a familiar task.
|
||||
Execute,
|
||||
/// Apply a procedure to an unfamiliar task.
|
||||
Implement,
|
||||
/// Distinguish relevant from irrelevant parts.
|
||||
Differentiate,
|
||||
/// Determine how elements fit a structure.
|
||||
Organize,
|
||||
/// Determine a point of view or intent.
|
||||
Attribute,
|
||||
/// Test for internal consistency.
|
||||
Check,
|
||||
/// Judge against external criteria.
|
||||
Critique,
|
||||
/// Propose alternative hypotheses.
|
||||
Generate,
|
||||
/// Devise a procedure.
|
||||
Plan,
|
||||
/// Construct a product.
|
||||
Produce,
|
||||
}
|
||||
|
||||
impl CognitiveProcess {
|
||||
/// The level this process belongs to.
|
||||
pub fn level(self) -> Level {
|
||||
for level in Level::ALL {
|
||||
if level.allows(self) {
|
||||
return level;
|
||||
}
|
||||
}
|
||||
// Unreachable: every variant appears in exactly one level's list.
|
||||
Level::Remember
|
||||
}
|
||||
|
||||
/// Every process, in level order.
|
||||
pub const ALL: [CognitiveProcess; 19] = [
|
||||
CognitiveProcess::Recognize,
|
||||
CognitiveProcess::Recall,
|
||||
CognitiveProcess::Interpret,
|
||||
CognitiveProcess::Exemplify,
|
||||
CognitiveProcess::Classify,
|
||||
CognitiveProcess::Summarize,
|
||||
CognitiveProcess::Infer,
|
||||
CognitiveProcess::Compare,
|
||||
CognitiveProcess::Explain,
|
||||
CognitiveProcess::Execute,
|
||||
CognitiveProcess::Implement,
|
||||
CognitiveProcess::Differentiate,
|
||||
CognitiveProcess::Organize,
|
||||
CognitiveProcess::Attribute,
|
||||
CognitiveProcess::Check,
|
||||
CognitiveProcess::Critique,
|
||||
CognitiveProcess::Generate,
|
||||
CognitiveProcess::Plan,
|
||||
CognitiveProcess::Produce,
|
||||
];
|
||||
|
||||
/// The snake_case token used in YAML.
|
||||
pub fn as_str(self) -> &'static str {
|
||||
use CognitiveProcess as P;
|
||||
match self {
|
||||
P::Recognize => "recognize",
|
||||
P::Recall => "recall",
|
||||
P::Interpret => "interpret",
|
||||
P::Exemplify => "exemplify",
|
||||
P::Classify => "classify",
|
||||
P::Summarize => "summarize",
|
||||
P::Infer => "infer",
|
||||
P::Compare => "compare",
|
||||
P::Explain => "explain",
|
||||
P::Execute => "execute",
|
||||
P::Implement => "implement",
|
||||
P::Differentiate => "differentiate",
|
||||
P::Organize => "organize",
|
||||
P::Attribute => "attribute",
|
||||
P::Check => "check",
|
||||
P::Critique => "critique",
|
||||
P::Generate => "generate",
|
||||
P::Plan => "plan",
|
||||
P::Produce => "produce",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for CognitiveProcess {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// The category of mistake a distractor is built to capture.
|
||||
///
|
||||
/// This does double duty. It disciplines authoring, because a distractor you
|
||||
/// cannot name an error for is probably filler. And it makes item analysis
|
||||
/// legible afterwards: a high selection rate on a `DroppedStep` option tells you
|
||||
/// where in a procedure students slip, which a bare letter never would.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ErrorType {
|
||||
/// Confused one remembered fact for a neighboring one.
|
||||
RecallConfusion,
|
||||
/// Swapped two terms that sound or look alike.
|
||||
TerminologySwap,
|
||||
/// Holds a specific, nameable wrong model.
|
||||
Misconception,
|
||||
/// Knows part of the idea but not all of it.
|
||||
IncompleteUnderstanding,
|
||||
/// Applied a valid rule outside its scope.
|
||||
Overgeneralization,
|
||||
/// True but irrelevant to the question asked.
|
||||
PlausibleIrrelevant,
|
||||
/// Omitted a step in a procedure.
|
||||
DroppedStep,
|
||||
/// Inverted the direction of a relationship.
|
||||
ReversedRelationship,
|
||||
/// Used a procedure that does not apply here.
|
||||
WrongProcedure,
|
||||
/// Right method, wrong sign or order of magnitude.
|
||||
SignOrMagnitudeError,
|
||||
/// Ignored a condition that interacts with the answer.
|
||||
IgnoresInteractingCondition,
|
||||
/// Correct as far as it goes, but not the best answer.
|
||||
CorrectButIncomplete,
|
||||
/// Took a heuristic shortcut that usually works.
|
||||
CommonShortcut,
|
||||
}
|
||||
|
||||
impl ErrorType {
|
||||
/// Every error type.
|
||||
pub const ALL: [ErrorType; 13] = [
|
||||
ErrorType::RecallConfusion,
|
||||
ErrorType::TerminologySwap,
|
||||
ErrorType::Misconception,
|
||||
ErrorType::IncompleteUnderstanding,
|
||||
ErrorType::Overgeneralization,
|
||||
ErrorType::PlausibleIrrelevant,
|
||||
ErrorType::DroppedStep,
|
||||
ErrorType::ReversedRelationship,
|
||||
ErrorType::WrongProcedure,
|
||||
ErrorType::SignOrMagnitudeError,
|
||||
ErrorType::IgnoresInteractingCondition,
|
||||
ErrorType::CorrectButIncomplete,
|
||||
ErrorType::CommonShortcut,
|
||||
];
|
||||
|
||||
/// The snake_case token used in YAML.
|
||||
pub fn as_str(self) -> &'static str {
|
||||
use ErrorType as E;
|
||||
match self {
|
||||
E::RecallConfusion => "recall_confusion",
|
||||
E::TerminologySwap => "terminology_swap",
|
||||
E::Misconception => "misconception",
|
||||
E::IncompleteUnderstanding => "incomplete_understanding",
|
||||
E::Overgeneralization => "overgeneralization",
|
||||
E::PlausibleIrrelevant => "plausible_irrelevant",
|
||||
E::DroppedStep => "dropped_step",
|
||||
E::ReversedRelationship => "reversed_relationship",
|
||||
E::WrongProcedure => "wrong_procedure",
|
||||
E::SignOrMagnitudeError => "sign_or_magnitude_error",
|
||||
E::IgnoresInteractingCondition => "ignores_interacting_condition",
|
||||
E::CorrectButIncomplete => "correct_but_incomplete",
|
||||
E::CommonShortcut => "common_shortcut",
|
||||
}
|
||||
}
|
||||
|
||||
/// A short instructor-facing gloss.
|
||||
pub fn gloss(self) -> &'static str {
|
||||
use ErrorType as E;
|
||||
match self {
|
||||
E::RecallConfusion => "confused with a neighboring fact",
|
||||
E::TerminologySwap => "swapped similar terms",
|
||||
E::Misconception => "specific wrong model",
|
||||
E::IncompleteUnderstanding => "partial grasp of the idea",
|
||||
E::Overgeneralization => "applied a rule outside its scope",
|
||||
E::PlausibleIrrelevant => "true but not what was asked",
|
||||
E::DroppedStep => "skipped a step",
|
||||
E::ReversedRelationship => "reversed the direction",
|
||||
E::WrongProcedure => "used the wrong procedure",
|
||||
E::SignOrMagnitudeError => "sign or magnitude slip",
|
||||
E::IgnoresInteractingCondition => "ignored an interacting condition",
|
||||
E::CorrectButIncomplete => "correct but not best",
|
||||
E::CommonShortcut => "took a familiar shortcut",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Where an item sits in the authoring workflow.
|
||||
///
|
||||
/// The state gates what the validator requires. A draft may be a bare idea; an
|
||||
/// approved item must be fully sourced and designed, because approval is what
|
||||
/// permits it onto a graded assessment. Retired items are kept forever so the
|
||||
/// bank is an append-only record of what you have asked students.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Status {
|
||||
/// Captured but not yet worked out.
|
||||
Draft,
|
||||
/// Written and awaiting review.
|
||||
InReview,
|
||||
/// Reviewed and sent back for changes.
|
||||
NeedsRevision,
|
||||
/// Cleared for use on a graded assessment.
|
||||
Approved,
|
||||
/// Withdrawn from use but retained for the record.
|
||||
Retired,
|
||||
}
|
||||
|
||||
impl Status {
|
||||
/// Whether an item in this state may appear on a graded assessment.
|
||||
pub fn is_usable(self) -> bool {
|
||||
matches!(self, Status::Approved)
|
||||
}
|
||||
|
||||
/// The snake_case token used in YAML.
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Status::Draft => "draft",
|
||||
Status::InReview => "in_review",
|
||||
Status::NeedsRevision => "needs_revision",
|
||||
Status::Approved => "approved",
|
||||
Status::Retired => "retired",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for Status {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// The response format of an item.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Format {
|
||||
/// Exactly one keyed option.
|
||||
SingleBestAnswer,
|
||||
/// One or more keyed options; the student must find every one.
|
||||
MultipleResponse,
|
||||
/// Two options, True and False.
|
||||
TrueFalse,
|
||||
}
|
||||
|
||||
impl Format {
|
||||
/// The QTI question type Canvas expects for this format.
|
||||
pub fn qti_type(self) -> &'static str {
|
||||
match self {
|
||||
Format::SingleBestAnswer => "multiple_choice_question",
|
||||
Format::MultipleResponse => "multiple_answers_question",
|
||||
Format::TrueFalse => "true_false_question",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// How strongly an item is expected to separate strong from weak students.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Discrimination {
|
||||
/// Most prepared students get it; it anchors rather than separates.
|
||||
Low,
|
||||
/// Separates somewhat.
|
||||
Moderate,
|
||||
/// Expected to separate sharply.
|
||||
High,
|
||||
}
|
||||
|
||||
impl Discrimination {
|
||||
/// The point-biserial band this expectation implies, as `(low, high)`.
|
||||
///
|
||||
/// Used to check an a priori expectation against the observed statistic.
|
||||
pub fn expected_band(self) -> (f64, f64) {
|
||||
match self {
|
||||
Discrimination::Low => (-1.0, 0.20),
|
||||
Discrimination::Moderate => (0.15, 0.40),
|
||||
Discrimination::High => (0.30, 1.0),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// What was done about an item after reviewing its statistics.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ReviewAction {
|
||||
/// Behaved as intended; leave it alone.
|
||||
Keep,
|
||||
/// Rewrite before reusing.
|
||||
Revise,
|
||||
/// Credit a defensible distractor for this administration.
|
||||
AwardPartialCredit,
|
||||
/// The key was wrong; fix it and rescore.
|
||||
CorrectKey,
|
||||
/// Withdraw from use.
|
||||
Retire,
|
||||
/// Keep but watch on the next administration.
|
||||
Monitor,
|
||||
}
|
||||
|
||||
/// A machine-detected problem with an item's observed behavior.
|
||||
///
|
||||
/// These are written by analysis, not by hand, and they are the queue you work
|
||||
/// through when deciding what to revise.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Flag {
|
||||
/// Weaker students outperformed stronger ones. Almost always a keying error
|
||||
/// or a genuinely ambiguous stem.
|
||||
NegativeDiscrimination,
|
||||
/// Barely separates students.
|
||||
LowDiscrimination,
|
||||
/// A distractor correlates with total score better than the key does.
|
||||
DistractorOutperformsKey,
|
||||
/// Strong students chose one particular distractor at a high rate, which is
|
||||
/// the signature of a second defensible reading.
|
||||
KeyUnderperforms,
|
||||
/// Nearly everyone answered correctly; carries little information.
|
||||
TooEasy,
|
||||
/// Nearly everyone answered incorrectly and it did not discriminate.
|
||||
TooHard,
|
||||
/// Many responses were faster than plausible reading time.
|
||||
HighRapidGuess,
|
||||
/// A distractor almost nobody chose; it is doing no work.
|
||||
NonfunctioningDistractor,
|
||||
/// Performance differed across groups after conditioning on ability.
|
||||
DifFlagged,
|
||||
/// Marked ambiguous by hand or inferred from partial credit awarded to a
|
||||
/// distractor during grading.
|
||||
Ambiguous,
|
||||
/// The observed difficulty was far from the difficulty you predicted.
|
||||
DesignMismatch,
|
||||
}
|
||||
|
||||
impl Flag {
|
||||
/// Every flag.
|
||||
pub const ALL: [Flag; 11] = [
|
||||
Flag::NegativeDiscrimination,
|
||||
Flag::LowDiscrimination,
|
||||
Flag::DistractorOutperformsKey,
|
||||
Flag::KeyUnderperforms,
|
||||
Flag::TooEasy,
|
||||
Flag::TooHard,
|
||||
Flag::HighRapidGuess,
|
||||
Flag::NonfunctioningDistractor,
|
||||
Flag::DifFlagged,
|
||||
Flag::Ambiguous,
|
||||
Flag::DesignMismatch,
|
||||
];
|
||||
|
||||
/// The flag's stable code, matching its YAML spelling.
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Flag::NegativeDiscrimination => "negative_discrimination",
|
||||
Flag::LowDiscrimination => "low_discrimination",
|
||||
Flag::DistractorOutperformsKey => "distractor_outperforms_key",
|
||||
Flag::KeyUnderperforms => "key_underperforms",
|
||||
Flag::TooEasy => "too_easy",
|
||||
Flag::TooHard => "too_hard",
|
||||
Flag::HighRapidGuess => "high_rapid_guess",
|
||||
Flag::NonfunctioningDistractor => "nonfunctioning_distractor",
|
||||
Flag::DifFlagged => "dif_flagged",
|
||||
Flag::Ambiguous => "ambiguous",
|
||||
Flag::DesignMismatch => "design_mismatch",
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the flag should stop an item from being reused as written.
|
||||
///
|
||||
/// Distinguishing blocking from advisory flags is what turns analysis into a
|
||||
/// workflow: a negative discrimination is a keying bug to fix before the item
|
||||
/// is ever given again, while an easy item is merely uninformative.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// `true` for flags that demand a revision.
|
||||
pub fn is_blocking(self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Flag::NegativeDiscrimination
|
||||
| Flag::DistractorOutperformsKey
|
||||
| Flag::Ambiguous
|
||||
| Flag::KeyUnderperforms
|
||||
)
|
||||
}
|
||||
|
||||
/// A short explanation of what the flag means and what to do about it.
|
||||
pub fn advice(self) -> &'static str {
|
||||
match self {
|
||||
Flag::NegativeDiscrimination => {
|
||||
"check the key first, then the stem for a second valid reading"
|
||||
}
|
||||
Flag::LowDiscrimination => "expected for anchors; investigate if the level is 3+",
|
||||
Flag::DistractorOutperformsKey => "the distractor may be the better answer",
|
||||
Flag::KeyUnderperforms => "strong students split; look for an ambiguity",
|
||||
Flag::TooEasy => "fine as an anchor, wasteful if you meant it to discriminate",
|
||||
Flag::TooHard => "check for a missing prerequisite or an unclear stem",
|
||||
Flag::HighRapidGuess => "position on the form or time pressure, not the item",
|
||||
Flag::NonfunctioningDistractor => "replace it with a plausible error",
|
||||
Flag::DifFlagged => "inspect wording for content unrelated to the objective",
|
||||
Flag::Ambiguous => "rewrite the stem to exclude the second reading",
|
||||
Flag::DesignMismatch => "update your expectation or revise the item",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn levels_order_by_demand() {
|
||||
assert!(Level::Remember < Level::Create);
|
||||
assert_eq!(Level::Apply.code(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_process_belongs_to_exactly_one_level() {
|
||||
let mut seen = Vec::new();
|
||||
for level in Level::ALL {
|
||||
for p in level.processes() {
|
||||
assert!(!seen.contains(p), "{p} appears under two levels");
|
||||
seen.push(*p);
|
||||
assert_eq!(p.level(), level);
|
||||
}
|
||||
}
|
||||
assert_eq!(seen.len(), 19, "all processes are assigned");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn level_process_pairing_is_checked() {
|
||||
assert!(Level::Apply.allows(CognitiveProcess::Implement));
|
||||
assert!(!Level::Apply.allows(CognitiveProcess::Recall));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn level_serializes_as_an_integer() {
|
||||
assert_eq!(serde_json::to_string(&Level::Apply).unwrap(), "3");
|
||||
assert_eq!(serde_json::from_str::<Level>("4").unwrap(), Level::Analyze);
|
||||
assert!(serde_json::from_str::<Level>("6").is_err());
|
||||
assert!(serde_json::from_str::<Level>("0").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn processes_use_snake_case() {
|
||||
assert_eq!(
|
||||
serde_json::to_string(&CognitiveProcess::Implement).unwrap(),
|
||||
"\"implement\""
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::from_str::<Status>("\"in_review\"").unwrap(),
|
||||
Status::InReview
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user