1000 lines
30 KiB
Rust
1000 lines
30 KiB
Rust
// SPDX-License-Identifier: Prosperity-3.0.0
|
|
// Copyright Scientific Computing Studio
|
|
// Source: https://git.scient.ing/education/coursebank
|
|
|
|
//! The bank file: a collection of items scoped to a topic or a lecture.
|
|
//!
|
|
//! One bank per topic (or per lecture, if that suits how you teach) is the unit
|
|
//! of authoring. Banks are small enough to review in a pull request, they let
|
|
//! two people write questions without colliding, and `bank.scope` records what
|
|
//! the file is *for* so `coursebank catalog` can tell you that you have eleven
|
|
//! items on enzyme kinetics and none on regulation.
|
|
//!
|
|
//! Validation here is split in two on purpose. [`BankFile::validate`] checks what
|
|
//! must be true for the file to be usable at all: ids are unique, a keyed answer
|
|
//! exists, an approved item is fully specified, a level and its cognitive process
|
|
//! agree. The softer question of whether an item is *well written* lives in
|
|
//! [`crate::lint`], because those checks are advisory and you should be able to
|
|
//! ship a file that trips a few of them.
|
|
|
|
use std::collections::BTreeMap;
|
|
use std::path::Path;
|
|
|
|
use serde::{Deserialize, Serialize};
|
|
|
|
use crate::course::{CourseFile, SCHEMA_VERSION};
|
|
use crate::date::Date;
|
|
use crate::error::Result;
|
|
use crate::item::Item;
|
|
use crate::taxonomy::{Format, Level, Status};
|
|
use crate::yaml;
|
|
|
|
/// A whole bank file.
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
#[serde(deny_unknown_fields)]
|
|
pub struct BankFile {
|
|
/// Schema version this file targets.
|
|
#[serde(
|
|
default = "default_version",
|
|
deserialize_with = "yaml::flexible_string"
|
|
)]
|
|
pub schema_version: String,
|
|
|
|
/// Bank identity and scope.
|
|
pub bank: BankMeta,
|
|
|
|
/// Values applied to every item in the file that does not set its own.
|
|
#[serde(default)]
|
|
pub defaults: BankDefaults,
|
|
|
|
/// The items.
|
|
#[serde(default)]
|
|
pub items: Vec<Item>,
|
|
}
|
|
|
|
/// Bank identity and scope.
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
#[serde(deny_unknown_fields)]
|
|
pub struct BankMeta {
|
|
/// Stable id, unique across the course. Item ids are namespaced by it.
|
|
pub id: String,
|
|
/// Human title.
|
|
pub title: String,
|
|
/// What this bank covers.
|
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
pub description: Option<String>,
|
|
/// What the bank is scoped to, so coverage can be reported against it.
|
|
#[serde(default)]
|
|
pub scope: Scope,
|
|
/// Who maintains it.
|
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
pub maintainer: Option<String>,
|
|
/// When it was created.
|
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
pub created: Option<Date>,
|
|
/// When it was last touched.
|
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
pub updated: Option<Date>,
|
|
}
|
|
|
|
/// What a bank is scoped to.
|
|
///
|
|
/// A bank may be scoped by lecture, by objective, by topic, or by none of them.
|
|
/// Declaring the scope is what lets the catalog report *gaps*: it can only tell
|
|
/// you that lecture 12 has no Apply-level items if it knows lecture 12 is
|
|
/// supposed to be covered here.
|
|
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
|
#[serde(deny_unknown_fields)]
|
|
pub struct Scope {
|
|
/// Lectures this bank draws from.
|
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
|
pub lectures: Vec<String>,
|
|
/// Objectives this bank is responsible for covering.
|
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
|
pub learning_objectives: Vec<String>,
|
|
/// Units this bank belongs to.
|
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
|
pub units: Vec<String>,
|
|
/// Topics this bank is about.
|
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
|
pub topics: Vec<String>,
|
|
}
|
|
|
|
/// Per-file defaults, so common metadata is written once.
|
|
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
|
#[serde(deny_unknown_fields)]
|
|
pub struct BankDefaults {
|
|
/// Default author for items in this file.
|
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
pub author: Option<String>,
|
|
/// Default point value.
|
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
pub points: Option<f64>,
|
|
/// Expected option count; the linter flags items that differ, since an
|
|
/// inconsistent option count across a form is itself a cue to students.
|
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
pub options_per_item: Option<usize>,
|
|
/// Topics added to every item in the file.
|
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
|
pub topics: Vec<String>,
|
|
/// Sources applied to items that declare none.
|
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
|
pub lectures: Vec<String>,
|
|
}
|
|
|
|
impl BankFile {
|
|
/// Loads a bank file from disk.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `path` - the YAML file.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The parsed bank.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns [`crate::error::Error::Io`] or [`crate::error::Error::Yaml`].
|
|
pub fn load(path: &Path) -> Result<BankFile> {
|
|
yaml::read(path)
|
|
}
|
|
|
|
/// Writes a bank file back out as YAML.
|
|
///
|
|
/// Round-tripping loses comments, which is why calibration is written by an
|
|
/// explicit `coursebank calibrate` step rather than as a side effect of
|
|
/// anything else: you should be able to see the diff it produces.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `path` - the destination.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns [`crate::error::Error::Io`] on a write failure.
|
|
pub fn save(&self, path: &Path) -> Result<()> {
|
|
yaml::write(path, self)
|
|
}
|
|
|
|
/// Applies file defaults to items that omit the corresponding field.
|
|
///
|
|
/// Called after loading so the rest of the crate never has to think about
|
|
/// defaults again.
|
|
pub fn apply_defaults(&mut self) {
|
|
let d = self.defaults.clone();
|
|
for item in &mut self.items {
|
|
if item.author.is_none() {
|
|
item.author = d.author.clone();
|
|
}
|
|
if item.points.is_none() {
|
|
item.points = d.points;
|
|
}
|
|
for t in &d.topics {
|
|
if !item.topics.contains(t) {
|
|
item.topics.push(t.clone());
|
|
}
|
|
}
|
|
if item.sources.is_empty() {
|
|
for lec in &d.lectures {
|
|
item.sources.push(crate::item::Source {
|
|
lecture: lec.clone(),
|
|
slides: Vec::new(),
|
|
readings: Vec::new(),
|
|
recording_seconds: None,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Loads a bank and applies its defaults.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `path` - the YAML file.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The parsed bank with defaults resolved.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Propagates load errors.
|
|
pub fn load_resolved(path: &Path) -> Result<BankFile> {
|
|
let mut b = BankFile::load(path)?;
|
|
b.apply_defaults();
|
|
Ok(b)
|
|
}
|
|
|
|
/// Checks every invariant that must hold for the file to be usable.
|
|
///
|
|
/// Returns all problems rather than the first, so one run fixes one file.
|
|
/// When a course file is supplied, cross-file references are checked too.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `course` - the course registry, for resolving objective and lecture
|
|
/// references. Pass `None` to check only what is local to the file.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// Every problem found, empty when the file is sound.
|
|
pub fn validate(&self, course: Option<&CourseFile>) -> Vec<String> {
|
|
let mut issues = Vec::new();
|
|
|
|
if self.bank.id.trim().is_empty() {
|
|
issues.push("bank.id is empty".into());
|
|
}
|
|
if self.bank.title.trim().is_empty() {
|
|
issues.push("bank.title is empty".into());
|
|
}
|
|
if let (Some(created), Some(updated)) = (self.bank.created, self.bank.updated) {
|
|
if updated < created {
|
|
issues.push(format!(
|
|
"bank.updated ({updated}) is before bank.created ({created})"
|
|
));
|
|
}
|
|
}
|
|
|
|
// Duplicate item ids inside the file.
|
|
let mut counts: BTreeMap<&str, usize> = BTreeMap::new();
|
|
for it in &self.items {
|
|
*counts.entry(it.id.as_str()).or_insert(0) += 1;
|
|
}
|
|
for (id, n) in &counts {
|
|
if *n > 1 {
|
|
issues.push(format!("duplicate item id `{id}` appears {n} times"));
|
|
}
|
|
}
|
|
|
|
if let Some(c) = course {
|
|
for lec in &self.bank.scope.lectures {
|
|
if !c.lectures.contains_key(lec) {
|
|
issues.push(format!("bank.scope: unknown lecture `{lec}`"));
|
|
}
|
|
}
|
|
for lo in &self.bank.scope.learning_objectives {
|
|
if !c.learning_objectives.contains_key(lo) {
|
|
issues.push(format!("bank.scope: unknown learning objective `{lo}`"));
|
|
}
|
|
}
|
|
}
|
|
|
|
for it in &self.items {
|
|
issues.extend(
|
|
validate_item(it, course, self.defaults.options_per_item)
|
|
.into_iter()
|
|
.map(|m| format!("{}: {m}", it.id)),
|
|
);
|
|
}
|
|
|
|
issues
|
|
}
|
|
|
|
/// Items that may be placed on a graded assessment.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// References to approved, unretired items.
|
|
pub fn assemblable(&self) -> Vec<&Item> {
|
|
self.items.iter().filter(|i| i.is_assemblable()).collect()
|
|
}
|
|
|
|
/// Counts of assemblable, non-bonus items by level.
|
|
///
|
|
/// This is the five-tuple you check a blueprint against.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// A map from level to count.
|
|
pub fn level_counts(&self) -> BTreeMap<Level, usize> {
|
|
let mut out: BTreeMap<Level, usize> = Level::ALL.iter().map(|l| (*l, 0)).collect();
|
|
for it in self.items.iter().filter(|i| i.is_assemblable() && !i.bonus) {
|
|
*out.entry(it.level).or_insert(0) += 1;
|
|
}
|
|
out
|
|
}
|
|
|
|
/// A skeleton bank file for `coursebank bank new`.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `id` - the bank id.
|
|
/// * `title` - the bank title.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// A bank with no items.
|
|
pub fn skeleton(id: &str, title: &str) -> BankFile {
|
|
BankFile {
|
|
schema_version: SCHEMA_VERSION.to_string(),
|
|
bank: BankMeta {
|
|
id: id.to_string(),
|
|
title: title.to_string(),
|
|
description: None,
|
|
scope: Scope::default(),
|
|
maintainer: None,
|
|
created: Some(Date::today()),
|
|
updated: Some(Date::today()),
|
|
},
|
|
defaults: BankDefaults::default(),
|
|
items: Vec::new(),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Validates one item.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `it` - the item.
|
|
/// * `course` - the course registry, when available.
|
|
/// * `expected_options` - the file's declared option count, when set.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// Problems found, without the item id prefix.
|
|
fn validate_item(
|
|
it: &Item,
|
|
course: Option<&CourseFile>,
|
|
expected_options: Option<usize>,
|
|
) -> Vec<String> {
|
|
let mut issues = Vec::new();
|
|
|
|
if it.id.trim().is_empty() {
|
|
issues.push("empty id".into());
|
|
}
|
|
if it.stem.trim().is_empty() {
|
|
issues.push("empty stem".into());
|
|
}
|
|
if it.version == 0 {
|
|
issues.push("version must be at least 1".into());
|
|
}
|
|
|
|
// --- options ----
|
|
// An open-response item takes no options; its answer lives in `solution`.
|
|
// Every other format needs at least two things to choose between.
|
|
if it.format.has_options() {
|
|
if it.options.len() < 2 {
|
|
issues.push(format!(
|
|
"needs at least 2 options, has {}",
|
|
it.options.len()
|
|
));
|
|
}
|
|
} else if !it.options.is_empty() {
|
|
issues.push(format!(
|
|
"{} items take no options, but {} were given; put the answer in `solution`",
|
|
it.format.as_str(),
|
|
it.options.len()
|
|
));
|
|
}
|
|
let mut seen: Vec<&str> = Vec::new();
|
|
for (i, o) in it.options.iter().enumerate() {
|
|
let pos = i + 1;
|
|
if o.text.trim().is_empty() {
|
|
issues.push(format!("option {pos}: empty text"));
|
|
}
|
|
let letter_ok = o.id.len() == 1
|
|
&& o.id
|
|
.chars()
|
|
.next()
|
|
.map(|c| c.is_ascii_uppercase() && c <= 'H')
|
|
.unwrap_or(false);
|
|
if !letter_ok {
|
|
issues.push(format!(
|
|
"option {pos}: id `{}` must be a single letter A through H",
|
|
o.id
|
|
));
|
|
}
|
|
if seen.contains(&o.id.as_str()) {
|
|
issues.push(format!("option {pos}: duplicate option id `{}`", o.id));
|
|
}
|
|
seen.push(&o.id);
|
|
|
|
let credit = o.credit();
|
|
if !(0.0..=1.0).contains(&credit) {
|
|
issues.push(format!(
|
|
"option {}: credit must be between 0 and 1, got {credit}",
|
|
o.id
|
|
));
|
|
}
|
|
// Partial credit must be argued for in writing, not remembered.
|
|
if o.is_partial() && o.defense.is_none() {
|
|
issues.push(format!(
|
|
"option {}: awards credit {credit} but gives no `defense`",
|
|
o.id
|
|
));
|
|
}
|
|
if o.is_partial() && !o.defensible {
|
|
issues.push(format!(
|
|
"option {}: awards credit {credit} but is not marked `defensible: true`",
|
|
o.id
|
|
));
|
|
}
|
|
if o.correct && credit == 0.0 {
|
|
issues.push(format!(
|
|
"option {}: keyed correct but earns no credit",
|
|
o.id
|
|
));
|
|
}
|
|
}
|
|
if let Some(n) = expected_options {
|
|
if it.options.len() != n && !it.options.is_empty() {
|
|
issues.push(format!(
|
|
"has {} options but the bank declares {n} per item",
|
|
it.options.len()
|
|
));
|
|
}
|
|
}
|
|
|
|
// --- key ---
|
|
let keys = it.key_indices();
|
|
match it.format {
|
|
Format::SingleBestAnswer => {
|
|
if keys.len() != 1 {
|
|
issues.push(format!(
|
|
"single_best_answer needs exactly one keyed option, has {}",
|
|
keys.len()
|
|
));
|
|
}
|
|
}
|
|
Format::MultipleResponse => {
|
|
if keys.is_empty() {
|
|
issues.push("multiple_response needs at least one keyed option".into());
|
|
}
|
|
if keys.len() == it.options.len() {
|
|
issues.push("multiple_response keys every option, so it asks nothing".into());
|
|
}
|
|
}
|
|
Format::TrueFalse => {
|
|
if it.options.len() != 2 {
|
|
issues.push(format!(
|
|
"true_false needs exactly 2 options, has {}",
|
|
it.options.len()
|
|
));
|
|
}
|
|
if keys.len() != 1 {
|
|
issues.push("true_false needs exactly one keyed option".into());
|
|
}
|
|
}
|
|
Format::OpenResponse => {
|
|
if !keys.is_empty() {
|
|
issues.push("open_response items have no keyed option".into());
|
|
}
|
|
}
|
|
}
|
|
|
|
// --- level and process must agree -------
|
|
if let Some(p) = it.cognitive_process {
|
|
if !it.level.allows(p) {
|
|
issues.push(format!(
|
|
"cognitive_process `{p}` belongs to level {} but the item is level {}",
|
|
p.level().code(),
|
|
it.level.code()
|
|
));
|
|
}
|
|
}
|
|
|
|
// --- design plausibility ------
|
|
if let Some(d) = &it.design {
|
|
if let Some(x) = d.expected_difficulty {
|
|
if !(0.0..=1.0).contains(&x) {
|
|
issues.push(format!(
|
|
"design.expected_difficulty must be between 0 and 1, got {x}"
|
|
));
|
|
}
|
|
}
|
|
if let Some(t) = d.expected_time_seconds {
|
|
if t <= 0.0 {
|
|
issues.push(format!(
|
|
"design.expected_time_seconds must be positive, got {t}"
|
|
));
|
|
}
|
|
}
|
|
}
|
|
|
|
// --- calibration plausibility ------
|
|
if let Some(c) = &it.calibration {
|
|
if let Some(p) = c.p_value {
|
|
if !(0.0..=1.0).contains(&p) {
|
|
issues.push(format!(
|
|
"calibration.p_value must be between 0 and 1, got {p}"
|
|
));
|
|
}
|
|
}
|
|
if let Some(r) = c.point_biserial {
|
|
if !(-1.0..=1.0).contains(&r) {
|
|
issues.push(format!(
|
|
"calibration.point_biserial must be between -1 and 1, got {r}"
|
|
));
|
|
}
|
|
}
|
|
for letter in c.option_stats.keys() {
|
|
if it.option(letter).is_none() {
|
|
issues.push(format!(
|
|
"calibration.option_stats has `{letter}`, which is not an option of this item"
|
|
));
|
|
}
|
|
}
|
|
if let Some(irt) = &c.irt {
|
|
if irt.a <= 0.0 {
|
|
issues.push(format!("calibration.irt.a must be positive, got {}", irt.a));
|
|
}
|
|
if let Some(cp) = irt.c {
|
|
if !(0.0..1.0).contains(&cp) {
|
|
issues.push(format!("calibration.irt.c must be in [0, 1), got {cp}"));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// --- history must be coherent ------
|
|
let mut last_version = 0u32;
|
|
for (i, h) in it.history.iter().enumerate() {
|
|
if h.version <= last_version {
|
|
issues.push(format!(
|
|
"history entry {} has version {} which does not increase",
|
|
i + 1,
|
|
h.version
|
|
));
|
|
}
|
|
last_version = h.version;
|
|
}
|
|
if !it.history.is_empty() && last_version > it.version {
|
|
issues.push(format!(
|
|
"history records version {last_version} but the item says version {}",
|
|
it.version
|
|
));
|
|
}
|
|
|
|
// --- retirement -----
|
|
if it.retired.is_some() && it.status != Status::Retired {
|
|
issues.push(format!(
|
|
"has a `retired` block but status is `{}`",
|
|
it.status
|
|
));
|
|
}
|
|
|
|
// --- approval gate -------
|
|
// Approval is what permits an item onto a graded assessment, so it is the
|
|
// right place to require that the item is fully sourced and designed.
|
|
if it.status == Status::Approved {
|
|
if it.cognitive_process.is_none() {
|
|
issues.push("approved items must declare a cognitive_process".into());
|
|
}
|
|
if it.learning_objectives.is_empty() {
|
|
issues.push("approved items must reference at least one learning objective".into());
|
|
}
|
|
if it.sources.is_empty() {
|
|
issues.push("approved items must cite at least one source".into());
|
|
}
|
|
if it.design.is_none() {
|
|
issues.push("approved items must carry a design block".into());
|
|
}
|
|
// An open-response item is graded from its solution, so approving one with
|
|
// neither a model answer nor a rubric would leave nothing to mark it by.
|
|
if !it.format.has_options() {
|
|
let gradeable = it
|
|
.solution
|
|
.as_ref()
|
|
.is_some_and(|s| s.model_answer.is_some() || !s.rubric.is_empty());
|
|
if !gradeable {
|
|
issues.push(
|
|
"approved open_response items need a solution with a model_answer or a rubric"
|
|
.into(),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
// --- cross-file references ----
|
|
if let Some(c) = course {
|
|
for lo in &it.learning_objectives {
|
|
match c.learning_objectives.get(lo) {
|
|
None => issues.push(format!("unknown learning objective `{lo}`")),
|
|
Some(obj) => {
|
|
if let Some(ceiling) = obj.level_ceiling {
|
|
if it.level > ceiling {
|
|
issues.push(format!(
|
|
"level {} exceeds the ceiling {} declared for objective `{lo}`",
|
|
it.level.code(),
|
|
ceiling.code()
|
|
));
|
|
}
|
|
}
|
|
if !obj.assessed {
|
|
issues.push(format!(
|
|
"objective `{lo}` is marked `assessed: false` but this item measures it"
|
|
));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
for s in &it.sources {
|
|
if !c.lectures.contains_key(&s.lecture) {
|
|
issues.push(format!("unknown lecture `{}`", s.lecture));
|
|
}
|
|
}
|
|
if let Some(st) = &it.stimulus {
|
|
if !c.stimuli.contains_key(st) {
|
|
issues.push(format!("unknown stimulus `{st}`"));
|
|
}
|
|
}
|
|
// A citation that names a reference key must name a real one, so a review
|
|
// pointer in the solutions document never resolves to nothing.
|
|
for citation in it.solution.iter().flat_map(|s| &s.review) {
|
|
if let Some(key) = &citation.reference {
|
|
if !c.references.contains_key(key) {
|
|
issues.push(format!("solution.review cites unknown reference `{key}`"));
|
|
}
|
|
}
|
|
}
|
|
if let Some(floor) = c.policy.partial_credit_floor_level {
|
|
for o in &it.options {
|
|
if o.is_partial() && it.level < floor {
|
|
issues.push(format!(
|
|
"option {} awards partial credit at level {}, below the course floor of {}",
|
|
o.id,
|
|
it.level.code(),
|
|
floor.code()
|
|
));
|
|
}
|
|
}
|
|
}
|
|
if !c.policy.allow_partial_credit && it.options.iter().any(|o| o.is_partial()) {
|
|
issues.push("awards partial credit, which the course policy disallows".into());
|
|
}
|
|
}
|
|
|
|
issues
|
|
}
|
|
|
|
fn default_version() -> String {
|
|
SCHEMA_VERSION.to_string()
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
fn bank(items_yaml: &str) -> BankFile {
|
|
let src = format!("bank:\n id: b\n title: Bank\ndefaults: {{}}\nitems:\n{items_yaml}");
|
|
let mut b: BankFile = serde_yaml_ng::from_str(&src).expect("bank parses");
|
|
b.apply_defaults();
|
|
b
|
|
}
|
|
|
|
#[test]
|
|
fn sound_bank_validates_clean() {
|
|
let b = bank(
|
|
r#"
|
|
- id: q-a-001
|
|
status: draft
|
|
level: 1
|
|
stem: What is x?
|
|
options:
|
|
- { id: A, text: right, correct: true }
|
|
- { id: B, text: wrong }
|
|
- { id: C, text: wrong too }
|
|
"#,
|
|
);
|
|
assert!(b.validate(None).is_empty(), "{:?}", b.validate(None));
|
|
}
|
|
|
|
#[test]
|
|
fn open_response_validates_without_options_and_rejects_them() {
|
|
// No options is fine, and no key is required.
|
|
let ok = bank(
|
|
r#"
|
|
- id: q-a-op-001
|
|
status: draft
|
|
level: 2
|
|
format: open_response
|
|
stem: Explain the first law.
|
|
solution:
|
|
model_answer: Energy is conserved.
|
|
"#,
|
|
);
|
|
assert!(ok.validate(None).is_empty(), "{:?}", ok.validate(None));
|
|
|
|
// Giving an open-response item options is the mistake, and so is approving
|
|
// one with nothing to grade it by.
|
|
let bad = bank(
|
|
r#"
|
|
- id: q-a-op-002
|
|
status: approved
|
|
level: 2
|
|
format: open_response
|
|
cognitive_process: explain
|
|
learning_objectives: [lo-x]
|
|
sources: [{ lecture: L1.1 }]
|
|
design: { rationale: r }
|
|
stem: Explain the first law.
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: B, text: b }
|
|
"#,
|
|
);
|
|
let issues = bad.validate(None);
|
|
assert!(
|
|
issues.iter().any(|i| i.contains("take no options")),
|
|
"{issues:?}"
|
|
);
|
|
assert!(
|
|
issues
|
|
.iter()
|
|
.any(|i| i.contains("model_answer or a rubric")),
|
|
"{issues:?}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn catches_missing_and_multiple_keys() {
|
|
let b = bank(
|
|
r#"
|
|
- id: q-a-001
|
|
status: draft
|
|
level: 1
|
|
stem: s
|
|
options:
|
|
- { id: A, text: a }
|
|
- { id: B, text: b }
|
|
- id: q-a-002
|
|
status: draft
|
|
level: 1
|
|
format: single_best_answer
|
|
stem: s
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: B, text: b, correct: true }
|
|
"#,
|
|
);
|
|
let issues = b.validate(None);
|
|
assert!(
|
|
issues
|
|
.iter()
|
|
.any(|i| i.contains("exactly one keyed option"))
|
|
);
|
|
assert_eq!(
|
|
issues
|
|
.iter()
|
|
.filter(|i| i.contains("exactly one keyed option"))
|
|
.count(),
|
|
2
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn catches_duplicate_ids_and_letters() {
|
|
let b = bank(
|
|
r#"
|
|
- id: q-a-001
|
|
status: draft
|
|
level: 1
|
|
stem: s
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: A, text: b }
|
|
- id: q-a-001
|
|
status: draft
|
|
level: 1
|
|
stem: s
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: B, text: b }
|
|
"#,
|
|
);
|
|
let issues = b.validate(None);
|
|
assert!(issues.iter().any(|i| i.contains("duplicate item id")));
|
|
assert!(issues.iter().any(|i| i.contains("duplicate option id")));
|
|
}
|
|
|
|
#[test]
|
|
fn level_and_process_must_agree() {
|
|
let b = bank(
|
|
r#"
|
|
- id: q-a-001
|
|
status: draft
|
|
level: 3
|
|
cognitive_process: recall
|
|
stem: s
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: B, text: b }
|
|
"#,
|
|
);
|
|
let issues = b.validate(None);
|
|
assert!(
|
|
issues.iter().any(|i| i.contains("belongs to level 1")),
|
|
"{issues:?}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn approval_requires_full_specification() {
|
|
let b = bank(
|
|
r#"
|
|
- id: q-a-001
|
|
status: approved
|
|
level: 1
|
|
stem: s
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: B, text: b }
|
|
"#,
|
|
);
|
|
let issues = b.validate(None);
|
|
for want in [
|
|
"cognitive_process",
|
|
"learning objective",
|
|
"source",
|
|
"design block",
|
|
] {
|
|
assert!(
|
|
issues.iter().any(|i| i.contains(want)),
|
|
"expected a complaint about {want}, got {issues:?}"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn partial_credit_needs_a_written_defense() {
|
|
let b = bank(
|
|
r#"
|
|
- id: q-a-001
|
|
status: draft
|
|
level: 5
|
|
stem: s
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: B, text: b, credit: 0.5 }
|
|
"#,
|
|
);
|
|
let issues = b.validate(None);
|
|
assert!(issues.iter().any(|i| i.contains("no `defense`")));
|
|
assert!(issues.iter().any(|i| i.contains("defensible: true")));
|
|
}
|
|
|
|
#[test]
|
|
fn defaults_fill_in_items() {
|
|
let src = r#"
|
|
bank: { id: b, title: Bank }
|
|
defaults:
|
|
author: Alex
|
|
points: 1.5
|
|
topics: [kinetics]
|
|
lectures: [L11]
|
|
items:
|
|
- id: q-a-001
|
|
status: draft
|
|
level: 1
|
|
stem: s
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: B, text: b }
|
|
- id: q-a-002
|
|
status: draft
|
|
level: 1
|
|
stem: s
|
|
author: Someone Else
|
|
topics: [kinetics]
|
|
sources: [{ lecture: L12 }]
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: B, text: b }
|
|
"#;
|
|
let mut b: BankFile = serde_yaml_ng::from_str(src).unwrap();
|
|
b.apply_defaults();
|
|
assert_eq!(b.items[0].author.as_deref(), Some("Alex"));
|
|
assert_eq!(b.items[0].points, Some(1.5));
|
|
assert_eq!(b.items[0].topics, vec!["kinetics"]);
|
|
assert_eq!(b.items[0].sources[0].lecture, "L11");
|
|
// Explicit values win, and topics are not duplicated.
|
|
assert_eq!(b.items[1].author.as_deref(), Some("Someone Else"));
|
|
assert_eq!(b.items[1].topics, vec!["kinetics"]);
|
|
assert_eq!(b.items[1].sources[0].lecture, "L12");
|
|
}
|
|
|
|
#[test]
|
|
fn cross_file_references_are_checked_against_the_course() {
|
|
let course: CourseFile = serde_yaml_ng::from_str(
|
|
r#"
|
|
course: { code: X, title: Y, term: Z }
|
|
lectures:
|
|
L11: { title: Kinetics }
|
|
learning_objectives:
|
|
lo-known: { text: Do the thing, level_ceiling: 2 }
|
|
"#,
|
|
)
|
|
.unwrap();
|
|
let b = bank(
|
|
r#"
|
|
- id: q-a-001
|
|
status: draft
|
|
level: 4
|
|
stem: s
|
|
learning_objectives: [lo-known, lo-unknown]
|
|
sources: [{ lecture: L99 }]
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: B, text: b }
|
|
"#,
|
|
);
|
|
let issues = b.validate(Some(&course));
|
|
assert!(
|
|
issues
|
|
.iter()
|
|
.any(|i| i.contains("unknown learning objective `lo-unknown`"))
|
|
);
|
|
assert!(issues.iter().any(|i| i.contains("unknown lecture `L99`")));
|
|
assert!(
|
|
issues.iter().any(|i| i.contains("exceeds the ceiling")),
|
|
"{issues:?}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn history_versions_must_increase() {
|
|
let b = bank(
|
|
r#"
|
|
- id: q-a-001
|
|
version: 2
|
|
status: draft
|
|
level: 1
|
|
stem: s
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: B, text: b }
|
|
history:
|
|
- { version: 2, date: 2026-01-01, change: second }
|
|
- { version: 1, date: 2026-01-02, change: first }
|
|
"#,
|
|
);
|
|
let issues = b.validate(None);
|
|
assert!(issues.iter().any(|i| i.contains("does not increase")));
|
|
}
|
|
|
|
#[test]
|
|
fn level_counts_exclude_drafts_and_bonuses() {
|
|
let b = bank(
|
|
r#"
|
|
- id: q-a-001
|
|
status: approved
|
|
level: 1
|
|
cognitive_process: recall
|
|
stem: s
|
|
learning_objectives: [lo]
|
|
sources: [{ lecture: L1 }]
|
|
design: { expected_difficulty: 0.8 }
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: B, text: b }
|
|
- id: q-a-002
|
|
status: draft
|
|
level: 1
|
|
stem: s
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: B, text: b }
|
|
- id: q-a-003
|
|
status: approved
|
|
level: 5
|
|
cognitive_process: generate
|
|
bonus: true
|
|
stem: s
|
|
learning_objectives: [lo]
|
|
sources: [{ lecture: L1 }]
|
|
design: { expected_difficulty: 0.3 }
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: B, text: b }
|
|
"#,
|
|
);
|
|
let counts = b.level_counts();
|
|
assert_eq!(counts[&Level::Remember], 1);
|
|
assert_eq!(counts[&Level::Create], 0, "bonus items are not scored");
|
|
assert_eq!(b.assemblable().len(), 2);
|
|
}
|
|
}
|