1107 lines
35 KiB
Rust
1107 lines
35 KiB
Rust
// SPDX-License-Identifier: Prosperity-3.0.0
|
|
// Copyright Scientific Computing Studio
|
|
// Source: https://git.scient.ing/education/coursebank
|
|
|
|
//! The bank file: a collection of items scoped to a topic or a lecture.
|
|
//!
|
|
//! One bank per topic (or per lecture, if that suits how you teach) is the unit
|
|
//! of authoring. Banks are small enough to review in a pull request, they let
|
|
//! two people write questions without colliding, and `bank.scope` records what
|
|
//! the file is *for* so `coursebank catalog` can tell you that you have eleven
|
|
//! items on enzyme kinetics and none on regulation.
|
|
//!
|
|
//! Validation here is split in two on purpose. [`BankFile::validate`] checks what
|
|
//! must be true for the file to be usable at all: ids are unique, a keyed answer
|
|
//! exists, an approved item is fully specified, a level and its cognitive process
|
|
//! agree. The softer question of whether an item is *well written* lives in
|
|
//! [`crate::lint`], because those checks are advisory and you should be able to
|
|
//! ship a file that trips a few of them.
|
|
|
|
use std::collections::BTreeMap;
|
|
use std::path::Path;
|
|
|
|
use serde::{Deserialize, Serialize};
|
|
|
|
use crate::course::{CourseFile, SCHEMA_VERSION};
|
|
use crate::date::Date;
|
|
use crate::error::Result;
|
|
use crate::item::Item;
|
|
use crate::taxonomy::{Format, Level, Status};
|
|
use crate::yaml;
|
|
|
|
/// A whole bank file.
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
#[serde(deny_unknown_fields)]
|
|
pub struct BankFile {
|
|
/// Schema version this file targets.
|
|
#[serde(
|
|
default = "default_version",
|
|
deserialize_with = "yaml::flexible_string"
|
|
)]
|
|
pub schema_version: String,
|
|
|
|
/// Bank identity and scope.
|
|
pub bank: BankMeta,
|
|
|
|
/// Values applied to every item in the file that does not set its own.
|
|
#[serde(default)]
|
|
pub defaults: BankDefaults,
|
|
|
|
/// The items.
|
|
#[serde(default)]
|
|
pub items: Vec<Item>,
|
|
}
|
|
|
|
/// Bank identity and scope.
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
#[serde(deny_unknown_fields)]
|
|
pub struct BankMeta {
|
|
/// Stable id, unique across the course. Item ids are namespaced by it.
|
|
pub id: String,
|
|
/// Human title.
|
|
pub title: String,
|
|
/// What this bank covers.
|
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
pub description: Option<String>,
|
|
/// What the bank is scoped to, so coverage can be reported against it.
|
|
#[serde(default)]
|
|
pub scope: Scope,
|
|
/// Who maintains it.
|
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
pub maintainer: Option<String>,
|
|
/// When it was created.
|
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
pub created: Option<Date>,
|
|
/// When it was last touched.
|
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
pub updated: Option<Date>,
|
|
}
|
|
|
|
/// What a bank is scoped to.
|
|
///
|
|
/// A bank may be scoped by lecture, by learning target, by topic, or by none of them.
|
|
/// Declaring the scope is what lets the catalog report *gaps*: it can only tell
|
|
/// you that lecture 12 has no Apply-level items if it knows lecture 12 is
|
|
/// supposed to be covered here.
|
|
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
|
#[serde(deny_unknown_fields)]
|
|
pub struct Scope {
|
|
/// Lectures this bank draws from.
|
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
|
pub lectures: Vec<String>,
|
|
/// Learning targets this bank is responsible for covering.
|
|
#[serde(
|
|
default,
|
|
alias = "learning_objectives",
|
|
skip_serializing_if = "Vec::is_empty"
|
|
)]
|
|
pub learning_targets: Vec<String>,
|
|
/// Units this bank belongs to.
|
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
|
pub units: Vec<String>,
|
|
/// Topics this bank is about.
|
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
|
pub topics: Vec<String>,
|
|
}
|
|
|
|
/// Per-file defaults, so common metadata is written once.
|
|
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
|
#[serde(deny_unknown_fields)]
|
|
pub struct BankDefaults {
|
|
/// Default author for items in this file.
|
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
pub author: Option<String>,
|
|
/// Default point value.
|
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
pub points: Option<f64>,
|
|
/// Expected option count; the linter flags items that differ, since an
|
|
/// inconsistent option count across a form is itself a cue to students.
|
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
pub options_per_item: Option<usize>,
|
|
/// Topics added to every item in the file.
|
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
|
pub topics: Vec<String>,
|
|
/// Sources applied to items that declare none.
|
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
|
pub lectures: Vec<String>,
|
|
}
|
|
|
|
impl BankFile {
|
|
/// Loads a bank file from disk.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `path` - the YAML file.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The parsed bank.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns [`crate::error::Error::Io`] or [`crate::error::Error::Yaml`].
|
|
pub fn load(path: &Path) -> Result<BankFile> {
|
|
yaml::read(path)
|
|
}
|
|
|
|
/// Writes a bank file back out as YAML.
|
|
///
|
|
/// Round-tripping loses comments, which is why calibration is written by an
|
|
/// explicit `coursebank calibrate` step rather than as a side effect of
|
|
/// anything else: you should be able to see the diff it produces.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `path` - the destination.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns [`crate::error::Error::Io`] on a write failure.
|
|
pub fn save(&self, path: &Path) -> Result<()> {
|
|
yaml::write(path, self)
|
|
}
|
|
|
|
/// Applies file defaults to items that omit the corresponding field.
|
|
///
|
|
/// Called after loading so the rest of the crate never has to think about
|
|
/// defaults again.
|
|
pub fn apply_defaults(&mut self) {
|
|
let d = self.defaults.clone();
|
|
for item in &mut self.items {
|
|
if item.author.is_none() {
|
|
item.author = d.author.clone();
|
|
}
|
|
if item.points.is_none() {
|
|
item.points = d.points;
|
|
}
|
|
for t in &d.topics {
|
|
if !item.topics.contains(t) {
|
|
item.topics.push(t.clone());
|
|
}
|
|
}
|
|
if item.sources.is_empty() {
|
|
for lec in &d.lectures {
|
|
item.sources.push(crate::item::Source {
|
|
lecture: lec.clone(),
|
|
slides: Vec::new(),
|
|
readings: Vec::new(),
|
|
recording_seconds: None,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Loads a bank and applies its defaults.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `path` - the YAML file.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The parsed bank with defaults resolved.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Propagates load errors.
|
|
pub fn load_resolved(path: &Path) -> Result<BankFile> {
|
|
let mut b = BankFile::load(path)?;
|
|
b.apply_defaults();
|
|
Ok(b)
|
|
}
|
|
|
|
/// Checks every invariant that must hold for the file to be usable.
|
|
///
|
|
/// Returns all problems rather than the first, so one run fixes one file.
|
|
/// When a course file is supplied, cross-file references are checked too.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `course` - the course registry, for resolving objective and lecture
|
|
/// references. Pass `None` to check only what is local to the file.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// Every problem found, empty when the file is sound.
|
|
pub fn validate(&self, course: Option<&CourseFile>) -> Vec<String> {
|
|
let mut issues = Vec::new();
|
|
|
|
if self.bank.id.trim().is_empty() {
|
|
issues.push("bank.id is empty".into());
|
|
}
|
|
if self.bank.title.trim().is_empty() {
|
|
issues.push("bank.title is empty".into());
|
|
}
|
|
if let (Some(created), Some(updated)) = (self.bank.created, self.bank.updated) {
|
|
if updated < created {
|
|
issues.push(format!(
|
|
"bank.updated ({updated}) is before bank.created ({created})"
|
|
));
|
|
}
|
|
}
|
|
|
|
// Duplicate item ids inside the file.
|
|
let mut counts: BTreeMap<&str, usize> = BTreeMap::new();
|
|
for it in &self.items {
|
|
*counts.entry(it.id.as_str()).or_insert(0) += 1;
|
|
}
|
|
for (id, n) in &counts {
|
|
if *n > 1 {
|
|
issues.push(format!("duplicate item id `{id}` appears {n} times"));
|
|
}
|
|
}
|
|
|
|
if let Some(c) = course {
|
|
for lec in &self.bank.scope.lectures {
|
|
if !c.lectures.contains_key(lec) {
|
|
issues.push(format!("bank.scope: unknown lecture `{lec}`"));
|
|
}
|
|
}
|
|
for target in &self.bank.scope.learning_targets {
|
|
if !c.is_target(target) && !c.is_objective(target) {
|
|
issues.push(format!("bank.scope: unknown learning target `{target}`"));
|
|
}
|
|
}
|
|
}
|
|
|
|
for it in &self.items {
|
|
issues.extend(
|
|
validate_item(it, course, self.defaults.options_per_item)
|
|
.into_iter()
|
|
.map(|m| format!("{}: {m}", it.id)),
|
|
);
|
|
}
|
|
|
|
issues
|
|
}
|
|
|
|
/// Items that may be placed on a graded assessment.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// References to approved, unretired items.
|
|
pub fn assemblable(&self) -> Vec<&Item> {
|
|
self.items.iter().filter(|i| i.is_assemblable()).collect()
|
|
}
|
|
|
|
/// Counts of assemblable, non-bonus items by level.
|
|
///
|
|
/// This is the five-tuple you check a blueprint against.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// A map from level to count.
|
|
pub fn level_counts(&self) -> BTreeMap<Level, usize> {
|
|
let mut out: BTreeMap<Level, usize> = Level::ALL.iter().map(|l| (*l, 0)).collect();
|
|
for it in self.items.iter().filter(|i| i.is_assemblable() && !i.bonus) {
|
|
*out.entry(it.level).or_insert(0) += 1;
|
|
}
|
|
out
|
|
}
|
|
|
|
/// A skeleton bank file for `coursebank bank new`.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `id` - the bank id.
|
|
/// * `title` - the bank title.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// A bank with no items.
|
|
pub fn skeleton(id: &str, title: &str) -> BankFile {
|
|
BankFile {
|
|
schema_version: SCHEMA_VERSION.to_string(),
|
|
bank: BankMeta {
|
|
id: id.to_string(),
|
|
title: title.to_string(),
|
|
description: None,
|
|
scope: Scope::default(),
|
|
maintainer: None,
|
|
created: Some(Date::today()),
|
|
updated: Some(Date::today()),
|
|
},
|
|
defaults: BankDefaults::default(),
|
|
items: Vec::new(),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Validates one item.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `it` - the item.
|
|
/// * `course` - the course registry, when available.
|
|
/// * `expected_options` - the file's declared option count, when set.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// Problems found, without the item id prefix.
|
|
fn validate_item(
|
|
it: &Item,
|
|
course: Option<&CourseFile>,
|
|
expected_options: Option<usize>,
|
|
) -> Vec<String> {
|
|
let mut issues = Vec::new();
|
|
|
|
if it.id.trim().is_empty() {
|
|
issues.push("empty id".into());
|
|
}
|
|
if it.stem.trim().is_empty() {
|
|
issues.push("empty stem".into());
|
|
}
|
|
if let Some(replaced) = &it.supersedes {
|
|
if *replaced == it.id {
|
|
issues.push("supersedes names this item".into());
|
|
}
|
|
}
|
|
|
|
// --- options ----
|
|
// An open-response item takes no options; its answer lives in `solution`.
|
|
// Every other format needs at least two things to choose between.
|
|
if it.format.has_options() {
|
|
if it.options.len() < 2 {
|
|
issues.push(format!(
|
|
"needs at least 2 options, has {}",
|
|
it.options.len()
|
|
));
|
|
}
|
|
} else if !it.options.is_empty() {
|
|
issues.push(format!(
|
|
"{} items take no options, but {} were given; put the answer in `solution`",
|
|
it.format.as_str(),
|
|
it.options.len()
|
|
));
|
|
}
|
|
let mut seen: Vec<&str> = Vec::new();
|
|
for (i, o) in it.options.iter().enumerate() {
|
|
let pos = i + 1;
|
|
if o.text.trim().is_empty() {
|
|
issues.push(format!("option {pos}: empty text"));
|
|
}
|
|
// Two forms are accepted: the 2.0 name, and the letter that preceded
|
|
// it. A bank migrates when `coursebank migrate options` is run on it,
|
|
// not when the tool is upgraded, so a course mid-migration still loads.
|
|
let slug_ok = o.id.strip_prefix("o-").is_some_and(|rest| {
|
|
!rest.is_empty()
|
|
&& !rest.starts_with('-')
|
|
&& !rest.ends_with('-')
|
|
&& !rest.contains("--")
|
|
&& rest
|
|
.chars()
|
|
.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-')
|
|
});
|
|
if !slug_ok && !Item::is_legacy_option_id(&o.id) {
|
|
issues.push(format!(
|
|
"option {pos}: id `{}` is neither a name such as `o-fourth-line` nor a pre-2.0 \
|
|
letter A through H",
|
|
o.id
|
|
));
|
|
}
|
|
if seen.contains(&o.id.as_str()) {
|
|
issues.push(format!("option {pos}: duplicate option id `{}`", o.id));
|
|
}
|
|
if let Some(retirement) = &o.retired {
|
|
if retirement.reason.trim().is_empty() {
|
|
issues.push(format!(
|
|
"option {pos}: retired without a reason. The reason is the finding — what \
|
|
the option did or failed to do — and it is the only part of a retirement \
|
|
that is worth anything later."
|
|
));
|
|
}
|
|
}
|
|
seen.push(&o.id);
|
|
|
|
let credit = o.credit();
|
|
if !(0.0..=1.0).contains(&credit) {
|
|
issues.push(format!(
|
|
"option {}: credit must be between 0 and 1, got {credit}",
|
|
o.id
|
|
));
|
|
}
|
|
// Partial credit must be argued for in writing, not remembered.
|
|
if o.is_partial() && o.defense.is_none() {
|
|
issues.push(format!(
|
|
"option {}: awards credit {credit} but gives no `defense`",
|
|
o.id
|
|
));
|
|
}
|
|
if o.is_partial() && !o.defensible {
|
|
issues.push(format!(
|
|
"option {}: awards credit {credit} but is not marked `defensible: true`",
|
|
o.id
|
|
));
|
|
}
|
|
if o.correct && credit == 0.0 {
|
|
issues.push(format!(
|
|
"option {}: keyed correct but earns no credit",
|
|
o.id
|
|
));
|
|
}
|
|
}
|
|
if let Some(n) = expected_options {
|
|
if it.options.len() != n && !it.options.is_empty() {
|
|
issues.push(format!(
|
|
"has {} options but the bank declares {n} per item",
|
|
it.options.len()
|
|
));
|
|
}
|
|
}
|
|
|
|
// --- key ---
|
|
// Counted over the pool that can still be drawn: a retired option is a
|
|
// record, not an offer.
|
|
let (live_keys, live_distractors) = it.pool();
|
|
let keys = it.key_indices();
|
|
match it.format {
|
|
Format::SingleBestAnswer => {
|
|
// Several defensible keys is a pool, not a bug — it is what lets you
|
|
// test whether "fourth" or "last of the four" is doing the work.
|
|
// Exactly one of them reaches a student, and that is the
|
|
// placement's business: see
|
|
// [`crate::catalog::Catalog::validate_record`].
|
|
if live_keys.is_empty() {
|
|
issues.push("single_best_answer needs at least one keyed option".into());
|
|
}
|
|
if live_distractors.is_empty() {
|
|
let retired = it.options.iter().any(|o| o.retired.is_some());
|
|
issues.push(if retired {
|
|
"every distractor is retired, so nothing can be drawn against the key".into()
|
|
} else {
|
|
"has no option that is not keyed correct, so it asks nothing".to_string()
|
|
});
|
|
}
|
|
}
|
|
Format::MultipleResponse => {
|
|
if keys.is_empty() {
|
|
issues.push("multiple_response needs at least one keyed option".into());
|
|
}
|
|
if keys.len() == it.options.len() {
|
|
issues.push("multiple_response keys every option, so it asks nothing".into());
|
|
}
|
|
}
|
|
Format::TrueFalse => {
|
|
if it.options.len() != 2 {
|
|
issues.push(format!(
|
|
"true_false needs exactly 2 options, has {}",
|
|
it.options.len()
|
|
));
|
|
}
|
|
if keys.len() != 1 {
|
|
issues.push("true_false needs exactly one keyed option".into());
|
|
}
|
|
}
|
|
Format::OpenResponse => {
|
|
if !keys.is_empty() {
|
|
issues.push("open_response items have no keyed option".into());
|
|
}
|
|
}
|
|
}
|
|
|
|
// --- level and process must agree -------
|
|
if let Some(p) = it.cognitive_process {
|
|
if !it.level.allows(p) {
|
|
issues.push(format!(
|
|
"cognitive_process `{p}` belongs to level {} but the item is level {}",
|
|
p.level().code(),
|
|
it.level.code()
|
|
));
|
|
}
|
|
}
|
|
|
|
// --- design plausibility ------
|
|
if let Some(d) = &it.design {
|
|
if let Some(x) = d.expected_difficulty {
|
|
if !(0.0..=1.0).contains(&x) {
|
|
issues.push(format!(
|
|
"design.expected_difficulty must be between 0 and 1, got {x}"
|
|
));
|
|
}
|
|
}
|
|
if let Some(t) = d.expected_time_seconds {
|
|
if t <= 0.0 {
|
|
issues.push(format!(
|
|
"design.expected_time_seconds must be positive, got {t}"
|
|
));
|
|
}
|
|
}
|
|
}
|
|
|
|
// --- calibration plausibility ------
|
|
if let Some(c) = &it.calibration {
|
|
if let Some(p) = c.p_value {
|
|
if !(0.0..=1.0).contains(&p) {
|
|
issues.push(format!(
|
|
"calibration.p_value must be between 0 and 1, got {p}"
|
|
));
|
|
}
|
|
}
|
|
if let Some(r) = c.point_biserial {
|
|
if !(-1.0..=1.0).contains(&r) {
|
|
issues.push(format!(
|
|
"calibration.point_biserial must be between -1 and 1, got {r}"
|
|
));
|
|
}
|
|
}
|
|
for option in c.option_stats.keys() {
|
|
if it.option(option).is_none() {
|
|
issues.push(format!(
|
|
"calibration.option_stats has `{option}`, which is not an option of this item"
|
|
));
|
|
}
|
|
}
|
|
if let Some(irt) = &c.irt {
|
|
if irt.a <= 0.0 {
|
|
issues.push(format!("calibration.irt.a must be positive, got {}", irt.a));
|
|
}
|
|
if let Some(cp) = irt.c {
|
|
if !(0.0..1.0).contains(&cp) {
|
|
issues.push(format!("calibration.irt.c must be in [0, 1), got {cp}"));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// --- retirement -----
|
|
if it.retired.is_some() && it.status != Status::Retired {
|
|
issues.push(format!(
|
|
"has a `retired` block but status is `{}`",
|
|
it.status
|
|
));
|
|
}
|
|
|
|
// --- approval gate -------
|
|
// Approval is what permits an item onto a graded assessment, so it is the
|
|
// right place to require that the item is fully sourced and designed.
|
|
if it.status == Status::Approved {
|
|
if it.cognitive_process.is_none() {
|
|
issues.push("approved items must declare a cognitive_process".into());
|
|
}
|
|
if it.learning_targets.is_empty() {
|
|
issues.push("approved items must reference at least one learning target".into());
|
|
}
|
|
if it.sources.is_empty() {
|
|
issues.push("approved items must cite at least one source".into());
|
|
}
|
|
if it.design.is_none() {
|
|
issues.push("approved items must carry a design block".into());
|
|
}
|
|
// An open-response item is graded from its solution, so approving one with
|
|
// neither a model answer nor a rubric would leave nothing to mark it by.
|
|
if !it.format.has_options() {
|
|
let gradeable = it
|
|
.solution
|
|
.as_ref()
|
|
.is_some_and(|s| s.model_answer.is_some() || !s.rubric.is_empty());
|
|
if !gradeable {
|
|
issues.push(
|
|
"approved open_response items need a solution with a model_answer or a rubric"
|
|
.into(),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
// --- cross-file references ----
|
|
if let Some(c) = course {
|
|
for tag in &it.learning_targets {
|
|
let is_target = c.is_target(tag);
|
|
if !is_target && !c.is_objective(tag) {
|
|
issues.push(format!("unknown learning target `{tag}`"));
|
|
continue;
|
|
}
|
|
// Items are tagged at the target tier. Tagging an objective that has
|
|
// targets would put the item in that objective's denominator without
|
|
// recording which performance the question actually asked for, and
|
|
// that record is what a report drills into and what coverage
|
|
// analysis counts. An objective with no targets stands as its own.
|
|
let its_targets = c.targets(tag);
|
|
if !its_targets.is_empty() {
|
|
issues.push(format!(
|
|
"`{tag}` is an objective with {} learning target(s); tag the specific \
|
|
target this item measures instead",
|
|
its_targets.len()
|
|
));
|
|
}
|
|
// The ceiling may be inherited from the objective, so a target that
|
|
// declares none of its own is still bounded.
|
|
if let Some(ceiling) = c.effective_level_ceiling(tag) {
|
|
if it.level > ceiling {
|
|
let declares_its_own = if is_target {
|
|
c.learning_targets
|
|
.get(tag)
|
|
.is_some_and(|t| t.level_ceiling.is_some())
|
|
} else {
|
|
true
|
|
};
|
|
let source = if declares_its_own {
|
|
format!("declared for `{tag}`")
|
|
} else {
|
|
format!(
|
|
"inherited by `{tag}` from its objective `{}`",
|
|
c.objective_for(tag)
|
|
)
|
|
};
|
|
issues.push(format!(
|
|
"level {} exceeds the ceiling {} {source}",
|
|
it.level.code(),
|
|
ceiling.code()
|
|
));
|
|
}
|
|
}
|
|
if !c.is_assessed(tag) {
|
|
let parked_above =
|
|
is_target && c.learning_targets.get(tag).is_some_and(|t| t.assessed);
|
|
let which = if parked_above {
|
|
format!(
|
|
"its objective `{}` is marked `assessed: false`",
|
|
c.objective_for(tag)
|
|
)
|
|
} else {
|
|
format!("`{tag}` is marked `assessed: false`")
|
|
};
|
|
issues.push(format!("{which} but this item measures it"));
|
|
}
|
|
}
|
|
for s in &it.sources {
|
|
if !c.lectures.contains_key(&s.lecture) {
|
|
issues.push(format!("unknown lecture `{}`", s.lecture));
|
|
}
|
|
}
|
|
if let Some(st) = &it.stimulus {
|
|
if !c.stimuli.contains_key(st) {
|
|
issues.push(format!("unknown stimulus `{st}`"));
|
|
}
|
|
}
|
|
// A citation that names a reference key must name a real one, so a review
|
|
// pointer in the solutions document never resolves to nothing.
|
|
for citation in it.solution.iter().flat_map(|s| &s.review) {
|
|
if let Some(key) = &citation.reference {
|
|
if !c.references.contains_key(key) {
|
|
issues.push(format!("solution.review cites unknown reference `{key}`"));
|
|
}
|
|
}
|
|
}
|
|
if let Some(floor) = c.policy.partial_credit_floor_level {
|
|
for o in &it.options {
|
|
if o.is_partial() && it.level < floor {
|
|
issues.push(format!(
|
|
"option {} awards partial credit at level {}, below the course floor of {}",
|
|
o.id,
|
|
it.level.code(),
|
|
floor.code()
|
|
));
|
|
}
|
|
}
|
|
}
|
|
if !c.policy.allow_partial_credit && it.options.iter().any(|o| o.is_partial()) {
|
|
issues.push("awards partial credit, which the course policy disallows".into());
|
|
}
|
|
}
|
|
|
|
issues
|
|
}
|
|
|
|
fn default_version() -> String {
|
|
SCHEMA_VERSION.to_string()
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
fn bank(items_yaml: &str) -> BankFile {
|
|
let src = format!("bank:\n id: b\n title: Bank\ndefaults: {{}}\nitems:\n{items_yaml}");
|
|
let mut b: BankFile = serde_yaml_ng::from_str(&src).expect("bank parses");
|
|
b.apply_defaults();
|
|
b
|
|
}
|
|
|
|
#[test]
|
|
fn sound_bank_validates_clean() {
|
|
let b = bank(
|
|
r#"
|
|
- id: q-a-001
|
|
status: draft
|
|
level: 1
|
|
stem: What is x?
|
|
options:
|
|
- { id: A, text: right, correct: true }
|
|
- { id: B, text: wrong }
|
|
- { id: C, text: wrong too }
|
|
"#,
|
|
);
|
|
assert!(b.validate(None).is_empty(), "{:?}", b.validate(None));
|
|
}
|
|
|
|
#[test]
|
|
fn open_response_validates_without_options_and_rejects_them() {
|
|
// No options is fine, and no key is required.
|
|
let ok = bank(
|
|
r#"
|
|
- id: q-a-op-001
|
|
status: draft
|
|
level: 2
|
|
format: open_response
|
|
stem: Explain the first law.
|
|
solution:
|
|
model_answer: Energy is conserved.
|
|
"#,
|
|
);
|
|
assert!(ok.validate(None).is_empty(), "{:?}", ok.validate(None));
|
|
|
|
// Giving an open-response item options is the mistake, and so is approving
|
|
// one with nothing to grade it by.
|
|
let bad = bank(
|
|
r#"
|
|
- id: q-a-op-002
|
|
status: approved
|
|
level: 2
|
|
format: open_response
|
|
cognitive_process: explain
|
|
learning_targets: [lo-x]
|
|
sources: [{ lecture: L1.1 }]
|
|
design: { rationale: r }
|
|
stem: Explain the first law.
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: B, text: b }
|
|
"#,
|
|
);
|
|
let issues = bad.validate(None);
|
|
assert!(
|
|
issues.iter().any(|i| i.contains("take no options")),
|
|
"{issues:?}"
|
|
);
|
|
assert!(
|
|
issues
|
|
.iter()
|
|
.any(|i| i.contains("model_answer or a rubric")),
|
|
"{issues:?}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn catches_missing_and_multiple_keys() {
|
|
let b = bank(
|
|
r#"
|
|
- id: q-a-001
|
|
status: draft
|
|
level: 1
|
|
stem: s
|
|
options:
|
|
- { id: A, text: a }
|
|
- { id: B, text: b }
|
|
- id: q-a-002
|
|
status: draft
|
|
level: 1
|
|
format: single_best_answer
|
|
stem: s
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: B, text: b, correct: true }
|
|
- id: q-a-003
|
|
status: draft
|
|
level: 1
|
|
format: single_best_answer
|
|
stem: s
|
|
options:
|
|
- { id: o-key-one, text: a, correct: true }
|
|
- { id: o-key-two, text: b, correct: true }
|
|
- { id: o-wrong-one, text: c }
|
|
- { id: o-wrong-two, text: d }
|
|
"#,
|
|
);
|
|
let issues = b.validate(None);
|
|
// No key at all is still a bank problem: nothing can be drawn from it.
|
|
assert!(
|
|
issues
|
|
.iter()
|
|
.any(|i| i.starts_with("q-a-001") && i.contains("at least one keyed option")),
|
|
"{issues:?}"
|
|
);
|
|
// Keying every option is still a bank problem, for the older reason: a
|
|
// question with nothing to choose against asks nothing.
|
|
assert!(
|
|
issues
|
|
.iter()
|
|
.any(|i| i.starts_with("q-a-002") && i.contains("asks nothing")),
|
|
"{issues:?}"
|
|
);
|
|
// Two defensible keys alongside real distractors is not a problem. It
|
|
// is the pool doing its job — it is what lets you test whether the
|
|
// wording of the key is what students are answering. Exactly one of
|
|
// them reaches a student, and that is checked against the placement
|
|
// that administers it, in `Catalog::validate_placement`.
|
|
assert!(
|
|
!issues.iter().any(|i| i.starts_with("q-a-003")
|
|
&& (i.contains("keyed option") || i.contains("asks nothing"))),
|
|
"{issues:?}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn catches_duplicate_ids_and_letters() {
|
|
let b = bank(
|
|
r#"
|
|
- id: q-a-001
|
|
status: draft
|
|
level: 1
|
|
stem: s
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: A, text: b }
|
|
- id: q-a-001
|
|
status: draft
|
|
level: 1
|
|
stem: s
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: B, text: b }
|
|
"#,
|
|
);
|
|
let issues = b.validate(None);
|
|
assert!(issues.iter().any(|i| i.contains("duplicate item id")));
|
|
assert!(issues.iter().any(|i| i.contains("duplicate option id")));
|
|
}
|
|
|
|
#[test]
|
|
fn level_and_process_must_agree() {
|
|
let b = bank(
|
|
r#"
|
|
- id: q-a-001
|
|
status: draft
|
|
level: 3
|
|
cognitive_process: recall
|
|
stem: s
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: B, text: b }
|
|
"#,
|
|
);
|
|
let issues = b.validate(None);
|
|
assert!(
|
|
issues.iter().any(|i| i.contains("belongs to level 1")),
|
|
"{issues:?}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn approval_requires_full_specification() {
|
|
let b = bank(
|
|
r#"
|
|
- id: q-a-001
|
|
status: approved
|
|
level: 1
|
|
stem: s
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: B, text: b }
|
|
"#,
|
|
);
|
|
let issues = b.validate(None);
|
|
for want in [
|
|
"cognitive_process",
|
|
"learning target",
|
|
"source",
|
|
"design block",
|
|
] {
|
|
assert!(
|
|
issues.iter().any(|i| i.contains(want)),
|
|
"expected a complaint about {want}, got {issues:?}"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn partial_credit_needs_a_written_defense() {
|
|
let b = bank(
|
|
r#"
|
|
- id: q-a-001
|
|
status: draft
|
|
level: 5
|
|
stem: s
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: B, text: b, credit: 0.5 }
|
|
"#,
|
|
);
|
|
let issues = b.validate(None);
|
|
assert!(issues.iter().any(|i| i.contains("no `defense`")));
|
|
assert!(issues.iter().any(|i| i.contains("defensible: true")));
|
|
}
|
|
|
|
#[test]
|
|
fn defaults_fill_in_items() {
|
|
let src = r#"
|
|
bank: { id: b, title: Bank }
|
|
defaults:
|
|
author: Alex
|
|
points: 1.5
|
|
topics: [kinetics]
|
|
lectures: [L11]
|
|
items:
|
|
- id: q-a-001
|
|
status: draft
|
|
level: 1
|
|
stem: s
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: B, text: b }
|
|
- id: q-a-002
|
|
status: draft
|
|
level: 1
|
|
stem: s
|
|
author: Someone Else
|
|
topics: [kinetics]
|
|
sources: [{ lecture: L12 }]
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: B, text: b }
|
|
"#;
|
|
let mut b: BankFile = serde_yaml_ng::from_str(src).unwrap();
|
|
b.apply_defaults();
|
|
assert_eq!(b.items[0].author.as_deref(), Some("Alex"));
|
|
assert_eq!(b.items[0].points, Some(1.5));
|
|
assert_eq!(b.items[0].topics, vec!["kinetics"]);
|
|
assert_eq!(b.items[0].sources[0].lecture, "L11");
|
|
// Explicit values win, and topics are not duplicated.
|
|
assert_eq!(b.items[1].author.as_deref(), Some("Someone Else"));
|
|
assert_eq!(b.items[1].topics, vec!["kinetics"]);
|
|
assert_eq!(b.items[1].sources[0].lecture, "L12");
|
|
}
|
|
|
|
#[test]
|
|
fn cross_file_references_are_checked_against_the_course() {
|
|
let course: CourseFile = serde_yaml_ng::from_str(
|
|
r#"
|
|
course: { code: X, title: Y, term: Z }
|
|
lectures:
|
|
L11: { title: Kinetics }
|
|
learning_objectives:
|
|
lo-known: { text: Do the thing, level_ceiling: 2 }
|
|
"#,
|
|
)
|
|
.unwrap();
|
|
let b = bank(
|
|
r#"
|
|
- id: q-a-001
|
|
status: draft
|
|
level: 4
|
|
stem: s
|
|
learning_targets: [lo-known, lo-unknown]
|
|
sources: [{ lecture: L99 }]
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: B, text: b }
|
|
"#,
|
|
);
|
|
let issues = b.validate(Some(&course));
|
|
assert!(
|
|
issues
|
|
.iter()
|
|
.any(|i| i.contains("unknown learning target `lo-unknown`"))
|
|
);
|
|
assert!(issues.iter().any(|i| i.contains("unknown lecture `L99`")));
|
|
assert!(
|
|
issues.iter().any(|i| i.contains("exceeds the ceiling")),
|
|
"{issues:?}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn an_item_must_tag_a_target_not_an_objective_that_has_them() {
|
|
let course: CourseFile = serde_yaml_ng::from_str(
|
|
r#"
|
|
course: { code: X, title: Y, term: Z }
|
|
learning_objectives:
|
|
lo-binding: { text: Quantify binding., level_ceiling: 3 }
|
|
lo-solo: { text: An objective with no targets. }
|
|
learning_targets:
|
|
t-kd: { text: Write the expression., objective: lo-binding }
|
|
"#,
|
|
)
|
|
.unwrap();
|
|
let b = bank(
|
|
r#"
|
|
- id: q-a-001
|
|
status: draft
|
|
level: 1
|
|
stem: s
|
|
learning_targets: [lo-binding]
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: B, text: b }
|
|
- id: q-a-002
|
|
status: draft
|
|
level: 1
|
|
stem: s
|
|
learning_targets: [t-kd, lo-solo]
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: B, text: b }
|
|
"#,
|
|
);
|
|
let issues = b.validate(Some(&course));
|
|
assert!(
|
|
issues
|
|
.iter()
|
|
.any(|i| i.contains("`lo-binding` is an objective with 1 learning target(s)")),
|
|
"got {issues:?}"
|
|
);
|
|
// A target is fine, and so is an objective that has no targets: it
|
|
// stands as its own, which is what lets a course migrate a unit at a
|
|
// time.
|
|
assert!(
|
|
!issues
|
|
.iter()
|
|
.any(|i| i.contains("`t-kd`") || i.contains("`lo-solo`")),
|
|
"got {issues:?}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn level_counts_exclude_drafts_and_bonuses() {
|
|
let b = bank(
|
|
r#"
|
|
- id: q-a-001
|
|
status: approved
|
|
level: 1
|
|
cognitive_process: recall
|
|
stem: s
|
|
learning_targets: [lo]
|
|
sources: [{ lecture: L1 }]
|
|
design: { expected_difficulty: 0.8 }
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: B, text: b }
|
|
- id: q-a-002
|
|
status: draft
|
|
level: 1
|
|
stem: s
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: B, text: b }
|
|
- id: q-a-003
|
|
status: approved
|
|
level: 5
|
|
cognitive_process: generate
|
|
bonus: true
|
|
stem: s
|
|
learning_targets: [lo]
|
|
sources: [{ lecture: L1 }]
|
|
design: { expected_difficulty: 0.3 }
|
|
options:
|
|
- { id: A, text: a, correct: true }
|
|
- { id: B, text: b }
|
|
"#,
|
|
);
|
|
let counts = b.level_counts();
|
|
assert_eq!(counts[&Level::Remember], 1);
|
|
assert_eq!(counts[&Level::Create], 0, "bonus items are not scored");
|
|
assert_eq!(b.assemblable().len(), 2);
|
|
}
|
|
}
|