956 lines
30 KiB
Rust
956 lines
30 KiB
Rust
// SPDX-License-Identifier: Prosperity-3.0.0
|
|
// Copyright Scientific Computing Studio
|
|
// Source: https://git.scient.ing/education/coursebank
|
|
|
|
//! Drawing an assessment from the item pool.
|
|
//!
|
|
//! Assembly is a constrained draw, not a random sample, and the constraints are
|
|
//! the point. A blueprint says how many items at each level; the pool says which
|
|
//! items are eligible; usage history says which ones students have seen recently.
|
|
//! What comes out is a form that matches the design you intended rather than
|
|
//! whichever questions happened to be at the top of the file.
|
|
//!
|
|
//! Two ordering rules do most of the work.
|
|
//!
|
|
//! *Objective minimums come first.* If a blueprint requires two items on
|
|
//! `lo-mm-kinetics`, those are placed before the level quotas are filled by
|
|
//! anything else, because a level quota can always be filled and a coverage
|
|
//! requirement often cannot. Filling in the other order strands the requirement.
|
|
//!
|
|
//! *Within a level, prefer the least recently used item.* Never-used items go
|
|
//! first, then the oldest, then the least often used. This spreads exposure
|
|
//! across the bank instead of wearing out your favorite twelve questions, and it
|
|
//! is the mechanism that makes writing new items pay off.
|
|
//!
|
|
//! Every tie is broken by a seeded shuffle, so a draw is reproducible from the
|
|
//! seed recorded in the assessment file.
|
|
|
|
use std::collections::{BTreeMap, BTreeSet};
|
|
|
|
use crate::assessment::{Assessment, AssessmentFile, Blueprint, Form, Kind, Placement, Platform};
|
|
use crate::catalog::Catalog;
|
|
use crate::course::{CourseFile, SCHEMA_VERSION};
|
|
use crate::date::Date;
|
|
use crate::error::{Error, Result};
|
|
use crate::history::History;
|
|
use crate::item::{Choice, Item};
|
|
use crate::rng::Rng;
|
|
use crate::taxonomy::{Format, Level};
|
|
|
|
/// The result of a draw.
|
|
#[derive(Debug, Clone)]
|
|
pub struct Selection {
|
|
/// Scored item ids, grouped and ordered by level.
|
|
pub scored: Vec<String>,
|
|
/// Bonus item ids.
|
|
pub bonus: Vec<String>,
|
|
/// Things the caller should know: quotas filled by relaxing a constraint,
|
|
/// levels that came up short, cooldowns that had to be ignored.
|
|
pub notes: Vec<String>,
|
|
}
|
|
|
|
impl Selection {
|
|
/// Every selected id, scored then bonus.
|
|
pub fn all(&self) -> Vec<String> {
|
|
let mut v = self.scored.clone();
|
|
v.extend(self.bonus.clone());
|
|
v
|
|
}
|
|
}
|
|
|
|
/// Draws an assessment from the pool according to a blueprint.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `catalog` - the loaded course.
|
|
/// * `blueprint` - the design to satisfy.
|
|
/// * `history` - usage history, for the least-recently-used preference.
|
|
/// * `as_of` - the date of the assessment, against which cooldowns are measured.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The selection, together with notes about any constraint that had to bend.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns [`Error::Infeasible`] when a level quota cannot be met even after
|
|
/// relaxing the reuse cooldown, with a message saying how many items were
|
|
/// available and how many were asked for.
|
|
pub fn select(
|
|
catalog: &Catalog,
|
|
blueprint: &Blueprint,
|
|
history: &History,
|
|
as_of: Date,
|
|
) -> Result<Selection> {
|
|
let seed = blueprint.seed.unwrap_or(0);
|
|
let mut notes = Vec::new();
|
|
|
|
// --- pool
|
|
let eligible: Vec<&crate::catalog::Entry> = catalog
|
|
.assemblable()
|
|
.into_iter()
|
|
.filter(|e| passes_filters(e, blueprint))
|
|
.collect();
|
|
|
|
if eligible.is_empty() {
|
|
return Err(Error::Infeasible(
|
|
"no approved items match the blueprint's bank, lecture, and topic filters".into(),
|
|
));
|
|
}
|
|
|
|
let cooldown = blueprint.cooldown_days.unwrap_or(0);
|
|
let mut chosen: Vec<String> = Vec::new();
|
|
let mut per_bank: BTreeMap<String, usize> = BTreeMap::new();
|
|
|
|
// --- objective minimums
|
|
// Placed first, because a coverage requirement is the constraint most likely
|
|
// to become unsatisfiable once the level quotas are full.
|
|
for (objective, needed) in &blueprint.objective_minimums {
|
|
let mut have = 0;
|
|
// A requirement written against an objective is satisfied by items on
|
|
// any of its targets, which is the only way a coverage requirement stays
|
|
// writable: a blueprint that had to name each target separately would be
|
|
// as long as the registry, and would need editing every time an
|
|
// objective gained one.
|
|
let mut candidates: Vec<&crate::catalog::Entry> = eligible
|
|
.iter()
|
|
.copied()
|
|
.filter(|e| {
|
|
e.item
|
|
.learning_targets
|
|
.iter()
|
|
.any(|t| t == objective || catalog.course.objective_for(t) == objective)
|
|
})
|
|
.filter(|e| !e.item.bonus)
|
|
.collect();
|
|
rank(&mut candidates, history, seed, "objective");
|
|
|
|
for e in candidates {
|
|
if have >= *needed {
|
|
break;
|
|
}
|
|
if chosen.contains(&e.uid) {
|
|
have += 1;
|
|
continue;
|
|
}
|
|
if !bank_has_room(&per_bank, &e.bank, blueprint) {
|
|
continue;
|
|
}
|
|
if cooldown > 0 && history.in_cooldown(&e.uid, cooldown, as_of) {
|
|
continue;
|
|
}
|
|
chosen.push(e.uid.clone());
|
|
*per_bank.entry(e.bank.clone()).or_insert(0) += 1;
|
|
have += 1;
|
|
}
|
|
if have < *needed {
|
|
notes.push(format!(
|
|
"objective `{objective}` requires {needed} item(s) but only {have} could be \
|
|
placed; write more items on it or lower the requirement"
|
|
));
|
|
}
|
|
}
|
|
|
|
// --- level quotas
|
|
let mut scored: Vec<String> = Vec::new();
|
|
for (level, want) in &blueprint.level_counts {
|
|
if *want == 0 {
|
|
continue;
|
|
}
|
|
let (picked, level_notes) = fill_level(
|
|
&eligible,
|
|
*level,
|
|
*want,
|
|
false,
|
|
history,
|
|
seed,
|
|
cooldown,
|
|
as_of,
|
|
&mut chosen,
|
|
&mut per_bank,
|
|
blueprint,
|
|
)?;
|
|
scored.extend(picked);
|
|
notes.extend(level_notes);
|
|
}
|
|
|
|
// Items placed to satisfy an objective minimum are scored items too, and
|
|
// they must appear exactly once.
|
|
for uid in &chosen {
|
|
if !scored.contains(uid) {
|
|
if let Some(e) = catalog.get(uid) {
|
|
if !e.item.bonus {
|
|
scored.push(uid.clone());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// --- bonus items
|
|
let mut bonus: Vec<String> = Vec::new();
|
|
for (level, want) in &blueprint.bonus_counts {
|
|
if *want == 0 {
|
|
continue;
|
|
}
|
|
let (picked, level_notes) = fill_level(
|
|
&eligible,
|
|
*level,
|
|
*want,
|
|
true,
|
|
history,
|
|
seed,
|
|
cooldown,
|
|
as_of,
|
|
&mut chosen,
|
|
&mut per_bank,
|
|
blueprint,
|
|
)?;
|
|
bonus.extend(picked);
|
|
notes.extend(level_notes);
|
|
}
|
|
|
|
// Order the scored items by level so the form ramps in difficulty. Students
|
|
// meet the recall items first, which is both kinder and better measurement:
|
|
// an early hard item costs time that later easy items cannot recover.
|
|
scored.sort_by_key(|uid| {
|
|
let e = catalog.get(uid);
|
|
(
|
|
e.map(|e| e.item.level.code()).unwrap_or(9),
|
|
e.map(|e| e.uid.clone()).unwrap_or_default(),
|
|
)
|
|
});
|
|
|
|
Ok(Selection {
|
|
scored,
|
|
bonus,
|
|
notes,
|
|
})
|
|
}
|
|
|
|
/// Fills one level's quota.
|
|
///
|
|
/// Cooldowns are relaxed rather than allowed to fail the draw, because an exam
|
|
/// that must be given on Thursday is better assembled from a recently used item
|
|
/// with a warning than not assembled at all.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `eligible` - the filtered pool.
|
|
/// * `level` - the level to fill.
|
|
/// * `want` - how many items are needed.
|
|
/// * `bonus` - whether to draw bonus items.
|
|
/// * `history` - usage history.
|
|
/// * `seed` - the tie-breaking seed.
|
|
/// * `cooldown` - the reuse cooldown in days, 0 to disable.
|
|
/// * `as_of` - the assessment date.
|
|
/// * `chosen` - ids already taken, updated in place.
|
|
/// * `per_bank` - per-bank counts, updated in place.
|
|
/// * `blueprint` - for the per-bank cap.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The ids picked and any notes.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns [`Error::Infeasible`] when the level cannot be filled at all.
|
|
#[allow(clippy::too_many_arguments)]
|
|
fn fill_level(
|
|
eligible: &[&crate::catalog::Entry],
|
|
level: Level,
|
|
want: usize,
|
|
bonus: bool,
|
|
history: &History,
|
|
seed: u64,
|
|
cooldown: i64,
|
|
as_of: Date,
|
|
chosen: &mut Vec<String>,
|
|
per_bank: &mut BTreeMap<String, usize>,
|
|
blueprint: &Blueprint,
|
|
) -> Result<(Vec<String>, Vec<String>)> {
|
|
let mut notes = Vec::new();
|
|
let mut candidates: Vec<&crate::catalog::Entry> = eligible
|
|
.iter()
|
|
.copied()
|
|
.filter(|e| e.item.level == level && e.item.bonus == bonus)
|
|
.collect();
|
|
|
|
let pool_size = candidates.len();
|
|
if pool_size < want {
|
|
return Err(Error::Infeasible(format!(
|
|
"level {} needs {want} {}item(s) but only {pool_size} approved item(s) are \
|
|
available; write more or lower the quota",
|
|
level.code(),
|
|
if bonus { "bonus " } else { "" }
|
|
)));
|
|
}
|
|
|
|
rank(
|
|
&mut candidates,
|
|
history,
|
|
seed,
|
|
&format!("L{}", level.code()),
|
|
);
|
|
|
|
let mut picked = Vec::new();
|
|
let mut skipped_for_cooldown = 0usize;
|
|
let mut skipped_for_bank = 0usize;
|
|
|
|
// Two passes: honor every constraint, then relax the cooldown if short.
|
|
for relax in [false, true] {
|
|
for e in &candidates {
|
|
if picked.len() >= want {
|
|
break;
|
|
}
|
|
if chosen.contains(&e.uid) {
|
|
continue;
|
|
}
|
|
if !bank_has_room(per_bank, &e.bank, blueprint) {
|
|
if !relax {
|
|
skipped_for_bank += 1;
|
|
}
|
|
continue;
|
|
}
|
|
if !relax && cooldown > 0 && history.in_cooldown(&e.uid, cooldown, as_of) {
|
|
skipped_for_cooldown += 1;
|
|
continue;
|
|
}
|
|
if relax && cooldown > 0 && history.in_cooldown(&e.uid, cooldown, as_of) {
|
|
notes.push(format!(
|
|
"level {}: reused `{}` inside the {cooldown}-day cooldown (last used {})",
|
|
level.code(),
|
|
e.uid,
|
|
history
|
|
.last_used(&e.uid)
|
|
.map(|d| d.to_string())
|
|
.unwrap_or_else(|| "unknown".into())
|
|
));
|
|
}
|
|
picked.push(e.uid.clone());
|
|
chosen.push(e.uid.clone());
|
|
*per_bank.entry(e.bank.clone()).or_insert(0) += 1;
|
|
}
|
|
if picked.len() >= want {
|
|
break;
|
|
}
|
|
}
|
|
|
|
if picked.len() < want {
|
|
return Err(Error::Infeasible(format!(
|
|
"level {} needs {want} item(s); {pool_size} exist but only {} could be placed \
|
|
({skipped_for_cooldown} blocked by the reuse cooldown, {skipped_for_bank} by the \
|
|
per-bank cap)",
|
|
level.code(),
|
|
picked.len()
|
|
)));
|
|
}
|
|
|
|
Ok((picked, notes))
|
|
}
|
|
|
|
/// Whether an entry passes the blueprint's inclusion filters.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `e` - the entry.
|
|
/// * `b` - the blueprint.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// `true` when the item is eligible.
|
|
fn passes_filters(e: &crate::catalog::Entry, b: &Blueprint) -> bool {
|
|
if !b.banks.is_empty() && !b.banks.contains(&e.bank) {
|
|
return false;
|
|
}
|
|
if !b.lectures.is_empty()
|
|
&& !e
|
|
.item
|
|
.sources
|
|
.iter()
|
|
.any(|s| b.lectures.contains(&s.lecture))
|
|
{
|
|
return false;
|
|
}
|
|
if !b.topics.is_empty() && !e.item.topics.iter().any(|t| b.topics.contains(t)) {
|
|
return false;
|
|
}
|
|
true
|
|
}
|
|
|
|
/// Whether a bank may contribute another item.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `per_bank` - counts so far.
|
|
/// * `bank` - the bank in question.
|
|
/// * `b` - the blueprint, for the cap.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// `true` when there is room.
|
|
fn bank_has_room(per_bank: &BTreeMap<String, usize>, bank: &str, b: &Blueprint) -> bool {
|
|
match b.max_per_bank {
|
|
Some(cap) => per_bank.get(bank).copied().unwrap_or(0) < cap,
|
|
None => true,
|
|
}
|
|
}
|
|
|
|
/// Orders candidates least-recently-used first, with a seeded tie-break.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `candidates` - the candidates to order, sorted in place.
|
|
/// * `history` - usage history.
|
|
/// * `seed` - the tie-breaking seed.
|
|
/// * `salt` - distinguishes the shuffles used for different levels, so two
|
|
/// levels drawing from overlapping pools do not tie-break identically.
|
|
fn rank(candidates: &mut Vec<&crate::catalog::Entry>, history: &History, seed: u64, salt: &str) {
|
|
// Shuffle first so the sort's stability turns into a random tie-break.
|
|
let mut rng = Rng::from_label(&format!("{seed}/{salt}"));
|
|
rng.shuffle(candidates);
|
|
|
|
candidates.sort_by_key(|e| {
|
|
let last = history.last_used(&e.uid);
|
|
(
|
|
// Never used sorts before ever used.
|
|
if last.is_some() { 1 } else { 0 },
|
|
last.map(|d| d.days_since_epoch()).unwrap_or(i64::MIN),
|
|
history.use_count(&e.uid),
|
|
)
|
|
});
|
|
}
|
|
|
|
/// Turns a selection into an assessment record ready to write.
|
|
///
|
|
/// The record captures the key and fingerprint of every item *as of now*, which
|
|
/// is what makes later analysis honest about drift.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `catalog` - the loaded course.
|
|
/// * `selection` - the draw.
|
|
/// * `id` - the assessment id.
|
|
/// * `title` - the printed title.
|
|
/// * `kind` - the kind of assessment.
|
|
/// * `date` - the administration date.
|
|
/// * `platform` - where it will be administered.
|
|
/// * `blueprint` - the blueprint used, recorded for later comparison.
|
|
/// * `forms` - how many alternate forms to declare.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The assessment record.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns [`Error::Unresolved`] if a selected id has vanished from the catalog.
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub fn to_record(
|
|
catalog: &Catalog,
|
|
selection: &Selection,
|
|
id: &str,
|
|
title: &str,
|
|
kind: Kind,
|
|
date: Date,
|
|
platform: Platform,
|
|
blueprint: &Blueprint,
|
|
forms: usize,
|
|
) -> Result<AssessmentFile> {
|
|
let default_points = catalog.course.policy.points_per_item;
|
|
let mut items = Vec::new();
|
|
|
|
for (number, (uid, is_bonus)) in (1u32..).zip(
|
|
selection
|
|
.scored
|
|
.iter()
|
|
.map(|u| (u, false))
|
|
.chain(selection.bonus.iter().map(|u| (u, true))),
|
|
) {
|
|
let e = catalog.require(uid)?;
|
|
let (key, distractors) = draw_options(
|
|
&e.item,
|
|
catalog.course.policy.options_per_item,
|
|
blueprint.seed.unwrap_or(0),
|
|
uid,
|
|
);
|
|
items.push(Placement {
|
|
number,
|
|
item: uid.clone(),
|
|
version: None,
|
|
stem_digest: Some(e.item.stem_digest()),
|
|
variant: Some(e.item.variant_digest(&key, &distractors)),
|
|
fingerprint: Some(e.item.fingerprint()),
|
|
points: Some(e.item.points(default_points)),
|
|
bonus: is_bonus || e.item.bonus,
|
|
distractors,
|
|
key,
|
|
level: Some(e.item.level),
|
|
learning_targets: e.item.learning_targets.clone(),
|
|
credit_overrides: BTreeMap::new(),
|
|
dropped: false,
|
|
dropped_as: None,
|
|
dropped_before_printing: false,
|
|
});
|
|
}
|
|
|
|
let form_list: Vec<Form> = (0..forms)
|
|
.map(|i| {
|
|
let label = form_label(i);
|
|
Form {
|
|
seed: Rng::from_label(&format!("{id}/form-{label}")).next_u64(),
|
|
id: label,
|
|
shuffle_items: false,
|
|
shuffle_options: true,
|
|
}
|
|
})
|
|
.collect();
|
|
|
|
Ok(AssessmentFile {
|
|
schema_version: SCHEMA_VERSION.to_string(),
|
|
assessment: Assessment {
|
|
id: id.to_string(),
|
|
title: title.to_string(),
|
|
term: Some(catalog.course.course.term.clone()),
|
|
kind,
|
|
date: Some(date),
|
|
platform,
|
|
minutes_allowed: None,
|
|
attempts: None,
|
|
shuffle: None,
|
|
scoring_policy: None,
|
|
instructions: None,
|
|
notes: None,
|
|
},
|
|
blueprint: Some(blueprint.clone()),
|
|
forms: form_list,
|
|
items,
|
|
})
|
|
}
|
|
|
|
/// The label for the nth form: A, B, ... Z, AA, AB, ...
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `i` - the zero-based form index.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The label.
|
|
fn form_label(i: usize) -> String {
|
|
let mut n = i;
|
|
let mut out = String::new();
|
|
loop {
|
|
out.insert(0, (b'A' + (n % 26) as u8) as char);
|
|
if n < 26 {
|
|
break;
|
|
}
|
|
n = n / 26 - 1;
|
|
}
|
|
out
|
|
}
|
|
|
|
/// The order items appear in on one form.
|
|
///
|
|
/// Permuting a form is a display concern, so it is computed on demand from the
|
|
/// recorded seed rather than stored. That keeps the record small and guarantees
|
|
/// every export of form B agrees.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `record` - the assessment record.
|
|
/// * `form` - the form to lay out.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// Placements in printed order for this form. The `number` field is left as
|
|
/// recorded, since it is the join key to grading data and must not change
|
|
/// between forms; use the position in the returned vector for what to print.
|
|
pub fn layout(record: &AssessmentFile, form: &Form) -> Vec<Placement> {
|
|
// Bonus items always come last, whatever the shuffle says: they are outside
|
|
// the scored total, and burying one mid-form invites students to spend time
|
|
// there that the graded questions needed.
|
|
let mut scored: Vec<Placement> = record.items.iter().filter(|p| !p.bonus).cloned().collect();
|
|
let bonus: Vec<Placement> = record.items.iter().filter(|p| p.bonus).cloned().collect();
|
|
|
|
if form.shuffle_items {
|
|
let mut rng = Rng::new(form.seed);
|
|
rng.shuffle(&mut scored);
|
|
}
|
|
|
|
scored.into_iter().chain(bonus).collect()
|
|
}
|
|
|
|
/// Draws the key and the distractors one placement administers.
|
|
///
|
|
/// Resolved here, at assembly, and written into the record as explicit lists.
|
|
/// Nothing downstream samples: an export that drew its own options would print
|
|
/// a different paper every time the bank was touched.
|
|
///
|
|
/// The draw is seeded on the blueprint and the item, so re-running `assemble`
|
|
/// with the same seed produces the same paper, and two items in one assessment
|
|
/// draw independently.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `item` - the item, whose options are a pool.
|
|
/// * `per_item` - how many options a form shows, from course policy.
|
|
/// * `seed` - the blueprint seed.
|
|
/// * `uid` - the item id, salting the draw.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The keyed ids and the distractor ids, each sorted, naming options of `item`.
|
|
/// Both empty for an item with no options, which is an open response.
|
|
pub fn draw_options(
|
|
item: &Item,
|
|
per_item: usize,
|
|
seed: u64,
|
|
uid: &str,
|
|
) -> (Vec<String>, Vec<String>) {
|
|
let (keys, distractors) = item.pool();
|
|
if keys.is_empty() && distractors.is_empty() {
|
|
return (Vec::new(), Vec::new());
|
|
}
|
|
|
|
// Multiple response keys every correct option; anything else keys one, and
|
|
// when the pool offers several defensible keys the draw picks one so that
|
|
// the record says which.
|
|
let wanted_keys = match item.format {
|
|
Format::MultipleResponse => keys.len(),
|
|
_ => 1.min(keys.len()),
|
|
};
|
|
let mut rng = Rng::from_label(&format!("{seed}/{uid}/options"));
|
|
|
|
let mut key_ids = pick(&keys, wanted_keys, &mut rng);
|
|
key_ids.sort();
|
|
|
|
// A pool with fewer usable distractors than the policy asks for is a
|
|
// finding, not a failure: the form comes out short and `lint` says so,
|
|
// rather than `assemble` refusing to build the assessment at all.
|
|
let wanted = per_item.saturating_sub(key_ids.len());
|
|
let mut distractor_ids = pick(&distractors, wanted.min(distractors.len()), &mut rng);
|
|
distractor_ids.sort();
|
|
|
|
(key_ids, distractor_ids)
|
|
}
|
|
|
|
/// Takes `n` options, preferring the ones that were designed rather than merely
|
|
/// written.
|
|
///
|
|
/// A distractor carrying a misconception and an error type is one you thought
|
|
/// about; one carrying neither is filler. When the pool is larger than the form,
|
|
/// the thought-about ones go on the paper. The shuffle comes first so that
|
|
/// options of equal standing are drawn by seed rather than by declaration
|
|
/// order.
|
|
fn pick(options: &[&Choice], n: usize, rng: &mut Rng) -> Vec<String> {
|
|
if n >= options.len() {
|
|
return options.iter().map(|o| o.id.clone()).collect();
|
|
}
|
|
let mut order: Vec<usize> = (0..options.len()).collect();
|
|
rng.shuffle(&mut order);
|
|
order.sort_by_key(|&i| {
|
|
let o = options[i];
|
|
u8::from(o.misconception.is_none()) + u8::from(o.error_type.is_none())
|
|
});
|
|
order
|
|
.into_iter()
|
|
.take(n)
|
|
.map(|i| options[i].id.clone())
|
|
.collect()
|
|
}
|
|
|
|
/// The option order for one item on one form.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `form` - the form.
|
|
/// * `uid` - the item's global id, which salts the permutation so two items on
|
|
/// the same form do not permute identically.
|
|
/// * `n` - the number of options.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// A permutation of `0..n`.
|
|
pub fn option_order(form: &Form, uid: &str, n: usize) -> Vec<usize> {
|
|
let mut order: Vec<usize> = (0..n).collect();
|
|
if form.shuffle_options && n > 1 {
|
|
let mut rng = Rng::from_label(&format!("{}/{}/{}", form.seed, form.id, uid));
|
|
rng.shuffle(&mut order);
|
|
}
|
|
order
|
|
}
|
|
|
|
/// Compares a record against its blueprint.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `record` - the assessment record.
|
|
/// * `course` - the course, for the objective each tagged target belongs to. An
|
|
/// objective minimum is satisfied by items on any of that objective's
|
|
/// targets, the same way [`select`] fills it.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// One message per discrepancy, empty when the form matches the design.
|
|
pub fn check_blueprint(record: &AssessmentFile, course: &CourseFile) -> Vec<String> {
|
|
let Some(bp) = &record.blueprint else {
|
|
return vec!["the record carries no blueprint to check against".into()];
|
|
};
|
|
let actual = record.level_counts();
|
|
let mut out = Vec::new();
|
|
for level in Level::ALL {
|
|
let want = bp.level_counts.get(&level).copied().unwrap_or(0);
|
|
let got = actual.get(&level).copied().unwrap_or(0);
|
|
if want != got {
|
|
out.push(format!(
|
|
"level {}: blueprint asks for {want}, the form has {got}",
|
|
level.code()
|
|
));
|
|
}
|
|
}
|
|
for (objective, needed) in &bp.objective_minimums {
|
|
let got = record
|
|
.items
|
|
.iter()
|
|
.filter(|p| {
|
|
p.learning_targets
|
|
.iter()
|
|
.any(|t| t == objective || course.objective_for(t) == objective)
|
|
})
|
|
.count();
|
|
if got < *needed {
|
|
out.push(format!(
|
|
"objective `{objective}`: blueprint asks for {needed} item(s), the form has {got}"
|
|
));
|
|
}
|
|
}
|
|
out
|
|
}
|
|
|
|
/// The set of learning targets an assessment covers, as tagged.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `record` - the assessment record.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The target ids, deduplicated.
|
|
pub fn covered_targets(record: &AssessmentFile) -> BTreeSet<String> {
|
|
record
|
|
.items
|
|
.iter()
|
|
.flat_map(|p| p.learning_targets.iter().cloned())
|
|
.collect()
|
|
}
|
|
|
|
/// The objectives an assessment covers, through the targets it measured.
|
|
///
|
|
/// The list a coverage claim should be made from: "this exam covered eleven of
|
|
/// the course's thirty-two objectives" is a sentence about the blueprint, while
|
|
/// the same count over targets is a sentence about how finely the course happens
|
|
/// to be subdivided.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `record` - the assessment record.
|
|
/// * `course` - the course, for the objective each tagged target belongs to.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The objective ids, deduplicated.
|
|
pub fn covered_objectives(record: &AssessmentFile, course: &CourseFile) -> BTreeSet<String> {
|
|
record
|
|
.items
|
|
.iter()
|
|
.flat_map(|p| p.learning_targets.iter())
|
|
.map(|t| course.objective_for(t).to_string())
|
|
.collect()
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn form_labels_extend_past_z() {
|
|
assert_eq!(form_label(0), "A");
|
|
assert_eq!(form_label(1), "B");
|
|
assert_eq!(form_label(25), "Z");
|
|
assert_eq!(form_label(26), "AA");
|
|
assert_eq!(form_label(27), "AB");
|
|
}
|
|
|
|
/// An item whose options are given as YAML, so the test needs no literal.
|
|
fn pool_item(options: &str) -> Item {
|
|
let src = format!(
|
|
r#"id: q-x
|
|
status: approved
|
|
level: 1
|
|
cognitive_process: recall
|
|
stem: Which line holds the quality scores?
|
|
learning_targets: [t-x]
|
|
sources: [{{ lecture: L1 }}]
|
|
options:
|
|
{options}"#
|
|
);
|
|
serde_yaml_ng::from_str(&src).expect("item parses")
|
|
}
|
|
|
|
const DESIGNED: &str = r#" - { id: o-key, text: right, correct: true }
|
|
- { id: o-designed-a, text: a, misconception: mistakes the separator, error_type: recall_confusion }
|
|
- { id: o-designed-b, text: b, misconception: confuses the two, error_type: recall_confusion }
|
|
- { id: o-filler-a, text: c }
|
|
- { id: o-filler-b, text: d }
|
|
"#;
|
|
|
|
#[test]
|
|
fn a_draw_prefers_designed_distractors_and_is_reproducible() {
|
|
let item = pool_item(DESIGNED);
|
|
|
|
let (key, distractors) = draw_options(&item, 3, 1103, "q-x");
|
|
assert_eq!(key, vec!["o-key".to_string()]);
|
|
assert_eq!(distractors.len(), 2);
|
|
// Thought-about distractors go on the paper before filler does.
|
|
assert!(
|
|
distractors.iter().all(|d| d.starts_with("o-designed")),
|
|
"{distractors:?}"
|
|
);
|
|
|
|
// Same seed, same paper.
|
|
assert_eq!(draw_options(&item, 3, 1103, "q-x"), (key, distractors));
|
|
|
|
// A retired option is not drawn, and the form comes out of the rest.
|
|
let retired = pool_item(&DESIGNED.replace(
|
|
"{ id: o-designed-a, text: a,",
|
|
"{ id: o-designed-a, text: a, retired: { 'on': 2026-09-20, reason: nonfunctioning },",
|
|
));
|
|
let (_, after) = draw_options(&retired, 3, 1103, "q-x");
|
|
assert!(!after.iter().any(|d| d == "o-designed-a"), "{after:?}");
|
|
}
|
|
|
|
#[test]
|
|
fn a_thin_pool_comes_out_short_rather_than_refusing_to_build() {
|
|
let item = pool_item(
|
|
" - { id: o-key, text: right, correct: true }\n - { id: o-one, text: wrong }\n",
|
|
);
|
|
let (key, distractors) = draw_options(&item, 4, 7, "q-y");
|
|
assert_eq!(key.len(), 1);
|
|
assert_eq!(
|
|
distractors.len(),
|
|
1,
|
|
"one usable distractor, so one is drawn"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn option_order_is_a_reproducible_permutation() {
|
|
let form = Form {
|
|
id: "A".into(),
|
|
seed: 12345,
|
|
shuffle_items: false,
|
|
shuffle_options: true,
|
|
};
|
|
let a = option_order(&form, "b::q-1", 5);
|
|
let b = option_order(&form, "b::q-1", 5);
|
|
assert_eq!(a, b, "same inputs give the same order");
|
|
|
|
let other = option_order(&form, "b::q-2", 5);
|
|
assert_ne!(a, other, "different items permute differently");
|
|
|
|
let mut sorted = a.clone();
|
|
sorted.sort_unstable();
|
|
assert_eq!(sorted, vec![0, 1, 2, 3, 4]);
|
|
}
|
|
|
|
#[test]
|
|
fn option_order_is_identity_when_shuffling_is_off() {
|
|
let form = Form {
|
|
id: "A".into(),
|
|
seed: 1,
|
|
shuffle_items: false,
|
|
shuffle_options: false,
|
|
};
|
|
assert_eq!(option_order(&form, "b::q-1", 4), vec![0, 1, 2, 3]);
|
|
}
|
|
|
|
#[test]
|
|
fn blueprint_check_reports_shortfalls() {
|
|
let record: AssessmentFile = serde_yaml_ng::from_str(
|
|
r#"
|
|
assessment: { id: x, title: X }
|
|
blueprint:
|
|
level_counts: { 1: 2, 3: 1 }
|
|
objective_minimums: { lo-key: 2 }
|
|
items:
|
|
- { number: 1, item: "b::q-1", level: 1, learning_targets: [lo-key] }
|
|
- { number: 2, item: "b::q-2", level: 1 }
|
|
"#,
|
|
)
|
|
.unwrap();
|
|
let course: CourseFile =
|
|
serde_yaml_ng::from_str("course: { code: C, title: T, term: M }").unwrap();
|
|
let issues = check_blueprint(&record, &course);
|
|
assert!(issues.iter().any(|i| i.contains("level 3")), "{issues:?}");
|
|
assert!(issues.iter().any(|i| i.contains("lo-key")), "{issues:?}");
|
|
// Level 1 matches, so it must not be reported.
|
|
assert!(!issues.iter().any(|i| i.contains("level 1")));
|
|
}
|
|
|
|
#[test]
|
|
fn an_objective_minimum_is_met_by_items_on_its_targets() {
|
|
// The blueprint names the objective a report will classify; the items
|
|
// are tagged with the specific performances they measure.
|
|
let record: AssessmentFile = serde_yaml_ng::from_str(
|
|
r#"
|
|
assessment: { id: x, title: X }
|
|
blueprint:
|
|
objective_minimums: { lo-binding: 2 }
|
|
items:
|
|
- { number: 1, item: "b::q-1", level: 1, learning_targets: [t-kd] }
|
|
- { number: 2, item: "b::q-2", level: 3, learning_targets: [t-plot] }
|
|
"#,
|
|
)
|
|
.unwrap();
|
|
let course: CourseFile = serde_yaml_ng::from_str(
|
|
r#"
|
|
course: { code: C, title: T, term: M }
|
|
learning_objectives:
|
|
lo-binding: { text: Quantify binding. }
|
|
learning_targets:
|
|
t-kd: { text: Write the expression., objective: lo-binding }
|
|
t-plot: { text: Read a plot., objective: lo-binding }
|
|
"#,
|
|
)
|
|
.unwrap();
|
|
let issues = check_blueprint(&record, &course);
|
|
assert!(
|
|
!issues.iter().any(|i| i.contains("lo-binding")),
|
|
"two items on its targets satisfy it: {issues:?}"
|
|
);
|
|
assert_eq!(
|
|
covered_objectives(&record, &course)
|
|
.into_iter()
|
|
.collect::<Vec<_>>(),
|
|
vec!["lo-binding"],
|
|
"coverage is claimed at the tier the blueprint is written in"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn covered_targets_deduplicates() {
|
|
let record: AssessmentFile = serde_yaml_ng::from_str(
|
|
r#"
|
|
assessment: { id: x, title: X }
|
|
items:
|
|
- { number: 1, item: "b::q-1", learning_targets: [lo-a, lo-b] }
|
|
- { number: 2, item: "b::q-2", learning_targets: [lo-a] }
|
|
"#,
|
|
)
|
|
.unwrap();
|
|
let set = covered_targets(&record);
|
|
assert_eq!(set.len(), 2);
|
|
assert!(set.contains("lo-a"));
|
|
}
|
|
}
|