education/coursebank
1
0
Fork 0
Code Issues Pull Requests 1 Actions Releases 1 Activity

feat: cooked up something fierce
Pipeline / check (pull_request) Successful in 3m9s
Details
Pipeline / docs (pull_request) Skipped
Details
Pipeline / nightly (pull_request) Skipped
Details
Pipeline / release (pull_request) Skipped
Details

Browse Source
This commit is contained in:
alexm
2026-09-26 18:22:25 -04:00
parent eabc98ad31
commit 5ac1e317c0
26 changed files with 1485 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/authoring/lint.rs
+161 -1
View File
@@ -88,6 +88,16 @@ pub enum Rule {
/// An expectation of low discrimination on a higher-level item.
ContradictoryDesign,
// --- the option pool ---
/// Fewer usable distractors than a form shows.
ThinOptionPool,
/// An option that has never been administered and has not been retired.
UnusedOption,
/// An option retired without saying what it did.
UnjustifiedRetirement,
/// A distractor that has never been chosen, in any set it appeared in.
NonfunctioningDistractor,
// --- evidence ---
/// Statistics describe an older version of the item.
StaleCalibration,
@@ -104,7 +114,7 @@ impl Rule {
///
/// Used by `--list-rules`, and by the test that keeps this list in step with
/// the enum.
pub const ALL: [Rule; 26] = [
pub const ALL: [Rule; 30] = [
Rule::KeyIsLongest,
Rule::UnevenOptionLength,
Rule::WordRepeatCue,
@@ -127,6 +137,10 @@ impl Rule {
Rule::WeakFormatForLevel,
Rule::ScoredBonusLevel,
Rule::ContradictoryDesign,
Rule::ThinOptionPool,
Rule::UnusedOption,
Rule::UnjustifiedRetirement,
Rule::NonfunctioningDistractor,
Rule::StaleCalibration,
Rule::DifficultyMissed,
Rule::DiscriminationMissed,
@@ -152,6 +166,9 @@ impl Rule {
// Statistics attached to text that has since changed are actively
// misleading, which is worse than absent.
R::StaleCalibration => Severity::High,
// An item that cannot fill a form is an item `assemble` will put on
// a paper short an option.
R::ThinOptionPool => Severity::High,
// An unanswerable question for a screen-reader user.
R::AssetWithoutAltText => Severity::High,
@@ -178,6 +195,13 @@ impl Rule {
| R::NoStudentFeedback
| R::DifficultyMissed
| R::DiscriminationMissed => Severity::Low,
// A distractor that draws nobody across several administrations is
// evidence to act on, not a style note.
R::NonfunctioningDistractor => Severity::Medium,
// Both are tidiness: the item still works, but its pool is
// carrying something nobody has accounted for.
R::UnusedOption | R::UnjustifiedRetirement => Severity::Low,
}
}
@@ -206,6 +230,10 @@ impl Rule {
Rule::WeakFormatForLevel => "complete-format-level",
Rule::ScoredBonusLevel => "complete-bonus-policy",
Rule::ContradictoryDesign => "complete-design-conflict",
Rule::ThinOptionPool => "pool-thin",
Rule::UnusedOption => "pool-unused",
Rule::UnjustifiedRetirement => "pool-unjustified-retirement",
Rule::NonfunctioningDistractor => "evidence-nonfunctioning",
Rule::StaleCalibration => "evidence-stale",
Rule::DifficultyMissed => "evidence-difficulty",
Rule::DiscriminationMissed => "evidence-discrimination",
@@ -238,9 +266,11 @@ impl Rule {
| Rule::WeakFormatForLevel
| Rule::ScoredBonusLevel
| Rule::ContradictoryDesign => "completeness",
Rule::ThinOptionPool | Rule::UnusedOption | Rule::UnjustifiedRetirement => "pool",
Rule::StaleCalibration
| Rule::DifficultyMissed
| Rule::DiscriminationMissed
| Rule::NonfunctioningDistractor
| Rule::DuplicateStem => "evidence",
}
}
@@ -270,6 +300,10 @@ impl Rule {
Rule::WeakFormatForLevel,
Rule::ScoredBonusLevel,
Rule::ContradictoryDesign,
Rule::ThinOptionPool,
Rule::UnusedOption,
Rule::UnjustifiedRetirement,
Rule::NonfunctioningDistractor,
Rule::StaleCalibration,
Rule::DifficultyMissed,
Rule::DiscriminationMissed,
@@ -302,6 +336,12 @@ impl Rule {
Rule::WeakFormatForLevel => "true/false at an analytic level",
Rule::ScoredBonusLevel => "a level the policy reserves for bonus is scored",
Rule::ContradictoryDesign => "low expected discrimination on a higher-level item",
Rule::ThinOptionPool => "fewer usable distractors than a form shows",
Rule::UnusedOption => "an option has never been administered and is not retired",
Rule::UnjustifiedRetirement => "an option was retired without saying what it did",
Rule::NonfunctioningDistractor => {
"a distractor has never been chosen in any set it appeared in"
}
Rule::StaleCalibration => "statistics describe an older version of the item",
Rule::DifficultyMissed => "observed difficulty was far from predicted",
Rule::DiscriminationMissed => "observed discrimination contradicted the prediction",
@@ -749,6 +789,87 @@ pub fn lint_item(entry: &Entry, course: &CourseFile, t: &Thresholds) -> Vec<Find
}
}
// --- the option pool
// These only make sense once options are a pool, and the pool is where an
// item's spare parts sit. A bank that never draws from it will not trip any
// of them.
if it.format.has_options() {
let (keys, distractors) = it.pool();
let wanted = course.policy.options_per_item.saturating_sub(1);
if distractors.len() < wanted {
push(
Rule::ThinOptionPool,
Severity::High,
format!(
"has {} usable distractor(s) but a form shows {}, so `assemble` will put \
this on a paper an option short",
distractors.len(),
course.policy.options_per_item
),
);
}
for option in &it.options {
match &option.retired {
Some(retirement) => {
if retirement.reason.trim().len() < 12 {
push(
Rule::UnjustifiedRetirement,
Severity::Low,
format!(
"option `{}` is retired with no real reason. The reason is the \
finding — what it drew, or failed to draw — and it is the only \
part of a retirement worth anything in two years",
option.id
),
);
}
}
None => {
// An option nobody has been shown is a draft, and a draft
// sitting in an approved item's pool will eventually be
// drawn onto a paper without ever having been reviewed
// against data.
if let Some(history) = it
.calibration
.as_ref()
.filter(|c| !c.options.is_empty())
.and_then(|c| c.options.get(&option.id))
{
if history.never_chosen && history.appearances > 1 {
push(
Rule::NonfunctioningDistractor,
Severity::Medium,
format!(
"option `{}` has appeared in {} option set(s) across {} \
examinees and has never been chosen. One administration \
would not show this; several do.",
option.id, history.appearances, history.n_examinees
),
);
}
} else if it
.calibration
.as_ref()
.is_some_and(|c| !c.options.is_empty())
{
push(
Rule::UnusedOption,
Severity::Low,
format!(
"option `{}` has never been administered. Either it is waiting \
its turn, or it was drafted and forgotten — retire it and say \
which.",
option.id
),
);
}
}
}
}
let _ = keys;
}
// --- evidence
if !it.calibration_is_current() {
push(
@@ -1215,6 +1336,45 @@ mod tests {
c
}
#[test]
fn a_pool_too_thin_to_fill_a_form_is_flagged() {
let c = codes(
r#"
id: q-a-001
status: draft
level: 2
stem: Which mechanism best explains the sigmoidal binding curve?
options:
- { id: o-shift, text: Ligand binding shifts the tetramer to a higher-affinity state, correct: true }
- { id: o-fixed, text: Each subunit binds with the same fixed affinity throughout }
"#,
);
// Policy shows four options and the pool can supply two, so `assemble`
// would put this on a paper two short.
assert!(c.contains(&"pool-thin"), "{c:?}");
}
#[test]
fn a_retirement_with_no_finding_is_flagged() {
let c = codes(
r#"
id: q-a-001
status: draft
level: 2
stem: Which mechanism best explains the sigmoidal binding curve?
options:
- { id: o-shift, text: Ligand binding shifts the tetramer to a higher-affinity state, correct: true }
- { id: o-fixed, text: Each subunit binds with the same fixed affinity throughout }
- { id: o-consumed, text: "Ligand is consumed as it binds, depleting the available pool" }
- { id: o-oxidation, text: The heme iron changes oxidation state upon binding }
- { id: o-cooperative, text: Subunits bind independently of one another, retired: { 'on': 2026-09-20, reason: bad } }
"#,
);
assert!(c.contains(&"pool-unjustified-retirement"), "{c:?}");
// Four live distractors is enough for a four-option form.
assert!(!c.contains(&"pool-thin"), "{c:?}");
}
#[test]
fn clean_item_passes() {
let c = codes(