1466 lines
47 KiB
Rust
1466 lines
47 KiB
Rust
// SPDX-License-Identifier: Prosperity-3.0.0
|
|
// Copyright Scientific Computing Studio
|
|
// Source: https://git.scient.ing/education/coursebank
|
|
|
|
//! Authoring-quality checks: is this item *well written*?
|
|
//!
|
|
//! Validation asks whether a file is usable. Linting asks the harder question,
|
|
//! and the answers are advisory: a lint finding is a prompt to look, not a
|
|
//! verdict. Some rules will fire on items you meant to write that way, which is
|
|
//! why every rule has a code you can silence.
|
|
//!
|
|
//! The rules fall into four families, and it is worth knowing which is which
|
|
//! because they have different reliability.
|
|
//!
|
|
//! *Cueing* rules are the most valuable, because they catch items that measure
|
|
//! test-taking rather than learning. If the key is reliably the longest option, a
|
|
//! student who knows nothing can beat the item. These rules are mechanical and
|
|
//! trustworthy.
|
|
//!
|
|
//! *Clarity* rules look for stems that do not pose a definite task, unemphasized
|
|
//! negation, and prose that reads well above the level of the course. They catch
|
|
//! real problems and also produce the most false positives.
|
|
//!
|
|
//! *Completeness* rules check that distractors are designed rather than filler,
|
|
//! and that there is something to say to a student who picks one. They are what
|
|
//! make the reporting features possible at all.
|
|
//!
|
|
//! *Evidence* rules compare what you predicted against what happened, and flag
|
|
//! statistics that describe a version of the item you have since edited.
|
|
|
|
use std::collections::{BTreeMap, BTreeSet};
|
|
|
|
use crate::catalog::{Catalog, Entry, Severity};
|
|
use crate::course::CourseFile;
|
|
use crate::item::Item;
|
|
use crate::taxonomy::{Discrimination, Format, Level, Status};
|
|
|
|
/// A lint rule.
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
|
pub enum Rule {
|
|
// --- cueing ---
|
|
/// The keyed option is conspicuously longer than the distractors.
|
|
KeyIsLongest,
|
|
/// Option lengths vary so much that length itself is informative.
|
|
UnevenOptionLength,
|
|
/// A distinctive word appears in the stem and only in the key.
|
|
WordRepeatCue,
|
|
/// An "all of the above" or "none of the above" option.
|
|
AllOrNoneOfTheAbove,
|
|
/// A distractor stated in absolute terms, which trained students discard.
|
|
AbsoluteDistractor,
|
|
/// One option's text is contained in another's.
|
|
OverlappingOptions,
|
|
/// Two options say close to the same thing.
|
|
NearDuplicateOptions,
|
|
/// The keyed letter is imbalanced across the bank.
|
|
KeyPositionImbalance,
|
|
/// An item's option count differs from its neighbors'.
|
|
InconsistentOptionCount,
|
|
|
|
// --- clarity ---
|
|
/// The stem does not pose a definite question or completion.
|
|
StemHasNoTask,
|
|
/// The stem is long enough that reading load competes with the construct.
|
|
StemTooLong,
|
|
/// Negation is present but not emphasized.
|
|
UnemphasizedNegation,
|
|
/// A vague quantifier that makes more than one option defensible.
|
|
VagueQualifier,
|
|
/// The prose reads well above the expected level.
|
|
HighReadingLoad,
|
|
/// A stem that asks which statement is "true" without a focus.
|
|
UnfocusedStem,
|
|
|
|
// --- completeness ---
|
|
/// A distractor with no named misconception or error type.
|
|
UndesignedDistractor,
|
|
/// A distractor with no explanation.
|
|
UnexplainedDistractor,
|
|
/// Nothing to show a student who chose this option.
|
|
NoStudentFeedback,
|
|
/// A figure with no alt text.
|
|
AssetWithoutAltText,
|
|
/// True/False used at an analytic level, where it carries little signal.
|
|
WeakFormatForLevel,
|
|
/// A level-5 item scored in the graded total against course policy.
|
|
ScoredBonusLevel,
|
|
/// An expectation of low discrimination on a higher-level item.
|
|
ContradictoryDesign,
|
|
|
|
// --- evidence ---
|
|
/// Statistics describe an older version of the item.
|
|
StaleCalibration,
|
|
/// Observed difficulty was far from predicted difficulty.
|
|
DifficultyMissed,
|
|
/// Observed discrimination contradicted the prediction.
|
|
DiscriminationMissed,
|
|
/// Two items in the course have nearly the same stem.
|
|
DuplicateStem,
|
|
}
|
|
|
|
impl Rule {
|
|
/// Every rule, grouped as the families appear above.
|
|
///
|
|
/// Used by `--list-rules`, and by the test that keeps this list in step with
|
|
/// the enum.
|
|
pub const ALL: [Rule; 26] = [
|
|
Rule::KeyIsLongest,
|
|
Rule::UnevenOptionLength,
|
|
Rule::WordRepeatCue,
|
|
Rule::AllOrNoneOfTheAbove,
|
|
Rule::AbsoluteDistractor,
|
|
Rule::OverlappingOptions,
|
|
Rule::NearDuplicateOptions,
|
|
Rule::KeyPositionImbalance,
|
|
Rule::InconsistentOptionCount,
|
|
Rule::StemHasNoTask,
|
|
Rule::StemTooLong,
|
|
Rule::UnemphasizedNegation,
|
|
Rule::VagueQualifier,
|
|
Rule::HighReadingLoad,
|
|
Rule::UnfocusedStem,
|
|
Rule::UndesignedDistractor,
|
|
Rule::UnexplainedDistractor,
|
|
Rule::NoStudentFeedback,
|
|
Rule::AssetWithoutAltText,
|
|
Rule::WeakFormatForLevel,
|
|
Rule::ScoredBonusLevel,
|
|
Rule::ContradictoryDesign,
|
|
Rule::StaleCalibration,
|
|
Rule::DifficultyMissed,
|
|
Rule::DiscriminationMissed,
|
|
Rule::DuplicateStem,
|
|
];
|
|
|
|
/// The rule's nominal severity, for `--list-rules` and `--min-severity`.
|
|
///
|
|
/// A finding may carry a different severity than its rule's nominal one when
|
|
/// the specific case is worse than usual — an unevenly sized option set is
|
|
/// low by default but medium when the spread is extreme. This value is the
|
|
/// rule's typical weight, which is what a listing should show.
|
|
///
|
|
/// The three cueing rules that rise to high are the ones a test-wise student
|
|
/// can exploit without knowing the material, which makes them scoring bugs
|
|
/// rather than style notes.
|
|
pub fn severity(self) -> Severity {
|
|
use Rule as R;
|
|
match self {
|
|
// A student who can find the key without the content is not being
|
|
// measured on the content.
|
|
R::KeyIsLongest | R::WordRepeatCue => Severity::High,
|
|
// Statistics attached to text that has since changed are actively
|
|
// misleading, which is worse than absent.
|
|
R::StaleCalibration => Severity::High,
|
|
// An unanswerable question for a screen-reader user.
|
|
R::AssetWithoutAltText => Severity::High,
|
|
|
|
R::AllOrNoneOfTheAbove
|
|
| R::OverlappingOptions
|
|
| R::NearDuplicateOptions
|
|
| R::KeyPositionImbalance
|
|
| R::StemHasNoTask
|
|
| R::UnemphasizedNegation
|
|
| R::VagueQualifier
|
|
| R::UnfocusedStem
|
|
| R::UndesignedDistractor
|
|
| R::WeakFormatForLevel
|
|
| R::ScoredBonusLevel
|
|
| R::ContradictoryDesign
|
|
| R::DuplicateStem => Severity::Medium,
|
|
|
|
R::UnevenOptionLength
|
|
| R::AbsoluteDistractor
|
|
| R::InconsistentOptionCount
|
|
| R::StemTooLong
|
|
| R::HighReadingLoad
|
|
| R::UnexplainedDistractor
|
|
| R::NoStudentFeedback
|
|
| R::DifficultyMissed
|
|
| R::DiscriminationMissed => Severity::Low,
|
|
}
|
|
}
|
|
|
|
/// The stable code used to silence the rule from the command line.
|
|
pub fn code(self) -> &'static str {
|
|
match self {
|
|
Rule::KeyIsLongest => "cue-key-longest",
|
|
Rule::UnevenOptionLength => "cue-uneven-length",
|
|
Rule::WordRepeatCue => "cue-word-repeat",
|
|
Rule::AllOrNoneOfTheAbove => "cue-all-of-the-above",
|
|
Rule::AbsoluteDistractor => "cue-absolute",
|
|
Rule::OverlappingOptions => "cue-overlap",
|
|
Rule::NearDuplicateOptions => "cue-near-duplicate",
|
|
Rule::KeyPositionImbalance => "cue-key-position",
|
|
Rule::InconsistentOptionCount => "cue-option-count",
|
|
Rule::StemHasNoTask => "clarity-no-task",
|
|
Rule::StemTooLong => "clarity-stem-length",
|
|
Rule::UnemphasizedNegation => "clarity-negation",
|
|
Rule::VagueQualifier => "clarity-vague",
|
|
Rule::HighReadingLoad => "clarity-reading-load",
|
|
Rule::UnfocusedStem => "clarity-unfocused",
|
|
Rule::UndesignedDistractor => "complete-distractor-design",
|
|
Rule::UnexplainedDistractor => "complete-distractor-explanation",
|
|
Rule::NoStudentFeedback => "complete-student-feedback",
|
|
Rule::AssetWithoutAltText => "complete-alt-text",
|
|
Rule::WeakFormatForLevel => "complete-format-level",
|
|
Rule::ScoredBonusLevel => "complete-bonus-policy",
|
|
Rule::ContradictoryDesign => "complete-design-conflict",
|
|
Rule::StaleCalibration => "evidence-stale",
|
|
Rule::DifficultyMissed => "evidence-difficulty",
|
|
Rule::DiscriminationMissed => "evidence-discrimination",
|
|
Rule::DuplicateStem => "evidence-duplicate-stem",
|
|
}
|
|
}
|
|
|
|
/// Which family the rule belongs to.
|
|
pub fn family(self) -> &'static str {
|
|
match self {
|
|
Rule::KeyIsLongest
|
|
| Rule::UnevenOptionLength
|
|
| Rule::WordRepeatCue
|
|
| Rule::AllOrNoneOfTheAbove
|
|
| Rule::AbsoluteDistractor
|
|
| Rule::OverlappingOptions
|
|
| Rule::NearDuplicateOptions
|
|
| Rule::KeyPositionImbalance
|
|
| Rule::InconsistentOptionCount => "cueing",
|
|
Rule::StemHasNoTask
|
|
| Rule::StemTooLong
|
|
| Rule::UnemphasizedNegation
|
|
| Rule::VagueQualifier
|
|
| Rule::HighReadingLoad
|
|
| Rule::UnfocusedStem => "clarity",
|
|
Rule::UndesignedDistractor
|
|
| Rule::UnexplainedDistractor
|
|
| Rule::NoStudentFeedback
|
|
| Rule::AssetWithoutAltText
|
|
| Rule::WeakFormatForLevel
|
|
| Rule::ScoredBonusLevel
|
|
| Rule::ContradictoryDesign => "completeness",
|
|
Rule::StaleCalibration
|
|
| Rule::DifficultyMissed
|
|
| Rule::DiscriminationMissed
|
|
| Rule::DuplicateStem => "evidence",
|
|
}
|
|
}
|
|
|
|
/// Every rule, for `coursebank lint --list-rules`.
|
|
pub fn all() -> Vec<Rule> {
|
|
vec![
|
|
Rule::KeyIsLongest,
|
|
Rule::UnevenOptionLength,
|
|
Rule::WordRepeatCue,
|
|
Rule::AllOrNoneOfTheAbove,
|
|
Rule::AbsoluteDistractor,
|
|
Rule::OverlappingOptions,
|
|
Rule::NearDuplicateOptions,
|
|
Rule::KeyPositionImbalance,
|
|
Rule::InconsistentOptionCount,
|
|
Rule::StemHasNoTask,
|
|
Rule::StemTooLong,
|
|
Rule::UnemphasizedNegation,
|
|
Rule::VagueQualifier,
|
|
Rule::HighReadingLoad,
|
|
Rule::UnfocusedStem,
|
|
Rule::UndesignedDistractor,
|
|
Rule::UnexplainedDistractor,
|
|
Rule::NoStudentFeedback,
|
|
Rule::AssetWithoutAltText,
|
|
Rule::WeakFormatForLevel,
|
|
Rule::ScoredBonusLevel,
|
|
Rule::ContradictoryDesign,
|
|
Rule::StaleCalibration,
|
|
Rule::DifficultyMissed,
|
|
Rule::DiscriminationMissed,
|
|
Rule::DuplicateStem,
|
|
]
|
|
}
|
|
|
|
/// What the rule is looking for, one line.
|
|
pub fn description(self) -> &'static str {
|
|
match self {
|
|
Rule::KeyIsLongest => "the key is much longer than every distractor",
|
|
Rule::UnevenOptionLength => "option lengths differ enough to be a cue",
|
|
Rule::WordRepeatCue => "a distinctive stem word appears only in the key",
|
|
Rule::AllOrNoneOfTheAbove => "an all-of-the-above or none-of-the-above option",
|
|
Rule::AbsoluteDistractor => "a distractor phrased in absolutes",
|
|
Rule::OverlappingOptions => "one option contains another",
|
|
Rule::NearDuplicateOptions => "two options are near duplicates",
|
|
Rule::KeyPositionImbalance => "keyed letters are unevenly distributed in the bank",
|
|
Rule::InconsistentOptionCount => "option count differs from the bank default",
|
|
Rule::StemHasNoTask => "the stem poses no definite task",
|
|
Rule::StemTooLong => "the stem is very long",
|
|
Rule::UnemphasizedNegation => "negation is not emphasized",
|
|
Rule::VagueQualifier => "a vague quantifier makes several options defensible",
|
|
Rule::HighReadingLoad => "reading level is well above the course",
|
|
Rule::UnfocusedStem => "the stem asks which statement is true, without a focus",
|
|
Rule::UndesignedDistractor => "a distractor names no misconception or error type",
|
|
Rule::UnexplainedDistractor => "a distractor has no explanation",
|
|
Rule::NoStudentFeedback => "nothing to show a student who chose this option",
|
|
Rule::AssetWithoutAltText => "a figure has no alt text",
|
|
Rule::WeakFormatForLevel => "true/false at an analytic level",
|
|
Rule::ScoredBonusLevel => "a level the policy reserves for bonus is scored",
|
|
Rule::ContradictoryDesign => "low expected discrimination on a higher-level item",
|
|
Rule::StaleCalibration => "statistics describe an older version of the item",
|
|
Rule::DifficultyMissed => "observed difficulty was far from predicted",
|
|
Rule::DiscriminationMissed => "observed discrimination contradicted the prediction",
|
|
Rule::DuplicateStem => "two items have nearly the same stem",
|
|
}
|
|
}
|
|
}
|
|
|
|
/// One lint finding.
|
|
#[derive(Debug, Clone)]
|
|
pub struct Finding {
|
|
/// The item's global id, or a bank id for bank-wide findings.
|
|
pub subject: String,
|
|
/// The rule that fired.
|
|
pub rule: Rule,
|
|
/// How much attention it deserves.
|
|
pub severity: Severity,
|
|
/// What was found, and where.
|
|
pub message: String,
|
|
}
|
|
|
|
/// Thresholds, so the judgment calls are visible and adjustable rather than
|
|
/// buried as literals in the middle of a function.
|
|
#[derive(Debug, Clone)]
|
|
pub struct Thresholds {
|
|
/// Stem word count above which the stem is called long.
|
|
pub stem_words: usize,
|
|
/// Ratio of longest to mean distractor length that counts as a length cue.
|
|
pub key_length_ratio: f64,
|
|
/// Ratio of longest to shortest option that counts as uneven.
|
|
pub option_spread_ratio: f64,
|
|
/// Jaccard similarity above which two options are near duplicates.
|
|
pub option_similarity: f64,
|
|
/// Jaccard similarity above which two stems are near duplicates.
|
|
pub stem_similarity: f64,
|
|
/// Flesch-Kincaid grade above which reading load is flagged.
|
|
pub reading_grade: f64,
|
|
/// The proportion of a bank's keys any one letter may hold.
|
|
pub key_share: f64,
|
|
/// The fewest items in a bank before key position is worth testing.
|
|
pub key_position_min_items: usize,
|
|
/// Absolute difference between predicted and observed difficulty that counts
|
|
/// as a missed prediction.
|
|
pub difficulty_tolerance: f64,
|
|
}
|
|
|
|
impl Default for Thresholds {
|
|
fn default() -> Thresholds {
|
|
Thresholds {
|
|
stem_words: 70,
|
|
key_length_ratio: 1.5,
|
|
option_spread_ratio: 3.0,
|
|
option_similarity: 0.8,
|
|
stem_similarity: 0.85,
|
|
reading_grade: 16.0,
|
|
key_share: 0.4,
|
|
key_position_min_items: 10,
|
|
difficulty_tolerance: 0.25,
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Lints every item in a course, plus the bank-wide and course-wide rules.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `catalog` - the loaded course.
|
|
/// * `t` - thresholds.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// Findings, ordered by severity then subject so the important ones come first.
|
|
pub fn lint_catalog(catalog: &Catalog, t: &Thresholds) -> Vec<Finding> {
|
|
let mut out = Vec::new();
|
|
|
|
for entry in &catalog.entries {
|
|
if entry.item.status == Status::Retired {
|
|
continue;
|
|
}
|
|
out.extend(lint_item(entry, &catalog.course, t));
|
|
}
|
|
|
|
out.extend(lint_key_positions(catalog, t));
|
|
out.extend(lint_option_counts(catalog));
|
|
out.extend(lint_duplicate_stems(catalog, t));
|
|
|
|
out.sort_by(|a, b| {
|
|
b.severity
|
|
.cmp(&a.severity)
|
|
.then(a.subject.cmp(&b.subject))
|
|
.then(a.rule.cmp(&b.rule))
|
|
});
|
|
out
|
|
}
|
|
|
|
/// Lints one item.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `entry` - the catalog entry.
|
|
/// * `course` - the course, for policy and expected reading level.
|
|
/// * `t` - thresholds.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// Findings for this item.
|
|
pub fn lint_item(entry: &Entry, course: &CourseFile, t: &Thresholds) -> Vec<Finding> {
|
|
let it = &entry.item;
|
|
let uid = entry.uid.clone();
|
|
let mut f = Vec::new();
|
|
let mut push = |rule: Rule, severity: Severity, message: String| {
|
|
f.push(Finding {
|
|
subject: uid.clone(),
|
|
rule,
|
|
severity,
|
|
message,
|
|
});
|
|
};
|
|
|
|
// ---------------------------------------------------------------- clarity
|
|
let stem = it.stem.trim();
|
|
let stem_lower = stem.to_lowercase();
|
|
let words: Vec<&str> = stem.split_whitespace().collect();
|
|
|
|
if !stem.contains('?') && !stem.ends_with(':') && !stem.ends_with("___") {
|
|
push(
|
|
Rule::StemHasNoTask,
|
|
Severity::Medium,
|
|
"the stem neither asks a question nor sets up a completion; a student \
|
|
has to infer what is being asked"
|
|
.to_string(),
|
|
);
|
|
}
|
|
if words.len() > t.stem_words {
|
|
push(
|
|
Rule::StemTooLong,
|
|
Severity::Low,
|
|
format!(
|
|
"the stem runs {} words; consider moving background into a shared stimulus",
|
|
words.len()
|
|
),
|
|
);
|
|
}
|
|
|
|
// Negation is legitimate but must be visible. Emphasis means the word is
|
|
// uppercase or wrapped in markup.
|
|
for neg in ["not", "except", "least", "never", "incorrect", "false"] {
|
|
if contains_word(&stem_lower, neg) && !negation_is_emphasized(stem, neg) {
|
|
push(
|
|
Rule::UnemphasizedNegation,
|
|
Severity::Medium,
|
|
format!(
|
|
"the stem turns on `{neg}` without emphasis; students skim past it. \
|
|
Write it as `{}` or bold it.",
|
|
neg.to_uppercase()
|
|
),
|
|
);
|
|
break;
|
|
}
|
|
}
|
|
|
|
for vague in [
|
|
"often",
|
|
"usually",
|
|
"generally",
|
|
"sometimes",
|
|
"may",
|
|
"might",
|
|
"several",
|
|
"many",
|
|
"frequently",
|
|
"typically",
|
|
] {
|
|
if contains_word(&stem_lower, vague) {
|
|
push(
|
|
Rule::VagueQualifier,
|
|
Severity::Low,
|
|
format!(
|
|
"`{vague}` in the stem can make more than one option defensible; \
|
|
pin the condition down"
|
|
),
|
|
);
|
|
break;
|
|
}
|
|
}
|
|
|
|
if stem_lower.contains("which of the following is true")
|
|
|| stem_lower.contains("which statement is true")
|
|
|| stem_lower.contains("which of the following statements is correct")
|
|
{
|
|
push(
|
|
Rule::UnfocusedStem,
|
|
Severity::Medium,
|
|
"an unfocused `which is true` stem tests scanning rather than a single idea; \
|
|
name the concept the item is about"
|
|
.to_string(),
|
|
);
|
|
}
|
|
|
|
let grade = flesch_kincaid_grade(stem);
|
|
if grade > t.reading_grade {
|
|
push(
|
|
Rule::HighReadingLoad,
|
|
Severity::Low,
|
|
format!(
|
|
"stem reads at about grade {grade:.0}; long sentences add reading load \
|
|
that is not part of what you are measuring"
|
|
),
|
|
);
|
|
}
|
|
|
|
// ----------------------------------------------------------------- cueing
|
|
let keys: Vec<&crate::item::Choice> = it.options.iter().filter(|o| o.correct).collect();
|
|
let distractors: Vec<&crate::item::Choice> = it.options.iter().filter(|o| !o.correct).collect();
|
|
|
|
if !keys.is_empty() && !distractors.is_empty() {
|
|
let key_len = keys
|
|
.iter()
|
|
.map(|o| o.text.trim().chars().count())
|
|
.max()
|
|
.unwrap_or(0) as f64;
|
|
let mean_distractor = distractors
|
|
.iter()
|
|
.map(|o| o.text.trim().chars().count() as f64)
|
|
.sum::<f64>()
|
|
/ distractors.len() as f64;
|
|
if mean_distractor > 0.0 && key_len / mean_distractor >= t.key_length_ratio {
|
|
push(
|
|
Rule::KeyIsLongest,
|
|
Severity::High,
|
|
format!(
|
|
"the key is {:.1}x the average distractor length ({key_len:.0} vs \
|
|
{mean_distractor:.0} characters); a test-wise student can pick it \
|
|
without knowing the content",
|
|
key_len / mean_distractor
|
|
),
|
|
);
|
|
}
|
|
}
|
|
|
|
let lens: Vec<usize> = it
|
|
.options
|
|
.iter()
|
|
.map(|o| o.text.trim().chars().count().max(1))
|
|
.collect();
|
|
if let (Some(&mx), Some(&mn)) = (lens.iter().max(), lens.iter().min()) {
|
|
if mn > 0 && (mx as f64) / (mn as f64) >= t.option_spread_ratio {
|
|
push(
|
|
Rule::UnevenOptionLength,
|
|
Severity::Low,
|
|
format!("option lengths run {mn} to {mx} characters; even them out"),
|
|
);
|
|
}
|
|
}
|
|
|
|
// A distinctive word shared by the stem and the key alone is a clang cue.
|
|
if let Some(word) = repeated_cue_word(it) {
|
|
push(
|
|
Rule::WordRepeatCue,
|
|
Severity::Medium,
|
|
format!(
|
|
"`{word}` appears in the stem and in the key but in no distractor; \
|
|
the echo points at the answer"
|
|
),
|
|
);
|
|
}
|
|
|
|
for o in &it.options {
|
|
let low = o.text.trim().to_lowercase();
|
|
if low.starts_with("all of the above")
|
|
|| low.starts_with("none of the above")
|
|
|| low.starts_with("both a and b")
|
|
|| low == "a and b"
|
|
|| low == "all of these"
|
|
|| low == "none of these"
|
|
{
|
|
push(
|
|
Rule::AllOrNoneOfTheAbove,
|
|
Severity::Medium,
|
|
format!(
|
|
"option {} is `{}`; partial knowledge answers it and shuffling \
|
|
answers in Canvas breaks it",
|
|
o.id,
|
|
o.text.trim()
|
|
),
|
|
);
|
|
}
|
|
if !o.correct {
|
|
for abs in [
|
|
"always",
|
|
"never",
|
|
"all ",
|
|
"none ",
|
|
"every ",
|
|
"no exceptions",
|
|
] {
|
|
if low.contains(abs) {
|
|
push(
|
|
Rule::AbsoluteDistractor,
|
|
Severity::Low,
|
|
format!(
|
|
"distractor {} is phrased absolutely (`{}`); students discard \
|
|
absolutes on principle",
|
|
o.id,
|
|
abs.trim()
|
|
),
|
|
);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
for i in 0..it.options.len() {
|
|
for j in (i + 1)..it.options.len() {
|
|
let a = it.options[i].text.trim().to_lowercase();
|
|
let b = it.options[j].text.trim().to_lowercase();
|
|
if a.is_empty() || b.is_empty() {
|
|
continue;
|
|
}
|
|
if a != b && (a.contains(&b) || b.contains(&a)) {
|
|
push(
|
|
Rule::OverlappingOptions,
|
|
Severity::Medium,
|
|
format!(
|
|
"option {} contains option {}; one cannot be right without the other",
|
|
if a.contains(&b) {
|
|
&it.options[i].id
|
|
} else {
|
|
&it.options[j].id
|
|
},
|
|
if a.contains(&b) {
|
|
&it.options[j].id
|
|
} else {
|
|
&it.options[i].id
|
|
}
|
|
),
|
|
);
|
|
} else {
|
|
let sim = jaccard(&tokens(&a), &tokens(&b));
|
|
if sim >= t.option_similarity {
|
|
push(
|
|
Rule::NearDuplicateOptions,
|
|
Severity::Medium,
|
|
format!(
|
|
"options {} and {} are {:.0}% the same; they are not two \
|
|
distinct ideas",
|
|
it.options[i].id,
|
|
it.options[j].id,
|
|
sim * 100.0
|
|
),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// ----------------------------------------------------------- completeness
|
|
// These are only worth insisting on once an item is meant to be used.
|
|
let is_ready = matches!(it.status, Status::Approved | Status::InReview);
|
|
if is_ready {
|
|
for o in &it.options {
|
|
if !o.correct {
|
|
if o.misconception.is_none() && o.error_type.is_none() {
|
|
push(
|
|
Rule::UndesignedDistractor,
|
|
Severity::Medium,
|
|
format!(
|
|
"distractor {} names no misconception or error_type; if you \
|
|
cannot say what mistake it captures, it is filler and its \
|
|
selection rate will tell you nothing",
|
|
o.id
|
|
),
|
|
);
|
|
}
|
|
if o.explanation.is_none() {
|
|
push(
|
|
Rule::UnexplainedDistractor,
|
|
Severity::Low,
|
|
format!("distractor {} has no explanation", o.id),
|
|
);
|
|
}
|
|
}
|
|
if o.student_text().is_none() {
|
|
push(
|
|
Rule::NoStudentFeedback,
|
|
Severity::Low,
|
|
format!(
|
|
"option {} carries no text a post-exam report could show a \
|
|
student who chose it",
|
|
o.id
|
|
),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
for a in &it.assets {
|
|
if a.alt
|
|
.as_deref()
|
|
.map(|s| s.trim().is_empty())
|
|
.unwrap_or(true)
|
|
{
|
|
push(
|
|
Rule::AssetWithoutAltText,
|
|
Severity::Medium,
|
|
format!("asset `{}` has no alt text", a.path),
|
|
);
|
|
}
|
|
}
|
|
|
|
if it.format == Format::TrueFalse && it.level >= Level::Analyze {
|
|
push(
|
|
Rule::WeakFormatForLevel,
|
|
Severity::Low,
|
|
format!(
|
|
"true/false at level {} gives a 50% floor and little diagnostic signal",
|
|
it.level.code()
|
|
),
|
|
);
|
|
}
|
|
|
|
if course.policy.bonus_levels.contains(&it.level) && !it.bonus {
|
|
push(
|
|
Rule::ScoredBonusLevel,
|
|
Severity::Medium,
|
|
format!(
|
|
"the course policy reserves level {} for bonus items, but this one is scored",
|
|
it.level.code()
|
|
),
|
|
);
|
|
}
|
|
|
|
if let Some(d) = &it.design {
|
|
if d.expected_discrimination == Some(Discrimination::Low) && it.level >= Level::Apply {
|
|
push(
|
|
Rule::ContradictoryDesign,
|
|
Severity::Low,
|
|
format!(
|
|
"a level {} item expected to discriminate poorly is doing the work of \
|
|
a level 1 anchor; check the level or the expectation",
|
|
it.level.code()
|
|
),
|
|
);
|
|
}
|
|
}
|
|
|
|
// --------------------------------------------------------------- evidence
|
|
if !it.calibration_is_current() {
|
|
push(
|
|
Rule::StaleCalibration,
|
|
Severity::High,
|
|
"the item was edited after it was calibrated, so its statistics and IRT \
|
|
parameters describe a question you no longer ask"
|
|
.to_string(),
|
|
);
|
|
}
|
|
|
|
if let (Some(design), Some(cal)) = (&it.design, &it.calibration) {
|
|
if let (Some(expected), Some(observed)) = (design.expected_difficulty, cal.p_value) {
|
|
let gap = (expected - observed).abs();
|
|
if gap >= t.difficulty_tolerance {
|
|
push(
|
|
Rule::DifficultyMissed,
|
|
Severity::Medium,
|
|
format!(
|
|
"you predicted {:.0}% correct and observed {:.0}%; either the \
|
|
cohort or the item is not what you thought",
|
|
expected * 100.0,
|
|
observed * 100.0
|
|
),
|
|
);
|
|
}
|
|
}
|
|
if let (Some(expected), Some(observed)) =
|
|
(design.expected_discrimination, cal.point_biserial)
|
|
{
|
|
let (lo, hi) = expected.expected_band();
|
|
if observed < lo || observed > hi {
|
|
push(
|
|
Rule::DiscriminationMissed,
|
|
Severity::Low,
|
|
format!(
|
|
"expected {:?} discrimination (roughly {lo:.2} to {hi:.2}) but \
|
|
observed a point-biserial of {observed:.2}",
|
|
expected
|
|
),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
f
|
|
}
|
|
|
|
/// Flags banks whose keyed letters cluster on one position.
|
|
///
|
|
/// Instructors reach for C. Students know it. This is a bank-level rule because
|
|
/// a single item cannot be imbalanced.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `catalog` - the loaded course.
|
|
/// * `t` - thresholds.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// One finding per offending bank.
|
|
fn lint_key_positions(catalog: &Catalog, t: &Thresholds) -> Vec<Finding> {
|
|
let mut by_bank: BTreeMap<&str, Vec<&Entry>> = BTreeMap::new();
|
|
for e in &catalog.entries {
|
|
if e.item.status == Status::Retired || e.item.is_multi_key() {
|
|
continue;
|
|
}
|
|
by_bank.entry(e.bank.as_str()).or_default().push(e);
|
|
}
|
|
|
|
let mut out = Vec::new();
|
|
for (bank, entries) in by_bank {
|
|
if entries.len() < t.key_position_min_items {
|
|
continue;
|
|
}
|
|
let mut counts: BTreeMap<String, usize> = BTreeMap::new();
|
|
for e in &entries {
|
|
for letter in e.item.key_letters() {
|
|
*counts.entry(letter).or_insert(0) += 1;
|
|
}
|
|
}
|
|
let total: usize = counts.values().sum();
|
|
if total == 0 {
|
|
continue;
|
|
}
|
|
for (letter, n) in &counts {
|
|
let share = *n as f64 / total as f64;
|
|
if share > t.key_share {
|
|
out.push(Finding {
|
|
subject: bank.to_string(),
|
|
rule: Rule::KeyPositionImbalance,
|
|
severity: Severity::Medium,
|
|
message: format!(
|
|
"{:.0}% of keys in this bank are `{letter}` ({n} of {total}); \
|
|
shuffle at export time or rebalance while authoring",
|
|
share * 100.0
|
|
),
|
|
});
|
|
}
|
|
}
|
|
}
|
|
out
|
|
}
|
|
|
|
/// Flags items whose option count differs from the mode of their bank.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `catalog` - the loaded course.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// One finding per odd item.
|
|
fn lint_option_counts(catalog: &Catalog) -> Vec<Finding> {
|
|
let mut by_bank: BTreeMap<&str, Vec<&Entry>> = BTreeMap::new();
|
|
for e in &catalog.entries {
|
|
if e.item.status == Status::Retired {
|
|
continue;
|
|
}
|
|
by_bank.entry(e.bank.as_str()).or_default().push(e);
|
|
}
|
|
|
|
let mut out = Vec::new();
|
|
for (_, entries) in by_bank {
|
|
if entries.len() < 4 {
|
|
continue;
|
|
}
|
|
let mut counts: BTreeMap<usize, usize> = BTreeMap::new();
|
|
for e in &entries {
|
|
*counts.entry(e.item.options.len()).or_insert(0) += 1;
|
|
}
|
|
let modal = counts
|
|
.iter()
|
|
.max_by_key(|(_, n)| **n)
|
|
.map(|(k, _)| *k)
|
|
.unwrap_or(0);
|
|
// Only complain when there is a clear norm to deviate from.
|
|
if counts.get(&modal).copied().unwrap_or(0) * 2 <= entries.len() {
|
|
continue;
|
|
}
|
|
for e in &entries {
|
|
if e.item.options.len() != modal {
|
|
out.push(Finding {
|
|
subject: e.uid.clone(),
|
|
rule: Rule::InconsistentOptionCount,
|
|
severity: Severity::Low,
|
|
message: format!(
|
|
"has {} options where the rest of the bank has {modal}; an odd \
|
|
count is itself a cue on a printed form",
|
|
e.item.options.len()
|
|
),
|
|
});
|
|
}
|
|
}
|
|
}
|
|
out
|
|
}
|
|
|
|
/// Flags pairs of items with nearly the same stem.
|
|
///
|
|
/// Two near-identical items are usually an accident of copying, and putting both
|
|
/// on one form double-counts a single piece of knowledge.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `catalog` - the loaded course.
|
|
/// * `t` - thresholds.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// One finding per duplicate pair.
|
|
fn lint_duplicate_stems(catalog: &Catalog, t: &Thresholds) -> Vec<Finding> {
|
|
let live: Vec<&Entry> = catalog
|
|
.entries
|
|
.iter()
|
|
.filter(|e| e.item.status != Status::Retired)
|
|
.collect();
|
|
|
|
let toks: Vec<BTreeSet<String>> = live.iter().map(|e| tokens(&e.item.stem)).collect();
|
|
let mut out = Vec::new();
|
|
for i in 0..live.len() {
|
|
for j in (i + 1)..live.len() {
|
|
if toks[i].len() < 4 || toks[j].len() < 4 {
|
|
continue;
|
|
}
|
|
let sim = jaccard(&toks[i], &toks[j]);
|
|
if sim >= t.stem_similarity {
|
|
out.push(Finding {
|
|
subject: live[i].uid.clone(),
|
|
rule: Rule::DuplicateStem,
|
|
severity: Severity::Medium,
|
|
message: format!(
|
|
"stem is {:.0}% the same as `{}`; putting both on one form \
|
|
double-counts one idea",
|
|
sim * 100.0,
|
|
live[j].uid
|
|
),
|
|
});
|
|
}
|
|
}
|
|
}
|
|
out
|
|
}
|
|
|
|
/// A word that appears in the stem and in the key but in no distractor.
|
|
///
|
|
/// Short and common words are ignored, since "the" appearing in both proves
|
|
/// nothing.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `it` - the item.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The cue word, when one exists.
|
|
fn repeated_cue_word(it: &Item) -> Option<String> {
|
|
let stem_words = tokens(&it.stem);
|
|
let key_words: BTreeSet<String> = it
|
|
.options
|
|
.iter()
|
|
.filter(|o| o.correct)
|
|
.flat_map(|o| tokens(&o.text))
|
|
.collect();
|
|
let distractor_words: BTreeSet<String> = it
|
|
.options
|
|
.iter()
|
|
.filter(|o| !o.correct)
|
|
.flat_map(|o| tokens(&o.text))
|
|
.collect();
|
|
|
|
for w in stem_words.intersection(&key_words) {
|
|
// A technical term is long; a function word is not.
|
|
if w.chars().count() >= 7 && !distractor_words.contains(w) {
|
|
return Some(w.clone());
|
|
}
|
|
}
|
|
None
|
|
}
|
|
|
|
/// Whether a negation word is emphasized in the stem.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `stem` - the stem as written.
|
|
/// * `neg` - the lowercase negation word.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// `true` when the word appears uppercased or inside emphasis markup.
|
|
fn negation_is_emphasized(stem: &str, neg: &str) -> bool {
|
|
let upper = neg.to_uppercase();
|
|
if stem.contains(&upper) {
|
|
return true;
|
|
}
|
|
// `*not*`, `**not**`, `_not_` all count as emphasis.
|
|
for wrap in ['*', '_'] {
|
|
let pattern = format!("{wrap}{neg}{wrap}");
|
|
if stem.to_lowercase().contains(&pattern) {
|
|
return true;
|
|
}
|
|
}
|
|
false
|
|
}
|
|
|
|
/// Whether a lowercase haystack contains a word as a whole word.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `haystack` - lowercase text.
|
|
/// * `needle` - the lowercase word.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// `true` on a whole-word match.
|
|
fn contains_word(haystack: &str, needle: &str) -> bool {
|
|
haystack
|
|
.split(|c: char| !c.is_alphanumeric() && c != '\'')
|
|
.any(|w| w == needle)
|
|
}
|
|
|
|
/// Lowercased alphabetic tokens of length 3 or more.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `text` - the text to tokenize.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The token set.
|
|
fn tokens(text: &str) -> BTreeSet<String> {
|
|
text.split(|c: char| !c.is_alphanumeric())
|
|
.map(|w| w.to_lowercase())
|
|
.filter(|w| w.chars().count() >= 3)
|
|
.collect()
|
|
}
|
|
|
|
/// Jaccard similarity of two token sets.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `a` - the first set.
|
|
/// * `b` - the second set.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// Similarity in `[0, 1]`; 0 when both are empty.
|
|
fn jaccard(a: &BTreeSet<String>, b: &BTreeSet<String>) -> f64 {
|
|
if a.is_empty() && b.is_empty() {
|
|
return 0.0;
|
|
}
|
|
let inter = a.intersection(b).count() as f64;
|
|
let union = a.union(b).count() as f64;
|
|
if union == 0.0 { 0.0 } else { inter / union }
|
|
}
|
|
|
|
/// The Flesch-Kincaid grade level of a passage.
|
|
///
|
|
/// This is a crude instrument, and it is used only to notice prose that is
|
|
/// *unusually* dense, never to prescribe a target.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `text` - the passage.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The estimated US grade level, 0 for empty input.
|
|
pub fn flesch_kincaid_grade(text: &str) -> f64 {
|
|
let words: Vec<&str> = text.split_whitespace().collect();
|
|
if words.is_empty() {
|
|
return 0.0;
|
|
}
|
|
let sentences = text
|
|
.chars()
|
|
.filter(|c| matches!(c, '.' | '?' | '!' | ';'))
|
|
.count()
|
|
.max(1);
|
|
let syllables: usize = words.iter().map(|w| syllables(w)).sum();
|
|
let w = words.len() as f64;
|
|
let s = sentences as f64;
|
|
let y = syllables as f64;
|
|
0.39 * (w / s) + 11.8 * (y / w) - 15.59
|
|
}
|
|
|
|
/// Estimates the syllable count of a word.
|
|
///
|
|
/// Counts vowel groups, drops a silent trailing `e`, and never returns 0.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `word` - the word.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The estimated syllable count, at least 1.
|
|
fn syllables(word: &str) -> usize {
|
|
let w: String = word
|
|
.chars()
|
|
.filter(|c| c.is_alphabetic())
|
|
.map(|c| c.to_ascii_lowercase())
|
|
.collect();
|
|
if w.is_empty() {
|
|
return 0;
|
|
}
|
|
let is_vowel = |c: char| matches!(c, 'a' | 'e' | 'i' | 'o' | 'u' | 'y');
|
|
let mut count = 0;
|
|
let mut prev_vowel = false;
|
|
for c in w.chars() {
|
|
let v = is_vowel(c);
|
|
if v && !prev_vowel {
|
|
count += 1;
|
|
}
|
|
prev_vowel = v;
|
|
}
|
|
if w.ends_with('e') && count > 1 {
|
|
count -= 1;
|
|
}
|
|
count.max(1)
|
|
}
|
|
|
|
/// Groups findings by rule for a summary table.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `findings` - the findings to summarize.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// A map from rule to count.
|
|
pub fn summarize(findings: &[Finding]) -> BTreeMap<Rule, usize> {
|
|
let mut out: BTreeMap<Rule, usize> = BTreeMap::new();
|
|
for f in findings {
|
|
*out.entry(f.rule).or_insert(0) += 1;
|
|
}
|
|
out
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn every_rule_appears_in_all_exactly_once() {
|
|
let mut seen = Rule::ALL.to_vec();
|
|
let before = seen.len();
|
|
seen.sort_by_key(|r| r.code());
|
|
seen.dedup_by_key(|r| r.code());
|
|
assert_eq!(seen.len(), before, "Rule::ALL has a duplicate");
|
|
}
|
|
|
|
#[test]
|
|
fn every_rule_has_a_unique_nonempty_code_and_description() {
|
|
let mut codes: Vec<&str> = Rule::ALL.iter().map(|r| r.code()).collect();
|
|
codes.sort_unstable();
|
|
let count = codes.len();
|
|
codes.dedup();
|
|
assert_eq!(codes.len(), count, "rule codes must be unique");
|
|
for rule in Rule::ALL {
|
|
assert!(!rule.code().is_empty());
|
|
assert!(!rule.description().is_empty(), "{:?}", rule);
|
|
// Codes are what users type into --ignore, so they must be shell-safe.
|
|
assert!(
|
|
rule.code()
|
|
.chars()
|
|
.all(|c| c.is_ascii_lowercase() || c == '-'),
|
|
"{} is not a plain lowercase code",
|
|
rule.code()
|
|
);
|
|
}
|
|
}
|
|
use crate::catalog::Entry;
|
|
use std::path::PathBuf;
|
|
|
|
fn course() -> CourseFile {
|
|
serde_yaml_ng::from_str("course: { code: X, title: Y, term: Z }").unwrap()
|
|
}
|
|
|
|
fn entry(yaml: &str) -> Entry {
|
|
let item: Item = serde_yaml_ng::from_str(yaml).expect("item parses");
|
|
Entry {
|
|
uid: format!("b::{}", item.id),
|
|
bank: "b".into(),
|
|
path: PathBuf::from("b.yaml"),
|
|
index: 0,
|
|
item,
|
|
}
|
|
}
|
|
|
|
fn codes(yaml: &str) -> Vec<&'static str> {
|
|
let e = entry(yaml);
|
|
let mut c: Vec<&'static str> = lint_item(&e, &course(), &Thresholds::default())
|
|
.iter()
|
|
.map(|f| f.rule.code())
|
|
.collect();
|
|
c.sort_unstable();
|
|
c.dedup();
|
|
c
|
|
}
|
|
|
|
#[test]
|
|
fn clean_item_passes() {
|
|
let c = codes(
|
|
r#"
|
|
id: q-a-001
|
|
status: draft
|
|
level: 2
|
|
stem: Which mechanism best explains the sigmoidal binding curve?
|
|
options:
|
|
- { id: A, text: Ligand binding shifts the tetramer to a higher-affinity state, correct: true }
|
|
- { id: B, text: Each subunit binds with the same fixed affinity throughout }
|
|
- { id: C, text: "Ligand is consumed as it binds, depleting the available pool" }
|
|
- { id: D, text: The heme iron changes oxidation state upon binding }
|
|
"#,
|
|
);
|
|
assert!(c.is_empty(), "expected no findings, got {c:?}");
|
|
}
|
|
|
|
#[test]
|
|
fn detects_the_length_cue() {
|
|
let c = codes(
|
|
r#"
|
|
id: q-a-001
|
|
status: draft
|
|
level: 1
|
|
stem: What is the hydrophobic effect?
|
|
options:
|
|
- { id: A, text: "The tendency of nonpolar groups to associate in water, driven mainly by the resulting increase in the entropy of the surrounding solvent shell", correct: true }
|
|
- { id: B, text: Van der Waals attraction }
|
|
- { id: C, text: Hydrogen bonding }
|
|
- { id: D, text: Heat release }
|
|
"#,
|
|
);
|
|
assert!(c.contains(&"cue-key-longest"), "{c:?}");
|
|
}
|
|
|
|
#[test]
|
|
fn detects_unemphasized_negation_and_accepts_emphasis() {
|
|
let bare = codes(
|
|
r#"
|
|
id: q-a-001
|
|
status: draft
|
|
level: 1
|
|
stem: Which of these is not a product of the reaction?
|
|
options:
|
|
- { id: A, text: aaaa, correct: true }
|
|
- { id: B, text: bbbb }
|
|
- { id: C, text: cccc }
|
|
"#,
|
|
);
|
|
assert!(bare.contains(&"clarity-negation"), "{bare:?}");
|
|
|
|
let emphasized = codes(
|
|
r#"
|
|
id: q-a-001
|
|
status: draft
|
|
level: 1
|
|
stem: Which of these is NOT a product of the reaction?
|
|
options:
|
|
- { id: A, text: aaaa, correct: true }
|
|
- { id: B, text: bbbb }
|
|
- { id: C, text: cccc }
|
|
"#,
|
|
);
|
|
assert!(!emphasized.contains(&"clarity-negation"), "{emphasized:?}");
|
|
}
|
|
|
|
#[test]
|
|
fn detects_all_of_the_above_and_overlap() {
|
|
let c = codes(
|
|
r#"
|
|
id: q-a-001
|
|
status: draft
|
|
level: 1
|
|
stem: Which applies?
|
|
options:
|
|
- { id: A, text: The enzyme is inhibited }
|
|
- { id: B, text: The enzyme is inhibited competitively }
|
|
- { id: C, text: All of the above, correct: true }
|
|
"#,
|
|
);
|
|
assert!(c.contains(&"cue-all-of-the-above"), "{c:?}");
|
|
assert!(c.contains(&"cue-overlap"), "{c:?}");
|
|
}
|
|
|
|
#[test]
|
|
fn detects_the_word_repeat_cue() {
|
|
let c = codes(
|
|
r#"
|
|
id: q-a-001
|
|
status: draft
|
|
level: 2
|
|
stem: Which process explains cooperativity in this system?
|
|
options:
|
|
- { id: A, text: Conformational coupling produces cooperativity, correct: true }
|
|
- { id: B, text: Independent binding at each site }
|
|
- { id: C, text: Substrate depletion during the assay }
|
|
"#,
|
|
);
|
|
assert!(c.contains(&"cue-word-repeat"), "{c:?}");
|
|
}
|
|
|
|
#[test]
|
|
fn stem_without_a_task_is_flagged_but_completions_are_not() {
|
|
assert!(
|
|
codes(
|
|
r#"
|
|
id: q-a-001
|
|
status: draft
|
|
level: 1
|
|
stem: The hydrophobic effect.
|
|
options:
|
|
- { id: A, text: aaaa, correct: true }
|
|
- { id: B, text: bbbb }
|
|
"#
|
|
)
|
|
.contains(&"clarity-no-task")
|
|
);
|
|
|
|
assert!(
|
|
!codes(
|
|
r#"
|
|
id: q-a-001
|
|
status: draft
|
|
level: 1
|
|
stem: "The initial velocity will most nearly:"
|
|
options:
|
|
- { id: A, text: aaaa, correct: true }
|
|
- { id: B, text: bbbb }
|
|
"#
|
|
)
|
|
.contains(&"clarity-no-task")
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn completeness_rules_only_apply_once_ready() {
|
|
let draft = codes(
|
|
r#"
|
|
id: q-a-001
|
|
status: draft
|
|
level: 1
|
|
stem: What is x?
|
|
options:
|
|
- { id: A, text: right, correct: true }
|
|
- { id: B, text: wrong }
|
|
"#,
|
|
);
|
|
assert!(!draft.contains(&"complete-distractor-design"));
|
|
|
|
let ready = codes(
|
|
r#"
|
|
id: q-a-001
|
|
status: in_review
|
|
level: 1
|
|
stem: What is x?
|
|
options:
|
|
- { id: A, text: right, correct: true }
|
|
- { id: B, text: wrong }
|
|
"#,
|
|
);
|
|
assert!(ready.contains(&"complete-distractor-design"), "{ready:?}");
|
|
}
|
|
|
|
#[test]
|
|
fn stale_calibration_is_high_severity() {
|
|
let e = entry(
|
|
r#"
|
|
id: q-a-001
|
|
status: approved
|
|
level: 1
|
|
stem: What is x?
|
|
options:
|
|
- { id: A, text: right, correct: true }
|
|
- { id: B, text: wrong }
|
|
calibration:
|
|
fingerprint: "0000000000000000"
|
|
p_value: 0.8
|
|
"#,
|
|
);
|
|
let f = lint_item(&e, &course(), &Thresholds::default());
|
|
let stale = f
|
|
.iter()
|
|
.find(|f| f.rule == Rule::StaleCalibration)
|
|
.expect("stale calibration detected");
|
|
assert_eq!(stale.severity, Severity::High);
|
|
}
|
|
|
|
#[test]
|
|
fn missed_predictions_are_reported() {
|
|
let e = entry(
|
|
r#"
|
|
id: q-a-001
|
|
status: approved
|
|
level: 3
|
|
stem: What is x?
|
|
options:
|
|
- { id: A, text: right, correct: true }
|
|
- { id: B, text: wrong }
|
|
design:
|
|
expected_difficulty: 0.85
|
|
expected_discrimination: high
|
|
calibration:
|
|
p_value: 0.30
|
|
point_biserial: 0.05
|
|
"#,
|
|
);
|
|
let mut c: Vec<&str> = lint_item(&e, &course(), &Thresholds::default())
|
|
.iter()
|
|
.map(|f| f.rule.code())
|
|
.collect();
|
|
c.sort_unstable();
|
|
assert!(c.contains(&"evidence-difficulty"), "{c:?}");
|
|
assert!(c.contains(&"evidence-discrimination"), "{c:?}");
|
|
}
|
|
|
|
#[test]
|
|
fn reading_grade_is_sane() {
|
|
let simple = flesch_kincaid_grade("The cat sat on the mat. It was a warm day.");
|
|
let dense = flesch_kincaid_grade(
|
|
"Notwithstanding the aforementioned considerations regarding \
|
|
thermodynamic favorability, the conformational equilibrium demonstrates \
|
|
substantial entropic contributions attributable to solvent reorganization.",
|
|
);
|
|
assert!(simple < 6.0, "simple prose scored {simple}");
|
|
assert!(dense > 16.0, "dense prose scored {dense}");
|
|
}
|
|
|
|
#[test]
|
|
fn syllable_estimates_are_close_enough() {
|
|
assert_eq!(syllables("cat"), 1);
|
|
assert_eq!(syllables("water"), 2);
|
|
assert_eq!(syllables("enzyme"), 2);
|
|
assert_eq!(syllables("a"), 1);
|
|
assert_eq!(syllables(""), 0);
|
|
}
|
|
|
|
#[test]
|
|
fn jaccard_and_tokens_behave() {
|
|
assert_eq!(jaccard(&tokens("the enzyme"), &tokens("the enzyme")), 1.0);
|
|
assert_eq!(jaccard(&tokens("alpha"), &tokens("beta")), 0.0);
|
|
// Short words are dropped, so "a b c" has no tokens.
|
|
assert!(tokens("a b c").is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn every_rule_has_a_unique_code() {
|
|
let mut codes: Vec<&str> = Rule::all().iter().map(|r| r.code()).collect();
|
|
let n = codes.len();
|
|
codes.sort_unstable();
|
|
codes.dedup();
|
|
assert_eq!(codes.len(), n, "rule codes must be unique");
|
|
}
|
|
}
|