// SPDX-License-Identifier: Prosperity-3.0.0 // Copyright Scientific Computing Studio // Source: https://git.scient.ing/education/coursebank //! Authoring-quality checks: is this item *well written*? //! //! Validation asks whether a file is usable. Linting asks the harder question, //! and the answers are advisory: a lint finding is a prompt to look, not a //! verdict. Some rules will fire on items you meant to write that way, which is //! why every rule has a code you can silence. //! //! The rules fall into four families, and it is worth knowing which is which //! because they have different reliability. //! //! *Cueing* rules are the most valuable, because they catch items that measure //! test-taking rather than learning. If the key is reliably the longest option, a //! student who knows nothing can beat the item. These rules are mechanical and //! trustworthy. //! //! *Clarity* rules look for stems that do not pose a definite task, unemphasized //! negation, and prose that reads well above the level of the course. They catch //! real problems and also produce the most false positives. //! //! *Completeness* rules check that distractors are designed rather than filler, //! and that there is something to say to a student who picks one. They are what //! make the reporting features possible at all. //! //! *Evidence* rules compare what you predicted against what happened, and flag //! statistics that describe a version of the item you have since edited. use std::collections::{BTreeMap, BTreeSet}; use crate::catalog::{Catalog, Entry, Severity}; use crate::course::CourseFile; use crate::item::Item; use crate::taxonomy::{Discrimination, Format, Level, Status}; /// A lint rule. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum Rule { // --- cueing --- /// The keyed option is conspicuously longer than the distractors. KeyIsLongest, /// Option lengths vary so much that length itself is informative. UnevenOptionLength, /// A distinctive word appears in the stem and only in the key. WordRepeatCue, /// An "all of the above" or "none of the above" option. AllOrNoneOfTheAbove, /// A distractor stated in absolute terms, which trained students discard. AbsoluteDistractor, /// One option's text is contained in another's. OverlappingOptions, /// Two options say close to the same thing. NearDuplicateOptions, /// The keyed letter is imbalanced across the bank. KeyPositionImbalance, /// An item's option count differs from its neighbors'. InconsistentOptionCount, // --- clarity --- /// The stem does not pose a definite question or completion. StemHasNoTask, /// The stem is long enough that reading load competes with the construct. StemTooLong, /// Negation is present but not emphasized. UnemphasizedNegation, /// A vague quantifier that makes more than one option defensible. VagueQualifier, /// The prose reads well above the expected level. HighReadingLoad, /// A stem that asks which statement is "true" without a focus. UnfocusedStem, // --- completeness --- /// A distractor with no named misconception or error type. UndesignedDistractor, /// A distractor with no explanation. UnexplainedDistractor, /// Nothing to show a student who chose this option. NoStudentFeedback, /// A figure with no alt text. AssetWithoutAltText, /// True/False used at an analytic level, where it carries little signal. WeakFormatForLevel, /// A level-5 item scored in the graded total against course policy. ScoredBonusLevel, /// An expectation of low discrimination on a higher-level item. ContradictoryDesign, // --- evidence --- /// Statistics describe an older version of the item. StaleCalibration, /// Observed difficulty was far from predicted difficulty. DifficultyMissed, /// Observed discrimination contradicted the prediction. DiscriminationMissed, /// Two items in the course have nearly the same stem. DuplicateStem, } impl Rule { /// Every rule, grouped as the families appear above. /// /// Used by `--list-rules`, and by the test that keeps this list in step with /// the enum. pub const ALL: [Rule; 26] = [ Rule::KeyIsLongest, Rule::UnevenOptionLength, Rule::WordRepeatCue, Rule::AllOrNoneOfTheAbove, Rule::AbsoluteDistractor, Rule::OverlappingOptions, Rule::NearDuplicateOptions, Rule::KeyPositionImbalance, Rule::InconsistentOptionCount, Rule::StemHasNoTask, Rule::StemTooLong, Rule::UnemphasizedNegation, Rule::VagueQualifier, Rule::HighReadingLoad, Rule::UnfocusedStem, Rule::UndesignedDistractor, Rule::UnexplainedDistractor, Rule::NoStudentFeedback, Rule::AssetWithoutAltText, Rule::WeakFormatForLevel, Rule::ScoredBonusLevel, Rule::ContradictoryDesign, Rule::StaleCalibration, Rule::DifficultyMissed, Rule::DiscriminationMissed, Rule::DuplicateStem, ]; /// The rule's nominal severity, for `--list-rules` and `--min-severity`. /// /// A finding may carry a different severity than its rule's nominal one when /// the specific case is worse than usual — an unevenly sized option set is /// low by default but medium when the spread is extreme. This value is the /// rule's typical weight, which is what a listing should show. /// /// The three cueing rules that rise to high are the ones a test-wise student /// can exploit without knowing the material, which makes them scoring bugs /// rather than style notes. pub fn severity(self) -> Severity { use Rule as R; match self { // A student who can find the key without the content is not being // measured on the content. R::KeyIsLongest | R::WordRepeatCue => Severity::High, // Statistics attached to text that has since changed are actively // misleading, which is worse than absent. R::StaleCalibration => Severity::High, // An unanswerable question for a screen-reader user. R::AssetWithoutAltText => Severity::High, R::AllOrNoneOfTheAbove | R::OverlappingOptions | R::NearDuplicateOptions | R::KeyPositionImbalance | R::StemHasNoTask | R::UnemphasizedNegation | R::VagueQualifier | R::UnfocusedStem | R::UndesignedDistractor | R::WeakFormatForLevel | R::ScoredBonusLevel | R::ContradictoryDesign | R::DuplicateStem => Severity::Medium, R::UnevenOptionLength | R::AbsoluteDistractor | R::InconsistentOptionCount | R::StemTooLong | R::HighReadingLoad | R::UnexplainedDistractor | R::NoStudentFeedback | R::DifficultyMissed | R::DiscriminationMissed => Severity::Low, } } /// The stable code used to silence the rule from the command line. pub fn code(self) -> &'static str { match self { Rule::KeyIsLongest => "cue-key-longest", Rule::UnevenOptionLength => "cue-uneven-length", Rule::WordRepeatCue => "cue-word-repeat", Rule::AllOrNoneOfTheAbove => "cue-all-of-the-above", Rule::AbsoluteDistractor => "cue-absolute", Rule::OverlappingOptions => "cue-overlap", Rule::NearDuplicateOptions => "cue-near-duplicate", Rule::KeyPositionImbalance => "cue-key-position", Rule::InconsistentOptionCount => "cue-option-count", Rule::StemHasNoTask => "clarity-no-task", Rule::StemTooLong => "clarity-stem-length", Rule::UnemphasizedNegation => "clarity-negation", Rule::VagueQualifier => "clarity-vague", Rule::HighReadingLoad => "clarity-reading-load", Rule::UnfocusedStem => "clarity-unfocused", Rule::UndesignedDistractor => "complete-distractor-design", Rule::UnexplainedDistractor => "complete-distractor-explanation", Rule::NoStudentFeedback => "complete-student-feedback", Rule::AssetWithoutAltText => "complete-alt-text", Rule::WeakFormatForLevel => "complete-format-level", Rule::ScoredBonusLevel => "complete-bonus-policy", Rule::ContradictoryDesign => "complete-design-conflict", Rule::StaleCalibration => "evidence-stale", Rule::DifficultyMissed => "evidence-difficulty", Rule::DiscriminationMissed => "evidence-discrimination", Rule::DuplicateStem => "evidence-duplicate-stem", } } /// Which family the rule belongs to. pub fn family(self) -> &'static str { match self { Rule::KeyIsLongest | Rule::UnevenOptionLength | Rule::WordRepeatCue | Rule::AllOrNoneOfTheAbove | Rule::AbsoluteDistractor | Rule::OverlappingOptions | Rule::NearDuplicateOptions | Rule::KeyPositionImbalance | Rule::InconsistentOptionCount => "cueing", Rule::StemHasNoTask | Rule::StemTooLong | Rule::UnemphasizedNegation | Rule::VagueQualifier | Rule::HighReadingLoad | Rule::UnfocusedStem => "clarity", Rule::UndesignedDistractor | Rule::UnexplainedDistractor | Rule::NoStudentFeedback | Rule::AssetWithoutAltText | Rule::WeakFormatForLevel | Rule::ScoredBonusLevel | Rule::ContradictoryDesign => "completeness", Rule::StaleCalibration | Rule::DifficultyMissed | Rule::DiscriminationMissed | Rule::DuplicateStem => "evidence", } } /// Every rule, for `coursebank lint --list-rules`. pub fn all() -> Vec { vec![ Rule::KeyIsLongest, Rule::UnevenOptionLength, Rule::WordRepeatCue, Rule::AllOrNoneOfTheAbove, Rule::AbsoluteDistractor, Rule::OverlappingOptions, Rule::NearDuplicateOptions, Rule::KeyPositionImbalance, Rule::InconsistentOptionCount, Rule::StemHasNoTask, Rule::StemTooLong, Rule::UnemphasizedNegation, Rule::VagueQualifier, Rule::HighReadingLoad, Rule::UnfocusedStem, Rule::UndesignedDistractor, Rule::UnexplainedDistractor, Rule::NoStudentFeedback, Rule::AssetWithoutAltText, Rule::WeakFormatForLevel, Rule::ScoredBonusLevel, Rule::ContradictoryDesign, Rule::StaleCalibration, Rule::DifficultyMissed, Rule::DiscriminationMissed, Rule::DuplicateStem, ] } /// What the rule is looking for, one line. pub fn description(self) -> &'static str { match self { Rule::KeyIsLongest => "the key is much longer than every distractor", Rule::UnevenOptionLength => "option lengths differ enough to be a cue", Rule::WordRepeatCue => "a distinctive stem word appears only in the key", Rule::AllOrNoneOfTheAbove => "an all-of-the-above or none-of-the-above option", Rule::AbsoluteDistractor => "a distractor phrased in absolutes", Rule::OverlappingOptions => "one option contains another", Rule::NearDuplicateOptions => "two options are near duplicates", Rule::KeyPositionImbalance => "keyed letters are unevenly distributed in the bank", Rule::InconsistentOptionCount => "option count differs from the bank default", Rule::StemHasNoTask => "the stem poses no definite task", Rule::StemTooLong => "the stem is very long", Rule::UnemphasizedNegation => "negation is not emphasized", Rule::VagueQualifier => "a vague quantifier makes several options defensible", Rule::HighReadingLoad => "reading level is well above the course", Rule::UnfocusedStem => "the stem asks which statement is true, without a focus", Rule::UndesignedDistractor => "a distractor names no misconception or error type", Rule::UnexplainedDistractor => "a distractor has no explanation", Rule::NoStudentFeedback => "nothing to show a student who chose this option", Rule::AssetWithoutAltText => "a figure has no alt text", Rule::WeakFormatForLevel => "true/false at an analytic level", Rule::ScoredBonusLevel => "a level the policy reserves for bonus is scored", Rule::ContradictoryDesign => "low expected discrimination on a higher-level item", Rule::StaleCalibration => "statistics describe an older version of the item", Rule::DifficultyMissed => "observed difficulty was far from predicted", Rule::DiscriminationMissed => "observed discrimination contradicted the prediction", Rule::DuplicateStem => "two items have nearly the same stem", } } } /// One lint finding. #[derive(Debug, Clone)] pub struct Finding { /// The item's global id, or a bank id for bank-wide findings. pub subject: String, /// The rule that fired. pub rule: Rule, /// How much attention it deserves. pub severity: Severity, /// What was found, and where. pub message: String, } /// Thresholds, so the judgment calls are visible and adjustable rather than /// buried as literals in the middle of a function. #[derive(Debug, Clone)] pub struct Thresholds { /// Stem word count above which the stem is called long. pub stem_words: usize, /// Ratio of longest to mean distractor length that counts as a length cue. pub key_length_ratio: f64, /// Ratio of longest to shortest option that counts as uneven. pub option_spread_ratio: f64, /// Jaccard similarity above which two options are near duplicates. pub option_similarity: f64, /// Jaccard similarity above which two stems are near duplicates. pub stem_similarity: f64, /// Flesch-Kincaid grade above which reading load is flagged. pub reading_grade: f64, /// The proportion of a bank's keys any one letter may hold. pub key_share: f64, /// The fewest items in a bank before key position is worth testing. pub key_position_min_items: usize, /// Absolute difference between predicted and observed difficulty that counts /// as a missed prediction. pub difficulty_tolerance: f64, } impl Default for Thresholds { fn default() -> Thresholds { Thresholds { stem_words: 70, key_length_ratio: 1.5, option_spread_ratio: 3.0, option_similarity: 0.8, stem_similarity: 0.85, reading_grade: 16.0, key_share: 0.4, key_position_min_items: 10, difficulty_tolerance: 0.25, } } } /// Lints every item in a course, plus the bank-wide and course-wide rules. /// /// # Arguments /// /// * `catalog` - the loaded course. /// * `t` - thresholds. /// /// # Returns /// /// Findings, ordered by severity then subject so the important ones come first. pub fn lint_catalog(catalog: &Catalog, t: &Thresholds) -> Vec { let mut out = Vec::new(); for entry in &catalog.entries { if entry.item.status == Status::Retired { continue; } out.extend(lint_item(entry, &catalog.course, t)); } out.extend(lint_key_positions(catalog, t)); out.extend(lint_option_counts(catalog)); out.extend(lint_duplicate_stems(catalog, t)); out.sort_by(|a, b| { b.severity .cmp(&a.severity) .then(a.subject.cmp(&b.subject)) .then(a.rule.cmp(&b.rule)) }); out } /// Lints one item. /// /// # Arguments /// /// * `entry` - the catalog entry. /// * `course` - the course, for policy and expected reading level. /// * `t` - thresholds. /// /// # Returns /// /// Findings for this item. pub fn lint_item(entry: &Entry, course: &CourseFile, t: &Thresholds) -> Vec { let it = &entry.item; let uid = entry.uid.clone(); let mut f = Vec::new(); let mut push = |rule: Rule, severity: Severity, message: String| { f.push(Finding { subject: uid.clone(), rule, severity, message, }); }; // ---------------------------------------------------------------- clarity let stem = it.stem.trim(); let stem_lower = stem.to_lowercase(); let words: Vec<&str> = stem.split_whitespace().collect(); if !stem.contains('?') && !stem.ends_with(':') && !stem.ends_with("___") { push( Rule::StemHasNoTask, Severity::Medium, "the stem neither asks a question nor sets up a completion; a student \ has to infer what is being asked" .to_string(), ); } if words.len() > t.stem_words { push( Rule::StemTooLong, Severity::Low, format!( "the stem runs {} words; consider moving background into a shared stimulus", words.len() ), ); } // Negation is legitimate but must be visible. Emphasis means the word is // uppercase or wrapped in markup. for neg in ["not", "except", "least", "never", "incorrect", "false"] { if contains_word(&stem_lower, neg) && !negation_is_emphasized(stem, neg) { push( Rule::UnemphasizedNegation, Severity::Medium, format!( "the stem turns on `{neg}` without emphasis; students skim past it. \ Write it as `{}` or bold it.", neg.to_uppercase() ), ); break; } } for vague in [ "often", "usually", "generally", "sometimes", "may", "might", "several", "many", "frequently", "typically", ] { if contains_word(&stem_lower, vague) { push( Rule::VagueQualifier, Severity::Low, format!( "`{vague}` in the stem can make more than one option defensible; \ pin the condition down" ), ); break; } } if stem_lower.contains("which of the following is true") || stem_lower.contains("which statement is true") || stem_lower.contains("which of the following statements is correct") { push( Rule::UnfocusedStem, Severity::Medium, "an unfocused `which is true` stem tests scanning rather than a single idea; \ name the concept the item is about" .to_string(), ); } let grade = flesch_kincaid_grade(stem); if grade > t.reading_grade { push( Rule::HighReadingLoad, Severity::Low, format!( "stem reads at about grade {grade:.0}; long sentences add reading load \ that is not part of what you are measuring" ), ); } // ----------------------------------------------------------------- cueing let keys: Vec<&crate::item::Choice> = it.options.iter().filter(|o| o.correct).collect(); let distractors: Vec<&crate::item::Choice> = it.options.iter().filter(|o| !o.correct).collect(); if !keys.is_empty() && !distractors.is_empty() { let key_len = keys .iter() .map(|o| o.text.trim().chars().count()) .max() .unwrap_or(0) as f64; let mean_distractor = distractors .iter() .map(|o| o.text.trim().chars().count() as f64) .sum::() / distractors.len() as f64; if mean_distractor > 0.0 && key_len / mean_distractor >= t.key_length_ratio { push( Rule::KeyIsLongest, Severity::High, format!( "the key is {:.1}x the average distractor length ({key_len:.0} vs \ {mean_distractor:.0} characters); a test-wise student can pick it \ without knowing the content", key_len / mean_distractor ), ); } } let lens: Vec = it .options .iter() .map(|o| o.text.trim().chars().count().max(1)) .collect(); if let (Some(&mx), Some(&mn)) = (lens.iter().max(), lens.iter().min()) { if mn > 0 && (mx as f64) / (mn as f64) >= t.option_spread_ratio { push( Rule::UnevenOptionLength, Severity::Low, format!("option lengths run {mn} to {mx} characters; even them out"), ); } } // A distinctive word shared by the stem and the key alone is a clang cue. if let Some(word) = repeated_cue_word(it) { push( Rule::WordRepeatCue, Severity::Medium, format!( "`{word}` appears in the stem and in the key but in no distractor; \ the echo points at the answer" ), ); } for o in &it.options { let low = o.text.trim().to_lowercase(); if low.starts_with("all of the above") || low.starts_with("none of the above") || low.starts_with("both a and b") || low == "a and b" || low == "all of these" || low == "none of these" { push( Rule::AllOrNoneOfTheAbove, Severity::Medium, format!( "option {} is `{}`; partial knowledge answers it and shuffling \ answers in Canvas breaks it", o.id, o.text.trim() ), ); } if !o.correct { for abs in [ "always", "never", "all ", "none ", "every ", "no exceptions", ] { if low.contains(abs) { push( Rule::AbsoluteDistractor, Severity::Low, format!( "distractor {} is phrased absolutely (`{}`); students discard \ absolutes on principle", o.id, abs.trim() ), ); break; } } } } for i in 0..it.options.len() { for j in (i + 1)..it.options.len() { let a = it.options[i].text.trim().to_lowercase(); let b = it.options[j].text.trim().to_lowercase(); if a.is_empty() || b.is_empty() { continue; } if a != b && (a.contains(&b) || b.contains(&a)) { push( Rule::OverlappingOptions, Severity::Medium, format!( "option {} contains option {}; one cannot be right without the other", if a.contains(&b) { &it.options[i].id } else { &it.options[j].id }, if a.contains(&b) { &it.options[j].id } else { &it.options[i].id } ), ); } else { let sim = jaccard(&tokens(&a), &tokens(&b)); if sim >= t.option_similarity { push( Rule::NearDuplicateOptions, Severity::Medium, format!( "options {} and {} are {:.0}% the same; they are not two \ distinct ideas", it.options[i].id, it.options[j].id, sim * 100.0 ), ); } } } } // ----------------------------------------------------------- completeness // These are only worth insisting on once an item is meant to be used. let is_ready = matches!(it.status, Status::Approved | Status::InReview); if is_ready { for o in &it.options { if !o.correct { if o.misconception.is_none() && o.error_type.is_none() { push( Rule::UndesignedDistractor, Severity::Medium, format!( "distractor {} names no misconception or error_type; if you \ cannot say what mistake it captures, it is filler and its \ selection rate will tell you nothing", o.id ), ); } if o.explanation.is_none() { push( Rule::UnexplainedDistractor, Severity::Low, format!("distractor {} has no explanation", o.id), ); } } if o.student_text().is_none() { push( Rule::NoStudentFeedback, Severity::Low, format!( "option {} carries no text a post-exam report could show a \ student who chose it", o.id ), ); } } } for a in &it.assets { if a.alt .as_deref() .map(|s| s.trim().is_empty()) .unwrap_or(true) { push( Rule::AssetWithoutAltText, Severity::Medium, format!("asset `{}` has no alt text", a.path), ); } } if it.format == Format::TrueFalse && it.level >= Level::Analyze { push( Rule::WeakFormatForLevel, Severity::Low, format!( "true/false at level {} gives a 50% floor and little diagnostic signal", it.level.code() ), ); } if course.policy.bonus_levels.contains(&it.level) && !it.bonus { push( Rule::ScoredBonusLevel, Severity::Medium, format!( "the course policy reserves level {} for bonus items, but this one is scored", it.level.code() ), ); } if let Some(d) = &it.design { if d.expected_discrimination == Some(Discrimination::Low) && it.level >= Level::Apply { push( Rule::ContradictoryDesign, Severity::Low, format!( "a level {} item expected to discriminate poorly is doing the work of \ a level 1 anchor; check the level or the expectation", it.level.code() ), ); } } // --------------------------------------------------------------- evidence if !it.calibration_is_current() { push( Rule::StaleCalibration, Severity::High, "the item was edited after it was calibrated, so its statistics and IRT \ parameters describe a question you no longer ask" .to_string(), ); } if let (Some(design), Some(cal)) = (&it.design, &it.calibration) { if let (Some(expected), Some(observed)) = (design.expected_difficulty, cal.p_value) { let gap = (expected - observed).abs(); if gap >= t.difficulty_tolerance { push( Rule::DifficultyMissed, Severity::Medium, format!( "you predicted {:.0}% correct and observed {:.0}%; either the \ cohort or the item is not what you thought", expected * 100.0, observed * 100.0 ), ); } } if let (Some(expected), Some(observed)) = (design.expected_discrimination, cal.point_biserial) { let (lo, hi) = expected.expected_band(); if observed < lo || observed > hi { push( Rule::DiscriminationMissed, Severity::Low, format!( "expected {:?} discrimination (roughly {lo:.2} to {hi:.2}) but \ observed a point-biserial of {observed:.2}", expected ), ); } } } f } /// Flags banks whose keyed letters cluster on one position. /// /// Instructors reach for C. Students know it. This is a bank-level rule because /// a single item cannot be imbalanced. /// /// # Arguments /// /// * `catalog` - the loaded course. /// * `t` - thresholds. /// /// # Returns /// /// One finding per offending bank. fn lint_key_positions(catalog: &Catalog, t: &Thresholds) -> Vec { let mut by_bank: BTreeMap<&str, Vec<&Entry>> = BTreeMap::new(); for e in &catalog.entries { if e.item.status == Status::Retired || e.item.is_multi_key() { continue; } by_bank.entry(e.bank.as_str()).or_default().push(e); } let mut out = Vec::new(); for (bank, entries) in by_bank { if entries.len() < t.key_position_min_items { continue; } let mut counts: BTreeMap = BTreeMap::new(); for e in &entries { for letter in e.item.key_letters() { *counts.entry(letter).or_insert(0) += 1; } } let total: usize = counts.values().sum(); if total == 0 { continue; } for (letter, n) in &counts { let share = *n as f64 / total as f64; if share > t.key_share { out.push(Finding { subject: bank.to_string(), rule: Rule::KeyPositionImbalance, severity: Severity::Medium, message: format!( "{:.0}% of keys in this bank are `{letter}` ({n} of {total}); \ shuffle at export time or rebalance while authoring", share * 100.0 ), }); } } } out } /// Flags items whose option count differs from the mode of their bank. /// /// # Arguments /// /// * `catalog` - the loaded course. /// /// # Returns /// /// One finding per odd item. fn lint_option_counts(catalog: &Catalog) -> Vec { let mut by_bank: BTreeMap<&str, Vec<&Entry>> = BTreeMap::new(); for e in &catalog.entries { if e.item.status == Status::Retired { continue; } by_bank.entry(e.bank.as_str()).or_default().push(e); } let mut out = Vec::new(); for (_, entries) in by_bank { if entries.len() < 4 { continue; } let mut counts: BTreeMap = BTreeMap::new(); for e in &entries { *counts.entry(e.item.options.len()).or_insert(0) += 1; } let modal = counts .iter() .max_by_key(|(_, n)| **n) .map(|(k, _)| *k) .unwrap_or(0); // Only complain when there is a clear norm to deviate from. if counts.get(&modal).copied().unwrap_or(0) * 2 <= entries.len() { continue; } for e in &entries { if e.item.options.len() != modal { out.push(Finding { subject: e.uid.clone(), rule: Rule::InconsistentOptionCount, severity: Severity::Low, message: format!( "has {} options where the rest of the bank has {modal}; an odd \ count is itself a cue on a printed form", e.item.options.len() ), }); } } } out } /// Flags pairs of items with nearly the same stem. /// /// Two near-identical items are usually an accident of copying, and putting both /// on one form double-counts a single piece of knowledge. /// /// # Arguments /// /// * `catalog` - the loaded course. /// * `t` - thresholds. /// /// # Returns /// /// One finding per duplicate pair. fn lint_duplicate_stems(catalog: &Catalog, t: &Thresholds) -> Vec { let live: Vec<&Entry> = catalog .entries .iter() .filter(|e| e.item.status != Status::Retired) .collect(); let toks: Vec> = live.iter().map(|e| tokens(&e.item.stem)).collect(); let mut out = Vec::new(); for i in 0..live.len() { for j in (i + 1)..live.len() { if toks[i].len() < 4 || toks[j].len() < 4 { continue; } let sim = jaccard(&toks[i], &toks[j]); if sim >= t.stem_similarity { out.push(Finding { subject: live[i].uid.clone(), rule: Rule::DuplicateStem, severity: Severity::Medium, message: format!( "stem is {:.0}% the same as `{}`; putting both on one form \ double-counts one idea", sim * 100.0, live[j].uid ), }); } } } out } /// A word that appears in the stem and in the key but in no distractor. /// /// Short and common words are ignored, since "the" appearing in both proves /// nothing. /// /// # Arguments /// /// * `it` - the item. /// /// # Returns /// /// The cue word, when one exists. fn repeated_cue_word(it: &Item) -> Option { let stem_words = tokens(&it.stem); let key_words: BTreeSet = it .options .iter() .filter(|o| o.correct) .flat_map(|o| tokens(&o.text)) .collect(); let distractor_words: BTreeSet = it .options .iter() .filter(|o| !o.correct) .flat_map(|o| tokens(&o.text)) .collect(); for w in stem_words.intersection(&key_words) { // A technical term is long; a function word is not. if w.chars().count() >= 7 && !distractor_words.contains(w) { return Some(w.clone()); } } None } /// Whether a negation word is emphasized in the stem. /// /// # Arguments /// /// * `stem` - the stem as written. /// * `neg` - the lowercase negation word. /// /// # Returns /// /// `true` when the word appears uppercased or inside emphasis markup. fn negation_is_emphasized(stem: &str, neg: &str) -> bool { let upper = neg.to_uppercase(); if stem.contains(&upper) { return true; } // `*not*`, `**not**`, `_not_` all count as emphasis. for wrap in ['*', '_'] { let pattern = format!("{wrap}{neg}{wrap}"); if stem.to_lowercase().contains(&pattern) { return true; } } false } /// Whether a lowercase haystack contains a word as a whole word. /// /// # Arguments /// /// * `haystack` - lowercase text. /// * `needle` - the lowercase word. /// /// # Returns /// /// `true` on a whole-word match. fn contains_word(haystack: &str, needle: &str) -> bool { haystack .split(|c: char| !c.is_alphanumeric() && c != '\'') .any(|w| w == needle) } /// Lowercased alphabetic tokens of length 3 or more. /// /// # Arguments /// /// * `text` - the text to tokenize. /// /// # Returns /// /// The token set. fn tokens(text: &str) -> BTreeSet { text.split(|c: char| !c.is_alphanumeric()) .map(|w| w.to_lowercase()) .filter(|w| w.chars().count() >= 3) .collect() } /// Jaccard similarity of two token sets. /// /// # Arguments /// /// * `a` - the first set. /// * `b` - the second set. /// /// # Returns /// /// Similarity in `[0, 1]`; 0 when both are empty. fn jaccard(a: &BTreeSet, b: &BTreeSet) -> f64 { if a.is_empty() && b.is_empty() { return 0.0; } let inter = a.intersection(b).count() as f64; let union = a.union(b).count() as f64; if union == 0.0 { 0.0 } else { inter / union } } /// The Flesch-Kincaid grade level of a passage. /// /// This is a crude instrument, and it is used only to notice prose that is /// *unusually* dense, never to prescribe a target. /// /// # Arguments /// /// * `text` - the passage. /// /// # Returns /// /// The estimated US grade level, 0 for empty input. pub fn flesch_kincaid_grade(text: &str) -> f64 { let words: Vec<&str> = text.split_whitespace().collect(); if words.is_empty() { return 0.0; } let sentences = text .chars() .filter(|c| matches!(c, '.' | '?' | '!' | ';')) .count() .max(1); let syllables: usize = words.iter().map(|w| syllables(w)).sum(); let w = words.len() as f64; let s = sentences as f64; let y = syllables as f64; 0.39 * (w / s) + 11.8 * (y / w) - 15.59 } /// Estimates the syllable count of a word. /// /// Counts vowel groups, drops a silent trailing `e`, and never returns 0. /// /// # Arguments /// /// * `word` - the word. /// /// # Returns /// /// The estimated syllable count, at least 1. fn syllables(word: &str) -> usize { let w: String = word .chars() .filter(|c| c.is_alphabetic()) .map(|c| c.to_ascii_lowercase()) .collect(); if w.is_empty() { return 0; } let is_vowel = |c: char| matches!(c, 'a' | 'e' | 'i' | 'o' | 'u' | 'y'); let mut count = 0; let mut prev_vowel = false; for c in w.chars() { let v = is_vowel(c); if v && !prev_vowel { count += 1; } prev_vowel = v; } if w.ends_with('e') && count > 1 { count -= 1; } count.max(1) } /// Groups findings by rule for a summary table. /// /// # Arguments /// /// * `findings` - the findings to summarize. /// /// # Returns /// /// A map from rule to count. pub fn summarize(findings: &[Finding]) -> BTreeMap { let mut out: BTreeMap = BTreeMap::new(); for f in findings { *out.entry(f.rule).or_insert(0) += 1; } out } #[cfg(test)] mod tests { use super::*; #[test] fn every_rule_appears_in_all_exactly_once() { let mut seen = Rule::ALL.to_vec(); let before = seen.len(); seen.sort_by_key(|r| r.code()); seen.dedup_by_key(|r| r.code()); assert_eq!(seen.len(), before, "Rule::ALL has a duplicate"); } #[test] fn every_rule_has_a_unique_nonempty_code_and_description() { let mut codes: Vec<&str> = Rule::ALL.iter().map(|r| r.code()).collect(); codes.sort_unstable(); let count = codes.len(); codes.dedup(); assert_eq!(codes.len(), count, "rule codes must be unique"); for rule in Rule::ALL { assert!(!rule.code().is_empty()); assert!(!rule.description().is_empty(), "{:?}", rule); // Codes are what users type into --ignore, so they must be shell-safe. assert!( rule.code() .chars() .all(|c| c.is_ascii_lowercase() || c == '-'), "{} is not a plain lowercase code", rule.code() ); } } use crate::catalog::Entry; use std::path::PathBuf; fn course() -> CourseFile { serde_yaml_ng::from_str("course: { code: X, title: Y, term: Z }").unwrap() } fn entry(yaml: &str) -> Entry { let item: Item = serde_yaml_ng::from_str(yaml).expect("item parses"); Entry { uid: format!("b::{}", item.id), bank: "b".into(), path: PathBuf::from("b.yaml"), index: 0, item, } } fn codes(yaml: &str) -> Vec<&'static str> { let e = entry(yaml); let mut c: Vec<&'static str> = lint_item(&e, &course(), &Thresholds::default()) .iter() .map(|f| f.rule.code()) .collect(); c.sort_unstable(); c.dedup(); c } #[test] fn clean_item_passes() { let c = codes( r#" id: q-a-001 status: draft level: 2 stem: Which mechanism best explains the sigmoidal binding curve? options: - { id: A, text: Ligand binding shifts the tetramer to a higher-affinity state, correct: true } - { id: B, text: Each subunit binds with the same fixed affinity throughout } - { id: C, text: "Ligand is consumed as it binds, depleting the available pool" } - { id: D, text: The heme iron changes oxidation state upon binding } "#, ); assert!(c.is_empty(), "expected no findings, got {c:?}"); } #[test] fn detects_the_length_cue() { let c = codes( r#" id: q-a-001 status: draft level: 1 stem: What is the hydrophobic effect? options: - { id: A, text: "The tendency of nonpolar groups to associate in water, driven mainly by the resulting increase in the entropy of the surrounding solvent shell", correct: true } - { id: B, text: Van der Waals attraction } - { id: C, text: Hydrogen bonding } - { id: D, text: Heat release } "#, ); assert!(c.contains(&"cue-key-longest"), "{c:?}"); } #[test] fn detects_unemphasized_negation_and_accepts_emphasis() { let bare = codes( r#" id: q-a-001 status: draft level: 1 stem: Which of these is not a product of the reaction? options: - { id: A, text: aaaa, correct: true } - { id: B, text: bbbb } - { id: C, text: cccc } "#, ); assert!(bare.contains(&"clarity-negation"), "{bare:?}"); let emphasized = codes( r#" id: q-a-001 status: draft level: 1 stem: Which of these is NOT a product of the reaction? options: - { id: A, text: aaaa, correct: true } - { id: B, text: bbbb } - { id: C, text: cccc } "#, ); assert!(!emphasized.contains(&"clarity-negation"), "{emphasized:?}"); } #[test] fn detects_all_of_the_above_and_overlap() { let c = codes( r#" id: q-a-001 status: draft level: 1 stem: Which applies? options: - { id: A, text: The enzyme is inhibited } - { id: B, text: The enzyme is inhibited competitively } - { id: C, text: All of the above, correct: true } "#, ); assert!(c.contains(&"cue-all-of-the-above"), "{c:?}"); assert!(c.contains(&"cue-overlap"), "{c:?}"); } #[test] fn detects_the_word_repeat_cue() { let c = codes( r#" id: q-a-001 status: draft level: 2 stem: Which process explains cooperativity in this system? options: - { id: A, text: Conformational coupling produces cooperativity, correct: true } - { id: B, text: Independent binding at each site } - { id: C, text: Substrate depletion during the assay } "#, ); assert!(c.contains(&"cue-word-repeat"), "{c:?}"); } #[test] fn stem_without_a_task_is_flagged_but_completions_are_not() { assert!( codes( r#" id: q-a-001 status: draft level: 1 stem: The hydrophobic effect. options: - { id: A, text: aaaa, correct: true } - { id: B, text: bbbb } "# ) .contains(&"clarity-no-task") ); assert!( !codes( r#" id: q-a-001 status: draft level: 1 stem: "The initial velocity will most nearly:" options: - { id: A, text: aaaa, correct: true } - { id: B, text: bbbb } "# ) .contains(&"clarity-no-task") ); } #[test] fn completeness_rules_only_apply_once_ready() { let draft = codes( r#" id: q-a-001 status: draft level: 1 stem: What is x? options: - { id: A, text: right, correct: true } - { id: B, text: wrong } "#, ); assert!(!draft.contains(&"complete-distractor-design")); let ready = codes( r#" id: q-a-001 status: in_review level: 1 stem: What is x? options: - { id: A, text: right, correct: true } - { id: B, text: wrong } "#, ); assert!(ready.contains(&"complete-distractor-design"), "{ready:?}"); } #[test] fn stale_calibration_is_high_severity() { let e = entry( r#" id: q-a-001 status: approved level: 1 stem: What is x? options: - { id: A, text: right, correct: true } - { id: B, text: wrong } calibration: fingerprint: "0000000000000000" p_value: 0.8 "#, ); let f = lint_item(&e, &course(), &Thresholds::default()); let stale = f .iter() .find(|f| f.rule == Rule::StaleCalibration) .expect("stale calibration detected"); assert_eq!(stale.severity, Severity::High); } #[test] fn missed_predictions_are_reported() { let e = entry( r#" id: q-a-001 status: approved level: 3 stem: What is x? options: - { id: A, text: right, correct: true } - { id: B, text: wrong } design: expected_difficulty: 0.85 expected_discrimination: high calibration: p_value: 0.30 point_biserial: 0.05 "#, ); let mut c: Vec<&str> = lint_item(&e, &course(), &Thresholds::default()) .iter() .map(|f| f.rule.code()) .collect(); c.sort_unstable(); assert!(c.contains(&"evidence-difficulty"), "{c:?}"); assert!(c.contains(&"evidence-discrimination"), "{c:?}"); } #[test] fn reading_grade_is_sane() { let simple = flesch_kincaid_grade("The cat sat on the mat. It was a warm day."); let dense = flesch_kincaid_grade( "Notwithstanding the aforementioned considerations regarding \ thermodynamic favorability, the conformational equilibrium demonstrates \ substantial entropic contributions attributable to solvent reorganization.", ); assert!(simple < 6.0, "simple prose scored {simple}"); assert!(dense > 16.0, "dense prose scored {dense}"); } #[test] fn syllable_estimates_are_close_enough() { assert_eq!(syllables("cat"), 1); assert_eq!(syllables("water"), 2); assert_eq!(syllables("enzyme"), 2); assert_eq!(syllables("a"), 1); assert_eq!(syllables(""), 0); } #[test] fn jaccard_and_tokens_behave() { assert_eq!(jaccard(&tokens("the enzyme"), &tokens("the enzyme")), 1.0); assert_eq!(jaccard(&tokens("alpha"), &tokens("beta")), 0.0); // Short words are dropped, so "a b c" has no tokens. assert!(tokens("a b c").is_empty()); } #[test] fn every_rule_has_a_unique_code() { let mut codes: Vec<&str> = Rule::all().iter().map(|r| r.code()).collect(); let n = codes.len(); codes.sort_unstable(); codes.dedup(); assert_eq!(codes.len(), n, "rule codes must be unique"); } }