// SPDX-License-Identifier: Prosperity-3.0.0 // Copyright Scientific Computing Studio // Source: https://git.scient.ing/education/coursebank //! What to tell a student, and what to tell yourself. //! //! [`crate::students`] computes mastery. [`crate::classical`] computes item //! statistics. Neither decides what belongs in a document, and that decision is //! not a formatting concern: it determines what a student is able to reconstruct //! from the page in front of them. //! //! So this module assembles two view models, and the interesting property of the //! first one is what it does not contain. //! //! # The withheld-question invariant //! //! [`StudentDiagnostic`] has no field for a stem and no field for option text. //! Not an empty one, not one gated behind a flag: the struct has no such field, so //! no template can print what it was never given. This is the same argument //! [`crate::typst::config::Reveal`] makes about the exam paper, for the same //! reason — a flag is one forgotten `if` away from a bad afternoon, and a //! diagnostic that carries the questions cannot be handed back before the makeup //! exam is given. //! //! What a student does get, per missed question: the number, its level, the //! objectives it measured and what those objectives ask, whether they answered //! it, the feedback written for the specific option they chose, the hint written //! for that same option, and the item's own `review` citations. That is why //! authoring distractors carefully pays off twice. //! //! Two further fields are available and off by default, because they are the two //! that trade a student's understanding against reusing the question. The //! misconception is written to you about the student; the worked solution is the //! solutions document. See [`Options::misconceptions`] and [`Options::solutions`]. //! //! It is worth being clear about what the default already discloses. The //! per-option feedback on a missed question routinely names the right answer, //! because that is what makes it useful. A report handed to sixty students is //! therefore already a partial answer key for the questions those students //! missed, with or without the two optional fields. //! //! Two consequences of that invariant are worth stating because they are easy to //! undo by accident: //! //! *Only student-facing feedback is used.* [`crate::item::Choice::student_text`] //! falls back to `explanation`, which is instructor-facing and routinely says //! which option is right. This module reads `feedback_student` and `misconception` //! and nothing else. //! //! *Feedback is looked up by bank letter, not printed letter.* On a shuffled form //! those differ, and looking up the printed letter returns another option's //! misconception — confident, specific, and about a question the student did not //! answer that way. See [`crate::decode`]. //! //! # What to study //! //! A list of missed objectives is a diagnosis, not a prescription. The study plan //! resolves each weak objective through the course's own reading registry, so a //! student is pointed at `KKW §6.2` with the sentence you wrote about what to take //! from it, rather than at the name of a chapter. //! //! [`LectureFocus`] answers the question a student actually asks, which is where //! to start. It ranks the lectures behind the missed questions by how many //! objectives went wrong in each, so a reading list of eleven sections becomes an //! ordered afternoon. use std::collections::{BTreeMap, BTreeSet}; use serde::Serialize; use crate::assessment::AssessmentFile; use crate::catalog::Catalog; use crate::classical::Analysis; use crate::course::{CourseFile, ReadingRole}; use crate::irt::Fit; use crate::item::Citation; use crate::responses::{Response, ResponseSet}; use crate::students::{Cohort, Mastery, StudentSummary}; use crate::taxonomy::{Level, Tier}; /// What to assemble. #[derive(Debug, Clone)] pub struct Options { /// Whether to compare the student to the class. pub comparison: bool, /// Whether to include the per-question map. /// /// The map names question numbers, never their content. It is what lets a /// student who has their paper back line the two up. pub questions: bool, /// Whether to include the feedback written for the option the student chose. pub feedback: bool, /// Whether to include the hint written for that option. /// /// On by default. A hint is the question you would ask a student who was /// reconsidering that option, so it gives them somewhere to start rather than /// a verdict to accept. pub hints: bool, /// Whether to name the misconception the chosen distractor was written to /// catch. /// /// Off by default, and not because it is unsafe. The text is written to you, /// about the student, in the third person, and next to the feedback written /// for them it reads like a chart note. pub misconceptions: bool, /// Whether to include the worked solution for a missed question. /// /// Off by default. [`crate::item::Solution::explanation`] is the derivation, /// the estimate, and the argument for the key over its neighbours: the body of /// the solutions document. Turning this on hands that to every student who /// missed the question, which is the right call for a question you will not /// use again and the wrong one for a bank you reuse each term. pub solutions: bool, /// How many objectives to build a study plan for. pub focus_limit: usize, /// How many readings to list per objective. pub readings_per_objective: usize, /// Whether to include the IRT ability estimate. pub ability: bool, } impl Default for Options { fn default() -> Options { Options { comparison: true, questions: true, feedback: true, hints: true, misconceptions: false, solutions: false, focus_limit: 4, readings_per_objective: 2, ability: false, } } } /// Everything one student's diagnostic says. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub struct StudentDiagnostic { /// The grouping key, which is a pseudonym when the store is pseudonymized. pub student_key: String, /// The display name, when identifiers were kept. #[serde(skip_serializing_if = "Option::is_none")] pub name: Option, /// The institutional id, when identifiers were kept. #[serde(skip_serializing_if = "Option::is_none")] pub sid: Option, /// Their email, when the export carried one and identifiers were kept. /// /// Absent rather than blank when the platform did not report it, so a /// template prints nothing instead of an empty label. #[serde(skip_serializing_if = "Option::is_none")] pub email: Option, /// Which form they sat. #[serde(skip_serializing_if = "Option::is_none")] pub form: Option, /// Their score. pub score: Score, /// How they compare to the class, when comparison is on. #[serde(skip_serializing_if = "Option::is_none")] pub standing: Option, /// Per-level performance. pub levels: Vec, /// Per-objective standing, in the course's own order. /// /// Objectives only, never their targets. This is the table that makes a /// claim, and a claim needs a denominator: an objective's row aggregates /// every item tagged to any of its targets, while a target's row usually /// rests on one question and could only ever read "not enough questions to /// say". Mixing the two produced a three-page table where most rows carried /// that mark and the few real classifications were lost among them. The /// specifics live in the two sections built for them: which lectures to go /// back to, and the notes on missed questions. pub objectives: Vec, /// Objectives they are clearly meeting, worst first among the confident ones. pub strengths: Vec, /// Objectives to work on, worst first. pub focus: Vec, /// One row per question, with no question in it. #[serde(skip_serializing_if = "Vec::is_empty")] pub questions: Vec, /// Questions thrown out, in number order. /// /// Reported separately from the question rows so the document can say once, /// in prose, what happened to them. The two kinds need different sentences: /// a removed question is gone from the denominator, and a full-credit /// question is still in it. #[serde(skip_serializing_if = "Vec::is_empty")] pub dropped_questions: Vec, /// Which lectures to go back to, the one that would repay the most time /// first. #[serde(skip_serializing_if = "Vec::is_empty")] pub review_lectures: Vec, /// What to read, grouped by objective. #[serde(skip_serializing_if = "Vec::is_empty")] pub study: Vec, } /// A score, with the denominators spelled out. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub struct Score { /// Points earned on scored items. pub points: f64, /// Points available on scored items. pub points_possible: f64, /// Percentage on scored items. pub percent: f64, /// Bonus points earned. pub bonus_points: f64, /// Items answered correctly. pub correct: usize, /// Scored items administered. pub n_items: usize, } /// Where a score sits in the class. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub struct Standing { /// The class mean percentage. pub class_mean: f64, /// The standard deviation of class percentages. pub class_sd: f64, /// A coarse band, never a rank. pub band: String, /// The IRT ability estimate, when asked for. #[serde(skip_serializing_if = "Option::is_none")] pub theta: Option, /// Its standard error. #[serde(skip_serializing_if = "Option::is_none")] pub theta_se: Option, } /// One cognitive level. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub struct LevelRow { /// The level code, 1 through 5. pub level: u8, /// The level name. pub name: String, /// What that level asks of a student, in one phrase. pub blurb: String, /// How many items at this level. pub n_items: usize, /// The student's rate. pub rate: f64, /// The class rate, when comparison is on. #[serde(skip_serializing_if = "Option::is_none")] pub class_rate: Option, /// A plain-language comparison, when comparison is on. #[serde(skip_serializing_if = "Option::is_none")] pub comparison: Option, } /// One objective's standing. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub struct ObjectiveRow { /// The objective id. pub id: String, /// The objective text. pub text: String, /// The unit it belongs to, when the course declares one. #[serde(skip_serializing_if = "Option::is_none")] pub unit: Option, /// How many items measured it. pub n_items: usize, /// Credit earned across them. pub credit: f64, /// The observed rate. pub rate: f64, /// The lower bound of the 95% Wilson interval. pub lower: f64, /// The upper bound. pub upper: f64, /// The class rate, when comparison is on. #[serde(skip_serializing_if = "Option::is_none")] pub class_rate: Option, /// The mastery classification: `meeting`, `developing`, `not yet`, or /// `not enough evidence`. pub status: String, /// A compact symbol for the same thing. pub symbol: String, /// Whether the interval, not just the estimate, clears the threshold. pub confident: bool, /// Whether too few items measured it to classify at all. This is a fact about /// the exam, and a report that says so is being honest rather than vague. pub thin_evidence: bool, /// The levels it was assessed at, as codes. pub levels: Vec, } /// An objective named in a list. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub struct ObjectiveRef { /// The objective id. pub id: String, /// The objective text. pub text: String, /// The observed rate. pub rate: f64, /// How many items measured it. pub n_items: usize, } /// What one question measured, named at both tiers. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub struct Measured { /// The objective this question's result rolls up to. /// /// `None` when the tagged id is an objective with no targets of its own, so /// that a report does not print the same sentence twice. #[serde(skip_serializing_if = "Option::is_none")] pub objective: Option, /// The target the question was written against. pub target: String, } /// One question, described without being reproduced. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub struct QuestionRow { /// The recorded question number, which is what is printed on the paper. pub number: u32, /// Where it sat on this student's form, when the forms differ. #[serde(skip_serializing_if = "Option::is_none")] pub position: Option, /// The level code. #[serde(skip_serializing_if = "Option::is_none")] pub level: Option, /// The learning targets it measured, by id. pub targets: Vec, /// What this question measured, at both tiers. /// /// Both, because each answers a different question a student has in front of /// a missed item. The target says what this question actually asked of them, /// which is the specific thing to go and practise. The objective says which /// row of the table above the mark landed in, which is how they tell whether /// one slip cost them a claim or whether it was one of several. Printing the /// target alone left them unable to connect the note to the table; printing /// the objective alone described something broader than the question. #[serde(skip_serializing_if = "Vec::is_empty")] pub measured: Vec, /// Whether it was answered correctly. #[serde(skip_serializing_if = "Option::is_none")] pub correct: Option, /// Credit earned, as a fraction. pub credit: f64, /// Whether it was a bonus question. #[serde(skip_serializing_if = "std::ops::Not::not")] pub bonus: bool, /// Whether it was dropped from scoring after the fact. /// /// A dropped question still appears in the map, because the student has the /// paper in front of them and will look for it. What it must not do is /// appear as an error they made. #[serde(skip_serializing_if = "std::ops::Not::not")] pub dropped: bool, /// Whether the student left it blank. pub blank: bool, /// The share of the class that answered it correctly, when comparison is on. #[serde(skip_serializing_if = "Option::is_none")] pub class_rate: Option, /// The feedback written for the option this student chose. #[serde(skip_serializing_if = "Option::is_none")] pub feedback: Option, /// The hint written for that option: where to look, not what the answer was. #[serde(skip_serializing_if = "Option::is_none")] pub hint: Option, /// The misconception that option was written to catch, when /// [`Options::misconceptions`] is on. #[serde(skip_serializing_if = "Option::is_none")] pub misconception: Option, /// The worked solution, when [`Options::solutions`] is on. #[serde(skip_serializing_if = "Option::is_none")] pub worked: Option, /// Where the material was taught. #[serde(skip_serializing_if = "Vec::is_empty")] pub taught_in: Vec, /// What to read again about this question, from the item's own `review` /// citations rather than from the objective's reading list. #[serde(skip_serializing_if = "Vec::is_empty")] pub review: Vec, } /// One question thrown out after the exam. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub struct DroppedQuestion { /// The recorded question number. pub number: u32, /// Whether the drop was applied by crediting every option, in which case the /// question is still in the points of record. pub full_credit: bool, } /// One citation to read again after missing a question. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub struct ItemReading { /// A short citation, e.g. `KKW §2.1`. pub citation: String, /// The work's full title, for a student who does not recognise the label. #[serde(skip_serializing_if = "Option::is_none")] pub title: Option, /// A link, when the citation resolves to one. #[serde(skip_serializing_if = "Option::is_none")] pub url: Option, } /// One lecture worth going back to, with the evidence for saying so. /// /// Ranked by how many *objectives* went wrong rather than how many questions /// did. Missing four questions on one objective is one thing to relearn; missing /// four questions across four objectives is four, and the second is the lecture /// to reread first even though the arithmetic looks identical. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub struct LectureFocus { /// The lecture id, e.g. `L1.4`. pub lecture: String, /// Its title. pub title: String, /// Where the slides live, when the course records that. #[serde(skip_serializing_if = "Option::is_none")] pub url: Option, /// How many distinct learning targets from this lecture were missed. The /// ranking key. /// /// Counting targets rather than objectives keeps the ranking informative: a /// lecture where four separate performances went wrong needs more time than /// one where a single performance was missed twice, and counting objectives /// would score those the same. pub n_targets: usize, /// How many questions from this lecture were missed. pub n_questions: usize, /// Which questions, so a student can line this up with their paper. #[serde(skip_serializing_if = "Vec::is_empty")] pub questions: Vec, /// The slides those questions came from, when the items record them. #[serde(skip_serializing_if = "Vec::is_empty")] pub slides: Vec, /// The learning targets that went wrong here, in the course's own words. /// /// Targets rather than objectives, because this section answers "what do I /// go and restudy". "You missed the objective on binding" sends a student to /// a whole lecture; "you missed reading a dissociation constant off an /// isotherm" sends them to one page of it. #[serde(skip_serializing_if = "Vec::is_empty")] pub targets: Vec, } /// What to read about one objective. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub struct StudyGroup { /// The objective id. pub objective: String, /// The objective text. pub text: String, /// The observed rate, so the list is ordered by need. pub rate: f64, /// The readings. pub readings: Vec, } /// One reading to revisit. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub struct StudyReading { /// A short citation, e.g. `KKW §6.2`. pub citation: String, /// The lecture it was assigned for. pub lecture: String, /// That lecture's title. pub lecture_title: String, /// A link, when the reference resolves to one. #[serde(skip_serializing_if = "Option::is_none")] pub url: Option, /// What to take from it, which is the sentence worth quoting at someone who /// missed the objective. #[serde(skip_serializing_if = "Option::is_none")] pub focus: Option, /// What the section covers. #[serde(skip_serializing_if = "Option::is_none")] pub summary: Option, /// Whether it was assigned or offered alongside. pub supplemental: bool, } /// Builds one student's diagnostic. /// /// # Arguments /// /// * `summary` - the student's computed summary. /// * `cohort` - the class context. /// * `catalog` - the loaded course, for objective text, feedback, and readings. /// * `set` - the responses, for this student's per-question rows. /// * `analysis` - the item analysis, for class rates per question. Optional. /// * `opts` - what to include. /// /// # Returns /// /// The diagnostic. pub fn student( summary: &StudentSummary, cohort: &Cohort, catalog: &Catalog, set: &ResponseSet, analysis: Option<&Analysis>, opts: &Options, ) -> StudentDiagnostic { let course = &catalog.course; let rows = set.for_student(&summary.student_key); let form = rows.first().and_then(|r| r.form.clone()); // The summary carries the name; the email only ever existed on the response // rows, and both are already absent from a pseudonymized store, so neither // needs a flag here. let name = summary .name .clone() .or_else(|| rows.iter().find_map(|r| r.name.clone())); let email = rows.iter().find_map(|r| r.email.clone()); let levels = summary .levels .iter() .map(|profile| LevelRow { level: profile.level.code(), name: profile.level.name().to_string(), blurb: profile.level.blurb().to_string(), n_items: profile.n_items, rate: profile.rate, class_rate: opts.comparison.then_some(profile.cohort_rate), comparison: opts.comparison.then(|| profile.comparison().to_string()), }) .collect(); let objectives: Vec = summary .objectives .iter() // Objectives only; see `StudentDiagnostic::objectives`. .filter(|mastery| mastery.tier == Tier::Objective) .map(|mastery| ObjectiveRow { id: mastery.id.clone(), text: mastery.text.clone(), unit: course.objective_unit(&mastery.id).map(str::to_string), n_items: mastery.n_items, credit: mastery.credit, rate: mastery.rate, lower: mastery.wilson_lower, upper: mastery.wilson_upper, class_rate: opts.comparison.then_some(mastery.cohort_rate), status: mastery.status.label().to_string(), symbol: mastery.status.symbol().to_string(), confident: mastery.confident, thin_evidence: mastery.status == Mastery::NotEnoughEvidence, levels: mastery.levels.iter().map(|l| l.code()).collect(), }) .collect(); let refs = |ids: &[String]| -> Vec { ids.iter() .filter_map(|id| objectives.iter().find(|o| &o.id == id)) .map(|o| ObjectiveRef { id: o.id.clone(), text: o.text.clone(), rate: o.rate, n_items: o.n_items, }) .collect() }; let strengths = refs(&summary.strengths); let focus = refs(&summary.focus); let class_rates: BTreeMap = analysis .map(|a| a.items.iter().map(|i| (i.number, i.p_value)).collect()) .unwrap_or_default(); let questions = if opts.questions { rows.iter() .map(|row| question_row(row, catalog, &class_rates, opts)) .collect() } else { Vec::new() }; let mut dropped_questions: Vec = rows .iter() .filter(|r| r.dropped) .map(|r| DroppedQuestion { number: r.item_number, full_credit: r.dropped_full_credit, }) .collect(); dropped_questions.sort_by_key(|d| d.number); dropped_questions.dedup_by_key(|d| d.number); // Built from the response rows rather than from `questions`, so a report with // `--no-questions` still says where to go back to; it just does not name the // question numbers. let review_lectures = lecture_focus(catalog, &rows, opts); let study = focus .iter() .take(opts.focus_limit) .map(|objective| StudyGroup { objective: objective.id.clone(), text: objective.text.clone(), rate: objective.rate, readings: readings_for(course, &objective.id, opts.readings_per_objective), }) .filter(|group| !group.readings.is_empty()) .collect(); StudentDiagnostic { student_key: summary.student_key.clone(), name, sid: summary.sid.clone(), email, form, score: Score { points: summary.points, points_possible: summary.points_possible, percent: summary.percent, bonus_points: summary.bonus_points, correct: summary.correct, n_items: summary.n_items, }, standing: opts.comparison.then(|| Standing { class_mean: cohort.mean_percent, class_sd: cohort.sd_percent, band: summary.band.clone(), theta: opts.ability.then_some(summary.theta).flatten(), theta_se: opts.ability.then_some(summary.theta_se).flatten(), }), levels, objectives, strengths, focus, questions, dropped_questions, review_lectures, study, } } /// Builds one question's row. /// /// The feedback lookup uses [`Response::chosen`], which prefers the bank letters /// written at ingest. Falling back to the printed letters is right for an /// unshuffled form and wrong for a shuffled one, which is why ingest translates /// rather than leaving it to here. fn question_row( row: &Response, catalog: &Catalog, class_rates: &BTreeMap, opts: &Options, ) -> QuestionRow { let blank = row.selected.is_empty() && row.eliminated.is_empty(); // A dropped question cannot be missed. Without `counts()` here, a question // thrown out after the exam still collects per-option feedback explaining an // error the student is no longer being charged for. let missed = row.credit < 0.999 && row.counts(); let mut feedback = None; let mut hint = None; let mut misconception = None; let mut worked = None; let mut taught_in = Vec::new(); let mut review = Vec::new(); if let Some(uid) = row.item_ref.as_deref() { if let Some(entry) = catalog.get(uid) { if missed { if let Some(letter) = row.chosen().first() { if let Some(choice) = entry.item.option(letter) { if opts.feedback { // `student_text` falls back to `explanation`, which is // written for a grader and often names the right // answer. A student report must not print it. feedback = choice .feedback_student .clone() .or_else(|| choice.misconception.clone()); } if opts.hints { hint = choice.hint.clone(); } // When an option carries no student feedback, the // fallback above already printed this text. The same // sentence twice under two labels reads as a bug. if opts.misconceptions && choice.misconception != feedback { misconception = choice.misconception.clone(); } } } if let Some(solution) = entry.item.solution.as_ref() { if opts.solutions { worked = solution.explanation.clone(); } review = item_readings(&catalog.course, &solution.review); } } for source in &entry.item.sources { let title = catalog .course .lectures .get(&source.lecture) .map(|l| l.title.clone()) .unwrap_or_else(|| source.lecture.clone()); if source.slides.is_empty() { taught_in.push(format!("{} ({})", title, source.lecture)); } else { let slides: Vec = source.slides.iter().map(|s| s.to_string()).collect(); taught_in.push(format!( "{} ({}), slide{} {}", title, source.lecture, if source.slides.len() == 1 { "" } else { "s" }, slides.join(", ") )); } } } } QuestionRow { number: row.item_number, position: row.form_position.filter(|p| *p != row.item_number), level: row.level.map(|l| l.code()), targets: row.learning_targets.clone(), measured: if missed { let course = &catalog.course; row.learning_targets .iter() .map(|id| { let objective = course.objective_for(id); Measured { // An objective with no targets of its own is tagged // directly, and then the two tiers are the same row. // Saying it twice would read as an error, so the // objective is left out. objective: (objective != id).then(|| course.text_for(objective)), target: course.text_for(id), } }) .collect() } else { // Only where it earns its space. Every question already carries its // target ids, and a correct answer needs no explaining. Vec::new() }, correct: row.correct, credit: row.credit, bonus: row.bonus, dropped: row.dropped, blank, class_rate: opts .comparison .then(|| class_rates.get(&row.item_number).copied()) .flatten(), feedback, hint, misconception, worked, taught_in, review, } } /// Resolves an item's `review` citations against the course reference registry. /// /// # Arguments /// /// * `course` - the course, for its reference labels and base URLs. /// * `citations` - the item's citations. /// /// # Returns /// /// One entry per citation that resolves to something printable. fn item_readings(course: &CourseFile, citations: &[Citation]) -> Vec { let mut out = Vec::new(); for citation in citations { let reference = citation .reference .as_deref() .and_then(|key| course.references.get(key).map(|r| (key, r))); let (label, title, url) = match reference { Some((key, reference)) => ( reference.label_or(key).to_string(), Some(reference.title.clone()), citation.href(reference), ), None => (citation.display(), None, citation.url.clone()), }; if label.is_empty() { continue; } let citation_text = match (&citation.text, &citation.locator) { (Some(text), _) => text.clone(), (None, Some(locator)) => format!("{label} {locator}"), (None, None) => label, }; out.push(ItemReading { citation: citation_text, title, url, }); } out } /// Ranks the lectures behind a student's missed questions. /// /// A lecture earns its place by how many distinct objectives went wrong in it, /// then by how many questions, then by id so the order is stable between runs. /// /// # Arguments /// /// * `catalog` - the loaded course, for objective and lecture titles. /// * `rows` - this student's responses. /// * `opts` - what to include; `questions` decides whether numbers are named. /// /// # Returns /// /// The lectures, the one that would repay the most time first. fn lecture_focus(catalog: &Catalog, rows: &[&Response], opts: &Options) -> Vec { /// What has accumulated for one lecture so far. #[derive(Default)] struct Tally { targets: BTreeSet, questions: BTreeSet, slides: BTreeSet, } let course = &catalog.course; let mut tallies: BTreeMap = BTreeMap::new(); for row in rows.iter().filter(|r| r.counts() && r.credit < 0.999) { // Two routes to a lecture, and both are wanted. The registry knows // which lectures develop a target; the item knows which lecture it was // written from, which is the finer answer when a target spans several. let mut lectures: BTreeSet = BTreeSet::new(); let mut slides: BTreeMap> = BTreeMap::new(); if let Some(entry) = row.item_ref.as_deref().and_then(|uid| catalog.get(uid)) { for source in &entry.item.sources { lectures.insert(source.lecture.clone()); slides .entry(source.lecture.clone()) .or_default() .extend(source.slides.iter().copied()); } } for target in &row.learning_targets { lectures.extend(course.lectures_for(target).iter().cloned()); } for lecture in lectures { let tally = tallies.entry(lecture.clone()).or_default(); tally.targets.extend(row.learning_targets.clone()); tally.questions.insert(row.item_number); if let Some(numbers) = slides.get(&lecture) { tally.slides.extend(numbers.iter().copied()); } } } let mut out: Vec = tallies .into_iter() .map(|(lecture, tally)| { let record = course.lectures.get(&lecture); LectureFocus { title: record .map(|l| l.title.clone()) .unwrap_or_else(|| lecture.clone()), url: record.and_then(|l| l.slides_url.clone()), n_targets: tally.targets.len(), n_questions: tally.questions.len(), questions: if opts.questions { tally.questions.iter().copied().collect() } else { Vec::new() }, slides: tally.slides.iter().copied().collect(), targets: tally.targets.iter().map(|id| course.text_for(id)).collect(), lecture, } }) .collect(); out.sort_by(|a, b| { b.n_targets .cmp(&a.n_targets) .then(b.n_questions.cmp(&a.n_questions)) .then(a.lecture.cmp(&b.lecture)) }); out } /// Resolves an objective to readings. /// /// # Arguments /// /// * `course` - the course registry. /// * `objective` - the objective id. /// * `limit` - how many readings to keep. /// /// # Returns /// /// The readings, assigned ones first. fn readings_for(course: &CourseFile, objective: &str, limit: usize) -> Vec { let mut out = Vec::new(); for (lecture_id, reading) in course.readings_for_objective(objective) { let lecture_title = course .lectures .get(lecture_id) .map(|l| l.title.clone()) .unwrap_or_else(|| lecture_id.to_string()); let (citation, url) = match reading.reference.as_deref() { Some(key) => match course.references.get(key) { Some(reference) => (reading.cite(key, reference), reading.resolve_url(reference)), None => ( reading .text .clone() .or_else(|| reading.locator.clone()) .unwrap_or_else(|| key.to_string()), reading.url.clone(), ), }, None => ( reading .text .clone() .or_else(|| reading.locator.clone()) .unwrap_or_else(|| lecture_title.clone()), reading.url.clone(), ), }; out.push(StudyReading { citation, lecture: lecture_id.to_string(), lecture_title, url, focus: reading.focus.clone(), summary: reading.summary.clone(), supplemental: reading.role == ReadingRole::Supplemental, }); } // Assigned before supplemental, otherwise the order the course declares. out.sort_by_key(|r| r.supplemental); out.truncate(limit); out } /// Everything the class diagnostic says. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub struct CohortDiagnostic { /// How many students sat it. pub n_students: usize, /// How many scored items. pub n_items: usize, /// The score distribution. pub distribution: Distribution, /// Whole-test reliability. pub reliability: ReliabilityRow, /// Per-level class performance. pub levels: Vec, /// Per-objective class performance, worst first. pub objectives: Vec, /// Objectives the class as a whole did not meet. pub gaps: Vec, /// The distribution binned by the course's letter-grade scale. Empty when /// `course.yaml` sets no scale, in which case the ten-point bins stand. #[serde(skip_serializing_if = "Vec::is_empty")] pub grades: Vec, /// Per-lecture class performance, worst first. #[serde(skip_serializing_if = "Vec::is_empty")] pub lectures: Vec, /// Per-question statistics. pub questions: Vec, /// Questions thrown out, which are therefore absent from every table above. /// Recorded so the report says why rather than leaving a gap in the /// numbering. #[serde(skip_serializing_if = "Vec::is_empty")] pub dropped_questions: Vec, /// What to do about each question that raised something. pub triage: Triage, /// How the authored expectations did. pub predictions: PredictionSummary, /// Questions worth revisiting before reuse, worst first. pub revise: Vec, /// The dropped items, described but not measured. /// /// Kept out of [`CohortDiagnostic::questions`] so that no statistic above /// silently includes an item that was thrown out, and reported alongside it /// in the evidence section so that dropping a question does not erase the /// evidence for having dropped it. pub dropped_detail: Vec, /// One row per form, when more than one was given. #[serde(skip_serializing_if = "Vec::is_empty")] pub forms: Vec, /// Where the form built differs from the blueprint it was drawn against. #[serde(skip_serializing_if = "Vec::is_empty")] pub blueprint: Vec, /// Response profiles the class falls into. #[serde(skip_serializing_if = "Vec::is_empty")] pub patterns: Vec, /// Cautions about the analysis itself. #[serde(skip_serializing_if = "Vec::is_empty")] pub warnings: Vec, } /// The score distribution, binned. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub struct Distribution { /// Mean percentage. pub mean: f64, /// Median percentage. pub median: f64, /// Standard deviation. pub sd: f64, /// Lowest percentage. pub min: f64, /// Highest percentage. pub max: f64, /// Counts in ten-point bins, from 0-9 through 90-100. pub bins: Vec, } /// One histogram bin. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub struct Bin { /// Inclusive lower bound, in percent. pub low: u32, /// Exclusive upper bound, in percent, except the last bin which includes 100. pub high: u32, /// How many students fell in it. pub count: usize, } /// Reliability, flattened for a template. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub struct ReliabilityRow { /// KR-20, when it could be computed. #[serde(skip_serializing_if = "Option::is_none")] pub alpha: Option, /// The standard error of measurement, in items. #[serde(skip_serializing_if = "Option::is_none")] pub sem: Option, /// Mean p-value across items. pub mean_p: f64, /// Mean point-biserial across items that had one. #[serde(skip_serializing_if = "Option::is_none")] pub mean_point_biserial: Option, /// What the alpha value means for a test this length in a class this size. pub interpretation: String, } /// One level, class-wide. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub struct CohortLevelRow { /// The level code. pub level: u8, /// The level name. pub name: String, /// How many items sat at this level. pub n_items: usize, /// The class rate. pub rate: f64, } /// One objective, class-wide. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub struct CohortObjectiveRow { /// The objective id. pub id: String, /// The objective text. pub text: String, /// How many items measured it. pub n_items: usize, /// The class rate. pub rate: f64, /// How many students met it. pub meeting: usize, /// How many students are developing on it. pub developing: usize, /// How many students are not yet meeting it. pub not_yet: usize, /// How many had too few items to classify. pub thin: usize, /// Whether the class rate is below the course's mastery threshold. pub below_threshold: bool, } /// One question, class-wide. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub struct CohortQuestionRow { /// The recorded question number. pub number: u32, /// The item's global id. #[serde(skip_serializing_if = "Option::is_none")] pub item: Option, /// The level code. #[serde(skip_serializing_if = "Option::is_none")] pub level: Option, /// The learning targets it measured, by id. pub targets: Vec, /// What those targets ask, in the course's own words. #[serde(skip_serializing_if = "Vec::is_empty")] pub target_texts: Vec, /// Whether the item was dropped from scoring. /// /// A dropped item carries no p, r, or D, and appears in none of the /// statistics above. It still appears in the evidence section, because the /// option spread that justified dropping it is the record of why, and that /// record should survive re-running the report afterwards. #[serde(skip_serializing_if = "std::ops::Not::not")] pub dropped: bool, /// Whether the drop was applied as full credit to everyone. #[serde(skip_serializing_if = "std::ops::Not::not")] pub dropped_full_credit: bool, /// The question as written. /// /// The instructor report reads better with it than without: a row of option /// shares says a distractor drew 44% of the class, and only the stem says /// whether that is a second defensible reading. Absent when the item has /// left the bank, and withheld from any report that is not the instructor /// copy. #[serde(skip_serializing_if = "Option::is_none")] pub stem: Option, /// Where the item was taught, as lecture titles and slide numbers. #[serde(skip_serializing_if = "Vec::is_empty")] pub taught_in: Vec, /// The lecture ids alone, for a table column where only `L1.4` fits. #[serde(skip_serializing_if = "Vec::is_empty")] pub lectures: Vec, /// Which difficulty band it fell in: `too easy`, `moderate`, or `hard`. pub difficulty_band: String, /// Which discrimination band it fell in, on the conventional cut points: /// `excellent`, `good`, `marginal`, `poor`, or `negative`. pub discrimination_band: String, /// Proportion correct. pub p_value: f64, /// Corrected item-total point-biserial. #[serde(skip_serializing_if = "Option::is_none")] pub point_biserial: Option, /// Upper minus lower group proportion correct. #[serde(skip_serializing_if = "Option::is_none")] pub discrimination: Option, /// Fraction who left it blank. pub blank_rate: f64, /// The keyed letters. pub key: Vec, /// Per-option selection, in letter order. pub options: Vec, /// Machine-detected problems. pub flags: Vec, /// What those flags mean. pub notes: Vec, /// How the item did against its author's expectation. Separate from `notes` /// because an unmet prediction on an uncalibrated item is a fact about the /// prediction. #[serde(skip_serializing_if = "Vec::is_empty")] pub prediction_notes: Vec, /// Whether that expectation rested on a prior calibration. pub calibrated: bool, /// Per-form proportion correct, when more than one form was given. A gap here /// on one question, with the rest of the exam in step, points at that /// question's permutation rather than at the cohort. #[serde(skip_serializing_if = "BTreeMap::is_empty")] pub by_form: BTreeMap, } /// One option's selection statistics. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub struct OptionRow { /// The bank letter, which is the one every statistic is keyed by. pub letter: String, /// The option as written, for a report that shows the question. /// /// Absent when the item is no longer in the bank, which is why this is an /// option rather than an empty string: a missing option and an empty one /// are different facts. #[serde(skip_serializing_if = "Option::is_none")] pub text: Option, /// What this option was lettered on each printed form, worst case one entry /// per form. /// /// Shuffling means the bank's option C is a different letter on every form, /// so a statistic reported against C cannot be checked against a student's /// paper without this map. It is the first thing anyone needs when a student /// brings a paper to office hours, and working it out by hand from a seal is /// the kind of task that gets done wrong once and then trusted. #[serde(skip_serializing_if = "Vec::is_empty")] pub printed: Vec, /// How many chose it. pub count: usize, /// The share who chose it. pub rate: f64, /// Whether it is keyed. pub is_key: bool, /// Correlation between choosing it and scoring well elsewhere. #[serde(skip_serializing_if = "Option::is_none")] pub point_biserial: Option, /// Whether it drew nobody, and is therefore doing no work. pub nonfunctioning: bool, } /// What one bank option was lettered on one form. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub struct PrintedLetter { /// The form id. pub form: String, /// The letter this option carried on that form's paper. pub letter: String, } /// One form's summary. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub struct FormRow { /// The form id. pub id: String, /// How many students sat it. pub n_students: usize, /// Their mean percentage. pub mean: f64, /// The standard deviation of their percentages. pub sd: f64, } /// One response profile. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub struct PatternRow { /// A label describing the pattern. pub label: String, /// How many students fit it. pub n_students: usize, /// Mean rate at each level, by level code. pub level_means: BTreeMap, } /// The class's scores binned by the course's own letter-grade scale. /// /// A ten-point histogram is the default because it needs no course /// configuration, but nobody acts on "nineteen students in the fifties". They /// act on "nineteen students are failing", and that sentence needs the scale /// from `course.yaml`. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub struct GradeRow { /// The letter. pub letter: String, /// The lowest percentage in the band. pub low: f64, /// The highest percentage in the band, which is just under the next band's /// floor, or 100 for the top band. pub high: f64, /// Grade points, when the scale records them. #[serde(skip_serializing_if = "Option::is_none")] pub gpa: Option, /// The attainment word, when the scale records one. #[serde(skip_serializing_if = "Option::is_none")] pub attainment: Option, /// The colour group, so A, A- and A+ can be tinted together. pub group: String, /// How many students landed in the band. pub count: usize, /// Their share of the class, in `0.0..=1.0`. pub share: f64, /// How many students are in this band or a higher one. pub at_or_above: usize, } /// One lecture's showing, aggregated from the items written against it. /// /// The objective table answers "which objective went wrong". This answers "which /// class meeting went wrong", which is the question that maps onto next week. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub struct CohortLectureRow { /// The lecture id. pub lecture: String, /// Its title. pub title: String, /// How many scored items traced back to it. pub n_items: usize, /// How many distinct objectives those items measured. pub n_objectives: usize, /// How many of those objectives the class did not meet. pub n_objectives_below: usize, /// Mean proportion correct across its items. pub rate: f64, /// The questions, so the row can be checked against the item table. pub questions: Vec, /// The worst objective under this lecture, by class rate. #[serde(skip_serializing_if = "Option::is_none")] pub worst_objective: Option, } /// What to do about one question, and why. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub struct TriageRow { /// The question number. pub number: u32, /// The item id. #[serde(skip_serializing_if = "Option::is_none")] pub item: Option, /// The level code. #[serde(skip_serializing_if = "Option::is_none")] pub level: Option, /// Proportion correct. pub p_value: f64, /// Corrected item-total correlation. #[serde(skip_serializing_if = "Option::is_none")] pub point_biserial: Option, /// Upper minus lower group. #[serde(skip_serializing_if = "Option::is_none")] pub discrimination: Option, /// What the question measured, in the course's words: its learning targets. #[serde(skip_serializing_if = "Vec::is_empty")] pub targets: Vec, /// Where it was taught. #[serde(skip_serializing_if = "Vec::is_empty")] pub taught_in: Vec, /// The specific option this recommendation is about, when it is about one. #[serde(skip_serializing_if = "Option::is_none")] pub option: Option, /// That option's share of responses. #[serde(skip_serializing_if = "Option::is_none")] pub option_share: Option, /// That option's correlation with total score. #[serde(skip_serializing_if = "Option::is_none")] pub option_point_biserial: Option, /// The evidence, one clause per line. pub reasons: Vec, } /// Every question sorted into what to do with it. /// /// The first four lists are decisions about items and are mutually exclusive: a /// question appears in the most severe one that fits, because there is no point /// rewriting a distractor on an item you are about to discard. `reteach` is not /// a decision about an item at all, so a question can appear there as well as in /// one of the others. #[derive(Debug, Clone, Default, Serialize)] #[serde(rename_all = "kebab-case")] pub struct Triage { /// Broken: the evidence says these did not measure what they were scored on. pub discard: Vec, /// A second defensible answer with statistical support behind it. pub rekey: Vec, /// Weak but salvageable, worth rewriting before reuse. pub revise: Vec, /// Sound items the class got wrong. A teaching finding, not an item finding. pub reteach: Vec, /// Items whose low discrimination is explained by their difficulty rather /// than by a fault. Listed so they are not mistaken for work to do. pub bounded: Vec, /// How many questions raised nothing at all. pub clean: usize, } /// How the authored expectations did against the data. /// /// This exists so that an uncalibrated bank does not produce one /// `design_mismatch` per item. Before an item has data, its expected difficulty /// is a prediction by its author, and the useful summary is whether those /// predictions run optimistic or pessimistic as a set. #[derive(Debug, Clone, Default, Serialize)] #[serde(rename_all = "kebab-case")] pub struct PredictionSummary { /// How many items recorded an expected difficulty. pub n_predicted: usize, /// How many of those expectations rest on a prior calibration. pub n_calibrated: usize, /// Mean of observed minus expected difficulty. Positive means the items came /// out easier than predicted. #[serde(skip_serializing_if = "Option::is_none")] pub mean_signed_error: Option, /// Mean absolute difficulty error, which is the size of a typical miss. #[serde(skip_serializing_if = "Option::is_none")] pub mean_abs_error: Option, /// How many landed inside the tolerance. pub n_within: usize, /// How many items recorded an expected discrimination band. pub n_band: usize, /// How many of those landed inside it. pub n_band_hit: usize, /// The largest single surprise, as `(question, expected, observed)`. #[serde(skip_serializing_if = "Option::is_none")] pub biggest_surprise: Option<(u32, f64, f64)>, } /// Builds the class diagnostic. /// /// # Arguments /// /// * `analysis` - the classical item analysis. /// * `cohort` - the per-student summaries and class rates. /// * `catalog` - the loaded course. /// * `record` - the assessment record, for the blueprint check. /// * `set` - the responses, for per-form and per-option breakdowns. /// * `fit` - an IRT fit, when one was computed. /// /// # Returns /// /// The diagnostic. pub fn cohort( analysis: &Analysis, cohort: &Cohort, catalog: &Catalog, record: &AssessmentFile, set: &ResponseSet, fit: Option<&Fit>, ) -> CohortDiagnostic { let _ = fit; let course = &catalog.course; let threshold = course.policy.mastery_threshold; let percents: Vec = cohort.students.iter().map(|s| s.percent).collect(); let level_counts: BTreeMap = { let mut counts: BTreeMap> = BTreeMap::new(); for row in set.rows.iter().filter(|r| r.counts()) { if let Some(level) = row.level { counts.entry(level).or_default().insert(row.item_number); } } counts.into_iter().map(|(k, v)| (k, v.len())).collect() }; let levels = Level::ALL .iter() .filter_map(|level| { let rate = cohort.level_rates.get(level).copied()?; Some(CohortLevelRow { level: level.code(), name: level.name().to_string(), n_items: level_counts.get(level).copied().unwrap_or(0), rate, }) }) .collect(); // Objective counts come from the responses so that an objective assessed by // two items is not reported as if it had one. Counted per objective, and by // item number, so a question tagged with two of that objective's targets is // one item here. let mut objective_items: BTreeMap> = BTreeMap::new(); for row in set.rows.iter().filter(|r| r.counts()) { for objective in &row.learning_targets { objective_items .entry(course.objective_for(objective).to_string()) .or_default() .insert(row.item_number); } } // Objectives, not targets: this table is the reteaching queue, and it is // only usable if it is short enough to read and each row rests on enough // items to believe. let mut objectives: Vec = cohort .objective_rates .iter() .map(|(id, rate)| { let mut meeting = 0; let mut developing = 0; let mut not_yet = 0; let mut thin = 0; for student in &cohort.students { if let Some(row) = student.objectives.iter().find(|o| &o.id == id) { match row.status { Mastery::Meeting => meeting += 1, Mastery::Developing => developing += 1, Mastery::NotYet => not_yet += 1, Mastery::NotEnoughEvidence => thin += 1, } } } CohortObjectiveRow { id: id.clone(), text: course.text_for(id), n_items: objective_items.get(id).map(|s| s.len()).unwrap_or(0), rate: *rate, meeting, developing, not_yet, thin, below_threshold: *rate < threshold, } }) .collect(); objectives.sort_by(|a, b| { a.rate .partial_cmp(&b.rate) .unwrap_or(std::cmp::Ordering::Equal) .then_with(|| a.id.cmp(&b.id)) }); let gaps: Vec = objectives .iter() .filter(|o| o.below_threshold) .cloned() .collect(); let by_form = per_form_p_values(set); let item_meta: BTreeMap, Vec)> = record .items .iter() .map(|p| { ( p.number, (p.level.map(|l| l.code()), p.learning_targets.clone()), ) }) .collect(); // The printed lettering per form, computed from the same two functions the // exporter and the seal use, so the letters here are the letters on the // paper rather than a second guess at them. let forms: Vec<&crate::assessment::Form> = record.forms.iter().collect(); let questions: Vec = analysis .items .iter() .map(|item| { let meta = item_meta.get(&item.number); let entry = item.item_ref.as_deref().and_then(|uid| catalog.get(uid)); CohortQuestionRow { number: item.number, item: item.item_ref.clone(), level: meta.and_then(|m| m.0), targets: meta.map(|m| m.1.clone()).unwrap_or_default(), target_texts: meta .map(|m| m.1.iter().map(|id| course.text_for(id)).collect()) .unwrap_or_default(), taught_in: item .item_ref .as_deref() .map(|uid| taught_in(catalog, uid)) .unwrap_or_default(), lectures: item .item_ref .as_deref() .and_then(|uid| catalog.get(uid)) .map(|entry| { entry .item .sources .iter() .map(|source| source.lecture.clone()) .collect() }) .unwrap_or_default(), dropped: false, dropped_full_credit: false, stem: entry.map(|e| e.item.stem.clone()), difficulty_band: difficulty_band(item.p_value).to_string(), discrimination_band: discrimination_band(item.point_biserial).to_string(), p_value: item.p_value, point_biserial: item.point_biserial, discrimination: item.discrimination_index, blank_rate: item.blank_rate, key: item.key.clone(), options: item .options .values() .map(|option| OptionRow { text: entry.and_then(|e| { e.item .options .iter() .find(|o| o.id == option.letter) .map(|o| o.text.clone()) }), printed: entry .map(|e| { printed_letters( &forms, &e.item, item.item_ref.as_deref(), &option.letter, ) }) .unwrap_or_default(), letter: option.letter.clone(), count: option.count, rate: option.rate, is_key: option.is_key, point_biserial: option.point_biserial, nonfunctioning: !option.is_key && option.rate <= 0.05, }) .collect(), flags: item.flags.iter().map(|f| f.as_str().to_string()).collect(), notes: item.notes.clone(), prediction_notes: item .prediction .as_ref() .map(|p| p.notes.clone()) .unwrap_or_default(), calibrated: item.prediction.as_ref().is_some_and(|p| p.calibrated), by_form: by_form.get(&item.number).cloned().unwrap_or_default(), } }) .collect(); let revise: Vec = analysis .revise_queue() .iter() .filter_map(|item| questions.iter().find(|q| q.number == item.number).cloned()) .collect(); let mut dropped_questions: Vec = set .all_items() .into_iter() .filter(|n| set.item_dropped(*n)) .map(|number| DroppedQuestion { number, full_credit: set.for_item(number).iter().any(|r| r.dropped_full_credit), }) .collect(); dropped_questions.sort_by_key(|d| d.number); // The dropped items, described from the responses rather than from the // scoring. Dropping an item overrides its credit, so p, r, and D are // meaningless for it and are left out. What students actually marked is // untouched by the drop, and that spread is the evidence that justified it. let mut dropped_detail: Vec = dropped_questions .iter() .map(|dropped| { let placement = record.placement(dropped.number); let uid = placement.map(|p| p.item.clone()); let entry = uid.as_deref().and_then(|uid| catalog.get(uid)); let responses = set.for_item(dropped.number); let answered = responses .iter() .filter(|r| !r.chosen().is_empty()) .count() .max(1); let keyed: BTreeSet = placement .map(|p| p.key.iter().cloned().collect()) .filter(|k: &BTreeSet| !k.is_empty()) .or_else(|| entry.map(|e| e.item.key_letters().into_iter().collect())) .unwrap_or_default(); // One row per option the item declares, so an option nobody // marked still shows as unchosen rather than vanishing. let options: Vec = entry .map(|e| { e.item .options .iter() .map(|option| { let count = responses .iter() .filter(|r| r.chosen().contains(&option.id)) .count(); OptionRow { text: Some(option.text.clone()), printed: printed_letters( &forms, &e.item, uid.as_deref(), &option.id, ), letter: option.id.clone(), count, rate: count as f64 / answered as f64, is_key: keyed.contains(&option.id), point_biserial: None, nonfunctioning: false, } }) .collect() }) .unwrap_or_default(); let targets = placement .map(|p| p.learning_targets.clone()) .unwrap_or_default(); CohortQuestionRow { number: dropped.number, item: uid.clone(), level: placement.and_then(|p| p.level).map(|l| l.code()), target_texts: targets.iter().map(|id| course.text_for(id)).collect(), targets, dropped: true, dropped_full_credit: dropped.full_credit, stem: entry.map(|e| e.item.stem.clone()), taught_in: uid .as_deref() .map(|uid| taught_in(catalog, uid)) .unwrap_or_default(), lectures: entry .map(|e| e.item.sources.iter().map(|s| s.lecture.clone()).collect()) .unwrap_or_default(), difficulty_band: String::new(), discrimination_band: String::new(), // The keyed share before the override, which is the closest // honest reading of how the item performed. It is not a // p-value: it counts marks, not credit. p_value: options .iter() .filter(|o| o.is_key) .map(|o| o.rate) .sum::() .min(1.0), point_biserial: None, discrimination: None, blank_rate: responses.iter().filter(|r| r.chosen().is_empty()).count() as f64 / responses.len().max(1) as f64, key: keyed.iter().cloned().collect(), options, flags: Vec::new(), notes: Vec::new(), prediction_notes: Vec::new(), calibrated: false, by_form: BTreeMap::new(), } }) .collect(); dropped_detail.sort_by_key(|q| q.number); let default_options = course.policy.options_per_item; let triage = triage(&questions, threshold, default_options); let predictions = prediction_summary(analysis); let grades = grade_rows(&course.policy, &percents); let lectures = lecture_rows(catalog, &questions, &objectives, threshold); CohortDiagnostic { n_students: cohort.students.len(), n_items: analysis.reliability.n_items, distribution: distribution(&percents), grades, lectures, dropped_questions, triage, predictions, reliability: ReliabilityRow { alpha: analysis.reliability.alpha, sem: analysis.reliability.sem, mean_p: analysis.reliability.mean_p, mean_point_biserial: analysis.reliability.mean_point_biserial, interpretation: analysis.reliability.interpretation(), }, levels, objectives, gaps, questions, revise, dropped_detail, forms: form_rows(set, cohort), blueprint: crate::select::check_blueprint(record, course), patterns: cohort .archetypes .iter() .map(|a| PatternRow { label: a.label.clone(), n_students: a.members.len(), level_means: a .level_means .iter() .map(|(level, mean)| (level.code(), *mean)) .collect(), }) .collect(), warnings: analysis.warnings.clone(), } } /// Where an item was taught, as lecture titles with slide numbers. /// /// # Arguments /// /// * `catalog` - the loaded course. /// * `uid` - the item's global id. /// /// # Returns /// /// One entry per source the item records. fn taught_in(catalog: &Catalog, uid: &str) -> Vec { let Some(entry) = catalog.get(uid) else { return Vec::new(); }; entry .item .sources .iter() .map(|source| { let title = catalog .course .lectures .get(&source.lecture) .map(|l| l.title.clone()) .unwrap_or_else(|| source.lecture.clone()); if source.slides.is_empty() { format!("{} ({})", title, source.lecture) } else { let slides: Vec = source.slides.iter().map(|s| s.to_string()).collect(); format!( "{} ({}), slide{} {}", title, source.lecture, if source.slides.len() == 1 { "" } else { "s" }, slides.join(", ") ) } }) .collect() } /// Where one bank option landed on each printed form. /// /// # Arguments /// /// * `forms` - the record's forms, in declaration order. /// * `item` - the bank item, for its option count and lettering. /// * `uid` - the item's global id, which salts the permutation. /// * `letter` - the bank letter to locate. /// /// # Returns /// /// One entry per form that permutes its options. Forms printing the bank order /// unchanged are left out, since an entry saying C was printed as C is noise on /// every row. fn printed_letters( forms: &[&crate::assessment::Form], item: &crate::item::Item, uid: Option<&str>, letter: &str, ) -> Vec { let Some(uid) = uid else { return Vec::new(); }; let Some(source) = item.options.iter().position(|o| o.id == letter) else { return Vec::new(); }; let n = item.options.len(); let mut out = Vec::new(); for form in forms { if !form.shuffle_options { continue; } // The same permutation the exporter and the seal use, so these are the // letters on the paper rather than a second guess at them. let order = crate::select::option_order(form, uid, n); // `order[position] == source` means the option printed in that slot is // the one being asked about. if let Some(position) = order.iter().position(|index| *index == source) { out.push(PrintedLetter { form: form.id.clone(), letter: crate::seal::printed_letter(position), }); } } out } /// The difficulty band a p-value falls in. /// /// Three bands rather than five. The only distinction that changes what you do /// is whether the item had room to discriminate at all, and that is a question /// about the middle versus the two ends. fn difficulty_band(p: f64) -> &'static str { if p >= 0.85 { "too easy" } else if p <= 0.35 { "hard" } else { "moderate" } } /// The discrimination band a point-biserial falls in. /// /// The cut points are the conventional ones from the item-analysis literature, /// usually attributed to Ebel: about 0.40 and above is excellent, 0.30 to 0.39 /// good, 0.20 to 0.29 marginal, and below 0.20 poor. They are rules of thumb /// rather than laws, and they must be read next to difficulty, because an item /// almost everyone passes or fails has little variance left to correlate with /// anything. fn discrimination_band(r: Option) -> &'static str { match r { None => "no variance", Some(r) if r < 0.0 => "negative", Some(r) if r < 0.20 => "poor", Some(r) if r < 0.30 => "marginal", Some(r) if r < 0.40 => "good", Some(_) => "excellent", } } /// Bins the class by the course's letter-grade scale. /// /// # Arguments /// /// * `policy` - the course policy, for its scale. /// * `percents` - one score per student, out of 100. /// /// # Returns /// /// One row per band, highest first. Empty when the course sets no scale, which /// is the signal for a report to fall back to ten-point bins. fn grade_rows(policy: &crate::course::Policy, percents: &[f64]) -> Vec { let bands = policy.bands(); if bands.is_empty() || percents.is_empty() { return Vec::new(); } let n = percents.len() as f64; let mut out: Vec = Vec::with_capacity(bands.len()); let mut running = 0usize; for (index, band) in bands.iter().enumerate() { // The ceiling is the floor of the band above, less the smallest step a // percentage is reported at, so the printed range reads the way a // syllabus writes it. let high = match index { 0 => 100.0, _ => bands[index - 1].min - 0.1, }; let count = percents .iter() .filter(|percent| { **percent + 1e-9 >= band.min && (index == 0 || **percent < bands[index - 1].min) }) .count(); running += count; out.push(GradeRow { letter: band.letter.clone(), low: band.min, high, gpa: band.gpa, attainment: band.attainment.clone(), group: band.group_key(), count, share: count as f64 / n, at_or_above: running, }); } out } /// Aggregates questions into per-lecture rows, worst first. /// /// # Arguments /// /// * `catalog` - the loaded course, for lecture titles and objective lectures. /// * `questions` - the per-question rows. /// * `objectives` - the per-objective rows, for the objective counts. /// * `threshold` - the mastery threshold. /// /// # Returns /// /// One row per lecture that any scored item traced back to. fn lecture_rows( catalog: &Catalog, questions: &[CohortQuestionRow], objectives: &[CohortObjectiveRow], threshold: f64, ) -> Vec { let course = &catalog.course; let mut items: BTreeMap> = BTreeMap::new(); // Keyed by objective, not by the target an item was tagged with: the rows // this is matched against are objective rows, so collecting target ids here // left every lookup empty and every count zero. let mut lecture_objectives: BTreeMap> = BTreeMap::new(); for question in questions { // The same two routes the student report uses: the item knows which // lecture it was written from, and the registry knows which lectures // develop the target. let mut lectures: BTreeSet = BTreeSet::new(); if let Some(entry) = question.item.as_deref().and_then(|uid| catalog.get(uid)) { for source in &entry.item.sources { lectures.insert(source.lecture.clone()); } } for target in &question.targets { lectures.extend(course.lectures_for(target).iter().cloned()); } for lecture in lectures { items.entry(lecture.clone()).or_default().push(question); lecture_objectives.entry(lecture).or_default().extend( question .targets .iter() .map(|t| course.objective_for(t).to_string()), ); } } let mut out: Vec = items .into_iter() .map(|(lecture, rows)| { let rate = rows.iter().map(|r| r.p_value).sum::() / rows.len() as f64; let ids = lecture_objectives .get(&lecture) .cloned() .unwrap_or_default(); let mine: Vec<&CohortObjectiveRow> = objectives.iter().filter(|o| ids.contains(&o.id)).collect(); CohortLectureRow { title: course .lectures .get(&lecture) .map(|l| l.title.clone()) .unwrap_or_else(|| lecture.clone()), n_items: rows.len(), n_objectives: ids.len(), n_objectives_below: mine.iter().filter(|o| o.rate < threshold).count(), rate, questions: rows.iter().map(|r| r.number).collect(), // `objectives` arrives sorted worst first, so the first match is // the weakest one under this lecture. worst_objective: mine.first().map(|o| o.text.clone()), lecture, } }) .collect(); out.sort_by(|a, b| { a.rate .partial_cmp(&b.rate) .unwrap_or(std::cmp::Ordering::Equal) .then_with(|| a.lecture.cmp(&b.lecture)) }); out } /// Summarizes how the authored expectations did. fn prediction_summary(analysis: &Analysis) -> PredictionSummary { let mut out = PredictionSummary::default(); let mut signed: Vec = Vec::new(); let mut biggest: Option<(u32, f64, f64)> = None; for item in &analysis.items { let Some(prediction) = &item.prediction else { continue; }; if prediction.calibrated { out.n_calibrated += 1; } if let (Some(expected), Some(error)) = (prediction.expected_p, prediction.p_error) { out.n_predicted += 1; signed.push(error); if prediction.p_within == Some(true) { out.n_within += 1; } if biggest .map(|(_, e, o)| (o - e).abs() < error.abs()) .unwrap_or(true) { biggest = Some((item.number, expected, item.p_value)); } } if prediction.expected_band.is_some() { out.n_band += 1; if prediction.band_hit == Some(true) { out.n_band_hit += 1; } } } if !signed.is_empty() { let n = signed.len() as f64; out.mean_signed_error = Some(signed.iter().sum::() / n); out.mean_abs_error = Some(signed.iter().map(|e| e.abs()).sum::() / n); } out.biggest_surprise = biggest; out } /// Sorts every question into what to do about it. /// /// The order of the tests is the order of severity, and the first match wins for /// the three item decisions. `reteach` is judged separately, because "the item /// worked and the class missed it" is not a competing diagnosis; it is a /// different kind of finding. /// /// # Arguments /// /// * `questions` - the per-question rows. /// * `threshold` - the mastery threshold, which sets what counts as a content /// gap worth reteaching. /// * `default_options` - the course's default option count, used for the chance /// rate when an item's own options cannot be counted. /// /// # Returns /// /// The buckets. fn triage(questions: &[CohortQuestionRow], threshold: f64, default_options: usize) -> Triage { let mut out = Triage::default(); for question in questions { let row = |reasons: Vec, option: Option<&OptionRow>| TriageRow { number: question.number, item: question.item.clone(), level: question.level, p_value: question.p_value, point_biserial: question.point_biserial, discrimination: question.discrimination, targets: question.target_texts.clone(), taught_in: question.taught_in.clone(), option: option.map(|o| o.letter.clone()), option_share: option.map(|o| o.rate), option_point_biserial: option.and_then(|o| o.point_biserial), reasons, }; let r = question.point_biserial; let key_r = question .options .iter() .filter(|o| o.is_key) .filter_map(|o| o.point_biserial) .fold(f64::NEG_INFINITY, f64::max); // Count single letters only, so a multiple-response combination row such // as `A+D` is not mistaken for a fifth option and does not deflate the // chance rate. let counted = question .options .iter() .filter(|o| o.letter.chars().count() == 1) .count(); let n_options = if counted >= 2 { counted } else { default_options.max(2) }; let chance = 1.0 / n_options as f64; // The best-supported alternative: chosen by a fifth of the class or more, // and correlating with total score at least as well as the key. The share // matters because a defensible reading that two students found is a // wording note, not a regrade. let challenger = question .options .iter() .filter(|o| !o.is_key && o.rate >= 0.20) .filter(|o| o.point_biserial.unwrap_or(f64::NEG_INFINITY) > 0.0) .filter(|o| { !key_r.is_finite() || o.point_biserial.unwrap_or(f64::NEG_INFINITY) >= key_r }) .max_by(|a, b| { a.point_biserial .unwrap_or(f64::NEG_INFINITY) .partial_cmp(&b.point_biserial.unwrap_or(f64::NEG_INFINITY)) .unwrap_or(std::cmp::Ordering::Equal) }); let mut placed = false; // 1. Discard. Negative discrimination means the students who knew the // material did worse on it, which no amount of rewording fixes after // the fact; scores already awarded on it are noise. if let Some(r) = r { if r < -0.05 { out.discard.push(row( vec![format!( "students who scored well overall did worse on this one (r = {r:+.2}). \ Whatever it measured, it was not what the rest of the exam measured." )], None, )); placed = true; } else if r < 0.05 && question.p_value <= chance + 0.05 { out.discard.push(row( vec![format!( "{:.0}% correct against {:.0}% for guessing, and no relationship to total \ score (r = {r:+.2}). The responses are indistinguishable from random.", question.p_value * 100.0, chance * 100.0 )], None, )); placed = true; } } // 2. Rekey or award partial credit. if !placed { if let Some(option) = challenger { let mut reasons = vec![format!( "option {} drew {:.0}% and tracks total score at least as well as the key \ ({:+.2} against {:+.2}).", option.letter, option.rate * 100.0, option.point_biserial.unwrap_or(0.0), if key_r.is_finite() { key_r } else { 0.0 } )]; if question.flags.iter().any(|f| f == "key_underperforms") { reasons.push( "the strongest students chose it more often than the key, which is the \ signature of two readings rather than of a guess." .to_string(), ); } reasons.push( "Either credit it for this administration or rewrite the stem to exclude it \ before reuse." .to_string(), ); out.rekey.push(row(reasons, Some(option))); placed = true; } } // 3. Revise, unless the weak discrimination is explained by difficulty. if !placed { let mut reasons: Vec = Vec::new(); let weak = r.map(|r| r < 0.20).unwrap_or(true); let bounded = weak && (question.p_value >= 0.85 || question.p_value <= 0.20); if weak && !bounded { reasons.push(format!( "at {:.0}% correct the item had room to separate students and did not \ (r = {}).", question.p_value * 100.0, r.map(|r| format!("{r:+.2}")) .unwrap_or_else(|| "n/a".into()) )); } let dead: Vec<&OptionRow> = question .options .iter() .filter(|o| o.nonfunctioning) .collect(); if !dead.is_empty() { reasons.push(format!( "option{} {} drew almost nobody, so the item is really a {}-way choice.", if dead.len() == 1 { "" } else { "s" }, dead.iter() .map(|o| o.letter.as_str()) .collect::>() .join(", "), n_options.saturating_sub(dead.len()).max(2) )); } if question.flags.iter().any(|f| f == "ambiguous") { reasons.push( "partial credit was awarded at grading time, which is a record that the item \ admitted more than one reading." .to_string(), ); } if bounded { out.bounded.push(row( vec![format!( "{:.0}% correct leaves little variance to correlate with, so r = {} is \ what this difficulty allows rather than a fault.", question.p_value * 100.0, r.map(|r| format!("{r:+.2}")) .unwrap_or_else(|| "n/a".into()) )], None, )); placed = true; } else if !reasons.is_empty() { out.revise.push(row(reasons, None)); placed = true; } } // 4. Reteach: the item did its job and the class still missed it. Judged // independently of the three above. let works = r.map(|r| r >= 0.20).unwrap_or(false); if works && question.p_value < threshold { out.reteach.push(row( vec![format!( "the item separated students cleanly (r = {}) and {:.0}% still missed it, so \ this is a gap in what the class knows rather than a fault in the question.", r.map(|r| format!("{r:+.2}")) .unwrap_or_else(|| "n/a".into()), (1.0 - question.p_value) * 100.0 )], None, )); } if !placed { out.clean += 1; } } // Worst first inside each bucket, so the top of every list is where to start. for bucket in [ &mut out.discard, &mut out.rekey, &mut out.revise, &mut out.bounded, ] { bucket.sort_by(|a, b| { a.point_biserial .unwrap_or(1.0) .partial_cmp(&b.point_biserial.unwrap_or(1.0)) .unwrap_or(std::cmp::Ordering::Equal) .then_with(|| a.number.cmp(&b.number)) }); } out.reteach.sort_by(|a, b| { a.p_value .partial_cmp(&b.p_value) .unwrap_or(std::cmp::Ordering::Equal) .then_with(|| a.number.cmp(&b.number)) }); out } /// Per-question proportion correct, split by form. fn per_form_p_values(set: &ResponseSet) -> BTreeMap> { let forms: BTreeSet<&str> = set.rows.iter().filter_map(|r| r.form.as_deref()).collect(); if forms.len() < 2 { return BTreeMap::new(); } let mut totals: BTreeMap<(u32, String), (usize, usize)> = BTreeMap::new(); for row in set.rows.iter().filter(|r| r.counts()) { let Some(form) = row.form.as_deref() else { continue; }; let entry = totals .entry((row.item_number, form.to_string())) .or_insert((0, 0)); entry.1 += 1; if row.correct == Some(true) { entry.0 += 1; } } let mut out: BTreeMap> = BTreeMap::new(); for ((number, form), (correct, n)) in totals { if n == 0 { continue; } out.entry(number) .or_default() .insert(form, correct as f64 / n as f64); } out } /// One row per form, when more than one was given. fn form_rows(set: &ResponseSet, cohort: &Cohort) -> Vec { let mut students_by_form: BTreeMap> = BTreeMap::new(); for row in &set.rows { if let Some(form) = row.form.as_deref() { students_by_form .entry(form.to_string()) .or_default() .insert(row.student_key.as_str()); } } if students_by_form.len() < 2 { return Vec::new(); } let percent: BTreeMap<&str, f64> = cohort .students .iter() .map(|s| (s.student_key.as_str(), s.percent)) .collect(); students_by_form .into_iter() .map(|(form, students)| { let values: Vec = students .iter() .filter_map(|s| percent.get(*s).copied()) .collect(); let n = values.len(); let mean = if n == 0 { 0.0 } else { values.iter().sum::() / n as f64 }; let sd = if n < 2 { 0.0 } else { let variance = values.iter().map(|v| (v - mean).powi(2)).sum::() / (n as f64 - 1.0); variance.sqrt() }; FormRow { id: form, n_students: n, mean, sd, } }) .collect() } /// Bins a set of percentages into a distribution. fn distribution(percents: &[f64]) -> Distribution { let mut sorted: Vec = percents.to_vec(); sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); let n = sorted.len(); let mean = if n == 0 { 0.0 } else { sorted.iter().sum::() / n as f64 }; let median = match n { 0 => 0.0, _ if n % 2 == 1 => sorted[n / 2], _ => (sorted[n / 2 - 1] + sorted[n / 2]) / 2.0, }; let sd = if n < 2 { 0.0 } else { (sorted.iter().map(|v| (v - mean).powi(2)).sum::() / (n as f64 - 1.0)).sqrt() }; let mut bins: Vec = (0..10) .map(|i| Bin { low: i * 10, high: if i == 9 { 100 } else { i * 10 + 10 }, count: 0, }) .collect(); for value in &sorted { let index = ((*value / 10.0).floor() as isize).clamp(0, 9) as usize; bins[index].count += 1; } Distribution { mean, median, sd, min: sorted.first().copied().unwrap_or(0.0), max: sorted.last().copied().unwrap_or(0.0), bins, } } #[cfg(test)] mod tests { use super::*; #[test] fn the_distribution_bins_a_hundred_into_the_last_bin() { let d = distribution(&[100.0, 95.0, 0.0, 42.0]); assert_eq!(d.bins[9].count, 2, "100 belongs with the nineties"); assert_eq!(d.bins[0].count, 1); assert_eq!(d.bins[4].count, 1); assert_eq!(d.max, 100.0); assert_eq!(d.min, 0.0); } #[test] fn the_median_averages_the_middle_pair() { assert_eq!(distribution(&[10.0, 20.0, 30.0, 40.0]).median, 25.0); assert_eq!(distribution(&[10.0, 20.0, 30.0]).median, 20.0); } #[test] fn an_empty_class_does_not_panic() { let d = distribution(&[]); assert_eq!(d.mean, 0.0); assert_eq!(d.bins.len(), 10); } #[test] fn the_student_diagnostic_has_no_field_for_question_content() { // A compile-time argument as much as a test: the struct has no stem and no // option text, so no template can print either one. If a field is ever // added, this serialization check is where the reason gets re-read. let json = serde_json::to_string(&StudentDiagnostic { student_key: "s-1".into(), name: None, sid: None, email: None, form: None, score: Score { points: 1.0, points_possible: 2.0, percent: 50.0, bonus_points: 0.0, correct: 1, n_items: 2, }, standing: None, levels: Vec::new(), objectives: Vec::new(), strengths: Vec::new(), focus: Vec::new(), questions: Vec::new(), dropped_questions: Vec::new(), review_lectures: Vec::new(), study: Vec::new(), }) .unwrap(); assert!(!json.contains("stem"), "{json}"); assert!(!json.contains("options"), "{json}"); } }