1143 lines
36 KiB
Rust
1143 lines
36 KiB
Rust
//! Turning responses into something you can say to a student.
|
|
//!
|
|
//! Item analysis tells you about items. This module tells you about people: which
|
|
//! objectives a student has actually met, which ones they are close on, and what
|
|
//! specifically to do next.
|
|
//!
|
|
//! # On declaring mastery from three questions
|
|
//!
|
|
//! The honest answer is that you often cannot. Two items on an objective give a
|
|
//! proportion with an enormous confidence interval — two out of two correct is
|
|
//! consistent with a true rate anywhere from about 0.55 upward. So this module
|
|
//! does three things instead of pretending otherwise.
|
|
//!
|
|
//! It refuses to classify at all below `min_items_for_mastery`, reporting "not
|
|
//! enough evidence", which is a finding about your blueprint rather than about the
|
|
//! student. It reports the Wilson score interval alongside every rate, because
|
|
//! Wilson behaves sensibly at the boundaries where the normal approximation
|
|
//! produces intervals extending past 1.0. And it separates the *classification*
|
|
//! (which uses the observed rate, so it is usable) from the *confidence* (which
|
|
//! uses the interval, so it is honest). A student can be "meeting" an objective
|
|
//! provisionally, and the report says so.
|
|
//!
|
|
//! # Comparison to the cohort
|
|
//!
|
|
//! Per-level performance is reported against the class rather than in absolute
|
|
//! terms, because "you got 60% of the Analyze items" means nothing to a student
|
|
//! without knowing that the class average was 55%. The comparison is descriptive,
|
|
//! not a curve.
|
|
|
|
use std::collections::{BTreeMap, BTreeSet};
|
|
|
|
use crate::course::{CourseFile, Policy};
|
|
use crate::responses::{Response, ResponseSet};
|
|
use crate::rng::Rng;
|
|
use crate::taxonomy::Level;
|
|
|
|
/// How well a student has met one objective.
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub enum Mastery {
|
|
/// Met the threshold.
|
|
Meeting,
|
|
/// Partway there.
|
|
Developing,
|
|
/// Not yet.
|
|
NotYet,
|
|
/// Too few items on this objective to say anything. This is a gap in the
|
|
/// assessment, not a judgment about the student.
|
|
NotEnoughEvidence,
|
|
}
|
|
|
|
impl Mastery {
|
|
/// A label for reports.
|
|
pub fn label(self) -> &'static str {
|
|
match self {
|
|
Mastery::Meeting => "meeting",
|
|
Mastery::Developing => "developing",
|
|
Mastery::NotYet => "not yet",
|
|
Mastery::NotEnoughEvidence => "not enough evidence",
|
|
}
|
|
}
|
|
|
|
/// A short symbol for compact tables.
|
|
pub fn symbol(self) -> &'static str {
|
|
match self {
|
|
Mastery::Meeting => "✓",
|
|
Mastery::Developing => "~",
|
|
Mastery::NotYet => "✗",
|
|
Mastery::NotEnoughEvidence => "?",
|
|
}
|
|
}
|
|
}
|
|
|
|
/// One student's standing on one objective.
|
|
#[derive(Debug, Clone)]
|
|
pub struct ObjectiveMastery {
|
|
/// The objective id.
|
|
pub objective: String,
|
|
/// The objective text, for reports.
|
|
pub text: String,
|
|
/// How many items on this objective the student saw.
|
|
pub n_items: usize,
|
|
/// How many they got right, counting partial credit.
|
|
pub credit: f64,
|
|
/// Observed rate, `credit / n_items`.
|
|
pub rate: f64,
|
|
/// Lower end of the Wilson interval.
|
|
pub wilson_lower: f64,
|
|
/// Upper end of the Wilson interval.
|
|
pub wilson_upper: f64,
|
|
/// The class's rate on the same objective.
|
|
pub cohort_rate: f64,
|
|
/// The classification.
|
|
pub status: Mastery,
|
|
/// Whether the interval, not just the point estimate, clears the threshold.
|
|
pub confident: bool,
|
|
/// Which levels the objective was assessed at, since meeting an objective at
|
|
/// Remember is a different claim from meeting it at Analyze.
|
|
pub levels: Vec<Level>,
|
|
}
|
|
|
|
/// One student's performance at one cognitive level.
|
|
#[derive(Debug, Clone)]
|
|
pub struct LevelProfile {
|
|
/// The level.
|
|
pub level: Level,
|
|
/// How many items at this level.
|
|
pub n_items: usize,
|
|
/// The student's rate.
|
|
pub rate: f64,
|
|
/// The class's rate.
|
|
pub cohort_rate: f64,
|
|
/// Difference from the class, in class standard deviations. `None` when the
|
|
/// class had no spread at this level.
|
|
pub z: Option<f64>,
|
|
}
|
|
|
|
impl LevelProfile {
|
|
/// A plain-language comparison to the class.
|
|
pub fn comparison(&self) -> &'static str {
|
|
match self.z {
|
|
Some(z) if z >= 1.0 => "well above the class",
|
|
Some(z) if z >= 0.4 => "above the class",
|
|
Some(z) if z > -0.4 => "about the same as the class",
|
|
Some(z) if z > -1.0 => "below the class",
|
|
Some(_) => "well below the class",
|
|
None => "the class did not vary here",
|
|
}
|
|
}
|
|
}
|
|
|
|
/// An item the student got wrong, with what to do about it.
|
|
#[derive(Debug, Clone)]
|
|
pub struct MissedItem {
|
|
/// The question number.
|
|
pub number: u32,
|
|
/// The item's global id.
|
|
pub item_ref: Option<String>,
|
|
/// What the student chose.
|
|
pub selected: Vec<String>,
|
|
/// Credit earned, since a partially credited response is not a clean miss.
|
|
pub credit: f64,
|
|
/// The level.
|
|
pub level: Option<Level>,
|
|
/// The objectives involved.
|
|
pub learning_objectives: Vec<String>,
|
|
/// The misconception the chosen distractor was written to detect.
|
|
pub misconception: Option<String>,
|
|
/// Feedback written for a student who chose that option.
|
|
pub feedback: Option<String>,
|
|
/// Where to go back to: lecture titles and slide numbers.
|
|
pub study: Vec<String>,
|
|
}
|
|
|
|
/// Everything needed to write one student's report.
|
|
#[derive(Debug, Clone)]
|
|
pub struct StudentSummary {
|
|
/// The grouping key.
|
|
pub student_key: String,
|
|
/// The name, when not pseudonymized.
|
|
pub name: Option<String>,
|
|
/// The student id, when not pseudonymized.
|
|
pub sid: Option<String>,
|
|
/// Points earned on scored items.
|
|
pub points: f64,
|
|
/// Points available on scored items.
|
|
pub points_possible: f64,
|
|
/// Percentage on scored items.
|
|
pub percent: f64,
|
|
/// Bonus points earned.
|
|
pub bonus_points: f64,
|
|
/// Items answered correctly.
|
|
pub correct: usize,
|
|
/// Items administered.
|
|
pub n_items: usize,
|
|
/// Where the score falls relative to the class, as a coarse band.
|
|
pub band: String,
|
|
/// IRT ability, when an IRT fit was supplied.
|
|
pub theta: Option<f64>,
|
|
/// Standard error of the ability estimate.
|
|
pub theta_se: Option<f64>,
|
|
/// Per-objective standing, in the course's objective order.
|
|
pub objectives: Vec<ObjectiveMastery>,
|
|
/// Per-level standing.
|
|
pub levels: Vec<LevelProfile>,
|
|
/// Objectives the student is clearly meeting.
|
|
pub strengths: Vec<String>,
|
|
/// Objectives to work on, worst first.
|
|
pub focus: Vec<String>,
|
|
/// Missed items with targeted guidance.
|
|
pub missed: Vec<MissedItem>,
|
|
}
|
|
|
|
impl StudentSummary {
|
|
/// The display name, falling back to the key.
|
|
pub fn display_name(&self) -> String {
|
|
self.name
|
|
.clone()
|
|
.unwrap_or_else(|| self.student_key.clone())
|
|
}
|
|
}
|
|
|
|
/// Class-level context.
|
|
#[derive(Debug, Clone)]
|
|
pub struct Cohort {
|
|
/// Per-student summaries, sorted by key.
|
|
pub students: Vec<StudentSummary>,
|
|
/// Class rate per objective.
|
|
pub objective_rates: BTreeMap<String, f64>,
|
|
/// Class rate per level.
|
|
pub level_rates: BTreeMap<Level, f64>,
|
|
/// Mean percentage.
|
|
pub mean_percent: f64,
|
|
/// Standard deviation of percentage.
|
|
pub sd_percent: f64,
|
|
/// Objectives the class as a whole did not meet, worst first. This is the
|
|
/// list that should change what you reteach.
|
|
pub class_gaps: Vec<(String, f64)>,
|
|
/// Optional grouping of students by response profile.
|
|
pub archetypes: Vec<Archetype>,
|
|
}
|
|
|
|
/// A cluster of students with a similar profile across levels.
|
|
#[derive(Debug, Clone)]
|
|
pub struct Archetype {
|
|
/// A label describing the pattern.
|
|
pub label: String,
|
|
/// The student keys in this cluster.
|
|
pub members: Vec<String>,
|
|
/// Mean rate at each level for this cluster.
|
|
pub level_means: BTreeMap<Level, f64>,
|
|
}
|
|
|
|
/// The Wilson score interval for a binomial proportion.
|
|
///
|
|
/// Preferred over the normal approximation because it stays inside `[0, 1]` and
|
|
/// behaves at the boundaries, which is exactly where classroom data lives: a
|
|
/// student who got three out of three needs an interval, and the textbook formula
|
|
/// gives width zero there.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `successes` - the number of successes, which may be fractional when partial
|
|
/// credit is involved.
|
|
/// * `n` - the number of trials.
|
|
/// * `z` - the standard normal quantile; 1.96 for a two-sided 95% interval.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The lower and upper bounds, or `(0.0, 1.0)` when there are no trials.
|
|
pub fn wilson(successes: f64, n: usize, z: f64) -> (f64, f64) {
|
|
if n == 0 {
|
|
return (0.0, 1.0);
|
|
}
|
|
let n = n as f64;
|
|
let p = (successes / n).clamp(0.0, 1.0);
|
|
let z2 = z * z;
|
|
let denominator = 1.0 + z2 / n;
|
|
let center = p + z2 / (2.0 * n);
|
|
let spread = z * ((p * (1.0 - p) / n) + z2 / (4.0 * n * n)).sqrt();
|
|
(
|
|
((center - spread) / denominator).clamp(0.0, 1.0),
|
|
((center + spread) / denominator).clamp(0.0, 1.0),
|
|
)
|
|
}
|
|
|
|
/// Builds per-student summaries for one administration.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `set` - the responses, already enriched with item metadata.
|
|
/// * `course` - the course, for objective text, order, and policy.
|
|
/// * `catalog` - the loaded course, for misconception feedback on missed items.
|
|
/// * `fit` - an optional IRT fit, whose abilities are attached when present.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The cohort.
|
|
pub fn summarize(
|
|
set: &ResponseSet,
|
|
course: &CourseFile,
|
|
catalog: Option<&crate::catalog::Catalog>,
|
|
fit: Option<&crate::irt::Fit>,
|
|
) -> Cohort {
|
|
let policy = &course.policy;
|
|
let students = set.students();
|
|
|
|
// Class rates first: every student's report is relative to these.
|
|
let objective_rates = rates_by_objective(&set.rows.iter().collect::<Vec<_>>());
|
|
let level_rates = rates_by_level(&set.rows.iter().collect::<Vec<_>>());
|
|
|
|
// Per-level spread across students, for the z comparisons.
|
|
let mut level_values: BTreeMap<Level, Vec<f64>> = BTreeMap::new();
|
|
for key in &students {
|
|
let rows = set.for_student(key);
|
|
for (level, rate) in rates_by_level(&rows) {
|
|
level_values.entry(level).or_default().push(rate);
|
|
}
|
|
}
|
|
let level_sd: BTreeMap<Level, f64> = level_values
|
|
.iter()
|
|
.map(|(level, values)| (*level, sd(values)))
|
|
.collect();
|
|
|
|
let percents: Vec<f64> = students
|
|
.iter()
|
|
.map(|key| {
|
|
let earned = set.scored_total(key);
|
|
let possible = set.points_available();
|
|
if possible > 0.0 {
|
|
100.0 * earned / possible
|
|
} else {
|
|
0.0
|
|
}
|
|
})
|
|
.collect();
|
|
let mean_percent = mean(&percents);
|
|
let sd_percent = sd(&percents);
|
|
|
|
let ability = fit.map(|f| f.ability_map()).unwrap_or_default();
|
|
let ability_se: BTreeMap<String, f64> = fit
|
|
.map(|f| {
|
|
f.abilities
|
|
.iter()
|
|
.map(|a| (a.student_key.clone(), a.se))
|
|
.collect()
|
|
})
|
|
.unwrap_or_default();
|
|
|
|
let order = course.objectives_in_order();
|
|
let mut summaries = Vec::with_capacity(students.len());
|
|
|
|
for (index, key) in students.iter().enumerate() {
|
|
let rows = set.for_student(key);
|
|
let points = set.scored_total(key);
|
|
let possible = set.points_available();
|
|
let percent = percents[index];
|
|
|
|
let correct = rows
|
|
.iter()
|
|
.filter(|r| r.counts() && r.correct == Some(true))
|
|
.count();
|
|
let n_items = rows.iter().filter(|r| r.counts()).count();
|
|
|
|
// Objectives, in the course's declared order so reports read the way the
|
|
// course is taught rather than alphabetically.
|
|
let per_objective = rates_by_objective(&rows);
|
|
let counts = counts_by_objective(&rows);
|
|
let mut objectives = Vec::new();
|
|
let mut seen: BTreeSet<&String> = BTreeSet::new();
|
|
for id in order.iter().chain(per_objective.keys()) {
|
|
if !seen.insert(id) {
|
|
continue;
|
|
}
|
|
let Some((n, credit)) = counts.get(id).copied() else {
|
|
continue;
|
|
};
|
|
objectives.push(objective_mastery(
|
|
id,
|
|
course,
|
|
n,
|
|
credit,
|
|
objective_rates.get(id).copied().unwrap_or(0.0),
|
|
&rows,
|
|
policy,
|
|
));
|
|
}
|
|
|
|
// Levels.
|
|
let student_levels = rates_by_level(&rows);
|
|
let level_counts = counts_by_level(&rows);
|
|
let levels: Vec<LevelProfile> = Level::ALL
|
|
.iter()
|
|
.filter_map(|level| {
|
|
let (n, _) = level_counts.get(level).copied()?;
|
|
if n == 0 {
|
|
return None;
|
|
}
|
|
let rate = student_levels.get(level).copied().unwrap_or(0.0);
|
|
let cohort_rate = level_rates.get(level).copied().unwrap_or(0.0);
|
|
let spread = level_sd.get(level).copied().unwrap_or(0.0);
|
|
Some(LevelProfile {
|
|
level: *level,
|
|
n_items: n,
|
|
rate,
|
|
cohort_rate,
|
|
z: if spread > 1e-9 {
|
|
Some((rate - cohort_rate) / spread)
|
|
} else {
|
|
None
|
|
},
|
|
})
|
|
})
|
|
.collect();
|
|
|
|
// Strengths and focus areas. Strengths need confidence, focus areas do
|
|
// not: telling a student to review something they may already know costs
|
|
// them an hour, while telling them they have mastered something they have
|
|
// not costs them the next exam.
|
|
let strengths: Vec<String> = objectives
|
|
.iter()
|
|
.filter(|o| o.status == Mastery::Meeting && o.confident)
|
|
.map(|o| o.objective.clone())
|
|
.collect();
|
|
let mut focus_pairs: Vec<(&ObjectiveMastery, f64)> = objectives
|
|
.iter()
|
|
.filter(|o| matches!(o.status, Mastery::NotYet | Mastery::Developing))
|
|
.map(|o| (o, o.rate))
|
|
.collect();
|
|
focus_pairs.sort_by(|a, b| {
|
|
a.1.partial_cmp(&b.1)
|
|
.unwrap_or(std::cmp::Ordering::Equal)
|
|
.then_with(|| a.0.objective.cmp(&b.0.objective))
|
|
});
|
|
let focus: Vec<String> = focus_pairs
|
|
.iter()
|
|
.map(|(o, _)| o.objective.clone())
|
|
.collect();
|
|
|
|
let missed = missed_items(&rows, catalog, course);
|
|
|
|
summaries.push(StudentSummary {
|
|
student_key: key.clone(),
|
|
name: rows.first().and_then(|r| r.name.clone()),
|
|
sid: rows.first().and_then(|r| r.sid.clone()),
|
|
points,
|
|
points_possible: possible,
|
|
percent,
|
|
bonus_points: set.bonus_total(key),
|
|
correct,
|
|
n_items,
|
|
band: band_for(percent, &percents),
|
|
theta: ability.get(key).copied(),
|
|
theta_se: ability_se.get(key).copied(),
|
|
objectives,
|
|
levels,
|
|
strengths,
|
|
focus,
|
|
missed,
|
|
});
|
|
}
|
|
|
|
// Class gaps: objectives where the whole class fell short. These are the ones
|
|
// to reteach rather than to send individual students away to review.
|
|
let mut class_gaps: Vec<(String, f64)> = objective_rates
|
|
.iter()
|
|
.filter(|(_, rate)| **rate < policy.mastery_threshold)
|
|
.map(|(id, rate)| (id.clone(), *rate))
|
|
.collect();
|
|
class_gaps.sort_by(|a, b| {
|
|
a.1.partial_cmp(&b.1)
|
|
.unwrap_or(std::cmp::Ordering::Equal)
|
|
.then_with(|| a.0.cmp(&b.0))
|
|
});
|
|
|
|
let archetypes = cluster(&summaries, 3);
|
|
|
|
Cohort {
|
|
students: summaries,
|
|
objective_rates,
|
|
level_rates,
|
|
mean_percent,
|
|
sd_percent,
|
|
class_gaps,
|
|
archetypes,
|
|
}
|
|
}
|
|
|
|
/// Builds one objective's mastery record.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `id` - the objective id.
|
|
/// * `course` - the course, for text and policy.
|
|
/// * `n` - items on this objective.
|
|
/// * `credit` - total credit earned.
|
|
/// * `cohort_rate` - the class rate.
|
|
/// * `rows` - the student's responses, for the level list.
|
|
/// * `policy` - the course policy.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The record.
|
|
fn objective_mastery(
|
|
id: &str,
|
|
course: &CourseFile,
|
|
n: usize,
|
|
credit: f64,
|
|
cohort_rate: f64,
|
|
rows: &[&Response],
|
|
policy: &Policy,
|
|
) -> ObjectiveMastery {
|
|
let rate = if n > 0 { credit / n as f64 } else { 0.0 };
|
|
let (lower, upper) = wilson(credit, n, 1.96);
|
|
|
|
let status = if n < policy.min_items_for_mastery.max(1) {
|
|
Mastery::NotEnoughEvidence
|
|
} else if rate >= policy.mastery_threshold {
|
|
Mastery::Meeting
|
|
} else if rate >= policy.mastery_threshold * 0.6 {
|
|
Mastery::Developing
|
|
} else {
|
|
Mastery::NotYet
|
|
};
|
|
|
|
let levels: Vec<Level> = rows
|
|
.iter()
|
|
.filter(|r| r.learning_objectives.iter().any(|o| o == id))
|
|
.filter_map(|r| r.level)
|
|
.collect::<BTreeSet<Level>>()
|
|
.into_iter()
|
|
.collect();
|
|
|
|
ObjectiveMastery {
|
|
objective: id.to_string(),
|
|
text: course.objective_text(id),
|
|
n_items: n,
|
|
credit,
|
|
rate,
|
|
wilson_lower: lower,
|
|
wilson_upper: upper,
|
|
cohort_rate,
|
|
status,
|
|
confident: lower >= policy.mastery_threshold,
|
|
levels,
|
|
}
|
|
}
|
|
|
|
/// Collects missed items with targeted guidance.
|
|
///
|
|
/// The guidance comes from the item's own authoring: the `misconception` recorded
|
|
/// on the distractor the student actually chose, and the lecture and slides the
|
|
/// item was written from. This is why authoring distractors deliberately pays off
|
|
/// twice — once when writing the item, and again in every report afterward.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `rows` - the student's responses.
|
|
/// * `catalog` - the loaded course.
|
|
/// * `course` - the course, for lecture titles.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The missed items, in question order.
|
|
fn missed_items(
|
|
rows: &[&Response],
|
|
catalog: Option<&crate::catalog::Catalog>,
|
|
course: &CourseFile,
|
|
) -> Vec<MissedItem> {
|
|
let mut out = Vec::new();
|
|
for r in rows {
|
|
if !r.counts() || r.credit >= 0.999 {
|
|
continue;
|
|
}
|
|
let mut misconception = None;
|
|
let mut feedback = None;
|
|
let mut study = Vec::new();
|
|
|
|
if let (Some(cat), Some(uid)) = (catalog, r.item_ref.as_deref()) {
|
|
if let Some(entry) = cat.get(uid) {
|
|
// Feedback for the specific option chosen, which is the whole
|
|
// point of recording per-distractor misconceptions.
|
|
if let Some(letter) = r.selected.first() {
|
|
if let Some(choice) = entry.item.option(letter) {
|
|
misconception = choice.misconception.clone();
|
|
feedback = choice.student_text().map(|s| s.to_string());
|
|
}
|
|
}
|
|
for source in &entry.item.sources {
|
|
let title = course
|
|
.lectures
|
|
.get(&source.lecture)
|
|
.map(|l| l.title.clone())
|
|
.unwrap_or_else(|| source.lecture.clone());
|
|
if source.slides.is_empty() {
|
|
study.push(title);
|
|
} else {
|
|
let slides: Vec<String> =
|
|
source.slides.iter().map(|s| s.to_string()).collect();
|
|
study.push(format!("{title}, slides {}", slides.join(", ")));
|
|
}
|
|
for reading in &source.readings {
|
|
study.push(reading.clone());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
out.push(MissedItem {
|
|
number: r.item_number,
|
|
item_ref: r.item_ref.clone(),
|
|
selected: r.selected.clone(),
|
|
credit: r.credit,
|
|
level: r.level,
|
|
learning_objectives: r.learning_objectives.clone(),
|
|
misconception,
|
|
feedback,
|
|
study,
|
|
});
|
|
}
|
|
out
|
|
}
|
|
|
|
/// Credit rate per objective over a set of responses.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `rows` - the responses.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The rate for each objective mentioned.
|
|
pub fn rates_by_objective(rows: &[&Response]) -> BTreeMap<String, f64> {
|
|
counts_by_objective(rows)
|
|
.into_iter()
|
|
.map(|(id, (n, credit))| {
|
|
let rate = if n > 0 { credit / n as f64 } else { 0.0 };
|
|
(id, rate)
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
/// Item counts and credit per objective.
|
|
///
|
|
/// An item tagged with two objectives counts toward both. That double counting is
|
|
/// intentional: the question "how is this student doing on kinetics" should use
|
|
/// every item that measured kinetics.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `rows` - the responses.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// `(item count, total credit)` per objective.
|
|
pub fn counts_by_objective(rows: &[&Response]) -> BTreeMap<String, (usize, f64)> {
|
|
let mut out: BTreeMap<String, (usize, f64)> = BTreeMap::new();
|
|
for r in rows {
|
|
if !r.counts() {
|
|
continue;
|
|
}
|
|
for objective in &r.learning_objectives {
|
|
let e = out.entry(objective.clone()).or_insert((0, 0.0));
|
|
e.0 += 1;
|
|
e.1 += r.credit.clamp(0.0, 1.0);
|
|
}
|
|
}
|
|
out
|
|
}
|
|
|
|
/// Credit rate per level.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `rows` - the responses.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The rate for each level present.
|
|
pub fn rates_by_level(rows: &[&Response]) -> BTreeMap<Level, f64> {
|
|
counts_by_level(rows)
|
|
.into_iter()
|
|
.map(|(level, (n, credit))| {
|
|
let rate = if n > 0 { credit / n as f64 } else { 0.0 };
|
|
(level, rate)
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
/// Item counts and credit per level.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `rows` - the responses.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// `(item count, total credit)` per level.
|
|
pub fn counts_by_level(rows: &[&Response]) -> BTreeMap<Level, (usize, f64)> {
|
|
let mut out: BTreeMap<Level, (usize, f64)> = BTreeMap::new();
|
|
for r in rows {
|
|
if !r.counts() {
|
|
continue;
|
|
}
|
|
if let Some(level) = r.level {
|
|
let e = out.entry(level).or_insert((0, 0.0));
|
|
e.0 += 1;
|
|
e.1 += r.credit.clamp(0.0, 1.0);
|
|
}
|
|
}
|
|
out
|
|
}
|
|
|
|
/// A coarse band for a score within a class.
|
|
///
|
|
/// Quartile bands rather than an exact percentile, because a percentile computed
|
|
/// from twenty-four students implies a precision it does not have, and because
|
|
/// telling a student they are "37th percentile" invites comparison in a way that
|
|
/// "middle half of the class" does not.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `percent` - the student's percentage.
|
|
/// * `all` - every student's percentage.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The band label.
|
|
fn band_for(percent: f64, all: &[f64]) -> String {
|
|
if all.len() < 4 {
|
|
return "the class is too small to place this meaningfully".to_string();
|
|
}
|
|
let mut sorted = all.to_vec();
|
|
sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
|
|
let below = sorted.iter().filter(|x| **x < percent).count() as f64;
|
|
let fraction = below / all.len() as f64;
|
|
if fraction >= 0.75 {
|
|
"top quarter of the class".to_string()
|
|
} else if fraction >= 0.5 {
|
|
"upper middle of the class".to_string()
|
|
} else if fraction >= 0.25 {
|
|
"lower middle of the class".to_string()
|
|
} else {
|
|
"bottom quarter of the class".to_string()
|
|
}
|
|
}
|
|
|
|
/// Groups students by their profile across levels.
|
|
///
|
|
/// This is descriptive, not diagnostic. It answers "are there recognizable
|
|
/// patterns in how this class is struggling" — for instance a group that handles
|
|
/// recall fine and falls apart on application, which calls for different
|
|
/// instruction than a group that is uniformly behind.
|
|
///
|
|
/// k-means with a seeded, deterministic initialization, so the same data always
|
|
/// produces the same groups.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `students` - the summaries.
|
|
/// * `k` - how many clusters to look for.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The clusters, largest first. Empty when there are too few students to bother.
|
|
pub fn cluster(students: &[StudentSummary], k: usize) -> Vec<Archetype> {
|
|
// Below about three students per cluster the groups are noise.
|
|
if students.len() < k * 3 || k == 0 {
|
|
return Vec::new();
|
|
}
|
|
|
|
// Feature vector: rate at each level that anyone was assessed on.
|
|
let levels: Vec<Level> = students
|
|
.iter()
|
|
.flat_map(|s| s.levels.iter().map(|l| l.level))
|
|
.collect::<BTreeSet<Level>>()
|
|
.into_iter()
|
|
.collect();
|
|
if levels.len() < 2 {
|
|
return Vec::new();
|
|
}
|
|
|
|
let points: Vec<Vec<f64>> = students
|
|
.iter()
|
|
.map(|s| {
|
|
levels
|
|
.iter()
|
|
.map(|level| {
|
|
s.levels
|
|
.iter()
|
|
.find(|l| l.level == *level)
|
|
.map(|l| l.rate)
|
|
.unwrap_or(0.0)
|
|
})
|
|
.collect()
|
|
})
|
|
.collect();
|
|
|
|
// Standardize each dimension so a level everyone did well on does not
|
|
// dominate the distance.
|
|
let mut standardized = points.clone();
|
|
for d in 0..levels.len() {
|
|
let column: Vec<f64> = points.iter().map(|p| p[d]).collect();
|
|
let m = mean(&column);
|
|
let s = sd(&column);
|
|
for (i, point) in standardized.iter_mut().enumerate() {
|
|
point[d] = if s > 1e-9 {
|
|
(points[i][d] - m) / s
|
|
} else {
|
|
0.0
|
|
};
|
|
}
|
|
}
|
|
|
|
// Seeded k-means++ initialization.
|
|
let mut rng = Rng::from_label("coursebank/archetypes");
|
|
let mut centers: Vec<Vec<f64>> =
|
|
vec![standardized[rng.below(standardized.len() as u64) as usize].clone()];
|
|
while centers.len() < k {
|
|
let distances: Vec<f64> = standardized
|
|
.iter()
|
|
.map(|p| {
|
|
centers
|
|
.iter()
|
|
.map(|c| squared_distance(p, c))
|
|
.fold(f64::INFINITY, f64::min)
|
|
})
|
|
.collect();
|
|
let total: f64 = distances.iter().sum();
|
|
if total <= 0.0 {
|
|
break;
|
|
}
|
|
let mut target = rng.unit() * total;
|
|
let mut chosen = standardized.len() - 1;
|
|
for (i, d) in distances.iter().enumerate() {
|
|
target -= d;
|
|
if target <= 0.0 {
|
|
chosen = i;
|
|
break;
|
|
}
|
|
}
|
|
centers.push(standardized[chosen].clone());
|
|
}
|
|
|
|
let mut assignment = vec![0usize; standardized.len()];
|
|
for _ in 0..50 {
|
|
let mut changed = false;
|
|
for (i, p) in standardized.iter().enumerate() {
|
|
let mut best = (0usize, f64::INFINITY);
|
|
for (c, center) in centers.iter().enumerate() {
|
|
let d = squared_distance(p, center);
|
|
if d < best.1 {
|
|
best = (c, d);
|
|
}
|
|
}
|
|
if assignment[i] != best.0 {
|
|
assignment[i] = best.0;
|
|
changed = true;
|
|
}
|
|
}
|
|
for (c, center) in centers.iter_mut().enumerate() {
|
|
let members: Vec<&Vec<f64>> = standardized
|
|
.iter()
|
|
.enumerate()
|
|
.filter(|(i, _)| assignment[*i] == c)
|
|
.map(|(_, p)| p)
|
|
.collect();
|
|
if members.is_empty() {
|
|
continue;
|
|
}
|
|
for d in 0..levels.len() {
|
|
center[d] = members.iter().map(|p| p[d]).sum::<f64>() / members.len() as f64;
|
|
}
|
|
}
|
|
if !changed {
|
|
break;
|
|
}
|
|
}
|
|
|
|
let mut out = Vec::new();
|
|
for c in 0..centers.len() {
|
|
let members: Vec<String> = students
|
|
.iter()
|
|
.enumerate()
|
|
.filter(|(i, _)| assignment[*i] == c)
|
|
.map(|(_, s)| s.student_key.clone())
|
|
.collect();
|
|
if members.is_empty() {
|
|
continue;
|
|
}
|
|
let mut level_means = BTreeMap::new();
|
|
for (d, level) in levels.iter().enumerate() {
|
|
let values: Vec<f64> = students
|
|
.iter()
|
|
.enumerate()
|
|
.filter(|(i, _)| assignment[*i] == c)
|
|
.map(|(i, _)| points[i][d])
|
|
.collect();
|
|
level_means.insert(*level, mean(&values));
|
|
}
|
|
out.push(Archetype {
|
|
label: label_for(&level_means),
|
|
members,
|
|
level_means,
|
|
});
|
|
}
|
|
out.sort_by(|a, b| b.members.len().cmp(&a.members.len()));
|
|
out
|
|
}
|
|
|
|
/// Names a cluster from its level profile.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `means` - mean rate at each level.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// A descriptive label.
|
|
fn label_for(means: &BTreeMap<Level, f64>) -> String {
|
|
let values: Vec<f64> = means.values().copied().collect();
|
|
let overall = mean(&values);
|
|
|
|
// Is the profile flat, or does it fall off with cognitive demand?
|
|
let low: Vec<f64> = means
|
|
.iter()
|
|
.filter(|(l, _)| l.code() <= 2)
|
|
.map(|(_, v)| *v)
|
|
.collect();
|
|
let high: Vec<f64> = means
|
|
.iter()
|
|
.filter(|(l, _)| l.code() >= 3)
|
|
.map(|(_, v)| *v)
|
|
.collect();
|
|
|
|
if !low.is_empty() && !high.is_empty() {
|
|
let drop = mean(&low) - mean(&high);
|
|
if drop > 0.25 {
|
|
return "knows the material, struggles to apply it".to_string();
|
|
}
|
|
if drop < -0.15 {
|
|
return "reasons well, gaps in recall".to_string();
|
|
}
|
|
}
|
|
|
|
if overall >= 0.85 {
|
|
"consistently strong".to_string()
|
|
} else if overall >= 0.65 {
|
|
"solid with scattered gaps".to_string()
|
|
} else {
|
|
"behind across the board".to_string()
|
|
}
|
|
}
|
|
|
|
/// Squared Euclidean distance.
|
|
fn squared_distance(a: &[f64], b: &[f64]) -> f64 {
|
|
a.iter().zip(b.iter()).map(|(x, y)| (x - y) * (x - y)).sum()
|
|
}
|
|
|
|
/// The arithmetic mean, zero for an empty slice.
|
|
fn mean(v: &[f64]) -> f64 {
|
|
if v.is_empty() {
|
|
0.0
|
|
} else {
|
|
v.iter().sum::<f64>() / v.len() as f64
|
|
}
|
|
}
|
|
|
|
/// The population standard deviation.
|
|
fn sd(v: &[f64]) -> f64 {
|
|
if v.len() < 2 {
|
|
return 0.0;
|
|
}
|
|
let m = mean(v);
|
|
(v.iter().map(|x| (x - m) * (x - m)).sum::<f64>() / v.len() as f64).sqrt()
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn wilson_stays_inside_zero_and_one() {
|
|
// Three out of three: the naive interval has zero width, Wilson does not.
|
|
let (lo, hi) = wilson(3.0, 3, 1.96);
|
|
assert!(lo > 0.0 && lo < 1.0, "lower bound {lo}");
|
|
assert_eq!(hi, 1.0);
|
|
assert!(lo < 0.5, "three items cannot establish a high rate: {lo}");
|
|
|
|
// Zero out of four.
|
|
let (lo, hi) = wilson(0.0, 4, 1.96);
|
|
assert_eq!(lo, 0.0);
|
|
assert!(hi > 0.0 && hi < 1.0);
|
|
|
|
// No data at all.
|
|
assert_eq!(wilson(0.0, 0, 1.96), (0.0, 1.0));
|
|
}
|
|
|
|
#[test]
|
|
fn wilson_narrows_as_n_grows() {
|
|
let (lo_small, hi_small) = wilson(8.0, 10, 1.96);
|
|
let (lo_big, hi_big) = wilson(80.0, 100, 1.96);
|
|
assert!(
|
|
(hi_big - lo_big) < (hi_small - lo_small),
|
|
"more data must give a tighter interval"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn two_items_never_claim_confident_mastery() {
|
|
// The classification may say "meeting", but confidence must not, because
|
|
// two items cannot establish a rate of 0.75.
|
|
let (lower, _) = wilson(2.0, 2, 1.96);
|
|
assert!(lower < 0.75, "got {lower}");
|
|
}
|
|
|
|
#[test]
|
|
fn bands_describe_position_coarsely() {
|
|
let all = vec![50.0, 60.0, 70.0, 80.0, 90.0, 95.0, 40.0, 30.0];
|
|
assert!(band_for(95.0, &all).contains("top"));
|
|
assert!(band_for(30.0, &all).contains("bottom"));
|
|
// Too few students to place anyone.
|
|
assert!(band_for(50.0, &[50.0, 60.0]).contains("too small"));
|
|
}
|
|
|
|
#[test]
|
|
fn objective_counts_credit_every_tagged_item() {
|
|
let rows = vec![
|
|
make("s1", 1, 1.0, &["lo-a", "lo-b"], Some(Level::Remember)),
|
|
make("s1", 2, 0.0, &["lo-a"], Some(Level::Apply)),
|
|
];
|
|
let refs: Vec<&Response> = rows.iter().collect();
|
|
let counts = counts_by_objective(&refs);
|
|
// lo-a saw both items; lo-b only the first.
|
|
assert_eq!(counts["lo-a"], (2, 1.0));
|
|
assert_eq!(counts["lo-b"], (1, 1.0));
|
|
let rates = rates_by_objective(&refs);
|
|
assert_eq!(rates["lo-a"], 0.5);
|
|
assert_eq!(rates["lo-b"], 1.0);
|
|
}
|
|
|
|
#[test]
|
|
fn level_rates_ignore_untagged_items() {
|
|
let rows = vec![
|
|
make("s1", 1, 1.0, &[], Some(Level::Remember)),
|
|
make("s1", 2, 0.0, &[], None),
|
|
];
|
|
let refs: Vec<&Response> = rows.iter().collect();
|
|
let counts = counts_by_level(&refs);
|
|
assert_eq!(counts.len(), 1);
|
|
assert_eq!(counts[&Level::Remember], (1, 1.0));
|
|
}
|
|
|
|
#[test]
|
|
fn mastery_labels_are_stable() {
|
|
assert_eq!(Mastery::Meeting.label(), "meeting");
|
|
assert_eq!(Mastery::NotEnoughEvidence.label(), "not enough evidence");
|
|
}
|
|
|
|
#[test]
|
|
fn clustering_needs_enough_students() {
|
|
assert!(cluster(&[], 3).is_empty());
|
|
let few: Vec<StudentSummary> = (0..4).map(|i| summary(&format!("s{i}"))).collect();
|
|
assert!(cluster(&few, 3).is_empty(), "four students, three clusters");
|
|
}
|
|
|
|
#[test]
|
|
fn clustering_is_deterministic_and_separates_profiles() {
|
|
// Half the class is strong on recall and weak on application; half is
|
|
// uniformly strong. Those are different problems.
|
|
let mut students = Vec::new();
|
|
for i in 0..12 {
|
|
let mut s = summary(&format!("s{i:02}"));
|
|
let (recall, apply) = if i < 6 { (0.95, 0.35) } else { (0.9, 0.85) };
|
|
s.levels = vec![
|
|
profile(Level::Remember, recall),
|
|
profile(Level::Apply, apply),
|
|
];
|
|
students.push(s);
|
|
}
|
|
let first = cluster(&students, 2);
|
|
let second = cluster(&students, 2);
|
|
assert_eq!(first.len(), 2);
|
|
assert_eq!(
|
|
first.iter().map(|a| a.members.clone()).collect::<Vec<_>>(),
|
|
second.iter().map(|a| a.members.clone()).collect::<Vec<_>>(),
|
|
"clustering must be reproducible"
|
|
);
|
|
// The two groups must not be mixed together.
|
|
let sizes: Vec<usize> = first.iter().map(|a| a.members.len()).collect();
|
|
assert_eq!(sizes, vec![6, 6], "got {sizes:?}");
|
|
assert!(first.iter().any(|a| a.label.contains("struggles to apply")));
|
|
}
|
|
|
|
fn make(
|
|
student: &str,
|
|
number: u32,
|
|
credit: f64,
|
|
objectives: &[&str],
|
|
level: Option<Level>,
|
|
) -> Response {
|
|
Response {
|
|
administration_id: "C/T/a".into(),
|
|
course: "C".into(),
|
|
term: "T".into(),
|
|
assessment_id: "a".into(),
|
|
date: None,
|
|
form: None,
|
|
student_key: student.into(),
|
|
sid: None,
|
|
name: None,
|
|
email: None,
|
|
section: None,
|
|
item_number: number,
|
|
item_ref: None,
|
|
item_version: None,
|
|
selected: vec!["A".into()],
|
|
eliminated: vec![],
|
|
correct: Some(credit >= 0.999),
|
|
credit,
|
|
points_possible: 1.0,
|
|
score: credit,
|
|
response_time_seconds: None,
|
|
level,
|
|
learning_objectives: objectives.iter().map(|s| s.to_string()).collect(),
|
|
topics: vec![],
|
|
bonus: false,
|
|
dropped: false,
|
|
}
|
|
}
|
|
|
|
fn summary(key: &str) -> StudentSummary {
|
|
StudentSummary {
|
|
student_key: key.to_string(),
|
|
name: None,
|
|
sid: None,
|
|
points: 0.0,
|
|
points_possible: 0.0,
|
|
percent: 0.0,
|
|
bonus_points: 0.0,
|
|
correct: 0,
|
|
n_items: 0,
|
|
band: String::new(),
|
|
theta: None,
|
|
theta_se: None,
|
|
objectives: Vec::new(),
|
|
levels: Vec::new(),
|
|
strengths: Vec::new(),
|
|
focus: Vec::new(),
|
|
missed: Vec::new(),
|
|
}
|
|
}
|
|
|
|
fn profile(level: Level, rate: f64) -> LevelProfile {
|
|
LevelProfile {
|
|
level,
|
|
n_items: 4,
|
|
rate,
|
|
cohort_rate: rate,
|
|
z: None,
|
|
}
|
|
}
|
|
}
|