1417 lines
47 KiB
Rust
1417 lines
47 KiB
Rust
// SPDX-License-Identifier: Prosperity-3.0.0
|
|
// Copyright Scientific Computing Studio
|
|
// Source: https://git.scient.ing/education/coursebank
|
|
|
|
//! Turning responses into something you can say to a student.
|
|
//!
|
|
//! Item analysis tells you about items. This module tells you about people: which
|
|
//! objectives a student has actually met, which ones they are close on, and what
|
|
//! specifically to do next.
|
|
//!
|
|
//! # On declaring mastery from three questions
|
|
//!
|
|
//! The honest answer is that you often cannot. Two items on an objective give a
|
|
//! proportion with an enormous confidence interval — two out of two correct is
|
|
//! consistent with a true rate anywhere from about 0.55 upward. So this module
|
|
//! does three things instead of pretending otherwise.
|
|
//!
|
|
//! It refuses to classify at all below `min_items_for_mastery`, reporting "not
|
|
//! enough evidence", which is a finding about your blueprint rather than about the
|
|
//! student. It reports the Wilson score interval alongside every rate, because
|
|
//! Wilson behaves sensibly at the boundaries where the normal approximation
|
|
//! produces intervals extending past 1.0. And it separates the *classification*
|
|
//! (which uses the observed rate, so it is usable) from the *confidence* (which
|
|
//! uses the interval, so it is honest). A student can be "meeting" an objective
|
|
//! provisionally, and the report says so.
|
|
//!
|
|
//! # Which tier gets classified
|
|
//!
|
|
//! The registry has two tiers, objectives and their targets (see
|
|
//! [`crate::course::Objective`]), and they are reported differently because the
|
|
//! evidence behind them differs in kind. An **objective** is classified: its
|
|
//! denominator is every item tagged to any of its targets, which is how an exam
|
|
//! that spends twelve questions across a topic gets to make one statement with a
|
|
//! real denominator instead of twelve statements with none.
|
|
//!
|
|
//! A **target** is not classified. It usually carries one or two items, and
|
|
//! `min_items_for_mastery` would mark almost all of them "not enough evidence",
|
|
//! which would be true but useless. So target rows report the observed rate as
|
|
//! itemized evidence for the objective's classification, and a report should
|
|
//! present them that way: not "you have not mastered this" but "here is what you
|
|
//! missed inside the objective above".
|
|
//!
|
|
//! An item tagged with two targets of the same objective counts *once* toward
|
|
//! that objective. Double counting is right across unrelated objectives, where
|
|
//! the question "how is this student doing on kinetics" should use every item
|
|
//! that measured kinetics, but within one denominator it would inflate both the
|
|
//! count and the confidence.
|
|
//!
|
|
//! # Comparison to the cohort
|
|
//!
|
|
//! Per-level performance is reported against the class rather than in absolute
|
|
//! terms, because "you got 60% of the Analyze items" means nothing to a student
|
|
//! without knowing that the class average was 55%. The comparison is descriptive,
|
|
//! not a curve.
|
|
|
|
use std::collections::{BTreeMap, BTreeSet};
|
|
|
|
use crate::course::CourseFile;
|
|
use crate::responses::{Response, ResponseSet};
|
|
use crate::rng::Rng;
|
|
use crate::taxonomy::{Level, Tier};
|
|
|
|
/// How well a student has met one objective.
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub enum Mastery {
|
|
/// Met the threshold.
|
|
Meeting,
|
|
/// Partway there.
|
|
Developing,
|
|
/// Not yet.
|
|
NotYet,
|
|
/// Too few items on this objective to say anything. This is a gap in the
|
|
/// assessment, not a judgment about the student.
|
|
NotEnoughEvidence,
|
|
}
|
|
|
|
impl Mastery {
|
|
/// A label for reports.
|
|
pub fn label(self) -> &'static str {
|
|
match self {
|
|
Mastery::Meeting => "meeting",
|
|
Mastery::Developing => "developing",
|
|
Mastery::NotYet => "not yet",
|
|
Mastery::NotEnoughEvidence => "not enough evidence",
|
|
}
|
|
}
|
|
|
|
/// A short symbol for compact tables.
|
|
pub fn symbol(self) -> &'static str {
|
|
match self {
|
|
Mastery::Meeting => "✓",
|
|
Mastery::Developing => "~",
|
|
Mastery::NotYet => "✗",
|
|
Mastery::NotEnoughEvidence => "?",
|
|
}
|
|
}
|
|
}
|
|
|
|
/// One student's standing on one registry entry, at either tier.
|
|
#[derive(Debug, Clone)]
|
|
pub struct ObjectiveMastery {
|
|
/// The registry id this row reports on, at either tier.
|
|
pub id: String,
|
|
/// The objective text, for reports.
|
|
pub text: String,
|
|
/// Which tier this row is, since only one of them is a classification.
|
|
pub tier: Tier,
|
|
/// The objective this row sits under, for a target row.
|
|
pub objective: Option<String>,
|
|
/// For an objective row, how many of its targets the exam reached.
|
|
///
|
|
/// A student report can say "four of the nine things under this objective
|
|
/// were tested", which is the honest scope of the claim. Zero for a target
|
|
/// row and for an objective with no targets.
|
|
pub targets_seen: usize,
|
|
/// For an objective row, how many targets it has in the registry.
|
|
pub targets_total: usize,
|
|
/// How many items the student saw. For an objective row, items tagged to any
|
|
/// of its targets, counted once each.
|
|
pub n_items: usize,
|
|
/// How many they got right, counting partial credit.
|
|
pub credit: f64,
|
|
/// Observed rate, `credit / n_items`.
|
|
pub rate: f64,
|
|
/// Lower end of the Wilson interval.
|
|
pub wilson_lower: f64,
|
|
/// Upper end of the Wilson interval.
|
|
pub wilson_upper: f64,
|
|
/// The class's rate on the same objective.
|
|
pub cohort_rate: f64,
|
|
/// The classification.
|
|
pub status: Mastery,
|
|
/// Whether the interval, not just the point estimate, clears the threshold.
|
|
pub confident: bool,
|
|
/// Which levels the objective was assessed at, since meeting an objective at
|
|
/// Remember is a different claim from meeting it at Analyze.
|
|
pub levels: Vec<Level>,
|
|
}
|
|
|
|
/// One student's performance at one cognitive level.
|
|
#[derive(Debug, Clone)]
|
|
pub struct LevelProfile {
|
|
/// The level.
|
|
pub level: Level,
|
|
/// How many items at this level.
|
|
pub n_items: usize,
|
|
/// The student's rate.
|
|
pub rate: f64,
|
|
/// The class's rate.
|
|
pub cohort_rate: f64,
|
|
/// Difference from the class, in class standard deviations. `None` when the
|
|
/// class had no spread at this level.
|
|
pub z: Option<f64>,
|
|
}
|
|
|
|
impl LevelProfile {
|
|
/// A plain-language comparison to the class.
|
|
pub fn comparison(&self) -> &'static str {
|
|
match self.z {
|
|
Some(z) if z >= 1.0 => "well above the class",
|
|
Some(z) if z >= 0.4 => "above the class",
|
|
Some(z) if z > -0.4 => "about the same as the class",
|
|
Some(z) if z > -1.0 => "below the class",
|
|
Some(_) => "well below the class",
|
|
None => "the class did not vary here",
|
|
}
|
|
}
|
|
}
|
|
|
|
/// An item the student got wrong, with what to do about it.
|
|
#[derive(Debug, Clone)]
|
|
pub struct MissedItem {
|
|
/// The question number.
|
|
pub number: u32,
|
|
/// The item's global id.
|
|
pub item_ref: Option<String>,
|
|
/// What the student chose.
|
|
pub selected: Vec<String>,
|
|
/// Credit earned, since a partially credited response is not a clean miss.
|
|
pub credit: f64,
|
|
/// The level.
|
|
pub level: Option<Level>,
|
|
/// The learning targets the question measured.
|
|
pub learning_targets: Vec<String>,
|
|
/// The misconception the chosen distractor was written to detect.
|
|
pub misconception: Option<String>,
|
|
/// Feedback written for a student who chose that option.
|
|
pub feedback: Option<String>,
|
|
/// Where to go back to: lecture titles and slide numbers.
|
|
pub study: Vec<String>,
|
|
}
|
|
|
|
/// Everything needed to write one student's report.
|
|
#[derive(Debug, Clone)]
|
|
pub struct StudentSummary {
|
|
/// The grouping key.
|
|
pub student_key: String,
|
|
/// The name, when not pseudonymized.
|
|
pub name: Option<String>,
|
|
/// The student id, when not pseudonymized.
|
|
pub sid: Option<String>,
|
|
/// Points earned on scored items.
|
|
pub points: f64,
|
|
/// Points available on scored items.
|
|
pub points_possible: f64,
|
|
/// Percentage on scored items.
|
|
pub percent: f64,
|
|
/// Bonus points earned.
|
|
pub bonus_points: f64,
|
|
/// Items answered correctly.
|
|
pub correct: usize,
|
|
/// Items administered.
|
|
pub n_items: usize,
|
|
/// Where the score falls relative to the class, as a coarse band.
|
|
pub band: String,
|
|
/// IRT ability, when an IRT fit was supplied.
|
|
pub theta: Option<f64>,
|
|
/// Standard error of the ability estimate.
|
|
pub theta_se: Option<f64>,
|
|
/// Per-objective standing, in the course's objective order.
|
|
pub objectives: Vec<ObjectiveMastery>,
|
|
/// Per-level standing.
|
|
pub levels: Vec<LevelProfile>,
|
|
/// Objectives the student is clearly meeting.
|
|
pub strengths: Vec<String>,
|
|
/// Objectives to work on, worst first.
|
|
pub focus: Vec<String>,
|
|
/// Missed items with targeted guidance.
|
|
pub missed: Vec<MissedItem>,
|
|
}
|
|
|
|
impl StudentSummary {
|
|
/// The display name, falling back to the key.
|
|
pub fn display_name(&self) -> String {
|
|
self.name
|
|
.clone()
|
|
.unwrap_or_else(|| self.student_key.clone())
|
|
}
|
|
}
|
|
|
|
/// Class-level context.
|
|
#[derive(Debug, Clone)]
|
|
pub struct Cohort {
|
|
/// Per-student summaries, sorted by key.
|
|
pub students: Vec<StudentSummary>,
|
|
/// Class rate per target, which is the tier items are tagged at. Use it to
|
|
/// drill into an objective the class missed.
|
|
pub target_rates: BTreeMap<String, f64>,
|
|
/// Class rate per objective, with each item counted once.
|
|
///
|
|
/// This is the class-level table worth acting on, and the one
|
|
/// [`Cohort::class_gaps`] is drawn from.
|
|
pub objective_rates: BTreeMap<String, f64>,
|
|
/// Class rate per level.
|
|
pub level_rates: BTreeMap<Level, f64>,
|
|
/// Mean percentage.
|
|
pub mean_percent: f64,
|
|
/// Standard deviation of percentage.
|
|
pub sd_percent: f64,
|
|
/// Objectives the class as a whole did not meet, worst first. This is the
|
|
/// list that should change what you reteach.
|
|
///
|
|
/// Objectives rather than targets, because a list of forty targets below
|
|
/// threshold is a list nobody reteaches from, and because a target that
|
|
/// carried one item on this exam does not support the claim that the class
|
|
/// missed it.
|
|
pub class_gaps: Vec<(String, f64)>,
|
|
/// Optional grouping of students by response profile.
|
|
pub archetypes: Vec<Archetype>,
|
|
}
|
|
|
|
/// A cluster of students with a similar profile across levels.
|
|
#[derive(Debug, Clone)]
|
|
pub struct Archetype {
|
|
/// A label describing the pattern.
|
|
pub label: String,
|
|
/// The student keys in this cluster.
|
|
pub members: Vec<String>,
|
|
/// Mean rate at each level for this cluster.
|
|
pub level_means: BTreeMap<Level, f64>,
|
|
}
|
|
|
|
/// The Wilson score interval for a binomial proportion.
|
|
///
|
|
/// Preferred over the normal approximation because it stays inside `[0, 1]` and
|
|
/// behaves at the boundaries, which is exactly where classroom data lives: a
|
|
/// student who got three out of three needs an interval, and the textbook formula
|
|
/// gives width zero there.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `successes` - the number of successes, which may be fractional when partial
|
|
/// credit is involved.
|
|
/// * `n` - the number of trials.
|
|
/// * `z` - the standard normal quantile; 1.96 for a two-sided 95% interval.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The lower and upper bounds, or `(0.0, 1.0)` when there are no trials.
|
|
pub fn wilson(successes: f64, n: usize, z: f64) -> (f64, f64) {
|
|
if n == 0 {
|
|
return (0.0, 1.0);
|
|
}
|
|
let n = n as f64;
|
|
let p = (successes / n).clamp(0.0, 1.0);
|
|
let z2 = z * z;
|
|
let denominator = 1.0 + z2 / n;
|
|
let center = p + z2 / (2.0 * n);
|
|
let spread = z * ((p * (1.0 - p) / n) + z2 / (4.0 * n * n)).sqrt();
|
|
(
|
|
((center - spread) / denominator).clamp(0.0, 1.0),
|
|
((center + spread) / denominator).clamp(0.0, 1.0),
|
|
)
|
|
}
|
|
|
|
/// Builds per-student summaries for one administration.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `set` - the responses, already enriched with item metadata.
|
|
/// * `course` - the course, for objective text, order, and policy.
|
|
/// * `catalog` - the loaded course, for misconception feedback on missed items.
|
|
/// * `fit` - an optional IRT fit, whose abilities are attached when present.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The cohort.
|
|
pub fn summarize(
|
|
set: &ResponseSet,
|
|
course: &CourseFile,
|
|
catalog: Option<&crate::catalog::Catalog>,
|
|
fit: Option<&crate::irt::Fit>,
|
|
) -> Cohort {
|
|
let policy = &course.policy;
|
|
let students = set.students();
|
|
|
|
// Class rates first: every student's report is relative to these.
|
|
let target_rates = rates_by_target(&set.rows.iter().collect::<Vec<_>>());
|
|
let objective_rates = rates_by_objective(&set.rows.iter().collect::<Vec<_>>(), course);
|
|
let level_rates = rates_by_level(&set.rows.iter().collect::<Vec<_>>());
|
|
|
|
// Per-level spread across students, for the z comparisons.
|
|
let mut level_values: BTreeMap<Level, Vec<f64>> = BTreeMap::new();
|
|
for key in &students {
|
|
let rows = set.for_student(key);
|
|
for (level, rate) in rates_by_level(&rows) {
|
|
level_values.entry(level).or_default().push(rate);
|
|
}
|
|
}
|
|
let level_sd: BTreeMap<Level, f64> = level_values
|
|
.iter()
|
|
.map(|(level, values)| (*level, sd(values)))
|
|
.collect();
|
|
|
|
let percents: Vec<f64> = students
|
|
.iter()
|
|
.map(|key| {
|
|
let earned = set.scored_total(key);
|
|
let possible = set.points_available();
|
|
if possible > 0.0 {
|
|
100.0 * earned / possible
|
|
} else {
|
|
0.0
|
|
}
|
|
})
|
|
.collect();
|
|
let mean_percent = mean(&percents);
|
|
let sd_percent = sd(&percents);
|
|
|
|
let ability = fit.map(|f| f.ability_map()).unwrap_or_default();
|
|
let ability_se: BTreeMap<String, f64> = fit
|
|
.map(|f| {
|
|
f.abilities
|
|
.iter()
|
|
.map(|a| (a.student_key.clone(), a.se))
|
|
.collect()
|
|
})
|
|
.unwrap_or_default();
|
|
|
|
let order = course.objectives_in_order();
|
|
let mut summaries = Vec::with_capacity(students.len());
|
|
|
|
for (index, key) in students.iter().enumerate() {
|
|
let rows = set.for_student(key);
|
|
let points = set.scored_total(key);
|
|
let possible = set.points_available();
|
|
let percent = percents[index];
|
|
|
|
let correct = rows
|
|
.iter()
|
|
.filter(|r| r.counts() && r.correct == Some(true))
|
|
.count();
|
|
let n_items = rows.iter().filter(|r| r.counts()).count();
|
|
|
|
// The registry in the course's declared order, so a report reads the way
|
|
// the course is taught rather than alphabetically. Each objective the
|
|
// exam reached is followed by the targets it reached, which is the order
|
|
// a report wants them in: the claim, then its evidence.
|
|
let target_counts = counts_by_target(&rows);
|
|
let objective_counts = counts_by_objective(&rows, course);
|
|
let mut objectives = Vec::new();
|
|
let mut seen: BTreeSet<String> = BTreeSet::new();
|
|
|
|
// `order` puts each objective ahead of its own targets, so walking it
|
|
// produces the tiering. Anything the exam measured that the registry
|
|
// does not know about is appended afterwards rather than dropped.
|
|
let measured: Vec<String> = target_counts.keys().cloned().collect();
|
|
for id in order.iter().cloned().chain(measured) {
|
|
if !seen.insert(id.clone()) {
|
|
continue;
|
|
}
|
|
if course.is_objective(&id) {
|
|
let Some((n, credit)) = objective_counts.get(&id).copied() else {
|
|
continue;
|
|
};
|
|
let targets = course.targets(&id);
|
|
let reached = targets
|
|
.iter()
|
|
.filter(|target| target_counts.contains_key(**target))
|
|
.count();
|
|
objectives.push(objective_mastery(
|
|
&id,
|
|
course,
|
|
n,
|
|
credit,
|
|
objective_rates.get(&id).copied().unwrap_or(0.0),
|
|
&rows,
|
|
policy.min_items_for_mastery.max(1),
|
|
reached,
|
|
targets.len(),
|
|
));
|
|
} else {
|
|
let Some((n, credit)) = target_counts.get(&id).copied() else {
|
|
continue;
|
|
};
|
|
// One item is the normal case for a target, so it is reported
|
|
// rather than withheld. The objective row above it carries the
|
|
// classification.
|
|
objectives.push(objective_mastery(
|
|
&id,
|
|
course,
|
|
n,
|
|
credit,
|
|
target_rates.get(&id).copied().unwrap_or(0.0),
|
|
&rows,
|
|
1,
|
|
0,
|
|
0,
|
|
));
|
|
}
|
|
}
|
|
|
|
// Levels.
|
|
let student_levels = rates_by_level(&rows);
|
|
let level_counts = counts_by_level(&rows);
|
|
let levels: Vec<LevelProfile> = Level::ALL
|
|
.iter()
|
|
.filter_map(|level| {
|
|
let (n, _) = level_counts.get(level).copied()?;
|
|
if n == 0 {
|
|
return None;
|
|
}
|
|
let rate = student_levels.get(level).copied().unwrap_or(0.0);
|
|
let cohort_rate = level_rates.get(level).copied().unwrap_or(0.0);
|
|
let spread = level_sd.get(level).copied().unwrap_or(0.0);
|
|
Some(LevelProfile {
|
|
level: *level,
|
|
n_items: n,
|
|
rate,
|
|
cohort_rate,
|
|
z: if spread > 1e-9 {
|
|
Some((rate - cohort_rate) / spread)
|
|
} else {
|
|
None
|
|
},
|
|
})
|
|
})
|
|
.collect();
|
|
|
|
// Strengths and focus areas. Strengths need confidence, focus areas do
|
|
// not: telling a student to review something they may already know costs
|
|
// them an hour, while telling them they have mastered something they have
|
|
// not costs them the next exam.
|
|
//
|
|
// Both lists are drawn from objective rows only. A focus list built from
|
|
// targets is as long as the exam and tells a student to review forty
|
|
// things, which is the same as telling them nothing; the objective list
|
|
// is short enough to act on, and the target rows underneath it say what
|
|
// to look at within each one.
|
|
let strengths: Vec<String> = objectives
|
|
.iter()
|
|
.filter(|o| o.tier == Tier::Objective && o.status == Mastery::Meeting && o.confident)
|
|
.map(|o| o.id.clone())
|
|
.collect();
|
|
let mut focus_pairs: Vec<(&ObjectiveMastery, f64)> = objectives
|
|
.iter()
|
|
.filter(|o| o.tier == Tier::Objective)
|
|
.filter(|o| matches!(o.status, Mastery::NotYet | Mastery::Developing))
|
|
.map(|o| (o, o.rate))
|
|
.collect();
|
|
focus_pairs.sort_by(|a, b| {
|
|
a.1.partial_cmp(&b.1)
|
|
.unwrap_or(std::cmp::Ordering::Equal)
|
|
.then_with(|| a.0.id.cmp(&b.0.id))
|
|
});
|
|
let focus: Vec<String> = focus_pairs.iter().map(|(o, _)| o.id.clone()).collect();
|
|
|
|
let missed = missed_items(&rows, catalog, course);
|
|
|
|
summaries.push(StudentSummary {
|
|
student_key: key.clone(),
|
|
name: rows.first().and_then(|r| r.name.clone()),
|
|
sid: rows.first().and_then(|r| r.sid.clone()),
|
|
points,
|
|
points_possible: possible,
|
|
percent,
|
|
bonus_points: set.bonus_total(key),
|
|
correct,
|
|
n_items,
|
|
band: band_for(percent, &percents),
|
|
theta: ability.get(key).copied(),
|
|
theta_se: ability_se.get(key).copied(),
|
|
objectives,
|
|
levels,
|
|
strengths,
|
|
focus,
|
|
missed,
|
|
});
|
|
}
|
|
|
|
// Class gaps: objectives where the whole class fell short. These are the ones
|
|
// to reteach rather than to send individual students away to review.
|
|
let mut class_gaps: Vec<(String, f64)> = objective_rates
|
|
.iter()
|
|
.filter(|(_, rate)| **rate < policy.mastery_threshold)
|
|
.map(|(id, rate)| (id.clone(), *rate))
|
|
.collect();
|
|
class_gaps.sort_by(|a, b| {
|
|
a.1.partial_cmp(&b.1)
|
|
.unwrap_or(std::cmp::Ordering::Equal)
|
|
.then_with(|| a.0.cmp(&b.0))
|
|
});
|
|
|
|
let archetypes = cluster(&summaries, 3);
|
|
|
|
Cohort {
|
|
students: summaries,
|
|
target_rates,
|
|
objective_rates,
|
|
level_rates,
|
|
mean_percent,
|
|
sd_percent,
|
|
class_gaps,
|
|
archetypes,
|
|
}
|
|
}
|
|
|
|
/// Builds one objective's record, at either tier.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `id` - the objective id.
|
|
/// * `course` - the course, for text, tier, and policy.
|
|
/// * `n` - items counting toward this row.
|
|
/// * `credit` - total credit earned.
|
|
/// * `cohort_rate` - the class rate for the same row.
|
|
/// * `rows` - the student's responses, for the level list.
|
|
/// * `min_items` - items required before the row is classified. The policy's
|
|
/// `min_items_for_mastery` for an objective row, and 1 for a target row, which
|
|
/// is evidence rather than a classification.
|
|
/// * `targets_seen` - targets this exam reached, for an objective row.
|
|
/// * `targets_total` - targets in the registry, for an objective row.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The record.
|
|
#[allow(clippy::too_many_arguments)]
|
|
fn objective_mastery(
|
|
id: &str,
|
|
course: &CourseFile,
|
|
n: usize,
|
|
credit: f64,
|
|
cohort_rate: f64,
|
|
rows: &[&Response],
|
|
min_items: usize,
|
|
targets_seen: usize,
|
|
targets_total: usize,
|
|
) -> ObjectiveMastery {
|
|
let policy = &course.policy;
|
|
let rate = if n > 0 { credit / n as f64 } else { 0.0 };
|
|
let (lower, upper) = wilson(credit, n, 1.96);
|
|
|
|
let status = if n < min_items.max(1) {
|
|
Mastery::NotEnoughEvidence
|
|
} else if rate >= policy.mastery_threshold {
|
|
Mastery::Meeting
|
|
} else if rate >= policy.mastery_threshold * 0.6 {
|
|
Mastery::Developing
|
|
} else {
|
|
Mastery::NotYet
|
|
};
|
|
|
|
// Levels the row was assessed at. For an objective row this is every level
|
|
// any of its targets was assessed at, which is what makes "met this
|
|
// objective" a checkable claim: meeting it on three Remember items is a
|
|
// different statement from meeting it on three Analyze items.
|
|
let levels: Vec<Level> = rows
|
|
.iter()
|
|
.filter(|r| {
|
|
r.learning_targets
|
|
.iter()
|
|
.any(|t| t == id || course.objective_for(t) == id)
|
|
})
|
|
.filter_map(|r| r.level)
|
|
.collect::<BTreeSet<Level>>()
|
|
.into_iter()
|
|
.collect();
|
|
|
|
ObjectiveMastery {
|
|
id: id.to_string(),
|
|
text: course.text_for(id),
|
|
tier: if course.is_objective(id) {
|
|
Tier::Objective
|
|
} else {
|
|
Tier::Target
|
|
},
|
|
objective: course
|
|
.is_target(id)
|
|
.then(|| course.objective_for(id).to_string()),
|
|
targets_seen,
|
|
targets_total,
|
|
n_items: n,
|
|
credit,
|
|
rate,
|
|
wilson_lower: lower,
|
|
wilson_upper: upper,
|
|
cohort_rate,
|
|
status,
|
|
confident: lower >= policy.mastery_threshold,
|
|
levels,
|
|
}
|
|
}
|
|
|
|
/// Collects missed items with targeted guidance.
|
|
///
|
|
/// The guidance comes from the item's own authoring: the `misconception` recorded
|
|
/// on the distractor the student actually chose, and the lecture and slides the
|
|
/// item was written from. This is why authoring distractors deliberately pays off
|
|
/// twice — once when writing the item, and again in every report afterward.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `rows` - the student's responses.
|
|
/// * `catalog` - the loaded course.
|
|
/// * `course` - the course, for lecture titles.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The missed items, in question order.
|
|
fn missed_items(
|
|
rows: &[&Response],
|
|
catalog: Option<&crate::catalog::Catalog>,
|
|
course: &CourseFile,
|
|
) -> Vec<MissedItem> {
|
|
let mut out = Vec::new();
|
|
for r in rows {
|
|
if !r.counts() || r.credit >= 0.999 {
|
|
continue;
|
|
}
|
|
let mut misconception = None;
|
|
let mut feedback = None;
|
|
let mut study = Vec::new();
|
|
|
|
if let (Some(cat), Some(uid)) = (catalog, r.item_ref.as_deref()) {
|
|
if let Some(entry) = cat.get(uid) {
|
|
// Feedback for the specific option chosen, which is the whole
|
|
// point of recording per-distractor misconceptions.
|
|
if let Some(letter) = r.chosen().first() {
|
|
if let Some(choice) = entry.item.option(letter) {
|
|
misconception = choice.misconception.clone();
|
|
feedback = choice.student_text().map(|s| s.to_string());
|
|
}
|
|
}
|
|
for source in &entry.item.sources {
|
|
let title = course
|
|
.lectures
|
|
.get(&source.lecture)
|
|
.map(|l| l.title.clone())
|
|
.unwrap_or_else(|| source.lecture.clone());
|
|
if source.slides.is_empty() {
|
|
study.push(title);
|
|
} else {
|
|
let slides: Vec<String> =
|
|
source.slides.iter().map(|s| s.to_string()).collect();
|
|
study.push(format!("{title}, slides {}", slides.join(", ")));
|
|
}
|
|
for reading in &source.readings {
|
|
study.push(reading.clone());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
out.push(MissedItem {
|
|
number: r.item_number,
|
|
item_ref: r.item_ref.clone(),
|
|
selected: r.chosen().to_vec(),
|
|
credit: r.credit,
|
|
level: r.level,
|
|
learning_targets: r.learning_targets.clone(),
|
|
misconception,
|
|
feedback,
|
|
study,
|
|
});
|
|
}
|
|
out
|
|
}
|
|
|
|
/// Credit rate per objective over a set of responses.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `rows` - the responses.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The rate for each target mentioned.
|
|
pub fn rates_by_target(rows: &[&Response]) -> BTreeMap<String, f64> {
|
|
counts_by_target(rows)
|
|
.into_iter()
|
|
.map(|(id, (n, credit))| {
|
|
let rate = if n > 0 { credit / n as f64 } else { 0.0 };
|
|
(id, rate)
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
/// Item counts and credit per target, as tagged.
|
|
///
|
|
/// An item tagged with two targets counts toward both. That double counting is
|
|
/// intentional: the question "how is this student doing on kinetics" should use
|
|
/// every item that measured kinetics. Roll-up to the objective, where the same
|
|
/// item must count once, is [`counts_by_objective`].
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `rows` - the responses.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// `(item count, total credit)` per target.
|
|
pub fn counts_by_target(rows: &[&Response]) -> BTreeMap<String, (usize, f64)> {
|
|
let mut out: BTreeMap<String, (usize, f64)> = BTreeMap::new();
|
|
for r in rows {
|
|
if !r.counts() {
|
|
continue;
|
|
}
|
|
for objective in &r.learning_targets {
|
|
let e = out.entry(objective.clone()).or_insert((0, 0.0));
|
|
e.0 += 1;
|
|
e.1 += r.credit.clamp(0.0, 1.0);
|
|
}
|
|
}
|
|
out
|
|
}
|
|
|
|
/// Item counts and credit per objective.
|
|
///
|
|
/// Each response contributes at most once to any one objective, even when it is
|
|
/// tagged with several of that objective's targets. Within a single denominator,
|
|
/// counting an item twice would inflate both the rate's weight and the
|
|
/// confidence interval's tightness, and the interval is the part of the report
|
|
/// that is supposed to stay honest. Across unrelated objectives an item still
|
|
/// counts toward each, as it does in [`counts_by_target`].
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `rows` - the responses.
|
|
/// * `course` - the course, for the objective each tagged target belongs to.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// `(item count, total credit)` per objective.
|
|
pub fn counts_by_objective(
|
|
rows: &[&Response],
|
|
course: &CourseFile,
|
|
) -> BTreeMap<String, (usize, f64)> {
|
|
let mut out: BTreeMap<String, (usize, f64)> = BTreeMap::new();
|
|
for r in rows {
|
|
if !r.counts() {
|
|
continue;
|
|
}
|
|
let objectives: BTreeSet<&str> = r
|
|
.learning_targets
|
|
.iter()
|
|
.map(|t| course.objective_for(t))
|
|
.collect();
|
|
for id in objectives {
|
|
let e = out.entry(id.to_string()).or_insert((0, 0.0));
|
|
e.0 += 1;
|
|
e.1 += r.credit.clamp(0.0, 1.0);
|
|
}
|
|
}
|
|
out
|
|
}
|
|
|
|
/// Credit rate per objective.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `rows` - the responses.
|
|
/// * `course` - the course, for the objective each tagged target belongs to.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The rate for each objective the responses reached.
|
|
pub fn rates_by_objective(rows: &[&Response], course: &CourseFile) -> BTreeMap<String, f64> {
|
|
counts_by_objective(rows, course)
|
|
.into_iter()
|
|
.map(|(id, (n, credit))| {
|
|
let rate = if n > 0 { credit / n as f64 } else { 0.0 };
|
|
(id, rate)
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
/// Credit rate per level.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `rows` - the responses.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The rate for each level present.
|
|
pub fn rates_by_level(rows: &[&Response]) -> BTreeMap<Level, f64> {
|
|
counts_by_level(rows)
|
|
.into_iter()
|
|
.map(|(level, (n, credit))| {
|
|
let rate = if n > 0 { credit / n as f64 } else { 0.0 };
|
|
(level, rate)
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
/// Item counts and credit per level.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `rows` - the responses.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// `(item count, total credit)` per level.
|
|
pub fn counts_by_level(rows: &[&Response]) -> BTreeMap<Level, (usize, f64)> {
|
|
let mut out: BTreeMap<Level, (usize, f64)> = BTreeMap::new();
|
|
for r in rows {
|
|
if !r.counts() {
|
|
continue;
|
|
}
|
|
if let Some(level) = r.level {
|
|
let e = out.entry(level).or_insert((0, 0.0));
|
|
e.0 += 1;
|
|
e.1 += r.credit.clamp(0.0, 1.0);
|
|
}
|
|
}
|
|
out
|
|
}
|
|
|
|
/// A coarse band for a score within a class.
|
|
///
|
|
/// Quartile bands rather than an exact percentile, because a percentile computed
|
|
/// from twenty-four students implies a precision it does not have, and because
|
|
/// telling a student they are "37th percentile" invites comparison in a way that
|
|
/// "middle half of the class" does not.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `percent` - the student's percentage.
|
|
/// * `all` - every student's percentage.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The band label.
|
|
fn band_for(percent: f64, all: &[f64]) -> String {
|
|
if all.len() < 4 {
|
|
return "the class is too small to place this meaningfully".to_string();
|
|
}
|
|
let mut sorted = all.to_vec();
|
|
sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
|
|
let below = sorted.iter().filter(|x| **x < percent).count() as f64;
|
|
let fraction = below / all.len() as f64;
|
|
if fraction >= 0.75 {
|
|
"top quarter of the class".to_string()
|
|
} else if fraction >= 0.5 {
|
|
"upper middle of the class".to_string()
|
|
} else if fraction >= 0.25 {
|
|
"lower middle of the class".to_string()
|
|
} else {
|
|
"bottom quarter of the class".to_string()
|
|
}
|
|
}
|
|
|
|
/// Groups students by their profile across levels.
|
|
///
|
|
/// This is descriptive, not diagnostic. It answers "are there recognizable
|
|
/// patterns in how this class is struggling" — for instance a group that handles
|
|
/// recall fine and falls apart on application, which calls for different
|
|
/// instruction than a group that is uniformly behind.
|
|
///
|
|
/// k-means with a seeded, deterministic initialization, so the same data always
|
|
/// produces the same groups.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `students` - the summaries.
|
|
/// * `k` - how many clusters to look for.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The clusters, largest first. Empty when there are too few students to bother.
|
|
pub fn cluster(students: &[StudentSummary], k: usize) -> Vec<Archetype> {
|
|
// Below about three students per cluster the groups are noise.
|
|
if students.len() < k * 3 || k == 0 {
|
|
return Vec::new();
|
|
}
|
|
|
|
// Feature vector: rate at each level that anyone was assessed on.
|
|
let levels: Vec<Level> = students
|
|
.iter()
|
|
.flat_map(|s| s.levels.iter().map(|l| l.level))
|
|
.collect::<BTreeSet<Level>>()
|
|
.into_iter()
|
|
.collect();
|
|
if levels.len() < 2 {
|
|
return Vec::new();
|
|
}
|
|
|
|
let points: Vec<Vec<f64>> = students
|
|
.iter()
|
|
.map(|s| {
|
|
levels
|
|
.iter()
|
|
.map(|level| {
|
|
s.levels
|
|
.iter()
|
|
.find(|l| l.level == *level)
|
|
.map(|l| l.rate)
|
|
.unwrap_or(0.0)
|
|
})
|
|
.collect()
|
|
})
|
|
.collect();
|
|
|
|
// Standardize each dimension so a level everyone did well on does not
|
|
// dominate the distance.
|
|
let mut standardized = points.clone();
|
|
for d in 0..levels.len() {
|
|
let column: Vec<f64> = points.iter().map(|p| p[d]).collect();
|
|
let m = mean(&column);
|
|
let s = sd(&column);
|
|
for (i, point) in standardized.iter_mut().enumerate() {
|
|
point[d] = if s > 1e-9 {
|
|
(points[i][d] - m) / s
|
|
} else {
|
|
0.0
|
|
};
|
|
}
|
|
}
|
|
|
|
// Seeded k-means++ initialization.
|
|
let mut rng = Rng::from_label("coursebank/archetypes");
|
|
let mut centers: Vec<Vec<f64>> =
|
|
vec![standardized[rng.below(standardized.len() as u64) as usize].clone()];
|
|
while centers.len() < k {
|
|
let distances: Vec<f64> = standardized
|
|
.iter()
|
|
.map(|p| {
|
|
centers
|
|
.iter()
|
|
.map(|c| squared_distance(p, c))
|
|
.fold(f64::INFINITY, f64::min)
|
|
})
|
|
.collect();
|
|
let total: f64 = distances.iter().sum();
|
|
if total <= 0.0 {
|
|
break;
|
|
}
|
|
let mut target = rng.unit() * total;
|
|
let mut chosen = standardized.len() - 1;
|
|
for (i, d) in distances.iter().enumerate() {
|
|
target -= d;
|
|
if target <= 0.0 {
|
|
chosen = i;
|
|
break;
|
|
}
|
|
}
|
|
centers.push(standardized[chosen].clone());
|
|
}
|
|
|
|
let mut assignment = vec![0usize; standardized.len()];
|
|
for _ in 0..50 {
|
|
let mut changed = false;
|
|
for (i, p) in standardized.iter().enumerate() {
|
|
let mut best = (0usize, f64::INFINITY);
|
|
for (c, center) in centers.iter().enumerate() {
|
|
let d = squared_distance(p, center);
|
|
if d < best.1 {
|
|
best = (c, d);
|
|
}
|
|
}
|
|
if assignment[i] != best.0 {
|
|
assignment[i] = best.0;
|
|
changed = true;
|
|
}
|
|
}
|
|
for (c, center) in centers.iter_mut().enumerate() {
|
|
let members: Vec<&Vec<f64>> = standardized
|
|
.iter()
|
|
.enumerate()
|
|
.filter(|(i, _)| assignment[*i] == c)
|
|
.map(|(_, p)| p)
|
|
.collect();
|
|
if members.is_empty() {
|
|
continue;
|
|
}
|
|
for d in 0..levels.len() {
|
|
center[d] = members.iter().map(|p| p[d]).sum::<f64>() / members.len() as f64;
|
|
}
|
|
}
|
|
if !changed {
|
|
break;
|
|
}
|
|
}
|
|
|
|
let mut out = Vec::new();
|
|
for c in 0..centers.len() {
|
|
let members: Vec<String> = students
|
|
.iter()
|
|
.enumerate()
|
|
.filter(|(i, _)| assignment[*i] == c)
|
|
.map(|(_, s)| s.student_key.clone())
|
|
.collect();
|
|
if members.is_empty() {
|
|
continue;
|
|
}
|
|
let mut level_means = BTreeMap::new();
|
|
for (d, level) in levels.iter().enumerate() {
|
|
let values: Vec<f64> = students
|
|
.iter()
|
|
.enumerate()
|
|
.filter(|(i, _)| assignment[*i] == c)
|
|
.map(|(i, _)| points[i][d])
|
|
.collect();
|
|
level_means.insert(*level, mean(&values));
|
|
}
|
|
out.push(Archetype {
|
|
label: label_for(&level_means),
|
|
members,
|
|
level_means,
|
|
});
|
|
}
|
|
out.sort_by_key(|b| std::cmp::Reverse(b.members.len()));
|
|
out
|
|
}
|
|
|
|
/// Names a cluster from its level profile.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `means` - mean rate at each level.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// A descriptive label.
|
|
fn label_for(means: &BTreeMap<Level, f64>) -> String {
|
|
let values: Vec<f64> = means.values().copied().collect();
|
|
let overall = mean(&values);
|
|
|
|
// Is the profile flat, or does it fall off with cognitive demand?
|
|
let low: Vec<f64> = means
|
|
.iter()
|
|
.filter(|(l, _)| l.code() <= 2)
|
|
.map(|(_, v)| *v)
|
|
.collect();
|
|
let high: Vec<f64> = means
|
|
.iter()
|
|
.filter(|(l, _)| l.code() >= 3)
|
|
.map(|(_, v)| *v)
|
|
.collect();
|
|
|
|
if !low.is_empty() && !high.is_empty() {
|
|
let drop = mean(&low) - mean(&high);
|
|
if drop > 0.25 {
|
|
return "knows the material, struggles to apply it".to_string();
|
|
}
|
|
if drop < -0.15 {
|
|
return "reasons well, gaps in recall".to_string();
|
|
}
|
|
}
|
|
|
|
if overall >= 0.85 {
|
|
"consistently strong".to_string()
|
|
} else if overall >= 0.65 {
|
|
"solid with scattered gaps".to_string()
|
|
} else {
|
|
"behind across the board".to_string()
|
|
}
|
|
}
|
|
|
|
/// Squared Euclidean distance.
|
|
fn squared_distance(a: &[f64], b: &[f64]) -> f64 {
|
|
a.iter().zip(b.iter()).map(|(x, y)| (x - y) * (x - y)).sum()
|
|
}
|
|
|
|
/// The arithmetic mean, zero for an empty slice.
|
|
fn mean(v: &[f64]) -> f64 {
|
|
if v.is_empty() {
|
|
0.0
|
|
} else {
|
|
v.iter().sum::<f64>() / v.len() as f64
|
|
}
|
|
}
|
|
|
|
/// The population standard deviation.
|
|
fn sd(v: &[f64]) -> f64 {
|
|
if v.len() < 2 {
|
|
return 0.0;
|
|
}
|
|
let m = mean(v);
|
|
(v.iter().map(|x| (x - m) * (x - m)).sum::<f64>() / v.len() as f64).sqrt()
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn wilson_stays_inside_zero_and_one() {
|
|
// Three out of three: the naive interval has zero width, Wilson does not.
|
|
let (lo, hi) = wilson(3.0, 3, 1.96);
|
|
assert!(lo > 0.0 && lo < 1.0, "lower bound {lo}");
|
|
assert_eq!(hi, 1.0);
|
|
assert!(lo < 0.5, "three items cannot establish a high rate: {lo}");
|
|
|
|
// Zero out of four.
|
|
let (lo, hi) = wilson(0.0, 4, 1.96);
|
|
assert_eq!(lo, 0.0);
|
|
assert!(hi > 0.0 && hi < 1.0);
|
|
|
|
// No data at all.
|
|
assert_eq!(wilson(0.0, 0, 1.96), (0.0, 1.0));
|
|
}
|
|
|
|
#[test]
|
|
fn wilson_narrows_as_n_grows() {
|
|
let (lo_small, hi_small) = wilson(8.0, 10, 1.96);
|
|
let (lo_big, hi_big) = wilson(80.0, 100, 1.96);
|
|
assert!(
|
|
(hi_big - lo_big) < (hi_small - lo_small),
|
|
"more data must give a tighter interval"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn two_items_never_claim_confident_mastery() {
|
|
// The classification may say "meeting", but confidence must not, because
|
|
// two items cannot establish a rate of 0.75.
|
|
let (lower, _) = wilson(2.0, 2, 1.96);
|
|
assert!(lower < 0.75, "got {lower}");
|
|
}
|
|
|
|
#[test]
|
|
fn bands_describe_position_coarsely() {
|
|
let all = vec![50.0, 60.0, 70.0, 80.0, 90.0, 95.0, 40.0, 30.0];
|
|
assert!(band_for(95.0, &all).contains("top"));
|
|
assert!(band_for(30.0, &all).contains("bottom"));
|
|
// Too few students to place anyone.
|
|
assert!(band_for(50.0, &[50.0, 60.0]).contains("too small"));
|
|
}
|
|
|
|
#[test]
|
|
fn target_counts_credit_every_tagged_item() {
|
|
let rows = [
|
|
make("s1", 1, 1.0, &["lo-a", "lo-b"], Some(Level::Remember)),
|
|
make("s1", 2, 0.0, &["lo-a"], Some(Level::Apply)),
|
|
];
|
|
let refs: Vec<&Response> = rows.iter().collect();
|
|
let counts = counts_by_target(&refs);
|
|
// lo-a saw both items; lo-b only the first.
|
|
assert_eq!(counts["lo-a"], (2, 1.0));
|
|
assert_eq!(counts["lo-b"], (1, 1.0));
|
|
let rates = rates_by_target(&refs);
|
|
assert_eq!(rates["lo-a"], 0.5);
|
|
assert_eq!(rates["lo-b"], 1.0);
|
|
}
|
|
|
|
/// A course with one objective over three targets, plus an objective with
|
|
/// no targets of its own.
|
|
fn tiered_course() -> CourseFile {
|
|
serde_yaml_ng::from_str(
|
|
r#"
|
|
course: { code: X, title: Y, term: Z }
|
|
policy: { mastery_threshold: 0.75, min_items_for_mastery: 2 }
|
|
learning_objectives:
|
|
lo-binding: { text: Quantify binding., order: 1 }
|
|
lo-standalone: { text: Untiered objective., order: 2 }
|
|
learning_targets:
|
|
t-kd: { text: Write the expression., objective: lo-binding, order: 1 }
|
|
t-plot: { text: Read a plot., objective: lo-binding, order: 2 }
|
|
t-window: { text: State the switching window., objective: lo-binding, order: 3 }
|
|
"#,
|
|
)
|
|
.expect("course parses")
|
|
}
|
|
|
|
#[test]
|
|
fn items_on_targets_roll_up_to_their_objective() {
|
|
let course = tiered_course();
|
|
let rows = [
|
|
make("s1", 1, 1.0, &["t-kd"], Some(Level::Remember)),
|
|
make("s1", 2, 0.0, &["t-plot"], Some(Level::Apply)),
|
|
make("s1", 3, 1.0, &["t-window"], Some(Level::Understand)),
|
|
make("s1", 4, 1.0, &["lo-standalone"], Some(Level::Remember)),
|
|
];
|
|
let refs: Vec<&Response> = rows.iter().collect();
|
|
|
|
let objectives = counts_by_objective(&refs, &course);
|
|
// Three items, two credited, in one denominator.
|
|
assert_eq!(objectives["lo-binding"], (3, 2.0));
|
|
assert_eq!(objectives["lo-standalone"], (1, 1.0));
|
|
// The targets are not themselves objective rows.
|
|
assert!(!objectives.contains_key("t-kd"));
|
|
|
|
// As-tagged counts are still available for the drill-down.
|
|
let tagged = counts_by_target(&refs);
|
|
assert_eq!(tagged["t-kd"], (1, 1.0));
|
|
assert_eq!(tagged.len(), 4);
|
|
|
|
let rates = rates_by_objective(&refs, &course);
|
|
assert!((rates["lo-binding"] - 2.0 / 3.0).abs() < 1e-9);
|
|
}
|
|
|
|
#[test]
|
|
fn one_item_counts_once_toward_its_objective() {
|
|
let course = tiered_course();
|
|
// A single question tagged with two targets of the same objective.
|
|
let rows = [make(
|
|
"s1",
|
|
1,
|
|
0.0,
|
|
&["t-kd", "t-plot"],
|
|
Some(Level::Understand),
|
|
)];
|
|
let refs: Vec<&Response> = rows.iter().collect();
|
|
|
|
let objectives = counts_by_objective(&refs, &course);
|
|
assert_eq!(
|
|
objectives["lo-binding"],
|
|
(1, 0.0),
|
|
"one question is one item in the objective's denominator"
|
|
);
|
|
// Whereas as-tagged counting credits both targets, as it always has.
|
|
let tagged = counts_by_target(&refs);
|
|
assert_eq!(tagged["t-kd"], (1, 0.0));
|
|
assert_eq!(tagged["t-plot"], (1, 0.0));
|
|
}
|
|
|
|
#[test]
|
|
fn an_untiered_course_rolls_up_to_itself() {
|
|
// Adopting the second tier is optional: with no targets declared, every
|
|
// entry is an objective and the rolled-up counts equal the tagged ones.
|
|
let course: CourseFile = serde_yaml_ng::from_str(
|
|
r#"
|
|
course: { code: X, title: Y, term: Z }
|
|
learning_objectives:
|
|
lo-a: { text: A }
|
|
lo-b: { text: B }
|
|
"#,
|
|
)
|
|
.expect("course parses");
|
|
let rows = [
|
|
make("s1", 1, 1.0, &["lo-a", "lo-b"], Some(Level::Remember)),
|
|
make("s1", 2, 0.0, &["lo-a"], Some(Level::Apply)),
|
|
];
|
|
let refs: Vec<&Response> = rows.iter().collect();
|
|
assert_eq!(counts_by_objective(&refs, &course), counts_by_target(&refs));
|
|
}
|
|
|
|
#[test]
|
|
fn level_rates_ignore_untagged_items() {
|
|
let rows = [
|
|
make("s1", 1, 1.0, &[], Some(Level::Remember)),
|
|
make("s1", 2, 0.0, &[], None),
|
|
];
|
|
let refs: Vec<&Response> = rows.iter().collect();
|
|
let counts = counts_by_level(&refs);
|
|
assert_eq!(counts.len(), 1);
|
|
assert_eq!(counts[&Level::Remember], (1, 1.0));
|
|
}
|
|
|
|
#[test]
|
|
fn mastery_labels_are_stable() {
|
|
assert_eq!(Mastery::Meeting.label(), "meeting");
|
|
assert_eq!(Mastery::NotEnoughEvidence.label(), "not enough evidence");
|
|
}
|
|
|
|
#[test]
|
|
fn clustering_needs_enough_students() {
|
|
assert!(cluster(&[], 3).is_empty());
|
|
let few: Vec<StudentSummary> = (0..4).map(|i| summary(&format!("s{i}"))).collect();
|
|
assert!(cluster(&few, 3).is_empty(), "four students, three clusters");
|
|
}
|
|
|
|
#[test]
|
|
fn clustering_is_deterministic_and_separates_profiles() {
|
|
// Half the class is strong on recall and weak on application; half is
|
|
// uniformly strong. Those are different problems.
|
|
let mut students = Vec::new();
|
|
for i in 0..12 {
|
|
let mut s = summary(&format!("s{i:02}"));
|
|
let (recall, apply) = if i < 6 { (0.95, 0.35) } else { (0.9, 0.85) };
|
|
s.levels = vec![
|
|
profile(Level::Remember, recall),
|
|
profile(Level::Apply, apply),
|
|
];
|
|
students.push(s);
|
|
}
|
|
let first = cluster(&students, 2);
|
|
let second = cluster(&students, 2);
|
|
assert_eq!(first.len(), 2);
|
|
assert_eq!(
|
|
first.iter().map(|a| a.members.clone()).collect::<Vec<_>>(),
|
|
second.iter().map(|a| a.members.clone()).collect::<Vec<_>>(),
|
|
"clustering must be reproducible"
|
|
);
|
|
// The two groups must not be mixed together.
|
|
let sizes: Vec<usize> = first.iter().map(|a| a.members.len()).collect();
|
|
assert_eq!(sizes, vec![6, 6], "got {sizes:?}");
|
|
assert!(first.iter().any(|a| a.label.contains("struggles to apply")));
|
|
}
|
|
|
|
fn make(
|
|
student: &str,
|
|
number: u32,
|
|
credit: f64,
|
|
objectives: &[&str],
|
|
level: Option<Level>,
|
|
) -> Response {
|
|
Response {
|
|
administration_id: "C/T/a".into(),
|
|
course: "C".into(),
|
|
term: "T".into(),
|
|
assessment_id: "a".into(),
|
|
date: None,
|
|
form: None,
|
|
form_position: None,
|
|
student_key: student.into(),
|
|
sid: None,
|
|
name: None,
|
|
email: None,
|
|
section: None,
|
|
item_number: number,
|
|
item_ref: None,
|
|
item_version: None,
|
|
variant: None,
|
|
selected: vec!["A".into()],
|
|
selected_source: vec![],
|
|
eliminated: vec![],
|
|
eliminated_source: vec![],
|
|
correct: Some(credit >= 0.999),
|
|
credit,
|
|
points_possible: 1.0,
|
|
score: credit,
|
|
response_time_seconds: None,
|
|
level,
|
|
learning_targets: objectives.iter().map(|s| s.to_string()).collect(),
|
|
topics: vec![],
|
|
bonus: false,
|
|
dropped: false,
|
|
dropped_full_credit: false,
|
|
}
|
|
}
|
|
|
|
fn summary(key: &str) -> StudentSummary {
|
|
StudentSummary {
|
|
student_key: key.to_string(),
|
|
name: None,
|
|
sid: None,
|
|
points: 0.0,
|
|
points_possible: 0.0,
|
|
percent: 0.0,
|
|
bonus_points: 0.0,
|
|
correct: 0,
|
|
n_items: 0,
|
|
band: String::new(),
|
|
theta: None,
|
|
theta_se: None,
|
|
objectives: Vec::new(),
|
|
levels: Vec::new(),
|
|
strengths: Vec::new(),
|
|
focus: Vec::new(),
|
|
missed: Vec::new(),
|
|
}
|
|
}
|
|
|
|
fn profile(level: Level, rate: f64) -> LevelProfile {
|
|
LevelProfile {
|
|
level,
|
|
n_items: 4,
|
|
rate,
|
|
cohort_rate: rate,
|
|
z: None,
|
|
}
|
|
}
|
|
}
|