//! Turning responses into something you can say to a student.
//!
//! Item analysis tells you about items. This module tells you about people: which
//! objectives a student has actually met, which ones they are close on, and what
//! specifically to do next.
//!
//! # On declaring mastery from three questions
//!
//! The honest answer is that you often cannot. Two items on an objective give a
//! proportion with an enormous confidence interval — two out of two correct is
//! consistent with a true rate anywhere from about 0.55 upward. So this module
//! does three things instead of pretending otherwise.
//!
//! It refuses to classify at all below `min_items_for_mastery`, reporting "not
//! enough evidence", which is a finding about your blueprint rather than about the
//! student. It reports the Wilson score interval alongside every rate, because
//! Wilson behaves sensibly at the boundaries where the normal approximation
//! produces intervals extending past 1.0. And it separates the *classification*
//! (which uses the observed rate, so it is usable) from the *confidence* (which
//! uses the interval, so it is honest). A student can be "meeting" an objective
//! provisionally, and the report says so.
//!
//! # Comparison to the cohort
//!
//! Per-level performance is reported against the class rather than in absolute
//! terms, because "you got 60% of the Analyze items" means nothing to a student
//! without knowing that the class average was 55%. The comparison is descriptive,
//! not a curve.

use std::collections::{BTreeMap, BTreeSet};

use crate::course::{CourseFile, Policy};
use crate::responses::{Response, ResponseSet};
use crate::rng::Rng;
use crate::taxonomy::Level;

/// How well a student has met one objective.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Mastery {
    /// Met the threshold.
    Meeting,
    /// Partway there.
    Developing,
    /// Not yet.
    NotYet,
    /// Too few items on this objective to say anything. This is a gap in the
    /// assessment, not a judgment about the student.
    NotEnoughEvidence,
}

impl Mastery {
    /// A label for reports.
    pub fn label(self) -> &'static str {
        match self {
            Mastery::Meeting => "meeting",
            Mastery::Developing => "developing",
            Mastery::NotYet => "not yet",
            Mastery::NotEnoughEvidence => "not enough evidence",
        }
    }

    /// A short symbol for compact tables.
    pub fn symbol(self) -> &'static str {
        match self {
            Mastery::Meeting => "✓",
            Mastery::Developing => "~",
            Mastery::NotYet => "✗",
            Mastery::NotEnoughEvidence => "?",
        }
    }
}

/// One student's standing on one objective.
#[derive(Debug, Clone)]
pub struct ObjectiveMastery {
    /// The objective id.
    pub objective: String,
    /// The objective text, for reports.
    pub text: String,
    /// How many items on this objective the student saw.
    pub n_items: usize,
    /// How many they got right, counting partial credit.
    pub credit: f64,
    /// Observed rate, `credit / n_items`.
    pub rate: f64,
    /// Lower end of the Wilson interval.
    pub wilson_lower: f64,
    /// Upper end of the Wilson interval.
    pub wilson_upper: f64,
    /// The class's rate on the same objective.
    pub cohort_rate: f64,
    /// The classification.
    pub status: Mastery,
    /// Whether the interval, not just the point estimate, clears the threshold.
    pub confident: bool,
    /// Which levels the objective was assessed at, since meeting an objective at
    /// Remember is a different claim from meeting it at Analyze.
    pub levels: Vec<Level>,
}

/// One student's performance at one cognitive level.
#[derive(Debug, Clone)]
pub struct LevelProfile {
    /// The level.
    pub level: Level,
    /// How many items at this level.
    pub n_items: usize,
    /// The student's rate.
    pub rate: f64,
    /// The class's rate.
    pub cohort_rate: f64,
    /// Difference from the class, in class standard deviations. `None` when the
    /// class had no spread at this level.
    pub z: Option<f64>,
}

impl LevelProfile {
    /// A plain-language comparison to the class.
    pub fn comparison(&self) -> &'static str {
        match self.z {
            Some(z) if z >= 1.0 => "well above the class",
            Some(z) if z >= 0.4 => "above the class",
            Some(z) if z > -0.4 => "about the same as the class",
            Some(z) if z > -1.0 => "below the class",
            Some(_) => "well below the class",
            None => "the class did not vary here",
        }
    }
}

/// An item the student got wrong, with what to do about it.
#[derive(Debug, Clone)]
pub struct MissedItem {
    /// The question number.
    pub number: u32,
    /// The item's global id.
    pub item_ref: Option<String>,
    /// What the student chose.
    pub selected: Vec<String>,
    /// Credit earned, since a partially credited response is not a clean miss.
    pub credit: f64,
    /// The level.
    pub level: Option<Level>,
    /// The objectives involved.
    pub learning_objectives: Vec<String>,
    /// The misconception the chosen distractor was written to detect.
    pub misconception: Option<String>,
    /// Feedback written for a student who chose that option.
    pub feedback: Option<String>,
    /// Where to go back to: lecture titles and slide numbers.
    pub study: Vec<String>,
}

/// Everything needed to write one student's report.
#[derive(Debug, Clone)]
pub struct StudentSummary {
    /// The grouping key.
    pub student_key: String,
    /// The name, when not pseudonymized.
    pub name: Option<String>,
    /// The student id, when not pseudonymized.
    pub sid: Option<String>,
    /// Points earned on scored items.
    pub points: f64,
    /// Points available on scored items.
    pub points_possible: f64,
    /// Percentage on scored items.
    pub percent: f64,
    /// Bonus points earned.
    pub bonus_points: f64,
    /// Items answered correctly.
    pub correct: usize,
    /// Items administered.
    pub n_items: usize,
    /// Where the score falls relative to the class, as a coarse band.
    pub band: String,
    /// IRT ability, when an IRT fit was supplied.
    pub theta: Option<f64>,
    /// Standard error of the ability estimate.
    pub theta_se: Option<f64>,
    /// Per-objective standing, in the course's objective order.
    pub objectives: Vec<ObjectiveMastery>,
    /// Per-level standing.
    pub levels: Vec<LevelProfile>,
    /// Objectives the student is clearly meeting.
    pub strengths: Vec<String>,
    /// Objectives to work on, worst first.
    pub focus: Vec<String>,
    /// Missed items with targeted guidance.
    pub missed: Vec<MissedItem>,
}

impl StudentSummary {
    /// The display name, falling back to the key.
    pub fn display_name(&self) -> String {
        self.name
            .clone()
            .unwrap_or_else(|| self.student_key.clone())
    }
}

/// Class-level context.
#[derive(Debug, Clone)]
pub struct Cohort {
    /// Per-student summaries, sorted by key.
    pub students: Vec<StudentSummary>,
    /// Class rate per objective.
    pub objective_rates: BTreeMap<String, f64>,
    /// Class rate per level.
    pub level_rates: BTreeMap<Level, f64>,
    /// Mean percentage.
    pub mean_percent: f64,
    /// Standard deviation of percentage.
    pub sd_percent: f64,
    /// Objectives the class as a whole did not meet, worst first. This is the
    /// list that should change what you reteach.
    pub class_gaps: Vec<(String, f64)>,
    /// Optional grouping of students by response profile.
    pub archetypes: Vec<Archetype>,
}

/// A cluster of students with a similar profile across levels.
#[derive(Debug, Clone)]
pub struct Archetype {
    /// A label describing the pattern.
    pub label: String,
    /// The student keys in this cluster.
    pub members: Vec<String>,
    /// Mean rate at each level for this cluster.
    pub level_means: BTreeMap<Level, f64>,
}

/// The Wilson score interval for a binomial proportion.
///
/// Preferred over the normal approximation because it stays inside `[0, 1]` and
/// behaves at the boundaries, which is exactly where classroom data lives: a
/// student who got three out of three needs an interval, and the textbook formula
/// gives width zero there.
///
/// # Arguments
///
/// * `successes` - the number of successes, which may be fractional when partial
///   credit is involved.
/// * `n` - the number of trials.
/// * `z` - the standard normal quantile; 1.96 for a two-sided 95% interval.
///
/// # Returns
///
/// The lower and upper bounds, or `(0.0, 1.0)` when there are no trials.
pub fn wilson(successes: f64, n: usize, z: f64) -> (f64, f64) {
    if n == 0 {
        return (0.0, 1.0);
    }
    let n = n as f64;
    let p = (successes / n).clamp(0.0, 1.0);
    let z2 = z * z;
    let denominator = 1.0 + z2 / n;
    let center = p + z2 / (2.0 * n);
    let spread = z * ((p * (1.0 - p) / n) + z2 / (4.0 * n * n)).sqrt();
    (
        ((center - spread) / denominator).clamp(0.0, 1.0),
        ((center + spread) / denominator).clamp(0.0, 1.0),
    )
}

/// Builds per-student summaries for one administration.
///
/// # Arguments
///
/// * `set` - the responses, already enriched with item metadata.
/// * `course` - the course, for objective text, order, and policy.
/// * `catalog` - the loaded course, for misconception feedback on missed items.
/// * `fit` - an optional IRT fit, whose abilities are attached when present.
///
/// # Returns
///
/// The cohort.
pub fn summarize(
    set: &ResponseSet,
    course: &CourseFile,
    catalog: Option<&crate::catalog::Catalog>,
    fit: Option<&crate::irt::Fit>,
) -> Cohort {
    let policy = &course.policy;
    let students = set.students();

    // Class rates first: every student's report is relative to these.
    let objective_rates = rates_by_objective(&set.rows.iter().collect::<Vec<_>>());
    let level_rates = rates_by_level(&set.rows.iter().collect::<Vec<_>>());

    // Per-level spread across students, for the z comparisons.
    let mut level_values: BTreeMap<Level, Vec<f64>> = BTreeMap::new();
    for key in &students {
        let rows = set.for_student(key);
        for (level, rate) in rates_by_level(&rows) {
            level_values.entry(level).or_default().push(rate);
        }
    }
    let level_sd: BTreeMap<Level, f64> = level_values
        .iter()
        .map(|(level, values)| (*level, sd(values)))
        .collect();

    let percents: Vec<f64> = students
        .iter()
        .map(|key| {
            let earned = set.scored_total(key);
            let possible = set.points_available();
            if possible > 0.0 {
                100.0 * earned / possible
            } else {
                0.0
            }
        })
        .collect();
    let mean_percent = mean(&percents);
    let sd_percent = sd(&percents);

    let ability = fit.map(|f| f.ability_map()).unwrap_or_default();
    let ability_se: BTreeMap<String, f64> = fit
        .map(|f| {
            f.abilities
                .iter()
                .map(|a| (a.student_key.clone(), a.se))
                .collect()
        })
        .unwrap_or_default();

    let order = course.objectives_in_order();
    let mut summaries = Vec::with_capacity(students.len());

    for (index, key) in students.iter().enumerate() {
        let rows = set.for_student(key);
        let points = set.scored_total(key);
        let possible = set.points_available();
        let percent = percents[index];

        let correct = rows
            .iter()
            .filter(|r| r.counts() && r.correct == Some(true))
            .count();
        let n_items = rows.iter().filter(|r| r.counts()).count();

        // Objectives, in the course's declared order so reports read the way the
        // course is taught rather than alphabetically.
        let per_objective = rates_by_objective(&rows);
        let counts = counts_by_objective(&rows);
        let mut objectives = Vec::new();
        let mut seen: BTreeSet<&String> = BTreeSet::new();
        for id in order.iter().chain(per_objective.keys()) {
            if !seen.insert(id) {
                continue;
            }
            let Some((n, credit)) = counts.get(id).copied() else {
                continue;
            };
            objectives.push(objective_mastery(
                id,
                course,
                n,
                credit,
                objective_rates.get(id).copied().unwrap_or(0.0),
                &rows,
                policy,
            ));
        }

        // Levels.
        let student_levels = rates_by_level(&rows);
        let level_counts = counts_by_level(&rows);
        let levels: Vec<LevelProfile> = Level::ALL
            .iter()
            .filter_map(|level| {
                let (n, _) = level_counts.get(level).copied()?;
                if n == 0 {
                    return None;
                }
                let rate = student_levels.get(level).copied().unwrap_or(0.0);
                let cohort_rate = level_rates.get(level).copied().unwrap_or(0.0);
                let spread = level_sd.get(level).copied().unwrap_or(0.0);
                Some(LevelProfile {
                    level: *level,
                    n_items: n,
                    rate,
                    cohort_rate,
                    z: if spread > 1e-9 {
                        Some((rate - cohort_rate) / spread)
                    } else {
                        None
                    },
                })
            })
            .collect();

        // Strengths and focus areas. Strengths need confidence, focus areas do
        // not: telling a student to review something they may already know costs
        // them an hour, while telling them they have mastered something they have
        // not costs them the next exam.
        let strengths: Vec<String> = objectives
            .iter()
            .filter(|o| o.status == Mastery::Meeting && o.confident)
            .map(|o| o.objective.clone())
            .collect();
        let mut focus_pairs: Vec<(&ObjectiveMastery, f64)> = objectives
            .iter()
            .filter(|o| matches!(o.status, Mastery::NotYet | Mastery::Developing))
            .map(|o| (o, o.rate))
            .collect();
        focus_pairs.sort_by(|a, b| {
            a.1.partial_cmp(&b.1)
                .unwrap_or(std::cmp::Ordering::Equal)
                .then_with(|| a.0.objective.cmp(&b.0.objective))
        });
        let focus: Vec<String> = focus_pairs
            .iter()
            .map(|(o, _)| o.objective.clone())
            .collect();

        let missed = missed_items(&rows, catalog, course);

        summaries.push(StudentSummary {
            student_key: key.clone(),
            name: rows.first().and_then(|r| r.name.clone()),
            sid: rows.first().and_then(|r| r.sid.clone()),
            points,
            points_possible: possible,
            percent,
            bonus_points: set.bonus_total(key),
            correct,
            n_items,
            band: band_for(percent, &percents),
            theta: ability.get(key).copied(),
            theta_se: ability_se.get(key).copied(),
            objectives,
            levels,
            strengths,
            focus,
            missed,
        });
    }

    // Class gaps: objectives where the whole class fell short. These are the ones
    // to reteach rather than to send individual students away to review.
    let mut class_gaps: Vec<(String, f64)> = objective_rates
        .iter()
        .filter(|(_, rate)| **rate < policy.mastery_threshold)
        .map(|(id, rate)| (id.clone(), *rate))
        .collect();
    class_gaps.sort_by(|a, b| {
        a.1.partial_cmp(&b.1)
            .unwrap_or(std::cmp::Ordering::Equal)
            .then_with(|| a.0.cmp(&b.0))
    });

    let archetypes = cluster(&summaries, 3);

    Cohort {
        students: summaries,
        objective_rates,
        level_rates,
        mean_percent,
        sd_percent,
        class_gaps,
        archetypes,
    }
}

/// Builds one objective's mastery record.
///
/// # Arguments
///
/// * `id` - the objective id.
/// * `course` - the course, for text and policy.
/// * `n` - items on this objective.
/// * `credit` - total credit earned.
/// * `cohort_rate` - the class rate.
/// * `rows` - the student's responses, for the level list.
/// * `policy` - the course policy.
///
/// # Returns
///
/// The record.
fn objective_mastery(
    id: &str,
    course: &CourseFile,
    n: usize,
    credit: f64,
    cohort_rate: f64,
    rows: &[&Response],
    policy: &Policy,
) -> ObjectiveMastery {
    let rate = if n > 0 { credit / n as f64 } else { 0.0 };
    let (lower, upper) = wilson(credit, n, 1.96);

    let status = if n < policy.min_items_for_mastery.max(1) {
        Mastery::NotEnoughEvidence
    } else if rate >= policy.mastery_threshold {
        Mastery::Meeting
    } else if rate >= policy.mastery_threshold * 0.6 {
        Mastery::Developing
    } else {
        Mastery::NotYet
    };

    let levels: Vec<Level> = rows
        .iter()
        .filter(|r| r.learning_objectives.iter().any(|o| o == id))
        .filter_map(|r| r.level)
        .collect::<BTreeSet<Level>>()
        .into_iter()
        .collect();

    ObjectiveMastery {
        objective: id.to_string(),
        text: course.objective_text(id),
        n_items: n,
        credit,
        rate,
        wilson_lower: lower,
        wilson_upper: upper,
        cohort_rate,
        status,
        confident: lower >= policy.mastery_threshold,
        levels,
    }
}

/// Collects missed items with targeted guidance.
///
/// The guidance comes from the item's own authoring: the `misconception` recorded
/// on the distractor the student actually chose, and the lecture and slides the
/// item was written from. This is why authoring distractors deliberately pays off
/// twice — once when writing the item, and again in every report afterward.
///
/// # Arguments
///
/// * `rows` - the student's responses.
/// * `catalog` - the loaded course.
/// * `course` - the course, for lecture titles.
///
/// # Returns
///
/// The missed items, in question order.
fn missed_items(
    rows: &[&Response],
    catalog: Option<&crate::catalog::Catalog>,
    course: &CourseFile,
) -> Vec<MissedItem> {
    let mut out = Vec::new();
    for r in rows {
        if !r.counts() || r.credit >= 0.999 {
            continue;
        }
        let mut misconception = None;
        let mut feedback = None;
        let mut study = Vec::new();

        if let (Some(cat), Some(uid)) = (catalog, r.item_ref.as_deref()) {
            if let Some(entry) = cat.get(uid) {
                // Feedback for the specific option chosen, which is the whole
                // point of recording per-distractor misconceptions.
                if let Some(letter) = r.selected.first() {
                    if let Some(choice) = entry.item.option(letter) {
                        misconception = choice.misconception.clone();
                        feedback = choice.student_text().map(|s| s.to_string());
                    }
                }
                for source in &entry.item.sources {
                    let title = course
                        .lectures
                        .get(&source.lecture)
                        .map(|l| l.title.clone())
                        .unwrap_or_else(|| source.lecture.clone());
                    if source.slides.is_empty() {
                        study.push(title);
                    } else {
                        let slides: Vec<String> =
                            source.slides.iter().map(|s| s.to_string()).collect();
                        study.push(format!("{title}, slides {}", slides.join(", ")));
                    }
                    for reading in &source.readings {
                        study.push(reading.clone());
                    }
                }
            }
        }

        out.push(MissedItem {
            number: r.item_number,
            item_ref: r.item_ref.clone(),
            selected: r.selected.clone(),
            credit: r.credit,
            level: r.level,
            learning_objectives: r.learning_objectives.clone(),
            misconception,
            feedback,
            study,
        });
    }
    out
}

/// Credit rate per objective over a set of responses.
///
/// # Arguments
///
/// * `rows` - the responses.
///
/// # Returns
///
/// The rate for each objective mentioned.
pub fn rates_by_objective(rows: &[&Response]) -> BTreeMap<String, f64> {
    counts_by_objective(rows)
        .into_iter()
        .map(|(id, (n, credit))| {
            let rate = if n > 0 { credit / n as f64 } else { 0.0 };
            (id, rate)
        })
        .collect()
}

/// Item counts and credit per objective.
///
/// An item tagged with two objectives counts toward both. That double counting is
/// intentional: the question "how is this student doing on kinetics" should use
/// every item that measured kinetics.
///
/// # Arguments
///
/// * `rows` - the responses.
///
/// # Returns
///
/// `(item count, total credit)` per objective.
pub fn counts_by_objective(rows: &[&Response]) -> BTreeMap<String, (usize, f64)> {
    let mut out: BTreeMap<String, (usize, f64)> = BTreeMap::new();
    for r in rows {
        if !r.counts() {
            continue;
        }
        for objective in &r.learning_objectives {
            let e = out.entry(objective.clone()).or_insert((0, 0.0));
            e.0 += 1;
            e.1 += r.credit.clamp(0.0, 1.0);
        }
    }
    out
}

/// Credit rate per level.
///
/// # Arguments
///
/// * `rows` - the responses.
///
/// # Returns
///
/// The rate for each level present.
pub fn rates_by_level(rows: &[&Response]) -> BTreeMap<Level, f64> {
    counts_by_level(rows)
        .into_iter()
        .map(|(level, (n, credit))| {
            let rate = if n > 0 { credit / n as f64 } else { 0.0 };
            (level, rate)
        })
        .collect()
}

/// Item counts and credit per level.
///
/// # Arguments
///
/// * `rows` - the responses.
///
/// # Returns
///
/// `(item count, total credit)` per level.
pub fn counts_by_level(rows: &[&Response]) -> BTreeMap<Level, (usize, f64)> {
    let mut out: BTreeMap<Level, (usize, f64)> = BTreeMap::new();
    for r in rows {
        if !r.counts() {
            continue;
        }
        if let Some(level) = r.level {
            let e = out.entry(level).or_insert((0, 0.0));
            e.0 += 1;
            e.1 += r.credit.clamp(0.0, 1.0);
        }
    }
    out
}

/// A coarse band for a score within a class.
///
/// Quartile bands rather than an exact percentile, because a percentile computed
/// from twenty-four students implies a precision it does not have, and because
/// telling a student they are "37th percentile" invites comparison in a way that
/// "middle half of the class" does not.
///
/// # Arguments
///
/// * `percent` - the student's percentage.
/// * `all` - every student's percentage.
///
/// # Returns
///
/// The band label.
fn band_for(percent: f64, all: &[f64]) -> String {
    if all.len() < 4 {
        return "the class is too small to place this meaningfully".to_string();
    }
    let mut sorted = all.to_vec();
    sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
    let below = sorted.iter().filter(|x| **x < percent).count() as f64;
    let fraction = below / all.len() as f64;
    if fraction >= 0.75 {
        "top quarter of the class".to_string()
    } else if fraction >= 0.5 {
        "upper middle of the class".to_string()
    } else if fraction >= 0.25 {
        "lower middle of the class".to_string()
    } else {
        "bottom quarter of the class".to_string()
    }
}

/// Groups students by their profile across levels.
///
/// This is descriptive, not diagnostic. It answers "are there recognizable
/// patterns in how this class is struggling" — for instance a group that handles
/// recall fine and falls apart on application, which calls for different
/// instruction than a group that is uniformly behind.
///
/// k-means with a seeded, deterministic initialization, so the same data always
/// produces the same groups.
///
/// # Arguments
///
/// * `students` - the summaries.
/// * `k` - how many clusters to look for.
///
/// # Returns
///
/// The clusters, largest first. Empty when there are too few students to bother.
pub fn cluster(students: &[StudentSummary], k: usize) -> Vec<Archetype> {
    // Below about three students per cluster the groups are noise.
    if students.len() < k * 3 || k == 0 {
        return Vec::new();
    }

    // Feature vector: rate at each level that anyone was assessed on.
    let levels: Vec<Level> = students
        .iter()
        .flat_map(|s| s.levels.iter().map(|l| l.level))
        .collect::<BTreeSet<Level>>()
        .into_iter()
        .collect();
    if levels.len() < 2 {
        return Vec::new();
    }

    let points: Vec<Vec<f64>> = students
        .iter()
        .map(|s| {
            levels
                .iter()
                .map(|level| {
                    s.levels
                        .iter()
                        .find(|l| l.level == *level)
                        .map(|l| l.rate)
                        .unwrap_or(0.0)
                })
                .collect()
        })
        .collect();

    // Standardize each dimension so a level everyone did well on does not
    // dominate the distance.
    let mut standardized = points.clone();
    for d in 0..levels.len() {
        let column: Vec<f64> = points.iter().map(|p| p[d]).collect();
        let m = mean(&column);
        let s = sd(&column);
        for (i, point) in standardized.iter_mut().enumerate() {
            point[d] = if s > 1e-9 {
                (points[i][d] - m) / s
            } else {
                0.0
            };
        }
    }

    // Seeded k-means++ initialization.
    let mut rng = Rng::from_label("coursebank/archetypes");
    let mut centers: Vec<Vec<f64>> =
        vec![standardized[rng.below(standardized.len() as u64) as usize].clone()];
    while centers.len() < k {
        let distances: Vec<f64> = standardized
            .iter()
            .map(|p| {
                centers
                    .iter()
                    .map(|c| squared_distance(p, c))
                    .fold(f64::INFINITY, f64::min)
            })
            .collect();
        let total: f64 = distances.iter().sum();
        if total <= 0.0 {
            break;
        }
        let mut target = rng.unit() * total;
        let mut chosen = standardized.len() - 1;
        for (i, d) in distances.iter().enumerate() {
            target -= d;
            if target <= 0.0 {
                chosen = i;
                break;
            }
        }
        centers.push(standardized[chosen].clone());
    }

    let mut assignment = vec![0usize; standardized.len()];
    for _ in 0..50 {
        let mut changed = false;
        for (i, p) in standardized.iter().enumerate() {
            let mut best = (0usize, f64::INFINITY);
            for (c, center) in centers.iter().enumerate() {
                let d = squared_distance(p, center);
                if d < best.1 {
                    best = (c, d);
                }
            }
            if assignment[i] != best.0 {
                assignment[i] = best.0;
                changed = true;
            }
        }
        for (c, center) in centers.iter_mut().enumerate() {
            let members: Vec<&Vec<f64>> = standardized
                .iter()
                .enumerate()
                .filter(|(i, _)| assignment[*i] == c)
                .map(|(_, p)| p)
                .collect();
            if members.is_empty() {
                continue;
            }
            for d in 0..levels.len() {
                center[d] = members.iter().map(|p| p[d]).sum::<f64>() / members.len() as f64;
            }
        }
        if !changed {
            break;
        }
    }

    let mut out = Vec::new();
    for c in 0..centers.len() {
        let members: Vec<String> = students
            .iter()
            .enumerate()
            .filter(|(i, _)| assignment[*i] == c)
            .map(|(_, s)| s.student_key.clone())
            .collect();
        if members.is_empty() {
            continue;
        }
        let mut level_means = BTreeMap::new();
        for (d, level) in levels.iter().enumerate() {
            let values: Vec<f64> = students
                .iter()
                .enumerate()
                .filter(|(i, _)| assignment[*i] == c)
                .map(|(i, _)| points[i][d])
                .collect();
            level_means.insert(*level, mean(&values));
        }
        out.push(Archetype {
            label: label_for(&level_means),
            members,
            level_means,
        });
    }
    out.sort_by(|a, b| b.members.len().cmp(&a.members.len()));
    out
}

/// Names a cluster from its level profile.
///
/// # Arguments
///
/// * `means` - mean rate at each level.
///
/// # Returns
///
/// A descriptive label.
fn label_for(means: &BTreeMap<Level, f64>) -> String {
    let values: Vec<f64> = means.values().copied().collect();
    let overall = mean(&values);

    // Is the profile flat, or does it fall off with cognitive demand?
    let low: Vec<f64> = means
        .iter()
        .filter(|(l, _)| l.code() <= 2)
        .map(|(_, v)| *v)
        .collect();
    let high: Vec<f64> = means
        .iter()
        .filter(|(l, _)| l.code() >= 3)
        .map(|(_, v)| *v)
        .collect();

    if !low.is_empty() && !high.is_empty() {
        let drop = mean(&low) - mean(&high);
        if drop > 0.25 {
            return "knows the material, struggles to apply it".to_string();
        }
        if drop < -0.15 {
            return "reasons well, gaps in recall".to_string();
        }
    }

    if overall >= 0.85 {
        "consistently strong".to_string()
    } else if overall >= 0.65 {
        "solid with scattered gaps".to_string()
    } else {
        "behind across the board".to_string()
    }
}

/// Squared Euclidean distance.
fn squared_distance(a: &[f64], b: &[f64]) -> f64 {
    a.iter().zip(b.iter()).map(|(x, y)| (x - y) * (x - y)).sum()
}

/// The arithmetic mean, zero for an empty slice.
fn mean(v: &[f64]) -> f64 {
    if v.is_empty() {
        0.0
    } else {
        v.iter().sum::<f64>() / v.len() as f64
    }
}

/// The population standard deviation.
fn sd(v: &[f64]) -> f64 {
    if v.len() < 2 {
        return 0.0;
    }
    let m = mean(v);
    (v.iter().map(|x| (x - m) * (x - m)).sum::<f64>() / v.len() as f64).sqrt()
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn wilson_stays_inside_zero_and_one() {
        // Three out of three: the naive interval has zero width, Wilson does not.
        let (lo, hi) = wilson(3.0, 3, 1.96);
        assert!(lo > 0.0 && lo < 1.0, "lower bound {lo}");
        assert_eq!(hi, 1.0);
        assert!(lo < 0.5, "three items cannot establish a high rate: {lo}");

        // Zero out of four.
        let (lo, hi) = wilson(0.0, 4, 1.96);
        assert_eq!(lo, 0.0);
        assert!(hi > 0.0 && hi < 1.0);

        // No data at all.
        assert_eq!(wilson(0.0, 0, 1.96), (0.0, 1.0));
    }

    #[test]
    fn wilson_narrows_as_n_grows() {
        let (lo_small, hi_small) = wilson(8.0, 10, 1.96);
        let (lo_big, hi_big) = wilson(80.0, 100, 1.96);
        assert!(
            (hi_big - lo_big) < (hi_small - lo_small),
            "more data must give a tighter interval"
        );
    }

    #[test]
    fn two_items_never_claim_confident_mastery() {
        // The classification may say "meeting", but confidence must not, because
        // two items cannot establish a rate of 0.75.
        let (lower, _) = wilson(2.0, 2, 1.96);
        assert!(lower < 0.75, "got {lower}");
    }

    #[test]
    fn bands_describe_position_coarsely() {
        let all = vec![50.0, 60.0, 70.0, 80.0, 90.0, 95.0, 40.0, 30.0];
        assert!(band_for(95.0, &all).contains("top"));
        assert!(band_for(30.0, &all).contains("bottom"));
        // Too few students to place anyone.
        assert!(band_for(50.0, &[50.0, 60.0]).contains("too small"));
    }

    #[test]
    fn objective_counts_credit_every_tagged_item() {
        let rows = vec![
            make("s1", 1, 1.0, &["lo-a", "lo-b"], Some(Level::Remember)),
            make("s1", 2, 0.0, &["lo-a"], Some(Level::Apply)),
        ];
        let refs: Vec<&Response> = rows.iter().collect();
        let counts = counts_by_objective(&refs);
        // lo-a saw both items; lo-b only the first.
        assert_eq!(counts["lo-a"], (2, 1.0));
        assert_eq!(counts["lo-b"], (1, 1.0));
        let rates = rates_by_objective(&refs);
        assert_eq!(rates["lo-a"], 0.5);
        assert_eq!(rates["lo-b"], 1.0);
    }

    #[test]
    fn level_rates_ignore_untagged_items() {
        let rows = vec![
            make("s1", 1, 1.0, &[], Some(Level::Remember)),
            make("s1", 2, 0.0, &[], None),
        ];
        let refs: Vec<&Response> = rows.iter().collect();
        let counts = counts_by_level(&refs);
        assert_eq!(counts.len(), 1);
        assert_eq!(counts[&Level::Remember], (1, 1.0));
    }

    #[test]
    fn mastery_labels_are_stable() {
        assert_eq!(Mastery::Meeting.label(), "meeting");
        assert_eq!(Mastery::NotEnoughEvidence.label(), "not enough evidence");
    }

    #[test]
    fn clustering_needs_enough_students() {
        assert!(cluster(&[], 3).is_empty());
        let few: Vec<StudentSummary> = (0..4).map(|i| summary(&format!("s{i}"))).collect();
        assert!(cluster(&few, 3).is_empty(), "four students, three clusters");
    }

    #[test]
    fn clustering_is_deterministic_and_separates_profiles() {
        // Half the class is strong on recall and weak on application; half is
        // uniformly strong. Those are different problems.
        let mut students = Vec::new();
        for i in 0..12 {
            let mut s = summary(&format!("s{i:02}"));
            let (recall, apply) = if i < 6 { (0.95, 0.35) } else { (0.9, 0.85) };
            s.levels = vec![
                profile(Level::Remember, recall),
                profile(Level::Apply, apply),
            ];
            students.push(s);
        }
        let first = cluster(&students, 2);
        let second = cluster(&students, 2);
        assert_eq!(first.len(), 2);
        assert_eq!(
            first.iter().map(|a| a.members.clone()).collect::<Vec<_>>(),
            second.iter().map(|a| a.members.clone()).collect::<Vec<_>>(),
            "clustering must be reproducible"
        );
        // The two groups must not be mixed together.
        let sizes: Vec<usize> = first.iter().map(|a| a.members.len()).collect();
        assert_eq!(sizes, vec![6, 6], "got {sizes:?}");
        assert!(first.iter().any(|a| a.label.contains("struggles to apply")));
    }

    fn make(
        student: &str,
        number: u32,
        credit: f64,
        objectives: &[&str],
        level: Option<Level>,
    ) -> Response {
        Response {
            administration_id: "C/T/a".into(),
            course: "C".into(),
            term: "T".into(),
            assessment_id: "a".into(),
            date: None,
            form: None,
            student_key: student.into(),
            sid: None,
            name: None,
            email: None,
            section: None,
            item_number: number,
            item_ref: None,
            item_version: None,
            selected: vec!["A".into()],
            eliminated: vec![],
            correct: Some(credit >= 0.999),
            credit,
            points_possible: 1.0,
            score: credit,
            response_time_seconds: None,
            level,
            learning_objectives: objectives.iter().map(|s| s.to_string()).collect(),
            topics: vec![],
            bonus: false,
            dropped: false,
        }
    }

    fn summary(key: &str) -> StudentSummary {
        StudentSummary {
            student_key: key.to_string(),
            name: None,
            sid: None,
            points: 0.0,
            points_possible: 0.0,
            percent: 0.0,
            bonus_points: 0.0,
            correct: 0,
            n_items: 0,
            band: String::new(),
            theta: None,
            theta_se: None,
            objectives: Vec::new(),
            levels: Vec::new(),
            strengths: Vec::new(),
            focus: Vec::new(),
            missed: Vec::new(),
        }
    }

    fn profile(level: Level, rate: f64) -> LevelProfile {
        LevelProfile {
            level,
            n_items: 4,
            rate,
            cohort_rate: rate,
            z: None,
        }
    }
}