feat: add learning objective targets
This commit is contained in:
@@ -1015,7 +1015,7 @@ mod tests {
|
||||
score: credit,
|
||||
response_time_seconds: None,
|
||||
level: None,
|
||||
learning_objectives: vec![],
|
||||
learning_targets: vec![],
|
||||
topics: vec![],
|
||||
bonus: false,
|
||||
dropped: false,
|
||||
|
||||
+24
-26
@@ -509,12 +509,9 @@ pub fn student(
|
||||
.objectives
|
||||
.iter()
|
||||
.map(|mastery| ObjectiveRow {
|
||||
id: mastery.objective.clone(),
|
||||
id: mastery.id.clone(),
|
||||
text: mastery.text.clone(),
|
||||
unit: course
|
||||
.learning_objectives
|
||||
.get(&mastery.objective)
|
||||
.and_then(|o| o.unit.clone()),
|
||||
unit: course.objective_unit(&mastery.id).map(str::to_string),
|
||||
n_items: mastery.n_items,
|
||||
credit: mastery.credit,
|
||||
rate: mastery.rate,
|
||||
@@ -700,11 +697,11 @@ fn question_row(
|
||||
number: row.item_number,
|
||||
position: row.form_position.filter(|p| *p != row.item_number),
|
||||
level: row.level.map(|l| l.code()),
|
||||
objectives: row.learning_objectives.clone(),
|
||||
objectives: row.learning_targets.clone(),
|
||||
objective_texts: if missed {
|
||||
row.learning_objectives
|
||||
row.learning_targets
|
||||
.iter()
|
||||
.map(|id| catalog.course.objective_text(id))
|
||||
.map(|id| catalog.course.text_for(id))
|
||||
.collect()
|
||||
} else {
|
||||
// Only where it earns its space. Every question already carries its
|
||||
@@ -830,15 +827,13 @@ fn lecture_focus(catalog: &Catalog, rows: &[&Response], opts: &Options) -> Vec<L
|
||||
.extend(source.slides.iter().copied());
|
||||
}
|
||||
}
|
||||
for objective in &row.learning_objectives {
|
||||
if let Some(entry) = course.learning_objectives.get(objective) {
|
||||
lectures.extend(entry.lectures.iter().cloned());
|
||||
}
|
||||
for target in &row.learning_targets {
|
||||
lectures.extend(course.lectures_for(target).iter().cloned());
|
||||
}
|
||||
|
||||
for lecture in lectures {
|
||||
let tally = tallies.entry(lecture.clone()).or_default();
|
||||
tally.objectives.extend(row.learning_objectives.clone());
|
||||
tally.objectives.extend(row.learning_targets.clone());
|
||||
tally.questions.insert(row.item_number);
|
||||
if let Some(numbers) = slides.get(&lecture) {
|
||||
tally.slides.extend(numbers.iter().copied());
|
||||
@@ -866,7 +861,7 @@ fn lecture_focus(catalog: &Catalog, rows: &[&Response], opts: &Options) -> Vec<L
|
||||
objectives: tally
|
||||
.objectives
|
||||
.iter()
|
||||
.map(|id| course.objective_text(id))
|
||||
.map(|id| course.text_for(id))
|
||||
.collect(),
|
||||
lecture,
|
||||
}
|
||||
@@ -1390,17 +1385,22 @@ pub fn cohort(
|
||||
.collect();
|
||||
|
||||
// Objective counts come from the responses so that an objective assessed by
|
||||
// two items is not reported as if it had one.
|
||||
// two items is not reported as if it had one. Counted per objective, and by
|
||||
// item number, so a question tagged with two of that objective's targets is
|
||||
// one item here.
|
||||
let mut objective_items: BTreeMap<String, BTreeSet<u32>> = BTreeMap::new();
|
||||
for row in set.rows.iter().filter(|r| r.counts()) {
|
||||
for objective in &row.learning_objectives {
|
||||
for objective in &row.learning_targets {
|
||||
objective_items
|
||||
.entry(objective.clone())
|
||||
.entry(course.objective_for(objective).to_string())
|
||||
.or_default()
|
||||
.insert(row.item_number);
|
||||
}
|
||||
}
|
||||
|
||||
// Objectives, not targets: this table is the reteaching queue, and it is
|
||||
// only usable if it is short enough to read and each row rests on enough
|
||||
// items to believe.
|
||||
let mut objectives: Vec<CohortObjectiveRow> = cohort
|
||||
.objective_rates
|
||||
.iter()
|
||||
@@ -1410,7 +1410,7 @@ pub fn cohort(
|
||||
let mut not_yet = 0;
|
||||
let mut thin = 0;
|
||||
for student in &cohort.students {
|
||||
if let Some(row) = student.objectives.iter().find(|o| &o.objective == id) {
|
||||
if let Some(row) = student.objectives.iter().find(|o| &o.id == id) {
|
||||
match row.status {
|
||||
Mastery::Meeting => meeting += 1,
|
||||
Mastery::Developing => developing += 1,
|
||||
@@ -1421,7 +1421,7 @@ pub fn cohort(
|
||||
}
|
||||
CohortObjectiveRow {
|
||||
id: id.clone(),
|
||||
text: course.objective_text(id),
|
||||
text: course.text_for(id),
|
||||
n_items: objective_items.get(id).map(|s| s.len()).unwrap_or(0),
|
||||
rate: *rate,
|
||||
meeting,
|
||||
@@ -1451,7 +1451,7 @@ pub fn cohort(
|
||||
.map(|p| {
|
||||
(
|
||||
p.number,
|
||||
(p.level.map(|l| l.code()), p.learning_objectives.clone()),
|
||||
(p.level.map(|l| l.code()), p.learning_targets.clone()),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
@@ -1467,7 +1467,7 @@ pub fn cohort(
|
||||
level: meta.and_then(|m| m.0),
|
||||
objectives: meta.map(|m| m.1.clone()).unwrap_or_default(),
|
||||
objective_texts: meta
|
||||
.map(|m| m.1.iter().map(|id| course.objective_text(id)).collect())
|
||||
.map(|m| m.1.iter().map(|id| course.text_for(id)).collect())
|
||||
.unwrap_or_default(),
|
||||
taught_in: item
|
||||
.item_ref
|
||||
@@ -1564,7 +1564,7 @@ pub fn cohort(
|
||||
questions,
|
||||
revise,
|
||||
forms: form_rows(set, cohort),
|
||||
blueprint: crate::select::check_blueprint(record),
|
||||
blueprint: crate::select::check_blueprint(record, course),
|
||||
patterns: cohort
|
||||
.archetypes
|
||||
.iter()
|
||||
@@ -1739,10 +1739,8 @@ fn lecture_rows(
|
||||
lectures.insert(source.lecture.clone());
|
||||
}
|
||||
}
|
||||
for objective in &question.objectives {
|
||||
if let Some(entry) = course.learning_objectives.get(objective) {
|
||||
lectures.extend(entry.lectures.iter().cloned());
|
||||
}
|
||||
for target in &question.objectives {
|
||||
lectures.extend(course.lectures_for(target).iter().cloned());
|
||||
}
|
||||
for lecture in lectures {
|
||||
items.entry(lecture.clone()).or_default().push(question);
|
||||
|
||||
+325
-60
@@ -24,6 +24,28 @@
|
||||
//! uses the interval, so it is honest). A student can be "meeting" an objective
|
||||
//! provisionally, and the report says so.
|
||||
//!
|
||||
//! # Which tier gets classified
|
||||
//!
|
||||
//! The registry has two tiers, objectives and their targets (see
|
||||
//! [`crate::course::Objective`]), and they are reported differently because the
|
||||
//! evidence behind them differs in kind. An **objective** is classified: its
|
||||
//! denominator is every item tagged to any of its targets, which is how an exam
|
||||
//! that spends twelve questions across a topic gets to make one statement with a
|
||||
//! real denominator instead of twelve statements with none.
|
||||
//!
|
||||
//! A **target** is not classified. It usually carries one or two items, and
|
||||
//! `min_items_for_mastery` would mark almost all of them "not enough evidence",
|
||||
//! which would be true but useless. So target rows report the observed rate as
|
||||
//! itemized evidence for the objective's classification, and a report should
|
||||
//! present them that way: not "you have not mastered this" but "here is what you
|
||||
//! missed inside the objective above".
|
||||
//!
|
||||
//! An item tagged with two targets of the same objective counts *once* toward
|
||||
//! that objective. Double counting is right across unrelated objectives, where
|
||||
//! the question "how is this student doing on kinetics" should use every item
|
||||
//! that measured kinetics, but within one denominator it would inflate both the
|
||||
//! count and the confidence.
|
||||
//!
|
||||
//! # Comparison to the cohort
|
||||
//!
|
||||
//! Per-level performance is reported against the class rather than in absolute
|
||||
@@ -33,10 +55,10 @@
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
use crate::course::{CourseFile, Policy};
|
||||
use crate::course::CourseFile;
|
||||
use crate::responses::{Response, ResponseSet};
|
||||
use crate::rng::Rng;
|
||||
use crate::taxonomy::Level;
|
||||
use crate::taxonomy::{Level, Tier};
|
||||
|
||||
/// How well a student has met one objective.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -74,14 +96,27 @@ impl Mastery {
|
||||
}
|
||||
}
|
||||
|
||||
/// One student's standing on one objective.
|
||||
/// One student's standing on one registry entry, at either tier.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ObjectiveMastery {
|
||||
/// The objective id.
|
||||
pub objective: String,
|
||||
/// The registry id this row reports on, at either tier.
|
||||
pub id: String,
|
||||
/// The objective text, for reports.
|
||||
pub text: String,
|
||||
/// How many items on this objective the student saw.
|
||||
/// Which tier this row is, since only one of them is a classification.
|
||||
pub tier: Tier,
|
||||
/// The objective this row sits under, for a target row.
|
||||
pub objective: Option<String>,
|
||||
/// For an objective row, how many of its targets the exam reached.
|
||||
///
|
||||
/// A student report can say "four of the nine things under this objective
|
||||
/// were tested", which is the honest scope of the claim. Zero for a target
|
||||
/// row and for an objective with no targets.
|
||||
pub targets_seen: usize,
|
||||
/// For an objective row, how many targets it has in the registry.
|
||||
pub targets_total: usize,
|
||||
/// How many items the student saw. For an objective row, items tagged to any
|
||||
/// of its targets, counted once each.
|
||||
pub n_items: usize,
|
||||
/// How many they got right, counting partial credit.
|
||||
pub credit: f64,
|
||||
@@ -145,8 +180,8 @@ pub struct MissedItem {
|
||||
pub credit: f64,
|
||||
/// The level.
|
||||
pub level: Option<Level>,
|
||||
/// The objectives involved.
|
||||
pub learning_objectives: Vec<String>,
|
||||
/// The learning targets the question measured.
|
||||
pub learning_targets: Vec<String>,
|
||||
/// The misconception the chosen distractor was written to detect.
|
||||
pub misconception: Option<String>,
|
||||
/// Feedback written for a student who chose that option.
|
||||
@@ -208,7 +243,13 @@ impl StudentSummary {
|
||||
pub struct Cohort {
|
||||
/// Per-student summaries, sorted by key.
|
||||
pub students: Vec<StudentSummary>,
|
||||
/// Class rate per objective.
|
||||
/// Class rate per target, which is the tier items are tagged at. Use it to
|
||||
/// drill into an objective the class missed.
|
||||
pub target_rates: BTreeMap<String, f64>,
|
||||
/// Class rate per objective, with each item counted once.
|
||||
///
|
||||
/// This is the class-level table worth acting on, and the one
|
||||
/// [`Cohort::class_gaps`] is drawn from.
|
||||
pub objective_rates: BTreeMap<String, f64>,
|
||||
/// Class rate per level.
|
||||
pub level_rates: BTreeMap<Level, f64>,
|
||||
@@ -218,6 +259,11 @@ pub struct Cohort {
|
||||
pub sd_percent: f64,
|
||||
/// Objectives the class as a whole did not meet, worst first. This is the
|
||||
/// list that should change what you reteach.
|
||||
///
|
||||
/// Objectives rather than targets, because a list of forty targets below
|
||||
/// threshold is a list nobody reteaches from, and because a target that
|
||||
/// carried one item on this exam does not support the claim that the class
|
||||
/// missed it.
|
||||
pub class_gaps: Vec<(String, f64)>,
|
||||
/// Optional grouping of students by response profile.
|
||||
pub archetypes: Vec<Archetype>,
|
||||
@@ -289,7 +335,8 @@ pub fn summarize(
|
||||
let students = set.students();
|
||||
|
||||
// Class rates first: every student's report is relative to these.
|
||||
let objective_rates = rates_by_objective(&set.rows.iter().collect::<Vec<_>>());
|
||||
let target_rates = rates_by_target(&set.rows.iter().collect::<Vec<_>>());
|
||||
let objective_rates = rates_by_objective(&set.rows.iter().collect::<Vec<_>>(), course);
|
||||
let level_rates = rates_by_level(&set.rows.iter().collect::<Vec<_>>());
|
||||
|
||||
// Per-level spread across students, for the z comparisons.
|
||||
@@ -345,28 +392,62 @@ pub fn summarize(
|
||||
.count();
|
||||
let n_items = rows.iter().filter(|r| r.counts()).count();
|
||||
|
||||
// Objectives, in the course's declared order so reports read the way the
|
||||
// course is taught rather than alphabetically.
|
||||
let per_objective = rates_by_objective(&rows);
|
||||
let counts = counts_by_objective(&rows);
|
||||
// The registry in the course's declared order, so a report reads the way
|
||||
// the course is taught rather than alphabetically. Each objective the
|
||||
// exam reached is followed by the targets it reached, which is the order
|
||||
// a report wants them in: the claim, then its evidence.
|
||||
let target_counts = counts_by_target(&rows);
|
||||
let objective_counts = counts_by_objective(&rows, course);
|
||||
let mut objectives = Vec::new();
|
||||
let mut seen: BTreeSet<&String> = BTreeSet::new();
|
||||
for id in order.iter().chain(per_objective.keys()) {
|
||||
if !seen.insert(id) {
|
||||
let mut seen: BTreeSet<String> = BTreeSet::new();
|
||||
|
||||
// `order` puts each objective ahead of its own targets, so walking it
|
||||
// produces the tiering. Anything the exam measured that the registry
|
||||
// does not know about is appended afterwards rather than dropped.
|
||||
let measured: Vec<String> = target_counts.keys().cloned().collect();
|
||||
for id in order.iter().cloned().chain(measured) {
|
||||
if !seen.insert(id.clone()) {
|
||||
continue;
|
||||
}
|
||||
let Some((n, credit)) = counts.get(id).copied() else {
|
||||
continue;
|
||||
};
|
||||
objectives.push(objective_mastery(
|
||||
id,
|
||||
course,
|
||||
n,
|
||||
credit,
|
||||
objective_rates.get(id).copied().unwrap_or(0.0),
|
||||
&rows,
|
||||
policy,
|
||||
));
|
||||
if course.is_objective(&id) {
|
||||
let Some((n, credit)) = objective_counts.get(&id).copied() else {
|
||||
continue;
|
||||
};
|
||||
let targets = course.targets(&id);
|
||||
let reached = targets
|
||||
.iter()
|
||||
.filter(|target| target_counts.contains_key(**target))
|
||||
.count();
|
||||
objectives.push(objective_mastery(
|
||||
&id,
|
||||
course,
|
||||
n,
|
||||
credit,
|
||||
objective_rates.get(&id).copied().unwrap_or(0.0),
|
||||
&rows,
|
||||
policy.min_items_for_mastery.max(1),
|
||||
reached,
|
||||
targets.len(),
|
||||
));
|
||||
} else {
|
||||
let Some((n, credit)) = target_counts.get(&id).copied() else {
|
||||
continue;
|
||||
};
|
||||
// One item is the normal case for a target, so it is reported
|
||||
// rather than withheld. The objective row above it carries the
|
||||
// classification.
|
||||
objectives.push(objective_mastery(
|
||||
&id,
|
||||
course,
|
||||
n,
|
||||
credit,
|
||||
target_rates.get(&id).copied().unwrap_or(0.0),
|
||||
&rows,
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// Levels.
|
||||
@@ -400,25 +481,29 @@ pub fn summarize(
|
||||
// not: telling a student to review something they may already know costs
|
||||
// them an hour, while telling them they have mastered something they have
|
||||
// not costs them the next exam.
|
||||
//
|
||||
// Both lists are drawn from objective rows only. A focus list built from
|
||||
// targets is as long as the exam and tells a student to review forty
|
||||
// things, which is the same as telling them nothing; the objective list
|
||||
// is short enough to act on, and the target rows underneath it say what
|
||||
// to look at within each one.
|
||||
let strengths: Vec<String> = objectives
|
||||
.iter()
|
||||
.filter(|o| o.status == Mastery::Meeting && o.confident)
|
||||
.map(|o| o.objective.clone())
|
||||
.filter(|o| o.tier == Tier::Objective && o.status == Mastery::Meeting && o.confident)
|
||||
.map(|o| o.id.clone())
|
||||
.collect();
|
||||
let mut focus_pairs: Vec<(&ObjectiveMastery, f64)> = objectives
|
||||
.iter()
|
||||
.filter(|o| o.tier == Tier::Objective)
|
||||
.filter(|o| matches!(o.status, Mastery::NotYet | Mastery::Developing))
|
||||
.map(|o| (o, o.rate))
|
||||
.collect();
|
||||
focus_pairs.sort_by(|a, b| {
|
||||
a.1.partial_cmp(&b.1)
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
.then_with(|| a.0.objective.cmp(&b.0.objective))
|
||||
.then_with(|| a.0.id.cmp(&b.0.id))
|
||||
});
|
||||
let focus: Vec<String> = focus_pairs
|
||||
.iter()
|
||||
.map(|(o, _)| o.objective.clone())
|
||||
.collect();
|
||||
let focus: Vec<String> = focus_pairs.iter().map(|(o, _)| o.id.clone()).collect();
|
||||
|
||||
let missed = missed_items(&rows, catalog, course);
|
||||
|
||||
@@ -460,6 +545,7 @@ pub fn summarize(
|
||||
|
||||
Cohort {
|
||||
students: summaries,
|
||||
target_rates,
|
||||
objective_rates,
|
||||
level_rates,
|
||||
mean_percent,
|
||||
@@ -469,21 +555,26 @@ pub fn summarize(
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds one objective's mastery record.
|
||||
/// Builds one objective's record, at either tier.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `id` - the objective id.
|
||||
/// * `course` - the course, for text and policy.
|
||||
/// * `n` - items on this objective.
|
||||
/// * `course` - the course, for text, tier, and policy.
|
||||
/// * `n` - items counting toward this row.
|
||||
/// * `credit` - total credit earned.
|
||||
/// * `cohort_rate` - the class rate.
|
||||
/// * `cohort_rate` - the class rate for the same row.
|
||||
/// * `rows` - the student's responses, for the level list.
|
||||
/// * `policy` - the course policy.
|
||||
/// * `min_items` - items required before the row is classified. The policy's
|
||||
/// `min_items_for_mastery` for an objective row, and 1 for a target row, which
|
||||
/// is evidence rather than a classification.
|
||||
/// * `targets_seen` - targets this exam reached, for an objective row.
|
||||
/// * `targets_total` - targets in the registry, for an objective row.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The record.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn objective_mastery(
|
||||
id: &str,
|
||||
course: &CourseFile,
|
||||
@@ -491,12 +582,15 @@ fn objective_mastery(
|
||||
credit: f64,
|
||||
cohort_rate: f64,
|
||||
rows: &[&Response],
|
||||
policy: &Policy,
|
||||
min_items: usize,
|
||||
targets_seen: usize,
|
||||
targets_total: usize,
|
||||
) -> ObjectiveMastery {
|
||||
let policy = &course.policy;
|
||||
let rate = if n > 0 { credit / n as f64 } else { 0.0 };
|
||||
let (lower, upper) = wilson(credit, n, 1.96);
|
||||
|
||||
let status = if n < policy.min_items_for_mastery.max(1) {
|
||||
let status = if n < min_items.max(1) {
|
||||
Mastery::NotEnoughEvidence
|
||||
} else if rate >= policy.mastery_threshold {
|
||||
Mastery::Meeting
|
||||
@@ -506,17 +600,35 @@ fn objective_mastery(
|
||||
Mastery::NotYet
|
||||
};
|
||||
|
||||
// Levels the row was assessed at. For an objective row this is every level
|
||||
// any of its targets was assessed at, which is what makes "met this
|
||||
// objective" a checkable claim: meeting it on three Remember items is a
|
||||
// different statement from meeting it on three Analyze items.
|
||||
let levels: Vec<Level> = rows
|
||||
.iter()
|
||||
.filter(|r| r.learning_objectives.iter().any(|o| o == id))
|
||||
.filter(|r| {
|
||||
r.learning_targets
|
||||
.iter()
|
||||
.any(|t| t == id || course.objective_for(t) == id)
|
||||
})
|
||||
.filter_map(|r| r.level)
|
||||
.collect::<BTreeSet<Level>>()
|
||||
.into_iter()
|
||||
.collect();
|
||||
|
||||
ObjectiveMastery {
|
||||
objective: id.to_string(),
|
||||
text: course.objective_text(id),
|
||||
id: id.to_string(),
|
||||
text: course.text_for(id),
|
||||
tier: if course.is_objective(id) {
|
||||
Tier::Objective
|
||||
} else {
|
||||
Tier::Target
|
||||
},
|
||||
objective: course
|
||||
.is_target(id)
|
||||
.then(|| course.objective_for(id).to_string()),
|
||||
targets_seen,
|
||||
targets_total,
|
||||
n_items: n,
|
||||
credit,
|
||||
rate,
|
||||
@@ -595,7 +707,7 @@ fn missed_items(
|
||||
selected: r.chosen().to_vec(),
|
||||
credit: r.credit,
|
||||
level: r.level,
|
||||
learning_objectives: r.learning_objectives.clone(),
|
||||
learning_targets: r.learning_targets.clone(),
|
||||
misconception,
|
||||
feedback,
|
||||
study,
|
||||
@@ -612,9 +724,9 @@ fn missed_items(
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The rate for each objective mentioned.
|
||||
pub fn rates_by_objective(rows: &[&Response]) -> BTreeMap<String, f64> {
|
||||
counts_by_objective(rows)
|
||||
/// The rate for each target mentioned.
|
||||
pub fn rates_by_target(rows: &[&Response]) -> BTreeMap<String, f64> {
|
||||
counts_by_target(rows)
|
||||
.into_iter()
|
||||
.map(|(id, (n, credit))| {
|
||||
let rate = if n > 0 { credit / n as f64 } else { 0.0 };
|
||||
@@ -623,11 +735,12 @@ pub fn rates_by_objective(rows: &[&Response]) -> BTreeMap<String, f64> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Item counts and credit per objective.
|
||||
/// Item counts and credit per target, as tagged.
|
||||
///
|
||||
/// An item tagged with two objectives counts toward both. That double counting is
|
||||
/// An item tagged with two targets counts toward both. That double counting is
|
||||
/// intentional: the question "how is this student doing on kinetics" should use
|
||||
/// every item that measured kinetics.
|
||||
/// every item that measured kinetics. Roll-up to the objective, where the same
|
||||
/// item must count once, is [`counts_by_objective`].
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
@@ -635,14 +748,14 @@ pub fn rates_by_objective(rows: &[&Response]) -> BTreeMap<String, f64> {
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// `(item count, total credit)` per objective.
|
||||
pub fn counts_by_objective(rows: &[&Response]) -> BTreeMap<String, (usize, f64)> {
|
||||
/// `(item count, total credit)` per target.
|
||||
pub fn counts_by_target(rows: &[&Response]) -> BTreeMap<String, (usize, f64)> {
|
||||
let mut out: BTreeMap<String, (usize, f64)> = BTreeMap::new();
|
||||
for r in rows {
|
||||
if !r.counts() {
|
||||
continue;
|
||||
}
|
||||
for objective in &r.learning_objectives {
|
||||
for objective in &r.learning_targets {
|
||||
let e = out.entry(objective.clone()).or_insert((0, 0.0));
|
||||
e.0 += 1;
|
||||
e.1 += r.credit.clamp(0.0, 1.0);
|
||||
@@ -651,6 +764,66 @@ pub fn counts_by_objective(rows: &[&Response]) -> BTreeMap<String, (usize, f64)>
|
||||
out
|
||||
}
|
||||
|
||||
/// Item counts and credit per objective.
|
||||
///
|
||||
/// Each response contributes at most once to any one objective, even when it is
|
||||
/// tagged with several of that objective's targets. Within a single denominator,
|
||||
/// counting an item twice would inflate both the rate's weight and the
|
||||
/// confidence interval's tightness, and the interval is the part of the report
|
||||
/// that is supposed to stay honest. Across unrelated objectives an item still
|
||||
/// counts toward each, as it does in [`counts_by_target`].
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `rows` - the responses.
|
||||
/// * `course` - the course, for the objective each tagged target belongs to.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// `(item count, total credit)` per objective.
|
||||
pub fn counts_by_objective(
|
||||
rows: &[&Response],
|
||||
course: &CourseFile,
|
||||
) -> BTreeMap<String, (usize, f64)> {
|
||||
let mut out: BTreeMap<String, (usize, f64)> = BTreeMap::new();
|
||||
for r in rows {
|
||||
if !r.counts() {
|
||||
continue;
|
||||
}
|
||||
let objectives: BTreeSet<&str> = r
|
||||
.learning_targets
|
||||
.iter()
|
||||
.map(|t| course.objective_for(t))
|
||||
.collect();
|
||||
for id in objectives {
|
||||
let e = out.entry(id.to_string()).or_insert((0, 0.0));
|
||||
e.0 += 1;
|
||||
e.1 += r.credit.clamp(0.0, 1.0);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Credit rate per objective.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `rows` - the responses.
|
||||
/// * `course` - the course, for the objective each tagged target belongs to.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The rate for each objective the responses reached.
|
||||
pub fn rates_by_objective(rows: &[&Response], course: &CourseFile) -> BTreeMap<String, f64> {
|
||||
counts_by_objective(rows, course)
|
||||
.into_iter()
|
||||
.map(|(id, (n, credit))| {
|
||||
let rate = if n > 0 { credit / n as f64 } else { 0.0 };
|
||||
(id, rate)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Credit rate per level.
|
||||
///
|
||||
/// # Arguments
|
||||
@@ -1007,21 +1180,113 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn objective_counts_credit_every_tagged_item() {
|
||||
fn target_counts_credit_every_tagged_item() {
|
||||
let rows = [
|
||||
make("s1", 1, 1.0, &["lo-a", "lo-b"], Some(Level::Remember)),
|
||||
make("s1", 2, 0.0, &["lo-a"], Some(Level::Apply)),
|
||||
];
|
||||
let refs: Vec<&Response> = rows.iter().collect();
|
||||
let counts = counts_by_objective(&refs);
|
||||
let counts = counts_by_target(&refs);
|
||||
// lo-a saw both items; lo-b only the first.
|
||||
assert_eq!(counts["lo-a"], (2, 1.0));
|
||||
assert_eq!(counts["lo-b"], (1, 1.0));
|
||||
let rates = rates_by_objective(&refs);
|
||||
let rates = rates_by_target(&refs);
|
||||
assert_eq!(rates["lo-a"], 0.5);
|
||||
assert_eq!(rates["lo-b"], 1.0);
|
||||
}
|
||||
|
||||
/// A course with one objective over three targets, plus an objective with
|
||||
/// no targets of its own.
|
||||
fn tiered_course() -> CourseFile {
|
||||
serde_yaml_ng::from_str(
|
||||
r#"
|
||||
course: { code: X, title: Y, term: Z }
|
||||
policy: { mastery_threshold: 0.75, min_items_for_mastery: 2 }
|
||||
learning_objectives:
|
||||
lo-binding: { text: Quantify binding., order: 1 }
|
||||
lo-standalone: { text: Untiered objective., order: 2 }
|
||||
learning_targets:
|
||||
t-kd: { text: Write the expression., objective: lo-binding, order: 1 }
|
||||
t-plot: { text: Read a plot., objective: lo-binding, order: 2 }
|
||||
t-window: { text: State the switching window., objective: lo-binding, order: 3 }
|
||||
"#,
|
||||
)
|
||||
.expect("course parses")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn items_on_targets_roll_up_to_their_objective() {
|
||||
let course = tiered_course();
|
||||
let rows = [
|
||||
make("s1", 1, 1.0, &["t-kd"], Some(Level::Remember)),
|
||||
make("s1", 2, 0.0, &["t-plot"], Some(Level::Apply)),
|
||||
make("s1", 3, 1.0, &["t-window"], Some(Level::Understand)),
|
||||
make("s1", 4, 1.0, &["lo-standalone"], Some(Level::Remember)),
|
||||
];
|
||||
let refs: Vec<&Response> = rows.iter().collect();
|
||||
|
||||
let objectives = counts_by_objective(&refs, &course);
|
||||
// Three items, two credited, in one denominator.
|
||||
assert_eq!(objectives["lo-binding"], (3, 2.0));
|
||||
assert_eq!(objectives["lo-standalone"], (1, 1.0));
|
||||
// The targets are not themselves objective rows.
|
||||
assert!(!objectives.contains_key("t-kd"));
|
||||
|
||||
// As-tagged counts are still available for the drill-down.
|
||||
let tagged = counts_by_target(&refs);
|
||||
assert_eq!(tagged["t-kd"], (1, 1.0));
|
||||
assert_eq!(tagged.len(), 4);
|
||||
|
||||
let rates = rates_by_objective(&refs, &course);
|
||||
assert!((rates["lo-binding"] - 2.0 / 3.0).abs() < 1e-9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_item_counts_once_toward_its_objective() {
|
||||
let course = tiered_course();
|
||||
// A single question tagged with two targets of the same objective.
|
||||
let rows = [make(
|
||||
"s1",
|
||||
1,
|
||||
0.0,
|
||||
&["t-kd", "t-plot"],
|
||||
Some(Level::Understand),
|
||||
)];
|
||||
let refs: Vec<&Response> = rows.iter().collect();
|
||||
|
||||
let objectives = counts_by_objective(&refs, &course);
|
||||
assert_eq!(
|
||||
objectives["lo-binding"],
|
||||
(1, 0.0),
|
||||
"one question is one item in the objective's denominator"
|
||||
);
|
||||
// Whereas as-tagged counting credits both targets, as it always has.
|
||||
let tagged = counts_by_target(&refs);
|
||||
assert_eq!(tagged["t-kd"], (1, 0.0));
|
||||
assert_eq!(tagged["t-plot"], (1, 0.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_untiered_course_rolls_up_to_itself() {
|
||||
// Adopting the second tier is optional: with no targets declared, every
|
||||
// entry is an objective and the rolled-up counts equal the tagged ones.
|
||||
let course: CourseFile = serde_yaml_ng::from_str(
|
||||
r#"
|
||||
course: { code: X, title: Y, term: Z }
|
||||
learning_objectives:
|
||||
lo-a: { text: A }
|
||||
lo-b: { text: B }
|
||||
"#,
|
||||
)
|
||||
.expect("course parses");
|
||||
let rows = [
|
||||
make("s1", 1, 1.0, &["lo-a", "lo-b"], Some(Level::Remember)),
|
||||
make("s1", 2, 0.0, &["lo-a"], Some(Level::Apply)),
|
||||
];
|
||||
let refs: Vec<&Response> = rows.iter().collect();
|
||||
assert_eq!(counts_by_objective(&refs, &course), counts_by_target(&refs));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn level_rates_ignore_untagged_items() {
|
||||
let rows = [
|
||||
@@ -1108,7 +1373,7 @@ mod tests {
|
||||
score: credit,
|
||||
response_time_seconds: None,
|
||||
level,
|
||||
learning_objectives: objectives.iter().map(|s| s.to_string()).collect(),
|
||||
learning_targets: objectives.iter().map(|s| s.to_string()).collect(),
|
||||
topics: vec![],
|
||||
bonus: false,
|
||||
dropped: false,
|
||||
|
||||
Reference in New Issue
Block a user