feat: add learning objective targets
Pipeline / check (pull_request) Failing after 3m25s
Pipeline / docs (pull_request) Skipped
Pipeline / nightly (pull_request) Skipped
Pipeline / release (pull_request) Skipped

This commit is contained in:
2026-09-21 14:02:34 -04:00
parent 9755417899
commit 8ec48fb185
32 changed files with 2396 additions and 448 deletions
+1 -1
View File
@@ -1015,7 +1015,7 @@ mod tests {
score: credit,
response_time_seconds: None,
level: None,
learning_objectives: vec![],
learning_targets: vec![],
topics: vec![],
bonus: false,
dropped: false,
+24 -26
View File
@@ -509,12 +509,9 @@ pub fn student(
.objectives
.iter()
.map(|mastery| ObjectiveRow {
id: mastery.objective.clone(),
id: mastery.id.clone(),
text: mastery.text.clone(),
unit: course
.learning_objectives
.get(&mastery.objective)
.and_then(|o| o.unit.clone()),
unit: course.objective_unit(&mastery.id).map(str::to_string),
n_items: mastery.n_items,
credit: mastery.credit,
rate: mastery.rate,
@@ -700,11 +697,11 @@ fn question_row(
number: row.item_number,
position: row.form_position.filter(|p| *p != row.item_number),
level: row.level.map(|l| l.code()),
objectives: row.learning_objectives.clone(),
objectives: row.learning_targets.clone(),
objective_texts: if missed {
row.learning_objectives
row.learning_targets
.iter()
.map(|id| catalog.course.objective_text(id))
.map(|id| catalog.course.text_for(id))
.collect()
} else {
// Only where it earns its space. Every question already carries its
@@ -830,15 +827,13 @@ fn lecture_focus(catalog: &Catalog, rows: &[&Response], opts: &Options) -> Vec<L
.extend(source.slides.iter().copied());
}
}
for objective in &row.learning_objectives {
if let Some(entry) = course.learning_objectives.get(objective) {
lectures.extend(entry.lectures.iter().cloned());
}
for target in &row.learning_targets {
lectures.extend(course.lectures_for(target).iter().cloned());
}
for lecture in lectures {
let tally = tallies.entry(lecture.clone()).or_default();
tally.objectives.extend(row.learning_objectives.clone());
tally.objectives.extend(row.learning_targets.clone());
tally.questions.insert(row.item_number);
if let Some(numbers) = slides.get(&lecture) {
tally.slides.extend(numbers.iter().copied());
@@ -866,7 +861,7 @@ fn lecture_focus(catalog: &Catalog, rows: &[&Response], opts: &Options) -> Vec<L
objectives: tally
.objectives
.iter()
.map(|id| course.objective_text(id))
.map(|id| course.text_for(id))
.collect(),
lecture,
}
@@ -1390,17 +1385,22 @@ pub fn cohort(
.collect();
// Objective counts come from the responses so that an objective assessed by
// two items is not reported as if it had one.
// two items is not reported as if it had one. Counted per objective, and by
// item number, so a question tagged with two of that objective's targets is
// one item here.
let mut objective_items: BTreeMap<String, BTreeSet<u32>> = BTreeMap::new();
for row in set.rows.iter().filter(|r| r.counts()) {
for objective in &row.learning_objectives {
for objective in &row.learning_targets {
objective_items
.entry(objective.clone())
.entry(course.objective_for(objective).to_string())
.or_default()
.insert(row.item_number);
}
}
// Objectives, not targets: this table is the reteaching queue, and it is
// only usable if it is short enough to read and each row rests on enough
// items to believe.
let mut objectives: Vec<CohortObjectiveRow> = cohort
.objective_rates
.iter()
@@ -1410,7 +1410,7 @@ pub fn cohort(
let mut not_yet = 0;
let mut thin = 0;
for student in &cohort.students {
if let Some(row) = student.objectives.iter().find(|o| &o.objective == id) {
if let Some(row) = student.objectives.iter().find(|o| &o.id == id) {
match row.status {
Mastery::Meeting => meeting += 1,
Mastery::Developing => developing += 1,
@@ -1421,7 +1421,7 @@ pub fn cohort(
}
CohortObjectiveRow {
id: id.clone(),
text: course.objective_text(id),
text: course.text_for(id),
n_items: objective_items.get(id).map(|s| s.len()).unwrap_or(0),
rate: *rate,
meeting,
@@ -1451,7 +1451,7 @@ pub fn cohort(
.map(|p| {
(
p.number,
(p.level.map(|l| l.code()), p.learning_objectives.clone()),
(p.level.map(|l| l.code()), p.learning_targets.clone()),
)
})
.collect();
@@ -1467,7 +1467,7 @@ pub fn cohort(
level: meta.and_then(|m| m.0),
objectives: meta.map(|m| m.1.clone()).unwrap_or_default(),
objective_texts: meta
.map(|m| m.1.iter().map(|id| course.objective_text(id)).collect())
.map(|m| m.1.iter().map(|id| course.text_for(id)).collect())
.unwrap_or_default(),
taught_in: item
.item_ref
@@ -1564,7 +1564,7 @@ pub fn cohort(
questions,
revise,
forms: form_rows(set, cohort),
blueprint: crate::select::check_blueprint(record),
blueprint: crate::select::check_blueprint(record, course),
patterns: cohort
.archetypes
.iter()
@@ -1739,10 +1739,8 @@ fn lecture_rows(
lectures.insert(source.lecture.clone());
}
}
for objective in &question.objectives {
if let Some(entry) = course.learning_objectives.get(objective) {
lectures.extend(entry.lectures.iter().cloned());
}
for target in &question.objectives {
lectures.extend(course.lectures_for(target).iter().cloned());
}
for lecture in lectures {
items.entry(lecture.clone()).or_default().push(question);
+325 -60
View File
@@ -24,6 +24,28 @@
//! uses the interval, so it is honest). A student can be "meeting" an objective
//! provisionally, and the report says so.
//!
//! # Which tier gets classified
//!
//! The registry has two tiers, objectives and their targets (see
//! [`crate::course::Objective`]), and they are reported differently because the
//! evidence behind them differs in kind. An **objective** is classified: its
//! denominator is every item tagged to any of its targets, which is how an exam
//! that spends twelve questions across a topic gets to make one statement with a
//! real denominator instead of twelve statements with none.
//!
//! A **target** is not classified. It usually carries one or two items, and
//! `min_items_for_mastery` would mark almost all of them "not enough evidence",
//! which would be true but useless. So target rows report the observed rate as
//! itemized evidence for the objective's classification, and a report should
//! present them that way: not "you have not mastered this" but "here is what you
//! missed inside the objective above".
//!
//! An item tagged with two targets of the same objective counts *once* toward
//! that objective. Double counting is right across unrelated objectives, where
//! the question "how is this student doing on kinetics" should use every item
//! that measured kinetics, but within one denominator it would inflate both the
//! count and the confidence.
//!
//! # Comparison to the cohort
//!
//! Per-level performance is reported against the class rather than in absolute
@@ -33,10 +55,10 @@
use std::collections::{BTreeMap, BTreeSet};
use crate::course::{CourseFile, Policy};
use crate::course::CourseFile;
use crate::responses::{Response, ResponseSet};
use crate::rng::Rng;
use crate::taxonomy::Level;
use crate::taxonomy::{Level, Tier};
/// How well a student has met one objective.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -74,14 +96,27 @@ impl Mastery {
}
}
/// One student's standing on one objective.
/// One student's standing on one registry entry, at either tier.
#[derive(Debug, Clone)]
pub struct ObjectiveMastery {
/// The objective id.
pub objective: String,
/// The registry id this row reports on, at either tier.
pub id: String,
/// The objective text, for reports.
pub text: String,
/// How many items on this objective the student saw.
/// Which tier this row is, since only one of them is a classification.
pub tier: Tier,
/// The objective this row sits under, for a target row.
pub objective: Option<String>,
/// For an objective row, how many of its targets the exam reached.
///
/// A student report can say "four of the nine things under this objective
/// were tested", which is the honest scope of the claim. Zero for a target
/// row and for an objective with no targets.
pub targets_seen: usize,
/// For an objective row, how many targets it has in the registry.
pub targets_total: usize,
/// How many items the student saw. For an objective row, items tagged to any
/// of its targets, counted once each.
pub n_items: usize,
/// How many they got right, counting partial credit.
pub credit: f64,
@@ -145,8 +180,8 @@ pub struct MissedItem {
pub credit: f64,
/// The level.
pub level: Option<Level>,
/// The objectives involved.
pub learning_objectives: Vec<String>,
/// The learning targets the question measured.
pub learning_targets: Vec<String>,
/// The misconception the chosen distractor was written to detect.
pub misconception: Option<String>,
/// Feedback written for a student who chose that option.
@@ -208,7 +243,13 @@ impl StudentSummary {
pub struct Cohort {
/// Per-student summaries, sorted by key.
pub students: Vec<StudentSummary>,
/// Class rate per objective.
/// Class rate per target, which is the tier items are tagged at. Use it to
/// drill into an objective the class missed.
pub target_rates: BTreeMap<String, f64>,
/// Class rate per objective, with each item counted once.
///
/// This is the class-level table worth acting on, and the one
/// [`Cohort::class_gaps`] is drawn from.
pub objective_rates: BTreeMap<String, f64>,
/// Class rate per level.
pub level_rates: BTreeMap<Level, f64>,
@@ -218,6 +259,11 @@ pub struct Cohort {
pub sd_percent: f64,
/// Objectives the class as a whole did not meet, worst first. This is the
/// list that should change what you reteach.
///
/// Objectives rather than targets, because a list of forty targets below
/// threshold is a list nobody reteaches from, and because a target that
/// carried one item on this exam does not support the claim that the class
/// missed it.
pub class_gaps: Vec<(String, f64)>,
/// Optional grouping of students by response profile.
pub archetypes: Vec<Archetype>,
@@ -289,7 +335,8 @@ pub fn summarize(
let students = set.students();
// Class rates first: every student's report is relative to these.
let objective_rates = rates_by_objective(&set.rows.iter().collect::<Vec<_>>());
let target_rates = rates_by_target(&set.rows.iter().collect::<Vec<_>>());
let objective_rates = rates_by_objective(&set.rows.iter().collect::<Vec<_>>(), course);
let level_rates = rates_by_level(&set.rows.iter().collect::<Vec<_>>());
// Per-level spread across students, for the z comparisons.
@@ -345,28 +392,62 @@ pub fn summarize(
.count();
let n_items = rows.iter().filter(|r| r.counts()).count();
// Objectives, in the course's declared order so reports read the way the
// course is taught rather than alphabetically.
let per_objective = rates_by_objective(&rows);
let counts = counts_by_objective(&rows);
// The registry in the course's declared order, so a report reads the way
// the course is taught rather than alphabetically. Each objective the
// exam reached is followed by the targets it reached, which is the order
// a report wants them in: the claim, then its evidence.
let target_counts = counts_by_target(&rows);
let objective_counts = counts_by_objective(&rows, course);
let mut objectives = Vec::new();
let mut seen: BTreeSet<&String> = BTreeSet::new();
for id in order.iter().chain(per_objective.keys()) {
if !seen.insert(id) {
let mut seen: BTreeSet<String> = BTreeSet::new();
// `order` puts each objective ahead of its own targets, so walking it
// produces the tiering. Anything the exam measured that the registry
// does not know about is appended afterwards rather than dropped.
let measured: Vec<String> = target_counts.keys().cloned().collect();
for id in order.iter().cloned().chain(measured) {
if !seen.insert(id.clone()) {
continue;
}
let Some((n, credit)) = counts.get(id).copied() else {
continue;
};
objectives.push(objective_mastery(
id,
course,
n,
credit,
objective_rates.get(id).copied().unwrap_or(0.0),
&rows,
policy,
));
if course.is_objective(&id) {
let Some((n, credit)) = objective_counts.get(&id).copied() else {
continue;
};
let targets = course.targets(&id);
let reached = targets
.iter()
.filter(|target| target_counts.contains_key(**target))
.count();
objectives.push(objective_mastery(
&id,
course,
n,
credit,
objective_rates.get(&id).copied().unwrap_or(0.0),
&rows,
policy.min_items_for_mastery.max(1),
reached,
targets.len(),
));
} else {
let Some((n, credit)) = target_counts.get(&id).copied() else {
continue;
};
// One item is the normal case for a target, so it is reported
// rather than withheld. The objective row above it carries the
// classification.
objectives.push(objective_mastery(
&id,
course,
n,
credit,
target_rates.get(&id).copied().unwrap_or(0.0),
&rows,
1,
0,
0,
));
}
}
// Levels.
@@ -400,25 +481,29 @@ pub fn summarize(
// not: telling a student to review something they may already know costs
// them an hour, while telling them they have mastered something they have
// not costs them the next exam.
//
// Both lists are drawn from objective rows only. A focus list built from
// targets is as long as the exam and tells a student to review forty
// things, which is the same as telling them nothing; the objective list
// is short enough to act on, and the target rows underneath it say what
// to look at within each one.
let strengths: Vec<String> = objectives
.iter()
.filter(|o| o.status == Mastery::Meeting && o.confident)
.map(|o| o.objective.clone())
.filter(|o| o.tier == Tier::Objective && o.status == Mastery::Meeting && o.confident)
.map(|o| o.id.clone())
.collect();
let mut focus_pairs: Vec<(&ObjectiveMastery, f64)> = objectives
.iter()
.filter(|o| o.tier == Tier::Objective)
.filter(|o| matches!(o.status, Mastery::NotYet | Mastery::Developing))
.map(|o| (o, o.rate))
.collect();
focus_pairs.sort_by(|a, b| {
a.1.partial_cmp(&b.1)
.unwrap_or(std::cmp::Ordering::Equal)
.then_with(|| a.0.objective.cmp(&b.0.objective))
.then_with(|| a.0.id.cmp(&b.0.id))
});
let focus: Vec<String> = focus_pairs
.iter()
.map(|(o, _)| o.objective.clone())
.collect();
let focus: Vec<String> = focus_pairs.iter().map(|(o, _)| o.id.clone()).collect();
let missed = missed_items(&rows, catalog, course);
@@ -460,6 +545,7 @@ pub fn summarize(
Cohort {
students: summaries,
target_rates,
objective_rates,
level_rates,
mean_percent,
@@ -469,21 +555,26 @@ pub fn summarize(
}
}
/// Builds one objective's mastery record.
/// Builds one objective's record, at either tier.
///
/// # Arguments
///
/// * `id` - the objective id.
/// * `course` - the course, for text and policy.
/// * `n` - items on this objective.
/// * `course` - the course, for text, tier, and policy.
/// * `n` - items counting toward this row.
/// * `credit` - total credit earned.
/// * `cohort_rate` - the class rate.
/// * `cohort_rate` - the class rate for the same row.
/// * `rows` - the student's responses, for the level list.
/// * `policy` - the course policy.
/// * `min_items` - items required before the row is classified. The policy's
/// `min_items_for_mastery` for an objective row, and 1 for a target row, which
/// is evidence rather than a classification.
/// * `targets_seen` - targets this exam reached, for an objective row.
/// * `targets_total` - targets in the registry, for an objective row.
///
/// # Returns
///
/// The record.
#[allow(clippy::too_many_arguments)]
fn objective_mastery(
id: &str,
course: &CourseFile,
@@ -491,12 +582,15 @@ fn objective_mastery(
credit: f64,
cohort_rate: f64,
rows: &[&Response],
policy: &Policy,
min_items: usize,
targets_seen: usize,
targets_total: usize,
) -> ObjectiveMastery {
let policy = &course.policy;
let rate = if n > 0 { credit / n as f64 } else { 0.0 };
let (lower, upper) = wilson(credit, n, 1.96);
let status = if n < policy.min_items_for_mastery.max(1) {
let status = if n < min_items.max(1) {
Mastery::NotEnoughEvidence
} else if rate >= policy.mastery_threshold {
Mastery::Meeting
@@ -506,17 +600,35 @@ fn objective_mastery(
Mastery::NotYet
};
// Levels the row was assessed at. For an objective row this is every level
// any of its targets was assessed at, which is what makes "met this
// objective" a checkable claim: meeting it on three Remember items is a
// different statement from meeting it on three Analyze items.
let levels: Vec<Level> = rows
.iter()
.filter(|r| r.learning_objectives.iter().any(|o| o == id))
.filter(|r| {
r.learning_targets
.iter()
.any(|t| t == id || course.objective_for(t) == id)
})
.filter_map(|r| r.level)
.collect::<BTreeSet<Level>>()
.into_iter()
.collect();
ObjectiveMastery {
objective: id.to_string(),
text: course.objective_text(id),
id: id.to_string(),
text: course.text_for(id),
tier: if course.is_objective(id) {
Tier::Objective
} else {
Tier::Target
},
objective: course
.is_target(id)
.then(|| course.objective_for(id).to_string()),
targets_seen,
targets_total,
n_items: n,
credit,
rate,
@@ -595,7 +707,7 @@ fn missed_items(
selected: r.chosen().to_vec(),
credit: r.credit,
level: r.level,
learning_objectives: r.learning_objectives.clone(),
learning_targets: r.learning_targets.clone(),
misconception,
feedback,
study,
@@ -612,9 +724,9 @@ fn missed_items(
///
/// # Returns
///
/// The rate for each objective mentioned.
pub fn rates_by_objective(rows: &[&Response]) -> BTreeMap<String, f64> {
counts_by_objective(rows)
/// The rate for each target mentioned.
pub fn rates_by_target(rows: &[&Response]) -> BTreeMap<String, f64> {
counts_by_target(rows)
.into_iter()
.map(|(id, (n, credit))| {
let rate = if n > 0 { credit / n as f64 } else { 0.0 };
@@ -623,11 +735,12 @@ pub fn rates_by_objective(rows: &[&Response]) -> BTreeMap<String, f64> {
.collect()
}
/// Item counts and credit per objective.
/// Item counts and credit per target, as tagged.
///
/// An item tagged with two objectives counts toward both. That double counting is
/// An item tagged with two targets counts toward both. That double counting is
/// intentional: the question "how is this student doing on kinetics" should use
/// every item that measured kinetics.
/// every item that measured kinetics. Roll-up to the objective, where the same
/// item must count once, is [`counts_by_objective`].
///
/// # Arguments
///
@@ -635,14 +748,14 @@ pub fn rates_by_objective(rows: &[&Response]) -> BTreeMap<String, f64> {
///
/// # Returns
///
/// `(item count, total credit)` per objective.
pub fn counts_by_objective(rows: &[&Response]) -> BTreeMap<String, (usize, f64)> {
/// `(item count, total credit)` per target.
pub fn counts_by_target(rows: &[&Response]) -> BTreeMap<String, (usize, f64)> {
let mut out: BTreeMap<String, (usize, f64)> = BTreeMap::new();
for r in rows {
if !r.counts() {
continue;
}
for objective in &r.learning_objectives {
for objective in &r.learning_targets {
let e = out.entry(objective.clone()).or_insert((0, 0.0));
e.0 += 1;
e.1 += r.credit.clamp(0.0, 1.0);
@@ -651,6 +764,66 @@ pub fn counts_by_objective(rows: &[&Response]) -> BTreeMap<String, (usize, f64)>
out
}
/// Item counts and credit per objective.
///
/// Each response contributes at most once to any one objective, even when it is
/// tagged with several of that objective's targets. Within a single denominator,
/// counting an item twice would inflate both the rate's weight and the
/// confidence interval's tightness, and the interval is the part of the report
/// that is supposed to stay honest. Across unrelated objectives an item still
/// counts toward each, as it does in [`counts_by_target`].
///
/// # Arguments
///
/// * `rows` - the responses.
/// * `course` - the course, for the objective each tagged target belongs to.
///
/// # Returns
///
/// `(item count, total credit)` per objective.
pub fn counts_by_objective(
rows: &[&Response],
course: &CourseFile,
) -> BTreeMap<String, (usize, f64)> {
let mut out: BTreeMap<String, (usize, f64)> = BTreeMap::new();
for r in rows {
if !r.counts() {
continue;
}
let objectives: BTreeSet<&str> = r
.learning_targets
.iter()
.map(|t| course.objective_for(t))
.collect();
for id in objectives {
let e = out.entry(id.to_string()).or_insert((0, 0.0));
e.0 += 1;
e.1 += r.credit.clamp(0.0, 1.0);
}
}
out
}
/// Credit rate per objective.
///
/// # Arguments
///
/// * `rows` - the responses.
/// * `course` - the course, for the objective each tagged target belongs to.
///
/// # Returns
///
/// The rate for each objective the responses reached.
pub fn rates_by_objective(rows: &[&Response], course: &CourseFile) -> BTreeMap<String, f64> {
counts_by_objective(rows, course)
.into_iter()
.map(|(id, (n, credit))| {
let rate = if n > 0 { credit / n as f64 } else { 0.0 };
(id, rate)
})
.collect()
}
/// Credit rate per level.
///
/// # Arguments
@@ -1007,21 +1180,113 @@ mod tests {
}
#[test]
fn objective_counts_credit_every_tagged_item() {
fn target_counts_credit_every_tagged_item() {
let rows = [
make("s1", 1, 1.0, &["lo-a", "lo-b"], Some(Level::Remember)),
make("s1", 2, 0.0, &["lo-a"], Some(Level::Apply)),
];
let refs: Vec<&Response> = rows.iter().collect();
let counts = counts_by_objective(&refs);
let counts = counts_by_target(&refs);
// lo-a saw both items; lo-b only the first.
assert_eq!(counts["lo-a"], (2, 1.0));
assert_eq!(counts["lo-b"], (1, 1.0));
let rates = rates_by_objective(&refs);
let rates = rates_by_target(&refs);
assert_eq!(rates["lo-a"], 0.5);
assert_eq!(rates["lo-b"], 1.0);
}
/// A course with one objective over three targets, plus an objective with
/// no targets of its own.
fn tiered_course() -> CourseFile {
serde_yaml_ng::from_str(
r#"
course: { code: X, title: Y, term: Z }
policy: { mastery_threshold: 0.75, min_items_for_mastery: 2 }
learning_objectives:
lo-binding: { text: Quantify binding., order: 1 }
lo-standalone: { text: Untiered objective., order: 2 }
learning_targets:
t-kd: { text: Write the expression., objective: lo-binding, order: 1 }
t-plot: { text: Read a plot., objective: lo-binding, order: 2 }
t-window: { text: State the switching window., objective: lo-binding, order: 3 }
"#,
)
.expect("course parses")
}
#[test]
fn items_on_targets_roll_up_to_their_objective() {
let course = tiered_course();
let rows = [
make("s1", 1, 1.0, &["t-kd"], Some(Level::Remember)),
make("s1", 2, 0.0, &["t-plot"], Some(Level::Apply)),
make("s1", 3, 1.0, &["t-window"], Some(Level::Understand)),
make("s1", 4, 1.0, &["lo-standalone"], Some(Level::Remember)),
];
let refs: Vec<&Response> = rows.iter().collect();
let objectives = counts_by_objective(&refs, &course);
// Three items, two credited, in one denominator.
assert_eq!(objectives["lo-binding"], (3, 2.0));
assert_eq!(objectives["lo-standalone"], (1, 1.0));
// The targets are not themselves objective rows.
assert!(!objectives.contains_key("t-kd"));
// As-tagged counts are still available for the drill-down.
let tagged = counts_by_target(&refs);
assert_eq!(tagged["t-kd"], (1, 1.0));
assert_eq!(tagged.len(), 4);
let rates = rates_by_objective(&refs, &course);
assert!((rates["lo-binding"] - 2.0 / 3.0).abs() < 1e-9);
}
#[test]
fn one_item_counts_once_toward_its_objective() {
let course = tiered_course();
// A single question tagged with two targets of the same objective.
let rows = [make(
"s1",
1,
0.0,
&["t-kd", "t-plot"],
Some(Level::Understand),
)];
let refs: Vec<&Response> = rows.iter().collect();
let objectives = counts_by_objective(&refs, &course);
assert_eq!(
objectives["lo-binding"],
(1, 0.0),
"one question is one item in the objective's denominator"
);
// Whereas as-tagged counting credits both targets, as it always has.
let tagged = counts_by_target(&refs);
assert_eq!(tagged["t-kd"], (1, 0.0));
assert_eq!(tagged["t-plot"], (1, 0.0));
}
#[test]
fn an_untiered_course_rolls_up_to_itself() {
// Adopting the second tier is optional: with no targets declared, every
// entry is an objective and the rolled-up counts equal the tagged ones.
let course: CourseFile = serde_yaml_ng::from_str(
r#"
course: { code: X, title: Y, term: Z }
learning_objectives:
lo-a: { text: A }
lo-b: { text: B }
"#,
)
.expect("course parses");
let rows = [
make("s1", 1, 1.0, &["lo-a", "lo-b"], Some(Level::Remember)),
make("s1", 2, 0.0, &["lo-a"], Some(Level::Apply)),
];
let refs: Vec<&Response> = rows.iter().collect();
assert_eq!(counts_by_objective(&refs, &course), counts_by_target(&refs));
}
#[test]
fn level_rates_ignore_untagged_items() {
let rows = [
@@ -1108,7 +1373,7 @@ mod tests {
score: credit,
response_time_seconds: None,
level,
learning_objectives: objectives.iter().map(|s| s.to_string()).collect(),
learning_targets: objectives.iter().map(|s| s.to_string()).collect(),
topics: vec![],
bonus: false,
dropped: false,