diff --git a/src/analysis/diagnostic.rs b/src/analysis/diagnostic.rs index ed63b11..7a63804 100644 --- a/src/analysis/diagnostic.rs +++ b/src/analysis/diagnostic.rs @@ -1020,6 +1020,13 @@ pub struct CohortDiagnostic { pub predictions: PredictionSummary, /// Questions worth revisiting before reuse, worst first. pub revise: Vec, + /// The dropped items, described but not measured. + /// + /// Kept out of [`CohortDiagnostic::questions`] so that no statistic above + /// silently includes an item that was thrown out, and reported alongside it + /// in the evidence section so that dropping a question does not erase the + /// evidence for having dropped it. + pub dropped_detail: Vec, /// One row per form, when more than one was given. #[serde(skip_serializing_if = "Vec::is_empty")] pub forms: Vec, @@ -1138,6 +1145,26 @@ pub struct CohortQuestionRow { /// What those targets ask, in the course's own words. #[serde(skip_serializing_if = "Vec::is_empty")] pub target_texts: Vec, + /// Whether the item was dropped from scoring. + /// + /// A dropped item carries no p, r, or D, and appears in none of the + /// statistics above. It still appears in the evidence section, because the + /// option spread that justified dropping it is the record of why, and that + /// record should survive re-running the report afterwards. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub dropped: bool, + /// Whether the drop was applied as full credit to everyone. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub dropped_full_credit: bool, + /// The question as written. + /// + /// The instructor report reads better with it than without: a row of option + /// shares says a distractor drew 44% of the class, and only the stem says + /// whether that is a second defensible reading. Absent when the item has + /// left the bank, and withheld from any report that is not the instructor + /// copy. + #[serde(skip_serializing_if = "Option::is_none")] + pub stem: Option, /// Where the item was taught, as lecture titles and slide numbers. #[serde(skip_serializing_if = "Vec::is_empty")] pub taught_in: Vec, @@ -1185,8 +1212,25 @@ pub struct CohortQuestionRow { #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub struct OptionRow { - /// The bank letter. + /// The bank letter, which is the one every statistic is keyed by. pub letter: String, + /// The option as written, for a report that shows the question. + /// + /// Absent when the item is no longer in the bank, which is why this is an + /// option rather than an empty string: a missing option and an empty one + /// are different facts. + #[serde(skip_serializing_if = "Option::is_none")] + pub text: Option, + /// What this option was lettered on each printed form, worst case one entry + /// per form. + /// + /// Shuffling means the bank's option C is a different letter on every form, + /// so a statistic reported against C cannot be checked against a student's + /// paper without this map. It is the first thing anyone needs when a student + /// brings a paper to office hours, and working it out by hand from a seal is + /// the kind of task that gets done wrong once and then trusted. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub printed: Vec, /// How many chose it. pub count: usize, /// The share who chose it. @@ -1200,6 +1244,16 @@ pub struct OptionRow { pub nonfunctioning: bool, } +/// What one bank option was lettered on one form. +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "kebab-case")] +pub struct PrintedLetter { + /// The form id. + pub form: String, + /// The letter this option carried on that form's paper. + pub letter: String, +} + /// One form's summary. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] @@ -1502,11 +1556,17 @@ pub fn cohort( }) .collect(); + // The printed lettering per form, computed from the same two functions the + // exporter and the seal use, so the letters here are the letters on the + // paper rather than a second guess at them. + let forms: Vec<&crate::assessment::Form> = record.forms.iter().collect(); + let questions: Vec = analysis .items .iter() .map(|item| { let meta = item_meta.get(&item.number); + let entry = item.item_ref.as_deref().and_then(|uid| catalog.get(uid)); CohortQuestionRow { number: item.number, item: item.item_ref.clone(), @@ -1533,6 +1593,9 @@ pub fn cohort( .collect() }) .unwrap_or_default(), + dropped: false, + dropped_full_credit: false, + stem: entry.map(|e| e.item.stem.clone()), difficulty_band: difficulty_band(item.p_value).to_string(), discrimination_band: discrimination_band(item.point_biserial).to_string(), p_value: item.p_value, @@ -1544,6 +1607,23 @@ pub fn cohort( .options .values() .map(|option| OptionRow { + text: entry.and_then(|e| { + e.item + .options + .iter() + .find(|o| o.id == option.letter) + .map(|o| o.text.clone()) + }), + printed: entry + .map(|e| { + printed_letters( + &forms, + &e.item, + item.item_ref.as_deref(), + &option.letter, + ) + }) + .unwrap_or_default(), letter: option.letter.clone(), count: option.count, rate: option.rate, @@ -1582,6 +1662,106 @@ pub fn cohort( .collect(); dropped_questions.sort_by_key(|d| d.number); + // The dropped items, described from the responses rather than from the + // scoring. Dropping an item overrides its credit, so p, r, and D are + // meaningless for it and are left out. What students actually marked is + // untouched by the drop, and that spread is the evidence that justified it. + let mut dropped_detail: Vec = dropped_questions + .iter() + .map(|dropped| { + let placement = record.placement(dropped.number); + let uid = placement.map(|p| p.item.clone()); + let entry = uid.as_deref().and_then(|uid| catalog.get(uid)); + let responses = set.for_item(dropped.number); + let answered = responses + .iter() + .filter(|r| !r.chosen().is_empty()) + .count() + .max(1); + let keyed: BTreeSet = placement + .map(|p| p.key.iter().cloned().collect()) + .filter(|k: &BTreeSet| !k.is_empty()) + .or_else(|| entry.map(|e| e.item.key_letters().into_iter().collect())) + .unwrap_or_default(); + + // One row per option the item declares, so an option nobody + // marked still shows as unchosen rather than vanishing. + let options: Vec = entry + .map(|e| { + e.item + .options + .iter() + .map(|option| { + let count = responses + .iter() + .filter(|r| r.chosen().iter().any(|l| *l == option.id)) + .count(); + OptionRow { + text: Some(option.text.clone()), + printed: printed_letters( + &forms, + &e.item, + uid.as_deref(), + &option.id, + ), + letter: option.id.clone(), + count, + rate: count as f64 / answered as f64, + is_key: keyed.contains(&option.id), + point_biserial: None, + nonfunctioning: false, + } + }) + .collect() + }) + .unwrap_or_default(); + + let targets = placement + .map(|p| p.learning_targets.clone()) + .unwrap_or_default(); + CohortQuestionRow { + number: dropped.number, + item: uid.clone(), + level: placement.and_then(|p| p.level).map(|l| l.code()), + target_texts: targets.iter().map(|id| course.text_for(id)).collect(), + targets, + dropped: true, + dropped_full_credit: dropped.full_credit, + stem: entry.map(|e| e.item.stem.clone()), + taught_in: uid + .as_deref() + .map(|uid| taught_in(catalog, uid)) + .unwrap_or_default(), + lectures: entry + .map(|e| e.item.sources.iter().map(|s| s.lecture.clone()).collect()) + .unwrap_or_default(), + difficulty_band: String::new(), + discrimination_band: String::new(), + // The keyed share before the override, which is the closest + // honest reading of how the item performed. It is not a + // p-value: it counts marks, not credit. + p_value: options + .iter() + .filter(|o| o.is_key) + .map(|o| o.rate) + .sum::() + .min(1.0), + point_biserial: None, + discrimination: None, + blank_rate: responses.iter().filter(|r| r.chosen().is_empty()).count() as f64 + / responses.len().max(1) as f64, + key: keyed.iter().cloned().collect(), + options, + flags: Vec::new(), + notes: Vec::new(), + prediction_notes: Vec::new(), + calibrated: false, + by_form: BTreeMap::new(), + } + }) + .collect(); + dropped_detail.sort_by_key(|q| q.number); + let default_options = course.policy.options_per_item; let triage = triage(&questions, threshold, default_options); let predictions = prediction_summary(analysis); @@ -1609,6 +1789,7 @@ pub fn cohort( gaps, questions, revise, + dropped_detail, forms: form_rows(set, cohort), blueprint: crate::select::check_blueprint(record, course), patterns: cohort @@ -1669,6 +1850,53 @@ fn taught_in(catalog: &Catalog, uid: &str) -> Vec { .collect() } +/// Where one bank option landed on each printed form. +/// +/// # Arguments +/// +/// * `forms` - the record's forms, in declaration order. +/// * `item` - the bank item, for its option count and lettering. +/// * `uid` - the item's global id, which salts the permutation. +/// * `letter` - the bank letter to locate. +/// +/// # Returns +/// +/// One entry per form that permutes its options. Forms printing the bank order +/// unchanged are left out, since an entry saying C was printed as C is noise on +/// every row. +fn printed_letters( + forms: &[&crate::assessment::Form], + item: &crate::item::Item, + uid: Option<&str>, + letter: &str, +) -> Vec { + let Some(uid) = uid else { + return Vec::new(); + }; + let Some(source) = item.options.iter().position(|o| o.id == letter) else { + return Vec::new(); + }; + let n = item.options.len(); + let mut out = Vec::new(); + for form in forms { + if !form.shuffle_options { + continue; + } + // The same permutation the exporter and the seal use, so these are the + // letters on the paper rather than a second guess at them. + let order = crate::select::option_order(form, uid, n); + // `order[position] == source` means the option printed in that slot is + // the one being asked about. + if let Some(position) = order.iter().position(|index| *index == source) { + out.push(PrintedLetter { + form: form.id.clone(), + letter: crate::seal::printed_letter(position), + }); + } + } + out +} + /// The difficulty band a p-value falls in. /// /// Three bands rather than five. The only distinction that changes what you do diff --git a/src/data/decode.rs b/src/data/decode.rs index cc9b209..e5d75af 100644 --- a/src/data/decode.rs +++ b/src/data/decode.rs @@ -651,8 +651,7 @@ pub fn apply(set: &mut ResponseSet, decoder: &FormDecoder, numbering: Numbering) if !colliding_positions.is_empty() { let list: Vec = colliding_positions.iter().map(|n| n.to_string()).collect(); let discarded = set.rows.len(); - set.rows - .retain(|row| row.form.as_deref() != Some(UNMAPPED)); + set.rows.retain(|row| row.form.as_deref() != Some(UNMAPPED)); warnings.push(format!( "form {}: {} response(s) at position(s) {} could not be translated, and their raw \ numbers are numbers this form does use. Keeping them would have given those \ diff --git a/src/export/typst/diagnostic.rs b/src/export/typst/diagnostic.rs index deaaf0b..850aee5 100644 --- a/src/export/typst/diagnostic.rs +++ b/src/export/typst/diagnostic.rs @@ -620,6 +620,18 @@ pub fn cohort_value(diagnostic: &CohortDiagnostic, config: &RenderConfig) -> Val .collect(), ), ); + // Separate from `questions` so no statistic can pick them up, and merged + // back in by the evidence section, which describes rather than measures. + out.insert( + "dropped-detail", + Value::Array( + diagnostic + .dropped_detail + .iter() + .map(|q| cohort_question_value(q, content)) + .collect(), + ), + ); out.insert( "grades", @@ -872,6 +884,18 @@ fn cohort_question_value(question: &CohortQuestionRow, content: bool) -> Value { .collect(), ), ); + value.insert_some( + "stem", + question + .stem + .as_ref() + .map(|text| markup_value(text, content)), + ); + value.insert("dropped", Value::Bool(question.dropped)); + value.insert( + "dropped-full-credit", + Value::Bool(question.dropped_full_credit), + ); value.insert( "taught-in", Value::Array(question.taught_in.iter().map(Value::str).collect()), @@ -902,6 +926,27 @@ fn cohort_question_value(question: &CohortQuestionRow, content: bool) -> Value { .map(|option| { let mut value = Value::dict(); value.insert("letter", Value::str(&option.letter)); + value.insert_some( + "text", + option.text.as_ref().map(|text| markup_value(text, content)), + ); + // The letter on each paper, so a statistic reported against + // the bank letter can be checked against a student's copy. + value.insert( + "printed", + Value::Array( + option + .printed + .iter() + .map(|printed| { + let mut pair = Value::dict(); + pair.insert("form", Value::str(&printed.form)); + pair.insert("letter", Value::str(&printed.letter)); + pair + }) + .collect(), + ), + ); value.insert("count", Value::Int(option.count as i64)); value.insert("rate", Value::Float(option.rate)); value.insert("is-key", Value::Bool(option.is_key)); diff --git a/src/export/typst/templates/cohort-report.typ b/src/export/typst/templates/cohort-report.typ index 1e01e4a..43536c5 100644 --- a/src/export/typst/templates/cohort-report.typ +++ b/src/export/typst/templates/cohort-report.typ @@ -121,11 +121,32 @@ discrimination: 0.10, blank-rate: 0.0, key: ("B",), + stem: [A sample question stem, shown so the option shares can be read against what was asked.], options: ( - (letter: "A", count: 9, rate: 0.375, is-key: false, point-biserial: 0.11, nonfunctioning: false), - (letter: "B", count: 10, rate: 0.417, is-key: true, point-biserial: 0.05, nonfunctioning: false), - (letter: "C", count: 5, rate: 0.208, is-key: false, point-biserial: -0.2, nonfunctioning: false), - (letter: "D", count: 0, rate: 0.0, is-key: false, nonfunctioning: true), + ( + letter: "A", + text: [A sample distractor built on a real misconception.], + printed: ((form: "A", letter: "C"), (form: "B", letter: "A")), + count: 9, rate: 0.375, is-key: false, point-biserial: 0.11, nonfunctioning: false, + ), + ( + letter: "B", + text: [A sample key.], + printed: ((form: "A", letter: "D"), (form: "B", letter: "C")), + count: 10, rate: 0.417, is-key: true, point-biserial: 0.05, nonfunctioning: false, + ), + ( + letter: "C", + text: [A second sample distractor.], + printed: ((form: "A", letter: "A"), (form: "B", letter: "D")), + count: 5, rate: 0.208, is-key: false, point-biserial: -0.2, nonfunctioning: false, + ), + ( + letter: "D", + text: [A distractor nobody chose.], + printed: ((form: "A", letter: "B"), (form: "B", letter: "B")), + count: 0, rate: 0.0, is-key: false, nonfunctioning: true, + ), ), flags: ("ambiguous",), notes: ([Distractor A drew as many strong students as the key.],), @@ -168,6 +189,33 @@ biggest-surprise: (number: 14, expected: 0.45, observed: 0.86), ), dropped-questions: (), + dropped-detail: ( + ( + number: 9, + item: "bank::q-sample-009", + level: 2, + targets: ("t-sample-gap",), + target-texts: ([A sample learning target.],), + dropped: true, + dropped-full-credit: true, + stem: [A sample question that was thrown out after the exam.], + taught-in: ("Entropy (L1.2)",), + lectures: ("L1.2",), + p: 0.18, + blank-rate: 0.0, + key: ("A",), + options: ( + (letter: "A", text: [The keyed option, which few chose.], count: 4, rate: 0.18, is-key: true, nonfunctioning: false), + (letter: "B", text: [The option most students read as correct.], count: 15, rate: 0.68, is-key: false, nonfunctioning: false), + (letter: "C", text: [A third option.], count: 3, rate: 0.14, is-key: false, nonfunctioning: false), + ), + flags: (), + notes: (), + prediction-notes: (), + calibrated: false, + by-form: (:), + ), + ), revise: (), forms: ( (id: "A", students: 12, mean: 73.5, sd: 10.2), @@ -1077,28 +1125,45 @@ ] // ───────────────────────────────────────────────────────────────────────────── -// The revise queue, with option tables +// Every question, with option tables // ───────────────────────────────────────────────────────────────────────────── -#let revise = cb-data.at("revise", default: ()) +// Every scored question, not only the flagged ones. A question that raised no +// flag still has an option spread worth seeing: it is the reference for what a +// healthy item looks like on this exam, and the place to check a query about a +// question nothing was wrong with. The flagged ones are already called out by +// name in the triage lists above, so nothing is lost by putting them back among +// their neighbours here. +// Dropped items come from their own list, since they carry no statistics and +// belong in none of the tables above. They belong here: the option spread is +// the evidence that justified dropping them, and re-running the report after a +// drop should not erase it. +#let evidence = questions + cb-data.at("dropped-detail", default: ()) -#if revise.len() > 0 [ +#if evidence.len() > 0 [ #pagebreak(weak: true) = The evidence, question by question #explain[ - The full breakdown for every flagged question: what the flags mean, and who - chose what. In question order, so you can find one by its number; the lists - above are the same questions sorted by what they ask of you. The r column - beside each option is the correlation between choosing that option and - scoring well on the rest of the exam, which is how a defensible distractor - announces itself. + The full breakdown for every question, flagged or not, including the ones + you dropped: what it asked, who chose what, and what any flags mean. A + dropped question is marked as such and carries no p, r, or D, because + dropping it overrode its credit. What students marked is untouched by the + drop, so its option spread still stands as the record of why it went. In question order, so you can find + one by its number; the lists above are the subset that needs a decision from + you, sorted by what they ask of you. The r column beside each option is the + correlation between choosing that option and scoring well on the rest of the + exam, which is how a defensible distractor announces itself. + + Each option shows the bank letter first, then what it was lettered on each + form (#raw("A:D") means the bank's option was printed as D on form A), so a + row here can be read against the paper a student is holding. ] // Question order here, not triage order. The lists above are for deciding // what to do; this section is for looking one question up while you do it, // and a reader with a number in hand should not have to scan every entry. - #for q in revise.sorted(key: q => q.number) [ + #for q in evidence.sorted(key: q => q.number) [ #block(breakable: false, above: entry-gap, width: 100%)[ #{ let parts = ( @@ -1106,6 +1171,11 @@ let bits = (text(weight: "bold", fill: accent)[Question #q.number],) let item = q.at("item", default: none) if item != none { bits.push(text(size: size-meta, fill: luma(120))[#item]) } + if q.at("dropped", default: false) { + bits.push(text(size: size-micro, weight: "bold", fill: luma(110))[ + DROPPED#if q.at("dropped-full-credit", default: false) [ · FULL CREDIT] + ]) + } let flags = q.at("flags", default: ()) if flags.len() > 0 { bits.push(stack(dir: ltr, spacing: 3pt, ..flags.map(flag-badge))) @@ -1117,6 +1187,20 @@ let context_ = question-context(q) if context_ != none { parts.push(context_) } + // The question itself. A row of option shares says a distractor drew + // 44% of the class; only the stem says whether that is a second + // defensible reading. Instructor copy only, which is the only place this + // section appears. + let stem = q.at("stem", default: none) + if stem != none { + parts.push(block( + width: 100%, + fill: luma(252), + stroke: (left: 2pt + luma(220)), + inset: (left: 8pt, rest: 6pt), + )[#text(size: size-small)[#markup(stem)]]) + } + for note in q.at("notes", default: ()) { parts.push(text(size: size-small)[— #markup(note)]) } @@ -1141,19 +1225,50 @@ } if show-options and q.at("options", default: ()).len() > 0 { + // The option column is two things: the bank letter every statistic is + // keyed by, and the letter the option actually carried on each paper. + // Without the second, a note about option D cannot be checked against + // the copy a student brings to office hours, since shuffling gives the + // same option a different letter on every form. + // + // The text goes last and takes the free column. It is the only cell + // whose length is not bounded, so anywhere else it either squeezes the + // numbers or wraps to three lines while they sit in a narrow gutter. + // Last, the numbers keep their natural widths and the text runs to the + // page edge. parts.push(table( - columns: (auto, auto, auto, 2.6cm, auto), + columns: (auto, auto, auto, 2.6cm, auto, 1fr), stroke: none, - align: (center + horizon, right + horizon, right + horizon, left + horizon, right + horizon), + align: ( + center + horizon, + right + horizon, + right + horizon, + left + horizon, + right + horizon, + left + top, + ), inset: (x: 5pt, y: 3.5pt), - table.header(th[OPT], th[n], th[SHARE], th[], th[r]), + table.header(th[OPT], th[n], th[SHARE], th[], th[r], th[TEXT]), ..q .options .map(option => ( { - if option.at("is-key", default: false) { + let printed = option.at("printed", default: ()) + let letter = if option.at("is-key", default: false) { text(weight: "bold", fill: ok-color)[#option.letter] } else { [#option.letter] } + if printed.len() == 0 { + letter + } else { + stack( + dir: ttb, + spacing: step * 0.25, + letter, + text(size: size-micro, fill: luma(125))[ + #printed.map(p => p.form + ":" + p.letter).join(" ") + ], + ) + } }, text(size: size-small)[#option.count], text(size: size-small)[#pct(option.rate)], @@ -1172,6 +1287,16 @@ text(size: size-small)[#signed(r)] } }, + { + let body = option.at("text", default: none) + if body == none { + text(size: size-micro, fill: thin-color)[not in the bank] + } else if option.at("is-key", default: false) { + text(size: size-small, fill: ok-color.darken(25%))[#markup(body)] + } else { + text(size: size-small)[#markup(body)] + } + }, )) .flatten(), ))