diff --git a/src/analysis/classical.rs b/src/analysis/classical.rs index 20c3b7d..319ed38 100644 --- a/src/analysis/classical.rs +++ b/src/analysis/classical.rs @@ -396,6 +396,39 @@ pub fn analyze( )); } + // A drop changes every student's percentage, so the two ways to get it wrong + // are worth saying out loud. Both are silent otherwise: the numbers simply + // come out different from the platform's. + if let Some(record) = record { + for placement in record.items.iter().filter(|p| p.dropped) { + let rows = set.for_item(placement.number); + if rows.is_empty() { + continue; + } + let all_credited = rows.iter().all(|r| r.credit >= 0.999); + if placement.dropped_with_credit() && !all_credited { + let short = rows.iter().filter(|r| r.credit < 0.999).count(); + warnings.push(format!( + "question {} is marked `dropped_as: full_credit`, but {short} of {} responses \ + carry less than full credit. Either the platform was not regraded or the \ + export predates the regrade; until one of those is fixed this report's \ + percentages will sit below the grade of record", + placement.number, + rows.len() + )); + } + if !placement.dropped_with_credit() && all_credited { + warnings.push(format!( + "question {} is dropped and every response carries full credit, which is what \ + crediting every option on the platform looks like. It is being removed from \ + the denominator here, so this report will read slightly lower than the \ + platform. Set `dropped_as: full_credit` if the platform kept the point", + placement.number + )); + } + } + } + let mut items = Vec::new(); let mut p_values = Vec::new(); let mut rpbs = Vec::new(); @@ -986,6 +1019,7 @@ mod tests { topics: vec![], bonus: false, dropped: false, + dropped_full_credit: false, } } diff --git a/src/analysis/diagnostic.rs b/src/analysis/diagnostic.rs index 9db39fa..a30b437 100644 --- a/src/analysis/diagnostic.rs +++ b/src/analysis/diagnostic.rs @@ -172,12 +172,14 @@ pub struct StudentDiagnostic { /// One row per question, with no question in it. #[serde(skip_serializing_if = "Vec::is_empty")] pub questions: Vec, - /// Questions dropped from scoring, in number order. + /// Questions thrown out, in number order. /// /// Reported separately from the question rows so the document can say once, - /// in prose, why the denominator is smaller than the paper was. + /// in prose, what happened to them. The two kinds need different sentences: + /// a removed question is gone from the denominator, and a full-credit + /// question is still in it. #[serde(skip_serializing_if = "Vec::is_empty")] - pub dropped_questions: Vec, + pub dropped_questions: Vec, /// Which lectures to go back to, the one that would repay the most time /// first. #[serde(skip_serializing_if = "Vec::is_empty")] @@ -360,6 +362,17 @@ pub struct QuestionRow { pub review: Vec, } +/// One question thrown out after the exam. +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "kebab-case")] +pub struct DroppedQuestion { + /// The recorded question number. + pub number: u32, + /// Whether the drop was applied by crediting every option, in which case the + /// question is still in the points of record. + pub full_credit: bool, +} + /// One citation to read again after missing a question. #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] @@ -543,13 +556,16 @@ pub fn student( Vec::new() }; - let mut dropped_questions: Vec = rows + let mut dropped_questions: Vec = rows .iter() .filter(|r| r.dropped) - .map(|r| r.item_number) + .map(|r| DroppedQuestion { + number: r.item_number, + full_credit: r.dropped_full_credit, + }) .collect(); - dropped_questions.sort_unstable(); - dropped_questions.dedup(); + dropped_questions.sort_by_key(|d| d.number); + dropped_questions.dedup_by_key(|d| d.number); // Built from the response rows rather than from `questions`, so a report with // `--no-questions` still says where to go back to; it just does not name the @@ -952,11 +968,11 @@ pub struct CohortDiagnostic { pub lectures: Vec, /// Per-question statistics. pub questions: Vec, - /// Questions dropped from scoring, which are therefore absent from every - /// table above. Recorded so the report says why rather than leaving a gap in - /// the numbering. + /// Questions thrown out, which are therefore absent from every table above. + /// Recorded so the report says why rather than leaving a gap in the + /// numbering. #[serde(skip_serializing_if = "Vec::is_empty")] - pub dropped_questions: Vec, + pub dropped_questions: Vec, /// What to do about each question that raised something. pub triage: Triage, /// How the authored expectations did. @@ -1509,12 +1525,16 @@ pub fn cohort( .filter_map(|item| questions.iter().find(|q| q.number == item.number).cloned()) .collect(); - let mut dropped_questions: Vec = set + let mut dropped_questions: Vec = set .all_items() .into_iter() .filter(|n| set.item_dropped(*n)) + .map(|number| DroppedQuestion { + number, + full_credit: set.for_item(number).iter().any(|r| r.dropped_full_credit), + }) .collect(); - dropped_questions.sort_unstable(); + dropped_questions.sort_by_key(|d| d.number); let default_options = course.policy.options_per_item; let triage = triage(&questions, threshold, default_options); diff --git a/src/analysis/students.rs b/src/analysis/students.rs index a36b874..c688252 100644 --- a/src/analysis/students.rs +++ b/src/analysis/students.rs @@ -1112,6 +1112,7 @@ mod tests { topics: vec![], bonus: false, dropped: false, + dropped_full_credit: false, } } diff --git a/src/authoring/select.rs b/src/authoring/select.rs index 2614f98..51d7264 100644 --- a/src/authoring/select.rs +++ b/src/authoring/select.rs @@ -468,6 +468,7 @@ pub fn to_record( learning_objectives: e.item.learning_objectives.clone(), credit_overrides: BTreeMap::new(), dropped: false, + dropped_as: None, }); } diff --git a/src/data/canvas.rs b/src/data/canvas.rs index 0d9e873..7f6a764 100644 --- a/src/data/canvas.rs +++ b/src/data/canvas.rs @@ -351,6 +351,7 @@ pub fn ingest( topics: Vec::new(), bonus: false, dropped: false, + dropped_full_credit: false, }); } } diff --git a/src/data/decode.rs b/src/data/decode.rs index 3f51421..84bdc93 100644 --- a/src/data/decode.rs +++ b/src/data/decode.rs @@ -709,6 +709,7 @@ mod tests { topics: Vec::new(), bonus: false, dropped: false, + dropped_full_credit: false, } } diff --git a/src/data/gradescope.rs b/src/data/gradescope.rs index 6f18c7a..671c0e8 100644 --- a/src/data/gradescope.rs +++ b/src/data/gradescope.rs @@ -659,6 +659,7 @@ pub fn to_responses(questions: &[Question], ctx: &Context) -> Import { topics: Vec::new(), bonus, dropped: false, + dropped_full_credit: false, }); } } diff --git a/src/data/responses.rs b/src/data/responses.rs index d7dd79a..5b03952 100644 --- a/src/data/responses.rs +++ b/src/data/responses.rs @@ -108,10 +108,26 @@ pub struct Response { pub bonus: bool, /// Whether the item was dropped after the fact. pub dropped: bool, + /// Whether that drop was applied by crediting every option, so the item is + /// still part of the points of record even though it is out of the + /// statistics. + /// + /// Written by [`ResponseSet::enrich`] from the placement's `dropped_as`. + #[serde(default)] + pub dropped_full_credit: bool, } impl Response { - /// Whether this row should count toward scored totals and item statistics. + /// Whether this row counts as evidence. + /// + /// Evidence means item statistics, objective mastery, level rates, and the + /// IRT fit. A dropped item is never evidence, however the drop was applied: + /// an item everyone was given has no variance to contribute and would only + /// flatter the objective it was written against. + /// + /// This is deliberately not the same question as [`Response::scores`]. The + /// two were one predicate until dropping a question stopped always meaning + /// removing it. /// /// # Returns /// @@ -120,6 +136,21 @@ impl Response { !self.bonus && !self.dropped } + /// Whether this row counts toward the points of record. + /// + /// A question dropped by crediting every option still sits in the student's + /// total on the platform, and a report that disagreed with the platform + /// about a student's percentage would be worse than no report. So a + /// full-credit drop stays in both the numerator and the denominator here, + /// while a removed drop leaves both. + /// + /// # Returns + /// + /// `true` when the row belongs in the score. + pub fn scores(&self) -> bool { + !self.bonus && (!self.dropped || self.dropped_full_credit) + } + /// The response coded for a dichotomous model. /// /// Partial credit is rounded toward the majority: a half-credit response is @@ -188,7 +219,11 @@ impl ResponseSet { set.into_iter().map(|s| s.to_string()).collect() } - /// The distinct item numbers that count toward the scored total, sorted. + /// The distinct item numbers that count as evidence, sorted. + /// + /// Named for the scored total it once described; it is the analysis matrix's + /// item list, so it uses [`Response::counts`] and excludes every dropped + /// item. [`ResponseSet::points_available`] is the scoring denominator. pub fn scored_items(&self) -> Vec { let set: BTreeSet = self .rows @@ -247,11 +282,12 @@ impl ResponseSet { /// /// # Returns /// - /// The sum of `score` over scored, undropped items. + /// The sum of `score` over the items that count toward the score, which + /// includes a question dropped by crediting every option. pub fn scored_total(&self, key: &str) -> f64 { self.rows .iter() - .filter(|r| r.student_key == key && r.counts()) + .filter(|r| r.student_key == key && r.scores()) .map(|r| r.score) .sum() } @@ -269,7 +305,7 @@ impl ResponseSet { /// for each item so a student who skipped an item still has a denominator. pub fn points_available(&self) -> f64 { let mut per_item: BTreeMap = BTreeMap::new(); - for r in self.rows.iter().filter(|r| r.counts()) { + for r in self.rows.iter().filter(|r| r.scores()) { let e = per_item.entry(r.item_number).or_insert(0.0); if r.points_possible > *e { *e = r.points_possible; @@ -382,6 +418,7 @@ impl ResponseSet { r.item_version = p.version; r.bonus = r.bonus || p.bonus; r.dropped = r.dropped || p.dropped; + r.dropped_full_credit = r.dropped_full_credit || p.dropped_with_credit(); if let Some(points) = p.points { // The record is authoritative for points as administered; the // export sometimes carries a stale maximum. @@ -636,6 +673,9 @@ pub struct FlatResponse { /// Comma-joined eliminated letters in the bank's lettering. #[serde(default)] pub eliminated_source: String, + /// Whether a dropped item was dropped by crediting every option. + #[serde(default)] + pub dropped_full_credit: bool, } impl FlatResponse { @@ -656,6 +696,7 @@ impl FlatResponse { assessment_id: r.assessment_id.clone(), date: r.date.map(|d| d.to_string()).unwrap_or_default(), form: r.form.clone().unwrap_or_default(), + dropped_full_credit: r.dropped_full_credit, form_position: r.form_position.unwrap_or(0), student_key: r.student_key.clone(), sid: r.sid.clone().unwrap_or_default(), @@ -709,6 +750,7 @@ impl FlatResponse { assessment_id: self.assessment_id.clone(), date: self.date.parse().ok(), form: none_if_empty(&self.form), + dropped_full_credit: self.dropped_full_credit, form_position: if self.form_position == 0 { None } else { @@ -792,6 +834,7 @@ mod tests { topics: vec![], bonus: false, dropped: false, + dropped_full_credit: false, } } diff --git a/src/data/store.rs b/src/data/store.rs index 10d4028..c851b2e 100644 --- a/src/data/store.rs +++ b/src/data/store.rs @@ -564,6 +564,7 @@ mod tests { topics: vec![], bonus: false, dropped: false, + dropped_full_credit: false, } } diff --git a/src/data/store_parquet.rs b/src/data/store_parquet.rs index b521325..0343cd9 100644 --- a/src/data/store_parquet.rs +++ b/src/data/store_parquet.rs @@ -62,6 +62,7 @@ pub fn schema() -> Schema { Field::new("bonus", DataType::Boolean, false), Field::new("dropped", DataType::Boolean, false), Field::new("form_position", DataType::UInt32, false), + Field::new("dropped_full_credit", DataType::Boolean, false), Field::new("selected_source", DataType::Utf8, false), Field::new("eliminated_source", DataType::Utf8, false), ]) @@ -124,6 +125,7 @@ fn to_batch(rows: &[FlatResponse]) -> Result { boolc(|r| r.bonus), boolc(|r| r.dropped), u32c(|r| r.form_position), + boolc(|r| r.dropped_full_credit), s(|r| &r.selected_source), s(|r| &r.eliminated_source), ]; @@ -256,6 +258,11 @@ fn from_batch(batch: &RecordBatch, path: &Path) -> Result> { .column_by_name(name) .and_then(|c| c.as_any().downcast_ref::()) }; + let optional_bools = |name: &str| -> Option<&BooleanArray> { + batch + .column_by_name(name) + .and_then(|c| c.as_any().downcast_ref::()) + }; let administration_id = strings("administration_id")?; let course = strings("course")?; @@ -284,6 +291,7 @@ fn from_batch(batch: &RecordBatch, path: &Path) -> Result> { let dropped = bools("dropped")?; let form_position = optional_uints("form_position"); + let dropped_full_credit = optional_bools("dropped_full_credit"); let selected_source = optional_strings("selected_source"); let eliminated_source = optional_strings("eliminated_source"); @@ -322,6 +330,7 @@ fn from_batch(batch: &RecordBatch, path: &Path) -> Result> { topics: topics.value(i).to_string(), bonus: bonus.value(i), dropped: dropped.value(i), + dropped_full_credit: dropped_full_credit.map(|a| a.value(i)).unwrap_or(false), }); } Ok(out) @@ -367,6 +376,7 @@ mod tests { topics: "kinetics".into(), bonus: false, dropped: false, + dropped_full_credit: false, form_position: 0, selected_source: String::new(), eliminated_source: String::new(), diff --git a/src/export/practice.rs b/src/export/practice.rs index ed156ff..b3a7a4c 100644 --- a/src/export/practice.rs +++ b/src/export/practice.rs @@ -602,6 +602,7 @@ items: learning_objectives: Vec::new(), credit_overrides: Default::default(), dropped: false, + dropped_as: None, }, Placement { number: 2, @@ -615,6 +616,7 @@ items: learning_objectives: Vec::new(), credit_overrides: Default::default(), dropped: false, + dropped_as: None, }, ], } diff --git a/src/export/qti.rs b/src/export/qti.rs index b4aeab5..4aaa520 100644 --- a/src/export/qti.rs +++ b/src/export/qti.rs @@ -1377,6 +1377,7 @@ items: learning_objectives: Vec::new(), credit_overrides: Default::default(), dropped: false, + dropped_as: None, }, Placement { number: 2, @@ -1390,6 +1391,7 @@ items: learning_objectives: Vec::new(), credit_overrides: Default::default(), dropped: false, + dropped_as: None, }, ], } diff --git a/src/export/site.rs b/src/export/site.rs index b396161..9e2608e 100644 --- a/src/export/site.rs +++ b/src/export/site.rs @@ -866,6 +866,7 @@ items: learning_objectives: Vec::new(), credit_overrides: Default::default(), dropped: false, + dropped_as: None, }, Placement { number: 2, @@ -879,6 +880,7 @@ items: learning_objectives: Vec::new(), credit_overrides: Default::default(), dropped: false, + dropped_as: None, }, ], } diff --git a/src/export/typst.rs b/src/export/typst.rs index a31b87a..014c669 100644 --- a/src/export/typst.rs +++ b/src/export/typst.rs @@ -427,6 +427,7 @@ mod tests { learning_objectives: Vec::new(), credit_overrides: Default::default(), dropped: true, + dropped_as: None, }, Placement { number: 2, @@ -440,6 +441,7 @@ mod tests { learning_objectives: Vec::new(), credit_overrides: Default::default(), dropped: false, + dropped_as: None, }, ], }; diff --git a/src/export/typst/diagnostic.rs b/src/export/typst/diagnostic.rs index a321ad3..47ee911 100644 --- a/src/export/typst/diagnostic.rs +++ b/src/export/typst/diagnostic.rs @@ -355,7 +355,12 @@ pub fn student_value(diagnostic: &StudentDiagnostic, config: &RenderConfig) -> V diagnostic .dropped_questions .iter() - .map(|n| Value::Int(*n as i64)) + .map(|dropped| { + let mut value = Value::dict(); + value.insert("number", Value::Int(dropped.number as i64)); + value.insert("full-credit", Value::Bool(dropped.full_credit)); + value + }) .collect(), ), ); @@ -630,7 +635,12 @@ pub fn cohort_value(diagnostic: &CohortDiagnostic, config: &RenderConfig) -> Val diagnostic .dropped_questions .iter() - .map(|n| Value::Int(*n as i64)) + .map(|dropped| { + let mut value = Value::dict(); + value.insert("number", Value::Int(dropped.number as i64)); + value.insert("full-credit", Value::Bool(dropped.full_credit)); + value + }) .collect(), ), ); diff --git a/src/export/typst/templates/cohort-report.typ b/src/export/typst/templates/cohort-report.typ index 420ae8e..720ac40 100644 --- a/src/export/typst/templates/cohort-report.typ +++ b/src/export/typst/templates/cohort-report.typ @@ -994,11 +994,21 @@ let dropped = cb-data.at("dropped-questions", default: ()) if dropped.len() > 0 [ #plural(dropped.len(), "Question", "Questions") - #dropped.map(str).join(", ") + #dropped.map(d => str(d.number)).join(", ") #plural(dropped.len(), "is", "are") marked dropped in the assessment record, so #plural(dropped.len(), "it is", "they are") absent from this table and from every statistic above. + #{ + let credited = dropped.filter(d => d.at("full-credit", default: false)) + if credited.len() > 0 [ + #plural(credited.len(), "Question", "Questions") + #credited.map(d => str(d.number)).join(", ") + #plural(credited.len(), "was", "were") credited to every student, so + #plural(credited.len(), "it remains", "they remain") in the score + denominator and the means above match the grade of record. + ] + } ] } ] @@ -1173,69 +1183,71 @@ #let predictions = cb-data.at("predictions", default: (:)) #if show-predictions and predictions.at("predicted", default: 0) > 0 [ - = How your predictions did + #block(breakable: false)[ + = How your predictions did - #let n = predictions.at("predicted", default: 0) - #let calibrated = predictions.at("calibrated", default: 0) - #let signed-error = predictions.at("mean-signed-error", default: none) - #let abs-error = predictions.at("mean-abs-error", default: none) + #let n = predictions.at("predicted", default: 0) + #let calibrated = predictions.at("calibrated", default: 0) + #let signed-error = predictions.at("mean-signed-error", default: none) + #let abs-error = predictions.at("mean-abs-error", default: none) - #explain[ - #if calibrated == 0 [ - None of these #n #plural(n, "expectation", "expectations") rests on prior - data, so they are predictions rather than calibrations. A prediction that - misses is a fact about the prediction: it does not flag the item, and it is - summarised here instead of appearing #n times in the tables above. Once - `coursebank calibrate` has written statistics back into the bank, a - subsequent miss means the cohort or the teaching moved, and it will be - flagged. - ] else [ - #calibrated of #n #plural(n, "expectation", "expectations") rests on a - prior calibration. Those are the ones whose misses are flagged on the item, - because a calibrated item that moves is telling you about this cohort. The - rest are predictions, and a miss corrects the prediction. + #explain[ + #if calibrated == 0 [ + None of these #n #plural(n, "expectation", "expectations") rests on prior + data, so they are predictions rather than calibrations. A prediction that + misses is a fact about the prediction: it does not flag the item, and it is + summarised here instead of appearing #n times in the tables above. Once + `coursebank calibrate` has written statistics back into the bank, a + subsequent miss means the cohort or the teaching moved, and it will be + flagged. + ] else [ + #calibrated of #n #plural(n, "expectation", "expectations") rests on a + prior calibration. Those are the ones whose misses are flagged on the item, + because a calibrated item that moves is telling you about this cohort. The + rest are predictions, and a miss corrects the prediction. + ] ] - ] - #grid( - columns: (1fr, 1fr, 1fr), - gutter: 9pt, - stat-card( - "difficulty bias", - if signed-error != none { - (if signed-error >= 0 { "+" } else { "" }) + str(calc.round(signed-error * 100)) + " pts" - } else { "n/a" }, - note: if signed-error != none and signed-error > 0 { - "items came out easier than you expected" - } else if signed-error != none { - "items came out harder than you expected" - } else { none }, - ), - stat-card( - "typical miss", - if abs-error != none { str(calc.round(abs-error * 100)) + " pts" } else { "n/a" }, - note: str(predictions.at("within", default: 0)) + " of " + str(n) + " inside tolerance", - ), - stat-card( - "discrimination band", - str(predictions.at("band-hit", default: 0)) + " / " + str(predictions.at("band", default: 0)), - note: "landed in the band you expected", - ), - ) + #grid( + columns: (1fr, 1fr, 1fr), + gutter: 9pt, + stat-card( + "difficulty bias", + if signed-error != none { + (if signed-error >= 0 { "+" } else { "" }) + str(calc.round(signed-error * 100)) + " pts" + } else { "n/a" }, + note: if signed-error != none and signed-error > 0 { + "items came out easier than you expected" + } else if signed-error != none { + "items came out harder than you expected" + } else { none }, + ), + stat-card( + "typical miss", + if abs-error != none { str(calc.round(abs-error * 100)) + " pts" } else { "n/a" }, + note: str(predictions.at("within", default: 0)) + " of " + str(n) + " inside tolerance", + ), + stat-card( + "discrimination band", + str(predictions.at("band-hit", default: 0)) + " / " + str(predictions.at("band", default: 0)), + note: "landed in the band you expected", + ), + ) - #{ - let surprise = predictions.at("biggest-surprise", default: none) - if surprise != none { - block(above: entry-gap)[ - #pad(right: prose-pad)[ - #text(size: size-lead, fill: luma(95))[ - The largest single gap was question #surprise.number, predicted at - #pct(surprise.expected) and observed at #pct(surprise.observed). + #{ + let surprise = predictions.at("biggest-surprise", default: none) + if surprise != none { + block(above: entry-gap)[ + #pad(right: prose-pad)[ + #text(size: size-lead, fill: luma(95))[ + The largest single gap was question #surprise.number, predicted at + #pct(surprise.expected) and observed at #pct(surprise.observed). + ] ] ] - ] + } } - } + ] ] // ───────────────────────────────────────────────────────────────────────────── diff --git a/src/export/typst/templates/student-report.typ b/src/export/typst/templates/student-report.typ index 7a898c8..22df326 100644 --- a/src/export/typst/templates/student-report.typ +++ b/src/export/typst/templates/student-report.typ @@ -143,6 +143,7 @@ review: ((citation: "KKW §6.2", title: "Molecules and Medicine", url: "https://example.edu/6/2"),), ), ), + dropped-questions: ((number: 35, full-credit: true),), review-lectures: ( ( lecture: "L1.1", @@ -458,15 +459,30 @@ #block(above: entry-gap)[ #pad(right: prose-pad)[ #text(size: size-lead)[ - #plural(dropped-questions.len(), "Question", "Questions") - #dropped-questions.map(str).join(", ") - #plural(dropped-questions.len(), "was", "were") - dropped after the exam and #plural(dropped-questions.len(), "is", "are") - not part of anyone's score. Your percentage above is out of the - #plural(dropped-questions.len(), "remaining question", "remaining questions"). - Nothing you wrote on - #plural(dropped-questions.len(), "it", "them") - counted for or against you. + #{ + // Two kinds of drop, and they need different sentences. A credited + // question is still in the denominator, so telling a student it was + // removed would not match the arithmetic they can do themselves. + let credited = dropped-questions.filter(d => d.at("full-credit", default: false)) + let removed = dropped-questions.filter(d => not d.at("full-credit", default: false)) + let numbers = list => list.map(d => str(d.number)).join(", ") + + if credited.len() > 0 [ + #plural(credited.len(), "Question", "Questions") #numbers(credited) + #plural(credited.len(), "was", "were") thrown out after the exam. + Everyone received full credit for + #plural(credited.len(), "it", "them"), so + #plural(credited.len(), "it is", "they are") still counted in the + score above and whatever you chose made no difference. + ] + if removed.len() > 0 [ + #plural(removed.len(), "Question", "Questions") #numbers(removed) + #plural(removed.len(), "was", "were") thrown out and removed from + scoring, so your percentage is out of the remaining questions. + Nothing you wrote on #plural(removed.len(), "it", "them") counted + either way. + ] + } ] ] ] @@ -822,7 +838,8 @@ #box(width: 0.7em, height: 0.7em, fill: ok-color.lighten(70%), radius: 2pt) right · #box(width: 0.7em, height: 0.7em, fill: mid-color.lighten(70%), radius: 2pt) part marks · #box(width: 0.7em, height: 0.7em, fill: bad-color.lighten(70%), radius: 2pt) not right · - #box(width: 0.7em, height: 0.7em, fill: luma(210), radius: 2pt) left blank + #box(width: 0.7em, height: 0.7em, fill: luma(210), radius: 2pt) left blank · + #box(width: 0.7em, height: 0.7em, fill: luma(150), radius: 2pt) dropped, not scored ] ] ] diff --git a/src/model/assessment.rs b/src/model/assessment.rs index e406959..51ec04c 100644 --- a/src/model/assessment.rs +++ b/src/model/assessment.rs @@ -281,6 +281,48 @@ pub struct Placement { /// Set when an item was dropped from scoring after administration. #[serde(default, skip_serializing_if = "is_false")] pub dropped: bool, + /// How the drop was applied on the grading platform. + /// + /// Two ways to throw a question out, and they produce different + /// percentages. [`DropStyle::Removed`] takes the item out of the numerator + /// and the denominator: a student with 27 of 35 scores 77.1%. + /// [`DropStyle::FullCredit`] is what you do when the grade of record lives + /// somewhere else and the question cannot be removed from it: every option + /// is keyed, everyone earns the point, and the same student scores 28 of 36, + /// or 77.8%. + /// + /// Defaults to [`DropStyle::Removed`], which is what `dropped: true` meant + /// before this field existed. Set it to `full_credit` when you have credited + /// every option in the platform, so that the report agrees with the grade the + /// student can see. + /// + /// Either way the item is out of the item statistics, the objective + /// evidence, and the IRT fit: an item everyone got right has no variance to + /// contribute. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub dropped_as: Option, +} + +/// How a dropped item was handled on the grading platform. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DropStyle { + /// Taken out of the numerator and the denominator. + Removed, + /// Every option credited, so the item stays in both. + FullCredit, +} + +impl Placement { + /// Whether this placement was dropped by crediting every option. + /// + /// # Returns + /// + /// `true` only when the item is dropped *and* the drop was applied as full + /// credit, so the item still belongs in the points of record. + pub fn dropped_with_credit(&self) -> bool { + self.dropped && self.dropped_as == Some(DropStyle::FullCredit) + } } impl AssessmentFile { @@ -451,6 +493,20 @@ impl AssessmentFile { )); } } + + if p.dropped_as.is_some() && !p.dropped { + issues.push(format!( + "question {}: `dropped_as` is set but `dropped` is not, so nothing is dropped", + p.number + )); + } + if p.dropped && !p.credit_overrides.is_empty() { + issues.push(format!( + "question {}: dropped and carrying credit overrides. Pick one: an override \ + rescores an option, a drop removes the question", + p.number + )); + } } let mut form_ids: Vec<&str> = Vec::new();