feat: improve dropped question support
This commit is contained in:
@@ -396,6 +396,39 @@ pub fn analyze(
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A drop changes every student's percentage, so the two ways to get it wrong
|
||||||
|
// are worth saying out loud. Both are silent otherwise: the numbers simply
|
||||||
|
// come out different from the platform's.
|
||||||
|
if let Some(record) = record {
|
||||||
|
for placement in record.items.iter().filter(|p| p.dropped) {
|
||||||
|
let rows = set.for_item(placement.number);
|
||||||
|
if rows.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let all_credited = rows.iter().all(|r| r.credit >= 0.999);
|
||||||
|
if placement.dropped_with_credit() && !all_credited {
|
||||||
|
let short = rows.iter().filter(|r| r.credit < 0.999).count();
|
||||||
|
warnings.push(format!(
|
||||||
|
"question {} is marked `dropped_as: full_credit`, but {short} of {} responses \
|
||||||
|
carry less than full credit. Either the platform was not regraded or the \
|
||||||
|
export predates the regrade; until one of those is fixed this report's \
|
||||||
|
percentages will sit below the grade of record",
|
||||||
|
placement.number,
|
||||||
|
rows.len()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if !placement.dropped_with_credit() && all_credited {
|
||||||
|
warnings.push(format!(
|
||||||
|
"question {} is dropped and every response carries full credit, which is what \
|
||||||
|
crediting every option on the platform looks like. It is being removed from \
|
||||||
|
the denominator here, so this report will read slightly lower than the \
|
||||||
|
platform. Set `dropped_as: full_credit` if the platform kept the point",
|
||||||
|
placement.number
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let mut items = Vec::new();
|
let mut items = Vec::new();
|
||||||
let mut p_values = Vec::new();
|
let mut p_values = Vec::new();
|
||||||
let mut rpbs = Vec::new();
|
let mut rpbs = Vec::new();
|
||||||
@@ -986,6 +1019,7 @@ mod tests {
|
|||||||
topics: vec![],
|
topics: vec![],
|
||||||
bonus: false,
|
bonus: false,
|
||||||
dropped: false,
|
dropped: false,
|
||||||
|
dropped_full_credit: false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+33
-13
@@ -172,12 +172,14 @@ pub struct StudentDiagnostic {
|
|||||||
/// One row per question, with no question in it.
|
/// One row per question, with no question in it.
|
||||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||||
pub questions: Vec<QuestionRow>,
|
pub questions: Vec<QuestionRow>,
|
||||||
/// Questions dropped from scoring, in number order.
|
/// Questions thrown out, in number order.
|
||||||
///
|
///
|
||||||
/// Reported separately from the question rows so the document can say once,
|
/// Reported separately from the question rows so the document can say once,
|
||||||
/// in prose, why the denominator is smaller than the paper was.
|
/// in prose, what happened to them. The two kinds need different sentences:
|
||||||
|
/// a removed question is gone from the denominator, and a full-credit
|
||||||
|
/// question is still in it.
|
||||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||||
pub dropped_questions: Vec<u32>,
|
pub dropped_questions: Vec<DroppedQuestion>,
|
||||||
/// Which lectures to go back to, the one that would repay the most time
|
/// Which lectures to go back to, the one that would repay the most time
|
||||||
/// first.
|
/// first.
|
||||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||||
@@ -360,6 +362,17 @@ pub struct QuestionRow {
|
|||||||
pub review: Vec<ItemReading>,
|
pub review: Vec<ItemReading>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// One question thrown out after the exam.
|
||||||
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
#[serde(rename_all = "kebab-case")]
|
||||||
|
pub struct DroppedQuestion {
|
||||||
|
/// The recorded question number.
|
||||||
|
pub number: u32,
|
||||||
|
/// Whether the drop was applied by crediting every option, in which case the
|
||||||
|
/// question is still in the points of record.
|
||||||
|
pub full_credit: bool,
|
||||||
|
}
|
||||||
|
|
||||||
/// One citation to read again after missing a question.
|
/// One citation to read again after missing a question.
|
||||||
#[derive(Debug, Clone, Serialize)]
|
#[derive(Debug, Clone, Serialize)]
|
||||||
#[serde(rename_all = "kebab-case")]
|
#[serde(rename_all = "kebab-case")]
|
||||||
@@ -543,13 +556,16 @@ pub fn student(
|
|||||||
Vec::new()
|
Vec::new()
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut dropped_questions: Vec<u32> = rows
|
let mut dropped_questions: Vec<DroppedQuestion> = rows
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|r| r.dropped)
|
.filter(|r| r.dropped)
|
||||||
.map(|r| r.item_number)
|
.map(|r| DroppedQuestion {
|
||||||
|
number: r.item_number,
|
||||||
|
full_credit: r.dropped_full_credit,
|
||||||
|
})
|
||||||
.collect();
|
.collect();
|
||||||
dropped_questions.sort_unstable();
|
dropped_questions.sort_by_key(|d| d.number);
|
||||||
dropped_questions.dedup();
|
dropped_questions.dedup_by_key(|d| d.number);
|
||||||
|
|
||||||
// Built from the response rows rather than from `questions`, so a report with
|
// Built from the response rows rather than from `questions`, so a report with
|
||||||
// `--no-questions` still says where to go back to; it just does not name the
|
// `--no-questions` still says where to go back to; it just does not name the
|
||||||
@@ -952,11 +968,11 @@ pub struct CohortDiagnostic {
|
|||||||
pub lectures: Vec<CohortLectureRow>,
|
pub lectures: Vec<CohortLectureRow>,
|
||||||
/// Per-question statistics.
|
/// Per-question statistics.
|
||||||
pub questions: Vec<CohortQuestionRow>,
|
pub questions: Vec<CohortQuestionRow>,
|
||||||
/// Questions dropped from scoring, which are therefore absent from every
|
/// Questions thrown out, which are therefore absent from every table above.
|
||||||
/// table above. Recorded so the report says why rather than leaving a gap in
|
/// Recorded so the report says why rather than leaving a gap in the
|
||||||
/// the numbering.
|
/// numbering.
|
||||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||||
pub dropped_questions: Vec<u32>,
|
pub dropped_questions: Vec<DroppedQuestion>,
|
||||||
/// What to do about each question that raised something.
|
/// What to do about each question that raised something.
|
||||||
pub triage: Triage,
|
pub triage: Triage,
|
||||||
/// How the authored expectations did.
|
/// How the authored expectations did.
|
||||||
@@ -1509,12 +1525,16 @@ pub fn cohort(
|
|||||||
.filter_map(|item| questions.iter().find(|q| q.number == item.number).cloned())
|
.filter_map(|item| questions.iter().find(|q| q.number == item.number).cloned())
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let mut dropped_questions: Vec<u32> = set
|
let mut dropped_questions: Vec<DroppedQuestion> = set
|
||||||
.all_items()
|
.all_items()
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter(|n| set.item_dropped(*n))
|
.filter(|n| set.item_dropped(*n))
|
||||||
|
.map(|number| DroppedQuestion {
|
||||||
|
number,
|
||||||
|
full_credit: set.for_item(number).iter().any(|r| r.dropped_full_credit),
|
||||||
|
})
|
||||||
.collect();
|
.collect();
|
||||||
dropped_questions.sort_unstable();
|
dropped_questions.sort_by_key(|d| d.number);
|
||||||
|
|
||||||
let default_options = course.policy.options_per_item;
|
let default_options = course.policy.options_per_item;
|
||||||
let triage = triage(&questions, threshold, default_options);
|
let triage = triage(&questions, threshold, default_options);
|
||||||
|
|||||||
@@ -1112,6 +1112,7 @@ mod tests {
|
|||||||
topics: vec![],
|
topics: vec![],
|
||||||
bonus: false,
|
bonus: false,
|
||||||
dropped: false,
|
dropped: false,
|
||||||
|
dropped_full_credit: false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -468,6 +468,7 @@ pub fn to_record(
|
|||||||
learning_objectives: e.item.learning_objectives.clone(),
|
learning_objectives: e.item.learning_objectives.clone(),
|
||||||
credit_overrides: BTreeMap::new(),
|
credit_overrides: BTreeMap::new(),
|
||||||
dropped: false,
|
dropped: false,
|
||||||
|
dropped_as: None,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -351,6 +351,7 @@ pub fn ingest(
|
|||||||
topics: Vec::new(),
|
topics: Vec::new(),
|
||||||
bonus: false,
|
bonus: false,
|
||||||
dropped: false,
|
dropped: false,
|
||||||
|
dropped_full_credit: false,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -709,6 +709,7 @@ mod tests {
|
|||||||
topics: Vec::new(),
|
topics: Vec::new(),
|
||||||
bonus: false,
|
bonus: false,
|
||||||
dropped: false,
|
dropped: false,
|
||||||
|
dropped_full_credit: false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -659,6 +659,7 @@ pub fn to_responses(questions: &[Question], ctx: &Context) -> Import {
|
|||||||
topics: Vec::new(),
|
topics: Vec::new(),
|
||||||
bonus,
|
bonus,
|
||||||
dropped: false,
|
dropped: false,
|
||||||
|
dropped_full_credit: false,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+48
-5
@@ -108,10 +108,26 @@ pub struct Response {
|
|||||||
pub bonus: bool,
|
pub bonus: bool,
|
||||||
/// Whether the item was dropped after the fact.
|
/// Whether the item was dropped after the fact.
|
||||||
pub dropped: bool,
|
pub dropped: bool,
|
||||||
|
/// Whether that drop was applied by crediting every option, so the item is
|
||||||
|
/// still part of the points of record even though it is out of the
|
||||||
|
/// statistics.
|
||||||
|
///
|
||||||
|
/// Written by [`ResponseSet::enrich`] from the placement's `dropped_as`.
|
||||||
|
#[serde(default)]
|
||||||
|
pub dropped_full_credit: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Response {
|
impl Response {
|
||||||
/// Whether this row should count toward scored totals and item statistics.
|
/// Whether this row counts as evidence.
|
||||||
|
///
|
||||||
|
/// Evidence means item statistics, objective mastery, level rates, and the
|
||||||
|
/// IRT fit. A dropped item is never evidence, however the drop was applied:
|
||||||
|
/// an item everyone was given has no variance to contribute and would only
|
||||||
|
/// flatter the objective it was written against.
|
||||||
|
///
|
||||||
|
/// This is deliberately not the same question as [`Response::scores`]. The
|
||||||
|
/// two were one predicate until dropping a question stopped always meaning
|
||||||
|
/// removing it.
|
||||||
///
|
///
|
||||||
/// # Returns
|
/// # Returns
|
||||||
///
|
///
|
||||||
@@ -120,6 +136,21 @@ impl Response {
|
|||||||
!self.bonus && !self.dropped
|
!self.bonus && !self.dropped
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Whether this row counts toward the points of record.
|
||||||
|
///
|
||||||
|
/// A question dropped by crediting every option still sits in the student's
|
||||||
|
/// total on the platform, and a report that disagreed with the platform
|
||||||
|
/// about a student's percentage would be worse than no report. So a
|
||||||
|
/// full-credit drop stays in both the numerator and the denominator here,
|
||||||
|
/// while a removed drop leaves both.
|
||||||
|
///
|
||||||
|
/// # Returns
|
||||||
|
///
|
||||||
|
/// `true` when the row belongs in the score.
|
||||||
|
pub fn scores(&self) -> bool {
|
||||||
|
!self.bonus && (!self.dropped || self.dropped_full_credit)
|
||||||
|
}
|
||||||
|
|
||||||
/// The response coded for a dichotomous model.
|
/// The response coded for a dichotomous model.
|
||||||
///
|
///
|
||||||
/// Partial credit is rounded toward the majority: a half-credit response is
|
/// Partial credit is rounded toward the majority: a half-credit response is
|
||||||
@@ -188,7 +219,11 @@ impl ResponseSet {
|
|||||||
set.into_iter().map(|s| s.to_string()).collect()
|
set.into_iter().map(|s| s.to_string()).collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The distinct item numbers that count toward the scored total, sorted.
|
/// The distinct item numbers that count as evidence, sorted.
|
||||||
|
///
|
||||||
|
/// Named for the scored total it once described; it is the analysis matrix's
|
||||||
|
/// item list, so it uses [`Response::counts`] and excludes every dropped
|
||||||
|
/// item. [`ResponseSet::points_available`] is the scoring denominator.
|
||||||
pub fn scored_items(&self) -> Vec<u32> {
|
pub fn scored_items(&self) -> Vec<u32> {
|
||||||
let set: BTreeSet<u32> = self
|
let set: BTreeSet<u32> = self
|
||||||
.rows
|
.rows
|
||||||
@@ -247,11 +282,12 @@ impl ResponseSet {
|
|||||||
///
|
///
|
||||||
/// # Returns
|
/// # Returns
|
||||||
///
|
///
|
||||||
/// The sum of `score` over scored, undropped items.
|
/// The sum of `score` over the items that count toward the score, which
|
||||||
|
/// includes a question dropped by crediting every option.
|
||||||
pub fn scored_total(&self, key: &str) -> f64 {
|
pub fn scored_total(&self, key: &str) -> f64 {
|
||||||
self.rows
|
self.rows
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|r| r.student_key == key && r.counts())
|
.filter(|r| r.student_key == key && r.scores())
|
||||||
.map(|r| r.score)
|
.map(|r| r.score)
|
||||||
.sum()
|
.sum()
|
||||||
}
|
}
|
||||||
@@ -269,7 +305,7 @@ impl ResponseSet {
|
|||||||
/// for each item so a student who skipped an item still has a denominator.
|
/// for each item so a student who skipped an item still has a denominator.
|
||||||
pub fn points_available(&self) -> f64 {
|
pub fn points_available(&self) -> f64 {
|
||||||
let mut per_item: BTreeMap<u32, f64> = BTreeMap::new();
|
let mut per_item: BTreeMap<u32, f64> = BTreeMap::new();
|
||||||
for r in self.rows.iter().filter(|r| r.counts()) {
|
for r in self.rows.iter().filter(|r| r.scores()) {
|
||||||
let e = per_item.entry(r.item_number).or_insert(0.0);
|
let e = per_item.entry(r.item_number).or_insert(0.0);
|
||||||
if r.points_possible > *e {
|
if r.points_possible > *e {
|
||||||
*e = r.points_possible;
|
*e = r.points_possible;
|
||||||
@@ -382,6 +418,7 @@ impl ResponseSet {
|
|||||||
r.item_version = p.version;
|
r.item_version = p.version;
|
||||||
r.bonus = r.bonus || p.bonus;
|
r.bonus = r.bonus || p.bonus;
|
||||||
r.dropped = r.dropped || p.dropped;
|
r.dropped = r.dropped || p.dropped;
|
||||||
|
r.dropped_full_credit = r.dropped_full_credit || p.dropped_with_credit();
|
||||||
if let Some(points) = p.points {
|
if let Some(points) = p.points {
|
||||||
// The record is authoritative for points as administered; the
|
// The record is authoritative for points as administered; the
|
||||||
// export sometimes carries a stale maximum.
|
// export sometimes carries a stale maximum.
|
||||||
@@ -636,6 +673,9 @@ pub struct FlatResponse {
|
|||||||
/// Comma-joined eliminated letters in the bank's lettering.
|
/// Comma-joined eliminated letters in the bank's lettering.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub eliminated_source: String,
|
pub eliminated_source: String,
|
||||||
|
/// Whether a dropped item was dropped by crediting every option.
|
||||||
|
#[serde(default)]
|
||||||
|
pub dropped_full_credit: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl FlatResponse {
|
impl FlatResponse {
|
||||||
@@ -656,6 +696,7 @@ impl FlatResponse {
|
|||||||
assessment_id: r.assessment_id.clone(),
|
assessment_id: r.assessment_id.clone(),
|
||||||
date: r.date.map(|d| d.to_string()).unwrap_or_default(),
|
date: r.date.map(|d| d.to_string()).unwrap_or_default(),
|
||||||
form: r.form.clone().unwrap_or_default(),
|
form: r.form.clone().unwrap_or_default(),
|
||||||
|
dropped_full_credit: r.dropped_full_credit,
|
||||||
form_position: r.form_position.unwrap_or(0),
|
form_position: r.form_position.unwrap_or(0),
|
||||||
student_key: r.student_key.clone(),
|
student_key: r.student_key.clone(),
|
||||||
sid: r.sid.clone().unwrap_or_default(),
|
sid: r.sid.clone().unwrap_or_default(),
|
||||||
@@ -709,6 +750,7 @@ impl FlatResponse {
|
|||||||
assessment_id: self.assessment_id.clone(),
|
assessment_id: self.assessment_id.clone(),
|
||||||
date: self.date.parse().ok(),
|
date: self.date.parse().ok(),
|
||||||
form: none_if_empty(&self.form),
|
form: none_if_empty(&self.form),
|
||||||
|
dropped_full_credit: self.dropped_full_credit,
|
||||||
form_position: if self.form_position == 0 {
|
form_position: if self.form_position == 0 {
|
||||||
None
|
None
|
||||||
} else {
|
} else {
|
||||||
@@ -792,6 +834,7 @@ mod tests {
|
|||||||
topics: vec![],
|
topics: vec![],
|
||||||
bonus: false,
|
bonus: false,
|
||||||
dropped: false,
|
dropped: false,
|
||||||
|
dropped_full_credit: false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -564,6 +564,7 @@ mod tests {
|
|||||||
topics: vec![],
|
topics: vec![],
|
||||||
bonus: false,
|
bonus: false,
|
||||||
dropped: false,
|
dropped: false,
|
||||||
|
dropped_full_credit: false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -62,6 +62,7 @@ pub fn schema() -> Schema {
|
|||||||
Field::new("bonus", DataType::Boolean, false),
|
Field::new("bonus", DataType::Boolean, false),
|
||||||
Field::new("dropped", DataType::Boolean, false),
|
Field::new("dropped", DataType::Boolean, false),
|
||||||
Field::new("form_position", DataType::UInt32, false),
|
Field::new("form_position", DataType::UInt32, false),
|
||||||
|
Field::new("dropped_full_credit", DataType::Boolean, false),
|
||||||
Field::new("selected_source", DataType::Utf8, false),
|
Field::new("selected_source", DataType::Utf8, false),
|
||||||
Field::new("eliminated_source", DataType::Utf8, false),
|
Field::new("eliminated_source", DataType::Utf8, false),
|
||||||
])
|
])
|
||||||
@@ -124,6 +125,7 @@ fn to_batch(rows: &[FlatResponse]) -> Result<RecordBatch> {
|
|||||||
boolc(|r| r.bonus),
|
boolc(|r| r.bonus),
|
||||||
boolc(|r| r.dropped),
|
boolc(|r| r.dropped),
|
||||||
u32c(|r| r.form_position),
|
u32c(|r| r.form_position),
|
||||||
|
boolc(|r| r.dropped_full_credit),
|
||||||
s(|r| &r.selected_source),
|
s(|r| &r.selected_source),
|
||||||
s(|r| &r.eliminated_source),
|
s(|r| &r.eliminated_source),
|
||||||
];
|
];
|
||||||
@@ -256,6 +258,11 @@ fn from_batch(batch: &RecordBatch, path: &Path) -> Result<Vec<FlatResponse>> {
|
|||||||
.column_by_name(name)
|
.column_by_name(name)
|
||||||
.and_then(|c| c.as_any().downcast_ref::<UInt32Array>())
|
.and_then(|c| c.as_any().downcast_ref::<UInt32Array>())
|
||||||
};
|
};
|
||||||
|
let optional_bools = |name: &str| -> Option<&BooleanArray> {
|
||||||
|
batch
|
||||||
|
.column_by_name(name)
|
||||||
|
.and_then(|c| c.as_any().downcast_ref::<BooleanArray>())
|
||||||
|
};
|
||||||
|
|
||||||
let administration_id = strings("administration_id")?;
|
let administration_id = strings("administration_id")?;
|
||||||
let course = strings("course")?;
|
let course = strings("course")?;
|
||||||
@@ -284,6 +291,7 @@ fn from_batch(batch: &RecordBatch, path: &Path) -> Result<Vec<FlatResponse>> {
|
|||||||
let dropped = bools("dropped")?;
|
let dropped = bools("dropped")?;
|
||||||
|
|
||||||
let form_position = optional_uints("form_position");
|
let form_position = optional_uints("form_position");
|
||||||
|
let dropped_full_credit = optional_bools("dropped_full_credit");
|
||||||
let selected_source = optional_strings("selected_source");
|
let selected_source = optional_strings("selected_source");
|
||||||
let eliminated_source = optional_strings("eliminated_source");
|
let eliminated_source = optional_strings("eliminated_source");
|
||||||
|
|
||||||
@@ -322,6 +330,7 @@ fn from_batch(batch: &RecordBatch, path: &Path) -> Result<Vec<FlatResponse>> {
|
|||||||
topics: topics.value(i).to_string(),
|
topics: topics.value(i).to_string(),
|
||||||
bonus: bonus.value(i),
|
bonus: bonus.value(i),
|
||||||
dropped: dropped.value(i),
|
dropped: dropped.value(i),
|
||||||
|
dropped_full_credit: dropped_full_credit.map(|a| a.value(i)).unwrap_or(false),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
Ok(out)
|
Ok(out)
|
||||||
@@ -367,6 +376,7 @@ mod tests {
|
|||||||
topics: "kinetics".into(),
|
topics: "kinetics".into(),
|
||||||
bonus: false,
|
bonus: false,
|
||||||
dropped: false,
|
dropped: false,
|
||||||
|
dropped_full_credit: false,
|
||||||
form_position: 0,
|
form_position: 0,
|
||||||
selected_source: String::new(),
|
selected_source: String::new(),
|
||||||
eliminated_source: String::new(),
|
eliminated_source: String::new(),
|
||||||
|
|||||||
@@ -602,6 +602,7 @@ items:
|
|||||||
learning_objectives: Vec::new(),
|
learning_objectives: Vec::new(),
|
||||||
credit_overrides: Default::default(),
|
credit_overrides: Default::default(),
|
||||||
dropped: false,
|
dropped: false,
|
||||||
|
dropped_as: None,
|
||||||
},
|
},
|
||||||
Placement {
|
Placement {
|
||||||
number: 2,
|
number: 2,
|
||||||
@@ -615,6 +616,7 @@ items:
|
|||||||
learning_objectives: Vec::new(),
|
learning_objectives: Vec::new(),
|
||||||
credit_overrides: Default::default(),
|
credit_overrides: Default::default(),
|
||||||
dropped: false,
|
dropped: false,
|
||||||
|
dropped_as: None,
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1377,6 +1377,7 @@ items:
|
|||||||
learning_objectives: Vec::new(),
|
learning_objectives: Vec::new(),
|
||||||
credit_overrides: Default::default(),
|
credit_overrides: Default::default(),
|
||||||
dropped: false,
|
dropped: false,
|
||||||
|
dropped_as: None,
|
||||||
},
|
},
|
||||||
Placement {
|
Placement {
|
||||||
number: 2,
|
number: 2,
|
||||||
@@ -1390,6 +1391,7 @@ items:
|
|||||||
learning_objectives: Vec::new(),
|
learning_objectives: Vec::new(),
|
||||||
credit_overrides: Default::default(),
|
credit_overrides: Default::default(),
|
||||||
dropped: false,
|
dropped: false,
|
||||||
|
dropped_as: None,
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -866,6 +866,7 @@ items:
|
|||||||
learning_objectives: Vec::new(),
|
learning_objectives: Vec::new(),
|
||||||
credit_overrides: Default::default(),
|
credit_overrides: Default::default(),
|
||||||
dropped: false,
|
dropped: false,
|
||||||
|
dropped_as: None,
|
||||||
},
|
},
|
||||||
Placement {
|
Placement {
|
||||||
number: 2,
|
number: 2,
|
||||||
@@ -879,6 +880,7 @@ items:
|
|||||||
learning_objectives: Vec::new(),
|
learning_objectives: Vec::new(),
|
||||||
credit_overrides: Default::default(),
|
credit_overrides: Default::default(),
|
||||||
dropped: false,
|
dropped: false,
|
||||||
|
dropped_as: None,
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -427,6 +427,7 @@ mod tests {
|
|||||||
learning_objectives: Vec::new(),
|
learning_objectives: Vec::new(),
|
||||||
credit_overrides: Default::default(),
|
credit_overrides: Default::default(),
|
||||||
dropped: true,
|
dropped: true,
|
||||||
|
dropped_as: None,
|
||||||
},
|
},
|
||||||
Placement {
|
Placement {
|
||||||
number: 2,
|
number: 2,
|
||||||
@@ -440,6 +441,7 @@ mod tests {
|
|||||||
learning_objectives: Vec::new(),
|
learning_objectives: Vec::new(),
|
||||||
credit_overrides: Default::default(),
|
credit_overrides: Default::default(),
|
||||||
dropped: false,
|
dropped: false,
|
||||||
|
dropped_as: None,
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -355,7 +355,12 @@ pub fn student_value(diagnostic: &StudentDiagnostic, config: &RenderConfig) -> V
|
|||||||
diagnostic
|
diagnostic
|
||||||
.dropped_questions
|
.dropped_questions
|
||||||
.iter()
|
.iter()
|
||||||
.map(|n| Value::Int(*n as i64))
|
.map(|dropped| {
|
||||||
|
let mut value = Value::dict();
|
||||||
|
value.insert("number", Value::Int(dropped.number as i64));
|
||||||
|
value.insert("full-credit", Value::Bool(dropped.full_credit));
|
||||||
|
value
|
||||||
|
})
|
||||||
.collect(),
|
.collect(),
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
@@ -630,7 +635,12 @@ pub fn cohort_value(diagnostic: &CohortDiagnostic, config: &RenderConfig) -> Val
|
|||||||
diagnostic
|
diagnostic
|
||||||
.dropped_questions
|
.dropped_questions
|
||||||
.iter()
|
.iter()
|
||||||
.map(|n| Value::Int(*n as i64))
|
.map(|dropped| {
|
||||||
|
let mut value = Value::dict();
|
||||||
|
value.insert("number", Value::Int(dropped.number as i64));
|
||||||
|
value.insert("full-credit", Value::Bool(dropped.full_credit));
|
||||||
|
value
|
||||||
|
})
|
||||||
.collect(),
|
.collect(),
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -994,11 +994,21 @@
|
|||||||
let dropped = cb-data.at("dropped-questions", default: ())
|
let dropped = cb-data.at("dropped-questions", default: ())
|
||||||
if dropped.len() > 0 [
|
if dropped.len() > 0 [
|
||||||
#plural(dropped.len(), "Question", "Questions")
|
#plural(dropped.len(), "Question", "Questions")
|
||||||
#dropped.map(str).join(", ")
|
#dropped.map(d => str(d.number)).join(", ")
|
||||||
#plural(dropped.len(), "is", "are")
|
#plural(dropped.len(), "is", "are")
|
||||||
marked dropped in the assessment record, so
|
marked dropped in the assessment record, so
|
||||||
#plural(dropped.len(), "it is", "they are")
|
#plural(dropped.len(), "it is", "they are")
|
||||||
absent from this table and from every statistic above.
|
absent from this table and from every statistic above.
|
||||||
|
#{
|
||||||
|
let credited = dropped.filter(d => d.at("full-credit", default: false))
|
||||||
|
if credited.len() > 0 [
|
||||||
|
#plural(credited.len(), "Question", "Questions")
|
||||||
|
#credited.map(d => str(d.number)).join(", ")
|
||||||
|
#plural(credited.len(), "was", "were") credited to every student, so
|
||||||
|
#plural(credited.len(), "it remains", "they remain") in the score
|
||||||
|
denominator and the means above match the grade of record.
|
||||||
|
]
|
||||||
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
@@ -1173,69 +1183,71 @@
|
|||||||
#let predictions = cb-data.at("predictions", default: (:))
|
#let predictions = cb-data.at("predictions", default: (:))
|
||||||
|
|
||||||
#if show-predictions and predictions.at("predicted", default: 0) > 0 [
|
#if show-predictions and predictions.at("predicted", default: 0) > 0 [
|
||||||
= How your predictions did
|
#block(breakable: false)[
|
||||||
|
= How your predictions did
|
||||||
|
|
||||||
#let n = predictions.at("predicted", default: 0)
|
#let n = predictions.at("predicted", default: 0)
|
||||||
#let calibrated = predictions.at("calibrated", default: 0)
|
#let calibrated = predictions.at("calibrated", default: 0)
|
||||||
#let signed-error = predictions.at("mean-signed-error", default: none)
|
#let signed-error = predictions.at("mean-signed-error", default: none)
|
||||||
#let abs-error = predictions.at("mean-abs-error", default: none)
|
#let abs-error = predictions.at("mean-abs-error", default: none)
|
||||||
|
|
||||||
#explain[
|
#explain[
|
||||||
#if calibrated == 0 [
|
#if calibrated == 0 [
|
||||||
None of these #n #plural(n, "expectation", "expectations") rests on prior
|
None of these #n #plural(n, "expectation", "expectations") rests on prior
|
||||||
data, so they are predictions rather than calibrations. A prediction that
|
data, so they are predictions rather than calibrations. A prediction that
|
||||||
misses is a fact about the prediction: it does not flag the item, and it is
|
misses is a fact about the prediction: it does not flag the item, and it is
|
||||||
summarised here instead of appearing #n times in the tables above. Once
|
summarised here instead of appearing #n times in the tables above. Once
|
||||||
`coursebank calibrate` has written statistics back into the bank, a
|
`coursebank calibrate` has written statistics back into the bank, a
|
||||||
subsequent miss means the cohort or the teaching moved, and it will be
|
subsequent miss means the cohort or the teaching moved, and it will be
|
||||||
flagged.
|
flagged.
|
||||||
] else [
|
] else [
|
||||||
#calibrated of #n #plural(n, "expectation", "expectations") rests on a
|
#calibrated of #n #plural(n, "expectation", "expectations") rests on a
|
||||||
prior calibration. Those are the ones whose misses are flagged on the item,
|
prior calibration. Those are the ones whose misses are flagged on the item,
|
||||||
because a calibrated item that moves is telling you about this cohort. The
|
because a calibrated item that moves is telling you about this cohort. The
|
||||||
rest are predictions, and a miss corrects the prediction.
|
rest are predictions, and a miss corrects the prediction.
|
||||||
|
]
|
||||||
]
|
]
|
||||||
]
|
|
||||||
|
|
||||||
#grid(
|
#grid(
|
||||||
columns: (1fr, 1fr, 1fr),
|
columns: (1fr, 1fr, 1fr),
|
||||||
gutter: 9pt,
|
gutter: 9pt,
|
||||||
stat-card(
|
stat-card(
|
||||||
"difficulty bias",
|
"difficulty bias",
|
||||||
if signed-error != none {
|
if signed-error != none {
|
||||||
(if signed-error >= 0 { "+" } else { "" }) + str(calc.round(signed-error * 100)) + " pts"
|
(if signed-error >= 0 { "+" } else { "" }) + str(calc.round(signed-error * 100)) + " pts"
|
||||||
} else { "n/a" },
|
} else { "n/a" },
|
||||||
note: if signed-error != none and signed-error > 0 {
|
note: if signed-error != none and signed-error > 0 {
|
||||||
"items came out easier than you expected"
|
"items came out easier than you expected"
|
||||||
} else if signed-error != none {
|
} else if signed-error != none {
|
||||||
"items came out harder than you expected"
|
"items came out harder than you expected"
|
||||||
} else { none },
|
} else { none },
|
||||||
),
|
),
|
||||||
stat-card(
|
stat-card(
|
||||||
"typical miss",
|
"typical miss",
|
||||||
if abs-error != none { str(calc.round(abs-error * 100)) + " pts" } else { "n/a" },
|
if abs-error != none { str(calc.round(abs-error * 100)) + " pts" } else { "n/a" },
|
||||||
note: str(predictions.at("within", default: 0)) + " of " + str(n) + " inside tolerance",
|
note: str(predictions.at("within", default: 0)) + " of " + str(n) + " inside tolerance",
|
||||||
),
|
),
|
||||||
stat-card(
|
stat-card(
|
||||||
"discrimination band",
|
"discrimination band",
|
||||||
str(predictions.at("band-hit", default: 0)) + " / " + str(predictions.at("band", default: 0)),
|
str(predictions.at("band-hit", default: 0)) + " / " + str(predictions.at("band", default: 0)),
|
||||||
note: "landed in the band you expected",
|
note: "landed in the band you expected",
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
#{
|
#{
|
||||||
let surprise = predictions.at("biggest-surprise", default: none)
|
let surprise = predictions.at("biggest-surprise", default: none)
|
||||||
if surprise != none {
|
if surprise != none {
|
||||||
block(above: entry-gap)[
|
block(above: entry-gap)[
|
||||||
#pad(right: prose-pad)[
|
#pad(right: prose-pad)[
|
||||||
#text(size: size-lead, fill: luma(95))[
|
#text(size: size-lead, fill: luma(95))[
|
||||||
The largest single gap was question #surprise.number, predicted at
|
The largest single gap was question #surprise.number, predicted at
|
||||||
#pct(surprise.expected) and observed at #pct(surprise.observed).
|
#pct(surprise.expected) and observed at #pct(surprise.observed).
|
||||||
|
]
|
||||||
]
|
]
|
||||||
]
|
]
|
||||||
]
|
}
|
||||||
}
|
}
|
||||||
}
|
]
|
||||||
]
|
]
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|||||||
@@ -143,6 +143,7 @@
|
|||||||
review: ((citation: "KKW §6.2", title: "Molecules and Medicine", url: "https://example.edu/6/2"),),
|
review: ((citation: "KKW §6.2", title: "Molecules and Medicine", url: "https://example.edu/6/2"),),
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
|
dropped-questions: ((number: 35, full-credit: true),),
|
||||||
review-lectures: (
|
review-lectures: (
|
||||||
(
|
(
|
||||||
lecture: "L1.1",
|
lecture: "L1.1",
|
||||||
@@ -458,15 +459,30 @@
|
|||||||
#block(above: entry-gap)[
|
#block(above: entry-gap)[
|
||||||
#pad(right: prose-pad)[
|
#pad(right: prose-pad)[
|
||||||
#text(size: size-lead)[
|
#text(size: size-lead)[
|
||||||
#plural(dropped-questions.len(), "Question", "Questions")
|
#{
|
||||||
#dropped-questions.map(str).join(", ")
|
// Two kinds of drop, and they need different sentences. A credited
|
||||||
#plural(dropped-questions.len(), "was", "were")
|
// question is still in the denominator, so telling a student it was
|
||||||
dropped after the exam and #plural(dropped-questions.len(), "is", "are")
|
// removed would not match the arithmetic they can do themselves.
|
||||||
not part of anyone's score. Your percentage above is out of the
|
let credited = dropped-questions.filter(d => d.at("full-credit", default: false))
|
||||||
#plural(dropped-questions.len(), "remaining question", "remaining questions").
|
let removed = dropped-questions.filter(d => not d.at("full-credit", default: false))
|
||||||
Nothing you wrote on
|
let numbers = list => list.map(d => str(d.number)).join(", ")
|
||||||
#plural(dropped-questions.len(), "it", "them")
|
|
||||||
counted for or against you.
|
if credited.len() > 0 [
|
||||||
|
#plural(credited.len(), "Question", "Questions") #numbers(credited)
|
||||||
|
#plural(credited.len(), "was", "were") thrown out after the exam.
|
||||||
|
Everyone received full credit for
|
||||||
|
#plural(credited.len(), "it", "them"), so
|
||||||
|
#plural(credited.len(), "it is", "they are") still counted in the
|
||||||
|
score above and whatever you chose made no difference.
|
||||||
|
]
|
||||||
|
if removed.len() > 0 [
|
||||||
|
#plural(removed.len(), "Question", "Questions") #numbers(removed)
|
||||||
|
#plural(removed.len(), "was", "were") thrown out and removed from
|
||||||
|
scoring, so your percentage is out of the remaining questions.
|
||||||
|
Nothing you wrote on #plural(removed.len(), "it", "them") counted
|
||||||
|
either way.
|
||||||
|
]
|
||||||
|
}
|
||||||
]
|
]
|
||||||
]
|
]
|
||||||
]
|
]
|
||||||
@@ -822,7 +838,8 @@
|
|||||||
#box(width: 0.7em, height: 0.7em, fill: ok-color.lighten(70%), radius: 2pt) right ·
|
#box(width: 0.7em, height: 0.7em, fill: ok-color.lighten(70%), radius: 2pt) right ·
|
||||||
#box(width: 0.7em, height: 0.7em, fill: mid-color.lighten(70%), radius: 2pt) part marks ·
|
#box(width: 0.7em, height: 0.7em, fill: mid-color.lighten(70%), radius: 2pt) part marks ·
|
||||||
#box(width: 0.7em, height: 0.7em, fill: bad-color.lighten(70%), radius: 2pt) not right ·
|
#box(width: 0.7em, height: 0.7em, fill: bad-color.lighten(70%), radius: 2pt) not right ·
|
||||||
#box(width: 0.7em, height: 0.7em, fill: luma(210), radius: 2pt) left blank
|
#box(width: 0.7em, height: 0.7em, fill: luma(210), radius: 2pt) left blank ·
|
||||||
|
#box(width: 0.7em, height: 0.7em, fill: luma(150), radius: 2pt) dropped, not scored
|
||||||
]
|
]
|
||||||
]
|
]
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -281,6 +281,48 @@ pub struct Placement {
|
|||||||
/// Set when an item was dropped from scoring after administration.
|
/// Set when an item was dropped from scoring after administration.
|
||||||
#[serde(default, skip_serializing_if = "is_false")]
|
#[serde(default, skip_serializing_if = "is_false")]
|
||||||
pub dropped: bool,
|
pub dropped: bool,
|
||||||
|
/// How the drop was applied on the grading platform.
|
||||||
|
///
|
||||||
|
/// Two ways to throw a question out, and they produce different
|
||||||
|
/// percentages. [`DropStyle::Removed`] takes the item out of the numerator
|
||||||
|
/// and the denominator: a student with 27 of 35 scores 77.1%.
|
||||||
|
/// [`DropStyle::FullCredit`] is what you do when the grade of record lives
|
||||||
|
/// somewhere else and the question cannot be removed from it: every option
|
||||||
|
/// is keyed, everyone earns the point, and the same student scores 28 of 36,
|
||||||
|
/// or 77.8%.
|
||||||
|
///
|
||||||
|
/// Defaults to [`DropStyle::Removed`], which is what `dropped: true` meant
|
||||||
|
/// before this field existed. Set it to `full_credit` when you have credited
|
||||||
|
/// every option in the platform, so that the report agrees with the grade the
|
||||||
|
/// student can see.
|
||||||
|
///
|
||||||
|
/// Either way the item is out of the item statistics, the objective
|
||||||
|
/// evidence, and the IRT fit: an item everyone got right has no variance to
|
||||||
|
/// contribute.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub dropped_as: Option<DropStyle>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// How a dropped item was handled on the grading platform.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum DropStyle {
|
||||||
|
/// Taken out of the numerator and the denominator.
|
||||||
|
Removed,
|
||||||
|
/// Every option credited, so the item stays in both.
|
||||||
|
FullCredit,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Placement {
|
||||||
|
/// Whether this placement was dropped by crediting every option.
|
||||||
|
///
|
||||||
|
/// # Returns
|
||||||
|
///
|
||||||
|
/// `true` only when the item is dropped *and* the drop was applied as full
|
||||||
|
/// credit, so the item still belongs in the points of record.
|
||||||
|
pub fn dropped_with_credit(&self) -> bool {
|
||||||
|
self.dropped && self.dropped_as == Some(DropStyle::FullCredit)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl AssessmentFile {
|
impl AssessmentFile {
|
||||||
@@ -451,6 +493,20 @@ impl AssessmentFile {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if p.dropped_as.is_some() && !p.dropped {
|
||||||
|
issues.push(format!(
|
||||||
|
"question {}: `dropped_as` is set but `dropped` is not, so nothing is dropped",
|
||||||
|
p.number
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if p.dropped && !p.credit_overrides.is_empty() {
|
||||||
|
issues.push(format!(
|
||||||
|
"question {}: dropped and carrying credit overrides. Pick one: an override \
|
||||||
|
rescores an option, a drop removes the question",
|
||||||
|
p.number
|
||||||
|
));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut form_ids: Vec<&str> = Vec::new();
|
let mut form_ids: Vec<&str> = Vec::new();
|
||||||
|
|||||||
Reference in New Issue
Block a user