feat: cooked up something fierce
This commit is contained in:
26 files changed
+1485
-44
No files matched your search
@@ -337,6 +337,7 @@ pub fn ingest(
|
||||
item_number: number,
|
||||
item_ref,
|
||||
item_version: None,
|
||||
variant: None,
|
||||
selected,
|
||||
selected_source: Vec::new(),
|
||||
eliminated: Vec::new(),
|
||||
|
||||
+4
-3
@@ -248,7 +248,8 @@ impl FormDecoder {
|
||||
for (index, placement) in printed.iter().enumerate() {
|
||||
let entry = catalog.require(&placement.item)?;
|
||||
let item = &entry.item;
|
||||
let order = select::option_order(form, &placement.item, item.options.len());
|
||||
let shown = item.administered(&placement.key, &placement.distractors);
|
||||
let order = select::option_order(form, &placement.item, shown.len());
|
||||
|
||||
let canonical_key: BTreeSet<String> = if placement.key.is_empty() {
|
||||
item.key_letters().into_iter().collect()
|
||||
@@ -260,8 +261,7 @@ impl FormDecoder {
|
||||
let mut to_printed = BTreeMap::new();
|
||||
let mut printed_key = Vec::new();
|
||||
for (position, source_index) in order.iter().enumerate() {
|
||||
let canonical = item
|
||||
.options
|
||||
let canonical = shown
|
||||
.get(*source_index)
|
||||
.map(|c| c.id.clone())
|
||||
.unwrap_or_else(|| printed_letter(*source_index));
|
||||
@@ -738,6 +738,7 @@ mod tests {
|
||||
form_position: None,
|
||||
item_ref: None,
|
||||
item_version: None,
|
||||
variant: None,
|
||||
selected: vec![selected.into()],
|
||||
eliminated: Vec::new(),
|
||||
selected_source: Vec::new(),
|
||||
|
||||
@@ -645,6 +645,7 @@ pub fn to_responses(questions: &[Question], ctx: &Context) -> Import {
|
||||
item_number: q.number,
|
||||
item_ref: None,
|
||||
item_version: None,
|
||||
variant: None,
|
||||
selected,
|
||||
selected_source: Vec::new(),
|
||||
eliminated,
|
||||
|
||||
@@ -75,6 +75,16 @@ pub struct Response {
|
||||
/// The item version as administered.
|
||||
pub item_version: Option<u32>,
|
||||
|
||||
/// The variant administered: which option set this row's student saw.
|
||||
///
|
||||
/// The grouping key for pooled statistics. Set at ingest from the
|
||||
/// assessment record, and stored so that anything reading the Parquet
|
||||
/// without this tool can group the same way — a store that only has
|
||||
/// `item_ref` cannot tell two option sets of one stem apart, and will
|
||||
/// average them.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub variant: Option<String>,
|
||||
|
||||
/// Option letters the student chose.
|
||||
pub selected: Vec<String>,
|
||||
/// The selected options in the bank's own lettering, written at ingest by
|
||||
@@ -314,6 +324,49 @@ impl ResponseSet {
|
||||
per_item.values().sum()
|
||||
}
|
||||
|
||||
/// How many rows each option set of each item has.
|
||||
///
|
||||
/// What a calibration report needs to be honest about sample size: an item
|
||||
/// administered three times with three different option sets has three
|
||||
/// cells, not one, and reporting "n = 72" of it would be wrong three ways.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Row counts keyed by item id and variant, with an empty variant for rows
|
||||
/// that carry none.
|
||||
pub fn variants(&self) -> BTreeMap<(String, String), usize> {
|
||||
let mut out: BTreeMap<(String, String), usize> = BTreeMap::new();
|
||||
for row in &self.rows {
|
||||
let Some(item) = &row.item_ref else { continue };
|
||||
let key = (item.clone(), row.variant.clone().unwrap_or_default());
|
||||
*out.entry(key).or_insert(0) += 1;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The rows for one option set of one item.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `item_ref` - the item id.
|
||||
/// * `variant` - the variant digest, or `None` for rows carrying none.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// A set holding only those rows, keeping the warnings of the original.
|
||||
pub fn for_variant(&self, item_ref: &str, variant: Option<&str>) -> ResponseSet {
|
||||
ResponseSet {
|
||||
rows: self
|
||||
.rows
|
||||
.iter()
|
||||
.filter(|r| r.item_ref.as_deref() == Some(item_ref))
|
||||
.filter(|r| r.variant.as_deref() == variant)
|
||||
.cloned()
|
||||
.collect(),
|
||||
warnings: self.warnings.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds the response matrix for psychometrics.
|
||||
///
|
||||
/// # Arguments
|
||||
@@ -416,6 +469,9 @@ impl ResponseSet {
|
||||
};
|
||||
r.item_ref = Some(p.item.clone());
|
||||
r.item_version = p.version;
|
||||
// Recorded when the record says so; derived from the option set
|
||||
// otherwise, which is the case for every administration before 2.0.
|
||||
r.variant = p.variant.clone();
|
||||
r.bonus = r.bonus || p.bonus;
|
||||
r.dropped = r.dropped || p.dropped;
|
||||
r.dropped_full_credit = r.dropped_full_credit || p.dropped_with_credit();
|
||||
@@ -434,6 +490,9 @@ impl ResponseSet {
|
||||
|
||||
if let Some(cat) = catalog {
|
||||
if let Some(entry) = cat.get(&p.item) {
|
||||
if r.variant.is_none() {
|
||||
r.variant = Some(p.variant_of(&entry.item));
|
||||
}
|
||||
r.level = Some(entry.item.level);
|
||||
r.learning_targets = if p.learning_targets.is_empty() {
|
||||
entry.item.learning_targets.clone()
|
||||
@@ -636,6 +695,8 @@ pub struct FlatResponse {
|
||||
pub item_ref: String,
|
||||
/// The item version, 0 when unknown.
|
||||
pub item_version: u32,
|
||||
/// The administered variant, empty when unknown.
|
||||
pub variant: String,
|
||||
/// Comma-joined selected letters.
|
||||
pub selected: String,
|
||||
/// Comma-joined eliminated letters.
|
||||
@@ -710,6 +771,7 @@ impl FlatResponse {
|
||||
item_number: r.item_number,
|
||||
item_ref: r.item_ref.clone().unwrap_or_default(),
|
||||
item_version: r.item_version.unwrap_or(0),
|
||||
variant: r.variant.clone().unwrap_or_default(),
|
||||
selected: r.selected.join(","),
|
||||
selected_source: r.selected_source.join(","),
|
||||
eliminated: r.eliminated.join(","),
|
||||
@@ -771,6 +833,7 @@ impl FlatResponse {
|
||||
// one ingested after it instead of splitting into two items.
|
||||
item_ref: none_if_empty(&self.item_ref)
|
||||
.map(|id| crate::item::canonical_id(&id).to_string()),
|
||||
variant: none_if_empty(&self.variant),
|
||||
item_version: if self.item_version == 0 {
|
||||
None
|
||||
} else {
|
||||
@@ -828,6 +891,7 @@ mod tests {
|
||||
item_number: number,
|
||||
item_ref: None,
|
||||
item_version: None,
|
||||
variant: None,
|
||||
selected: vec!["A".into()],
|
||||
selected_source: vec![],
|
||||
eliminated: vec![],
|
||||
@@ -846,6 +910,31 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rows_group_by_the_option_set_they_administered() {
|
||||
let mut set = ResponseSet::new();
|
||||
for (student, variant) in [("s1", "v1"), ("s2", "v1"), ("s3", "v2")] {
|
||||
let mut r = row(student, 1, 1.0);
|
||||
r.item_ref = Some("q-x".into());
|
||||
r.variant = Some(variant.into());
|
||||
set.rows.push(r);
|
||||
}
|
||||
// A row from before the column existed.
|
||||
let mut old = row("s4", 1, 1.0);
|
||||
old.item_ref = Some("q-x".into());
|
||||
set.rows.push(old);
|
||||
|
||||
let counts = set.variants();
|
||||
assert_eq!(counts[&("q-x".to_string(), "v1".to_string())], 2);
|
||||
assert_eq!(counts[&("q-x".to_string(), "v2".to_string())], 1);
|
||||
// Unknown groups on its own rather than joining either set.
|
||||
assert_eq!(counts[&("q-x".to_string(), String::new())], 1);
|
||||
|
||||
assert_eq!(set.for_variant("q-x", Some("v1")).rows.len(), 2);
|
||||
assert_eq!(set.for_variant("q-x", None).rows.len(), 1);
|
||||
assert_eq!(set.for_variant("q-other", Some("v1")).rows.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matrix_is_students_by_items() {
|
||||
let mut set = ResponseSet::new();
|
||||
|
||||
@@ -614,6 +614,7 @@ mod tests {
|
||||
item_number: number,
|
||||
item_ref: Some("bank::q-x-001".into()),
|
||||
item_version: Some(2),
|
||||
variant: None,
|
||||
selected: vec!["C".into()],
|
||||
selected_source: vec![],
|
||||
eliminated: vec![],
|
||||
|
||||
@@ -49,6 +49,7 @@ pub fn schema() -> Schema {
|
||||
Field::new("item_number", DataType::UInt32, false),
|
||||
Field::new("item_ref", DataType::Utf8, false),
|
||||
Field::new("item_version", DataType::UInt32, false),
|
||||
Field::new("variant", DataType::Utf8, false),
|
||||
Field::new("selected", DataType::Utf8, false),
|
||||
Field::new("eliminated", DataType::Utf8, false),
|
||||
Field::new("correct", DataType::Utf8, false),
|
||||
@@ -114,6 +115,7 @@ fn to_batch(rows: &[FlatResponse]) -> Result<RecordBatch> {
|
||||
u32c(|r| r.item_number),
|
||||
s(|r| &r.item_ref),
|
||||
u32c(|r| r.item_version),
|
||||
s(|r| &r.variant),
|
||||
s(|r| &r.selected),
|
||||
s(|r| &r.eliminated),
|
||||
s(|r| &r.correct),
|
||||
@@ -279,6 +281,9 @@ fn from_batch(batch: &RecordBatch, path: &Path) -> Result<Vec<FlatResponse>> {
|
||||
let item_number = uints("item_number")?;
|
||||
let item_ref = strings("item_ref")?;
|
||||
let item_version = uints("item_version")?;
|
||||
// Added after the first stores were written, so absent rather than fatal in
|
||||
// a file from before 2.0; `coursebank migrate variants` fills it in.
|
||||
let variant = optional_strings("variant");
|
||||
let selected = strings("selected")?;
|
||||
let eliminated = strings("eliminated")?;
|
||||
let correct = strings("correct")?;
|
||||
@@ -314,6 +319,7 @@ fn from_batch(batch: &RecordBatch, path: &Path) -> Result<Vec<FlatResponse>> {
|
||||
item_number: item_number.value(i),
|
||||
item_ref: item_ref.value(i).to_string(),
|
||||
item_version: item_version.value(i),
|
||||
variant: variant.map(|c| c.value(i).to_string()).unwrap_or_default(),
|
||||
selected: selected.value(i).to_string(),
|
||||
selected_source: selected_source
|
||||
.map(|a| a.value(i).to_string())
|
||||
@@ -366,6 +372,7 @@ mod tests {
|
||||
item_number: number,
|
||||
item_ref: "bank::q-a-001".into(),
|
||||
item_version: 3,
|
||||
variant: "4c81fa".into(),
|
||||
selected: "C".into(),
|
||||
eliminated: String::new(),
|
||||
correct: "1".into(),
|
||||
|
||||
Reference in new issue
Block a user