Files
coursebank/src/data/store_parquet.rs
T
alexm abae39bd54
Pipeline / check (pull_request) Failing after 12m11s
Pipeline / docs (pull_request) Skipped
Pipeline / nightly (pull_request) Skipped
Pipeline / release (pull_request) Skipped
chore: relicense under Apache-2.0
Replace the Prosperity Public License 3.0.0 with the Apache License 2.0:
LICENSE.md, a new NOTICE.md, the Cargo license field, every SPDX header,
and the README's license and contributing sections.
2026-11-18 18:57:08 -05:00

445 lines
16 KiB
Rust

// SPDX-License-Identifier: Apache-2.0
// Copyright Scienting
// Source: https://git.scient.ing/education/coursebank
//! Parquet reading and writing, isolated so the rest of the crate never touches
//! Arrow types.
//!
//! Every use of `arrow` and `parquet` in this crate is in this file. That is
//! deliberate: those two crates move fast and release breaking versions together,
//! and confining them to one module means a version bump is a single-file edit
//! rather than a refactor. It also means `--no-default-features` compiles the
//! whole tool without them.
//!
//! The schema mirrors [`FlatResponse`] field for field. Optional values are
//! written with the same sentinels the CSV path uses — empty string, `0` for an
//! unknown level — rather than nulls, so that a Parquet file and a CSV file of
//! the same data load identically in pandas.
use std::path::Path;
use std::sync::Arc;
use arrow_array::{ArrayRef, BooleanArray, Float64Array, RecordBatch, StringArray, UInt32Array};
use arrow_schema::{DataType, Field, Schema};
use parquet::arrow::ArrowWriter;
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
use parquet::basic::Compression;
use parquet::file::properties::WriterProperties;
use crate::error::{Error, Result};
use crate::responses::FlatResponse;
/// The Arrow schema for stored responses.
///
/// # Returns
///
/// The schema, whose field order matches [`FlatResponse`].
pub fn schema() -> Schema {
Schema::new(vec![
Field::new("administration_id", DataType::Utf8, false),
Field::new("course", DataType::Utf8, false),
Field::new("term", DataType::Utf8, false),
Field::new("assessment_id", DataType::Utf8, false),
Field::new("date", DataType::Utf8, false),
Field::new("form", DataType::Utf8, false),
Field::new("student_key", DataType::Utf8, false),
Field::new("sid", DataType::Utf8, false),
Field::new("email", DataType::Utf8, false),
Field::new("section", DataType::Utf8, false),
Field::new("item_number", DataType::UInt32, false),
Field::new("item_ref", DataType::Utf8, false),
Field::new("item_version", DataType::UInt32, false),
Field::new("variant", DataType::Utf8, false),
Field::new("selected", DataType::Utf8, false),
Field::new("eliminated", DataType::Utf8, false),
Field::new("correct", DataType::Utf8, false),
Field::new("credit", DataType::Float64, false),
Field::new("points_possible", DataType::Float64, false),
Field::new("score", DataType::Float64, false),
Field::new("response_time_seconds", DataType::Utf8, false),
Field::new("level", DataType::UInt32, false),
// The stored column name predates the objective/target rename and is left
// alone: an existing parquet file has to keep loading.
Field::new("learning_objectives", DataType::Utf8, false),
Field::new("topics", DataType::Utf8, false),
Field::new("bonus", DataType::Boolean, false),
Field::new("dropped", DataType::Boolean, false),
Field::new("form_position", DataType::UInt32, false),
Field::new("dropped_full_credit", DataType::Boolean, false),
Field::new("selected_source", DataType::Utf8, false),
Field::new("eliminated_source", DataType::Utf8, false),
])
}
/// Builds a record batch from flat responses.
///
/// # Arguments
///
/// * `rows` - the rows.
///
/// # Returns
///
/// The batch.
///
/// # Errors
///
/// Returns [`Error::Other`] when Arrow rejects the column set, which would mean
/// this function and [`schema`] have drifted apart.
fn to_batch(rows: &[FlatResponse]) -> Result<RecordBatch> {
let s = |f: fn(&FlatResponse) -> &str| -> ArrayRef {
Arc::new(StringArray::from(rows.iter().map(f).collect::<Vec<&str>>()))
};
let f64c = |f: fn(&FlatResponse) -> f64| -> ArrayRef {
Arc::new(Float64Array::from(rows.iter().map(f).collect::<Vec<f64>>()))
};
let u32c = |f: fn(&FlatResponse) -> u32| -> ArrayRef {
Arc::new(UInt32Array::from(rows.iter().map(f).collect::<Vec<u32>>()))
};
let boolc = |f: fn(&FlatResponse) -> bool| -> ArrayRef {
Arc::new(BooleanArray::from(
rows.iter().map(f).collect::<Vec<bool>>(),
))
};
let columns: Vec<ArrayRef> = vec![
s(|r| &r.administration_id),
s(|r| &r.course),
s(|r| &r.term),
s(|r| &r.assessment_id),
s(|r| &r.date),
s(|r| &r.form),
s(|r| &r.student_key),
s(|r| &r.sid),
s(|r| &r.email),
s(|r| &r.section),
u32c(|r| r.item_number),
s(|r| &r.item_ref),
u32c(|r| r.item_version),
s(|r| &r.variant),
s(|r| &r.selected),
s(|r| &r.eliminated),
s(|r| &r.correct),
f64c(|r| r.credit),
f64c(|r| r.points_possible),
f64c(|r| r.score),
s(|r| &r.response_time_seconds),
u32c(|r| r.level as u32),
s(|r| &r.learning_targets),
s(|r| &r.topics),
boolc(|r| r.bonus),
boolc(|r| r.dropped),
u32c(|r| r.form_position),
boolc(|r| r.dropped_full_credit),
s(|r| &r.selected_source),
s(|r| &r.eliminated_source),
];
RecordBatch::try_new(Arc::new(schema()), columns).map_err(|e| {
Error::Other(format!(
"internal error building a Parquet batch: {e}. This means the Arrow schema and the \
column builders in store_parquet.rs have drifted apart"
))
})
}
/// Writes flat responses to a Parquet file.
///
/// # Arguments
///
/// * `path` - the destination.
/// * `rows` - the rows.
///
/// # Errors
///
/// Returns [`Error::Io`] on a write failure and [`Error::Other`] when the
/// Parquet writer rejects the batch.
pub fn write(path: &Path, rows: &[FlatResponse]) -> Result<()> {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent).map_err(|e| Error::io(parent, e))?;
}
let batch = to_batch(rows)?;
let file = std::fs::File::create(path).map_err(|e| Error::io(path, e))?;
// Snappy rather than zstd: it is the format's most universally readable
// codec, and these files are small enough that the ratio difference is
// irrelevant next to being openable by an old pandas.
let props = WriterProperties::builder()
.set_compression(Compression::SNAPPY)
.build();
let mut writer = ArrowWriter::try_new(file, Arc::new(schema()), Some(props))
.map_err(|e| Error::Other(format!("cannot open {} for Parquet: {e}", path.display())))?;
writer
.write(&batch)
.map_err(|e| Error::Other(format!("cannot write {}: {e}", path.display())))?;
writer
.close()
.map_err(|e| Error::Other(format!("cannot finish {}: {e}", path.display())))?;
Ok(())
}
/// Reads flat responses from a Parquet file.
///
/// # Arguments
///
/// * `path` - the file.
///
/// # Returns
///
/// The rows.
///
/// # Errors
///
/// Returns [`Error::Io`] when the file cannot be opened and [`Error::Other`]
/// when its schema does not match what this crate writes.
pub fn read(path: &Path) -> Result<Vec<FlatResponse>> {
let file = std::fs::File::open(path).map_err(|e| Error::io(path, e))?;
let builder = ParquetRecordBatchReaderBuilder::try_new(file)
.map_err(|e| Error::Other(format!("cannot read {} as Parquet: {e}", path.display())))?;
let reader = builder
.build()
.map_err(|e| Error::Other(format!("cannot read {}: {e}", path.display())))?;
let mut out = Vec::new();
for batch in reader {
let batch =
batch.map_err(|e| Error::Other(format!("cannot read {}: {e}", path.display())))?;
out.extend(from_batch(&batch, path)?);
}
Ok(out)
}
/// Converts a record batch back into flat responses.
///
/// # Arguments
///
/// * `batch` - the batch.
/// * `path` - the source file, for error messages.
///
/// # Returns
///
/// The rows.
///
/// # Errors
///
/// Returns [`Error::Other`] when a column is missing or has the wrong type.
fn from_batch(batch: &RecordBatch, path: &Path) -> Result<Vec<FlatResponse>> {
let strings = |name: &str| -> Result<&StringArray> {
batch
.column_by_name(name)
.and_then(|c| c.as_any().downcast_ref::<StringArray>())
.ok_or_else(|| column_error(name, "string", path))
};
let floats = |name: &str| -> Result<&Float64Array> {
batch
.column_by_name(name)
.and_then(|c| c.as_any().downcast_ref::<Float64Array>())
.ok_or_else(|| column_error(name, "float64", path))
};
let uints = |name: &str| -> Result<&UInt32Array> {
batch
.column_by_name(name)
.and_then(|c| c.as_any().downcast_ref::<UInt32Array>())
.ok_or_else(|| column_error(name, "uint32", path))
};
let bools = |name: &str| -> Result<&BooleanArray> {
batch
.column_by_name(name)
.and_then(|c| c.as_any().downcast_ref::<BooleanArray>())
.ok_or_else(|| column_error(name, "boolean", path))
};
// The three decode columns are read optionally rather than required, so a
// file written before seals existed still loads. A missing column is not a
// foreign file; it is last term's data.
let optional_strings = |name: &str| -> Option<&StringArray> {
batch
.column_by_name(name)
.and_then(|c| c.as_any().downcast_ref::<StringArray>())
};
let optional_uints = |name: &str| -> Option<&UInt32Array> {
batch
.column_by_name(name)
.and_then(|c| c.as_any().downcast_ref::<UInt32Array>())
};
let optional_bools = |name: &str| -> Option<&BooleanArray> {
batch
.column_by_name(name)
.and_then(|c| c.as_any().downcast_ref::<BooleanArray>())
};
let administration_id = strings("administration_id")?;
let course = strings("course")?;
let term = strings("term")?;
let assessment_id = strings("assessment_id")?;
let date = strings("date")?;
let form = strings("form")?;
let student_key = strings("student_key")?;
let sid = strings("sid")?;
let email = strings("email")?;
let section = strings("section")?;
let item_number = uints("item_number")?;
let item_ref = strings("item_ref")?;
let item_version = uints("item_version")?;
// Added after the first stores were written, so absent rather than fatal in
// a file from before 2.0; `coursebank migrate variants` fills it in.
let variant = optional_strings("variant");
let selected = strings("selected")?;
let eliminated = strings("eliminated")?;
let correct = strings("correct")?;
let credit = floats("credit")?;
let points_possible = floats("points_possible")?;
let score = floats("score")?;
let response_time_seconds = strings("response_time_seconds")?;
let level = uints("level")?;
let learning_targets = strings("learning_objectives")?;
let topics = strings("topics")?;
let bonus = bools("bonus")?;
let dropped = bools("dropped")?;
let form_position = optional_uints("form_position");
let dropped_full_credit = optional_bools("dropped_full_credit");
let selected_source = optional_strings("selected_source");
let eliminated_source = optional_strings("eliminated_source");
let mut out = Vec::with_capacity(batch.num_rows());
for i in 0..batch.num_rows() {
out.push(FlatResponse {
administration_id: administration_id.value(i).to_string(),
course: course.value(i).to_string(),
term: term.value(i).to_string(),
assessment_id: assessment_id.value(i).to_string(),
date: date.value(i).to_string(),
form: form.value(i).to_string(),
form_position: form_position.map(|a| a.value(i)).unwrap_or(0),
student_key: student_key.value(i).to_string(),
sid: sid.value(i).to_string(),
email: email.value(i).to_string(),
section: section.value(i).to_string(),
item_number: item_number.value(i),
item_ref: item_ref.value(i).to_string(),
item_version: item_version.value(i),
variant: variant.map(|c| c.value(i).to_string()).unwrap_or_default(),
selected: selected.value(i).to_string(),
selected_source: selected_source
.map(|a| a.value(i).to_string())
.unwrap_or_default(),
eliminated: eliminated.value(i).to_string(),
eliminated_source: eliminated_source
.map(|a| a.value(i).to_string())
.unwrap_or_default(),
correct: correct.value(i).to_string(),
credit: credit.value(i),
points_possible: points_possible.value(i),
score: score.value(i),
response_time_seconds: response_time_seconds.value(i).to_string(),
level: level.value(i) as u8,
learning_targets: learning_targets.value(i).to_string(),
topics: topics.value(i).to_string(),
bonus: bonus.value(i),
dropped: dropped.value(i),
dropped_full_credit: dropped_full_credit.map(|a| a.value(i)).unwrap_or(false),
});
}
Ok(out)
}
/// Builds the error for a missing or mistyped column.
fn column_error(name: &str, expected: &str, path: &Path) -> Error {
Error::Other(format!(
"{} is missing the `{name}` column or it is not {expected}; this file was probably not \
written by coursebank",
path.display()
))
}
#[cfg(test)]
mod tests {
use super::*;
fn flat(student: &str, number: u32) -> FlatResponse {
FlatResponse {
administration_id: "C/2026s/e1".into(),
course: "C".into(),
term: "2026s".into(),
assessment_id: "e1".into(),
date: "2026-04-01".into(),
form: String::new(),
student_key: student.into(),
sid: "1234567".into(),
email: String::new(),
section: "L01".into(),
item_number: number,
item_ref: "bank::q-a-001".into(),
item_version: 3,
variant: "4c81fa".into(),
selected: "C".into(),
eliminated: String::new(),
correct: "1".into(),
credit: 1.0,
points_possible: 1.5,
score: 1.5,
response_time_seconds: "42.0".into(),
level: 3,
learning_targets: "lo-a,lo-b".into(),
topics: "kinetics".into(),
bonus: false,
dropped: false,
dropped_full_credit: false,
form_position: 0,
selected_source: String::new(),
eliminated_source: String::new(),
}
}
#[test]
fn schema_matches_the_column_builders() {
let rows = vec![flat("s1", 1)];
let batch = to_batch(&rows).expect("schema and builders agree");
assert_eq!(batch.num_columns(), schema().fields().len());
assert_eq!(batch.num_rows(), 1);
}
#[test]
fn round_trips_through_a_file() {
let dir = std::env::temp_dir().join(format!("cb-pq-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("e1.parquet");
let rows = vec![flat("s1", 1), flat("s2", 2)];
write(&path, &rows).unwrap();
let back = read(&path).unwrap();
assert_eq!(back.len(), 2);
assert_eq!(back[0].student_key, "s1");
assert_eq!(back[1].item_number, 2);
assert_eq!(back[0].learning_targets, "lo-a,lo-b");
assert_eq!(back[0].credit, 1.0);
assert_eq!(back[0].level, 3);
assert!(!back[0].bonus);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn an_empty_batch_is_writable() {
let dir = std::env::temp_dir().join(format!("cb-pq-empty-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("empty.parquet");
write(&path, &[]).unwrap();
assert_eq!(read(&path).unwrap().len(), 0);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn a_foreign_file_is_rejected_clearly() {
let dir = std::env::temp_dir().join(format!("cb-pq-bad-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("not.parquet");
std::fs::write(&path, b"this is not parquet").unwrap();
let err = read(&path).unwrap_err();
assert!(err.to_string().contains("as Parquet"));
std::fs::remove_dir_all(&dir).ok();
}
}