Files
coursebank/src/data/store_parquet.rs
T
alexmandClaude c8d6f4a5dd
Pipeline / check (pull_request) Successful in 12m21s
Pipeline / docs (pull_request) Skipped
Pipeline / nightly (pull_request) Skipped
Pipeline / release (pull_request) Skipped
Name Scienting as the copyright holder and contributor
Replace 'Scientific Computing Studio' in the source headers, the Cargo.toml
authors, and the LICENSE contributor line. pixi.toml still has the old
authors line.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014zNf4shD4zaCKLucZWPYL9
2026-10-20 00:21:19 -04:00

445 lines
16 KiB
Rust

// SPDX-License-Identifier: Prosperity-3.0.0
// Copyright Scienting
// Source: https://git.scient.ing/education/coursebank
//! Parquet reading and writing, isolated so the rest of the crate never touches
//! Arrow types.
//!
//! Every use of `arrow` and `parquet` in this crate is in this file. That is
//! deliberate: those two crates move fast and release breaking versions together,
//! and confining them to one module means a version bump is a single-file edit
//! rather than a refactor. It also means `--no-default-features` compiles the
//! whole tool without them.
//!
//! The schema mirrors [`FlatResponse`] field for field. Optional values are
//! written with the same sentinels the CSV path uses — empty string, `0` for an
//! unknown level — rather than nulls, so that a Parquet file and a CSV file of
//! the same data load identically in pandas.
use std::path::Path;
use std::sync::Arc;
use arrow_array::{ArrayRef, BooleanArray, Float64Array, RecordBatch, StringArray, UInt32Array};
use arrow_schema::{DataType, Field, Schema};
use parquet::arrow::ArrowWriter;
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
use parquet::basic::Compression;
use parquet::file::properties::WriterProperties;
use crate::error::{Error, Result};
use crate::responses::FlatResponse;
/// The Arrow schema for stored responses.
///
/// # Returns
///
/// The schema, whose field order matches [`FlatResponse`].
pub fn schema() -> Schema {
Schema::new(vec![
Field::new("administration_id", DataType::Utf8, false),
Field::new("course", DataType::Utf8, false),
Field::new("term", DataType::Utf8, false),
Field::new("assessment_id", DataType::Utf8, false),
Field::new("date", DataType::Utf8, false),
Field::new("form", DataType::Utf8, false),
Field::new("student_key", DataType::Utf8, false),
Field::new("sid", DataType::Utf8, false),
Field::new("email", DataType::Utf8, false),
Field::new("section", DataType::Utf8, false),
Field::new("item_number", DataType::UInt32, false),
Field::new("item_ref", DataType::Utf8, false),
Field::new("item_version", DataType::UInt32, false),
Field::new("variant", DataType::Utf8, false),
Field::new("selected", DataType::Utf8, false),
Field::new("eliminated", DataType::Utf8, false),
Field::new("correct", DataType::Utf8, false),
Field::new("credit", DataType::Float64, false),
Field::new("points_possible", DataType::Float64, false),
Field::new("score", DataType::Float64, false),
Field::new("response_time_seconds", DataType::Utf8, false),
Field::new("level", DataType::UInt32, false),
// The stored column name predates the objective/target rename and is left
// alone: an existing parquet file has to keep loading.
Field::new("learning_objectives", DataType::Utf8, false),
Field::new("topics", DataType::Utf8, false),
Field::new("bonus", DataType::Boolean, false),
Field::new("dropped", DataType::Boolean, false),
Field::new("form_position", DataType::UInt32, false),
Field::new("dropped_full_credit", DataType::Boolean, false),
Field::new("selected_source", DataType::Utf8, false),
Field::new("eliminated_source", DataType::Utf8, false),
])
}
/// Builds a record batch from flat responses.
///
/// # Arguments
///
/// * `rows` - the rows.
///
/// # Returns
///
/// The batch.
///
/// # Errors
///
/// Returns [`Error::Other`] when Arrow rejects the column set, which would mean
/// this function and [`schema`] have drifted apart.
fn to_batch(rows: &[FlatResponse]) -> Result<RecordBatch> {
let s = |f: fn(&FlatResponse) -> &str| -> ArrayRef {
Arc::new(StringArray::from(rows.iter().map(f).collect::<Vec<&str>>()))
};
let f64c = |f: fn(&FlatResponse) -> f64| -> ArrayRef {
Arc::new(Float64Array::from(rows.iter().map(f).collect::<Vec<f64>>()))
};
let u32c = |f: fn(&FlatResponse) -> u32| -> ArrayRef {
Arc::new(UInt32Array::from(rows.iter().map(f).collect::<Vec<u32>>()))
};
let boolc = |f: fn(&FlatResponse) -> bool| -> ArrayRef {
Arc::new(BooleanArray::from(
rows.iter().map(f).collect::<Vec<bool>>(),
))
};
let columns: Vec<ArrayRef> = vec![
s(|r| &r.administration_id),
s(|r| &r.course),
s(|r| &r.term),
s(|r| &r.assessment_id),
s(|r| &r.date),
s(|r| &r.form),
s(|r| &r.student_key),
s(|r| &r.sid),
s(|r| &r.email),
s(|r| &r.section),
u32c(|r| r.item_number),
s(|r| &r.item_ref),
u32c(|r| r.item_version),
s(|r| &r.variant),
s(|r| &r.selected),
s(|r| &r.eliminated),
s(|r| &r.correct),
f64c(|r| r.credit),
f64c(|r| r.points_possible),
f64c(|r| r.score),
s(|r| &r.response_time_seconds),
u32c(|r| r.level as u32),
s(|r| &r.learning_targets),
s(|r| &r.topics),
boolc(|r| r.bonus),
boolc(|r| r.dropped),
u32c(|r| r.form_position),
boolc(|r| r.dropped_full_credit),
s(|r| &r.selected_source),
s(|r| &r.eliminated_source),
];
RecordBatch::try_new(Arc::new(schema()), columns).map_err(|e| {
Error::Other(format!(
"internal error building a Parquet batch: {e}. This means the Arrow schema and the \
column builders in store_parquet.rs have drifted apart"
))
})
}
/// Writes flat responses to a Parquet file.
///
/// # Arguments
///
/// * `path` - the destination.
/// * `rows` - the rows.
///
/// # Errors
///
/// Returns [`Error::Io`] on a write failure and [`Error::Other`] when the
/// Parquet writer rejects the batch.
pub fn write(path: &Path, rows: &[FlatResponse]) -> Result<()> {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent).map_err(|e| Error::io(parent, e))?;
}
let batch = to_batch(rows)?;
let file = std::fs::File::create(path).map_err(|e| Error::io(path, e))?;
// Snappy rather than zstd: it is the format's most universally readable
// codec, and these files are small enough that the ratio difference is
// irrelevant next to being openable by an old pandas.
let props = WriterProperties::builder()
.set_compression(Compression::SNAPPY)
.build();
let mut writer = ArrowWriter::try_new(file, Arc::new(schema()), Some(props))
.map_err(|e| Error::Other(format!("cannot open {} for Parquet: {e}", path.display())))?;
writer
.write(&batch)
.map_err(|e| Error::Other(format!("cannot write {}: {e}", path.display())))?;
writer
.close()
.map_err(|e| Error::Other(format!("cannot finish {}: {e}", path.display())))?;
Ok(())
}
/// Reads flat responses from a Parquet file.
///
/// # Arguments
///
/// * `path` - the file.
///
/// # Returns
///
/// The rows.
///
/// # Errors
///
/// Returns [`Error::Io`] when the file cannot be opened and [`Error::Other`]
/// when its schema does not match what this crate writes.
pub fn read(path: &Path) -> Result<Vec<FlatResponse>> {
let file = std::fs::File::open(path).map_err(|e| Error::io(path, e))?;
let builder = ParquetRecordBatchReaderBuilder::try_new(file)
.map_err(|e| Error::Other(format!("cannot read {} as Parquet: {e}", path.display())))?;
let reader = builder
.build()
.map_err(|e| Error::Other(format!("cannot read {}: {e}", path.display())))?;
let mut out = Vec::new();
for batch in reader {
let batch =
batch.map_err(|e| Error::Other(format!("cannot read {}: {e}", path.display())))?;
out.extend(from_batch(&batch, path)?);
}
Ok(out)
}
/// Converts a record batch back into flat responses.
///
/// # Arguments
///
/// * `batch` - the batch.
/// * `path` - the source file, for error messages.
///
/// # Returns
///
/// The rows.
///
/// # Errors
///
/// Returns [`Error::Other`] when a column is missing or has the wrong type.
fn from_batch(batch: &RecordBatch, path: &Path) -> Result<Vec<FlatResponse>> {
let strings = |name: &str| -> Result<&StringArray> {
batch
.column_by_name(name)
.and_then(|c| c.as_any().downcast_ref::<StringArray>())
.ok_or_else(|| column_error(name, "string", path))
};
let floats = |name: &str| -> Result<&Float64Array> {
batch
.column_by_name(name)
.and_then(|c| c.as_any().downcast_ref::<Float64Array>())
.ok_or_else(|| column_error(name, "float64", path))
};
let uints = |name: &str| -> Result<&UInt32Array> {
batch
.column_by_name(name)
.and_then(|c| c.as_any().downcast_ref::<UInt32Array>())
.ok_or_else(|| column_error(name, "uint32", path))
};
let bools = |name: &str| -> Result<&BooleanArray> {
batch
.column_by_name(name)
.and_then(|c| c.as_any().downcast_ref::<BooleanArray>())
.ok_or_else(|| column_error(name, "boolean", path))
};
// The three decode columns are read optionally rather than required, so a
// file written before seals existed still loads. A missing column is not a
// foreign file; it is last term's data.
let optional_strings = |name: &str| -> Option<&StringArray> {
batch
.column_by_name(name)
.and_then(|c| c.as_any().downcast_ref::<StringArray>())
};
let optional_uints = |name: &str| -> Option<&UInt32Array> {
batch
.column_by_name(name)
.and_then(|c| c.as_any().downcast_ref::<UInt32Array>())
};
let optional_bools = |name: &str| -> Option<&BooleanArray> {
batch
.column_by_name(name)
.and_then(|c| c.as_any().downcast_ref::<BooleanArray>())
};
let administration_id = strings("administration_id")?;
let course = strings("course")?;
let term = strings("term")?;
let assessment_id = strings("assessment_id")?;
let date = strings("date")?;
let form = strings("form")?;
let student_key = strings("student_key")?;
let sid = strings("sid")?;
let email = strings("email")?;
let section = strings("section")?;
let item_number = uints("item_number")?;
let item_ref = strings("item_ref")?;
let item_version = uints("item_version")?;
// Added after the first stores were written, so absent rather than fatal in
// a file from before 2.0; `coursebank migrate variants` fills it in.
let variant = optional_strings("variant");
let selected = strings("selected")?;
let eliminated = strings("eliminated")?;
let correct = strings("correct")?;
let credit = floats("credit")?;
let points_possible = floats("points_possible")?;
let score = floats("score")?;
let response_time_seconds = strings("response_time_seconds")?;
let level = uints("level")?;
let learning_targets = strings("learning_objectives")?;
let topics = strings("topics")?;
let bonus = bools("bonus")?;
let dropped = bools("dropped")?;
let form_position = optional_uints("form_position");
let dropped_full_credit = optional_bools("dropped_full_credit");
let selected_source = optional_strings("selected_source");
let eliminated_source = optional_strings("eliminated_source");
let mut out = Vec::with_capacity(batch.num_rows());
for i in 0..batch.num_rows() {
out.push(FlatResponse {
administration_id: administration_id.value(i).to_string(),
course: course.value(i).to_string(),
term: term.value(i).to_string(),
assessment_id: assessment_id.value(i).to_string(),
date: date.value(i).to_string(),
form: form.value(i).to_string(),
form_position: form_position.map(|a| a.value(i)).unwrap_or(0),
student_key: student_key.value(i).to_string(),
sid: sid.value(i).to_string(),
email: email.value(i).to_string(),
section: section.value(i).to_string(),
item_number: item_number.value(i),
item_ref: item_ref.value(i).to_string(),
item_version: item_version.value(i),
variant: variant.map(|c| c.value(i).to_string()).unwrap_or_default(),
selected: selected.value(i).to_string(),
selected_source: selected_source
.map(|a| a.value(i).to_string())
.unwrap_or_default(),
eliminated: eliminated.value(i).to_string(),
eliminated_source: eliminated_source
.map(|a| a.value(i).to_string())
.unwrap_or_default(),
correct: correct.value(i).to_string(),
credit: credit.value(i),
points_possible: points_possible.value(i),
score: score.value(i),
response_time_seconds: response_time_seconds.value(i).to_string(),
level: level.value(i) as u8,
learning_targets: learning_targets.value(i).to_string(),
topics: topics.value(i).to_string(),
bonus: bonus.value(i),
dropped: dropped.value(i),
dropped_full_credit: dropped_full_credit.map(|a| a.value(i)).unwrap_or(false),
});
}
Ok(out)
}
/// Builds the error for a missing or mistyped column.
fn column_error(name: &str, expected: &str, path: &Path) -> Error {
Error::Other(format!(
"{} is missing the `{name}` column or it is not {expected}; this file was probably not \
written by coursebank",
path.display()
))
}
#[cfg(test)]
mod tests {
use super::*;
fn flat(student: &str, number: u32) -> FlatResponse {
FlatResponse {
administration_id: "C/2026s/e1".into(),
course: "C".into(),
term: "2026s".into(),
assessment_id: "e1".into(),
date: "2026-04-01".into(),
form: String::new(),
student_key: student.into(),
sid: "1234567".into(),
email: String::new(),
section: "L01".into(),
item_number: number,
item_ref: "bank::q-a-001".into(),
item_version: 3,
variant: "4c81fa".into(),
selected: "C".into(),
eliminated: String::new(),
correct: "1".into(),
credit: 1.0,
points_possible: 1.5,
score: 1.5,
response_time_seconds: "42.0".into(),
level: 3,
learning_targets: "lo-a,lo-b".into(),
topics: "kinetics".into(),
bonus: false,
dropped: false,
dropped_full_credit: false,
form_position: 0,
selected_source: String::new(),
eliminated_source: String::new(),
}
}
#[test]
fn schema_matches_the_column_builders() {
let rows = vec![flat("s1", 1)];
let batch = to_batch(&rows).expect("schema and builders agree");
assert_eq!(batch.num_columns(), schema().fields().len());
assert_eq!(batch.num_rows(), 1);
}
#[test]
fn round_trips_through_a_file() {
let dir = std::env::temp_dir().join(format!("cb-pq-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("e1.parquet");
let rows = vec![flat("s1", 1), flat("s2", 2)];
write(&path, &rows).unwrap();
let back = read(&path).unwrap();
assert_eq!(back.len(), 2);
assert_eq!(back[0].student_key, "s1");
assert_eq!(back[1].item_number, 2);
assert_eq!(back[0].learning_targets, "lo-a,lo-b");
assert_eq!(back[0].credit, 1.0);
assert_eq!(back[0].level, 3);
assert!(!back[0].bonus);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn an_empty_batch_is_writable() {
let dir = std::env::temp_dir().join(format!("cb-pq-empty-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("empty.parquet");
write(&path, &[]).unwrap();
assert_eq!(read(&path).unwrap().len(), 0);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn a_foreign_file_is_rejected_clearly() {
let dir = std::env::temp_dir().join(format!("cb-pq-bad-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("not.parquet");
std::fs::write(&path, b"this is not parquet").unwrap();
let err = read(&path).unwrap_err();
assert!(err.to_string().contains("as Parquet"));
std::fs::remove_dir_all(&dir).ok();
}
}