Replace 'Scientific Computing Studio' in the source headers, the Cargo.toml authors, and the LICENSE contributor line. pixi.toml still has the old authors line. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014zNf4shD4zaCKLucZWPYL9
445 lines
16 KiB
Rust
445 lines
16 KiB
Rust
// SPDX-License-Identifier: Prosperity-3.0.0
|
|
// Copyright Scienting
|
|
// Source: https://git.scient.ing/education/coursebank
|
|
|
|
//! Parquet reading and writing, isolated so the rest of the crate never touches
|
|
//! Arrow types.
|
|
//!
|
|
//! Every use of `arrow` and `parquet` in this crate is in this file. That is
|
|
//! deliberate: those two crates move fast and release breaking versions together,
|
|
//! and confining them to one module means a version bump is a single-file edit
|
|
//! rather than a refactor. It also means `--no-default-features` compiles the
|
|
//! whole tool without them.
|
|
//!
|
|
//! The schema mirrors [`FlatResponse`] field for field. Optional values are
|
|
//! written with the same sentinels the CSV path uses — empty string, `0` for an
|
|
//! unknown level — rather than nulls, so that a Parquet file and a CSV file of
|
|
//! the same data load identically in pandas.
|
|
|
|
use std::path::Path;
|
|
use std::sync::Arc;
|
|
|
|
use arrow_array::{ArrayRef, BooleanArray, Float64Array, RecordBatch, StringArray, UInt32Array};
|
|
use arrow_schema::{DataType, Field, Schema};
|
|
use parquet::arrow::ArrowWriter;
|
|
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
|
|
use parquet::basic::Compression;
|
|
use parquet::file::properties::WriterProperties;
|
|
|
|
use crate::error::{Error, Result};
|
|
use crate::responses::FlatResponse;
|
|
|
|
/// The Arrow schema for stored responses.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The schema, whose field order matches [`FlatResponse`].
|
|
pub fn schema() -> Schema {
|
|
Schema::new(vec![
|
|
Field::new("administration_id", DataType::Utf8, false),
|
|
Field::new("course", DataType::Utf8, false),
|
|
Field::new("term", DataType::Utf8, false),
|
|
Field::new("assessment_id", DataType::Utf8, false),
|
|
Field::new("date", DataType::Utf8, false),
|
|
Field::new("form", DataType::Utf8, false),
|
|
Field::new("student_key", DataType::Utf8, false),
|
|
Field::new("sid", DataType::Utf8, false),
|
|
Field::new("email", DataType::Utf8, false),
|
|
Field::new("section", DataType::Utf8, false),
|
|
Field::new("item_number", DataType::UInt32, false),
|
|
Field::new("item_ref", DataType::Utf8, false),
|
|
Field::new("item_version", DataType::UInt32, false),
|
|
Field::new("variant", DataType::Utf8, false),
|
|
Field::new("selected", DataType::Utf8, false),
|
|
Field::new("eliminated", DataType::Utf8, false),
|
|
Field::new("correct", DataType::Utf8, false),
|
|
Field::new("credit", DataType::Float64, false),
|
|
Field::new("points_possible", DataType::Float64, false),
|
|
Field::new("score", DataType::Float64, false),
|
|
Field::new("response_time_seconds", DataType::Utf8, false),
|
|
Field::new("level", DataType::UInt32, false),
|
|
// The stored column name predates the objective/target rename and is left
|
|
// alone: an existing parquet file has to keep loading.
|
|
Field::new("learning_objectives", DataType::Utf8, false),
|
|
Field::new("topics", DataType::Utf8, false),
|
|
Field::new("bonus", DataType::Boolean, false),
|
|
Field::new("dropped", DataType::Boolean, false),
|
|
Field::new("form_position", DataType::UInt32, false),
|
|
Field::new("dropped_full_credit", DataType::Boolean, false),
|
|
Field::new("selected_source", DataType::Utf8, false),
|
|
Field::new("eliminated_source", DataType::Utf8, false),
|
|
])
|
|
}
|
|
|
|
/// Builds a record batch from flat responses.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `rows` - the rows.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The batch.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns [`Error::Other`] when Arrow rejects the column set, which would mean
|
|
/// this function and [`schema`] have drifted apart.
|
|
fn to_batch(rows: &[FlatResponse]) -> Result<RecordBatch> {
|
|
let s = |f: fn(&FlatResponse) -> &str| -> ArrayRef {
|
|
Arc::new(StringArray::from(rows.iter().map(f).collect::<Vec<&str>>()))
|
|
};
|
|
let f64c = |f: fn(&FlatResponse) -> f64| -> ArrayRef {
|
|
Arc::new(Float64Array::from(rows.iter().map(f).collect::<Vec<f64>>()))
|
|
};
|
|
let u32c = |f: fn(&FlatResponse) -> u32| -> ArrayRef {
|
|
Arc::new(UInt32Array::from(rows.iter().map(f).collect::<Vec<u32>>()))
|
|
};
|
|
let boolc = |f: fn(&FlatResponse) -> bool| -> ArrayRef {
|
|
Arc::new(BooleanArray::from(
|
|
rows.iter().map(f).collect::<Vec<bool>>(),
|
|
))
|
|
};
|
|
|
|
let columns: Vec<ArrayRef> = vec![
|
|
s(|r| &r.administration_id),
|
|
s(|r| &r.course),
|
|
s(|r| &r.term),
|
|
s(|r| &r.assessment_id),
|
|
s(|r| &r.date),
|
|
s(|r| &r.form),
|
|
s(|r| &r.student_key),
|
|
s(|r| &r.sid),
|
|
s(|r| &r.email),
|
|
s(|r| &r.section),
|
|
u32c(|r| r.item_number),
|
|
s(|r| &r.item_ref),
|
|
u32c(|r| r.item_version),
|
|
s(|r| &r.variant),
|
|
s(|r| &r.selected),
|
|
s(|r| &r.eliminated),
|
|
s(|r| &r.correct),
|
|
f64c(|r| r.credit),
|
|
f64c(|r| r.points_possible),
|
|
f64c(|r| r.score),
|
|
s(|r| &r.response_time_seconds),
|
|
u32c(|r| r.level as u32),
|
|
s(|r| &r.learning_targets),
|
|
s(|r| &r.topics),
|
|
boolc(|r| r.bonus),
|
|
boolc(|r| r.dropped),
|
|
u32c(|r| r.form_position),
|
|
boolc(|r| r.dropped_full_credit),
|
|
s(|r| &r.selected_source),
|
|
s(|r| &r.eliminated_source),
|
|
];
|
|
|
|
RecordBatch::try_new(Arc::new(schema()), columns).map_err(|e| {
|
|
Error::Other(format!(
|
|
"internal error building a Parquet batch: {e}. This means the Arrow schema and the \
|
|
column builders in store_parquet.rs have drifted apart"
|
|
))
|
|
})
|
|
}
|
|
|
|
/// Writes flat responses to a Parquet file.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `path` - the destination.
|
|
/// * `rows` - the rows.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns [`Error::Io`] on a write failure and [`Error::Other`] when the
|
|
/// Parquet writer rejects the batch.
|
|
pub fn write(path: &Path, rows: &[FlatResponse]) -> Result<()> {
|
|
if let Some(parent) = path.parent() {
|
|
std::fs::create_dir_all(parent).map_err(|e| Error::io(parent, e))?;
|
|
}
|
|
let batch = to_batch(rows)?;
|
|
let file = std::fs::File::create(path).map_err(|e| Error::io(path, e))?;
|
|
|
|
// Snappy rather than zstd: it is the format's most universally readable
|
|
// codec, and these files are small enough that the ratio difference is
|
|
// irrelevant next to being openable by an old pandas.
|
|
let props = WriterProperties::builder()
|
|
.set_compression(Compression::SNAPPY)
|
|
.build();
|
|
|
|
let mut writer = ArrowWriter::try_new(file, Arc::new(schema()), Some(props))
|
|
.map_err(|e| Error::Other(format!("cannot open {} for Parquet: {e}", path.display())))?;
|
|
writer
|
|
.write(&batch)
|
|
.map_err(|e| Error::Other(format!("cannot write {}: {e}", path.display())))?;
|
|
writer
|
|
.close()
|
|
.map_err(|e| Error::Other(format!("cannot finish {}: {e}", path.display())))?;
|
|
Ok(())
|
|
}
|
|
|
|
/// Reads flat responses from a Parquet file.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `path` - the file.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The rows.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns [`Error::Io`] when the file cannot be opened and [`Error::Other`]
|
|
/// when its schema does not match what this crate writes.
|
|
pub fn read(path: &Path) -> Result<Vec<FlatResponse>> {
|
|
let file = std::fs::File::open(path).map_err(|e| Error::io(path, e))?;
|
|
let builder = ParquetRecordBatchReaderBuilder::try_new(file)
|
|
.map_err(|e| Error::Other(format!("cannot read {} as Parquet: {e}", path.display())))?;
|
|
let reader = builder
|
|
.build()
|
|
.map_err(|e| Error::Other(format!("cannot read {}: {e}", path.display())))?;
|
|
|
|
let mut out = Vec::new();
|
|
for batch in reader {
|
|
let batch =
|
|
batch.map_err(|e| Error::Other(format!("cannot read {}: {e}", path.display())))?;
|
|
out.extend(from_batch(&batch, path)?);
|
|
}
|
|
Ok(out)
|
|
}
|
|
|
|
/// Converts a record batch back into flat responses.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `batch` - the batch.
|
|
/// * `path` - the source file, for error messages.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The rows.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns [`Error::Other`] when a column is missing or has the wrong type.
|
|
fn from_batch(batch: &RecordBatch, path: &Path) -> Result<Vec<FlatResponse>> {
|
|
let strings = |name: &str| -> Result<&StringArray> {
|
|
batch
|
|
.column_by_name(name)
|
|
.and_then(|c| c.as_any().downcast_ref::<StringArray>())
|
|
.ok_or_else(|| column_error(name, "string", path))
|
|
};
|
|
let floats = |name: &str| -> Result<&Float64Array> {
|
|
batch
|
|
.column_by_name(name)
|
|
.and_then(|c| c.as_any().downcast_ref::<Float64Array>())
|
|
.ok_or_else(|| column_error(name, "float64", path))
|
|
};
|
|
let uints = |name: &str| -> Result<&UInt32Array> {
|
|
batch
|
|
.column_by_name(name)
|
|
.and_then(|c| c.as_any().downcast_ref::<UInt32Array>())
|
|
.ok_or_else(|| column_error(name, "uint32", path))
|
|
};
|
|
let bools = |name: &str| -> Result<&BooleanArray> {
|
|
batch
|
|
.column_by_name(name)
|
|
.and_then(|c| c.as_any().downcast_ref::<BooleanArray>())
|
|
.ok_or_else(|| column_error(name, "boolean", path))
|
|
};
|
|
|
|
// The three decode columns are read optionally rather than required, so a
|
|
// file written before seals existed still loads. A missing column is not a
|
|
// foreign file; it is last term's data.
|
|
let optional_strings = |name: &str| -> Option<&StringArray> {
|
|
batch
|
|
.column_by_name(name)
|
|
.and_then(|c| c.as_any().downcast_ref::<StringArray>())
|
|
};
|
|
let optional_uints = |name: &str| -> Option<&UInt32Array> {
|
|
batch
|
|
.column_by_name(name)
|
|
.and_then(|c| c.as_any().downcast_ref::<UInt32Array>())
|
|
};
|
|
let optional_bools = |name: &str| -> Option<&BooleanArray> {
|
|
batch
|
|
.column_by_name(name)
|
|
.and_then(|c| c.as_any().downcast_ref::<BooleanArray>())
|
|
};
|
|
|
|
let administration_id = strings("administration_id")?;
|
|
let course = strings("course")?;
|
|
let term = strings("term")?;
|
|
let assessment_id = strings("assessment_id")?;
|
|
let date = strings("date")?;
|
|
let form = strings("form")?;
|
|
let student_key = strings("student_key")?;
|
|
let sid = strings("sid")?;
|
|
let email = strings("email")?;
|
|
let section = strings("section")?;
|
|
let item_number = uints("item_number")?;
|
|
let item_ref = strings("item_ref")?;
|
|
let item_version = uints("item_version")?;
|
|
// Added after the first stores were written, so absent rather than fatal in
|
|
// a file from before 2.0; `coursebank migrate variants` fills it in.
|
|
let variant = optional_strings("variant");
|
|
let selected = strings("selected")?;
|
|
let eliminated = strings("eliminated")?;
|
|
let correct = strings("correct")?;
|
|
let credit = floats("credit")?;
|
|
let points_possible = floats("points_possible")?;
|
|
let score = floats("score")?;
|
|
let response_time_seconds = strings("response_time_seconds")?;
|
|
let level = uints("level")?;
|
|
let learning_targets = strings("learning_objectives")?;
|
|
let topics = strings("topics")?;
|
|
let bonus = bools("bonus")?;
|
|
let dropped = bools("dropped")?;
|
|
|
|
let form_position = optional_uints("form_position");
|
|
let dropped_full_credit = optional_bools("dropped_full_credit");
|
|
let selected_source = optional_strings("selected_source");
|
|
let eliminated_source = optional_strings("eliminated_source");
|
|
|
|
let mut out = Vec::with_capacity(batch.num_rows());
|
|
for i in 0..batch.num_rows() {
|
|
out.push(FlatResponse {
|
|
administration_id: administration_id.value(i).to_string(),
|
|
course: course.value(i).to_string(),
|
|
term: term.value(i).to_string(),
|
|
assessment_id: assessment_id.value(i).to_string(),
|
|
date: date.value(i).to_string(),
|
|
form: form.value(i).to_string(),
|
|
form_position: form_position.map(|a| a.value(i)).unwrap_or(0),
|
|
student_key: student_key.value(i).to_string(),
|
|
sid: sid.value(i).to_string(),
|
|
email: email.value(i).to_string(),
|
|
section: section.value(i).to_string(),
|
|
item_number: item_number.value(i),
|
|
item_ref: item_ref.value(i).to_string(),
|
|
item_version: item_version.value(i),
|
|
variant: variant.map(|c| c.value(i).to_string()).unwrap_or_default(),
|
|
selected: selected.value(i).to_string(),
|
|
selected_source: selected_source
|
|
.map(|a| a.value(i).to_string())
|
|
.unwrap_or_default(),
|
|
eliminated: eliminated.value(i).to_string(),
|
|
eliminated_source: eliminated_source
|
|
.map(|a| a.value(i).to_string())
|
|
.unwrap_or_default(),
|
|
correct: correct.value(i).to_string(),
|
|
credit: credit.value(i),
|
|
points_possible: points_possible.value(i),
|
|
score: score.value(i),
|
|
response_time_seconds: response_time_seconds.value(i).to_string(),
|
|
level: level.value(i) as u8,
|
|
learning_targets: learning_targets.value(i).to_string(),
|
|
topics: topics.value(i).to_string(),
|
|
bonus: bonus.value(i),
|
|
dropped: dropped.value(i),
|
|
dropped_full_credit: dropped_full_credit.map(|a| a.value(i)).unwrap_or(false),
|
|
});
|
|
}
|
|
Ok(out)
|
|
}
|
|
|
|
/// Builds the error for a missing or mistyped column.
|
|
fn column_error(name: &str, expected: &str, path: &Path) -> Error {
|
|
Error::Other(format!(
|
|
"{} is missing the `{name}` column or it is not {expected}; this file was probably not \
|
|
written by coursebank",
|
|
path.display()
|
|
))
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
fn flat(student: &str, number: u32) -> FlatResponse {
|
|
FlatResponse {
|
|
administration_id: "C/2026s/e1".into(),
|
|
course: "C".into(),
|
|
term: "2026s".into(),
|
|
assessment_id: "e1".into(),
|
|
date: "2026-04-01".into(),
|
|
form: String::new(),
|
|
student_key: student.into(),
|
|
sid: "1234567".into(),
|
|
email: String::new(),
|
|
section: "L01".into(),
|
|
item_number: number,
|
|
item_ref: "bank::q-a-001".into(),
|
|
item_version: 3,
|
|
variant: "4c81fa".into(),
|
|
selected: "C".into(),
|
|
eliminated: String::new(),
|
|
correct: "1".into(),
|
|
credit: 1.0,
|
|
points_possible: 1.5,
|
|
score: 1.5,
|
|
response_time_seconds: "42.0".into(),
|
|
level: 3,
|
|
learning_targets: "lo-a,lo-b".into(),
|
|
topics: "kinetics".into(),
|
|
bonus: false,
|
|
dropped: false,
|
|
dropped_full_credit: false,
|
|
form_position: 0,
|
|
selected_source: String::new(),
|
|
eliminated_source: String::new(),
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn schema_matches_the_column_builders() {
|
|
let rows = vec![flat("s1", 1)];
|
|
let batch = to_batch(&rows).expect("schema and builders agree");
|
|
assert_eq!(batch.num_columns(), schema().fields().len());
|
|
assert_eq!(batch.num_rows(), 1);
|
|
}
|
|
|
|
#[test]
|
|
fn round_trips_through_a_file() {
|
|
let dir = std::env::temp_dir().join(format!("cb-pq-{}", std::process::id()));
|
|
std::fs::create_dir_all(&dir).unwrap();
|
|
let path = dir.join("e1.parquet");
|
|
|
|
let rows = vec![flat("s1", 1), flat("s2", 2)];
|
|
write(&path, &rows).unwrap();
|
|
let back = read(&path).unwrap();
|
|
|
|
assert_eq!(back.len(), 2);
|
|
assert_eq!(back[0].student_key, "s1");
|
|
assert_eq!(back[1].item_number, 2);
|
|
assert_eq!(back[0].learning_targets, "lo-a,lo-b");
|
|
assert_eq!(back[0].credit, 1.0);
|
|
assert_eq!(back[0].level, 3);
|
|
assert!(!back[0].bonus);
|
|
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
#[test]
|
|
fn an_empty_batch_is_writable() {
|
|
let dir = std::env::temp_dir().join(format!("cb-pq-empty-{}", std::process::id()));
|
|
std::fs::create_dir_all(&dir).unwrap();
|
|
let path = dir.join("empty.parquet");
|
|
write(&path, &[]).unwrap();
|
|
assert_eq!(read(&path).unwrap().len(), 0);
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
#[test]
|
|
fn a_foreign_file_is_rejected_clearly() {
|
|
let dir = std::env::temp_dir().join(format!("cb-pq-bad-{}", std::process::id()));
|
|
std::fs::create_dir_all(&dir).unwrap();
|
|
let path = dir.join("not.parquet");
|
|
std::fs::write(&path, b"this is not parquet").unwrap();
|
|
let err = read(&path).unwrap_err();
|
|
assert!(err.to_string().contains("as Parquet"));
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
}
|