1 parent
228a0da47f
commit
abc0bdf621
79 files changed
+31370
No files matched your search
@@ -0,0 +1,391 @@
|
||||
// SPDX-License-Identifier: Prosperity-3.0.0
|
||||
// Copyright Scientific Computing Studio
|
||||
// Source: https://git.scient.ing/education/coursebank
|
||||
|
||||
//! Parquet reading and writing, isolated so the rest of the crate never touches
|
||||
//! Arrow types.
|
||||
//!
|
||||
//! Every use of `arrow` and `parquet` in this crate is in this file. That is
|
||||
//! deliberate: those two crates move fast and release breaking versions together,
|
||||
//! and confining them to one module means a version bump is a single-file edit
|
||||
//! rather than a refactor. It also means `--no-default-features` compiles the
|
||||
//! whole tool without them.
|
||||
//!
|
||||
//! The schema mirrors [`FlatResponse`] field for field. Optional values are
|
||||
//! written with the same sentinels the CSV path uses — empty string, `0` for an
|
||||
//! unknown level — rather than nulls, so that a Parquet file and a CSV file of
|
||||
//! the same data load identically in pandas.
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow_array::{ArrayRef, BooleanArray, Float64Array, RecordBatch, StringArray, UInt32Array};
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
use parquet::arrow::ArrowWriter;
|
||||
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
|
||||
use parquet::basic::Compression;
|
||||
use parquet::file::properties::WriterProperties;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::responses::FlatResponse;
|
||||
|
||||
/// The Arrow schema for stored responses.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The schema, whose field order matches [`FlatResponse`].
|
||||
pub fn schema() -> Schema {
|
||||
Schema::new(vec![
|
||||
Field::new("administration_id", DataType::Utf8, false),
|
||||
Field::new("course", DataType::Utf8, false),
|
||||
Field::new("term", DataType::Utf8, false),
|
||||
Field::new("assessment_id", DataType::Utf8, false),
|
||||
Field::new("date", DataType::Utf8, false),
|
||||
Field::new("form", DataType::Utf8, false),
|
||||
Field::new("student_key", DataType::Utf8, false),
|
||||
Field::new("sid", DataType::Utf8, false),
|
||||
Field::new("email", DataType::Utf8, false),
|
||||
Field::new("section", DataType::Utf8, false),
|
||||
Field::new("item_number", DataType::UInt32, false),
|
||||
Field::new("item_ref", DataType::Utf8, false),
|
||||
Field::new("item_version", DataType::UInt32, false),
|
||||
Field::new("selected", DataType::Utf8, false),
|
||||
Field::new("eliminated", DataType::Utf8, false),
|
||||
Field::new("correct", DataType::Utf8, false),
|
||||
Field::new("credit", DataType::Float64, false),
|
||||
Field::new("points_possible", DataType::Float64, false),
|
||||
Field::new("score", DataType::Float64, false),
|
||||
Field::new("response_time_seconds", DataType::Utf8, false),
|
||||
Field::new("level", DataType::UInt32, false),
|
||||
Field::new("learning_objectives", DataType::Utf8, false),
|
||||
Field::new("topics", DataType::Utf8, false),
|
||||
Field::new("bonus", DataType::Boolean, false),
|
||||
Field::new("dropped", DataType::Boolean, false),
|
||||
])
|
||||
}
|
||||
|
||||
/// Builds a record batch from flat responses.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `rows` - the rows.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The batch.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`Error::Other`] when Arrow rejects the column set, which would mean
|
||||
/// this function and [`schema`] have drifted apart.
|
||||
fn to_batch(rows: &[FlatResponse]) -> Result<RecordBatch> {
|
||||
let s = |f: fn(&FlatResponse) -> &str| -> ArrayRef {
|
||||
Arc::new(StringArray::from(rows.iter().map(f).collect::<Vec<&str>>()))
|
||||
};
|
||||
let f64c = |f: fn(&FlatResponse) -> f64| -> ArrayRef {
|
||||
Arc::new(Float64Array::from(rows.iter().map(f).collect::<Vec<f64>>()))
|
||||
};
|
||||
let u32c = |f: fn(&FlatResponse) -> u32| -> ArrayRef {
|
||||
Arc::new(UInt32Array::from(rows.iter().map(f).collect::<Vec<u32>>()))
|
||||
};
|
||||
let boolc = |f: fn(&FlatResponse) -> bool| -> ArrayRef {
|
||||
Arc::new(BooleanArray::from(
|
||||
rows.iter().map(f).collect::<Vec<bool>>(),
|
||||
))
|
||||
};
|
||||
|
||||
let columns: Vec<ArrayRef> = vec![
|
||||
s(|r| &r.administration_id),
|
||||
s(|r| &r.course),
|
||||
s(|r| &r.term),
|
||||
s(|r| &r.assessment_id),
|
||||
s(|r| &r.date),
|
||||
s(|r| &r.form),
|
||||
s(|r| &r.student_key),
|
||||
s(|r| &r.sid),
|
||||
s(|r| &r.email),
|
||||
s(|r| &r.section),
|
||||
u32c(|r| r.item_number),
|
||||
s(|r| &r.item_ref),
|
||||
u32c(|r| r.item_version),
|
||||
s(|r| &r.selected),
|
||||
s(|r| &r.eliminated),
|
||||
s(|r| &r.correct),
|
||||
f64c(|r| r.credit),
|
||||
f64c(|r| r.points_possible),
|
||||
f64c(|r| r.score),
|
||||
s(|r| &r.response_time_seconds),
|
||||
u32c(|r| r.level as u32),
|
||||
s(|r| &r.learning_objectives),
|
||||
s(|r| &r.topics),
|
||||
boolc(|r| r.bonus),
|
||||
boolc(|r| r.dropped),
|
||||
];
|
||||
|
||||
RecordBatch::try_new(Arc::new(schema()), columns).map_err(|e| {
|
||||
Error::Other(format!(
|
||||
"internal error building a Parquet batch: {e}. This means the Arrow schema and the \
|
||||
column builders in store_parquet.rs have drifted apart"
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
/// Writes flat responses to a Parquet file.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `path` - the destination.
|
||||
/// * `rows` - the rows.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`Error::Io`] on a write failure and [`Error::Other`] when the
|
||||
/// Parquet writer rejects the batch.
|
||||
pub fn write(path: &Path, rows: &[FlatResponse]) -> Result<()> {
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent).map_err(|e| Error::io(parent, e))?;
|
||||
}
|
||||
let batch = to_batch(rows)?;
|
||||
let file = std::fs::File::create(path).map_err(|e| Error::io(path, e))?;
|
||||
|
||||
// Snappy rather than zstd: it is the format's most universally readable
|
||||
// codec, and these files are small enough that the ratio difference is
|
||||
// irrelevant next to being openable by an old pandas.
|
||||
let props = WriterProperties::builder()
|
||||
.set_compression(Compression::SNAPPY)
|
||||
.build();
|
||||
|
||||
let mut writer = ArrowWriter::try_new(file, Arc::new(schema()), Some(props))
|
||||
.map_err(|e| Error::Other(format!("cannot open {} for Parquet: {e}", path.display())))?;
|
||||
writer
|
||||
.write(&batch)
|
||||
.map_err(|e| Error::Other(format!("cannot write {}: {e}", path.display())))?;
|
||||
writer
|
||||
.close()
|
||||
.map_err(|e| Error::Other(format!("cannot finish {}: {e}", path.display())))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reads flat responses from a Parquet file.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `path` - the file.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The rows.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`Error::Io`] when the file cannot be opened and [`Error::Other`]
|
||||
/// when its schema does not match what this crate writes.
|
||||
pub fn read(path: &Path) -> Result<Vec<FlatResponse>> {
|
||||
let file = std::fs::File::open(path).map_err(|e| Error::io(path, e))?;
|
||||
let builder = ParquetRecordBatchReaderBuilder::try_new(file)
|
||||
.map_err(|e| Error::Other(format!("cannot read {} as Parquet: {e}", path.display())))?;
|
||||
let reader = builder
|
||||
.build()
|
||||
.map_err(|e| Error::Other(format!("cannot read {}: {e}", path.display())))?;
|
||||
|
||||
let mut out = Vec::new();
|
||||
for batch in reader {
|
||||
let batch =
|
||||
batch.map_err(|e| Error::Other(format!("cannot read {}: {e}", path.display())))?;
|
||||
out.extend(from_batch(&batch, path)?);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Converts a record batch back into flat responses.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `batch` - the batch.
|
||||
/// * `path` - the source file, for error messages.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The rows.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`Error::Other`] when a column is missing or has the wrong type.
|
||||
fn from_batch(batch: &RecordBatch, path: &Path) -> Result<Vec<FlatResponse>> {
|
||||
let strings = |name: &str| -> Result<&StringArray> {
|
||||
batch
|
||||
.column_by_name(name)
|
||||
.and_then(|c| c.as_any().downcast_ref::<StringArray>())
|
||||
.ok_or_else(|| column_error(name, "string", path))
|
||||
};
|
||||
let floats = |name: &str| -> Result<&Float64Array> {
|
||||
batch
|
||||
.column_by_name(name)
|
||||
.and_then(|c| c.as_any().downcast_ref::<Float64Array>())
|
||||
.ok_or_else(|| column_error(name, "float64", path))
|
||||
};
|
||||
let uints = |name: &str| -> Result<&UInt32Array> {
|
||||
batch
|
||||
.column_by_name(name)
|
||||
.and_then(|c| c.as_any().downcast_ref::<UInt32Array>())
|
||||
.ok_or_else(|| column_error(name, "uint32", path))
|
||||
};
|
||||
let bools = |name: &str| -> Result<&BooleanArray> {
|
||||
batch
|
||||
.column_by_name(name)
|
||||
.and_then(|c| c.as_any().downcast_ref::<BooleanArray>())
|
||||
.ok_or_else(|| column_error(name, "boolean", path))
|
||||
};
|
||||
|
||||
let administration_id = strings("administration_id")?;
|
||||
let course = strings("course")?;
|
||||
let term = strings("term")?;
|
||||
let assessment_id = strings("assessment_id")?;
|
||||
let date = strings("date")?;
|
||||
let form = strings("form")?;
|
||||
let student_key = strings("student_key")?;
|
||||
let sid = strings("sid")?;
|
||||
let email = strings("email")?;
|
||||
let section = strings("section")?;
|
||||
let item_number = uints("item_number")?;
|
||||
let item_ref = strings("item_ref")?;
|
||||
let item_version = uints("item_version")?;
|
||||
let selected = strings("selected")?;
|
||||
let eliminated = strings("eliminated")?;
|
||||
let correct = strings("correct")?;
|
||||
let credit = floats("credit")?;
|
||||
let points_possible = floats("points_possible")?;
|
||||
let score = floats("score")?;
|
||||
let response_time_seconds = strings("response_time_seconds")?;
|
||||
let level = uints("level")?;
|
||||
let learning_objectives = strings("learning_objectives")?;
|
||||
let topics = strings("topics")?;
|
||||
let bonus = bools("bonus")?;
|
||||
let dropped = bools("dropped")?;
|
||||
|
||||
let mut out = Vec::with_capacity(batch.num_rows());
|
||||
for i in 0..batch.num_rows() {
|
||||
out.push(FlatResponse {
|
||||
administration_id: administration_id.value(i).to_string(),
|
||||
course: course.value(i).to_string(),
|
||||
term: term.value(i).to_string(),
|
||||
assessment_id: assessment_id.value(i).to_string(),
|
||||
date: date.value(i).to_string(),
|
||||
form: form.value(i).to_string(),
|
||||
student_key: student_key.value(i).to_string(),
|
||||
sid: sid.value(i).to_string(),
|
||||
email: email.value(i).to_string(),
|
||||
section: section.value(i).to_string(),
|
||||
item_number: item_number.value(i),
|
||||
item_ref: item_ref.value(i).to_string(),
|
||||
item_version: item_version.value(i),
|
||||
selected: selected.value(i).to_string(),
|
||||
eliminated: eliminated.value(i).to_string(),
|
||||
correct: correct.value(i).to_string(),
|
||||
credit: credit.value(i),
|
||||
points_possible: points_possible.value(i),
|
||||
score: score.value(i),
|
||||
response_time_seconds: response_time_seconds.value(i).to_string(),
|
||||
level: level.value(i) as u8,
|
||||
learning_objectives: learning_objectives.value(i).to_string(),
|
||||
topics: topics.value(i).to_string(),
|
||||
bonus: bonus.value(i),
|
||||
dropped: dropped.value(i),
|
||||
});
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Builds the error for a missing or mistyped column.
|
||||
fn column_error(name: &str, expected: &str, path: &Path) -> Error {
|
||||
Error::Other(format!(
|
||||
"{} is missing the `{name}` column or it is not {expected}; this file was probably not \
|
||||
written by coursebank",
|
||||
path.display()
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn flat(student: &str, number: u32) -> FlatResponse {
|
||||
FlatResponse {
|
||||
administration_id: "C/2026s/e1".into(),
|
||||
course: "C".into(),
|
||||
term: "2026s".into(),
|
||||
assessment_id: "e1".into(),
|
||||
date: "2026-04-01".into(),
|
||||
form: String::new(),
|
||||
student_key: student.into(),
|
||||
sid: "1234567".into(),
|
||||
email: String::new(),
|
||||
section: "L01".into(),
|
||||
item_number: number,
|
||||
item_ref: "bank::q-a-001".into(),
|
||||
item_version: 3,
|
||||
selected: "C".into(),
|
||||
eliminated: String::new(),
|
||||
correct: "1".into(),
|
||||
credit: 1.0,
|
||||
points_possible: 1.5,
|
||||
score: 1.5,
|
||||
response_time_seconds: "42.0".into(),
|
||||
level: 3,
|
||||
learning_objectives: "lo-a,lo-b".into(),
|
||||
topics: "kinetics".into(),
|
||||
bonus: false,
|
||||
dropped: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn schema_matches_the_column_builders() {
|
||||
let rows = vec![flat("s1", 1)];
|
||||
let batch = to_batch(&rows).expect("schema and builders agree");
|
||||
assert_eq!(batch.num_columns(), schema().fields().len());
|
||||
assert_eq!(batch.num_rows(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn round_trips_through_a_file() {
|
||||
let dir = std::env::temp_dir().join(format!("cb-pq-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let path = dir.join("e1.parquet");
|
||||
|
||||
let rows = vec![flat("s1", 1), flat("s2", 2)];
|
||||
write(&path, &rows).unwrap();
|
||||
let back = read(&path).unwrap();
|
||||
|
||||
assert_eq!(back.len(), 2);
|
||||
assert_eq!(back[0].student_key, "s1");
|
||||
assert_eq!(back[1].item_number, 2);
|
||||
assert_eq!(back[0].learning_objectives, "lo-a,lo-b");
|
||||
assert_eq!(back[0].credit, 1.0);
|
||||
assert_eq!(back[0].level, 3);
|
||||
assert!(!back[0].bonus);
|
||||
|
||||
std::fs::remove_dir_all(&dir).ok();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_batch_is_writable() {
|
||||
let dir = std::env::temp_dir().join(format!("cb-pq-empty-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let path = dir.join("empty.parquet");
|
||||
write(&path, &[]).unwrap();
|
||||
assert_eq!(read(&path).unwrap().len(), 0);
|
||||
std::fs::remove_dir_all(&dir).ok();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_foreign_file_is_rejected_clearly() {
|
||||
let dir = std::env::temp_dir().join(format!("cb-pq-bad-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let path = dir.join("not.parquet");
|
||||
std::fs::write(&path, b"this is not parquet").unwrap();
|
||||
let err = read(&path).unwrap_err();
|
||||
assert!(err.to_string().contains("as Parquet"));
|
||||
std::fs::remove_dir_all(&dir).ok();
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user