Dev (#1)
Sync README to GitHub / sync (push) Successful in 12s
CI / check (push) Successful in 8m31s
Deploy docs / deploy (push) Successful in 6m44s
Nightly / nightly (push) Successful in 10m33s

Reviewed-on: #1
This commit was merged in pull request #1.
This commit is contained in:
alexm committed 2026-08-07 15:48:11 -04:00
1 parent 228a0da47f
commit abc0bdf621
79 files changed
+31370

No files matched your search

+391
View File
@@ -0,0 +1,391 @@
// SPDX-License-Identifier: Prosperity-3.0.0
// Copyright Scientific Computing Studio
// Source: https://git.scient.ing/education/coursebank
//! Parquet reading and writing, isolated so the rest of the crate never touches
//! Arrow types.
//!
//! Every use of `arrow` and `parquet` in this crate is in this file. That is
//! deliberate: those two crates move fast and release breaking versions together,
//! and confining them to one module means a version bump is a single-file edit
//! rather than a refactor. It also means `--no-default-features` compiles the
//! whole tool without them.
//!
//! The schema mirrors [`FlatResponse`] field for field. Optional values are
//! written with the same sentinels the CSV path uses — empty string, `0` for an
//! unknown level — rather than nulls, so that a Parquet file and a CSV file of
//! the same data load identically in pandas.
use std::path::Path;
use std::sync::Arc;
use arrow_array::{ArrayRef, BooleanArray, Float64Array, RecordBatch, StringArray, UInt32Array};
use arrow_schema::{DataType, Field, Schema};
use parquet::arrow::ArrowWriter;
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
use parquet::basic::Compression;
use parquet::file::properties::WriterProperties;
use crate::error::{Error, Result};
use crate::responses::FlatResponse;
/// The Arrow schema for stored responses.
///
/// # Returns
///
/// The schema, whose field order matches [`FlatResponse`].
pub fn schema() -> Schema {
Schema::new(vec![
Field::new("administration_id", DataType::Utf8, false),
Field::new("course", DataType::Utf8, false),
Field::new("term", DataType::Utf8, false),
Field::new("assessment_id", DataType::Utf8, false),
Field::new("date", DataType::Utf8, false),
Field::new("form", DataType::Utf8, false),
Field::new("student_key", DataType::Utf8, false),
Field::new("sid", DataType::Utf8, false),
Field::new("email", DataType::Utf8, false),
Field::new("section", DataType::Utf8, false),
Field::new("item_number", DataType::UInt32, false),
Field::new("item_ref", DataType::Utf8, false),
Field::new("item_version", DataType::UInt32, false),
Field::new("selected", DataType::Utf8, false),
Field::new("eliminated", DataType::Utf8, false),
Field::new("correct", DataType::Utf8, false),
Field::new("credit", DataType::Float64, false),
Field::new("points_possible", DataType::Float64, false),
Field::new("score", DataType::Float64, false),
Field::new("response_time_seconds", DataType::Utf8, false),
Field::new("level", DataType::UInt32, false),
Field::new("learning_objectives", DataType::Utf8, false),
Field::new("topics", DataType::Utf8, false),
Field::new("bonus", DataType::Boolean, false),
Field::new("dropped", DataType::Boolean, false),
])
}
/// Builds a record batch from flat responses.
///
/// # Arguments
///
/// * `rows` - the rows.
///
/// # Returns
///
/// The batch.
///
/// # Errors
///
/// Returns [`Error::Other`] when Arrow rejects the column set, which would mean
/// this function and [`schema`] have drifted apart.
fn to_batch(rows: &[FlatResponse]) -> Result<RecordBatch> {
let s = |f: fn(&FlatResponse) -> &str| -> ArrayRef {
Arc::new(StringArray::from(rows.iter().map(f).collect::<Vec<&str>>()))
};
let f64c = |f: fn(&FlatResponse) -> f64| -> ArrayRef {
Arc::new(Float64Array::from(rows.iter().map(f).collect::<Vec<f64>>()))
};
let u32c = |f: fn(&FlatResponse) -> u32| -> ArrayRef {
Arc::new(UInt32Array::from(rows.iter().map(f).collect::<Vec<u32>>()))
};
let boolc = |f: fn(&FlatResponse) -> bool| -> ArrayRef {
Arc::new(BooleanArray::from(
rows.iter().map(f).collect::<Vec<bool>>(),
))
};
let columns: Vec<ArrayRef> = vec![
s(|r| &r.administration_id),
s(|r| &r.course),
s(|r| &r.term),
s(|r| &r.assessment_id),
s(|r| &r.date),
s(|r| &r.form),
s(|r| &r.student_key),
s(|r| &r.sid),
s(|r| &r.email),
s(|r| &r.section),
u32c(|r| r.item_number),
s(|r| &r.item_ref),
u32c(|r| r.item_version),
s(|r| &r.selected),
s(|r| &r.eliminated),
s(|r| &r.correct),
f64c(|r| r.credit),
f64c(|r| r.points_possible),
f64c(|r| r.score),
s(|r| &r.response_time_seconds),
u32c(|r| r.level as u32),
s(|r| &r.learning_objectives),
s(|r| &r.topics),
boolc(|r| r.bonus),
boolc(|r| r.dropped),
];
RecordBatch::try_new(Arc::new(schema()), columns).map_err(|e| {
Error::Other(format!(
"internal error building a Parquet batch: {e}. This means the Arrow schema and the \
column builders in store_parquet.rs have drifted apart"
))
})
}
/// Writes flat responses to a Parquet file.
///
/// # Arguments
///
/// * `path` - the destination.
/// * `rows` - the rows.
///
/// # Errors
///
/// Returns [`Error::Io`] on a write failure and [`Error::Other`] when the
/// Parquet writer rejects the batch.
pub fn write(path: &Path, rows: &[FlatResponse]) -> Result<()> {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent).map_err(|e| Error::io(parent, e))?;
}
let batch = to_batch(rows)?;
let file = std::fs::File::create(path).map_err(|e| Error::io(path, e))?;
// Snappy rather than zstd: it is the format's most universally readable
// codec, and these files are small enough that the ratio difference is
// irrelevant next to being openable by an old pandas.
let props = WriterProperties::builder()
.set_compression(Compression::SNAPPY)
.build();
let mut writer = ArrowWriter::try_new(file, Arc::new(schema()), Some(props))
.map_err(|e| Error::Other(format!("cannot open {} for Parquet: {e}", path.display())))?;
writer
.write(&batch)
.map_err(|e| Error::Other(format!("cannot write {}: {e}", path.display())))?;
writer
.close()
.map_err(|e| Error::Other(format!("cannot finish {}: {e}", path.display())))?;
Ok(())
}
/// Reads flat responses from a Parquet file.
///
/// # Arguments
///
/// * `path` - the file.
///
/// # Returns
///
/// The rows.
///
/// # Errors
///
/// Returns [`Error::Io`] when the file cannot be opened and [`Error::Other`]
/// when its schema does not match what this crate writes.
pub fn read(path: &Path) -> Result<Vec<FlatResponse>> {
let file = std::fs::File::open(path).map_err(|e| Error::io(path, e))?;
let builder = ParquetRecordBatchReaderBuilder::try_new(file)
.map_err(|e| Error::Other(format!("cannot read {} as Parquet: {e}", path.display())))?;
let reader = builder
.build()
.map_err(|e| Error::Other(format!("cannot read {}: {e}", path.display())))?;
let mut out = Vec::new();
for batch in reader {
let batch =
batch.map_err(|e| Error::Other(format!("cannot read {}: {e}", path.display())))?;
out.extend(from_batch(&batch, path)?);
}
Ok(out)
}
/// Converts a record batch back into flat responses.
///
/// # Arguments
///
/// * `batch` - the batch.
/// * `path` - the source file, for error messages.
///
/// # Returns
///
/// The rows.
///
/// # Errors
///
/// Returns [`Error::Other`] when a column is missing or has the wrong type.
fn from_batch(batch: &RecordBatch, path: &Path) -> Result<Vec<FlatResponse>> {
let strings = |name: &str| -> Result<&StringArray> {
batch
.column_by_name(name)
.and_then(|c| c.as_any().downcast_ref::<StringArray>())
.ok_or_else(|| column_error(name, "string", path))
};
let floats = |name: &str| -> Result<&Float64Array> {
batch
.column_by_name(name)
.and_then(|c| c.as_any().downcast_ref::<Float64Array>())
.ok_or_else(|| column_error(name, "float64", path))
};
let uints = |name: &str| -> Result<&UInt32Array> {
batch
.column_by_name(name)
.and_then(|c| c.as_any().downcast_ref::<UInt32Array>())
.ok_or_else(|| column_error(name, "uint32", path))
};
let bools = |name: &str| -> Result<&BooleanArray> {
batch
.column_by_name(name)
.and_then(|c| c.as_any().downcast_ref::<BooleanArray>())
.ok_or_else(|| column_error(name, "boolean", path))
};
let administration_id = strings("administration_id")?;
let course = strings("course")?;
let term = strings("term")?;
let assessment_id = strings("assessment_id")?;
let date = strings("date")?;
let form = strings("form")?;
let student_key = strings("student_key")?;
let sid = strings("sid")?;
let email = strings("email")?;
let section = strings("section")?;
let item_number = uints("item_number")?;
let item_ref = strings("item_ref")?;
let item_version = uints("item_version")?;
let selected = strings("selected")?;
let eliminated = strings("eliminated")?;
let correct = strings("correct")?;
let credit = floats("credit")?;
let points_possible = floats("points_possible")?;
let score = floats("score")?;
let response_time_seconds = strings("response_time_seconds")?;
let level = uints("level")?;
let learning_objectives = strings("learning_objectives")?;
let topics = strings("topics")?;
let bonus = bools("bonus")?;
let dropped = bools("dropped")?;
let mut out = Vec::with_capacity(batch.num_rows());
for i in 0..batch.num_rows() {
out.push(FlatResponse {
administration_id: administration_id.value(i).to_string(),
course: course.value(i).to_string(),
term: term.value(i).to_string(),
assessment_id: assessment_id.value(i).to_string(),
date: date.value(i).to_string(),
form: form.value(i).to_string(),
student_key: student_key.value(i).to_string(),
sid: sid.value(i).to_string(),
email: email.value(i).to_string(),
section: section.value(i).to_string(),
item_number: item_number.value(i),
item_ref: item_ref.value(i).to_string(),
item_version: item_version.value(i),
selected: selected.value(i).to_string(),
eliminated: eliminated.value(i).to_string(),
correct: correct.value(i).to_string(),
credit: credit.value(i),
points_possible: points_possible.value(i),
score: score.value(i),
response_time_seconds: response_time_seconds.value(i).to_string(),
level: level.value(i) as u8,
learning_objectives: learning_objectives.value(i).to_string(),
topics: topics.value(i).to_string(),
bonus: bonus.value(i),
dropped: dropped.value(i),
});
}
Ok(out)
}
/// Builds the error for a missing or mistyped column.
fn column_error(name: &str, expected: &str, path: &Path) -> Error {
Error::Other(format!(
"{} is missing the `{name}` column or it is not {expected}; this file was probably not \
written by coursebank",
path.display()
))
}
#[cfg(test)]
mod tests {
use super::*;
fn flat(student: &str, number: u32) -> FlatResponse {
FlatResponse {
administration_id: "C/2026s/e1".into(),
course: "C".into(),
term: "2026s".into(),
assessment_id: "e1".into(),
date: "2026-04-01".into(),
form: String::new(),
student_key: student.into(),
sid: "1234567".into(),
email: String::new(),
section: "L01".into(),
item_number: number,
item_ref: "bank::q-a-001".into(),
item_version: 3,
selected: "C".into(),
eliminated: String::new(),
correct: "1".into(),
credit: 1.0,
points_possible: 1.5,
score: 1.5,
response_time_seconds: "42.0".into(),
level: 3,
learning_objectives: "lo-a,lo-b".into(),
topics: "kinetics".into(),
bonus: false,
dropped: false,
}
}
#[test]
fn schema_matches_the_column_builders() {
let rows = vec![flat("s1", 1)];
let batch = to_batch(&rows).expect("schema and builders agree");
assert_eq!(batch.num_columns(), schema().fields().len());
assert_eq!(batch.num_rows(), 1);
}
#[test]
fn round_trips_through_a_file() {
let dir = std::env::temp_dir().join(format!("cb-pq-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("e1.parquet");
let rows = vec![flat("s1", 1), flat("s2", 2)];
write(&path, &rows).unwrap();
let back = read(&path).unwrap();
assert_eq!(back.len(), 2);
assert_eq!(back[0].student_key, "s1");
assert_eq!(back[1].item_number, 2);
assert_eq!(back[0].learning_objectives, "lo-a,lo-b");
assert_eq!(back[0].credit, 1.0);
assert_eq!(back[0].level, 3);
assert!(!back[0].bonus);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn an_empty_batch_is_writable() {
let dir = std::env::temp_dir().join(format!("cb-pq-empty-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("empty.parquet");
write(&path, &[]).unwrap();
assert_eq!(read(&path).unwrap().len(), 0);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn a_foreign_file_is_rejected_clearly() {
let dir = std::env::temp_dir().join(format!("cb-pq-bad-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("not.parquet");
std::fs::write(&path, b"this is not parquet").unwrap();
let err = read(&path).unwrap_err();
assert!(err.to_string().contains("as Parquet"));
std::fs::remove_dir_all(&dir).ok();
}
}