// SPDX-License-Identifier: Prosperity-3.0.0 // Copyright Scienting // Source: https://git.scient.ing/education/coursebank //! Parquet reading and writing, isolated so the rest of the crate never touches //! Arrow types. //! //! Every use of `arrow` and `parquet` in this crate is in this file. That is //! deliberate: those two crates move fast and release breaking versions together, //! and confining them to one module means a version bump is a single-file edit //! rather than a refactor. It also means `--no-default-features` compiles the //! whole tool without them. //! //! The schema mirrors [`FlatResponse`] field for field. Optional values are //! written with the same sentinels the CSV path uses — empty string, `0` for an //! unknown level — rather than nulls, so that a Parquet file and a CSV file of //! the same data load identically in pandas. use std::path::Path; use std::sync::Arc; use arrow_array::{ArrayRef, BooleanArray, Float64Array, RecordBatch, StringArray, UInt32Array}; use arrow_schema::{DataType, Field, Schema}; use parquet::arrow::ArrowWriter; use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; use parquet::basic::Compression; use parquet::file::properties::WriterProperties; use crate::error::{Error, Result}; use crate::responses::FlatResponse; /// The Arrow schema for stored responses. /// /// # Returns /// /// The schema, whose field order matches [`FlatResponse`]. pub fn schema() -> Schema { Schema::new(vec![ Field::new("administration_id", DataType::Utf8, false), Field::new("course", DataType::Utf8, false), Field::new("term", DataType::Utf8, false), Field::new("assessment_id", DataType::Utf8, false), Field::new("date", DataType::Utf8, false), Field::new("form", DataType::Utf8, false), Field::new("student_key", DataType::Utf8, false), Field::new("sid", DataType::Utf8, false), Field::new("email", DataType::Utf8, false), Field::new("section", DataType::Utf8, false), Field::new("item_number", DataType::UInt32, false), Field::new("item_ref", DataType::Utf8, false), Field::new("item_version", DataType::UInt32, false), Field::new("variant", DataType::Utf8, false), Field::new("selected", DataType::Utf8, false), Field::new("eliminated", DataType::Utf8, false), Field::new("correct", DataType::Utf8, false), Field::new("credit", DataType::Float64, false), Field::new("points_possible", DataType::Float64, false), Field::new("score", DataType::Float64, false), Field::new("response_time_seconds", DataType::Utf8, false), Field::new("level", DataType::UInt32, false), // The stored column name predates the objective/target rename and is left // alone: an existing parquet file has to keep loading. Field::new("learning_objectives", DataType::Utf8, false), Field::new("topics", DataType::Utf8, false), Field::new("bonus", DataType::Boolean, false), Field::new("dropped", DataType::Boolean, false), Field::new("form_position", DataType::UInt32, false), Field::new("dropped_full_credit", DataType::Boolean, false), Field::new("selected_source", DataType::Utf8, false), Field::new("eliminated_source", DataType::Utf8, false), ]) } /// Builds a record batch from flat responses. /// /// # Arguments /// /// * `rows` - the rows. /// /// # Returns /// /// The batch. /// /// # Errors /// /// Returns [`Error::Other`] when Arrow rejects the column set, which would mean /// this function and [`schema`] have drifted apart. fn to_batch(rows: &[FlatResponse]) -> Result { let s = |f: fn(&FlatResponse) -> &str| -> ArrayRef { Arc::new(StringArray::from(rows.iter().map(f).collect::>())) }; let f64c = |f: fn(&FlatResponse) -> f64| -> ArrayRef { Arc::new(Float64Array::from(rows.iter().map(f).collect::>())) }; let u32c = |f: fn(&FlatResponse) -> u32| -> ArrayRef { Arc::new(UInt32Array::from(rows.iter().map(f).collect::>())) }; let boolc = |f: fn(&FlatResponse) -> bool| -> ArrayRef { Arc::new(BooleanArray::from( rows.iter().map(f).collect::>(), )) }; let columns: Vec = vec![ s(|r| &r.administration_id), s(|r| &r.course), s(|r| &r.term), s(|r| &r.assessment_id), s(|r| &r.date), s(|r| &r.form), s(|r| &r.student_key), s(|r| &r.sid), s(|r| &r.email), s(|r| &r.section), u32c(|r| r.item_number), s(|r| &r.item_ref), u32c(|r| r.item_version), s(|r| &r.variant), s(|r| &r.selected), s(|r| &r.eliminated), s(|r| &r.correct), f64c(|r| r.credit), f64c(|r| r.points_possible), f64c(|r| r.score), s(|r| &r.response_time_seconds), u32c(|r| r.level as u32), s(|r| &r.learning_targets), s(|r| &r.topics), boolc(|r| r.bonus), boolc(|r| r.dropped), u32c(|r| r.form_position), boolc(|r| r.dropped_full_credit), s(|r| &r.selected_source), s(|r| &r.eliminated_source), ]; RecordBatch::try_new(Arc::new(schema()), columns).map_err(|e| { Error::Other(format!( "internal error building a Parquet batch: {e}. This means the Arrow schema and the \ column builders in store_parquet.rs have drifted apart" )) }) } /// Writes flat responses to a Parquet file. /// /// # Arguments /// /// * `path` - the destination. /// * `rows` - the rows. /// /// # Errors /// /// Returns [`Error::Io`] on a write failure and [`Error::Other`] when the /// Parquet writer rejects the batch. pub fn write(path: &Path, rows: &[FlatResponse]) -> Result<()> { if let Some(parent) = path.parent() { std::fs::create_dir_all(parent).map_err(|e| Error::io(parent, e))?; } let batch = to_batch(rows)?; let file = std::fs::File::create(path).map_err(|e| Error::io(path, e))?; // Snappy rather than zstd: it is the format's most universally readable // codec, and these files are small enough that the ratio difference is // irrelevant next to being openable by an old pandas. let props = WriterProperties::builder() .set_compression(Compression::SNAPPY) .build(); let mut writer = ArrowWriter::try_new(file, Arc::new(schema()), Some(props)) .map_err(|e| Error::Other(format!("cannot open {} for Parquet: {e}", path.display())))?; writer .write(&batch) .map_err(|e| Error::Other(format!("cannot write {}: {e}", path.display())))?; writer .close() .map_err(|e| Error::Other(format!("cannot finish {}: {e}", path.display())))?; Ok(()) } /// Reads flat responses from a Parquet file. /// /// # Arguments /// /// * `path` - the file. /// /// # Returns /// /// The rows. /// /// # Errors /// /// Returns [`Error::Io`] when the file cannot be opened and [`Error::Other`] /// when its schema does not match what this crate writes. pub fn read(path: &Path) -> Result> { let file = std::fs::File::open(path).map_err(|e| Error::io(path, e))?; let builder = ParquetRecordBatchReaderBuilder::try_new(file) .map_err(|e| Error::Other(format!("cannot read {} as Parquet: {e}", path.display())))?; let reader = builder .build() .map_err(|e| Error::Other(format!("cannot read {}: {e}", path.display())))?; let mut out = Vec::new(); for batch in reader { let batch = batch.map_err(|e| Error::Other(format!("cannot read {}: {e}", path.display())))?; out.extend(from_batch(&batch, path)?); } Ok(out) } /// Converts a record batch back into flat responses. /// /// # Arguments /// /// * `batch` - the batch. /// * `path` - the source file, for error messages. /// /// # Returns /// /// The rows. /// /// # Errors /// /// Returns [`Error::Other`] when a column is missing or has the wrong type. fn from_batch(batch: &RecordBatch, path: &Path) -> Result> { let strings = |name: &str| -> Result<&StringArray> { batch .column_by_name(name) .and_then(|c| c.as_any().downcast_ref::()) .ok_or_else(|| column_error(name, "string", path)) }; let floats = |name: &str| -> Result<&Float64Array> { batch .column_by_name(name) .and_then(|c| c.as_any().downcast_ref::()) .ok_or_else(|| column_error(name, "float64", path)) }; let uints = |name: &str| -> Result<&UInt32Array> { batch .column_by_name(name) .and_then(|c| c.as_any().downcast_ref::()) .ok_or_else(|| column_error(name, "uint32", path)) }; let bools = |name: &str| -> Result<&BooleanArray> { batch .column_by_name(name) .and_then(|c| c.as_any().downcast_ref::()) .ok_or_else(|| column_error(name, "boolean", path)) }; // The three decode columns are read optionally rather than required, so a // file written before seals existed still loads. A missing column is not a // foreign file; it is last term's data. let optional_strings = |name: &str| -> Option<&StringArray> { batch .column_by_name(name) .and_then(|c| c.as_any().downcast_ref::()) }; let optional_uints = |name: &str| -> Option<&UInt32Array> { batch .column_by_name(name) .and_then(|c| c.as_any().downcast_ref::()) }; let optional_bools = |name: &str| -> Option<&BooleanArray> { batch .column_by_name(name) .and_then(|c| c.as_any().downcast_ref::()) }; let administration_id = strings("administration_id")?; let course = strings("course")?; let term = strings("term")?; let assessment_id = strings("assessment_id")?; let date = strings("date")?; let form = strings("form")?; let student_key = strings("student_key")?; let sid = strings("sid")?; let email = strings("email")?; let section = strings("section")?; let item_number = uints("item_number")?; let item_ref = strings("item_ref")?; let item_version = uints("item_version")?; // Added after the first stores were written, so absent rather than fatal in // a file from before 2.0; `coursebank migrate variants` fills it in. let variant = optional_strings("variant"); let selected = strings("selected")?; let eliminated = strings("eliminated")?; let correct = strings("correct")?; let credit = floats("credit")?; let points_possible = floats("points_possible")?; let score = floats("score")?; let response_time_seconds = strings("response_time_seconds")?; let level = uints("level")?; let learning_targets = strings("learning_objectives")?; let topics = strings("topics")?; let bonus = bools("bonus")?; let dropped = bools("dropped")?; let form_position = optional_uints("form_position"); let dropped_full_credit = optional_bools("dropped_full_credit"); let selected_source = optional_strings("selected_source"); let eliminated_source = optional_strings("eliminated_source"); let mut out = Vec::with_capacity(batch.num_rows()); for i in 0..batch.num_rows() { out.push(FlatResponse { administration_id: administration_id.value(i).to_string(), course: course.value(i).to_string(), term: term.value(i).to_string(), assessment_id: assessment_id.value(i).to_string(), date: date.value(i).to_string(), form: form.value(i).to_string(), form_position: form_position.map(|a| a.value(i)).unwrap_or(0), student_key: student_key.value(i).to_string(), sid: sid.value(i).to_string(), email: email.value(i).to_string(), section: section.value(i).to_string(), item_number: item_number.value(i), item_ref: item_ref.value(i).to_string(), item_version: item_version.value(i), variant: variant.map(|c| c.value(i).to_string()).unwrap_or_default(), selected: selected.value(i).to_string(), selected_source: selected_source .map(|a| a.value(i).to_string()) .unwrap_or_default(), eliminated: eliminated.value(i).to_string(), eliminated_source: eliminated_source .map(|a| a.value(i).to_string()) .unwrap_or_default(), correct: correct.value(i).to_string(), credit: credit.value(i), points_possible: points_possible.value(i), score: score.value(i), response_time_seconds: response_time_seconds.value(i).to_string(), level: level.value(i) as u8, learning_targets: learning_targets.value(i).to_string(), topics: topics.value(i).to_string(), bonus: bonus.value(i), dropped: dropped.value(i), dropped_full_credit: dropped_full_credit.map(|a| a.value(i)).unwrap_or(false), }); } Ok(out) } /// Builds the error for a missing or mistyped column. fn column_error(name: &str, expected: &str, path: &Path) -> Error { Error::Other(format!( "{} is missing the `{name}` column or it is not {expected}; this file was probably not \ written by coursebank", path.display() )) } #[cfg(test)] mod tests { use super::*; fn flat(student: &str, number: u32) -> FlatResponse { FlatResponse { administration_id: "C/2026s/e1".into(), course: "C".into(), term: "2026s".into(), assessment_id: "e1".into(), date: "2026-04-01".into(), form: String::new(), student_key: student.into(), sid: "1234567".into(), email: String::new(), section: "L01".into(), item_number: number, item_ref: "bank::q-a-001".into(), item_version: 3, variant: "4c81fa".into(), selected: "C".into(), eliminated: String::new(), correct: "1".into(), credit: 1.0, points_possible: 1.5, score: 1.5, response_time_seconds: "42.0".into(), level: 3, learning_targets: "lo-a,lo-b".into(), topics: "kinetics".into(), bonus: false, dropped: false, dropped_full_credit: false, form_position: 0, selected_source: String::new(), eliminated_source: String::new(), } } #[test] fn schema_matches_the_column_builders() { let rows = vec![flat("s1", 1)]; let batch = to_batch(&rows).expect("schema and builders agree"); assert_eq!(batch.num_columns(), schema().fields().len()); assert_eq!(batch.num_rows(), 1); } #[test] fn round_trips_through_a_file() { let dir = std::env::temp_dir().join(format!("cb-pq-{}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); let path = dir.join("e1.parquet"); let rows = vec![flat("s1", 1), flat("s2", 2)]; write(&path, &rows).unwrap(); let back = read(&path).unwrap(); assert_eq!(back.len(), 2); assert_eq!(back[0].student_key, "s1"); assert_eq!(back[1].item_number, 2); assert_eq!(back[0].learning_targets, "lo-a,lo-b"); assert_eq!(back[0].credit, 1.0); assert_eq!(back[0].level, 3); assert!(!back[0].bonus); std::fs::remove_dir_all(&dir).ok(); } #[test] fn an_empty_batch_is_writable() { let dir = std::env::temp_dir().join(format!("cb-pq-empty-{}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); let path = dir.join("empty.parquet"); write(&path, &[]).unwrap(); assert_eq!(read(&path).unwrap().len(), 0); std::fs::remove_dir_all(&dir).ok(); } #[test] fn a_foreign_file_is_rejected_clearly() { let dir = std::env::temp_dir().join(format!("cb-pq-bad-{}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); let path = dir.join("not.parquet"); std::fs::write(&path, b"this is not parquet").unwrap(); let err = read(&path).unwrap_err(); assert!(err.to_string().contains("as Parquet")); std::fs::remove_dir_all(&dir).ok(); } }