// SPDX-License-Identifier: Prosperity-3.0.0 // Copyright Scienting // Source: https://git.scient.ing/education/coursebank //! The sealed administration: what the students actually held in their hands. //! //! An assessment record says which item sat at question 14. It does not say what //! that item *said*, and it does not say that on form B question 14's option C was //! the bank's option A. Both facts are derivable — the first from the bank, the //! second from the form seed — right up until the bank changes, and then they are //! silently derivable to the wrong answer. //! //! A seal freezes them. It is written once, before the exam is administered, and //! it holds three things the record does not: //! //! * **The content as administered.** Stem and option text, verbatim, with a //! fingerprint on each. Reword a distractor next term and the seal still shows //! what this cohort was asked. //! * **The permutation, expanded.** For every form and every printed question, the //! map from printed letter to bank letter, plus the key as printed. This is what //! turns "the student chose C" into "the student chose the bank's option A", //! which is the only form of that sentence worth storing. //! * **Digests over both.** One per form, one per item, and one over the whole //! file. Re-derive any of them and compare; a mismatch names what moved. //! //! The seal is not a cache. Nothing reads it for speed. It exists so that the //! question "is what I am about to analyze the same thing the students saw?" has //! an answer that does not depend on the bank having stayed still. //! //! # Where it lives //! //! `seals/.yaml`, in its own directory rather than beside the //! assessment record, because [`crate::assessment::AssessmentFile::load_all`] //! parses every `.yaml` in `assessments/` and a seal is not an assessment. //! //! # The order of operations //! //! ```text //! assemble ──▶ export typst ──▶ seal ──▶ print ──▶ administer ──▶ ingest //! │ │ //! └────────── verified against ──────┘ //! ``` //! //! Seal after exporting and before printing. Sealing earlier is harmless; //! sealing after ingest is not, because a seal written from a drifted bank //! records the drift as truth. use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; use crate::assessment::{AssessmentFile, Form, Placement}; use crate::catalog::{Catalog, Severity}; use crate::course::SCHEMA_VERSION; use crate::date::Date; use crate::error::{Error, Result}; use crate::hash::{fingerprint, hex, sha256}; use crate::item::Item; use crate::layout::Layout; use crate::select; use crate::taxonomy::Level; use crate::yaml; /// The directory holding seals, relative to the course root. pub const SEAL_DIR: &str = "seals"; /// The prefix on a digest string, naming the algorithm so a future change is /// visible in the file rather than inferred from the length. const DIGEST_PREFIX: &str = "sha256:"; /// A whole seal file. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct SealFile { /// Schema version this file targets. #[serde( default = "default_version", deserialize_with = "yaml::flexible_string" )] pub schema_version: String, /// Identity, provenance, and the digest over everything else. pub seal: SealMeta, /// The items as administered, by recorded question number. #[serde(default)] pub items: Vec, /// One entry per form, holding that form's expanded permutation. #[serde(default)] pub forms: Vec, } /// Identity and provenance. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct SealMeta { /// The assessment id this seals. pub assessment: String, /// The assessment title, denormalized so the file reads standalone. pub title: String, /// The course code. pub course: String, /// The term. pub term: String, /// The administration date, when the record carried one. #[serde(default, skip_serializing_if = "Option::is_none")] pub date: Option, /// The day the seal was written. pub sealed_on: Date, /// The tool and version that wrote it. pub generator: Generator, /// Whether option and stem text were stored, or only their fingerprints. #[serde(default = "yes")] pub content: bool, /// Recorded question numbers that were already dropped when the seal was /// written, and therefore were not printed. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub dropped: Vec, /// The digest over every item and form below. /// /// Written last and checked first. [`SealFile::recompute_digest`] rebuilds it /// from the file's own contents; [`SealFile::verify`] rebuilds a seal from the /// live course and compares this against it. pub digest: String, /// The digests this file had before it was rewritten, oldest first. /// /// A seal is meant to be tamper-evident, which puts a migration that /// renames an id inside it in an awkward position: the id is part of /// [`SealFile::digest_input`], so rewriting the file invalidates the digest /// that vouched for it, and recomputing it quietly would produce a record /// that looks untouched and is not. /// /// So a migration does both — recomputes the digest and leaves the old one /// here. What the seal then says is the honest thing: these are the /// contents, this is what they hash to, and here is what the file hashed to /// before each rewrite. Anyone holding an earlier copy, a backup or a git /// revision, can check it against the right entry. /// /// Deliberately outside [`SealFile::digest_input`]: a field that recorded /// past digests and was itself covered by the current one could not be /// appended to without invalidating what it describes. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub superseded_digests: Vec, } /// What wrote a seal. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Generator { /// The tool name. pub tool: String, /// The tool version. pub version: String, } /// One item, frozen as administered. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct SealedItem { /// The recorded question number, which is the join key to grading exports. pub number: u32, /// The item's global id. pub item: String, /// The item version as administered. /// Retained only so a pre-2.0 seal still loads. Ignored when reasoning /// about the item, but still written: [`SealFile::digest_input`] covers it, /// so dropping it on a round trip would invalidate the digest of every /// administration sealed before 2.0. #[serde(default, skip_serializing_if = "Option::is_none")] pub version: Option, /// The stem's digest as administered. #[serde(default, skip_serializing_if = "Option::is_none")] pub stem_digest: Option, /// The item's content fingerprint, the same one /// [`crate::item::Item::fingerprint`] computes, so a seal and a bank can be /// compared without re-hashing either by hand. pub fingerprint: String, /// Points as administered. pub points: f64, /// Whether it was scored as bonus. #[serde(default, skip_serializing_if = "is_false")] pub bonus: bool, /// The cognitive level as administered. #[serde(default, skip_serializing_if = "Option::is_none")] pub level: Option, /// Objectives as administered. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub learning_targets: Vec, /// The keyed letters in the bank's own lettering, before any shuffle. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub key: Vec, /// The stem as administered. Absent when the seal was written without content. #[serde(default, skip_serializing_if = "Option::is_none")] pub stem: Option, /// The options in bank order. #[serde(default)] pub options: Vec, } /// One option, frozen as administered. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct SealedOption { /// The letter this option carries in the bank. pub letter: String, /// Whether it was keyed correct. #[serde(default, skip_serializing_if = "is_false")] pub correct: bool, /// Credit it earned, when the bank set one explicitly. #[serde(default, skip_serializing_if = "Option::is_none")] pub credit: Option, /// A fingerprint of the option text, so drift is detectable even when the /// seal was written without content. pub digest: String, /// The option text as administered. Absent when the seal was written without /// content. #[serde(default, skip_serializing_if = "Option::is_none")] pub text: Option, } /// One form, with its permutation expanded rather than implied by a seed. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct SealedForm { /// The form id, e.g. `A`. pub id: String, /// The seed the permutation came from, kept so it can be re-derived. pub seed: u64, /// Whether item order was permuted. #[serde(default)] pub shuffle_items: bool, /// Whether option order was permuted. #[serde(default)] pub shuffle_options: bool, /// A digest over this form's question list alone, so a single form can be /// checked without reading the rest of the file. pub digest: String, /// The printed questions, in printed order. #[serde(default)] pub questions: Vec, } /// One question as printed on one form. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct SealedQuestion { /// Where it sat on the page, counting from 1. This is what a grading export /// names its file after. pub position: u32, /// The recorded question number, which is what the assessment record and the /// response store use. pub number: u32, /// The item's global id, repeated here so a form block reads on its own. pub item: String, /// The keyed letters *as printed on this form*. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub printed_key: Vec, /// The letter map, in printed order: `printed` is what the student saw, /// `canonical` is the bank's letter for the same option. #[serde(default)] pub options: Vec, } /// One printed letter and the bank letter it stands for. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct LetterMap { /// The letter as printed on this form. pub printed: String, /// The letter the same option carries in the bank. pub canonical: String, } /// What to put in a seal. #[derive(Debug, Clone)] pub struct Options { /// Whether to store stem and option text, not just fingerprints. /// /// On by default. The whole point of a seal is to answer "what did they /// actually see" after the bank has moved on, and a fingerprint answers only /// "not this". pub content: bool, /// Which forms to seal. Empty means every form the record declares, or a /// single unshuffled form `A` when it declares none. pub forms: Vec, } impl Default for Options { fn default() -> Options { Options { content: true, forms: Vec::new(), } } } /// One way a live course has drifted from a seal. #[derive(Debug, Clone)] pub struct Drift { /// How much it matters. pub severity: Severity, /// A machine-readable code, so a check can be silenced or grepped. pub code: &'static str, /// What moved, in a sentence. pub message: String, } impl Drift { /// Builds a drift finding. /// /// # Arguments /// /// * `severity` - how much it matters. /// * `code` - the stable code. /// * `message` - the explanation. /// /// # Returns /// /// The finding. fn new(severity: Severity, code: &'static str, message: impl Into) -> Drift { Drift { severity, code, message: message.into(), } } /// Whether this finding should stop an analysis rather than annotate it. pub fn is_blocking(&self) -> bool { self.severity == Severity::High } } impl std::fmt::Display for Drift { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "[{}] {}", self.code, self.message) } } /// The printed label for a zero-based option position. /// /// This mirrors [`crate::typst::config::LetterStyle::Upper`], which is the /// lettering every bundled exam template uses and the only lettering a Gradescope /// rubric column can carry. A course that prints numeric or Roman option labels /// must map them back to letters before ingest; the seal always speaks letters. /// /// # Arguments /// /// * `position` - the printed position, counting from zero. /// /// # Returns /// /// `A`, `B`, ... `Z`, `AA`. pub fn printed_letter(position: usize) -> String { let mut n = position; let mut letters = Vec::new(); loop { letters.push((b'A' + (n % 26) as u8) as char); if n < 26 { break; } n = n / 26 - 1; } letters.iter().rev().collect() } /// The forms to seal, resolving an empty request and a record with no forms. /// /// # Arguments /// /// * `record` - the assessment record. /// * `wanted` - form ids, or empty for all of them. /// /// # Returns /// /// The forms, in the record's order. /// /// # Errors /// /// Returns [`Error::Usage`] when a named form is not declared. fn forms_to_seal(record: &AssessmentFile, wanted: &[String]) -> Result> { if record.forms.is_empty() { // A record with no declared forms was still printed once, and that single // printing is a form: unshuffled, seed zero. Sealing it costs nothing and // means the ingest path has one shape rather than two. return Ok(vec![Form { id: "A".to_string(), seed: 0, shuffle_items: false, shuffle_options: false, }]); } if wanted.is_empty() { return Ok(record.forms.clone()); } let mut out = Vec::new(); for id in wanted { let form = record .forms .iter() .find(|f| f.id.eq_ignore_ascii_case(id)) .ok_or_else(|| { Error::usage(format!( "no form `{id}` on this assessment; it declares {}", record .forms .iter() .map(|f| f.id.as_str()) .collect::>() .join(", ") )) })?; out.push(form.clone()); } Ok(out) } /// Builds a seal from the live course. /// /// # Arguments /// /// * `catalog` - the loaded course. /// * `record` - the assessment record. /// * `opts` - what to include. /// /// # Returns /// /// The seal, with every digest computed. /// /// # Errors /// /// Returns [`Error::Unresolved`] when a placement references an item that is not /// in the bank, and [`Error::Usage`] when a requested form is not declared. pub fn build(catalog: &Catalog, record: &AssessmentFile, opts: &Options) -> Result { let default_points = catalog.course.policy.points_per_item; let forms = forms_to_seal(record, &opts.forms)?; let mut items = Vec::new(); for placement in &record.items { let entry = catalog.require(&placement.item)?; items.push(sealed_item(placement, &entry.item, default_points, opts)); } items.sort_by_key(|i| i.number); let mut sealed_forms = Vec::new(); for form in &forms { sealed_forms.push(sealed_form(catalog, record, form)?); } let dropped: Vec = record .items .iter() .filter(|p| p.dropped) .map(|p| p.number) .collect(); let mut file = SealFile { schema_version: SCHEMA_VERSION.to_string(), seal: SealMeta { assessment: record.assessment.id.clone(), title: record.assessment.title.clone(), course: catalog.course.course.code.clone(), term: record .assessment .term .clone() .unwrap_or_else(|| catalog.course.course.term.clone()), date: record.assessment.date, sealed_on: Date::today(), generator: Generator { tool: "coursebank".to_string(), version: crate::VERSION.to_string(), }, content: opts.content, dropped, digest: String::new(), superseded_digests: Vec::new(), }, items, forms: sealed_forms, }; file.seal.digest = file.recompute_digest(); Ok(file) } /// Freezes one item. fn sealed_item( placement: &Placement, item: &Item, default_points: f64, opts: &Options, ) -> SealedItem { let options: Vec = item .options .iter() .map(|choice| SealedOption { letter: choice.id.clone(), correct: choice.correct, credit: choice.credit, digest: fingerprint([choice.id.as_str(), choice.text.trim()]), text: if opts.content { Some(choice.text.clone()) } else { None }, }) .collect(); SealedItem { number: placement.number, item: placement.item.clone(), version: None, stem_digest: Some(item.stem_digest()), fingerprint: item.fingerprint(), points: placement .points .unwrap_or_else(|| item.points(default_points)), bonus: placement.bonus, level: placement.level.or(Some(item.level)), learning_targets: if placement.learning_targets.is_empty() { item.learning_targets.clone() } else { placement.learning_targets.clone() }, key: if placement.key.is_empty() { item.key_letters() } else { placement.key.clone() }, stem: if opts.content { Some(item.stem.clone()) } else { None }, options, } } /// Expands one form's permutation. /// /// The expansion comes from [`select::layout`] and [`select::option_order`], the /// same two functions every export calls, so a seal cannot describe a paper the /// exporter would not have produced. fn sealed_form(catalog: &Catalog, record: &AssessmentFile, form: &Form) -> Result { let mut questions = Vec::new(); // Only an item pulled before printing takes no printed position. An item // dropped from scoring after the exam was on the page and keeps its place, // so re-sealing after a drop describes the same paper the students held // rather than renumbering everything after it. let printed: Vec = select::layout(record, form) .into_iter() .filter(|p| p.was_printed()) .collect(); for (index, placement) in printed.iter().enumerate() { let entry = catalog.require(&placement.item)?; let item = &entry.item; // The pool is not the paper: seal what this placement administered. let shown = item.administered(&placement.key, &placement.distractors); let n = shown.len(); let order = select::option_order(form, &placement.item, n); let canonical_key: BTreeSet = if placement.key.is_empty() { item.key_letters().into_iter().collect() } else { placement.key.iter().cloned().collect() }; let mut options = Vec::with_capacity(n); let mut printed_key = Vec::new(); for (position, source_index) in order.iter().enumerate() { let canonical = shown .get(*source_index) .map(|c| c.id.clone()) .unwrap_or_else(|| printed_letter(*source_index)); let printed = printed_letter(position); if canonical_key.contains(&canonical) { printed_key.push(printed.clone()); } options.push(LetterMap { printed, canonical }); } questions.push(SealedQuestion { position: index as u32 + 1, number: placement.number, item: placement.item.clone(), printed_key, options, }); } let mut sealed = SealedForm { id: form.id.clone(), seed: form.seed, shuffle_items: form.shuffle_items, shuffle_options: form.shuffle_options, digest: String::new(), questions, }; sealed.digest = form_digest(&sealed); Ok(sealed) } /// The digest over one form's question list. fn form_digest(form: &SealedForm) -> String { let mut buf = String::new(); buf.push_str(&format!( "form\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}{}\n", form.id, form.seed, form.shuffle_items, form.shuffle_options )); for q in &form.questions { buf.push_str(&format!( "q\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}", q.position, q.number, q.item, q.printed_key.join(",") )); for map in &q.options { buf.push_str(&format!("{}>{};", map.printed, map.canonical)); } buf.push('\n'); } digest_of(&buf) } /// A digest string over some canonical text. fn digest_of(text: &str) -> String { format!("{DIGEST_PREFIX}{}", hex(&sha256(text.as_bytes()))) } impl SealFile { /// The path a seal lives at. /// /// # Arguments /// /// * `layout` - the course layout. /// * `assessment_id` - the assessment id. /// /// # Returns /// /// `seals/.yaml` under the course root. pub fn path(layout: &Layout, assessment_id: &str) -> PathBuf { layout .root .join(SEAL_DIR) .join(format!("{assessment_id}.yaml")) } /// Loads a seal. /// /// # Arguments /// /// * `path` - the YAML file. /// /// # Returns /// /// The parsed seal. /// /// # Errors /// /// Returns a load error. pub fn load(path: &Path) -> Result { yaml::read(path) } /// Loads every seal in a directory, oldest administration first. /// /// # Arguments /// /// * `dir` - the seals directory. /// /// # Returns /// /// The seals, empty when the directory does not exist. /// /// # Errors /// /// Propagates load failures, including a seal that does not parse. pub fn load_all(dir: &Path) -> Result> { let mut out = Vec::new(); for path in crate::yaml::list_yaml(dir)? { out.push(SealFile::load(&path)?); } out.sort_by(|a, b| { a.seal .date .cmp(&b.seal.date) .then(a.seal.assessment.cmp(&b.seal.assessment)) }); Ok(out) } /// Loads the seal for an assessment, if one has been written. /// /// Absence is not an error. A course that has never sealed anything should /// still ingest, with the permutation derived from the record instead. /// /// # Arguments /// /// * `layout` - the course layout. /// * `assessment_id` - the assessment id. /// /// # Returns /// /// The seal, or `None`. /// /// # Errors /// /// Returns a load error when the file exists but does not parse. pub fn find(layout: &Layout, assessment_id: &str) -> Result> { let path = SealFile::path(layout, assessment_id); if path.is_file() { Ok(Some(SealFile::load(&path)?)) } else { Ok(None) } } /// Writes the seal out. /// /// # Arguments /// /// * `path` - the destination. /// /// # Errors /// /// Returns [`Error::Io`] on a write failure. pub fn save(&self, path: &Path) -> Result<()> { yaml::write(path, self) } /// One form's block. /// /// # Arguments /// /// * `id` - the form id, matched case-insensitively. /// /// # Returns /// /// The form, or `None`. pub fn form(&self, id: &str) -> Option<&SealedForm> { self.forms.iter().find(|f| f.id.eq_ignore_ascii_case(id)) } /// One item's block, by recorded question number. /// /// # Arguments /// /// * `number` - the recorded number. /// /// # Returns /// /// The item, or `None`. pub fn item(&self, number: u32) -> Option<&SealedItem> { self.items.iter().find(|i| i.number == number) } /// The canonical text this file's digest is computed over. /// /// Built field by field rather than by serializing the struct, so a change to /// serde attributes, key order, or YAML quoting cannot invalidate every seal /// ever written. Unit separators keep concatenation unambiguous. /// /// # Returns /// /// The digest input. pub fn digest_input(&self) -> String { let mut buf = String::new(); buf.push_str(&format!( "seal\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}{}\n", self.schema_version, self.seal.assessment, self.seal.course, self.seal.term )); for item in &self.items { // Appended only when present: a seal written before 2.0 has to keep // producing the input it was digested from, or every older // administration fails verification. if let Some(stem) = &item.stem_digest { buf.push_str(&format!("stem\u{1f}{}\u{1f}{stem}\n", item.number)); } buf.push_str(&format!( "item\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}", item.number, item.item, item.version.unwrap_or(0), item.fingerprint, item.points, item.bonus, item.level.map(|l| l.code()).unwrap_or(0), item.key.join(",") )); for option in &item.options { buf.push_str(&format!( "{}:{}:{};", option.letter, option.correct, option.digest )); } buf.push('\n'); } for form in &self.forms { buf.push_str(&format!("formref\u{1f}{}\u{1f}{}\n", form.id, form.digest)); } buf } /// Renames the items this seal froze, keeping the digest honest. /// /// The only sanctioned way to rewrite a seal. It recomputes the digest, /// because the ids are part of what the digest covers, and records the /// previous one in [`SealMeta::superseded_digests`], because a recomputed /// digest with no trace of the recompute is a record that claims never to /// have been touched. /// /// # Arguments /// /// * `rename` - a map from old item id to new. /// /// # Returns /// /// How many placements were renamed. Zero leaves the file alone, digest /// included. pub fn rename_items(&mut self, rename: &BTreeMap) -> usize { let mut renamed = 0; for item in &mut self.items { if let Some(new) = rename.get(&item.item) { item.item = new.clone(); renamed += 1; } } if renamed == 0 { return 0; } let previous = std::mem::take(&mut self.seal.digest); self.seal.superseded_digests.push(previous); self.seal.digest = self.recompute_digest(); renamed } /// Recomputes this file's digest from its own contents. /// /// # Returns /// /// The digest string. pub fn recompute_digest(&self) -> String { digest_of(&self.digest_input()) } /// Checks that the file has not been edited since it was written. /// /// This catches a hand-edit of the seal itself, which is a different failure /// from the bank drifting and deserves a different message. /// /// # Returns /// /// Findings, empty when the file is internally consistent. pub fn check_self(&self) -> Vec { let mut out = Vec::new(); let expected = self.recompute_digest(); if expected != self.seal.digest { out.push(Drift::new( Severity::High, "seal-digest", format!( "the seal's own digest does not match its contents ({} recorded, {} \ computed); the file was edited after it was written", short(&self.seal.digest), short(&expected) ), )); } for form in &self.forms { let expected = form_digest(form); if expected != form.digest { out.push(Drift::new( Severity::High, "seal-form-digest", format!( "form {}'s permutation block was edited after sealing", form.id ), )); } } out } /// Compares the seal against the live course. /// /// Everything this reports is a reason to distrust an analysis that pools the /// sealed administration with anything else, and most of it is a reason to /// distrust per-option feedback sent to a student. /// /// # Arguments /// /// * `catalog` - the loaded course, as it stands now. /// * `record` - the assessment record, as it stands now. /// /// # Returns /// /// Findings, worst first, empty when nothing has moved. pub fn verify(&self, catalog: &Catalog, record: &AssessmentFile) -> Vec { let mut out = self.check_self(); let rebuilt = build( catalog, record, &Options { content: self.seal.content, forms: self.forms.iter().map(|f| f.id.clone()).collect(), }, ); let rebuilt = match rebuilt { Ok(r) => r, Err(e) => { out.push(Drift::new( Severity::High, "seal-unbuildable", format!("the course no longer produces this assessment at all: {e}"), )); return out; } }; if rebuilt.seal.digest == self.seal.digest { return out; } // The digests differ, so say what differs rather than that they do. let sealed: BTreeMap = self.items.iter().map(|i| (i.number, i)).collect(); let live: BTreeMap = rebuilt.items.iter().map(|i| (i.number, i)).collect(); for (number, was) in &sealed { let Some(now) = live.get(number) else { out.push(Drift::new( Severity::High, "item-removed", format!( "question {number} ({}) is no longer on the assessment record", was.item ), )); continue; }; if was.item != now.item { out.push(Drift::new( Severity::High, "item-replaced", format!("question {number} was {} and is now {}", was.item, now.item), )); continue; } if was.fingerprint != now.fingerprint { out.push(Drift::new( Severity::High, "item-edited", format!( "question {number} ({}) was edited after administration; the students saw \ a different version of the stem or options", was.item ), )); } if was.key != now.key { out.push(Drift::new( Severity::High, "key-changed", format!( "question {number} ({}) was keyed {} and is now keyed {}", was.item, was.key.join(""), now.key.join("") ), )); } if was.options.len() != now.options.len() { out.push(Drift::new( Severity::High, "option-count-changed", format!( "question {number} ({}) had {} options and now has {}, so every printed \ letter on every form means something else", was.item, was.options.len(), now.options.len() ), )); } if was.points != now.points { out.push(Drift::new( Severity::Medium, "points-changed", format!( "question {number} was worth {} point(s) and is now worth {}", was.points, now.points ), )); } if was.learning_targets != now.learning_targets { out.push(Drift::new( Severity::Low, "objectives-retagged", format!( "question {number} ({}) was retagged; per-objective results for this \ administration were computed against the old tags", was.item ), )); } if was.level != now.level { out.push(Drift::new( Severity::Low, "level-changed", format!("question {number} ({}) changed level", was.item), )); } } for number in live.keys() { if !sealed.contains_key(number) { out.push(Drift::new( Severity::Medium, "item-added", format!("question {number} was added to the record after sealing"), )); } } for form in &self.forms { let Some(now) = rebuilt.form(&form.id) else { out.push(Drift::new( Severity::High, "form-removed", format!("form {} is no longer declared on the record", form.id), )); continue; }; if now.digest != form.digest { let reason = if now.seed != form.seed { format!( "its seed changed from {} to {}, so every option moved", form.seed, now.seed ) } else if now.shuffle_options != form.shuffle_options || now.shuffle_items != form.shuffle_items { "its shuffle settings changed".to_string() } else { "the items behind it changed".to_string() }; out.push(Drift::new( Severity::High, "form-permutation-changed", format!( "form {} no longer permutes the way it did when it was printed: {reason}", form.id ), )); } } out.sort_by_key(|d| std::cmp::Reverse(d.severity)); out } } /// How a form's printed answer key is distributed. /// /// A shuffle is uniform per item and says nothing about the sequence it produces /// across a form. Over thirty-odd questions it will occasionally produce a run /// students notice: six consecutive questions keying to `A` is unremarkable /// probabilistically and very remarkable at a desk, where it reads as a mistake /// and makes a student who is sure of all six doubt the last three. /// /// Sealing is the moment to look, because it is the last moment before printing. /// Changing a form's seed reshuffles it; nothing else has to change. #[derive(Debug, Clone)] pub struct Balance { /// The form id. pub form: String, /// How many questions key to each printed letter. pub counts: BTreeMap, /// The longest run of consecutive questions sharing a printed key, as /// `(letter, first number, last number)`. pub longest_run: Option<(String, u32, u32)>, /// How long that run was. Stored rather than derived from the two numbers, /// which are recorded numbers and need not be contiguous. pub run: usize, /// How many questions were counted. pub total: usize, } impl Balance { /// The length of the longest run. pub fn run_length(&self) -> usize { self.run } /// Advisories worth printing before the form is printed. /// /// Deliberately quiet. Both thresholds are set where the pattern stops being /// something only a statistician would see: a run of five, and a letter /// carrying more than twice its share. /// /// # Returns /// /// One message per observation, empty when the form looks unremarkable. pub fn notes(&self) -> Vec { let mut out = Vec::new(); if self.total == 0 { return out; } if self.run_length() >= 5 { if let Some((letter, first, last)) = &self.longest_run { out.push(format!( "form {}: questions {first} through {last} all key to {letter}. That is {} in \ a row, which students read as an error in the key. Re-seed the form if you \ have not printed it yet", self.form, self.run_length() )); } } let share = self.total as f64 / self.counts.len().max(1) as f64; for (letter, count) in &self.counts { if *count as f64 > share * 2.0 && *count >= 4 { out.push(format!( "form {}: {count} of {} questions key to {letter}", self.form, self.total )); } } out } } /// Measures one form's printed key. /// /// # Arguments /// /// * `form` - the sealed form. /// /// # Returns /// /// The balance. Questions with no single keyed letter, such as a /// multiple-response item, are skipped rather than guessed at. pub fn balance(form: &SealedForm) -> Balance { let mut counts: BTreeMap = BTreeMap::new(); let mut sequence: Vec<(u32, String)> = Vec::new(); for question in &form.questions { if question.printed_key.len() != 1 { continue; } let letter = question.printed_key[0].clone(); *counts.entry(letter.clone()).or_insert(0) += 1; sequence.push((question.number, letter)); } let mut longest: Option<(String, u32, u32)> = None; let mut best = 0usize; let mut run = 1usize; for i in 1..sequence.len() { if sequence[i].1 == sequence[i - 1].1 { run += 1; if run > best { best = run; longest = Some(( sequence[i].1.clone(), sequence[i + 1 - run].0, sequence[i].0, )); } } else { run = 1; } } Balance { form: form.id.clone(), total: sequence.len(), counts, longest_run: longest, run: best.max(if sequence.is_empty() { 0 } else { 1 }), } } /// The first eight hex characters of a digest, for messages. /// /// # Arguments /// /// * `digest` - the full digest string. /// /// # Returns /// /// A short form such as `sha256:1a2b3c4d`. pub fn short(digest: &str) -> String { match digest.split_once(':') { Some((algorithm, hex)) => { format!("{algorithm}:{}", &hex[..hex.len().min(8)]) } None => digest.chars().take(8).collect(), } } fn default_version() -> String { SCHEMA_VERSION.to_string() } fn yes() -> bool { true } fn is_false(b: &bool) -> bool { !*b } #[cfg(test)] mod tests { use super::*; #[test] fn printed_letters_run_past_z() { assert_eq!(printed_letter(0), "A"); assert_eq!(printed_letter(3), "D"); assert_eq!(printed_letter(25), "Z"); assert_eq!(printed_letter(26), "AA"); } fn sample() -> SealFile { let mut file = SealFile { schema_version: "1.0".into(), seal: SealMeta { assessment: "e1".into(), title: "Exam 1".into(), course: "BIOSC 1000".into(), term: "2026f".into(), date: None, sealed_on: Date::today(), generator: Generator { tool: "coursebank".into(), version: "test".into(), }, content: true, dropped: Vec::new(), digest: String::new(), superseded_digests: Vec::new(), }, items: vec![SealedItem { number: 1, item: "b::q-1".into(), version: Some(1), stem_digest: None, fingerprint: "abc".into(), points: 1.0, bonus: false, level: Some(Level::Remember), learning_targets: vec!["lo-a".into()], key: vec!["B".into()], stem: Some("Stem.".into()), options: vec![ SealedOption { letter: "A".into(), correct: false, credit: None, digest: "d1".into(), text: Some("First".into()), }, SealedOption { letter: "B".into(), correct: true, credit: None, digest: "d2".into(), text: Some("Second".into()), }, ], }], forms: vec![SealedForm { id: "A".into(), seed: 7, shuffle_items: false, shuffle_options: true, digest: String::new(), questions: vec![SealedQuestion { position: 1, number: 1, item: "b::q-1".into(), printed_key: vec!["A".into()], options: vec![ LetterMap { printed: "A".into(), canonical: "B".into(), }, LetterMap { printed: "B".into(), canonical: "A".into(), }, ], }], }], }; file.forms[0].digest = form_digest(&file.forms[0]); file.seal.digest = file.recompute_digest(); file } #[test] fn a_fresh_seal_is_internally_consistent() { assert!(sample().check_self().is_empty()); } #[test] fn editing_the_file_breaks_its_digest() { let mut file = sample(); file.items[0].key = vec!["A".into()]; let findings = file.check_self(); assert!( findings.iter().any(|d| d.code == "seal-digest"), "{findings:?}" ); } #[test] fn editing_a_permutation_breaks_that_forms_digest() { let mut file = sample(); file.forms[0].questions[0].options[0].canonical = "A".into(); let findings = file.check_self(); assert!( findings.iter().any(|d| d.code == "seal-form-digest"), "{findings:?}" ); } #[test] fn the_digest_is_order_sensitive() { let file = sample(); let mut swapped = file.clone(); swapped.items[0].options.swap(0, 1); assert_ne!(file.recompute_digest(), swapped.recompute_digest()); } fn form_keyed(letters: &[&str]) -> SealedForm { SealedForm { id: "B".into(), seed: 1, shuffle_items: false, shuffle_options: true, digest: String::new(), questions: letters .iter() .enumerate() .map(|(i, letter)| SealedQuestion { position: i as u32 + 1, number: i as u32 + 1, item: format!("b::q-{i}"), printed_key: vec![letter.to_string()], options: Vec::new(), }) .collect(), } } #[test] fn a_long_run_of_one_letter_is_named_with_its_question_numbers() { // Form B of a real exam did this: questions 2 through 7 all keyed to A. let form = form_keyed(&["C", "A", "A", "A", "A", "A", "A", "C", "B"]); let balance = balance(&form); assert_eq!(balance.run_length(), 6); assert_eq!( balance.longest_run, Some(("A".to_string(), 2, 7)), "the run is reported by question number, not by index" ); let notes = balance.notes(); assert!(notes.iter().any(|n| n.contains("2 through 7")), "{notes:?}"); } #[test] fn an_ordinary_form_says_nothing() { let form = form_keyed(&["A", "B", "C", "D", "A", "C", "B", "D", "C", "A", "D", "B"]); assert!(balance(&form).notes().is_empty()); } #[test] fn a_form_with_no_questions_does_not_panic() { let balance = balance(&form_keyed(&[])); assert_eq!(balance.run_length(), 0); assert!(balance.notes().is_empty()); } #[test] fn short_digests_keep_the_algorithm() { assert_eq!(short("sha256:0123456789abcdef"), "sha256:01234567"); } #[test] fn renaming_an_item_recomputes_the_digest_and_says_it_did() { let mut seal = sample(); seal.seal.digest = seal.recompute_digest(); let original = seal.seal.digest.clone(); assert!(seal.check_self().is_empty()); let mut rename = BTreeMap::new(); rename.insert(seal.items[0].item.clone(), "q-renamed".to_string()); assert_eq!(seal.rename_items(&rename), 1); // The digest covers the ids, so it has to move — and the file has to // admit that it moved rather than looking untouched. assert_eq!(seal.items[0].item, "q-renamed"); assert_ne!(seal.seal.digest, original); assert_eq!(seal.seal.superseded_digests, vec![original]); assert!(seal.check_self().is_empty(), "{:?}", seal.check_self()); // A rename that matches nothing leaves the file entirely alone. let steady = seal.seal.digest.clone(); assert_eq!(seal.rename_items(&BTreeMap::new()), 0); assert_eq!(seal.seal.digest, steady); assert_eq!(seal.seal.superseded_digests.len(), 1); } #[test] fn round_trips_through_yaml() { let file = sample(); let text = serde_yaml_ng::to_string(&file).unwrap(); let back: SealFile = serde_yaml_ng::from_str(&text).unwrap(); assert_eq!(back.seal.digest, file.seal.digest); assert!(back.check_self().is_empty()); assert_eq!(back.form("a").map(|f| f.seed), Some(7)); } }