Files
coursebank/src/model/seal.rs
T
alexmandClaude c8d6f4a5dd
Pipeline / check (pull_request) Successful in 12m21s
Pipeline / docs (pull_request) Skipped
Pipeline / nightly (pull_request) Skipped
Pipeline / release (pull_request) Skipped
Name Scienting as the copyright holder and contributor
Replace 'Scientific Computing Studio' in the source headers, the Cargo.toml
authors, and the LICENSE contributor line. pixi.toml still has the old
authors line.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014zNf4shD4zaCKLucZWPYL9
2026-10-20 00:21:19 -04:00

1436 lines
48 KiB
Rust

// SPDX-License-Identifier: Prosperity-3.0.0
// Copyright Scienting
// Source: https://git.scient.ing/education/coursebank
//! The sealed administration: what the students actually held in their hands.
//!
//! An assessment record says which item sat at question 14. It does not say what
//! that item *said*, and it does not say that on form B question 14's option C was
//! the bank's option A. Both facts are derivable — the first from the bank, the
//! second from the form seed — right up until the bank changes, and then they are
//! silently derivable to the wrong answer.
//!
//! A seal freezes them. It is written once, before the exam is administered, and
//! it holds three things the record does not:
//!
//! * **The content as administered.** Stem and option text, verbatim, with a
//! fingerprint on each. Reword a distractor next term and the seal still shows
//! what this cohort was asked.
//! * **The permutation, expanded.** For every form and every printed question, the
//! map from printed letter to bank letter, plus the key as printed. This is what
//! turns "the student chose C" into "the student chose the bank's option A",
//! which is the only form of that sentence worth storing.
//! * **Digests over both.** One per form, one per item, and one over the whole
//! file. Re-derive any of them and compare; a mismatch names what moved.
//!
//! The seal is not a cache. Nothing reads it for speed. It exists so that the
//! question "is what I am about to analyze the same thing the students saw?" has
//! an answer that does not depend on the bank having stayed still.
//!
//! # Where it lives
//!
//! `seals/<assessment-id>.yaml`, in its own directory rather than beside the
//! assessment record, because [`crate::assessment::AssessmentFile::load_all`]
//! parses every `.yaml` in `assessments/` and a seal is not an assessment.
//!
//! # The order of operations
//!
//! ```text
//! assemble ──▶ export typst ──▶ seal ──▶ print ──▶ administer ──▶ ingest
//! │ │
//! └────────── verified against ──────┘
//! ```
//!
//! Seal after exporting and before printing. Sealing earlier is harmless;
//! sealing after ingest is not, because a seal written from a drifted bank
//! records the drift as truth.
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
use crate::assessment::{AssessmentFile, Form, Placement};
use crate::catalog::{Catalog, Severity};
use crate::course::SCHEMA_VERSION;
use crate::date::Date;
use crate::error::{Error, Result};
use crate::hash::{fingerprint, hex, sha256};
use crate::item::Item;
use crate::layout::Layout;
use crate::select;
use crate::taxonomy::Level;
use crate::yaml;
/// The directory holding seals, relative to the course root.
pub const SEAL_DIR: &str = "seals";
/// The prefix on a digest string, naming the algorithm so a future change is
/// visible in the file rather than inferred from the length.
const DIGEST_PREFIX: &str = "sha256:";
/// A whole seal file.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct SealFile {
/// Schema version this file targets.
#[serde(
default = "default_version",
deserialize_with = "yaml::flexible_string"
)]
pub schema_version: String,
/// Identity, provenance, and the digest over everything else.
pub seal: SealMeta,
/// The items as administered, by recorded question number.
#[serde(default)]
pub items: Vec<SealedItem>,
/// One entry per form, holding that form's expanded permutation.
#[serde(default)]
pub forms: Vec<SealedForm>,
}
/// Identity and provenance.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct SealMeta {
/// The assessment id this seals.
pub assessment: String,
/// The assessment title, denormalized so the file reads standalone.
pub title: String,
/// The course code.
pub course: String,
/// The term.
pub term: String,
/// The administration date, when the record carried one.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub date: Option<Date>,
/// The day the seal was written.
pub sealed_on: Date,
/// The tool and version that wrote it.
pub generator: Generator,
/// Whether option and stem text were stored, or only their fingerprints.
#[serde(default = "yes")]
pub content: bool,
/// Recorded question numbers that were already dropped when the seal was
/// written, and therefore were not printed.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dropped: Vec<u32>,
/// The digest over every item and form below.
///
/// Written last and checked first. [`SealFile::recompute_digest`] rebuilds it
/// from the file's own contents; [`SealFile::verify`] rebuilds a seal from the
/// live course and compares this against it.
pub digest: String,
/// The digests this file had before it was rewritten, oldest first.
///
/// A seal is meant to be tamper-evident, which puts a migration that
/// renames an id inside it in an awkward position: the id is part of
/// [`SealFile::digest_input`], so rewriting the file invalidates the digest
/// that vouched for it, and recomputing it quietly would produce a record
/// that looks untouched and is not.
///
/// So a migration does both — recomputes the digest and leaves the old one
/// here. What the seal then says is the honest thing: these are the
/// contents, this is what they hash to, and here is what the file hashed to
/// before each rewrite. Anyone holding an earlier copy, a backup or a git
/// revision, can check it against the right entry.
///
/// Deliberately outside [`SealFile::digest_input`]: a field that recorded
/// past digests and was itself covered by the current one could not be
/// appended to without invalidating what it describes.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub superseded_digests: Vec<String>,
}
/// What wrote a seal.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Generator {
/// The tool name.
pub tool: String,
/// The tool version.
pub version: String,
}
/// One item, frozen as administered.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct SealedItem {
/// The recorded question number, which is the join key to grading exports.
pub number: u32,
/// The item's global id.
pub item: String,
/// The item version as administered.
/// Retained only so a pre-2.0 seal still loads. Ignored when reasoning
/// about the item, but still written: [`SealFile::digest_input`] covers it,
/// so dropping it on a round trip would invalidate the digest of every
/// administration sealed before 2.0.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub version: Option<u32>,
/// The stem's digest as administered.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub stem_digest: Option<String>,
/// The item's content fingerprint, the same one
/// [`crate::item::Item::fingerprint`] computes, so a seal and a bank can be
/// compared without re-hashing either by hand.
pub fingerprint: String,
/// Points as administered.
pub points: f64,
/// Whether it was scored as bonus.
#[serde(default, skip_serializing_if = "is_false")]
pub bonus: bool,
/// The cognitive level as administered.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub level: Option<Level>,
/// Objectives as administered.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub learning_targets: Vec<String>,
/// The keyed letters in the bank's own lettering, before any shuffle.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub key: Vec<String>,
/// The stem as administered. Absent when the seal was written without content.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub stem: Option<String>,
/// The options in bank order.
#[serde(default)]
pub options: Vec<SealedOption>,
}
/// One option, frozen as administered.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct SealedOption {
/// The letter this option carries in the bank.
pub letter: String,
/// Whether it was keyed correct.
#[serde(default, skip_serializing_if = "is_false")]
pub correct: bool,
/// Credit it earned, when the bank set one explicitly.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub credit: Option<f64>,
/// A fingerprint of the option text, so drift is detectable even when the
/// seal was written without content.
pub digest: String,
/// The option text as administered. Absent when the seal was written without
/// content.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub text: Option<String>,
}
/// One form, with its permutation expanded rather than implied by a seed.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct SealedForm {
/// The form id, e.g. `A`.
pub id: String,
/// The seed the permutation came from, kept so it can be re-derived.
pub seed: u64,
/// Whether item order was permuted.
#[serde(default)]
pub shuffle_items: bool,
/// Whether option order was permuted.
#[serde(default)]
pub shuffle_options: bool,
/// A digest over this form's question list alone, so a single form can be
/// checked without reading the rest of the file.
pub digest: String,
/// The printed questions, in printed order.
#[serde(default)]
pub questions: Vec<SealedQuestion>,
}
/// One question as printed on one form.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct SealedQuestion {
/// Where it sat on the page, counting from 1. This is what a grading export
/// names its file after.
pub position: u32,
/// The recorded question number, which is what the assessment record and the
/// response store use.
pub number: u32,
/// The item's global id, repeated here so a form block reads on its own.
pub item: String,
/// The keyed letters *as printed on this form*.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub printed_key: Vec<String>,
/// The letter map, in printed order: `printed` is what the student saw,
/// `canonical` is the bank's letter for the same option.
#[serde(default)]
pub options: Vec<LetterMap>,
}
/// One printed letter and the bank letter it stands for.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct LetterMap {
/// The letter as printed on this form.
pub printed: String,
/// The letter the same option carries in the bank.
pub canonical: String,
}
/// What to put in a seal.
#[derive(Debug, Clone)]
pub struct Options {
/// Whether to store stem and option text, not just fingerprints.
///
/// On by default. The whole point of a seal is to answer "what did they
/// actually see" after the bank has moved on, and a fingerprint answers only
/// "not this".
pub content: bool,
/// Which forms to seal. Empty means every form the record declares, or a
/// single unshuffled form `A` when it declares none.
pub forms: Vec<String>,
}
impl Default for Options {
fn default() -> Options {
Options {
content: true,
forms: Vec::new(),
}
}
}
/// One way a live course has drifted from a seal.
#[derive(Debug, Clone)]
pub struct Drift {
/// How much it matters.
pub severity: Severity,
/// A machine-readable code, so a check can be silenced or grepped.
pub code: &'static str,
/// What moved, in a sentence.
pub message: String,
}
impl Drift {
/// Builds a drift finding.
///
/// # Arguments
///
/// * `severity` - how much it matters.
/// * `code` - the stable code.
/// * `message` - the explanation.
///
/// # Returns
///
/// The finding.
fn new(severity: Severity, code: &'static str, message: impl Into<String>) -> Drift {
Drift {
severity,
code,
message: message.into(),
}
}
/// Whether this finding should stop an analysis rather than annotate it.
pub fn is_blocking(&self) -> bool {
self.severity == Severity::High
}
}
impl std::fmt::Display for Drift {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "[{}] {}", self.code, self.message)
}
}
/// The printed label for a zero-based option position.
///
/// This mirrors [`crate::typst::config::LetterStyle::Upper`], which is the
/// lettering every bundled exam template uses and the only lettering a Gradescope
/// rubric column can carry. A course that prints numeric or Roman option labels
/// must map them back to letters before ingest; the seal always speaks letters.
///
/// # Arguments
///
/// * `position` - the printed position, counting from zero.
///
/// # Returns
///
/// `A`, `B`, ... `Z`, `AA`.
pub fn printed_letter(position: usize) -> String {
let mut n = position;
let mut letters = Vec::new();
loop {
letters.push((b'A' + (n % 26) as u8) as char);
if n < 26 {
break;
}
n = n / 26 - 1;
}
letters.iter().rev().collect()
}
/// The forms to seal, resolving an empty request and a record with no forms.
///
/// # Arguments
///
/// * `record` - the assessment record.
/// * `wanted` - form ids, or empty for all of them.
///
/// # Returns
///
/// The forms, in the record's order.
///
/// # Errors
///
/// Returns [`Error::Usage`] when a named form is not declared.
fn forms_to_seal(record: &AssessmentFile, wanted: &[String]) -> Result<Vec<Form>> {
if record.forms.is_empty() {
// A record with no declared forms was still printed once, and that single
// printing is a form: unshuffled, seed zero. Sealing it costs nothing and
// means the ingest path has one shape rather than two.
return Ok(vec![Form {
id: "A".to_string(),
seed: 0,
shuffle_items: false,
shuffle_options: false,
}]);
}
if wanted.is_empty() {
return Ok(record.forms.clone());
}
let mut out = Vec::new();
for id in wanted {
let form = record
.forms
.iter()
.find(|f| f.id.eq_ignore_ascii_case(id))
.ok_or_else(|| {
Error::usage(format!(
"no form `{id}` on this assessment; it declares {}",
record
.forms
.iter()
.map(|f| f.id.as_str())
.collect::<Vec<_>>()
.join(", ")
))
})?;
out.push(form.clone());
}
Ok(out)
}
/// Builds a seal from the live course.
///
/// # Arguments
///
/// * `catalog` - the loaded course.
/// * `record` - the assessment record.
/// * `opts` - what to include.
///
/// # Returns
///
/// The seal, with every digest computed.
///
/// # Errors
///
/// Returns [`Error::Unresolved`] when a placement references an item that is not
/// in the bank, and [`Error::Usage`] when a requested form is not declared.
pub fn build(catalog: &Catalog, record: &AssessmentFile, opts: &Options) -> Result<SealFile> {
let default_points = catalog.course.policy.points_per_item;
let forms = forms_to_seal(record, &opts.forms)?;
let mut items = Vec::new();
for placement in &record.items {
let entry = catalog.require(&placement.item)?;
items.push(sealed_item(placement, &entry.item, default_points, opts));
}
items.sort_by_key(|i| i.number);
let mut sealed_forms = Vec::new();
for form in &forms {
sealed_forms.push(sealed_form(catalog, record, form)?);
}
let dropped: Vec<u32> = record
.items
.iter()
.filter(|p| p.dropped)
.map(|p| p.number)
.collect();
let mut file = SealFile {
schema_version: SCHEMA_VERSION.to_string(),
seal: SealMeta {
assessment: record.assessment.id.clone(),
title: record.assessment.title.clone(),
course: catalog.course.course.code.clone(),
term: record
.assessment
.term
.clone()
.unwrap_or_else(|| catalog.course.course.term.clone()),
date: record.assessment.date,
sealed_on: Date::today(),
generator: Generator {
tool: "coursebank".to_string(),
version: crate::VERSION.to_string(),
},
content: opts.content,
dropped,
digest: String::new(),
superseded_digests: Vec::new(),
},
items,
forms: sealed_forms,
};
file.seal.digest = file.recompute_digest();
Ok(file)
}
/// Freezes one item.
fn sealed_item(
placement: &Placement,
item: &Item,
default_points: f64,
opts: &Options,
) -> SealedItem {
let options: Vec<SealedOption> = item
.options
.iter()
.map(|choice| SealedOption {
letter: choice.id.clone(),
correct: choice.correct,
credit: choice.credit,
digest: fingerprint([choice.id.as_str(), choice.text.trim()]),
text: if opts.content {
Some(choice.text.clone())
} else {
None
},
})
.collect();
SealedItem {
number: placement.number,
item: placement.item.clone(),
version: None,
stem_digest: Some(item.stem_digest()),
fingerprint: item.fingerprint(),
points: placement
.points
.unwrap_or_else(|| item.points(default_points)),
bonus: placement.bonus,
level: placement.level.or(Some(item.level)),
learning_targets: if placement.learning_targets.is_empty() {
item.learning_targets.clone()
} else {
placement.learning_targets.clone()
},
key: if placement.key.is_empty() {
item.key_letters()
} else {
placement.key.clone()
},
stem: if opts.content {
Some(item.stem.clone())
} else {
None
},
options,
}
}
/// Expands one form's permutation.
///
/// The expansion comes from [`select::layout`] and [`select::option_order`], the
/// same two functions every export calls, so a seal cannot describe a paper the
/// exporter would not have produced.
fn sealed_form(catalog: &Catalog, record: &AssessmentFile, form: &Form) -> Result<SealedForm> {
let mut questions = Vec::new();
// Only an item pulled before printing takes no printed position. An item
// dropped from scoring after the exam was on the page and keeps its place,
// so re-sealing after a drop describes the same paper the students held
// rather than renumbering everything after it.
let printed: Vec<Placement> = select::layout(record, form)
.into_iter()
.filter(|p| p.was_printed())
.collect();
for (index, placement) in printed.iter().enumerate() {
let entry = catalog.require(&placement.item)?;
let item = &entry.item;
// The pool is not the paper: seal what this placement administered.
let shown = item.administered(&placement.key, &placement.distractors);
let n = shown.len();
let order = select::option_order(form, &placement.item, n);
let canonical_key: BTreeSet<String> = if placement.key.is_empty() {
item.key_letters().into_iter().collect()
} else {
placement.key.iter().cloned().collect()
};
let mut options = Vec::with_capacity(n);
let mut printed_key = Vec::new();
for (position, source_index) in order.iter().enumerate() {
let canonical = shown
.get(*source_index)
.map(|c| c.id.clone())
.unwrap_or_else(|| printed_letter(*source_index));
let printed = printed_letter(position);
if canonical_key.contains(&canonical) {
printed_key.push(printed.clone());
}
options.push(LetterMap { printed, canonical });
}
questions.push(SealedQuestion {
position: index as u32 + 1,
number: placement.number,
item: placement.item.clone(),
printed_key,
options,
});
}
let mut sealed = SealedForm {
id: form.id.clone(),
seed: form.seed,
shuffle_items: form.shuffle_items,
shuffle_options: form.shuffle_options,
digest: String::new(),
questions,
};
sealed.digest = form_digest(&sealed);
Ok(sealed)
}
/// The digest over one form's question list.
fn form_digest(form: &SealedForm) -> String {
let mut buf = String::new();
buf.push_str(&format!(
"form\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}{}\n",
form.id, form.seed, form.shuffle_items, form.shuffle_options
));
for q in &form.questions {
buf.push_str(&format!(
"q\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}",
q.position,
q.number,
q.item,
q.printed_key.join(",")
));
for map in &q.options {
buf.push_str(&format!("{}>{};", map.printed, map.canonical));
}
buf.push('\n');
}
digest_of(&buf)
}
/// A digest string over some canonical text.
fn digest_of(text: &str) -> String {
format!("{DIGEST_PREFIX}{}", hex(&sha256(text.as_bytes())))
}
impl SealFile {
/// The path a seal lives at.
///
/// # Arguments
///
/// * `layout` - the course layout.
/// * `assessment_id` - the assessment id.
///
/// # Returns
///
/// `seals/<id>.yaml` under the course root.
pub fn path(layout: &Layout, assessment_id: &str) -> PathBuf {
layout
.root
.join(SEAL_DIR)
.join(format!("{assessment_id}.yaml"))
}
/// Loads a seal.
///
/// # Arguments
///
/// * `path` - the YAML file.
///
/// # Returns
///
/// The parsed seal.
///
/// # Errors
///
/// Returns a load error.
pub fn load(path: &Path) -> Result<SealFile> {
yaml::read(path)
}
/// Loads every seal in a directory, oldest administration first.
///
/// # Arguments
///
/// * `dir` - the seals directory.
///
/// # Returns
///
/// The seals, empty when the directory does not exist.
///
/// # Errors
///
/// Propagates load failures, including a seal that does not parse.
pub fn load_all(dir: &Path) -> Result<Vec<SealFile>> {
let mut out = Vec::new();
for path in crate::yaml::list_yaml(dir)? {
out.push(SealFile::load(&path)?);
}
out.sort_by(|a, b| {
a.seal
.date
.cmp(&b.seal.date)
.then(a.seal.assessment.cmp(&b.seal.assessment))
});
Ok(out)
}
/// Loads the seal for an assessment, if one has been written.
///
/// Absence is not an error. A course that has never sealed anything should
/// still ingest, with the permutation derived from the record instead.
///
/// # Arguments
///
/// * `layout` - the course layout.
/// * `assessment_id` - the assessment id.
///
/// # Returns
///
/// The seal, or `None`.
///
/// # Errors
///
/// Returns a load error when the file exists but does not parse.
pub fn find(layout: &Layout, assessment_id: &str) -> Result<Option<SealFile>> {
let path = SealFile::path(layout, assessment_id);
if path.is_file() {
Ok(Some(SealFile::load(&path)?))
} else {
Ok(None)
}
}
/// Writes the seal out.
///
/// # Arguments
///
/// * `path` - the destination.
///
/// # Errors
///
/// Returns [`Error::Io`] on a write failure.
pub fn save(&self, path: &Path) -> Result<()> {
yaml::write(path, self)
}
/// One form's block.
///
/// # Arguments
///
/// * `id` - the form id, matched case-insensitively.
///
/// # Returns
///
/// The form, or `None`.
pub fn form(&self, id: &str) -> Option<&SealedForm> {
self.forms.iter().find(|f| f.id.eq_ignore_ascii_case(id))
}
/// One item's block, by recorded question number.
///
/// # Arguments
///
/// * `number` - the recorded number.
///
/// # Returns
///
/// The item, or `None`.
pub fn item(&self, number: u32) -> Option<&SealedItem> {
self.items.iter().find(|i| i.number == number)
}
/// The canonical text this file's digest is computed over.
///
/// Built field by field rather than by serializing the struct, so a change to
/// serde attributes, key order, or YAML quoting cannot invalidate every seal
/// ever written. Unit separators keep concatenation unambiguous.
///
/// # Returns
///
/// The digest input.
pub fn digest_input(&self) -> String {
let mut buf = String::new();
buf.push_str(&format!(
"seal\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}{}\n",
self.schema_version, self.seal.assessment, self.seal.course, self.seal.term
));
for item in &self.items {
// Appended only when present: a seal written before 2.0 has to keep
// producing the input it was digested from, or every older
// administration fails verification.
if let Some(stem) = &item.stem_digest {
buf.push_str(&format!("stem\u{1f}{}\u{1f}{stem}\n", item.number));
}
buf.push_str(&format!(
"item\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}{}\u{1f}",
item.number,
item.item,
item.version.unwrap_or(0),
item.fingerprint,
item.points,
item.bonus,
item.level.map(|l| l.code()).unwrap_or(0),
item.key.join(",")
));
for option in &item.options {
buf.push_str(&format!(
"{}:{}:{};",
option.letter, option.correct, option.digest
));
}
buf.push('\n');
}
for form in &self.forms {
buf.push_str(&format!("formref\u{1f}{}\u{1f}{}\n", form.id, form.digest));
}
buf
}
/// Renames the items this seal froze, keeping the digest honest.
///
/// The only sanctioned way to rewrite a seal. It recomputes the digest,
/// because the ids are part of what the digest covers, and records the
/// previous one in [`SealMeta::superseded_digests`], because a recomputed
/// digest with no trace of the recompute is a record that claims never to
/// have been touched.
///
/// # Arguments
///
/// * `rename` - a map from old item id to new.
///
/// # Returns
///
/// How many placements were renamed. Zero leaves the file alone, digest
/// included.
pub fn rename_items(&mut self, rename: &BTreeMap<String, String>) -> usize {
let mut renamed = 0;
for item in &mut self.items {
if let Some(new) = rename.get(&item.item) {
item.item = new.clone();
renamed += 1;
}
}
if renamed == 0 {
return 0;
}
let previous = std::mem::take(&mut self.seal.digest);
self.seal.superseded_digests.push(previous);
self.seal.digest = self.recompute_digest();
renamed
}
/// Recomputes this file's digest from its own contents.
///
/// # Returns
///
/// The digest string.
pub fn recompute_digest(&self) -> String {
digest_of(&self.digest_input())
}
/// Checks that the file has not been edited since it was written.
///
/// This catches a hand-edit of the seal itself, which is a different failure
/// from the bank drifting and deserves a different message.
///
/// # Returns
///
/// Findings, empty when the file is internally consistent.
pub fn check_self(&self) -> Vec<Drift> {
let mut out = Vec::new();
let expected = self.recompute_digest();
if expected != self.seal.digest {
out.push(Drift::new(
Severity::High,
"seal-digest",
format!(
"the seal's own digest does not match its contents ({} recorded, {} \
computed); the file was edited after it was written",
short(&self.seal.digest),
short(&expected)
),
));
}
for form in &self.forms {
let expected = form_digest(form);
if expected != form.digest {
out.push(Drift::new(
Severity::High,
"seal-form-digest",
format!(
"form {}'s permutation block was edited after sealing",
form.id
),
));
}
}
out
}
/// Compares the seal against the live course.
///
/// Everything this reports is a reason to distrust an analysis that pools the
/// sealed administration with anything else, and most of it is a reason to
/// distrust per-option feedback sent to a student.
///
/// # Arguments
///
/// * `catalog` - the loaded course, as it stands now.
/// * `record` - the assessment record, as it stands now.
///
/// # Returns
///
/// Findings, worst first, empty when nothing has moved.
pub fn verify(&self, catalog: &Catalog, record: &AssessmentFile) -> Vec<Drift> {
let mut out = self.check_self();
let rebuilt = build(
catalog,
record,
&Options {
content: self.seal.content,
forms: self.forms.iter().map(|f| f.id.clone()).collect(),
},
);
let rebuilt = match rebuilt {
Ok(r) => r,
Err(e) => {
out.push(Drift::new(
Severity::High,
"seal-unbuildable",
format!("the course no longer produces this assessment at all: {e}"),
));
return out;
}
};
if rebuilt.seal.digest == self.seal.digest {
return out;
}
// The digests differ, so say what differs rather than that they do.
let sealed: BTreeMap<u32, &SealedItem> = self.items.iter().map(|i| (i.number, i)).collect();
let live: BTreeMap<u32, &SealedItem> =
rebuilt.items.iter().map(|i| (i.number, i)).collect();
for (number, was) in &sealed {
let Some(now) = live.get(number) else {
out.push(Drift::new(
Severity::High,
"item-removed",
format!(
"question {number} ({}) is no longer on the assessment record",
was.item
),
));
continue;
};
if was.item != now.item {
out.push(Drift::new(
Severity::High,
"item-replaced",
format!("question {number} was {} and is now {}", was.item, now.item),
));
continue;
}
if was.fingerprint != now.fingerprint {
out.push(Drift::new(
Severity::High,
"item-edited",
format!(
"question {number} ({}) was edited after administration; the students saw \
a different version of the stem or options",
was.item
),
));
}
if was.key != now.key {
out.push(Drift::new(
Severity::High,
"key-changed",
format!(
"question {number} ({}) was keyed {} and is now keyed {}",
was.item,
was.key.join(""),
now.key.join("")
),
));
}
if was.options.len() != now.options.len() {
out.push(Drift::new(
Severity::High,
"option-count-changed",
format!(
"question {number} ({}) had {} options and now has {}, so every printed \
letter on every form means something else",
was.item,
was.options.len(),
now.options.len()
),
));
}
if was.points != now.points {
out.push(Drift::new(
Severity::Medium,
"points-changed",
format!(
"question {number} was worth {} point(s) and is now worth {}",
was.points, now.points
),
));
}
if was.learning_targets != now.learning_targets {
out.push(Drift::new(
Severity::Low,
"objectives-retagged",
format!(
"question {number} ({}) was retagged; per-objective results for this \
administration were computed against the old tags",
was.item
),
));
}
if was.level != now.level {
out.push(Drift::new(
Severity::Low,
"level-changed",
format!("question {number} ({}) changed level", was.item),
));
}
}
for number in live.keys() {
if !sealed.contains_key(number) {
out.push(Drift::new(
Severity::Medium,
"item-added",
format!("question {number} was added to the record after sealing"),
));
}
}
for form in &self.forms {
let Some(now) = rebuilt.form(&form.id) else {
out.push(Drift::new(
Severity::High,
"form-removed",
format!("form {} is no longer declared on the record", form.id),
));
continue;
};
if now.digest != form.digest {
let reason = if now.seed != form.seed {
format!(
"its seed changed from {} to {}, so every option moved",
form.seed, now.seed
)
} else if now.shuffle_options != form.shuffle_options
|| now.shuffle_items != form.shuffle_items
{
"its shuffle settings changed".to_string()
} else {
"the items behind it changed".to_string()
};
out.push(Drift::new(
Severity::High,
"form-permutation-changed",
format!(
"form {} no longer permutes the way it did when it was printed: {reason}",
form.id
),
));
}
}
out.sort_by_key(|d| std::cmp::Reverse(d.severity));
out
}
}
/// How a form's printed answer key is distributed.
///
/// A shuffle is uniform per item and says nothing about the sequence it produces
/// across a form. Over thirty-odd questions it will occasionally produce a run
/// students notice: six consecutive questions keying to `A` is unremarkable
/// probabilistically and very remarkable at a desk, where it reads as a mistake
/// and makes a student who is sure of all six doubt the last three.
///
/// Sealing is the moment to look, because it is the last moment before printing.
/// Changing a form's seed reshuffles it; nothing else has to change.
#[derive(Debug, Clone)]
pub struct Balance {
/// The form id.
pub form: String,
/// How many questions key to each printed letter.
pub counts: BTreeMap<String, usize>,
/// The longest run of consecutive questions sharing a printed key, as
/// `(letter, first number, last number)`.
pub longest_run: Option<(String, u32, u32)>,
/// How long that run was. Stored rather than derived from the two numbers,
/// which are recorded numbers and need not be contiguous.
pub run: usize,
/// How many questions were counted.
pub total: usize,
}
impl Balance {
/// The length of the longest run.
pub fn run_length(&self) -> usize {
self.run
}
/// Advisories worth printing before the form is printed.
///
/// Deliberately quiet. Both thresholds are set where the pattern stops being
/// something only a statistician would see: a run of five, and a letter
/// carrying more than twice its share.
///
/// # Returns
///
/// One message per observation, empty when the form looks unremarkable.
pub fn notes(&self) -> Vec<String> {
let mut out = Vec::new();
if self.total == 0 {
return out;
}
if self.run_length() >= 5 {
if let Some((letter, first, last)) = &self.longest_run {
out.push(format!(
"form {}: questions {first} through {last} all key to {letter}. That is {} in \
a row, which students read as an error in the key. Re-seed the form if you \
have not printed it yet",
self.form,
self.run_length()
));
}
}
let share = self.total as f64 / self.counts.len().max(1) as f64;
for (letter, count) in &self.counts {
if *count as f64 > share * 2.0 && *count >= 4 {
out.push(format!(
"form {}: {count} of {} questions key to {letter}",
self.form, self.total
));
}
}
out
}
}
/// Measures one form's printed key.
///
/// # Arguments
///
/// * `form` - the sealed form.
///
/// # Returns
///
/// The balance. Questions with no single keyed letter, such as a
/// multiple-response item, are skipped rather than guessed at.
pub fn balance(form: &SealedForm) -> Balance {
let mut counts: BTreeMap<String, usize> = BTreeMap::new();
let mut sequence: Vec<(u32, String)> = Vec::new();
for question in &form.questions {
if question.printed_key.len() != 1 {
continue;
}
let letter = question.printed_key[0].clone();
*counts.entry(letter.clone()).or_insert(0) += 1;
sequence.push((question.number, letter));
}
let mut longest: Option<(String, u32, u32)> = None;
let mut best = 0usize;
let mut run = 1usize;
for i in 1..sequence.len() {
if sequence[i].1 == sequence[i - 1].1 {
run += 1;
if run > best {
best = run;
longest = Some((
sequence[i].1.clone(),
sequence[i + 1 - run].0,
sequence[i].0,
));
}
} else {
run = 1;
}
}
Balance {
form: form.id.clone(),
total: sequence.len(),
counts,
longest_run: longest,
run: best.max(if sequence.is_empty() { 0 } else { 1 }),
}
}
/// The first eight hex characters of a digest, for messages.
///
/// # Arguments
///
/// * `digest` - the full digest string.
///
/// # Returns
///
/// A short form such as `sha256:1a2b3c4d`.
pub fn short(digest: &str) -> String {
match digest.split_once(':') {
Some((algorithm, hex)) => {
format!("{algorithm}:{}", &hex[..hex.len().min(8)])
}
None => digest.chars().take(8).collect(),
}
}
fn default_version() -> String {
SCHEMA_VERSION.to_string()
}
fn yes() -> bool {
true
}
fn is_false(b: &bool) -> bool {
!*b
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn printed_letters_run_past_z() {
assert_eq!(printed_letter(0), "A");
assert_eq!(printed_letter(3), "D");
assert_eq!(printed_letter(25), "Z");
assert_eq!(printed_letter(26), "AA");
}
fn sample() -> SealFile {
let mut file = SealFile {
schema_version: "1.0".into(),
seal: SealMeta {
assessment: "e1".into(),
title: "Exam 1".into(),
course: "BIOSC 1000".into(),
term: "2026f".into(),
date: None,
sealed_on: Date::today(),
generator: Generator {
tool: "coursebank".into(),
version: "test".into(),
},
content: true,
dropped: Vec::new(),
digest: String::new(),
superseded_digests: Vec::new(),
},
items: vec![SealedItem {
number: 1,
item: "b::q-1".into(),
version: Some(1),
stem_digest: None,
fingerprint: "abc".into(),
points: 1.0,
bonus: false,
level: Some(Level::Remember),
learning_targets: vec!["lo-a".into()],
key: vec!["B".into()],
stem: Some("Stem.".into()),
options: vec![
SealedOption {
letter: "A".into(),
correct: false,
credit: None,
digest: "d1".into(),
text: Some("First".into()),
},
SealedOption {
letter: "B".into(),
correct: true,
credit: None,
digest: "d2".into(),
text: Some("Second".into()),
},
],
}],
forms: vec![SealedForm {
id: "A".into(),
seed: 7,
shuffle_items: false,
shuffle_options: true,
digest: String::new(),
questions: vec![SealedQuestion {
position: 1,
number: 1,
item: "b::q-1".into(),
printed_key: vec!["A".into()],
options: vec![
LetterMap {
printed: "A".into(),
canonical: "B".into(),
},
LetterMap {
printed: "B".into(),
canonical: "A".into(),
},
],
}],
}],
};
file.forms[0].digest = form_digest(&file.forms[0]);
file.seal.digest = file.recompute_digest();
file
}
#[test]
fn a_fresh_seal_is_internally_consistent() {
assert!(sample().check_self().is_empty());
}
#[test]
fn editing_the_file_breaks_its_digest() {
let mut file = sample();
file.items[0].key = vec!["A".into()];
let findings = file.check_self();
assert!(
findings.iter().any(|d| d.code == "seal-digest"),
"{findings:?}"
);
}
#[test]
fn editing_a_permutation_breaks_that_forms_digest() {
let mut file = sample();
file.forms[0].questions[0].options[0].canonical = "A".into();
let findings = file.check_self();
assert!(
findings.iter().any(|d| d.code == "seal-form-digest"),
"{findings:?}"
);
}
#[test]
fn the_digest_is_order_sensitive() {
let file = sample();
let mut swapped = file.clone();
swapped.items[0].options.swap(0, 1);
assert_ne!(file.recompute_digest(), swapped.recompute_digest());
}
fn form_keyed(letters: &[&str]) -> SealedForm {
SealedForm {
id: "B".into(),
seed: 1,
shuffle_items: false,
shuffle_options: true,
digest: String::new(),
questions: letters
.iter()
.enumerate()
.map(|(i, letter)| SealedQuestion {
position: i as u32 + 1,
number: i as u32 + 1,
item: format!("b::q-{i}"),
printed_key: vec![letter.to_string()],
options: Vec::new(),
})
.collect(),
}
}
#[test]
fn a_long_run_of_one_letter_is_named_with_its_question_numbers() {
// Form B of a real exam did this: questions 2 through 7 all keyed to A.
let form = form_keyed(&["C", "A", "A", "A", "A", "A", "A", "C", "B"]);
let balance = balance(&form);
assert_eq!(balance.run_length(), 6);
assert_eq!(
balance.longest_run,
Some(("A".to_string(), 2, 7)),
"the run is reported by question number, not by index"
);
let notes = balance.notes();
assert!(notes.iter().any(|n| n.contains("2 through 7")), "{notes:?}");
}
#[test]
fn an_ordinary_form_says_nothing() {
let form = form_keyed(&["A", "B", "C", "D", "A", "C", "B", "D", "C", "A", "D", "B"]);
assert!(balance(&form).notes().is_empty());
}
#[test]
fn a_form_with_no_questions_does_not_panic() {
let balance = balance(&form_keyed(&[]));
assert_eq!(balance.run_length(), 0);
assert!(balance.notes().is_empty());
}
#[test]
fn short_digests_keep_the_algorithm() {
assert_eq!(short("sha256:0123456789abcdef"), "sha256:01234567");
}
#[test]
fn renaming_an_item_recomputes_the_digest_and_says_it_did() {
let mut seal = sample();
seal.seal.digest = seal.recompute_digest();
let original = seal.seal.digest.clone();
assert!(seal.check_self().is_empty());
let mut rename = BTreeMap::new();
rename.insert(seal.items[0].item.clone(), "q-renamed".to_string());
assert_eq!(seal.rename_items(&rename), 1);
// The digest covers the ids, so it has to move — and the file has to
// admit that it moved rather than looking untouched.
assert_eq!(seal.items[0].item, "q-renamed");
assert_ne!(seal.seal.digest, original);
assert_eq!(seal.seal.superseded_digests, vec![original]);
assert!(seal.check_self().is_empty(), "{:?}", seal.check_self());
// A rename that matches nothing leaves the file entirely alone.
let steady = seal.seal.digest.clone();
assert_eq!(seal.rename_items(&BTreeMap::new()), 0);
assert_eq!(seal.seal.digest, steady);
assert_eq!(seal.seal.superseded_digests.len(), 1);
}
#[test]
fn round_trips_through_yaml() {
let file = sample();
let text = serde_yaml_ng::to_string(&file).unwrap();
let back: SealFile = serde_yaml_ng::from_str(&text).unwrap();
assert_eq!(back.seal.digest, file.seal.digest);
assert!(back.check_self().is_empty());
assert_eq!(back.form("a").map(|f| f.seed), Some(7));
}
}