feat: improve worksheet

This commit is contained in:
alexm committed 2026-08-10 01:52:15 -04:00
1 parent f475c630e0
commit 327ac371e4
17 files changed
+1469 -49

No files matched your search

+96 -11
View File
@@ -352,10 +352,20 @@ fn validate_item(
issues.push("version must be at least 1".into());
}
// --- options -----------------------------------------------------------
if it.options.len() < 2 {
// --- options ---
// An open-response item takes no options; its answer lives in `solution`.
// Every other format needs at least two things to choose between.
if it.format.has_options() {
if it.options.len() < 2 {
issues.push(format!(
"needs at least 2 options, has {}",
it.options.len()
));
}
} else if !it.options.is_empty() {
issues.push(format!(
"needs at least 2 options, has {}",
"{} items take no options, but {} were given; put the answer in `solution`",
it.format.as_str(),
it.options.len()
));
}
@@ -418,7 +428,7 @@ fn validate_item(
}
}
// --- key ---------------------------------------------------------------
// --- key -----
let keys = it.key_indices();
match it.format {
Format::SingleBestAnswer => {
@@ -448,9 +458,14 @@ fn validate_item(
issues.push("true_false needs exactly one keyed option".into());
}
}
Format::OpenResponse => {
if !keys.is_empty() {
issues.push("open_response items have no keyed option".into());
}
}
}
// --- level and process must agree -------------------------------------
// --- level and process must agree --------
if let Some(p) = it.cognitive_process {
if !it.level.allows(p) {
issues.push(format!(
@@ -461,7 +476,7 @@ fn validate_item(
}
}
// --- design plausibility ----------------------------------------------
// --- design plausibility -----------
if let Some(d) = &it.design {
if let Some(x) = d.expected_difficulty {
if !(0.0..=1.0).contains(&x) {
@@ -479,7 +494,7 @@ fn validate_item(
}
}
// --- calibration plausibility -----------------------------------------
// --- calibration plausibility ------
if let Some(c) = &it.calibration {
if let Some(p) = c.p_value {
if !(0.0..=1.0).contains(&p) {
@@ -514,7 +529,7 @@ fn validate_item(
}
}
// --- history must be coherent -----------------------------------------
// --- history must be coherent ------
let mut last_version = 0u32;
for (i, h) in it.history.iter().enumerate() {
if h.version <= last_version {
@@ -533,7 +548,7 @@ fn validate_item(
));
}
// --- retirement -------------------------------------------------------
// --- retirement ----
if it.retired.is_some() && it.status != Status::Retired {
issues.push(format!(
"has a `retired` block but status is `{}`",
@@ -541,7 +556,7 @@ fn validate_item(
));
}
// --- approval gate ----------------------------------------------------
// --- approval gate -------
// Approval is what permits an item onto a graded assessment, so it is the
// right place to require that the item is fully sourced and designed.
if it.status == Status::Approved {
@@ -557,9 +572,23 @@ fn validate_item(
if it.design.is_none() {
issues.push("approved items must carry a design block".into());
}
// An open-response item is graded from its solution, so approving one with
// neither a model answer nor a rubric would leave nothing to mark it by.
if !it.format.has_options() {
let gradeable = it
.solution
.as_ref()
.is_some_and(|s| s.model_answer.is_some() || !s.rubric.is_empty());
if !gradeable {
issues.push(
"approved open_response items need a solution with a model_answer or a rubric"
.into(),
);
}
}
}
// --- cross-file references --------------------------------------------
// --- cross-file references ---------
if let Some(c) = course {
for lo in &it.learning_objectives {
match c.learning_objectives.get(lo) {
@@ -592,6 +621,15 @@ fn validate_item(
issues.push(format!("unknown stimulus `{st}`"));
}
}
// A citation that names a reference key must name a real one, so a review
// pointer in the solutions document never resolves to nothing.
for citation in it.solution.iter().flat_map(|s| &s.review) {
if let Some(key) = &citation.reference {
if !c.references.contains_key(key) {
issues.push(format!("solution.review cites unknown reference `{key}`"));
}
}
}
if let Some(floor) = c.policy.partial_credit_floor_level {
for o in &it.options {
if o.is_partial() && it.level < floor {
@@ -644,6 +682,53 @@ mod tests {
assert!(b.validate(None).is_empty(), "{:?}", b.validate(None));
}
#[test]
fn open_response_validates_without_options_and_rejects_them() {
// No options is fine, and no key is required.
let ok = bank(
r#"
- id: q-a-op-001
status: draft
level: 2
format: open_response
stem: Explain the first law.
solution:
model_answer: Energy is conserved.
"#,
);
assert!(ok.validate(None).is_empty(), "{:?}", ok.validate(None));
// Giving an open-response item options is the mistake, and so is approving
// one with nothing to grade it by.
let bad = bank(
r#"
- id: q-a-op-002
status: approved
level: 2
format: open_response
cognitive_process: explain
learning_objectives: [lo-x]
sources: [{ lecture: L1.1 }]
design: { rationale: r }
stem: Explain the first law.
options:
- { id: A, text: a, correct: true }
- { id: B, text: b }
"#,
);
let issues = bad.validate(None);
assert!(
issues.iter().any(|i| i.contains("take no options")),
"{issues:?}"
);
assert!(
issues
.iter()
.any(|i| i.contains("model_answer or a rubric")),
"{issues:?}"
);
}
#[test]
fn catches_missing_and_multiple_keys() {
let b = bank(
+312 -1
View File
@@ -19,7 +19,11 @@
//! it. That keeps bank files readable and reviewable in a pull request while
//! still letting statistics accumulate across terms.
use serde::{Deserialize, Serialize};
use std::fmt;
use serde::de::{self, MapAccess, Visitor};
use serde::ser::SerializeMap;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use crate::date::Date;
use crate::hash::fingerprint;
@@ -79,8 +83,25 @@ pub struct Item {
/// The answer options in canonical order. Shuffling happens at export time
/// per form, never here, so the bank stays diffable.
///
/// Empty for a [`Format::OpenResponse`] item, which is answered in free text
/// and graded from its [`Solution`] instead. A choice format must still supply
/// at least two, which [`crate::bank::BankFile::validate`] enforces; leaving
/// them out is reported there, with every other problem, rather than failing
/// the parse on its own.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub options: Vec<Choice>,
/// The worked solution: a model answer, an explanation a student can learn
/// from, and, for an open-response item, the rubric it is graded against.
///
/// This is what the solutions document renders and what a paper answer key
/// prints. It is withheld from any question paper and from the exam payload,
/// the same way an option's `correct` flag is, so a document built for the
/// student cannot leak it.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub solution: Option<Solution>,
/// Objectives this item measures, as ids into the course registry.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub learning_objectives: Vec<String>,
@@ -239,6 +260,214 @@ pub struct Source {
pub recording_seconds: Option<u32>,
}
/// A pointer from an item into the course reference registry: where to read more,
/// or what to revisit after missing the item.
///
/// It holds a citation key and a locator rather than a restated citation, so a
/// reference is written once in `course.yaml` and a changed edition is a single
/// edit. The exporters resolve it against
/// [`crate::course::CourseFile::references`] into a short label such as `KKW §6.1`,
/// linked when the location resolves to a URL. This is the same pointer a lecture
/// [`crate::course::Reading`] uses, kept lean here because an item cites a reading;
/// it does not restate one.
///
/// A citation may also be written as a bare string, which lands unparsed in `text`
/// and serializes back out as a string, so a bank that stored readings as plain
/// strings keeps loading and round-trips byte-for-byte.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Citation {
/// Citation key into the course reference registry.
pub reference: Option<String>,
/// Where inside the work: `§6.1`, `pp. 212-219`, `fig. 4`.
pub locator: Option<String>,
/// Appended to the reference's `base_url` to reach this location.
pub path: Option<String>,
/// A full URL, when the location is not under the reference's `base_url`.
pub url: Option<String>,
/// A citation written as a bare string, held unparsed.
pub text: Option<String>,
}
impl Citation {
/// A short display string that needs no reference lookup.
///
/// Prefers the unparsed `text`, then the key and locator. A caller that holds
/// the course, such as an exporter, can resolve a nicer label and a link; this
/// is the fallback for one that does not.
///
/// # Returns
///
/// The display string, empty when the citation carries nothing.
pub fn display(&self) -> String {
if let Some(text) = &self.text {
return text.clone();
}
match (&self.reference, &self.locator) {
(Some(k), Some(l)) => format!("{k} {l}"),
(Some(k), None) => k.clone(),
(None, Some(l)) => l.clone(),
(None, None) => String::new(),
}
}
}
/// Writes a citation as a mapping, or as a bare string when that is all it holds.
impl Serialize for Citation {
fn serialize<S: Serializer>(&self, s: S) -> std::result::Result<S::Ok, S::Error> {
if let Some(text) = &self.text {
if self.reference.is_none()
&& self.locator.is_none()
&& self.path.is_none()
&& self.url.is_none()
{
return s.serialize_str(text);
}
}
let mut map = s.serialize_map(None)?;
if let Some(v) = &self.reference {
map.serialize_entry("ref", v)?;
}
if let Some(v) = &self.locator {
map.serialize_entry("locator", v)?;
}
if let Some(v) = &self.path {
map.serialize_entry("path", v)?;
}
if let Some(v) = &self.url {
map.serialize_entry("url", v)?;
}
if let Some(v) = &self.text {
map.serialize_entry("text", v)?;
}
map.end()
}
}
/// Accepts a citation written either as a mapping or as a bare string.
impl<'de> Deserialize<'de> for Citation {
fn deserialize<D: Deserializer<'de>>(d: D) -> std::result::Result<Citation, D::Error> {
/// The mapping form, with the field set kept in one place.
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Mapping {
#[serde(rename = "ref", default)]
reference: Option<String>,
#[serde(default)]
locator: Option<String>,
#[serde(default)]
path: Option<String>,
#[serde(default)]
url: Option<String>,
#[serde(default)]
text: Option<String>,
}
struct V;
impl<'a> Visitor<'a> for V {
type Value = Citation;
fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("a citation mapping with a `ref`, or a plain citation string")
}
fn visit_str<E: de::Error>(self, v: &str) -> std::result::Result<Citation, E> {
Ok(Citation {
text: Some(v.to_string()),
..Citation::default()
})
}
fn visit_map<M: MapAccess<'a>>(
self,
map: M,
) -> std::result::Result<Citation, M::Error> {
let m = Mapping::deserialize(de::value::MapAccessDeserializer::new(map))?;
Ok(Citation {
reference: m.reference,
locator: m.locator,
path: m.path,
url: m.url,
text: m.text,
})
}
}
d.deserialize_any(V)
}
}
/// One line of a grading rubric for an open-response item.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct RubricCriterion {
/// What earns the points, e.g. "states H = U + PV" or "compares to ~2.5 kJ/mol".
pub description: String,
/// Points for this line. Absent lets a grader decide; when present, the lines
/// are meant to sum to the item's point value.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub points: Option<f64>,
}
/// The worked solution to an item: what the answer is, why, and how it is graded.
///
/// One place, versioned with the question, holds everything a student learns from
/// after the fact and everything a grader marks an open response against. For a
/// choice item the per-option [`Choice::explanation`] says why each option is right
/// or wrong; the solution adds the single worked line of reasoning a solutions
/// document leads with. For an [`Format::OpenResponse`] item the solution is the
/// whole answer, because there are no options to annotate.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Solution {
/// The model answer, in the authoring markup. For an open-response item this is
/// the response a full-credit student would write; for a choice item it is an
/// optional one-line statement of the key in words.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub model_answer: Option<String>,
/// The worked reasoning a student can learn from: the derivation, the estimate,
/// the argument for the key over its neighbours. This is the body of the
/// solutions document.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub explanation: Option<String>,
/// How an open response is graded, one criterion per line.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub rubric: Vec<RubricCriterion>,
/// Responses a short constructed answer would be accepted as. Shown in the
/// solutions document as accepted answers, and the hook for automated grading
/// later.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub accepted: Vec<String>,
/// Where to look again after missing this item, as citations into the course
/// reference registry. Resolved and linked by the exporters.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub review: Vec<Citation>,
}
impl Solution {
/// Whether the solution carries anything worth rendering.
///
/// Used to decide whether a solutions entry has a body to print, so an item
/// with an empty `solution:` block is treated as having none.
pub fn is_empty(&self) -> bool {
self.model_answer.is_none()
&& self.explanation.is_none()
&& self.rubric.is_empty()
&& self.accepted.is_empty()
&& self.review.is_empty()
}
/// Total of the rubric line points, when every line carries one.
///
/// # Returns
///
/// The sum, or `None` if any line omits its points or the rubric is empty.
pub fn rubric_points(&self) -> Option<f64> {
if self.rubric.is_empty() {
return None;
}
self.rubric.iter().map(|c| c.points).sum::<Option<f64>>()
}
}
/// A figure or data file reproduced with an item.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
@@ -473,6 +702,7 @@ impl Item {
stimulus: None,
stem: stem.to_string(),
options,
solution: None,
learning_objectives: Vec::new(),
sources: Vec::new(),
topics: Vec::new(),
@@ -538,6 +768,14 @@ impl Item {
self.key_indices().len() > 1
}
/// Whether the item presents selectable options, per its [`Format`].
///
/// `false` for an [`Format::OpenResponse`] item. Callers that would otherwise
/// index `options` or read a key should branch on this first.
pub fn has_options(&self) -> bool {
self.format.has_options()
}
/// The display title, falling back to a truncated stem.
///
/// # Returns
@@ -821,4 +1059,77 @@ options:
IrtModel::ThreePl
);
}
#[test]
fn open_response_item_parses_without_options() {
let it = item(
r#"
id: q-enthalpy-op-001
status: draft
level: 2
format: open_response
stem: Explain why, at constant pressure, the heat exchanged equals the enthalpy change.
solution:
model_answer: >-
At constant pressure the P dV expansion work is folded into H = U + PV, so the
heat q_p equals the change in H.
rubric:
- { description: "states H = U + PV", points: 1 }
- { description: "identifies q_p with the enthalpy change", points: 1 }
review:
- { ref: kuriyan2012molecules, locator: "§6.4", path: "6/A/#4" }
"#,
);
assert_eq!(it.format, Format::OpenResponse);
assert!(it.options.is_empty());
assert!(!it.has_options());
assert!(it.key_letters().is_empty());
let sol = it.solution.as_ref().expect("has a solution");
assert!(!sol.is_empty());
assert_eq!(sol.rubric_points(), Some(2.0));
assert_eq!(sol.review.len(), 1);
assert_eq!(
sol.review[0].reference.as_deref(),
Some("kuriyan2012molecules")
);
}
#[test]
fn a_solution_serializes_only_what_it_holds() {
let it = item(
r#"
id: q-demo-op-002
status: draft
level: 2
format: open_response
stem: State the first law.
solution:
model_answer: The total energy of an isolated system is constant.
"#,
);
let yaml = serde_yaml_ng::to_string(&it).expect("serializes");
// Open-response items carry no options key, and an empty rubric is omitted.
assert!(!yaml.contains("options:"), "no options key:\n{yaml}");
assert!(!yaml.contains("rubric"), "empty rubric omitted:\n{yaml}");
assert!(yaml.contains("model_answer:"));
}
#[test]
fn a_citation_round_trips_as_string_or_mapping() {
// A bare string stays a bare string.
let bare: Citation = serde_yaml_ng::from_str("\"KKW §6.4 (course reserve)\"").unwrap();
assert_eq!(bare.text.as_deref(), Some("KKW §6.4 (course reserve)"));
assert_eq!(bare.display(), "KKW §6.4 (course reserve)");
let back = serde_yaml_ng::to_string(&bare).unwrap();
assert_eq!(back.trim(), "KKW §6.4 (course reserve)");
// A mapping keeps its fields, and `ref` is the key's YAML spelling.
let mapped: Citation =
serde_yaml_ng::from_str("{ ref: kuriyan2012molecules, locator: \"§6.4\" }").unwrap();
assert_eq!(mapped.reference.as_deref(), Some("kuriyan2012molecules"));
assert_eq!(mapped.display(), "kuriyan2012molecules §6.4");
let back = serde_yaml_ng::to_string(&mapped).unwrap();
assert!(back.contains("ref: kuriyan2012molecules"));
assert!(back.contains("locator:"));
}
}
+50
View File
@@ -423,17 +423,53 @@ pub enum Format {
MultipleResponse,
/// Two options, True and False.
TrueFalse,
/// A free-text answer the student writes rather than selects.
///
/// It carries no options and is not machine-scored. What a grader marks it
/// against, and what a solutions document shows, lives in the item's
/// [`crate::item::Solution`]: a model answer and, when the item is worth more
/// than a point, a rubric. This is the format for "explain", "derive", and
/// "estimate" prompts that a set of distractors would trivialize.
OpenResponse,
}
impl Format {
/// Every response format.
pub const ALL: [Format; 4] = [
Format::SingleBestAnswer,
Format::MultipleResponse,
Format::TrueFalse,
Format::OpenResponse,
];
/// The QTI question type Canvas expects for this format.
pub fn qti_type(self) -> &'static str {
match self {
Format::SingleBestAnswer => "multiple_choice_question",
Format::MultipleResponse => "multiple_answers_question",
Format::TrueFalse => "true_false_question",
Format::OpenResponse => "essay_question",
}
}
/// The snake_case token used in YAML.
pub fn as_str(self) -> &'static str {
match self {
Format::SingleBestAnswer => "single_best_answer",
Format::MultipleResponse => "multiple_response",
Format::TrueFalse => "true_false",
Format::OpenResponse => "open_response",
}
}
/// Whether items in this format present selectable options.
///
/// `false` only for [`Format::OpenResponse`]. Validation, assembly, and the
/// exporters branch on this rather than on the variant, so the day a second
/// free-text format is added it inherits the no-options handling for free.
pub fn has_options(self) -> bool {
!matches!(self, Format::OpenResponse)
}
}
/// How strongly an item is expected to separate strong from weak students.
@@ -633,4 +669,18 @@ mod tests {
Status::InReview
);
}
#[test]
fn open_response_is_the_only_format_without_options() {
assert_eq!(
serde_json::from_str::<Format>("\"open_response\"").unwrap(),
Format::OpenResponse
);
assert_eq!(Format::OpenResponse.as_str(), "open_response");
assert_eq!(Format::OpenResponse.qti_type(), "essay_question");
assert!(!Format::OpenResponse.has_options());
for f in Format::ALL {
assert_eq!(f.has_options(), f != Format::OpenResponse);
}
}
}