feat: improve worksheet
This commit is contained in:
+96
-11
@@ -352,10 +352,20 @@ fn validate_item(
|
||||
issues.push("version must be at least 1".into());
|
||||
}
|
||||
|
||||
// --- options -----------------------------------------------------------
|
||||
if it.options.len() < 2 {
|
||||
// --- options ---
|
||||
// An open-response item takes no options; its answer lives in `solution`.
|
||||
// Every other format needs at least two things to choose between.
|
||||
if it.format.has_options() {
|
||||
if it.options.len() < 2 {
|
||||
issues.push(format!(
|
||||
"needs at least 2 options, has {}",
|
||||
it.options.len()
|
||||
));
|
||||
}
|
||||
} else if !it.options.is_empty() {
|
||||
issues.push(format!(
|
||||
"needs at least 2 options, has {}",
|
||||
"{} items take no options, but {} were given; put the answer in `solution`",
|
||||
it.format.as_str(),
|
||||
it.options.len()
|
||||
));
|
||||
}
|
||||
@@ -418,7 +428,7 @@ fn validate_item(
|
||||
}
|
||||
}
|
||||
|
||||
// --- key ---------------------------------------------------------------
|
||||
// --- key -----
|
||||
let keys = it.key_indices();
|
||||
match it.format {
|
||||
Format::SingleBestAnswer => {
|
||||
@@ -448,9 +458,14 @@ fn validate_item(
|
||||
issues.push("true_false needs exactly one keyed option".into());
|
||||
}
|
||||
}
|
||||
Format::OpenResponse => {
|
||||
if !keys.is_empty() {
|
||||
issues.push("open_response items have no keyed option".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- level and process must agree -------------------------------------
|
||||
// --- level and process must agree --------
|
||||
if let Some(p) = it.cognitive_process {
|
||||
if !it.level.allows(p) {
|
||||
issues.push(format!(
|
||||
@@ -461,7 +476,7 @@ fn validate_item(
|
||||
}
|
||||
}
|
||||
|
||||
// --- design plausibility ----------------------------------------------
|
||||
// --- design plausibility -----------
|
||||
if let Some(d) = &it.design {
|
||||
if let Some(x) = d.expected_difficulty {
|
||||
if !(0.0..=1.0).contains(&x) {
|
||||
@@ -479,7 +494,7 @@ fn validate_item(
|
||||
}
|
||||
}
|
||||
|
||||
// --- calibration plausibility -----------------------------------------
|
||||
// --- calibration plausibility ------
|
||||
if let Some(c) = &it.calibration {
|
||||
if let Some(p) = c.p_value {
|
||||
if !(0.0..=1.0).contains(&p) {
|
||||
@@ -514,7 +529,7 @@ fn validate_item(
|
||||
}
|
||||
}
|
||||
|
||||
// --- history must be coherent -----------------------------------------
|
||||
// --- history must be coherent ------
|
||||
let mut last_version = 0u32;
|
||||
for (i, h) in it.history.iter().enumerate() {
|
||||
if h.version <= last_version {
|
||||
@@ -533,7 +548,7 @@ fn validate_item(
|
||||
));
|
||||
}
|
||||
|
||||
// --- retirement -------------------------------------------------------
|
||||
// --- retirement ----
|
||||
if it.retired.is_some() && it.status != Status::Retired {
|
||||
issues.push(format!(
|
||||
"has a `retired` block but status is `{}`",
|
||||
@@ -541,7 +556,7 @@ fn validate_item(
|
||||
));
|
||||
}
|
||||
|
||||
// --- approval gate ----------------------------------------------------
|
||||
// --- approval gate -------
|
||||
// Approval is what permits an item onto a graded assessment, so it is the
|
||||
// right place to require that the item is fully sourced and designed.
|
||||
if it.status == Status::Approved {
|
||||
@@ -557,9 +572,23 @@ fn validate_item(
|
||||
if it.design.is_none() {
|
||||
issues.push("approved items must carry a design block".into());
|
||||
}
|
||||
// An open-response item is graded from its solution, so approving one with
|
||||
// neither a model answer nor a rubric would leave nothing to mark it by.
|
||||
if !it.format.has_options() {
|
||||
let gradeable = it
|
||||
.solution
|
||||
.as_ref()
|
||||
.is_some_and(|s| s.model_answer.is_some() || !s.rubric.is_empty());
|
||||
if !gradeable {
|
||||
issues.push(
|
||||
"approved open_response items need a solution with a model_answer or a rubric"
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- cross-file references --------------------------------------------
|
||||
// --- cross-file references ---------
|
||||
if let Some(c) = course {
|
||||
for lo in &it.learning_objectives {
|
||||
match c.learning_objectives.get(lo) {
|
||||
@@ -592,6 +621,15 @@ fn validate_item(
|
||||
issues.push(format!("unknown stimulus `{st}`"));
|
||||
}
|
||||
}
|
||||
// A citation that names a reference key must name a real one, so a review
|
||||
// pointer in the solutions document never resolves to nothing.
|
||||
for citation in it.solution.iter().flat_map(|s| &s.review) {
|
||||
if let Some(key) = &citation.reference {
|
||||
if !c.references.contains_key(key) {
|
||||
issues.push(format!("solution.review cites unknown reference `{key}`"));
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(floor) = c.policy.partial_credit_floor_level {
|
||||
for o in &it.options {
|
||||
if o.is_partial() && it.level < floor {
|
||||
@@ -644,6 +682,53 @@ mod tests {
|
||||
assert!(b.validate(None).is_empty(), "{:?}", b.validate(None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_response_validates_without_options_and_rejects_them() {
|
||||
// No options is fine, and no key is required.
|
||||
let ok = bank(
|
||||
r#"
|
||||
- id: q-a-op-001
|
||||
status: draft
|
||||
level: 2
|
||||
format: open_response
|
||||
stem: Explain the first law.
|
||||
solution:
|
||||
model_answer: Energy is conserved.
|
||||
"#,
|
||||
);
|
||||
assert!(ok.validate(None).is_empty(), "{:?}", ok.validate(None));
|
||||
|
||||
// Giving an open-response item options is the mistake, and so is approving
|
||||
// one with nothing to grade it by.
|
||||
let bad = bank(
|
||||
r#"
|
||||
- id: q-a-op-002
|
||||
status: approved
|
||||
level: 2
|
||||
format: open_response
|
||||
cognitive_process: explain
|
||||
learning_objectives: [lo-x]
|
||||
sources: [{ lecture: L1.1 }]
|
||||
design: { rationale: r }
|
||||
stem: Explain the first law.
|
||||
options:
|
||||
- { id: A, text: a, correct: true }
|
||||
- { id: B, text: b }
|
||||
"#,
|
||||
);
|
||||
let issues = bad.validate(None);
|
||||
assert!(
|
||||
issues.iter().any(|i| i.contains("take no options")),
|
||||
"{issues:?}"
|
||||
);
|
||||
assert!(
|
||||
issues
|
||||
.iter()
|
||||
.any(|i| i.contains("model_answer or a rubric")),
|
||||
"{issues:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn catches_missing_and_multiple_keys() {
|
||||
let b = bank(
|
||||
|
||||
Reference in New Issue
Block a user