1242 lines
46 KiB
Rust
1242 lines
46 KiB
Rust
// SPDX-License-Identifier: Prosperity-3.0.0
|
|
// Copyright Scientific Computing Studio
|
|
// Source: https://git.scient.ing/education/coursebank
|
|
|
|
//! Emitting JSON Schema for the YAML formats.
|
|
//!
|
|
//! The point of this module is autocomplete. Every editor with a YAML language
|
|
//! server reads a `# yaml-language-server: $schema=...` modeline, and once it does,
|
|
//! writing an item becomes a matter of tabbing through valid `cognitive_process`
|
|
//! values instead of looking them up. That is a much better authoring experience
|
|
//! than running a validator afterward and reading a list of typos.
|
|
//!
|
|
//! The schemas are written by hand rather than derived from the Rust types. That is
|
|
//! a real cost — two definitions to keep in step — bought for two reasons: the
|
|
//! schema can carry prose descriptions aimed at whoever is writing the item, which
|
|
//! is what shows up in editor tooltips, and it can encode `enum` value lists that a
|
|
//! generic derivation would emit as bare strings. The crate's own validation
|
|
//! remains authoritative; the schema is for the editor.
|
|
|
|
use serde_json::{Value, json};
|
|
|
|
use crate::course::SCHEMA_VERSION;
|
|
use crate::error::Result;
|
|
use crate::taxonomy::{CognitiveProcess, ErrorType, Flag, Level};
|
|
|
|
/// The base URL schemas refer to each other by.
|
|
const BASE: &str = "https://coursebank.dev/schema";
|
|
|
|
/// The three schema kinds.
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub enum Kind {
|
|
/// `course.yaml`.
|
|
Course,
|
|
/// `banks/*.yaml`.
|
|
Bank,
|
|
/// `assessments/*.yaml`.
|
|
Assessment,
|
|
}
|
|
|
|
impl Kind {
|
|
/// All three kinds.
|
|
pub const ALL: [Kind; 3] = [Kind::Course, Kind::Bank, Kind::Assessment];
|
|
|
|
/// The file name a schema is written to.
|
|
pub fn filename(self) -> &'static str {
|
|
match self {
|
|
Kind::Course => "course.schema.json",
|
|
Kind::Bank => "bank.schema.json",
|
|
Kind::Assessment => "assessment.schema.json",
|
|
}
|
|
}
|
|
|
|
/// The modeline that points an editor at this schema.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `relative_path` - the path from the YAML file to the schema directory.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// A comment line to place at the top of the YAML file.
|
|
pub fn modeline(self, relative_path: &str) -> String {
|
|
format!(
|
|
"# yaml-language-server: $schema={}/{}",
|
|
relative_path.trim_end_matches('/'),
|
|
self.filename()
|
|
)
|
|
}
|
|
}
|
|
|
|
/// Builds a schema.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `kind` - which schema to build.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The schema as JSON.
|
|
pub fn schema(kind: Kind) -> Value {
|
|
match kind {
|
|
Kind::Course => course_schema(),
|
|
Kind::Bank => bank_schema(),
|
|
Kind::Assessment => assessment_schema(),
|
|
}
|
|
}
|
|
|
|
/// Writes all three schemas to a directory.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `dir` - the destination directory.
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The paths written.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns [`crate::error::Error::Io`] on a write failure.
|
|
pub fn write_all(dir: &std::path::Path) -> Result<Vec<std::path::PathBuf>> {
|
|
std::fs::create_dir_all(dir).map_err(|e| crate::error::Error::io(dir, e))?;
|
|
let mut written = Vec::new();
|
|
for kind in Kind::ALL {
|
|
let path = dir.join(kind.filename());
|
|
crate::yaml::write_json(&path, &schema(kind))?;
|
|
written.push(path);
|
|
}
|
|
Ok(written)
|
|
}
|
|
|
|
/// The string values of an enum, for a schema `enum` list.
|
|
fn strings<T: AsRef<str>>(values: &[T]) -> Value {
|
|
Value::Array(
|
|
values
|
|
.iter()
|
|
.map(|v| Value::String(v.as_ref().to_string()))
|
|
.collect(),
|
|
)
|
|
}
|
|
|
|
/// A schema fragment for a required non-empty string.
|
|
fn text(description: &str) -> Value {
|
|
json!({ "type": "string", "minLength": 1, "description": description })
|
|
}
|
|
|
|
/// A schema fragment for an array of strings.
|
|
fn string_array(description: &str) -> Value {
|
|
json!({
|
|
"type": "array",
|
|
"items": { "type": "string" },
|
|
"description": description
|
|
})
|
|
}
|
|
|
|
/// A schema fragment for a proportion in `[0, 1]`.
|
|
fn proportion(description: &str) -> Value {
|
|
json!({
|
|
"type": "number",
|
|
"minimum": 0.0,
|
|
"maximum": 1.0,
|
|
"description": description
|
|
})
|
|
}
|
|
|
|
/// A schema fragment for a `YYYY-MM-DD` date.
|
|
fn date(description: &str) -> Value {
|
|
json!({
|
|
"type": "string",
|
|
"pattern": "^\\d{4}-\\d{2}-\\d{2}$",
|
|
"description": description
|
|
})
|
|
}
|
|
|
|
/// The level enum, with the taxonomy spelled out in the description so it appears
|
|
/// in editor tooltips.
|
|
fn level() -> Value {
|
|
let descriptions: Vec<String> = Level::ALL
|
|
.iter()
|
|
.map(|l| format!("{} = {} ({})", l.code(), l.name(), l.blurb()))
|
|
.collect();
|
|
json!({
|
|
"type": "integer",
|
|
"minimum": 1,
|
|
"maximum": 5,
|
|
"description": format!("Cognitive level. {}", descriptions.join("; "))
|
|
})
|
|
}
|
|
|
|
/// The cognitive process enum, grouped by level in the description.
|
|
fn cognitive_process() -> Value {
|
|
let all: Vec<&str> = CognitiveProcess::ALL.iter().map(|p| p.as_str()).collect();
|
|
let by_level: Vec<String> = Level::ALL
|
|
.iter()
|
|
.map(|l| {
|
|
let names: Vec<&str> = l.processes().iter().map(|p| p.as_str()).collect();
|
|
format!("level {}: {}", l.code(), names.join(", "))
|
|
})
|
|
.collect();
|
|
json!({
|
|
"type": "string",
|
|
"enum": strings(&all),
|
|
"description": format!(
|
|
"The specific cognitive operation. Must belong to the item's level — {}.",
|
|
by_level.join("; ")
|
|
)
|
|
})
|
|
}
|
|
|
|
/// The distractor error-type enum, with each gloss in the description.
|
|
fn error_type() -> Value {
|
|
let all: Vec<&str> = ErrorType::ALL.iter().map(|e| e.as_str()).collect();
|
|
let glosses: Vec<String> = ErrorType::ALL
|
|
.iter()
|
|
.map(|e| format!("{} — {}", e.as_str(), e.gloss()))
|
|
.collect();
|
|
json!({
|
|
"type": "string",
|
|
"enum": strings(&all),
|
|
"description": format!(
|
|
"What kind of mistake this distractor is designed to catch. {}",
|
|
glosses.join("; ")
|
|
)
|
|
})
|
|
}
|
|
|
|
/// The schema for the `course` identity block.
|
|
fn course_identity_schema() -> Value {
|
|
json!({
|
|
"type": "object",
|
|
"required": ["code", "title", "term"],
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"code": text("Course code, e.g. BIOSC 1540."),
|
|
"title": text("Course title."),
|
|
"term": text("Term, e.g., 2026s."),
|
|
"institution": { "type": "string" },
|
|
"instructors": string_array("Instructor names."),
|
|
"slug": {
|
|
"type": "string",
|
|
"description": "Short identifier used in file names and administration ids. \
|
|
Derived from the code if omitted."
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The schema for the course-wide policy block.
|
|
fn policy_schema() -> Value {
|
|
json!({
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"description": "Course-wide defaults and rules that validation enforces.",
|
|
"properties": {
|
|
"points_per_item": {
|
|
"type": "number",
|
|
"exclusiveMinimum": 0.0,
|
|
"description": "Default points for an item that does not set its own."
|
|
},
|
|
"options_per_item": {
|
|
"type": "integer",
|
|
"minimum": 2,
|
|
"description": "Expected option count; the linter flags items that differ."
|
|
},
|
|
"bonus_levels": {
|
|
"type": "array",
|
|
"items": level(),
|
|
"description": "Levels a bonus item may be drawn from."
|
|
},
|
|
"allow_partial_credit": {
|
|
"type": "boolean",
|
|
"description": "Whether any option may carry partial credit. When false, \
|
|
validation rejects items that do."
|
|
},
|
|
"partial_credit_floor_level": level(),
|
|
"mastery_threshold": proportion(
|
|
"Rate at which an objective counts as met. 0.75 is a common choice."
|
|
),
|
|
"min_items_for_mastery": {
|
|
"type": "integer",
|
|
"minimum": 1,
|
|
"description": "Below this many items on an objective, reports say 'not enough \
|
|
evidence' rather than classifying."
|
|
},
|
|
"grade_scale": {
|
|
"type": "array",
|
|
"description": "Letter-grade bands. Only the lower bound of each is recorded; a \
|
|
band runs up to the next one. Set this and a class report bins \
|
|
scores by letter rather than by ten-point interval.",
|
|
"items": {
|
|
"type": "object",
|
|
"required": ["letter", "min"],
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"letter": {
|
|
"type": "string",
|
|
"description": "The letter as it appears on a transcript."
|
|
},
|
|
"min": {
|
|
"type": "number",
|
|
"minimum": 0,
|
|
"maximum": 100,
|
|
"description": "Lowest percentage earning this letter, inclusive."
|
|
},
|
|
"gpa": {
|
|
"type": "number",
|
|
"minimum": 0,
|
|
"description": "Grade points the band carries."
|
|
},
|
|
"attainment": {
|
|
"type": "string",
|
|
"description": "The attainment word attached to the band."
|
|
},
|
|
"group": {
|
|
"type": "string",
|
|
"description": "Colour group for reports; defaults to the letter's \
|
|
first character."
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The schema for one unit.
|
|
fn unit_schema() -> Value {
|
|
json!({
|
|
"type": "object",
|
|
"required": ["id", "title"],
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"id": text("Unit id."),
|
|
"title": text("Unit title."),
|
|
"description": { "type": "string" }
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The schema for one lecture.
|
|
fn lecture_schema() -> Value {
|
|
json!({
|
|
"type": "object",
|
|
"required": ["title"],
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"title": text("Lecture title."),
|
|
"date": date("Date delivered."),
|
|
"unit": { "type": "string", "description": "Unit id." },
|
|
"slides_url": { "type": "string" },
|
|
"readings": {
|
|
"type": "array",
|
|
"description": "Readings assigned with this lecture, in the order you assign \
|
|
them. A plain string is the pre-schema form and still loads.",
|
|
"items": reading_schema()
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The schema for one learning objective.
|
|
fn objective_schema() -> Value {
|
|
json!({
|
|
"type": "object",
|
|
"required": ["text"],
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"text": text("The objective as a student would read it. Start with a verb."),
|
|
"unit": { "type": "string" },
|
|
"lectures": string_array("Lecture ids that cover this."),
|
|
"order": {
|
|
"type": "integer",
|
|
"minimum": 1,
|
|
"description": "Position in teaching order, low first. A lecture page numbers \
|
|
objectives by this; without it they sort by id, which puts \
|
|
an objective before its own prerequisite."
|
|
},
|
|
"level_ceiling": level(),
|
|
"prerequisites": string_array(
|
|
"Objective ids that must come first. Cycles are rejected."
|
|
),
|
|
"tags": string_array("Free-form tags."),
|
|
"assessed": {
|
|
"type": "boolean",
|
|
"description": "Set false for an objective you teach but do not test; coverage \
|
|
reporting will stop flagging it as a gap."
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The schema for one cited work.
|
|
fn reference_schema() -> Value {
|
|
json!({
|
|
"type": "object",
|
|
"required": ["title"],
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"label": text("Short form a reading list shows, such as KKW. One work per label."),
|
|
"kind": {
|
|
"type": "string",
|
|
"enum": strings(&[
|
|
"book", "chapter", "article", "preprint", "thesis",
|
|
"website", "software", "dataset", "video", "other"
|
|
]),
|
|
"description": "Kind of work, following BibTeX entry types."
|
|
},
|
|
"role": {
|
|
"type": "string",
|
|
"enum": strings(&["required", "supplemental"]),
|
|
"description": "required for a course text; supplemental for background."
|
|
},
|
|
"title": text("Full title."),
|
|
"authors": string_array("Authors as `Family, Given`, in printed order."),
|
|
"year": { "type": "integer", "description": "Year of publication." },
|
|
"edition": { "type": "string", "description": "Edition as printed: 7th." },
|
|
"publisher": { "type": "string" },
|
|
"container": { "type": "string", "description": "Journal, edited volume, or series." },
|
|
"volume": { "type": "string" },
|
|
"issue": { "type": "string" },
|
|
"pages": { "type": "string", "description": "Pages of the work, not of a reading." },
|
|
"doi": { "type": "string", "description": "Bare DOI: 10.1038/nature12373." },
|
|
"isbn": { "type": "string" },
|
|
"url": { "type": "string", "description": "Canonical URL for the whole work." },
|
|
"base_url": {
|
|
"type": "string",
|
|
"description": "Prefix a reading's `path` is joined to, so the citation key \
|
|
appears once instead of once per reading."
|
|
},
|
|
"note": { "type": "string", "description": "Access notes: reserve shelf, license." }
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The schema for one reading: a location inside a reference, and what it is for.
|
|
fn reading_schema() -> Value {
|
|
json!({
|
|
"oneOf": [
|
|
{ "type": "string", "description": "The pre-schema form: a citation, unparsed." },
|
|
reading_mapping_schema()
|
|
]
|
|
})
|
|
}
|
|
|
|
/// The mapping form of a reading.
|
|
fn reading_mapping_schema() -> Value {
|
|
json!({
|
|
"type": "object",
|
|
"required": ["ref"],
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"ref": text("Citation key into `references`."),
|
|
"locator": text("Where inside the work: §6.1, pp. 212-219, ch. 3."),
|
|
"path": {
|
|
"type": "string",
|
|
"description": "Joined to the reference's base_url to reach this location."
|
|
},
|
|
"url": {
|
|
"type": "string",
|
|
"description": "Full URL, when base_url does not cover the location."
|
|
},
|
|
"role": {
|
|
"type": "string",
|
|
"enum": strings(&["assigned", "supplemental"]),
|
|
"description": "supplemental means offered but not separately assessed."
|
|
},
|
|
"objectives": string_array(
|
|
"Objective ids this reading serves. A student who misses one of these is \
|
|
pointed here, so the list is what makes study guidance specific."
|
|
),
|
|
"summary": text("What the section contains."),
|
|
"focus": text("What to take from it. This is the sentence a student report quotes."),
|
|
"skip": text("What to gloss, and why it is out of scope."),
|
|
"text": text("A pre-schema citation string, held unparsed.")
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The schema for one shared stimulus.
|
|
fn stimulus_schema() -> Value {
|
|
json!({
|
|
"type": "object",
|
|
"required": ["body"],
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"body": text("The stimulus text, in coursebank markup."),
|
|
"asset": { "type": "string", "description": "Path to an image." },
|
|
"caption": { "type": "string" }
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The course schema.
|
|
fn course_schema() -> Value {
|
|
json!({
|
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
"$id": format!("{BASE}/course.schema.json"),
|
|
"title": "coursebank course file",
|
|
"description": "Course identity, policy, and the registries that item and assessment \
|
|
files reference by id.",
|
|
"type": "object",
|
|
"required": ["course"],
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"schema_version": {
|
|
"type": ["string", "number"],
|
|
"description": format!("Format version; currently {SCHEMA_VERSION}.")
|
|
},
|
|
"course": course_identity_schema(),
|
|
"policy": policy_schema(),
|
|
"units": {
|
|
"type": "array",
|
|
"description": "Course units, in teaching order. That order drives report layout.",
|
|
"items": unit_schema()
|
|
},
|
|
"lectures": {
|
|
"type": "object",
|
|
"description": "Lectures by id. Items cite these so reports can tell a student \
|
|
where to go back to.",
|
|
"additionalProperties": lecture_schema()
|
|
},
|
|
"learning_objectives": {
|
|
"type": "object",
|
|
"description": "Objectives by id. Everything downstream — coverage, mastery, \
|
|
student reports — keys off these.",
|
|
"additionalProperties": objective_schema()
|
|
},
|
|
"stimuli": {
|
|
"type": "object",
|
|
"description": "Shared passages, figures, or data that several items refer to.",
|
|
"additionalProperties": stimulus_schema()
|
|
},
|
|
"references": {
|
|
"type": "object",
|
|
"description": "Works the course cites, by citation key. Readings point in \
|
|
here, so an edition change is one edit.",
|
|
"additionalProperties": reference_schema()
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
/// One option's schema.
|
|
///
|
|
/// Split out from [`item_schema`] rather than inlined, because `serde_json`'s
|
|
/// `json!` macro recurses once per nesting level *and* once per key-value pair. A
|
|
/// single literal describing the whole item exceeded the default recursion limit of
|
|
/// 128, so each subtree gets its own shallow invocation. Keeping them small also
|
|
/// means adding a field later cannot silently reintroduce the problem.
|
|
fn option_schema() -> Value {
|
|
json!({
|
|
"type": "object",
|
|
"required": ["id", "text"],
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"id": {
|
|
"type": "string",
|
|
"pattern": "^[A-H]$",
|
|
"description": "Option letter. Identity, not print position — shuffled forms \
|
|
relabel on the way out."
|
|
},
|
|
"text": text("The option as a student reads it."),
|
|
"correct": { "type": "boolean" },
|
|
"credit": proportion("Partial credit. Requires defensible: true and a defense."),
|
|
"explanation": {
|
|
"type": "string",
|
|
"description": "Why this option is right or wrong. For you, not the student."
|
|
},
|
|
"hint": { "type": "string" },
|
|
"misconception": {
|
|
"type": "string",
|
|
"description": "The specific wrong belief that leads here. This text is what \
|
|
student reports use, so write it as a completion of 'students \
|
|
pick this when ...'."
|
|
},
|
|
"error_type": error_type(),
|
|
"defensible": {
|
|
"type": "boolean",
|
|
"description": "This option has a reading under which it is arguably correct. \
|
|
Required before granting partial credit."
|
|
},
|
|
"defense": {
|
|
"type": "string",
|
|
"description": "The argument for that reading. Required when defensible is true."
|
|
},
|
|
"feedback_student": {
|
|
"type": "string",
|
|
"description": "Shown to a student who chose this, on Canvas and in reports."
|
|
},
|
|
"selection_rate_expected": proportion(
|
|
"How often you expect this to be chosen. Compared against reality."
|
|
)
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The schema for where an item's material was taught.
|
|
fn source_schema() -> Value {
|
|
json!({
|
|
"type": "object",
|
|
"required": ["lecture"],
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"lecture": text("Lecture id from course.yaml."),
|
|
"slides": { "type": "array", "items": { "type": "integer", "minimum": 1 } },
|
|
"readings": string_array("Specific readings."),
|
|
"recording_seconds": { "type": "integer", "minimum": 0 }
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The schema for an attached figure.
|
|
fn asset_schema() -> Value {
|
|
json!({
|
|
"type": "object",
|
|
"required": ["path"],
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"path": text("Path to the image, relative to the course root."),
|
|
"alt": {
|
|
"type": "string",
|
|
"description": "Alt text. Required in practice: the linter flags an asset without \
|
|
it, because an exam question a screen reader cannot convey is not \
|
|
answerable."
|
|
},
|
|
"caption": { "type": "string" }
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The worked solution, and for an open-response item how it is graded.
|
|
fn solution_schema() -> Value {
|
|
json!({
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"description": "The answer, the reasoning, and the rubric. Rendered in the solutions \
|
|
document and the answer key, never in a question paper.",
|
|
"properties": {
|
|
"model_answer": {
|
|
"type": "string",
|
|
"description": "For an open-response item, the response a full-credit student \
|
|
writes; for a choice item, an optional one-line statement of the key."
|
|
},
|
|
"explanation": {
|
|
"type": "string",
|
|
"description": "The worked reasoning a student learns from. The body of the \
|
|
solutions entry."
|
|
},
|
|
"rubric": { "type": "array", "items": rubric_criterion_schema() },
|
|
"accepted": {
|
|
"type": "array",
|
|
"items": { "type": "string" },
|
|
"description": "Responses a short constructed answer is accepted as."
|
|
},
|
|
"review": {
|
|
"type": "array",
|
|
"items": citation_schema(),
|
|
"description": "Where to look again after missing this item."
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
/// One rubric line for an open-response item.
|
|
fn rubric_criterion_schema() -> Value {
|
|
json!({
|
|
"type": "object",
|
|
"required": ["description"],
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"description": text("What earns the points on this line."),
|
|
"points": { "type": "number", "minimum": 0.0 }
|
|
}
|
|
})
|
|
}
|
|
|
|
/// A citation into the reference registry, written as an object or a bare string.
|
|
fn citation_schema() -> Value {
|
|
json!({
|
|
"oneOf": [
|
|
{ "type": "string", "description": "A citation, unparsed." },
|
|
citation_mapping_schema()
|
|
]
|
|
})
|
|
}
|
|
|
|
/// The object form of a citation.
|
|
fn citation_mapping_schema() -> Value {
|
|
json!({
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"ref": text("Citation key into `references`."),
|
|
"locator": { "type": "string", "description": "Where inside the work: §6.1, pp. 4-9." },
|
|
"path": { "type": "string", "description": "Joined to the reference base_url." },
|
|
"url": { "type": "string" },
|
|
"text": { "type": "string" }
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The schema for authored design intent.
|
|
fn design_schema() -> Value {
|
|
json!({
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"description": "What you expected before giving the item. Kept separate from observed \
|
|
statistics so the two can be compared.",
|
|
"properties": {
|
|
"expected_difficulty": proportion("Fraction you expect to answer correctly."),
|
|
"expected_discrimination": {
|
|
"type": "string",
|
|
"enum": ["low", "moderate", "high"]
|
|
},
|
|
"expected_time_seconds": { "type": "number", "exclusiveMinimum": 0.0 },
|
|
"rationale": {
|
|
"type": "string",
|
|
"description": "Why this item exists and what it is meant to catch."
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The schema for one option's observed statistics.
|
|
fn option_stat_schema() -> Value {
|
|
json!({
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"selection_rate": proportion("How often chosen."),
|
|
"point_biserial": { "type": "number", "minimum": -1.0, "maximum": 1.0 },
|
|
"upper_group_rate": proportion("Rate in the top 27%."),
|
|
"lower_group_rate": proportion("Rate in the bottom 27%.")
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The schema for fitted IRT parameters.
|
|
fn irt_schema() -> Value {
|
|
json!({
|
|
"type": "object",
|
|
"required": ["a", "b"],
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"model": { "type": "string", "enum": ["rasch", "2pl", "3pl"] },
|
|
"a": { "type": "number", "description": "Discrimination." },
|
|
"b": { "type": "number", "description": "Difficulty." },
|
|
"c": proportion("Lower asymptote, 3PL only."),
|
|
"se_a": { "type": "number", "minimum": 0.0 },
|
|
"se_b": { "type": "number", "minimum": 0.0 },
|
|
"n": { "type": "integer", "minimum": 0 },
|
|
"bayesian": {
|
|
"type": "boolean",
|
|
"description": "Whether priors were used. On a class-sized sample they should be."
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The schema for pooled observed statistics.
|
|
fn calibration_schema() -> Value {
|
|
let flags: Vec<&str> = Flag::ALL.iter().map(|f| f.as_str()).collect();
|
|
json!({
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"description": "Written by `coursebank calibrate`, not by hand. Pooled across \
|
|
administrations.",
|
|
"properties": {
|
|
"administrations": string_array("Administration ids pooled here."),
|
|
"updated": date("When calibration last ran."),
|
|
"fingerprint": {
|
|
"type": "string",
|
|
"description": "Hash of what a student saw. When it stops matching the item, these \
|
|
statistics describe a different question."
|
|
},
|
|
"n_examinees": { "type": "integer", "minimum": 0 },
|
|
"p_value": proportion("Observed proportion correct."),
|
|
"point_biserial": { "type": "number", "minimum": -1.0, "maximum": 1.0 },
|
|
"discrimination_index": { "type": "number", "minimum": -1.0, "maximum": 1.0 },
|
|
"mean_response_time_seconds": { "type": "number", "minimum": 0.0 },
|
|
"rapid_guess_rate": proportion("Fraction answered faster than readable."),
|
|
"option_stats": {
|
|
"type": "object",
|
|
"additionalProperties": option_stat_schema()
|
|
},
|
|
"irt": irt_schema(),
|
|
"flags": {
|
|
"type": "array",
|
|
"items": { "type": "string", "enum": strings(&flags) }
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The schema for a recorded review decision.
|
|
fn review_schema() -> Value {
|
|
json!({
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"reviewed_by": { "type": "string" },
|
|
"reviewed_on": date("Review date."),
|
|
"action": {
|
|
"type": "string",
|
|
"enum": ["keep", "revise", "award_partial_credit", "correct_key", "retire",
|
|
"monitor"]
|
|
},
|
|
"notes": { "type": "string" }
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The schema for one revision-history entry.
|
|
fn history_schema() -> Value {
|
|
json!({
|
|
"type": "object",
|
|
"required": ["version", "date", "change"],
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"version": { "type": "integer", "minimum": 1 },
|
|
"date": date("When the change was made."),
|
|
"author": { "type": "string" },
|
|
"change": text("What changed and why.")
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The schema for a retirement record.
|
|
fn retirement_schema() -> Value {
|
|
json!({
|
|
"type": "object",
|
|
"required": ["on", "reason"],
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"on": date("Retirement date."),
|
|
"reason": text("Why it was retired."),
|
|
"replaced_by": { "type": "string", "description": "Successor item id." }
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The identity and classification half of the item schema.
|
|
///
|
|
/// Split from [`item_content_properties`] purely to keep each `json!` invocation
|
|
/// short; the two are merged into one `properties` object by [`item_schema`].
|
|
fn item_identity_properties() -> Value {
|
|
json!({
|
|
"id": {
|
|
"type": "string",
|
|
"pattern": "^q-[a-z0-9]+(-[a-z0-9]+)*-[0-9]{3}$",
|
|
"description": "Item id, e.g. q-glycolysis-014. Stable forever: assessment records \
|
|
and stored responses refer to it."
|
|
},
|
|
"version": {
|
|
"type": "integer",
|
|
"minimum": 1,
|
|
"description": "Bump when you change what a student sees. Recorded on every \
|
|
administration so drift is detectable."
|
|
},
|
|
"status": {
|
|
"type": "string",
|
|
"enum": ["draft", "in_review", "needs_revision", "approved", "retired"],
|
|
"description": "Only approved items can be drawn into an assessment."
|
|
},
|
|
"level": level(),
|
|
"cognitive_process": cognitive_process(),
|
|
"format": {
|
|
"type": "string",
|
|
"enum": ["single_best_answer", "multiple_response", "true_false", "open_response"],
|
|
"description": "single_best_answer keys exactly one option; multiple_response keys \
|
|
two or more; open_response takes no options and is graded from its \
|
|
solution."
|
|
},
|
|
"bonus": { "type": "boolean" },
|
|
"points": { "type": "number", "exclusiveMinimum": 0.0 },
|
|
"author": { "type": "string" },
|
|
"notes_private": {
|
|
"type": "string",
|
|
"description": "Never exported anywhere a student can see."
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The content and evidence half of the item schema.
|
|
fn item_content_properties() -> Value {
|
|
json!({
|
|
"title": {
|
|
"type": "string",
|
|
"description": "Short internal label. Never shown to students."
|
|
},
|
|
"stimulus": { "type": "string", "description": "Stimulus id from course.yaml." },
|
|
"stem": text(
|
|
"The question. Ask something specific; the linter flags stems with no task in them."
|
|
),
|
|
"options": {
|
|
"type": "array",
|
|
"minItems": 2,
|
|
"maxItems": 8,
|
|
"description": "Absent for an open_response item; at least two for any choice format.",
|
|
"items": option_schema()
|
|
},
|
|
"solution": solution_schema(),
|
|
"learning_objectives": string_array(
|
|
"Objective ids this item measures. Reports aggregate on these, so an item with none \
|
|
contributes to nothing."
|
|
),
|
|
"sources": {
|
|
"type": "array",
|
|
"description": "Where the material was taught. Drives the 'review this' lines in \
|
|
student reports.",
|
|
"items": source_schema()
|
|
},
|
|
"topics": string_array("Free-form topics, used for blueprint filtering."),
|
|
"prerequisites": string_array("Objective ids a student needs before this item."),
|
|
"assets": { "type": "array", "items": asset_schema() },
|
|
"design": design_schema(),
|
|
"calibration": calibration_schema(),
|
|
"review": review_schema(),
|
|
"history": {
|
|
"type": "array",
|
|
"description": "One entry per version. Versions must increase.",
|
|
"items": history_schema()
|
|
},
|
|
"retired": retirement_schema()
|
|
})
|
|
}
|
|
|
|
/// The item schema fragment, shared by the bank schema.
|
|
///
|
|
/// Assembled from the helpers above rather than written as one literal. See
|
|
/// [`option_schema`] for why.
|
|
fn item_schema() -> Value {
|
|
let mut properties = serde_json::Map::new();
|
|
for half in [item_identity_properties(), item_content_properties()] {
|
|
if let Value::Object(map) = half {
|
|
properties.extend(map);
|
|
}
|
|
}
|
|
json!({
|
|
"type": "object",
|
|
"required": ["id", "level", "stem"],
|
|
"additionalProperties": false,
|
|
"properties": Value::Object(properties)
|
|
})
|
|
}
|
|
|
|
/// The schema for a bank's `bank` metadata block.
|
|
fn bank_meta_schema() -> Value {
|
|
json!({
|
|
"type": "object",
|
|
"required": ["id", "title"],
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"id": text("Bank id, unique within the course."),
|
|
"title": text("Human-readable title."),
|
|
"description": { "type": "string" },
|
|
"scope": bank_scope_schema()
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The schema for what a bank is meant to cover.
|
|
fn bank_scope_schema() -> Value {
|
|
json!({
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"description": "What this bank is meant to cover. Validation warns when an item strays \
|
|
outside it.",
|
|
"properties": {
|
|
"lectures": string_array("Lecture ids."),
|
|
"learning_objectives": string_array("Objective ids."),
|
|
"units": string_array("Unit ids."),
|
|
"topics": string_array("Topics.")
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The schema for per-file item defaults.
|
|
fn bank_defaults_schema() -> Value {
|
|
json!({
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"description": "Applied to every item in the file that does not set the field. Saves \
|
|
repeating yourself; the item always wins.",
|
|
"properties": {
|
|
"author": { "type": "string" },
|
|
"points": { "type": "number", "exclusiveMinimum": 0.0 },
|
|
"options_per_item": { "type": "integer", "minimum": 2 },
|
|
"topics": string_array("Topics added to every item."),
|
|
"lectures": string_array("Lecture ids for items with no sources of their own.")
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The bank schema.
|
|
fn bank_schema() -> Value {
|
|
json!({
|
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
"$id": format!("{BASE}/bank.schema.json"),
|
|
"title": "coursebank item bank",
|
|
"description": "A collection of items. Split banks by unit or topic; ids must be unique \
|
|
across the whole course, not just within a file.",
|
|
"type": "object",
|
|
"required": ["bank", "items"],
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"schema_version": { "type": ["string", "number"] },
|
|
"bank": bank_meta_schema(),
|
|
"defaults": bank_defaults_schema(),
|
|
"items": { "type": "array", "items": item_schema() }
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The schema for the `assessment` metadata block.
|
|
fn assessment_meta_schema() -> Value {
|
|
json!({
|
|
"type": "object",
|
|
"required": ["id", "title"],
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"id": text("Assessment id, e.g. exam-4. Used in administration ids."),
|
|
"title": text("Printed title."),
|
|
"term": { "type": "string", "description": "Defaults to the course term." },
|
|
"kind": {
|
|
"type": "string",
|
|
"enum": ["exam", "quiz", "homework", "practice", "final"],
|
|
"description": "practice is excluded from calibration by default, since \
|
|
conditions differ too much to pool."
|
|
},
|
|
"date": date("Administration date. Drives reuse cooldowns."),
|
|
"platform": { "type": "string", "enum": ["paper", "canvas", "other"] },
|
|
"minutes_allowed": { "type": "number", "exclusiveMinimum": 0.0 },
|
|
"attempts": {
|
|
"type": "integer",
|
|
"description": "Canvas attempt limit; -1 for unlimited."
|
|
},
|
|
"shuffle": { "type": "boolean" },
|
|
"scoring_policy": { "type": "string", "enum": ["keep_highest", "keep_latest"] },
|
|
"instructions": { "type": "string" },
|
|
"notes": { "type": "string" }
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The schema for the blueprint an assessment was drawn to.
|
|
fn blueprint_schema() -> Value {
|
|
json!({
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"description": "The design the form was drawn to satisfy. Kept so the form can be checked \
|
|
against the intent afterward.",
|
|
"properties": {
|
|
"level_counts": {
|
|
"type": "object",
|
|
"description": "How many scored items at each level, keyed by level number.",
|
|
"additionalProperties": { "type": "integer", "minimum": 0 }
|
|
},
|
|
"bonus_counts": {
|
|
"type": "object",
|
|
"additionalProperties": { "type": "integer", "minimum": 0 }
|
|
},
|
|
"objective_minimums": {
|
|
"type": "object",
|
|
"description": "Minimum items per objective. Placed before level quotas, because \
|
|
a coverage requirement is the constraint most likely to become \
|
|
unsatisfiable.",
|
|
"additionalProperties": { "type": "integer", "minimum": 0 }
|
|
},
|
|
"lectures": string_array("Restrict the draw to these lectures."),
|
|
"topics": string_array("Restrict the draw to these topics."),
|
|
"banks": string_array("Restrict the draw to these banks."),
|
|
"max_per_bank": { "type": "integer", "minimum": 1 },
|
|
"cooldown_days": {
|
|
"type": "integer",
|
|
"minimum": 0,
|
|
"description": "Avoid items used within this many days. Relaxed with a warning \
|
|
rather than failing the draw."
|
|
},
|
|
"seed": {
|
|
"type": "integer",
|
|
"minimum": 0,
|
|
"description": "Makes the draw reproducible."
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The schema for one alternate form.
|
|
fn form_schema() -> Value {
|
|
json!({
|
|
"type": "object",
|
|
"required": ["id", "seed"],
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"id": text("Form label, e.g. A."),
|
|
"seed": { "type": "integer", "minimum": 0 },
|
|
"shuffle_items": { "type": "boolean" },
|
|
"shuffle_options": { "type": "boolean" }
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The schema for one question placement.
|
|
fn placement_schema() -> Value {
|
|
json!({
|
|
"type": "object",
|
|
"required": ["number", "item"],
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"number": {
|
|
"type": "integer",
|
|
"minimum": 1,
|
|
"description": "The question number as administered. This is the join key to \
|
|
Gradescope and Canvas exports, so it must not change after the \
|
|
fact."
|
|
},
|
|
"item": text("Item reference, `bank::item-id` or a bare item id."),
|
|
"version": { "type": "integer", "minimum": 1 },
|
|
"fingerprint": {
|
|
"type": "string",
|
|
"description": "What the item looked like when given. Validation warns if the item \
|
|
has since changed."
|
|
},
|
|
"points": { "type": "number", "minimum": 0.0 },
|
|
"bonus": { "type": "boolean" },
|
|
"key": string_array("Keyed option letters as administered."),
|
|
"level": level(),
|
|
"learning_objectives": string_array("Objectives as administered."),
|
|
"credit_overrides": {
|
|
"type": "object",
|
|
"description": "Partial credit decided after the fact, by option letter. Recording \
|
|
it here keeps the rescoring decision with the administration it \
|
|
applies to.",
|
|
"additionalProperties": { "type": "number", "minimum": 0.0, "maximum": 1.0 }
|
|
},
|
|
"dropped": {
|
|
"type": "boolean",
|
|
"description": "Excluded from scoring and from statistics."
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The assessment schema.
|
|
fn assessment_schema() -> Value {
|
|
json!({
|
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
"$id": format!("{BASE}/assessment.schema.json"),
|
|
"title": "coursebank assessment record",
|
|
"description": "A record of what was given, to whom, and when. This is the join between \
|
|
item banks and grading data, and it is the source of truth for reuse \
|
|
history — there is no separate ledger to drift out of step.",
|
|
"type": "object",
|
|
"required": ["assessment"],
|
|
"additionalProperties": false,
|
|
"properties": {
|
|
"schema_version": { "type": ["string", "number"] },
|
|
"assessment": assessment_meta_schema(),
|
|
"blueprint": blueprint_schema(),
|
|
"forms": {
|
|
"type": "array",
|
|
"description": "Alternate forms. Option order is derived from the seed rather than \
|
|
stored, so every export of a form agrees.",
|
|
"items": form_schema()
|
|
},
|
|
"items": {
|
|
"type": "array",
|
|
"description": "One entry per question, in number order.",
|
|
"items": placement_schema()
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn every_schema_is_well_formed() {
|
|
for kind in Kind::ALL {
|
|
let s = schema(kind);
|
|
assert!(s["$schema"].is_string(), "{kind:?} needs a $schema");
|
|
assert!(s["$id"].is_string(), "{kind:?} needs an $id");
|
|
assert_eq!(s["type"], "object");
|
|
assert!(
|
|
s["additionalProperties"] == false,
|
|
"{kind:?} must reject unknown keys, matching the Rust deserializer"
|
|
);
|
|
// Round-trips as JSON.
|
|
let text = serde_json::to_string(&s).unwrap();
|
|
let back: Value = serde_json::from_str(&text).unwrap();
|
|
assert_eq!(back, s);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn the_level_enum_lists_all_five() {
|
|
let l = level();
|
|
assert_eq!(l["minimum"], 1);
|
|
assert_eq!(l["maximum"], 5);
|
|
let description = l["description"].as_str().unwrap();
|
|
for level in Level::ALL {
|
|
assert!(
|
|
description.contains(level.name()),
|
|
"missing {}",
|
|
level.name()
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn the_process_enum_matches_the_taxonomy() {
|
|
let p = cognitive_process();
|
|
let listed = p["enum"].as_array().unwrap();
|
|
assert_eq!(listed.len(), CognitiveProcess::ALL.len());
|
|
assert!(listed.contains(&Value::String("differentiate".into())));
|
|
}
|
|
|
|
#[test]
|
|
fn the_item_schema_constrains_ids_and_options() {
|
|
let item = item_schema();
|
|
let props = &item["properties"];
|
|
assert!(props["id"]["pattern"].as_str().unwrap().starts_with("^q-"));
|
|
assert_eq!(props["options"]["minItems"], 2);
|
|
assert_eq!(props["options"]["maxItems"], 8);
|
|
assert_eq!(
|
|
props["options"]["items"]["properties"]["id"]["pattern"],
|
|
"^[A-H]$"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn modelines_point_at_the_right_file() {
|
|
assert_eq!(
|
|
Kind::Bank.modeline("../.coursebank/schema"),
|
|
"# yaml-language-server: $schema=../.coursebank/schema/bank.schema.json"
|
|
);
|
|
// A trailing slash must not double up.
|
|
assert!(
|
|
Kind::Course
|
|
.modeline("schema/")
|
|
.ends_with("schema/course.schema.json")
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn schemas_write_to_disk() {
|
|
let dir = std::env::temp_dir().join(format!("cb-schema-{}", std::process::id()));
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
let written = write_all(&dir).unwrap();
|
|
assert_eq!(written.len(), 3);
|
|
for path in &written {
|
|
assert!(path.exists());
|
|
let text = std::fs::read_to_string(path).unwrap();
|
|
let _: Value = serde_json::from_str(&text).expect("valid JSON on disk");
|
|
}
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
}
|