feat(migrate): glossary from 1.0 terms.yaml and values.yaml

`coursebank migrate glossary --terms terms.yaml --values values.yaml`
writes glossary.yaml. Ids gain their term- and val- prefixes, course
becomes term and meaning becomes definition, a value's term: gains the
prefix of the id it names, and first and lectures are dropped; each one
the targets now contradict is reported, as is each entry no target lists
yet and each field 2.1 has no place for.

Like the other migrations it copies lines, so comments, block scalars,
and flow mappings survive. Every entry is read back and compared with the
same conversion done on the parsed source, falling back to the serializer
when the copy differs. An existing glossary.yaml is added to, never
rewritten, and an entry the course already declares is skipped, so a
second run is a no-op.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
alexmandClaude committed 2026-11-16 12:23:39 -05:00
1 parent 50a1e083c1
commit 6c07122fc5
5 files changed
+1221

No files matched your search

+19
View File
@@ -200,6 +200,25 @@ pub(crate) enum MigrateCommand {
#[arg(long)]
dry_run: bool,
},
/// Bring a 1.0 terms.yaml and values.yaml into the 2.1 glossary.
///
/// Writes glossary.yaml. Ids gain their `term-` and `val-` prefixes,
/// `course` becomes `term` and `meaning` becomes `definition`, and `first`
/// and `lectures` are dropped, since 2.1 derives them from the targets
/// that list an entry; each one that disagrees is reported. The old files
/// are left in place, and an entry the course already declares is
/// skipped, so a second run changes nothing.
Glossary {
/// The 1.0 terms file.
#[arg(long, value_name = "FILE")]
terms: Option<PathBuf>,
/// The 1.0 values file.
#[arg(long, value_name = "FILE")]
values: Option<PathBuf>,
/// Show what would be written, and write nothing.
#[arg(long)]
dry_run: bool,
},
/// Rewrite pre-2.0 `bank::item` ids as the item ids they name.
///
/// Touches assessment records, seals, and the response store. Everything
+65
View File
@@ -118,9 +118,74 @@ pub(crate) fn migrate(cli: &Cli, sub: &MigrateCommand) -> Result<Outcome> {
MigrateCommand::References { dry_run } => migrate_references(cli, *dry_run),
MigrateCommand::Order { dry_run } => migrate_order(cli, *dry_run),
MigrateCommand::Counters { dry_run } => migrate_counters(cli, *dry_run),
MigrateCommand::Glossary {
terms,
values,
dry_run,
} => migrate_glossary(cli, terms.as_deref(), values.as_deref(), *dry_run),
}
}
/// Brings a 1.0 terms.yaml and values.yaml into glossary.yaml.
fn migrate_glossary(
cli: &Cli,
terms: Option<&Path>,
values: Option<&Path>,
dry_run: bool,
) -> Result<Outcome> {
let report = migrate::glossary(&cli.course, terms, values, !dry_run)?;
let list = |heading: &str, lines: &[String]| {
if !lines.is_empty() {
println!("{heading}");
for line in lines {
println!(" {line}");
}
println!();
}
};
let renamed: Vec<String> = report
.renamed
.iter()
.map(|(old, new)| format!("{old} -> {new}"))
.collect();
list("renamed:", &renamed);
list(
"dropped `first` and `lectures` that the targets now contradict:",
&report.derived,
);
list("dropped, no 2.1 field:", &report.dropped);
list("relinked:", &report.relinked);
list("no target lists these yet:", &report.unlisted);
let present: Vec<String> = report
.present
.iter()
.map(|(id, path)| format!("{id} ({})", path.display()))
.collect();
list("already declared, skipped:", &present);
let Some(text) = &report.text else {
println!("nothing to do: the course already declares every entry");
return Ok(Outcome::Ok);
};
if dry_run {
println!(
"{} would read:\n\n{text}\nnothing written",
report.path.display()
);
return Ok(Outcome::Ok);
}
println!(
"wrote {} entr{} to {}\n\nNext:\n coursebank validate\n coursebank lint \
# glossary-unused lists entries nothing points at yet\n\n\
The old files are untouched; delete them once the pipelines read the registry.",
report.added.len(),
if report.added.len() == 1 { "y" } else { "ies" },
report.path.display()
);
Ok(Outcome::Ok)
}
/// Drops the trailing counter from every item id.
fn migrate_counters(cli: &Cli, dry_run: bool) -> Result<Outcome> {
let (rename, touched) = migrate::counters(&cli.course, !dry_run)?;
+4
View File
@@ -22,6 +22,7 @@
//! | [`references`] | prose notes into citation fields | a DOI in a note is a link nobody can follow |
//! | [`order`] | removes the `order:` integers | objective order is derived, and targets have none |
//! | [`counters`] | drops the `-001` from item ids | a per-bank sequence number is not part of a name |
//! | [`glossary()`] | 1.0 `terms.yaml` and `values.yaml` into `glossary.yaml` | an entry is defined once, by id, where targets can point at it |
//!
//! # What the option migration does not touch
//!
@@ -74,6 +75,9 @@ use crate::layout::Layout;
use crate::store::{self, Store};
use crate::yaml;
mod glossary;
pub use glossary::{GlossaryMigration, glossary};
/// The width past which a flow sequence is written as a block list instead.
const FLOW_WIDTH: usize = 96;
+975
View File
@@ -0,0 +1,975 @@
//! `migrate glossary`: a 1.0 `terms.yaml` and `values.yaml` into the 2.1
//! registries.
//!
//! The same textual approach as the rest of [`crate::migrate`]. Each entry's
//! lines are copied and re-indented from a list item to a map entry, with three
//! edits: the `id` becomes the key and gains its prefix, `course` and `meaning`
//! are renamed, and `first` and `lectures`, which 2.1 derives from the targets
//! that list an entry, are dropped. Comments, folded scalars, and flow mappings
//! like `{ term: ten-fold }` survive.
//!
//! Every copied entry is parsed back and compared with the same conversion done
//! on the parsed source. An entry whose copied text does not read back the same
//! (a flow-mapping list item, a quoted key) is written by the serializer
//! instead, so the file is right even where it is not pretty. The whole file is
//! checked the same way before anything is written, and the course is loaded
//! again after.
//!
//! An entry the course already declares, in `glossary.yaml` or any other
//! fragment, is skipped, so a second run is a no-op.
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use serde::Serialize;
use serde::de::DeserializeOwned;
use serde_yaml_ng::{Mapping, Value as Yaml};
use super::{indent_of, key_at, relative, scalar};
use crate::course::CourseFile;
use crate::course::fragment::{Fragment, Section};
use crate::course::glossary::{self, RefKind, TERM_PREFIX, Term, VALUE_PREFIX, Value};
use crate::error::{Error, Result};
use crate::layout::Layout;
use crate::yaml;
/// What `migrate glossary` writes, and what it found on the way.
#[derive(Debug, Clone, Default)]
pub struct GlossaryMigration {
/// The file it writes, relative to the course root.
pub path: PathBuf,
/// The file's new contents; `None` when there is nothing to add.
pub text: Option<String>,
/// Ids added, terms first, each in source order.
pub added: Vec<String>,
/// Old id to new id, for every added entry whose id gained a prefix.
pub renamed: Vec<(String, String)>,
/// Entries the course already declares, with the file that declares them.
pub present: Vec<(String, PathBuf)>,
/// One message per dropped `first` or `lectures` that differs from what
/// the targets now say.
pub derived: Vec<String>,
/// Added entries that no target lists yet.
pub unlisted: Vec<String>,
/// One message per field dropped because 2.1 has no place for it.
pub dropped: Vec<String>,
/// One message per value whose `term:` was rewritten to the prefixed id.
pub relinked: Vec<String>,
}
/// Which 1.0 file an entry came from.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Kind {
Term,
Value,
}
impl Kind {
/// The top-level key, the same in 1.0 and 2.1.
fn section(self) -> &'static str {
match self {
Kind::Term => "terms",
Kind::Value => "values",
}
}
/// The prefix a 2.1 id carries.
fn prefix(self) -> &'static str {
match self {
Kind::Term => TERM_PREFIX,
Kind::Value => VALUE_PREFIX,
}
}
/// The registry an id of this kind lives in.
fn registry(self) -> Section {
match self {
Kind::Term => Section::Terms,
Kind::Value => Section::Values,
}
}
/// The 1.0 field that 2.1 derives from the targets instead.
fn derived(self) -> &'static str {
match self {
Kind::Term => "first",
Kind::Value => "lectures",
}
}
/// The 2.1 name of a renamed 1.0 field.
fn renamed(self, key: &str) -> Option<&'static str> {
match (self, key) {
(Kind::Term, "course") => Some("term"),
(Kind::Term, "meaning") => Some("definition"),
_ => None,
}
}
/// Every field the 2.1 entry has; anything else is dropped and reported.
fn fields(self) -> &'static [&'static str] {
match self {
Kind::Term => &[
"term",
"definition",
"kind",
"textbook",
"variants",
"symbol",
"units",
"exam_wording",
"status",
"tags",
"notes_private",
],
Kind::Value => &[
"name",
"value",
"range",
"units",
"say",
"exact",
"term",
"source",
"conflicts",
"recall",
"status",
"tags",
"notes_private",
],
}
}
}
/// One 1.0 entry, read leniently and converted to 2.1 fields.
#[derive(Debug, Clone)]
struct Entry {
/// The id as the 1.0 file wrote it.
old: String,
/// The 2.1 id.
id: String,
/// The 2.1 fields, renamed, unknown ones gone.
fields: Mapping,
/// What the dropped `first` or `lectures` said.
derived: Option<Yaml>,
/// One message per unknown field dropped.
dropped: Vec<String>,
/// The value's `term:` before it gained its prefix.
relinked: Option<String>,
}
/// One list item in a 1.0 file, as lines.
#[derive(Debug, Clone, Default)]
struct Item {
/// Comment lines directly above the item.
lead: Vec<String>,
/// The `- ` line and everything under it.
lines: Vec<String>,
/// The column of the item's dash.
dash: usize,
}
/// Writes a 1.0 `terms.yaml` and `values.yaml` into the course's
/// `glossary.yaml`.
///
/// Ids gain their `term-` and `val-` prefixes, `course` becomes `term` and
/// `meaning` becomes `definition`, a value's `term:` gains the prefix of the id
/// it names, and `first` and `lectures` are dropped: 2.1 derives them from the
/// targets that list an entry, and each one that disagrees with what the
/// targets now say is reported. A field 2.1 has no place for is dropped and
/// reported. The old files are left alone.
///
/// An existing `glossary.yaml` is added to, never rewritten: new entries go at
/// the end of its `terms:` and `values:` sections, and everything already in
/// it stays byte for byte.
///
/// # Arguments
///
/// * `root` - the course directory.
/// * `terms` - the 1.0 terms file, if migrating one.
/// * `values` - the 1.0 values file, if migrating one.
/// * `write` - whether to write, or only to report.
///
/// # Returns
///
/// What was (or would be) written, and the report.
///
/// # Errors
///
/// Returns [`Error::Usage`] when neither file is given, [`Error::Invalid`] for
/// entries that cannot be migrated (no id, an id that would not be valid, a
/// field 2.1 would reject), and [`Error::Other`] if the written file would not
/// read back as migrated. Nothing is written in any of those cases.
pub fn glossary(
root: &Path,
terms: Option<&Path>,
values: Option<&Path>,
write: bool,
) -> Result<GlossaryMigration> {
if terms.is_none() && values.is_none() {
return Err(Error::usage(
"nothing to migrate: give --terms, --values, or both",
));
}
let layout = Layout::new(root);
let course = CourseFile::load_dir(root)?;
let path = layout.glossary_file();
let mut report = GlossaryMigration {
path: relative(root, &path),
..GlossaryMigration::default()
};
let mut new_terms: BTreeMap<String, Term> = BTreeMap::new();
let mut new_values: BTreeMap<String, Value> = BTreeMap::new();
let mut sections: Vec<(Kind, Vec<Vec<String>>)> = Vec::new();
if let Some(file) = terms {
let chunks = convert(
root,
file,
Kind::Term,
&course,
&mut report,
&mut new_terms,
|f| &f.terms,
)?;
sections.push((Kind::Term, chunks));
}
if let Some(file) = values {
let chunks = convert(
root,
file,
Kind::Value,
&course,
&mut report,
&mut new_values,
|f| &f.values,
)?;
sections.push((Kind::Value, chunks));
}
if report.added.is_empty() {
return Ok(report);
}
let before = if path.is_file() {
Some(std::fs::read_to_string(&path).map_err(|e| Error::io(&path, e))?)
} else {
None
};
let names: Vec<String> = [terms, values]
.into_iter()
.flatten()
.map(file_name)
.collect();
let mut text = before.clone().unwrap_or_else(|| header(&names));
for (kind, chunks) in &sections {
if !chunks.is_empty() {
text = insert(&text, kind.section(), chunks)?;
}
}
verify(before.as_deref(), &text, &new_terms, &new_values)?;
if write {
yaml::write_text(&path, &text)?;
// The fragment reads back; this checks the course still assembles with
// it, and puts the old file back when it does not.
if let Err(e) = CourseFile::load_dir(root) {
match &before {
Some(old) => yaml::write_text(&path, old)?,
None => std::fs::remove_file(&path).map_err(|e| Error::io(&path, e))?,
}
return Err(e);
}
}
report.text = Some(text);
Ok(report)
}
/// Converts one 1.0 file into the entry chunks to write.
///
/// Fills in the report and `expected`, the typed entries the written file must
/// read back as.
fn convert<T>(
root: &Path,
file: &Path,
kind: Kind,
course: &CourseFile,
report: &mut GlossaryMigration,
expected: &mut BTreeMap<String, T>,
pick: fn(&Fragment) -> &BTreeMap<String, T>,
) -> Result<Vec<Vec<String>>>
where
T: DeserializeOwned + Serialize + PartialEq + Clone,
{
let text = std::fs::read_to_string(file).map_err(|e| Error::io(file, e))?;
let doc: Yaml = serde_yaml_ng::from_str(&text).map_err(|source| Error::Yaml {
path: file.to_path_buf(),
source,
})?;
let name = file_name(file);
let entries = entries(&doc, kind, &name)?;
// The text items line up with the parsed entries one for one, unless the
// list is written in some form the scanner does not follow; then every
// entry is serialized.
let items = items(&text, kind.section());
let items = (items.len() == entries.len()).then_some(items);
let first_use = course.glossary_first_use();
let mut problems = Vec::new();
let mut chunks = Vec::new();
for (n, entry) in entries.iter().enumerate() {
if let Some((section, at)) = course.origin(&entry.id) {
let at = relative(root, at);
if section == kind.registry() {
report.present.push((entry.id.clone(), at));
} else {
problems.push(format!(
"{name}: `{}` would become `{}`, which {} already uses under `{}`",
entry.old,
entry.id,
at.display(),
section.key()
));
}
continue;
}
let typed: T = match serde_yaml_ng::from_value(Yaml::Mapping(entry.fields.clone())) {
Ok(typed) => typed,
Err(e) => {
problems.push(format!("{name}: `{}`: {e}", entry.old));
continue;
}
};
let copied = items
.as_ref()
.and_then(|items| copied(&items[n], kind, entry))
.filter(|chunk| reads_back(chunk, kind, &entry.id, &typed, pick));
let chunk = match copied {
Some(chunk) => chunk,
None => serialized(&entry.id, &typed)?,
};
if entry.old != entry.id {
report.renamed.push((entry.old.clone(), entry.id.clone()));
}
report.dropped.extend(entry.dropped.iter().cloned());
if let Some(old) = &entry.relinked {
report.relinked.push(format!(
"`{}`: `term: {old}` now reads `term: {TERM_PREFIX}{old}`",
entry.id
));
}
if let Some(message) = derived_message(kind, entry, course) {
report.derived.push(message);
}
if !first_use.contains_key(entry.id.as_str()) {
report.unlisted.push(entry.id.clone());
}
report.added.push(entry.id.clone());
expected.insert(entry.id.clone(), typed);
chunks.push(chunk);
}
if !problems.is_empty() {
return Err(Error::Invalid(problems));
}
Ok(chunks)
}
/// Reads the entries of one 1.0 file.
///
/// Lenient about everything 2.1 does not need: a missing schema version, an id
/// written as a number, fields 2.1 has no place for. Strict about what would
/// make a wrong file: an entry with no id, an id that would not be valid with
/// its prefix, two entries that would share an id, and a field given twice
/// under its old and new names.
fn entries(doc: &Yaml, kind: Kind, file: &str) -> Result<Vec<Entry>> {
let section = kind.section();
let list = match doc.get(section) {
None | Some(Yaml::Null) => return Ok(Vec::new()),
Some(Yaml::Sequence(list)) => list,
Some(_) => {
return Err(Error::usage(format!(
"{file}: `{section}` is not a list; 1.0 writes one `- id:` item per entry"
)));
}
};
let mut out: Vec<Entry> = Vec::new();
let mut problems = Vec::new();
for (n, item) in list.iter().enumerate() {
let Yaml::Mapping(map) = item else {
problems.push(format!(
"{file}: {section} entry {} is not a mapping",
n + 1
));
continue;
};
let Some(old) = map.get("id").and_then(plain) else {
problems.push(format!("{file}: {section} entry {} has no `id`", n + 1));
continue;
};
let id = if old.starts_with(kind.prefix()) {
old.clone()
} else {
format!("{}{old}", kind.prefix())
};
if !glossary::is_prefixed_id(&id, kind.prefix()) {
problems.push(format!(
"{file}: `{old}` would become `{id}`, which is not a valid id; rename it in {file} first"
));
continue;
}
if out.iter().any(|e| e.id == id) {
problems.push(format!("{file}: two entries would both be `{id}`"));
continue;
}
let mut entry = Entry {
old: old.clone(),
id,
fields: Mapping::new(),
derived: None,
dropped: Vec::new(),
relinked: None,
};
for (key, value) in map {
let Some(key) = key.as_str() else {
entry
.dropped
.push(format!("`{}`: a key that is not a string", entry.id));
continue;
};
if key == "id" {
continue;
}
if key == kind.derived() {
entry.derived = Some(value.clone());
continue;
}
let name = kind.renamed(key).unwrap_or(key);
if !kind.fields().contains(&name) {
entry.dropped.push(format!(
"`{}`: `{key}`, which 2.1 has no field for",
entry.id
));
continue;
}
let mut value = value.clone();
if kind == Kind::Value
&& name == "term"
&& let Some(term) = value.as_str()
&& !term.starts_with(TERM_PREFIX)
{
entry.relinked = Some(term.to_string());
value = Yaml::String(format!("{TERM_PREFIX}{term}"));
}
if entry
.fields
.insert(Yaml::String(name.to_string()), value)
.is_some()
{
problems.push(format!(
"{file}: `{old}` gives `{name}` twice, once as `{key}`"
));
}
}
out.push(entry);
}
if !problems.is_empty() {
return Err(Error::Invalid(problems));
}
Ok(out)
}
/// A scalar as a string: strings as they are, numbers and booleans as written.
fn plain(value: &Yaml) -> Option<String> {
match value {
Yaml::String(s) => Some(s.clone()),
Yaml::Number(n) => Some(n.to_string()),
Yaml::Bool(b) => Some(b.to_string()),
_ => None,
}
}
/// The list items under a top-level key, as lines.
fn items(text: &str, section: &str) -> Vec<Item> {
let lines: Vec<&str> = text.lines().collect();
let Some(start) = lines
.iter()
.position(|l| key_at(l, 0).as_deref() == Some(section))
else {
return Vec::new();
};
let body: Vec<&str> = lines[start + 1..]
.iter()
.take_while(|l| key_at(l, 0).is_none())
.copied()
.collect();
let starts_item = |line: &str| {
let trimmed = line.trim_start();
trimmed.starts_with("- ") || trimmed == "-"
};
let Some(dash) = body
.iter()
.find(|l| starts_item(l))
.and_then(|l| indent_of(l))
else {
return Vec::new();
};
let mut out: Vec<Item> = Vec::new();
let mut lead: Vec<String> = Vec::new();
for line in body {
if indent_of(line) == Some(dash) && starts_item(line) {
if let Some(previous) = out.last_mut() {
lead = trailing_comments(&mut previous.lines);
}
out.push(Item {
lead: std::mem::take(&mut lead),
lines: vec![line.to_string()],
dash,
});
continue;
}
match out.last_mut() {
Some(item) => item.lines.push(line.to_string()),
None if line.trim_start().starts_with('#') => lead.push(line.to_string()),
None => {}
}
}
if let Some(last) = out.last_mut() {
trailing_comments(&mut last.lines);
}
out
}
/// Takes a run of trailing blank and comment lines off an item; returns the
/// comments, which belong to whatever follows.
fn trailing_comments(lines: &mut Vec<String>) -> Vec<String> {
let mut tail = Vec::new();
while lines.len() > 1 {
let last = lines.last().map(|l| l.trim()).unwrap_or_default();
if last.is_empty() || last.starts_with('#') {
tail.push(lines.pop().unwrap_or_default());
} else {
break;
}
}
tail.reverse();
tail.into_iter()
.filter(|l| l.trim().starts_with('#'))
.collect()
}
/// An item's lines as an entry: ` term-x:` and its fields four deep.
///
/// `None` when the item is not written in a form this follows line by line,
/// for the caller to serialize instead.
fn copied(item: &Item, kind: Kind, entry: &Entry) -> Option<Vec<String>> {
// The dash becomes a space, so the first field sits where the rest do.
let first = item.lines.first()?;
let after = first.get(item.dash + 1..)?;
let mut lines: Vec<String> = Vec::new();
let indent = if after.trim().is_empty() {
// A bare `-` with the fields on the lines below it.
lines.extend(item.lines[1..].iter().cloned());
lines.iter().find_map(|l| indent_of(l))?
} else {
lines.push(format!("{} {after}", &first[..item.dash]));
lines.extend(item.lines[1..].iter().cloned());
indent_of(&lines[0])?
};
// A flow mapping is one value, not lines of fields.
if lines
.iter()
.find(|l| indent_of(l).is_some())
.is_some_and(|l| l.trim_start().starts_with(['{', '[']))
{
return None;
}
// Split into fields: a key at the field indent starts one; comments and
// deeper lines continue it.
let mut fields: Vec<(String, Vec<String>)> = Vec::new();
let mut lead: Vec<String> = item.lead.clone();
for line in lines {
match key_at(&line, indent) {
Some(key) => fields.push((key, vec![line])),
None => match fields.last_mut() {
Some((_, field)) => field.push(line),
None if line.trim().is_empty() => {}
None if line.trim_start().starts_with('#') => lead.push(line),
None => return None,
},
}
}
if fields.is_empty() {
return None;
}
let mut out: Vec<String> = lead.iter().map(|l| format!(" {}", l.trim())).collect();
out.push(format!(" {}:", entry.id));
for (key, mut field) in fields {
if key == "id" || key == kind.derived() {
continue;
}
let name = kind.renamed(&key).unwrap_or(&key);
if !kind.fields().contains(&name) {
continue;
}
if name != key {
let line = &field[0];
if !line[indent..].starts_with(key.as_str()) {
return None;
}
field[0] = format!("{}{name}{}", &line[..indent], &line[indent + key.len()..]);
}
if kind == Kind::Value && key == "term" && entry.relinked.is_some() {
// Only a one-line plain scalar is rewritten in place.
let term = scalar(&field[0], indent, "term")?;
if field.len() > 1 || entry.relinked.as_deref() != Some(term.as_str()) {
return None;
}
field[0] = format!("{}term: {TERM_PREFIX}{term}", &field[0][..indent]);
}
out.extend(field.iter().map(|l| shift(l, indent, 4)));
}
while out.last().is_some_and(|l| l.trim().is_empty()) {
out.pop();
}
Some(out)
}
/// Moves a line from one indentation to another, keeping what is under it
/// relative.
fn shift(line: &str, from: usize, to: usize) -> String {
if line.trim().is_empty() {
return String::new();
}
if to >= from {
return format!("{}{line}", " ".repeat(to - from));
}
let leading = line.len() - line.trim_start().len();
line[leading.min(from - to)..].to_string()
}
/// Whether a chunk reads back as the entry it was copied from.
fn reads_back<T: DeserializeOwned + PartialEq>(
chunk: &[String],
kind: Kind,
id: &str,
expected: &T,
pick: fn(&Fragment) -> &BTreeMap<String, T>,
) -> bool {
let text = format!("{}:\n{}\n", kind.section(), chunk.join("\n"));
serde_yaml_ng::from_str::<Fragment>(&text)
.ok()
.is_some_and(|fragment| pick(&fragment).get(id) == Some(expected))
}
/// An entry written by the serializer, for when its lines could not be copied.
fn serialized<T: Serialize>(id: &str, entry: &T) -> Result<Vec<String>> {
let text = yaml::to_string(entry)?;
let mut out = vec![format!(" {id}:")];
out.extend(
text.lines()
.filter(|l| *l != "---")
.map(|l| format!(" {l}")),
);
Ok(out)
}
/// What a dropped `first` or `lectures` said, when the targets now say
/// something else.
fn derived_message(kind: Kind, entry: &Entry, course: &CourseFile) -> Option<String> {
let old = entry.derived.as_ref()?;
let said: Vec<String> = match old {
Yaml::Sequence(list) => list.iter().filter_map(plain).collect(),
other => plain(other).into_iter().collect(),
};
let now: Vec<&str> = match kind {
Kind::Term => course.first_lecture(&entry.id).into_iter().collect(),
Kind::Value => lectures_listing(course, &entry.id, RefKind::Value),
};
let same = said.iter().map(String::as_str).collect::<BTreeSet<_>>()
== now.iter().copied().collect::<BTreeSet<_>>();
if same {
return None;
}
let derived = if now.is_empty() {
"none".to_string()
} else if kind == Kind::Term {
now.join(", ")
} else {
format!("[{}]", now.join(", "))
};
let was = if kind == Kind::Term {
said.join(", ")
} else {
format!("[{}]", said.join(", "))
};
Some(format!(
"`{}`: `{}` was {was}; derived from the targets: {derived}",
entry.id,
kind.derived()
))
}
/// Every lecture, in order, with a target that lists an entry.
fn lectures_listing<'a>(course: &'a CourseFile, id: &str, kind: RefKind) -> Vec<&'a str> {
course
.lectures
.keys()
.map(String::as_str)
.filter(|lecture| {
course
.lecture_targets_in_order(lecture)
.iter()
.filter_map(|t| course.learning_targets.get(*t))
.any(|target| target.listed(kind).iter().any(|x| x == id))
})
.collect()
}
/// A path's file name, for messages.
fn file_name(path: &Path) -> String {
path.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_else(|| path.display().to_string())
}
/// The comment a new `glossary.yaml` starts with.
fn header(sources: &[String]) -> String {
format!(
"# glossary.yaml (coursebank 2.1)\n\
# Entries brought in from {} by `coursebank migrate glossary`. Move each one to the\n\
# objective file whose targets first need it; keep the cross-cutting ones here.\n",
sources.join(" and ")
)
}
/// Adds entry chunks at the end of a top-level section, or a new section at
/// the end of the file.
fn insert(text: &str, section: &str, chunks: &[Vec<String>]) -> Result<String> {
let mut lines: Vec<String> = text.lines().map(str::to_string).collect();
let new: Vec<String> = chunks.iter().flatten().cloned().collect();
match lines
.iter()
.position(|l| key_at(l, 0).as_deref() == Some(section))
{
Some(at) => {
let colon = lines[at].find(':').unwrap_or(lines[at].len());
let rest = lines[at][colon + 1..].trim().to_string();
if !(rest.is_empty() || rest.starts_with('#')) {
if rest == "{}" {
lines[at] = format!("{section}:");
} else {
return Err(Error::usage(format!(
"glossary.yaml writes `{section}:` on one line; write it as a block, one \
entry per key, and run this again"
)));
}
}
let mut end = at + 1;
while end < lines.len() && key_at(&lines[end], 0).is_none() {
end += 1;
}
// Blank lines and column-0 comments at the end of the section
// introduce whatever follows it.
while end > at + 1 {
let last = &lines[end - 1];
let comment = indent_of(last) == Some(0) && last.starts_with('#');
if last.trim().is_empty() || comment {
end -= 1;
} else {
break;
}
}
lines.splice(end..end, new);
}
None => {
while lines.last().is_some_and(|l| l.trim().is_empty()) {
lines.pop();
}
let has_content = lines.iter().any(|l| {
let t = l.trim();
!t.is_empty() && !t.starts_with('#')
});
if has_content {
lines.push(String::new());
}
lines.push(format!("{section}:"));
lines.extend(new);
}
}
Ok(format!("{}\n", lines.join("\n")))
}
/// Checks that the new file reads back as the old one plus the migrated
/// entries, and nothing else.
fn verify(
before: Option<&str>,
after: &str,
terms: &BTreeMap<String, Term>,
values: &BTreeMap<String, Value>,
) -> Result<()> {
let unreadable = |e: serde_yaml_ng::Error| {
Error::other(format!(
"the glossary.yaml this would write does not load ({e}); nothing was written"
))
};
let mut parsed: Fragment = serde_yaml_ng::from_str(after).map_err(unreadable)?;
for (id, term) in terms {
if parsed.terms.remove(id).as_ref() != Some(term) {
return Err(Error::other(format!(
"`{id}` does not read back as migrated; nothing was written"
)));
}
}
for (id, value) in values {
if parsed.values.remove(id).as_ref() != Some(value) {
return Err(Error::other(format!(
"`{id}` does not read back as migrated; nothing was written"
)));
}
}
let old: Fragment = match before {
Some(text) => serde_yaml_ng::from_str(text).map_err(unreadable)?,
None => Fragment::default(),
};
if yaml::to_string(&parsed)? != yaml::to_string(&old)? {
return Err(Error::other(
"migrating would change entries glossary.yaml already has; nothing was written",
));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
const TERMS: &str = "schema_version: \"1.0\"
terms:
# A coined one.
- id: decade
course: decade
variants: [decades]
kind: coined
meaning: >-
a factor of ten in a concentration
ratio
first: L2.1
color: blue
- id: term-gap
course: gap
meaning: a dash
";
fn parsed(text: &str, kind: Kind) -> Vec<Entry> {
let doc: Yaml = serde_yaml_ng::from_str(text).unwrap();
entries(&doc, kind, "terms.yaml").unwrap()
}
#[test]
fn entries_take_their_prefix_once_and_lose_derived_and_unknown_fields() {
let entries = parsed(TERMS, Kind::Term);
assert_eq!(entries[0].id, "term-decade");
assert_eq!(entries[1].id, "term-gap");
assert_eq!(entries[0].derived, Some(Yaml::String("L2.1".into())));
assert_eq!(entries[0].dropped.len(), 1);
assert!(entries[0].dropped[0].contains("`color`"));
assert!(entries[0].fields.contains_key("definition"));
assert!(!entries[0].fields.contains_key("meaning"));
}
#[test]
fn copied_lines_keep_comments_and_block_scalars() {
let entries = parsed(TERMS, Kind::Term);
let items = items(TERMS, "terms");
assert_eq!(items.len(), 2);
let chunk = copied(&items[0], Kind::Term, &entries[0]).unwrap();
assert_eq!(
chunk,
[
" # A coined one.",
" term-decade:",
" term: decade",
" variants: [decades]",
" kind: coined",
" definition: >-",
" a factor of ten in a concentration",
" ratio",
]
);
let term: Term =
serde_yaml_ng::from_value(Yaml::Mapping(entries[0].fields.clone())).unwrap();
assert!(reads_back(&chunk, Kind::Term, "term-decade", &term, |f| &f.terms));
}
#[test]
fn items_at_column_zero_are_reindented() {
let text = "terms:\n- id: gap\n course: gap\n meaning: a dash\n";
let entries = parsed(text, Kind::Term);
let items = items(text, "terms");
let chunk = copied(&items[0], Kind::Term, &entries[0]).unwrap();
assert_eq!(
chunk,
[" term-gap:", " term: gap", " definition: a dash"]
);
}
#[test]
fn a_value_names_its_term_by_the_new_id() {
let text = "values:\n - id: k\n name: k\n say: about 6\n term: decade\n";
let entries = parsed(text, Kind::Value);
assert_eq!(entries[0].relinked.as_deref(), Some("decade"));
let items = items(text, "values");
let chunk = copied(&items[0], Kind::Value, &entries[0]).unwrap();
assert_eq!(chunk.last().unwrap(), " term: term-decade");
}
#[test]
fn a_flow_item_falls_back_to_the_serializer() {
let text = "terms:\n - {id: gap, course: gap, meaning: a dash}\n";
let entries = parsed(text, Kind::Term);
let items = items(text, "terms");
assert!(copied(&items[0], Kind::Term, &entries[0]).is_none());
let term: Term =
serde_yaml_ng::from_value(Yaml::Mapping(entries[0].fields.clone())).unwrap();
let chunk = serialized("term-gap", &term).unwrap();
assert!(reads_back(&chunk, Kind::Term, "term-gap", &term, |f| &f.terms));
}
#[test]
fn bad_and_duplicate_ids_are_refused() {
let doc: Yaml = serde_yaml_ng::from_str(
"terms:\n - id: Two Words\n course: x\n meaning: y\n - id: a\n course: a\n meaning: b\n - id: term-a\n course: a\n meaning: b\n - course: z\n",
)
.unwrap();
let Err(Error::Invalid(problems)) = entries(&doc, Kind::Term, "terms.yaml") else {
panic!("expected problems");
};
assert_eq!(problems.len(), 3, "{problems:?}");
}
#[test]
fn insert_appends_to_a_section_or_adds_one() {
let text = "# keep\nterms:\n term-a:\n term: a\n definition: b\n\n# about values\nvalues: {}\n";
let chunk = vec![" term-c:".to_string(), " term: c".to_string()];
let out = insert(text, "terms", std::slice::from_ref(&chunk)).unwrap();
assert!(out.contains(" definition: b\n term-c:\n term: c\n\n# about values\n"));
let value = vec![" val-x:".to_string(), " name: x".to_string()];
let out = insert(&out, "values", &[value]).unwrap();
assert!(out.ends_with("values:\n val-x:\n name: x\n"));
let fresh = insert("# header\n", "terms", &[chunk]).unwrap();
assert_eq!(fresh, "# header\nterms:\n term-c:\n term: c\n");
}
#[test]
fn shift_moves_whole_blocks() {
assert_eq!(shift(" deeper", 2, 4), " deeper");
assert_eq!(shift(" deeper", 6, 4), " deeper");
assert_eq!(shift(" ", 2, 4), "");
}
}
+158
View File
@@ -0,0 +1,158 @@
// SPDX-License-Identifier: Prosperity-3.0.0
// Copyright Scienting
// Source: https://git.scient.ing/education/coursebank
//! `coursebank migrate glossary`, against `docs/design/2.1/examples/migration/`.
//!
//! The example course's own `glossary.yaml` is that migration's output after a
//! person linked the value to its term, so each test starts from the example
//! with it removed.
mod common;
use std::path::{Path, PathBuf};
use std::process::Command;
use common::{design, edit, v21_course};
use coursebank::course::fragment::Fragment;
use coursebank::migrate;
/// The example course without its glossary.
fn course(tag: &str) -> PathBuf {
let root = v21_course(tag);
std::fs::remove_file(root.join("glossary.yaml")).unwrap();
root
}
fn source(name: &str) -> PathBuf {
design().join("examples/migration").join(name)
}
fn fragment(path: &Path) -> Fragment {
serde_yaml_ng::from_str(&std::fs::read_to_string(path).unwrap()).unwrap()
}
fn run(root: &Path, write: bool) -> migrate::GlossaryMigration {
migrate::glossary(
root,
Some(&source("terms.yaml")),
Some(&source("values.yaml")),
write,
)
.unwrap()
}
#[test]
fn migration_reproduces_the_expected_glossary() {
let root = course("mig-expected");
let report = run(&root, true);
let written = fragment(&root.join("glossary.yaml"));
let expected = fragment(&source("glossary.expected.yaml"));
assert_eq!(written.terms, expected.terms);
assert_eq!(written.values, expected.values);
// The report the expected file's header comment describes.
assert_eq!(
report.renamed,
[
("decade".to_string(), "term-decade".to_string()),
("per-decade".to_string(), "val-per-decade".to_string()),
]
);
assert_eq!(report.derived.len(), 2, "{:?}", report.derived);
assert!(report.derived[0].contains("`first` was L2.1"));
assert!(report.derived[1].contains("`lectures` was [L2.1, L2.4, L2.5]"));
assert_eq!(report.unlisted, ["term-decade", "val-per-decade"]);
assert!(report.dropped.is_empty() && report.relinked.is_empty());
// The course still validates with the written file.
let issues = coursebank::Catalog::load(&root)
.unwrap()
.validate()
.unwrap();
assert!(issues.is_empty(), "{issues:#?}");
}
#[test]
fn migrating_twice_is_a_no_op() {
let root = course("mig-twice");
run(&root, true);
let before = std::fs::read(root.join("glossary.yaml")).unwrap();
let again = run(&root, true);
assert!(again.text.is_none());
assert!(again.added.is_empty() && again.renamed.is_empty());
assert_eq!(again.present.len(), 2);
assert_eq!(std::fs::read(root.join("glossary.yaml")).unwrap(), before);
// The example as shipped already holds both, linked: also a no-op.
let shipped = v21_course("mig-shipped");
assert!(run(&shipped, true).text.is_none());
}
#[test]
fn a_dropped_first_is_compared_with_the_targets() {
let root = course("mig-derived");
edit(
&root,
"objectives/lo-alignment-scoring.yaml",
"terms: [term-alignment-matrix, term-gap,",
"terms: [term-decade, term-alignment-matrix, term-gap,",
);
let report = run(&root, false);
assert!(
report.derived[0].contains("`first` was L2.1; derived from the targets: l09"),
"{:?}",
report.derived
);
assert_eq!(report.unlisted, ["val-per-decade"]);
assert!(!root.join("glossary.yaml").exists(), "a dry run wrote");
}
#[test]
fn an_existing_glossary_is_added_to_not_rewritten() {
let root = course("mig-existing");
let before = "# Cross-cutting terms.\nterms:\n term-other:\n term: other\n \
definition: Something else.\n";
std::fs::write(root.join("glossary.yaml"), before).unwrap();
let report = run(&root, true);
let after = std::fs::read_to_string(root.join("glossary.yaml")).unwrap();
assert!(after.starts_with(before), "{after}");
assert!(after.contains("\nvalues:\n val-per-decade:\n"));
assert_eq!(report.added, ["term-decade", "val-per-decade"]);
let written = fragment(&root.join("glossary.yaml"));
assert_eq!(written.terms.len(), 2);
assert_eq!(written.values.len(), 1);
}
#[test]
fn the_cli_reports_and_a_dry_run_writes_nothing() {
let root = course("mig-cli");
let terms = source("terms.yaml");
let values = source("values.yaml");
let out = Command::new(env!("CARGO_BIN_EXE_coursebank"))
.arg("-C")
.arg(&root)
.args(["migrate", "glossary", "--dry-run", "--terms"])
.arg(&terms)
.arg("--values")
.arg(&values)
.output()
.unwrap();
let stdout = String::from_utf8_lossy(&out.stdout);
assert!(out.status.success(), "{stdout}");
assert!(stdout.contains("decade -> term-decade"));
assert!(stdout.contains("nothing written"));
assert!(!root.join("glossary.yaml").exists());
let none = Command::new(env!("CARGO_BIN_EXE_coursebank"))
.arg("-C")
.arg(&root)
.args(["migrate", "glossary"])
.output()
.unwrap();
assert!(!none.status.success());
}