diff --git a/src/cli.rs b/src/cli.rs index bd0f2f5..bda53db 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -200,6 +200,25 @@ pub(crate) enum MigrateCommand { #[arg(long)] dry_run: bool, }, + /// Bring a 1.0 terms.yaml and values.yaml into the 2.1 glossary. + /// + /// Writes glossary.yaml. Ids gain their `term-` and `val-` prefixes, + /// `course` becomes `term` and `meaning` becomes `definition`, and `first` + /// and `lectures` are dropped, since 2.1 derives them from the targets + /// that list an entry; each one that disagrees is reported. The old files + /// are left in place, and an entry the course already declares is + /// skipped, so a second run changes nothing. + Glossary { + /// The 1.0 terms file. + #[arg(long, value_name = "FILE")] + terms: Option, + /// The 1.0 values file. + #[arg(long, value_name = "FILE")] + values: Option, + /// Show what would be written, and write nothing. + #[arg(long)] + dry_run: bool, + }, /// Rewrite pre-2.0 `bank::item` ids as the item ids they name. /// /// Touches assessment records, seals, and the response store. Everything diff --git a/src/commands/project.rs b/src/commands/project.rs index ce1aa4c..f1ff8a8 100644 --- a/src/commands/project.rs +++ b/src/commands/project.rs @@ -118,9 +118,74 @@ pub(crate) fn migrate(cli: &Cli, sub: &MigrateCommand) -> Result { MigrateCommand::References { dry_run } => migrate_references(cli, *dry_run), MigrateCommand::Order { dry_run } => migrate_order(cli, *dry_run), MigrateCommand::Counters { dry_run } => migrate_counters(cli, *dry_run), + MigrateCommand::Glossary { + terms, + values, + dry_run, + } => migrate_glossary(cli, terms.as_deref(), values.as_deref(), *dry_run), } } +/// Brings a 1.0 terms.yaml and values.yaml into glossary.yaml. +fn migrate_glossary( + cli: &Cli, + terms: Option<&Path>, + values: Option<&Path>, + dry_run: bool, +) -> Result { + let report = migrate::glossary(&cli.course, terms, values, !dry_run)?; + + let list = |heading: &str, lines: &[String]| { + if !lines.is_empty() { + println!("{heading}"); + for line in lines { + println!(" {line}"); + } + println!(); + } + }; + let renamed: Vec = report + .renamed + .iter() + .map(|(old, new)| format!("{old} -> {new}")) + .collect(); + list("renamed:", &renamed); + list( + "dropped `first` and `lectures` that the targets now contradict:", + &report.derived, + ); + list("dropped, no 2.1 field:", &report.dropped); + list("relinked:", &report.relinked); + list("no target lists these yet:", &report.unlisted); + let present: Vec = report + .present + .iter() + .map(|(id, path)| format!("{id} ({})", path.display())) + .collect(); + list("already declared, skipped:", &present); + + let Some(text) = &report.text else { + println!("nothing to do: the course already declares every entry"); + return Ok(Outcome::Ok); + }; + if dry_run { + println!( + "{} would read:\n\n{text}\nnothing written", + report.path.display() + ); + return Ok(Outcome::Ok); + } + println!( + "wrote {} entr{} to {}\n\nNext:\n coursebank validate\n coursebank lint \ + # glossary-unused lists entries nothing points at yet\n\n\ + The old files are untouched; delete them once the pipelines read the registry.", + report.added.len(), + if report.added.len() == 1 { "y" } else { "ies" }, + report.path.display() + ); + Ok(Outcome::Ok) +} + /// Drops the trailing counter from every item id. fn migrate_counters(cli: &Cli, dry_run: bool) -> Result { let (rename, touched) = migrate::counters(&cli.course, !dry_run)?; diff --git a/src/migrate.rs b/src/migrate.rs index 9b0e170..7d8ae04 100644 --- a/src/migrate.rs +++ b/src/migrate.rs @@ -22,6 +22,7 @@ //! | [`references`] | prose notes into citation fields | a DOI in a note is a link nobody can follow | //! | [`order`] | removes the `order:` integers | objective order is derived, and targets have none | //! | [`counters`] | drops the `-001` from item ids | a per-bank sequence number is not part of a name | +//! | [`glossary()`] | 1.0 `terms.yaml` and `values.yaml` into `glossary.yaml` | an entry is defined once, by id, where targets can point at it | //! //! # What the option migration does not touch //! @@ -74,6 +75,9 @@ use crate::layout::Layout; use crate::store::{self, Store}; use crate::yaml; +mod glossary; +pub use glossary::{GlossaryMigration, glossary}; + /// The width past which a flow sequence is written as a block list instead. const FLOW_WIDTH: usize = 96; diff --git a/src/migrate/glossary.rs b/src/migrate/glossary.rs new file mode 100644 index 0000000..6d777c5 --- /dev/null +++ b/src/migrate/glossary.rs @@ -0,0 +1,975 @@ +//! `migrate glossary`: a 1.0 `terms.yaml` and `values.yaml` into the 2.1 +//! registries. +//! +//! The same textual approach as the rest of [`crate::migrate`]. Each entry's +//! lines are copied and re-indented from a list item to a map entry, with three +//! edits: the `id` becomes the key and gains its prefix, `course` and `meaning` +//! are renamed, and `first` and `lectures`, which 2.1 derives from the targets +//! that list an entry, are dropped. Comments, folded scalars, and flow mappings +//! like `{ term: ten-fold }` survive. +//! +//! Every copied entry is parsed back and compared with the same conversion done +//! on the parsed source. An entry whose copied text does not read back the same +//! (a flow-mapping list item, a quoted key) is written by the serializer +//! instead, so the file is right even where it is not pretty. The whole file is +//! checked the same way before anything is written, and the course is loaded +//! again after. +//! +//! An entry the course already declares, in `glossary.yaml` or any other +//! fragment, is skipped, so a second run is a no-op. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +use serde::Serialize; +use serde::de::DeserializeOwned; +use serde_yaml_ng::{Mapping, Value as Yaml}; + +use super::{indent_of, key_at, relative, scalar}; +use crate::course::CourseFile; +use crate::course::fragment::{Fragment, Section}; +use crate::course::glossary::{self, RefKind, TERM_PREFIX, Term, VALUE_PREFIX, Value}; +use crate::error::{Error, Result}; +use crate::layout::Layout; +use crate::yaml; + +/// What `migrate glossary` writes, and what it found on the way. +#[derive(Debug, Clone, Default)] +pub struct GlossaryMigration { + /// The file it writes, relative to the course root. + pub path: PathBuf, + /// The file's new contents; `None` when there is nothing to add. + pub text: Option, + /// Ids added, terms first, each in source order. + pub added: Vec, + /// Old id to new id, for every added entry whose id gained a prefix. + pub renamed: Vec<(String, String)>, + /// Entries the course already declares, with the file that declares them. + pub present: Vec<(String, PathBuf)>, + /// One message per dropped `first` or `lectures` that differs from what + /// the targets now say. + pub derived: Vec, + /// Added entries that no target lists yet. + pub unlisted: Vec, + /// One message per field dropped because 2.1 has no place for it. + pub dropped: Vec, + /// One message per value whose `term:` was rewritten to the prefixed id. + pub relinked: Vec, +} + +/// Which 1.0 file an entry came from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Kind { + Term, + Value, +} + +impl Kind { + /// The top-level key, the same in 1.0 and 2.1. + fn section(self) -> &'static str { + match self { + Kind::Term => "terms", + Kind::Value => "values", + } + } + + /// The prefix a 2.1 id carries. + fn prefix(self) -> &'static str { + match self { + Kind::Term => TERM_PREFIX, + Kind::Value => VALUE_PREFIX, + } + } + + /// The registry an id of this kind lives in. + fn registry(self) -> Section { + match self { + Kind::Term => Section::Terms, + Kind::Value => Section::Values, + } + } + + /// The 1.0 field that 2.1 derives from the targets instead. + fn derived(self) -> &'static str { + match self { + Kind::Term => "first", + Kind::Value => "lectures", + } + } + + /// The 2.1 name of a renamed 1.0 field. + fn renamed(self, key: &str) -> Option<&'static str> { + match (self, key) { + (Kind::Term, "course") => Some("term"), + (Kind::Term, "meaning") => Some("definition"), + _ => None, + } + } + + /// Every field the 2.1 entry has; anything else is dropped and reported. + fn fields(self) -> &'static [&'static str] { + match self { + Kind::Term => &[ + "term", + "definition", + "kind", + "textbook", + "variants", + "symbol", + "units", + "exam_wording", + "status", + "tags", + "notes_private", + ], + Kind::Value => &[ + "name", + "value", + "range", + "units", + "say", + "exact", + "term", + "source", + "conflicts", + "recall", + "status", + "tags", + "notes_private", + ], + } + } +} + +/// One 1.0 entry, read leniently and converted to 2.1 fields. +#[derive(Debug, Clone)] +struct Entry { + /// The id as the 1.0 file wrote it. + old: String, + /// The 2.1 id. + id: String, + /// The 2.1 fields, renamed, unknown ones gone. + fields: Mapping, + /// What the dropped `first` or `lectures` said. + derived: Option, + /// One message per unknown field dropped. + dropped: Vec, + /// The value's `term:` before it gained its prefix. + relinked: Option, +} + +/// One list item in a 1.0 file, as lines. +#[derive(Debug, Clone, Default)] +struct Item { + /// Comment lines directly above the item. + lead: Vec, + /// The `- ` line and everything under it. + lines: Vec, + /// The column of the item's dash. + dash: usize, +} + +/// Writes a 1.0 `terms.yaml` and `values.yaml` into the course's +/// `glossary.yaml`. +/// +/// Ids gain their `term-` and `val-` prefixes, `course` becomes `term` and +/// `meaning` becomes `definition`, a value's `term:` gains the prefix of the id +/// it names, and `first` and `lectures` are dropped: 2.1 derives them from the +/// targets that list an entry, and each one that disagrees with what the +/// targets now say is reported. A field 2.1 has no place for is dropped and +/// reported. The old files are left alone. +/// +/// An existing `glossary.yaml` is added to, never rewritten: new entries go at +/// the end of its `terms:` and `values:` sections, and everything already in +/// it stays byte for byte. +/// +/// # Arguments +/// +/// * `root` - the course directory. +/// * `terms` - the 1.0 terms file, if migrating one. +/// * `values` - the 1.0 values file, if migrating one. +/// * `write` - whether to write, or only to report. +/// +/// # Returns +/// +/// What was (or would be) written, and the report. +/// +/// # Errors +/// +/// Returns [`Error::Usage`] when neither file is given, [`Error::Invalid`] for +/// entries that cannot be migrated (no id, an id that would not be valid, a +/// field 2.1 would reject), and [`Error::Other`] if the written file would not +/// read back as migrated. Nothing is written in any of those cases. +pub fn glossary( + root: &Path, + terms: Option<&Path>, + values: Option<&Path>, + write: bool, +) -> Result { + if terms.is_none() && values.is_none() { + return Err(Error::usage( + "nothing to migrate: give --terms, --values, or both", + )); + } + let layout = Layout::new(root); + let course = CourseFile::load_dir(root)?; + let path = layout.glossary_file(); + let mut report = GlossaryMigration { + path: relative(root, &path), + ..GlossaryMigration::default() + }; + + let mut new_terms: BTreeMap = BTreeMap::new(); + let mut new_values: BTreeMap = BTreeMap::new(); + let mut sections: Vec<(Kind, Vec>)> = Vec::new(); + if let Some(file) = terms { + let chunks = convert( + root, + file, + Kind::Term, + &course, + &mut report, + &mut new_terms, + |f| &f.terms, + )?; + sections.push((Kind::Term, chunks)); + } + if let Some(file) = values { + let chunks = convert( + root, + file, + Kind::Value, + &course, + &mut report, + &mut new_values, + |f| &f.values, + )?; + sections.push((Kind::Value, chunks)); + } + if report.added.is_empty() { + return Ok(report); + } + + let before = if path.is_file() { + Some(std::fs::read_to_string(&path).map_err(|e| Error::io(&path, e))?) + } else { + None + }; + let names: Vec = [terms, values] + .into_iter() + .flatten() + .map(file_name) + .collect(); + let mut text = before.clone().unwrap_or_else(|| header(&names)); + for (kind, chunks) in §ions { + if !chunks.is_empty() { + text = insert(&text, kind.section(), chunks)?; + } + } + verify(before.as_deref(), &text, &new_terms, &new_values)?; + + if write { + yaml::write_text(&path, &text)?; + // The fragment reads back; this checks the course still assembles with + // it, and puts the old file back when it does not. + if let Err(e) = CourseFile::load_dir(root) { + match &before { + Some(old) => yaml::write_text(&path, old)?, + None => std::fs::remove_file(&path).map_err(|e| Error::io(&path, e))?, + } + return Err(e); + } + } + report.text = Some(text); + Ok(report) +} + +/// Converts one 1.0 file into the entry chunks to write. +/// +/// Fills in the report and `expected`, the typed entries the written file must +/// read back as. +fn convert( + root: &Path, + file: &Path, + kind: Kind, + course: &CourseFile, + report: &mut GlossaryMigration, + expected: &mut BTreeMap, + pick: fn(&Fragment) -> &BTreeMap, +) -> Result>> +where + T: DeserializeOwned + Serialize + PartialEq + Clone, +{ + let text = std::fs::read_to_string(file).map_err(|e| Error::io(file, e))?; + let doc: Yaml = serde_yaml_ng::from_str(&text).map_err(|source| Error::Yaml { + path: file.to_path_buf(), + source, + })?; + let name = file_name(file); + let entries = entries(&doc, kind, &name)?; + + // The text items line up with the parsed entries one for one, unless the + // list is written in some form the scanner does not follow; then every + // entry is serialized. + let items = items(&text, kind.section()); + let items = (items.len() == entries.len()).then_some(items); + let first_use = course.glossary_first_use(); + + let mut problems = Vec::new(); + let mut chunks = Vec::new(); + for (n, entry) in entries.iter().enumerate() { + if let Some((section, at)) = course.origin(&entry.id) { + let at = relative(root, at); + if section == kind.registry() { + report.present.push((entry.id.clone(), at)); + } else { + problems.push(format!( + "{name}: `{}` would become `{}`, which {} already uses under `{}`", + entry.old, + entry.id, + at.display(), + section.key() + )); + } + continue; + } + let typed: T = match serde_yaml_ng::from_value(Yaml::Mapping(entry.fields.clone())) { + Ok(typed) => typed, + Err(e) => { + problems.push(format!("{name}: `{}`: {e}", entry.old)); + continue; + } + }; + let copied = items + .as_ref() + .and_then(|items| copied(&items[n], kind, entry)) + .filter(|chunk| reads_back(chunk, kind, &entry.id, &typed, pick)); + let chunk = match copied { + Some(chunk) => chunk, + None => serialized(&entry.id, &typed)?, + }; + + if entry.old != entry.id { + report.renamed.push((entry.old.clone(), entry.id.clone())); + } + report.dropped.extend(entry.dropped.iter().cloned()); + if let Some(old) = &entry.relinked { + report.relinked.push(format!( + "`{}`: `term: {old}` now reads `term: {TERM_PREFIX}{old}`", + entry.id + )); + } + if let Some(message) = derived_message(kind, entry, course) { + report.derived.push(message); + } + if !first_use.contains_key(entry.id.as_str()) { + report.unlisted.push(entry.id.clone()); + } + report.added.push(entry.id.clone()); + expected.insert(entry.id.clone(), typed); + chunks.push(chunk); + } + if !problems.is_empty() { + return Err(Error::Invalid(problems)); + } + Ok(chunks) +} + +/// Reads the entries of one 1.0 file. +/// +/// Lenient about everything 2.1 does not need: a missing schema version, an id +/// written as a number, fields 2.1 has no place for. Strict about what would +/// make a wrong file: an entry with no id, an id that would not be valid with +/// its prefix, two entries that would share an id, and a field given twice +/// under its old and new names. +fn entries(doc: &Yaml, kind: Kind, file: &str) -> Result> { + let section = kind.section(); + let list = match doc.get(section) { + None | Some(Yaml::Null) => return Ok(Vec::new()), + Some(Yaml::Sequence(list)) => list, + Some(_) => { + return Err(Error::usage(format!( + "{file}: `{section}` is not a list; 1.0 writes one `- id:` item per entry" + ))); + } + }; + + let mut out: Vec = Vec::new(); + let mut problems = Vec::new(); + for (n, item) in list.iter().enumerate() { + let Yaml::Mapping(map) = item else { + problems.push(format!( + "{file}: {section} entry {} is not a mapping", + n + 1 + )); + continue; + }; + let Some(old) = map.get("id").and_then(plain) else { + problems.push(format!("{file}: {section} entry {} has no `id`", n + 1)); + continue; + }; + let id = if old.starts_with(kind.prefix()) { + old.clone() + } else { + format!("{}{old}", kind.prefix()) + }; + if !glossary::is_prefixed_id(&id, kind.prefix()) { + problems.push(format!( + "{file}: `{old}` would become `{id}`, which is not a valid id; rename it in {file} first" + )); + continue; + } + if out.iter().any(|e| e.id == id) { + problems.push(format!("{file}: two entries would both be `{id}`")); + continue; + } + + let mut entry = Entry { + old: old.clone(), + id, + fields: Mapping::new(), + derived: None, + dropped: Vec::new(), + relinked: None, + }; + for (key, value) in map { + let Some(key) = key.as_str() else { + entry + .dropped + .push(format!("`{}`: a key that is not a string", entry.id)); + continue; + }; + if key == "id" { + continue; + } + if key == kind.derived() { + entry.derived = Some(value.clone()); + continue; + } + let name = kind.renamed(key).unwrap_or(key); + if !kind.fields().contains(&name) { + entry.dropped.push(format!( + "`{}`: `{key}`, which 2.1 has no field for", + entry.id + )); + continue; + } + let mut value = value.clone(); + if kind == Kind::Value + && name == "term" + && let Some(term) = value.as_str() + && !term.starts_with(TERM_PREFIX) + { + entry.relinked = Some(term.to_string()); + value = Yaml::String(format!("{TERM_PREFIX}{term}")); + } + if entry + .fields + .insert(Yaml::String(name.to_string()), value) + .is_some() + { + problems.push(format!( + "{file}: `{old}` gives `{name}` twice, once as `{key}`" + )); + } + } + out.push(entry); + } + if !problems.is_empty() { + return Err(Error::Invalid(problems)); + } + Ok(out) +} + +/// A scalar as a string: strings as they are, numbers and booleans as written. +fn plain(value: &Yaml) -> Option { + match value { + Yaml::String(s) => Some(s.clone()), + Yaml::Number(n) => Some(n.to_string()), + Yaml::Bool(b) => Some(b.to_string()), + _ => None, + } +} + +/// The list items under a top-level key, as lines. +fn items(text: &str, section: &str) -> Vec { + let lines: Vec<&str> = text.lines().collect(); + let Some(start) = lines + .iter() + .position(|l| key_at(l, 0).as_deref() == Some(section)) + else { + return Vec::new(); + }; + let body: Vec<&str> = lines[start + 1..] + .iter() + .take_while(|l| key_at(l, 0).is_none()) + .copied() + .collect(); + let starts_item = |line: &str| { + let trimmed = line.trim_start(); + trimmed.starts_with("- ") || trimmed == "-" + }; + let Some(dash) = body + .iter() + .find(|l| starts_item(l)) + .and_then(|l| indent_of(l)) + else { + return Vec::new(); + }; + + let mut out: Vec = Vec::new(); + let mut lead: Vec = Vec::new(); + for line in body { + if indent_of(line) == Some(dash) && starts_item(line) { + if let Some(previous) = out.last_mut() { + lead = trailing_comments(&mut previous.lines); + } + out.push(Item { + lead: std::mem::take(&mut lead), + lines: vec![line.to_string()], + dash, + }); + continue; + } + match out.last_mut() { + Some(item) => item.lines.push(line.to_string()), + None if line.trim_start().starts_with('#') => lead.push(line.to_string()), + None => {} + } + } + if let Some(last) = out.last_mut() { + trailing_comments(&mut last.lines); + } + out +} + +/// Takes a run of trailing blank and comment lines off an item; returns the +/// comments, which belong to whatever follows. +fn trailing_comments(lines: &mut Vec) -> Vec { + let mut tail = Vec::new(); + while lines.len() > 1 { + let last = lines.last().map(|l| l.trim()).unwrap_or_default(); + if last.is_empty() || last.starts_with('#') { + tail.push(lines.pop().unwrap_or_default()); + } else { + break; + } + } + tail.reverse(); + tail.into_iter() + .filter(|l| l.trim().starts_with('#')) + .collect() +} + +/// An item's lines as an entry: ` term-x:` and its fields four deep. +/// +/// `None` when the item is not written in a form this follows line by line, +/// for the caller to serialize instead. +fn copied(item: &Item, kind: Kind, entry: &Entry) -> Option> { + // The dash becomes a space, so the first field sits where the rest do. + let first = item.lines.first()?; + let after = first.get(item.dash + 1..)?; + let mut lines: Vec = Vec::new(); + let indent = if after.trim().is_empty() { + // A bare `-` with the fields on the lines below it. + lines.extend(item.lines[1..].iter().cloned()); + lines.iter().find_map(|l| indent_of(l))? + } else { + lines.push(format!("{} {after}", &first[..item.dash])); + lines.extend(item.lines[1..].iter().cloned()); + indent_of(&lines[0])? + }; + // A flow mapping is one value, not lines of fields. + if lines + .iter() + .find(|l| indent_of(l).is_some()) + .is_some_and(|l| l.trim_start().starts_with(['{', '['])) + { + return None; + } + + // Split into fields: a key at the field indent starts one; comments and + // deeper lines continue it. + let mut fields: Vec<(String, Vec)> = Vec::new(); + let mut lead: Vec = item.lead.clone(); + for line in lines { + match key_at(&line, indent) { + Some(key) => fields.push((key, vec![line])), + None => match fields.last_mut() { + Some((_, field)) => field.push(line), + None if line.trim().is_empty() => {} + None if line.trim_start().starts_with('#') => lead.push(line), + None => return None, + }, + } + } + if fields.is_empty() { + return None; + } + + let mut out: Vec = lead.iter().map(|l| format!(" {}", l.trim())).collect(); + out.push(format!(" {}:", entry.id)); + for (key, mut field) in fields { + if key == "id" || key == kind.derived() { + continue; + } + let name = kind.renamed(&key).unwrap_or(&key); + if !kind.fields().contains(&name) { + continue; + } + if name != key { + let line = &field[0]; + if !line[indent..].starts_with(key.as_str()) { + return None; + } + field[0] = format!("{}{name}{}", &line[..indent], &line[indent + key.len()..]); + } + if kind == Kind::Value && key == "term" && entry.relinked.is_some() { + // Only a one-line plain scalar is rewritten in place. + let term = scalar(&field[0], indent, "term")?; + if field.len() > 1 || entry.relinked.as_deref() != Some(term.as_str()) { + return None; + } + field[0] = format!("{}term: {TERM_PREFIX}{term}", &field[0][..indent]); + } + out.extend(field.iter().map(|l| shift(l, indent, 4))); + } + while out.last().is_some_and(|l| l.trim().is_empty()) { + out.pop(); + } + Some(out) +} + +/// Moves a line from one indentation to another, keeping what is under it +/// relative. +fn shift(line: &str, from: usize, to: usize) -> String { + if line.trim().is_empty() { + return String::new(); + } + if to >= from { + return format!("{}{line}", " ".repeat(to - from)); + } + let leading = line.len() - line.trim_start().len(); + line[leading.min(from - to)..].to_string() +} + +/// Whether a chunk reads back as the entry it was copied from. +fn reads_back( + chunk: &[String], + kind: Kind, + id: &str, + expected: &T, + pick: fn(&Fragment) -> &BTreeMap, +) -> bool { + let text = format!("{}:\n{}\n", kind.section(), chunk.join("\n")); + serde_yaml_ng::from_str::(&text) + .ok() + .is_some_and(|fragment| pick(&fragment).get(id) == Some(expected)) +} + +/// An entry written by the serializer, for when its lines could not be copied. +fn serialized(id: &str, entry: &T) -> Result> { + let text = yaml::to_string(entry)?; + let mut out = vec![format!(" {id}:")]; + out.extend( + text.lines() + .filter(|l| *l != "---") + .map(|l| format!(" {l}")), + ); + Ok(out) +} + +/// What a dropped `first` or `lectures` said, when the targets now say +/// something else. +fn derived_message(kind: Kind, entry: &Entry, course: &CourseFile) -> Option { + let old = entry.derived.as_ref()?; + let said: Vec = match old { + Yaml::Sequence(list) => list.iter().filter_map(plain).collect(), + other => plain(other).into_iter().collect(), + }; + let now: Vec<&str> = match kind { + Kind::Term => course.first_lecture(&entry.id).into_iter().collect(), + Kind::Value => lectures_listing(course, &entry.id, RefKind::Value), + }; + let same = said.iter().map(String::as_str).collect::>() + == now.iter().copied().collect::>(); + if same { + return None; + } + let derived = if now.is_empty() { + "none".to_string() + } else if kind == Kind::Term { + now.join(", ") + } else { + format!("[{}]", now.join(", ")) + }; + let was = if kind == Kind::Term { + said.join(", ") + } else { + format!("[{}]", said.join(", ")) + }; + Some(format!( + "`{}`: `{}` was {was}; derived from the targets: {derived}", + entry.id, + kind.derived() + )) +} + +/// Every lecture, in order, with a target that lists an entry. +fn lectures_listing<'a>(course: &'a CourseFile, id: &str, kind: RefKind) -> Vec<&'a str> { + course + .lectures + .keys() + .map(String::as_str) + .filter(|lecture| { + course + .lecture_targets_in_order(lecture) + .iter() + .filter_map(|t| course.learning_targets.get(*t)) + .any(|target| target.listed(kind).iter().any(|x| x == id)) + }) + .collect() +} + +/// A path's file name, for messages. +fn file_name(path: &Path) -> String { + path.file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_else(|| path.display().to_string()) +} + +/// The comment a new `glossary.yaml` starts with. +fn header(sources: &[String]) -> String { + format!( + "# glossary.yaml (coursebank 2.1)\n\ + # Entries brought in from {} by `coursebank migrate glossary`. Move each one to the\n\ + # objective file whose targets first need it; keep the cross-cutting ones here.\n", + sources.join(" and ") + ) +} + +/// Adds entry chunks at the end of a top-level section, or a new section at +/// the end of the file. +fn insert(text: &str, section: &str, chunks: &[Vec]) -> Result { + let mut lines: Vec = text.lines().map(str::to_string).collect(); + let new: Vec = chunks.iter().flatten().cloned().collect(); + match lines + .iter() + .position(|l| key_at(l, 0).as_deref() == Some(section)) + { + Some(at) => { + let colon = lines[at].find(':').unwrap_or(lines[at].len()); + let rest = lines[at][colon + 1..].trim().to_string(); + if !(rest.is_empty() || rest.starts_with('#')) { + if rest == "{}" { + lines[at] = format!("{section}:"); + } else { + return Err(Error::usage(format!( + "glossary.yaml writes `{section}:` on one line; write it as a block, one \ + entry per key, and run this again" + ))); + } + } + let mut end = at + 1; + while end < lines.len() && key_at(&lines[end], 0).is_none() { + end += 1; + } + // Blank lines and column-0 comments at the end of the section + // introduce whatever follows it. + while end > at + 1 { + let last = &lines[end - 1]; + let comment = indent_of(last) == Some(0) && last.starts_with('#'); + if last.trim().is_empty() || comment { + end -= 1; + } else { + break; + } + } + lines.splice(end..end, new); + } + None => { + while lines.last().is_some_and(|l| l.trim().is_empty()) { + lines.pop(); + } + let has_content = lines.iter().any(|l| { + let t = l.trim(); + !t.is_empty() && !t.starts_with('#') + }); + if has_content { + lines.push(String::new()); + } + lines.push(format!("{section}:")); + lines.extend(new); + } + } + Ok(format!("{}\n", lines.join("\n"))) +} + +/// Checks that the new file reads back as the old one plus the migrated +/// entries, and nothing else. +fn verify( + before: Option<&str>, + after: &str, + terms: &BTreeMap, + values: &BTreeMap, +) -> Result<()> { + let unreadable = |e: serde_yaml_ng::Error| { + Error::other(format!( + "the glossary.yaml this would write does not load ({e}); nothing was written" + )) + }; + let mut parsed: Fragment = serde_yaml_ng::from_str(after).map_err(unreadable)?; + for (id, term) in terms { + if parsed.terms.remove(id).as_ref() != Some(term) { + return Err(Error::other(format!( + "`{id}` does not read back as migrated; nothing was written" + ))); + } + } + for (id, value) in values { + if parsed.values.remove(id).as_ref() != Some(value) { + return Err(Error::other(format!( + "`{id}` does not read back as migrated; nothing was written" + ))); + } + } + let old: Fragment = match before { + Some(text) => serde_yaml_ng::from_str(text).map_err(unreadable)?, + None => Fragment::default(), + }; + if yaml::to_string(&parsed)? != yaml::to_string(&old)? { + return Err(Error::other( + "migrating would change entries glossary.yaml already has; nothing was written", + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + const TERMS: &str = "schema_version: \"1.0\" +terms: + # A coined one. + - id: decade + course: decade + variants: [decades] + kind: coined + meaning: >- + a factor of ten in a concentration + ratio + first: L2.1 + color: blue + - id: term-gap + course: gap + meaning: a dash +"; + + fn parsed(text: &str, kind: Kind) -> Vec { + let doc: Yaml = serde_yaml_ng::from_str(text).unwrap(); + entries(&doc, kind, "terms.yaml").unwrap() + } + + #[test] + fn entries_take_their_prefix_once_and_lose_derived_and_unknown_fields() { + let entries = parsed(TERMS, Kind::Term); + assert_eq!(entries[0].id, "term-decade"); + assert_eq!(entries[1].id, "term-gap"); + assert_eq!(entries[0].derived, Some(Yaml::String("L2.1".into()))); + assert_eq!(entries[0].dropped.len(), 1); + assert!(entries[0].dropped[0].contains("`color`")); + assert!(entries[0].fields.contains_key("definition")); + assert!(!entries[0].fields.contains_key("meaning")); + } + + #[test] + fn copied_lines_keep_comments_and_block_scalars() { + let entries = parsed(TERMS, Kind::Term); + let items = items(TERMS, "terms"); + assert_eq!(items.len(), 2); + let chunk = copied(&items[0], Kind::Term, &entries[0]).unwrap(); + assert_eq!( + chunk, + [ + " # A coined one.", + " term-decade:", + " term: decade", + " variants: [decades]", + " kind: coined", + " definition: >-", + " a factor of ten in a concentration", + " ratio", + ] + ); + let term: Term = + serde_yaml_ng::from_value(Yaml::Mapping(entries[0].fields.clone())).unwrap(); + assert!(reads_back(&chunk, Kind::Term, "term-decade", &term, |f| &f.terms)); + } + + #[test] + fn items_at_column_zero_are_reindented() { + let text = "terms:\n- id: gap\n course: gap\n meaning: a dash\n"; + let entries = parsed(text, Kind::Term); + let items = items(text, "terms"); + let chunk = copied(&items[0], Kind::Term, &entries[0]).unwrap(); + assert_eq!( + chunk, + [" term-gap:", " term: gap", " definition: a dash"] + ); + } + + #[test] + fn a_value_names_its_term_by_the_new_id() { + let text = "values:\n - id: k\n name: k\n say: about 6\n term: decade\n"; + let entries = parsed(text, Kind::Value); + assert_eq!(entries[0].relinked.as_deref(), Some("decade")); + let items = items(text, "values"); + let chunk = copied(&items[0], Kind::Value, &entries[0]).unwrap(); + assert_eq!(chunk.last().unwrap(), " term: term-decade"); + } + + #[test] + fn a_flow_item_falls_back_to_the_serializer() { + let text = "terms:\n - {id: gap, course: gap, meaning: a dash}\n"; + let entries = parsed(text, Kind::Term); + let items = items(text, "terms"); + assert!(copied(&items[0], Kind::Term, &entries[0]).is_none()); + let term: Term = + serde_yaml_ng::from_value(Yaml::Mapping(entries[0].fields.clone())).unwrap(); + let chunk = serialized("term-gap", &term).unwrap(); + assert!(reads_back(&chunk, Kind::Term, "term-gap", &term, |f| &f.terms)); + } + + #[test] + fn bad_and_duplicate_ids_are_refused() { + let doc: Yaml = serde_yaml_ng::from_str( + "terms:\n - id: Two Words\n course: x\n meaning: y\n - id: a\n course: a\n meaning: b\n - id: term-a\n course: a\n meaning: b\n - course: z\n", + ) + .unwrap(); + let Err(Error::Invalid(problems)) = entries(&doc, Kind::Term, "terms.yaml") else { + panic!("expected problems"); + }; + assert_eq!(problems.len(), 3, "{problems:?}"); + } + + #[test] + fn insert_appends_to_a_section_or_adds_one() { + let text = "# keep\nterms:\n term-a:\n term: a\n definition: b\n\n# about values\nvalues: {}\n"; + let chunk = vec![" term-c:".to_string(), " term: c".to_string()]; + let out = insert(text, "terms", std::slice::from_ref(&chunk)).unwrap(); + assert!(out.contains(" definition: b\n term-c:\n term: c\n\n# about values\n")); + let value = vec![" val-x:".to_string(), " name: x".to_string()]; + let out = insert(&out, "values", &[value]).unwrap(); + assert!(out.ends_with("values:\n val-x:\n name: x\n")); + let fresh = insert("# header\n", "terms", &[chunk]).unwrap(); + assert_eq!(fresh, "# header\nterms:\n term-c:\n term: c\n"); + } + + #[test] + fn shift_moves_whole_blocks() { + assert_eq!(shift(" deeper", 2, 4), " deeper"); + assert_eq!(shift(" deeper", 6, 4), " deeper"); + assert_eq!(shift(" ", 2, 4), ""); + } +} diff --git a/tests/migrate_2_1.rs b/tests/migrate_2_1.rs new file mode 100644 index 0000000..23d10bb --- /dev/null +++ b/tests/migrate_2_1.rs @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: Prosperity-3.0.0 +// Copyright Scienting +// Source: https://git.scient.ing/education/coursebank + +//! `coursebank migrate glossary`, against `docs/design/2.1/examples/migration/`. +//! +//! The example course's own `glossary.yaml` is that migration's output after a +//! person linked the value to its term, so each test starts from the example +//! with it removed. + +mod common; + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use common::{design, edit, v21_course}; +use coursebank::course::fragment::Fragment; +use coursebank::migrate; + +/// The example course without its glossary. +fn course(tag: &str) -> PathBuf { + let root = v21_course(tag); + std::fs::remove_file(root.join("glossary.yaml")).unwrap(); + root +} + +fn source(name: &str) -> PathBuf { + design().join("examples/migration").join(name) +} + +fn fragment(path: &Path) -> Fragment { + serde_yaml_ng::from_str(&std::fs::read_to_string(path).unwrap()).unwrap() +} + +fn run(root: &Path, write: bool) -> migrate::GlossaryMigration { + migrate::glossary( + root, + Some(&source("terms.yaml")), + Some(&source("values.yaml")), + write, + ) + .unwrap() +} + +#[test] +fn migration_reproduces_the_expected_glossary() { + let root = course("mig-expected"); + let report = run(&root, true); + + let written = fragment(&root.join("glossary.yaml")); + let expected = fragment(&source("glossary.expected.yaml")); + assert_eq!(written.terms, expected.terms); + assert_eq!(written.values, expected.values); + + // The report the expected file's header comment describes. + assert_eq!( + report.renamed, + [ + ("decade".to_string(), "term-decade".to_string()), + ("per-decade".to_string(), "val-per-decade".to_string()), + ] + ); + assert_eq!(report.derived.len(), 2, "{:?}", report.derived); + assert!(report.derived[0].contains("`first` was L2.1")); + assert!(report.derived[1].contains("`lectures` was [L2.1, L2.4, L2.5]")); + assert_eq!(report.unlisted, ["term-decade", "val-per-decade"]); + assert!(report.dropped.is_empty() && report.relinked.is_empty()); + + // The course still validates with the written file. + let issues = coursebank::Catalog::load(&root) + .unwrap() + .validate() + .unwrap(); + assert!(issues.is_empty(), "{issues:#?}"); +} + +#[test] +fn migrating_twice_is_a_no_op() { + let root = course("mig-twice"); + run(&root, true); + let before = std::fs::read(root.join("glossary.yaml")).unwrap(); + + let again = run(&root, true); + assert!(again.text.is_none()); + assert!(again.added.is_empty() && again.renamed.is_empty()); + assert_eq!(again.present.len(), 2); + assert_eq!(std::fs::read(root.join("glossary.yaml")).unwrap(), before); + + // The example as shipped already holds both, linked: also a no-op. + let shipped = v21_course("mig-shipped"); + assert!(run(&shipped, true).text.is_none()); +} + +#[test] +fn a_dropped_first_is_compared_with_the_targets() { + let root = course("mig-derived"); + edit( + &root, + "objectives/lo-alignment-scoring.yaml", + "terms: [term-alignment-matrix, term-gap,", + "terms: [term-decade, term-alignment-matrix, term-gap,", + ); + let report = run(&root, false); + assert!( + report.derived[0].contains("`first` was L2.1; derived from the targets: l09"), + "{:?}", + report.derived + ); + assert_eq!(report.unlisted, ["val-per-decade"]); + assert!(!root.join("glossary.yaml").exists(), "a dry run wrote"); +} + +#[test] +fn an_existing_glossary_is_added_to_not_rewritten() { + let root = course("mig-existing"); + let before = "# Cross-cutting terms.\nterms:\n term-other:\n term: other\n \ + definition: Something else.\n"; + std::fs::write(root.join("glossary.yaml"), before).unwrap(); + + let report = run(&root, true); + let after = std::fs::read_to_string(root.join("glossary.yaml")).unwrap(); + assert!(after.starts_with(before), "{after}"); + assert!(after.contains("\nvalues:\n val-per-decade:\n")); + assert_eq!(report.added, ["term-decade", "val-per-decade"]); + + let written = fragment(&root.join("glossary.yaml")); + assert_eq!(written.terms.len(), 2); + assert_eq!(written.values.len(), 1); +} + +#[test] +fn the_cli_reports_and_a_dry_run_writes_nothing() { + let root = course("mig-cli"); + let terms = source("terms.yaml"); + let values = source("values.yaml"); + let out = Command::new(env!("CARGO_BIN_EXE_coursebank")) + .arg("-C") + .arg(&root) + .args(["migrate", "glossary", "--dry-run", "--terms"]) + .arg(&terms) + .arg("--values") + .arg(&values) + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&out.stdout); + assert!(out.status.success(), "{stdout}"); + assert!(stdout.contains("decade -> term-decade")); + assert!(stdout.contains("nothing written")); + assert!(!root.join("glossary.yaml").exists()); + + let none = Command::new(env!("CARGO_BIN_EXE_coursebank")) + .arg("-C") + .arg(&root) + .args(["migrate", "glossary"]) + .output() + .unwrap(); + assert!(!none.status.success()); +}