WIP: Prelim use #2

Draft
alexm wants to merge 40 commits from prelim-use into main
pull from: prelim-use
5 changed files with 1221 additions and 0 deletions
Showing only changes of commit 6c07122fc5 - Show all commits

No files matched your search

+19
View File
@@ -200,6 +200,25 @@ pub(crate) enum MigrateCommand {
#[arg(long)]
dry_run: bool,
},
/// Bring a 1.0 terms.yaml and values.yaml into the 2.1 glossary.
///
/// Writes glossary.yaml. Ids gain their `term-` and `val-` prefixes,
/// `course` becomes `term` and `meaning` becomes `definition`, and `first`
/// and `lectures` are dropped, since 2.1 derives them from the targets
/// that list an entry; each one that disagrees is reported. The old files
/// are left in place, and an entry the course already declares is
/// skipped, so a second run changes nothing.
Glossary {
/// The 1.0 terms file.
#[arg(long, value_name = "FILE")]
terms: Option<PathBuf>,
/// The 1.0 values file.
#[arg(long, value_name = "FILE")]
values: Option<PathBuf>,
/// Show what would be written, and write nothing.
#[arg(long)]
dry_run: bool,
},
/// Rewrite pre-2.0 `bank::item` ids as the item ids they name.
///
/// Touches assessment records, seals, and the response store. Everything
+65
View File
@@ -118,9 +118,74 @@ pub(crate) fn migrate(cli: &Cli, sub: &MigrateCommand) -> Result<Outcome> {
MigrateCommand::References { dry_run } => migrate_references(cli, *dry_run),
MigrateCommand::Order { dry_run } => migrate_order(cli, *dry_run),
MigrateCommand::Counters { dry_run } => migrate_counters(cli, *dry_run),
MigrateCommand::Glossary {
terms,
values,
dry_run,
} => migrate_glossary(cli, terms.as_deref(), values.as_deref(), *dry_run),
}
}
/// Brings a 1.0 terms.yaml and values.yaml into glossary.yaml.
fn migrate_glossary(
cli: &Cli,
terms: Option<&Path>,
values: Option<&Path>,
dry_run: bool,
) -> Result<Outcome> {
let report = migrate::glossary(&cli.course, terms, values, !dry_run)?;
let list = |heading: &str, lines: &[String]| {
if !lines.is_empty() {
println!("{heading}");
for line in lines {
println!(" {line}");
}
println!();
}
};
let renamed: Vec<String> = report
.renamed
.iter()
.map(|(old, new)| format!("{old} -> {new}"))
.collect();
list("renamed:", &renamed);
list(
"dropped `first` and `lectures` that the targets now contradict:",
&report.derived,
);
list("dropped, no 2.1 field:", &report.dropped);
list("relinked:", &report.relinked);
list("no target lists these yet:", &report.unlisted);
let present: Vec<String> = report
.present
.iter()
.map(|(id, path)| format!("{id} ({})", path.display()))
.collect();
list("already declared, skipped:", &present);
let Some(text) = &report.text else {
println!("nothing to do: the course already declares every entry");
return Ok(Outcome::Ok);
};
if dry_run {
println!(
"{} would read:\n\n{text}\nnothing written",
report.path.display()
);
return Ok(Outcome::Ok);
}
println!(
"wrote {} entr{} to {}\n\nNext:\n coursebank validate\n coursebank lint \
# glossary-unused lists entries nothing points at yet\n\n\
The old files are untouched; delete them once the pipelines read the registry.",
report.added.len(),
if report.added.len() == 1 { "y" } else { "ies" },
report.path.display()
);
Ok(Outcome::Ok)
}
/// Drops the trailing counter from every item id.
fn migrate_counters(cli: &Cli, dry_run: bool) -> Result<Outcome> {
let (rename, touched) = migrate::counters(&cli.course, !dry_run)?;
+4
View File
@@ -22,6 +22,7 @@
//! | [`references`] | prose notes into citation fields | a DOI in a note is a link nobody can follow |
//! | [`order`] | removes the `order:` integers | objective order is derived, and targets have none |
//! | [`counters`] | drops the `-001` from item ids | a per-bank sequence number is not part of a name |
//! | [`glossary()`] | 1.0 `terms.yaml` and `values.yaml` into `glossary.yaml` | an entry is defined once, by id, where targets can point at it |
//!
//! # What the option migration does not touch
//!
@@ -74,6 +75,9 @@ use crate::layout::Layout;
use crate::store::{self, Store};
use crate::yaml;
mod glossary;
pub use glossary::{GlossaryMigration, glossary};
/// The width past which a flow sequence is written as a block list instead.
const FLOW_WIDTH: usize = 96;
+975
View File
@@ -0,0 +1,975 @@
//! `migrate glossary`: a 1.0 `terms.yaml` and `values.yaml` into the 2.1
//! registries.
//!
//! The same textual approach as the rest of [`crate::migrate`]. Each entry's
//! lines are copied and re-indented from a list item to a map entry, with three
//! edits: the `id` becomes the key and gains its prefix, `course` and `meaning`
//! are renamed, and `first` and `lectures`, which 2.1 derives from the targets
//! that list an entry, are dropped. Comments, folded scalars, and flow mappings
//! like `{ term: ten-fold }` survive.
//!
//! Every copied entry is parsed back and compared with the same conversion done
//! on the parsed source. An entry whose copied text does not read back the same
//! (a flow-mapping list item, a quoted key) is written by the serializer
//! instead, so the file is right even where it is not pretty. The whole file is
//! checked the same way before anything is written, and the course is loaded
//! again after.
//!
//! An entry the course already declares, in `glossary.yaml` or any other
//! fragment, is skipped, so a second run is a no-op.
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use serde::Serialize;
use serde::de::DeserializeOwned;
use serde_yaml_ng::{Mapping, Value as Yaml};
use super::{indent_of, key_at, relative, scalar};
use crate::course::CourseFile;
use crate::course::fragment::{Fragment, Section};
use crate::course::glossary::{self, RefKind, TERM_PREFIX, Term, VALUE_PREFIX, Value};
use crate::error::{Error, Result};
use crate::layout::Layout;
use crate::yaml;
/// What `migrate glossary` writes, and what it found on the way.
#[derive(Debug, Clone, Default)]
pub struct GlossaryMigration {
/// The file it writes, relative to the course root.
pub path: PathBuf,
/// The file's new contents; `None` when there is nothing to add.
pub text: Option<String>,
/// Ids added, terms first, each in source order.
pub added: Vec<String>,
/// Old id to new id, for every added entry whose id gained a prefix.
pub renamed: Vec<(String, String)>,
/// Entries the course already declares, with the file that declares them.
pub present: Vec<(String, PathBuf)>,
/// One message per dropped `first` or `lectures` that differs from what
/// the targets now say.
pub derived: Vec<String>,
/// Added entries that no target lists yet.
pub unlisted: Vec<String>,
/// One message per field dropped because 2.1 has no place for it.
pub dropped: Vec<String>,
/// One message per value whose `term:` was rewritten to the prefixed id.
pub relinked: Vec<String>,
}
/// Which 1.0 file an entry came from.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Kind {
Term,
Value,
}
impl Kind {
/// The top-level key, the same in 1.0 and 2.1.
fn section(self) -> &'static str {
match self {
Kind::Term => "terms",
Kind::Value => "values",
}
}
/// The prefix a 2.1 id carries.
fn prefix(self) -> &'static str {
match self {
Kind::Term => TERM_PREFIX,
Kind::Value => VALUE_PREFIX,
}
}
/// The registry an id of this kind lives in.
fn registry(self) -> Section {
match self {
Kind::Term => Section::Terms,
Kind::Value => Section::Values,
}
}
/// The 1.0 field that 2.1 derives from the targets instead.
fn derived(self) -> &'static str {
match self {
Kind::Term => "first",
Kind::Value => "lectures",
}
}
/// The 2.1 name of a renamed 1.0 field.
fn renamed(self, key: &str) -> Option<&'static str> {
match (self, key) {
(Kind::Term, "course") => Some("term"),
(Kind::Term, "meaning") => Some("definition"),
_ => None,
}
}
/// Every field the 2.1 entry has; anything else is dropped and reported.
fn fields(self) -> &'static [&'static str] {
match self {
Kind::Term => &[
"term",
"definition",
"kind",
"textbook",
"variants",
"symbol",
"units",
"exam_wording",
"status",
"tags",
"notes_private",
],
Kind::Value => &[
"name",
"value",
"range",
"units",
"say",
"exact",
"term",
"source",
"conflicts",
"recall",
"status",
"tags",
"notes_private",
],
}
}
}
/// One 1.0 entry, read leniently and converted to 2.1 fields.
#[derive(Debug, Clone)]
struct Entry {
/// The id as the 1.0 file wrote it.
old: String,
/// The 2.1 id.
id: String,
/// The 2.1 fields, renamed, unknown ones gone.
fields: Mapping,
/// What the dropped `first` or `lectures` said.
derived: Option<Yaml>,
/// One message per unknown field dropped.
dropped: Vec<String>,
/// The value's `term:` before it gained its prefix.
relinked: Option<String>,
}
/// One list item in a 1.0 file, as lines.
#[derive(Debug, Clone, Default)]
struct Item {
/// Comment lines directly above the item.
lead: Vec<String>,
/// The `- ` line and everything under it.
lines: Vec<String>,
/// The column of the item's dash.
dash: usize,
}
/// Writes a 1.0 `terms.yaml` and `values.yaml` into the course's
/// `glossary.yaml`.
///
/// Ids gain their `term-` and `val-` prefixes, `course` becomes `term` and
/// `meaning` becomes `definition`, a value's `term:` gains the prefix of the id
/// it names, and `first` and `lectures` are dropped: 2.1 derives them from the
/// targets that list an entry, and each one that disagrees with what the
/// targets now say is reported. A field 2.1 has no place for is dropped and
/// reported. The old files are left alone.
///
/// An existing `glossary.yaml` is added to, never rewritten: new entries go at
/// the end of its `terms:` and `values:` sections, and everything already in
/// it stays byte for byte.
///
/// # Arguments
///
/// * `root` - the course directory.
/// * `terms` - the 1.0 terms file, if migrating one.
/// * `values` - the 1.0 values file, if migrating one.
/// * `write` - whether to write, or only to report.
///
/// # Returns
///
/// What was (or would be) written, and the report.
///
/// # Errors
///
/// Returns [`Error::Usage`] when neither file is given, [`Error::Invalid`] for
/// entries that cannot be migrated (no id, an id that would not be valid, a
/// field 2.1 would reject), and [`Error::Other`] if the written file would not
/// read back as migrated. Nothing is written in any of those cases.
pub fn glossary(
root: &Path,
terms: Option<&Path>,
values: Option<&Path>,
write: bool,
) -> Result<GlossaryMigration> {
if terms.is_none() && values.is_none() {
return Err(Error::usage(
"nothing to migrate: give --terms, --values, or both",
));
}
let layout = Layout::new(root);
let course = CourseFile::load_dir(root)?;
let path = layout.glossary_file();
let mut report = GlossaryMigration {
path: relative(root, &path),
..GlossaryMigration::default()
};
let mut new_terms: BTreeMap<String, Term> = BTreeMap::new();
let mut new_values: BTreeMap<String, Value> = BTreeMap::new();
let mut sections: Vec<(Kind, Vec<Vec<String>>)> = Vec::new();
if let Some(file) = terms {
let chunks = convert(
root,
file,
Kind::Term,
&course,
&mut report,
&mut new_terms,
|f| &f.terms,
)?;
sections.push((Kind::Term, chunks));
}
if let Some(file) = values {
let chunks = convert(
root,
file,
Kind::Value,
&course,
&mut report,
&mut new_values,
|f| &f.values,
)?;
sections.push((Kind::Value, chunks));
}
if report.added.is_empty() {
return Ok(report);
}
let before = if path.is_file() {
Some(std::fs::read_to_string(&path).map_err(|e| Error::io(&path, e))?)
} else {
None
};
let names: Vec<String> = [terms, values]
.into_iter()
.flatten()
.map(file_name)
.collect();
let mut text = before.clone().unwrap_or_else(|| header(&names));
for (kind, chunks) in &sections {
if !chunks.is_empty() {
text = insert(&text, kind.section(), chunks)?;
}
}
verify(before.as_deref(), &text, &new_terms, &new_values)?;
if write {
yaml::write_text(&path, &text)?;
// The fragment reads back; this checks the course still assembles with
// it, and puts the old file back when it does not.
if let Err(e) = CourseFile::load_dir(root) {
match &before {
Some(old) => yaml::write_text(&path, old)?,
None => std::fs::remove_file(&path).map_err(|e| Error::io(&path, e))?,
}
return Err(e);
}
}
report.text = Some(text);
Ok(report)
}
/// Converts one 1.0 file into the entry chunks to write.
///
/// Fills in the report and `expected`, the typed entries the written file must
/// read back as.
fn convert<T>(
root: &Path,
file: &Path,
kind: Kind,
course: &CourseFile,
report: &mut GlossaryMigration,
expected: &mut BTreeMap<String, T>,
pick: fn(&Fragment) -> &BTreeMap<String, T>,
) -> Result<Vec<Vec<String>>>
where
T: DeserializeOwned + Serialize + PartialEq + Clone,
{
let text = std::fs::read_to_string(file).map_err(|e| Error::io(file, e))?;
let doc: Yaml = serde_yaml_ng::from_str(&text).map_err(|source| Error::Yaml {
path: file.to_path_buf(),
source,
})?;
let name = file_name(file);
let entries = entries(&doc, kind, &name)?;
// The text items line up with the parsed entries one for one, unless the
// list is written in some form the scanner does not follow; then every
// entry is serialized.
let items = items(&text, kind.section());
let items = (items.len() == entries.len()).then_some(items);
let first_use = course.glossary_first_use();
let mut problems = Vec::new();
let mut chunks = Vec::new();
for (n, entry) in entries.iter().enumerate() {
if let Some((section, at)) = course.origin(&entry.id) {
let at = relative(root, at);
if section == kind.registry() {
report.present.push((entry.id.clone(), at));
} else {
problems.push(format!(
"{name}: `{}` would become `{}`, which {} already uses under `{}`",
entry.old,
entry.id,
at.display(),
section.key()
));
}
continue;
}
let typed: T = match serde_yaml_ng::from_value(Yaml::Mapping(entry.fields.clone())) {
Ok(typed) => typed,
Err(e) => {
problems.push(format!("{name}: `{}`: {e}", entry.old));
continue;
}
};
let copied = items
.as_ref()
.and_then(|items| copied(&items[n], kind, entry))
.filter(|chunk| reads_back(chunk, kind, &entry.id, &typed, pick));
let chunk = match copied {
Some(chunk) => chunk,
None => serialized(&entry.id, &typed)?,
};
if entry.old != entry.id {
report.renamed.push((entry.old.clone(), entry.id.clone()));
}
report.dropped.extend(entry.dropped.iter().cloned());
if let Some(old) = &entry.relinked {
report.relinked.push(format!(
"`{}`: `term: {old}` now reads `term: {TERM_PREFIX}{old}`",
entry.id
));
}
if let Some(message) = derived_message(kind, entry, course) {
report.derived.push(message);
}
if !first_use.contains_key(entry.id.as_str()) {
report.unlisted.push(entry.id.clone());
}
report.added.push(entry.id.clone());
expected.insert(entry.id.clone(), typed);
chunks.push(chunk);
}
if !problems.is_empty() {
return Err(Error::Invalid(problems));
}
Ok(chunks)
}
/// Reads the entries of one 1.0 file.
///
/// Lenient about everything 2.1 does not need: a missing schema version, an id
/// written as a number, fields 2.1 has no place for. Strict about what would
/// make a wrong file: an entry with no id, an id that would not be valid with
/// its prefix, two entries that would share an id, and a field given twice
/// under its old and new names.
fn entries(doc: &Yaml, kind: Kind, file: &str) -> Result<Vec<Entry>> {
let section = kind.section();
let list = match doc.get(section) {
None | Some(Yaml::Null) => return Ok(Vec::new()),
Some(Yaml::Sequence(list)) => list,
Some(_) => {
return Err(Error::usage(format!(
"{file}: `{section}` is not a list; 1.0 writes one `- id:` item per entry"
)));
}
};
let mut out: Vec<Entry> = Vec::new();
let mut problems = Vec::new();
for (n, item) in list.iter().enumerate() {
let Yaml::Mapping(map) = item else {
problems.push(format!(
"{file}: {section} entry {} is not a mapping",
n + 1
));
continue;
};
let Some(old) = map.get("id").and_then(plain) else {
problems.push(format!("{file}: {section} entry {} has no `id`", n + 1));
continue;
};
let id = if old.starts_with(kind.prefix()) {
old.clone()
} else {
format!("{}{old}", kind.prefix())
};
if !glossary::is_prefixed_id(&id, kind.prefix()) {
problems.push(format!(
"{file}: `{old}` would become `{id}`, which is not a valid id; rename it in {file} first"
));
continue;
}
if out.iter().any(|e| e.id == id) {
problems.push(format!("{file}: two entries would both be `{id}`"));
continue;
}
let mut entry = Entry {
old: old.clone(),
id,
fields: Mapping::new(),
derived: None,
dropped: Vec::new(),
relinked: None,
};
for (key, value) in map {
let Some(key) = key.as_str() else {
entry
.dropped
.push(format!("`{}`: a key that is not a string", entry.id));
continue;
};
if key == "id" {
continue;
}
if key == kind.derived() {
entry.derived = Some(value.clone());
continue;
}
let name = kind.renamed(key).unwrap_or(key);
if !kind.fields().contains(&name) {
entry.dropped.push(format!(
"`{}`: `{key}`, which 2.1 has no field for",
entry.id
));
continue;
}
let mut value = value.clone();
if kind == Kind::Value
&& name == "term"
&& let Some(term) = value.as_str()
&& !term.starts_with(TERM_PREFIX)
{
entry.relinked = Some(term.to_string());
value = Yaml::String(format!("{TERM_PREFIX}{term}"));
}
if entry
.fields
.insert(Yaml::String(name.to_string()), value)
.is_some()
{
problems.push(format!(
"{file}: `{old}` gives `{name}` twice, once as `{key}`"
));
}
}
out.push(entry);
}
if !problems.is_empty() {
return Err(Error::Invalid(problems));
}
Ok(out)
}
/// A scalar as a string: strings as they are, numbers and booleans as written.
fn plain(value: &Yaml) -> Option<String> {
match value {
Yaml::String(s) => Some(s.clone()),
Yaml::Number(n) => Some(n.to_string()),
Yaml::Bool(b) => Some(b.to_string()),
_ => None,
}
}
/// The list items under a top-level key, as lines.
fn items(text: &str, section: &str) -> Vec<Item> {
let lines: Vec<&str> = text.lines().collect();
let Some(start) = lines
.iter()
.position(|l| key_at(l, 0).as_deref() == Some(section))
else {
return Vec::new();
};
let body: Vec<&str> = lines[start + 1..]
.iter()
.take_while(|l| key_at(l, 0).is_none())
.copied()
.collect();
let starts_item = |line: &str| {
let trimmed = line.trim_start();
trimmed.starts_with("- ") || trimmed == "-"
};
let Some(dash) = body
.iter()
.find(|l| starts_item(l))
.and_then(|l| indent_of(l))
else {
return Vec::new();
};
let mut out: Vec<Item> = Vec::new();
let mut lead: Vec<String> = Vec::new();
for line in body {
if indent_of(line) == Some(dash) && starts_item(line) {
if let Some(previous) = out.last_mut() {
lead = trailing_comments(&mut previous.lines);
}
out.push(Item {
lead: std::mem::take(&mut lead),
lines: vec![line.to_string()],
dash,
});
continue;
}
match out.last_mut() {
Some(item) => item.lines.push(line.to_string()),
None if line.trim_start().starts_with('#') => lead.push(line.to_string()),
None => {}
}
}
if let Some(last) = out.last_mut() {
trailing_comments(&mut last.lines);
}
out
}
/// Takes a run of trailing blank and comment lines off an item; returns the
/// comments, which belong to whatever follows.
fn trailing_comments(lines: &mut Vec<String>) -> Vec<String> {
let mut tail = Vec::new();
while lines.len() > 1 {
let last = lines.last().map(|l| l.trim()).unwrap_or_default();
if last.is_empty() || last.starts_with('#') {
tail.push(lines.pop().unwrap_or_default());
} else {
break;
}
}
tail.reverse();
tail.into_iter()
.filter(|l| l.trim().starts_with('#'))
.collect()
}
/// An item's lines as an entry: ` term-x:` and its fields four deep.
///
/// `None` when the item is not written in a form this follows line by line,
/// for the caller to serialize instead.
fn copied(item: &Item, kind: Kind, entry: &Entry) -> Option<Vec<String>> {
// The dash becomes a space, so the first field sits where the rest do.
let first = item.lines.first()?;
let after = first.get(item.dash + 1..)?;
let mut lines: Vec<String> = Vec::new();
let indent = if after.trim().is_empty() {
// A bare `-` with the fields on the lines below it.
lines.extend(item.lines[1..].iter().cloned());
lines.iter().find_map(|l| indent_of(l))?
} else {
lines.push(format!("{} {after}", &first[..item.dash]));
lines.extend(item.lines[1..].iter().cloned());
indent_of(&lines[0])?
};
// A flow mapping is one value, not lines of fields.
if lines
.iter()
.find(|l| indent_of(l).is_some())
.is_some_and(|l| l.trim_start().starts_with(['{', '[']))
{
return None;
}
// Split into fields: a key at the field indent starts one; comments and
// deeper lines continue it.
let mut fields: Vec<(String, Vec<String>)> = Vec::new();
let mut lead: Vec<String> = item.lead.clone();
for line in lines {
match key_at(&line, indent) {
Some(key) => fields.push((key, vec![line])),
None => match fields.last_mut() {
Some((_, field)) => field.push(line),
None if line.trim().is_empty() => {}
None if line.trim_start().starts_with('#') => lead.push(line),
None => return None,
},
}
}
if fields.is_empty() {
return None;
}
let mut out: Vec<String> = lead.iter().map(|l| format!(" {}", l.trim())).collect();
out.push(format!(" {}:", entry.id));
for (key, mut field) in fields {
if key == "id" || key == kind.derived() {
continue;
}
let name = kind.renamed(&key).unwrap_or(&key);
if !kind.fields().contains(&name) {
continue;
}
if name != key {
let line = &field[0];
if !line[indent..].starts_with(key.as_str()) {
return None;
}
field[0] = format!("{}{name}{}", &line[..indent], &line[indent + key.len()..]);
}
if kind == Kind::Value && key == "term" && entry.relinked.is_some() {
// Only a one-line plain scalar is rewritten in place.
let term = scalar(&field[0], indent, "term")?;
if field.len() > 1 || entry.relinked.as_deref() != Some(term.as_str()) {
return None;
}
field[0] = format!("{}term: {TERM_PREFIX}{term}", &field[0][..indent]);
}
out.extend(field.iter().map(|l| shift(l, indent, 4)));
}
while out.last().is_some_and(|l| l.trim().is_empty()) {
out.pop();
}
Some(out)
}
/// Moves a line from one indentation to another, keeping what is under it
/// relative.
fn shift(line: &str, from: usize, to: usize) -> String {
if line.trim().is_empty() {
return String::new();
}
if to >= from {
return format!("{}{line}", " ".repeat(to - from));
}
let leading = line.len() - line.trim_start().len();
line[leading.min(from - to)..].to_string()
}
/// Whether a chunk reads back as the entry it was copied from.
fn reads_back<T: DeserializeOwned + PartialEq>(
chunk: &[String],
kind: Kind,
id: &str,
expected: &T,
pick: fn(&Fragment) -> &BTreeMap<String, T>,
) -> bool {
let text = format!("{}:\n{}\n", kind.section(), chunk.join("\n"));
serde_yaml_ng::from_str::<Fragment>(&text)
.ok()
.is_some_and(|fragment| pick(&fragment).get(id) == Some(expected))
}
/// An entry written by the serializer, for when its lines could not be copied.
fn serialized<T: Serialize>(id: &str, entry: &T) -> Result<Vec<String>> {
let text = yaml::to_string(entry)?;
let mut out = vec![format!(" {id}:")];
out.extend(
text.lines()
.filter(|l| *l != "---")
.map(|l| format!(" {l}")),
);
Ok(out)
}
/// What a dropped `first` or `lectures` said, when the targets now say
/// something else.
fn derived_message(kind: Kind, entry: &Entry, course: &CourseFile) -> Option<String> {
let old = entry.derived.as_ref()?;
let said: Vec<String> = match old {
Yaml::Sequence(list) => list.iter().filter_map(plain).collect(),
other => plain(other).into_iter().collect(),
};
let now: Vec<&str> = match kind {
Kind::Term => course.first_lecture(&entry.id).into_iter().collect(),
Kind::Value => lectures_listing(course, &entry.id, RefKind::Value),
};
let same = said.iter().map(String::as_str).collect::<BTreeSet<_>>()
== now.iter().copied().collect::<BTreeSet<_>>();
if same {
return None;
}
let derived = if now.is_empty() {
"none".to_string()
} else if kind == Kind::Term {
now.join(", ")
} else {
format!("[{}]", now.join(", "))
};
let was = if kind == Kind::Term {
said.join(", ")
} else {
format!("[{}]", said.join(", "))
};
Some(format!(
"`{}`: `{}` was {was}; derived from the targets: {derived}",
entry.id,
kind.derived()
))
}
/// Every lecture, in order, with a target that lists an entry.
fn lectures_listing<'a>(course: &'a CourseFile, id: &str, kind: RefKind) -> Vec<&'a str> {
course
.lectures
.keys()
.map(String::as_str)
.filter(|lecture| {
course
.lecture_targets_in_order(lecture)
.iter()
.filter_map(|t| course.learning_targets.get(*t))
.any(|target| target.listed(kind).iter().any(|x| x == id))
})
.collect()
}
/// A path's file name, for messages.
fn file_name(path: &Path) -> String {
path.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_else(|| path.display().to_string())
}
/// The comment a new `glossary.yaml` starts with.
fn header(sources: &[String]) -> String {
format!(
"# glossary.yaml (coursebank 2.1)\n\
# Entries brought in from {} by `coursebank migrate glossary`. Move each one to the\n\
# objective file whose targets first need it; keep the cross-cutting ones here.\n",
sources.join(" and ")
)
}
/// Adds entry chunks at the end of a top-level section, or a new section at
/// the end of the file.
fn insert(text: &str, section: &str, chunks: &[Vec<String>]) -> Result<String> {
let mut lines: Vec<String> = text.lines().map(str::to_string).collect();
let new: Vec<String> = chunks.iter().flatten().cloned().collect();
match lines
.iter()
.position(|l| key_at(l, 0).as_deref() == Some(section))
{
Some(at) => {
let colon = lines[at].find(':').unwrap_or(lines[at].len());
let rest = lines[at][colon + 1..].trim().to_string();
if !(rest.is_empty() || rest.starts_with('#')) {
if rest == "{}" {
lines[at] = format!("{section}:");
} else {
return Err(Error::usage(format!(
"glossary.yaml writes `{section}:` on one line; write it as a block, one \
entry per key, and run this again"
)));
}
}
let mut end = at + 1;
while end < lines.len() && key_at(&lines[end], 0).is_none() {
end += 1;
}
// Blank lines and column-0 comments at the end of the section
// introduce whatever follows it.
while end > at + 1 {
let last = &lines[end - 1];
let comment = indent_of(last) == Some(0) && last.starts_with('#');
if last.trim().is_empty() || comment {
end -= 1;
} else {
break;
}
}
lines.splice(end..end, new);
}
None => {
while lines.last().is_some_and(|l| l.trim().is_empty()) {
lines.pop();
}
let has_content = lines.iter().any(|l| {
let t = l.trim();
!t.is_empty() && !t.starts_with('#')
});
if has_content {
lines.push(String::new());
}
lines.push(format!("{section}:"));
lines.extend(new);
}
}
Ok(format!("{}\n", lines.join("\n")))
}
/// Checks that the new file reads back as the old one plus the migrated
/// entries, and nothing else.
fn verify(
before: Option<&str>,
after: &str,
terms: &BTreeMap<String, Term>,
values: &BTreeMap<String, Value>,
) -> Result<()> {
let unreadable = |e: serde_yaml_ng::Error| {
Error::other(format!(
"the glossary.yaml this would write does not load ({e}); nothing was written"
))
};
let mut parsed: Fragment = serde_yaml_ng::from_str(after).map_err(unreadable)?;
for (id, term) in terms {
if parsed.terms.remove(id).as_ref() != Some(term) {
return Err(Error::other(format!(
"`{id}` does not read back as migrated; nothing was written"
)));
}
}
for (id, value) in values {
if parsed.values.remove(id).as_ref() != Some(value) {
return Err(Error::other(format!(
"`{id}` does not read back as migrated; nothing was written"
)));
}
}
let old: Fragment = match before {
Some(text) => serde_yaml_ng::from_str(text).map_err(unreadable)?,
None => Fragment::default(),
};
if yaml::to_string(&parsed)? != yaml::to_string(&old)? {
return Err(Error::other(
"migrating would change entries glossary.yaml already has; nothing was written",
));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
const TERMS: &str = "schema_version: \"1.0\"
terms:
# A coined one.
- id: decade
course: decade
variants: [decades]
kind: coined
meaning: >-
a factor of ten in a concentration
ratio
first: L2.1
color: blue
- id: term-gap
course: gap
meaning: a dash
";
fn parsed(text: &str, kind: Kind) -> Vec<Entry> {
let doc: Yaml = serde_yaml_ng::from_str(text).unwrap();
entries(&doc, kind, "terms.yaml").unwrap()
}
#[test]
fn entries_take_their_prefix_once_and_lose_derived_and_unknown_fields() {
let entries = parsed(TERMS, Kind::Term);
assert_eq!(entries[0].id, "term-decade");
assert_eq!(entries[1].id, "term-gap");
assert_eq!(entries[0].derived, Some(Yaml::String("L2.1".into())));
assert_eq!(entries[0].dropped.len(), 1);
assert!(entries[0].dropped[0].contains("`color`"));
assert!(entries[0].fields.contains_key("definition"));
assert!(!entries[0].fields.contains_key("meaning"));
}
#[test]
fn copied_lines_keep_comments_and_block_scalars() {
let entries = parsed(TERMS, Kind::Term);
let items = items(TERMS, "terms");
assert_eq!(items.len(), 2);
let chunk = copied(&items[0], Kind::Term, &entries[0]).unwrap();
assert_eq!(
chunk,
[
" # A coined one.",
" term-decade:",
" term: decade",
" variants: [decades]",
" kind: coined",
" definition: >-",
" a factor of ten in a concentration",
" ratio",
]
);
let term: Term =
serde_yaml_ng::from_value(Yaml::Mapping(entries[0].fields.clone())).unwrap();
assert!(reads_back(&chunk, Kind::Term, "term-decade", &term, |f| &f.terms));
}
#[test]
fn items_at_column_zero_are_reindented() {
let text = "terms:\n- id: gap\n course: gap\n meaning: a dash\n";
let entries = parsed(text, Kind::Term);
let items = items(text, "terms");
let chunk = copied(&items[0], Kind::Term, &entries[0]).unwrap();
assert_eq!(
chunk,
[" term-gap:", " term: gap", " definition: a dash"]
);
}
#[test]
fn a_value_names_its_term_by_the_new_id() {
let text = "values:\n - id: k\n name: k\n say: about 6\n term: decade\n";
let entries = parsed(text, Kind::Value);
assert_eq!(entries[0].relinked.as_deref(), Some("decade"));
let items = items(text, "values");
let chunk = copied(&items[0], Kind::Value, &entries[0]).unwrap();
assert_eq!(chunk.last().unwrap(), " term: term-decade");
}
#[test]
fn a_flow_item_falls_back_to_the_serializer() {
let text = "terms:\n - {id: gap, course: gap, meaning: a dash}\n";
let entries = parsed(text, Kind::Term);
let items = items(text, "terms");
assert!(copied(&items[0], Kind::Term, &entries[0]).is_none());
let term: Term =
serde_yaml_ng::from_value(Yaml::Mapping(entries[0].fields.clone())).unwrap();
let chunk = serialized("term-gap", &term).unwrap();
assert!(reads_back(&chunk, Kind::Term, "term-gap", &term, |f| &f.terms));
}
#[test]
fn bad_and_duplicate_ids_are_refused() {
let doc: Yaml = serde_yaml_ng::from_str(
"terms:\n - id: Two Words\n course: x\n meaning: y\n - id: a\n course: a\n meaning: b\n - id: term-a\n course: a\n meaning: b\n - course: z\n",
)
.unwrap();
let Err(Error::Invalid(problems)) = entries(&doc, Kind::Term, "terms.yaml") else {
panic!("expected problems");
};
assert_eq!(problems.len(), 3, "{problems:?}");
}
#[test]
fn insert_appends_to_a_section_or_adds_one() {
let text = "# keep\nterms:\n term-a:\n term: a\n definition: b\n\n# about values\nvalues: {}\n";
let chunk = vec![" term-c:".to_string(), " term: c".to_string()];
let out = insert(text, "terms", std::slice::from_ref(&chunk)).unwrap();
assert!(out.contains(" definition: b\n term-c:\n term: c\n\n# about values\n"));
let value = vec![" val-x:".to_string(), " name: x".to_string()];
let out = insert(&out, "values", &[value]).unwrap();
assert!(out.ends_with("values:\n val-x:\n name: x\n"));
let fresh = insert("# header\n", "terms", &[chunk]).unwrap();
assert_eq!(fresh, "# header\nterms:\n term-c:\n term: c\n");
}
#[test]
fn shift_moves_whole_blocks() {
assert_eq!(shift(" deeper", 2, 4), " deeper");
assert_eq!(shift(" deeper", 6, 4), " deeper");
assert_eq!(shift(" ", 2, 4), "");
}
}
+158
View File
@@ -0,0 +1,158 @@
// SPDX-License-Identifier: Prosperity-3.0.0
// Copyright Scienting
// Source: https://git.scient.ing/education/coursebank
//! `coursebank migrate glossary`, against `docs/design/2.1/examples/migration/`.
//!
//! The example course's own `glossary.yaml` is that migration's output after a
//! person linked the value to its term, so each test starts from the example
//! with it removed.
mod common;
use std::path::{Path, PathBuf};
use std::process::Command;
use common::{design, edit, v21_course};
use coursebank::course::fragment::Fragment;
use coursebank::migrate;
/// The example course without its glossary.
fn course(tag: &str) -> PathBuf {
let root = v21_course(tag);
std::fs::remove_file(root.join("glossary.yaml")).unwrap();
root
}
fn source(name: &str) -> PathBuf {
design().join("examples/migration").join(name)
}
fn fragment(path: &Path) -> Fragment {
serde_yaml_ng::from_str(&std::fs::read_to_string(path).unwrap()).unwrap()
}
fn run(root: &Path, write: bool) -> migrate::GlossaryMigration {
migrate::glossary(
root,
Some(&source("terms.yaml")),
Some(&source("values.yaml")),
write,
)
.unwrap()
}
#[test]
fn migration_reproduces_the_expected_glossary() {
let root = course("mig-expected");
let report = run(&root, true);
let written = fragment(&root.join("glossary.yaml"));
let expected = fragment(&source("glossary.expected.yaml"));
assert_eq!(written.terms, expected.terms);
assert_eq!(written.values, expected.values);
// The report the expected file's header comment describes.
assert_eq!(
report.renamed,
[
("decade".to_string(), "term-decade".to_string()),
("per-decade".to_string(), "val-per-decade".to_string()),
]
);
assert_eq!(report.derived.len(), 2, "{:?}", report.derived);
assert!(report.derived[0].contains("`first` was L2.1"));
assert!(report.derived[1].contains("`lectures` was [L2.1, L2.4, L2.5]"));
assert_eq!(report.unlisted, ["term-decade", "val-per-decade"]);
assert!(report.dropped.is_empty() && report.relinked.is_empty());
// The course still validates with the written file.
let issues = coursebank::Catalog::load(&root)
.unwrap()
.validate()
.unwrap();
assert!(issues.is_empty(), "{issues:#?}");
}
#[test]
fn migrating_twice_is_a_no_op() {
let root = course("mig-twice");
run(&root, true);
let before = std::fs::read(root.join("glossary.yaml")).unwrap();
let again = run(&root, true);
assert!(again.text.is_none());
assert!(again.added.is_empty() && again.renamed.is_empty());
assert_eq!(again.present.len(), 2);
assert_eq!(std::fs::read(root.join("glossary.yaml")).unwrap(), before);
// The example as shipped already holds both, linked: also a no-op.
let shipped = v21_course("mig-shipped");
assert!(run(&shipped, true).text.is_none());
}
#[test]
fn a_dropped_first_is_compared_with_the_targets() {
let root = course("mig-derived");
edit(
&root,
"objectives/lo-alignment-scoring.yaml",
"terms: [term-alignment-matrix, term-gap,",
"terms: [term-decade, term-alignment-matrix, term-gap,",
);
let report = run(&root, false);
assert!(
report.derived[0].contains("`first` was L2.1; derived from the targets: l09"),
"{:?}",
report.derived
);
assert_eq!(report.unlisted, ["val-per-decade"]);
assert!(!root.join("glossary.yaml").exists(), "a dry run wrote");
}
#[test]
fn an_existing_glossary_is_added_to_not_rewritten() {
let root = course("mig-existing");
let before = "# Cross-cutting terms.\nterms:\n term-other:\n term: other\n \
definition: Something else.\n";
std::fs::write(root.join("glossary.yaml"), before).unwrap();
let report = run(&root, true);
let after = std::fs::read_to_string(root.join("glossary.yaml")).unwrap();
assert!(after.starts_with(before), "{after}");
assert!(after.contains("\nvalues:\n val-per-decade:\n"));
assert_eq!(report.added, ["term-decade", "val-per-decade"]);
let written = fragment(&root.join("glossary.yaml"));
assert_eq!(written.terms.len(), 2);
assert_eq!(written.values.len(), 1);
}
#[test]
fn the_cli_reports_and_a_dry_run_writes_nothing() {
let root = course("mig-cli");
let terms = source("terms.yaml");
let values = source("values.yaml");
let out = Command::new(env!("CARGO_BIN_EXE_coursebank"))
.arg("-C")
.arg(&root)
.args(["migrate", "glossary", "--dry-run", "--terms"])
.arg(&terms)
.arg("--values")
.arg(&values)
.output()
.unwrap();
let stdout = String::from_utf8_lossy(&out.stdout);
assert!(out.status.success(), "{stdout}");
assert!(stdout.contains("decade -> term-decade"));
assert!(stdout.contains("nothing written"));
assert!(!root.join("glossary.yaml").exists());
let none = Command::new(env!("CARGO_BIN_EXE_coursebank"))
.arg("-C")
.arg(&root)
.args(["migrate", "glossary"])
.output()
.unwrap();
assert!(!none.status.success());
}