// SPDX-License-Identifier: Prosperity-3.0.0
// Copyright Scientific Computing Studio
// Source: https://git.scient.ing/education/coursebank

//! Reading Canvas's "Student Analysis" quiz export.
//!
//! Canvas exports one wide CSV for the whole quiz. After the student columns come
//! two columns per question: one headed `<question id>: <question text>` holding
//! the answer text the student chose, and one immediately after it holding the
//! points earned, headed with the points possible.
//!
//! The awkward part is that Canvas records answer text, not option letters. So
//! recovering "this student chose C" means matching the exported text back against
//! the item's options. That matching is done on normalized text because the text
//! makes a round trip through the QTI export and Canvas's own HTML sanitizer, and
//! comes back with different markup than it left with.
//!
//! When a match cannot be made, the row keeps its score and loses its option
//! letter, with a warning naming the question. Totals and per-objective mastery
//! still work, and only distractor analysis is affected.
//!
//! Questions are aligned to item numbers by matching stem text the same way, with
//! column order as the fallback. Order alone would be wrong for any quiz where
//! Canvas shuffled the questions.

use std::collections::BTreeMap;
use std::path::Path;

use crate::assessment::AssessmentFile;
use crate::catalog::Catalog;
use crate::date::Date;
use crate::error::{Error, Result};
use crate::responses::{Response, ResponseSet, administration_id};

/// What the ingest needs that the export does not carry.
pub type Context = crate::gradescope::Context;

/// One question column pair found in the header.
#[derive(Debug, Clone)]
struct QuestionColumn {
    /// Index of the answer-text column.
    text_index: usize,
    /// Index of the points-earned column, when the header had one.
    points_index: Option<usize>,
    /// The Canvas question id, from the column header.
    ///
    /// Kept for diagnostics rather than for matching: when a column cannot be
    /// traced to the assessment record, or an answer cannot be traced to an option,
    /// this id is what lets you find the question in Canvas and see what differs.
    /// Matching itself goes by question text, because Canvas ids change when a quiz
    /// is copied to a new term.
    canvas_id: String,
    /// The question text from the header.
    question_text: String,
    /// Points possible, parsed from the points column's header.
    points_possible: f64,
    /// The item number this maps to, once resolved.
    item_number: Option<u32>,
}

/// Splits a question column header into its id and text.
///
/// # Arguments
///
/// * `header` - the column header.
///
/// # Returns
///
/// The id and text, or `None` when the header is not a question column.
fn split_question_header(header: &str) -> Option<(String, String)> {
    let (id, rest) = header.split_once(':')?;
    let id = id.trim();
    if id.is_empty() || !id.chars().all(|c| c.is_ascii_digit()) {
        return None;
    }
    Some((id.to_string(), rest.trim().to_string()))
}

/// Normalizes text for comparison.
///
/// Strips HTML tags, decodes the handful of entities that survive a QTI round
/// trip, drops punctuation, lowercases, and collapses whitespace.
///
/// # Arguments
///
/// * `s` - the text.
///
/// # Returns
///
/// The normalized form.
pub fn normalize(s: &str) -> String {
    // Strip tags.
    let mut stripped = String::with_capacity(s.len());
    let mut in_tag = false;
    for ch in s.chars() {
        match ch {
            '<' => in_tag = true,
            '>' => {
                in_tag = false;
                stripped.push(' ');
            }
            _ if in_tag => {}
            _ => stripped.push(ch),
        }
    }

    // Decode the entities that actually appear.
    let decoded = stripped
        .replace("&nbsp;", " ")
        .replace("&amp;", "&")
        .replace("&lt;", "<")
        .replace("&gt;", ">")
        .replace("&quot;", "\"")
        .replace("&apos;", "'")
        .replace("&#39;", "'")
        .replace("&rarr;", "->")
        .replace("&harr;", "<->")
        .replace("&minus;", "-");

    let mut out = String::with_capacity(decoded.len());
    let mut last_space = true;
    for ch in decoded.chars() {
        let c = ch.to_ascii_lowercase();
        if c.is_alphanumeric() {
            out.push(c);
            last_space = false;
        } else if (c.is_whitespace() || c == '-' || c == '_') && !last_space {
            out.push(' ');
            last_space = true;
        }
        // Everything else — punctuation, entity leftovers — is dropped.
    }
    out.trim().to_string()
}

/// Reads a Canvas Student Analysis export.
///
/// # Arguments
///
/// * `path` - the CSV file.
/// * `ctx` - identifying information.
/// * `record` - the assessment record, used to align questions to item numbers.
/// * `catalog` - the loaded course, used to recover option letters from answer
///   text. Without it, scores are still ingested but letters are not.
///
/// # Returns
///
/// The normalized responses.
///
/// # Errors
///
/// Returns [`Error::Csv`] on a malformed file and [`Error::Invalid`] when no
/// question columns can be found.
pub fn ingest(
    path: &Path,
    ctx: &Context,
    record: Option<&AssessmentFile>,
    catalog: Option<&Catalog>,
) -> Result<ResponseSet> {
    let mut reader = csv::ReaderBuilder::new()
        .flexible(true)
        .has_headers(true)
        .from_path(path)
        .map_err(|e| Error::Csv {
            path: path.to_path_buf(),
            source: e,
        })?;

    let header = reader
        .headers()
        .map_err(|e| Error::Csv {
            path: path.to_path_buf(),
            source: e,
        })?
        .clone();
    let header: Vec<String> = header.iter().map(|h| h.trim().to_string()).collect();

    let find = |name: &str| header.iter().position(|h| h.eq_ignore_ascii_case(name));
    let name_col = find("name");
    let id_col = find("id");
    let sis_col = find("sis_id").or_else(|| find("sis id"));
    let section_col = find("section");
    let submitted_col = find("submitted");
    let attempt_col = find("attempt");

    // Locate the question column pairs.
    let mut questions: Vec<QuestionColumn> = Vec::new();
    for (i, h) in header.iter().enumerate() {
        if let Some((canvas_id, question_text)) = split_question_header(h) {
            // The next column holds points earned; its header is the points
            // possible. Canvas writes it as a bare number.
            let (points_index, points_possible) = match header.get(i + 1) {
                Some(next) => match next.trim().parse::<f64>() {
                    Ok(p) => (Some(i + 1), p),
                    Err(_) => (None, 0.0),
                },
                None => (None, 0.0),
            };
            questions.push(QuestionColumn {
                text_index: i,
                points_index,
                canvas_id,
                question_text,
                points_possible,
                item_number: None,
            });
        }
    }

    if questions.is_empty() {
        return Err(Error::Invalid(vec![format!(
            "{} has no question columns; a Canvas Student Analysis export heads each question \
             `<id>: <question text>`. Did you export the Item Analysis instead?",
            path.display()
        )]));
    }

    let mut warnings = Vec::new();
    align_questions(&mut questions, record, catalog, &mut warnings);

    // Build the answer-text lookup once per question: normalized option text to
    // option letter.
    let mut answer_lookup: BTreeMap<usize, BTreeMap<String, String>> = BTreeMap::new();
    let mut item_meta: BTreeMap<usize, (Option<String>, Vec<String>)> = BTreeMap::new();
    if let (Some(cat), Some(rec)) = (catalog, record) {
        for (qi, q) in questions.iter().enumerate() {
            let Some(number) = q.item_number else {
                continue;
            };
            let Some(placement) = rec.placement(number) else {
                continue;
            };
            let Some(entry) = cat.get(&placement.item) else {
                continue;
            };
            let mut map = BTreeMap::new();
            for opt in &entry.item.options {
                map.insert(normalize(&opt.text), opt.id.clone());
            }
            answer_lookup.insert(qi, map);
            item_meta.insert(qi, (Some(placement.item.clone()), placement.key.clone()));
        }
    }

    let admin = administration_id(&ctx.course, &ctx.term, &ctx.assessment_id);
    let mut set = ResponseSet::new();
    let mut unmatched_answers: BTreeMap<(u32, String), usize> = BTreeMap::new();

    for rec in reader.records() {
        let rec = rec.map_err(|e| Error::Csv {
            path: path.to_path_buf(),
            source: e,
        })?;
        let cell = |i: Option<usize>| -> Option<String> {
            i.and_then(|i| rec.get(i))
                .map(|s| s.trim().to_string())
                .filter(|s| !s.is_empty())
        };

        let sid = cell(sis_col).or_else(|| cell(id_col));
        let name = cell(name_col);
        if sid.is_none() && name.is_none() {
            continue;
        }
        // Canvas emits a "Test Student" row for anyone who previewed the quiz.
        if name.as_deref() == Some("Test Student") {
            continue;
        }
        let student_key = sid
            .clone()
            .or_else(|| name.clone())
            .unwrap_or_else(|| "unknown".to_string());
        let section = cell(section_col);
        let submitted = cell(submitted_col);
        let attempt = cell(attempt_col).and_then(|a| a.parse::<u32>().ok());
        // Only the graded attempt is exported per row, but a student who never
        // submitted still gets a row; those carry no responses.
        if submitted.is_none() && attempt.is_none() {
            continue;
        }

        for (qi, q) in questions.iter().enumerate() {
            let Some(number) = q.item_number else {
                continue;
            };
            let answer_text = rec.get(q.text_index).map(|s| s.trim()).unwrap_or("");
            let score = q
                .points_index
                .and_then(|i| rec.get(i))
                .and_then(|s| s.trim().parse::<f64>().ok())
                .unwrap_or(0.0);

            let points = q.points_possible;
            let credit = if points > 0.0 { score / points } else { 0.0 };

            let mut selected = Vec::new();
            if !answer_text.is_empty() {
                let normalized = normalize(answer_text);
                match answer_lookup.get(&qi).and_then(|m| m.get(&normalized)) {
                    Some(letter) => selected.push(letter.clone()),
                    None => {
                        *unmatched_answers
                            .entry((number, q.canvas_id.clone()))
                            .or_insert(0) += 1;
                    }
                }
            }

            let (item_ref, key) = item_meta.get(&qi).cloned().unwrap_or((None, Vec::new()));

            let correct = if answer_text.is_empty() {
                None
            } else if points > 0.0 {
                Some(credit >= 0.999)
            } else if !key.is_empty() && !selected.is_empty() {
                Some(selected == key)
            } else {
                None
            };

            set.rows.push(Response {
                administration_id: admin.clone(),
                course: ctx.course.clone(),
                term: ctx.term.clone(),
                assessment_id: ctx.assessment_id.clone(),
                date: ctx.date,
                form: ctx.form.clone(),
                student_key: student_key.clone(),
                sid: sid.clone(),
                name: name.clone(),
                email: None,
                section: section.clone(),
                item_number: number,
                item_ref,
                item_version: None,
                selected,
                eliminated: Vec::new(),
                correct,
                credit,
                points_possible: points,
                score,
                response_time_seconds: None,
                level: None,
                learning_objectives: Vec::new(),
                topics: Vec::new(),
                bonus: false,
                dropped: false,
            });
        }
    }

    for ((number, canvas_id), count) in unmatched_answers {
        warnings.push(format!(
            "question {number} (Canvas id {canvas_id}): {count} answer(s) did not match any option \
             text, so those responses have a score but no option letter; distractor analysis for \
             this item will be incomplete. The usual cause is the option text being edited in \
             Canvas after import"
        ));
    }

    if set.rows.is_empty() {
        warnings.push(
            "no student rows were found; the export may contain only the header, or every row \
             may be an unsubmitted attempt"
                .to_string(),
        );
    }

    set.warnings.extend(warnings);
    Ok(set)
}

/// Assigns an item number to each question column.
///
/// Matching on stem text is preferred over column order because Canvas shuffles
/// questions when the quiz says to, and the export follows the shuffled order.
///
/// # Arguments
///
/// * `questions` - the columns, updated in place.
/// * `record` - the assessment record.
/// * `catalog` - the loaded course.
/// * `warnings` - collects anything ambiguous.
fn align_questions(
    questions: &mut [QuestionColumn],
    record: Option<&AssessmentFile>,
    catalog: Option<&Catalog>,
    warnings: &mut Vec<String>,
) {
    let Some(rec) = record else {
        // Without a record, the only sensible assumption is column order.
        for (i, q) in questions.iter_mut().enumerate() {
            q.item_number = Some(i as u32 + 1);
        }
        warnings.push(
            "no assessment record was supplied, so questions were matched to item numbers by \
             column order; pass --assessment to match on question text instead"
                .to_string(),
        );
        return;
    };

    // Normalized stem to item number, when the catalog is available.
    let mut by_stem: BTreeMap<String, Vec<u32>> = BTreeMap::new();
    if let Some(cat) = catalog {
        for p in &rec.items {
            if let Some(entry) = cat.get(&p.item) {
                by_stem
                    .entry(normalize(&entry.item.stem))
                    .or_default()
                    .push(p.number);
            }
        }
    }

    let mut used: Vec<u32> = Vec::new();
    let mut unmatched: Vec<String> = Vec::new();

    for (i, q) in questions.iter_mut().enumerate() {
        let normalized = normalize(&q.question_text);
        let candidates = by_stem.get(&normalized);
        match candidates {
            Some(numbers) => {
                // Prefer a number not already claimed, so two items sharing a stem
                // do not collapse onto one.
                match numbers.iter().find(|n| !used.contains(n)) {
                    Some(n) => {
                        q.item_number = Some(*n);
                        used.push(*n);
                    }
                    None => {
                        q.item_number = Some(numbers[0]);
                    }
                }
            }
            None => {
                // Fall back to position, which is right for an unshuffled quiz.
                let fallback = rec.items.get(i).map(|p| p.number).unwrap_or(i as u32 + 1);
                if !used.contains(&fallback) {
                    used.push(fallback);
                }
                q.item_number = Some(fallback);
                // Report the Canvas question id, not just a count. The id is what
                // you can search for in Canvas to see which question this was, and
                // an unmatched column usually means the text was edited there after
                // import — so being able to find it is the whole remedy.
                unmatched.push(format!("{} (assumed question {fallback})", q.canvas_id));
            }
        }
    }

    if !unmatched.is_empty() {
        warnings.push(format!(
            "{} of {} question column(s) could not be matched to the assessment record by question \
             text and were matched by column order instead. Canvas question id(s): {}. Check that \
             the record describes this quiz, and that the question text was not edited in Canvas \
             after import",
            unmatched.len(),
            questions.len(),
            unmatched.join(", ")
        ));
    }
}

/// Guesses a date from a Canvas timestamp column.
///
/// # Arguments
///
/// * `s` - the timestamp, e.g. `2026-04-01 14:03:22 UTC`.
///
/// # Returns
///
/// The date, when the leading token parses as one.
pub fn date_from_timestamp(s: &str) -> Option<Date> {
    s.split_whitespace().next()?.parse().ok()
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn recognizes_question_headers() {
        assert_eq!(
            split_question_header("123456: Which enzyme catalyzes the step?"),
            Some((
                "123456".to_string(),
                "Which enzyme catalyzes the step?".to_string()
            ))
        );
        assert_eq!(split_question_header("name"), None);
        assert_eq!(split_question_header("n correct"), None);
        // A colon in prose is not a question column.
        assert_eq!(split_question_header("Note: read carefully"), None);
    }

    #[test]
    fn normalization_survives_a_qti_round_trip() {
        // Option text authored as `K#sub[m] increases` is exported as HTML and
        // comes back from Canvas wrapped in tags. Both must normalize to the same
        // thing as the HTML we sent, or letters cannot be recovered.
        let sent = "K<sub>m</sub> increases";
        let returned = "<p>K<sub>m</sub> increases</p>";
        assert_eq!(normalize(sent), normalize(returned));
        assert_eq!(normalize(returned), "k m increases");
        assert_eq!(normalize("K&nbsp;m increases"), "k m increases");
    }

    #[test]
    fn normalization_ignores_punctuation_and_case() {
        assert_eq!(
            normalize("The Rate, Increases!"),
            normalize("the rate increases")
        );
        assert_eq!(normalize("  spaced   out  "), "spaced out");
        assert_eq!(normalize("<b>bold</b> text"), "bold text");
    }

    #[test]
    fn timestamps_yield_dates() {
        assert_eq!(
            date_from_timestamp("2026-04-01 14:03:22 UTC").map(|d| d.to_string()),
            Some("2026-04-01".to_string())
        );
        assert_eq!(date_from_timestamp("not a date"), None);
    }

    #[test]
    fn falls_back_to_column_order_without_a_record() {
        let mut questions = vec![
            QuestionColumn {
                text_index: 8,
                points_index: Some(9),
                canvas_id: "1".into(),
                question_text: "Q one".into(),
                points_possible: 1.0,
                item_number: None,
            },
            QuestionColumn {
                text_index: 10,
                points_index: Some(11),
                canvas_id: "2".into(),
                question_text: "Q two".into(),
                points_possible: 1.0,
                item_number: None,
            },
        ];
        let mut warnings = Vec::new();
        align_questions(&mut questions, None, None, &mut warnings);
        assert_eq!(questions[0].item_number, Some(1));
        assert_eq!(questions[1].item_number, Some(2));
        assert_eq!(warnings.len(), 1);
    }

    #[test]
    fn parses_a_realistic_export() {
        let dir = std::env::temp_dir().join(format!("cb-canvas-{}", std::process::id()));
        std::fs::create_dir_all(&dir).unwrap();
        let path = dir.join("analysis.csv");
        std::fs::write(
            &path,
            "name,id,sis_id,section,submitted,attempt,\
             1001: Which enzyme?,1.0,1002: Which pathway?,1.0,n correct,n incorrect,score\n\
             Ada Lovelace,9001,1234567,L01,2026-04-01 14:00:00 UTC,1,Hexokinase,1.0,Glycolysis,\
             1.0,2,0,2.0\n\
             Alan Turing,9002,7654321,L01,2026-04-01 14:05:00 UTC,1,Pyruvate kinase,0.0,\
             Glycolysis,1.0,1,1,1.0\n\
             Test Student,9999,,L01,2026-04-01 13:00:00 UTC,1,Hexokinase,1.0,Glycolysis,1.0,2,0,\
             2.0\n",
        )
        .unwrap();

        let ctx = Context {
            course: "BIOSC1540".into(),
            term: "2026s".into(),
            assessment_id: "quiz-1".into(),
            date: None,
            form: None,
        };
        let set = ingest(&path, &ctx, None, None).unwrap();

        // Two real students, two questions each. The preview row is discarded.
        assert_eq!(set.rows.len(), 4, "{:?}", set.warnings);
        assert!(
            set.rows
                .iter()
                .all(|r| r.name.as_deref() != Some("Test Student"))
        );
        assert_eq!(set.scored_total("1234567"), 2.0);
        assert_eq!(set.scored_total("7654321"), 1.0);

        let alan_q1 = set
            .rows
            .iter()
            .find(|r| r.student_key == "7654321" && r.item_number == 1)
            .unwrap();
        assert_eq!(alan_q1.correct, Some(false));
        assert_eq!(alan_q1.credit, 0.0);
        // No catalog was supplied, so no letter could be recovered.
        assert!(alan_q1.selected.is_empty());

        std::fs::remove_dir_all(&dir).ok();
    }

    #[test]
    fn rejects_the_wrong_export_kind() {
        let dir = std::env::temp_dir().join(format!("cb-canvas-bad-{}", std::process::id()));
        std::fs::create_dir_all(&dir).unwrap();
        let path = dir.join("item.csv");
        std::fs::write(&path, "question,discrimination index\nQ1,0.4\n").unwrap();
        let ctx = Context {
            course: "C".into(),
            term: "T".into(),
            assessment_id: "a".into(),
            date: None,
            form: None,
        };
        let err = ingest(&path, &ctx, None, None).unwrap_err();
        assert!(err.to_string().contains("Student Analysis"));
        std::fs::remove_dir_all(&dir).ok();
    }
}