diff --git a/proto/anki/card_rendering.proto b/proto/anki/card_rendering.proto index 145e4b0dbe7..4035ae68b05 100644 --- a/proto/anki/card_rendering.proto +++ b/proto/anki/card_rendering.proto @@ -165,6 +165,7 @@ message HtmlToTextLineRequest { message CompareAnswerRequest { string expected = 1; string provided = 2; + bool combining = 3; } message ExtractClozeForTypingRequest { diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index 6ae37befe90..66b2fb61857 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -1152,8 +1152,12 @@ def render_markdown(self, text: str, sanitize: bool = True) -> str: "Not intended for public consumption at this time." return self._backend.render_markdown(markdown=text, sanitize=sanitize) - def compare_answer(self, expected: str, provided: str) -> str: - return self._backend.compare_answer(expected=expected, provided=provided) + def compare_answer( + self, expected: str, provided: str, combining: bool = True + ) -> str: + return self._backend.compare_answer( + expected=expected, provided=provided, combining=combining + ) def extract_cloze_for_typing(self, text: str, ordinal: int) -> str: return self._backend.extract_cloze_for_typing(text=text, ordinal=ordinal) diff --git a/qt/aqt/reviewer.py b/qt/aqt/reviewer.py index 4a16f7b47f9..b5a6e4d6fe0 100644 --- a/qt/aqt/reviewer.py +++ b/qt/aqt/reviewer.py @@ -152,6 +152,7 @@ def __init__(self, mw: AnkiQt) -> None: self.previous_card: Card | None = None self._answeredIds: list[CardId] = [] self._recordedAudio: str | None = None + self.combining: bool = True self.typeCorrect: str | None = None # web init happens before this is set self.state: Literal["question", "answer", "transition"] | None = None self._refresh_needed: RefreshNeeded | None = None @@ -699,6 +700,7 @@ def typeAnsFilter(self, buf: str) -> str: return self.typeAnsAnswerFilter(buf) def typeAnsQuestionFilter(self, buf: str) -> str: + self.combining = True self.typeCorrect = None clozeIdx = None m = re.search(self.typeAnsPat, buf) @@ -711,6 +713,9 @@ def typeAnsQuestionFilter(self, buf: str) -> str: clozeIdx = self.card.ord + 1 fld = fld.split(":")[1] # loop through fields for a match + if fld.startswith("nc:"): + self.combining = False + fld = fld.split(":")[1] for f in self.card.note_type()["flds"]: if f["name"] == fld: self.typeCorrect = self.card.note()[f["name"]] @@ -750,7 +755,7 @@ def typeAnsAnswerFilter(self, buf: str) -> str: hadHR = len(buf) != origSize expected = self.typeCorrect provided = self.typedAnswer - output = self.mw.col.compare_answer(expected, provided) + output = self.mw.col.compare_answer(expected, provided, self.combining) # and update the type answer area def repl(match: Match) -> str: diff --git a/rslib/src/card_rendering/service.rs b/rslib/src/card_rendering/service.rs index 7e0f9ba67a9..8d15857258c 100644 --- a/rslib/src/card_rendering/service.rs +++ b/rslib/src/card_rendering/service.rs @@ -167,7 +167,7 @@ impl crate::services::CardRenderingService for Collection { &mut self, input: anki_proto::card_rendering::CompareAnswerRequest, ) -> Result { - Ok(compare_answer(&input.expected, &input.provided).into()) + Ok(compare_answer(&input.expected, &input.provided, input.combining).into()) } fn extract_cloze_for_typing( diff --git a/rslib/src/template_filters.rs b/rslib/src/template_filters.rs index b6408d965a4..f55d4586267 100644 --- a/rslib/src/template_filters.rs +++ b/rslib/src/template_filters.rs @@ -33,6 +33,8 @@ pub(crate) fn apply_filters<'a>( // type:cloze is handled specially let filters = if filters == ["cloze", "type"] { &["type-cloze"] + } else if filters == ["nc", "type"] { + &["type-nc"] } else { filters }; @@ -80,6 +82,7 @@ fn apply_filter( "kana" => kana_filter(text), "type" => type_filter(field_name), "type-cloze" => type_cloze_filter(field_name), + "type-nc" => type_nc_filter(field_name), "hint" => hint_filter(text, field_name), "cloze" => cloze_filter(text, context), "cloze-only" => cloze_only_filter(text, context), @@ -171,6 +174,10 @@ fn type_cloze_filter<'a>(field_name: &str) -> Cow<'a, str> { format!("[[type:cloze:{}]]", field_name).into() } +fn type_nc_filter<'a>(field_name: &str) -> Cow<'a, str> { + format!("[[type:nc:{}]]", field_name).into() +} + fn hint_filter<'a>(text: &'a str, field_name: &str) -> Cow<'a, str> { if text.trim().is_empty() { return text.into(); @@ -238,6 +245,7 @@ field fn typing() { assert_eq!(type_filter("Front"), "[[type:Front]]"); assert_eq!(type_cloze_filter("Front"), "[[type:cloze:Front]]"); + assert_eq!(type_nc_filter("Front"), "[[type:nc:Front]]"); let ctx = RenderContext { fields: &Default::default(), nonempty_fields: &Default::default(), @@ -249,6 +257,10 @@ field apply_filters("ignored", &["cloze", "type"], "Text", &ctx), ("[[type:cloze:Text]]".into(), vec![]) ); + assert_eq!( + apply_filters("ignored", &["nc", "type"], "Text", &ctx), + ("[[type:nc:Text]]".into(), vec![]) + ); } #[test] diff --git a/rslib/src/text.rs b/rslib/src/text.rs index b32ef45c1f8..7f741540cd2 100644 --- a/rslib/src/text.rs +++ b/rslib/src/text.rs @@ -13,6 +13,7 @@ use regex::Regex; use unicase::eq as uni_eq; use unicode_normalization::char::is_combining_mark; use unicode_normalization::is_nfc; +use unicode_normalization::is_nfkd; use unicode_normalization::is_nfkd_quick; use unicode_normalization::IsNormalized; use unicode_normalization::UnicodeNormalization; @@ -367,10 +368,9 @@ pub(crate) fn sanitize_html_no_images(html: &str) -> String { } pub(crate) fn normalize_to_nfc(s: &str) -> Cow { - if !is_nfc(s) { - s.chars().nfc().collect::().into() - } else { - s.into() + match is_nfc(s) { + false => s.chars().nfc().collect::().into(), + true => s.into(), } } @@ -380,6 +380,13 @@ pub(crate) fn ensure_string_in_nfc(s: &mut String) { } } +pub(crate) fn normalize_to_nfkd(s: &str) -> Cow { + match is_nfkd(s) { + false => s.chars().nfkd().collect::().into(), + true => s.into(), + } +} + static EXTRA_NO_COMBINING_REPLACEMENTS: phf::Map = phf::phf_map! { '€' => "E", 'Æ' => "AE", diff --git a/rslib/src/typeanswer.rs b/rslib/src/typeanswer.rs index 81df07880d0..6ea165e54aa 100644 --- a/rslib/src/typeanswer.rs +++ b/rslib/src/typeanswer.rs @@ -11,6 +11,7 @@ use unic_ucd_category::GeneralCategory; use crate::card_rendering::strip_av_tags; use crate::text::normalize_to_nfc; +use crate::text::normalize_to_nfkd; use crate::text::strip_html; static LINEBREAKS: Lazy = Lazy::new(|| { @@ -34,42 +35,37 @@ macro_rules! format_typeans { } // Public API -pub fn compare_answer(expected: &str, provided: &str) -> String { +pub fn compare_answer(expected: &str, provided: &str, combining: bool) -> String { if provided.is_empty() { format_typeans!(htmlescape::encode_minimal(expected)) - } else { + } else if combining { Diff::new(expected, provided).to_html() + } else { + DiffNonCombining::new(expected, provided).to_html() } } -struct Diff { - provided: Vec, - expected: Vec, - expected_original: String, -} +// Core Logic +trait DiffTrait { + fn get_provided(&self) -> &[char]; + fn get_expected(&self) -> &[char]; + fn get_expected_original(&self) -> &str; -impl Diff { - fn new(expected: &str, provided: &str) -> Self { - Self { - provided: normalize_to_nfc(provided).chars().collect(), - expected: normalize_to_nfc(&prepare_expected(expected)) - .chars() - .collect(), - expected_original: expected.to_string(), - } - } + fn new(expected: &str, provided: &str) -> Self; + fn normalize_expected(expected: &str) -> Vec; + fn normalize_provided(provided: &str) -> Vec; // Entry Point fn to_html(&self) -> String { - if self.provided == self.expected { + if self.get_provided() == self.get_expected() { format_typeans!(format!( "{}", - self.expected_original + self.get_expected_original() )) } else { let output = self.to_tokens(); let provided_html = render_tokens(&output.provided_tokens); - let expected_html = render_tokens(&output.expected_tokens); + let expected_html = self.render_expected_tokens(&output.expected_tokens); format_typeans!(format!( "{provided_html}
↓
{expected_html}" @@ -78,7 +74,7 @@ impl Diff { } fn to_tokens(&self) -> DiffTokens { - let mut matcher = SequenceMatcher::new(&self.provided, &self.expected); + let mut matcher = SequenceMatcher::new(self.get_provided(), self.get_expected()); let mut provided_tokens = Vec::new(); let mut expected_tokens = Vec::new(); @@ -105,19 +101,20 @@ impl Diff { _ => unreachable!(), } } + DiffTokens { provided_tokens, expected_tokens, } } - // Utility Functions + fn render_expected_tokens(&self, tokens: &[DiffToken]) -> String; + fn slice_expected(&self, opcode: &Opcode) -> String { - get_slice(&self.expected, opcode.second_start, opcode.second_end) + get_slice(self.get_expected(), opcode.second_start, opcode.second_end) } - fn slice_provided(&self, opcode: &Opcode) -> String { - get_slice(&self.provided, opcode.first_start, opcode.first_end) + get_slice(self.get_provided(), opcode.first_start, opcode.first_end) } } @@ -131,7 +128,6 @@ fn prepare_expected(expected: &str) -> String { strip_html(&no_linebreaks).trim().to_string() } -// Render Functions fn render_tokens(tokens: &[DiffToken]) -> String { tokens.iter().fold(String::new(), |mut acc, token| { let isolated_text = isolate_leading_mark(&token.text); @@ -156,6 +152,117 @@ fn isolate_leading_mark(text: &str) -> Cow { } } +// Default Comparison +struct Diff { + provided: Vec, + expected: Vec, + expected_original: String, +} + +impl DiffTrait for Diff { + fn get_provided(&self) -> &[char] { + &self.provided + } + fn get_expected(&self) -> &[char] { + &self.expected + } + fn get_expected_original(&self) -> &str { + &self.expected_original + } + + fn new(expected: &str, provided: &str) -> Self { + Self { + provided: Self::normalize_provided(provided), + expected: Self::normalize_expected(expected), + expected_original: expected.to_string(), + } + } + fn normalize_expected(expected: &str) -> Vec { + normalize_to_nfc(&prepare_expected(expected)) + .chars() + .collect() + } + fn normalize_provided(provided: &str) -> Vec { + normalize_to_nfc(provided).chars().collect() + } + + fn render_expected_tokens(&self, tokens: &[DiffToken]) -> String { + render_tokens(tokens) + } +} + +// Non-Combining Comparison +struct DiffNonCombining { + base: Diff, + expected_split: Vec, +} + +impl DiffTrait for DiffNonCombining { + fn get_provided(&self) -> &[char] { + &self.base.provided + } + fn get_expected(&self) -> &[char] { + &self.base.expected + } + fn get_expected_original(&self) -> &str { + &self.base.expected_original + } + + fn new(expected: &str, provided: &str) -> Self { + // filter out combining elements + let mut expected_stripped = String::new(); + // tokenized into "char+combining" for final rendering + let mut expected_split: Vec = Vec::new(); + for c in Self::normalize_expected(expected) { + if unicode_normalization::char::is_combining_mark(c) { + if let Some(last) = expected_split.last_mut() { + last.push(c); + } + } else { + expected_stripped.push(c); + expected_split.push(c.to_string()); + } + } + + Self { + base: Diff { + provided: Self::normalize_provided(provided), + expected: expected_stripped.chars().collect(), + expected_original: expected.to_string(), + }, + expected_split, + } + } + fn normalize_expected(expected: &str) -> Vec { + normalize_to_nfkd(&prepare_expected(expected)) + .chars() + .collect() + } + fn normalize_provided(provided: &str) -> Vec { + normalize_to_nfkd(provided) + .chars() + .filter(|c| !unicode_normalization::char::is_combining_mark(*c)) + .collect() + } + + // Since the combining characters are still required learning content, use + // expected_split to show them directly in the "expected" line, rather than + // having to otherwise e.g. include their field twice in the note template. + fn render_expected_tokens(&self, tokens: &[DiffToken]) -> String { + let mut idx = 0; + tokens.iter().fold(String::new(), |mut acc, token| { + let end = idx + token.text.chars().count(); + let txt = self.expected_split[idx..end].concat(); + idx = end; + let encoded_text = htmlescape::encode_minimal(&txt); + let class = token.to_class(); + acc.push_str(&format!("{encoded_text}")); + acc + }) + } +} + +// Utility Items #[derive(Debug, PartialEq, Eq)] struct DiffTokens { provided_tokens: Vec, @@ -179,19 +286,15 @@ impl DiffToken { fn new(kind: DiffTokenKind, text: String) -> Self { Self { kind, text } } - fn good(text: String) -> Self { Self::new(DiffTokenKind::Good, text) } - fn bad(text: String) -> Self { Self::new(DiffTokenKind::Bad, text) } - fn missing(text: String) -> Self { Self::new(DiffTokenKind::Missing, text) } - fn to_class(&self) -> &'static str { match self.kind { DiffTokenKind::Good => "typeGood", @@ -300,7 +403,7 @@ mod test { #[test] fn empty_input_shows_as_code() { - let ctx = compare_answer("123", ""); + let ctx = compare_answer("123", "", true); assert_eq!(ctx, "123"); }