diff --git a/src/harmonization_framework/primitives/normalize.py b/src/harmonization_framework/primitives/normalize.py index e3680ab..99a630b 100644 --- a/src/harmonization_framework/primitives/normalize.py +++ b/src/harmonization_framework/primitives/normalize.py @@ -51,9 +51,21 @@ def transform(self, value: str) -> str: def remove_accents(self, value: str) -> str: """ - Remove accents and diacritics using NFKC normalization. + Remove accents and diacritics from `value`. + + Uses NFKD (compatibility decomposition) so that pre-composed + characters like 'é' (U+00E9) are split into a base letter plus a + combining mark before the mark is dropped. The previous NFKC form + was a composing normalization and so left composed accents in place + — strings like "café" came back unchanged. + + NFKD over NFD was chosen because the compatibility variant also + folds presentational variants useful in harmonization (e.g. + the ligature 'fi' becomes 'fi', superscripts collapse to digits). + Callers needing to preserve those should pre-normalize input to + NFC and apply this transform afterwards. """ - normalized = unicodedata.normalize("NFKC", value) + normalized = unicodedata.normalize("NFKD", value) return "".join(char for char in normalized if not unicodedata.combining(char)) def remove_punctuation(self, value: str) -> str: diff --git a/tests/test_primitives_serialization.py b/tests/test_primitives_serialization.py index 450b3e6..5f28de8 100644 --- a/tests/test_primitives_serialization.py +++ b/tests/test_primitives_serialization.py @@ -301,6 +301,51 @@ def test_normalize_text_serialization_and_transform(): assert primitive.transform(["AbC", "DeF"]) == ["abc", "def"] +def test_normalize_text_remove_accents_handles_composed_input(): + # Pre-composed accented characters: 'é' is one code point (U+00E9). + # The bug we are fixing was that NFKC normalization left these intact, + # so the combining-mark filter never removed the accent. + primitive = NormalizeText(Normalization.ACCENT) + assert primitive.transform("café") == "cafe" + assert primitive.transform("résumé") == "resume" + assert primitive.transform("naïve") == "naive" + assert primitive.transform("piñata") == "pinata" + assert primitive.transform("über") == "uber" + assert primitive.transform("Ångström") == "Angstrom" + + +def test_normalize_text_remove_accents_handles_decomposed_input(): + # If the caller hands us a pre-decomposed string (base + combining mark), + # accent removal should still work — NFKD is idempotent on already-NFKD input. + decomposed_cafe = "café" # "café" with explicit combining acute + primitive = NormalizeText(Normalization.ACCENT) + assert primitive.transform(decomposed_cafe) == "cafe" + + +def test_normalize_text_remove_accents_leaves_unaccented_text_unchanged(): + primitive = NormalizeText(Normalization.ACCENT) + assert primitive.transform("hello world") == "hello world" + assert primitive.transform("ABC123") == "ABC123" + + +def test_normalize_text_remove_accents_folds_compatibility_forms(): + # NFKD (compatibility decomposition) is intentional: it folds presentational + # variants like the ligature 'fi' to 'fi' and superscript digits to plain + # digits, which is what harmonization usually wants. + primitive = NormalizeText(Normalization.ACCENT) + assert primitive.transform("file") == "file" + assert primitive.transform("x²") == "x2" + + +def test_normalize_text_remove_accents_roundtrip_via_serialization(): + primitive = NormalizeText(Normalization.ACCENT) + payload = primitive.to_dict() + assert payload == {"operation": "normalize_text", "normalization": "remove_accents"} + + roundtrip = NormalizeText.from_serialization(payload) + assert roundtrip.transform("café") == "cafe" + + def test_convert_date_serialization_and_transform(): primitive = ConvertDate("%Y-%m-%d", "%m/%d/%Y") payload = primitive.to_dict()