From 2b53256d433c8e5ec60b9da670701d5db16bedcf Mon Sep 17 00:00:00 2001 From: agent Date: Wed, 28 Oct 2026 21:29:52 -0500 Subject: [PATCH 1/2] test(normalization): reproduce alphanumeric footnote corruption --- tests/test_regression_fixes.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/test_regression_fixes.py b/tests/test_regression_fixes.py index 1c52870..9a8c01d 100644 --- a/tests/test_regression_fixes.py +++ b/tests/test_regression_fixes.py @@ -84,6 +84,40 @@ def test_footnote_removal(): assert "[1]" not in normalized +@pytest.mark.parametrize("mode", ["spacy", "off"]) +@pytest.mark.parametrize( + "text", + [ + "F1 FP1 Q1 Q2 Q3 CO2 H2O 3D MP3 x86.", + "word1 awesome1 version2 item23.", + "H2O2 B2B ISO9001 F1-car model2_rev3.", + ], +) +def test_footnote_cleanup_preserves_alphanumeric_terms(text, mode): + assert normalize(text, {"normalization_apostrophe_mode": mode}) == text + + +@pytest.mark.parametrize("mode", ["spacy", "off"]) +def test_footnote_cleanup_removes_explicit_markers_only(mode): + assert normalize( + "F1[1] and H2O[23][2]. Evidence[123].", + {"normalization_apostrophe_mode": mode}, + ) == "F1 and H2O. Evidence." + + +@pytest.mark.parametrize("mode", ["spacy", "off"]) +def test_footnote_cleanup_disabled_preserves_bracket_markers(mode): + text = "F1[1] and H2O[23][2]." + assert normalize( + text, + { + "normalization_apostrophe_mode": mode, + "normalization_footnotes": False, + "normalization_numbers": False, + }, + ) == "F1[1] and H2O[23] [2]." + + def test_manual_override_normalization(): from abogen.entity_analysis import normalize_manual_override_token From 4dd0bf0ff20cd67ac6d45835e5cd89731f0880c2 Mon Sep 17 00:00:00 2001 From: agent Date: Wed, 28 Oct 2026 21:31:41 -0500 Subject: [PATCH 2/2] fix(normalization): preserve unmarked alphanumeric terms Only remove explicit bracketed footnote markers. Plain letter-digit text cannot distinguish identifiers from flattened footnotes. Update the old suffix-removal expectation and cover the intentional superscript compatibility change. --- abogen/kokoro_text_normalization.py | 6 +----- tests/test_regression_fixes.py | 21 ++++++++++++++++----- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/abogen/kokoro_text_normalization.py b/abogen/kokoro_text_normalization.py index 962dc79..5cd47bc 100644 --- a/abogen/kokoro_text_normalization.py +++ b/abogen/kokoro_text_normalization.py @@ -153,7 +153,6 @@ _CURRENCY_RE = re.compile( _URL_RE = re.compile( r"(https?://)?(www\.)?(?P[a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)+)(/[^\s]*)?" ) -_FOOTNOTE_RE = re.compile(r"([a-zA-Z]+)(\d+)") _BRACKET_FOOTNOTE_RE = re.compile(r"\[\d+\]") _ISO_DATE_RE = re.compile( @@ -2002,16 +2001,13 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str: spoken = domain.replace(".", " dot ") return spoken - def _remove_footnote(match: re.Match[str]) -> str: - return match.group(1) - normalized = text # Apply URL replacement first (independent of number conversion). normalized = _URL_RE.sub(_replace_url, normalized) if getattr(cfg, "remove_footnotes", False): - normalized = _FOOTNOTE_RE.sub(_remove_footnote, normalized) + # Unmarked letter-digit text is ambiguous (word1, F1, H2O). normalized = _BRACKET_FOOTNOTE_RE.sub("", normalized) if cfg.convert_currency: diff --git a/tests/test_regression_fixes.py b/tests/test_regression_fixes.py index 9a8c01d..6897410 100644 --- a/tests/test_regression_fixes.py +++ b/tests/test_regression_fixes.py @@ -71,11 +71,10 @@ def test_roman_numerals_world_war(): def test_footnote_removal(): - # Bob is awesome1. -> Bob is awesome. + # A plain digit suffix cannot reliably be identified as a footnote. normalized = normalize("Bob is awesome1.") print(f"Bob is awesome1. -> {normalized}") - assert "bob is awesome." in normalized.lower() - assert "1" not in normalized + assert normalized == "Bob is awesome1." # Citation needed[1]. -> Citation needed. normalized = normalize("Citation needed[1].") @@ -98,13 +97,25 @@ def test_footnote_cleanup_preserves_alphanumeric_terms(text, mode): @pytest.mark.parametrize("mode", ["spacy", "off"]) -def test_footnote_cleanup_removes_explicit_markers_only(mode): +@pytest.mark.parametrize("convert_numbers", [True, False]) +def test_footnote_cleanup_removes_explicit_markers_only(mode, convert_numbers): assert normalize( "F1[1] and H2O[23][2]. Evidence[123].", - {"normalization_apostrophe_mode": mode}, + { + "normalization_apostrophe_mode": mode, + "normalization_numbers": convert_numbers, + }, ) == "F1 and H2O. Evidence." +@pytest.mark.parametrize("mode", ["spacy", "off"]) +def test_superscript_digits_are_normalized_without_guessing_footnotes(mode): + # NFKC flattens superscripts before footnote cleanup sees the text. + assert normalize( + "Evidence¹ and Evidence²³.", {"normalization_apostrophe_mode": mode} + ) == "Evidence1 and Evidence23." + + @pytest.mark.parametrize("mode", ["spacy", "off"]) def test_footnote_cleanup_disabled_preserves_bracket_markers(mode): text = "F1[1] and H2O[23][2]."