diff --git a/abogen/kokoro_text_normalization.py b/abogen/kokoro_text_normalization.py index 962dc79..5cd47bc 100644 --- a/abogen/kokoro_text_normalization.py +++ b/abogen/kokoro_text_normalization.py @@ -153,7 +153,6 @@ _CURRENCY_RE = re.compile( _URL_RE = re.compile( r"(https?://)?(www\.)?(?P[a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)+)(/[^\s]*)?" ) -_FOOTNOTE_RE = re.compile(r"([a-zA-Z]+)(\d+)") _BRACKET_FOOTNOTE_RE = re.compile(r"\[\d+\]") _ISO_DATE_RE = re.compile( @@ -2002,16 +2001,13 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str: spoken = domain.replace(".", " dot ") return spoken - def _remove_footnote(match: re.Match[str]) -> str: - return match.group(1) - normalized = text # Apply URL replacement first (independent of number conversion). normalized = _URL_RE.sub(_replace_url, normalized) if getattr(cfg, "remove_footnotes", False): - normalized = _FOOTNOTE_RE.sub(_remove_footnote, normalized) + # Unmarked letter-digit text is ambiguous (word1, F1, H2O). normalized = _BRACKET_FOOTNOTE_RE.sub("", normalized) if cfg.convert_currency: diff --git a/tests/test_regression_fixes.py b/tests/test_regression_fixes.py index 1c52870..6897410 100644 --- a/tests/test_regression_fixes.py +++ b/tests/test_regression_fixes.py @@ -71,11 +71,10 @@ def test_roman_numerals_world_war(): def test_footnote_removal(): - # Bob is awesome1. -> Bob is awesome. + # A plain digit suffix cannot reliably be identified as a footnote. normalized = normalize("Bob is awesome1.") print(f"Bob is awesome1. -> {normalized}") - assert "bob is awesome." in normalized.lower() - assert "1" not in normalized + assert normalized == "Bob is awesome1." # Citation needed[1]. -> Citation needed. normalized = normalize("Citation needed[1].") @@ -84,6 +83,52 @@ def test_footnote_removal(): assert "[1]" not in normalized +@pytest.mark.parametrize("mode", ["spacy", "off"]) +@pytest.mark.parametrize( + "text", + [ + "F1 FP1 Q1 Q2 Q3 CO2 H2O 3D MP3 x86.", + "word1 awesome1 version2 item23.", + "H2O2 B2B ISO9001 F1-car model2_rev3.", + ], +) +def test_footnote_cleanup_preserves_alphanumeric_terms(text, mode): + assert normalize(text, {"normalization_apostrophe_mode": mode}) == text + + +@pytest.mark.parametrize("mode", ["spacy", "off"]) +@pytest.mark.parametrize("convert_numbers", [True, False]) +def test_footnote_cleanup_removes_explicit_markers_only(mode, convert_numbers): + assert normalize( + "F1[1] and H2O[23][2]. Evidence[123].", + { + "normalization_apostrophe_mode": mode, + "normalization_numbers": convert_numbers, + }, + ) == "F1 and H2O. Evidence." + + +@pytest.mark.parametrize("mode", ["spacy", "off"]) +def test_superscript_digits_are_normalized_without_guessing_footnotes(mode): + # NFKC flattens superscripts before footnote cleanup sees the text. + assert normalize( + "Evidence¹ and Evidence²³.", {"normalization_apostrophe_mode": mode} + ) == "Evidence1 and Evidence23." + + +@pytest.mark.parametrize("mode", ["spacy", "off"]) +def test_footnote_cleanup_disabled_preserves_bracket_markers(mode): + text = "F1[1] and H2O[23][2]." + assert normalize( + text, + { + "normalization_apostrophe_mode": mode, + "normalization_footnotes": False, + "normalization_numbers": False, + }, + ) == "F1[1] and H2O[23] [2]." + + def test_manual_override_normalization(): from abogen.entity_analysis import normalize_manual_override_token