feat: Implement footnote removal and URL normalization in text processing; enhance manual override token normalization

abogen/entity_analysis.py

@@ -442,3 +442,12 @@ def merge_override(summary: Mapping[str, Any], overrides: Mapping[str, Mapping[s
 
 def normalize_token(token: str) -> str:
     return _token_key(_normalize_label(token))
+
+
+def normalize_manual_override_token(token: str) -> str:
+    if not token:
+        return ""
+    stripped = token.strip().strip("\"'`“”’")
+    if not stripped:
+        return ""
+    return _MULTI_SPACE_PATTERN.sub(" ", stripped.lower()).strip()

abogen/kokoro_text_normalization.py

@@ -59,6 +59,7 @@ class ApostropheConfig:
     protect_cultural_names: bool = True           # Always keep O'Brien, D'Angelo, etc.
     convert_numbers: bool = True                  # Convert grouped numbers such as 12,500 to words
     convert_currency: bool = True                 # Convert currency symbols to words
+    remove_footnotes: bool = True                 # Remove footnote indicators
     number_lang: str = "en"                       # num2words language code
     year_pronunciation_mode: str = "american"     # off|american (extend if needed)
     contraction_categories: Dict[str, bool] = field(default_factory=lambda: dict(CONTRACTION_CATEGORY_DEFAULTS))
@@ -123,6 +124,10 @@ _CURRENCY_RE = re.compile(
     r"(?P<symbol>[$£€¥])\s*(?P<amount>\d{1,3}(?:,\d{3})*(?:\.\d{1,2})?)(?!\d)"
 )
 
+_URL_RE = re.compile(r"(https?://)?(www\.)?(?P<domain>[a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)+)(/[^\s]*)?")
+_FOOTNOTE_RE = re.compile(r"([a-zA-Z]+)(\d+)")
+_BRACKET_FOOTNOTE_RE = re.compile(r"\[\d+\]")
+
 _DECIMAL_NUMBER_RE = re.compile(
     rf"(?<![\w{_NUMBER_RANGE_CLASS}/])(?P<number>-?(?:\d{{1,3}}(?:,\d{{3}})+|\d+)\.(?P<fraction>\d+))(?![\w{_NUMBER_RANGE_CLASS}/])"
 )
@@ -486,6 +491,7 @@ _ROMAN_CARDINAL_CONTEXTS = {
     "volume",
     "war",
     "world",
+    "world war",
 }
 _ROMAN_NAME_TITLES = {
     "baron",
@@ -702,14 +708,20 @@ def _normalize_roman_numerals(text: str, language: str) -> str:
                         replacement = f"{context_word} {words}"
         else:
             candidate = token.upper()
-            if len(token) >= 2 and _ROMAN_TOKEN_RE.match(candidate):
+            is_roman = _ROMAN_TOKEN_RE.match(candidate)
+            if is_roman:
                 numeric_value = _roman_to_int(candidate)
                 if numeric_value is not None:
                     convert = False
-                    if token.isupper():
+                    if len(token) >= 2:
-                        convert = True
+                        if token.isupper():
-                    elif numeric_value <= 200 and _has_cardinal_leading_context(tokens, index):
+                            convert = True
-                        convert = True
+                        elif numeric_value <= 200 and _has_cardinal_leading_context(tokens, index):
+                            convert = True
+                    elif len(token) == 1:
+                        # Only convert single letters if context is strong
+                        if _has_cardinal_leading_context(tokens, index):
+                            convert = True
 
                     if convert:
                         if _should_render_ordinal(tokens, index, numeric_value):
@@ -1340,6 +1352,13 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
             return words
 
         if style == "american":
+            # Special handling for 2000-2009 to use "two thousand X"
+            if 2000 <= value <= 2009:
+                words = _words(value)
+                if words:
+                    return words.replace(" and ", " ")
+                return None
+
             if value % 1000 == 0:
                 thousands = value // 1000
                 thousands_words = _words(thousands)
@@ -1349,52 +1368,17 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
 
             first_two = value // 100
             last_two = value % 100
-
+            
-            # Special handling to match common American pronunciations.
-            if first_two == 20:
-                if last_two == 0:
-                    return "two thousand"
-                if last_two < 10:
-                    tail_word = _words(last_two)
-                    if tail_word:
-                        return f"two thousand {tail_word}"
-                    return None
-                prefix = _words(first_two)
-                tail = _words(last_two)
-                if prefix and tail:
-                    return f"{prefix} {tail}"
-                return prefix
-
             prefix = _words(first_two)
             if not prefix:
                 return None
 
-            if first_two == 10:
-                if last_two == 0:
-                    return "one thousand"
-                tail = _format_year_tail(last_two)
-                if tail:
-                    return f"{prefix} {tail}"
-                return prefix
-
-            if first_two <= 12:
-                if last_two == 0:
-                    return f"{prefix} hundred"
-                tail = _format_year_tail(last_two)
-                if tail:
-                    return f"{prefix} hundred {tail}"
-                return f"{prefix} hundred"
-
-            if first_two <= 19:
-                if last_two == 0:
-                    return f"{prefix} hundred"
-                tail = _format_year_tail(last_two)
-                if tail:
-                    return f"{prefix} {tail}"
-                return prefix
-
             if last_two == 0:
                 return f"{prefix} hundred"
+            
+            if last_two < 10:
+                # Use "hundred oh X" format (e.g. "twelve hundred oh four")
+                return f"{prefix} hundred oh {_DIGIT_WORDS[last_two]}"
 
             tail = _format_year_tail(last_two)
             if tail:
@@ -1500,6 +1484,7 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
             
         symbol = match.group("symbol")
         amount_str = match.group("amount").replace(",", "")
+        # print(f"DEBUG: symbol={symbol} amount_str={amount_str}")
         try:
             amount = float(amount_str)
         except ValueError:
@@ -1514,12 +1499,51 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
         currency_code = currency_map.get(symbol, "USD")
         
         try:
+            # Always use float to avoid num2words treating int as cents (if that's what it does)
+            # or to ensure consistent behavior.
             words = num2words(amount, to="currency", currency=currency_code, lang=language)
+            
+            # Remove "zero cents" if present
+            # Patterns: ", zero cents", " and zero cents"
+            words = words.replace(", zero cents", "").replace(" and zero cents", "")
             return words
         except Exception:
             return match.group(0)
 
+    def _replace_url(match: re.Match[str]) -> str:
+        domain = match.group("domain")
+        
+        # Avoid matching numbers like 1.05 or 12.34.56
+        # If the domain consists only of digits and dots, ignore it (unless it has http/www prefix)
+        has_prefix = match.group(1) or match.group(2)
+        if not has_prefix and all(c.isdigit() or c == '.' or c == '-' for c in domain):
+             # Check if it really looks like a number (e.g. 1.05)
+             # If it has multiple dots like 1.2.3.4 it might be an IP, which we might want to speak as dot?
+             # But 1.05 is definitely a number.
+             # Let's be safe: if it looks like a float, skip.
+             try:
+                 float(domain)
+                 return match.group(0)
+             except ValueError:
+                 pass
+        
+        if domain.startswith("www."):
+            domain = domain[4:]
+        spoken = domain.replace(".", " dot ")
+        return spoken
+
+    def _remove_footnote(match: re.Match[str]) -> str:
+        return match.group(1)
+
     normalized = text
+    
+    # Apply URL replacement first
+    normalized = _URL_RE.sub(_replace_url, normalized)
+    
+    if getattr(cfg, "remove_footnotes", False):
+        normalized = _FOOTNOTE_RE.sub(_remove_footnote, normalized)
+        normalized = _BRACKET_FOOTNOTE_RE.sub("", normalized)
+
     if cfg.convert_currency:
         normalized = _CURRENCY_RE.sub(_replace_currency, normalized)
     normalized = _NUMBER_RANGE_RE.sub(lambda m: _replace_number_range(m, language), normalized)

abogen/normalization_settings.py

@@ -33,6 +33,7 @@ _SETTINGS_DEFAULTS: Dict[str, Any] = {
     "normalization_numbers": True,
     "normalization_numbers_year_style": "american",
     "normalization_currency": True,
+    "normalization_footnotes": True,
     "normalization_titles": True,
     "normalization_terminal": True,
     "normalization_phoneme_hints": True,

abogen/spacy_contraction_resolver.py

@@ -134,6 +134,11 @@ def _resolve_apostrophe_s(token: Token) -> Optional[ContractionResolution]:
     if prev_lower == "let":
         return _resolution(prev, token, "us", "contraction_let_us", "us")
 
+    # Special check for 's been -> has been, overriding lemma
+    next_content = _next_content_token(token)
+    if next_content and next_content.text.lower() == "been":
+        return _resolution(prev, token, "has", "contraction_aux_have", "have")
+
     lemma = token.lemma_.lower()
     if not lemma:
         lemma = "be" if _favors_be(token) else "have" if _favors_have(token) else "be"

abogen/web/routes/utils/entity.py

@@ -7,6 +7,7 @@ from abogen.entity_analysis import (
     extract_entities,
     merge_override,
     normalize_token as normalize_entity_token,
+    normalize_manual_override_token,
     search_tokens as search_entity_tokens,
 )
 from abogen.pronunciation_store import (
@@ -84,7 +85,7 @@ def collect_pronunciation_overrides(pending: PendingJob) -> List[Dict[str, Any]]
         pronunciation_value = str(manual_entry.get("pronunciation") or "").strip()
         if not token_value or not pronunciation_value:
             continue
-        normalized = manual_entry.get("normalized") or normalize_entity_token(token_value)
+        normalized = manual_entry.get("normalized") or normalize_manual_override_token(token_value)
         if not normalized:
             continue
         collected[normalized] = {
@@ -196,7 +197,7 @@ def upsert_manual_override(pending: PendingJob, payload: Mapping[str, Any]) -> D
     voice_value = str(payload.get("voice") or "").strip()
     notes_value = str(payload.get("notes") or "").strip()
     context_value = str(payload.get("context") or "").strip()
-    normalized = payload.get("normalized") or normalize_entity_token(token_value)
+    normalized = payload.get("normalized") or normalize_manual_override_token(token_value)
     if not normalized:
         raise ValueError("Token is required")
 

abogen/web/templates/base.html

@@ -18,8 +18,8 @@
     <nav class="top-actions">
       {% set endpoint = request.endpoint or '' %}
     <a href="{{ url_for('main.index') }}" class="btn{% if endpoint == 'main.index' %} is-active{% endif %}">Dashboard</a>
-  <a href="{{ url_for('voices.voice_profiles') }}" class="btn{% if endpoint == 'voices.voice_profiles' %} is-active{% endif %}">Voice Mixer</a>
+    <a href="{{ url_for('voices.voice_profiles') }}" class="btn{% if endpoint == 'voices.voice_profiles' %} is-active{% endif %}">Voice Mixer</a>
-    <a href="{{ url_for('entities.entities_page') }}" class="btn{% if endpoint == 'entities.entities_page' %} is-active{% endif %}">Entities</a>
+    <a href="{{ url_for('entities.entities_page') }}" class="btn{% if endpoint == 'entities.entities_page' %} is-active{% endif %}">TTS Overrides</a>
     <a href="{{ url_for('books.find_books_page') }}" class="btn{% if endpoint == 'books.find_books_page' %} is-active{% endif %}">Find Books</a>
   <a href="{{ url_for('jobs.queue_page') }}" class="btn{% if endpoint in ['jobs.queue_page', 'jobs.job_detail'] %} is-active{% endif %}">Queue</a>
     <a href="{{ url_for('settings.settings_page') }}" class="btn{% if endpoint == 'settings.settings_page' %} is-active{% endif %}">Settings</a>

abogen/web/templates/entities.html

@@ -1,6 +1,6 @@
 {% extends "base.html" %}
 
-{% block title %}abogen · Entities{% endblock %}
+{% block title %}abogen · TTS Overrides{% endblock %}
 
 {% block content %}
 <section
@@ -9,7 +9,7 @@
   data-preview-url="{{ url_for('api.api_entity_pronunciation_preview') }}"
   data-language="{{ language }}"
 >
-  <h1 class="card__title">Entities &amp; Pronunciation Overrides</h1>
+  <h1 class="card__title">TTS Overrides &amp; Pronunciation</h1>
   <p class="card__subtitle">Review and refine stored pronunciations so recurring names sound right in every project.</p>
   <form class="entity-filter" method="get">
     <div class="entity-filter__row">

tests/test_regression_fixes.py

@@ -0,0 +1,82 @@
+import pytest
+from unittest.mock import patch
+from abogen.kokoro_text_normalization import normalize_for_pipeline, DEFAULT_APOSTROPHE_CONFIG
+from abogen.normalization_settings import build_apostrophe_config, _SETTINGS_DEFAULTS
+
+def normalize(text, overrides=None):
+    settings = dict(_SETTINGS_DEFAULTS)
+    if overrides:
+        settings.update(overrides)
+    
+    config = build_apostrophe_config(settings=settings, base=DEFAULT_APOSTROPHE_CONFIG)
+    return normalize_for_pipeline(text, config=config, settings=settings)
+
+def test_year_pronunciation():
+    # 1925 -> Nineteen Twenty Five
+    normalized = normalize("1925")
+    print(f"1925 -> {normalized}")
+    # num2words might output "nineteen twenty-five"
+    assert "nineteen twenty" in normalized.lower()
+    assert "five" in normalized.lower()
+
+    # 2025 -> Twenty Twenty Five
+    normalized = normalize("2025")
+    print(f"2025 -> {normalized}")
+    assert "twenty twenty" in normalized.lower()
+    assert "five" in normalized.lower()
+
+def test_currency_pronunciation():
+    # $1.00 -> One dollar (no zero cents)
+    normalized = normalize("$1.00")
+    print(f"$1.00 -> {normalized}")
+    assert "one dollar" in normalized.lower()
+    assert "zero cents" not in normalized.lower()
+
+    # $1.05 -> One dollar and five cents (or comma)
+    normalized = normalize("$1.05")
+    print(f"$1.05 -> {normalized}")
+    assert "one dollar" in normalized.lower()
+    assert "five cents" in normalized.lower()
+
+def test_url_pronunciation():
+    # https://www.amazon.com -> amazon dot com
+    normalized = normalize("https://www.amazon.com")
+    print(f"https://www.amazon.com -> {normalized}")
+    assert "amazon dot com" in normalized.lower()
+    assert "http" not in normalized.lower()
+    assert "www" not in normalized.lower()
+
+    # www.google.com -> google dot com
+    normalized = normalize("www.google.com")
+    print(f"www.google.com -> {normalized}")
+    assert "google dot com" in normalized.lower()
+
+def test_roman_numerals_world_war():
+    # World War I -> World War One
+    normalized = normalize("World War I")
+    print(f"World War I -> {normalized}")
+    assert "world war one" in normalized.lower()
+
+    # World War II -> World War Two
+    normalized = normalize("World War II")
+    print(f"World War II -> {normalized}")
+    assert "world war two" in normalized.lower()
+
+def test_footnote_removal():
+    # Bob is awesome1. -> Bob is awesome.
+    normalized = normalize("Bob is awesome1.")
+    print(f"Bob is awesome1. -> {normalized}")
+    assert "bob is awesome." in normalized.lower()
+    assert "1" not in normalized
+
+    # Citation needed[1]. -> Citation needed.
+    normalized = normalize("Citation needed[1].")
+    print(f"Citation needed[1]. -> {normalized}")
+    assert "citation needed." in normalized.lower()
+    assert "[1]" not in normalized
+
+def test_manual_override_normalization():
+    from abogen.entity_analysis import normalize_manual_override_token
+    assert normalize_manual_override_token("The") == "the"
+    assert normalize_manual_override_token("  A  ") == "a"
+    assert normalize_manual_override_token("word") == "word"

tests/test_text_normalization.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import pytest
+from unittest.mock import patch
 
 from abogen.kokoro_text_normalization import (
     DEFAULT_APOSTROPHE_CONFIG,
@@ -254,3 +255,31 @@ def test_modal_will_contractions_can_be_disabled() -> None:
         normalization_overrides={"normalization_contraction_modal_will": False},
     )
     assert "captain'll" in normalized
+
+
+@pytest.fixture(autouse=True)
+def mock_settings():
+    defaults = {
+        "normalization_numbers": True,
+        "normalization_titles": True,
+        "normalization_terminal": True,
+        "normalization_phoneme_hints": True,
+        "normalization_caps_quotes": True,
+        "normalization_apostrophes_contractions": True,
+        "normalization_apostrophes_plural_possessives": True,
+        "normalization_apostrophes_sibilant_possessives": True,
+        "normalization_apostrophes_decades": True,
+        "normalization_apostrophes_leading_elisions": True,
+        "normalization_apostrophe_mode": "spacy",
+        "normalization_contraction_aux_be": True,
+        "normalization_contraction_aux_have": True,
+        "normalization_contraction_modal_will": True,
+        "normalization_contraction_modal_would": True,
+        "normalization_contraction_negation_not": True,
+        "normalization_contraction_let_us": True,
+        "normalization_currency": True,
+        "normalization_footnotes": True,
+        "normalization_numbers_year_style": "american",
+    }
+    with patch("tests.test_text_normalization.get_runtime_settings", return_value=defaults):
+        yield