feat: Add _chunk_text_for_tts function to prioritize raw text for TTS synthesis and implement related tests

abogen/web/conversion_runner.py

@@ -1264,6 +1264,25 @@ def _safe_int(value: Any, default: int = 0) -> int:
         return default
 
 
+def _chunk_text_for_tts(entry: Mapping[str, Any]) -> str:
+    """Choose the best source text for synthesis.
+
+    We must prefer the raw chunk text (`text` / `original_text`) so manual/pronunciation
+    overrides can match against the original tokens (e.g. censored words like `Unfu*k`).
+    `normalized_text` may have already been run through `normalize_for_pipeline`, which
+    can remove punctuation and prevent overrides from triggering.
+    """
+
+    if not isinstance(entry, Mapping):
+        return ""
+    return str(
+        entry.get("text")
+        or entry.get("original_text")
+        or entry.get("normalized_text")
+        or ""
+    ).strip()
+
+
 def _escape_ffmetadata_value(value: str) -> str:
     escaped = str(value).replace("\\", "\\\\").replace("\n", "\\n")
     escaped = escaped.replace("=", "\\=").replace(";", "\\;").replace("#", "\\#")
@@ -1986,11 +2005,7 @@ def run_conversion_job(job: Job) -> None:
                         level="debug",
                     )
                 for chunk_entry in chunks_for_chapter:
-                    chunk_text = str(
-                        chunk_entry.get("normalized_text")
-                        or chunk_entry.get("text")
-                        or ""
-                    ).strip()
+                    chunk_text = _chunk_text_for_tts(chunk_entry)
                     if not chunk_text:
                         continue
 

tests/test_chunk_text_for_tts_prefers_raw.py

@@ -0,0 +1,25 @@
+from abogen.web.conversion_runner import _chunk_text_for_tts
+
+
+def test_chunk_text_for_tts_prefers_text_over_normalized_text():
+    entry = {
+        # Simulate a pre-normalized chunk that lost the asterisk.
+        "normalized_text": "Unfuk",
+        # Raw chunk should preserve censored token for manual overrides.
+        "text": "Unfu*k",
+    }
+
+    assert _chunk_text_for_tts(entry) == "Unfu*k"
+
+
+def test_chunk_text_for_tts_falls_back_to_original_text_then_normalized_text():
+    entry = {
+        "original_text": "Hello * world",
+        "normalized_text": "Hello world",
+    }
+    assert _chunk_text_for_tts(entry) == "Hello * world"
+
+    entry2 = {
+        "normalized_text": "Only normalized",
+    }
+    assert _chunk_text_for_tts(entry2) == "Only normalized"