feat: Enhance text normalization with support for internet slang expansion, currency formatting, and date handling; update debug WAVs interface and settings

abogen/web/debug_tts_runner.py

@@ -26,6 +26,7 @@ class DebugWavArtifact:
     label: str
     filename: str
     code: Optional[str] = None
+    text: Optional[str] = None
 
 
 def _resolve_voice_setting(value: str) -> tuple[str, Optional[str], Optional[str]]:
@@ -65,6 +66,21 @@ def _extract_cases_from_text(text: str) -> List[Tuple[str, str]]:
     return cases
 
 
+def _spoken_id(code: str) -> str:
+    # Make IDs pronounceable and stable (avoid reading as a word).
+    out: List[str] = []
+    for ch in str(code or ""):
+        if ch == "_":
+            out.append(" ")
+        elif ch.isalnum():
+            out.append(ch)
+        else:
+            out.append(" ")
+    # Add spaces between alnum to encourage letter-by-letter reading.
+    spaced = " ".join("".join(out).split())
+    return spaced
+
+
 def run_debug_tts_wavs(
     *,
     output_root: Path,
@@ -96,6 +112,14 @@ def run_debug_tts_wavs(
     combined_text = extraction.combined_text or "\n\n".join((c.text or "") for c in extraction.chapters)
     cases = _extract_cases_from_text(combined_text)
 
+    # Prefer the canonical sample catalog for text (EPUB extraction may include headings).
+    try:
+        from abogen.debug_tts_samples import DEBUG_TTS_SAMPLES
+
+        sample_text_by_code = {sample.code: sample.text for sample in DEBUG_TTS_SAMPLES}
+    except Exception:
+        sample_text_by_code = {}
+
     expected = list(iter_expected_codes())
     found_codes = {code for code, _ in cases}
     missing = [code for code in expected if code not in found_codes]
@@ -162,11 +186,15 @@ def run_debug_tts_wavs(
     overall_path = run_dir / "overall.wav"
     overall_audio: List[np.ndarray] = []
 
-    def synth(text: str) -> np.ndarray:
-        normalized = normalize_for_pipeline(
-            text,
-            config=apostrophe_config,
-            settings=normalization_settings,
+    def synth(text: str, *, apply_normalization: bool = True) -> np.ndarray:
+        normalized = (
+            normalize_for_pipeline(
+                text,
+                config=apostrophe_config,
+                settings=normalization_settings,
+            )
+            if apply_normalization
+            else str(text or "")
         )
         parts: List[np.ndarray] = []
         for segment in pipeline(
@@ -182,19 +210,26 @@ def run_debug_tts_wavs(
             return np.zeros(0, dtype="float32")
         return np.concatenate(parts).astype("float32", copy=False)
 
+    pause_1s = np.zeros(int(1.0 * SAMPLE_RATE), dtype="float32")
+    between_cases = np.zeros(int(0.35 * SAMPLE_RATE), dtype="float32")
+
     # Per sample
     for code, snippet in cases:
+        snippet = sample_text_by_code.get(code, snippet)
         if not snippet:
             continue
-        audio = synth(snippet)
+        id_audio = synth(_spoken_id(code), apply_normalization=False)
+        text_audio = synth(snippet, apply_normalization=True)
+        audio = np.concatenate([id_audio, pause_1s, text_audio]).astype("float32", copy=False)
         filename = f"case_{code}.wav"
         path = run_dir / filename
         # Write float32 PCM WAV.
         import soundfile as sf
 
         sf.write(path, audio, SAMPLE_RATE, subtype="FLOAT")
-        artifacts.append(DebugWavArtifact(label=f"{code}", filename=filename, code=code))
+        artifacts.append(DebugWavArtifact(label=f"{code}", filename=filename, code=code, text=snippet))
         overall_audio.append(audio)
+        overall_audio.append(between_cases)
 
     # Overall
     if overall_audio:
@@ -204,7 +239,7 @@ def run_debug_tts_wavs(
     import soundfile as sf
 
     sf.write(overall_path, combined, SAMPLE_RATE, subtype="FLOAT")
-    artifacts.insert(0, DebugWavArtifact(label="Overall", filename="overall.wav", code=None))
+    artifacts.insert(0, DebugWavArtifact(label="Overall", filename="overall.wav", code=None, text=None))
 
     manifest = {
         "run_id": run_id,

abogen/web/routes/settings.py

@@ -80,9 +80,17 @@ def update_settings() -> ResponseReturnValue:
         maximum=25,
     )
 
+    def _extract_checkbox(name: str, default: bool) -> bool:
+        values = form.getlist(name) if hasattr(form, "getlist") else []
+        if values:
+            return coerce_bool(values[-1], default)
+        if hasattr(form, "__contains__") and name in form:
+            return False
+        return default
+
     # Normalization settings
     for key in _NORMALIZATION_BOOLEAN_KEYS:
-        current[key] = coerce_bool(form.get(key), False)
+        current[key] = _extract_checkbox(key, bool(current.get(key, True)))
     for key in _NORMALIZATION_STRING_KEYS:
         current[key] = (form.get(key) or "").strip()
 
@@ -236,7 +244,8 @@ def download_debug_wav(run_id: str, filename: str) -> ResponseReturnValue:
     safe_name = (filename or "").strip()
     if not safe_run or not safe_name or "/" in safe_name or "\\" in safe_name:
         abort(404)
-    if not safe_name.lower().endswith(".wav") and safe_name != "manifest.json":
+    is_wav = safe_name.lower().endswith(".wav")
+    if not is_wav and safe_name != "manifest.json":
         abort(404)
 
     root = Path(current_app.config.get("OUTPUT_FOLDER") or get_user_output_path("web"))
@@ -247,4 +256,12 @@ def download_debug_wav(run_id: str, filename: str) -> ResponseReturnValue:
     expected_dir = (root / "debug" / safe_run).resolve()
     if expected_dir not in path.parents:
         abort(404)
-    return send_file(path, as_attachment=True, download_name=path.name)
+    wants_download = str(request.args.get("download") or "").strip().lower() in {"1", "true", "yes"}
+    mimetype = "audio/wav" if is_wav else "application/json"
+    # Inline playback should work for WAVs; allow explicit downloads via ?download=1.
+    return send_file(
+        path,
+        mimetype=mimetype,
+        as_attachment=wants_download,
+        download_name=path.name,
+    )

abogen/web/routes/utils/settings.py

@@ -47,6 +47,9 @@ _NORMALIZATION_BOOLEAN_KEYS = {
     "normalization_terminal",
     "normalization_phoneme_hints",
     "normalization_caps_quotes",
+    "normalization_currency",
+    "normalization_footnotes",
+    "normalization_internet_slang",
     "normalization_apostrophes_contractions",
     "normalization_apostrophes_plural_possessives",
     "normalization_apostrophes_sibilant_possessives",
@@ -58,8 +61,6 @@ _NORMALIZATION_BOOLEAN_KEYS = {
     "normalization_contraction_modal_would",
     "normalization_contraction_negation_not",
     "normalization_contraction_let_us",
-    "normalization_currency",
-    "normalization_footnotes",
 }
 
 _NORMALIZATION_STRING_KEYS = {
@@ -84,6 +85,9 @@ BOOLEAN_SETTINGS = {
     "normalization_terminal",
     "normalization_phoneme_hints",
     "normalization_caps_quotes",
+    "normalization_currency",
+    "normalization_footnotes",
+    "normalization_internet_slang",
     "normalization_apostrophes_contractions",
     "normalization_apostrophes_plural_possessives",
     "normalization_apostrophes_sibilant_possessives",
@@ -107,6 +111,7 @@ _NORMALIZATION_GROUPS = [
             {"key": "normalization_numbers", "label": "Convert grouped numbers to words"},
             {"key": "normalization_currency", "label": "Convert currency symbols ($10 → ten dollars)"},
             {"key": "normalization_titles", "label": "Expand titles and suffixes (Dr., St., Jr., …)"},
+            {"key": "normalization_internet_slang", "label": "Expand internet slang (pls → please)"},
             {"key": "normalization_footnotes", "label": "Remove footnote indicators ([1], [2])"},
             {"key": "normalization_terminal", "label": "Ensure sentences end with terminal punctuation"},
             {"key": "normalization_caps_quotes", "label": "Convert ALL CAPS dialogue inside quotes"},
@@ -195,10 +200,13 @@ def settings_defaults() -> Dict[str, Any]:
         "llm_prompt": llm_env_defaults.get("llm_prompt", DEFAULT_LLM_PROMPT),
         "llm_context_mode": llm_env_defaults.get("llm_context_mode", "sentence"),
         "normalization_numbers": True,
+        "normalization_currency": True,
+        "normalization_footnotes": True,
         "normalization_titles": True,
         "normalization_terminal": True,
         "normalization_phoneme_hints": True,
         "normalization_caps_quotes": True,
+        "normalization_internet_slang": False,
         "normalization_apostrophes_contractions": True,
         "normalization_apostrophes_plural_possessives": True,
         "normalization_apostrophes_sibilant_possessives": True,

abogen/web/templates/debug_wavs.html

@@ -7,10 +7,7 @@
   <h1 class="card__title">Debug WAVs</h1>
   <p class="tag">Run ID: <code>{{ run_id }}</code></p>
 
-  <div class="field field--stack">
-    <audio data-role="debug-audio" controls preload="none" style="width: 100%;"></audio>
-    <p class="hint">Click an ID to play its WAV. “Overall” is the concatenation of all samples.</p>
-  </div>
+  <p class="hint">Each clip reads: ID, one second pause, then the reference text.</p>
 
   {% if artifacts %}
   <div class="field field--wide">
@@ -18,15 +15,15 @@
     <ul>
       {% for item in artifacts %}
       <li>
-        <button
-          type="button"
-          class="button button--ghost"
-          data-role="debug-play"
-          data-audio-src="{{ item.url }}"
-        >
-          {{ item.label }}
-        </button>
-        <a class="muted" href="{{ item.url }}" download>{{ item.filename }}</a>
+        <div class="field field--stack" style="margin: 0;">
+          <div>
+            <strong>{{ item.label }}</strong>
+            {% if item.text %}
+              — <span class="muted">{{ item.text }}</span>
+            {% endif %}
+          </div>
+          <audio controls preload="none" src="{{ item.url }}"></audio>
+        </div>
       </li>
       {% endfor %}
     </ul>
@@ -37,25 +34,4 @@
     <a href="{{ url_for('settings.settings_page', _anchor='debug') }}" class="button button--ghost">Back to Settings</a>
   </div>
 </section>
-
-<script>
-  (function () {
-    const audio = document.querySelector('[data-role="debug-audio"]');
-    const buttons = Array.from(document.querySelectorAll('[data-role="debug-play"]'));
-    if (!audio || !buttons.length) {
-      return;
-    }
-
-    buttons.forEach((button) => {
-      button.addEventListener('click', () => {
-        const src = button.dataset.audioSrc;
-        if (!src) return;
-        if (audio.src !== src) {
-          audio.src = src;
-        }
-        audio.play();
-      });
-    });
-  })();
-</script>
 {% endblock %}