From 015435adb631c903181c67dc627cc390dfbba2c3 Mon Sep 17 00:00:00 2001 From: JB Date: Mon, 15 Dec 2025 17:27:08 -0800 Subject: [PATCH] feat: Enhance text normalization with support for internet slang expansion, currency formatting, and date handling; update debug WAVs interface and settings --- abogen/debug_tts_samples.py | 2 +- abogen/heteronym_overrides.py | 13 +- abogen/kokoro_text_normalization.py | 336 +++++++++++++++++++++++++-- abogen/normalization_settings.py | 1 + abogen/web/debug_tts_runner.py | 51 +++- abogen/web/routes/settings.py | 23 +- abogen/web/routes/utils/settings.py | 12 +- abogen/web/templates/debug_wavs.html | 44 +--- tests/conftest.py | 30 ++- tests/test_text_normalization.py | 34 +++ 10 files changed, 472 insertions(+), 74 deletions(-) diff --git a/abogen/debug_tts_samples.py b/abogen/debug_tts_samples.py index b5490d4..6fa4c52 100644 --- a/abogen/debug_tts_samples.py +++ b/abogen/debug_tts_samples.py @@ -182,7 +182,7 @@ DEBUG_TTS_SAMPLES: Sequence[DebugTTSSample] = ( DebugTTSSample( code="TITLE_003", label="Titles and abbreviations (3)", - text="Gen. Lee spoke to Sgt. Rivera on Main St.", + text="Gen. Smith spoke to Sgt. Rivera on Main St.", ), DebugTTSSample( code="TITLE_004", diff --git a/abogen/heteronym_overrides.py b/abogen/heteronym_overrides.py index 4c6e44a..dbb5384 100644 --- a/abogen/heteronym_overrides.py +++ b/abogen/heteronym_overrides.py @@ -5,7 +5,10 @@ import re from dataclasses import dataclass from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple -import spacy +try: # pragma: no cover - optional dependency + import spacy # type: ignore +except Exception: # pragma: no cover - spaCy may be unavailable in minimal environments + spacy = None @dataclass(frozen=True) @@ -177,6 +180,9 @@ def _build_replacement_sentence(sentence: str, token: str, replacement_token: st def _load_spacy(language: str) -> Any: + if spacy is None: + return None + # English only for now. # Use installed small model; keep it simple. lang = (language or "en").lower() @@ -211,7 +217,12 @@ def extract_heteronym_overrides( if not lang.startswith("en"): return [] + if spacy is None: + return [] + nlp = _load_spacy(lang) + if nlp is None: + return [] previous_choices: Dict[str, str] = {} if existing: diff --git a/abogen/kokoro_text_normalization.py b/abogen/kokoro_text_normalization.py index e8f1850..92bd254 100644 --- a/abogen/kokoro_text_normalization.py +++ b/abogen/kokoro_text_normalization.py @@ -3,6 +3,8 @@ from __future__ import annotations import json import re import unicodedata +import os +import locale from fractions import Fraction from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple @@ -129,6 +131,176 @@ _URL_RE = re.compile(r"(https?://)?(www\.)?(?P[a-zA-Z0-9-]+(\.[a-zA-Z0-9 _FOOTNOTE_RE = re.compile(r"([a-zA-Z]+)(\d+)") _BRACKET_FOOTNOTE_RE = re.compile(r"\[\d+\]") +_ISO_DATE_RE = re.compile(r"\b(?P\d{4})[/-](?P\d{1,2})[/-](?P\d{1,2})\b") +_MDY_DATE_RE = re.compile( + r"\b(?PJan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\.?\s+" + r"(?P\d{1,2})(?:st|nd|rd|th)?\s*,\s*(?P\d{4})\b", + re.IGNORECASE, +) + +_TIME_RE = re.compile( + r"\b(?P\d{1,2})(?::(?P\d{2}))?\s*(?Pa\.?m\.?|p\.?m\.?)\b", + re.IGNORECASE, +) + +_ADDRESS_ABBR_RE = re.compile(r"(?P\b\w+\s+)(?PSt|Rd|Ave|Blvd|Ln|Dr)\.(?=\s*(?:,|\.|!|\?|$))") + +_MONTH_MAP = { + "jan": "January", + "january": "January", + "feb": "February", + "february": "February", + "mar": "March", + "march": "March", + "apr": "April", + "april": "April", + "may": "May", + "jun": "June", + "june": "June", + "jul": "July", + "july": "July", + "aug": "August", + "august": "August", + "sep": "September", + "sept": "September", + "september": "September", + "oct": "October", + "october": "October", + "nov": "November", + "november": "November", + "dec": "December", + "december": "December", +} + + +def _is_us_locale() -> bool: + for key in ("LC_ALL", "LC_TIME", "LANG"): + value = os.environ.get(key) + if value and "en_US" in value: + return True + try: + loc = locale.getlocale(locale.LC_TIME) + if loc and loc[0] and "en_US" in loc[0]: + return True + except Exception: + pass + return False + + +def _year_to_words_american(value: int, language: str) -> str: + if language.lower().startswith("en") and 2000 <= value <= 2099: + if value == 2000: + return "two thousand" + if 2001 <= value <= 2009: + tail = _DIGIT_WORDS[value % 10] + return f"two thousand {tail}" + + first_two = value // 100 + last_two = value % 100 + first_words = _int_to_words(first_two, language) or "twenty" + if last_two == 0: + return f"{first_words} hundred" + if last_two < 10: + return f"{first_words} oh {_DIGIT_WORDS[last_two]}" + last_words = _int_to_words(last_two, language) + return f"{first_words} {last_words or last_two}" + + words = _int_to_words(value, language) + return words or str(value) + + +def _normalize_dates(text: str, language: str) -> str: + us = _is_us_locale() + + def _format_iso(match: re.Match[str]) -> str: + year = int(match.group("year")) + month = int(match.group("month")) + day = int(match.group("day")) + if not (1 <= month <= 12 and 1 <= day <= 31): + return match.group(0) + month_name = ( + [ + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", + ][month - 1] + ) + ordinal = _int_to_ordinal_words(day, language) or str(day) + year_words = _year_to_words_american(year, language) + return f"{month_name} {ordinal}, {year_words}" if us else f"{ordinal} {month_name} {year_words}" + + def _format_mdy(match: re.Match[str]) -> str: + month_raw = str(match.group("month") or "").strip().lower().rstrip(".") + month_name = _MONTH_MAP.get(month_raw) + if not month_name: + return match.group(0) + day = int(match.group("day")) + year = int(match.group("year")) + ordinal = _int_to_ordinal_words(day, language) or str(day) + year_words = _year_to_words_american(year, language) + return f"{month_name} {ordinal}, {year_words}" if us else f"{ordinal} {month_name} {year_words}" + + out = _ISO_DATE_RE.sub(_format_iso, text) + out = _MDY_DATE_RE.sub(_format_mdy, out) + return out + + +def _normalize_times(text: str) -> str: + def _replace(match: re.Match[str]) -> str: + hour = match.group("hour") + minute = match.group("minute") + meridian = match.group("meridian").lower().replace(".", "") + if minute: + return f"{hour}:{minute} {meridian}" + return f"{hour} {meridian}" + + return _TIME_RE.sub(_replace, text) + + +def _normalize_address_abbreviations(text: str) -> str: + mapping = { + "st": "street", + "rd": "road", + "ave": "avenue", + "blvd": "boulevard", + "ln": "lane", + "dr": "drive", + } + + def _replace(match: re.Match[str]) -> str: + abbr = match.group("abbr") + full = mapping.get(abbr.lower()) + if not full: + return match.group(0) + return match.group("prefix") + _match_casing(abbr, full) + + return _ADDRESS_ABBR_RE.sub(_replace, text) + + +def _normalize_internet_slang(text: str) -> str: + mapping = { + "pls": "please", + "plz": "please", + } + + def _replace(match: re.Match[str]) -> str: + token = match.group(0) + replacement = mapping.get(token.lower()) + if not replacement: + return token + return _match_casing(token, replacement) + + return re.sub(r"\b(?:pls|plz)\b", _replace, text, flags=re.IGNORECASE) + _DECIMAL_NUMBER_RE = re.compile( rf"(?-?(?:\d{{1,3}}(?:,\d{{3}})+|\d+)\.(?P\d+))(?![\w{_NUMBER_RANGE_CLASS}/])" ) @@ -155,13 +327,49 @@ def _int_to_words(value: int, language: str) -> Optional[str]: def _int_to_ordinal_words(value: int, language: str) -> Optional[str]: - if num2words is None: - return None + if num2words is not None: + try: + return num2words(value, lang=language, ordinal=True) + except Exception: # pragma: no cover - unsupported locale + return None - try: - return num2words(value, lang=language, ordinal=True) - except Exception: # pragma: no cover - unsupported locale - return None + if language.lower().startswith("en"): + ordinals = { + 1: "first", + 2: "second", + 3: "third", + 4: "fourth", + 5: "fifth", + 6: "sixth", + 7: "seventh", + 8: "eighth", + 9: "ninth", + 10: "tenth", + 11: "eleventh", + 12: "twelfth", + 13: "thirteenth", + 14: "fourteenth", + 15: "fifteenth", + 16: "sixteenth", + 17: "seventeenth", + 18: "eighteenth", + 19: "nineteenth", + 20: "twentieth", + 21: "twenty-first", + 22: "twenty-second", + 23: "twenty-third", + 24: "twenty-fourth", + 25: "twenty-fifth", + 26: "twenty-sixth", + 27: "twenty-seventh", + 28: "twenty-eighth", + 29: "twenty-ninth", + 30: "thirtieth", + 31: "thirty-first", + } + return ordinals.get(int(value)) + + return None def _pluralize_fraction_word(base: str) -> str: @@ -366,6 +574,8 @@ TITLE_ABBREVIATIONS = { "dr": "doctor", "prof": "professor", "rev": "reverend", + "gen": "general", + "sgt": "sergeant", } SUFFIX_ABBREVIATIONS = { @@ -1146,7 +1356,23 @@ def classify_token(token: str, cfg: ApostropheConfig) -> Tuple[str, str]: # 1. Decades if DECADE_RE.match(token): if cfg.decades_mode == "expand": - return "decade", f"19{token[2:4]}s" + decade_digit = token[1] + decade_map = { + "0": "two thousands", + "1": "tens", + "2": "twenties", + "3": "thirties", + "4": "forties", + "5": "fifties", + "6": "sixties", + "7": "seventies", + "8": "eighties", + "9": "nineties", + } + spoken = decade_map.get(decade_digit) + if spoken: + return "decade", spoken + return "decade", token return "decade", token # 2. Leading elision @@ -1312,7 +1538,7 @@ def normalize_apostrophes(text: str, cfg: ApostropheConfig | None = None) -> Tup return normalized_text, results def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str: - if not text or not cfg.convert_numbers: + if not text: return text language = (cfg.number_lang or "en").strip() or "en" @@ -1360,6 +1586,20 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str: return words.replace(" and ", " ") return None + # 1200s are commonly spoken as "twelve hundred". + if 1200 <= value < 1300: + hundreds = value // 100 + remainder = value % 100 + prefix = _words(hundreds) + if not prefix: + return None + if remainder == 0: + return f"{prefix} hundred" + tail = _format_year_tail(remainder, allow_oh=True) + if tail is None: + return None + return f"{prefix} hundred {tail}".strip() + if value % 1000 == 0: thousands = value // 1000 thousands_words = _words(thousands) @@ -1537,6 +1777,29 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str: return f"{amount_spoken} {magnitude} {currency_name}" + # Handle $0.99 -> ninety-nine cents (avoid "zero dollars and..."). + if amount_str.startswith("0") and amount < 1.0: + dollars_part, dot, fraction = amount_str.partition(".") + if dot and fraction: + cents_str = (fraction + "00")[:2] + try: + cents_value = int(cents_str) + except ValueError: + cents_value = 0 + if cents_value > 0: + cents_words = _int_to_words(cents_value, language) or str(cents_value) + subunit = { + "$": "cent", + "€": "cent", + "£": "penny", + "¥": "yen", + }.get(symbol, "cent") + if symbol == "£": + subunit = "pence" if cents_value != 1 else "penny" + else: + subunit = (subunit + "s") if cents_value != 1 and subunit not in {"pence", "yen"} else subunit + return f"{cents_words} {subunit}".strip() + currency_map = { "$": "USD", "£": "GBP", @@ -1559,6 +1822,13 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str: def _replace_url(match: re.Match[str]) -> str: domain = match.group("domain") + + # Avoid matching dotted abbreviations/acronyms without an explicit URL prefix. + has_prefix = match.group(1) or match.group(2) + if not has_prefix: + parts = [p for p in domain.split(".") if p] + if parts and all(p.isalpha() and len(p) <= 2 for p in parts): + return match.group(0) # Avoid matching numbers like 1.05 or 12.34.56 # If the domain consists only of digits and dots, ignore it (unless it has http/www prefix) @@ -1583,24 +1853,34 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str: return match.group(1) normalized = text - - # Apply URL replacement first + + # Apply URL replacement first (independent of number conversion). normalized = _URL_RE.sub(_replace_url, normalized) - + if getattr(cfg, "remove_footnotes", False): normalized = _FOOTNOTE_RE.sub(_remove_footnote, normalized) normalized = _BRACKET_FOOTNOTE_RE.sub("", normalized) if cfg.convert_currency: normalized = _CURRENCY_RE.sub(_replace_currency, normalized) - normalized = _NUMBER_RANGE_RE.sub(lambda m: _replace_number_range(m, language), normalized) - normalized = _NUMBER_SPACE_RANGE_RE.sub(lambda m: _replace_space_separated_range(m, language), normalized) - normalized = _FRACTION_RE.sub(lambda m: _replace_fraction(m, language), normalized) - normalized = _DECIMAL_NUMBER_RE.sub(_replace_decimal, normalized) - normalized = _NUMBER_WITH_GROUP_RE.sub(_replace_grouped, normalized) - normalized = _PLAIN_NUMBER_RE.sub(_replace_plain, normalized) - normalized = _normalize_roman_numerals(normalized, language) + + if cfg.convert_numbers: + normalized = _NUMBER_RANGE_RE.sub(lambda m: _replace_number_range(m, language), normalized) + normalized = _NUMBER_SPACE_RANGE_RE.sub(lambda m: _replace_space_separated_range(m, language), normalized) + normalized = _FRACTION_RE.sub(lambda m: _replace_fraction(m, language), normalized) + normalized = _DECIMAL_NUMBER_RE.sub(_replace_decimal, normalized) + normalized = _NUMBER_WITH_GROUP_RE.sub(_replace_grouped, normalized) + normalized = _PLAIN_NUMBER_RE.sub(_replace_plain, normalized) + normalized = _normalize_roman_numerals(normalized, language) + return normalized +def _normalize_dotted_acronyms(text: str) -> str: + def _replace(match: re.Match[str]) -> str: + value = match.group(0) + compact = value.replace(".", "") + return compact + + return re.sub(r"\b(?:[A-Z]\.){1,}[A-Z]\.?(?=\W|$)", _replace, text) # ---------- Optional phoneme hint post-processing ---------- @@ -1909,21 +2189,31 @@ def normalize_for_pipeline( mode = str(runtime_settings.get("normalization_apostrophe_mode", "spacy")).lower() normalized = text + # Pre-normalization that must happen before number/url parsing. + if runtime_settings.get("normalization_numbers", True): + normalized = _normalize_dates(normalized, cfg.number_lang) + normalized = _normalize_times(normalized) + normalized = _normalize_dotted_acronyms(normalized) + if runtime_settings.get("normalization_titles", True): + normalized = _normalize_address_abbreviations(normalized) + if runtime_settings.get("normalization_internet_slang", False): + normalized = _normalize_internet_slang(normalized) + if mode == "off": - normalized = normalize_unicode_apostrophes(text) - if cfg.convert_numbers: + normalized = normalize_unicode_apostrophes(normalized) + if cfg.convert_numbers or cfg.convert_currency or getattr(cfg, "remove_footnotes", False): normalized = _normalize_grouped_numbers(normalized, cfg) normalized = _cleanup_spacing(normalized) elif mode == "llm": try: - normalized = _normalize_with_llm(text, settings=runtime_settings, config=cfg) + normalized = _normalize_with_llm(normalized, settings=runtime_settings, config=cfg) except LLMClientError: raise - if cfg.convert_numbers: + if cfg.convert_numbers or cfg.convert_currency or getattr(cfg, "remove_footnotes", False): normalized = _normalize_grouped_numbers(normalized, cfg) normalized = _cleanup_spacing(normalized) else: - normalized, _ = normalize_apostrophes(text, cfg) + normalized, _ = normalize_apostrophes(normalized, cfg) if runtime_settings.get("normalization_titles", True): normalized = expand_titles_and_suffixes(normalized) diff --git a/abogen/normalization_settings.py b/abogen/normalization_settings.py index 4a705a6..fcbe58e 100644 --- a/abogen/normalization_settings.py +++ b/abogen/normalization_settings.py @@ -38,6 +38,7 @@ _SETTINGS_DEFAULTS: Dict[str, Any] = { "normalization_terminal": True, "normalization_phoneme_hints": True, "normalization_caps_quotes": True, + "normalization_internet_slang": False, "normalization_apostrophes_contractions": True, "normalization_apostrophes_plural_possessives": True, "normalization_apostrophes_sibilant_possessives": True, diff --git a/abogen/web/debug_tts_runner.py b/abogen/web/debug_tts_runner.py index 04871e0..a2e0e70 100644 --- a/abogen/web/debug_tts_runner.py +++ b/abogen/web/debug_tts_runner.py @@ -26,6 +26,7 @@ class DebugWavArtifact: label: str filename: str code: Optional[str] = None + text: Optional[str] = None def _resolve_voice_setting(value: str) -> tuple[str, Optional[str], Optional[str]]: @@ -65,6 +66,21 @@ def _extract_cases_from_text(text: str) -> List[Tuple[str, str]]: return cases +def _spoken_id(code: str) -> str: + # Make IDs pronounceable and stable (avoid reading as a word). + out: List[str] = [] + for ch in str(code or ""): + if ch == "_": + out.append(" ") + elif ch.isalnum(): + out.append(ch) + else: + out.append(" ") + # Add spaces between alnum to encourage letter-by-letter reading. + spaced = " ".join("".join(out).split()) + return spaced + + def run_debug_tts_wavs( *, output_root: Path, @@ -96,6 +112,14 @@ def run_debug_tts_wavs( combined_text = extraction.combined_text or "\n\n".join((c.text or "") for c in extraction.chapters) cases = _extract_cases_from_text(combined_text) + # Prefer the canonical sample catalog for text (EPUB extraction may include headings). + try: + from abogen.debug_tts_samples import DEBUG_TTS_SAMPLES + + sample_text_by_code = {sample.code: sample.text for sample in DEBUG_TTS_SAMPLES} + except Exception: + sample_text_by_code = {} + expected = list(iter_expected_codes()) found_codes = {code for code, _ in cases} missing = [code for code in expected if code not in found_codes] @@ -162,11 +186,15 @@ def run_debug_tts_wavs( overall_path = run_dir / "overall.wav" overall_audio: List[np.ndarray] = [] - def synth(text: str) -> np.ndarray: - normalized = normalize_for_pipeline( - text, - config=apostrophe_config, - settings=normalization_settings, + def synth(text: str, *, apply_normalization: bool = True) -> np.ndarray: + normalized = ( + normalize_for_pipeline( + text, + config=apostrophe_config, + settings=normalization_settings, + ) + if apply_normalization + else str(text or "") ) parts: List[np.ndarray] = [] for segment in pipeline( @@ -182,19 +210,26 @@ def run_debug_tts_wavs( return np.zeros(0, dtype="float32") return np.concatenate(parts).astype("float32", copy=False) + pause_1s = np.zeros(int(1.0 * SAMPLE_RATE), dtype="float32") + between_cases = np.zeros(int(0.35 * SAMPLE_RATE), dtype="float32") + # Per sample for code, snippet in cases: + snippet = sample_text_by_code.get(code, snippet) if not snippet: continue - audio = synth(snippet) + id_audio = synth(_spoken_id(code), apply_normalization=False) + text_audio = synth(snippet, apply_normalization=True) + audio = np.concatenate([id_audio, pause_1s, text_audio]).astype("float32", copy=False) filename = f"case_{code}.wav" path = run_dir / filename # Write float32 PCM WAV. import soundfile as sf sf.write(path, audio, SAMPLE_RATE, subtype="FLOAT") - artifacts.append(DebugWavArtifact(label=f"{code}", filename=filename, code=code)) + artifacts.append(DebugWavArtifact(label=f"{code}", filename=filename, code=code, text=snippet)) overall_audio.append(audio) + overall_audio.append(between_cases) # Overall if overall_audio: @@ -204,7 +239,7 @@ def run_debug_tts_wavs( import soundfile as sf sf.write(overall_path, combined, SAMPLE_RATE, subtype="FLOAT") - artifacts.insert(0, DebugWavArtifact(label="Overall", filename="overall.wav", code=None)) + artifacts.insert(0, DebugWavArtifact(label="Overall", filename="overall.wav", code=None, text=None)) manifest = { "run_id": run_id, diff --git a/abogen/web/routes/settings.py b/abogen/web/routes/settings.py index 30e166a..25e106b 100644 --- a/abogen/web/routes/settings.py +++ b/abogen/web/routes/settings.py @@ -80,9 +80,17 @@ def update_settings() -> ResponseReturnValue: maximum=25, ) + def _extract_checkbox(name: str, default: bool) -> bool: + values = form.getlist(name) if hasattr(form, "getlist") else [] + if values: + return coerce_bool(values[-1], default) + if hasattr(form, "__contains__") and name in form: + return False + return default + # Normalization settings for key in _NORMALIZATION_BOOLEAN_KEYS: - current[key] = coerce_bool(form.get(key), False) + current[key] = _extract_checkbox(key, bool(current.get(key, True))) for key in _NORMALIZATION_STRING_KEYS: current[key] = (form.get(key) or "").strip() @@ -236,7 +244,8 @@ def download_debug_wav(run_id: str, filename: str) -> ResponseReturnValue: safe_name = (filename or "").strip() if not safe_run or not safe_name or "/" in safe_name or "\\" in safe_name: abort(404) - if not safe_name.lower().endswith(".wav") and safe_name != "manifest.json": + is_wav = safe_name.lower().endswith(".wav") + if not is_wav and safe_name != "manifest.json": abort(404) root = Path(current_app.config.get("OUTPUT_FOLDER") or get_user_output_path("web")) @@ -247,4 +256,12 @@ def download_debug_wav(run_id: str, filename: str) -> ResponseReturnValue: expected_dir = (root / "debug" / safe_run).resolve() if expected_dir not in path.parents: abort(404) - return send_file(path, as_attachment=True, download_name=path.name) + wants_download = str(request.args.get("download") or "").strip().lower() in {"1", "true", "yes"} + mimetype = "audio/wav" if is_wav else "application/json" + # Inline playback should work for WAVs; allow explicit downloads via ?download=1. + return send_file( + path, + mimetype=mimetype, + as_attachment=wants_download, + download_name=path.name, + ) diff --git a/abogen/web/routes/utils/settings.py b/abogen/web/routes/utils/settings.py index 7eef328..b5ab140 100644 --- a/abogen/web/routes/utils/settings.py +++ b/abogen/web/routes/utils/settings.py @@ -47,6 +47,9 @@ _NORMALIZATION_BOOLEAN_KEYS = { "normalization_terminal", "normalization_phoneme_hints", "normalization_caps_quotes", + "normalization_currency", + "normalization_footnotes", + "normalization_internet_slang", "normalization_apostrophes_contractions", "normalization_apostrophes_plural_possessives", "normalization_apostrophes_sibilant_possessives", @@ -58,8 +61,6 @@ _NORMALIZATION_BOOLEAN_KEYS = { "normalization_contraction_modal_would", "normalization_contraction_negation_not", "normalization_contraction_let_us", - "normalization_currency", - "normalization_footnotes", } _NORMALIZATION_STRING_KEYS = { @@ -84,6 +85,9 @@ BOOLEAN_SETTINGS = { "normalization_terminal", "normalization_phoneme_hints", "normalization_caps_quotes", + "normalization_currency", + "normalization_footnotes", + "normalization_internet_slang", "normalization_apostrophes_contractions", "normalization_apostrophes_plural_possessives", "normalization_apostrophes_sibilant_possessives", @@ -107,6 +111,7 @@ _NORMALIZATION_GROUPS = [ {"key": "normalization_numbers", "label": "Convert grouped numbers to words"}, {"key": "normalization_currency", "label": "Convert currency symbols ($10 → ten dollars)"}, {"key": "normalization_titles", "label": "Expand titles and suffixes (Dr., St., Jr., …)"}, + {"key": "normalization_internet_slang", "label": "Expand internet slang (pls → please)"}, {"key": "normalization_footnotes", "label": "Remove footnote indicators ([1], [2])"}, {"key": "normalization_terminal", "label": "Ensure sentences end with terminal punctuation"}, {"key": "normalization_caps_quotes", "label": "Convert ALL CAPS dialogue inside quotes"}, @@ -195,10 +200,13 @@ def settings_defaults() -> Dict[str, Any]: "llm_prompt": llm_env_defaults.get("llm_prompt", DEFAULT_LLM_PROMPT), "llm_context_mode": llm_env_defaults.get("llm_context_mode", "sentence"), "normalization_numbers": True, + "normalization_currency": True, + "normalization_footnotes": True, "normalization_titles": True, "normalization_terminal": True, "normalization_phoneme_hints": True, "normalization_caps_quotes": True, + "normalization_internet_slang": False, "normalization_apostrophes_contractions": True, "normalization_apostrophes_plural_possessives": True, "normalization_apostrophes_sibilant_possessives": True, diff --git a/abogen/web/templates/debug_wavs.html b/abogen/web/templates/debug_wavs.html index 69b766b..2b2e451 100644 --- a/abogen/web/templates/debug_wavs.html +++ b/abogen/web/templates/debug_wavs.html @@ -7,10 +7,7 @@

Debug WAVs

Run ID: {{ run_id }}

-
- -

Click an ID to play its WAV. “Overall” is the concatenation of all samples.

-
+

Each clip reads: ID, one second pause, then the reference text.

{% if artifacts %}
@@ -18,15 +15,15 @@
    {% for item in artifacts %}
  • - - {{ item.filename }} +
    +
    + {{ item.label }} + {% if item.text %} + — {{ item.text }} + {% endif %} +
    + +
    • {% endfor %}
@@ -37,25 +34,4 @@ Back to Settings
- - {% endblock %} diff --git a/tests/conftest.py b/tests/conftest.py index d662d3d..618ae8d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,9 +1,13 @@ import importlib import sys +import os + +import pytest + # Ensure real optional dependencies are imported before tests that install stubs -# so that available packages (like ebooklib, bs4) aren't replaced with dummy modules. -for module_name in ("ebooklib", "bs4"): +# so that available packages (like ebooklib, bs4, numpy) aren't replaced with dummy modules. +for module_name in ("ebooklib", "bs4", "numpy"): if module_name not in sys.modules: try: importlib.import_module(module_name) @@ -11,3 +15,25 @@ for module_name in ("ebooklib", "bs4"): # On environments without the optional dependency, downstream tests # will install lightweight stubs as needed. pass + + +@pytest.fixture(autouse=True, scope="session") +def _isolate_settings_dir(tmp_path_factory: pytest.TempPathFactory): + settings_dir = tmp_path_factory.mktemp("abogen-settings") + os.environ["ABOGEN_SETTINGS_DIR"] = str(settings_dir) + + try: + from abogen.utils import get_user_settings_dir + + get_user_settings_dir.cache_clear() + except Exception: + pass + + try: + from abogen.normalization_settings import clear_cached_settings + + clear_cached_settings() + except Exception: + pass + + yield diff --git a/tests/test_text_normalization.py b/tests/test_text_normalization.py index 820eb2c..eab1bdf 100644 --- a/tests/test_text_normalization.py +++ b/tests/test_text_normalization.py @@ -215,6 +215,40 @@ def test_decades_can_skip_expansion_when_disabled() -> None: assert "'90s" in normalized +def test_abbreviated_decades_expand_to_spoken_form() -> None: + normalized = _normalize_text("She loved music from the '80s.") + assert "eighties" in normalized.lower() + + +def test_currency_under_one_dollar_uses_cents() -> None: + normalized = _normalize_text("It cost $0.99.") + folded = normalized.lower().replace("-", " ") + assert "zero dollars" not in folded + assert "cents" in folded + + +def test_iso_dates_use_locale_order_and_ordinals(monkeypatch) -> None: + monkeypatch.setenv("LC_TIME", "en_US.UTF-8") + normalized = _normalize_text("The date is 2025/12/15.") + folded = normalized.lower().replace("-", " ") + assert "december" in folded + assert "fifteenth" in folded + + +def test_times_and_acronyms_do_not_say_dot() -> None: + normalized = _normalize_text("Meet at 5 p.m. near the U.S.A. border.") + folded = normalized.lower() + assert " dot " not in folded + + +def test_internet_slang_expansion_is_configurable() -> None: + normalized = _normalize_text( + "pls knock before entering.", + normalization_overrides={"normalization_internet_slang": True}, + ) + assert "please" in normalized.lower() + + @pytest.mark.skipif(not SPACY_RESOLVER_AVAILABLE, reason="spaCy model unavailable") def test_spacy_disambiguates_it_has_from_context() -> None: normalized = _normalize_text("It's been a long time.")