From 387d324ea07a44e8eaf9df816faecc87e9600798 Mon Sep 17 00:00:00 2001 From: JB Date: Tue, 11 Nov 2025 05:25:29 -0800 Subject: [PATCH] feat: Add year pronunciation handling and roman numeral normalization to text processing --- abogen/kokoro_text_normalization.py | 297 ++++++++++++++++++++++++++++ abogen/normalization_settings.py | 2 + tests/test_text_normalization.py | 42 ++++ 3 files changed, 341 insertions(+) diff --git a/abogen/kokoro_text_normalization.py b/abogen/kokoro_text_normalization.py index 9517683..1cc3ab3 100644 --- a/abogen/kokoro_text_normalization.py +++ b/abogen/kokoro_text_normalization.py @@ -49,6 +49,7 @@ class ApostropheConfig: protect_cultural_names: bool = True # Always keep O'Brien, D'Angelo, etc. convert_numbers: bool = True # Convert grouped numbers such as 12,500 to words number_lang: str = "en" # num2words language code + year_pronunciation_mode: str = "american" # off|american (extend if needed) contraction_categories: Dict[str, bool] = field(default_factory=lambda: dict(CONTRACTION_CATEGORY_DEFAULTS)) def is_contraction_enabled(self, category: str) -> bool: @@ -430,6 +431,105 @@ _ROMAN_COMPOSE_ORDER = [ _ROMAN_PREFIX_RE = re.compile(r"^(?P[IVXLCDM]+)(?P[\s\.:,;\-–—]*)", re.IGNORECASE) +_ROMAN_TOKEN_RE = re.compile(r"^[IVXLCDM]+$") +_ROMAN_CARDINAL_CONTEXTS = { + "act", + "appendix", + "article", + "battle", + "book", + "campaign", + "chapter", + "episode", + "film", + "final", + "fantasy", + "game", + "installment", + "lesson", + "level", + "mission", + "movement", + "opus", + "operation", + "page", + "part", + "phase", + "psalm", + "round", + "scene", + "season", + "section", + "series", + "song", + "super", + "bowl", + "stage", + "step", + "track", + "volume", + "war", + "world", +} +_ROMAN_NAME_TITLES = { + "baron", + "baroness", + "captain", + "cardinal", + "count", + "countess", + "duchess", + "duke", + "emperor", + "empress", + "general", + "governor", + "king", + "lord", + "lady", + "major", + "pope", + "president", + "prince", + "princess", + "queen", + "saint", + "sir", +} +_ROMAN_NAME_CONNECTORS = { + "de", + "del", + "della", + "der", + "di", + "dos", + "la", + "le", + "of", + "the", + "van", + "von", +} +_ROMAN_BREAK_TOKENS = { + ",", + ".", + "!", + "?", + ";", + ":", + "(", + ")", + "[", + "]", + "{", + "}", + "—", + "–", + "-", + "'", + '"', +} + def _roman_to_int(token: str) -> Optional[int]: if not token: @@ -463,6 +563,100 @@ def _int_to_roman(value: int) -> str: return "".join(parts) +def _is_titlecase_token(token: str) -> bool: + cleaned = token.replace("'", "").replace("-", "") + if not cleaned: + return False + if not cleaned[0].isalpha() or not cleaned[0].isupper(): + return False + tail = cleaned[1:] + return not tail or tail.islower() + + +def _token_is_cardinal_context(token: str) -> bool: + return token.lower() in _ROMAN_CARDINAL_CONTEXTS + + +def _should_render_ordinal( + tokens: Sequence[Tuple[str, int, int]], + index: int, + value: int, +) -> bool: + # Treat trailing roman numerals in name-like sequences as ordinals while + # leaving enumerated headings or series labels as cardinals. + if value <= 0: + return False + if index <= 0: + return False + + uppercase_count = 0 + title_count = 0 + j = index - 1 + while j >= 0: + token, *_ = tokens[j] + lowered = token.lower() + + if lowered in _ROMAN_CARDINAL_CONTEXTS: + return False + if lowered in _ROMAN_BREAK_TOKENS or token.isdigit(): + break + if lowered in _ROMAN_NAME_CONNECTORS: + j -= 1 + continue + if _is_titlecase_token(token): + uppercase_count += 1 + if lowered in _ROMAN_NAME_TITLES: + title_count += 1 + j -= 1 + continue + break + + if not uppercase_count: + return False + + if title_count: + return value <= 50 + + if uppercase_count >= 2: + return value <= 20 + + return False + + +def _normalize_roman_numerals(text: str, language: str) -> str: + if not text: + return text + + tokens = tokenize_with_spans(text) + if not tokens: + return text + + parts: List[str] = [] + cursor = 0 + + for index, (token, start, end) in enumerate(tokens): + parts.append(text[cursor:start]) + replacement = token + + if len(token) >= 2 and token.isupper() and _ROMAN_TOKEN_RE.match(token): + numeric_value = _roman_to_int(token) + if numeric_value is not None: + if _should_render_ordinal(tokens, index, numeric_value): + ordinal = _int_to_ordinal_words(numeric_value, language) + if ordinal: + replacement = f"the {ordinal}" + else: + words = _int_to_words(numeric_value, language) + if words: + replacement = words + + parts.append(replacement) + cursor = end + + parts.append(text[cursor:]) + return "".join(parts) + + def normalize_roman_numeral_titles( titles: Sequence[str], *, @@ -890,6 +1084,105 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str: language = (cfg.number_lang or "en").strip() or "en" + def _year_mode() -> str: + mode = (cfg.year_pronunciation_mode or "").strip().lower() + if mode in {"", "none", "off", "disabled"}: + return "off" + if mode not in {"american"}: + return "off" + return mode + + year_mode = _year_mode() + + def _format_year_tail(value: int, *, allow_oh: bool = True) -> Optional[str]: + if value == 0: + return "" + if value < 10: + if allow_oh: + return f"oh {_DIGIT_WORDS[value]}" + return _DIGIT_WORDS[value] + words = _int_to_words(value, language) + if not words: + return None + return words + + def _format_year_like(token: str, value: int) -> Optional[str]: + if year_mode == "off" or num2words is None: + return None + if len(token) != 4 or not token.isdigit(): + return None + if value < 1000 or value > 9999: + return None + style = year_mode + + def _words(value_to_convert: int) -> Optional[str]: + words = _int_to_words(value_to_convert, language) + return words + + if style == "american": + if value % 1000 == 0: + thousands = value // 1000 + thousands_words = _words(thousands) + if thousands_words: + return f"{thousands_words} thousand" + return None + + first_two = value // 100 + last_two = value % 100 + + # Special handling to match common American pronunciations. + if first_two == 20: + if last_two == 0: + return "two thousand" + if last_two < 10: + tail_word = _words(last_two) + if tail_word: + return f"two thousand {tail_word}" + return None + prefix = _words(first_two) + tail = _words(last_two) + if prefix and tail: + return f"{prefix} {tail}" + return prefix + + prefix = _words(first_two) + if not prefix: + return None + + if first_two == 10: + if last_two == 0: + return "one thousand" + tail = _format_year_tail(last_two) + if tail: + return f"{prefix} {tail}" + return prefix + + if first_two <= 12: + if last_two == 0: + return f"{prefix} hundred" + tail = _format_year_tail(last_two) + if tail: + return f"{prefix} hundred {tail}" + return f"{prefix} hundred" + + if first_two <= 19: + if last_two == 0: + return f"{prefix} hundred" + tail = _format_year_tail(last_two) + if tail: + return f"{prefix} {tail}" + return prefix + + if last_two == 0: + return f"{prefix} hundred" + + tail = _format_year_tail(last_two) + if tail: + return f"{prefix} {tail}" + return prefix + + return None + def _replace_grouped(match: re.Match[str]) -> str: token = match.group(1) value = _coerce_int_token(token) @@ -927,6 +1220,9 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str: value = _coerce_int_token(token) if value is None: return token + year_like = _format_year_like(token, value) + if year_like: + return year_like if num2words is None: return str(value) words = _int_to_words(value, language) @@ -985,6 +1281,7 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str: normalized = _DECIMAL_NUMBER_RE.sub(_replace_decimal, normalized) normalized = _NUMBER_WITH_GROUP_RE.sub(_replace_grouped, normalized) normalized = _PLAIN_NUMBER_RE.sub(_replace_plain, normalized) + normalized = _normalize_roman_numerals(normalized, language) return normalized # ---------- Optional phoneme hint post-processing ---------- diff --git a/abogen/normalization_settings.py b/abogen/normalization_settings.py index 4a2fb3c..e9392c7 100644 --- a/abogen/normalization_settings.py +++ b/abogen/normalization_settings.py @@ -31,6 +31,7 @@ _SETTINGS_DEFAULTS: Dict[str, Any] = { "llm_prompt": DEFAULT_LLM_PROMPT, "llm_context_mode": "sentence", "normalization_numbers": True, + "normalization_numbers_year_style": "american", "normalization_titles": True, "normalization_terminal": True, "normalization_phoneme_hints": True, @@ -170,6 +171,7 @@ def build_apostrophe_config( ) -> ApostropheConfig: config = replace(base or ApostropheConfig()) config.convert_numbers = bool(settings.get("normalization_numbers", True)) + config.year_pronunciation_mode = str(settings.get("normalization_numbers_year_style", "american") or "").strip().lower() config.add_phoneme_hints = bool(settings.get("normalization_phoneme_hints", True)) config.contraction_mode = "expand" if settings.get("normalization_apostrophes_contractions", True) else "keep" config.plural_possessive_mode = ( diff --git a/tests/test_text_normalization.py b/tests/test_text_normalization.py index c4705e7..93bd772 100644 --- a/tests/test_text_normalization.py +++ b/tests/test_text_normalization.py @@ -101,6 +101,48 @@ def test_space_separated_numbers_become_ranges() -> None: assert "pages twelve to fourteen" in normalized.lower() +def test_year_like_numbers_use_common_pronunciation() -> None: + normalized = _normalize_for_pipeline("In 1924 the journey began") + folded = normalized.lower().replace("-", " ") + assert "nineteen twenty four" in folded + + +def test_early_century_years_use_hundred_format() -> None: + normalized = _normalize_for_pipeline("In 1204 the city fell") + assert "twelve hundred" in normalized.lower() + assert "oh four" in normalized.lower() + + +def test_roman_numerals_in_titles_are_converted() -> None: + normalized = _normalize_for_pipeline("Chapter IV begins now") + assert "chapter four" in normalized.lower() + + +def test_roman_numeral_suffixes_use_ordinals() -> None: + normalized = _normalize_for_pipeline("Bob Smith II arrived late") + assert "bob smith the second" in normalized.lower() + + +def test_recent_years_split_twenty_style() -> None: + normalized = _normalize_for_pipeline("In 2025 we planned ahead") + folded = normalized.lower().replace("-", " ") + assert "twenty twenty five" in folded + + +def test_two_thousands_use_two_thousand_prefix() -> None: + normalized = _normalize_for_pipeline("In 2005 we celebrated") + assert "two thousand five" in normalized.lower() + + +def test_year_style_can_be_disabled() -> None: + normalized = _normalize_for_pipeline( + "In 2025 we planned ahead", + normalization_overrides={"normalization_numbers_year_style": "off"}, + ) + folded = normalized.lower().replace("-", " ") + assert "twenty twenty five" not in folded + + def test_contractions_can_be_kept_when_override_disabled() -> None: normalized = _normalize_for_pipeline( "It's a good day.",