diff --git a/abogen/entity_analysis.py b/abogen/entity_analysis.py index 0681d3a..34ef2d2 100644 --- a/abogen/entity_analysis.py +++ b/abogen/entity_analysis.py @@ -442,3 +442,12 @@ def merge_override(summary: Mapping[str, Any], overrides: Mapping[str, Mapping[s def normalize_token(token: str) -> str: return _token_key(_normalize_label(token)) + + +def normalize_manual_override_token(token: str) -> str: + if not token: + return "" + stripped = token.strip().strip("\"'`“”’") + if not stripped: + return "" + return _MULTI_SPACE_PATTERN.sub(" ", stripped.lower()).strip() diff --git a/abogen/kokoro_text_normalization.py b/abogen/kokoro_text_normalization.py index a3138c7..47cfe11 100644 --- a/abogen/kokoro_text_normalization.py +++ b/abogen/kokoro_text_normalization.py @@ -59,6 +59,7 @@ class ApostropheConfig: protect_cultural_names: bool = True # Always keep O'Brien, D'Angelo, etc. convert_numbers: bool = True # Convert grouped numbers such as 12,500 to words convert_currency: bool = True # Convert currency symbols to words + remove_footnotes: bool = True # Remove footnote indicators number_lang: str = "en" # num2words language code year_pronunciation_mode: str = "american" # off|american (extend if needed) contraction_categories: Dict[str, bool] = field(default_factory=lambda: dict(CONTRACTION_CATEGORY_DEFAULTS)) @@ -123,6 +124,10 @@ _CURRENCY_RE = re.compile( r"(?P[$£€¥])\s*(?P\d{1,3}(?:,\d{3})*(?:\.\d{1,2})?)(?!\d)" ) +_URL_RE = re.compile(r"(https?://)?(www\.)?(?P[a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)+)(/[^\s]*)?") +_FOOTNOTE_RE = re.compile(r"([a-zA-Z]+)(\d+)") +_BRACKET_FOOTNOTE_RE = re.compile(r"\[\d+\]") + _DECIMAL_NUMBER_RE = re.compile( rf"(?-?(?:\d{{1,3}}(?:,\d{{3}})+|\d+)\.(?P\d+))(?![\w{_NUMBER_RANGE_CLASS}/])" ) @@ -486,6 +491,7 @@ _ROMAN_CARDINAL_CONTEXTS = { "volume", "war", "world", + "world war", } _ROMAN_NAME_TITLES = { "baron", @@ -702,14 +708,20 @@ def _normalize_roman_numerals(text: str, language: str) -> str: replacement = f"{context_word} {words}" else: candidate = token.upper() - if len(token) >= 2 and _ROMAN_TOKEN_RE.match(candidate): + is_roman = _ROMAN_TOKEN_RE.match(candidate) + if is_roman: numeric_value = _roman_to_int(candidate) if numeric_value is not None: convert = False - if token.isupper(): - convert = True - elif numeric_value <= 200 and _has_cardinal_leading_context(tokens, index): - convert = True + if len(token) >= 2: + if token.isupper(): + convert = True + elif numeric_value <= 200 and _has_cardinal_leading_context(tokens, index): + convert = True + elif len(token) == 1: + # Only convert single letters if context is strong + if _has_cardinal_leading_context(tokens, index): + convert = True if convert: if _should_render_ordinal(tokens, index, numeric_value): @@ -1340,6 +1352,13 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str: return words if style == "american": + # Special handling for 2000-2009 to use "two thousand X" + if 2000 <= value <= 2009: + words = _words(value) + if words: + return words.replace(" and ", " ") + return None + if value % 1000 == 0: thousands = value // 1000 thousands_words = _words(thousands) @@ -1349,52 +1368,17 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str: first_two = value // 100 last_two = value % 100 - - # Special handling to match common American pronunciations. - if first_two == 20: - if last_two == 0: - return "two thousand" - if last_two < 10: - tail_word = _words(last_two) - if tail_word: - return f"two thousand {tail_word}" - return None - prefix = _words(first_two) - tail = _words(last_two) - if prefix and tail: - return f"{prefix} {tail}" - return prefix - + prefix = _words(first_two) if not prefix: return None - if first_two == 10: - if last_two == 0: - return "one thousand" - tail = _format_year_tail(last_two) - if tail: - return f"{prefix} {tail}" - return prefix - - if first_two <= 12: - if last_two == 0: - return f"{prefix} hundred" - tail = _format_year_tail(last_two) - if tail: - return f"{prefix} hundred {tail}" - return f"{prefix} hundred" - - if first_two <= 19: - if last_two == 0: - return f"{prefix} hundred" - tail = _format_year_tail(last_two) - if tail: - return f"{prefix} {tail}" - return prefix - if last_two == 0: return f"{prefix} hundred" + + if last_two < 10: + # Use "hundred oh X" format (e.g. "twelve hundred oh four") + return f"{prefix} hundred oh {_DIGIT_WORDS[last_two]}" tail = _format_year_tail(last_two) if tail: @@ -1500,6 +1484,7 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str: symbol = match.group("symbol") amount_str = match.group("amount").replace(",", "") + # print(f"DEBUG: symbol={symbol} amount_str={amount_str}") try: amount = float(amount_str) except ValueError: @@ -1514,12 +1499,51 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str: currency_code = currency_map.get(symbol, "USD") try: + # Always use float to avoid num2words treating int as cents (if that's what it does) + # or to ensure consistent behavior. words = num2words(amount, to="currency", currency=currency_code, lang=language) + + # Remove "zero cents" if present + # Patterns: ", zero cents", " and zero cents" + words = words.replace(", zero cents", "").replace(" and zero cents", "") return words except Exception: return match.group(0) + def _replace_url(match: re.Match[str]) -> str: + domain = match.group("domain") + + # Avoid matching numbers like 1.05 or 12.34.56 + # If the domain consists only of digits and dots, ignore it (unless it has http/www prefix) + has_prefix = match.group(1) or match.group(2) + if not has_prefix and all(c.isdigit() or c == '.' or c == '-' for c in domain): + # Check if it really looks like a number (e.g. 1.05) + # If it has multiple dots like 1.2.3.4 it might be an IP, which we might want to speak as dot? + # But 1.05 is definitely a number. + # Let's be safe: if it looks like a float, skip. + try: + float(domain) + return match.group(0) + except ValueError: + pass + + if domain.startswith("www."): + domain = domain[4:] + spoken = domain.replace(".", " dot ") + return spoken + + def _remove_footnote(match: re.Match[str]) -> str: + return match.group(1) + normalized = text + + # Apply URL replacement first + normalized = _URL_RE.sub(_replace_url, normalized) + + if getattr(cfg, "remove_footnotes", False): + normalized = _FOOTNOTE_RE.sub(_remove_footnote, normalized) + normalized = _BRACKET_FOOTNOTE_RE.sub("", normalized) + if cfg.convert_currency: normalized = _CURRENCY_RE.sub(_replace_currency, normalized) normalized = _NUMBER_RANGE_RE.sub(lambda m: _replace_number_range(m, language), normalized) diff --git a/abogen/normalization_settings.py b/abogen/normalization_settings.py index a2a2b75..22a2c09 100644 --- a/abogen/normalization_settings.py +++ b/abogen/normalization_settings.py @@ -33,6 +33,7 @@ _SETTINGS_DEFAULTS: Dict[str, Any] = { "normalization_numbers": True, "normalization_numbers_year_style": "american", "normalization_currency": True, + "normalization_footnotes": True, "normalization_titles": True, "normalization_terminal": True, "normalization_phoneme_hints": True, diff --git a/abogen/spacy_contraction_resolver.py b/abogen/spacy_contraction_resolver.py index 3b9e679..d85ca7e 100644 --- a/abogen/spacy_contraction_resolver.py +++ b/abogen/spacy_contraction_resolver.py @@ -134,6 +134,11 @@ def _resolve_apostrophe_s(token: Token) -> Optional[ContractionResolution]: if prev_lower == "let": return _resolution(prev, token, "us", "contraction_let_us", "us") + # Special check for 's been -> has been, overriding lemma + next_content = _next_content_token(token) + if next_content and next_content.text.lower() == "been": + return _resolution(prev, token, "has", "contraction_aux_have", "have") + lemma = token.lemma_.lower() if not lemma: lemma = "be" if _favors_be(token) else "have" if _favors_have(token) else "be" diff --git a/abogen/web/routes/utils/entity.py b/abogen/web/routes/utils/entity.py index df92492..7b81373 100644 --- a/abogen/web/routes/utils/entity.py +++ b/abogen/web/routes/utils/entity.py @@ -7,6 +7,7 @@ from abogen.entity_analysis import ( extract_entities, merge_override, normalize_token as normalize_entity_token, + normalize_manual_override_token, search_tokens as search_entity_tokens, ) from abogen.pronunciation_store import ( @@ -84,7 +85,7 @@ def collect_pronunciation_overrides(pending: PendingJob) -> List[Dict[str, Any]] pronunciation_value = str(manual_entry.get("pronunciation") or "").strip() if not token_value or not pronunciation_value: continue - normalized = manual_entry.get("normalized") or normalize_entity_token(token_value) + normalized = manual_entry.get("normalized") or normalize_manual_override_token(token_value) if not normalized: continue collected[normalized] = { @@ -196,7 +197,7 @@ def upsert_manual_override(pending: PendingJob, payload: Mapping[str, Any]) -> D voice_value = str(payload.get("voice") or "").strip() notes_value = str(payload.get("notes") or "").strip() context_value = str(payload.get("context") or "").strip() - normalized = payload.get("normalized") or normalize_entity_token(token_value) + normalized = payload.get("normalized") or normalize_manual_override_token(token_value) if not normalized: raise ValueError("Token is required") diff --git a/abogen/web/templates/base.html b/abogen/web/templates/base.html index b04c60c..6235880 100644 --- a/abogen/web/templates/base.html +++ b/abogen/web/templates/base.html @@ -18,8 +18,8 @@