diff --git a/abogen/kokoro_text_normalization.py b/abogen/kokoro_text_normalization.py index cb37940..5adc2c2 100644 --- a/abogen/kokoro_text_normalization.py +++ b/abogen/kokoro_text_normalization.py @@ -14,6 +14,8 @@ except Exception: # pragma: no cover - graceful degradation if TYPE_CHECKING: # pragma: no cover - type checking only from abogen.llm_client import LLMCompletion +from abogen.spacy_contraction_resolver import resolve_ambiguous_contractions + # ---------- Configuration Dataclass ---------- @dataclass @@ -27,7 +29,7 @@ class ApostropheConfig: acronym_possessive_mode: str = "keep" # keep|collapse_add_s decades_mode: str = "expand" # keep|expand leading_elision_mode: str = "expand" # keep|expand - ambiguous_past_modal_mode: str = "keep" # keep|expand_prefer_would|expand_prefer_had + ambiguous_past_modal_mode: str = "contextual" # keep|expand_prefer_would|expand_prefer_had|contextual add_phoneme_hints: bool = True # Whether to emit markers like ‹IZ› fantasy_marker: str = "‹FAP›" # Marker inserted if fantasy_mode == mark sibilant_iz_marker: str = "‹IZ›" # Marker for /ɪz/ insertion @@ -41,15 +43,6 @@ class ApostropheConfig: # Common contraction expansions (straightforward unambiguous) CONTRACTIONS_EXACT = { - "it's": "it is", - "that's": "that is", - "what's": "what is", - "where's": "where is", - "who's": "who is", - "when's": "when is", - "how's": "how is", - "there's": "there is", - "here's": "here is", "let's": "let us", "i'm": "i am", "you're": "you are", @@ -65,12 +58,6 @@ CONTRACTIONS_EXACT = { "she'll": "she will", "we'll": "we will", "they'll": "they will", - "i'd": "i would", # ambiguous (had/would), treat default - "you'd": "you would", - "he'd": "he would", - "she'd": "she would", - "we'd": "we would", - "they'd": "they would", "can't": "can not", # or "cannot" "won't": "will not", "don't": "do not", @@ -238,6 +225,16 @@ def _replace_fraction(match: re.Match[str], language: str) -> str: AMBIGUOUS_D_BASES = {"i","you","he","she","we","they"} AMBIGUOUS_S_BASES = {"it","that","what","where","who","when","how","there","here"} + +def _is_ambiguous_d(token: str) -> bool: + low = token.lower() + return low.endswith("'d") and low[:-2] in AMBIGUOUS_D_BASES + + +def _is_ambiguous_s(token: str) -> bool: + low = token.lower() + return low.endswith("'s") and low[:-2] in AMBIGUOUS_S_BASES + # Irregular possessives that are not formed by simple + 's logic IRREGULAR_POSSESSIVES = { "children's": "children's", @@ -321,6 +318,10 @@ def tokenize(text: str) -> List[str]: return WORD_TOKEN_RE.findall(text) +def tokenize_with_spans(text: str) -> List[Tuple[str, int, int]]: + return [(match.group(0), match.start(), match.end()) for match in WORD_TOKEN_RE.finditer(text)] + + def _cleanup_spacing(text: str) -> str: if not text: return text @@ -571,40 +572,40 @@ def classify_token(token: str, cfg: ApostropheConfig) -> Tuple[str, str]: return "leading_elision", LEADING_ELISION[low] return "leading_elision", token - # 3. Exact contraction + # 3. Ambiguous 'd contractions + if _is_ambiguous_d(token): + base = low[:-2] + mode = cfg.ambiguous_past_modal_mode + if cfg.contraction_mode == "collapse": + return "ambiguous_contraction_d", base + "d" + if cfg.contraction_mode == "expand": + if mode == "expand_prefer_would": + return "ambiguous_contraction_d", base + " would" + if mode == "expand_prefer_had": + return "ambiguous_contraction_d", base + " had" + if mode == "contextual": + return "ambiguous_contraction_d", base + " would" + return "ambiguous_contraction_d", token + + # 4. Ambiguous 's contractions + if _is_ambiguous_s(token): + base = low[:-2] + if cfg.contraction_mode == "expand": + return "ambiguous_contraction_s", base + " is" + if cfg.contraction_mode == "collapse": + return "ambiguous_contraction_s", base + "s" + return "ambiguous_contraction_s", token + + # 5. Exact contraction if low in CONTRACTIONS_EXACT: if cfg.contraction_mode == "expand": return "contraction", CONTRACTIONS_EXACT[low] elif cfg.contraction_mode == "collapse": - # collapse: remove apostrophe only (it's -> its) + # collapse: remove apostrophe only (he's -> hes) return "contraction", low.replace("'", "") else: return "contraction", token - # 4. Ambiguous 'd - if low.endswith("'d"): - base = low[:-2] - if base in AMBIGUOUS_D_BASES: - if cfg.ambiguous_past_modal_mode == "expand_prefer_would": - return "ambiguous_contraction_d", base + " would" - elif cfg.ambiguous_past_modal_mode == "expand_prefer_had": - return "ambiguous_contraction_d", base + " had" - elif cfg.contraction_mode == "collapse": - return "ambiguous_contraction_d", base + "d" - return "ambiguous_contraction_d", token - - # 5. Ambiguous 's - if low.endswith("'s"): - base = low[:-2] - if base in AMBIGUOUS_S_BASES: - # treat as contraction 'is' under chosen mode - if cfg.contraction_mode == "expand": - return "ambiguous_contraction_s", base + " is" - elif cfg.contraction_mode == "collapse": - return "ambiguous_contraction_s", base + "s" - else: - return "ambiguous_contraction_s", token - # 6. Irregular possessives (keep or expand logic) if low in IRREGULAR_POSSESSIVES: if cfg.irregular_possessive_mode == "keep": @@ -684,13 +685,38 @@ def normalize_apostrophes(text: str, cfg: ApostropheConfig | None = None) -> Tup text = normalize_unicode_apostrophes(text) text = _normalize_grouped_numbers(text, cfg) - tokens = tokenize(text) + token_entries = tokenize_with_spans(text) - results = [] + use_contextual_s = cfg.contraction_mode == "expand" + use_contextual_d = cfg.contraction_mode == "expand" and cfg.ambiguous_past_modal_mode == "contextual" + + need_contextual = False + if (use_contextual_s or use_contextual_d) and token_entries: + for token_value, _, _ in token_entries: + if use_contextual_s and _is_ambiguous_s(token_value): + need_contextual = True + break + if use_contextual_d and _is_ambiguous_d(token_value): + need_contextual = True + break + + contextual_resolutions = resolve_ambiguous_contractions(text) if need_contextual else {} + + results: List[Tuple[str, str, str]] = [] normalized_tokens: List[str] = [] - for tok in tokens: + for tok, start, end in token_entries: category, norm = classify_token(tok, cfg) + + resolution = contextual_resolutions.get((start, end)) if contextual_resolutions else None + if resolution is not None: + if resolution.category == "ambiguous_contraction_s" and use_contextual_s: + category = resolution.category + norm = resolution.expansion + elif resolution.category == "ambiguous_contraction_d" and use_contextual_d: + category = resolution.category + norm = resolution.expansion + results.append((tok, category, norm)) normalized_tokens.append(norm) diff --git a/abogen/normalization_settings.py b/abogen/normalization_settings.py index 5524100..4d6941b 100644 --- a/abogen/normalization_settings.py +++ b/abogen/normalization_settings.py @@ -167,6 +167,7 @@ def build_apostrophe_config( config.leading_elision_mode = ( "expand" if settings.get("normalization_apostrophes_leading_elisions", True) else "keep" ) + config.ambiguous_past_modal_mode = "contextual" if config.contraction_mode == "expand" else "keep" return config diff --git a/abogen/spacy_contraction_resolver.py b/abogen/spacy_contraction_resolver.py new file mode 100644 index 0000000..00442cb --- /dev/null +++ b/abogen/spacy_contraction_resolver.py @@ -0,0 +1,231 @@ +from __future__ import annotations + +import os +import logging +from dataclasses import dataclass +from functools import lru_cache +from typing import Any, Dict, Optional, Tuple + +try: # pragma: no cover - optional dependency + import spacy +except Exception: # pragma: no cover - spaCy unavailable at runtime + spacy = None + +# Lazy spaCy type hints to avoid a hard dependency at import time. +Language = Any # type: ignore[assignment] +Token = Any # type: ignore[assignment] + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class ContractionResolution: + start: int + end: int + surface: str + expansion: str + category: str + lemma: str + + @property + def span(self) -> Tuple[int, int]: + return self.start, self.end + + +_DEFAULT_MODEL = os.environ.get("ABOGEN_SPACY_MODEL", "en_core_web_sm") + + +@lru_cache(maxsize=1) +def _load_spacy_model(model: str = _DEFAULT_MODEL) -> Optional[Language]: + if spacy is None: + logger.debug("spaCy is not installed; skipping contraction disambiguation") + return None + + try: + nlp = spacy.load(model) + except Exception as exc: # pragma: no cover - depends on environment + logger.warning("Failed to load spaCy model '%s': %s", model, exc) + return None + return nlp + + +def resolve_ambiguous_contractions(text: str, *, model: Optional[str] = None) -> Dict[Tuple[int, int], ContractionResolution]: + """Use spaCy to disambiguate ambiguous contractions in *text*. + + Returns a mapping from (start, end) spans to their resolved expansion. + Only ambiguous `'s` and `'d` contractions are considered. + """ + if not text: + return {} + + nlp = _load_spacy_model(model or _DEFAULT_MODEL) + if nlp is None: + return {} + + doc = nlp(text) + resolutions: Dict[Tuple[int, int], ContractionResolution] = {} + for token in doc: + if token.text == "'s": + resolution = _resolve_apostrophe_s(token) + elif token.text == "'d": + resolution = _resolve_apostrophe_d(token) + else: + resolution = None + + if resolution is None: + continue + + if resolution.span not in resolutions: + resolutions[resolution.span] = resolution + return resolutions + + +def _resolution(prev: Token, token: Token, expansion_word: str, category: str, lemma_hint: str) -> Optional[ContractionResolution]: + if token is None or prev is None: + return None + + if prev.idx + len(prev.text) != token.idx: + # Not a contiguous contraction (whitespace or punctuation in between) + return None + + surface_start = prev.idx + surface_end = token.idx + len(token.text) + surface_text = token.doc.text[surface_start:surface_end] + + expansion = _assemble_expansion(prev.text, surface_text, expansion_word) + return ContractionResolution( + start=surface_start, + end=surface_end, + surface=surface_text, + expansion=expansion, + category=category, + lemma=lemma_hint, + ) + + +def _assemble_expansion(base_text: str, surface_text: str, expansion_word: str) -> str: + """Combine *base_text* with *expansion_word*, preserving coarse casing.""" + if not expansion_word: + return base_text + + if surface_text.isupper() and expansion_word.isalpha(): + adjusted = expansion_word.upper() + elif len(surface_text) > 2 and surface_text[:-2].istitle() and expansion_word: + # Surface like "It's" -> keep appended word lowercase + adjusted = expansion_word.lower() + else: + adjusted = expansion_word + + return f"{base_text} {adjusted}".strip() + + +def _resolve_apostrophe_s(token: Token) -> Optional[ContractionResolution]: + prev = token.nbor(-1) if token.i > 0 else None + if prev is None: + return None + + # Possessive marker e.g., dog's + if token.tag_ == "POS" or token.lemma_ == "'s": + return None + + prev_lower = prev.lemma_.lower() + surface = token.doc.text[prev.idx : token.idx + len(token.text)] + + if prev_lower == "let": + return _resolution(prev, token, "us", "contraction", "us") + + lemma = token.lemma_.lower() + if not lemma: + lemma = "be" if _favors_be(token) else "have" if _favors_have(token) else "be" + + if lemma == "be": + return _resolution(prev, token, "is", "ambiguous_contraction_s", "be") + if lemma == "have": + return _resolution(prev, token, "has", "ambiguous_contraction_s", "have") + + if _favors_have(token): + return _resolution(prev, token, "has", "ambiguous_contraction_s", "have") + + if _favors_be(token): + return _resolution(prev, token, "is", "ambiguous_contraction_s", "be") + + # Default to copula expansion. + return _resolution(prev, token, "is", "ambiguous_contraction_s", lemma or "be") + + +def _resolve_apostrophe_d(token: Token) -> Optional[ContractionResolution]: + prev = token.nbor(-1) if token.i > 0 else None + if prev is None: + return None + + if token.morph.get("VerbForm") == ["Part"]: + # spaCy sometimes tags possessives oddly; guard anyway + return None + + lemma = token.lemma_.lower() + tense = set(token.morph.get("Tense")) + + if "Past" in tense and lemma in {"have", "had"}: + return _resolution(prev, token, "had", "ambiguous_contraction_d", "have") + + if token.tag_ == "MD" or lemma in {"will", "would", "shall"}: + return _resolution(prev, token, "would", "ambiguous_contraction_d", lemma or "will") + + head = token.head if token.head is not None else None + if head is not None and head.i > token.i: + head_tag = head.tag_ + head_lemma = head.lemma_.lower() + if head_tag in {"VBN", "VBD"} or head_lemma in {"gone", "been", "had"}: + return _resolution(prev, token, "had", "ambiguous_contraction_d", "have") + if head_lemma == "better": + return _resolution(prev, token, "had", "ambiguous_contraction_d", "have") + if head_tag == "VB" or head.pos_ in {"VERB", "AUX"}: + return _resolution(prev, token, "would", "ambiguous_contraction_d", lemma or "will") + + next_content = _next_content_token(token) + if next_content is not None: + next_tag = next_content.tag_ + if next_tag in {"VBN", "VBD"} or next_content.lemma_.lower() in {"been", "gone", "had"}: + return _resolution(prev, token, "had", "ambiguous_contraction_d", "have") + if next_content.lemma_.lower() == "better": + return _resolution(prev, token, "had", "ambiguous_contraction_d", "have") + if next_tag == "VB": + return _resolution(prev, token, "would", "ambiguous_contraction_d", lemma or "will") + + # Fallback: if lemma hints at perfect aspect use "had", otherwise "would". + if lemma in {"have", "had"}: + return _resolution(prev, token, "had", "ambiguous_contraction_d", lemma) + return _resolution(prev, token, "would", "ambiguous_contraction_d", lemma or "will") + + +def _next_content_token(token: Token) -> Optional[Token]: + doc = token.doc + for candidate in doc[token.i + 1 :]: + if candidate.is_space: + continue + if candidate.is_punct and candidate.text not in {"-"}: + break + if candidate.text in {"'", ""}: + continue + return candidate + return None + + +def _favors_have(token: Token) -> bool: + next_content = _next_content_token(token) + if next_content is None: + return False + if next_content.tag_ in {"VBN"}: + return True + if next_content.lemma_.lower() in {"been", "gone", "had"}: + return True + return False + + +def _favors_be(token: Token) -> bool: + next_content = _next_content_token(token) + if next_content is None: + return True + if next_content.tag_ in {"VBG", "JJ", "RB", "DT", "IN"}: + return True + return False \ No newline at end of file diff --git a/tests/test_text_normalization.py b/tests/test_text_normalization.py index 76ac32e..1864908 100644 --- a/tests/test_text_normalization.py +++ b/tests/test_text_normalization.py @@ -1,7 +1,13 @@ from __future__ import annotations +import pytest + from abogen.kokoro_text_normalization import normalize_roman_numeral_titles from abogen.web.conversion_runner import _normalize_for_pipeline +from abogen.spacy_contraction_resolver import resolve_ambiguous_contractions + + +SPACY_RESOLVER_AVAILABLE = bool(resolve_ambiguous_contractions("It's been a long time.")) def test_title_abbreviations_are_expanded(): @@ -103,3 +109,27 @@ def test_decades_can_skip_expansion_when_disabled() -> None: normalization_overrides={"normalization_apostrophes_decades": False}, ) assert "'90s" in normalized + + +@pytest.mark.skipif(not SPACY_RESOLVER_AVAILABLE, reason="spaCy model unavailable") +def test_spacy_disambiguates_it_has_from_context() -> None: + normalized = _normalize_for_pipeline("It's been a long time.") + assert "It has been a long time." == normalized + + +@pytest.mark.skipif(not SPACY_RESOLVER_AVAILABLE, reason="spaCy model unavailable") +def test_spacy_disambiguates_it_is_from_context() -> None: + normalized = _normalize_for_pipeline("It's cold outside.") + assert "It is cold outside." == normalized + + +@pytest.mark.skipif(not SPACY_RESOLVER_AVAILABLE, reason="spaCy model unavailable") +def test_spacy_disambiguates_she_had() -> None: + normalized = _normalize_for_pipeline("She'd left before dawn.") + assert "She had left before dawn." == normalized + + +@pytest.mark.skipif(not SPACY_RESOLVER_AVAILABLE, reason="spaCy model unavailable") +def test_spacy_disambiguates_she_would() -> None: + normalized = _normalize_for_pipeline("She'd go if invited.") + assert "She would go if invited." == normalized