feat: Implement contraction resolution using spaCy for ambiguous contractions

abogen/kokoro_text_normalization.py

@@ -14,6 +14,8 @@ except Exception:  # pragma: no cover - graceful degradation
 if TYPE_CHECKING:  # pragma: no cover - type checking only
     from abogen.llm_client import LLMCompletion
 
+from abogen.spacy_contraction_resolver import resolve_ambiguous_contractions
+
 # ---------- Configuration Dataclass ----------
 
 @dataclass
@@ -27,7 +29,7 @@ class ApostropheConfig:
     acronym_possessive_mode: str = "keep"         # keep|collapse_add_s
     decades_mode: str = "expand"                  # keep|expand
     leading_elision_mode: str = "expand"          # keep|expand
-    ambiguous_past_modal_mode: str = "keep"       # keep|expand_prefer_would|expand_prefer_had
+    ambiguous_past_modal_mode: str = "contextual" # keep|expand_prefer_would|expand_prefer_had|contextual
     add_phoneme_hints: bool = True                # Whether to emit markers like ‹IZ›
     fantasy_marker: str = "‹FAP›"                 # Marker inserted if fantasy_mode == mark
     sibilant_iz_marker: str = "‹IZ›"              # Marker for /ɪz/ insertion
@@ -41,15 +43,6 @@ class ApostropheConfig:
 
 # Common contraction expansions (straightforward unambiguous)
 CONTRACTIONS_EXACT = {
-    "it's": "it is",
-    "that's": "that is",
-    "what's": "what is",
-    "where's": "where is",
-    "who's": "who is",
-    "when's": "when is",
-    "how's": "how is",
-    "there's": "there is",
-    "here's": "here is",
     "let's": "let us",
     "i'm": "i am",
     "you're": "you are",
@@ -65,12 +58,6 @@ CONTRACTIONS_EXACT = {
     "she'll": "she will",
     "we'll": "we will",
     "they'll": "they will",
-    "i'd": "i would",     # ambiguous (had/would), treat default
-    "you'd": "you would",
-    "he'd": "he would",
-    "she'd": "she would",
-    "we'd": "we would",
-    "they'd": "they would",
     "can't": "can not",   # or "cannot"
     "won't": "will not",
     "don't": "do not",
@@ -238,6 +225,16 @@ def _replace_fraction(match: re.Match[str], language: str) -> str:
 AMBIGUOUS_D_BASES = {"i","you","he","she","we","they"}
 AMBIGUOUS_S_BASES = {"it","that","what","where","who","when","how","there","here"}
 
+
+def _is_ambiguous_d(token: str) -> bool:
+    low = token.lower()
+    return low.endswith("'d") and low[:-2] in AMBIGUOUS_D_BASES
+
+
+def _is_ambiguous_s(token: str) -> bool:
+    low = token.lower()
+    return low.endswith("'s") and low[:-2] in AMBIGUOUS_S_BASES
+
 # Irregular possessives that are not formed by simple + 's logic
 IRREGULAR_POSSESSIVES = {
     "children's": "children's",
@@ -321,6 +318,10 @@ def tokenize(text: str) -> List[str]:
     return WORD_TOKEN_RE.findall(text)
 
 
+def tokenize_with_spans(text: str) -> List[Tuple[str, int, int]]:
+    return [(match.group(0), match.start(), match.end()) for match in WORD_TOKEN_RE.finditer(text)]
+
+
 def _cleanup_spacing(text: str) -> str:
     if not text:
         return text
@@ -571,40 +572,40 @@ def classify_token(token: str, cfg: ApostropheConfig) -> Tuple[str, str]:
             return "leading_elision", LEADING_ELISION[low]
         return "leading_elision", token
 
-    # 3. Exact contraction
+    # 3. Ambiguous 'd contractions
+    if _is_ambiguous_d(token):
+        base = low[:-2]
+        mode = cfg.ambiguous_past_modal_mode
+        if cfg.contraction_mode == "collapse":
+            return "ambiguous_contraction_d", base + "d"
+        if cfg.contraction_mode == "expand":
+            if mode == "expand_prefer_would":
+                return "ambiguous_contraction_d", base + " would"
+            if mode == "expand_prefer_had":
+                return "ambiguous_contraction_d", base + " had"
+            if mode == "contextual":
+                return "ambiguous_contraction_d", base + " would"
+        return "ambiguous_contraction_d", token
+
+    # 4. Ambiguous 's contractions
+    if _is_ambiguous_s(token):
+        base = low[:-2]
+        if cfg.contraction_mode == "expand":
+            return "ambiguous_contraction_s", base + " is"
+        if cfg.contraction_mode == "collapse":
+            return "ambiguous_contraction_s", base + "s"
+        return "ambiguous_contraction_s", token
+
+    # 5. Exact contraction
     if low in CONTRACTIONS_EXACT:
         if cfg.contraction_mode == "expand":
             return "contraction", CONTRACTIONS_EXACT[low]
         elif cfg.contraction_mode == "collapse":
-            # collapse: remove apostrophe only (it's -> its)
+            # collapse: remove apostrophe only (he's -> hes)
             return "contraction", low.replace("'", "")
         else:
             return "contraction", token
 
-    # 4. Ambiguous 'd
-    if low.endswith("'d"):
-        base = low[:-2]
-        if base in AMBIGUOUS_D_BASES:
-            if cfg.ambiguous_past_modal_mode == "expand_prefer_would":
-                return "ambiguous_contraction_d", base + " would"
-            elif cfg.ambiguous_past_modal_mode == "expand_prefer_had":
-                return "ambiguous_contraction_d", base + " had"
-            elif cfg.contraction_mode == "collapse":
-                return "ambiguous_contraction_d", base + "d"
-            return "ambiguous_contraction_d", token
-
-    # 5. Ambiguous 's
-    if low.endswith("'s"):
-        base = low[:-2]
-        if base in AMBIGUOUS_S_BASES:
-            # treat as contraction 'is' under chosen mode
-            if cfg.contraction_mode == "expand":
-                return "ambiguous_contraction_s", base + " is"
-            elif cfg.contraction_mode == "collapse":
-                return "ambiguous_contraction_s", base + "s"
-            else:
-                return "ambiguous_contraction_s", token
-
     # 6. Irregular possessives (keep or expand logic)
     if low in IRREGULAR_POSSESSIVES:
         if cfg.irregular_possessive_mode == "keep":
@@ -684,13 +685,38 @@ def normalize_apostrophes(text: str, cfg: ApostropheConfig | None = None) -> Tup
 
     text = normalize_unicode_apostrophes(text)
     text = _normalize_grouped_numbers(text, cfg)
-    tokens = tokenize(text)
+    token_entries = tokenize_with_spans(text)
 
-    results = []
+    use_contextual_s = cfg.contraction_mode == "expand"
+    use_contextual_d = cfg.contraction_mode == "expand" and cfg.ambiguous_past_modal_mode == "contextual"
+
+    need_contextual = False
+    if (use_contextual_s or use_contextual_d) and token_entries:
+        for token_value, _, _ in token_entries:
+            if use_contextual_s and _is_ambiguous_s(token_value):
+                need_contextual = True
+                break
+            if use_contextual_d and _is_ambiguous_d(token_value):
+                need_contextual = True
+                break
+
+    contextual_resolutions = resolve_ambiguous_contractions(text) if need_contextual else {}
+
+    results: List[Tuple[str, str, str]] = []
     normalized_tokens: List[str] = []
 
-    for tok in tokens:
+    for tok, start, end in token_entries:
         category, norm = classify_token(tok, cfg)
+
+        resolution = contextual_resolutions.get((start, end)) if contextual_resolutions else None
+        if resolution is not None:
+            if resolution.category == "ambiguous_contraction_s" and use_contextual_s:
+                category = resolution.category
+                norm = resolution.expansion
+            elif resolution.category == "ambiguous_contraction_d" and use_contextual_d:
+                category = resolution.category
+                norm = resolution.expansion
+
         results.append((tok, category, norm))
         normalized_tokens.append(norm)
 

abogen/normalization_settings.py

@@ -167,6 +167,7 @@ def build_apostrophe_config(
     config.leading_elision_mode = (
         "expand" if settings.get("normalization_apostrophes_leading_elisions", True) else "keep"
     )
+    config.ambiguous_past_modal_mode = "contextual" if config.contraction_mode == "expand" else "keep"
     return config
 
 

abogen/spacy_contraction_resolver.py

@@ -0,0 +1,231 @@
+from __future__ import annotations
+
+import os
+import logging
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Any, Dict, Optional, Tuple
+
+try:  # pragma: no cover - optional dependency
+    import spacy
+except Exception:  # pragma: no cover - spaCy unavailable at runtime
+    spacy = None
+
+# Lazy spaCy type hints to avoid a hard dependency at import time.
+Language = Any  # type: ignore[assignment]
+Token = Any  # type: ignore[assignment]
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class ContractionResolution:
+    start: int
+    end: int
+    surface: str
+    expansion: str
+    category: str
+    lemma: str
+
+    @property
+    def span(self) -> Tuple[int, int]:
+        return self.start, self.end
+
+
+_DEFAULT_MODEL = os.environ.get("ABOGEN_SPACY_MODEL", "en_core_web_sm")
+
+
+@lru_cache(maxsize=1)
+def _load_spacy_model(model: str = _DEFAULT_MODEL) -> Optional[Language]:
+    if spacy is None:
+        logger.debug("spaCy is not installed; skipping contraction disambiguation")
+        return None
+
+    try:
+        nlp = spacy.load(model)
+    except Exception as exc:  # pragma: no cover - depends on environment
+        logger.warning("Failed to load spaCy model '%s': %s", model, exc)
+        return None
+    return nlp
+
+
+def resolve_ambiguous_contractions(text: str, *, model: Optional[str] = None) -> Dict[Tuple[int, int], ContractionResolution]:
+    """Use spaCy to disambiguate ambiguous contractions in *text*.
+
+    Returns a mapping from (start, end) spans to their resolved expansion.
+    Only ambiguous `'s` and `'d` contractions are considered.
+    """
+    if not text:
+        return {}
+
+    nlp = _load_spacy_model(model or _DEFAULT_MODEL)
+    if nlp is None:
+        return {}
+
+    doc = nlp(text)
+    resolutions: Dict[Tuple[int, int], ContractionResolution] = {}
+    for token in doc:
+        if token.text == "'s":
+            resolution = _resolve_apostrophe_s(token)
+        elif token.text == "'d":
+            resolution = _resolve_apostrophe_d(token)
+        else:
+            resolution = None
+
+        if resolution is None:
+            continue
+
+        if resolution.span not in resolutions:
+            resolutions[resolution.span] = resolution
+    return resolutions
+
+
+def _resolution(prev: Token, token: Token, expansion_word: str, category: str, lemma_hint: str) -> Optional[ContractionResolution]:
+    if token is None or prev is None:
+        return None
+
+    if prev.idx + len(prev.text) != token.idx:
+        # Not a contiguous contraction (whitespace or punctuation in between)
+        return None
+
+    surface_start = prev.idx
+    surface_end = token.idx + len(token.text)
+    surface_text = token.doc.text[surface_start:surface_end]
+
+    expansion = _assemble_expansion(prev.text, surface_text, expansion_word)
+    return ContractionResolution(
+        start=surface_start,
+        end=surface_end,
+        surface=surface_text,
+        expansion=expansion,
+        category=category,
+        lemma=lemma_hint,
+    )
+
+
+def _assemble_expansion(base_text: str, surface_text: str, expansion_word: str) -> str:
+    """Combine *base_text* with *expansion_word*, preserving coarse casing."""
+    if not expansion_word:
+        return base_text
+
+    if surface_text.isupper() and expansion_word.isalpha():
+        adjusted = expansion_word.upper()
+    elif len(surface_text) > 2 and surface_text[:-2].istitle() and expansion_word:
+        # Surface like "It's" -> keep appended word lowercase
+        adjusted = expansion_word.lower()
+    else:
+        adjusted = expansion_word
+
+    return f"{base_text} {adjusted}".strip()
+
+
+def _resolve_apostrophe_s(token: Token) -> Optional[ContractionResolution]:
+    prev = token.nbor(-1) if token.i > 0 else None
+    if prev is None:
+        return None
+
+    # Possessive marker e.g., dog's
+    if token.tag_ == "POS" or token.lemma_ == "'s":
+        return None
+
+    prev_lower = prev.lemma_.lower()
+    surface = token.doc.text[prev.idx : token.idx + len(token.text)]
+
+    if prev_lower == "let":
+        return _resolution(prev, token, "us", "contraction", "us")
+
+    lemma = token.lemma_.lower()
+    if not lemma:
+        lemma = "be" if _favors_be(token) else "have" if _favors_have(token) else "be"
+
+    if lemma == "be":
+        return _resolution(prev, token, "is", "ambiguous_contraction_s", "be")
+    if lemma == "have":
+        return _resolution(prev, token, "has", "ambiguous_contraction_s", "have")
+
+    if _favors_have(token):
+        return _resolution(prev, token, "has", "ambiguous_contraction_s", "have")
+
+    if _favors_be(token):
+        return _resolution(prev, token, "is", "ambiguous_contraction_s", "be")
+
+    # Default to copula expansion.
+    return _resolution(prev, token, "is", "ambiguous_contraction_s", lemma or "be")
+
+
+def _resolve_apostrophe_d(token: Token) -> Optional[ContractionResolution]:
+    prev = token.nbor(-1) if token.i > 0 else None
+    if prev is None:
+        return None
+
+    if token.morph.get("VerbForm") == ["Part"]:
+        # spaCy sometimes tags possessives oddly; guard anyway
+        return None
+
+    lemma = token.lemma_.lower()
+    tense = set(token.morph.get("Tense"))
+
+    if "Past" in tense and lemma in {"have", "had"}:
+        return _resolution(prev, token, "had", "ambiguous_contraction_d", "have")
+
+    if token.tag_ == "MD" or lemma in {"will", "would", "shall"}:
+        return _resolution(prev, token, "would", "ambiguous_contraction_d", lemma or "will")
+
+    head = token.head if token.head is not None else None
+    if head is not None and head.i > token.i:
+        head_tag = head.tag_
+        head_lemma = head.lemma_.lower()
+        if head_tag in {"VBN", "VBD"} or head_lemma in {"gone", "been", "had"}:
+            return _resolution(prev, token, "had", "ambiguous_contraction_d", "have")
+        if head_lemma == "better":
+            return _resolution(prev, token, "had", "ambiguous_contraction_d", "have")
+        if head_tag == "VB" or head.pos_ in {"VERB", "AUX"}:
+            return _resolution(prev, token, "would", "ambiguous_contraction_d", lemma or "will")
+
+    next_content = _next_content_token(token)
+    if next_content is not None:
+        next_tag = next_content.tag_
+        if next_tag in {"VBN", "VBD"} or next_content.lemma_.lower() in {"been", "gone", "had"}:
+            return _resolution(prev, token, "had", "ambiguous_contraction_d", "have")
+        if next_content.lemma_.lower() == "better":
+            return _resolution(prev, token, "had", "ambiguous_contraction_d", "have")
+        if next_tag == "VB":
+            return _resolution(prev, token, "would", "ambiguous_contraction_d", lemma or "will")
+
+    # Fallback: if lemma hints at perfect aspect use "had", otherwise "would".
+    if lemma in {"have", "had"}:
+        return _resolution(prev, token, "had", "ambiguous_contraction_d", lemma)
+    return _resolution(prev, token, "would", "ambiguous_contraction_d", lemma or "will")
+
+
+def _next_content_token(token: Token) -> Optional[Token]:
+    doc = token.doc
+    for candidate in doc[token.i + 1 :]:
+        if candidate.is_space:
+            continue
+        if candidate.is_punct and candidate.text not in {"-"}:
+            break
+        if candidate.text in {"'", ""}:
+            continue
+        return candidate
+    return None
+
+
+def _favors_have(token: Token) -> bool:
+    next_content = _next_content_token(token)
+    if next_content is None:
+        return False
+    if next_content.tag_ in {"VBN"}:
+        return True
+    if next_content.lemma_.lower() in {"been", "gone", "had"}:
+        return True
+    return False
+
+
+def _favors_be(token: Token) -> bool:
+    next_content = _next_content_token(token)
+    if next_content is None:
+        return True
+    if next_content.tag_ in {"VBG", "JJ", "RB", "DT", "IN"}:
+        return True
+    return False

tests/test_text_normalization.py

@@ -1,7 +1,13 @@
 from __future__ import annotations
 
+import pytest
+
 from abogen.kokoro_text_normalization import normalize_roman_numeral_titles
 from abogen.web.conversion_runner import _normalize_for_pipeline
+from abogen.spacy_contraction_resolver import resolve_ambiguous_contractions
+
+
+SPACY_RESOLVER_AVAILABLE = bool(resolve_ambiguous_contractions("It's been a long time."))
 
 
 def test_title_abbreviations_are_expanded():
@@ -103,3 +109,27 @@ def test_decades_can_skip_expansion_when_disabled() -> None:
         normalization_overrides={"normalization_apostrophes_decades": False},
     )
     assert "'90s" in normalized
+
+
+@pytest.mark.skipif(not SPACY_RESOLVER_AVAILABLE, reason="spaCy model unavailable")
+def test_spacy_disambiguates_it_has_from_context() -> None:
+    normalized = _normalize_for_pipeline("It's been a long time.")
+    assert "It has been a long time." == normalized
+
+
+@pytest.mark.skipif(not SPACY_RESOLVER_AVAILABLE, reason="spaCy model unavailable")
+def test_spacy_disambiguates_it_is_from_context() -> None:
+    normalized = _normalize_for_pipeline("It's cold outside.")
+    assert "It is cold outside." == normalized
+
+
+@pytest.mark.skipif(not SPACY_RESOLVER_AVAILABLE, reason="spaCy model unavailable")
+def test_spacy_disambiguates_she_had() -> None:
+    normalized = _normalize_for_pipeline("She'd left before dawn.")
+    assert "She had left before dawn." == normalized
+
+
+@pytest.mark.skipif(not SPACY_RESOLVER_AVAILABLE, reason="spaCy model unavailable")
+def test_spacy_disambiguates_she_would() -> None:
+    normalized = _normalize_for_pipeline("She'd go if invited.")
+    assert "She would go if invited." == normalized