mirror of
https://github.com/denizsafak/abogen.git
synced 2026-07-18 13:40:27 +02:00
feat: Add contraction handling options and UI for normalization settings
This commit is contained in:
JB
+233
-101
@@ -4,7 +4,7 @@ import json
|
||||
import re
|
||||
import unicodedata
|
||||
from fractions import Fraction
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple
|
||||
try: # pragma: no cover - optional dependency guard
|
||||
from num2words import num2words
|
||||
@@ -16,6 +16,17 @@ if TYPE_CHECKING: # pragma: no cover - type checking only
|
||||
|
||||
from abogen.spacy_contraction_resolver import resolve_ambiguous_contractions
|
||||
|
||||
# ---------- Contraction Category Defaults ----------
|
||||
|
||||
CONTRACTION_CATEGORY_DEFAULTS: Dict[str, bool] = {
|
||||
"contraction_aux_be": True,
|
||||
"contraction_aux_have": True,
|
||||
"contraction_modal_will": True,
|
||||
"contraction_modal_would": True,
|
||||
"contraction_negation_not": True,
|
||||
"contraction_let_us": True,
|
||||
}
|
||||
|
||||
# ---------- Configuration Dataclass ----------
|
||||
|
||||
@dataclass
|
||||
@@ -38,44 +49,44 @@ class ApostropheConfig:
|
||||
protect_cultural_names: bool = True # Always keep O'Brien, D'Angelo, etc.
|
||||
convert_numbers: bool = True # Convert grouped numbers such as 12,500 to words
|
||||
number_lang: str = "en" # num2words language code
|
||||
contraction_categories: Dict[str, bool] = field(default_factory=lambda: dict(CONTRACTION_CATEGORY_DEFAULTS))
|
||||
|
||||
def is_contraction_enabled(self, category: str) -> bool:
|
||||
return self.contraction_categories.get(category, True)
|
||||
|
||||
# ---------- Dictionaries / Patterns ----------
|
||||
|
||||
# Common contraction expansions (straightforward unambiguous)
|
||||
CONTRACTIONS_EXACT = {
|
||||
"let's": "let us",
|
||||
"i'm": "i am",
|
||||
"you're": "you are",
|
||||
"we're": "we are",
|
||||
"they're": "they are",
|
||||
"i've": "i have",
|
||||
"you've": "you have",
|
||||
"we've": "we have",
|
||||
"they've": "they have",
|
||||
"i'll": "i will",
|
||||
"you'll": "you will",
|
||||
"he'll": "he will",
|
||||
"she'll": "she will",
|
||||
"we'll": "we will",
|
||||
"they'll": "they will",
|
||||
"can't": "can not", # or "cannot"
|
||||
"won't": "will not",
|
||||
"don't": "do not",
|
||||
"doesn't": "does not",
|
||||
"didn't": "did not",
|
||||
"isn't": "is not",
|
||||
"aren't": "are not",
|
||||
"wasn't": "was not",
|
||||
"weren't": "were not",
|
||||
"haven't": "have not",
|
||||
"hasn't": "has not",
|
||||
"hadn't": "had not",
|
||||
"couldn't": "could not",
|
||||
"shouldn't": "should not",
|
||||
"wouldn't": "would not",
|
||||
"mustn't": "must not",
|
||||
"mightn't": "might not",
|
||||
"shan't": "shall not",
|
||||
# Common contraction expansions (type + expansion words)
|
||||
CONTRACTION_LEXICON: Dict[str, Tuple[str, Tuple[str, ...]]] = {
|
||||
"let's": ("contraction_let_us", ("let", "us")),
|
||||
"can't": ("contraction_negation_not", ("can", "not")),
|
||||
"won't": ("contraction_negation_not", ("will", "not")),
|
||||
"don't": ("contraction_negation_not", ("do", "not")),
|
||||
"doesn't": ("contraction_negation_not", ("does", "not")),
|
||||
"didn't": ("contraction_negation_not", ("did", "not")),
|
||||
"isn't": ("contraction_negation_not", ("is", "not")),
|
||||
"aren't": ("contraction_negation_not", ("are", "not")),
|
||||
"wasn't": ("contraction_negation_not", ("was", "not")),
|
||||
"weren't": ("contraction_negation_not", ("were", "not")),
|
||||
"haven't": ("contraction_negation_not", ("have", "not")),
|
||||
"hasn't": ("contraction_negation_not", ("has", "not")),
|
||||
"hadn't": ("contraction_negation_not", ("had", "not")),
|
||||
"couldn't": ("contraction_negation_not", ("could", "not")),
|
||||
"shouldn't": ("contraction_negation_not", ("should", "not")),
|
||||
"wouldn't": ("contraction_negation_not", ("would", "not")),
|
||||
"mustn't": ("contraction_negation_not", ("must", "not")),
|
||||
"mightn't": ("contraction_negation_not", ("might", "not")),
|
||||
"shan't": ("contraction_negation_not", ("shall", "not")),
|
||||
}
|
||||
|
||||
SUFFIX_CONTRACTION_RULES: Tuple[Tuple[str, str, str], ...] = (
|
||||
("'ll", "will", "contraction_modal_will"),
|
||||
("'re", "are", "contraction_aux_be"),
|
||||
("'ve", "have", "contraction_aux_have"),
|
||||
)
|
||||
|
||||
SUFFIX_CONTRACTION_BASES: Dict[str, Tuple[str, ...]] = {
|
||||
"'m": ("i",),
|
||||
}
|
||||
|
||||
# For ambiguous 'd and 's we handle separately
|
||||
@@ -223,7 +234,22 @@ def _replace_fraction(match: re.Match[str], language: str) -> str:
|
||||
return match.group(0)
|
||||
return spoken
|
||||
AMBIGUOUS_D_BASES = {"i","you","he","she","we","they"}
|
||||
AMBIGUOUS_S_BASES = {"it","that","what","where","who","when","how","there","here"}
|
||||
AMBIGUOUS_S_BASES = {
|
||||
"it",
|
||||
"that",
|
||||
"what",
|
||||
"where",
|
||||
"who",
|
||||
"when",
|
||||
"how",
|
||||
"there",
|
||||
"here",
|
||||
"he",
|
||||
"she",
|
||||
"we",
|
||||
"they",
|
||||
"you",
|
||||
}
|
||||
|
||||
|
||||
def _is_ambiguous_d(token: str) -> bool:
|
||||
@@ -544,25 +570,127 @@ def is_cultural_name(token: str, cfg: ApostropheConfig) -> bool:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _case_preserving_words(original: str, words: Sequence[str]) -> str:
|
||||
if not words:
|
||||
return ""
|
||||
if original.isupper():
|
||||
return " ".join(word.upper() for word in words)
|
||||
|
||||
if original[:1].isupper():
|
||||
adjusted = [words[0].capitalize()]
|
||||
if len(words) > 1:
|
||||
adjusted.extend(words[1:])
|
||||
return " ".join(adjusted)
|
||||
|
||||
return " ".join(words)
|
||||
|
||||
|
||||
def _apply_contraction_policy(
|
||||
token: str,
|
||||
*,
|
||||
category: str,
|
||||
cfg: ApostropheConfig,
|
||||
expand: Callable[[], str],
|
||||
collapse: Optional[str] = None,
|
||||
) -> str:
|
||||
mode = cfg.contraction_mode
|
||||
if mode == "collapse":
|
||||
return collapse if collapse is not None else token.replace("'", "")
|
||||
if mode != "expand":
|
||||
return token
|
||||
if not cfg.is_contraction_enabled(category):
|
||||
return token
|
||||
return expand()
|
||||
|
||||
|
||||
def _assemble_contraction_expansion(base_text: str, surface_text: str, expansion_word: str) -> str:
|
||||
if not expansion_word:
|
||||
return base_text
|
||||
|
||||
if surface_text.isupper() and expansion_word.isalpha():
|
||||
adjusted = expansion_word.upper()
|
||||
elif len(surface_text) > 2 and surface_text[:-2].istitle() and expansion_word:
|
||||
adjusted = expansion_word.lower()
|
||||
else:
|
||||
adjusted = expansion_word
|
||||
|
||||
return f"{base_text} {adjusted}".strip()
|
||||
|
||||
|
||||
def _classify_ambiguous_d(token: str, cfg: ApostropheConfig) -> Tuple[str, str]:
|
||||
base = token[:-2]
|
||||
collapse_value = base + "d"
|
||||
|
||||
if cfg.contraction_mode == "collapse":
|
||||
return "contraction_modal_would", collapse_value
|
||||
if cfg.contraction_mode != "expand":
|
||||
return "contraction_modal_would", token
|
||||
|
||||
mode = cfg.ambiguous_past_modal_mode
|
||||
if mode == "expand_prefer_had":
|
||||
candidates = [
|
||||
("contraction_aux_have", "had"),
|
||||
("contraction_modal_would", "would"),
|
||||
]
|
||||
elif mode == "expand_prefer_would":
|
||||
candidates = [
|
||||
("contraction_modal_would", "would"),
|
||||
("contraction_aux_have", "had"),
|
||||
]
|
||||
else: # contextual
|
||||
candidates = [
|
||||
("contraction_modal_would", "would"),
|
||||
("contraction_aux_have", "had"),
|
||||
]
|
||||
|
||||
for category, word in candidates:
|
||||
if not cfg.is_contraction_enabled(category):
|
||||
continue
|
||||
expanded = _assemble_contraction_expansion(base, token, word)
|
||||
return category, expanded
|
||||
|
||||
# If every category is disabled, leave the token as-is but report default category
|
||||
return candidates[0][0], token
|
||||
|
||||
|
||||
def _classify_ambiguous_s(token: str, cfg: ApostropheConfig) -> Tuple[str, str]:
|
||||
base = token[:-2]
|
||||
|
||||
if cfg.contraction_mode == "collapse":
|
||||
return "contraction_aux_be", base + "s"
|
||||
if cfg.contraction_mode != "expand":
|
||||
return "contraction_aux_be", token
|
||||
|
||||
candidates = [
|
||||
("contraction_aux_be", "is"),
|
||||
("contraction_aux_have", "has"),
|
||||
]
|
||||
|
||||
for category, word in candidates:
|
||||
if not cfg.is_contraction_enabled(category):
|
||||
continue
|
||||
expanded = _assemble_contraction_expansion(base, token, word)
|
||||
return category, expanded
|
||||
|
||||
return candidates[0][0], token
|
||||
|
||||
def classify_token(token: str, cfg: ApostropheConfig) -> Tuple[str, str]:
|
||||
"""
|
||||
Classify apostrophe usage and propose normalized form.
|
||||
Returns (category, normalized_token_or_same).
|
||||
Categories: contraction, ambiguous_contraction_s, ambiguous_contraction_d,
|
||||
plural_possessive, irregular_possessive, sibilant_possessive,
|
||||
singular_possessive, acronym_possessive, decade, leading_elision,
|
||||
fantasy_internal, other
|
||||
Categories include: contraction_* variants, plural_possessive, irregular_possessive,
|
||||
sibilant_possessive, singular_possessive, acronym_possessive, decade, leading_elision,
|
||||
fantasy_internal, cultural_name, other.
|
||||
"""
|
||||
if "'" not in token:
|
||||
return "other", token
|
||||
|
||||
raw = token
|
||||
low = token.lower()
|
||||
|
||||
# 1. Decades
|
||||
if DECADE_RE.match(token):
|
||||
if cfg.decades_mode == "expand":
|
||||
# '90s -> 1990s (you could also choose 90s)
|
||||
return "decade", f"19{token[2:4]}s"
|
||||
return "decade", token
|
||||
|
||||
@@ -574,103 +702,108 @@ def classify_token(token: str, cfg: ApostropheConfig) -> Tuple[str, str]:
|
||||
|
||||
# 3. Ambiguous 'd contractions
|
||||
if _is_ambiguous_d(token):
|
||||
base = low[:-2]
|
||||
mode = cfg.ambiguous_past_modal_mode
|
||||
if cfg.contraction_mode == "collapse":
|
||||
return "ambiguous_contraction_d", base + "d"
|
||||
if cfg.contraction_mode == "expand":
|
||||
if mode == "expand_prefer_would":
|
||||
return "ambiguous_contraction_d", base + " would"
|
||||
if mode == "expand_prefer_had":
|
||||
return "ambiguous_contraction_d", base + " had"
|
||||
if mode == "contextual":
|
||||
return "ambiguous_contraction_d", base + " would"
|
||||
return "ambiguous_contraction_d", token
|
||||
return _classify_ambiguous_d(token, cfg)
|
||||
|
||||
# 4. Ambiguous 's contractions
|
||||
if _is_ambiguous_s(token):
|
||||
base = low[:-2]
|
||||
if cfg.contraction_mode == "expand":
|
||||
return "ambiguous_contraction_s", base + " is"
|
||||
if cfg.contraction_mode == "collapse":
|
||||
return "ambiguous_contraction_s", base + "s"
|
||||
return "ambiguous_contraction_s", token
|
||||
return _classify_ambiguous_s(token, cfg)
|
||||
|
||||
# 5. Exact contraction
|
||||
if low in CONTRACTIONS_EXACT:
|
||||
if cfg.contraction_mode == "expand":
|
||||
return "contraction", CONTRACTIONS_EXACT[low]
|
||||
elif cfg.contraction_mode == "collapse":
|
||||
# collapse: remove apostrophe only (he's -> hes)
|
||||
return "contraction", low.replace("'", "")
|
||||
else:
|
||||
return "contraction", token
|
||||
# 5. Lexicon-based contractions
|
||||
lex_entry = CONTRACTION_LEXICON.get(low)
|
||||
if lex_entry is not None:
|
||||
category, words = lex_entry
|
||||
|
||||
# 6. Irregular possessives (keep or expand logic)
|
||||
def _expand() -> str:
|
||||
return _case_preserving_words(token, words)
|
||||
|
||||
collapse_value = token.replace("'", "")
|
||||
normalized = _apply_contraction_policy(token, category=category, cfg=cfg, expand=_expand, collapse=collapse_value)
|
||||
return category, normalized
|
||||
|
||||
# 6. Suffix contractions ('m handled separately)
|
||||
if low.endswith("'m") and low[:-2] in SUFFIX_CONTRACTION_BASES.get("'m", ()): # pronoun I'm
|
||||
|
||||
def _expand_m() -> str:
|
||||
base = token[:-2]
|
||||
return _assemble_contraction_expansion(base, token, "am")
|
||||
|
||||
normalized = _apply_contraction_policy(
|
||||
token,
|
||||
category="contraction_aux_be",
|
||||
cfg=cfg,
|
||||
expand=_expand_m,
|
||||
collapse=token.replace("'", ""),
|
||||
)
|
||||
return "contraction_aux_be", normalized
|
||||
|
||||
for suffix, append_word, category in SUFFIX_CONTRACTION_RULES:
|
||||
if low.endswith(suffix) and len(token) > len(suffix):
|
||||
base = token[: -len(suffix)]
|
||||
|
||||
def _expand_suffix() -> str:
|
||||
return _assemble_contraction_expansion(base, token, append_word)
|
||||
|
||||
normalized = _apply_contraction_policy(
|
||||
token,
|
||||
category=category,
|
||||
cfg=cfg,
|
||||
expand=_expand_suffix,
|
||||
collapse=token.replace("'", ""),
|
||||
)
|
||||
return category, normalized
|
||||
|
||||
# 7. Irregular possessives (keep or expand logic)
|
||||
if low in IRREGULAR_POSSESSIVES:
|
||||
if cfg.irregular_possessive_mode == "keep":
|
||||
return "irregular_possessive", token
|
||||
else:
|
||||
# 'expand': we might keep same or optionally add marker
|
||||
return "irregular_possessive", token
|
||||
return "irregular_possessive", token
|
||||
|
||||
# 7. Plural possessive pattern dogs'
|
||||
# 8. Plural possessive pattern dogs'
|
||||
if re.match(r"^[A-Za-z0-9]+s'$", token):
|
||||
if cfg.plural_possessive_mode == "collapse":
|
||||
return "plural_possessive", token[:-1] # remove trailing apostrophe
|
||||
return "plural_possessive", token[:-1]
|
||||
return "plural_possessive", token
|
||||
|
||||
# 8. Acronym possessive NASA's
|
||||
# 9. Acronym possessive NASA's
|
||||
if ACRONYM_POSSESSIVE_RE.match(token):
|
||||
if cfg.acronym_possessive_mode == "collapse_add_s":
|
||||
return "acronym_possessive", token.replace("'", "")
|
||||
return "acronym_possessive", token
|
||||
|
||||
# 9. Sibilant singular possessive boss's, church's
|
||||
# 10. Sibilant singular possessive boss's, church's
|
||||
if low.endswith("'s"):
|
||||
base = token[:-2]
|
||||
if SIBILANT_END_RE.search(base):
|
||||
if cfg.sibilant_possessive_mode == "keep":
|
||||
return "sibilant_possessive", token
|
||||
elif cfg.sibilant_possessive_mode == "approx":
|
||||
# convert to base + "es" (boss's -> bosses)
|
||||
# risk: loses possessive semantics visually
|
||||
if cfg.sibilant_possessive_mode == "approx":
|
||||
return "sibilant_possessive", base + "es"
|
||||
elif cfg.sibilant_possessive_mode == "mark":
|
||||
# remove apostrophe, add IZ marker
|
||||
if cfg.sibilant_possessive_mode == "mark":
|
||||
normalized = base
|
||||
if cfg.add_phoneme_hints:
|
||||
normalized += cfg.sibilant_iz_marker
|
||||
else:
|
||||
normalized += "es"
|
||||
normalized += cfg.sibilant_iz_marker if cfg.add_phoneme_hints else "es"
|
||||
return "sibilant_possessive", normalized
|
||||
|
||||
# 10. Generic singular possessive (\w+'s)
|
||||
# 11. Generic singular possessive (\w+'s)
|
||||
if re.match(r"^[A-Za-z0-9]+'s$", token):
|
||||
if cfg.possessive_mode == "collapse":
|
||||
# Just remove apostrophe
|
||||
return "singular_possessive", token.replace("'", "")
|
||||
return "singular_possessive", token
|
||||
|
||||
# 11. Cultural names or fantasy internal
|
||||
# 12. Cultural names or fantasy internal
|
||||
if is_cultural_name(token, cfg):
|
||||
return "cultural_name", token
|
||||
|
||||
# 12. Fantasy internal apostrophes
|
||||
if INTERNAL_APOSTROPHE_RE.search(token):
|
||||
if cfg.fantasy_mode == "keep":
|
||||
return "fantasy_internal", token
|
||||
elif cfg.fantasy_mode == "mark":
|
||||
if cfg.fantasy_mode == "mark":
|
||||
out = token + (cfg.fantasy_marker if cfg.add_phoneme_hints else "")
|
||||
return "fantasy_internal", out
|
||||
elif cfg.fantasy_mode == "collapse_internal":
|
||||
# Remove internal apostrophes only
|
||||
if cfg.fantasy_mode == "collapse_internal":
|
||||
inner = re.sub(r"(?<=\w)'+(?=\w)", cfg.joiner, token)
|
||||
return "fantasy_internal", inner
|
||||
|
||||
# 13. Fallback: treat as other (maybe stray apostrophe)
|
||||
if cfg.fantasy_mode == "collapse_internal":
|
||||
# Remove any internal apostrophes
|
||||
return "other", token.replace("'", cfg.joiner)
|
||||
return "other", token
|
||||
|
||||
@@ -709,13 +842,12 @@ def normalize_apostrophes(text: str, cfg: ApostropheConfig | None = None) -> Tup
|
||||
category, norm = classify_token(tok, cfg)
|
||||
|
||||
resolution = contextual_resolutions.get((start, end)) if contextual_resolutions else None
|
||||
if resolution is not None:
|
||||
if resolution.category == "ambiguous_contraction_s" and use_contextual_s:
|
||||
category = resolution.category
|
||||
norm = resolution.expansion
|
||||
elif resolution.category == "ambiguous_contraction_d" and use_contextual_d:
|
||||
if resolution is not None and cfg.contraction_mode == "expand":
|
||||
if cfg.is_contraction_enabled(resolution.category):
|
||||
category = resolution.category
|
||||
norm = resolution.expansion
|
||||
else:
|
||||
norm = tok
|
||||
|
||||
results.append((tok, category, norm))
|
||||
normalized_tokens.append(norm)
|
||||
|
||||
@@ -5,7 +5,7 @@ from dataclasses import replace
|
||||
from functools import lru_cache
|
||||
from typing import Any, Dict, Mapping, Optional
|
||||
|
||||
from abogen.kokoro_text_normalization import ApostropheConfig
|
||||
from abogen.kokoro_text_normalization import ApostropheConfig, CONTRACTION_CATEGORY_DEFAULTS
|
||||
from abogen.llm_client import LLMConfiguration
|
||||
from abogen.utils import load_config
|
||||
|
||||
@@ -40,6 +40,21 @@ _SETTINGS_DEFAULTS: Dict[str, Any] = {
|
||||
"normalization_apostrophes_decades": True,
|
||||
"normalization_apostrophes_leading_elisions": True,
|
||||
"normalization_apostrophe_mode": "spacy",
|
||||
"normalization_contraction_aux_be": True,
|
||||
"normalization_contraction_aux_have": True,
|
||||
"normalization_contraction_modal_will": True,
|
||||
"normalization_contraction_modal_would": True,
|
||||
"normalization_contraction_negation_not": True,
|
||||
"normalization_contraction_let_us": True,
|
||||
}
|
||||
|
||||
_CONTRACTION_SETTING_MAP: Dict[str, str] = {
|
||||
"normalization_contraction_aux_be": "contraction_aux_be",
|
||||
"normalization_contraction_aux_have": "contraction_aux_have",
|
||||
"normalization_contraction_modal_will": "contraction_modal_will",
|
||||
"normalization_contraction_modal_would": "contraction_modal_would",
|
||||
"normalization_contraction_negation_not": "contraction_negation_not",
|
||||
"normalization_contraction_let_us": "contraction_let_us",
|
||||
}
|
||||
|
||||
_ENVIRONMENT_KEYS: Dict[str, str] = {
|
||||
@@ -168,6 +183,12 @@ def build_apostrophe_config(
|
||||
"expand" if settings.get("normalization_apostrophes_leading_elisions", True) else "keep"
|
||||
)
|
||||
config.ambiguous_past_modal_mode = "contextual" if config.contraction_mode == "expand" else "keep"
|
||||
category_flags = dict(CONTRACTION_CATEGORY_DEFAULTS)
|
||||
for setting_key, category in _CONTRACTION_SETTING_MAP.items():
|
||||
default_value = bool(_SETTINGS_DEFAULTS.get(setting_key, True))
|
||||
raw_value = settings.get(setting_key, default_value)
|
||||
category_flags[category] = _coerce_bool(raw_value, default_value)
|
||||
config.contraction_categories = category_flags
|
||||
return config
|
||||
|
||||
|
||||
|
||||
@@ -132,25 +132,25 @@ def _resolve_apostrophe_s(token: Token) -> Optional[ContractionResolution]:
|
||||
surface = token.doc.text[prev.idx : token.idx + len(token.text)]
|
||||
|
||||
if prev_lower == "let":
|
||||
return _resolution(prev, token, "us", "contraction", "us")
|
||||
return _resolution(prev, token, "us", "contraction_let_us", "us")
|
||||
|
||||
lemma = token.lemma_.lower()
|
||||
if not lemma:
|
||||
lemma = "be" if _favors_be(token) else "have" if _favors_have(token) else "be"
|
||||
|
||||
if lemma == "be":
|
||||
return _resolution(prev, token, "is", "ambiguous_contraction_s", "be")
|
||||
return _resolution(prev, token, "is", "contraction_aux_be", "be")
|
||||
if lemma == "have":
|
||||
return _resolution(prev, token, "has", "ambiguous_contraction_s", "have")
|
||||
return _resolution(prev, token, "has", "contraction_aux_have", "have")
|
||||
|
||||
if _favors_have(token):
|
||||
return _resolution(prev, token, "has", "ambiguous_contraction_s", "have")
|
||||
return _resolution(prev, token, "has", "contraction_aux_have", "have")
|
||||
|
||||
if _favors_be(token):
|
||||
return _resolution(prev, token, "is", "ambiguous_contraction_s", "be")
|
||||
return _resolution(prev, token, "is", "contraction_aux_be", "be")
|
||||
|
||||
# Default to copula expansion.
|
||||
return _resolution(prev, token, "is", "ambiguous_contraction_s", lemma or "be")
|
||||
return _resolution(prev, token, "is", "contraction_aux_be", lemma or "be")
|
||||
|
||||
|
||||
def _resolve_apostrophe_d(token: Token) -> Optional[ContractionResolution]:
|
||||
@@ -164,38 +164,35 @@ def _resolve_apostrophe_d(token: Token) -> Optional[ContractionResolution]:
|
||||
|
||||
lemma = token.lemma_.lower()
|
||||
tense = set(token.morph.get("Tense"))
|
||||
next_content = _next_content_token(token)
|
||||
prefers_had = _context_prefers_had(token)
|
||||
|
||||
if prefers_had:
|
||||
return _resolution(prev, token, "had", "contraction_aux_have", "have")
|
||||
|
||||
if "Past" in tense and lemma in {"have", "had"}:
|
||||
return _resolution(prev, token, "had", "ambiguous_contraction_d", "have")
|
||||
return _resolution(prev, token, "had", "contraction_aux_have", "have")
|
||||
|
||||
if token.tag_ == "MD" or lemma in {"will", "would", "shall"}:
|
||||
return _resolution(prev, token, "would", "ambiguous_contraction_d", lemma or "will")
|
||||
|
||||
head = token.head if token.head is not None else None
|
||||
if head is not None and head.i > token.i:
|
||||
head_tag = head.tag_
|
||||
head_lemma = head.lemma_.lower()
|
||||
if head_tag in {"VBN", "VBD"} or head_lemma in {"gone", "been", "had"}:
|
||||
return _resolution(prev, token, "had", "ambiguous_contraction_d", "have")
|
||||
if head_lemma == "better":
|
||||
return _resolution(prev, token, "had", "ambiguous_contraction_d", "have")
|
||||
if head_tag == "VB" or head.pos_ in {"VERB", "AUX"}:
|
||||
return _resolution(prev, token, "would", "ambiguous_contraction_d", lemma or "will")
|
||||
|
||||
next_content = _next_content_token(token)
|
||||
if next_content is not None:
|
||||
next_tag = next_content.tag_
|
||||
if next_tag in {"VBN", "VBD"} or next_content.lemma_.lower() in {"been", "gone", "had"}:
|
||||
return _resolution(prev, token, "had", "ambiguous_contraction_d", "have")
|
||||
if next_content.lemma_.lower() == "better":
|
||||
return _resolution(prev, token, "had", "ambiguous_contraction_d", "have")
|
||||
if next_tag == "VB":
|
||||
return _resolution(prev, token, "would", "ambiguous_contraction_d", lemma or "will")
|
||||
next_lemma = next_content.lemma_.lower()
|
||||
else:
|
||||
next_tag = ""
|
||||
next_lemma = ""
|
||||
|
||||
if next_tag == "VB":
|
||||
return _resolution(prev, token, "would", "contraction_modal_would", lemma or "will")
|
||||
|
||||
if token.tag_ == "MD" or lemma in {"will", "would", "shall"}:
|
||||
return _resolution(prev, token, "would", "contraction_modal_would", lemma or "will")
|
||||
|
||||
if next_lemma in {"been", "gone", "had", "better"} or next_tag in {"VBN", "VBD"}:
|
||||
return _resolution(prev, token, "had", "contraction_aux_have", "have")
|
||||
|
||||
# Fallback: if lemma hints at perfect aspect use "had", otherwise "would".
|
||||
if lemma in {"have", "had"}:
|
||||
return _resolution(prev, token, "had", "ambiguous_contraction_d", lemma)
|
||||
return _resolution(prev, token, "would", "ambiguous_contraction_d", lemma or "will")
|
||||
return _resolution(prev, token, "had", "contraction_aux_have", lemma)
|
||||
|
||||
return _resolution(prev, token, "would", "contraction_modal_would", lemma or "will")
|
||||
|
||||
|
||||
def _next_content_token(token: Token) -> Optional[Token]:
|
||||
@@ -228,4 +225,29 @@ def _favors_be(token: Token) -> bool:
|
||||
return True
|
||||
if next_content.tag_ in {"VBG", "JJ", "RB", "DT", "IN"}:
|
||||
return True
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def _context_prefers_had(token: Token) -> bool:
|
||||
head = token.head if token.head is not None else None
|
||||
if head is not None and head.i > token.i:
|
||||
head_tag = head.tag_
|
||||
head_lemma = head.lemma_.lower()
|
||||
if head_tag in {"VBN", "VBD"} or head_lemma in {"gone", "been", "had"}:
|
||||
return True
|
||||
if head_lemma == "better":
|
||||
return True
|
||||
|
||||
next_content = _next_content_token(token)
|
||||
if next_content is None:
|
||||
return False
|
||||
next_tag = next_content.tag_
|
||||
next_lemma = next_content.lemma_.lower()
|
||||
if next_tag in {"VBN", "VBD"}:
|
||||
return True
|
||||
if next_lemma in {"been", "gone", "had"}:
|
||||
return True
|
||||
if next_lemma == "better":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@@ -2100,6 +2100,12 @@ BOOLEAN_SETTINGS = {
|
||||
"normalization_apostrophes_sibilant_possessives",
|
||||
"normalization_apostrophes_decades",
|
||||
"normalization_apostrophes_leading_elisions",
|
||||
"normalization_contraction_aux_be",
|
||||
"normalization_contraction_aux_have",
|
||||
"normalization_contraction_modal_will",
|
||||
"normalization_contraction_modal_would",
|
||||
"normalization_contraction_negation_not",
|
||||
"normalization_contraction_let_us",
|
||||
}
|
||||
|
||||
FLOAT_SETTINGS = {"silence_between_chapters", "chapter_intro_delay", "llm_timeout"}
|
||||
@@ -2111,6 +2117,12 @@ _APOSTROPHE_OVERRIDE_KEYS = (
|
||||
"normalization_apostrophes_sibilant_possessives",
|
||||
"normalization_apostrophes_decades",
|
||||
"normalization_apostrophes_leading_elisions",
|
||||
"normalization_contraction_aux_be",
|
||||
"normalization_contraction_aux_have",
|
||||
"normalization_contraction_modal_will",
|
||||
"normalization_contraction_modal_would",
|
||||
"normalization_contraction_negation_not",
|
||||
"normalization_contraction_let_us",
|
||||
)
|
||||
|
||||
|
||||
@@ -3376,6 +3388,12 @@ def api_normalization_preview() -> ResponseReturnValue:
|
||||
"normalization_apostrophes_sibilant_possessives",
|
||||
"normalization_apostrophes_decades",
|
||||
"normalization_apostrophes_leading_elisions",
|
||||
"normalization_contraction_aux_be",
|
||||
"normalization_contraction_aux_have",
|
||||
"normalization_contraction_modal_will",
|
||||
"normalization_contraction_modal_would",
|
||||
"normalization_contraction_negation_not",
|
||||
"normalization_contraction_let_us",
|
||||
)
|
||||
for key in boolean_keys:
|
||||
if key in normalization_payload:
|
||||
|
||||
@@ -28,6 +28,11 @@ let folderModalOpener = null;
|
||||
let folderModalPreviousFocus = null;
|
||||
let audiobookshelfFolderSource = [];
|
||||
|
||||
const contractionModal = document.querySelector('[data-role="contraction-modal"]');
|
||||
const contractionModalOverlay = contractionModal ? contractionModal.querySelector('[data-role="contraction-modal-overlay"]') : null;
|
||||
let contractionModalOpener = null;
|
||||
let contractionModalPreviousFocus = null;
|
||||
|
||||
function setStatus(target, message, state) {
|
||||
if (!target) {
|
||||
return;
|
||||
@@ -177,6 +182,66 @@ function closeFolderModal(event) {
|
||||
folderModalOpener = null;
|
||||
}
|
||||
|
||||
function openContractionModal(opener) {
|
||||
if (!contractionModal) {
|
||||
return;
|
||||
}
|
||||
contractionModalOpener = opener || null;
|
||||
contractionModalPreviousFocus = document.activeElement instanceof HTMLElement ? document.activeElement : null;
|
||||
contractionModal.hidden = false;
|
||||
contractionModal.dataset.open = 'true';
|
||||
document.body.classList.add('modal-open');
|
||||
const focusTarget = contractionModal.querySelector('input, button, select, textarea');
|
||||
if (focusTarget instanceof HTMLElement) {
|
||||
focusTarget.focus({ preventScroll: true });
|
||||
}
|
||||
}
|
||||
|
||||
function closeContractionModal(event) {
|
||||
if (event) {
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
}
|
||||
if (!contractionModal || contractionModal.hidden) {
|
||||
return;
|
||||
}
|
||||
contractionModal.dataset.open = 'false';
|
||||
contractionModal.hidden = true;
|
||||
document.body.classList.remove('modal-open');
|
||||
const focusTarget =
|
||||
(contractionModalPreviousFocus && typeof contractionModalPreviousFocus.focus === 'function'
|
||||
? contractionModalPreviousFocus
|
||||
: contractionModalOpener) || null;
|
||||
if (focusTarget && typeof focusTarget.focus === 'function') {
|
||||
focusTarget.focus({ preventScroll: true });
|
||||
}
|
||||
contractionModalPreviousFocus = null;
|
||||
contractionModalOpener = null;
|
||||
}
|
||||
|
||||
function initContractionModal() {
|
||||
if (!contractionModal) {
|
||||
return;
|
||||
}
|
||||
const openButton = document.querySelector('[data-action="contraction-modal-open"]');
|
||||
if (openButton) {
|
||||
openButton.addEventListener('click', () => openContractionModal(openButton));
|
||||
}
|
||||
const closeButtons = contractionModal.querySelectorAll('[data-action="contraction-modal-close"]');
|
||||
closeButtons.forEach((button) => {
|
||||
button.addEventListener('click', closeContractionModal);
|
||||
});
|
||||
if (contractionModalOverlay) {
|
||||
contractionModalOverlay.addEventListener('click', closeContractionModal);
|
||||
}
|
||||
contractionModal.addEventListener('keydown', (event) => {
|
||||
if (event.key === 'Escape') {
|
||||
event.preventDefault();
|
||||
closeContractionModal(event);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function renderFolderList(query) {
|
||||
if (!folderList) {
|
||||
return;
|
||||
@@ -461,6 +526,9 @@ async function previewLLM(button) {
|
||||
}
|
||||
|
||||
function collectNormalizationSettings() {
|
||||
if (!form) {
|
||||
return null;
|
||||
}
|
||||
const normalization = {
|
||||
normalization_numbers: Boolean(form.querySelector('input[name="normalization_numbers"]')?.checked),
|
||||
normalization_titles: Boolean(form.querySelector('input[name="normalization_titles"]')?.checked),
|
||||
@@ -471,6 +539,12 @@ function collectNormalizationSettings() {
|
||||
normalization_apostrophes_sibilant_possessives: Boolean(form.querySelector('input[name="normalization_apostrophes_sibilant_possessives"]')?.checked),
|
||||
normalization_apostrophes_decades: Boolean(form.querySelector('input[name="normalization_apostrophes_decades"]')?.checked),
|
||||
normalization_apostrophes_leading_elisions: Boolean(form.querySelector('input[name="normalization_apostrophes_leading_elisions"]')?.checked),
|
||||
normalization_contraction_aux_be: Boolean(form.querySelector('input[name="normalization_contraction_aux_be"]')?.checked),
|
||||
normalization_contraction_aux_have: Boolean(form.querySelector('input[name="normalization_contraction_aux_have"]')?.checked),
|
||||
normalization_contraction_modal_will: Boolean(form.querySelector('input[name="normalization_contraction_modal_will"]')?.checked),
|
||||
normalization_contraction_modal_would: Boolean(form.querySelector('input[name="normalization_contraction_modal_would"]')?.checked),
|
||||
normalization_contraction_negation_not: Boolean(form.querySelector('input[name="normalization_contraction_negation_not"]')?.checked),
|
||||
normalization_contraction_let_us: Boolean(form.querySelector('input[name="normalization_contraction_let_us"]')?.checked),
|
||||
normalization_apostrophe_mode: form.querySelector('input[name="normalization_apostrophe_mode"]:checked')?.value || 'spacy',
|
||||
};
|
||||
return normalization;
|
||||
@@ -687,10 +761,13 @@ async function previewNormalization(button) {
|
||||
normalizationAudio.removeAttribute('src');
|
||||
}
|
||||
setStatus(status, 'Building preview…');
|
||||
button.disabled = true;
|
||||
const normalization = collectNormalizationSettings();
|
||||
if (!normalization) {
|
||||
setStatus(status, 'Unable to gather normalization settings.', 'error');
|
||||
return;
|
||||
}
|
||||
const llmFields = collectLLMFields();
|
||||
try {
|
||||
const normalization = collectNormalizationSettings();
|
||||
const llmFields = collectLLMFields();
|
||||
const response = await fetch('/api/normalization/preview', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
@@ -797,5 +874,6 @@ if (form) {
|
||||
initSampleSelector();
|
||||
initActions();
|
||||
initFolderPicker();
|
||||
initContractionModal();
|
||||
initLLMStateWatchers();
|
||||
}
|
||||
|
||||
@@ -324,6 +324,10 @@
|
||||
<span>Expand leading elisions ('tis -> it is)</span>
|
||||
</label>
|
||||
</div>
|
||||
<div class="field field--inline field--actions">
|
||||
<button type="button" class="button button--ghost button--small" data-action="contraction-modal-open">Advanced…</button>
|
||||
<p class="hint">Choose which contraction families are expanded.</p>
|
||||
</div>
|
||||
<div class="field">
|
||||
<span class="field__label">Apostrophe strategy</span>
|
||||
<div class="choices choices--inline">
|
||||
@@ -494,6 +498,49 @@
|
||||
</section>
|
||||
</div>
|
||||
|
||||
<div class="modal" data-role="contraction-modal" hidden>
|
||||
<div class="modal__overlay" data-role="contraction-modal-overlay" tabindex="-1"></div>
|
||||
<div class="modal__content card card--modal" role="dialog" aria-modal="true" aria-labelledby="contraction-modal-title">
|
||||
<header class="modal__header">
|
||||
<p class="modal__eyebrow">Normalization</p>
|
||||
<h2 class="modal__title" id="contraction-modal-title">Contraction Options</h2>
|
||||
<button type="button" class="button button--ghost button--small" data-action="contraction-modal-close" aria-label="Close contraction options">Close</button>
|
||||
</header>
|
||||
<div class="modal__body">
|
||||
<p class="hint">Enable or disable specific contraction families. These controls apply when contraction expansion is enabled.</p>
|
||||
<div class="field field--choices">
|
||||
<label class="toggle-pill">
|
||||
<input type="checkbox" name="normalization_contraction_aux_be" value="true" {% if settings.normalization_contraction_aux_be %}checked{% endif %}>
|
||||
<span>Be verbs ('m, 're, 's -> am/is/are)</span>
|
||||
</label>
|
||||
<label class="toggle-pill">
|
||||
<input type="checkbox" name="normalization_contraction_aux_have" value="true" {% if settings.normalization_contraction_aux_have %}checked{% endif %}>
|
||||
<span>Have auxiliaries ('ve, 'd -> have/had)</span>
|
||||
</label>
|
||||
<label class="toggle-pill">
|
||||
<input type="checkbox" name="normalization_contraction_modal_will" value="true" {% if settings.normalization_contraction_modal_will %}checked{% endif %}>
|
||||
<span>Will modals ('ll -> will)</span>
|
||||
</label>
|
||||
<label class="toggle-pill">
|
||||
<input type="checkbox" name="normalization_contraction_modal_would" value="true" {% if settings.normalization_contraction_modal_would %}checked{% endif %}>
|
||||
<span>Would conditionals ('d -> would)</span>
|
||||
</label>
|
||||
<label class="toggle-pill">
|
||||
<input type="checkbox" name="normalization_contraction_negation_not" value="true" {% if settings.normalization_contraction_negation_not %}checked{% endif %}>
|
||||
<span>Negative forms (can't -> can not)</span>
|
||||
</label>
|
||||
<label class="toggle-pill">
|
||||
<input type="checkbox" name="normalization_contraction_let_us" value="true" {% if settings.normalization_contraction_let_us %}checked{% endif %}>
|
||||
<span>Let us (let's -> let us)</span>
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
<footer class="modal__footer">
|
||||
<button type="button" class="button button--ghost" data-action="contraction-modal-close">Done</button>
|
||||
</footer>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="modal" data-role="audiobookshelf-folder-modal" hidden>
|
||||
<div class="modal__overlay" data-role="audiobookshelf-folder-overlay" tabindex="-1"></div>
|
||||
<div class="modal__content card card--modal folder-picker-modal" role="dialog" aria-modal="true" aria-labelledby="audiobookshelf-folder-picker-title">
|
||||
|
||||
Reference in New Issue
Block a user