mirror of
https://github.com/denizsafak/abogen.git
synced 2026-07-18 13:40:27 +02:00
feat: Add contraction handling options and UI for normalization settings
This commit is contained in:
+232
-100
@@ -4,7 +4,7 @@ import json
|
|||||||
import re
|
import re
|
||||||
import unicodedata
|
import unicodedata
|
||||||
from fractions import Fraction
|
from fractions import Fraction
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass, field
|
||||||
from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple
|
from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple
|
||||||
try: # pragma: no cover - optional dependency guard
|
try: # pragma: no cover - optional dependency guard
|
||||||
from num2words import num2words
|
from num2words import num2words
|
||||||
@@ -16,6 +16,17 @@ if TYPE_CHECKING: # pragma: no cover - type checking only
|
|||||||
|
|
||||||
from abogen.spacy_contraction_resolver import resolve_ambiguous_contractions
|
from abogen.spacy_contraction_resolver import resolve_ambiguous_contractions
|
||||||
|
|
||||||
|
# ---------- Contraction Category Defaults ----------
|
||||||
|
|
||||||
|
CONTRACTION_CATEGORY_DEFAULTS: Dict[str, bool] = {
|
||||||
|
"contraction_aux_be": True,
|
||||||
|
"contraction_aux_have": True,
|
||||||
|
"contraction_modal_will": True,
|
||||||
|
"contraction_modal_would": True,
|
||||||
|
"contraction_negation_not": True,
|
||||||
|
"contraction_let_us": True,
|
||||||
|
}
|
||||||
|
|
||||||
# ---------- Configuration Dataclass ----------
|
# ---------- Configuration Dataclass ----------
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -38,44 +49,44 @@ class ApostropheConfig:
|
|||||||
protect_cultural_names: bool = True # Always keep O'Brien, D'Angelo, etc.
|
protect_cultural_names: bool = True # Always keep O'Brien, D'Angelo, etc.
|
||||||
convert_numbers: bool = True # Convert grouped numbers such as 12,500 to words
|
convert_numbers: bool = True # Convert grouped numbers such as 12,500 to words
|
||||||
number_lang: str = "en" # num2words language code
|
number_lang: str = "en" # num2words language code
|
||||||
|
contraction_categories: Dict[str, bool] = field(default_factory=lambda: dict(CONTRACTION_CATEGORY_DEFAULTS))
|
||||||
|
|
||||||
|
def is_contraction_enabled(self, category: str) -> bool:
|
||||||
|
return self.contraction_categories.get(category, True)
|
||||||
|
|
||||||
# ---------- Dictionaries / Patterns ----------
|
# ---------- Dictionaries / Patterns ----------
|
||||||
|
|
||||||
# Common contraction expansions (straightforward unambiguous)
|
# Common contraction expansions (type + expansion words)
|
||||||
CONTRACTIONS_EXACT = {
|
CONTRACTION_LEXICON: Dict[str, Tuple[str, Tuple[str, ...]]] = {
|
||||||
"let's": "let us",
|
"let's": ("contraction_let_us", ("let", "us")),
|
||||||
"i'm": "i am",
|
"can't": ("contraction_negation_not", ("can", "not")),
|
||||||
"you're": "you are",
|
"won't": ("contraction_negation_not", ("will", "not")),
|
||||||
"we're": "we are",
|
"don't": ("contraction_negation_not", ("do", "not")),
|
||||||
"they're": "they are",
|
"doesn't": ("contraction_negation_not", ("does", "not")),
|
||||||
"i've": "i have",
|
"didn't": ("contraction_negation_not", ("did", "not")),
|
||||||
"you've": "you have",
|
"isn't": ("contraction_negation_not", ("is", "not")),
|
||||||
"we've": "we have",
|
"aren't": ("contraction_negation_not", ("are", "not")),
|
||||||
"they've": "they have",
|
"wasn't": ("contraction_negation_not", ("was", "not")),
|
||||||
"i'll": "i will",
|
"weren't": ("contraction_negation_not", ("were", "not")),
|
||||||
"you'll": "you will",
|
"haven't": ("contraction_negation_not", ("have", "not")),
|
||||||
"he'll": "he will",
|
"hasn't": ("contraction_negation_not", ("has", "not")),
|
||||||
"she'll": "she will",
|
"hadn't": ("contraction_negation_not", ("had", "not")),
|
||||||
"we'll": "we will",
|
"couldn't": ("contraction_negation_not", ("could", "not")),
|
||||||
"they'll": "they will",
|
"shouldn't": ("contraction_negation_not", ("should", "not")),
|
||||||
"can't": "can not", # or "cannot"
|
"wouldn't": ("contraction_negation_not", ("would", "not")),
|
||||||
"won't": "will not",
|
"mustn't": ("contraction_negation_not", ("must", "not")),
|
||||||
"don't": "do not",
|
"mightn't": ("contraction_negation_not", ("might", "not")),
|
||||||
"doesn't": "does not",
|
"shan't": ("contraction_negation_not", ("shall", "not")),
|
||||||
"didn't": "did not",
|
}
|
||||||
"isn't": "is not",
|
|
||||||
"aren't": "are not",
|
SUFFIX_CONTRACTION_RULES: Tuple[Tuple[str, str, str], ...] = (
|
||||||
"wasn't": "was not",
|
("'ll", "will", "contraction_modal_will"),
|
||||||
"weren't": "were not",
|
("'re", "are", "contraction_aux_be"),
|
||||||
"haven't": "have not",
|
("'ve", "have", "contraction_aux_have"),
|
||||||
"hasn't": "has not",
|
)
|
||||||
"hadn't": "had not",
|
|
||||||
"couldn't": "could not",
|
SUFFIX_CONTRACTION_BASES: Dict[str, Tuple[str, ...]] = {
|
||||||
"shouldn't": "should not",
|
"'m": ("i",),
|
||||||
"wouldn't": "would not",
|
|
||||||
"mustn't": "must not",
|
|
||||||
"mightn't": "might not",
|
|
||||||
"shan't": "shall not",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# For ambiguous 'd and 's we handle separately
|
# For ambiguous 'd and 's we handle separately
|
||||||
@@ -223,7 +234,22 @@ def _replace_fraction(match: re.Match[str], language: str) -> str:
|
|||||||
return match.group(0)
|
return match.group(0)
|
||||||
return spoken
|
return spoken
|
||||||
AMBIGUOUS_D_BASES = {"i","you","he","she","we","they"}
|
AMBIGUOUS_D_BASES = {"i","you","he","she","we","they"}
|
||||||
AMBIGUOUS_S_BASES = {"it","that","what","where","who","when","how","there","here"}
|
AMBIGUOUS_S_BASES = {
|
||||||
|
"it",
|
||||||
|
"that",
|
||||||
|
"what",
|
||||||
|
"where",
|
||||||
|
"who",
|
||||||
|
"when",
|
||||||
|
"how",
|
||||||
|
"there",
|
||||||
|
"here",
|
||||||
|
"he",
|
||||||
|
"she",
|
||||||
|
"we",
|
||||||
|
"they",
|
||||||
|
"you",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def _is_ambiguous_d(token: str) -> bool:
|
def _is_ambiguous_d(token: str) -> bool:
|
||||||
@@ -544,25 +570,127 @@ def is_cultural_name(token: str, cfg: ApostropheConfig) -> bool:
|
|||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _case_preserving_words(original: str, words: Sequence[str]) -> str:
|
||||||
|
if not words:
|
||||||
|
return ""
|
||||||
|
if original.isupper():
|
||||||
|
return " ".join(word.upper() for word in words)
|
||||||
|
|
||||||
|
if original[:1].isupper():
|
||||||
|
adjusted = [words[0].capitalize()]
|
||||||
|
if len(words) > 1:
|
||||||
|
adjusted.extend(words[1:])
|
||||||
|
return " ".join(adjusted)
|
||||||
|
|
||||||
|
return " ".join(words)
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_contraction_policy(
|
||||||
|
token: str,
|
||||||
|
*,
|
||||||
|
category: str,
|
||||||
|
cfg: ApostropheConfig,
|
||||||
|
expand: Callable[[], str],
|
||||||
|
collapse: Optional[str] = None,
|
||||||
|
) -> str:
|
||||||
|
mode = cfg.contraction_mode
|
||||||
|
if mode == "collapse":
|
||||||
|
return collapse if collapse is not None else token.replace("'", "")
|
||||||
|
if mode != "expand":
|
||||||
|
return token
|
||||||
|
if not cfg.is_contraction_enabled(category):
|
||||||
|
return token
|
||||||
|
return expand()
|
||||||
|
|
||||||
|
|
||||||
|
def _assemble_contraction_expansion(base_text: str, surface_text: str, expansion_word: str) -> str:
|
||||||
|
if not expansion_word:
|
||||||
|
return base_text
|
||||||
|
|
||||||
|
if surface_text.isupper() and expansion_word.isalpha():
|
||||||
|
adjusted = expansion_word.upper()
|
||||||
|
elif len(surface_text) > 2 and surface_text[:-2].istitle() and expansion_word:
|
||||||
|
adjusted = expansion_word.lower()
|
||||||
|
else:
|
||||||
|
adjusted = expansion_word
|
||||||
|
|
||||||
|
return f"{base_text} {adjusted}".strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _classify_ambiguous_d(token: str, cfg: ApostropheConfig) -> Tuple[str, str]:
|
||||||
|
base = token[:-2]
|
||||||
|
collapse_value = base + "d"
|
||||||
|
|
||||||
|
if cfg.contraction_mode == "collapse":
|
||||||
|
return "contraction_modal_would", collapse_value
|
||||||
|
if cfg.contraction_mode != "expand":
|
||||||
|
return "contraction_modal_would", token
|
||||||
|
|
||||||
|
mode = cfg.ambiguous_past_modal_mode
|
||||||
|
if mode == "expand_prefer_had":
|
||||||
|
candidates = [
|
||||||
|
("contraction_aux_have", "had"),
|
||||||
|
("contraction_modal_would", "would"),
|
||||||
|
]
|
||||||
|
elif mode == "expand_prefer_would":
|
||||||
|
candidates = [
|
||||||
|
("contraction_modal_would", "would"),
|
||||||
|
("contraction_aux_have", "had"),
|
||||||
|
]
|
||||||
|
else: # contextual
|
||||||
|
candidates = [
|
||||||
|
("contraction_modal_would", "would"),
|
||||||
|
("contraction_aux_have", "had"),
|
||||||
|
]
|
||||||
|
|
||||||
|
for category, word in candidates:
|
||||||
|
if not cfg.is_contraction_enabled(category):
|
||||||
|
continue
|
||||||
|
expanded = _assemble_contraction_expansion(base, token, word)
|
||||||
|
return category, expanded
|
||||||
|
|
||||||
|
# If every category is disabled, leave the token as-is but report default category
|
||||||
|
return candidates[0][0], token
|
||||||
|
|
||||||
|
|
||||||
|
def _classify_ambiguous_s(token: str, cfg: ApostropheConfig) -> Tuple[str, str]:
|
||||||
|
base = token[:-2]
|
||||||
|
|
||||||
|
if cfg.contraction_mode == "collapse":
|
||||||
|
return "contraction_aux_be", base + "s"
|
||||||
|
if cfg.contraction_mode != "expand":
|
||||||
|
return "contraction_aux_be", token
|
||||||
|
|
||||||
|
candidates = [
|
||||||
|
("contraction_aux_be", "is"),
|
||||||
|
("contraction_aux_have", "has"),
|
||||||
|
]
|
||||||
|
|
||||||
|
for category, word in candidates:
|
||||||
|
if not cfg.is_contraction_enabled(category):
|
||||||
|
continue
|
||||||
|
expanded = _assemble_contraction_expansion(base, token, word)
|
||||||
|
return category, expanded
|
||||||
|
|
||||||
|
return candidates[0][0], token
|
||||||
|
|
||||||
def classify_token(token: str, cfg: ApostropheConfig) -> Tuple[str, str]:
|
def classify_token(token: str, cfg: ApostropheConfig) -> Tuple[str, str]:
|
||||||
"""
|
"""
|
||||||
Classify apostrophe usage and propose normalized form.
|
Classify apostrophe usage and propose normalized form.
|
||||||
Returns (category, normalized_token_or_same).
|
Returns (category, normalized_token_or_same).
|
||||||
Categories: contraction, ambiguous_contraction_s, ambiguous_contraction_d,
|
Categories include: contraction_* variants, plural_possessive, irregular_possessive,
|
||||||
plural_possessive, irregular_possessive, sibilant_possessive,
|
sibilant_possessive, singular_possessive, acronym_possessive, decade, leading_elision,
|
||||||
singular_possessive, acronym_possessive, decade, leading_elision,
|
fantasy_internal, cultural_name, other.
|
||||||
fantasy_internal, other
|
|
||||||
"""
|
"""
|
||||||
if "'" not in token:
|
if "'" not in token:
|
||||||
return "other", token
|
return "other", token
|
||||||
|
|
||||||
raw = token
|
|
||||||
low = token.lower()
|
low = token.lower()
|
||||||
|
|
||||||
# 1. Decades
|
# 1. Decades
|
||||||
if DECADE_RE.match(token):
|
if DECADE_RE.match(token):
|
||||||
if cfg.decades_mode == "expand":
|
if cfg.decades_mode == "expand":
|
||||||
# '90s -> 1990s (you could also choose 90s)
|
|
||||||
return "decade", f"19{token[2:4]}s"
|
return "decade", f"19{token[2:4]}s"
|
||||||
return "decade", token
|
return "decade", token
|
||||||
|
|
||||||
@@ -574,103 +702,108 @@ def classify_token(token: str, cfg: ApostropheConfig) -> Tuple[str, str]:
|
|||||||
|
|
||||||
# 3. Ambiguous 'd contractions
|
# 3. Ambiguous 'd contractions
|
||||||
if _is_ambiguous_d(token):
|
if _is_ambiguous_d(token):
|
||||||
base = low[:-2]
|
return _classify_ambiguous_d(token, cfg)
|
||||||
mode = cfg.ambiguous_past_modal_mode
|
|
||||||
if cfg.contraction_mode == "collapse":
|
|
||||||
return "ambiguous_contraction_d", base + "d"
|
|
||||||
if cfg.contraction_mode == "expand":
|
|
||||||
if mode == "expand_prefer_would":
|
|
||||||
return "ambiguous_contraction_d", base + " would"
|
|
||||||
if mode == "expand_prefer_had":
|
|
||||||
return "ambiguous_contraction_d", base + " had"
|
|
||||||
if mode == "contextual":
|
|
||||||
return "ambiguous_contraction_d", base + " would"
|
|
||||||
return "ambiguous_contraction_d", token
|
|
||||||
|
|
||||||
# 4. Ambiguous 's contractions
|
# 4. Ambiguous 's contractions
|
||||||
if _is_ambiguous_s(token):
|
if _is_ambiguous_s(token):
|
||||||
base = low[:-2]
|
return _classify_ambiguous_s(token, cfg)
|
||||||
if cfg.contraction_mode == "expand":
|
|
||||||
return "ambiguous_contraction_s", base + " is"
|
|
||||||
if cfg.contraction_mode == "collapse":
|
|
||||||
return "ambiguous_contraction_s", base + "s"
|
|
||||||
return "ambiguous_contraction_s", token
|
|
||||||
|
|
||||||
# 5. Exact contraction
|
# 5. Lexicon-based contractions
|
||||||
if low in CONTRACTIONS_EXACT:
|
lex_entry = CONTRACTION_LEXICON.get(low)
|
||||||
if cfg.contraction_mode == "expand":
|
if lex_entry is not None:
|
||||||
return "contraction", CONTRACTIONS_EXACT[low]
|
category, words = lex_entry
|
||||||
elif cfg.contraction_mode == "collapse":
|
|
||||||
# collapse: remove apostrophe only (he's -> hes)
|
|
||||||
return "contraction", low.replace("'", "")
|
|
||||||
else:
|
|
||||||
return "contraction", token
|
|
||||||
|
|
||||||
# 6. Irregular possessives (keep or expand logic)
|
def _expand() -> str:
|
||||||
|
return _case_preserving_words(token, words)
|
||||||
|
|
||||||
|
collapse_value = token.replace("'", "")
|
||||||
|
normalized = _apply_contraction_policy(token, category=category, cfg=cfg, expand=_expand, collapse=collapse_value)
|
||||||
|
return category, normalized
|
||||||
|
|
||||||
|
# 6. Suffix contractions ('m handled separately)
|
||||||
|
if low.endswith("'m") and low[:-2] in SUFFIX_CONTRACTION_BASES.get("'m", ()): # pronoun I'm
|
||||||
|
|
||||||
|
def _expand_m() -> str:
|
||||||
|
base = token[:-2]
|
||||||
|
return _assemble_contraction_expansion(base, token, "am")
|
||||||
|
|
||||||
|
normalized = _apply_contraction_policy(
|
||||||
|
token,
|
||||||
|
category="contraction_aux_be",
|
||||||
|
cfg=cfg,
|
||||||
|
expand=_expand_m,
|
||||||
|
collapse=token.replace("'", ""),
|
||||||
|
)
|
||||||
|
return "contraction_aux_be", normalized
|
||||||
|
|
||||||
|
for suffix, append_word, category in SUFFIX_CONTRACTION_RULES:
|
||||||
|
if low.endswith(suffix) and len(token) > len(suffix):
|
||||||
|
base = token[: -len(suffix)]
|
||||||
|
|
||||||
|
def _expand_suffix() -> str:
|
||||||
|
return _assemble_contraction_expansion(base, token, append_word)
|
||||||
|
|
||||||
|
normalized = _apply_contraction_policy(
|
||||||
|
token,
|
||||||
|
category=category,
|
||||||
|
cfg=cfg,
|
||||||
|
expand=_expand_suffix,
|
||||||
|
collapse=token.replace("'", ""),
|
||||||
|
)
|
||||||
|
return category, normalized
|
||||||
|
|
||||||
|
# 7. Irregular possessives (keep or expand logic)
|
||||||
if low in IRREGULAR_POSSESSIVES:
|
if low in IRREGULAR_POSSESSIVES:
|
||||||
if cfg.irregular_possessive_mode == "keep":
|
if cfg.irregular_possessive_mode == "keep":
|
||||||
return "irregular_possessive", token
|
return "irregular_possessive", token
|
||||||
else:
|
|
||||||
# 'expand': we might keep same or optionally add marker
|
|
||||||
return "irregular_possessive", token
|
return "irregular_possessive", token
|
||||||
|
|
||||||
# 7. Plural possessive pattern dogs'
|
# 8. Plural possessive pattern dogs'
|
||||||
if re.match(r"^[A-Za-z0-9]+s'$", token):
|
if re.match(r"^[A-Za-z0-9]+s'$", token):
|
||||||
if cfg.plural_possessive_mode == "collapse":
|
if cfg.plural_possessive_mode == "collapse":
|
||||||
return "plural_possessive", token[:-1] # remove trailing apostrophe
|
return "plural_possessive", token[:-1]
|
||||||
return "plural_possessive", token
|
return "plural_possessive", token
|
||||||
|
|
||||||
# 8. Acronym possessive NASA's
|
# 9. Acronym possessive NASA's
|
||||||
if ACRONYM_POSSESSIVE_RE.match(token):
|
if ACRONYM_POSSESSIVE_RE.match(token):
|
||||||
if cfg.acronym_possessive_mode == "collapse_add_s":
|
if cfg.acronym_possessive_mode == "collapse_add_s":
|
||||||
return "acronym_possessive", token.replace("'", "")
|
return "acronym_possessive", token.replace("'", "")
|
||||||
return "acronym_possessive", token
|
return "acronym_possessive", token
|
||||||
|
|
||||||
# 9. Sibilant singular possessive boss's, church's
|
# 10. Sibilant singular possessive boss's, church's
|
||||||
if low.endswith("'s"):
|
if low.endswith("'s"):
|
||||||
base = token[:-2]
|
base = token[:-2]
|
||||||
if SIBILANT_END_RE.search(base):
|
if SIBILANT_END_RE.search(base):
|
||||||
if cfg.sibilant_possessive_mode == "keep":
|
if cfg.sibilant_possessive_mode == "keep":
|
||||||
return "sibilant_possessive", token
|
return "sibilant_possessive", token
|
||||||
elif cfg.sibilant_possessive_mode == "approx":
|
if cfg.sibilant_possessive_mode == "approx":
|
||||||
# convert to base + "es" (boss's -> bosses)
|
|
||||||
# risk: loses possessive semantics visually
|
|
||||||
return "sibilant_possessive", base + "es"
|
return "sibilant_possessive", base + "es"
|
||||||
elif cfg.sibilant_possessive_mode == "mark":
|
if cfg.sibilant_possessive_mode == "mark":
|
||||||
# remove apostrophe, add IZ marker
|
|
||||||
normalized = base
|
normalized = base
|
||||||
if cfg.add_phoneme_hints:
|
normalized += cfg.sibilant_iz_marker if cfg.add_phoneme_hints else "es"
|
||||||
normalized += cfg.sibilant_iz_marker
|
|
||||||
else:
|
|
||||||
normalized += "es"
|
|
||||||
return "sibilant_possessive", normalized
|
return "sibilant_possessive", normalized
|
||||||
|
|
||||||
# 10. Generic singular possessive (\w+'s)
|
# 11. Generic singular possessive (\w+'s)
|
||||||
if re.match(r"^[A-Za-z0-9]+'s$", token):
|
if re.match(r"^[A-Za-z0-9]+'s$", token):
|
||||||
if cfg.possessive_mode == "collapse":
|
if cfg.possessive_mode == "collapse":
|
||||||
# Just remove apostrophe
|
|
||||||
return "singular_possessive", token.replace("'", "")
|
return "singular_possessive", token.replace("'", "")
|
||||||
return "singular_possessive", token
|
return "singular_possessive", token
|
||||||
|
|
||||||
# 11. Cultural names or fantasy internal
|
# 12. Cultural names or fantasy internal
|
||||||
if is_cultural_name(token, cfg):
|
if is_cultural_name(token, cfg):
|
||||||
return "cultural_name", token
|
return "cultural_name", token
|
||||||
|
|
||||||
# 12. Fantasy internal apostrophes
|
|
||||||
if INTERNAL_APOSTROPHE_RE.search(token):
|
if INTERNAL_APOSTROPHE_RE.search(token):
|
||||||
if cfg.fantasy_mode == "keep":
|
if cfg.fantasy_mode == "keep":
|
||||||
return "fantasy_internal", token
|
return "fantasy_internal", token
|
||||||
elif cfg.fantasy_mode == "mark":
|
if cfg.fantasy_mode == "mark":
|
||||||
out = token + (cfg.fantasy_marker if cfg.add_phoneme_hints else "")
|
out = token + (cfg.fantasy_marker if cfg.add_phoneme_hints else "")
|
||||||
return "fantasy_internal", out
|
return "fantasy_internal", out
|
||||||
elif cfg.fantasy_mode == "collapse_internal":
|
if cfg.fantasy_mode == "collapse_internal":
|
||||||
# Remove internal apostrophes only
|
|
||||||
inner = re.sub(r"(?<=\w)'+(?=\w)", cfg.joiner, token)
|
inner = re.sub(r"(?<=\w)'+(?=\w)", cfg.joiner, token)
|
||||||
return "fantasy_internal", inner
|
return "fantasy_internal", inner
|
||||||
|
|
||||||
# 13. Fallback: treat as other (maybe stray apostrophe)
|
|
||||||
if cfg.fantasy_mode == "collapse_internal":
|
if cfg.fantasy_mode == "collapse_internal":
|
||||||
# Remove any internal apostrophes
|
|
||||||
return "other", token.replace("'", cfg.joiner)
|
return "other", token.replace("'", cfg.joiner)
|
||||||
return "other", token
|
return "other", token
|
||||||
|
|
||||||
@@ -709,13 +842,12 @@ def normalize_apostrophes(text: str, cfg: ApostropheConfig | None = None) -> Tup
|
|||||||
category, norm = classify_token(tok, cfg)
|
category, norm = classify_token(tok, cfg)
|
||||||
|
|
||||||
resolution = contextual_resolutions.get((start, end)) if contextual_resolutions else None
|
resolution = contextual_resolutions.get((start, end)) if contextual_resolutions else None
|
||||||
if resolution is not None:
|
if resolution is not None and cfg.contraction_mode == "expand":
|
||||||
if resolution.category == "ambiguous_contraction_s" and use_contextual_s:
|
if cfg.is_contraction_enabled(resolution.category):
|
||||||
category = resolution.category
|
|
||||||
norm = resolution.expansion
|
|
||||||
elif resolution.category == "ambiguous_contraction_d" and use_contextual_d:
|
|
||||||
category = resolution.category
|
category = resolution.category
|
||||||
norm = resolution.expansion
|
norm = resolution.expansion
|
||||||
|
else:
|
||||||
|
norm = tok
|
||||||
|
|
||||||
results.append((tok, category, norm))
|
results.append((tok, category, norm))
|
||||||
normalized_tokens.append(norm)
|
normalized_tokens.append(norm)
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ from dataclasses import replace
|
|||||||
from functools import lru_cache
|
from functools import lru_cache
|
||||||
from typing import Any, Dict, Mapping, Optional
|
from typing import Any, Dict, Mapping, Optional
|
||||||
|
|
||||||
from abogen.kokoro_text_normalization import ApostropheConfig
|
from abogen.kokoro_text_normalization import ApostropheConfig, CONTRACTION_CATEGORY_DEFAULTS
|
||||||
from abogen.llm_client import LLMConfiguration
|
from abogen.llm_client import LLMConfiguration
|
||||||
from abogen.utils import load_config
|
from abogen.utils import load_config
|
||||||
|
|
||||||
@@ -40,6 +40,21 @@ _SETTINGS_DEFAULTS: Dict[str, Any] = {
|
|||||||
"normalization_apostrophes_decades": True,
|
"normalization_apostrophes_decades": True,
|
||||||
"normalization_apostrophes_leading_elisions": True,
|
"normalization_apostrophes_leading_elisions": True,
|
||||||
"normalization_apostrophe_mode": "spacy",
|
"normalization_apostrophe_mode": "spacy",
|
||||||
|
"normalization_contraction_aux_be": True,
|
||||||
|
"normalization_contraction_aux_have": True,
|
||||||
|
"normalization_contraction_modal_will": True,
|
||||||
|
"normalization_contraction_modal_would": True,
|
||||||
|
"normalization_contraction_negation_not": True,
|
||||||
|
"normalization_contraction_let_us": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
_CONTRACTION_SETTING_MAP: Dict[str, str] = {
|
||||||
|
"normalization_contraction_aux_be": "contraction_aux_be",
|
||||||
|
"normalization_contraction_aux_have": "contraction_aux_have",
|
||||||
|
"normalization_contraction_modal_will": "contraction_modal_will",
|
||||||
|
"normalization_contraction_modal_would": "contraction_modal_would",
|
||||||
|
"normalization_contraction_negation_not": "contraction_negation_not",
|
||||||
|
"normalization_contraction_let_us": "contraction_let_us",
|
||||||
}
|
}
|
||||||
|
|
||||||
_ENVIRONMENT_KEYS: Dict[str, str] = {
|
_ENVIRONMENT_KEYS: Dict[str, str] = {
|
||||||
@@ -168,6 +183,12 @@ def build_apostrophe_config(
|
|||||||
"expand" if settings.get("normalization_apostrophes_leading_elisions", True) else "keep"
|
"expand" if settings.get("normalization_apostrophes_leading_elisions", True) else "keep"
|
||||||
)
|
)
|
||||||
config.ambiguous_past_modal_mode = "contextual" if config.contraction_mode == "expand" else "keep"
|
config.ambiguous_past_modal_mode = "contextual" if config.contraction_mode == "expand" else "keep"
|
||||||
|
category_flags = dict(CONTRACTION_CATEGORY_DEFAULTS)
|
||||||
|
for setting_key, category in _CONTRACTION_SETTING_MAP.items():
|
||||||
|
default_value = bool(_SETTINGS_DEFAULTS.get(setting_key, True))
|
||||||
|
raw_value = settings.get(setting_key, default_value)
|
||||||
|
category_flags[category] = _coerce_bool(raw_value, default_value)
|
||||||
|
config.contraction_categories = category_flags
|
||||||
return config
|
return config
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -132,25 +132,25 @@ def _resolve_apostrophe_s(token: Token) -> Optional[ContractionResolution]:
|
|||||||
surface = token.doc.text[prev.idx : token.idx + len(token.text)]
|
surface = token.doc.text[prev.idx : token.idx + len(token.text)]
|
||||||
|
|
||||||
if prev_lower == "let":
|
if prev_lower == "let":
|
||||||
return _resolution(prev, token, "us", "contraction", "us")
|
return _resolution(prev, token, "us", "contraction_let_us", "us")
|
||||||
|
|
||||||
lemma = token.lemma_.lower()
|
lemma = token.lemma_.lower()
|
||||||
if not lemma:
|
if not lemma:
|
||||||
lemma = "be" if _favors_be(token) else "have" if _favors_have(token) else "be"
|
lemma = "be" if _favors_be(token) else "have" if _favors_have(token) else "be"
|
||||||
|
|
||||||
if lemma == "be":
|
if lemma == "be":
|
||||||
return _resolution(prev, token, "is", "ambiguous_contraction_s", "be")
|
return _resolution(prev, token, "is", "contraction_aux_be", "be")
|
||||||
if lemma == "have":
|
if lemma == "have":
|
||||||
return _resolution(prev, token, "has", "ambiguous_contraction_s", "have")
|
return _resolution(prev, token, "has", "contraction_aux_have", "have")
|
||||||
|
|
||||||
if _favors_have(token):
|
if _favors_have(token):
|
||||||
return _resolution(prev, token, "has", "ambiguous_contraction_s", "have")
|
return _resolution(prev, token, "has", "contraction_aux_have", "have")
|
||||||
|
|
||||||
if _favors_be(token):
|
if _favors_be(token):
|
||||||
return _resolution(prev, token, "is", "ambiguous_contraction_s", "be")
|
return _resolution(prev, token, "is", "contraction_aux_be", "be")
|
||||||
|
|
||||||
# Default to copula expansion.
|
# Default to copula expansion.
|
||||||
return _resolution(prev, token, "is", "ambiguous_contraction_s", lemma or "be")
|
return _resolution(prev, token, "is", "contraction_aux_be", lemma or "be")
|
||||||
|
|
||||||
|
|
||||||
def _resolve_apostrophe_d(token: Token) -> Optional[ContractionResolution]:
|
def _resolve_apostrophe_d(token: Token) -> Optional[ContractionResolution]:
|
||||||
@@ -164,38 +164,35 @@ def _resolve_apostrophe_d(token: Token) -> Optional[ContractionResolution]:
|
|||||||
|
|
||||||
lemma = token.lemma_.lower()
|
lemma = token.lemma_.lower()
|
||||||
tense = set(token.morph.get("Tense"))
|
tense = set(token.morph.get("Tense"))
|
||||||
|
next_content = _next_content_token(token)
|
||||||
|
prefers_had = _context_prefers_had(token)
|
||||||
|
|
||||||
|
if prefers_had:
|
||||||
|
return _resolution(prev, token, "had", "contraction_aux_have", "have")
|
||||||
|
|
||||||
if "Past" in tense and lemma in {"have", "had"}:
|
if "Past" in tense and lemma in {"have", "had"}:
|
||||||
return _resolution(prev, token, "had", "ambiguous_contraction_d", "have")
|
return _resolution(prev, token, "had", "contraction_aux_have", "have")
|
||||||
|
|
||||||
if token.tag_ == "MD" or lemma in {"will", "would", "shall"}:
|
|
||||||
return _resolution(prev, token, "would", "ambiguous_contraction_d", lemma or "will")
|
|
||||||
|
|
||||||
head = token.head if token.head is not None else None
|
|
||||||
if head is not None and head.i > token.i:
|
|
||||||
head_tag = head.tag_
|
|
||||||
head_lemma = head.lemma_.lower()
|
|
||||||
if head_tag in {"VBN", "VBD"} or head_lemma in {"gone", "been", "had"}:
|
|
||||||
return _resolution(prev, token, "had", "ambiguous_contraction_d", "have")
|
|
||||||
if head_lemma == "better":
|
|
||||||
return _resolution(prev, token, "had", "ambiguous_contraction_d", "have")
|
|
||||||
if head_tag == "VB" or head.pos_ in {"VERB", "AUX"}:
|
|
||||||
return _resolution(prev, token, "would", "ambiguous_contraction_d", lemma or "will")
|
|
||||||
|
|
||||||
next_content = _next_content_token(token)
|
|
||||||
if next_content is not None:
|
if next_content is not None:
|
||||||
next_tag = next_content.tag_
|
next_tag = next_content.tag_
|
||||||
if next_tag in {"VBN", "VBD"} or next_content.lemma_.lower() in {"been", "gone", "had"}:
|
next_lemma = next_content.lemma_.lower()
|
||||||
return _resolution(prev, token, "had", "ambiguous_contraction_d", "have")
|
else:
|
||||||
if next_content.lemma_.lower() == "better":
|
next_tag = ""
|
||||||
return _resolution(prev, token, "had", "ambiguous_contraction_d", "have")
|
next_lemma = ""
|
||||||
|
|
||||||
if next_tag == "VB":
|
if next_tag == "VB":
|
||||||
return _resolution(prev, token, "would", "ambiguous_contraction_d", lemma or "will")
|
return _resolution(prev, token, "would", "contraction_modal_would", lemma or "will")
|
||||||
|
|
||||||
|
if token.tag_ == "MD" or lemma in {"will", "would", "shall"}:
|
||||||
|
return _resolution(prev, token, "would", "contraction_modal_would", lemma or "will")
|
||||||
|
|
||||||
|
if next_lemma in {"been", "gone", "had", "better"} or next_tag in {"VBN", "VBD"}:
|
||||||
|
return _resolution(prev, token, "had", "contraction_aux_have", "have")
|
||||||
|
|
||||||
# Fallback: if lemma hints at perfect aspect use "had", otherwise "would".
|
|
||||||
if lemma in {"have", "had"}:
|
if lemma in {"have", "had"}:
|
||||||
return _resolution(prev, token, "had", "ambiguous_contraction_d", lemma)
|
return _resolution(prev, token, "had", "contraction_aux_have", lemma)
|
||||||
return _resolution(prev, token, "would", "ambiguous_contraction_d", lemma or "will")
|
|
||||||
|
return _resolution(prev, token, "would", "contraction_modal_would", lemma or "will")
|
||||||
|
|
||||||
|
|
||||||
def _next_content_token(token: Token) -> Optional[Token]:
|
def _next_content_token(token: Token) -> Optional[Token]:
|
||||||
@@ -229,3 +226,28 @@ def _favors_be(token: Token) -> bool:
|
|||||||
if next_content.tag_ in {"VBG", "JJ", "RB", "DT", "IN"}:
|
if next_content.tag_ in {"VBG", "JJ", "RB", "DT", "IN"}:
|
||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _context_prefers_had(token: Token) -> bool:
|
||||||
|
head = token.head if token.head is not None else None
|
||||||
|
if head is not None and head.i > token.i:
|
||||||
|
head_tag = head.tag_
|
||||||
|
head_lemma = head.lemma_.lower()
|
||||||
|
if head_tag in {"VBN", "VBD"} or head_lemma in {"gone", "been", "had"}:
|
||||||
|
return True
|
||||||
|
if head_lemma == "better":
|
||||||
|
return True
|
||||||
|
|
||||||
|
next_content = _next_content_token(token)
|
||||||
|
if next_content is None:
|
||||||
|
return False
|
||||||
|
next_tag = next_content.tag_
|
||||||
|
next_lemma = next_content.lemma_.lower()
|
||||||
|
if next_tag in {"VBN", "VBD"}:
|
||||||
|
return True
|
||||||
|
if next_lemma in {"been", "gone", "had"}:
|
||||||
|
return True
|
||||||
|
if next_lemma == "better":
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|||||||
@@ -2100,6 +2100,12 @@ BOOLEAN_SETTINGS = {
|
|||||||
"normalization_apostrophes_sibilant_possessives",
|
"normalization_apostrophes_sibilant_possessives",
|
||||||
"normalization_apostrophes_decades",
|
"normalization_apostrophes_decades",
|
||||||
"normalization_apostrophes_leading_elisions",
|
"normalization_apostrophes_leading_elisions",
|
||||||
|
"normalization_contraction_aux_be",
|
||||||
|
"normalization_contraction_aux_have",
|
||||||
|
"normalization_contraction_modal_will",
|
||||||
|
"normalization_contraction_modal_would",
|
||||||
|
"normalization_contraction_negation_not",
|
||||||
|
"normalization_contraction_let_us",
|
||||||
}
|
}
|
||||||
|
|
||||||
FLOAT_SETTINGS = {"silence_between_chapters", "chapter_intro_delay", "llm_timeout"}
|
FLOAT_SETTINGS = {"silence_between_chapters", "chapter_intro_delay", "llm_timeout"}
|
||||||
@@ -2111,6 +2117,12 @@ _APOSTROPHE_OVERRIDE_KEYS = (
|
|||||||
"normalization_apostrophes_sibilant_possessives",
|
"normalization_apostrophes_sibilant_possessives",
|
||||||
"normalization_apostrophes_decades",
|
"normalization_apostrophes_decades",
|
||||||
"normalization_apostrophes_leading_elisions",
|
"normalization_apostrophes_leading_elisions",
|
||||||
|
"normalization_contraction_aux_be",
|
||||||
|
"normalization_contraction_aux_have",
|
||||||
|
"normalization_contraction_modal_will",
|
||||||
|
"normalization_contraction_modal_would",
|
||||||
|
"normalization_contraction_negation_not",
|
||||||
|
"normalization_contraction_let_us",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -3376,6 +3388,12 @@ def api_normalization_preview() -> ResponseReturnValue:
|
|||||||
"normalization_apostrophes_sibilant_possessives",
|
"normalization_apostrophes_sibilant_possessives",
|
||||||
"normalization_apostrophes_decades",
|
"normalization_apostrophes_decades",
|
||||||
"normalization_apostrophes_leading_elisions",
|
"normalization_apostrophes_leading_elisions",
|
||||||
|
"normalization_contraction_aux_be",
|
||||||
|
"normalization_contraction_aux_have",
|
||||||
|
"normalization_contraction_modal_will",
|
||||||
|
"normalization_contraction_modal_would",
|
||||||
|
"normalization_contraction_negation_not",
|
||||||
|
"normalization_contraction_let_us",
|
||||||
)
|
)
|
||||||
for key in boolean_keys:
|
for key in boolean_keys:
|
||||||
if key in normalization_payload:
|
if key in normalization_payload:
|
||||||
|
|||||||
@@ -28,6 +28,11 @@ let folderModalOpener = null;
|
|||||||
let folderModalPreviousFocus = null;
|
let folderModalPreviousFocus = null;
|
||||||
let audiobookshelfFolderSource = [];
|
let audiobookshelfFolderSource = [];
|
||||||
|
|
||||||
|
const contractionModal = document.querySelector('[data-role="contraction-modal"]');
|
||||||
|
const contractionModalOverlay = contractionModal ? contractionModal.querySelector('[data-role="contraction-modal-overlay"]') : null;
|
||||||
|
let contractionModalOpener = null;
|
||||||
|
let contractionModalPreviousFocus = null;
|
||||||
|
|
||||||
function setStatus(target, message, state) {
|
function setStatus(target, message, state) {
|
||||||
if (!target) {
|
if (!target) {
|
||||||
return;
|
return;
|
||||||
@@ -177,6 +182,66 @@ function closeFolderModal(event) {
|
|||||||
folderModalOpener = null;
|
folderModalOpener = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function openContractionModal(opener) {
|
||||||
|
if (!contractionModal) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
contractionModalOpener = opener || null;
|
||||||
|
contractionModalPreviousFocus = document.activeElement instanceof HTMLElement ? document.activeElement : null;
|
||||||
|
contractionModal.hidden = false;
|
||||||
|
contractionModal.dataset.open = 'true';
|
||||||
|
document.body.classList.add('modal-open');
|
||||||
|
const focusTarget = contractionModal.querySelector('input, button, select, textarea');
|
||||||
|
if (focusTarget instanceof HTMLElement) {
|
||||||
|
focusTarget.focus({ preventScroll: true });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function closeContractionModal(event) {
|
||||||
|
if (event) {
|
||||||
|
event.preventDefault();
|
||||||
|
event.stopPropagation();
|
||||||
|
}
|
||||||
|
if (!contractionModal || contractionModal.hidden) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
contractionModal.dataset.open = 'false';
|
||||||
|
contractionModal.hidden = true;
|
||||||
|
document.body.classList.remove('modal-open');
|
||||||
|
const focusTarget =
|
||||||
|
(contractionModalPreviousFocus && typeof contractionModalPreviousFocus.focus === 'function'
|
||||||
|
? contractionModalPreviousFocus
|
||||||
|
: contractionModalOpener) || null;
|
||||||
|
if (focusTarget && typeof focusTarget.focus === 'function') {
|
||||||
|
focusTarget.focus({ preventScroll: true });
|
||||||
|
}
|
||||||
|
contractionModalPreviousFocus = null;
|
||||||
|
contractionModalOpener = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function initContractionModal() {
|
||||||
|
if (!contractionModal) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const openButton = document.querySelector('[data-action="contraction-modal-open"]');
|
||||||
|
if (openButton) {
|
||||||
|
openButton.addEventListener('click', () => openContractionModal(openButton));
|
||||||
|
}
|
||||||
|
const closeButtons = contractionModal.querySelectorAll('[data-action="contraction-modal-close"]');
|
||||||
|
closeButtons.forEach((button) => {
|
||||||
|
button.addEventListener('click', closeContractionModal);
|
||||||
|
});
|
||||||
|
if (contractionModalOverlay) {
|
||||||
|
contractionModalOverlay.addEventListener('click', closeContractionModal);
|
||||||
|
}
|
||||||
|
contractionModal.addEventListener('keydown', (event) => {
|
||||||
|
if (event.key === 'Escape') {
|
||||||
|
event.preventDefault();
|
||||||
|
closeContractionModal(event);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
function renderFolderList(query) {
|
function renderFolderList(query) {
|
||||||
if (!folderList) {
|
if (!folderList) {
|
||||||
return;
|
return;
|
||||||
@@ -461,6 +526,9 @@ async function previewLLM(button) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function collectNormalizationSettings() {
|
function collectNormalizationSettings() {
|
||||||
|
if (!form) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
const normalization = {
|
const normalization = {
|
||||||
normalization_numbers: Boolean(form.querySelector('input[name="normalization_numbers"]')?.checked),
|
normalization_numbers: Boolean(form.querySelector('input[name="normalization_numbers"]')?.checked),
|
||||||
normalization_titles: Boolean(form.querySelector('input[name="normalization_titles"]')?.checked),
|
normalization_titles: Boolean(form.querySelector('input[name="normalization_titles"]')?.checked),
|
||||||
@@ -471,6 +539,12 @@ function collectNormalizationSettings() {
|
|||||||
normalization_apostrophes_sibilant_possessives: Boolean(form.querySelector('input[name="normalization_apostrophes_sibilant_possessives"]')?.checked),
|
normalization_apostrophes_sibilant_possessives: Boolean(form.querySelector('input[name="normalization_apostrophes_sibilant_possessives"]')?.checked),
|
||||||
normalization_apostrophes_decades: Boolean(form.querySelector('input[name="normalization_apostrophes_decades"]')?.checked),
|
normalization_apostrophes_decades: Boolean(form.querySelector('input[name="normalization_apostrophes_decades"]')?.checked),
|
||||||
normalization_apostrophes_leading_elisions: Boolean(form.querySelector('input[name="normalization_apostrophes_leading_elisions"]')?.checked),
|
normalization_apostrophes_leading_elisions: Boolean(form.querySelector('input[name="normalization_apostrophes_leading_elisions"]')?.checked),
|
||||||
|
normalization_contraction_aux_be: Boolean(form.querySelector('input[name="normalization_contraction_aux_be"]')?.checked),
|
||||||
|
normalization_contraction_aux_have: Boolean(form.querySelector('input[name="normalization_contraction_aux_have"]')?.checked),
|
||||||
|
normalization_contraction_modal_will: Boolean(form.querySelector('input[name="normalization_contraction_modal_will"]')?.checked),
|
||||||
|
normalization_contraction_modal_would: Boolean(form.querySelector('input[name="normalization_contraction_modal_would"]')?.checked),
|
||||||
|
normalization_contraction_negation_not: Boolean(form.querySelector('input[name="normalization_contraction_negation_not"]')?.checked),
|
||||||
|
normalization_contraction_let_us: Boolean(form.querySelector('input[name="normalization_contraction_let_us"]')?.checked),
|
||||||
normalization_apostrophe_mode: form.querySelector('input[name="normalization_apostrophe_mode"]:checked')?.value || 'spacy',
|
normalization_apostrophe_mode: form.querySelector('input[name="normalization_apostrophe_mode"]:checked')?.value || 'spacy',
|
||||||
};
|
};
|
||||||
return normalization;
|
return normalization;
|
||||||
@@ -687,10 +761,13 @@ async function previewNormalization(button) {
|
|||||||
normalizationAudio.removeAttribute('src');
|
normalizationAudio.removeAttribute('src');
|
||||||
}
|
}
|
||||||
setStatus(status, 'Building preview…');
|
setStatus(status, 'Building preview…');
|
||||||
button.disabled = true;
|
|
||||||
try {
|
|
||||||
const normalization = collectNormalizationSettings();
|
const normalization = collectNormalizationSettings();
|
||||||
|
if (!normalization) {
|
||||||
|
setStatus(status, 'Unable to gather normalization settings.', 'error');
|
||||||
|
return;
|
||||||
|
}
|
||||||
const llmFields = collectLLMFields();
|
const llmFields = collectLLMFields();
|
||||||
|
try {
|
||||||
const response = await fetch('/api/normalization/preview', {
|
const response = await fetch('/api/normalization/preview', {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: { 'Content-Type': 'application/json' },
|
headers: { 'Content-Type': 'application/json' },
|
||||||
@@ -797,5 +874,6 @@ if (form) {
|
|||||||
initSampleSelector();
|
initSampleSelector();
|
||||||
initActions();
|
initActions();
|
||||||
initFolderPicker();
|
initFolderPicker();
|
||||||
|
initContractionModal();
|
||||||
initLLMStateWatchers();
|
initLLMStateWatchers();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -324,6 +324,10 @@
|
|||||||
<span>Expand leading elisions ('tis -> it is)</span>
|
<span>Expand leading elisions ('tis -> it is)</span>
|
||||||
</label>
|
</label>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="field field--inline field--actions">
|
||||||
|
<button type="button" class="button button--ghost button--small" data-action="contraction-modal-open">Advanced…</button>
|
||||||
|
<p class="hint">Choose which contraction families are expanded.</p>
|
||||||
|
</div>
|
||||||
<div class="field">
|
<div class="field">
|
||||||
<span class="field__label">Apostrophe strategy</span>
|
<span class="field__label">Apostrophe strategy</span>
|
||||||
<div class="choices choices--inline">
|
<div class="choices choices--inline">
|
||||||
@@ -494,6 +498,49 @@
|
|||||||
</section>
|
</section>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div class="modal" data-role="contraction-modal" hidden>
|
||||||
|
<div class="modal__overlay" data-role="contraction-modal-overlay" tabindex="-1"></div>
|
||||||
|
<div class="modal__content card card--modal" role="dialog" aria-modal="true" aria-labelledby="contraction-modal-title">
|
||||||
|
<header class="modal__header">
|
||||||
|
<p class="modal__eyebrow">Normalization</p>
|
||||||
|
<h2 class="modal__title" id="contraction-modal-title">Contraction Options</h2>
|
||||||
|
<button type="button" class="button button--ghost button--small" data-action="contraction-modal-close" aria-label="Close contraction options">Close</button>
|
||||||
|
</header>
|
||||||
|
<div class="modal__body">
|
||||||
|
<p class="hint">Enable or disable specific contraction families. These controls apply when contraction expansion is enabled.</p>
|
||||||
|
<div class="field field--choices">
|
||||||
|
<label class="toggle-pill">
|
||||||
|
<input type="checkbox" name="normalization_contraction_aux_be" value="true" {% if settings.normalization_contraction_aux_be %}checked{% endif %}>
|
||||||
|
<span>Be verbs ('m, 're, 's -> am/is/are)</span>
|
||||||
|
</label>
|
||||||
|
<label class="toggle-pill">
|
||||||
|
<input type="checkbox" name="normalization_contraction_aux_have" value="true" {% if settings.normalization_contraction_aux_have %}checked{% endif %}>
|
||||||
|
<span>Have auxiliaries ('ve, 'd -> have/had)</span>
|
||||||
|
</label>
|
||||||
|
<label class="toggle-pill">
|
||||||
|
<input type="checkbox" name="normalization_contraction_modal_will" value="true" {% if settings.normalization_contraction_modal_will %}checked{% endif %}>
|
||||||
|
<span>Will modals ('ll -> will)</span>
|
||||||
|
</label>
|
||||||
|
<label class="toggle-pill">
|
||||||
|
<input type="checkbox" name="normalization_contraction_modal_would" value="true" {% if settings.normalization_contraction_modal_would %}checked{% endif %}>
|
||||||
|
<span>Would conditionals ('d -> would)</span>
|
||||||
|
</label>
|
||||||
|
<label class="toggle-pill">
|
||||||
|
<input type="checkbox" name="normalization_contraction_negation_not" value="true" {% if settings.normalization_contraction_negation_not %}checked{% endif %}>
|
||||||
|
<span>Negative forms (can't -> can not)</span>
|
||||||
|
</label>
|
||||||
|
<label class="toggle-pill">
|
||||||
|
<input type="checkbox" name="normalization_contraction_let_us" value="true" {% if settings.normalization_contraction_let_us %}checked{% endif %}>
|
||||||
|
<span>Let us (let's -> let us)</span>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<footer class="modal__footer">
|
||||||
|
<button type="button" class="button button--ghost" data-action="contraction-modal-close">Done</button>
|
||||||
|
</footer>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div class="modal" data-role="audiobookshelf-folder-modal" hidden>
|
<div class="modal" data-role="audiobookshelf-folder-modal" hidden>
|
||||||
<div class="modal__overlay" data-role="audiobookshelf-folder-overlay" tabindex="-1"></div>
|
<div class="modal__overlay" data-role="audiobookshelf-folder-overlay" tabindex="-1"></div>
|
||||||
<div class="modal__content card card--modal folder-picker-modal" role="dialog" aria-modal="true" aria-labelledby="audiobookshelf-folder-picker-title">
|
<div class="modal__content card card--modal folder-picker-modal" role="dialog" aria-modal="true" aria-labelledby="audiobookshelf-folder-picker-title">
|
||||||
|
|||||||
@@ -133,3 +133,21 @@ def test_spacy_disambiguates_she_had() -> None:
|
|||||||
def test_spacy_disambiguates_she_would() -> None:
|
def test_spacy_disambiguates_she_would() -> None:
|
||||||
normalized = _normalize_for_pipeline("She'd go if invited.")
|
normalized = _normalize_for_pipeline("She'd go if invited.")
|
||||||
assert "She would go if invited." == normalized
|
assert "She would go if invited." == normalized
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(not SPACY_RESOLVER_AVAILABLE, reason="spaCy model unavailable")
|
||||||
|
def test_sample_sentence_handles_complex_contractions() -> None:
|
||||||
|
sample = "I've heard the captain'll arrive by dusk, but they'd said the same yesterday."
|
||||||
|
normalized = _normalize_for_pipeline(sample)
|
||||||
|
assert (
|
||||||
|
"I have heard the captain will arrive by dusk, but they had said the same yesterday." == normalized
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_modal_will_contractions_can_be_disabled() -> None:
|
||||||
|
sample = "The captain'll arrive at dawn."
|
||||||
|
normalized = _normalize_for_pipeline(
|
||||||
|
sample,
|
||||||
|
normalization_overrides={"normalization_contraction_modal_will": False},
|
||||||
|
)
|
||||||
|
assert "captain'll" in normalized
|
||||||
|
|||||||
Reference in New Issue
Block a user