feat: Add stubs for soundfile and static_ffmpeg modules to facilitate testing

This commit is contained in:
JB
2025-11-11 17:00:58 -08:00
parent 504b5ab5e5
commit 028384e6ee
9 changed files with 424 additions and 54 deletions
+217 -11
View File
@@ -530,6 +530,12 @@ _ROMAN_BREAK_TOKENS = {
'"',
}
_ROMAN_CONTEXT_PASSTHROUGH = {"-", "", "", ":"}
_ROMAN_CONTEXT_COMPOUND_RE = re.compile(
r"^(?P<context>[A-Za-z]+)(?P<sep>[-–—:])(?P<roman>[IVXLCDM]+)$",
re.IGNORECASE,
)
def _roman_to_int(token: str) -> Optional[int]:
if not token:
@@ -577,6 +583,29 @@ def _token_is_cardinal_context(token: str) -> bool:
return token.lower() in _ROMAN_CARDINAL_CONTEXTS
def _has_cardinal_leading_context(tokens: Sequence[Tuple[str, int, int]], index: int) -> bool:
j = index - 1
while j >= 0:
token, *_ = tokens[j]
stripped = token.strip()
if not stripped:
j -= 1
continue
lowered = stripped.lower()
if lowered in _ROMAN_CONTEXT_PASSTHROUGH:
j -= 1
continue
if lowered in _ROMAN_BREAK_TOKENS:
return False
cleaned = lowered.strip("()[]{}\"'.,;!?")
if cleaned in _ROMAN_CARDINAL_CONTEXTS:
return True
if cleaned:
return False
j -= 1
return False
def _should_render_ordinal(
tokens: Sequence[Tuple[str, int, int]],
index: int,
@@ -638,17 +667,43 @@ def _normalize_roman_numerals(text: str, language: str) -> str:
parts.append(text[cursor:start])
replacement = token
if len(token) >= 2 and token.isupper() and _ROMAN_TOKEN_RE.match(token):
numeric_value = _roman_to_int(token)
if numeric_value is not None:
if _should_render_ordinal(tokens, index, numeric_value):
ordinal = _int_to_ordinal_words(numeric_value, language)
if ordinal:
replacement = f"the {ordinal}"
else:
words = _int_to_words(numeric_value, language)
if words:
replacement = words
compound_match = _ROMAN_CONTEXT_COMPOUND_RE.match(token)
if compound_match:
context_word = compound_match.group("context")
separator = compound_match.group("sep")
roman_part = compound_match.group("roman")
numeric_value = _roman_to_int(roman_part.upper())
if (
numeric_value is not None
and numeric_value <= 200
and context_word.lower() in _ROMAN_CARDINAL_CONTEXTS
):
words = _int_to_words(numeric_value, language)
if words:
if separator == ":":
replacement = f"{context_word}: {words}"
else:
replacement = f"{context_word} {words}"
else:
candidate = token.upper()
if len(token) >= 2 and _ROMAN_TOKEN_RE.match(candidate):
numeric_value = _roman_to_int(candidate)
if numeric_value is not None:
convert = False
if token.isupper():
convert = True
elif numeric_value <= 200 and _has_cardinal_leading_context(tokens, index):
convert = True
if convert:
if _should_render_ordinal(tokens, index, numeric_value):
ordinal = _int_to_ordinal_words(numeric_value, language)
if ordinal:
replacement = f"the {ordinal}"
else:
words = _int_to_words(numeric_value, language)
if words:
replacement = words
parts.append(replacement)
cursor = end
@@ -657,6 +712,155 @@ def _normalize_roman_numerals(text: str, language: str) -> str:
return "".join(parts)
_ACRONYM_ALLOWLIST = {
"AI",
"API",
"CPU",
"DIY",
"GPU",
"HTML",
"HTTP",
"HTTPS",
"ID",
"JSON",
"MP3",
"MP4",
"M4B",
"NASA",
"OCR",
"PDF",
"SQL",
"TV",
"TTS",
"UK",
"UN",
"UFO",
"OK",
"URL",
"USA",
"US",
"VR",
}
_ROMAN_NUMERAL_LETTERS = frozenset("IVXLCDM")
_CAPS_WORD_PATTERN = re.compile(r"[A-Z][A-Z0-9'\u2019-]*")
_WORD_PATTERN = re.compile(r"[A-Za-z][A-Za-z0-9'\u2019-]*")
_QUOTE_PAIRS = {
'"': '"',
"": "",
"": "",
"«": "»",
"": "",
}
def _should_preserve_caps_word(word: str) -> bool:
letters = "".join(ch for ch in word if ch.isalpha())
if not letters:
return False
base = letters
if word.endswith(("'S", "S")) and len(letters) > 1:
base = letters[:-1]
upper_base = base.upper()
if upper_base in _ACRONYM_ALLOWLIST:
return True
if all(ch in _ROMAN_NUMERAL_LETTERS for ch in letters.upper()) and len(letters) <= 7:
return True
return False
def _should_normalize_caps_segment(segment: str) -> bool:
letters = [ch for ch in segment if ch.isalpha()]
if not letters:
return False
if any(ch.islower() for ch in letters):
return False
if len(letters) <= 1:
return False
if not any(ch.isspace() for ch in segment) and len(letters) <= 4:
return False
return True
def _normalize_caps_segment(segment: str) -> str:
if not segment:
return segment
preserve: Dict[str, str] = {}
for match in _CAPS_WORD_PATTERN.finditer(segment):
word = match.group(0)
if _should_preserve_caps_word(word):
preserve[word.lower()] = word
lowered = segment.lower()
result_chars: List[str] = []
capitalize_next = True
for char in lowered:
if capitalize_next and char.isalpha():
result_chars.append(char.upper())
capitalize_next = False
else:
result_chars.append(char)
if char.isalpha():
capitalize_next = False
if char in ".!?":
capitalize_next = True
elif char in "\n":
capitalize_next = True
def _restore(match: re.Match[str]) -> str:
token = match.group(0)
lookup = preserve.get(token.lower())
if lookup:
return lookup
lower = token.lower()
if lower == "i":
return "I"
if lower.startswith("i'") or lower.startswith("i\u2019"):
return "I" + token[1:]
return token
return _WORD_PATTERN.sub(_restore, "".join(result_chars))
def _normalize_all_caps_quotes(text: str) -> str:
if not text:
return text
builder: List[str] = []
index = 0
length = len(text)
while index < length:
char = text[index]
closing = _QUOTE_PAIRS.get(char)
if not closing:
builder.append(char)
index += 1
continue
cursor = index + 1
while cursor < length and text[cursor] != closing:
cursor += 1
if cursor >= length:
builder.append(text[index:])
break
body = text[index + 1 : cursor]
if _should_normalize_caps_segment(body):
normalized = _normalize_caps_segment(body)
builder.append(char + normalized + closing)
else:
builder.append(text[index : cursor + 1])
index = cursor + 1
if index < length:
builder.append(text[index:])
return "".join(builder)
def normalize_roman_numeral_titles(
titles: Sequence[str],
*,
@@ -1611,6 +1815,8 @@ def normalize_for_pipeline(
normalized = expand_titles_and_suffixes(normalized)
if runtime_settings.get("normalization_terminal", True):
normalized = ensure_terminal_punctuation(normalized)
if runtime_settings.get("normalization_caps_quotes", True):
normalized = _normalize_all_caps_quotes(normalized)
if cfg.add_phoneme_hints:
normalized = apply_phoneme_hints(normalized, iz_marker=cfg.sibilant_iz_marker)
+1
View File
@@ -35,6 +35,7 @@ _SETTINGS_DEFAULTS: Dict[str, Any] = {
"normalization_titles": True,
"normalization_terminal": True,
"normalization_phoneme_hints": True,
"normalization_caps_quotes": True,
"normalization_apostrophes_contractions": True,
"normalization_apostrophes_plural_possessives": True,
"normalization_apostrophes_sibilant_possessives": True,
+3
View File
@@ -2095,6 +2095,7 @@ BOOLEAN_SETTINGS = {
"normalization_titles",
"normalization_terminal",
"normalization_phoneme_hints",
"normalization_caps_quotes",
"normalization_apostrophes_contractions",
"normalization_apostrophes_plural_possessives",
"normalization_apostrophes_sibilant_possessives",
@@ -2123,6 +2124,7 @@ _APOSTROPHE_OVERRIDE_KEYS = (
"normalization_contraction_modal_would",
"normalization_contraction_negation_not",
"normalization_contraction_let_us",
"normalization_caps_quotes",
)
@@ -2192,6 +2194,7 @@ def _settings_defaults() -> Dict[str, Any]:
"normalization_titles": True,
"normalization_terminal": True,
"normalization_phoneme_hints": True,
"normalization_caps_quotes": True,
"normalization_apostrophes_contractions": True,
"normalization_apostrophes_plural_possessives": True,
"normalization_apostrophes_sibilant_possessives": True,
+24 -3
View File
@@ -116,9 +116,30 @@
body: JSON.stringify(payload),
});
const data = await response.json();
if (!response.ok || data.error) {
throw new Error(data.error || 'Preview failed.');
const contentType = response.headers.get('Content-Type') || '';
let data = null;
if (contentType.includes('application/json')) {
try {
data = await response.json();
} catch (parseError) {
if (!response.ok) {
throw new Error('Preview failed.');
}
throw parseError instanceof Error ? parseError : new Error('Preview failed.');
}
} else {
if (!response.ok) {
const fallback = await response.text().catch(() => '');
throw new Error(fallback || 'Preview failed.');
}
throw new Error('Preview failed.');
}
if (!response.ok || (data && data.error)) {
throw new Error((data && data.error) || 'Preview failed.');
}
if (!data || typeof data !== 'object') {
throw new Error('Preview failed.');
}
if (!data.audio_base64) {
throw new Error('Preview did not return audio.');
+47 -9
View File
@@ -108,9 +108,19 @@
<span>Voice override</span>
<select name="voice">
<option value="" selected>No override</option>
{% for voice in options.voice_catalog %}
<option value="{{ voice.id }}">{{ voice.display_name }} - {{ voice.language_label }} - {{ voice.gender }}</option>
{% endfor %}
{% if options.voice_profile_options %}
<optgroup label="Saved mixes">
{% for profile in options.voice_profile_options %}
{% set profile_value = 'profile:' ~ profile.name %}
<option value="{{ profile_value }}">{{ profile.name }}{% if profile.language %} · {{ profile.language|upper }}{% endif %}</option>
{% endfor %}
</optgroup>
{% endif %}
<optgroup label="Stock voices">
{% for voice in options.voice_catalog %}
<option value="{{ voice.id }}">{{ voice.display_name }} - {{ voice.language_label }} - {{ voice.gender }}</option>
{% endfor %}
</optgroup>
</select>
</label>
</div>
@@ -166,17 +176,45 @@
>
</td>
<td data-label="Voice">
{% set current_voice = override.voice or '' %}
{% set known_voice = namespace(value=False) %}
{% if current_voice %}
{% if current_voice in options.voice_catalog_map %}
{% set known_voice.value = True %}
{% else %}
{% for profile in options.voice_profile_options %}
{% if current_voice == 'profile:' ~ profile.name %}
{% set known_voice.value = True %}
{% endif %}
{% endfor %}
{% endif %}
{% endif %}
<select
class="overrides-table__select"
name="voice"
form="{{ form_id }}"
>
<option value="" {% if not override.voice %}selected{% endif %}>No override</option>
{% for voice in options.voice_catalog %}
<option value="{{ voice.id }}" {% if override.voice == voice.id %}selected{% endif %}>
{{ voice.display_name }} - {{ voice.language_label }} - {{ voice.gender }}
</option>
{% endfor %}
<option value="" {% if not current_voice %}selected{% endif %}>No override</option>
{% if options.voice_profile_options %}
<optgroup label="Saved mixes">
{% for profile in options.voice_profile_options %}
{% set profile_value = 'profile:' ~ profile.name %}
<option value="{{ profile_value }}" {% if current_voice == profile_value %}selected{% endif %}>
{{ profile.name }}{% if profile.language %} · {{ profile.language|upper }}{% endif %}
</option>
{% endfor %}
</optgroup>
{% endif %}
<optgroup label="Stock voices">
{% for voice in options.voice_catalog %}
<option value="{{ voice.id }}" {% if current_voice == voice.id %}selected{% endif %}>
{{ voice.display_name }} - {{ voice.language_label }} - {{ voice.gender }}
</option>
{% endfor %}
</optgroup>
{% if current_voice and not known_voice.value %}
<option value="{{ current_voice }}" selected>{{ current_voice }}</option>
{% endif %}
</select>
</td>
<td data-label="Usage">{{ override.usage_count }}</td>
@@ -127,7 +127,12 @@
{% set narrator_voice = '' %}
{% endif %}
{% set normalization_overrides = pending.normalization_overrides if pending and pending.normalization_overrides else {} %}
{% set apostrophe_toggles = [
{% set normalization_toggles = [
{
'name': 'normalization_caps_quotes',
'label': "Convert ALL CAPS dialogue inside quotes",
'value': normalization_overrides.get('normalization_caps_quotes', settings_dict.get('normalization_caps_quotes', True)),
},
{
'name': 'normalization_apostrophes_contractions',
'label': "Expand contractions (it's -> it is)",
@@ -298,7 +303,7 @@
<section class="form-section">
<h3 class="form-section__title">Text normalization</h3>
<div class="field-grid field-grid--compact">
{% for toggle in apostrophe_toggles %}
{% for toggle in normalization_toggles %}
<div class="field field--stack">
<label class="toggle-pill">
<input type="hidden" name="{{ toggle.name }}" value="false">
+4
View File
@@ -302,6 +302,10 @@
<input type="checkbox" name="normalization_phoneme_hints" value="true" {% if settings.normalization_phoneme_hints %}checked{% endif %}>
<span>Add phoneme hints for possessives</span>
</label>
<label class="toggle-pill">
<input type="checkbox" name="normalization_caps_quotes" value="true" {% if settings.normalization_caps_quotes %}checked{% endif %}>
<span>Convert ALL CAPS dialogue inside quotes</span>
</label>
<span class="field__caption">Apostrophes</span>
<label class="toggle-pill">
<input type="checkbox" name="normalization_apostrophes_contractions" value="true" {% if settings.normalization_apostrophes_contractions %}checked{% endif %}>
+46
View File
@@ -0,0 +1,46 @@
"""Test package initialization.
Provides lightweight fallbacks for optional dependencies so unit tests can run
without the full runtime stack.
"""
from __future__ import annotations
import sys
from types import ModuleType
def _soundfile_write_stub(file_obj, data, samplerate, format="WAV", **_kwargs): # pragma: no cover - stub
"""Minimal stand-in for soundfile.write used in tests.
The real library streams waveform data to disk. Our tests don't exercise
audio synthesis, so it's safe to accept the call and write nothing.
"""
if hasattr(file_obj, "write"):
try:
file_obj.write(b"")
except Exception:
# Ignore errors from exotic buffers; the real implementation would
# write binary samples, so a no-op keeps behavior predictable.
pass
if "soundfile" not in sys.modules: # pragma: no cover - import guard
stub = ModuleType("soundfile")
stub.write = _soundfile_write_stub # type: ignore[attr-defined]
sys.modules["soundfile"] = stub
def _static_ffmpeg_add_paths_stub(*_args, **_kwargs) -> None: # pragma: no cover - stub
"""Placeholder for static_ffmpeg.add_paths used in tests."""
if "static_ffmpeg" not in sys.modules: # pragma: no cover - import guard
ffmpeg_module = ModuleType("static_ffmpeg")
ffmpeg_module.add_paths = _static_ffmpeg_add_paths_stub # type: ignore[attr-defined]
ffmpeg_run = ModuleType("static_ffmpeg.run")
ffmpeg_run.LOCK_FILE = "" # type: ignore[attr-defined]
ffmpeg_module.run = ffmpeg_run # type: ignore[attr-defined]
sys.modules["static_ffmpeg"] = ffmpeg_module
sys.modules["static_ffmpeg.run"] = ffmpeg_run
+75 -29
View File
@@ -2,17 +2,33 @@ from __future__ import annotations
import pytest
from abogen.kokoro_text_normalization import normalize_roman_numeral_titles
from abogen.web.conversion_runner import _normalize_for_pipeline
from abogen.kokoro_text_normalization import (
DEFAULT_APOSTROPHE_CONFIG,
normalize_for_pipeline,
normalize_roman_numeral_titles,
)
from abogen.normalization_settings import (
apply_overrides as apply_normalization_overrides,
build_apostrophe_config,
get_runtime_settings,
)
from abogen.spacy_contraction_resolver import resolve_ambiguous_contractions
SPACY_RESOLVER_AVAILABLE = bool(resolve_ambiguous_contractions("It's been a long time."))
def _normalize_text(text: str, *, normalization_overrides: dict[str, object] | None = None) -> str:
runtime_settings = get_runtime_settings()
if normalization_overrides:
runtime_settings = apply_normalization_overrides(runtime_settings, normalization_overrides)
config = build_apostrophe_config(settings=runtime_settings, base=DEFAULT_APOSTROPHE_CONFIG)
return normalize_for_pipeline(text, config=config, settings=runtime_settings)
def test_title_abbreviations_are_expanded():
text = "Dr. Watson met Mr. Holmes and Ms. Hudson."
normalized = _normalize_for_pipeline(text)
normalized = _normalize_text(text)
assert "Doctor" in normalized
assert "Mister" in normalized
assert "Miz" in normalized
@@ -20,25 +36,25 @@ def test_title_abbreviations_are_expanded():
def test_suffix_abbreviations_are_expanded_with_case_preserved():
text = "John Doe Jr. spoke to JANE DOE SR. about the estate."
normalized = _normalize_for_pipeline(text)
normalized = _normalize_text(text)
assert "John Doe Junior" in normalized
assert "JANE DOE SENIOR" in normalized
def test_missing_terminal_punctuation_is_added():
normalized = _normalize_for_pipeline("Chapter 1")
normalized = _normalize_text("Chapter 1")
assert normalized.endswith(".")
def test_terminal_punctuation_respects_closing_quotes():
normalized = _normalize_for_pipeline('"Chapter 1"')
normalized = _normalize_text('"Chapter 1"')
compact = normalized.replace(" ", "")
assert compact.endswith('."')
def test_normalization_preserves_spacing_around_quotes_and_hyphen():
sample = "“Still,” said Château-Renaud, “Dr. dAvrigny, who attends my mother, declares he is in despair about it."
normalized = _normalize_for_pipeline(sample)
normalized = _normalize_text(sample)
assert normalized.startswith(
"“Still,” said Château-Renaud, “Doctor d'Avrigny, who attends my mother, declares he is in despair about it."
@@ -72,70 +88,100 @@ def test_normalize_roman_titles_preserves_separators() -> None:
def test_grouped_numbers_are_spelled_out() -> None:
normalized = _normalize_for_pipeline("The vault holds 35,000 credits")
normalized = _normalize_text("The vault holds 35,000 credits")
assert "thirty-five thousand" in normalized.lower()
def test_numeric_ranges_are_spoken_with_to() -> None:
normalized = _normalize_for_pipeline("Chapters 1-3")
normalized = _normalize_text("Chapters 1-3")
assert "one to three" in normalized.lower()
def test_simple_fractions_are_spoken() -> None:
normalized = _normalize_for_pipeline("Add 1/2 cup of sugar")
normalized = _normalize_text("Add 1/2 cup of sugar")
assert "one half" in normalized.lower()
def test_plain_numbers_are_spelled_out() -> None:
normalized = _normalize_for_pipeline("He rolled a 42.")
normalized = _normalize_text("He rolled a 42.")
assert "forty-two" in normalized.lower()
def test_decimal_numbers_include_point() -> None:
normalized = _normalize_for_pipeline("Book 4.5 of the series.")
normalized = _normalize_text("Book 4.5 of the series.")
assert "four point five" in normalized.lower()
def test_space_separated_numbers_become_ranges() -> None:
normalized = _normalize_for_pipeline("Read pages 12 14 tonight.")
normalized = _normalize_text("Read pages 12 14 tonight.")
assert "pages twelve to fourteen" in normalized.lower()
def test_year_like_numbers_use_common_pronunciation() -> None:
normalized = _normalize_for_pipeline("In 1924 the journey began")
normalized = _normalize_text("In 1924 the journey began")
folded = normalized.lower().replace("-", " ")
assert "nineteen twenty four" in folded
def test_early_century_years_use_hundred_format() -> None:
normalized = _normalize_for_pipeline("In 1204 the city fell")
normalized = _normalize_text("In 1204 the city fell")
assert "twelve hundred" in normalized.lower()
assert "oh four" in normalized.lower()
def test_roman_numerals_in_titles_are_converted() -> None:
normalized = _normalize_for_pipeline("Chapter IV begins now")
normalized = _normalize_text("Chapter IV begins now")
assert "chapter four" in normalized.lower()
def test_roman_numeral_suffixes_use_ordinals() -> None:
normalized = _normalize_for_pipeline("Bob Smith II arrived late")
normalized = _normalize_text("Bob Smith II arrived late")
assert "bob smith the second" in normalized.lower()
def test_lowercase_roman_after_part_converts_to_cardinal() -> None:
normalized = _normalize_text("We studied part iii of the manuscript.")
assert "part three" in normalized.lower()
def test_hyphenated_phase_with_roman_is_converted() -> None:
normalized = _normalize_text("They executed phase-IV without delay.")
assert "phase four" in normalized.lower()
def test_all_caps_quotes_are_sentence_cased() -> None:
normalized = _normalize_text('"THIS IS A TEST."')
cleaned = normalized.replace('" ', '"')
assert '"This is a test."' in cleaned
def test_caps_quote_preserves_acronyms() -> None:
normalized = _normalize_text("“THE NASA TEAM ARRIVED.”")
assert "“The NASA team arrived.”" in normalized
def test_caps_quote_normalization_respects_override() -> None:
normalized = _normalize_text(
'"KEEP SHOUTING."',
normalization_overrides={"normalization_caps_quotes": False},
)
cleaned = normalized.replace('" ', '"')
assert '"KEEP SHOUTING."' in cleaned
def test_recent_years_split_twenty_style() -> None:
normalized = _normalize_for_pipeline("In 2025 we planned ahead")
normalized = _normalize_text("In 2025 we planned ahead")
folded = normalized.lower().replace("-", " ")
assert "twenty twenty five" in folded
def test_two_thousands_use_two_thousand_prefix() -> None:
normalized = _normalize_for_pipeline("In 2005 we celebrated")
normalized = _normalize_text("In 2005 we celebrated")
assert "two thousand five" in normalized.lower()
def test_year_style_can_be_disabled() -> None:
normalized = _normalize_for_pipeline(
normalized = _normalize_text(
"In 2025 we planned ahead",
normalization_overrides={"normalization_numbers_year_style": "off"},
)
@@ -144,7 +190,7 @@ def test_year_style_can_be_disabled() -> None:
def test_contractions_can_be_kept_when_override_disabled() -> None:
normalized = _normalize_for_pipeline(
normalized = _normalize_text(
"It's a good day.",
normalization_overrides={"normalization_apostrophes_contractions": False},
)
@@ -152,7 +198,7 @@ def test_contractions_can_be_kept_when_override_disabled() -> None:
def test_sibilant_possessives_remain_when_marking_disabled() -> None:
normalized = _normalize_for_pipeline(
normalized = _normalize_text(
"The boss's chair wobbled.",
normalization_overrides={"normalization_apostrophes_sibilant_possessives": False},
)
@@ -161,7 +207,7 @@ def test_sibilant_possessives_remain_when_marking_disabled() -> None:
def test_decades_can_skip_expansion_when_disabled() -> None:
normalized = _normalize_for_pipeline(
normalized = _normalize_text(
"Classic hits from the '90s filled the hall.",
normalization_overrides={"normalization_apostrophes_decades": False},
)
@@ -170,32 +216,32 @@ def test_decades_can_skip_expansion_when_disabled() -> None:
@pytest.mark.skipif(not SPACY_RESOLVER_AVAILABLE, reason="spaCy model unavailable")
def test_spacy_disambiguates_it_has_from_context() -> None:
normalized = _normalize_for_pipeline("It's been a long time.")
normalized = _normalize_text("It's been a long time.")
assert "It has been a long time." == normalized
@pytest.mark.skipif(not SPACY_RESOLVER_AVAILABLE, reason="spaCy model unavailable")
def test_spacy_disambiguates_it_is_from_context() -> None:
normalized = _normalize_for_pipeline("It's cold outside.")
normalized = _normalize_text("It's cold outside.")
assert "It is cold outside." == normalized
@pytest.mark.skipif(not SPACY_RESOLVER_AVAILABLE, reason="spaCy model unavailable")
def test_spacy_disambiguates_she_had() -> None:
normalized = _normalize_for_pipeline("She'd left before dawn.")
normalized = _normalize_text("She'd left before dawn.")
assert "She had left before dawn." == normalized
@pytest.mark.skipif(not SPACY_RESOLVER_AVAILABLE, reason="spaCy model unavailable")
def test_spacy_disambiguates_she_would() -> None:
normalized = _normalize_for_pipeline("She'd go if invited.")
normalized = _normalize_text("She'd go if invited.")
assert "She would go if invited." == normalized
@pytest.mark.skipif(not SPACY_RESOLVER_AVAILABLE, reason="spaCy model unavailable")
def test_sample_sentence_handles_complex_contractions() -> None:
sample = "I've heard the captain'll arrive by dusk, but they'd said the same yesterday."
normalized = _normalize_for_pipeline(sample)
normalized = _normalize_text(sample)
assert (
"I have heard the captain will arrive by dusk, but they had said the same yesterday." == normalized
)
@@ -203,7 +249,7 @@ def test_sample_sentence_handles_complex_contractions() -> None:
def test_modal_will_contractions_can_be_disabled() -> None:
sample = "The captain'll arrive at dawn."
normalized = _normalize_for_pipeline(
normalized = _normalize_text(
sample,
normalization_overrides={"normalization_contraction_modal_will": False},
)