mirror of
https://github.com/denizsafak/abogen.git
synced 2026-07-18 13:40:27 +02:00
feat: Add stubs for soundfile and static_ffmpeg modules to facilitate testing
This commit is contained in:
@@ -530,6 +530,12 @@ _ROMAN_BREAK_TOKENS = {
|
||||
'"',
|
||||
}
|
||||
|
||||
_ROMAN_CONTEXT_PASSTHROUGH = {"-", "–", "—", ":"}
|
||||
_ROMAN_CONTEXT_COMPOUND_RE = re.compile(
|
||||
r"^(?P<context>[A-Za-z]+)(?P<sep>[-–—:])(?P<roman>[IVXLCDM]+)$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _roman_to_int(token: str) -> Optional[int]:
|
||||
if not token:
|
||||
@@ -577,6 +583,29 @@ def _token_is_cardinal_context(token: str) -> bool:
|
||||
return token.lower() in _ROMAN_CARDINAL_CONTEXTS
|
||||
|
||||
|
||||
def _has_cardinal_leading_context(tokens: Sequence[Tuple[str, int, int]], index: int) -> bool:
|
||||
j = index - 1
|
||||
while j >= 0:
|
||||
token, *_ = tokens[j]
|
||||
stripped = token.strip()
|
||||
if not stripped:
|
||||
j -= 1
|
||||
continue
|
||||
lowered = stripped.lower()
|
||||
if lowered in _ROMAN_CONTEXT_PASSTHROUGH:
|
||||
j -= 1
|
||||
continue
|
||||
if lowered in _ROMAN_BREAK_TOKENS:
|
||||
return False
|
||||
cleaned = lowered.strip("()[]{}\"'.,;!?")
|
||||
if cleaned in _ROMAN_CARDINAL_CONTEXTS:
|
||||
return True
|
||||
if cleaned:
|
||||
return False
|
||||
j -= 1
|
||||
return False
|
||||
|
||||
|
||||
def _should_render_ordinal(
|
||||
tokens: Sequence[Tuple[str, int, int]],
|
||||
index: int,
|
||||
@@ -638,9 +667,35 @@ def _normalize_roman_numerals(text: str, language: str) -> str:
|
||||
parts.append(text[cursor:start])
|
||||
replacement = token
|
||||
|
||||
if len(token) >= 2 and token.isupper() and _ROMAN_TOKEN_RE.match(token):
|
||||
numeric_value = _roman_to_int(token)
|
||||
compound_match = _ROMAN_CONTEXT_COMPOUND_RE.match(token)
|
||||
if compound_match:
|
||||
context_word = compound_match.group("context")
|
||||
separator = compound_match.group("sep")
|
||||
roman_part = compound_match.group("roman")
|
||||
numeric_value = _roman_to_int(roman_part.upper())
|
||||
if (
|
||||
numeric_value is not None
|
||||
and numeric_value <= 200
|
||||
and context_word.lower() in _ROMAN_CARDINAL_CONTEXTS
|
||||
):
|
||||
words = _int_to_words(numeric_value, language)
|
||||
if words:
|
||||
if separator == ":":
|
||||
replacement = f"{context_word}: {words}"
|
||||
else:
|
||||
replacement = f"{context_word} {words}"
|
||||
else:
|
||||
candidate = token.upper()
|
||||
if len(token) >= 2 and _ROMAN_TOKEN_RE.match(candidate):
|
||||
numeric_value = _roman_to_int(candidate)
|
||||
if numeric_value is not None:
|
||||
convert = False
|
||||
if token.isupper():
|
||||
convert = True
|
||||
elif numeric_value <= 200 and _has_cardinal_leading_context(tokens, index):
|
||||
convert = True
|
||||
|
||||
if convert:
|
||||
if _should_render_ordinal(tokens, index, numeric_value):
|
||||
ordinal = _int_to_ordinal_words(numeric_value, language)
|
||||
if ordinal:
|
||||
@@ -657,6 +712,155 @@ def _normalize_roman_numerals(text: str, language: str) -> str:
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
_ACRONYM_ALLOWLIST = {
|
||||
"AI",
|
||||
"API",
|
||||
"CPU",
|
||||
"DIY",
|
||||
"GPU",
|
||||
"HTML",
|
||||
"HTTP",
|
||||
"HTTPS",
|
||||
"ID",
|
||||
"JSON",
|
||||
"MP3",
|
||||
"MP4",
|
||||
"M4B",
|
||||
"NASA",
|
||||
"OCR",
|
||||
"PDF",
|
||||
"SQL",
|
||||
"TV",
|
||||
"TTS",
|
||||
"UK",
|
||||
"UN",
|
||||
"UFO",
|
||||
"OK",
|
||||
"URL",
|
||||
"USA",
|
||||
"US",
|
||||
"VR",
|
||||
}
|
||||
_ROMAN_NUMERAL_LETTERS = frozenset("IVXLCDM")
|
||||
_CAPS_WORD_PATTERN = re.compile(r"[A-Z][A-Z0-9'\u2019-]*")
|
||||
_WORD_PATTERN = re.compile(r"[A-Za-z][A-Za-z0-9'\u2019-]*")
|
||||
_QUOTE_PAIRS = {
|
||||
'"': '"',
|
||||
"“": "”",
|
||||
"„": "“",
|
||||
"«": "»",
|
||||
"‹": "›",
|
||||
}
|
||||
|
||||
|
||||
def _should_preserve_caps_word(word: str) -> bool:
|
||||
letters = "".join(ch for ch in word if ch.isalpha())
|
||||
if not letters:
|
||||
return False
|
||||
base = letters
|
||||
if word.endswith(("'S", "’S")) and len(letters) > 1:
|
||||
base = letters[:-1]
|
||||
upper_base = base.upper()
|
||||
if upper_base in _ACRONYM_ALLOWLIST:
|
||||
return True
|
||||
if all(ch in _ROMAN_NUMERAL_LETTERS for ch in letters.upper()) and len(letters) <= 7:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _should_normalize_caps_segment(segment: str) -> bool:
|
||||
letters = [ch for ch in segment if ch.isalpha()]
|
||||
if not letters:
|
||||
return False
|
||||
if any(ch.islower() for ch in letters):
|
||||
return False
|
||||
if len(letters) <= 1:
|
||||
return False
|
||||
if not any(ch.isspace() for ch in segment) and len(letters) <= 4:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _normalize_caps_segment(segment: str) -> str:
|
||||
if not segment:
|
||||
return segment
|
||||
|
||||
preserve: Dict[str, str] = {}
|
||||
for match in _CAPS_WORD_PATTERN.finditer(segment):
|
||||
word = match.group(0)
|
||||
if _should_preserve_caps_word(word):
|
||||
preserve[word.lower()] = word
|
||||
|
||||
lowered = segment.lower()
|
||||
result_chars: List[str] = []
|
||||
capitalize_next = True
|
||||
for char in lowered:
|
||||
if capitalize_next and char.isalpha():
|
||||
result_chars.append(char.upper())
|
||||
capitalize_next = False
|
||||
else:
|
||||
result_chars.append(char)
|
||||
if char.isalpha():
|
||||
capitalize_next = False
|
||||
if char in ".!?":
|
||||
capitalize_next = True
|
||||
elif char in "\n":
|
||||
capitalize_next = True
|
||||
|
||||
def _restore(match: re.Match[str]) -> str:
|
||||
token = match.group(0)
|
||||
lookup = preserve.get(token.lower())
|
||||
if lookup:
|
||||
return lookup
|
||||
lower = token.lower()
|
||||
if lower == "i":
|
||||
return "I"
|
||||
if lower.startswith("i'") or lower.startswith("i\u2019"):
|
||||
return "I" + token[1:]
|
||||
return token
|
||||
|
||||
return _WORD_PATTERN.sub(_restore, "".join(result_chars))
|
||||
|
||||
|
||||
def _normalize_all_caps_quotes(text: str) -> str:
|
||||
if not text:
|
||||
return text
|
||||
|
||||
builder: List[str] = []
|
||||
index = 0
|
||||
length = len(text)
|
||||
|
||||
while index < length:
|
||||
char = text[index]
|
||||
closing = _QUOTE_PAIRS.get(char)
|
||||
if not closing:
|
||||
builder.append(char)
|
||||
index += 1
|
||||
continue
|
||||
|
||||
cursor = index + 1
|
||||
while cursor < length and text[cursor] != closing:
|
||||
cursor += 1
|
||||
|
||||
if cursor >= length:
|
||||
builder.append(text[index:])
|
||||
break
|
||||
|
||||
body = text[index + 1 : cursor]
|
||||
if _should_normalize_caps_segment(body):
|
||||
normalized = _normalize_caps_segment(body)
|
||||
builder.append(char + normalized + closing)
|
||||
else:
|
||||
builder.append(text[index : cursor + 1])
|
||||
|
||||
index = cursor + 1
|
||||
|
||||
if index < length:
|
||||
builder.append(text[index:])
|
||||
|
||||
return "".join(builder)
|
||||
|
||||
|
||||
def normalize_roman_numeral_titles(
|
||||
titles: Sequence[str],
|
||||
*,
|
||||
@@ -1611,6 +1815,8 @@ def normalize_for_pipeline(
|
||||
normalized = expand_titles_and_suffixes(normalized)
|
||||
if runtime_settings.get("normalization_terminal", True):
|
||||
normalized = ensure_terminal_punctuation(normalized)
|
||||
if runtime_settings.get("normalization_caps_quotes", True):
|
||||
normalized = _normalize_all_caps_quotes(normalized)
|
||||
|
||||
if cfg.add_phoneme_hints:
|
||||
normalized = apply_phoneme_hints(normalized, iz_marker=cfg.sibilant_iz_marker)
|
||||
|
||||
@@ -35,6 +35,7 @@ _SETTINGS_DEFAULTS: Dict[str, Any] = {
|
||||
"normalization_titles": True,
|
||||
"normalization_terminal": True,
|
||||
"normalization_phoneme_hints": True,
|
||||
"normalization_caps_quotes": True,
|
||||
"normalization_apostrophes_contractions": True,
|
||||
"normalization_apostrophes_plural_possessives": True,
|
||||
"normalization_apostrophes_sibilant_possessives": True,
|
||||
|
||||
@@ -2095,6 +2095,7 @@ BOOLEAN_SETTINGS = {
|
||||
"normalization_titles",
|
||||
"normalization_terminal",
|
||||
"normalization_phoneme_hints",
|
||||
"normalization_caps_quotes",
|
||||
"normalization_apostrophes_contractions",
|
||||
"normalization_apostrophes_plural_possessives",
|
||||
"normalization_apostrophes_sibilant_possessives",
|
||||
@@ -2123,6 +2124,7 @@ _APOSTROPHE_OVERRIDE_KEYS = (
|
||||
"normalization_contraction_modal_would",
|
||||
"normalization_contraction_negation_not",
|
||||
"normalization_contraction_let_us",
|
||||
"normalization_caps_quotes",
|
||||
)
|
||||
|
||||
|
||||
@@ -2192,6 +2194,7 @@ def _settings_defaults() -> Dict[str, Any]:
|
||||
"normalization_titles": True,
|
||||
"normalization_terminal": True,
|
||||
"normalization_phoneme_hints": True,
|
||||
"normalization_caps_quotes": True,
|
||||
"normalization_apostrophes_contractions": True,
|
||||
"normalization_apostrophes_plural_possessives": True,
|
||||
"normalization_apostrophes_sibilant_possessives": True,
|
||||
|
||||
@@ -116,9 +116,30 @@
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
if (!response.ok || data.error) {
|
||||
throw new Error(data.error || 'Preview failed.');
|
||||
const contentType = response.headers.get('Content-Type') || '';
|
||||
let data = null;
|
||||
if (contentType.includes('application/json')) {
|
||||
try {
|
||||
data = await response.json();
|
||||
} catch (parseError) {
|
||||
if (!response.ok) {
|
||||
throw new Error('Preview failed.');
|
||||
}
|
||||
throw parseError instanceof Error ? parseError : new Error('Preview failed.');
|
||||
}
|
||||
} else {
|
||||
if (!response.ok) {
|
||||
const fallback = await response.text().catch(() => '');
|
||||
throw new Error(fallback || 'Preview failed.');
|
||||
}
|
||||
throw new Error('Preview failed.');
|
||||
}
|
||||
|
||||
if (!response.ok || (data && data.error)) {
|
||||
throw new Error((data && data.error) || 'Preview failed.');
|
||||
}
|
||||
if (!data || typeof data !== 'object') {
|
||||
throw new Error('Preview failed.');
|
||||
}
|
||||
if (!data.audio_base64) {
|
||||
throw new Error('Preview did not return audio.');
|
||||
|
||||
@@ -108,9 +108,19 @@
|
||||
<span>Voice override</span>
|
||||
<select name="voice">
|
||||
<option value="" selected>No override</option>
|
||||
{% if options.voice_profile_options %}
|
||||
<optgroup label="Saved mixes">
|
||||
{% for profile in options.voice_profile_options %}
|
||||
{% set profile_value = 'profile:' ~ profile.name %}
|
||||
<option value="{{ profile_value }}">{{ profile.name }}{% if profile.language %} · {{ profile.language|upper }}{% endif %}</option>
|
||||
{% endfor %}
|
||||
</optgroup>
|
||||
{% endif %}
|
||||
<optgroup label="Stock voices">
|
||||
{% for voice in options.voice_catalog %}
|
||||
<option value="{{ voice.id }}">{{ voice.display_name }} - {{ voice.language_label }} - {{ voice.gender }}</option>
|
||||
{% endfor %}
|
||||
</optgroup>
|
||||
</select>
|
||||
</label>
|
||||
</div>
|
||||
@@ -166,17 +176,45 @@
|
||||
>
|
||||
</td>
|
||||
<td data-label="Voice">
|
||||
{% set current_voice = override.voice or '' %}
|
||||
{% set known_voice = namespace(value=False) %}
|
||||
{% if current_voice %}
|
||||
{% if current_voice in options.voice_catalog_map %}
|
||||
{% set known_voice.value = True %}
|
||||
{% else %}
|
||||
{% for profile in options.voice_profile_options %}
|
||||
{% if current_voice == 'profile:' ~ profile.name %}
|
||||
{% set known_voice.value = True %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
<select
|
||||
class="overrides-table__select"
|
||||
name="voice"
|
||||
form="{{ form_id }}"
|
||||
>
|
||||
<option value="" {% if not override.voice %}selected{% endif %}>No override</option>
|
||||
<option value="" {% if not current_voice %}selected{% endif %}>No override</option>
|
||||
{% if options.voice_profile_options %}
|
||||
<optgroup label="Saved mixes">
|
||||
{% for profile in options.voice_profile_options %}
|
||||
{% set profile_value = 'profile:' ~ profile.name %}
|
||||
<option value="{{ profile_value }}" {% if current_voice == profile_value %}selected{% endif %}>
|
||||
{{ profile.name }}{% if profile.language %} · {{ profile.language|upper }}{% endif %}
|
||||
</option>
|
||||
{% endfor %}
|
||||
</optgroup>
|
||||
{% endif %}
|
||||
<optgroup label="Stock voices">
|
||||
{% for voice in options.voice_catalog %}
|
||||
<option value="{{ voice.id }}" {% if override.voice == voice.id %}selected{% endif %}>
|
||||
<option value="{{ voice.id }}" {% if current_voice == voice.id %}selected{% endif %}>
|
||||
{{ voice.display_name }} - {{ voice.language_label }} - {{ voice.gender }}
|
||||
</option>
|
||||
{% endfor %}
|
||||
</optgroup>
|
||||
{% if current_voice and not known_voice.value %}
|
||||
<option value="{{ current_voice }}" selected>{{ current_voice }}</option>
|
||||
{% endif %}
|
||||
</select>
|
||||
</td>
|
||||
<td data-label="Usage">{{ override.usage_count }}</td>
|
||||
|
||||
@@ -127,7 +127,12 @@
|
||||
{% set narrator_voice = '' %}
|
||||
{% endif %}
|
||||
{% set normalization_overrides = pending.normalization_overrides if pending and pending.normalization_overrides else {} %}
|
||||
{% set apostrophe_toggles = [
|
||||
{% set normalization_toggles = [
|
||||
{
|
||||
'name': 'normalization_caps_quotes',
|
||||
'label': "Convert ALL CAPS dialogue inside quotes",
|
||||
'value': normalization_overrides.get('normalization_caps_quotes', settings_dict.get('normalization_caps_quotes', True)),
|
||||
},
|
||||
{
|
||||
'name': 'normalization_apostrophes_contractions',
|
||||
'label': "Expand contractions (it's -> it is)",
|
||||
@@ -298,7 +303,7 @@
|
||||
<section class="form-section">
|
||||
<h3 class="form-section__title">Text normalization</h3>
|
||||
<div class="field-grid field-grid--compact">
|
||||
{% for toggle in apostrophe_toggles %}
|
||||
{% for toggle in normalization_toggles %}
|
||||
<div class="field field--stack">
|
||||
<label class="toggle-pill">
|
||||
<input type="hidden" name="{{ toggle.name }}" value="false">
|
||||
|
||||
@@ -302,6 +302,10 @@
|
||||
<input type="checkbox" name="normalization_phoneme_hints" value="true" {% if settings.normalization_phoneme_hints %}checked{% endif %}>
|
||||
<span>Add phoneme hints for possessives</span>
|
||||
</label>
|
||||
<label class="toggle-pill">
|
||||
<input type="checkbox" name="normalization_caps_quotes" value="true" {% if settings.normalization_caps_quotes %}checked{% endif %}>
|
||||
<span>Convert ALL CAPS dialogue inside quotes</span>
|
||||
</label>
|
||||
<span class="field__caption">Apostrophes</span>
|
||||
<label class="toggle-pill">
|
||||
<input type="checkbox" name="normalization_apostrophes_contractions" value="true" {% if settings.normalization_apostrophes_contractions %}checked{% endif %}>
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Test package initialization.
|
||||
|
||||
Provides lightweight fallbacks for optional dependencies so unit tests can run
|
||||
without the full runtime stack.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from types import ModuleType
|
||||
|
||||
|
||||
def _soundfile_write_stub(file_obj, data, samplerate, format="WAV", **_kwargs): # pragma: no cover - stub
|
||||
"""Minimal stand-in for soundfile.write used in tests.
|
||||
|
||||
The real library streams waveform data to disk. Our tests don't exercise
|
||||
audio synthesis, so it's safe to accept the call and write nothing.
|
||||
"""
|
||||
|
||||
if hasattr(file_obj, "write"):
|
||||
try:
|
||||
file_obj.write(b"")
|
||||
except Exception:
|
||||
# Ignore errors from exotic buffers; the real implementation would
|
||||
# write binary samples, so a no-op keeps behavior predictable.
|
||||
pass
|
||||
|
||||
|
||||
if "soundfile" not in sys.modules: # pragma: no cover - import guard
|
||||
stub = ModuleType("soundfile")
|
||||
stub.write = _soundfile_write_stub # type: ignore[attr-defined]
|
||||
sys.modules["soundfile"] = stub
|
||||
|
||||
|
||||
def _static_ffmpeg_add_paths_stub(*_args, **_kwargs) -> None: # pragma: no cover - stub
|
||||
"""Placeholder for static_ffmpeg.add_paths used in tests."""
|
||||
|
||||
|
||||
if "static_ffmpeg" not in sys.modules: # pragma: no cover - import guard
|
||||
ffmpeg_module = ModuleType("static_ffmpeg")
|
||||
ffmpeg_module.add_paths = _static_ffmpeg_add_paths_stub # type: ignore[attr-defined]
|
||||
ffmpeg_run = ModuleType("static_ffmpeg.run")
|
||||
ffmpeg_run.LOCK_FILE = "" # type: ignore[attr-defined]
|
||||
ffmpeg_module.run = ffmpeg_run # type: ignore[attr-defined]
|
||||
sys.modules["static_ffmpeg"] = ffmpeg_module
|
||||
sys.modules["static_ffmpeg.run"] = ffmpeg_run
|
||||
@@ -2,17 +2,33 @@ from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from abogen.kokoro_text_normalization import normalize_roman_numeral_titles
|
||||
from abogen.web.conversion_runner import _normalize_for_pipeline
|
||||
from abogen.kokoro_text_normalization import (
|
||||
DEFAULT_APOSTROPHE_CONFIG,
|
||||
normalize_for_pipeline,
|
||||
normalize_roman_numeral_titles,
|
||||
)
|
||||
from abogen.normalization_settings import (
|
||||
apply_overrides as apply_normalization_overrides,
|
||||
build_apostrophe_config,
|
||||
get_runtime_settings,
|
||||
)
|
||||
from abogen.spacy_contraction_resolver import resolve_ambiguous_contractions
|
||||
|
||||
|
||||
SPACY_RESOLVER_AVAILABLE = bool(resolve_ambiguous_contractions("It's been a long time."))
|
||||
|
||||
|
||||
def _normalize_text(text: str, *, normalization_overrides: dict[str, object] | None = None) -> str:
|
||||
runtime_settings = get_runtime_settings()
|
||||
if normalization_overrides:
|
||||
runtime_settings = apply_normalization_overrides(runtime_settings, normalization_overrides)
|
||||
config = build_apostrophe_config(settings=runtime_settings, base=DEFAULT_APOSTROPHE_CONFIG)
|
||||
return normalize_for_pipeline(text, config=config, settings=runtime_settings)
|
||||
|
||||
|
||||
def test_title_abbreviations_are_expanded():
|
||||
text = "Dr. Watson met Mr. Holmes and Ms. Hudson."
|
||||
normalized = _normalize_for_pipeline(text)
|
||||
normalized = _normalize_text(text)
|
||||
assert "Doctor" in normalized
|
||||
assert "Mister" in normalized
|
||||
assert "Miz" in normalized
|
||||
@@ -20,25 +36,25 @@ def test_title_abbreviations_are_expanded():
|
||||
|
||||
def test_suffix_abbreviations_are_expanded_with_case_preserved():
|
||||
text = "John Doe Jr. spoke to JANE DOE SR. about the estate."
|
||||
normalized = _normalize_for_pipeline(text)
|
||||
normalized = _normalize_text(text)
|
||||
assert "John Doe Junior" in normalized
|
||||
assert "JANE DOE SENIOR" in normalized
|
||||
|
||||
|
||||
def test_missing_terminal_punctuation_is_added():
|
||||
normalized = _normalize_for_pipeline("Chapter 1")
|
||||
normalized = _normalize_text("Chapter 1")
|
||||
assert normalized.endswith(".")
|
||||
|
||||
|
||||
def test_terminal_punctuation_respects_closing_quotes():
|
||||
normalized = _normalize_for_pipeline('"Chapter 1"')
|
||||
normalized = _normalize_text('"Chapter 1"')
|
||||
compact = normalized.replace(" ", "")
|
||||
assert compact.endswith('."')
|
||||
|
||||
|
||||
def test_normalization_preserves_spacing_around_quotes_and_hyphen():
|
||||
sample = "“Still,” said Château-Renaud, “Dr. d’Avrigny, who attends my mother, declares he is in despair about it."
|
||||
normalized = _normalize_for_pipeline(sample)
|
||||
normalized = _normalize_text(sample)
|
||||
|
||||
assert normalized.startswith(
|
||||
"“Still,” said Château-Renaud, “Doctor d'Avrigny, who attends my mother, declares he is in despair about it."
|
||||
@@ -72,70 +88,100 @@ def test_normalize_roman_titles_preserves_separators() -> None:
|
||||
|
||||
|
||||
def test_grouped_numbers_are_spelled_out() -> None:
|
||||
normalized = _normalize_for_pipeline("The vault holds 35,000 credits")
|
||||
normalized = _normalize_text("The vault holds 35,000 credits")
|
||||
assert "thirty-five thousand" in normalized.lower()
|
||||
|
||||
|
||||
def test_numeric_ranges_are_spoken_with_to() -> None:
|
||||
normalized = _normalize_for_pipeline("Chapters 1-3")
|
||||
normalized = _normalize_text("Chapters 1-3")
|
||||
assert "one to three" in normalized.lower()
|
||||
|
||||
|
||||
def test_simple_fractions_are_spoken() -> None:
|
||||
normalized = _normalize_for_pipeline("Add 1/2 cup of sugar")
|
||||
normalized = _normalize_text("Add 1/2 cup of sugar")
|
||||
assert "one half" in normalized.lower()
|
||||
|
||||
|
||||
def test_plain_numbers_are_spelled_out() -> None:
|
||||
normalized = _normalize_for_pipeline("He rolled a 42.")
|
||||
normalized = _normalize_text("He rolled a 42.")
|
||||
assert "forty-two" in normalized.lower()
|
||||
|
||||
|
||||
def test_decimal_numbers_include_point() -> None:
|
||||
normalized = _normalize_for_pipeline("Book 4.5 of the series.")
|
||||
normalized = _normalize_text("Book 4.5 of the series.")
|
||||
assert "four point five" in normalized.lower()
|
||||
|
||||
|
||||
def test_space_separated_numbers_become_ranges() -> None:
|
||||
normalized = _normalize_for_pipeline("Read pages 12 14 tonight.")
|
||||
normalized = _normalize_text("Read pages 12 14 tonight.")
|
||||
assert "pages twelve to fourteen" in normalized.lower()
|
||||
|
||||
|
||||
def test_year_like_numbers_use_common_pronunciation() -> None:
|
||||
normalized = _normalize_for_pipeline("In 1924 the journey began")
|
||||
normalized = _normalize_text("In 1924 the journey began")
|
||||
folded = normalized.lower().replace("-", " ")
|
||||
assert "nineteen twenty four" in folded
|
||||
|
||||
|
||||
def test_early_century_years_use_hundred_format() -> None:
|
||||
normalized = _normalize_for_pipeline("In 1204 the city fell")
|
||||
normalized = _normalize_text("In 1204 the city fell")
|
||||
assert "twelve hundred" in normalized.lower()
|
||||
assert "oh four" in normalized.lower()
|
||||
|
||||
|
||||
def test_roman_numerals_in_titles_are_converted() -> None:
|
||||
normalized = _normalize_for_pipeline("Chapter IV begins now")
|
||||
normalized = _normalize_text("Chapter IV begins now")
|
||||
assert "chapter four" in normalized.lower()
|
||||
|
||||
|
||||
def test_roman_numeral_suffixes_use_ordinals() -> None:
|
||||
normalized = _normalize_for_pipeline("Bob Smith II arrived late")
|
||||
normalized = _normalize_text("Bob Smith II arrived late")
|
||||
assert "bob smith the second" in normalized.lower()
|
||||
|
||||
|
||||
def test_lowercase_roman_after_part_converts_to_cardinal() -> None:
|
||||
normalized = _normalize_text("We studied part iii of the manuscript.")
|
||||
assert "part three" in normalized.lower()
|
||||
|
||||
|
||||
def test_hyphenated_phase_with_roman_is_converted() -> None:
|
||||
normalized = _normalize_text("They executed phase-IV without delay.")
|
||||
assert "phase four" in normalized.lower()
|
||||
|
||||
|
||||
def test_all_caps_quotes_are_sentence_cased() -> None:
|
||||
normalized = _normalize_text('"THIS IS A TEST."')
|
||||
cleaned = normalized.replace('" ', '"')
|
||||
assert '"This is a test."' in cleaned
|
||||
|
||||
|
||||
def test_caps_quote_preserves_acronyms() -> None:
|
||||
normalized = _normalize_text("“THE NASA TEAM ARRIVED.”")
|
||||
assert "“The NASA team arrived.”" in normalized
|
||||
|
||||
|
||||
def test_caps_quote_normalization_respects_override() -> None:
|
||||
normalized = _normalize_text(
|
||||
'"KEEP SHOUTING."',
|
||||
normalization_overrides={"normalization_caps_quotes": False},
|
||||
)
|
||||
cleaned = normalized.replace('" ', '"')
|
||||
assert '"KEEP SHOUTING."' in cleaned
|
||||
|
||||
|
||||
def test_recent_years_split_twenty_style() -> None:
|
||||
normalized = _normalize_for_pipeline("In 2025 we planned ahead")
|
||||
normalized = _normalize_text("In 2025 we planned ahead")
|
||||
folded = normalized.lower().replace("-", " ")
|
||||
assert "twenty twenty five" in folded
|
||||
|
||||
|
||||
def test_two_thousands_use_two_thousand_prefix() -> None:
|
||||
normalized = _normalize_for_pipeline("In 2005 we celebrated")
|
||||
normalized = _normalize_text("In 2005 we celebrated")
|
||||
assert "two thousand five" in normalized.lower()
|
||||
|
||||
|
||||
def test_year_style_can_be_disabled() -> None:
|
||||
normalized = _normalize_for_pipeline(
|
||||
normalized = _normalize_text(
|
||||
"In 2025 we planned ahead",
|
||||
normalization_overrides={"normalization_numbers_year_style": "off"},
|
||||
)
|
||||
@@ -144,7 +190,7 @@ def test_year_style_can_be_disabled() -> None:
|
||||
|
||||
|
||||
def test_contractions_can_be_kept_when_override_disabled() -> None:
|
||||
normalized = _normalize_for_pipeline(
|
||||
normalized = _normalize_text(
|
||||
"It's a good day.",
|
||||
normalization_overrides={"normalization_apostrophes_contractions": False},
|
||||
)
|
||||
@@ -152,7 +198,7 @@ def test_contractions_can_be_kept_when_override_disabled() -> None:
|
||||
|
||||
|
||||
def test_sibilant_possessives_remain_when_marking_disabled() -> None:
|
||||
normalized = _normalize_for_pipeline(
|
||||
normalized = _normalize_text(
|
||||
"The boss's chair wobbled.",
|
||||
normalization_overrides={"normalization_apostrophes_sibilant_possessives": False},
|
||||
)
|
||||
@@ -161,7 +207,7 @@ def test_sibilant_possessives_remain_when_marking_disabled() -> None:
|
||||
|
||||
|
||||
def test_decades_can_skip_expansion_when_disabled() -> None:
|
||||
normalized = _normalize_for_pipeline(
|
||||
normalized = _normalize_text(
|
||||
"Classic hits from the '90s filled the hall.",
|
||||
normalization_overrides={"normalization_apostrophes_decades": False},
|
||||
)
|
||||
@@ -170,32 +216,32 @@ def test_decades_can_skip_expansion_when_disabled() -> None:
|
||||
|
||||
@pytest.mark.skipif(not SPACY_RESOLVER_AVAILABLE, reason="spaCy model unavailable")
|
||||
def test_spacy_disambiguates_it_has_from_context() -> None:
|
||||
normalized = _normalize_for_pipeline("It's been a long time.")
|
||||
normalized = _normalize_text("It's been a long time.")
|
||||
assert "It has been a long time." == normalized
|
||||
|
||||
|
||||
@pytest.mark.skipif(not SPACY_RESOLVER_AVAILABLE, reason="spaCy model unavailable")
|
||||
def test_spacy_disambiguates_it_is_from_context() -> None:
|
||||
normalized = _normalize_for_pipeline("It's cold outside.")
|
||||
normalized = _normalize_text("It's cold outside.")
|
||||
assert "It is cold outside." == normalized
|
||||
|
||||
|
||||
@pytest.mark.skipif(not SPACY_RESOLVER_AVAILABLE, reason="spaCy model unavailable")
|
||||
def test_spacy_disambiguates_she_had() -> None:
|
||||
normalized = _normalize_for_pipeline("She'd left before dawn.")
|
||||
normalized = _normalize_text("She'd left before dawn.")
|
||||
assert "She had left before dawn." == normalized
|
||||
|
||||
|
||||
@pytest.mark.skipif(not SPACY_RESOLVER_AVAILABLE, reason="spaCy model unavailable")
|
||||
def test_spacy_disambiguates_she_would() -> None:
|
||||
normalized = _normalize_for_pipeline("She'd go if invited.")
|
||||
normalized = _normalize_text("She'd go if invited.")
|
||||
assert "She would go if invited." == normalized
|
||||
|
||||
|
||||
@pytest.mark.skipif(not SPACY_RESOLVER_AVAILABLE, reason="spaCy model unavailable")
|
||||
def test_sample_sentence_handles_complex_contractions() -> None:
|
||||
sample = "I've heard the captain'll arrive by dusk, but they'd said the same yesterday."
|
||||
normalized = _normalize_for_pipeline(sample)
|
||||
normalized = _normalize_text(sample)
|
||||
assert (
|
||||
"I have heard the captain will arrive by dusk, but they had said the same yesterday." == normalized
|
||||
)
|
||||
@@ -203,7 +249,7 @@ def test_sample_sentence_handles_complex_contractions() -> None:
|
||||
|
||||
def test_modal_will_contractions_can_be_disabled() -> None:
|
||||
sample = "The captain'll arrive at dawn."
|
||||
normalized = _normalize_for_pipeline(
|
||||
normalized = _normalize_text(
|
||||
sample,
|
||||
normalization_overrides={"normalization_contraction_modal_will": False},
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user