refactor: Language Enum with ISO codes

- Language enum: en-US, en-GB, es, fr, hi, it, ja, pt-BR, zh
- Engine-specific mappings (kokoro → single-letter) live in pipeline_factory and synthesize
- spacy_utils uses Language enum keys for model mapping
- split_pattern uses Language enum properties (is_cjk)
- Updated all tests to use ISO codes
This commit is contained in:
Artem Akymenko
2026-07-22 10:54:39 +00:00
parent 4aef73ff85
commit 0805e9fdae
11 changed files with 206 additions and 50 deletions
+2 -2
View File
@@ -13,7 +13,7 @@ from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Dict, List, Optional
from abogen.domain.enums import OutputFormat, SaveMode, SubtitleFormat, SubtitleMode
from abogen.domain.enums import Language, OutputFormat, SaveMode, SubtitleFormat, SubtitleMode
@dataclass
@@ -30,7 +30,7 @@ class ConversionRequest:
original_filename: str = ""
# --- TTS Settings ---
language: str = "a"
language: Language = Language.EN_US
tts_provider: str = "kokoro"
voice: str = "M1"
voice_profile: Optional[str] = None
+1 -1
View File
@@ -135,7 +135,7 @@ def _prepare_tts_context(
# Compute split pattern
split_pattern = get_split_pattern(
str(request.language or "a"),
request.language or Language.EN_US,
request.subtitle_mode or SubtitleMode.DISABLED,
)
+54
View File
@@ -123,3 +123,57 @@ class InputFormat(str, Enum):
return cls(suffix)
except ValueError:
raise ValueError(f"Unsupported input format: {path.suffix!r}. Supported: {[m.value for m in cls]}")
class Language(str, Enum):
"""TTS language code (ISO 639-1 with region where needed).
Each engine (Kokoro, Supertonic) maps these to its own
internal language identifiers.
"""
EN_US = "en-US"
EN_GB = "en-GB"
ES = "es"
FR = "fr"
HI = "hi"
IT = "it"
JA = "ja"
PT_BR = "pt-BR"
ZH = "zh"
@property
def display_name(self) -> str:
"""Human-readable language name."""
_names = {
"en-US": "American English",
"en-GB": "British English",
"es": "Spanish",
"fr": "French",
"hi": "Hindi",
"it": "Italian",
"ja": "Japanese",
"pt-BR": "Brazilian Portuguese",
"zh": "Mandarin Chinese",
}
return _names[self.value]
@property
def is_cjk(self) -> bool:
"""True for CJK languages (Chinese, Japanese)."""
return self in (self.ZH, self.JA)
@property
def supports_subtitle_tokens(self) -> bool:
"""True if this language generates timestamped tokens for subtitles."""
return self in (self.EN_US, self.EN_GB)
@classmethod
def from_str(cls, value: str) -> Language:
"""Parse from user input: ISO code, case-insensitive."""
if isinstance(value, Language):
return value
normalized = value.strip()
for member in cls:
if member.value.lower() == normalized.lower():
return member
raise ValueError(f"Invalid Language: {value!r}. Valid: {[m.value for m in cls]}")
+22 -1
View File
@@ -9,9 +9,23 @@ from __future__ import annotations
from typing import Any, Dict, Optional
from abogen.domain.device import select_device
from abogen.domain.enums import Language
from abogen.domain.voice_resolution import initialize_voice_cache
from abogen.tts_plugin.utils import create_pipeline, is_plugin_registered
# Kokoro-specific language mapping (engine's responsibility)
_KOKORO_LANG_MAP = {
Language.EN_US: "a",
Language.EN_GB: "b",
Language.ES: "e",
Language.FR: "f",
Language.HI: "h",
Language.IT: "i",
Language.JA: "j",
Language.PT_BR: "p",
Language.ZH: "z",
}
def resolve_device(use_gpu: bool) -> str:
"""Determine compute device from job and global config flags."""
@@ -36,11 +50,18 @@ def create_pipeline_for_job(
if not is_plugin_registered(provider):
provider = "kokoro"
# Convert Language enum to Kokoro single-letter code
try:
lang = Language.from_str(language) if not isinstance(language, Language) else language
except ValueError:
lang = Language.EN_US # fallback for unknown languages
kokoro_code = _KOKORO_LANG_MAP.get(lang, "a")
if provider == "supertonic":
return create_pipeline("supertonic")
device = resolve_device(use_gpu)
return create_pipeline("kokoro", lang_code=language, device=device)
return create_pipeline("kokoro", lang_code=kokoro_code, device=device)
def dispose_pipelines(pipelines: Dict[str, Any]) -> None:
+16 -7
View File
@@ -3,7 +3,7 @@ from __future__ import annotations
"""Unified split pattern logic extracted from 3 copies."""
import re
from abogen.domain.enums import SubtitleMode
from abogen.domain.enums import Language, SubtitleMode
PUNCTUATION_SENTENCE = r".!?。!?"
PUNCTUATION_SENTENCE_COMMA = r".!?,。!?、,"
@@ -19,23 +19,32 @@ def get_split_pattern(language: str, subtitle_mode: str) -> str:
Returns:
Split pattern string
"""
try:
lang = Language.from_str(language) if not isinstance(language, Language) else language
except ValueError:
lang = None # unknown language — treat as non-English, non-CJK
try:
mode = SubtitleMode.from_str(subtitle_mode) if not isinstance(subtitle_mode, SubtitleMode) else subtitle_mode
except ValueError:
mode = SubtitleMode.DISABLED
# For English, always use newline splitting only
if language in ("a", "b"):
if lang in (Language.EN_US, Language.EN_GB):
return "\n"
# Determine spacing pattern based on language
spacing = r"\s*" if language in ("z", "j") else r"\s+"
spacing = r"\s*" if lang and lang.is_cjk else r"\s+"
# For CJK languages, when subtitle mode is Disabled or Line, prefer
# punctuation-based splitting instead of plain newline splitting.
if subtitle_mode in (SubtitleMode.DISABLED, SubtitleMode.LINE) and language in ("z", "j"):
if mode in (SubtitleMode.DISABLED, SubtitleMode.LINE) and lang and lang.is_cjk:
return rf"(?<=[{PUNCTUATION_SENTENCE}]){spacing}|\n+"
if subtitle_mode == SubtitleMode.LINE:
if mode == SubtitleMode.LINE:
return "\n"
elif subtitle_mode == SubtitleMode.SENTENCE:
elif mode == SubtitleMode.SENTENCE:
return rf"(?<=[{PUNCTUATION_SENTENCE}]){spacing}|\n+"
elif subtitle_mode == SubtitleMode.SENTENCE_COMMA:
elif mode == SubtitleMode.SENTENCE_COMMA:
return rf"(?<=[{PUNCTUATION_SENTENCE_COMMA}]){spacing}|\n+"
else:
return r"\n+"
+2 -2
View File
@@ -10,7 +10,7 @@ from __future__ import annotations
import re
from typing import List, Optional, Tuple
from abogen.domain.enums import SubtitleMode
from abogen.domain.enums import Language, SubtitleMode
# Punctuation constants for sentence splitting
@@ -53,7 +53,7 @@ def process_subtitle_tokens(
use_spacy_for_english = (
use_spacy_segmentation
and subtitle_mode not in [SubtitleMode.DISABLED, SubtitleMode.LINE]
and lang_code in ["a", "b"]
and lang_code in [Language.EN_US, Language.EN_GB]
and subtitle_mode in [SubtitleMode.SENTENCE, SubtitleMode.SENTENCE_COMMA]
)
+20 -11
View File
@@ -2,21 +2,23 @@
Lazy-loaded spaCy utilities for sentence segmentation.
"""
from abogen.domain.enums import Language
# Cached spaCy module and models (lazy loaded)
_spacy = None
_nlp_cache = {}
# Language code to spaCy model mapping
SPACY_MODELS = {
"a": "en_core_web_sm", # American English
"b": "en_core_web_sm", # British English
"e": "es_core_news_sm", # Spanish
"f": "fr_core_news_sm", # French
"i": "it_core_news_sm", # Italian
"p": "pt_core_news_sm", # Brazilian Portuguese
"z": "zh_core_web_sm", # Mandarin Chinese
"j": "ja_core_news_sm", # Japanese
"h": "xx_sent_ud_sm", # Hindi (multi-language model)
Language.EN_US: "en_core_web_sm",
Language.EN_GB: "en_core_web_sm",
Language.ES: "es_core_news_sm",
Language.FR: "fr_core_news_sm",
Language.IT: "it_core_news_sm",
Language.PT_BR: "pt_core_news_sm",
Language.ZH: "zh_core_web_sm",
Language.JA: "ja_core_news_sm",
Language.HI: "xx_sent_ud_sm",
}
@@ -36,10 +38,9 @@ def _load_spacy():
def get_spacy_model(lang_code, log_callback=None):
"""
Get or load a spaCy model for the given language code.
Downloads the model automatically if not available.
Args:
lang_code: Language code (a, b, e, f, etc.)
lang_code: Language code or Language enum (e.g., "a", "en-US", Language.EN_US)
log_callback: Optional function to log messages
Returns:
@@ -58,6 +59,14 @@ def get_spacy_model(lang_code, log_callback=None):
else:
print(msg)
# Normalize to Language enum
if not isinstance(lang_code, Language):
try:
lang_code = Language.from_str(lang_code)
except ValueError:
log(f"\nspaCy: Unknown language '{lang_code}'...")
return None
# Check if model is cached
if lang_code in _nlp_cache:
return _nlp_cache[lang_code]
+23 -2
View File
@@ -7,8 +7,22 @@ from flask import current_app, send_file
from flask.typing import ResponseReturnValue
from abogen.domain.device import select_device as _select_device
from abogen.domain.enums import Language
from abogen.domain.split_pattern import get_split_pattern
# Kokoro-specific language mapping (engine's responsibility)
_KOKORO_LANG_MAP = {
Language.EN_US: "a",
Language.EN_GB: "b",
Language.ES: "e",
Language.FR: "f",
Language.HI: "h",
Language.IT: "i",
Language.JA: "j",
Language.PT_BR: "p",
Language.ZH: "z",
}
SAMPLE_RATE = 24000
@@ -45,14 +59,21 @@ def _resolve_pipeline(language: str, use_gpu: bool) -> Tuple[Any, bool]:
def get_preview_pipeline(language: str, device: str) -> Any:
key = (language, device)
# Convert Language enum to Kokoro single-letter code
try:
lang = Language.from_str(language) if not isinstance(language, Language) else language
except ValueError:
lang = Language.EN_US
kokoro_code = _KOKORO_LANG_MAP.get(lang, "a")
key = (kokoro_code, device)
with _preview_pipeline_lock:
pipeline = _preview_pipelines.get(key)
if pipeline is not None:
return pipeline
from abogen.tts_plugin.utils import create_pipeline
pipeline = create_pipeline("kokoro", lang_code=language, device=device)
pipeline = create_pipeline("kokoro", lang_code=kokoro_code, device=device)
_preview_pipelines[key] = pipeline
return pipeline