Dymas/abogen
1
0
Fork 0
mirror of https://github.com/denizsafak/abogen.git synced 2026-09-20 11:40:57 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix(segmentation): process TTS segments per sentence, fix all subtitle modes

Browse Source 
Sentence modes processed all text as a whole: Pipeline.__call__ merged every
engine segment back into one (whole text, no per-token timings), producing a
single giant subtitle and whole-text progress logs.

- tts_plugin/types: add TokenTiming, AudioSegment, SynthesizedAudio.segments
- tts_plugin/utils: Pipeline yields one Segment per engine segment (with
  tokens); merged fallback only when engine provides none
- kokoro engine: expose per-segment graphemes/audio + per-word token timings
- supertonic engine: expose per-segment graphemes/audio (no tokens)
- split_pattern: English Sentence/Sentence+Comma engine split is newline-only
  (boundaries applied at subtitle time via spaCy); non-English Sentence+Comma
  with spaCy ON uses spaCy pre-segmentation + newline engine split (no
  commas); spaCy-off fallback keeps comma pattern
- tts_segments: restore inter-segment whitespace on real per-word token
  boundaries only (never FakeToken fallbacks)
- _to_language_enum: accept Language enum input (str(enum) is "Language.ES",
  silently resolved to EN_US and disabled spaCy pre-TTS for every language
  in WebUI)
- pyqt/conversion, utils: replace print with logging
- add AGENTS.md documenting the segmentation/subtitle contract for future
  sessions
- tests: update English split-pattern expectations (1566 passing)
This commit is contained in:
Deniz Şafak
2026-08-20 23:00:15 +03:00
parent 823f5be029
commit 5432de7ac5
10 changed files with 247 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
abogen/domain/conversion_pipeline.py
+26 -9
View File
@@ -60,7 +60,7 @@ def spacy_pre_tts_segmentation(
text_segments is a list of sentences (always at least one element).
active_split_pattern is the regex to use for TTS backend splitting.
"""
from abogen.domain.split_pattern import PUNCTUATION_COMMAS, get_split_pattern
from abogen.domain.split_pattern import get_split_pattern
def _log(msg: str) -> None:
if log_callback:
@@ -99,20 +99,19 @@ def spacy_pre_tts_segmentation(
_log(f"spaCy: Text segmented into {len(spacy_sentences)} sentences...")
# Compute split_pattern override based on subtitle mode
spacing_pattern = r"\s*" if lang_enum in _CJK_LANGS else r"\s+"
if subtitle_mode_str == "Sentence + Comma":
active_split = r"(?<=[{}]){}|\n+".format(PUNCTUATION_COMMAS, spacing_pattern)
else:
# Sentence mode: spaCy already split, only split on newlines
active_split = "\n"
# spaCy already split at sentence boundaries; the engine only needs to
# split on newlines. Commas are never used in the engine split pattern
# for non-English (Sentence + Comma splits at commas only at subtitle
# time, like English).
active_split = "\n"
return spacy_sentences, active_split
def _to_language_enum(lang_code: Any) -> Language:
"""Convert lang_code to Language enum (ISO code or Language enum)."""
if isinstance(lang_code, Language):
return lang_code
try:
return Language.from_str(str(lang_code))
except ValueError:
@@ -174,6 +173,8 @@ def tts_segments(
segment_iter = backend(text, **kwargs)
chunk_start = current_time
prev_tokens: Optional[List[Dict[str, Any]]] = None
prev_was_fallback = True
for segment in segment_iter:
graphemes_raw = getattr(segment, "graphemes", "") or ""
@@ -186,8 +187,10 @@ def tts_segments(
duration = len(audio) / SAMPLE_RATE
tokens_list = getattr(segment, "tokens", [])
was_fallback = False
if not tokens_list and graphemes:
tokens_list = [FakeToken(graphemes, 0, duration)]
was_fallback = True
tokens = [
{
@@ -199,6 +202,18 @@ def tts_segments(
for tok in tokens_list
]
# When the engine splits text on a punctuation pattern, the
# whitespace between segments is consumed by the split. Restore a
# trailing space on the boundary token of the previous segment so
# subtitle processing sees the original spacing (only for real
# per-word tokens; FakeToken fallbacks split via their own logic).
if (
not prev_was_fallback
and prev_tokens
and not prev_tokens[-1].get("whitespace")
):
prev_tokens[-1]["whitespace"] = " "
yield SegmentResult(
graphemes=graphemes,
audio=audio,
@@ -207,6 +222,8 @@ def tts_segments(
tokens=tokens,
)
prev_tokens = tokens
prev_was_fallback = was_fallback
chunk_start += duration
abogen/domain/split_pattern.py
+11 -2
View File
@@ -27,9 +27,18 @@ def get_split_pattern(language: Language, subtitle_mode: str) -> str:
except ValueError:
mode = SubtitleMode.DISABLED
# For English, always use newline splitting only
# English: spaCy is NOT used for pre-TTS segmentation (it is only used
# for post-TTS subtitle boundaries), so sentence boundaries for English
# are applied at subtitle time, not in the TTS engine. Disabled, Line,
# Sentence, and Sentence + Comma all keep newline-only engine splitting.
if language in (Language.EN_US, Language.EN_GB):
return "\n"
if mode in (
SubtitleMode.DISABLED,
SubtitleMode.LINE,
SubtitleMode.SENTENCE,
SubtitleMode.SENTENCE_COMMA,
):
return "\n"
# Determine spacing pattern based on language
spacing = r"\s*" if language.is_cjk else r"\s+"
abogen/pyqt/conversion.py
+13 -16
View File
@@ -1,5 +1,6 @@
import os
import time
import logging
import hashlib # For generating unique cache filenames
from pathlib import Path
from platformdirs import user_desktop_dir
@@ -50,6 +51,8 @@ import abogen.hf_tracker as hf_tracker
import static_ffmpeg
import threading # for efficient waiting
logger = logging.getLogger(__name__)
# Configuration constants
@@ -64,7 +67,6 @@ from abogen.subtitle_utils import (
sanitize_name_for_os,
split_text_by_voice_markers
)
from abogen.domain.split_pattern import PUNCTUATION_COMMAS
class CountdownDialog(QDialog):
"""Base dialog with auto-accept countdown functionality"""
@@ -348,7 +350,7 @@ class ConversionThread(QThread):
return samples_processed
def run(self):
print(
logger.info(
f"\nVoice: {self.voice}\nLanguage: {self.lang_code}\nSpeed: {self.speed}\nGPU: {self.use_gpu}\nFile: {self.file_name}\nSubtitle mode: {self.subtitle_mode}\nOutput format: {self.output_format}\nSave option: {self.save_option}\n"
)
try:
@@ -873,7 +875,6 @@ class ConversionThread(QThread):
)
spacy_sentences = None
active_split_pattern = self.split_pattern
spacing_pattern = r"\s*" if self.lang_code in (Language.JA, Language.ZH) else r"\s+"
# Pre-load spaCy model for English if it will be needed for subtitle generation
if (
@@ -914,15 +915,11 @@ class ConversionThread(QThread):
"grey",
)
)
# For Sentence + Comma mode, still split on commas within spaCy sentences
if self.subtitle_mode == "Sentence + Comma":
active_split_pattern = r"(?<=[{}]){}|\n+".format(
PUNCTUATION_COMMAS, spacing_pattern
)
else:
active_split_pattern = (
"\n" # Use newline splitting for Sentence mode
)
# spaCy already split at sentence boundaries; the
# engine only splits on newlines. Commas are never
# used in the engine split pattern (Sentence +
# Comma splits at commas only at subtitle time).
active_split_pattern = "\n"
else:
self.log_updated.emit(
("\nspaCy: Fallback to default segmentation...", "grey")
@@ -933,10 +930,10 @@ class ConversionThread(QThread):
# Print active split pattern used by the TTS engine once for this batch
try:
print(f"Using split pattern: {active_split_pattern!r}")
logger.info(f"Using split pattern: {active_split_pattern!r}")
except Exception:
# Print must never break processing
print("Using split pattern: (unprintable)")
# Logging must never break processing
logger.warning("Using split pattern: (unprintable)")
for text_segment in text_segments:
def _qt_check_cancel() -> bool:
@@ -1445,7 +1442,7 @@ class VoicePreviewThread(QThread):
return os.path.join(self.cache_dir, filename)
def run(self):
print(
logger.info(
f"\nVoice: {self.voice}\nLanguage: {self.lang_code}\nSpeed: {self.speed}\nGPU: {self.use_gpu}\n"
)
abogen/tts_plugin/types.py
+42
View File
@@ -79,6 +79,44 @@ class SynthesisRequest:
format: AudioFormat
@dataclass(frozen=True)
class TokenTiming:
"""Per-token timing within a synthesized segment.
Attributes:
text: Token text.
whitespace: Whitespace following the token ("" if none).
start: Start time in seconds (relative to segment start).
end: End time in seconds (relative to segment start).
"""
text: str
whitespace: str = ""
start: float = 0.0
end: float = 0.0
@dataclass(frozen=True)
class AudioSegment:
"""One contiguous synthesized segment (sentence-level chunk).
Engines that split the input text (via ``split_pattern``) expose each
chunk as its own AudioSegment so hosts can report per-sentence progress
and build subtitles from per-token timings.
Attributes:
graphemes: The text this segment was synthesized from.
audio: Raw float32 PCM audio bytes for this segment.
sample_rate: Sample rate of ``audio``.
tokens: Per-token timing details, when the engine provides them.
"""
graphemes: str
audio: bytes
sample_rate: int
tokens: tuple[TokenTiming, ...] = ()
@dataclass(frozen=True)
class SynthesizedAudio:
"""Immutable value object for synthesized audio result.
@@ -87,11 +125,15 @@ class SynthesizedAudio:
data: Raw audio bytes.
format: Audio format of the result.
duration: Duration of the audio.
segments: Per-segment details when the engine split the text into
sentence-level chunks (empty for engines that only produce a
single merged result).
"""
data: bytes
format: AudioFormat
duration: Duration
segments: tuple[AudioSegment, ...] = ()
@dataclass(frozen=True)
abogen/tts_plugin/utils.py
+25 -2
View File
@@ -169,15 +169,38 @@ class Pipeline:
)
result = session.synthesize(request)
audio_array = np.frombuffer(result.data, dtype=np.float32)
from dataclasses import dataclass
from dataclasses import dataclass, field
@dataclass
class Token:
text: str
whitespace: str = ""
start_ts: float = 0.0
end_ts: float = 0.0
@dataclass
class Segment:
graphemes: str
audio: np.ndarray
tokens: list[Any] = field(default_factory=list)
if result.segments:
for seg in result.segments:
audio_array = np.frombuffer(seg.audio, dtype=np.float32)
tokens = [
Token(
text=tok.text,
whitespace=tok.whitespace,
start_ts=tok.start,
end_ts=tok.end,
)
for tok in seg.tokens
]
yield Segment(graphemes=seg.graphemes, audio=audio_array, tokens=tokens)
return
audio_array = np.frombuffer(result.data, dtype=np.float32)
yield Segment(graphemes=text, audio=audio_array)
def load_single_voice(self, voice_name: str) -> Any:
abogen/utils.py
+5 -7
View File
@@ -16,6 +16,8 @@ from functools import lru_cache
from dotenv import load_dotenv, find_dotenv
logger = logging.getLogger(__name__)
def _load_environment() -> None:
explicit_path = os.environ.get("ABOGEN_ENV_FILE")
@@ -441,10 +443,6 @@ default_encoding = sys.getfilesystemencoding()
def create_process(cmd, stdin=None, text=True, capture_output=False):
import logging
logger = logging.getLogger(__name__)
# Configure root logger to output to console if not already configured
root = logging.getLogger()
if not root.handlers:
@@ -493,8 +491,8 @@ def create_process(cmd, stdin=None, text=True, capture_output=False):
}
)
# Print the command being executed
print(f"Executing: {cmd if isinstance(cmd, str) else ' '.join(cmd)}")
# Log the command being executed
logger.info(f"Executing: {cmd if isinstance(cmd, str) else ' '.join(cmd)}")
proc = subprocess.Popen(cmd, **kwargs)
@@ -615,7 +613,7 @@ def prevent_sleep_start():
)
else:
# Non-systemd distro or systemd tools not installed: skip inhibition rather than crash
print(
logger.warning(
"systemd-inhibit not found: skipping sleep inhibition on this Linux system."
)