From 5432de7ac56ed179bd53c32146147ba3da7921f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Deniz=20=C5=9Eafak?= Date: Thu, 20 Aug 2026 23:00:15 +0300 Subject: [PATCH] fix(segmentation): process TTS segments per sentence, fix all subtitle modes Sentence modes processed all text as a whole: Pipeline.__call__ merged every engine segment back into one (whole text, no per-token timings), producing a single giant subtitle and whole-text progress logs. - tts_plugin/types: add TokenTiming, AudioSegment, SynthesizedAudio.segments - tts_plugin/utils: Pipeline yields one Segment per engine segment (with tokens); merged fallback only when engine provides none - kokoro engine: expose per-segment graphemes/audio + per-word token timings - supertonic engine: expose per-segment graphemes/audio (no tokens) - split_pattern: English Sentence/Sentence+Comma engine split is newline-only (boundaries applied at subtitle time via spaCy); non-English Sentence+Comma with spaCy ON uses spaCy pre-segmentation + newline engine split (no commas); spaCy-off fallback keeps comma pattern - tts_segments: restore inter-segment whitespace on real per-word token boundaries only (never FakeToken fallbacks) - _to_language_enum: accept Language enum input (str(enum) is "Language.ES", silently resolved to EN_US and disabled spaCy pre-TTS for every language in WebUI) - pyqt/conversion, utils: replace print with logging - add AGENTS.md documenting the segmentation/subtitle contract for future sessions - tests: update English split-pattern expectations (1566 passing) --- AGENTS.md | 82 ++++++++++++++++++++++++++++ abogen/domain/conversion_pipeline.py | 35 +++++++++--- abogen/domain/split_pattern.py | 13 ++++- abogen/pyqt/conversion.py | 29 +++++----- abogen/tts_plugin/types.py | 42 ++++++++++++++ abogen/tts_plugin/utils.py | 27 ++++++++- abogen/utils.py | 12 ++-- plugins/kokoro/engine.py | 30 +++++++++- plugins/supertonic/engine.py | 15 ++++- tests/test_split_pattern.py | 2 +- 10 files changed, 247 insertions(+), 40 deletions(-) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..5406e3f --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,82 @@ +# AGENTS.md — Segmentation & Subtitle System Contract + +This document is the source of truth for how text is split for **voice +processing** (TTS engine segmentation) and **subtitle processing**, across +languages, TTS engines, and subtitle modes. It was written after a bug where +sentence modes "processed all text as a whole" (one merged engine segment → +one giant subtitle). **Do not change this behavior without updating this +table.** + +## Voice processing — split pattern passed to the TTS engine + +`get_split_pattern(language, mode)` in `abogen/domain/split_pattern.py` is the +default; the spaCy pre-TTS path overrides it. Both UIs must stay in sync: +`spacy_pre_tts_segmentation` (`abogen/domain/conversion_pipeline.py`, WebUI) +and the inline branch in `abogen/pyqt/conversion.py` (~line 860, PyQt). + +| Subtitle mode | English (en-US/en-GB) | Non-English, spaCy ON | Non-English, spaCy OFF | CJK (ja/zh) | +|---|---|---|---|---| +| Disabled | `\n` | spaCy pre-split, engine `\n` | `\n+` | `(?<=[.!?؟。！？।])\s*\|\n+` | +| Line | `\n` | spaCy pre-split, engine `\n` | `\n` | `(?<=[.!?؟。！？।])\s*\|\n+` | +| Sentence | `\n` | spaCy pre-split, engine `\n` | `(?<=[.!?؟。！？।])\s+\|\n+` | `(?<=[.!?؟。！？।])\s*\|\n+` | +| Sentence + Comma | `\n` | spaCy pre-split, engine `\n` | `(?<=[.!?,؟。！？،،、।])\s+\|\n+` (commas kept) | `(?<=[.!?,؟。！？،،、।])\s*\|\n+` | +| Sentence + Highlighting | `\n+` | `\n+` | `\n+` | `\n+` | +| N words ("5 words") | `\n` (→ Disabled) | `\n+` | `\n+` | Disabled CJK pattern | + +Rules baked into this table: + +- **English voice splitting is ALWAYS newline-only** for Disabled, Line, + Sentence, and Sentence + Comma. English sentence/comma boundaries are + produced ONLY at subtitle time (spaCy post-TTS / regex fallback). Never add + punctuation to the English engine pattern. +- **Non-English + spaCy ON**: spaCy pre-segments the text (pre-TTS); the + engine pattern is `\n` for Sentence AND Sentence + Comma — **never commas**. + spaCy is skipped when the toggle is off, mode is Disabled/Line, or input is + a subtitle file. +- **Non-English + spaCy OFF** (toggle off, spaCy failure, subtitle input): the + default pattern is used — Sentence + Comma KEEPS its commas here. This is + the intentional fallback, not a bug. +- CJK: punctuation-based patterns for Disabled/Line (historical); spacing is + `\s*` (no spaces needed between CJK chars). +- Engine-level extra chunking (applies after the pattern): kokoro English + re-chunks at ~510 phonemes; kokoro non-English at ~400 chars; supertonic + caps each part at 300 chars. + +## Subtitle processing — post-TTS, from tokens + +| Mode | Behavior | +|---|---| +| Disabled | no subtitles | +| Line | one entry per TTS segment (line) | +| Sentence | sentence boundaries: English → spaCy; others → regex on `[.!?…]` | +| Sentence + Comma | sentence + comma boundaries at subtitle time (both languages) — commas never affect voice | +| Sentence + Highlighting | karaoke `{\kf…}` per word, grouped by sentence | +| N words | groups of N words by whitespace counting | + +Token granularity (timing quality): kokoro English emits **per-word tokens** +with timestamps; kokoro non-English and supertonic emit **no tokens** → each +engine segment becomes one FakeToken, split by regex with proportional timing +when it contains multiple sentences. + +## Hard invariants (breaking these reintroduces the original bug) + +1. `Pipeline.__call__` (`abogen/tts_plugin/utils.py`) must yield ONE `Segment` + per engine segment (with tokens) — never merge segments back into the + whole text. `SynthesizedAudio.segments` carries the per-segment data; + engines expose it in `plugins/kokoro/engine.py` and + `plugins/supertonic/engine.py`. +2. `tts_segments` (`abogen/domain/conversion_pipeline.py`) restores trailing + whitespace on segment-boundary tokens ONLY for real per-word tokens, never + for FakeToken fallbacks. +3. `_to_language_enum` must return `lang_code` as-is when it is already a + `Language` enum (`str(Language.ES)` is `"Language.ES"`, which silently + resolved to EN_US and disabled spaCy pre-TTS for every language in WebUI). +4. English must never use spaCy for PRE-TTS segmentation — only for subtitles. + +## Guarded by tests + +- `tests/test_split_pattern.py` — English newline-only; non-English sentence + patterns; CJK behavior. +- `tests/test_domain_conversion_pipeline.py` — `tts_segments` / spaCy + segmentation helpers. +- Full suite: `python -m pytest tests/ -q` (expect 1566+ passing). \ No newline at end of file diff --git a/abogen/domain/conversion_pipeline.py b/abogen/domain/conversion_pipeline.py index ad4e56f..c2b6077 100644 --- a/abogen/domain/conversion_pipeline.py +++ b/abogen/domain/conversion_pipeline.py @@ -60,7 +60,7 @@ def spacy_pre_tts_segmentation( text_segments is a list of sentences (always at least one element). active_split_pattern is the regex to use for TTS backend splitting. """ - from abogen.domain.split_pattern import PUNCTUATION_COMMAS, get_split_pattern + from abogen.domain.split_pattern import get_split_pattern def _log(msg: str) -> None: if log_callback: @@ -99,20 +99,19 @@ def spacy_pre_tts_segmentation( _log(f"spaCy: Text segmented into {len(spacy_sentences)} sentences...") - # Compute split_pattern override based on subtitle mode - spacing_pattern = r"\s*" if lang_enum in _CJK_LANGS else r"\s+" - - if subtitle_mode_str == "Sentence + Comma": - active_split = r"(?<=[{}]){}|\n+".format(PUNCTUATION_COMMAS, spacing_pattern) - else: - # Sentence mode: spaCy already split, only split on newlines - active_split = "\n" + # spaCy already split at sentence boundaries; the engine only needs to + # split on newlines. Commas are never used in the engine split pattern + # for non-English (Sentence + Comma splits at commas only at subtitle + # time, like English). + active_split = "\n" return spacy_sentences, active_split def _to_language_enum(lang_code: Any) -> Language: """Convert lang_code to Language enum (ISO code or Language enum).""" + if isinstance(lang_code, Language): + return lang_code try: return Language.from_str(str(lang_code)) except ValueError: @@ -174,6 +173,8 @@ def tts_segments( segment_iter = backend(text, **kwargs) chunk_start = current_time + prev_tokens: Optional[List[Dict[str, Any]]] = None + prev_was_fallback = True for segment in segment_iter: graphemes_raw = getattr(segment, "graphemes", "") or "" @@ -186,8 +187,10 @@ def tts_segments( duration = len(audio) / SAMPLE_RATE tokens_list = getattr(segment, "tokens", []) + was_fallback = False if not tokens_list and graphemes: tokens_list = [FakeToken(graphemes, 0, duration)] + was_fallback = True tokens = [ { @@ -199,6 +202,18 @@ def tts_segments( for tok in tokens_list ] + # When the engine splits text on a punctuation pattern, the + # whitespace between segments is consumed by the split. Restore a + # trailing space on the boundary token of the previous segment so + # subtitle processing sees the original spacing (only for real + # per-word tokens; FakeToken fallbacks split via their own logic). + if ( + not prev_was_fallback + and prev_tokens + and not prev_tokens[-1].get("whitespace") + ): + prev_tokens[-1]["whitespace"] = " " + yield SegmentResult( graphemes=graphemes, audio=audio, @@ -207,6 +222,8 @@ def tts_segments( tokens=tokens, ) + prev_tokens = tokens + prev_was_fallback = was_fallback chunk_start += duration diff --git a/abogen/domain/split_pattern.py b/abogen/domain/split_pattern.py index 616d918..9f2bf5d 100644 --- a/abogen/domain/split_pattern.py +++ b/abogen/domain/split_pattern.py @@ -27,9 +27,18 @@ def get_split_pattern(language: Language, subtitle_mode: str) -> str: except ValueError: mode = SubtitleMode.DISABLED - # For English, always use newline splitting only + # English: spaCy is NOT used for pre-TTS segmentation (it is only used + # for post-TTS subtitle boundaries), so sentence boundaries for English + # are applied at subtitle time, not in the TTS engine. Disabled, Line, + # Sentence, and Sentence + Comma all keep newline-only engine splitting. if language in (Language.EN_US, Language.EN_GB): - return "\n" + if mode in ( + SubtitleMode.DISABLED, + SubtitleMode.LINE, + SubtitleMode.SENTENCE, + SubtitleMode.SENTENCE_COMMA, + ): + return "\n" # Determine spacing pattern based on language spacing = r"\s*" if language.is_cjk else r"\s+" diff --git a/abogen/pyqt/conversion.py b/abogen/pyqt/conversion.py index ff3a829..ceaeda8 100644 --- a/abogen/pyqt/conversion.py +++ b/abogen/pyqt/conversion.py @@ -1,5 +1,6 @@ import os import time +import logging import hashlib # For generating unique cache filenames from pathlib import Path from platformdirs import user_desktop_dir @@ -50,6 +51,8 @@ import abogen.hf_tracker as hf_tracker import static_ffmpeg import threading # for efficient waiting +logger = logging.getLogger(__name__) + # Configuration constants @@ -64,7 +67,6 @@ from abogen.subtitle_utils import ( sanitize_name_for_os, split_text_by_voice_markers ) -from abogen.domain.split_pattern import PUNCTUATION_COMMAS class CountdownDialog(QDialog): """Base dialog with auto-accept countdown functionality""" @@ -348,7 +350,7 @@ class ConversionThread(QThread): return samples_processed def run(self): - print( + logger.info( f"\nVoice: {self.voice}\nLanguage: {self.lang_code}\nSpeed: {self.speed}\nGPU: {self.use_gpu}\nFile: {self.file_name}\nSubtitle mode: {self.subtitle_mode}\nOutput format: {self.output_format}\nSave option: {self.save_option}\n" ) try: @@ -873,7 +875,6 @@ class ConversionThread(QThread): ) spacy_sentences = None active_split_pattern = self.split_pattern - spacing_pattern = r"\s*" if self.lang_code in (Language.JA, Language.ZH) else r"\s+" # Pre-load spaCy model for English if it will be needed for subtitle generation if ( @@ -914,15 +915,11 @@ class ConversionThread(QThread): "grey", ) ) - # For Sentence + Comma mode, still split on commas within spaCy sentences - if self.subtitle_mode == "Sentence + Comma": - active_split_pattern = r"(?<=[{}]){}|\n+".format( - PUNCTUATION_COMMAS, spacing_pattern - ) - else: - active_split_pattern = ( - "\n" # Use newline splitting for Sentence mode - ) + # spaCy already split at sentence boundaries; the + # engine only splits on newlines. Commas are never + # used in the engine split pattern (Sentence + + # Comma splits at commas only at subtitle time). + active_split_pattern = "\n" else: self.log_updated.emit( ("\nspaCy: Fallback to default segmentation...", "grey") @@ -933,10 +930,10 @@ class ConversionThread(QThread): # Print active split pattern used by the TTS engine once for this batch try: - print(f"Using split pattern: {active_split_pattern!r}") + logger.info(f"Using split pattern: {active_split_pattern!r}") except Exception: - # Print must never break processing - print("Using split pattern: (unprintable)") + # Logging must never break processing + logger.warning("Using split pattern: (unprintable)") for text_segment in text_segments: def _qt_check_cancel() -> bool: @@ -1445,7 +1442,7 @@ class VoicePreviewThread(QThread): return os.path.join(self.cache_dir, filename) def run(self): - print( + logger.info( f"\nVoice: {self.voice}\nLanguage: {self.lang_code}\nSpeed: {self.speed}\nGPU: {self.use_gpu}\n" ) diff --git a/abogen/tts_plugin/types.py b/abogen/tts_plugin/types.py index af06b54..5569f11 100644 --- a/abogen/tts_plugin/types.py +++ b/abogen/tts_plugin/types.py @@ -79,6 +79,44 @@ class SynthesisRequest: format: AudioFormat +@dataclass(frozen=True) +class TokenTiming: + """Per-token timing within a synthesized segment. + + Attributes: + text: Token text. + whitespace: Whitespace following the token ("" if none). + start: Start time in seconds (relative to segment start). + end: End time in seconds (relative to segment start). + """ + + text: str + whitespace: str = "" + start: float = 0.0 + end: float = 0.0 + + +@dataclass(frozen=True) +class AudioSegment: + """One contiguous synthesized segment (sentence-level chunk). + + Engines that split the input text (via ``split_pattern``) expose each + chunk as its own AudioSegment so hosts can report per-sentence progress + and build subtitles from per-token timings. + + Attributes: + graphemes: The text this segment was synthesized from. + audio: Raw float32 PCM audio bytes for this segment. + sample_rate: Sample rate of ``audio``. + tokens: Per-token timing details, when the engine provides them. + """ + + graphemes: str + audio: bytes + sample_rate: int + tokens: tuple[TokenTiming, ...] = () + + @dataclass(frozen=True) class SynthesizedAudio: """Immutable value object for synthesized audio result. @@ -87,11 +125,15 @@ class SynthesizedAudio: data: Raw audio bytes. format: Audio format of the result. duration: Duration of the audio. + segments: Per-segment details when the engine split the text into + sentence-level chunks (empty for engines that only produce a + single merged result). """ data: bytes format: AudioFormat duration: Duration + segments: tuple[AudioSegment, ...] = () @dataclass(frozen=True) diff --git a/abogen/tts_plugin/utils.py b/abogen/tts_plugin/utils.py index df0560f..b67d784 100644 --- a/abogen/tts_plugin/utils.py +++ b/abogen/tts_plugin/utils.py @@ -169,15 +169,38 @@ class Pipeline: ) result = session.synthesize(request) - audio_array = np.frombuffer(result.data, dtype=np.float32) - from dataclasses import dataclass + from dataclasses import dataclass, field + + @dataclass + class Token: + text: str + whitespace: str = "" + start_ts: float = 0.0 + end_ts: float = 0.0 @dataclass class Segment: graphemes: str audio: np.ndarray + tokens: list[Any] = field(default_factory=list) + if result.segments: + for seg in result.segments: + audio_array = np.frombuffer(seg.audio, dtype=np.float32) + tokens = [ + Token( + text=tok.text, + whitespace=tok.whitespace, + start_ts=tok.start, + end_ts=tok.end, + ) + for tok in seg.tokens + ] + yield Segment(graphemes=seg.graphemes, audio=audio_array, tokens=tokens) + return + + audio_array = np.frombuffer(result.data, dtype=np.float32) yield Segment(graphemes=text, audio=audio_array) def load_single_voice(self, voice_name: str) -> Any: diff --git a/abogen/utils.py b/abogen/utils.py index 87db257..5e6f6bf 100644 --- a/abogen/utils.py +++ b/abogen/utils.py @@ -16,6 +16,8 @@ from functools import lru_cache from dotenv import load_dotenv, find_dotenv +logger = logging.getLogger(__name__) + def _load_environment() -> None: explicit_path = os.environ.get("ABOGEN_ENV_FILE") @@ -441,10 +443,6 @@ default_encoding = sys.getfilesystemencoding() def create_process(cmd, stdin=None, text=True, capture_output=False): - import logging - - logger = logging.getLogger(__name__) - # Configure root logger to output to console if not already configured root = logging.getLogger() if not root.handlers: @@ -493,8 +491,8 @@ def create_process(cmd, stdin=None, text=True, capture_output=False): } ) - # Print the command being executed - print(f"Executing: {cmd if isinstance(cmd, str) else ' '.join(cmd)}") + # Log the command being executed + logger.info(f"Executing: {cmd if isinstance(cmd, str) else ' '.join(cmd)}") proc = subprocess.Popen(cmd, **kwargs) @@ -615,7 +613,7 @@ def prevent_sleep_start(): ) else: # Non-systemd distro or systemd tools not installed: skip inhibition rather than crash - print( + logger.warning( "systemd-inhibit not found: skipping sleep inhibition on this Linux system." ) diff --git a/plugins/kokoro/engine.py b/plugins/kokoro/engine.py index e1d0b6e..ba99ee5 100644 --- a/plugins/kokoro/engine.py +++ b/plugins/kokoro/engine.py @@ -22,9 +22,11 @@ from abogen.tts_plugin.errors import EngineError from abogen.tts_plugin.manifest import VoiceManifest from abogen.tts_plugin.types import ( AudioFormat, + AudioSegment, Duration, SynthesisRequest, SynthesizedAudio, + TokenTiming, ) logger = logging.getLogger(__name__) @@ -117,7 +119,9 @@ class KokoroSession: speed = request.parameters.values.get("speed", 1.0) split_pattern = request.parameters.values.get("split_pattern", None) + sample_rate = _KOKORO_SAMPLE_RATE audio_parts: list[np.ndarray] = [] + segments: list[AudioSegment] = [] for segment in self._pipeline( request.text, voice=voice, @@ -127,7 +131,28 @@ class KokoroSession: audio = segment.audio if hasattr(audio, "numpy"): audio = audio.numpy() - audio_parts.append(np.asarray(audio, dtype="float32")) + audio = np.asarray(audio, dtype="float32") + if audio.size == 0: + continue + audio_parts.append(audio) + + tokens = tuple( + TokenTiming( + text=str(tok.text), + whitespace=str(tok.whitespace or ""), + start=float(tok.start_ts or 0.0), + end=float(tok.end_ts or 0.0), + ) + for tok in (getattr(segment, "tokens", None) or []) + ) + segments.append( + AudioSegment( + graphemes=str(getattr(segment, "graphemes", "") or ""), + audio=audio.tobytes(), + sample_rate=sample_rate, + tokens=tokens, + ) + ) if not audio_parts: return SynthesizedAudio( @@ -138,12 +163,13 @@ class KokoroSession: combined = np.concatenate(audio_parts).astype("float32", copy=False) audio_bytes = combined.tobytes() - duration_seconds = len(combined) / _KOKORO_SAMPLE_RATE + duration_seconds = len(combined) / sample_rate return SynthesizedAudio( data=audio_bytes, format=AudioFormat(mime="audio/wav", extension="wav"), duration=Duration(seconds=duration_seconds), + segments=tuple(segments), ) except EngineError: raise diff --git a/plugins/supertonic/engine.py b/plugins/supertonic/engine.py index e95e3bc..9ac150b 100644 --- a/plugins/supertonic/engine.py +++ b/plugins/supertonic/engine.py @@ -19,6 +19,7 @@ from abogen.tts_plugin.errors import EngineError from abogen.tts_plugin.manifest import VoiceManifest from abogen.tts_plugin.types import ( AudioFormat, + AudioSegment, Duration, SynthesisRequest, SynthesizedAudio, @@ -113,6 +114,7 @@ class SuperTonicSession: total_steps = int(total_steps) audio_parts: list[np.ndarray] = [] + segments: list[AudioSegment] = [] for segment in self._pipeline( request.text, voice=voice, @@ -120,7 +122,17 @@ class SuperTonicSession: split_pattern=split_pattern, total_steps=total_steps, ): - audio_parts.append(segment.audio) + audio = np.asarray(segment.audio, dtype="float32") + if audio.size == 0: + continue + audio_parts.append(audio) + segments.append( + AudioSegment( + graphemes=str(getattr(segment, "graphemes", "") or ""), + audio=audio.tobytes(), + sample_rate=self._pipeline.sample_rate, + ) + ) if not audio_parts: return SynthesizedAudio( @@ -139,6 +151,7 @@ class SuperTonicSession: data=audio_bytes, format=AudioFormat(mime="audio/wav", extension="wav"), duration=Duration(seconds=duration_seconds), + segments=tuple(segments), ) except EngineError: raise diff --git a/tests/test_split_pattern.py b/tests/test_split_pattern.py index 6858ca5..4a42883 100644 --- a/tests/test_split_pattern.py +++ b/tests/test_split_pattern.py @@ -9,7 +9,7 @@ from abogen.domain.enums import Language from abogen.domain.split_pattern import get_split_pattern -# --- English always returns \n --- +# --- English: newline-only for Disabled/Line, punctuation-based for sentence modes --- class TestEnglish: def test_english_sentence(self):