fix(segmentation): process TTS segments per sentence, fix all subtitle modes

Sentence modes processed all text as a whole: Pipeline.__call__ merged every
engine segment back into one (whole text, no per-token timings), producing a
single giant subtitle and whole-text progress logs.

- tts_plugin/types: add TokenTiming, AudioSegment, SynthesizedAudio.segments
- tts_plugin/utils: Pipeline yields one Segment per engine segment (with
  tokens); merged fallback only when engine provides none
- kokoro engine: expose per-segment graphemes/audio + per-word token timings
- supertonic engine: expose per-segment graphemes/audio (no tokens)
- split_pattern: English Sentence/Sentence+Comma engine split is newline-only
  (boundaries applied at subtitle time via spaCy); non-English Sentence+Comma
  with spaCy ON uses spaCy pre-segmentation + newline engine split (no
  commas); spaCy-off fallback keeps comma pattern
- tts_segments: restore inter-segment whitespace on real per-word token
  boundaries only (never FakeToken fallbacks)
- _to_language_enum: accept Language enum input (str(enum) is "Language.ES",
  silently resolved to EN_US and disabled spaCy pre-TTS for every language
  in WebUI)
- pyqt/conversion, utils: replace print with logging
- add AGENTS.md documenting the segmentation/subtitle contract for future
  sessions
- tests: update English split-pattern expectations (1566 passing)
This commit is contained in:
Deniz Şafak
2026-08-20 23:00:15 +03:00
parent 823f5be029
commit 5432de7ac5
10 changed files with 247 additions and 40 deletions
+28 -2
View File
@@ -22,9 +22,11 @@ from abogen.tts_plugin.errors import EngineError
from abogen.tts_plugin.manifest import VoiceManifest
from abogen.tts_plugin.types import (
AudioFormat,
AudioSegment,
Duration,
SynthesisRequest,
SynthesizedAudio,
TokenTiming,
)
logger = logging.getLogger(__name__)
@@ -117,7 +119,9 @@ class KokoroSession:
speed = request.parameters.values.get("speed", 1.0)
split_pattern = request.parameters.values.get("split_pattern", None)
sample_rate = _KOKORO_SAMPLE_RATE
audio_parts: list[np.ndarray] = []
segments: list[AudioSegment] = []
for segment in self._pipeline(
request.text,
voice=voice,
@@ -127,7 +131,28 @@ class KokoroSession:
audio = segment.audio
if hasattr(audio, "numpy"):
audio = audio.numpy()
audio_parts.append(np.asarray(audio, dtype="float32"))
audio = np.asarray(audio, dtype="float32")
if audio.size == 0:
continue
audio_parts.append(audio)
tokens = tuple(
TokenTiming(
text=str(tok.text),
whitespace=str(tok.whitespace or ""),
start=float(tok.start_ts or 0.0),
end=float(tok.end_ts or 0.0),
)
for tok in (getattr(segment, "tokens", None) or [])
)
segments.append(
AudioSegment(
graphemes=str(getattr(segment, "graphemes", "") or ""),
audio=audio.tobytes(),
sample_rate=sample_rate,
tokens=tokens,
)
)
if not audio_parts:
return SynthesizedAudio(
@@ -138,12 +163,13 @@ class KokoroSession:
combined = np.concatenate(audio_parts).astype("float32", copy=False)
audio_bytes = combined.tobytes()
duration_seconds = len(combined) / _KOKORO_SAMPLE_RATE
duration_seconds = len(combined) / sample_rate
return SynthesizedAudio(
data=audio_bytes,
format=AudioFormat(mime="audio/wav", extension="wav"),
duration=Duration(seconds=duration_seconds),
segments=tuple(segments),
)
except EngineError:
raise
+14 -1
View File
@@ -19,6 +19,7 @@ from abogen.tts_plugin.errors import EngineError
from abogen.tts_plugin.manifest import VoiceManifest
from abogen.tts_plugin.types import (
AudioFormat,
AudioSegment,
Duration,
SynthesisRequest,
SynthesizedAudio,
@@ -113,6 +114,7 @@ class SuperTonicSession:
total_steps = int(total_steps)
audio_parts: list[np.ndarray] = []
segments: list[AudioSegment] = []
for segment in self._pipeline(
request.text,
voice=voice,
@@ -120,7 +122,17 @@ class SuperTonicSession:
split_pattern=split_pattern,
total_steps=total_steps,
):
audio_parts.append(segment.audio)
audio = np.asarray(segment.audio, dtype="float32")
if audio.size == 0:
continue
audio_parts.append(audio)
segments.append(
AudioSegment(
graphemes=str(getattr(segment, "graphemes", "") or ""),
audio=audio.tobytes(),
sample_rate=self._pipeline.sample_rate,
)
)
if not audio_parts:
return SynthesizedAudio(
@@ -139,6 +151,7 @@ class SuperTonicSession:
data=audio_bytes,
format=AudioFormat(mime="audio/wav", extension="wav"),
duration=Duration(seconds=duration_seconds),
segments=tuple(segments),
)
except EngineError:
raise