From 294069e53e30bf114e5901b9cc65287dcbf8065a Mon Sep 17 00:00:00 2001 From: Artem Akymenko Date: Sat, 18 Jul 2026 06:36:19 +0000 Subject: [PATCH] =?UTF-8?q?refactor(webui):=20token-level=20subtitle=20pro?= =?UTF-8?q?cessing=20via=20process=5Fsubtitle=5Ftokens=20=E2=80=94=20P0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before: WebUI wrote one subtitle entry per TTS segment (no sentence grouping, no comma splitting, no karaoke highlighting). The subtitle modes 'Sentence', 'Sentence + Comma', and 'Sentence + Highlighting' produced broken output. After: emit_text() accumulates tokens_with_timestamps from each segment's .tokens attribute, then flushes them through domain.subtitle_generation.process_subtitle_tokens() at the end. This gives the WebUI the same subtitle quality as the PyQt desktop GUI: - Sentence mode: groups tokens into sentences - Sentence + Comma: splits on commas within sentences - Sentence + Highlighting: karaoke timing per word - Word-count mode: groups by N words Also removed the duplicate _to_float32 function from synthesize.py (now imports from domain.audio_helpers). 1053 tests pass. --- abogen/webui/conversion_runner.py | 47 +++++++++++++++++++++---- abogen/webui/routes/utils/synthesize.py | 16 --------- 2 files changed, 41 insertions(+), 22 deletions(-) diff --git a/abogen/webui/conversion_runner.py b/abogen/webui/conversion_runner.py index 8373788..464fa2c 100644 --- a/abogen/webui/conversion_runner.py +++ b/abogen/webui/conversion_runner.py @@ -114,6 +114,7 @@ from abogen.domain.output_paths import ( from abogen.domain.device import select_device as _select_device from abogen.domain.split_pattern import get_split_pattern from abogen.domain.progress import ProgressTracker, calc_etr_str +from abogen.domain.subtitle_generation import process_subtitle_tokens from abogen.domain.audio_helpers import ( build_ffmpeg_command as _build_ffmpeg_command, to_float32 as _to_float32, @@ -491,6 +492,9 @@ def run_conversion_job(job: Job) -> None: ) try: + # Accumulate tokens for subtitle processing (token-level grouping) + accumulated_tokens: List[dict] = [] + for segment in segment_iter: canceller() graphemes_raw = getattr(segment, "graphemes", "") or "" @@ -507,6 +511,7 @@ def run_conversion_job(job: Job) -> None: audio_sink.write(audio) duration = len(audio) / SAMPLE_RATE + chunk_start = current_time processed_chars += len(graphemes) job.processed_characters = processed_chars if job.total_characters: @@ -523,16 +528,46 @@ def run_conversion_job(job: Job) -> None: prefix = f"{preview_prefix} · " if preview_prefix else "" job.add_log(f"{prefix}{processed_chars:,}/{job.total_characters or '—'}: {preview_text[:80]}") - if subtitle_writer and audio_sink and graphemes: - subtitle_writer.write_entry( - start=current_time, - end=current_time + duration, - text=graphemes, - ) + # Accumulate tokens from this segment for subtitle processing + if subtitle_writer and audio_sink: + tokens_list = getattr(segment, "tokens", []) + + # Fallback for languages without token support: create a single token + if not tokens_list and graphemes: + class _FakeToken: + def __init__(self, text, start, end): + self.text = text + self.start_ts = start + self.end_ts = end + self.whitespace = "" + tokens_list = [_FakeToken(graphemes, 0, duration)] + + for tok in tokens_list: + accumulated_tokens.append({ + "start": chunk_start + (tok.start_ts or 0), + "end": chunk_start + (tok.end_ts or 0), + "text": tok.text, + "whitespace": tok.whitespace, + }) if audio_sink: current_time += duration + # Flush accumulated tokens through process_subtitle_tokens + if subtitle_writer and audio_sink and accumulated_tokens: + new_entries: List[tuple] = [] + process_subtitle_tokens( + accumulated_tokens, + new_entries, + job.max_subtitle_words, + job.subtitle_mode, + job.language, + use_spacy_segmentation=False, + fallback_end_time=current_time, + ) + for start, end, text in new_entries: + subtitle_writer.write_entry(start=start, end=end, text=text) + except OverflowError as exc: job.add_log( f"Skipped chunk — number too large for TTS conversion: {exc}", diff --git a/abogen/webui/routes/utils/synthesize.py b/abogen/webui/routes/utils/synthesize.py index 3f3cb7b..5f1feb7 100644 --- a/abogen/webui/routes/utils/synthesize.py +++ b/abogen/webui/routes/utils/synthesize.py @@ -44,22 +44,6 @@ def _resolve_pipeline(language: str, use_gpu: bool) -> Tuple[Any, bool]: raise RuntimeError("Preview pipeline is unavailable") from last_error -def _to_float32(audio_segment) -> np.ndarray: - if audio_segment is None: - return np.zeros(0, dtype="float32") - - tensor = audio_segment - if hasattr(tensor, "detach"): - tensor = tensor.detach() - if hasattr(tensor, "cpu"): - try: - tensor = tensor.cpu() - except Exception: - pass - if hasattr(tensor, "numpy"): - return np.asarray(tensor.numpy(), dtype="float32").reshape(-1) - return np.asarray(tensor, dtype="float32").reshape(-1) - def get_preview_pipeline(language: str, device: str) -> Any: key = (language, device) with _preview_pipeline_lock: