From 675ed437c8a29dec2bd7030973c7687cb595eb64 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 13 Dec 2025 17:06:55 +0800 Subject: [PATCH] Extracts out subtitle_utils from conversion.py The goal of this change is to move any non PyQt related logic from conversion.py into its own subtitle_utils.py. Together with this change there was an opportunity to pull together some duplicate text processing that was also in utils.py --- abogen/book_handler.py | 8 +- abogen/conversion.py | 437 +--------------------------------- abogen/gui.py | 8 +- abogen/queue_manager_gui.py | 2 +- abogen/subtitle_utils.py | 459 ++++++++++++++++++++++++++++++++++++ abogen/utils.py | 39 --- 6 files changed, 483 insertions(+), 470 deletions(-) create mode 100644 abogen/subtitle_utils.py diff --git a/abogen/book_handler.py b/abogen/book_handler.py index cb406d8..b8073c1 100644 --- a/abogen/book_handler.py +++ b/abogen/book_handler.py @@ -28,11 +28,15 @@ from PyQt6.QtCore import ( QSize, ) from abogen.utils import ( - clean_text, - calculate_text_length, detect_encoding, get_resource_path, ) + +from abogen.subtitle_utils import ( + clean_text, + calculate_text_length, +) + import os import logging # Add logging import urllib.parse diff --git a/abogen/conversion.py b/abogen/conversion.py index c0afb2b..2195b8a 100644 --- a/abogen/conversion.py +++ b/abogen/conversion.py @@ -7,7 +7,6 @@ from PyQt6.QtCore import QThread, pyqtSignal, Qt, QTimer from PyQt6.QtWidgets import QCheckBox, QVBoxLayout, QDialog, QLabel, QDialogButtonBox import soundfile as sf from abogen.utils import ( - clean_text, create_process, get_user_cache_path, detect_encoding, @@ -33,428 +32,17 @@ _USER_RESPONSE_TIMEOUT = ( 0.1 # Timeout in seconds for checking user response/cancellation ) -# Pre-compile frequently used regex patterns for better performance -_METADATA_TAG_PATTERN = re.compile(r"<]*>>") -_CHAPTER_MARKER_PATTERN = re.compile(r"<]*>>") -_HTML_TAG_PATTERN = re.compile(r"<[^>]+>") -_VOICE_TAG_PATTERN = re.compile(r"{[^}]+}") -_ASS_STYLING_PATTERN = re.compile(r"\{[^}]+\}") -_ASS_NEWLINE_N_PATTERN = re.compile(r"\\N") -_ASS_NEWLINE_LOWER_N_PATTERN = re.compile(r"\\n") -_CHAPTER_MARKER_SEARCH_PATTERN = re.compile(r"<>") -_WEBVTT_HEADER_PATTERN = re.compile(r"^WEBVTT.*?\n", re.MULTILINE) -_VTT_STYLE_PATTERN = re.compile(r"STYLE\s*\n.*?(?=\n\n|$)", re.DOTALL) -_VTT_NOTE_PATTERN = re.compile(r"NOTE\s*\n.*?(?=\n\n|$)", re.DOTALL) -_DOUBLE_NEWLINE_SPLIT_PATTERN = re.compile(r"\n\s*\n") -_VTT_TIMESTAMP_PATTERN = re.compile(r"([\d:.]+)\s*-->\s*([\d:.]+)") -_TIMESTAMP_ONLY_PATTERN = re.compile(r"^(\d{1,2}:\d{2}:\d{2}(?:[.,]\d{1,3})?)$") -_WINDOWS_ILLEGAL_CHARS_PATTERN = re.compile(r'[<>:"/\\|?*]') -_CONTROL_CHARS_PATTERN = re.compile(r"[\x00-\x1f]") -_LINUX_CONTROL_CHARS_PATTERN = re.compile( - r"[\x01-\x1f]" -) # Linux: exclude \x00 for separate handling -_MACOS_ILLEGAL_CHARS_PATTERN = re.compile(r"[:]") -_LINUX_ILLEGAL_CHARS_PATTERN = re.compile(r"[/\x00]") - - -def clean_subtitle_text(text): - """Remove chapter markers and metadata tags from subtitle text.""" - # Use pre-compiled patterns for better performance - text = _METADATA_TAG_PATTERN.sub("", text) - text = _CHAPTER_MARKER_PATTERN.sub("", text) - return text.strip() - - -def parse_srt_file(file_path): - """ - Parse an SRT subtitle file and return a list of subtitle entries. - - Args: - file_path: Path to the SRT file - - Returns: - List of tuples: [(start_time_seconds, end_time_seconds, text), ...] - """ - encoding = detect_encoding(file_path) - with open(file_path, "r", encoding=encoding, errors="replace") as f: - content = f.read() - - # Split by double newlines to get individual subtitle blocks - blocks = re.split(r"\n\s*\n", content.strip()) - - subtitles = [] - for block in blocks: - if not block.strip(): - continue - - lines = block.strip().split("\n") - if len(lines) < 3: - continue - - # First line is index, second line is timestamp, rest is text - try: - timestamp_line = lines[1] - match = re.match( - r"(\d{2}:\d{2}:\d{2},\d{3})\s*-->\s*(\d{2}:\d{2}:\d{2},\d{3})", - timestamp_line, - ) - if not match: - continue - - start_str = match.group(1) - end_str = match.group(2) - text = "\n".join(lines[2:]) - - # Convert timestamp to seconds - def time_to_seconds(t): - h, m, s_ms = t.split(":") - s, ms = s_ms.split(",") - return int(h) * 3600 + int(m) * 60 + int(s) + int(ms) / 1000.0 - - start_sec = time_to_seconds(start_str) - end_sec = time_to_seconds(end_str) - - # Clean text of any styling tags using pre-compiled pattern - text = _HTML_TAG_PATTERN.sub("", text) - # Remove chapter markers and metadata tags - text = clean_subtitle_text(text) - - if text: # Only add non-empty subtitles - subtitles.append((start_sec, end_sec, text)) - except (ValueError, IndexError): - continue - - return subtitles - - -def parse_vtt_file(file_path): - """ - Parse a VTT (WebVTT) subtitle file and return a list of subtitle entries. - - Args: - file_path: Path to the VTT file - - Returns: - List of tuples: [(start_time_seconds, end_time_seconds, text), ...] - """ - encoding = detect_encoding(file_path) - with open(file_path, "r", encoding=encoding, errors="replace") as f: - content = f.read() - - # Remove WEBVTT header and any style/note blocks using pre-compiled patterns - content = _WEBVTT_HEADER_PATTERN.sub("", content) - content = _VTT_STYLE_PATTERN.sub("", content) - content = _VTT_NOTE_PATTERN.sub("", content) - - # Split by double newlines to get individual subtitle blocks using pre-compiled pattern - blocks = _DOUBLE_NEWLINE_SPLIT_PATTERN.split(content.strip()) - - subtitles = [] - for block in blocks: - if not block.strip(): - continue - - lines = block.strip().split("\n") - if len(lines) < 2: - continue - - # VTT can have optional identifier on first line, timestamp on second or first - timestamp_line = None - text_start_idx = 0 - - # Check if first line is timestamp - if "-->" in lines[0]: - timestamp_line = lines[0] - text_start_idx = 1 - elif len(lines) > 1 and "-->" in lines[1]: - timestamp_line = lines[1] - text_start_idx = 2 - else: - continue - - try: - # VTT format: 00:00:00.000 --> 00:00:05.000 or 00:00.000 --> 00:05.000 - # Use pre-compiled pattern - match = _VTT_TIMESTAMP_PATTERN.match(timestamp_line) - if not match: - continue - - start_str = match.group(1) - end_str = match.group(2) - text = "\n".join(lines[text_start_idx:]) - - # Convert timestamp to seconds - def time_to_seconds(t): - parts = t.split(":") - if len(parts) == 3: # HH:MM:SS.mmm - h, m, s = parts - s, ms = s.split(".") - return int(h) * 3600 + int(m) * 60 + int(s) + int(ms) / 1000.0 - elif len(parts) == 2: # MM:SS.mmm - m, s = parts - s, ms = s.split(".") - return int(m) * 60 + int(s) + int(ms) / 1000.0 - return 0 - - start_sec = time_to_seconds(start_str) - end_sec = time_to_seconds(end_str) - - # Clean text of any styling tags and cue settings using pre-compiled patterns - text = _HTML_TAG_PATTERN.sub("", text) - text = _VOICE_TAG_PATTERN.sub("", text) # Remove voice tags - # Remove chapter markers and metadata tags - text = clean_subtitle_text(text) - - if text: # Only add non-empty subtitles - subtitles.append((start_sec, end_sec, text)) - except (ValueError, IndexError, AttributeError): - continue - - return subtitles - - -def detect_timestamps_in_text(file_path): - """Detect if text file contains timestamp markers (HH:MM:SS or HH:MM:SS,ms format) on separate lines.""" - try: - encoding = detect_encoding(file_path) - with open(file_path, "r", encoding=encoding, errors="replace") as f: - lines = [ - line.strip() for line in f.readlines()[:50] if line.strip() - ] # Check first 50 non-empty lines - - # Count lines that are ONLY timestamps (no other text) - # Supports HH:MM:SS or HH:MM:SS,ms format - # Use pre-compiled pattern for better performance - timestamp_lines = sum( - 1 for line in lines if _TIMESTAMP_ONLY_PATTERN.match(line) - ) - - # Must have at least 2 timestamp-only lines and they should be >5% of total lines - return timestamp_lines >= 2 and (timestamp_lines / max(len(lines), 1)) > 0.05 - except Exception: - return False - - -def parse_timestamp_text_file(file_path): - """Parse text file with timestamps. Returns list of (start_time, end_time, text) tuples. - Supports HH:MM:SS or HH:MM:SS,ms format. Returns time in seconds as float.""" - encoding = detect_encoding(file_path) - with open(file_path, "r", encoding=encoding, errors="replace") as f: - content = f.read() - - # Split by timestamp pattern (supports HH:MM:SS or HH:MM:SS,ms) - pattern = r"^(\d{1,2}:\d{2}:\d{2}(?:[.,]\d{1,3})?)$" - lines = content.split("\n") - - def parse_time(time_str): - """Convert HH:MM:SS or HH:MM:SS,ms to seconds as float.""" - time_str = time_str.replace(",", ".") - parts = time_str.split(":") - return float(int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2])) - - entries = [] - current_time = None - current_text = [] - pre_timestamp_text = [] # Text before first timestamp - - for line in lines: - match = re.match(pattern, line.strip()) - if match: - # Save previous entry - if current_time is not None and current_text: - text = "\n".join(current_text).strip() - if text: - entries.append((current_time, text)) - elif current_time is None and pre_timestamp_text: - # First timestamp found, save pre-timestamp text with time 0 - text = "\n".join(pre_timestamp_text).strip() - if text: - entries.append((0.0, text)) - pre_timestamp_text = [] - - # Start new entry - time_str = match.group(1) - current_time = parse_time(time_str) - current_text = [] - elif current_time is not None: - current_text.append(line) - else: - # Text before first timestamp - pre_timestamp_text.append(line) - - # Save last entry - if current_time is not None and current_text: - text = "\n".join(current_text).strip() - if text: - entries.append((current_time, text)) - elif not entries and pre_timestamp_text: - # No timestamps found at all, treat entire file as starting at 0 - text = "\n".join(pre_timestamp_text).strip() - if text: - entries.append((0.0, text)) - - # Convert to subtitle format with end times - subtitles = [] - for i, (start_time, text) in enumerate(entries): - end_time = entries[i + 1][0] if i + 1 < len(entries) else None - # Remove chapter markers and metadata tags - text = clean_subtitle_text(text) - if text: # Only add non-empty entries - subtitles.append((start_time, end_time, text)) - - return subtitles - - -def parse_ass_file(file_path): - """ - Parse an ASS/SSA subtitle file and return a list of subtitle entries. - - Args: - file_path: Path to the ASS/SSA file - - Returns: - List of tuples: [(start_time_seconds, end_time_seconds, text), ...] - """ - encoding = detect_encoding(file_path) - with open(file_path, "r", encoding=encoding, errors="replace") as f: - lines = f.readlines() - - subtitles = [] - in_events = False - format_indices = {} - - for line in lines: - line = line.strip() - - if line.startswith("[Events]"): - in_events = True - continue - - if line.startswith("[") and in_events: - # New section, stop processing - break - - if in_events and line.startswith("Format:"): - # Parse format line to know column positions - parts = line.split(":", 1)[1].strip().split(",") - for i, part in enumerate(parts): - format_indices[part.strip().lower()] = i - continue - - if in_events and (line.startswith("Dialogue:") or line.startswith("Comment:")): - if line.startswith("Comment:"): - continue # Skip comments - - parts = line.split(":", 1)[1].strip().split(",", len(format_indices) - 1) - - if ( - "start" in format_indices - and "end" in format_indices - and "text" in format_indices - ): - start_str = parts[format_indices["start"]].strip() - end_str = parts[format_indices["end"]].strip() - text = parts[format_indices["text"]].strip() - - # Convert timestamp to seconds (ASS format: H:MM:SS.CS where CS is centiseconds) - def ass_time_to_seconds(t): - parts = t.split(":") - if len(parts) == 3: - h, m, s = parts - s_parts = s.split(".") - seconds = float(s_parts[0]) - centiseconds = float(s_parts[1]) if len(s_parts) > 1 else 0 - return ( - int(h) * 3600 + int(m) * 60 + seconds + centiseconds / 100.0 - ) - return 0 - - start_sec = ass_time_to_seconds(start_str) - end_sec = ass_time_to_seconds(end_str) - - # Clean text of ASS styling tags using pre-compiled patterns - text = _ASS_STYLING_PATTERN.sub("", text) # Remove {tags} - text = _ASS_NEWLINE_N_PATTERN.sub("\n", text) # Convert \N to newline - text = _ASS_NEWLINE_LOWER_N_PATTERN.sub( - "\n", text - ) # Convert \n to newline - # Remove chapter markers and metadata tags - text = clean_subtitle_text(text) - - if text: # Only add non-empty subtitles - subtitles.append((start_sec, end_sec, text)) - - return subtitles - - -def get_sample_voice_text(lang_code): - return SAMPLE_VOICE_TEXTS.get(lang_code, SAMPLE_VOICE_TEXTS["a"]) - - -def sanitize_name_for_os(name, is_folder=True): - """ - Sanitize a filename or folder name based on the operating system. - - Args: - name: The name to sanitize - is_folder: Whether this is a folder name (default: True) - - Returns: - Sanitized name safe for the current OS - """ - if not name: - return "audiobook" - - system = platform.system() - - if system == "Windows": - # Windows illegal characters: < > : " / \ | ? * - # Also can't end with space or dot - # Use pre-compiled pattern for better performance - sanitized = _WINDOWS_ILLEGAL_CHARS_PATTERN.sub("_", name) - # Remove control characters (0-31) - sanitized = _CONTROL_CHARS_PATTERN.sub("_", sanitized) - # Remove trailing spaces and dots - sanitized = sanitized.rstrip(". ") - # Windows reserved names (CON, PRN, AUX, NUL, COM1-9, LPT1-9) - reserved = ( - ["CON", "PRN", "AUX", "NUL"] - + [f"COM{i}" for i in range(1, 10)] - + [f"LPT{i}" for i in range(1, 10)] - ) - if sanitized.upper() in reserved or sanitized.upper().split(".")[0] in reserved: - sanitized = f"_{sanitized}" - elif system == "Darwin": # macOS - # macOS illegal characters: : (colon is converted to / by the system) - # Also can't start with dot (hidden file) for folders typically - # Use pre-compiled pattern for better performance - sanitized = _MACOS_ILLEGAL_CHARS_PATTERN.sub("_", name) - # Remove control characters - sanitized = _CONTROL_CHARS_PATTERN.sub("_", sanitized) - # Avoid leading dot for folders (creates hidden folders) - if is_folder and sanitized.startswith("."): - sanitized = "_" + sanitized[1:] - else: # Linux and others - # Linux illegal characters: / and null character - # Though / is illegal, most other chars are technically allowed - # Use pre-compiled pattern for better performance - sanitized = _LINUX_ILLEGAL_CHARS_PATTERN.sub("_", name) - # Remove other control characters for safety (excluding \x00 which is already handled) - sanitized = _LINUX_CONTROL_CHARS_PATTERN.sub("_", sanitized) - # Avoid leading dot for folders (creates hidden folders) - if is_folder and sanitized.startswith("."): - sanitized = "_" + sanitized[1:] - - # Ensure the name is not empty after sanitization - if not sanitized or sanitized.strip() == "": - sanitized = "audiobook" - - # Limit length to 255 characters (common limit across filesystems) - if len(sanitized) > 255: - sanitized = sanitized[:255].rstrip(". ") - - return sanitized - +from abogen.subtitle_utils import ( + clean_text, + parse_srt_file, + parse_vtt_file, + detect_timestamps_in_text, + parse_timestamp_text_file, + parse_ass_file, + get_sample_voice_text, + sanitize_name_for_os, + _CHAPTER_MARKER_SEARCH_PATTERN, +) class CountdownDialog(QDialog): """Base dialog with auto-accept countdown functionality""" @@ -936,9 +524,6 @@ class ConversionThread(QThread): # Clean up text using utility function text = clean_text(text) - # Remove metadata markers from the text to be processed - # Use pre-compiled pattern for better performance - text = _METADATA_TAG_PATTERN.sub("", text) # --- Chapter splitting logic --- # Use pre-compiled pattern for better performance diff --git a/abogen/gui.py b/abogen/gui.py index 9fcab99..95a779a 100644 --- a/abogen/gui.py +++ b/abogen/gui.py @@ -62,14 +62,18 @@ from abogen.utils import ( load_config, save_config, get_gpu_acceleration, - clean_text, prevent_sleep_start, prevent_sleep_end, - calculate_text_length, get_resource_path, get_user_cache_path, LoadPipelineThread, ) + +from abogen.subtitle_utils import ( + clean_text, + calculate_text_length, +) + from abogen.conversion import ConversionThread, VoicePreviewThread, PlayAudioThread from abogen.book_handler import HandlerDialog from abogen.constants import ( diff --git a/abogen/queue_manager_gui.py b/abogen/queue_manager_gui.py index 8341ba4..384be79 100644 --- a/abogen/queue_manager_gui.py +++ b/abogen/queue_manager_gui.py @@ -502,7 +502,7 @@ class QueueManager(QDialog): return attrs def add_files_from_paths(self, file_paths): - from abogen.utils import calculate_text_length + from abogen.subtitle_utils import calculate_text_length from PyQt6.QtWidgets import QMessageBox import os diff --git a/abogen/subtitle_utils.py b/abogen/subtitle_utils.py new file mode 100644 index 0000000..52054da --- /dev/null +++ b/abogen/subtitle_utils.py @@ -0,0 +1,459 @@ +import re +import platform +from abogen.utils import detect_encoding +from abogen.constants import SAMPLE_VOICE_TEXTS + +# Pre-compile frequently used regex patterns for better performance +_METADATA_TAG_PATTERN = re.compile(r"<]*>>") +_WHITESPACE_PATTERN = re.compile(r"[^\S\n]+") +_MULTIPLE_NEWLINES_PATTERN = re.compile(r"\n{3,}") +_SINGLE_NEWLINE_PATTERN = re.compile(r"(?]*>>") +_HTML_TAG_PATTERN = re.compile(r"<[^>]+>") +_VOICE_TAG_PATTERN = re.compile(r"{[^}]+}") +_ASS_STYLING_PATTERN = re.compile(r"\{[^}]+\}") +_ASS_NEWLINE_N_PATTERN = re.compile(r"\\N") +_ASS_NEWLINE_LOWER_N_PATTERN = re.compile(r"\\n") +_CHAPTER_MARKER_SEARCH_PATTERN = re.compile(r"<>") +_WEBVTT_HEADER_PATTERN = re.compile(r"^WEBVTT.*?\n", re.MULTILINE) +_VTT_STYLE_PATTERN = re.compile(r"STYLE\s*\n.*?(?=\n\n|$)", re.DOTALL) +_VTT_NOTE_PATTERN = re.compile(r"NOTE\s*\n.*?(?=\n\n|$)", re.DOTALL) +_DOUBLE_NEWLINE_SPLIT_PATTERN = re.compile(r"\n\s*\n") +_VTT_TIMESTAMP_PATTERN = re.compile(r"([\d:.]+)\s*-->\s*([\d:.]+)") +_TIMESTAMP_ONLY_PATTERN = re.compile(r"^(\d{1,2}:\d{2}:\d{2}(?:[.,]\d{1,3})?)$") +_WINDOWS_ILLEGAL_CHARS_PATTERN = re.compile(r'[<>:"/\\|?*]') +_CONTROL_CHARS_PATTERN = re.compile(r"[\x00-\x1f]") +_LINUX_CONTROL_CHARS_PATTERN = re.compile( + r"[\x01-\x1f]" +) # Linux: exclude \x00 for separate handling +_MACOS_ILLEGAL_CHARS_PATTERN = re.compile(r"[:]") +_LINUX_ILLEGAL_CHARS_PATTERN = re.compile(r"[/\x00]") + + +def clean_subtitle_text(text): + """Remove chapter markers and metadata tags from subtitle text.""" + # Use pre-compiled patterns for better performance + text = _METADATA_TAG_PATTERN.sub("", text) + text = _CHAPTER_MARKER_PATTERN.sub("", text) + return text.strip() + +def calculate_text_length(text): + # Use pre-compiled patterns for better performance + # Ignore chapter markers and metadata patterns in a single pass + text = _CHAPTER_MARKER_PATTERN.sub("", text) + text = _METADATA_TAG_PATTERN.sub("", text) + # Ignore newlines and leading/trailing spaces + text = text.replace("\n", "").strip() + # Calculate character count + char_count = len(text) + return char_count + +def clean_text(text, *args, **kwargs): + # Remove metadata tags first + text = _METADATA_TAG_PATTERN.sub("", text) + # Load replace_single_newlines from config + cfg = load_config() + replace_single_newlines = cfg.get("replace_single_newlines", True) + # Collapse all whitespace (excluding newlines) into single spaces per line and trim edges + # Use pre-compiled pattern for better performance + lines = [_WHITESPACE_PATTERN.sub(" ", line).strip() for line in text.splitlines()] + text = "\n".join(lines) + # Standardize paragraph breaks (multiple newlines become exactly two) and trim overall whitespace + # Use pre-compiled pattern for better performance + text = _MULTIPLE_NEWLINES_PATTERN.sub("\n\n", text).strip() + # Optionally replace single newlines with spaces, but preserve double newlines + if replace_single_newlines: + # Use pre-compiled pattern for better performance + text = _SINGLE_NEWLINE_PATTERN.sub(" ", text) + return text + + +def parse_srt_file(file_path): + """ + Parse an SRT subtitle file and return a list of subtitle entries. + + Args: + file_path: Path to the SRT file + + Returns: + List of tuples: [(start_time_seconds, end_time_seconds, text), ...] + """ + encoding = detect_encoding(file_path) + with open(file_path, "r", encoding=encoding, errors="replace") as f: + content = f.read() + + # Split by double newlines to get individual subtitle blocks + blocks = re.split(r"\n\s*\n", content.strip()) + + subtitles = [] + for block in blocks: + if not block.strip(): + continue + + lines = block.strip().split("\n") + if len(lines) < 3: + continue + + # First line is index, second line is timestamp, rest is text + try: + timestamp_line = lines[1] + match = re.match( + r"(\d{2}:\d{2}:\d{2},\d{3})\s*-->\s*(\d{2}:\d{2}:\d{2},\d{3})", + timestamp_line, + ) + if not match: + continue + + start_str = match.group(1) + end_str = match.group(2) + text = "\n".join(lines[2:]) + + # Convert timestamp to seconds + def time_to_seconds(t): + h, m, s_ms = t.split(":") + s, ms = s_ms.split(",") + return int(h) * 3600 + int(m) * 60 + int(s) + int(ms) / 1000.0 + + start_sec = time_to_seconds(start_str) + end_sec = time_to_seconds(end_str) + + # Clean text of any styling tags using pre-compiled pattern + text = _HTML_TAG_PATTERN.sub("", text) + # Remove chapter markers and metadata tags + text = clean_subtitle_text(text) + + if text: # Only add non-empty subtitles + subtitles.append((start_sec, end_sec, text)) + except (ValueError, IndexError): + continue + + return subtitles + + +def parse_vtt_file(file_path): + """ + Parse a VTT (WebVTT) subtitle file and return a list of subtitle entries. + + Args: + file_path: Path to the VTT file + + Returns: + List of tuples: [(start_time_seconds, end_time_seconds, text), ...] + """ + encoding = detect_encoding(file_path) + with open(file_path, "r", encoding=encoding, errors="replace") as f: + content = f.read() + + # Remove WEBVTT header and any style/note blocks using pre-compiled patterns + content = _WEBVTT_HEADER_PATTERN.sub("", content) + content = _VTT_STYLE_PATTERN.sub("", content) + content = _VTT_NOTE_PATTERN.sub("", content) + + # Split by double newlines to get individual subtitle blocks using pre-compiled pattern + blocks = _DOUBLE_NEWLINE_SPLIT_PATTERN.split(content.strip()) + + subtitles = [] + for block in blocks: + if not block.strip(): + continue + + lines = block.strip().split("\n") + if len(lines) < 2: + continue + + # VTT can have optional identifier on first line, timestamp on second or first + timestamp_line = None + text_start_idx = 0 + + # Check if first line is timestamp + if "-->" in lines[0]: + timestamp_line = lines[0] + text_start_idx = 1 + elif len(lines) > 1 and "-->" in lines[1]: + timestamp_line = lines[1] + text_start_idx = 2 + else: + continue + + try: + # VTT format: 00:00:00.000 --> 00:00:05.000 or 00:00.000 --> 00:05.000 + # Use pre-compiled pattern + match = _VTT_TIMESTAMP_PATTERN.match(timestamp_line) + if not match: + continue + + start_str = match.group(1) + end_str = match.group(2) + text = "\n".join(lines[text_start_idx:]) + + # Convert timestamp to seconds + def time_to_seconds(t): + parts = t.split(":") + if len(parts) == 3: # HH:MM:SS.mmm + h, m, s = parts + s, ms = s.split(".") + return int(h) * 3600 + int(m) * 60 + int(s) + int(ms) / 1000.0 + elif len(parts) == 2: # MM:SS.mmm + m, s = parts + s, ms = s.split(".") + return int(m) * 60 + int(s) + int(ms) / 1000.0 + return 0 + + start_sec = time_to_seconds(start_str) + end_sec = time_to_seconds(end_str) + + # Clean text of any styling tags and cue settings using pre-compiled patterns + text = _HTML_TAG_PATTERN.sub("", text) + text = _VOICE_TAG_PATTERN.sub("", text) # Remove voice tags + # Remove chapter markers and metadata tags + text = clean_subtitle_text(text) + + if text: # Only add non-empty subtitles + subtitles.append((start_sec, end_sec, text)) + except (ValueError, IndexError, AttributeError): + continue + + return subtitles + + +def detect_timestamps_in_text(file_path): + """Detect if text file contains timestamp markers (HH:MM:SS or HH:MM:SS,ms format) on separate lines.""" + try: + encoding = detect_encoding(file_path) + with open(file_path, "r", encoding=encoding, errors="replace") as f: + lines = [ + line.strip() for line in f.readlines()[:50] if line.strip() + ] # Check first 50 non-empty lines + + # Count lines that are ONLY timestamps (no other text) + # Supports HH:MM:SS or HH:MM:SS,ms format + # Use pre-compiled pattern for better performance + timestamp_lines = sum( + 1 for line in lines if _TIMESTAMP_ONLY_PATTERN.match(line) + ) + + # Must have at least 2 timestamp-only lines and they should be >5% of total lines + return timestamp_lines >= 2 and (timestamp_lines / max(len(lines), 1)) > 0.05 + except Exception: + return False + + +def parse_timestamp_text_file(file_path): + """Parse text file with timestamps. Returns list of (start_time, end_time, text) tuples. + Supports HH:MM:SS or HH:MM:SS,ms format. Returns time in seconds as float.""" + encoding = detect_encoding(file_path) + with open(file_path, "r", encoding=encoding, errors="replace") as f: + content = f.read() + + # Split by timestamp pattern (supports HH:MM:SS or HH:MM:SS,ms) + pattern = r"^(\d{1,2}:\d{2}:\d{2}(?:[.,]\d{1,3})?)$" + lines = content.split("\n") + + def parse_time(time_str): + """Convert HH:MM:SS or HH:MM:SS,ms to seconds as float.""" + time_str = time_str.replace(",", ".") + parts = time_str.split(":") + return float(int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2])) + + entries = [] + current_time = None + current_text = [] + pre_timestamp_text = [] # Text before first timestamp + + for line in lines: + match = re.match(pattern, line.strip()) + if match: + # Save previous entry + if current_time is not None and current_text: + text = "\n".join(current_text).strip() + if text: + entries.append((current_time, text)) + elif current_time is None and pre_timestamp_text: + # First timestamp found, save pre-timestamp text with time 0 + text = "\n".join(pre_timestamp_text).strip() + if text: + entries.append((0.0, text)) + pre_timestamp_text = [] + + # Start new entry + time_str = match.group(1) + current_time = parse_time(time_str) + current_text = [] + elif current_time is not None: + current_text.append(line) + else: + # Text before first timestamp + pre_timestamp_text.append(line) + + # Save last entry + if current_time is not None and current_text: + text = "\n".join(current_text).strip() + if text: + entries.append((current_time, text)) + elif not entries and pre_timestamp_text: + # No timestamps found at all, treat entire file as starting at 0 + text = "\n".join(pre_timestamp_text).strip() + if text: + entries.append((0.0, text)) + + # Convert to subtitle format with end times + subtitles = [] + for i, (start_time, text) in enumerate(entries): + end_time = entries[i + 1][0] if i + 1 < len(entries) else None + # Remove chapter markers and metadata tags + text = clean_subtitle_text(text) + if text: # Only add non-empty entries + subtitles.append((start_time, end_time, text)) + + return subtitles + + +def parse_ass_file(file_path): + """ + Parse an ASS/SSA subtitle file and return a list of subtitle entries. + + Args: + file_path: Path to the ASS/SSA file + + Returns: + List of tuples: [(start_time_seconds, end_time_seconds, text), ...] + """ + encoding = detect_encoding(file_path) + with open(file_path, "r", encoding=encoding, errors="replace") as f: + lines = f.readlines() + + subtitles = [] + in_events = False + format_indices = {} + + for line in lines: + line = line.strip() + + if line.startswith("[Events]"): + in_events = True + continue + + if line.startswith("[") and in_events: + # New section, stop processing + break + + if in_events and line.startswith("Format:"): + # Parse format line to know column positions + parts = line.split(":", 1)[1].strip().split(",") + for i, part in enumerate(parts): + format_indices[part.strip().lower()] = i + continue + + if in_events and (line.startswith("Dialogue:") or line.startswith("Comment:")): + if line.startswith("Comment:"): + continue # Skip comments + + parts = line.split(":", 1)[1].strip().split(",", len(format_indices) - 1) + + if ( + "start" in format_indices + and "end" in format_indices + and "text" in format_indices + ): + start_str = parts[format_indices["start"]].strip() + end_str = parts[format_indices["end"]].strip() + text = parts[format_indices["text"]].strip() + + # Convert timestamp to seconds (ASS format: H:MM:SS.CS where CS is centiseconds) + def ass_time_to_seconds(t): + parts = t.split(":") + if len(parts) == 3: + h, m, s = parts + s_parts = s.split(".") + seconds = float(s_parts[0]) + centiseconds = float(s_parts[1]) if len(s_parts) > 1 else 0 + return ( + int(h) * 3600 + int(m) * 60 + seconds + centiseconds / 100.0 + ) + return 0 + + start_sec = ass_time_to_seconds(start_str) + end_sec = ass_time_to_seconds(end_str) + + # Clean text of ASS styling tags using pre-compiled patterns + text = _ASS_STYLING_PATTERN.sub("", text) # Remove {tags} + text = _ASS_NEWLINE_N_PATTERN.sub("\n", text) # Convert \N to newline + text = _ASS_NEWLINE_LOWER_N_PATTERN.sub( + "\n", text + ) # Convert \n to newline + # Remove chapter markers and metadata tags + text = clean_subtitle_text(text) + + if text: # Only add non-empty subtitles + subtitles.append((start_sec, end_sec, text)) + + return subtitles + + +def get_sample_voice_text(lang_code): + return SAMPLE_VOICE_TEXTS.get(lang_code, SAMPLE_VOICE_TEXTS["a"]) + + +def sanitize_name_for_os(name, is_folder=True): + """ + Sanitize a filename or folder name based on the operating system. + + Args: + name: The name to sanitize + is_folder: Whether this is a folder name (default: True) + + Returns: + Sanitized name safe for the current OS + """ + if not name: + return "audiobook" + + system = platform.system() + + if system == "Windows": + # Windows illegal characters: < > : " / \ | ? * + # Also can't end with space or dot + # Use pre-compiled pattern for better performance + sanitized = _WINDOWS_ILLEGAL_CHARS_PATTERN.sub("_", name) + # Remove control characters (0-31) + sanitized = _CONTROL_CHARS_PATTERN.sub("_", sanitized) + # Remove trailing spaces and dots + sanitized = sanitized.rstrip(". ") + # Windows reserved names (CON, PRN, AUX, NUL, COM1-9, LPT1-9) + reserved = ( + ["CON", "PRN", "AUX", "NUL"] + + [f"COM{i}" for i in range(1, 10)] + + [f"LPT{i}" for i in range(1, 10)] + ) + if sanitized.upper() in reserved or sanitized.upper().split(".")[0] in reserved: + sanitized = f"_{sanitized}" + elif system == "Darwin": # macOS + # macOS illegal characters: : (colon is converted to / by the system) + # Also can't start with dot (hidden file) for folders typically + # Use pre-compiled pattern for better performance + sanitized = _MACOS_ILLEGAL_CHARS_PATTERN.sub("_", name) + # Remove control characters + sanitized = _CONTROL_CHARS_PATTERN.sub("_", sanitized) + # Avoid leading dot for folders (creates hidden folders) + if is_folder and sanitized.startswith("."): + sanitized = "_" + sanitized[1:] + else: # Linux and others + # Linux illegal characters: / and null character + # Though / is illegal, most other chars are technically allowed + # Use pre-compiled pattern for better performance + sanitized = _LINUX_ILLEGAL_CHARS_PATTERN.sub("_", name) + # Remove other control characters for safety (excluding \x00 which is already handled) + sanitized = _LINUX_CONTROL_CHARS_PATTERN.sub("_", sanitized) + # Avoid leading dot for folders (creates hidden folders) + if is_folder and sanitized.startswith("."): + sanitized = "_" + sanitized[1:] + + # Ensure the name is not empty after sanitization + if not sanitized or sanitized.strip() == "": + sanitized = "audiobook" + + # Limit length to 255 characters (common limit across filesystems) + if len(sanitized) > 255: + sanitized = sanitized[:255].rstrip(". ") + + return sanitized diff --git a/abogen/utils.py b/abogen/utils.py index ba6e30e..f5d3633 100644 --- a/abogen/utils.py +++ b/abogen/utils.py @@ -10,14 +10,6 @@ from threading import Thread warnings.filterwarnings("ignore") -# Pre-compile frequently used regex patterns for better performance -_WHITESPACE_PATTERN = re.compile(r"[^\S\n]+") -_MULTIPLE_NEWLINES_PATTERN = re.compile(r"\n{3,}") -_SINGLE_NEWLINE_PATTERN = re.compile(r"(?>") -_METADATA_PATTERN = re.compile(r"<]*>>") - - def detect_encoding(file_path): import chardet import charset_normalizer @@ -129,28 +121,8 @@ def get_user_cache_path(folder=None): _sleep_procs = {"Darwin": None, "Linux": None} # Store sleep prevention processes - -def clean_text(text, *args, **kwargs): - # Load replace_single_newlines from config - cfg = load_config() - replace_single_newlines = cfg.get("replace_single_newlines", True) - # Collapse all whitespace (excluding newlines) into single spaces per line and trim edges - # Use pre-compiled pattern for better performance - lines = [_WHITESPACE_PATTERN.sub(" ", line).strip() for line in text.splitlines()] - text = "\n".join(lines) - # Standardize paragraph breaks (multiple newlines become exactly two) and trim overall whitespace - # Use pre-compiled pattern for better performance - text = _MULTIPLE_NEWLINES_PATTERN.sub("\n\n", text).strip() - # Optionally replace single newlines with spaces, but preserve double newlines - if replace_single_newlines: - # Use pre-compiled pattern for better performance - text = _SINGLE_NEWLINE_PATTERN.sub(" ", text) - return text - - default_encoding = sys.getfilesystemencoding() - def create_process(cmd, stdin=None, text=True, capture_output=False): import logging @@ -252,17 +224,6 @@ def save_config(config): pass -def calculate_text_length(text): - # Use pre-compiled patterns for better performance - # Ignore chapter markers and metadata patterns in a single pass - text = _CHAPTER_MARKER_PATTERN.sub("", text) - text = _METADATA_PATTERN.sub("", text) - # Ignore newlines and leading/trailing spaces - text = text.replace("\n", "").strip() - # Calculate character count - char_count = len(text) - return char_count - def get_gpu_acceleration(enabled): """