Merge pull request #103 from denizsafak/copilot/improve-slow-code-efficiency

Optimize regex compilation and eliminate busy-wait loops

CHANGELOG.md

@@ -1,3 +1,6 @@
+# 1.2.4 (Pre-release)
+- Optimized regex compilation and eliminated busy-wait loops.
+
 # 1.2.3
 - Same as 1.2.2, re-released to fix an issue with subtitle timing when using timestamp-based text files.
 

abogen/book_handler.py

@@ -44,6 +44,16 @@ logging.basicConfig(
     level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
 )
 
+# Pre-compile frequently used regex patterns for better performance
+_BRACKETED_NUMBERS_PATTERN = re.compile(r"\[\s*\d+\s*\]")
+_STANDALONE_PAGE_NUMBERS_PATTERN = re.compile(r"^\s*\d+\s*$", re.MULTILINE)
+_PAGE_NUMBERS_AT_END_PATTERN = re.compile(r"\s+\d+\s*$", re.MULTILINE)
+_PAGE_NUMBERS_WITH_DASH_PATTERN = re.compile(r"\s+[-–—]\s*\d+\s*[-–—]?\s*$", re.MULTILINE)
+_HTML_TAG_PATTERN = re.compile(r"<[^>]+>")
+_LEADING_DASH_PATTERN = re.compile(r"^\s*[-–—]\s*")
+_LEADING_SIMPLE_DASH_PATTERN = re.compile(r"^\s*-\s*")
+
+
 
 class HandlerDialog(QDialog):
     # Class variables to remember checkbox states between dialog instances
@@ -428,19 +438,12 @@ class HandlerDialog(QDialog):
         """Pre-process all page contents from PDF document"""
         for page_num in range(len(self.pdf_doc)):
             text = clean_text(self.pdf_doc[page_num].get_text())
-            # Remove bracketed numbers (citations, footnotes)
+            # Remove bracketed numbers, page numbers, etc. using pre-compiled patterns
-            text = re.sub(r"\[\s*\d+\s*\]", "", text)
+            # Combine all regex operations for better performance
-
+            text = _BRACKETED_NUMBERS_PATTERN.sub("", text)
-            # Remove standalone page numbers (numbers alone on a line)
+            text = _STANDALONE_PAGE_NUMBERS_PATTERN.sub("", text)
-            text = re.sub(r"^\s*\d+\s*$", "", text, flags=re.MULTILINE)
+            text = _PAGE_NUMBERS_AT_END_PATTERN.sub("", text)
-
+            text = _PAGE_NUMBERS_WITH_DASH_PATTERN.sub("", text)
-            # Remove page numbers at the end of paragraphs
-            # This pattern looks for digits surrounded by whitespace at the end of paragraphs
-            text = re.sub(r"\s+\d+\s*$", "", text, flags=re.MULTILINE)
-
-            # Also remove page numbers followed by a hyphen or dash at paragraph end
-            # (common in headers/footers like "- 42 -")
-            text = re.sub(r"\s+[-–—]\s*\d+\s*[-–—]?\s*$", "", text, flags=re.MULTILINE)
 
             page_id = f"page_{page_num + 1}"
             self.content_texts[page_id] = text
@@ -569,7 +572,7 @@ class HandlerDialog(QDialog):
                 text = clean_text(soup.get_text()).strip()
                 if text:
                     self.content_texts[doc_href] = text
-                    self.content_lengths[doc_href] = len(text)
+                    self.content_lengths[doc_href] = calculate_text_length(text)
 
                     title = None
                     if soup.title and soup.title.string:
@@ -892,7 +895,7 @@ class HandlerDialog(QDialog):
                 text = clean_text(slice_soup.get_text()).strip()
                 if text:
                     self.content_texts[current_src] = text
-                    self.content_lengths[current_src] = len(text)
+                    self.content_lengths[current_src] = calculate_text_length(text)
                 else:
                     self.content_texts[current_src] = ""
                     self.content_lengths[current_src] = 0
@@ -2015,7 +2018,8 @@ class HandlerDialog(QDialog):
         html_content += "<hr/>"
 
         if self.book_metadata["description"]:
-            desc = re.sub(r"<[^>]+>", "", self.book_metadata["description"])
+            # Use pre-compiled pattern for better performance
+            desc = _HTML_TAG_PATTERN.sub("", self.book_metadata["description"])
             html_content += f"<h3>Description:</h3><p>{desc}</p>"
 
         if self.file_type == "pdf":
@@ -2296,8 +2300,8 @@ class HandlerDialog(QDialog):
             text = self.content_texts.get(identifier)
             if text and text.strip():
                 title = item.text(0)
-                # Remove leading dashes from title
+                # Remove leading dashes from title using pre-compiled pattern
-                title = re.sub(r"^\s*[-–—]\s*", "", title).strip()
+                title = _LEADING_DASH_PATTERN.sub("", title).strip()
                 marker = f"<<CHAPTER_MARKER:{title}>>"
                 chapter_texts.append(marker + "\n" + text)
 
@@ -2331,7 +2335,8 @@ class HandlerDialog(QDialog):
             text = self.content_texts.get(identifier)
             if text and text.strip():
                 title = item.text(0)
-                title = re.sub(r"^\s*[-–—]\s*", "", title).strip()
+                # Use pre-compiled pattern for better performance
+                title = _LEADING_DASH_PATTERN.sub("", title).strip()
                 marker = f"<<CHAPTER_MARKER:{title}>>"
                 chapter_texts.append(marker + "\n" + text)
 
@@ -2401,12 +2406,14 @@ class HandlerDialog(QDialog):
                             combined_text += "\n\n" + child_text
                         included_text_ids.add(child_id)
                     if combined_text.strip():
-                        title = re.sub(r"^\s*-\s*", "", parent_title).strip()
+                        # Use pre-compiled pattern for better performance
+                        title = _LEADING_SIMPLE_DASH_PATTERN.sub("", parent_title).strip()
                         marker = f"<<CHAPTER_MARKER:{title}>>"
                         section_titles.append((title, marker + "\n" + combined_text))
                         included_text_ids.add(parent_id)
                 elif not parent_checked and checked_children:
-                    title = re.sub(r"^\s*-\s*", "", parent_title).strip()
+                    # Use pre-compiled pattern for better performance
+                    title = _LEADING_SIMPLE_DASH_PATTERN.sub("", parent_title).strip()
                     marker = f"<<CHAPTER_MARKER:{title}>>"
                     for idx, (child, child_id) in enumerate(checked_children):
                         text = self.content_texts.get(child_id, "")
@@ -2426,7 +2433,8 @@ class HandlerDialog(QDialog):
                     text = self.content_texts.get(identifier, "")
                     if text:
                         title = item.text(0)
-                        title = re.sub(r"^\s*-\s*", "", title).strip()
+                        # Use pre-compiled pattern for better performance
+                        title = _LEADING_SIMPLE_DASH_PATTERN.sub("", title).strip()
                         marker = f"<<CHAPTER_MARKER:{title}>>"
                         section_titles.append((title, marker + "\n" + text))
                         included_text_ids.add(identifier)

abogen/conversion.py

@@ -28,13 +28,38 @@ import threading  # for efficient waiting
 import subprocess
 import platform
 
+# Configuration constants
+_USER_RESPONSE_TIMEOUT = 0.1  # Timeout in seconds for checking user response/cancellation
+
+# Pre-compile frequently used regex patterns for better performance
+_METADATA_TAG_PATTERN = re.compile(r"<<METADATA_[^:]+:[^>]*>>")
+_CHAPTER_MARKER_PATTERN = re.compile(r"<<CHAPTER_MARKER:[^>]*>>")
+_HTML_TAG_PATTERN = re.compile(r"<[^>]+>")
+_VOICE_TAG_PATTERN = re.compile(r"{[^}]+}")
+_ASS_STYLING_PATTERN = re.compile(r"\{[^}]+\}")
+_ASS_NEWLINE_N_PATTERN = re.compile(r"\\N")
+_ASS_NEWLINE_n_PATTERN = re.compile(r"\\n")
+_BRACKETED_NUMBERS_PATTERN = re.compile(r"\[\s*\d+\s*\]")
+_CHAPTER_MARKER_SEARCH_PATTERN = re.compile(r"<<CHAPTER_MARKER:(.*?)>>")
+_WEBVTT_HEADER_PATTERN = re.compile(r"^WEBVTT.*?\n", re.MULTILINE)
+_VTT_STYLE_PATTERN = re.compile(r"STYLE\s*\n.*?(?=\n\n|$)", re.DOTALL)
+_VTT_NOTE_PATTERN = re.compile(r"NOTE\s*\n.*?(?=\n\n|$)", re.DOTALL)
+_DOUBLE_NEWLINE_SPLIT_PATTERN = re.compile(r"\n\s*\n")
+_SRT_TIMESTAMP_PATTERN = re.compile(r"(\d{2}:\d{2}:\d{2},\d{3})\s*-->\s*(\d{2}:\d{2}:\d{2},\d{3})")
+_VTT_TIMESTAMP_PATTERN = re.compile(r"([\d:.]+)\s*-->\s*([\d:.]+)")
+_TIMESTAMP_ONLY_PATTERN = re.compile(r"^(\d{1,2}:\d{2}:\d{2}(?:,\d{1,3})?)$")
+_WINDOWS_ILLEGAL_CHARS_PATTERN = re.compile(r'[<>:"/\\|?*]')
+_CONTROL_CHARS_PATTERN = re.compile(r"[\x00-\x1f]")
+_LINUX_CONTROL_CHARS_PATTERN = re.compile(r"[\x01-\x1f]")  # Linux: exclude \x00 for separate handling
+_MACOS_ILLEGAL_CHARS_PATTERN = re.compile(r"[:]")
+_LINUX_ILLEGAL_CHARS_PATTERN = re.compile(r"[/\x00]")
+
 
 def clean_subtitle_text(text):
     """Remove chapter markers and metadata tags from subtitle text."""
-    # Remove metadata tags
+    # Use pre-compiled patterns for better performance
-    text = re.sub(r"<<METADATA_[^:]+:[^>]*>>", "", text)
+    text = _METADATA_TAG_PATTERN.sub("", text)
-    # Remove chapter markers
+    text = _CHAPTER_MARKER_PATTERN.sub("", text)
-    text = re.sub(r"<<CHAPTER_MARKER:[^>]*>>", "", text)
     return text.strip()
 
 
@@ -87,8 +112,8 @@ def parse_srt_file(file_path):
             start_sec = time_to_seconds(start_str)
             end_sec = time_to_seconds(end_str)
 
-            # Clean text of any styling tags
+            # Clean text of any styling tags using pre-compiled pattern
-            text = re.sub(r"<[^>]+>", "", text)
+            text = _HTML_TAG_PATTERN.sub("", text)
             # Remove chapter markers and metadata tags
             text = clean_subtitle_text(text)
 
@@ -114,13 +139,13 @@ def parse_vtt_file(file_path):
     with open(file_path, "r", encoding=encoding, errors="replace") as f:
         content = f.read()
 
-    # Remove WEBVTT header and any style/note blocks
+    # Remove WEBVTT header and any style/note blocks using pre-compiled patterns
-    content = re.sub(r"^WEBVTT.*?\n", "", content, flags=re.MULTILINE)
+    content = _WEBVTT_HEADER_PATTERN.sub("", content)
-    content = re.sub(r"STYLE\s*\n.*?(?=\n\n|$)", "", content, flags=re.DOTALL)
+    content = _VTT_STYLE_PATTERN.sub("", content)
-    content = re.sub(r"NOTE\s*\n.*?(?=\n\n|$)", "", content, flags=re.DOTALL)
+    content = _VTT_NOTE_PATTERN.sub("", content)
 
-    # Split by double newlines to get individual subtitle blocks
+    # Split by double newlines to get individual subtitle blocks using pre-compiled pattern
-    blocks = re.split(r"\n\s*\n", content.strip())
+    blocks = _DOUBLE_NEWLINE_SPLIT_PATTERN.split(content.strip())
 
     subtitles = []
     for block in blocks:
@@ -147,7 +172,8 @@ def parse_vtt_file(file_path):
 
         try:
             # VTT format: 00:00:00.000 --> 00:00:05.000 or 00:00.000 --> 00:05.000
-            match = re.match(r"([\d:.]+)\s*-->\s*([\d:.]+)", timestamp_line)
+            # Use pre-compiled pattern
+            match = _VTT_TIMESTAMP_PATTERN.match(timestamp_line)
             if not match:
                 continue
 
@@ -171,9 +197,9 @@ def parse_vtt_file(file_path):
             start_sec = time_to_seconds(start_str)
             end_sec = time_to_seconds(end_str)
 
-            # Clean text of any styling tags and cue settings
+            # Clean text of any styling tags and cue settings using pre-compiled patterns
-            text = re.sub(r"<[^>]+>", "", text)
+            text = _HTML_TAG_PATTERN.sub("", text)
-            text = re.sub(r"{[^}]+}", "", text)  # Remove voice tags
+            text = _VOICE_TAG_PATTERN.sub("", text)  # Remove voice tags
             # Remove chapter markers and metadata tags
             text = clean_subtitle_text(text)
 
@@ -196,8 +222,8 @@ def detect_timestamps_in_text(file_path):
 
         # Count lines that are ONLY timestamps (no other text)
         # Supports HH:MM:SS or HH:MM:SS,ms format
-        timestamp_pattern = r"^(\d{1,2}:\d{2}:\d{2}(?:,\d{1,3})?)$"
+        # Use pre-compiled pattern for better performance
-        timestamp_lines = sum(1 for line in lines if re.match(timestamp_pattern, line))
+        timestamp_lines = sum(1 for line in lines if _TIMESTAMP_ONLY_PATTERN.match(line))
 
         # Must have at least 2 timestamp-only lines and they should be >5% of total lines
         return timestamp_lines >= 2 and (timestamp_lines / max(len(lines), 1)) > 0.05
@@ -348,10 +374,10 @@ def parse_ass_file(file_path):
                 start_sec = ass_time_to_seconds(start_str)
                 end_sec = ass_time_to_seconds(end_str)
 
-                # Clean text of ASS styling tags
+                # Clean text of ASS styling tags using pre-compiled patterns
-                text = re.sub(r"\{[^}]+\}", "", text)  # Remove {tags}
+                text = _ASS_STYLING_PATTERN.sub("", text)  # Remove {tags}
-                text = re.sub(r"\\N", "\n", text)  # Convert \N to newline
+                text = _ASS_NEWLINE_N_PATTERN.sub("\n", text)  # Convert \N to newline
-                text = re.sub(r"\\n", "\n", text)  # Convert \n to newline
+                text = _ASS_NEWLINE_n_PATTERN.sub("\n", text)  # Convert \n to newline
                 # Remove chapter markers and metadata tags
                 text = clean_subtitle_text(text)
 
@@ -384,9 +410,10 @@ def sanitize_name_for_os(name, is_folder=True):
     if system == "Windows":
         # Windows illegal characters: < > : " / \ | ? *
         # Also can't end with space or dot
-        sanitized = re.sub(r'[<>:"/\\|?*]', "_", name)
+        # Use pre-compiled pattern for better performance
+        sanitized = _WINDOWS_ILLEGAL_CHARS_PATTERN.sub("_", name)
         # Remove control characters (0-31)
-        sanitized = re.sub(r"[\x00-\x1f]", "_", sanitized)
+        sanitized = _CONTROL_CHARS_PATTERN.sub("_", sanitized)
         # Remove trailing spaces and dots
         sanitized = sanitized.rstrip(". ")
         # Windows reserved names (CON, PRN, AUX, NUL, COM1-9, LPT1-9)
@@ -400,18 +427,20 @@ def sanitize_name_for_os(name, is_folder=True):
     elif system == "Darwin":  # macOS
         # macOS illegal characters: : (colon is converted to / by the system)
         # Also can't start with dot (hidden file) for folders typically
-        sanitized = re.sub(r"[:]", "_", name)
+        # Use pre-compiled pattern for better performance
+        sanitized = _MACOS_ILLEGAL_CHARS_PATTERN.sub("_", name)
         # Remove control characters
-        sanitized = re.sub(r"[\x00-\x1f]", "_", sanitized)
+        sanitized = _CONTROL_CHARS_PATTERN.sub("_", sanitized)
         # Avoid leading dot for folders (creates hidden folders)
         if is_folder and sanitized.startswith("."):
             sanitized = "_" + sanitized[1:]
     else:  # Linux and others
         # Linux illegal characters: / and null character
         # Though / is illegal, most other chars are technically allowed
-        sanitized = re.sub(r"[/\x00]", "_", name)
+        # Use pre-compiled pattern for better performance
-        # Remove other control characters for safety
+        sanitized = _LINUX_ILLEGAL_CHARS_PATTERN.sub("_", name)
-        sanitized = re.sub(r"[\x01-\x1f]", "_", sanitized)
+        # Remove other control characters for safety (excluding \x00 which is already handled)
+        sanitized = _LINUX_CONTROL_CHARS_PATTERN.sub("_", sanitized)
         # Avoid leading dot for folders (creates hidden folders)
         if is_folder and sanitized.startswith("."):
             sanitized = "_" + sanitized[1:]
@@ -628,6 +657,7 @@ class ConversionThread(QThread):
     ):  # Add use_gpu parameter
         super().__init__()
         self._chapter_options_event = threading.Event()
+        self._timestamp_response_event = threading.Event()
         self.np = np_module
         self.KPipeline = kpipeline_class
         self.file_name = file_name
@@ -847,15 +877,19 @@ class ConversionThread(QThread):
                     self.log_updated.emit("\nDetected timestamps in text file")
                     # Signal to ask user (-1 indicates timestamp detection)
                     self.chapters_detected.emit(-1)
-                    # Wait for user response
+                    # Wait for user response using event with timeout for responsive cancellation
-                    while not hasattr(self, "_timestamp_response"):
+                    while not self._timestamp_response_event.wait(timeout=_USER_RESPONSE_TIMEOUT):
                         if self.cancel_requested:
                             self.conversion_finished.emit("Cancelled", None)
                             return
-                        time.sleep(0.1)
+                    # Check cancellation one more time after event is set
+                    if self.cancel_requested:
+                        self.conversion_finished.emit("Cancelled", None)
+                        return
                     if not self._timestamp_response:
                         is_timestamp_text = False
                     delattr(self, "_timestamp_response")
+                    self._timestamp_response_event.clear()
 
             # Process subtitle files separately
             if is_subtitle_file or is_timestamp_text:
@@ -875,12 +909,12 @@ class ConversionThread(QThread):
             text = clean_text(text)
 
             # Remove metadata markers from the text to be processed
-            metadata_pattern = r"<<METADATA_[^:]+:[^>]*>>"
+            # Use pre-compiled pattern for better performance
-            text = re.sub(metadata_pattern, "", text)
+            text = _METADATA_TAG_PATTERN.sub("", text)
 
             # --- Chapter splitting logic ---
-            chapter_pattern = r"<<CHAPTER_MARKER:(.*?)>>"
+            # Use pre-compiled pattern for better performance
-            chapter_splits = list(re.finditer(chapter_pattern, text))
+            chapter_splits = list(_CHAPTER_MARKER_SEARCH_PATTERN.finditer(text))
             chapters = []
             if chapter_splits:
                 # prepend Introduction for content before first marker
@@ -984,10 +1018,12 @@ class ConversionThread(QThread):
                     parent_dir, f"{sanitized_base_name}{suffix}_chapters"
                 )
                 # Only check for files with allowed extensions (extension without dot, case-insensitive)
+                # Use generator expression to avoid processing all files upfront
+                file_parts = (os.path.splitext(fname) for fname in os.listdir(parent_dir))
                 clash = any(
-                    os.path.splitext(fname)[0] == f"{sanitized_base_name}{suffix}"
+                    name == f"{sanitized_base_name}{suffix}"
-                    and os.path.splitext(fname)[1][1:].lower() in allowed_exts
+                    and ext[1:].lower() in allowed_exts
-                    for fname in os.listdir(parent_dir)
+                    for name, ext in file_parts
                 )
                 if not os.path.exists(chapters_out_dir_candidate) and not clash:
                     break
@@ -1727,10 +1763,12 @@ class ConversionThread(QThread):
             allowed_exts = set(SUPPORTED_SOUND_FORMATS + SUPPORTED_SUBTITLE_FORMATS)
             while True:
                 suffix = f"_{counter}" if counter > 1 else ""
+                # Use generator expression to avoid processing all files upfront
+                file_parts = (os.path.splitext(f) for f in os.listdir(parent_dir))
                 if not any(
-                    os.path.splitext(f)[0] == f"{sanitized_base_name}{suffix}"
+                    name == f"{sanitized_base_name}{suffix}"
-                    and os.path.splitext(f)[1][1:].lower() in allowed_exts
+                    and ext[1:].lower() in allowed_exts
-                    for f in os.listdir(parent_dir)
+                    for name, ext in file_parts
                 ):
                     break
                 counter += 1
@@ -2158,6 +2196,7 @@ class ConversionThread(QThread):
     def set_timestamp_response(self, treat_as_subtitle):
         """Set whether to treat timestamp text file as subtitle."""
         self._timestamp_response = treat_as_subtitle
+        self._timestamp_response_event.set()
 
     def _extract_and_add_metadata_tags_to_ffmpeg_cmd(self):
         """Extract metadata tags from text content and add them to ffmpeg command"""

abogen/utils.py

@@ -10,6 +10,13 @@ from threading import Thread
 
 warnings.filterwarnings("ignore")
 
+# Pre-compile frequently used regex patterns for better performance
+_WHITESPACE_PATTERN = re.compile(r"[^\S\n]+")
+_MULTIPLE_NEWLINES_PATTERN = re.compile(r"\n{3,}")
+_SINGLE_NEWLINE_PATTERN = re.compile(r"(?<!\n)\n(?!\n)")
+_CHAPTER_MARKER_PATTERN = re.compile(r"<<CHAPTER_MARKER:.*?>>")
+_METADATA_PATTERN = re.compile(r"<<METADATA_[^:]+:[^>]*>>")
+
 
 def detect_encoding(file_path):
     import chardet
@@ -128,13 +135,16 @@ def clean_text(text, *args, **kwargs):
     cfg = load_config()
     replace_single_newlines = cfg.get("replace_single_newlines", False)
     # Collapse all whitespace (excluding newlines) into single spaces per line and trim edges
-    lines = [re.sub(r"[^\S\n]+", " ", line).strip() for line in text.splitlines()]
+    # Use pre-compiled pattern for better performance
+    lines = [_WHITESPACE_PATTERN.sub(" ", line).strip() for line in text.splitlines()]
     text = "\n".join(lines)
     # Standardize paragraph breaks (multiple newlines become exactly two) and trim overall whitespace
-    text = re.sub(r"\n{3,}", "\n\n", text).strip()
+    # Use pre-compiled pattern for better performance
+    text = _MULTIPLE_NEWLINES_PATTERN.sub("\n\n", text).strip()
     # Optionally replace single newlines with spaces, but preserve double newlines
     if replace_single_newlines:
-        text = re.sub(r"(?<!\n)\n(?!\n)", " ", text)
+        # Use pre-compiled pattern for better performance
+        text = _SINGLE_NEWLINE_PATTERN.sub(" ", text)
     return text
 
 
@@ -243,14 +253,12 @@ def save_config(config):
 
 
 def calculate_text_length(text):
-    # Ignore chapter markers
+    # Use pre-compiled patterns for better performance
-    text = re.sub(r"<<CHAPTER_MARKER:.*?>>", "", text)
+    # Ignore chapter markers and metadata patterns in a single pass
-    # Ignore metadata patterns
+    text = _CHAPTER_MARKER_PATTERN.sub("", text)
-    text = re.sub(r"<<METADATA_[^:]+:[^>]*>>", "", text)
+    text = _METADATA_PATTERN.sub("", text)
-    # Ignore newlines
+    # Ignore newlines and leading/trailing spaces
-    text = text.replace("\n", "")
+    text = text.replace("\n", "").strip()
-    # Ignore leading/trailing spaces
-    text = text.strip()
     # Calculate character count
     char_count = len(text)
     return char_count