Dymas/abogen
1
0
Fork 0
mirror of https://github.com/denizsafak/abogen.git synced 2026-07-18 13:40:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Optimize regex patterns by pre-compiling frequently used patterns

Browse Source 
Co-authored-by: denizsafak <39929354+denizsafak@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2025-11-20 23:11:06 +00:00
co-authored by denizsafak
parent cd2e5c9150
commit 7f75aa8209
3 changed files with 107 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
abogen/book_handler.py
+30 -22
View File
@@ -44,6 +44,16 @@ logging.basicConfig(
level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
)
# Pre-compile frequently used regex patterns for better performance
_BRACKETED_NUMBERS_PATTERN = re.compile(r"\[\s*\d+\s*\]")
_STANDALONE_PAGE_NUMBERS_PATTERN = re.compile(r"^\s*\d+\s*$", re.MULTILINE)
_PAGE_NUMBERS_AT_END_PATTERN = re.compile(r"\s+\d+\s*$", re.MULTILINE)
_PAGE_NUMBERS_WITH_DASH_PATTERN = re.compile(r"\s+[-–—]\s*\d+\s*[-–—]?\s*$", re.MULTILINE)
_HTML_TAG_PATTERN = re.compile(r"<[^>]+>")
_LEADING_DASH_PATTERN = re.compile(r"^\s*[-–—]\s*")
_LEADING_SIMPLE_DASH_PATTERN = re.compile(r"^\s*-\s*")
class HandlerDialog(QDialog):
# Class variables to remember checkbox states between dialog instances
@@ -428,19 +438,12 @@ class HandlerDialog(QDialog):
"""Pre-process all page contents from PDF document"""
for page_num in range(len(self.pdf_doc)):
text = clean_text(self.pdf_doc[page_num].get_text())
# Remove bracketed numbers (citations, footnotes)
text = re.sub(r"\[\s*\d+\s*\]", "", text)
# Remove standalone page numbers (numbers alone on a line)
text = re.sub(r"^\s*\d+\s*$", "", text, flags=re.MULTILINE)
# Remove page numbers at the end of paragraphs
# This pattern looks for digits surrounded by whitespace at the end of paragraphs
text = re.sub(r"\s+\d+\s*$", "", text, flags=re.MULTILINE)
# Also remove page numbers followed by a hyphen or dash at paragraph end
# (common in headers/footers like "- 42 -")
text = re.sub(r"\s+[-–—]\s*\d+\s*[-–—]?\s*$", "", text, flags=re.MULTILINE)
# Remove bracketed numbers, page numbers, etc. using pre-compiled patterns
# Combine all regex operations for better performance
text = _BRACKETED_NUMBERS_PATTERN.sub("", text)
text = _STANDALONE_PAGE_NUMBERS_PATTERN.sub("", text)
text = _PAGE_NUMBERS_AT_END_PATTERN.sub("", text)
text = _PAGE_NUMBERS_WITH_DASH_PATTERN.sub("", text)
page_id = f"page_{page_num + 1}"
self.content_texts[page_id] = text
@@ -569,7 +572,7 @@ class HandlerDialog(QDialog):
text = clean_text(soup.get_text()).strip()
if text:
self.content_texts[doc_href] = text
self.content_lengths[doc_href] = len(text)
self.content_lengths[doc_href] = calculate_text_length(text)
title = None
if soup.title and soup.title.string:
@@ -892,7 +895,7 @@ class HandlerDialog(QDialog):
text = clean_text(slice_soup.get_text()).strip()
if text:
self.content_texts[current_src] = text
self.content_lengths[current_src] = len(text)
self.content_lengths[current_src] = calculate_text_length(text)
else:
self.content_texts[current_src] = ""
self.content_lengths[current_src] = 0
@@ -2015,7 +2018,8 @@ class HandlerDialog(QDialog):
html_content += "<hr/>"
if self.book_metadata["description"]:
desc = re.sub(r"<[^>]+>", "", self.book_metadata["description"])
# Use pre-compiled pattern for better performance
desc = _HTML_TAG_PATTERN.sub("", self.book_metadata["description"])
html_content += f"<h3>Description:</h3><p>{desc}</p>"
if self.file_type == "pdf":
@@ -2296,8 +2300,8 @@ class HandlerDialog(QDialog):
text = self.content_texts.get(identifier)
if text and text.strip():
title = item.text(0)
# Remove leading dashes from title
title = re.sub(r"^\s*[-–—]\s*", "", title).strip()
# Remove leading dashes from title using pre-compiled pattern
title = _LEADING_DASH_PATTERN.sub("", title).strip()
marker = f"<<CHAPTER_MARKER:{title}>>"
chapter_texts.append(marker + "\n" + text)
@@ -2331,7 +2335,8 @@ class HandlerDialog(QDialog):
text = self.content_texts.get(identifier)
if text and text.strip():
title = item.text(0)
title = re.sub(r"^\s*[-–—]\s*", "", title).strip()
# Use pre-compiled pattern for better performance
title = _LEADING_DASH_PATTERN.sub("", title).strip()
marker = f"<<CHAPTER_MARKER:{title}>>"
chapter_texts.append(marker + "\n" + text)
@@ -2401,12 +2406,14 @@ class HandlerDialog(QDialog):
combined_text += "\n\n" + child_text
included_text_ids.add(child_id)
if combined_text.strip():
title = re.sub(r"^\s*-\s*", "", parent_title).strip()
# Use pre-compiled pattern for better performance
title = _LEADING_SIMPLE_DASH_PATTERN.sub("", parent_title).strip()
marker = f"<<CHAPTER_MARKER:{title}>>"
section_titles.append((title, marker + "\n" + combined_text))
included_text_ids.add(parent_id)
elif not parent_checked and checked_children:
title = re.sub(r"^\s*-\s*", "", parent_title).strip()
# Use pre-compiled pattern for better performance
title = _LEADING_SIMPLE_DASH_PATTERN.sub("", parent_title).strip()
marker = f"<<CHAPTER_MARKER:{title}>>"
for idx, (child, child_id) in enumerate(checked_children):
text = self.content_texts.get(child_id, "")
@@ -2426,7 +2433,8 @@ class HandlerDialog(QDialog):
text = self.content_texts.get(identifier, "")
if text:
title = item.text(0)
title = re.sub(r"^\s*-\s*", "", title).strip()
# Use pre-compiled pattern for better performance
title = _LEADING_SIMPLE_DASH_PATTERN.sub("", title).strip()
marker = f"<<CHAPTER_MARKER:{title}>>"
section_titles.append((title, marker + "\n" + text))
included_text_ids.add(identifier)
abogen/conversion.py
+58 -33
View File
@@ -3,7 +3,7 @@ import re
import time
import hashlib # For generating unique cache filenames
from platformdirs import user_desktop_dir
from PyQt6.QtCore import QThread, pyqtSignal, Qt, QTimer
from PyQt6.QtCore import QThread, pyqtSignal, Qt, QTimer, QEventLoop
from PyQt6.QtWidgets import QCheckBox, QVBoxLayout, QDialog, QLabel, QDialogButtonBox
import soundfile as sf
from abogen.utils import (
@@ -28,13 +28,34 @@ import threading # for efficient waiting
import subprocess
import platform
# Pre-compile frequently used regex patterns for better performance
_METADATA_TAG_PATTERN = re.compile(r"<<METADATA_[^:]+:[^>]*>>")
_CHAPTER_MARKER_PATTERN = re.compile(r"<<CHAPTER_MARKER:[^>]*>>")
_HTML_TAG_PATTERN = re.compile(r"<[^>]+>")
_VOICE_TAG_PATTERN = re.compile(r"{[^}]+}")
_ASS_STYLING_PATTERN = re.compile(r"\{[^}]+\}")
_ASS_NEWLINE_N_PATTERN = re.compile(r"\\N")
_ASS_NEWLINE_n_PATTERN = re.compile(r"\\n")
_BRACKETED_NUMBERS_PATTERN = re.compile(r"\[\s*\d+\s*\]")
_CHAPTER_MARKER_SEARCH_PATTERN = re.compile(r"<<CHAPTER_MARKER:(.*?)>>")
_WEBVTT_HEADER_PATTERN = re.compile(r"^WEBVTT.*?\n", re.MULTILINE)
_VTT_STYLE_PATTERN = re.compile(r"STYLE\s*\n.*?(?=\n\n|$)", re.DOTALL)
_VTT_NOTE_PATTERN = re.compile(r"NOTE\s*\n.*?(?=\n\n|$)", re.DOTALL)
_DOUBLE_NEWLINE_SPLIT_PATTERN = re.compile(r"\n\s*\n")
_SRT_TIMESTAMP_PATTERN = re.compile(r"(\d{2}:\d{2}:\d{2},\d{3})\s*-->\s*(\d{2}:\d{2}:\d{2},\d{3})")
_VTT_TIMESTAMP_PATTERN = re.compile(r"([\d:.]+)\s*-->\s*([\d:.]+)")
_TIMESTAMP_ONLY_PATTERN = re.compile(r"^(\d{1,2}:\d{2}:\d{2}(?:,\d{1,3})?)$")
_WINDOWS_ILLEGAL_CHARS_PATTERN = re.compile(r'[<>:"/\\|?*]')
_CONTROL_CHARS_PATTERN = re.compile(r"[\x00-\x1f]")
_MACOS_ILLEGAL_CHARS_PATTERN = re.compile(r"[:]")
_LINUX_ILLEGAL_CHARS_PATTERN = re.compile(r"[/\x00]")
def clean_subtitle_text(text):
"""Remove chapter markers and metadata tags from subtitle text."""
# Remove metadata tags
text = re.sub(r"<<METADATA_[^:]+:[^>]*>>", "", text)
# Remove chapter markers
text = re.sub(r"<<CHAPTER_MARKER:[^>]*>>", "", text)
# Use pre-compiled patterns for better performance
text = _METADATA_TAG_PATTERN.sub("", text)
text = _CHAPTER_MARKER_PATTERN.sub("", text)
return text.strip()
@@ -87,8 +108,8 @@ def parse_srt_file(file_path):
start_sec = time_to_seconds(start_str)
end_sec = time_to_seconds(end_str)
# Clean text of any styling tags
text = re.sub(r"<[^>]+>", "", text)
# Clean text of any styling tags using pre-compiled pattern
text = _HTML_TAG_PATTERN.sub("", text)
# Remove chapter markers and metadata tags
text = clean_subtitle_text(text)
@@ -114,13 +135,13 @@ def parse_vtt_file(file_path):
with open(file_path, "r", encoding=encoding, errors="replace") as f:
content = f.read()
# Remove WEBVTT header and any style/note blocks
content = re.sub(r"^WEBVTT.*?\n", "", content, flags=re.MULTILINE)
content = re.sub(r"STYLE\s*\n.*?(?=\n\n|$)", "", content, flags=re.DOTALL)
content = re.sub(r"NOTE\s*\n.*?(?=\n\n|$)", "", content, flags=re.DOTALL)
# Remove WEBVTT header and any style/note blocks using pre-compiled patterns
content = _WEBVTT_HEADER_PATTERN.sub("", content)
content = _VTT_STYLE_PATTERN.sub("", content)
content = _VTT_NOTE_PATTERN.sub("", content)
# Split by double newlines to get individual subtitle blocks
blocks = re.split(r"\n\s*\n", content.strip())
# Split by double newlines to get individual subtitle blocks using pre-compiled pattern
blocks = _DOUBLE_NEWLINE_SPLIT_PATTERN.split(content.strip())
subtitles = []
for block in blocks:
@@ -147,7 +168,8 @@ def parse_vtt_file(file_path):
try:
# VTT format: 00:00:00.000 --> 00:00:05.000 or 00:00.000 --> 00:05.000
match = re.match(r"([\d:.]+)\s*-->\s*([\d:.]+)", timestamp_line)
# Use pre-compiled pattern
match = _VTT_TIMESTAMP_PATTERN.match(timestamp_line)
if not match:
continue
@@ -171,9 +193,9 @@ def parse_vtt_file(file_path):
start_sec = time_to_seconds(start_str)
end_sec = time_to_seconds(end_str)
# Clean text of any styling tags and cue settings
text = re.sub(r"<[^>]+>", "", text)
text = re.sub(r"{[^}]+}", "", text) # Remove voice tags
# Clean text of any styling tags and cue settings using pre-compiled patterns
text = _HTML_TAG_PATTERN.sub("", text)
text = _VOICE_TAG_PATTERN.sub("", text) # Remove voice tags
# Remove chapter markers and metadata tags
text = clean_subtitle_text(text)
@@ -196,8 +218,8 @@ def detect_timestamps_in_text(file_path):
# Count lines that are ONLY timestamps (no other text)
# Supports HH:MM:SS or HH:MM:SS,ms format
timestamp_pattern = r"^(\d{1,2}:\d{2}:\d{2}(?:,\d{1,3})?)$"
timestamp_lines = sum(1 for line in lines if re.match(timestamp_pattern, line))
# Use pre-compiled pattern for better performance
timestamp_lines = sum(1 for line in lines if _TIMESTAMP_ONLY_PATTERN.match(line))
# Must have at least 2 timestamp-only lines and they should be >5% of total lines
return timestamp_lines >= 2 and (timestamp_lines / max(len(lines), 1)) > 0.05
@@ -348,10 +370,10 @@ def parse_ass_file(file_path):
start_sec = ass_time_to_seconds(start_str)
end_sec = ass_time_to_seconds(end_str)
# Clean text of ASS styling tags
text = re.sub(r"\{[^}]+\}", "", text) # Remove {tags}
text = re.sub(r"\\N", "\n", text) # Convert \N to newline
text = re.sub(r"\\n", "\n", text) # Convert \n to newline
# Clean text of ASS styling tags using pre-compiled patterns
text = _ASS_STYLING_PATTERN.sub("", text) # Remove {tags}
text = _ASS_NEWLINE_N_PATTERN.sub("\n", text) # Convert \N to newline
text = _ASS_NEWLINE_n_PATTERN.sub("\n", text) # Convert \n to newline
# Remove chapter markers and metadata tags
text = clean_subtitle_text(text)
@@ -384,9 +406,10 @@ def sanitize_name_for_os(name, is_folder=True):
if system == "Windows":
# Windows illegal characters: < > : " / \ | ? *
# Also can't end with space or dot
sanitized = re.sub(r'[<>:"/\\|?*]', "_", name)
# Use pre-compiled pattern for better performance
sanitized = _WINDOWS_ILLEGAL_CHARS_PATTERN.sub("_", name)
# Remove control characters (0-31)
sanitized = re.sub(r"[\x00-\x1f]", "_", sanitized)
sanitized = _CONTROL_CHARS_PATTERN.sub("_", sanitized)
# Remove trailing spaces and dots
sanitized = sanitized.rstrip(". ")
# Windows reserved names (CON, PRN, AUX, NUL, COM1-9, LPT1-9)
@@ -400,18 +423,20 @@ def sanitize_name_for_os(name, is_folder=True):
elif system == "Darwin": # macOS
# macOS illegal characters: : (colon is converted to / by the system)
# Also can't start with dot (hidden file) for folders typically
sanitized = re.sub(r"[:]", "_", name)
# Use pre-compiled pattern for better performance
sanitized = _MACOS_ILLEGAL_CHARS_PATTERN.sub("_", name)
# Remove control characters
sanitized = re.sub(r"[\x00-\x1f]", "_", sanitized)
sanitized = _CONTROL_CHARS_PATTERN.sub("_", sanitized)
# Avoid leading dot for folders (creates hidden folders)
if is_folder and sanitized.startswith("."):
sanitized = "_" + sanitized[1:]
else: # Linux and others
# Linux illegal characters: / and null character
# Though / is illegal, most other chars are technically allowed
sanitized = re.sub(r"[/\x00]", "_", name)
# Use pre-compiled pattern for better performance
sanitized = _LINUX_ILLEGAL_CHARS_PATTERN.sub("_", name)
# Remove other control characters for safety
sanitized = re.sub(r"[\x01-\x1f]", "_", sanitized)
sanitized = _CONTROL_CHARS_PATTERN.sub("_", sanitized)
# Avoid leading dot for folders (creates hidden folders)
if is_folder and sanitized.startswith("."):
sanitized = "_" + sanitized[1:]
@@ -875,12 +900,12 @@ class ConversionThread(QThread):
text = clean_text(text)
# Remove metadata markers from the text to be processed
metadata_pattern = r"<<METADATA_[^:]+:[^>]*>>"
text = re.sub(metadata_pattern, "", text)
# Use pre-compiled pattern for better performance
text = _METADATA_TAG_PATTERN.sub("", text)
# --- Chapter splitting logic ---
chapter_pattern = r"<<CHAPTER_MARKER:(.*?)>>"
chapter_splits = list(re.finditer(chapter_pattern, text))
# Use pre-compiled pattern for better performance
chapter_splits = list(_CHAPTER_MARKER_SEARCH_PATTERN.finditer(text))
chapters = []
if chapter_splits:
# prepend Introduction for content before first marker
abogen/utils.py
+19 -11
View File
@@ -10,6 +10,13 @@ from threading import Thread
warnings.filterwarnings("ignore")
# Pre-compile frequently used regex patterns for better performance
_WHITESPACE_PATTERN = re.compile(r"[^\S\n]+")
_MULTIPLE_NEWLINES_PATTERN = re.compile(r"\n{3,}")
_SINGLE_NEWLINE_PATTERN = re.compile(r"(?<!\n)\n(?!\n)")
_CHAPTER_MARKER_PATTERN = re.compile(r"<<CHAPTER_MARKER:.*?>>")
_METADATA_PATTERN = re.compile(r"<<METADATA_[^:]+:[^>]*>>")
def detect_encoding(file_path):
import chardet
@@ -128,13 +135,16 @@ def clean_text(text, *args, **kwargs):
cfg = load_config()
replace_single_newlines = cfg.get("replace_single_newlines", False)
# Collapse all whitespace (excluding newlines) into single spaces per line and trim edges
lines = [re.sub(r"[^\S\n]+", " ", line).strip() for line in text.splitlines()]
# Use pre-compiled pattern for better performance
lines = [_WHITESPACE_PATTERN.sub(" ", line).strip() for line in text.splitlines()]
text = "\n".join(lines)
# Standardize paragraph breaks (multiple newlines become exactly two) and trim overall whitespace
text = re.sub(r"\n{3,}", "\n\n", text).strip()
# Use pre-compiled pattern for better performance
text = _MULTIPLE_NEWLINES_PATTERN.sub("\n\n", text).strip()
# Optionally replace single newlines with spaces, but preserve double newlines
if replace_single_newlines:
text = re.sub(r"(?<!\n)\n(?!\n)", " ", text)
# Use pre-compiled pattern for better performance
text = _SINGLE_NEWLINE_PATTERN.sub(" ", text)
return text
@@ -243,14 +253,12 @@ def save_config(config):
def calculate_text_length(text):
# Ignore chapter markers
text = re.sub(r"<<CHAPTER_MARKER:.*?>>", "", text)
# Ignore metadata patterns
text = re.sub(r"<<METADATA_[^:]+:[^>]*>>", "", text)
# Ignore newlines
text = text.replace("\n", "")
# Ignore leading/trailing spaces
text = text.strip()
# Use pre-compiled patterns for better performance
# Ignore chapter markers and metadata patterns in a single pass
text = _CHAPTER_MARKER_PATTERN.sub("", text)
text = _METADATA_PATTERN.sub("", text)
# Ignore newlines and leading/trailing spaces
text = text.replace("\n", "").strip()
# Calculate character count
char_count = len(text)
return char_count