Dymas/abogen
1
0
Fork 0
mirror of https://github.com/denizsafak/abogen.git synced 2026-07-18 13:40:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Optimize regex patterns by pre-compiling frequently used patterns

Browse Source 
Co-authored-by: denizsafak <39929354+denizsafak@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2025-11-20 23:11:06 +00:00
co-authored by denizsafak
parent cd2e5c9150
commit 7f75aa8209
3 changed files with 107 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
abogen/book_handler.py
+30 -22
View File
@@ -44,6 +44,16 @@ logging.basicConfig(
level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
) )
# Pre-compile frequently used regex patterns for better performance
_BRACKETED_NUMBERS_PATTERN = re.compile(r"\[\s*\d+\s*\]")
_STANDALONE_PAGE_NUMBERS_PATTERN = re.compile(r"^\s*\d+\s*$", re.MULTILINE)
_PAGE_NUMBERS_AT_END_PATTERN = re.compile(r"\s+\d+\s*$", re.MULTILINE)
_PAGE_NUMBERS_WITH_DASH_PATTERN = re.compile(r"\s+[-–—]\s*\d+\s*[-–—]?\s*$", re.MULTILINE)
_HTML_TAG_PATTERN = re.compile(r"<[^>]+>")
_LEADING_DASH_PATTERN = re.compile(r"^\s*[-–—]\s*")
_LEADING_SIMPLE_DASH_PATTERN = re.compile(r"^\s*-\s*")
class HandlerDialog(QDialog): class HandlerDialog(QDialog):
# Class variables to remember checkbox states between dialog instances # Class variables to remember checkbox states between dialog instances
@@ -428,19 +438,12 @@ class HandlerDialog(QDialog):
"""Pre-process all page contents from PDF document""" """Pre-process all page contents from PDF document"""
for page_num in range(len(self.pdf_doc)): for page_num in range(len(self.pdf_doc)):
text = clean_text(self.pdf_doc[page_num].get_text()) text = clean_text(self.pdf_doc[page_num].get_text())
# Remove bracketed numbers (citations, footnotes) # Remove bracketed numbers, page numbers, etc. using pre-compiled patterns
text = re.sub(r"\[\s*\d+\s*\]", "", text) # Combine all regex operations for better performance
text = _BRACKETED_NUMBERS_PATTERN.sub("", text)
# Remove standalone page numbers (numbers alone on a line) text = _STANDALONE_PAGE_NUMBERS_PATTERN.sub("", text)
text = re.sub(r"^\s*\d+\s*$", "", text, flags=re.MULTILINE) text = _PAGE_NUMBERS_AT_END_PATTERN.sub("", text)
text = _PAGE_NUMBERS_WITH_DASH_PATTERN.sub("", text)
# Remove page numbers at the end of paragraphs
# This pattern looks for digits surrounded by whitespace at the end of paragraphs
text = re.sub(r"\s+\d+\s*$", "", text, flags=re.MULTILINE)
# Also remove page numbers followed by a hyphen or dash at paragraph end
# (common in headers/footers like "- 42 -")
text = re.sub(r"\s+[-–—]\s*\d+\s*[-–—]?\s*$", "", text, flags=re.MULTILINE)
page_id = f"page_{page_num + 1}" page_id = f"page_{page_num + 1}"
self.content_texts[page_id] = text self.content_texts[page_id] = text
@@ -569,7 +572,7 @@ class HandlerDialog(QDialog):
text = clean_text(soup.get_text()).strip() text = clean_text(soup.get_text()).strip()
if text: if text:
self.content_texts[doc_href] = text self.content_texts[doc_href] = text
self.content_lengths[doc_href] = len(text) self.content_lengths[doc_href] = calculate_text_length(text)
title = None title = None
if soup.title and soup.title.string: if soup.title and soup.title.string:
@@ -892,7 +895,7 @@ class HandlerDialog(QDialog):
text = clean_text(slice_soup.get_text()).strip() text = clean_text(slice_soup.get_text()).strip()
if text: if text:
self.content_texts[current_src] = text self.content_texts[current_src] = text
self.content_lengths[current_src] = len(text) self.content_lengths[current_src] = calculate_text_length(text)
else: else:
self.content_texts[current_src] = "" self.content_texts[current_src] = ""
self.content_lengths[current_src] = 0 self.content_lengths[current_src] = 0
@@ -2015,7 +2018,8 @@ class HandlerDialog(QDialog):
html_content += "<hr/>" html_content += "<hr/>"
if self.book_metadata["description"]: if self.book_metadata["description"]:
desc = re.sub(r"<[^>]+>", "", self.book_metadata["description"]) # Use pre-compiled pattern for better performance
desc = _HTML_TAG_PATTERN.sub("", self.book_metadata["description"])
html_content += f"<h3>Description:</h3><p>{desc}</p>" html_content += f"<h3>Description:</h3><p>{desc}</p>"
if self.file_type == "pdf": if self.file_type == "pdf":
@@ -2296,8 +2300,8 @@ class HandlerDialog(QDialog):
text = self.content_texts.get(identifier) text = self.content_texts.get(identifier)
if text and text.strip(): if text and text.strip():
title = item.text(0) title = item.text(0)
# Remove leading dashes from title # Remove leading dashes from title using pre-compiled pattern
title = re.sub(r"^\s*[-–—]\s*", "", title).strip() title = _LEADING_DASH_PATTERN.sub("", title).strip()
marker = f"<<CHAPTER_MARKER:{title}>>" marker = f"<<CHAPTER_MARKER:{title}>>"
chapter_texts.append(marker + "\n" + text) chapter_texts.append(marker + "\n" + text)
@@ -2331,7 +2335,8 @@ class HandlerDialog(QDialog):
text = self.content_texts.get(identifier) text = self.content_texts.get(identifier)
if text and text.strip(): if text and text.strip():
title = item.text(0) title = item.text(0)
title = re.sub(r"^\s*[-–—]\s*", "", title).strip() # Use pre-compiled pattern for better performance
title = _LEADING_DASH_PATTERN.sub("", title).strip()
marker = f"<<CHAPTER_MARKER:{title}>>" marker = f"<<CHAPTER_MARKER:{title}>>"
chapter_texts.append(marker + "\n" + text) chapter_texts.append(marker + "\n" + text)
@@ -2401,12 +2406,14 @@ class HandlerDialog(QDialog):
combined_text += "\n\n" + child_text combined_text += "\n\n" + child_text
included_text_ids.add(child_id) included_text_ids.add(child_id)
if combined_text.strip(): if combined_text.strip():
title = re.sub(r"^\s*-\s*", "", parent_title).strip() # Use pre-compiled pattern for better performance
title = _LEADING_SIMPLE_DASH_PATTERN.sub("", parent_title).strip()
marker = f"<<CHAPTER_MARKER:{title}>>" marker = f"<<CHAPTER_MARKER:{title}>>"
section_titles.append((title, marker + "\n" + combined_text)) section_titles.append((title, marker + "\n" + combined_text))
included_text_ids.add(parent_id) included_text_ids.add(parent_id)
elif not parent_checked and checked_children: elif not parent_checked and checked_children:
title = re.sub(r"^\s*-\s*", "", parent_title).strip() # Use pre-compiled pattern for better performance
title = _LEADING_SIMPLE_DASH_PATTERN.sub("", parent_title).strip()
marker = f"<<CHAPTER_MARKER:{title}>>" marker = f"<<CHAPTER_MARKER:{title}>>"
for idx, (child, child_id) in enumerate(checked_children): for idx, (child, child_id) in enumerate(checked_children):
text = self.content_texts.get(child_id, "") text = self.content_texts.get(child_id, "")
@@ -2426,7 +2433,8 @@ class HandlerDialog(QDialog):
text = self.content_texts.get(identifier, "") text = self.content_texts.get(identifier, "")
if text: if text:
title = item.text(0) title = item.text(0)
title = re.sub(r"^\s*-\s*", "", title).strip() # Use pre-compiled pattern for better performance
title = _LEADING_SIMPLE_DASH_PATTERN.sub("", title).strip()
marker = f"<<CHAPTER_MARKER:{title}>>" marker = f"<<CHAPTER_MARKER:{title}>>"
section_titles.append((title, marker + "\n" + text)) section_titles.append((title, marker + "\n" + text))
included_text_ids.add(identifier) included_text_ids.add(identifier)
abogen/conversion.py
+58 -33
View File
@@ -3,7 +3,7 @@ import re
import time import time
import hashlib # For generating unique cache filenames import hashlib # For generating unique cache filenames
from platformdirs import user_desktop_dir from platformdirs import user_desktop_dir
from PyQt6.QtCore import QThread, pyqtSignal, Qt, QTimer from PyQt6.QtCore import QThread, pyqtSignal, Qt, QTimer, QEventLoop
from PyQt6.QtWidgets import QCheckBox, QVBoxLayout, QDialog, QLabel, QDialogButtonBox from PyQt6.QtWidgets import QCheckBox, QVBoxLayout, QDialog, QLabel, QDialogButtonBox
import soundfile as sf import soundfile as sf
from abogen.utils import ( from abogen.utils import (
@@ -28,13 +28,34 @@ import threading # for efficient waiting
import subprocess import subprocess
import platform import platform
# Pre-compile frequently used regex patterns for better performance
_METADATA_TAG_PATTERN = re.compile(r"<<METADATA_[^:]+:[^>]*>>")
_CHAPTER_MARKER_PATTERN = re.compile(r"<<CHAPTER_MARKER:[^>]*>>")
_HTML_TAG_PATTERN = re.compile(r"<[^>]+>")
_VOICE_TAG_PATTERN = re.compile(r"{[^}]+}")
_ASS_STYLING_PATTERN = re.compile(r"\{[^}]+\}")
_ASS_NEWLINE_N_PATTERN = re.compile(r"\\N")
_ASS_NEWLINE_n_PATTERN = re.compile(r"\\n")
_BRACKETED_NUMBERS_PATTERN = re.compile(r"\[\s*\d+\s*\]")
_CHAPTER_MARKER_SEARCH_PATTERN = re.compile(r"<<CHAPTER_MARKER:(.*?)>>")
_WEBVTT_HEADER_PATTERN = re.compile(r"^WEBVTT.*?\n", re.MULTILINE)
_VTT_STYLE_PATTERN = re.compile(r"STYLE\s*\n.*?(?=\n\n|$)", re.DOTALL)
_VTT_NOTE_PATTERN = re.compile(r"NOTE\s*\n.*?(?=\n\n|$)", re.DOTALL)
_DOUBLE_NEWLINE_SPLIT_PATTERN = re.compile(r"\n\s*\n")
_SRT_TIMESTAMP_PATTERN = re.compile(r"(\d{2}:\d{2}:\d{2},\d{3})\s*-->\s*(\d{2}:\d{2}:\d{2},\d{3})")
_VTT_TIMESTAMP_PATTERN = re.compile(r"([\d:.]+)\s*-->\s*([\d:.]+)")
_TIMESTAMP_ONLY_PATTERN = re.compile(r"^(\d{1,2}:\d{2}:\d{2}(?:,\d{1,3})?)$")
_WINDOWS_ILLEGAL_CHARS_PATTERN = re.compile(r'[<>:"/\\|?*]')
_CONTROL_CHARS_PATTERN = re.compile(r"[\x00-\x1f]")
_MACOS_ILLEGAL_CHARS_PATTERN = re.compile(r"[:]")
_LINUX_ILLEGAL_CHARS_PATTERN = re.compile(r"[/\x00]")
def clean_subtitle_text(text): def clean_subtitle_text(text):
"""Remove chapter markers and metadata tags from subtitle text.""" """Remove chapter markers and metadata tags from subtitle text."""
# Remove metadata tags # Use pre-compiled patterns for better performance
text = re.sub(r"<<METADATA_[^:]+:[^>]*>>", "", text) text = _METADATA_TAG_PATTERN.sub("", text)
# Remove chapter markers text = _CHAPTER_MARKER_PATTERN.sub("", text)
text = re.sub(r"<<CHAPTER_MARKER:[^>]*>>", "", text)
return text.strip() return text.strip()
@@ -87,8 +108,8 @@ def parse_srt_file(file_path):
start_sec = time_to_seconds(start_str) start_sec = time_to_seconds(start_str)
end_sec = time_to_seconds(end_str) end_sec = time_to_seconds(end_str)
# Clean text of any styling tags # Clean text of any styling tags using pre-compiled pattern
text = re.sub(r"<[^>]+>", "", text) text = _HTML_TAG_PATTERN.sub("", text)
# Remove chapter markers and metadata tags # Remove chapter markers and metadata tags
text = clean_subtitle_text(text) text = clean_subtitle_text(text)
@@ -114,13 +135,13 @@ def parse_vtt_file(file_path):
with open(file_path, "r", encoding=encoding, errors="replace") as f: with open(file_path, "r", encoding=encoding, errors="replace") as f:
content = f.read() content = f.read()
# Remove WEBVTT header and any style/note blocks # Remove WEBVTT header and any style/note blocks using pre-compiled patterns
content = re.sub(r"^WEBVTT.*?\n", "", content, flags=re.MULTILINE) content = _WEBVTT_HEADER_PATTERN.sub("", content)
content = re.sub(r"STYLE\s*\n.*?(?=\n\n|$)", "", content, flags=re.DOTALL) content = _VTT_STYLE_PATTERN.sub("", content)
content = re.sub(r"NOTE\s*\n.*?(?=\n\n|$)", "", content, flags=re.DOTALL) content = _VTT_NOTE_PATTERN.sub("", content)
# Split by double newlines to get individual subtitle blocks # Split by double newlines to get individual subtitle blocks using pre-compiled pattern
blocks = re.split(r"\n\s*\n", content.strip()) blocks = _DOUBLE_NEWLINE_SPLIT_PATTERN.split(content.strip())
subtitles = [] subtitles = []
for block in blocks: for block in blocks:
@@ -147,7 +168,8 @@ def parse_vtt_file(file_path):
try: try:
# VTT format: 00:00:00.000 --> 00:00:05.000 or 00:00.000 --> 00:05.000 # VTT format: 00:00:00.000 --> 00:00:05.000 or 00:00.000 --> 00:05.000
match = re.match(r"([\d:.]+)\s*-->\s*([\d:.]+)", timestamp_line) # Use pre-compiled pattern
match = _VTT_TIMESTAMP_PATTERN.match(timestamp_line)
if not match: if not match:
continue continue
@@ -171,9 +193,9 @@ def parse_vtt_file(file_path):
start_sec = time_to_seconds(start_str) start_sec = time_to_seconds(start_str)
end_sec = time_to_seconds(end_str) end_sec = time_to_seconds(end_str)
# Clean text of any styling tags and cue settings # Clean text of any styling tags and cue settings using pre-compiled patterns
text = re.sub(r"<[^>]+>", "", text) text = _HTML_TAG_PATTERN.sub("", text)
text = re.sub(r"{[^}]+}", "", text) # Remove voice tags text = _VOICE_TAG_PATTERN.sub("", text) # Remove voice tags
# Remove chapter markers and metadata tags # Remove chapter markers and metadata tags
text = clean_subtitle_text(text) text = clean_subtitle_text(text)
@@ -196,8 +218,8 @@ def detect_timestamps_in_text(file_path):
# Count lines that are ONLY timestamps (no other text) # Count lines that are ONLY timestamps (no other text)
# Supports HH:MM:SS or HH:MM:SS,ms format # Supports HH:MM:SS or HH:MM:SS,ms format
timestamp_pattern = r"^(\d{1,2}:\d{2}:\d{2}(?:,\d{1,3})?)$" # Use pre-compiled pattern for better performance
timestamp_lines = sum(1 for line in lines if re.match(timestamp_pattern, line)) timestamp_lines = sum(1 for line in lines if _TIMESTAMP_ONLY_PATTERN.match(line))
# Must have at least 2 timestamp-only lines and they should be >5% of total lines # Must have at least 2 timestamp-only lines and they should be >5% of total lines
return timestamp_lines >= 2 and (timestamp_lines / max(len(lines), 1)) > 0.05 return timestamp_lines >= 2 and (timestamp_lines / max(len(lines), 1)) > 0.05
@@ -348,10 +370,10 @@ def parse_ass_file(file_path):
start_sec = ass_time_to_seconds(start_str) start_sec = ass_time_to_seconds(start_str)
end_sec = ass_time_to_seconds(end_str) end_sec = ass_time_to_seconds(end_str)
# Clean text of ASS styling tags # Clean text of ASS styling tags using pre-compiled patterns
text = re.sub(r"\{[^}]+\}", "", text) # Remove {tags} text = _ASS_STYLING_PATTERN.sub("", text) # Remove {tags}
text = re.sub(r"\\N", "\n", text) # Convert \N to newline text = _ASS_NEWLINE_N_PATTERN.sub("\n", text) # Convert \N to newline
text = re.sub(r"\\n", "\n", text) # Convert \n to newline text = _ASS_NEWLINE_n_PATTERN.sub("\n", text) # Convert \n to newline
# Remove chapter markers and metadata tags # Remove chapter markers and metadata tags
text = clean_subtitle_text(text) text = clean_subtitle_text(text)
@@ -384,9 +406,10 @@ def sanitize_name_for_os(name, is_folder=True):
if system == "Windows": if system == "Windows":
# Windows illegal characters: < > : " / \ | ? * # Windows illegal characters: < > : " / \ | ? *
# Also can't end with space or dot # Also can't end with space or dot
sanitized = re.sub(r'[<>:"/\\|?*]', "_", name) # Use pre-compiled pattern for better performance
sanitized = _WINDOWS_ILLEGAL_CHARS_PATTERN.sub("_", name)
# Remove control characters (0-31) # Remove control characters (0-31)
sanitized = re.sub(r"[\x00-\x1f]", "_", sanitized) sanitized = _CONTROL_CHARS_PATTERN.sub("_", sanitized)
# Remove trailing spaces and dots # Remove trailing spaces and dots
sanitized = sanitized.rstrip(". ") sanitized = sanitized.rstrip(". ")
# Windows reserved names (CON, PRN, AUX, NUL, COM1-9, LPT1-9) # Windows reserved names (CON, PRN, AUX, NUL, COM1-9, LPT1-9)
@@ -400,18 +423,20 @@ def sanitize_name_for_os(name, is_folder=True):
elif system == "Darwin": # macOS elif system == "Darwin": # macOS
# macOS illegal characters: : (colon is converted to / by the system) # macOS illegal characters: : (colon is converted to / by the system)
# Also can't start with dot (hidden file) for folders typically # Also can't start with dot (hidden file) for folders typically
sanitized = re.sub(r"[:]", "_", name) # Use pre-compiled pattern for better performance
sanitized = _MACOS_ILLEGAL_CHARS_PATTERN.sub("_", name)
# Remove control characters # Remove control characters
sanitized = re.sub(r"[\x00-\x1f]", "_", sanitized) sanitized = _CONTROL_CHARS_PATTERN.sub("_", sanitized)
# Avoid leading dot for folders (creates hidden folders) # Avoid leading dot for folders (creates hidden folders)
if is_folder and sanitized.startswith("."): if is_folder and sanitized.startswith("."):
sanitized = "_" + sanitized[1:] sanitized = "_" + sanitized[1:]
else: # Linux and others else: # Linux and others
# Linux illegal characters: / and null character # Linux illegal characters: / and null character
# Though / is illegal, most other chars are technically allowed # Though / is illegal, most other chars are technically allowed
sanitized = re.sub(r"[/\x00]", "_", name) # Use pre-compiled pattern for better performance
sanitized = _LINUX_ILLEGAL_CHARS_PATTERN.sub("_", name)
# Remove other control characters for safety # Remove other control characters for safety
sanitized = re.sub(r"[\x01-\x1f]", "_", sanitized) sanitized = _CONTROL_CHARS_PATTERN.sub("_", sanitized)
# Avoid leading dot for folders (creates hidden folders) # Avoid leading dot for folders (creates hidden folders)
if is_folder and sanitized.startswith("."): if is_folder and sanitized.startswith("."):
sanitized = "_" + sanitized[1:] sanitized = "_" + sanitized[1:]
@@ -875,12 +900,12 @@ class ConversionThread(QThread):
text = clean_text(text) text = clean_text(text)
# Remove metadata markers from the text to be processed # Remove metadata markers from the text to be processed
metadata_pattern = r"<<METADATA_[^:]+:[^>]*>>" # Use pre-compiled pattern for better performance
text = re.sub(metadata_pattern, "", text) text = _METADATA_TAG_PATTERN.sub("", text)
# --- Chapter splitting logic --- # --- Chapter splitting logic ---
chapter_pattern = r"<<CHAPTER_MARKER:(.*?)>>" # Use pre-compiled pattern for better performance
chapter_splits = list(re.finditer(chapter_pattern, text)) chapter_splits = list(_CHAPTER_MARKER_SEARCH_PATTERN.finditer(text))
chapters = [] chapters = []
if chapter_splits: if chapter_splits:
# prepend Introduction for content before first marker # prepend Introduction for content before first marker
abogen/utils.py
+19 -11
View File
@@ -10,6 +10,13 @@ from threading import Thread
warnings.filterwarnings("ignore") warnings.filterwarnings("ignore")
# Pre-compile frequently used regex patterns for better performance
_WHITESPACE_PATTERN = re.compile(r"[^\S\n]+")
_MULTIPLE_NEWLINES_PATTERN = re.compile(r"\n{3,}")
_SINGLE_NEWLINE_PATTERN = re.compile(r"(?<!\n)\n(?!\n)")
_CHAPTER_MARKER_PATTERN = re.compile(r"<<CHAPTER_MARKER:.*?>>")
_METADATA_PATTERN = re.compile(r"<<METADATA_[^:]+:[^>]*>>")
def detect_encoding(file_path): def detect_encoding(file_path):
import chardet import chardet
@@ -128,13 +135,16 @@ def clean_text(text, *args, **kwargs):
cfg = load_config() cfg = load_config()
replace_single_newlines = cfg.get("replace_single_newlines", False) replace_single_newlines = cfg.get("replace_single_newlines", False)
# Collapse all whitespace (excluding newlines) into single spaces per line and trim edges # Collapse all whitespace (excluding newlines) into single spaces per line and trim edges
lines = [re.sub(r"[^\S\n]+", " ", line).strip() for line in text.splitlines()] # Use pre-compiled pattern for better performance
lines = [_WHITESPACE_PATTERN.sub(" ", line).strip() for line in text.splitlines()]
text = "\n".join(lines) text = "\n".join(lines)
# Standardize paragraph breaks (multiple newlines become exactly two) and trim overall whitespace # Standardize paragraph breaks (multiple newlines become exactly two) and trim overall whitespace
text = re.sub(r"\n{3,}", "\n\n", text).strip() # Use pre-compiled pattern for better performance
text = _MULTIPLE_NEWLINES_PATTERN.sub("\n\n", text).strip()
# Optionally replace single newlines with spaces, but preserve double newlines # Optionally replace single newlines with spaces, but preserve double newlines
if replace_single_newlines: if replace_single_newlines:
text = re.sub(r"(?<!\n)\n(?!\n)", " ", text) # Use pre-compiled pattern for better performance
text = _SINGLE_NEWLINE_PATTERN.sub(" ", text)
return text return text
@@ -243,14 +253,12 @@ def save_config(config):
def calculate_text_length(text): def calculate_text_length(text):
# Ignore chapter markers # Use pre-compiled patterns for better performance
text = re.sub(r"<<CHAPTER_MARKER:.*?>>", "", text) # Ignore chapter markers and metadata patterns in a single pass
# Ignore metadata patterns text = _CHAPTER_MARKER_PATTERN.sub("", text)
text = re.sub(r"<<METADATA_[^:]+:[^>]*>>", "", text) text = _METADATA_PATTERN.sub("", text)
# Ignore newlines # Ignore newlines and leading/trailing spaces
text = text.replace("\n", "") text = text.replace("\n", "").strip()
# Ignore leading/trailing spaces
text = text.strip()
# Calculate character count # Calculate character count
char_count = len(text) char_count = len(text)
return char_count return char_count