mirror of
https://github.com/denizsafak/abogen.git
synced 2026-07-18 05:40:26 +02:00
Merge pull request #103 from denizsafak/copilot/improve-slow-code-efficiency
Optimize regex compilation and eliminate busy-wait loops
This commit is contained in:
Deniz Şafak
@@ -1,3 +1,6 @@
|
||||
# 1.2.4 (Pre-release)
|
||||
- Optimized regex compilation and eliminated busy-wait loops.
|
||||
|
||||
# 1.2.3
|
||||
- Same as 1.2.2, re-released to fix an issue with subtitle timing when using timestamp-based text files.
|
||||
|
||||
|
||||
+30
-22
@@ -44,6 +44,16 @@ logging.basicConfig(
|
||||
level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
|
||||
)
|
||||
|
||||
# Pre-compile frequently used regex patterns for better performance
|
||||
_BRACKETED_NUMBERS_PATTERN = re.compile(r"\[\s*\d+\s*\]")
|
||||
_STANDALONE_PAGE_NUMBERS_PATTERN = re.compile(r"^\s*\d+\s*$", re.MULTILINE)
|
||||
_PAGE_NUMBERS_AT_END_PATTERN = re.compile(r"\s+\d+\s*$", re.MULTILINE)
|
||||
_PAGE_NUMBERS_WITH_DASH_PATTERN = re.compile(r"\s+[-–—]\s*\d+\s*[-–—]?\s*$", re.MULTILINE)
|
||||
_HTML_TAG_PATTERN = re.compile(r"<[^>]+>")
|
||||
_LEADING_DASH_PATTERN = re.compile(r"^\s*[-–—]\s*")
|
||||
_LEADING_SIMPLE_DASH_PATTERN = re.compile(r"^\s*-\s*")
|
||||
|
||||
|
||||
|
||||
class HandlerDialog(QDialog):
|
||||
# Class variables to remember checkbox states between dialog instances
|
||||
@@ -428,19 +438,12 @@ class HandlerDialog(QDialog):
|
||||
"""Pre-process all page contents from PDF document"""
|
||||
for page_num in range(len(self.pdf_doc)):
|
||||
text = clean_text(self.pdf_doc[page_num].get_text())
|
||||
# Remove bracketed numbers (citations, footnotes)
|
||||
text = re.sub(r"\[\s*\d+\s*\]", "", text)
|
||||
|
||||
# Remove standalone page numbers (numbers alone on a line)
|
||||
text = re.sub(r"^\s*\d+\s*$", "", text, flags=re.MULTILINE)
|
||||
|
||||
# Remove page numbers at the end of paragraphs
|
||||
# This pattern looks for digits surrounded by whitespace at the end of paragraphs
|
||||
text = re.sub(r"\s+\d+\s*$", "", text, flags=re.MULTILINE)
|
||||
|
||||
# Also remove page numbers followed by a hyphen or dash at paragraph end
|
||||
# (common in headers/footers like "- 42 -")
|
||||
text = re.sub(r"\s+[-–—]\s*\d+\s*[-–—]?\s*$", "", text, flags=re.MULTILINE)
|
||||
# Remove bracketed numbers, page numbers, etc. using pre-compiled patterns
|
||||
# Combine all regex operations for better performance
|
||||
text = _BRACKETED_NUMBERS_PATTERN.sub("", text)
|
||||
text = _STANDALONE_PAGE_NUMBERS_PATTERN.sub("", text)
|
||||
text = _PAGE_NUMBERS_AT_END_PATTERN.sub("", text)
|
||||
text = _PAGE_NUMBERS_WITH_DASH_PATTERN.sub("", text)
|
||||
|
||||
page_id = f"page_{page_num + 1}"
|
||||
self.content_texts[page_id] = text
|
||||
@@ -569,7 +572,7 @@ class HandlerDialog(QDialog):
|
||||
text = clean_text(soup.get_text()).strip()
|
||||
if text:
|
||||
self.content_texts[doc_href] = text
|
||||
self.content_lengths[doc_href] = len(text)
|
||||
self.content_lengths[doc_href] = calculate_text_length(text)
|
||||
|
||||
title = None
|
||||
if soup.title and soup.title.string:
|
||||
@@ -892,7 +895,7 @@ class HandlerDialog(QDialog):
|
||||
text = clean_text(slice_soup.get_text()).strip()
|
||||
if text:
|
||||
self.content_texts[current_src] = text
|
||||
self.content_lengths[current_src] = len(text)
|
||||
self.content_lengths[current_src] = calculate_text_length(text)
|
||||
else:
|
||||
self.content_texts[current_src] = ""
|
||||
self.content_lengths[current_src] = 0
|
||||
@@ -2015,7 +2018,8 @@ class HandlerDialog(QDialog):
|
||||
html_content += "<hr/>"
|
||||
|
||||
if self.book_metadata["description"]:
|
||||
desc = re.sub(r"<[^>]+>", "", self.book_metadata["description"])
|
||||
# Use pre-compiled pattern for better performance
|
||||
desc = _HTML_TAG_PATTERN.sub("", self.book_metadata["description"])
|
||||
html_content += f"<h3>Description:</h3><p>{desc}</p>"
|
||||
|
||||
if self.file_type == "pdf":
|
||||
@@ -2296,8 +2300,8 @@ class HandlerDialog(QDialog):
|
||||
text = self.content_texts.get(identifier)
|
||||
if text and text.strip():
|
||||
title = item.text(0)
|
||||
# Remove leading dashes from title
|
||||
title = re.sub(r"^\s*[-–—]\s*", "", title).strip()
|
||||
# Remove leading dashes from title using pre-compiled pattern
|
||||
title = _LEADING_DASH_PATTERN.sub("", title).strip()
|
||||
marker = f"<<CHAPTER_MARKER:{title}>>"
|
||||
chapter_texts.append(marker + "\n" + text)
|
||||
|
||||
@@ -2331,7 +2335,8 @@ class HandlerDialog(QDialog):
|
||||
text = self.content_texts.get(identifier)
|
||||
if text and text.strip():
|
||||
title = item.text(0)
|
||||
title = re.sub(r"^\s*[-–—]\s*", "", title).strip()
|
||||
# Use pre-compiled pattern for better performance
|
||||
title = _LEADING_DASH_PATTERN.sub("", title).strip()
|
||||
marker = f"<<CHAPTER_MARKER:{title}>>"
|
||||
chapter_texts.append(marker + "\n" + text)
|
||||
|
||||
@@ -2401,12 +2406,14 @@ class HandlerDialog(QDialog):
|
||||
combined_text += "\n\n" + child_text
|
||||
included_text_ids.add(child_id)
|
||||
if combined_text.strip():
|
||||
title = re.sub(r"^\s*-\s*", "", parent_title).strip()
|
||||
# Use pre-compiled pattern for better performance
|
||||
title = _LEADING_SIMPLE_DASH_PATTERN.sub("", parent_title).strip()
|
||||
marker = f"<<CHAPTER_MARKER:{title}>>"
|
||||
section_titles.append((title, marker + "\n" + combined_text))
|
||||
included_text_ids.add(parent_id)
|
||||
elif not parent_checked and checked_children:
|
||||
title = re.sub(r"^\s*-\s*", "", parent_title).strip()
|
||||
# Use pre-compiled pattern for better performance
|
||||
title = _LEADING_SIMPLE_DASH_PATTERN.sub("", parent_title).strip()
|
||||
marker = f"<<CHAPTER_MARKER:{title}>>"
|
||||
for idx, (child, child_id) in enumerate(checked_children):
|
||||
text = self.content_texts.get(child_id, "")
|
||||
@@ -2426,7 +2433,8 @@ class HandlerDialog(QDialog):
|
||||
text = self.content_texts.get(identifier, "")
|
||||
if text:
|
||||
title = item.text(0)
|
||||
title = re.sub(r"^\s*-\s*", "", title).strip()
|
||||
# Use pre-compiled pattern for better performance
|
||||
title = _LEADING_SIMPLE_DASH_PATTERN.sub("", title).strip()
|
||||
marker = f"<<CHAPTER_MARKER:{title}>>"
|
||||
section_titles.append((title, marker + "\n" + text))
|
||||
included_text_ids.add(identifier)
|
||||
|
||||
+81
-42
@@ -28,13 +28,38 @@ import threading # for efficient waiting
|
||||
import subprocess
|
||||
import platform
|
||||
|
||||
# Configuration constants
|
||||
_USER_RESPONSE_TIMEOUT = 0.1 # Timeout in seconds for checking user response/cancellation
|
||||
|
||||
# Pre-compile frequently used regex patterns for better performance
|
||||
_METADATA_TAG_PATTERN = re.compile(r"<<METADATA_[^:]+:[^>]*>>")
|
||||
_CHAPTER_MARKER_PATTERN = re.compile(r"<<CHAPTER_MARKER:[^>]*>>")
|
||||
_HTML_TAG_PATTERN = re.compile(r"<[^>]+>")
|
||||
_VOICE_TAG_PATTERN = re.compile(r"{[^}]+}")
|
||||
_ASS_STYLING_PATTERN = re.compile(r"\{[^}]+\}")
|
||||
_ASS_NEWLINE_N_PATTERN = re.compile(r"\\N")
|
||||
_ASS_NEWLINE_n_PATTERN = re.compile(r"\\n")
|
||||
_BRACKETED_NUMBERS_PATTERN = re.compile(r"\[\s*\d+\s*\]")
|
||||
_CHAPTER_MARKER_SEARCH_PATTERN = re.compile(r"<<CHAPTER_MARKER:(.*?)>>")
|
||||
_WEBVTT_HEADER_PATTERN = re.compile(r"^WEBVTT.*?\n", re.MULTILINE)
|
||||
_VTT_STYLE_PATTERN = re.compile(r"STYLE\s*\n.*?(?=\n\n|$)", re.DOTALL)
|
||||
_VTT_NOTE_PATTERN = re.compile(r"NOTE\s*\n.*?(?=\n\n|$)", re.DOTALL)
|
||||
_DOUBLE_NEWLINE_SPLIT_PATTERN = re.compile(r"\n\s*\n")
|
||||
_SRT_TIMESTAMP_PATTERN = re.compile(r"(\d{2}:\d{2}:\d{2},\d{3})\s*-->\s*(\d{2}:\d{2}:\d{2},\d{3})")
|
||||
_VTT_TIMESTAMP_PATTERN = re.compile(r"([\d:.]+)\s*-->\s*([\d:.]+)")
|
||||
_TIMESTAMP_ONLY_PATTERN = re.compile(r"^(\d{1,2}:\d{2}:\d{2}(?:,\d{1,3})?)$")
|
||||
_WINDOWS_ILLEGAL_CHARS_PATTERN = re.compile(r'[<>:"/\\|?*]')
|
||||
_CONTROL_CHARS_PATTERN = re.compile(r"[\x00-\x1f]")
|
||||
_LINUX_CONTROL_CHARS_PATTERN = re.compile(r"[\x01-\x1f]") # Linux: exclude \x00 for separate handling
|
||||
_MACOS_ILLEGAL_CHARS_PATTERN = re.compile(r"[:]")
|
||||
_LINUX_ILLEGAL_CHARS_PATTERN = re.compile(r"[/\x00]")
|
||||
|
||||
|
||||
def clean_subtitle_text(text):
|
||||
"""Remove chapter markers and metadata tags from subtitle text."""
|
||||
# Remove metadata tags
|
||||
text = re.sub(r"<<METADATA_[^:]+:[^>]*>>", "", text)
|
||||
# Remove chapter markers
|
||||
text = re.sub(r"<<CHAPTER_MARKER:[^>]*>>", "", text)
|
||||
# Use pre-compiled patterns for better performance
|
||||
text = _METADATA_TAG_PATTERN.sub("", text)
|
||||
text = _CHAPTER_MARKER_PATTERN.sub("", text)
|
||||
return text.strip()
|
||||
|
||||
|
||||
@@ -87,8 +112,8 @@ def parse_srt_file(file_path):
|
||||
start_sec = time_to_seconds(start_str)
|
||||
end_sec = time_to_seconds(end_str)
|
||||
|
||||
# Clean text of any styling tags
|
||||
text = re.sub(r"<[^>]+>", "", text)
|
||||
# Clean text of any styling tags using pre-compiled pattern
|
||||
text = _HTML_TAG_PATTERN.sub("", text)
|
||||
# Remove chapter markers and metadata tags
|
||||
text = clean_subtitle_text(text)
|
||||
|
||||
@@ -114,13 +139,13 @@ def parse_vtt_file(file_path):
|
||||
with open(file_path, "r", encoding=encoding, errors="replace") as f:
|
||||
content = f.read()
|
||||
|
||||
# Remove WEBVTT header and any style/note blocks
|
||||
content = re.sub(r"^WEBVTT.*?\n", "", content, flags=re.MULTILINE)
|
||||
content = re.sub(r"STYLE\s*\n.*?(?=\n\n|$)", "", content, flags=re.DOTALL)
|
||||
content = re.sub(r"NOTE\s*\n.*?(?=\n\n|$)", "", content, flags=re.DOTALL)
|
||||
# Remove WEBVTT header and any style/note blocks using pre-compiled patterns
|
||||
content = _WEBVTT_HEADER_PATTERN.sub("", content)
|
||||
content = _VTT_STYLE_PATTERN.sub("", content)
|
||||
content = _VTT_NOTE_PATTERN.sub("", content)
|
||||
|
||||
# Split by double newlines to get individual subtitle blocks
|
||||
blocks = re.split(r"\n\s*\n", content.strip())
|
||||
# Split by double newlines to get individual subtitle blocks using pre-compiled pattern
|
||||
blocks = _DOUBLE_NEWLINE_SPLIT_PATTERN.split(content.strip())
|
||||
|
||||
subtitles = []
|
||||
for block in blocks:
|
||||
@@ -147,7 +172,8 @@ def parse_vtt_file(file_path):
|
||||
|
||||
try:
|
||||
# VTT format: 00:00:00.000 --> 00:00:05.000 or 00:00.000 --> 00:05.000
|
||||
match = re.match(r"([\d:.]+)\s*-->\s*([\d:.]+)", timestamp_line)
|
||||
# Use pre-compiled pattern
|
||||
match = _VTT_TIMESTAMP_PATTERN.match(timestamp_line)
|
||||
if not match:
|
||||
continue
|
||||
|
||||
@@ -171,9 +197,9 @@ def parse_vtt_file(file_path):
|
||||
start_sec = time_to_seconds(start_str)
|
||||
end_sec = time_to_seconds(end_str)
|
||||
|
||||
# Clean text of any styling tags and cue settings
|
||||
text = re.sub(r"<[^>]+>", "", text)
|
||||
text = re.sub(r"{[^}]+}", "", text) # Remove voice tags
|
||||
# Clean text of any styling tags and cue settings using pre-compiled patterns
|
||||
text = _HTML_TAG_PATTERN.sub("", text)
|
||||
text = _VOICE_TAG_PATTERN.sub("", text) # Remove voice tags
|
||||
# Remove chapter markers and metadata tags
|
||||
text = clean_subtitle_text(text)
|
||||
|
||||
@@ -196,8 +222,8 @@ def detect_timestamps_in_text(file_path):
|
||||
|
||||
# Count lines that are ONLY timestamps (no other text)
|
||||
# Supports HH:MM:SS or HH:MM:SS,ms format
|
||||
timestamp_pattern = r"^(\d{1,2}:\d{2}:\d{2}(?:,\d{1,3})?)$"
|
||||
timestamp_lines = sum(1 for line in lines if re.match(timestamp_pattern, line))
|
||||
# Use pre-compiled pattern for better performance
|
||||
timestamp_lines = sum(1 for line in lines if _TIMESTAMP_ONLY_PATTERN.match(line))
|
||||
|
||||
# Must have at least 2 timestamp-only lines and they should be >5% of total lines
|
||||
return timestamp_lines >= 2 and (timestamp_lines / max(len(lines), 1)) > 0.05
|
||||
@@ -348,10 +374,10 @@ def parse_ass_file(file_path):
|
||||
start_sec = ass_time_to_seconds(start_str)
|
||||
end_sec = ass_time_to_seconds(end_str)
|
||||
|
||||
# Clean text of ASS styling tags
|
||||
text = re.sub(r"\{[^}]+\}", "", text) # Remove {tags}
|
||||
text = re.sub(r"\\N", "\n", text) # Convert \N to newline
|
||||
text = re.sub(r"\\n", "\n", text) # Convert \n to newline
|
||||
# Clean text of ASS styling tags using pre-compiled patterns
|
||||
text = _ASS_STYLING_PATTERN.sub("", text) # Remove {tags}
|
||||
text = _ASS_NEWLINE_N_PATTERN.sub("\n", text) # Convert \N to newline
|
||||
text = _ASS_NEWLINE_n_PATTERN.sub("\n", text) # Convert \n to newline
|
||||
# Remove chapter markers and metadata tags
|
||||
text = clean_subtitle_text(text)
|
||||
|
||||
@@ -384,9 +410,10 @@ def sanitize_name_for_os(name, is_folder=True):
|
||||
if system == "Windows":
|
||||
# Windows illegal characters: < > : " / \ | ? *
|
||||
# Also can't end with space or dot
|
||||
sanitized = re.sub(r'[<>:"/\\|?*]', "_", name)
|
||||
# Use pre-compiled pattern for better performance
|
||||
sanitized = _WINDOWS_ILLEGAL_CHARS_PATTERN.sub("_", name)
|
||||
# Remove control characters (0-31)
|
||||
sanitized = re.sub(r"[\x00-\x1f]", "_", sanitized)
|
||||
sanitized = _CONTROL_CHARS_PATTERN.sub("_", sanitized)
|
||||
# Remove trailing spaces and dots
|
||||
sanitized = sanitized.rstrip(". ")
|
||||
# Windows reserved names (CON, PRN, AUX, NUL, COM1-9, LPT1-9)
|
||||
@@ -400,18 +427,20 @@ def sanitize_name_for_os(name, is_folder=True):
|
||||
elif system == "Darwin": # macOS
|
||||
# macOS illegal characters: : (colon is converted to / by the system)
|
||||
# Also can't start with dot (hidden file) for folders typically
|
||||
sanitized = re.sub(r"[:]", "_", name)
|
||||
# Use pre-compiled pattern for better performance
|
||||
sanitized = _MACOS_ILLEGAL_CHARS_PATTERN.sub("_", name)
|
||||
# Remove control characters
|
||||
sanitized = re.sub(r"[\x00-\x1f]", "_", sanitized)
|
||||
sanitized = _CONTROL_CHARS_PATTERN.sub("_", sanitized)
|
||||
# Avoid leading dot for folders (creates hidden folders)
|
||||
if is_folder and sanitized.startswith("."):
|
||||
sanitized = "_" + sanitized[1:]
|
||||
else: # Linux and others
|
||||
# Linux illegal characters: / and null character
|
||||
# Though / is illegal, most other chars are technically allowed
|
||||
sanitized = re.sub(r"[/\x00]", "_", name)
|
||||
# Remove other control characters for safety
|
||||
sanitized = re.sub(r"[\x01-\x1f]", "_", sanitized)
|
||||
# Use pre-compiled pattern for better performance
|
||||
sanitized = _LINUX_ILLEGAL_CHARS_PATTERN.sub("_", name)
|
||||
# Remove other control characters for safety (excluding \x00 which is already handled)
|
||||
sanitized = _LINUX_CONTROL_CHARS_PATTERN.sub("_", sanitized)
|
||||
# Avoid leading dot for folders (creates hidden folders)
|
||||
if is_folder and sanitized.startswith("."):
|
||||
sanitized = "_" + sanitized[1:]
|
||||
@@ -628,6 +657,7 @@ class ConversionThread(QThread):
|
||||
): # Add use_gpu parameter
|
||||
super().__init__()
|
||||
self._chapter_options_event = threading.Event()
|
||||
self._timestamp_response_event = threading.Event()
|
||||
self.np = np_module
|
||||
self.KPipeline = kpipeline_class
|
||||
self.file_name = file_name
|
||||
@@ -847,15 +877,19 @@ class ConversionThread(QThread):
|
||||
self.log_updated.emit("\nDetected timestamps in text file")
|
||||
# Signal to ask user (-1 indicates timestamp detection)
|
||||
self.chapters_detected.emit(-1)
|
||||
# Wait for user response
|
||||
while not hasattr(self, "_timestamp_response"):
|
||||
# Wait for user response using event with timeout for responsive cancellation
|
||||
while not self._timestamp_response_event.wait(timeout=_USER_RESPONSE_TIMEOUT):
|
||||
if self.cancel_requested:
|
||||
self.conversion_finished.emit("Cancelled", None)
|
||||
return
|
||||
# Check cancellation one more time after event is set
|
||||
if self.cancel_requested:
|
||||
self.conversion_finished.emit("Cancelled", None)
|
||||
return
|
||||
time.sleep(0.1)
|
||||
if not self._timestamp_response:
|
||||
is_timestamp_text = False
|
||||
delattr(self, "_timestamp_response")
|
||||
self._timestamp_response_event.clear()
|
||||
|
||||
# Process subtitle files separately
|
||||
if is_subtitle_file or is_timestamp_text:
|
||||
@@ -875,12 +909,12 @@ class ConversionThread(QThread):
|
||||
text = clean_text(text)
|
||||
|
||||
# Remove metadata markers from the text to be processed
|
||||
metadata_pattern = r"<<METADATA_[^:]+:[^>]*>>"
|
||||
text = re.sub(metadata_pattern, "", text)
|
||||
# Use pre-compiled pattern for better performance
|
||||
text = _METADATA_TAG_PATTERN.sub("", text)
|
||||
|
||||
# --- Chapter splitting logic ---
|
||||
chapter_pattern = r"<<CHAPTER_MARKER:(.*?)>>"
|
||||
chapter_splits = list(re.finditer(chapter_pattern, text))
|
||||
# Use pre-compiled pattern for better performance
|
||||
chapter_splits = list(_CHAPTER_MARKER_SEARCH_PATTERN.finditer(text))
|
||||
chapters = []
|
||||
if chapter_splits:
|
||||
# prepend Introduction for content before first marker
|
||||
@@ -984,10 +1018,12 @@ class ConversionThread(QThread):
|
||||
parent_dir, f"{sanitized_base_name}{suffix}_chapters"
|
||||
)
|
||||
# Only check for files with allowed extensions (extension without dot, case-insensitive)
|
||||
# Use generator expression to avoid processing all files upfront
|
||||
file_parts = (os.path.splitext(fname) for fname in os.listdir(parent_dir))
|
||||
clash = any(
|
||||
os.path.splitext(fname)[0] == f"{sanitized_base_name}{suffix}"
|
||||
and os.path.splitext(fname)[1][1:].lower() in allowed_exts
|
||||
for fname in os.listdir(parent_dir)
|
||||
name == f"{sanitized_base_name}{suffix}"
|
||||
and ext[1:].lower() in allowed_exts
|
||||
for name, ext in file_parts
|
||||
)
|
||||
if not os.path.exists(chapters_out_dir_candidate) and not clash:
|
||||
break
|
||||
@@ -1727,10 +1763,12 @@ class ConversionThread(QThread):
|
||||
allowed_exts = set(SUPPORTED_SOUND_FORMATS + SUPPORTED_SUBTITLE_FORMATS)
|
||||
while True:
|
||||
suffix = f"_{counter}" if counter > 1 else ""
|
||||
# Use generator expression to avoid processing all files upfront
|
||||
file_parts = (os.path.splitext(f) for f in os.listdir(parent_dir))
|
||||
if not any(
|
||||
os.path.splitext(f)[0] == f"{sanitized_base_name}{suffix}"
|
||||
and os.path.splitext(f)[1][1:].lower() in allowed_exts
|
||||
for f in os.listdir(parent_dir)
|
||||
name == f"{sanitized_base_name}{suffix}"
|
||||
and ext[1:].lower() in allowed_exts
|
||||
for name, ext in file_parts
|
||||
):
|
||||
break
|
||||
counter += 1
|
||||
@@ -2158,6 +2196,7 @@ class ConversionThread(QThread):
|
||||
def set_timestamp_response(self, treat_as_subtitle):
|
||||
"""Set whether to treat timestamp text file as subtitle."""
|
||||
self._timestamp_response = treat_as_subtitle
|
||||
self._timestamp_response_event.set()
|
||||
|
||||
def _extract_and_add_metadata_tags_to_ffmpeg_cmd(self):
|
||||
"""Extract metadata tags from text content and add them to ffmpeg command"""
|
||||
|
||||
+19
-11
@@ -10,6 +10,13 @@ from threading import Thread
|
||||
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
# Pre-compile frequently used regex patterns for better performance
|
||||
_WHITESPACE_PATTERN = re.compile(r"[^\S\n]+")
|
||||
_MULTIPLE_NEWLINES_PATTERN = re.compile(r"\n{3,}")
|
||||
_SINGLE_NEWLINE_PATTERN = re.compile(r"(?<!\n)\n(?!\n)")
|
||||
_CHAPTER_MARKER_PATTERN = re.compile(r"<<CHAPTER_MARKER:.*?>>")
|
||||
_METADATA_PATTERN = re.compile(r"<<METADATA_[^:]+:[^>]*>>")
|
||||
|
||||
|
||||
def detect_encoding(file_path):
|
||||
import chardet
|
||||
@@ -128,13 +135,16 @@ def clean_text(text, *args, **kwargs):
|
||||
cfg = load_config()
|
||||
replace_single_newlines = cfg.get("replace_single_newlines", False)
|
||||
# Collapse all whitespace (excluding newlines) into single spaces per line and trim edges
|
||||
lines = [re.sub(r"[^\S\n]+", " ", line).strip() for line in text.splitlines()]
|
||||
# Use pre-compiled pattern for better performance
|
||||
lines = [_WHITESPACE_PATTERN.sub(" ", line).strip() for line in text.splitlines()]
|
||||
text = "\n".join(lines)
|
||||
# Standardize paragraph breaks (multiple newlines become exactly two) and trim overall whitespace
|
||||
text = re.sub(r"\n{3,}", "\n\n", text).strip()
|
||||
# Use pre-compiled pattern for better performance
|
||||
text = _MULTIPLE_NEWLINES_PATTERN.sub("\n\n", text).strip()
|
||||
# Optionally replace single newlines with spaces, but preserve double newlines
|
||||
if replace_single_newlines:
|
||||
text = re.sub(r"(?<!\n)\n(?!\n)", " ", text)
|
||||
# Use pre-compiled pattern for better performance
|
||||
text = _SINGLE_NEWLINE_PATTERN.sub(" ", text)
|
||||
return text
|
||||
|
||||
|
||||
@@ -243,14 +253,12 @@ def save_config(config):
|
||||
|
||||
|
||||
def calculate_text_length(text):
|
||||
# Ignore chapter markers
|
||||
text = re.sub(r"<<CHAPTER_MARKER:.*?>>", "", text)
|
||||
# Ignore metadata patterns
|
||||
text = re.sub(r"<<METADATA_[^:]+:[^>]*>>", "", text)
|
||||
# Ignore newlines
|
||||
text = text.replace("\n", "")
|
||||
# Ignore leading/trailing spaces
|
||||
text = text.strip()
|
||||
# Use pre-compiled patterns for better performance
|
||||
# Ignore chapter markers and metadata patterns in a single pass
|
||||
text = _CHAPTER_MARKER_PATTERN.sub("", text)
|
||||
text = _METADATA_PATTERN.sub("", text)
|
||||
# Ignore newlines and leading/trailing spaces
|
||||
text = text.replace("\n", "").strip()
|
||||
# Calculate character count
|
||||
char_count = len(text)
|
||||
return char_count
|
||||
|
||||
Reference in New Issue
Block a user