From 1fdd1c85402f623c5db206024601b5327e6ed6af Mon Sep 17 00:00:00 2001 From: Mohan Krishnan Date: Mon, 22 Dec 2025 13:32:40 +0800 Subject: [PATCH] Extracts book_parser.py from book_handler.py Extracts out a BaseBookParser, PDFParser and MarkdownParser class from book_handler logic The basic contract is that the parser classes populate 4 attributes that are used by the book_handler logic - content_texts - content_lengths - book_metatdata - processed_nav_structures (used by the method get_chapters on the BaseBookParser class) Adds tests to validate the changes as well in tests/ Run tests from the abogen dir as follows ```bash python -m unittest discover ``` --- abogen/book_handler.py | 1115 ++--------------- abogen/book_parser.py | 917 ++++++++++++++ tests/__init__.py | 0 tests/test_book_handler_regression.py | 84 ++ tests/test_book_parser.py | 210 ++++ tests/test_epub_content_slicing.py | 199 +++ tests/test_epub_heuristic_nav.py | 117 ++ tests/test_epub_html_nav_parsing.py | 184 +++ .../test_epub_missing_file_error_handling.py | 100 ++ tests/test_epub_ncx_parsing.py | 140 +++ tests/test_epub_standard_nav.py | 130 ++ 11 files changed, 2194 insertions(+), 1002 deletions(-) create mode 100644 abogen/book_parser.py create mode 100644 tests/__init__.py create mode 100644 tests/test_book_handler_regression.py create mode 100644 tests/test_book_parser.py create mode 100644 tests/test_epub_content_slicing.py create mode 100644 tests/test_epub_heuristic_nav.py create mode 100644 tests/test_epub_html_nav_parsing.py create mode 100644 tests/test_epub_missing_file_error_handling.py create mode 100644 tests/test_epub_ncx_parsing.py create mode 100644 tests/test_epub_standard_nav.py diff --git a/abogen/book_handler.py b/abogen/book_handler.py index b8073c1..4fa63ba 100644 --- a/abogen/book_handler.py +++ b/abogen/book_handler.py @@ -1,8 +1,5 @@ import re -import ebooklib import base64 -import fitz # PyMuPDF for PDF support -from ebooklib import epub from bs4 import BeautifulSoup, NavigableString from PyQt6.QtGui import QMovie from PyQt6.QtWidgets import ( @@ -31,6 +28,7 @@ from abogen.utils import ( detect_encoding, get_resource_path, ) +from abogen.book_parser import get_book_parser from abogen.subtitle_utils import ( clean_text, @@ -38,9 +36,8 @@ from abogen.subtitle_utils import ( ) import os -import logging # Add logging +import logging import urllib.parse -import markdown import textwrap # Setup logging @@ -48,13 +45,6 @@ logging.basicConfig( level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" ) -# Pre-compile frequently used regex patterns for better performance -_BRACKETED_NUMBERS_PATTERN = re.compile(r"\[\s*\d+\s*\]") -_STANDALONE_PAGE_NUMBERS_PATTERN = re.compile(r"^\s*\d+\s*$", re.MULTILINE) -_PAGE_NUMBERS_AT_END_PATTERN = re.compile(r"\s+\d+\s*$", re.MULTILINE) -_PAGE_NUMBERS_WITH_DASH_PATTERN = re.compile( - r"\s+[-–—]\s*\d+\s*[-–—]?\s*$", re.MULTILINE -) _HTML_TAG_PATTERN = re.compile(r"<[^>]+>") _LEADING_DASH_PATTERN = re.compile(r"^\s*[-–—]\s*") _LEADING_SIMPLE_DASH_PATTERN = re.compile(r"^\s*-\s*") @@ -106,23 +96,22 @@ class HandlerDialog(QDialog): # Normalize path book_path = os.path.normpath(os.path.abspath(book_path)) - - # Determine file type if not explicitly provided - if file_type: - self.file_type = file_type - elif book_path.lower().endswith(".pdf"): - self.file_type = "pdf" - elif book_path.lower().endswith((".md", ".markdown")): - self.file_type = "markdown" - else: - self.file_type = "epub" self.book_path = book_path + # Initialize Parser + try: + # Factory handles file type detection if file_type is None + self.parser = get_book_parser(book_path, file_type=file_type) + # Parser loads automatically in init now + except Exception as e: + logging.error(f"Failed to initialize parser for {book_path}: {e}") + raise + # Extract book name from file path book_name = os.path.splitext(os.path.basename(book_path))[0] # Set window title based on file type and book name - item_type = "Chapters" if self.file_type in ["epub", "markdown"] else "Pages" + item_type = "Chapters" if self.parser.file_type in ["epub", "markdown"] else "Pages" self.setWindowTitle(f"Select {item_type} - {book_name}") self.resize(1200, 900) self._block_signals = False # Flag to prevent recursive signals @@ -138,57 +127,8 @@ class HandlerDialog(QDialog): self.merge_chapters_at_end = HandlerDialog._merge_chapters_at_end self.save_as_project = HandlerDialog._save_as_project - # Load the book based on file type - try: - if self.file_type == "epub": - self.book = epub.read_epub(book_path) - elif self.file_type == "markdown": - self.book = None # Markdown doesn't use ebooklib - else: - self.book = None - except KeyError as e: - logging.error( - f"EPUB file is missing a referenced file: {e}. Skipping missing file." - ) - # Try to patch ebooklib to skip missing files (monkey-patch read_file) - import types - - orig_read_file = None - try: - from ebooklib import epub as _epub_module - - reader_class = _epub_module.EpubReader - orig_read_file = reader_class.read_file - - def safe_read_file(self, name): - try: - return orig_read_file(self, name) - except KeyError: - logging.warning( - f"Missing file in EPUB: {name}. Returning empty bytes." - ) - return b"" - - reader_class.read_file = safe_read_file - self.book = epub.read_epub(book_path) - reader_class.read_file = orig_read_file # Restore - except Exception as patch_e: - logging.error(f"Failed to patch ebooklib for missing files: {patch_e}") - raise e - self.pdf_doc = fitz.open(book_path) if self.file_type == "pdf" else None - self.markdown_text = None - if self.file_type == "markdown": - try: - encoding = detect_encoding(book_path) - with open(book_path, "r", encoding=encoding, errors="replace") as f: - self.markdown_text = f.read() - except Exception as e: - logging.error(f"Error reading markdown file: {e}") - self.markdown_text = "" - self.markdown_toc = [] # For storing parsed markdown TOC - - # Extract book metadata - self.book_metadata = self._extract_book_metadata() + # Initialize metadata dict; will be populated in _preprocess_content by the background loader + self.book_metadata = {} # Initialize UI elements that are used in other methods self.save_chapters_checkbox = None @@ -207,6 +147,8 @@ class HandlerDialog(QDialog): # For storing content and lengths (will be filled by background loader) self.content_texts = {} self.content_lengths = {} + # Also maintain refs for structure + self.processed_nav_structure = [] # Add a placeholder "Information" item so the tree isn't empty immediately info_item = QTreeWidgetItem(self.treeWidget, ["Information"]) @@ -396,787 +338,51 @@ class HandlerDialog(QDialog): cfg = load_config() replace_single_newlines = cfg.get("replace_single_newlines", True) - cache_key = (self.book_path, mod_time, self.file_type, replace_single_newlines) + cache_key = (self.book_path, mod_time, self.parser.file_type, replace_single_newlines) # Check if content is already cached if cache_key in HandlerDialog._content_cache: cached_data = HandlerDialog._content_cache[cache_key] self.content_texts = cached_data["content_texts"] self.content_lengths = cached_data["content_lengths"] - if "doc_content" in cached_data: - self.doc_content = cached_data["doc_content"] - if "markdown_toc" in cached_data: - self.markdown_toc = cached_data["markdown_toc"] + if "processed_nav_structure" in cached_data: + self.processed_nav_structure = cached_data["processed_nav_structure"] + if "book_metadata" in cached_data: + self.book_metadata = cached_data["book_metadata"] + + # Apply to parser so it stays in sync if used elsewhere + self.parser.content_texts = self.content_texts + self.parser.content_lengths = self.content_lengths + self.parser.processed_nav_structure = self.processed_nav_structure + self.parser.book_metadata = self.book_metadata + logging.info(f"Using cached content for {os.path.basename(self.book_path)}") return # Process content if not cached - if self.file_type == "epub": - try: - self._process_epub_content_nav() # Use the new navigation-based method - except Exception as e: - logging.error( - f"Error processing EPUB with navigation: {e}. Falling back to TOC/spine.", - exc_info=True, - ) - # Fallback to a simpler spine-based processing if nav fails - self._process_epub_content_spine_fallback() - elif self.file_type == "markdown": - self._preprocess_markdown_content() - else: - self._preprocess_pdf_content() + try: + self.parser.process_content(replace_single_newlines=replace_single_newlines) + self.content_texts = self.parser.content_texts + self.content_lengths = self.parser.content_lengths + self.processed_nav_structure = self.parser.processed_nav_structure + self.book_metadata = self.parser.get_metadata() + except Exception as e: + logging.error(f"Error processing content: {e}", exc_info=True) + # Handle empty/failure case + self.content_texts = {} + self.content_lengths = {} # Cache the processed content cache_data = { "content_texts": self.content_texts, "content_lengths": self.content_lengths, + "processed_nav_structure": self.processed_nav_structure, + "book_metadata": self.book_metadata, } - if hasattr(self, "doc_content"): - cache_data["doc_content"] = self.doc_content - if hasattr(self, "markdown_toc"): - cache_data["markdown_toc"] = self.markdown_toc HandlerDialog._content_cache[cache_key] = cache_data logging.info(f"Cached content for {os.path.basename(self.book_path)}") - def _preprocess_pdf_content(self): - """Pre-process all page contents from PDF document""" - for page_num in range(len(self.pdf_doc)): - text = clean_text(self.pdf_doc[page_num].get_text()) - # Remove bracketed numbers, page numbers, etc. using pre-compiled patterns - # Combine all regex operations for better performance - text = _BRACKETED_NUMBERS_PATTERN.sub("", text) - text = _STANDALONE_PAGE_NUMBERS_PATTERN.sub("", text) - text = _PAGE_NUMBERS_AT_END_PATTERN.sub("", text) - text = _PAGE_NUMBERS_WITH_DASH_PATTERN.sub("", text) - - page_id = f"page_{page_num + 1}" - self.content_texts[page_id] = text - self.content_lengths[page_id] = calculate_text_length(text) - - def _preprocess_markdown_content(self): - if not self.markdown_text: - return - - # Generate TOC from the original (dedented) markdown BEFORE cleaning, - # so header ids/anchors are preserved for reliable position detection. - original_text = textwrap.dedent(self.markdown_text) - md = markdown.Markdown(extensions=["toc", "fenced_code"]) - html = md.convert(original_text) - self.markdown_toc = md.toc_tokens - - # Use cleaned text for stored content/length calculations - cleaned_full_text = clean_text(original_text) - - soup = BeautifulSoup(html, "html.parser") - self.content_texts = {} - self.content_lengths = {} - - if not self.markdown_toc: - chapter_id = "markdown_content" - self.content_texts[chapter_id] = cleaned_full_text - self.content_lengths[chapter_id] = calculate_text_length(cleaned_full_text) - return - - all_headers = [] - - def flatten_toc(toc_list): - for header in toc_list: - all_headers.append(header) - if header.get("children"): - flatten_toc(header["children"]) - - flatten_toc(self.markdown_toc) - - header_positions = [] - for header in all_headers: - header_id = header["id"] - id_pattern = f'id="{header_id}"' - pos = html.find(id_pattern) - if pos != -1: - tag_start = html.rfind("<", 0, pos) - header_positions.append( - {"id": header_id, "start": tag_start, "name": header["name"]} - ) - header_positions.sort(key=lambda x: x["start"]) - - for i, header_pos in enumerate(header_positions): - header_id = header_pos["id"] - header_name = header_pos["name"] - content_start = header_pos["start"] - content_end = ( - header_positions[i + 1]["start"] - if i + 1 < len(header_positions) - else len(html) - ) - section_html = html[content_start:content_end] - section_soup = BeautifulSoup(section_html, "html.parser") - header_tag = section_soup.find(attrs={"id": header_id}) - if header_tag: - header_tag.decompose() - # Clean section text for storage/lengths - section_text = clean_text(section_soup.get_text()).strip() - chapter_id = header_id - if section_text: - full_content = f"{header_name}\n\n{section_text}" - self.content_texts[chapter_id] = full_content - self.content_lengths[chapter_id] = calculate_text_length(full_content) - else: - self.content_texts[chapter_id] = header_name - self.content_lengths[chapter_id] = calculate_text_length(header_name) - - def _process_epub_content_spine_fallback(self): - """Fallback EPUB processing based purely on spine order.""" - logging.info("Using spine fallback for EPUB processing.") - self.doc_content = {} - spine_docs = [] - for spine_item_tuple in self.book.spine: - item_id = spine_item_tuple[0] - item = self.book.get_item_with_id(item_id) - if item: - spine_docs.append(item.get_name()) - else: - logging.warning(f"Spine item with id '{item_id}' not found.") - - # Cache content - for item in self.book.get_items_of_type(ebooklib.ITEM_DOCUMENT): - href = item.get_name() - if href in spine_docs: - try: - html_content = item.get_content().decode("utf-8", errors="ignore") - self.doc_content[href] = html_content - except Exception as e: - logging.error(f"Error decoding content for {href}: {e}") - self.doc_content[href] = "" - - # Create a simple TOC based on spine order - synthetic_toc = [] - self.content_texts = {} - self.content_lengths = {} - for i, doc_href in enumerate(spine_docs): - html_content = self.doc_content.get(doc_href, "") - if html_content: - soup = BeautifulSoup(html_content, "html.parser") - - # Handle ordered lists by prepending numbers to list items - for ol in soup.find_all("ol"): - # Get start attribute or default to 1 - start = int(ol.get("start", 1)) - for i, li in enumerate(ol.find_all("li", recursive=False)): - # Insert the number at the beginning of the list item - number_text = f"{start + i}) " - if li.string: - li.string.replace_with(number_text + li.string) - else: - li.insert(0, NavigableString(number_text)) - - # Remove sup and sub tags - for tag in soup.find_all(["sup", "sub"]): - tag.decompose() - - text = clean_text(soup.get_text()).strip() - if text: - self.content_texts[doc_href] = text - self.content_lengths[doc_href] = calculate_text_length(text) - - title = None - if soup.title and soup.title.string: - title = soup.title.string.strip() - elif (h1 := soup.find("h1")) and h1.get_text(strip=True): - title = h1.get_text(strip=True) - - if not title: - title = f"Untitled Chapter {i + 1}" - synthetic_toc.append( - (epub.Link(doc_href, title, doc_href), []) - ) # Wrap in tuple and empty list for compatibility - - # Replace book.toc with the synthetic one if it was empty or fallback was triggered - if not self.book.toc or not hasattr( - self, "processed_nav_structure" - ): # Check if nav processing failed - self.book.toc = synthetic_toc - logging.info(f"Generated synthetic TOC with {len(synthetic_toc)} entries.") - - def _process_epub_content_nav(self): - """ - Process EPUB content using ITEM_NAVIGATION (NAV HTML) or ITEM_NCX. - Globally orders navigation entries and slices content between them. - """ - logging.info( - "Attempting to process EPUB using navigation document (NAV/NCX)..." - ) - nav_item = None - nav_type = None - - # 1. Check ITEM_NAVIGATION for actual NAV HTML (.xhtml/.html) - nav_items = list(self.book.get_items_of_type(ebooklib.ITEM_NAVIGATION)) - if nav_items: - # Prefer files explicitly named 'nav.xhtml' or similar - preferred_nav = next( - ( - item - for item in nav_items - if "nav" in item.get_name().lower() - and item.get_name().lower().endswith((".xhtml", ".html")) - ), - None, - ) - if preferred_nav: - nav_item = preferred_nav - nav_type = "html" - logging.info(f"Found preferred NAV HTML item: {nav_item.get_name()}") - else: - # Check if any ITEM_NAVIGATION is actually HTML - html_nav = next( - ( - item - for item in nav_items - if item.get_name().lower().endswith((".xhtml", ".html")) - ), - None, - ) - if html_nav: - nav_item = html_nav - nav_type = "html" - logging.info( - f"Found NAV HTML item in ITEM_NAVIGATION: {html_nav.get_name()}" - ) - - # 2. If no NAV HTML found via ITEM_NAVIGATION, check if ITEM_NAVIGATION points to NCX - if not nav_item and nav_items: - ncx_in_nav = next( - ( - item - for item in nav_items - if item.get_name().lower().endswith(".ncx") - ), - None, - ) - if ncx_in_nav: - nav_item = ncx_in_nav - nav_type = "ncx" - logging.info( - f"Found NCX item via ITEM_NAVIGATION: {ncx_in_nav.get_name()}" - ) - - # 3. If still no nav_item, check for NCX or fallback to NAV HTML in all ITEM_DOCUMENTs - ncx_constant = getattr(epub, "ITEM_NCX", None) - if not nav_item and ncx_constant is not None: - ncx_items = list(self.book.get_items_of_type(ncx_constant)) - if ncx_items: - nav_item = ncx_items[0] - nav_type = "ncx" - logging.info(f"Found NCX item via ITEM_NCX: {nav_item.get_name()}") - # Fallback: search all ITEM_DOCUMENTs for a NAV HTML with