v1.0.2

CHANGELOG.md

@@ -1,5 +1,6 @@
-- Added `abogen-cli` command for better troubleshooting and error handling.
+- Enhanced EPUB handling by treating all items in chapter list (including anchors) as chapters, improving navigation and organization for poorly structured books, mentioned by @Darthagnon in #4
-- Switched from setuptools to hatchling for packaging.
+- Fixed the issue with some chapters in EPUB files had missing content.
-- Added classifiers to the package metadata.
+- Fixed the issue with some EPUB files only having one chapter caused the program to ignore the entire book.
-- Fixed "No module named 'docopt'" and "setuptools.build_meta" import errors while using .bat installer in Windows, mentioned by @nigelp in https://github.com/denizsafak/abogen/issues/2 
+- Fixed "utf-8' codec can't decode byte" error, mentioned by @nigelp in #3
+- Added "Replace single newlines with spaces" option in the menu. This can be useful for texts that have imaginary line breaks.
 - Improvements in code and documentation.

README.md

@@ -1,7 +1,8 @@
 # abogen <img width="40px" title="abogen icon" src="https://raw.githubusercontent.com/denizsafak/abogen/refs/heads/main/abogen/assets/icon.ico" align="right" style="padding-left: 10px; padding-top:5px;">
 
 [![Build Status](https://github.com/denizsafak/abogen/actions/workflows/test_pip.yml/badge.svg)](https://github.com/denizsafak/abogen/actions)
-![Python Versions](https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12-blue?logo=python)
+[![GitHub Release](https://img.shields.io/github/v/release/denizsafak/abogen)](https://github.com/denizsafak/abogen/releases/latest)
+![abogen PyPi Python Versions](https://img.shields.io/pypi/pyversions/abogen)
 [![Operating Systems](https://img.shields.io/badge/os-windows%20%7C%20linux%20%7C%20macos%20-blue)](https://github.com/denizsafak/abogen/releases/latest)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 [![License: MIT](https://img.shields.io/badge/License-MIT-maroon.svg)](https://opensource.org/licenses/MIT)
@@ -55,7 +56,7 @@ sudo dnf install espeak-ng
 # Install abogen
 pip install abogen
 ```
-> If you get "No matching distribution found" error, try installing it on supported Python (3.10 to 3.12). You can use [pyenv](https://github.com/pyenv/pyenv) to manage multiple Python versions easily in Linux.
+> If you get "No matching distribution found" error, try installing it on supported Python (3.10 to 3.12). You can use [pyenv](https://github.com/pyenv/pyenv) to manage multiple Python versions easily in Linux. Watch this [video](https://www.youtube.com/watch?v=MVyb-nI4KyI) by NetworkChuck for a quick guide.
 
 Then simply run by typing:
 
@@ -87,6 +88,7 @@ Here’s Abogen in action: in this demo, it processes ∼3,000 characters of tex
 - **Save location**: `Save next to input file`, `Save to desktop`, or `Choose output folder`
 - **Chapter Control**: Select specific `chapters` from ePUBs or `chapters + pages` from PDFs.
 - **Options**:
+    - **Replace single newlines with spaces**: Replaces single newlines with spaces in the text. This is useful for texts that have imaginary line breaks.
     - **Configure max words per subtitle**: Automatically configures the maximum number of words per subtitle entry.
     - **Create desktop shortcut**: Creates a shortcut on your desktop for easy access.
     - **Open config.json directory**: Opens the directory where the configuration file is stored.
@@ -116,6 +118,11 @@ keep-open=yes
 --audio-device=openal
 --sub-margin-x=235
 --sub-pos=60
+# --- Audio Quality ---
+audio-spdif=ac3,dts,eac3,truehd,dts-hd
+audio-channels=auto
+audio-samplerate=48000
+volume-max=200
 ```
 
 ## `Similar Projects`

WINDOWS_INSTALL.bat

@@ -46,8 +46,7 @@ if exist "%VERSION_FILE%" (
     set "LOCAL_VERSION_CLEAN=!LOCAL_VERSION:.=!"
 
     :: First verify GitHub is accessible by checking HTTP status code
-    curl -s -o nul -w "%%{http_code}" "%VERSION_URL%" > "%TEMP%\abogen_http_status.txt"
+    for /f %%i in ('curl -s -o nul -w "%%{http_code}" "%VERSION_URL%"') do set abogen_http_status=%%i
-    set /p abogen_http_status=<"%TEMP%\abogen_http_status.txt"
     
     if not "!abogen_http_status!"=="200" (
         echo Failed to access GitHub repository ^(HTTP status: !abogen_http_status!^). Continuing with current version.
@@ -55,14 +54,8 @@ if exist "%VERSION_FILE%" (
     )
     
     :: Get the remote version (only if GitHub is accessible)
-    curl -s -o "%TEMP%\abogen_remote_version.txt" "%VERSION_URL%"
+    for /f "delims=" %%i in ('curl -s "%VERSION_URL%"') do set abogen_remote_version=%%i
-    findstr "^" "%TEMP%\abogen_remote_version.txt" >nul 2>&1
-    if errorlevel 1 (
-        echo Failed to retrieve version information. Continuing with current version.
-        goto continue_with_current_version
-    )
     
-    set /p abogen_remote_version=<"%TEMP%\abogen_remote_version.txt"
     if "!abogen_remote_version!"=="" (
         echo Empty version information received. Continuing with current version.
         goto continue_with_current_version
@@ -97,8 +90,7 @@ if exist "%VERSION_FILE%" (
         echo Downloading the latest update...
         
         :: Test the zip URL before downloading
-        curl -s -o nul -w "%%{http_code}" "%UPDATE_ZIP_URL%" > "%TEMP%\abogen_zip_status.txt"
+        for /f %%i in ('curl -s -o nul -w "%%{http_code}" "%UPDATE_ZIP_URL%"') do set abogen_zip_status=%%i
-        set /p abogen_zip_status=<"%TEMP%\abogen_zip_status.txt"
         
         if not "!abogen_zip_status!"=="302" (
             echo Failed to access update zip file ^(HTTP status: !abogen_zip_status!^). Continuing with current version.
@@ -137,7 +129,7 @@ if exist "%VERSION_FILE%" (
         )
     ) else (
         echo Current version: !LOCAL_VERSION!, Remote version: !abogen_remote_version!
-        echo You are usering the latest version.
+        echo You are using the latest version.
     )
 ) else (
     echo VERSION file not found. Cannot check for updates.

abogen/VERSION

@@ -1 +1 @@
-1.0.0
+1.0.2

abogen/book_handler.py

@@ -19,6 +19,7 @@ from PyQt5.QtWidgets import (
     QPushButton,
     QCheckBox,
     QMenu,
+    QLabel,
 )
 from PyQt5.QtCore import Qt
 from utils import clean_text, calculate_text_length
@@ -128,74 +129,234 @@ class HandlerDialog(QDialog):
     def _preprocess_content(self):
         """Pre-process content from the document"""
         if self.file_type == "epub":
-            self._preprocess_epub_content()
+            # Always process EPUB content using the anchor-based approach
+            self._process_epub_content()
         else:
             self._preprocess_pdf_content()
 
-    def _preprocess_epub_content(self):
+    def _process_epub_content(self, split_anchors=True):
-        """Extract EPUB content with preserved formatting."""
+        """
-        book, spine = self.book, [
+        Process EPUB content by globally ordering TOC entries and slicing content between them.
-            item.get_name()
+        Ensures all content between defined TOC start points is captured.
-            for item in self.book.get_items_of_type(ebooklib.ITEM_DOCUMENT)
+        """
-        ]
+        # split_anchors parameter kept for compatibility but always treated as True
+        book = self.book
+
+        # 1. Cache all document HTML and determine spine order
+        self.doc_content = {}
+        # Correctly get hrefs from spine items
+        spine_docs = []
+        for spine_item_tuple in book.spine:
+            item_id = spine_item_tuple[0]
+            item = book.get_item_with_id(item_id)
+            if item:
+                spine_docs.append(item.get_name())  # Use get_name() for href
+            else:
+                print(f"Warning: Spine item with id '{item_id}' not found in book items.")
+
+        doc_order = {href: i for i, href in enumerate(spine_docs)}
+
+        for item in book.get_items_of_type(ebooklib.ITEM_DOCUMENT):
+            href = item.get_name()
+            if href in doc_order:  # Only process docs in spine
+                try:
+                    html_content = item.get_content().decode('utf-8', errors='ignore')
+                    self.doc_content[href] = html_content
+                except Exception:
+                    self.doc_content[href] = ""  # Handle decoding errors
+
+        # 2. Get all TOC entries with hrefs and determine their positions
+        toc_entries_with_pos = []
+
+        def find_position(doc_href, fragment_id):
+            if doc_href not in self.doc_content:
+                return -1
+            html_content = self.doc_content[doc_href]
+            if not fragment_id:  # No fragment, position is 0
+                return 0
+
+            # Find position of fragment identifier (id= or name=)
+            id_match_str = f'id="{fragment_id}"'
+            name_match_str = f'name="{fragment_id}"'
+
+            id_pos = html_content.find(id_match_str)
+            name_pos = html_content.find(name_match_str)
+
+            pos = -1
+            if id_pos != -1 and name_pos != -1:
+                pos = min(id_pos, name_pos)
+            elif id_pos != -1:
+                pos = id_pos
+            elif name_pos != -1:
+                pos = name_pos
+            else:
+                return -1  # Anchor not found by simple string search
+
+            # Backtrack to the start of the tag '<'
+            tag_start_pos = html_content.rfind('<', 0, pos)
+            return tag_start_pos if tag_start_pos != -1 else 0  # Default to 0 if '<' not found
+
+        def collect_toc_entries(entries):
+            collected = []
+            for entry in entries:
+                href, title = None, "Unknown"
+                children = []
+                entry_obj = None  # Store the original entry object
+
+                if isinstance(entry, ebooklib.epub.Link):
+                    href, title = entry.href, entry.title or entry.href
+                    entry_obj = entry
+                elif isinstance(entry, tuple) and len(entry) >= 1:
+                    section_or_link = entry[0]
+                    entry_obj = section_or_link
+                    if isinstance(section_or_link, ebooklib.epub.Section):
+                        title = section_or_link.title
+                        href = getattr(section_or_link, "href", None)
+                    elif isinstance(section_or_link, ebooklib.epub.Link):
+                        href, title = section_or_link.href, section_or_link.title or section_or_link.href
+
+                    if len(entry) > 1 and isinstance(entry[1], list):
+                        children = entry[1]
 
-        # Get TOC entries with titles
-        toc = []
-        for entry in book.toc:
-            if isinstance(entry, ebooklib.epub.Link):
-                toc.append((entry.href.split("#")[0], entry.title))
-            elif isinstance(entry, tuple) and hasattr(entry[0], "href"):
-                href = getattr(entry[0], "href", None)
                 if href:
-                    toc.append(
+                    base_href, fragment = href.split('#', 1) if '#' in href else (href, None)
-                        (
+                    if base_href in doc_order:  # Only consider entries pointing to spine documents
-                            href.split("#")[0],
+                        position = find_position(base_href, fragment)
-                            entry[0].title if hasattr(entry[0], "title") else None,
+                        if position != -1:  # Only add if position is valid
-                        )
+                            collected.append({
-                    )
+                                "href": href,  # Use the original href from TOC as the key
+                                "title": title,
+                                "doc_href": base_href,
+                                "position": position,
+                                "doc_order": doc_order[base_href]
+                            })
 
-        # Find chapter ranges and process content
+                if children:
-        toc = [(h, t) for h, t in toc if h in spine and t]
+                    collected.extend(collect_toc_entries(children))
-        chapter_indices = [spine.index(h) for h, _ in toc if h in spine] + [len(spine)]
+            return collected
 
-        def process_html(item):
+        all_toc_entries = collect_toc_entries(self.book.toc)
-            if not item:
-                return ""
-            soup = BeautifulSoup(item.get_content(), "html.parser")
-            for br in soup.find_all("br"):
-                br.replace_with("\n")
-            for p in soup.find_all(["p", "div"]):
-                p.append("\n\n")
-            return re.sub(r"\n{3,}", "\n\n", clean_text(soup.get_text())).strip()
 
-        # Process chapters
+        # Handle case where book has no TOC or empty TOC
-        for i in range(len(toc)):
+        if not all_toc_entries:
-            href, title = toc[i]
+            # Create a synthetic TOC entry for the first spine document
-            start, end = chapter_indices[i], chapter_indices[i + 1]
+            if spine_docs:
+                # Process all content as a single chapter
+                all_content_html = ""
+                for doc_href in spine_docs:
+                    all_content_html += self.doc_content.get(doc_href, "")
 
-            texts = [
+                if all_content_html:
-                process_html(book.get_item_with_href(spine[idx]))
+                    soup = BeautifulSoup(all_content_html, 'html.parser')
-                for idx in range(start, end)
+                    text = clean_text(soup.get_text()).strip()
-            ]
-            texts = [t for t in texts if t]
                     
-            full_text = "\n\n".join(texts)
+                    # Use the first spine document as the identifier
-            if title and full_text.lower().startswith(title.lower()):
+                    first_doc = spine_docs[0]
-                full_text = (
+                    self.content_texts[first_doc] = text
-                    full_text[: len(title)].rstrip()
+                    self.content_lengths[first_doc] = len(text)
-                    + "\n\n"
-                    + full_text[len(title) :].lstrip()
-                )
                     
-            self.content_texts[href] = full_text
+                    # Create a synthetic TOC entry for tree building
-            self.content_lengths[href] = calculate_text_length(full_text)
+                    self.book.toc = [(epub.Link(first_doc, "Main Content", first_doc),)]
+            return
 
-        # Process any spine items not in TOC
+        # 3. Sort TOC entries globally
-        for href in spine:
+        all_toc_entries.sort(key=lambda x: (x["doc_order"], x["position"]))
-            if href not in self.content_texts:
+
-                text = process_html(book.get_item_with_href(href))
+        # 4. Slice content between sorted entries
-                self.content_texts[href] = text
+        self.content_texts = {}
-                self.content_lengths[href] = calculate_text_length(text)
+        self.content_lengths = {}
+        num_entries = len(all_toc_entries)
+
+        for i in range(num_entries):
+            current_entry = all_toc_entries[i]
+            current_href = current_entry["href"]
+            current_doc = current_entry["doc_href"]
+            current_pos = current_entry["position"]
+            current_doc_html = self.doc_content.get(current_doc, "")
+
+            start_slice_pos = current_pos
+            slice_html = ""
+
+            # Find the start of the next TOC entry
+            next_entry = all_toc_entries[i + 1] if (i + 1) < num_entries else None
+
+            if next_entry:
+                next_doc = next_entry["doc_href"]
+                next_pos = next_entry["position"]
+
+                if current_doc == next_doc:
+                    # Next entry is in the same document
+                    slice_html = current_doc_html[start_slice_pos:next_pos]
+                else:
+                    # Next entry is in a different document
+                    # Take content from current position to end of current document
+                    slice_html = current_doc_html[start_slice_pos:]
+                    # Include content from intermediate documents in the spine
+                    current_doc_index = current_entry["doc_order"]
+                    next_doc_index = next_entry["doc_order"]
+                    for doc_idx in range(current_doc_index + 1, next_doc_index):
+                        intermediate_doc_href = spine_docs[doc_idx]
+                        slice_html += self.doc_content.get(intermediate_doc_href, "")
+                    # Add content from the beginning of the next document up to the next entry's position
+                    next_doc_html = self.doc_content.get(next_doc, "")
+                    slice_html += next_doc_html[:next_pos]
+            else:
+                # This is the last TOC entry
+                # Take content from current position to end of current document
+                slice_html = current_doc_html[start_slice_pos:]
+                # Include content from all remaining documents in the spine
+                current_doc_index = current_entry["doc_order"]
+                for doc_idx in range(current_doc_index + 1, len(spine_docs)):
+                    intermediate_doc_href = spine_docs[doc_idx]
+                    slice_html += self.doc_content.get(intermediate_doc_href, "")
+
+            # 5. Extract text and store
+            slice_soup = BeautifulSoup(slice_html, 'html.parser')
+            
+            # Remove sup and sub tags from the HTML before extracting text
+            for tag in slice_soup.find_all(['sup', 'sub']):
+                tag.decompose()
+                
+            text = clean_text(slice_soup.get_text()).strip()
+            self.content_texts[current_href] = text  # Store using the original TOC href
+            self.content_lengths[current_href] = len(text)
+
+        # 6. Handle content BEFORE the first TOC entry
+        if all_toc_entries:
+            first_entry = all_toc_entries[0]
+            first_doc_href = first_entry["doc_href"]
+            first_pos = first_entry["position"]
+            first_doc_order = first_entry["doc_order"]
+            prefix_html = ""
+            # Include content from documents before the first entry's document
+            for doc_idx in range(first_doc_order):
+                intermediate_doc_href = spine_docs[doc_idx]
+                prefix_html += self.doc_content.get(intermediate_doc_href, "")
+            # Include content from the start of the first entry's document up to its position
+            first_doc_html = self.doc_content.get(first_doc_href, "")
+            prefix_html += first_doc_html[:first_pos]
+
+            if prefix_html.strip():
+                prefix_soup = BeautifulSoup(prefix_html, 'html.parser')
+                # Remove sup and sub tags
+                for tag in prefix_soup.find_all(['sup', 'sub']):
+                    tag.decompose()
+                prefix_text = clean_text(prefix_soup.get_text()).strip()
+                
+                if prefix_text:
+                    # Create a new chapter for content before the first TOC entry
+                    # Use a synthetic href to avoid collision with real TOC entries
+                    prefix_chapter_href = "prefix_content_chapter"
+                    self.content_texts[prefix_chapter_href] = prefix_text
+                    self.content_lengths[prefix_chapter_href] = len(prefix_text)
+                    
+                    # Add a new entry to the TOC for the prefix content
+                    prefix_link = epub.Link(prefix_chapter_href, "Introduction", prefix_chapter_href)
+                    # Insert at beginning of TOC
+                    if isinstance(self.book.toc, list):
+                        self.book.toc.insert(0, (prefix_link,))
+                    else:
+                        self.book.toc = [(prefix_link,)] + (self.book.toc or [])
 
     def _preprocess_pdf_content(self):
         """Pre-process all page contents from PDF document"""
@@ -228,16 +389,21 @@ class HandlerDialog(QDialog):
 
     def _build_epub_tree(self):
         """Build the tree for EPUB files from TOC"""
-        checkable_hrefs = set()
+        self.treeWidget.clear()
+        info_item = QTreeWidgetItem(self.treeWidget, ["Information"])
+        info_item.setData(0, Qt.UserRole, "info:bookinfo")
+        info_item.setFlags(info_item.flags() & ~Qt.ItemIsUserCheckable)
+        font = info_item.font(0)
+        font.setBold(True)
+        info_item.setFont(0, font)
 
+        # Regular tree building
         def build_tree(toc_entries, parent_item):
             for entry in toc_entries:
                 href, title, children = None, "Unknown", []
-
                 if isinstance(entry, ebooklib.epub.Link):
                     href, title = entry.href, entry.title or entry.href
                 elif isinstance(entry, tuple) and len(entry) >= 1:
-                    # Handle nested sections
                     section_or_link = entry[0]
                     if isinstance(section_or_link, ebooklib.epub.Section):
                         title = section_or_link.title
@@ -247,44 +413,28 @@ class HandlerDialog(QDialog):
                             section_or_link.href,
                             section_or_link.title or section_or_link.href,
                         )
-
                     if len(entry) > 1 and isinstance(entry[1], list):
                         children = entry[1]
                 else:
-                    continue  # Skip unknown entry types
+                    continue
-
-                # Use the href without fragment for content lookup
-                lookup_href = href.split("#")[0] if href else None
 
+                # Create tree item
                 item = QTreeWidgetItem(parent_item, [title])
-                item.setData(0, Qt.UserRole, href)  # Store original href
+                item.setData(0, Qt.UserRole, href)
 
-                # Make item checkable if it has content or children
+                # Make item checkable if it has content
-                has_content = (
+                has_content = href and href in self.content_texts and self.content_texts[href].strip()
-                    lookup_href
+                if has_content or children:
-                    and lookup_href in self.content_texts
-                    and self.content_texts[lookup_href]
-                )
-                is_potentially_checkable = has_content or children
-
-                # Only make the first occurrence of a given lookup_href checkable
-                if is_potentially_checkable and (
-                    not lookup_href or lookup_href not in checkable_hrefs
-                ):
                     item.setFlags(item.flags() | Qt.ItemIsUserCheckable)
-                    if lookup_href:
-                        checkable_hrefs.add(lookup_href)
-                    # Set initial state based on provided checks
                     is_checked = href and href in self.checked_chapters
                     item.setCheckState(0, Qt.Checked if is_checked else Qt.Unchecked)
                 else:
-                    # Not checkable if duplicate content or no content/children
                     item.setFlags(item.flags() & ~Qt.ItemIsUserCheckable)
 
+                # Process children
                 if children:
                     build_tree(children, item)
 
-        # Build the tree from the TOC
         build_tree(self.book.toc, self.treeWidget)
 
     def _build_pdf_tree(self):
@@ -501,6 +651,20 @@ class HandlerDialog(QDialog):
         self.previewEdit.setMinimumWidth(300)
         self.previewEdit.setStyleSheet("QTextEdit { border: none; }")
 
+        # Create informative text label below preview
+        self.previewInfoLabel = QLabel("*Note: You can modify the content later using the \"Edit\" button in the input box or by accessing the temporary files directory through settings.", self)
+        self.previewInfoLabel.setWordWrap(True)
+        self.previewInfoLabel.setStyleSheet("QLabel { color: #666; font-style: italic; }")
+
+        # Right panel layout (preview and info label)
+        previewLayout = QVBoxLayout()
+        previewLayout.setContentsMargins(0, 0, 0, 0)
+        previewLayout.addWidget(self.previewEdit, 1)
+        previewLayout.addWidget(self.previewInfoLabel, 0)
+        
+        rightWidget = QWidget()
+        rightWidget.setLayout(previewLayout)
+
         # Dialog buttons
         buttons = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel, self)
         buttons.accepted.connect(self.accept)
@@ -589,7 +753,7 @@ class HandlerDialog(QDialog):
         # Create splitter for left panel and preview
         self.splitter = QSplitter(Qt.Horizontal)
         self.splitter.addWidget(leftWidget)
-        self.splitter.addWidget(self.previewEdit)
+        self.splitter.addWidget(rightWidget)  # Now using rightWidget that includes preview and label
         self.splitter.setSizes([280, 420])
 
         # Set main layout
@@ -858,20 +1022,19 @@ class HandlerDialog(QDialog):
         # Get content based on file type
         text = None
         if self.file_type == "epub":
-            lookup_href = identifier.split("#")[0] if identifier else None
+            # For EPUB, always use the exact href from the TOC
-            text = self.content_texts.get(lookup_href, None)
+            text = self.content_texts.get(identifier)
         else:  # PDF
-            text = self.content_texts.get(identifier, None)
+            text = self.content_texts.get(identifier)
 
-        # Display content or placeholder text
+        # Display content or placeholder text - never remove titles
         if text is None:
-            self.previewEdit.setPlainText(
+            title = current.text(0)
-                "(Section title - select a sub-item for content)"
+            # Add title to preview even if no content
-                if current.childCount() > 0
+            self.previewEdit.setPlainText(f"{title}\n\n(No content available for this item)")
-                else "(No content associated with this entry)"
-            )
         elif not text.strip():
-            self.previewEdit.setPlainText("(Item is empty)")
+            title = current.text(0)
+            self.previewEdit.setPlainText(f"{title}\n\n(This item is empty)")
         else:
             self.previewEdit.setPlainText(text)
 
@@ -1011,35 +1174,29 @@ class HandlerDialog(QDialog):
     def _get_epub_selected_text(self):
         """Get selected text from EPUB content"""
         all_checked_hrefs = set()
-        included_text_hrefs = set()
         chapter_titles = []
 
-        # Collect all checked hrefs
+        # Collect all checked hrefs in tree order to preserve chapter sequence
+        ordered_checked_items = []
         iterator = QTreeWidgetItemIterator(self.treeWidget)
         while iterator.value():
             item = iterator.value()
             if item.checkState(0) == Qt.Checked:
                 href = item.data(0, Qt.UserRole)
-                if href:
+                if href and href != "info:bookinfo":
                     all_checked_hrefs.add(href)
+                    ordered_checked_items.append((item, href))
             iterator += 1
 
-        # Include content for all checked items
+        # Process checked items in order
-        iterator = QTreeWidgetItemIterator(self.treeWidget)
+        for item, href in ordered_checked_items:
-        while iterator.value():
+            # Always use the exact href (including fragment) from the TOC
-            item = iterator.value()
+            text = self.content_texts.get(href)
-            if item.checkState(0) == Qt.Checked:
+            if text and text.strip():
-                href = item.data(0, Qt.UserRole)
+                title = item.text(0)
-                if href:
+                title = re.sub(r"^\s*-\s*", "", title).strip()
-                    lookup_href = href.split("#")[0]
+                marker = f"<<CHAPTER_MARKER:{title}>>"
-                    text = self.content_texts.get(lookup_href, "")
+                chapter_titles.append((title, marker + "\n" + text))
-                    if text and lookup_href not in included_text_hrefs:
-                        title = item.text(0)
-                        title = re.sub(r"^\s*-\s*", "", title).strip()
-                        marker = f"<<CHAPTER_MARKER:{title}>>"
-                        chapter_titles.append((title, marker + "\n" + text))
-                        included_text_hrefs.add(lookup_href)
-            iterator += 1
 
         return "\n\n".join([t[1] for t in chapter_titles]), all_checked_hrefs
 
@@ -1080,7 +1237,7 @@ class HandlerDialog(QDialog):
             return "\n\n".join(all_content), all_checked_identifiers
 
         # For PDFs with bookmarks, process content with parent-child relationships
-        # New logic: if only child pages are selected (not parent), use parent's name as chapter marker at first selected child
+        # If only child pages are selected (not parent), use parent's name as chapter marker at first selected child
         iterator = QTreeWidgetItemIterator(self.treeWidget)
         while iterator.value():
             item = iterator.value()

abogen/constants.py

@@ -1,4 +1,4 @@
-from utils import get_version, get_user_config_path
+from utils import get_version
 
 # Program Information
 PROGRAM_NAME = "abogen"

abogen/conversion.py

@@ -2,6 +2,8 @@ import os
 import re
 import tempfile
 import time
+import chardet
+import charset_normalizer
 from PyQt5.QtCore import QThread, pyqtSignal, Qt
 from PyQt5.QtWidgets import QCheckBox, QVBoxLayout, QDialog, QLabel, QDialogButtonBox
 import soundfile as sf
@@ -12,6 +14,21 @@ from constants import PROGRAM_NAME, LANGUAGE_DESCRIPTIONS, SAMPLE_VOICE_TEXTS
 def get_sample_voice_text(lang_code):
     return SAMPLE_VOICE_TEXTS.get(lang_code, SAMPLE_VOICE_TEXTS["a"])
 
+def detect_encoding(file_path):
+    with open(file_path, "rb") as f:
+        raw_data = f.read()
+    detected_encoding = None
+    for detectors in (charset_normalizer, chardet):
+        try:
+            result = detectors.detect(raw_data)["encoding"]
+        except Exception:
+            continue
+        if result is not None:
+            detected_encoding = result
+            break
+    encoding = detected_encoding if detected_encoding else "utf-8"
+    return encoding.lower()
+
 
 class ChapterOptionsDialog(QDialog):
     def __init__(self, chapter_count, parent=None):
@@ -153,6 +170,10 @@ class ConversionThread(QThread):
             self.log_updated.emit(f"- Subtitle mode: {self.subtitle_mode}")
             self.log_updated.emit(f"- Output format: {self.output_format}")
             self.log_updated.emit(f"- Save option: {self.save_option}")
+            if self.replace_single_newlines:
+                self.log_updated.emit(
+                    f"- Replace single newlines: Yes"
+                )
 
             # Display save_chapters_separately flag if it's set
             if hasattr(self, "save_chapters_separately"):
@@ -183,7 +204,8 @@ class ConversionThread(QThread):
             if self.is_direct_text:
                 text = self.file_name  # Treat file_name as direct text input
             else:
-                with open(self.file_name, "r", encoding="utf-8") as file:
+                encoding = detect_encoding(self.file_name)
+                with open(self.file_name, "r", encoding=encoding, errors="replace") as file:
                     text = file.read()
 
             # Clean up text using utility function
@@ -455,7 +477,7 @@ class ConversionThread(QThread):
                         chapter_srt_path = os.path.join(
                             chapters_out_dir, f"{chapter_filename}.srt"
                         )
-                        with open(chapter_srt_path, "w", encoding="utf-8") as srt_file:
+                        with open(chapter_srt_path, "w", encoding="utf-8", errors="replace") as srt_file:
                             for i, (start, end, text) in enumerate(
                                 chapter_subtitle_entries, 1
                             ):
@@ -496,7 +518,7 @@ class ConversionThread(QThread):
                 srt_path = os.path.splitext(out_path)[0] + ".srt"
                 sf.write(out_path, audio, 24000, format=self.output_format)
                 if self.subtitle_mode != "Disabled":
-                    with open(srt_path, "w", encoding="utf-8") as srt_file:
+                    with open(srt_path, "w", encoding="utf-8", errors="replace") as srt_file:
                         for i, (start, end, text) in enumerate(subtitle_entries, 1):
                             srt_file.write(
                                 f"{i}\n{self._srt_time(start)} --> {self._srt_time(end)}\n{text}\n\n"

abogen/gui.py

@@ -467,6 +467,7 @@ class abogen(QWidget):
         self.use_gpu = self.config.get(
             "use_gpu", True
         )  # Load GPU setting with default True
+        self.replace_single_newlines = self.config.get("replace_single_newlines", False)
         self._pending_close_event = None
         self.gpu_ok = False  # Initialize GPU availability status
 
@@ -1059,6 +1060,8 @@ class abogen(QWidget):
             self.conversion_thread.file_size_str = file_size_str
             # Pass max_subtitle_words from config
             self.conversion_thread.max_subtitle_words = self.max_subtitle_words
+            # Pass replace_single_newlines setting
+            self.conversion_thread.replace_single_newlines = self.replace_single_newlines
             # Pass chapter count for EPUB or PDF files
             if self.selected_file_type in ["epub", "pdf"] and hasattr(
                 self, "selected_chapters"
@@ -1499,6 +1502,13 @@ class abogen(QWidget):
         """Show a dropdown menu for settings options."""
         menu = QMenu(self)
 
+        # Add replace single newlines option
+        newline_action = QAction("Replace single newlines with spaces", self)
+        newline_action.setCheckable(True)
+        newline_action.setChecked(self.replace_single_newlines)
+        newline_action.triggered.connect(self.toggle_replace_single_newlines)
+        menu.addAction(newline_action)
+
         # Add max words per subtitle option
         max_words_action = QAction("Configure max words per subtitle", self)
         max_words_action.triggered.connect(self.set_max_subtitle_words)
@@ -1545,6 +1555,11 @@ class abogen(QWidget):
 
         menu.exec_(self.settings_btn.mapToGlobal(QPoint(0, self.settings_btn.height())))
 
+    def toggle_replace_single_newlines(self, checked):
+        self.replace_single_newlines = checked
+        self.config["replace_single_newlines"] = checked
+        save_config(self.config)
+
     def reveal_config_in_explorer(self):
         """Open the configuration file location in file explorer."""
         from utils import get_user_config_path

abogen/utils.py

@@ -69,15 +69,18 @@ def get_user_config_path():
 _sleep_procs = {"Darwin": None, "Linux": None}  # Store sleep prevention processes
 
 
-def clean_text(text):
+def clean_text(text, *args, **kwargs):
-    # Trim spaces and tabs at the start and end of each line, preserving blank lines
+    # Load replace_single_newlines from config
-    text = "\n".join(line.strip() for line in text.splitlines())
+    cfg = load_config()
+    replace_single_newlines = cfg.get("replace_single_newlines", False)
+    # Collapse all whitespace (excluding newlines) into single spaces per line and trim edges
+    lines = [re.sub(r"[^\S\n]+", " ", line).strip() for line in text.splitlines()]
+    text = "\n".join(lines)
     # Standardize paragraph breaks (multiple newlines become exactly two) and trim overall whitespace
     text = re.sub(r"\n{3,}", "\n\n", text).strip()
-    # Replace single newlines with spaces, but preserve double newlines
+    # Optionally replace single newlines with spaces, but preserve double newlines
-    # text = re.sub(r"(?<!\n)\n(?!\n)", " ", text)
+    if replace_single_newlines:
-    # Collapse multiple spaces and tabs into a single space
+        text = re.sub(r"(?<!\n)\n(?!\n)", " ", text)
-    text = re.sub(r"[ \t]+", " ", text)
     return text
 
 

demo/README.md

@@ -40,11 +40,6 @@ Run this FFmpeg command to create the tiny video:
 ffmpeg -loop 1 -framerate 24 -i bg.jpg -i audio.wav -vf "ass=your_subtitle_demo.ass" -c:v libvpx-vp9 -b:v 0 -crf 30 -c:a libopus -shortest demo.webm
 ```
 
-That's it! The magic happens because:
-- We use a single static background image instead of many frames
-- The subtitles are stored as text (vector data), not as pixels
-- VP9 video codec with Opus audio provides excellent compression
-
 ## For Higher Quality (But Larger) Video (.mp4)
 
 If you need better quality for distribution, use this command instead:

pyproject.toml

@@ -19,7 +19,9 @@ dependencies = [
     "beautifulsoup4>=4.13.4",
     "PyMuPDF>=1.25.5",
     "soundfile>=0.13.1",
-    "pygame>=2.6.1"
+    "pygame>=2.6.1",
+    "charset_normalizer>=3.4.1",
+    "chardet>=5.2.0"
 ]
 
 classifiers = [
@@ -52,10 +54,9 @@ abogen-cli = "abogen.main:main"
 exclude = [
     "/.github",
     "/demo",
-    "/demo",
+    "/abogen/resources",
     "/abogen/assets/create_shortcuts.bat",
     "WINDOWS_INSTALL.bat",
-    
 ]
 
 [tool.hatch.build.targets.wheel]