Dymas/abogen
1
0
Fork 0
mirror of https://github.com/denizsafak/abogen.git synced 2026-07-18 13:40:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

Improved the markdown logic

Browse Source
This commit is contained in:
Deniz Şafak
2025-09-17 03:32:13 +03:00
parent 196cd93008
commit 219e2fc6ae
6 changed files with 116 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+2
View File
@@ -1,4 +1,6 @@
# 1.1.8 (pre-release) # 1.1.8 (pre-release)
- Added `.md` (Markdown) file extension support by @brianxiadong in PR #75
- Improved the markdown logic to better handle various markdown structures and edge cases.
- Fixed `No Qt platform plugin could be initialized` error, mentioned in #59 by @sunrainxyz - Fixed `No Qt platform plugin could be initialized` error, mentioned in #59 by @sunrainxyz
# 1.1.7 # 1.1.7
abogen/book_handler.py
+86 -103
View File
@@ -22,10 +22,12 @@ from PyQt5.QtWidgets import (
QLabel, QLabel,
) )
from PyQt5.QtCore import Qt from PyQt5.QtCore import Qt
from abogen.utils import clean_text, calculate_text_length from abogen.utils import clean_text, calculate_text_length, detect_encoding
import os import os
import logging # Add logging import logging # Add logging
import urllib.parse import urllib.parse
import markdown
import textwrap
# Setup logging # Setup logging
logging.basicConfig( logging.basicConfig(
@@ -112,12 +114,13 @@ class HandlerDialog(QDialog):
self.markdown_text = None self.markdown_text = None
if self.file_type == "markdown": if self.file_type == "markdown":
try: try:
encoding = self._detect_encoding(book_path) encoding = detect_encoding(book_path)
with open(book_path, "r", encoding=encoding, errors="replace") as f: with open(book_path, "r", encoding=encoding, errors="replace") as f:
self.markdown_text = f.read() self.markdown_text = f.read()
except Exception as e: except Exception as e:
logging.error(f"Error reading markdown file: {e}") logging.error(f"Error reading markdown file: {e}")
self.markdown_text = "" self.markdown_text = ""
self.markdown_toc = [] # For storing parsed markdown TOC
# Extract book metadata # Extract book metadata
self.book_metadata = self._extract_book_metadata() self.book_metadata = self._extract_book_metadata()
@@ -184,17 +187,6 @@ class HandlerDialog(QDialog):
# Update checkbox states # Update checkbox states
self._update_checkbox_states() self._update_checkbox_states()
def _detect_encoding(self, file_path):
"""Detect encoding of a text file"""
with open(file_path, "rb") as f:
raw_data = f.read()
try:
import chardet
result = chardet.detect(raw_data)
return result.get("encoding", "utf-8")
except ImportError:
return "utf-8"
def _preprocess_content(self): def _preprocess_content(self):
"""Pre-process content from the document""" """Pre-process content from the document"""
if self.file_type == "epub": if self.file_type == "epub":
@@ -233,66 +225,69 @@ class HandlerDialog(QDialog):
page_id = f"page_{page_num + 1}" page_id = f"page_{page_num + 1}"
self.content_texts[page_id] = text self.content_texts[page_id] = text
self.content_lengths[page_id] = calculate_text_length(text) self.content_lengths[page_id] = calculate_text_length(text)
def _preprocess_markdown_content(self): def _preprocess_markdown_content(self):
"""Pre-process markdown content and extract chapters/sections"""
if not self.markdown_text: if not self.markdown_text:
return return
# Split markdown by headers to create chapters
import re
# Clean the text first
text = clean_text(self.markdown_text) text = clean_text(self.markdown_text)
text = textwrap.dedent(text)
md = markdown.Markdown(extensions=['toc', 'fenced_code'])
html = md.convert(text)
self.markdown_toc = md.toc_tokens
# Find all markdown headers (# ## ### etc.) soup = BeautifulSoup(html, 'html.parser')
header_pattern = r'^(#{1,6})\s+(.+)$'
headers = list(re.finditer(header_pattern, text, re.MULTILINE))
self.content_texts = {} self.content_texts = {}
self.content_lengths = {} self.content_lengths = {}
if not headers: if not self.markdown_toc:
# No headers found, treat entire content as single chapter
chapter_id = "markdown_content" chapter_id = "markdown_content"
self.content_texts[chapter_id] = text self.content_texts[chapter_id] = text
self.content_lengths[chapter_id] = calculate_text_length(text) self.content_lengths[chapter_id] = calculate_text_length(text)
return return
# Process headers and extract content between them all_headers = []
for i, header_match in enumerate(headers): def flatten_toc(toc_list):
header_level = len(header_match.group(1)) # Number of # symbols for header in toc_list:
header_text = header_match.group(2).strip() all_headers.append(header)
header_start = header_match.start() if header.get('children'):
flatten_toc(header['children'])
flatten_toc(self.markdown_toc)
# Find content between this header and the next one of same or higher level header_positions = []
content_start = header_match.end() for header in all_headers:
content_end = len(text) header_id = header['id']
id_pattern = f'id="{header_id}"'
pos = html.find(id_pattern)
if pos != -1:
tag_start = html.rfind('<', 0, pos)
header_positions.append({
'id': header_id,
'start': tag_start,
'name': header['name']
})
header_positions.sort(key=lambda x: x['start'])
# Look for next header of same or higher level (fewer # symbols) for i, header_pos in enumerate(header_positions):
for j in range(i + 1, len(headers)): header_id = header_pos['id']
next_header = headers[j] header_name = header_pos['name']
next_level = len(next_header.group(1)) content_start = header_pos['start']
if next_level <= header_level: content_end = header_positions[i + 1]['start'] if i + 1 < len(header_positions) else len(html)
content_end = next_header.start() section_html = html[content_start:content_end]
break section_soup = BeautifulSoup(section_html, 'html.parser')
header_tag = section_soup.find(attrs={'id': header_id})
# Extract content for this chapter if header_tag:
chapter_content = text[content_start:content_end].strip() header_tag.decompose()
section_text = clean_text(section_soup.get_text()).strip()
# Create unique identifier for this chapter chapter_id = header_id
chapter_id = f"markdown_chapter_{i + 1}" if section_text:
full_content = f"{header_name}\n\n{section_text}"
# Store the chapter content
if chapter_content:
# Include the header in the content
full_content = f"{header_text}\n\n{chapter_content}"
self.content_texts[chapter_id] = full_content self.content_texts[chapter_id] = full_content
self.content_lengths[chapter_id] = calculate_text_length(full_content) self.content_lengths[chapter_id] = calculate_text_length(full_content)
else: else:
# Even if no content, store the header self.content_texts[chapter_id] = header_name
self.content_texts[chapter_id] = header_text self.content_lengths[chapter_id] = calculate_text_length(header_name)
self.content_lengths[chapter_id] = calculate_text_length(header_text)
def _process_epub_content_spine_fallback(self): def _process_epub_content_spine_fallback(self):
"""Fallback EPUB processing based purely on spine order.""" """Fallback EPUB processing based purely on spine order."""
@@ -1189,61 +1184,47 @@ class HandlerDialog(QDialog):
page_item.setFlags(page_item.flags() & ~Qt.ItemIsUserCheckable) page_item.setFlags(page_item.flags() & ~Qt.ItemIsUserCheckable)
def _build_markdown_tree(self): def _build_markdown_tree(self):
"""Build tree structure for markdown file based on headers""" """Build tree structure for markdown file based on parsed TOC."""
if not self.content_texts: if not self.markdown_text:
return return
# If only one content item (no headers), create simple structure if not self.markdown_toc:
if len(self.content_texts) == 1: # Handle case with no headers (single content block)
chapter_id = list(self.content_texts.keys())[0] if self.content_texts:
title = "Content" chapter_id = list(self.content_texts.keys())[0]
title = "Content"
item = QTreeWidgetItem(self.treeWidget, [title]) item = QTreeWidgetItem(self.treeWidget, [title])
item.setData(0, Qt.UserRole, chapter_id) item.setData(0, Qt.UserRole, chapter_id)
if self.content_lengths.get(chapter_id, 0) > 0:
if self.content_lengths.get(chapter_id, 0) > 0: item.setFlags(item.flags() | Qt.ItemIsUserCheckable)
item.setFlags(item.flags() | Qt.ItemIsUserCheckable) is_checked = chapter_id in self.checked_chapters
is_checked = chapter_id in self.checked_chapters item.setCheckState(0, Qt.Checked if is_checked else Qt.Unchecked)
item.setCheckState(0, Qt.Checked if is_checked else Qt.Unchecked) else:
else: item.setFlags(item.flags() & ~Qt.ItemIsUserCheckable)
item.setFlags(item.flags() & ~Qt.ItemIsUserCheckable)
return return
# Build hierarchical tree based on markdown headers def build_from_toc(toc_list, parent_item):
# First, parse the original markdown to get header structure for header in toc_list:
import re title = header['name']
header_pattern = r'^(#{1,6})\s+(.+)$' chapter_id = header['id']
headers = list(re.finditer(header_pattern, self.markdown_text, re.MULTILINE))
# Create tree items for each header item = QTreeWidgetItem(parent_item, [title])
stack = [] # Stack to track parent items at different levels item.setData(0, Qt.UserRole, chapter_id)
for i, header_match in enumerate(headers): has_content = self.content_lengths.get(chapter_id, 0) > 0
header_level = len(header_match.group(1)) has_children = bool(header.get('children'))
header_text = header_match.group(2).strip()
chapter_id = f"markdown_chapter_{i + 1}"
# Pop stack until we find appropriate parent level if has_content or has_children:
while stack and stack[-1][0] >= header_level: item.setFlags(item.flags() | Qt.ItemIsUserCheckable)
stack.pop() is_checked = chapter_id in self.checked_chapters
item.setCheckState(0, Qt.Checked if is_checked else Qt.Unchecked)
else:
item.setFlags(item.flags() & ~Qt.ItemIsUserCheckable)
# Determine parent item if has_children:
parent_item = stack[-1][1] if stack else self.treeWidget build_from_toc(header['children'], item)
# Create tree item build_from_toc(self.markdown_toc, self.treeWidget)
item = QTreeWidgetItem(parent_item, [header_text])
item.setData(0, Qt.UserRole, chapter_id)
# Set checkable state based on content
if self.content_lengths.get(chapter_id, 0) > 0:
item.setFlags(item.flags() | Qt.ItemIsUserCheckable)
is_checked = chapter_id in self.checked_chapters
item.setCheckState(0, Qt.Checked if is_checked else Qt.Unchecked)
else:
item.setFlags(item.flags() & ~Qt.ItemIsUserCheckable)
# Add to stack for potential children
stack.append((header_level, item))
def _build_pdf_pages_tree(self): def _build_pdf_pages_tree(self):
pages_item = QTreeWidgetItem(self.treeWidget, ["Pages"]) pages_item = QTreeWidgetItem(self.treeWidget, ["Pages"])
@@ -1843,10 +1824,11 @@ class HandlerDialog(QDialog):
logging.warning(f"Error parsing markdown frontmatter: {e}") logging.warning(f"Error parsing markdown frontmatter: {e}")
# Fallback: use first H1 header as title if no frontmatter title # Fallback: use first H1 header as title if no frontmatter title
if not metadata["title"]: if not metadata["title"] and self.markdown_toc:
first_h1_match = re.search(r'^#\s+(.+)$', self.markdown_text, re.MULTILINE) # Find the first level 1 header
if first_h1_match: first_h1 = next((h for h in self.markdown_toc if h['level'] == 1), None)
metadata["title"] = first_h1_match.group(1).strip() if first_h1:
metadata["title"] = first_h1['name']
else: else:
pdf_info = self.pdf_doc.metadata pdf_info = self.pdf_doc.metadata
if pdf_info: if pdf_info:
@@ -1949,6 +1931,7 @@ class HandlerDialog(QDialog):
item_order_counter += 1 item_order_counter += 1
if item.checkState(0) == Qt.Checked: if item.checkState(0) == Qt.Checked:
identifier = item.data(0, Qt.UserRole) identifier = item.data(0, Qt.UserRole)
if identifier and identifier != "info:bookinfo": if identifier and identifier != "info:bookinfo":
all_checked_identifiers.add(identifier) all_checked_identifiers.add(identifier)
ordered_checked_items.append((item_order_counter, item, identifier)) ordered_checked_items.append((item_order_counter, item, identifier))
abogen/conversion.py
+1 -19
View File
@@ -1,14 +1,12 @@
import os import os
import re import re
import time import time
import chardet
import charset_normalizer
import hashlib # For generating unique cache filenames import hashlib # For generating unique cache filenames
from platformdirs import user_desktop_dir from platformdirs import user_desktop_dir
from PyQt5.QtCore import QThread, pyqtSignal, Qt, QTimer from PyQt5.QtCore import QThread, pyqtSignal, Qt, QTimer
from PyQt5.QtWidgets import QCheckBox, QVBoxLayout, QDialog, QLabel, QDialogButtonBox from PyQt5.QtWidgets import QCheckBox, QVBoxLayout, QDialog, QLabel, QDialogButtonBox
import soundfile as sf import soundfile as sf
from abogen.utils import clean_text, create_process, get_user_cache_path from abogen.utils import clean_text, create_process, get_user_cache_path, detect_encoding
from abogen.constants import ( from abogen.constants import (
LANGUAGE_DESCRIPTIONS, LANGUAGE_DESCRIPTIONS,
SAMPLE_VOICE_TEXTS, SAMPLE_VOICE_TEXTS,
@@ -30,22 +28,6 @@ def get_sample_voice_text(lang_code):
return SAMPLE_VOICE_TEXTS.get(lang_code, SAMPLE_VOICE_TEXTS["a"]) return SAMPLE_VOICE_TEXTS.get(lang_code, SAMPLE_VOICE_TEXTS["a"])
def detect_encoding(file_path):
with open(file_path, "rb") as f:
raw_data = f.read()
detected_encoding = None
for detectors in (charset_normalizer, chardet):
try:
result = detectors.detect(raw_data)["encoding"]
except Exception:
continue
if result is not None:
detected_encoding = result
break
encoding = detected_encoding if detected_encoding else "utf-8"
return encoding.lower()
class ChapterOptionsDialog(QDialog): class ChapterOptionsDialog(QDialog):
def __init__(self, chapter_count, parent=None): def __init__(self, chapter_count, parent=None):
super().__init__(parent) super().__init__(parent)
abogen/gui.py
+8 -8
View File
@@ -294,13 +294,13 @@ class InputBox(QLabel):
f"QLabel {{ {self.STYLE_ACTIVE} }} QLabel:hover {{ {self.STYLE_ACTIVE_HOVER} }}" f"QLabel {{ {self.STYLE_ACTIVE} }} QLabel:hover {{ {self.STYLE_ACTIVE_HOVER} }}"
) )
self.clear_btn.show() self.clear_btn.show()
is_document = self.window().selected_file_type in ["epub", "pdf"] is_document = self.window().selected_file_type in ["epub", "pdf", "md", "markdown"]
self.chapters_btn.setVisible(is_document) self.chapters_btn.setVisible(is_document)
if is_document: if is_document:
chapter_count = len(self.window().selected_chapters) chapter_count = len(self.window().selected_chapters)
file_type = self.window().selected_file_type file_type = self.window().selected_file_type
# Adjust button text based on file type # Adjust button text based on file type
if file_type == "epub": if file_type == "epub" or file_type == "md" or file_type == "markdown":
self.chapters_btn.setText(f"Chapters ({chapter_count})") self.chapters_btn.setText(f"Chapters ({chapter_count})")
else: # PDF - always use Pages else: # PDF - always use Pages
self.chapters_btn.setText(f"Pages ({chapter_count})") self.chapters_btn.setText(f"Pages ({chapter_count})")
@@ -313,7 +313,7 @@ class InputBox(QLabel):
# For epub/pdf files, show edit if we have a selected_file (temp txt) # For epub/pdf files, show edit if we have a selected_file (temp txt)
if ( if (
self.window().selected_file_type in ["epub", "pdf"] self.window().selected_file_type in ["epub", "pdf", "md", "markdown", "md", "markdown"]
and self.window().selected_file and self.window().selected_file
): ):
should_show_edit = True should_show_edit = True
@@ -462,7 +462,7 @@ class InputBox(QLabel):
def on_chapters_clicked(self): def on_chapters_clicked(self):
win = self.window() win = self.window()
if win.selected_file_type in ["epub", "pdf"] and win.selected_book_path: if win.selected_file_type in ["epub", "pdf", "md", "markdown"] and win.selected_book_path:
# Call open_book_file which shows the dialog and updates selected_chapters # Call open_book_file which shows the dialog and updates selected_chapters
if win.open_book_file(win.selected_book_path): if win.open_book_file(win.selected_book_path):
# Refresh the info label and button text after dialog closes # Refresh the info label and button text after dialog closes
@@ -474,7 +474,7 @@ class InputBox(QLabel):
def on_edit_clicked(self): def on_edit_clicked(self):
win = self.window() win = self.window()
# For PDFs and EPUBs, use the temporary text file # For PDFs and EPUBs, use the temporary text file
if win.selected_file_type in ["epub", "pdf"] and win.selected_file: if win.selected_file_type in ["epub", "pdf", "md", "markdown"] and win.selected_file:
# Use the temporary .txt file that was generated # Use the temporary .txt file that was generated
win.open_textbox_dialog(win.selected_file) win.open_textbox_dialog(win.selected_file)
else: else:
@@ -611,7 +611,7 @@ class TextboxDialog(QDialog):
hasattr(main_window, "displayed_file_path") hasattr(main_window, "displayed_file_path")
and main_window.displayed_file_path and main_window.displayed_file_path
): ):
if main_window.selected_file_type in ["epub", "pdf"]: if main_window.selected_file_type in ["epub", "pdf", "md", "markdown"]:
# Use the base name of the displayed file but change extension to .txt # Use the base name of the displayed file but change extension to .txt
base_name = os.path.splitext(main_window.displayed_file_path)[0] base_name = os.path.splitext(main_window.displayed_file_path)[0]
initial_path = base_name + ".txt" initial_path = base_name + ".txt"
@@ -1639,7 +1639,7 @@ class abogen(QWidget):
def add_to_queue(self): def add_to_queue(self):
# For epub/pdf, always use the converted txt file (selected_file) # For epub/pdf, always use the converted txt file (selected_file)
if self.selected_file_type in ["epub", "pdf"]: if self.selected_file_type in ["epub", "pdf", "md", "markdown"]:
file_to_queue = self.selected_file file_to_queue = self.selected_file
else: else:
file_to_queue = ( file_to_queue = (
@@ -1875,7 +1875,7 @@ class abogen(QWidget):
"subtitle_format", "ass_centered_narrow" "subtitle_format", "ass_centered_narrow"
) )
# Pass chapter count for EPUB or PDF files # Pass chapter count for EPUB or PDF files
if self.selected_file_type in ["epub", "pdf"] and hasattr( if self.selected_file_type in ["epub", "pdf", "md", "markdown"] and hasattr(
self, "selected_chapters" self, "selected_chapters"
): ):
self.conversion_thread.chapter_count = len(self.selected_chapters) self.conversion_thread.chapter_count = len(self.selected_chapters)
abogen/utils.py
+17
View File
@@ -11,6 +11,23 @@ from threading import Thread
warnings.filterwarnings("ignore") warnings.filterwarnings("ignore")
def detect_encoding(file_path):
import chardet
import charset_normalizer
with open(file_path, "rb") as f:
raw_data = f.read()
detected_encoding = None
for detectors in (charset_normalizer, chardet):
try:
result = detectors.detect(raw_data)["encoding"]
except Exception:
continue
if result is not None:
detected_encoding = result
break
encoding = detected_encoding if detected_encoding else "utf-8"
return encoding.lower()
def get_resource_path(package, resource): def get_resource_path(package, resource):
""" """
Get the path to a resource file, with fallback to local file system. Get the path to a resource file, with fallback to local file system.
pyproject.toml
+2 -1
View File
@@ -24,7 +24,8 @@ dependencies = [
"pygame>=2.6.1", "pygame>=2.6.1",
"charset_normalizer>=3.4.1", "charset_normalizer>=3.4.1",
"chardet>=5.2.0", "chardet>=5.2.0",
"static_ffmpeg>=2.13" "static_ffmpeg>=2.13",
"Markdown>=3.9"
] ]
classifiers = [ classifiers = [