diff --git a/CHANGELOG.md b/CHANGELOG.md index 138bbd1..ce96a61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1 @@ -- Added `abogen-cli` command for better troubleshooting and error handling. -- Switched from setuptools to hatchling for packaging. -- Added classifiers to the package metadata. -- Fixed "No module named 'docopt'" and "setuptools.build_meta" import errors while using .bat installer in Windows, mentioned by @nigelp in https://github.com/denizsafak/abogen/issues/2 -- Improvements in code and documentation. \ No newline at end of file +- Fixed "'utf-8' codec can't decode byte" error, mentioned in #3 by @nigelp \ No newline at end of file diff --git a/abogen/conversion.py b/abogen/conversion.py index 81888b4..f14c1e3 100644 --- a/abogen/conversion.py +++ b/abogen/conversion.py @@ -2,6 +2,8 @@ import os import re import tempfile import time +import chardet +import charset_normalizer from PyQt5.QtCore import QThread, pyqtSignal, Qt from PyQt5.QtWidgets import QCheckBox, QVBoxLayout, QDialog, QLabel, QDialogButtonBox import soundfile as sf @@ -12,6 +14,21 @@ from constants import PROGRAM_NAME, LANGUAGE_DESCRIPTIONS, SAMPLE_VOICE_TEXTS def get_sample_voice_text(lang_code): return SAMPLE_VOICE_TEXTS.get(lang_code, SAMPLE_VOICE_TEXTS["a"]) +def detect_encoding(file_path): + with open(file_path, "rb") as f: + raw_data = f.read() + detected_encoding = None + for detectors in (charset_normalizer, chardet): + try: + result = detectors.detect(raw_data)["encoding"] + except Exception: + continue + if result is not None: + detected_encoding = result + break + encoding = detected_encoding if detected_encoding else "utf-8" + return encoding.lower() + class ChapterOptionsDialog(QDialog): def __init__(self, chapter_count, parent=None): @@ -183,7 +200,8 @@ class ConversionThread(QThread): if self.is_direct_text: text = self.file_name # Treat file_name as direct text input else: - with open(self.file_name, "r", encoding="utf-8") as file: + encoding = detect_encoding(self.file_name) + with open(self.file_name, "r", encoding=encoding, errors="replace") as file: text = file.read() # Clean up text using utility function @@ -455,7 +473,7 @@ class ConversionThread(QThread): chapter_srt_path = os.path.join( chapters_out_dir, f"{chapter_filename}.srt" ) - with open(chapter_srt_path, "w", encoding="utf-8") as srt_file: + with open(chapter_srt_path, "w", encoding="utf-8", errors="replace") as srt_file: for i, (start, end, text) in enumerate( chapter_subtitle_entries, 1 ): @@ -496,7 +514,7 @@ class ConversionThread(QThread): srt_path = os.path.splitext(out_path)[0] + ".srt" sf.write(out_path, audio, 24000, format=self.output_format) if self.subtitle_mode != "Disabled": - with open(srt_path, "w", encoding="utf-8") as srt_file: + with open(srt_path, "w", encoding="utf-8", errors="replace") as srt_file: for i, (start, end, text) in enumerate(subtitle_entries, 1): srt_file.write( f"{i}\n{self._srt_time(start)} --> {self._srt_time(end)}\n{text}\n\n" diff --git a/pyproject.toml b/pyproject.toml index 308fd64..03491fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,9 @@ dependencies = [ "beautifulsoup4>=4.13.4", "PyMuPDF>=1.25.5", "soundfile>=0.13.1", - "pygame>=2.6.1" + "pygame>=2.6.1", + "charset_normalizer>=3.4.1", + "chardet>=5.2.0" ] classifiers = [ @@ -52,10 +54,9 @@ abogen-cli = "abogen.main:main" exclude = [ "/.github", "/demo", - "/demo", + "/abogen/resources", "/abogen/assets/create_shortcuts.bat", "WINDOWS_INSTALL.bat", - ] [tool.hatch.build.targets.wheel]