Fixed "'utf-8' codec can't decode byte" error

CHANGELOG.md

@@ -1,5 +1 @@
-- Added `abogen-cli` command for better troubleshooting and error handling.
+- Fixed "'utf-8' codec can't decode byte" error, mentioned in #3 by @nigelp
-- Switched from setuptools to hatchling for packaging.
-- Added classifiers to the package metadata.
-- Fixed "No module named 'docopt'" and "setuptools.build_meta" import errors while using .bat installer in Windows, mentioned by @nigelp in https://github.com/denizsafak/abogen/issues/2 
-- Improvements in code and documentation.

abogen/conversion.py

@@ -2,6 +2,8 @@ import os
 import re
 import tempfile
 import time
+import chardet
+import charset_normalizer
 from PyQt5.QtCore import QThread, pyqtSignal, Qt
 from PyQt5.QtWidgets import QCheckBox, QVBoxLayout, QDialog, QLabel, QDialogButtonBox
 import soundfile as sf
@@ -12,6 +14,21 @@ from constants import PROGRAM_NAME, LANGUAGE_DESCRIPTIONS, SAMPLE_VOICE_TEXTS
 def get_sample_voice_text(lang_code):
     return SAMPLE_VOICE_TEXTS.get(lang_code, SAMPLE_VOICE_TEXTS["a"])
 
+def detect_encoding(file_path):
+    with open(file_path, "rb") as f:
+        raw_data = f.read()
+    detected_encoding = None
+    for detectors in (charset_normalizer, chardet):
+        try:
+            result = detectors.detect(raw_data)["encoding"]
+        except Exception:
+            continue
+        if result is not None:
+            detected_encoding = result
+            break
+    encoding = detected_encoding if detected_encoding else "utf-8"
+    return encoding.lower()
+
 
 class ChapterOptionsDialog(QDialog):
     def __init__(self, chapter_count, parent=None):
@@ -183,7 +200,8 @@ class ConversionThread(QThread):
             if self.is_direct_text:
                 text = self.file_name  # Treat file_name as direct text input
             else:
-                with open(self.file_name, "r", encoding="utf-8") as file:
+                encoding = detect_encoding(self.file_name)
+                with open(self.file_name, "r", encoding=encoding, errors="replace") as file:
                     text = file.read()
 
             # Clean up text using utility function
@@ -455,7 +473,7 @@ class ConversionThread(QThread):
                         chapter_srt_path = os.path.join(
                             chapters_out_dir, f"{chapter_filename}.srt"
                         )
-                        with open(chapter_srt_path, "w", encoding="utf-8") as srt_file:
+                        with open(chapter_srt_path, "w", encoding="utf-8", errors="replace") as srt_file:
                             for i, (start, end, text) in enumerate(
                                 chapter_subtitle_entries, 1
                             ):
@@ -496,7 +514,7 @@ class ConversionThread(QThread):
                 srt_path = os.path.splitext(out_path)[0] + ".srt"
                 sf.write(out_path, audio, 24000, format=self.output_format)
                 if self.subtitle_mode != "Disabled":
-                    with open(srt_path, "w", encoding="utf-8") as srt_file:
+                    with open(srt_path, "w", encoding="utf-8", errors="replace") as srt_file:
                         for i, (start, end, text) in enumerate(subtitle_entries, 1):
                             srt_file.write(
                                 f"{i}\n{self._srt_time(start)} --> {self._srt_time(end)}\n{text}\n\n"

pyproject.toml

@@ -19,7 +19,9 @@ dependencies = [
     "beautifulsoup4>=4.13.4",
     "PyMuPDF>=1.25.5",
     "soundfile>=0.13.1",
-    "pygame>=2.6.1"
+    "pygame>=2.6.1",
+    "charset_normalizer>=3.4.1",
+    "chardet>=5.2.0"
 ]
 
 classifiers = [
@@ -52,10 +54,9 @@ abogen-cli = "abogen.main:main"
 exclude = [
     "/.github",
     "/demo",
-    "/demo",
+    "/abogen/resources",
     "/abogen/assets/create_shortcuts.bat",
     "WINDOWS_INSTALL.bat",
-    
 ]
 
 [tool.hatch.build.targets.wheel]