Merge pull request #5 from jborza/main

Voice mixing - creating voice formulas

abogen/conversion.py

@@ -9,6 +9,7 @@ from PyQt5.QtWidgets import QCheckBox, QVBoxLayout, QDialog, QLabel, QDialogButt
 import soundfile as sf
 from utils import clean_text
 from constants import PROGRAM_NAME, LANGUAGE_DESCRIPTIONS, SAMPLE_VOICE_TEXTS
+from voice_formulas import get_new_voice
 
 
 def get_sample_voice_text(lang_code):
@@ -283,6 +284,14 @@ class ConversionThread(QThread):
                 parent_dir = os.path.dirname(base_path)
             else:
                 parent_dir = self.output_folder or os.getcwd()
+            # Ensure the output folder exists, error if it doesn't
+            if not os.path.exists(parent_dir):
+                self.log_updated.emit(
+                    (
+                        f"Output folder does not exist: {parent_dir}",
+                        "red",
+                    )
+                )
             # Find a unique suffix for both folder and merged file, always
             counter = 1
             while True:
@@ -342,9 +351,16 @@ class ConversionThread(QThread):
 
                 # Set split_pattern to \n+ which will split on one or more newlines
                 split_pattern = r"\n+"
+                
+                # Check if the voice is a formula and load it if necessary
+                if '*' in self.voice:
+                    loaded_voice = get_new_voice(tts, self.voice, self.use_gpu)
+                else:
+                    loaded_voice = self.voice
+                
                 for result in tts(
                     chapter_text,
-                    voice=self.voice,
+                    voice=loaded_voice,
                     speed=self.speed,
                     split_pattern=split_pattern,
                 ):

abogen/gui.py

@@ -3,6 +3,7 @@ import time
 import tempfile
 import platform
 import base64
+import re
 from PyQt5.QtWidgets import (
     QApplication,
     QWidget,
@@ -70,6 +71,7 @@ from constants import (
     SUPPORTED_LANGUAGES_FOR_SUBTITLE_GENERATION,
 )
 from threading import Thread
+from voice_formula_gui import VoiceFormulaDialog
 
 # Import ctypes for Windows-specific taskbar icon
 if platform.system() == "Windows":
@@ -458,6 +460,7 @@ class abogen(QWidget):
         self.last_output_path = None
         self.selected_voice = self.config.get("selected_voice", "af_heart")
         self.selected_lang = self.selected_voice[0]
+        self.mixed_voice_state = None  # Store the mixed voice state
         self.is_converting = False
         self.subtitle_mode = self.config.get("subtitle_mode", "Sentence")
         self.max_subtitle_words = self.config.get(
@@ -569,6 +572,15 @@ class abogen(QWidget):
         )
         voice_row.addWidget(self.voice_combo)
         
+        # Voice formula button
+        self.btn_voice_formula_mixer = QPushButton(self)
+        self.btn_voice_formula_mixer.setText("🛠") # TODO add voice formula icon
+        self.btn_voice_formula_mixer.setToolTip("Mix and match voices")
+        self.btn_voice_formula_mixer.setFixedSize(40, 36)
+        self.btn_voice_formula_mixer.setStyleSheet("QPushButton { padding: 6px 12px; }")
+        self.btn_voice_formula_mixer.clicked.connect(self.show_voice_formula_dialog)
+        voice_row.addWidget(self.btn_voice_formula_mixer)
+
         # Play/Stop icons
         def make_icon(color, shape):
             pix = QPixmap(20, 20)
@@ -1039,11 +1051,21 @@ class abogen(QWidget):
                 else self.subtitle_mode
             )
             
+            # if voice formula is not None, use the selected voice
+            if self.mixed_voice_state:
+                formula_components = [f"{weight} * {name}" for name, weight in self.mixed_voice_state]
+                voice_formula = " + ".join(filter(None, formula_components))
+            else:
+                voice_formula = self.selected_voice
+            # selected language - use the first voice of the mix
+            match = re.search(r'\b([a-z])', voice_formula)
+            selected_lang = match.group(1)
+
             self.conversion_thread = ConversionThread(
                 self.selected_file,
-                self.selected_lang,
+                selected_lang,
                 speed,
-                self.selected_voice,
+                voice_formula,
                 self.save_option,
                 self.selected_output_folder,
                 subtitle_mode=actual_subtitle_mode,
@@ -1657,6 +1679,16 @@ class abogen(QWidget):
         self.config["check_updates"] = checked
         save_config(self.config)
         
+    def show_voice_formula_dialog(self):
+        # get the current voice mix
+        if self.mixed_voice_state is None:
+            # if no voice mix is set, use the selected voice
+            self.mixed_voice_state = [(self.selected_voice, 1.0)]
+        
+        dialog = VoiceFormulaDialog(self, initial_state=self.mixed_voice_state)
+        if dialog.exec_() == QDialog.Accepted:
+            self.mixed_voice_state = dialog.get_selected_voices()
+       
     def show_about_dialog(self):
         """Show an About dialog with program information including GitHub link."""
         # Get application icon for dialog

abogen/voice_formula_gui.py

@@ -0,0 +1,189 @@
+from PyQt5.QtWidgets import (
+    QDialog,
+    QVBoxLayout,
+    QCheckBox,
+    QLabel,
+    QHBoxLayout,
+    QDoubleSpinBox,
+    QSlider,
+    QScrollArea,
+    QWidget,
+    QPushButton,
+    QSizePolicy
+)
+from PyQt5.QtCore import (
+    Qt,
+    QTimer
+)
+from constants import (
+    VOICES_INTERNAL, 
+    FLAGS
+)
+
+# Constants for voice names and flags
+VOICE_MIXER_WIDTH = 160
+FEMALE = "👩‍🦰"
+MALE = "👨"
+
+class VoiceMixer(QWidget):
+    def __init__(self, voice_name, language_icon, initial_status=False, initial_weight=0.0):
+        super().__init__()
+
+        self.voice_name = voice_name
+
+        # Set fixed width for this widget
+        self.setFixedWidth(VOICE_MIXER_WIDTH)
+        # TODO Set CSS for rounded corners
+        # self.setObjectName("VoiceMixer")
+        # self.setStyleSheet(self.ROUNDED_CSS)
+
+        # Main Layout
+        layout = QVBoxLayout()
+
+        # Checkbox at the top
+        self.checkbox = QCheckBox()
+        self.checkbox.setChecked(initial_status)
+        self.checkbox.stateChanged.connect(self.toggle_inputs)
+        layout.addWidget(self.checkbox, alignment=Qt.AlignCenter)
+
+        voice_gender = self.get_voice_gender()
+        name = voice_name[3:].capitalize()
+        name_label = QLabel(f"{language_icon} {voice_gender} {name}")
+        name_layout = QHBoxLayout()
+        name_layout.addWidget(name_label)
+        name_layout.setAlignment(name_label, Qt.AlignCenter)
+        layout.addLayout(name_layout)
+
+        # Input and Slider
+        self.spin_box = QDoubleSpinBox()
+        self.spin_box.setRange(0, 1)
+        self.spin_box.setSingleStep(0.01)
+        self.spin_box.setDecimals(2)
+        self.spin_box.setValue(initial_weight)
+
+        self.slider = QSlider(Qt.Vertical)  # Set slider orientation to vertical
+        self.slider.setRange(0, 100)
+        self.slider.setValue(int(initial_weight * 100))
+        self.slider.setFixedHeight(180)
+        self.slider.valueChanged.connect(
+            lambda val: self.spin_box.setValue(val / 100)
+        )
+        self.spin_box.valueChanged.connect(
+            lambda val: self.slider.setValue(int(val * 100))
+        )
+
+        slider_layout = QVBoxLayout()
+        slider_layout.addWidget(self.spin_box)
+        slider_layout.addWidget(QLabel("1", alignment=Qt.AlignCenter))
+        slider_layout.addWidget(self.slider, alignment=Qt.AlignCenter)
+        slider_layout.addWidget(QLabel("0", alignment=Qt.AlignCenter))
+
+        layout.addLayout(slider_layout)
+        self.setLayout(layout)
+
+        # Disable inputs initially if the checkbox is unchecked
+        self.toggle_inputs()
+        
+    def toggle_inputs(self):
+        """Enable or disable inputs based on the checkbox state."""
+        is_enabled = self.checkbox.isChecked()
+        self.spin_box.setEnabled(is_enabled)
+        self.slider.setEnabled(is_enabled)
+
+    def get_voice_gender(self):
+        if self.voice_name in VOICES_INTERNAL:
+            gender = self.voice_name[1]
+            return FEMALE if gender == "f" else MALE
+        return ""
+
+    def get_formula_component(self):
+        if self.checkbox.isChecked():
+            weight = self.spin_box.value()
+            return f"{weight:.3f} * {self.voice_name.lower().replace(' ', '_')}"
+        return ""
+
+
+    def get_voice_weight(self):
+        """Return the voice and its weight if selected."""
+        if self.checkbox.isChecked():
+            return self.voice_name, self.spin_box.value()
+        return None
+
+class VoiceFormulaDialog(QDialog):
+    def __init__(self, parent=None, initial_state=None):
+        super().__init__(parent)
+
+        self.setWindowTitle("Voice Mixer")
+        self.setFixedSize(1000, 500)
+        self.voice_mixers = []
+
+        # Main Layout
+        main_layout = QVBoxLayout()
+
+        # Header Label
+        header_label = QLabel("Select Voices For the Mix and Adjust Weights")
+        main_layout.addWidget(header_label)
+
+        # Scroll Area for Voice Panels
+        self.scroll_area = QScrollArea()
+        self.scroll_area.setWidgetResizable(True)
+        self.scroll_area.setFixedSize(1000, 400)  # Keep scroll area height within 500
+        self.voice_list_widget = QWidget()
+        self.voice_list_layout = QHBoxLayout()  
+        self.voice_list_widget.setLayout(self.voice_list_layout)
+        self.voice_list_widget.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
+        self.scroll_area.setWidget(self.voice_list_widget)
+        main_layout.addWidget(self.scroll_area)
+
+        # Add buttons
+        button_layout = QHBoxLayout()
+        ok_button = QPushButton("OK")
+        cancel_button = QPushButton("Cancel")
+
+        # Connect buttons to appropriate slots
+        ok_button.clicked.connect(self.accept)      
+        cancel_button.clicked.connect(self.reject)  
+
+        button_layout.addStretch()  # Push buttons to the right
+        button_layout.addWidget(ok_button)
+        button_layout.addWidget(cancel_button)
+
+        main_layout.addLayout(button_layout)
+
+        self.setLayout(main_layout)
+        
+        self.add_voices(initial_state)
+
+    def add_voices(self, initial_state):
+        """Add voice mixers to the dialog based on the initial state and scroll to first enabled one."""
+        first_enabled_voice = None 
+
+        for voice in VOICES_INTERNAL:
+            flag = FLAGS.get(voice[0], "")
+            matching_voice = next((item for item in initial_state if item[0] == voice), None)
+            initial_status = matching_voice is not None
+            initial_weight = matching_voice[1] if matching_voice else 1.0
+            voice_mixer = self.add_voice(voice, flag, initial_status, initial_weight)
+            # remember the first enabled voice
+            if initial_status and first_enabled_voice is None:
+                first_enabled_voice = voice_mixer
+        
+        if first_enabled_voice:
+            self.scroll_to_voice(first_enabled_voice)
+        
+    def add_voice(self, voice_name, language_icon, initial_status=False, initial_weight=1.0):
+        voice_mixer = VoiceMixer(voice_name, language_icon, initial_status, initial_weight)
+        self.voice_mixers.append(voice_mixer)
+        self.voice_list_layout.addWidget(voice_mixer)
+        return voice_mixer
+
+    def scroll_to_voice(self, voice_mixer):
+        """Scroll the QScrollArea to ensure the given VoiceMixer is visible."""        
+        QTimer.singleShot(0, lambda: self.scroll_area.ensureWidgetVisible(voice_mixer))
+
+    def get_selected_voices(self):
+        """Return the list of selected voices and their weights."""
+        selected_voices = [
+            mixer.get_voice_weight() for mixer in self.voice_mixers
+        ]
+        return [voice for voice in selected_voices if voice]  # Filter out None

abogen/voice_formulas.py

@@ -0,0 +1,51 @@
+import re
+from constants import VOICES_INTERNAL
+
+# Calls parsing and loads the voice to gpu or cpu
+def get_new_voice(pipeline, formula, use_gpu):
+    try:
+        weighted_voice = parse_voice_formula(pipeline, formula)            
+        device = "cuda" if use_gpu else "cpu"
+        return weighted_voice.to(device)
+    except Exception as e:
+        raise ValueError(f"Failed to create voice: {str(e)}")
+    
+# Parse the formula and get the combined voice tensor        
+def parse_voice_formula(pipeline, formula):
+    if not formula.strip():
+        raise ValueError("Empty voice formula")
+        
+    # Initialize the weighted sum
+    weighted_sum = None
+    
+    total_weight = calculate_sum_from_formula(formula)
+
+    # Split the formula into terms
+    voices = formula.split('+')
+    
+    for term in voices:
+        # Parse each term (format: "0.333 * voice_name")
+        weight, voice_name = term.strip().split('*')
+        weight = float(weight.strip())
+        # normalize the weight
+        weight /= total_weight if total_weight > 0 else 1.0
+        voice_name = voice_name.strip()
+        
+        # Get the voice tensor
+        if voice_name not in VOICES_INTERNAL:
+            raise ValueError(f"Unknown voice: {voice_name}")
+            
+        voice_tensor = pipeline.load_single_voice(voice_name)
+        
+        # Add to weighted sum
+        if weighted_sum is None:
+            weighted_sum = weight * voice_tensor
+        else:
+            weighted_sum += weight * voice_tensor
+            
+    return weighted_sum
+
+def calculate_sum_from_formula(formula):
+    weights = re.findall(r'([\d.]+) \*', formula)
+    total_sum = sum(float(weight) for weight in weights)
+    return total_sum