diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..341d28b --- /dev/null +++ b/.env.example @@ -0,0 +1,44 @@ +# Copy this file to `.env` and customize the paths to match your environment. +# Relative paths are resolved from the repository root when running locally. +# Each `*_DIR` value below points to a directory on the host. Docker Compose +# mounts it into the container at the standard path noted in the comment. + +# Host directory that stores JSON settings. Mounted to /config in Docker. +ABOGEN_SETTINGS_DIR=./config + +# Host directory for rendered audio/subtitle files. Mounted to /data/outputs +# in Docker. +ABOGEN_OUTPUT_DIR=./storage/output + +# Temporary working directory. When running in Docker, keep this inside the +# mounted data volume (or another writable host path) so non-root users can +# write to it. Only audio conversion scratch files are staged here by default; +# other library caches remain inside the container volume. For local +# (non-Docker) usage, change this to a path that makes sense on your machine or +# comment it out to fall back to the OS cache directory. Mounted to /data/cache +# in Docker. +ABOGEN_TEMP_DIR=./storage/tmp + +# UID/GID used when running the Docker container. 1000:1000 matches most Linux hosts. +# Find your current values with: +# id -u # UID +# id -g # GID +ABOGEN_UID=1000 +ABOGEN_GID=1000 + +# Network mode for the Docker container. Options: +# bridge (default) - Isolated container network, uses port mapping +# host - Container uses host's network directly, required for +# accessing LAN resources like Calibre OPDS servers +# ABOGEN_NETWORK_MODE=host + +# Optional: Seed the web UI with working defaults for the LLM-powered +# text normalization features. Leave these blank to configure everything +# from the Settings page. +ABOGEN_LLM_BASE_URL=http://localhost:11434 # Supply the server root; /v1 is added automatically. +ABOGEN_LLM_API_KEY=ollama +ABOGEN_LLM_MODEL=llama3.1:8b +ABOGEN_LLM_TIMEOUT=45 +ABOGEN_LLM_CONTEXT_MODE=sentence +# For custom prompts, keep the text on a single line or escape newlines. +#ABOGEN_LLM_PROMPT=Provide regex replacements for any apostrophes in {{sentence}} using apply_regex_replacements. diff --git a/.gitignore b/.gitignore index c99cf9f..bd660ac 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ __pycache__/ env/ venv/ .env/ +.env .venv/ test/ @@ -30,6 +31,10 @@ python_embedded/ # abogen *config.json +config/ +storage/ build/ dist/ .old/ +test_assets/ +dev_notes/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 90311a9..2e78e11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +# Unreleased + +# 1.3.0 +- Added an EPUB 3 packaging pipeline that builds media-overlay EPUBs from generated audio and chunk metadata. +- Persisted chunk timing metadata in job artifacts and exercised the exporter with automated tests. +- Added Flask-based Web UI (`abogen-web`) for Docker and headless server deployments. +- Reorganized codebase to support both PyQt6 desktop GUI and Web UI from a shared core. +- Added Supertonic TTS engine support with GPU acceleration. +- Added entity analysis and pronunciation override system for proper nouns. +- Added speaker/role assignment for multi-voice "theatrical" audiobooks. +- Added Calibre OPDS and Audiobookshelf integration. + # 1.2.5 - Added new option: `Override item settings with current selection` in the queue manager. When enabled, all items in the queue will be processed using the current global settings selected in the main GUI, overriding their individual settings. When disabled, each item will retain its own specific settings. - Fixed `Error "Could not load the Qt platform plugin "xcb"` error that occurred in some Linux distributions due to missing `libxcb-cursor0` library by conditionally loading the bundled library when the system version is unavailable, issue mentioned by @bmcgonag in #101. @@ -53,7 +65,7 @@ - Fixed `/` and `\` path display by normalizing paths. - Fixed book reprocessing issue where books were being processed every time the chapters window was opened, improving performance when reopening the same book. - Fixed taskbar icon not appearing correctly in Windows. -- Fixed “Go to folder” button not opening the chapter output directory when only separate chapters were generated. +- Fixed "Go to folder" button not opening the chapter output directory when only separate chapters were generated. - Improvements in code and documentation. # 1.1.9 @@ -182,7 +194,7 @@ - Improved invalid profile handling in the voice mixer. # v1.0.3 -- Added voice mixing, allowing multiple voices to be combined into a single “Mixed Voice”, a feature mentioned by @PulsarFTW in #1. Special thanks to @jborza for making this possible through his contributions in #5. +- Added voice mixing, allowing multiple voices to be combined into a single "Mixed Voice", a feature mentioned by @PulsarFTW in #1. Special thanks to @jborza for making this possible through his contributions in #5. - Added profile system to voice mixer, allowing users to create and manage multiple voice profiles. - Improvements in the voice mixer, mostly for organizing controls and enhancing user experience. - Added icons for flags and genders in the GUI, making it easier to identify different options. diff --git a/README.md b/README.md index 3f1a810..ddb3f79 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Abogen is a powerful text-to-speech conversion tool that makes it easy to turn e https://github.com/user-attachments/assets/094ba3df-7d66-494a-bc31-0e4b41d0b865 -> This demo was generated in just 5 seconds, producing ∼1 minute of audio with perfectly synced subtitles. To create a similar video, see [the demo guide](https://github.com/denizsafak/abogen/tree/main/demo). +> This demo was generated in just 5 seconds, producing ∼1 minute of audio with perfectly synced subtitles. To create a similar video, see [the demo guide](https://github.com/denizsafak/abogen/tree/main/demo). ## `How to install?` Abogen Compatible PyPi Python Versions @@ -166,13 +166,30 @@ pip3 install --pre torch torchvision torchaudio --index-url https://download.pyt > Special thanks to [@hg000125](https://github.com/hg000125) for his contribution in [#23](https://github.com/denizsafak/abogen/issues/23). AMD GPU support is possible thanks to his work. + +## Interfaces + +Abogen offers **two interfaces**, but currently they have different feature sets. The **Web UI** contains newer features that are still being integrated into the desktop application. + +| Command | Interface | Features | +|---------|-----------|----------| +| `abogen` | PyQt6 Desktop GUI | Stable core features | +| `abogen-web` | Flask Web UI | Core features + **Supertonic TTS**, **LLM Normalization**, **Audiobookshelf Integration** and more! | + +> **Note:** The Web UI is under active development. We are working to integrate these new features into the PyQt desktop app. until then, the Web UI provides the most feature-rich experience. + +> Special thanks to [@jeremiahsb](https://github.com/jeremiahsb) for making this possible! I was honestly surprised by his [massive contribution](https://github.com/denizsafak/abogen/pull/120) (>55,000 lines!) that brought the entire Web UI to life. + +# 🖥️ Desktop Application (PyQt) + ## `How to run?` -You can simply run this command to start Abogen: +You can simply run this command to start Abogen Desktop GUI: ```bash abogen ``` + > [!TIP] > If you installed Abogen using the Windows installer `(WINDOWS_INSTALL.bat)`, It should have created a shortcut in the same folder, or your desktop. You can run it from there. If you lost the shortcut, Abogen is located in `python_embedded/Scripts/abogen.exe`. You can run it from there directly. @@ -189,7 +206,7 @@ abogen ## `In action` -Here’s Abogen in action: in this demo, it processes ∼3,000 characters of text in just 11 seconds and turns it into 3 minutes and 28 seconds of audio, and I have a low-end **RTX 2060 Mobile laptop GPU**. Your results may vary depending on your hardware. +Here’s Abogen in action: in this demo, it processes ∼3,000 characters of text in just 11 seconds and turns it into 3 minutes and 28 seconds of audio, and I have a low-end **RTX 2060 Mobile laptop GPU**. Your results may vary depending on your hardware. ## `Configuration` @@ -262,6 +279,172 @@ Abogen will process each item in the queue automatically, saving outputs as conf > Special thanks to [@jborza](https://github.com/jborza) for adding queue mode in PR [#35](https://github.com/denizsafak/abogen/pull/35) +--- +# 🌐 Web Application (WebUI) + +## `How to run?` + +Run this command to start the Web UI: + +```bash +abogen-web +``` +Then open http://localhost:8808 and drag in your documents. Jobs run in the background worker and the browser updates automatically. + + + +## `Using the web UI` +1. Upload a document (drag & drop or use the upload button). +2. Choose voice, language, speed, subtitle style, and output format. +3. Click **Create job**. The job immediately appears in the queue. +4. Watch progress and logs update live. Download audio/subtitle assets when complete. +5. Cancel or delete jobs any time. Download logs for troubleshooting. + +Multiple jobs can run sequentially; the worker processes them in order. + +## `Container image` +You can build a lightweight container image directly from the repository root: + +```bash +docker build -t abogen . +mkdir -p ~/abogen-data/uploads ~/abogen-data/outputs +docker run --rm \ + -p 8808:8808 \ + -v ~/abogen-data:/data \ + --name abogen \ + abogen +``` + +Browse to http://localhost:8808. Uploaded source files are stored in `/data/uploads` and rendered audio/subtitles appear in `/data/outputs`. + +### Container environment variables +| Variable | Default | Purpose | +|----------|---------|---------| +| `ABOGEN_HOST` | `0.0.0.0` | Bind address for the Flask server | +| `ABOGEN_PORT` | `8808` | HTTP port | +| `ABOGEN_DEBUG` | `false` | Enable Flask debug mode | +| `ABOGEN_UPLOAD_ROOT` | `/data/uploads` | Directory where uploaded files are stored | +| `ABOGEN_OUTPUT_ROOT` | `/data/outputs` | Directory for generated audio and subtitles (legacy alias of `ABOGEN_OUTPUT_DIR`) | +| `ABOGEN_OUTPUT_DIR` | `/data/outputs` | Container path for rendered audio/subtitles | +| `ABOGEN_SETTINGS_DIR` | `/config` | Container path for JSON settings/configuration | +| `ABOGEN_TEMP_DIR` | `/data/cache` (Docker) or platform cache dir | Container path for temporary audio working files | +| `ABOGEN_UID` | `1000` | UID that the container should run as (matches host user) | +| `ABOGEN_GID` | `1000` | GID that the container should run as (matches host group) | +| `ABOGEN_LLM_BASE_URL` | `""` | OpenAI-compatible endpoint used to seed the Settings → LLM panel | +| `ABOGEN_LLM_API_KEY` | `""` | API key passed to the endpoint above | +| `ABOGEN_LLM_MODEL` | `""` | Default model selected when you refresh the model list | +| `ABOGEN_LLM_TIMEOUT` | `30` | Timeout (seconds) for server-side LLM requests | +| `ABOGEN_LLM_CONTEXT_MODE` | `sentence` | Default prompt context window (`sentence`, `paragraph`, `document`) | +| `ABOGEN_LLM_PROMPT` | `""` | Custom normalization prompt template seeded into the UI | + +Set any of these with `-e VAR=value` when starting the container. + +To discover your local UID/GID for matching file permissions inside the container, run: + +```bash +id -u +id -g +``` + +Use those values to populate `ABOGEN_UID` / `ABOGEN_GID` in your `.env` file. + +When running via Docker Compose, set `ABOGEN_SETTINGS_DIR`, +`ABOGEN_OUTPUT_DIR`, and `ABOGEN_TEMP_DIR` in your `.env` file to the host +directories you want mounted into the container. Compose maps them to +`/config`, `/data/outputs`, and `/data/cache` respectively while exporting +those in-container paths to the application. Non-audio caches (e.g., Hugging +Face downloads) stick to the container's internal cache under `/tmp/abogen-home/.cache` +by default, so only conversion scratch data touches the mounted `ABOGEN_TEMP_DIR`. +Ensure each host directory exists and is writable by the UID/GID you configure +before starting the stack. + +### Docker Compose (GPU by default) +The repo includes `docker-compose.yaml`, which targets GPU hosts out of the box. Install the NVIDIA Container Toolkit and run: + +```bash +docker compose up -d --build +``` + +Key build/runtime knobs: + +- `TORCH_VERSION` – pin a specific PyTorch release that matches your driver (leave blank for the latest on the configured index). +- `TORCH_INDEX_URL` – swap out the PyTorch download index when targeting a different CUDA build. +- `ABOGEN_DATA` – host path that stores uploads/outputs (defaults to `./data`). + +CPU-only deployment: comment out the `deploy.resources.reservations.devices` block (and the optional `runtime: nvidia` line) inside the compose file. Compose will then run without requesting a GPU. If you prefer the classic CLI: + +```bash +docker build -f abogen/Dockerfile -t abogen-gpu . +docker run --rm \ + --gpus all \ + -p 8808:8808 \ + -v ~/abogen-data:/data \ + abogen-gpu +``` + +## `LLM-assisted text normalization` +Abogen can hand tricky apostrophes and contractions to an OpenAI-compatible large language model. Configure it from **Settings → LLM**: + +1. Enter the base URL for your endpoint (Ollama, OpenAI proxy, etc.) and an API key if required. Use the server root (for Ollama: `http://localhost:11434`)—Abogen appends `/v1/...` automatically, but it also accepts inputs that already end in `/v1`. +2. Click **Refresh models** to load the catalog, pick a default model, and adjust the timeout or prompt template. +3. Use the preview box to test the prompt, then save the settings. The Normalization panel can synthesize a short audio preview with the current configuration. + +When you are running inside Docker or a CI pipeline, seed the form automatically with `ABOGEN_LLM_*` variables in your `.env` file. The `.env.example` file includes sample values for a local Ollama server. + +## `Audiobookshelf integration` +Abogen can push finished audiobooks directly into Audiobookshelf. Configure this under **Settings → Integrations → Audiobookshelf** by providing: + +- **Base URL** – the HTTPS origin (and optional path prefix) where your Audiobookshelf server is reachable, for example `https://abs.example.com` or `https://media.example.com/abs`. Do **not** append `/api`. +- **Library ID** – the identifier of the target Audiobookshelf library (copy it from the library’s settings page in ABS). +- **Folder (name or ID)** – the destination folder inside that library. Enter the folder name exactly as it appears in Audiobookshelf (Abogen resolves it to the correct ID automatically), paste the raw `folderId`, or click **Browse folders** to fetch the available folders and populate the field. +- **API token** – a personal access token generated in Audiobookshelf under *Account → API tokens*. + +You can enable automatic uploads for future jobs or trigger individual uploads from the queue once the connection succeeds. + +### Reverse proxy checklist (Nginx Proxy Manager) +When Audiobookshelf sits behind Nginx Proxy Manager (NPM), make sure the API paths and headers reach the backend untouched: + +1. Create a **Proxy Host** that points to your ABS container or host (default forward port `13378`). +2. Under the **SSL** tab, enable your certificate and tick **Force SSL** if you want HTTPS only. +3. In the **Advanced** tab, append the snippet below so bearer tokens, client IPs, and large uploads survive the proxy hop: + ```nginx + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Port $server_port; + proxy_set_header Authorization $http_authorization; + client_max_body_size 5g; + proxy_read_timeout 300s; + proxy_connect_timeout 300s; + ``` +4. Disable **Block Common Exploits** (it strips Authorization headers in some NPM builds). +5. Enable **Websockets Support** on the main proxy screen (Audiobookshelf uses it for the web UI, and it keeps the reverse proxy configuration consistent). +6. If you publish Audiobookshelf under a path prefix (for example `/abs`), add a **Custom Location** with `Location: /abs/` and set the **Forward Path** to `/`. That rewrite strips the `/abs` prefix before traffic reaches Audiobookshelf so `/abs/api/...` on the internet becomes `/api/...` on the backend. Use the same prefixed URL in Abogen’s “Base URL” field. + +After saving the proxy host, test the API from the machine running Abogen: + +```bash +curl -i "https://abs.example.com/api/libraries" \ + -H "Authorization: Bearer YOUR_API_TOKEN" +``` + +If you still receive `Cannot GET /api/...`, the proxy is rewriting paths. Double-check the **Custom Locations** table (the `Forward Path` column should be empty for `/abs/`) and review the NPM access/error logs while issuing the curl request to confirm the backend sees the full `/api/libraries` URL. + +A JSON response confirming the libraries list means the proxy is routing API calls correctly. You can then use **Browse folders** to confirm the library contents, run **Test connection** in Abogen’s settings (it verifies the library and resolves the folder), and use the “Send to Audiobookshelf” button on completed jobs. + +## `JSON endpoints` +Need machine-readable status updates? The dashboard calls a small set of helper endpoints you can reuse: +- `GET /api/jobs/` returns job metadata, progress, and log lines in JSON. +- `GET /partials/jobs` renders the live job list as HTML (htmx uses this for polling). +- `GET /partials/jobs//logs` renders just the log window. + +More automation hooks are planned; contributions are very welcome if you need additional routes. + +--- +# Core Features (Available in Both) + ## `About Chapter Markers` When you process ePUB, PDF or markdown files, Abogen converts them into text files stored in your cache directory. When you click "Edit," you're actually modifying these converted text files. In these text files, you'll notice tags that look like this: @@ -335,6 +518,9 @@ For a complete list of supported languages and voices, refer to Kokoro's [VOICES > See [How to fix Japanese audio not working?](#japanese-audio-not-working) +--- +# Guides & Troubleshooting + ## `MPV Config` I highly recommend using [MPV](https://mpv.io/installation/) to play your audio files, as it supports displaying subtitles even without a video track. Here's my `mpv.conf`: ``` @@ -353,54 +539,6 @@ audio-samplerate=48000 volume-max=200 ``` -## `Docker Guide` -If you want to run Abogen in a Docker container: -1) [Download the repository](https://github.com/denizsafak/abogen/archive/refs/heads/main.zip) and extract, or clone it using git. -2) Go to `abogen` folder. You should see `Dockerfile` there. -3) Open your terminal in that directory and run the following commands: - -```bash -# Build the Docker image: -docker build --progress plain -t abogen . - -# Note that building the image may take a while. -# After building is complete, run the Docker container: - -# Windows -docker run --name abogen -v %cd%:/shared -p 5800:5800 -p 5900:5900 --gpus all abogen - -# Linux -docker run --name abogen -v $(pwd):/shared -p 5800:5800 -p 5900:5900 --gpus all abogen - -# MacOS -docker run --name abogen -v $(pwd):/shared -p 5800:5800 -p 5900:5900 abogen - -# We expose port 5800 for use by a web browser, 5900 if you want to connect with a VNC client. -``` - -Abogen launches automatically inside the container. -- You can access it via a web browser at [http://localhost:5800](http://localhost:5800) or connect to it using a VNC client at `localhost:5900`. -- You can use `/shared` directory to share files between your host and the container. -- For later use, start it with `docker start abogen` and stop it with `docker stop abogen`. -- Pass in `-e WEB_AUDIO="1"` for `docker run` to enable audio. - -Known issues: -- Audio preview is not working inside container (ALSA error) if using a VNC client. -- `Open cache directory` and `Open configuration directory` options in settings not working. (Tried pcmanfm, did not work with Abogen). - -> Special thanks to [@geo38](https://www.reddit.com/user/geo38/) from Reddit, who provided the Dockerfile and instructions in [this comment](https://www.reddit.com/r/selfhosted/comments/1k8x1yo/comment/mpe0bz8/). - -## `🌐 Web Application` - -A web-based version of Abogen has been developed by [@jeremiahsb](https://github.com/jeremiahsb). - -**Access the repository here:** [jeremiahsb/abogen](https://github.com/jeremiahsb/abogen) - -> [!NOTE] -> I intend to merge this implementation into the main repository in the future once existing conflicts are resolved. Until then, please be aware that the web version is maintained independently and may not always be in sync with the latest updates in this repository. - -> Special thanks to [@jeremiahsb](https://github.com/jeremiahsb) for implementing the web app! - ## `Similar Projects` Abogen is a standalone project, but it is inspired by and shares some similarities with other projects. Here are a few: - [audiblez](https://github.com/santinic/audiblez): Generate audiobooks from e-books. **(Has CLI and GUI support)** @@ -570,6 +708,7 @@ abogen # Opens the GUI Feel free to explore the code and make any changes you like. ## `Credits` +- Web UI implementation by [@jeremiahsb](https://github.com/jeremiahsb) - Abogen uses [Kokoro](https://github.com/hexgrad/kokoro) for its high-quality, natural-sounding text-to-speech synthesis. Huge thanks to the Kokoro team for making this possible. - Thanks to the [spaCy](https://spacy.io/) project for its sentence-segmentation tools, which help Abogen produce cleaner, more natural sentence segmentation. - Thanks to [@wojiushixiaobai](https://github.com/wojiushixiaobai) for [Embedded Python](https://github.com/wojiushixiaobai/Python-Embed-Win64) packages. These modified packages include pip pre-installed, enabling Abogen to function as a standalone application without requiring users to separately install Python in Windows. diff --git a/WINDOWS_INSTALL.bat b/WINDOWS_INSTALL.bat index 81a1f02..d414e87 100644 --- a/WINDOWS_INSTALL.bat +++ b/WINDOWS_INSTALL.bat @@ -310,7 +310,7 @@ for /f %%i in ('%PYTHON_CONSOLE_PATH% -c "from abogen.is_nvidia import check; pr echo. echo Checking CUDA availability... if /I "%IS_NVIDIA%"=="true" ( - for /f %%i in ('%PYTHON_CONSOLE_PATH% -c "from torch.cuda import is_available; print(is_available())"') do set cuda_available=%%i + for /f %%i in ('%PYTHON_CONSOLE_PATH% %PROJECTFOLDER%\check_cuda.py') do set cuda_available=%%i if "%cuda_available%"=="False" ( echo "Installing PyTorch with CUDA (12.8) support..." diff --git a/abogen/Dockerfile b/abogen/Dockerfile deleted file mode 100644 index e24a50b..0000000 --- a/abogen/Dockerfile +++ /dev/null @@ -1,44 +0,0 @@ -# Special thanks to @geo38 from Reddit, who provided this Dockerfile: -# https://www.reddit.com/r/selfhosted/comments/1k8x1yo/comment/mpe0bz8/ - -# Use a docker base image that runs a window manager that can be viewed -# outside the image with a web browser or VNC client. -# https://github.com/jlesage/docker-baseimage-gui -FROM jlesage/baseimage-gui:debian-12-v4 - -# Load stuff needed by abogen -RUN apt-get update \ - && apt-get install -y \ - python3 \ - python3-venv \ - python3-pip \ - python3-pyqt6 \ - espeak-ng \ - libxcb-cursor0 \ - libgl1 \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -# The base image will run /startapp.sh on launch. -# -# The base image runs that script as user 'app' uid=1000. That user -# does not exist in the base image but is created at run time. -# -# We need to install abogen in python venv (requirement of newer python3). -# -# The python venv has to be writable by the 'app' user as abogen dynamically -# installs python packages, so create the venv as that user -# -# We intend to share the /shared directory with the host using a bind volume -# in order to access any source files and the created files. - -RUN echo '#!/bin/bash\nsource /app/venv/bin/activate\nexec abogen' > /startapp.sh \ - && chmod 555 /startapp.sh \ - && mkdir /app /shared \ - && chown 1000:1000 /app /shared \ - && chmod 755 /app /shared -USER 1000:1000 -RUN python3 -m venv /app/venv -RUN /bin/bash -c "source /app/venv/bin/activate && pip install abogen" -# Change back to user ROOT as the startup scripts inside base image needs it -USER root diff --git a/abogen/VERSION b/abogen/VERSION index 3a1f10e..589268e 100644 --- a/abogen/VERSION +++ b/abogen/VERSION @@ -1 +1 @@ -1.2.5 \ No newline at end of file +1.3.0 \ No newline at end of file diff --git a/abogen/book_parser.py b/abogen/book_parser.py new file mode 100644 index 0000000..a5da16a --- /dev/null +++ b/abogen/book_parser.py @@ -0,0 +1,1066 @@ +import os +import re +import logging +import textwrap +import urllib.parse +from abc import ABC, abstractmethod + +import ebooklib +from ebooklib import epub +from bs4 import BeautifulSoup, NavigableString +import fitz # PyMuPDF +import markdown + +from abogen.utils import detect_encoding +from abogen.subtitle_utils import clean_text, calculate_text_length + +# Pre-compile frequently used regex patterns +_BRACKETED_NUMBERS_PATTERN = re.compile(r"\[\s*\d+\s*\]") +_STANDALONE_PAGE_NUMBERS_PATTERN = re.compile(r"^\s*\d+\s*$", re.MULTILINE) +_PAGE_NUMBERS_AT_END_PATTERN = re.compile(r"\s+\d+\s*$", re.MULTILINE) +_PAGE_NUMBERS_WITH_DASH_PATTERN = re.compile( + r"\s+[-–—]\s*\d+\s*[-–—]?\s*$", re.MULTILINE +) + + +class BaseBookParser(ABC): + """ + Abstract base class for parsing different book formats. + """ + + def __init__(self, book_path): + self.book_path = os.path.normpath(os.path.abspath(book_path)) + self.content_texts = {} + self.content_lengths = {} + self.book_metadata = {} + # Unified structure for navigation: list of dicts + # { 'title': str, 'src': str, 'children': [], 'has_content': bool } + self.processed_nav_structure = [] + self.load() + + @abstractmethod + def load(self): + """Load the book file.""" + pass + + def close(self): + """Close any open file handles.""" + pass + + def __enter__(self): + # Already loaded in __init__, or lazily. + # Just ensure we have resources if needed, or do nothing. + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + @abstractmethod + def process_content(self, replace_single_newlines=True): + """Process the book content to extract text and structure.""" + pass + + @property + @abstractmethod + def file_type(self): + """Return the type of the file (pdf, epub, markdown).""" + pass + + def get_chapters(self): + """Return a list of chapter IDs and Names.""" + chapters = [] + if self.processed_nav_structure: + + def flatten_nav(nodes): + for node in nodes: + if node.get("has_content"): + chapters.append((node["src"], node["title"])) + if node.get("children"): + flatten_nav(node["children"]) + + flatten_nav(self.processed_nav_structure) + else: + # Fallback for simple content without nav structure + for ch_id, content in self.content_texts.items(): + # This could be improved, but serves as a generic fallback + chapters.append((ch_id, ch_id)) + return chapters + + def get_formatted_text(self): + """ + Returns the full text of the book formatted with chapter markers. + """ + chapters = self.get_chapters() + full_text = [] + + for chapter_id, chapter_name in chapters: + text = self.content_texts.get(chapter_id, "") + if text: + full_text.append(f"\n<>\n") + full_text.append(text) + + return "\n".join(full_text) + + def get_metadata(self): + """Return extracted metadata.""" + return self.book_metadata + + +class PdfParser(BaseBookParser): + def __init__(self, book_path): + self.pdf_doc = None + super().__init__(book_path) + + @property + def file_type(self): + return "pdf" + + def load(self): + try: + self.pdf_doc = fitz.open(self.book_path) + except Exception as e: + logging.error(f"Error loading PDF {self.book_path}: {e}") + raise + + def close(self): + if self.pdf_doc: + self.pdf_doc.close() + self.pdf_doc = None + + def _extract_book_metadata(self): + # PDF metadata extraction can be added here if needed + # For now, base class metadata is empty dict + pass + + def process_content(self, replace_single_newlines=True): + if not self.pdf_doc: + self.load() + + # 1. Extract text from all pages first + for page_num in range(len(self.pdf_doc)): + text = clean_text(self.pdf_doc[page_num].get_text()) + + # Clean up common PDF artifacts: + text = _BRACKETED_NUMBERS_PATTERN.sub("", text) + text = _STANDALONE_PAGE_NUMBERS_PATTERN.sub("", text) + text = _PAGE_NUMBERS_AT_END_PATTERN.sub("", text) + text = _PAGE_NUMBERS_WITH_DASH_PATTERN.sub("", text) + + page_id = f"page_{page_num + 1}" + self.content_texts[page_id] = text + self.content_lengths[page_id] = calculate_text_length(text) + + # 2. Build Navigation Structure + toc = self.pdf_doc.get_toc() + + if not toc: + # Fallback: Flat list of pages if no TOC + self.processed_nav_structure = [] + pages_node = { + "title": "Pages", + "src": None, + "children": [], + "has_content": False + } + # Add all pages as children + for page_num in range(len(self.pdf_doc)): + page_id = f"page_{page_num + 1}" + title = self._get_page_title(page_num, self.content_texts.get(page_id, "")) + pages_node["children"].append({ + "title": title, + "src": page_id, + "children": [], + "has_content": True + }) + self.processed_nav_structure.append(pages_node) + else: + self.processed_nav_structure = self._build_structure_from_toc(toc) + + return self.content_texts, self.content_lengths + + def _get_page_title(self, page_num, text): + title = f"Page {page_num + 1}" + if text: + first_line = text.split("\n", 1)[0].strip() + if first_line and len(first_line) < 100: + title += f" - {first_line}" + return title + + def _build_structure_from_toc(self, toc): + # 1. Flatten TOC to easier list (page_num, title, level) + # fitz TOC is [[lvl, title, page, dest], ...] + + bookmarks = [] + for entry in toc: + lvl, title, page = entry[:3] + if isinstance(page, int): + page_idx = page - 1 + else: + # Handle potential complex destinations if necessary, but usually simple int + # PyMuPDF docs say int. + page_idx = -1 + + if page_idx >= 0: + bookmarks.append({"level": lvl, "title": title, "page": page_idx}) + + + root_children = [] + stack = [] # Stack of (level, list_to_append_to) + stack.append((0, root_children)) + + # Step 1: Build the Skeleton Tree from TOC + # And keep a flat list of these nodes to associate with pages. + + processed_nodes = [] # List of (page_idx, node_dict) + + for entry in bookmarks: + node = { + "title": entry["title"], + "src": f"page_{entry['page'] + 1}", + "children": [], + "has_content": True + } + + # Find parent + level = entry["level"] + + # Adjust stack + while stack and stack[-1][0] >= level: + stack.pop() + + parent_list = stack[-1][1] + parent_list.append(node) + + stack.append((level, node["children"])) + processed_nodes.append((entry["page"], node)) + + # Step 3: Add gap pages. + # Sort processed_nodes by page index to find ranges. + sorted_bookmarks = sorted(processed_nodes, key=lambda x: x[0]) + + # Set of pages that are "bookmarks" + bookmarked_pages = set(p for p, n in sorted_bookmarks) + + current_node = None + # We need a way to look up bookmarks starting at p + bookmarks_by_page = {} + for p, node in processed_nodes: + if p not in bookmarks_by_page: + bookmarks_by_page[p] = [] + bookmarks_by_page[p].append(node) + + + # Let's iterate. + for page_num in range(len(self.pdf_doc)): + page_id = f"page_{page_num + 1}" + + # Check if this page STARTS bookmarks + if page_num in bookmarks_by_page: + + starts = bookmarks_by_page[page_num] + current_node = starts[-1] + + continue + + # If page is NOT a bookmark, it's a "gap page". + # Add as child to current_node + title = self._get_page_title(page_num, self.content_texts.get(page_id, "")) + page_node = { + "title": title, + "src": page_id, + "children": [], + "has_content": True + } + + if current_node: + current_node["children"].append(page_node) + else: + # No preceding bookmark. Add to root. + root_children.append(page_node) + + return root_children + + +class MarkdownParser(BaseBookParser): + def __init__(self, book_path): + self.markdown_text = None + super().__init__(book_path) + + @property + def file_type(self): + return "markdown" + + def load(self): + try: + encoding = detect_encoding(self.book_path) + with open(self.book_path, "r", encoding=encoding, errors="replace") as f: + self.markdown_text = f.read() + except Exception as e: + logging.error(f"Error reading markdown file: {e}") + self.markdown_text = "" + + def process_content(self, replace_single_newlines=True): + if self.markdown_text is None: + self.load() + + self._process_markdown_content() + return self.content_texts, self.content_lengths + + def _convert_markdown_toc_to_nav(self, toc_tokens): + nav_nodes = [] + for token in toc_tokens: + node = { + "title": token["name"], + "src": token["id"], + "children": self._convert_markdown_toc_to_nav( + token.get("children", []) + ), + "has_content": True, + } + nav_nodes.append(node) + return nav_nodes + + def _process_markdown_content(self): + if not self.markdown_text: + return + + original_text = textwrap.dedent(self.markdown_text) + md = markdown.Markdown(extensions=["toc", "fenced_code"]) + html = md.convert(original_text) + markdown_toc = md.toc_tokens + + # Convert markdown TOC tokens to our unified navigation structure + self.processed_nav_structure = self._convert_markdown_toc_to_nav(markdown_toc) + + cleaned_full_text = clean_text(original_text) + + # If no TOC found, treat as single chapter + if not self.processed_nav_structure: + chapter_id = "markdown_content" + self.content_texts[chapter_id] = cleaned_full_text + self.content_lengths[chapter_id] = calculate_text_length(cleaned_full_text) + return + + soup = BeautifulSoup(html, "html.parser") + + all_headers = [] + + def flatten_nav_internal(nodes): + for node in nodes: + all_headers.append(node) + if node.get("children"): + flatten_nav_internal(node["children"]) + + flatten_nav_internal(self.processed_nav_structure) + + header_positions = [] + for node in all_headers: + header_id = node["src"] + id_pattern = f'id="{header_id}"' + pos = html.find(id_pattern) + if pos != -1: + tag_start = html.rfind("<", 0, pos) + header_positions.append( + {"id": header_id, "start": tag_start, "name": node["title"]} + ) + header_positions.sort(key=lambda x: x["start"]) + + for i, header_pos in enumerate(header_positions): + header_id = header_pos["id"] + header_name = header_pos["name"] + content_start = header_pos["start"] + + content_end = ( + header_positions[i + 1]["start"] + if i + 1 < len(header_positions) + else len(html) + ) + section_html = html[content_start:content_end] + section_soup = BeautifulSoup(section_html, "html.parser") + + header_tag = section_soup.find(attrs={"id": header_id}) + if header_tag: + header_tag.decompose() + + section_text = clean_text(section_soup.get_text()).strip() + chapter_id = header_id + if section_text: + full_content = f"{header_name}\n\n{section_text}" + self.content_texts[chapter_id] = full_content + self.content_lengths[chapter_id] = calculate_text_length(full_content) + else: + self.content_texts[chapter_id] = header_name + self.content_lengths[chapter_id] = calculate_text_length(header_name) + + def get_chapters(self): + chapters = super().get_chapters() + if not chapters and "markdown_content" in self.content_texts: + chapters.append(("markdown_content", "Content")) + return chapters + + +class EpubParser(BaseBookParser): + def __init__(self, book_path): + self.book = None + self.doc_content = {} + super().__init__(book_path) + + @property + def file_type(self): + return "epub" + + def load(self): + try: + self.book = epub.read_epub(self.book_path) + except KeyError as e: + # TODO: should we just patch the ebooklib pre-emptively to avoid the need to catch this exception? + logging.warning(f"EPUB missing referenced file: {e}. Attempting to patch.") + # Patch ebooklib to skip missing files + import types + from ebooklib import epub as _epub_module + + reader_class = _epub_module.EpubReader + orig_read_file = reader_class.read_file + + def safe_read_file(self, name): + try: + return orig_read_file(self, name) + except KeyError: + logging.warning( + f"Missing file in EPUB: {name}. Returning empty bytes." + ) + return b"" + + reader_class.read_file = safe_read_file + try: + self.book = epub.read_epub(self.book_path) + finally: + reader_class.read_file = orig_read_file + + def process_content(self, replace_single_newlines=True): + if not self.book: + self.load() + + self.book_metadata = self._extract_book_metadata() + try: + nav_item, nav_type = self._identify_nav_item() + self._execute_nav_parsing_logic(nav_item, nav_type) + except Exception as e: + logging.warning(f"EPUB nav processing failed: {e}. Falling back to spine.") + self._process_epub_content_spine_fallback() + + return self.content_texts, self.content_lengths + + def _extract_book_metadata(self): + metadata = {} + if not self.book: + return metadata + + try: + metadata["title"] = self.book.get_metadata("DC", "title")[0][0] + except Exception: + metadata["title"] = os.path.splitext(os.path.basename(self.book_path))[0] + + try: + metadata["author"] = self.book.get_metadata("DC", "creator")[0][0] + except Exception: + metadata["author"] = "Unknown Author" + + try: + metadata["language"] = self.book.get_metadata("DC", "language")[0][0] + except Exception: + metadata["language"] = "en" + + return metadata + + def _find_doc_key(self, base_href, doc_order, doc_order_decoded): + candidates = [ + base_href, + urllib.parse.unquote(base_href), + ] + base_name = os.path.basename(base_href).lower() + for k in list(doc_order.keys()) + list(doc_order_decoded.keys()): + if os.path.basename(k).lower() == base_name: + candidates.append(k) + for candidate in candidates: + if candidate in doc_order: + return candidate, doc_order[candidate] + elif candidate in doc_order_decoded: + return candidate, doc_order_decoded[candidate] + return None, None + + def _find_position_robust(self, doc_href, fragment_id): + if doc_href not in self.doc_content: + logging.warning(f"Document '{doc_href}' not found in cached content.") + return 0 + html_content = self.doc_content[doc_href] + if not fragment_id: + return 0 + + try: + temp_soup = BeautifulSoup(f"
{html_content}
", "html.parser") + target_element = temp_soup.find(id=fragment_id) + if target_element: + tag_str = str(target_element) + pos = html_content.find(tag_str[: min(len(tag_str), 200)]) + if pos != -1: + return pos + except Exception as e: + logging.warning(f"BeautifulSoup failed to find id='{fragment_id}': {e}") + + safe_fragment_id = re.escape(fragment_id) + id_name_pattern = re.compile( + f"<[^>]+(?:id|name)\\s*=\\s*[\"']{safe_fragment_id}[\"']", re.IGNORECASE + ) + match = id_name_pattern.search(html_content) + if match: + return match.start() + + id_match_str = f'id="{fragment_id}"' + name_match_str = f'name="{fragment_id}"' + id_pos = html_content.find(id_match_str) + name_pos = html_content.find(name_match_str) + + pos = -1 + if id_pos != -1 and name_pos != -1: + pos = min(id_pos, name_pos) + elif id_pos != -1: + pos = id_pos + elif name_pos != -1: + pos = name_pos + + if pos != -1: + tag_start_pos = html_content.rfind("<", 0, pos) + final_pos = tag_start_pos if tag_start_pos != -1 else 0 + return final_pos + + logging.warning( + f"Anchor '{fragment_id}' not found in {doc_href}. Defaulting to position 0." + ) + return 0 + + def _parse_ncx_navpoint( + self, + nav_point, + ordered_entries, + doc_order, + doc_order_decoded, + tree_structure_list, + find_position_func, + ): + """ + Recursive parsing of NCX navigation nodes. + + Logic tested by: tests/test_epub_ncx_parsing.py + """ + nav_label = nav_point.find("navLabel") + content = nav_point.find("content") + title = ( + nav_label.find("text").get_text(strip=True) + if nav_label and nav_label.find("text") + else "Untitled Section" + ) + src = content["src"] if content and "src" in content.attrs else None + + current_entry_node = {"title": title, "src": src, "children": []} + + if src: + base_href, fragment = src.split("#", 1) if "#" in src else (src, None) + doc_key, doc_idx = self._find_doc_key( + base_href, doc_order, doc_order_decoded + ) + if not doc_key: + current_entry_node["has_content"] = False + else: + position = find_position_func(doc_key, fragment) + entry_data = { + "src": src, + "title": title, + "doc_href": doc_key, + "position": position, + "doc_order": doc_idx, + } + ordered_entries.append(entry_data) + current_entry_node["has_content"] = True + else: + current_entry_node["has_content"] = False + + child_navpoints = nav_point.find_all("navPoint", recursive=False) + if child_navpoints: + for child_np in child_navpoints: + self._parse_ncx_navpoint( + child_np, + ordered_entries, + doc_order, + doc_order_decoded, + current_entry_node["children"], + find_position_func, + ) + + if title and ( + current_entry_node.get("has_content", False) + or current_entry_node["children"] + ): + tree_structure_list.append(current_entry_node) + + def _extract_nav_li_title(self, li_element, link_element=None, span_element=None): + """Helper to extract title from a nav
  • element, handling various structures.""" + title = "Untitled Section" + + if link_element: + title = link_element.get_text(strip=True) or title + elif span_element: + title = span_element.get_text(strip=True) or title + + # Fallback to direct text if title is empty or default + # If we used link/span but got empty string, we try fallback. + # If we didn't use link/span, we try fallback. + if not title.strip() or title == "Untitled Section": + li_text = "".join( + t for t in li_element.contents if isinstance(t, NavigableString) + ).strip() + if li_text: + title = li_text + + # Second fallback: if we have a span but title is still empty, try span text again + # (covered by logic above mostly, but mirroring original logic's intense fallback) + if (not title.strip() or title == "Untitled Section") and span_element: + title = span_element.get_text(strip=True) or title + + return title + + def _parse_html_nav_li( + self, + li_element, + ordered_entries, + doc_order, + doc_order_decoded, + tree_structure_list, + find_position_func, + ): + """ + Recursive parsing of HTML5 Navigation (li) nodes. + + Logic tested by: tests/test_epub_html_nav_parsing.py + """ + link = li_element.find("a", recursive=False) + span_text = li_element.find("span", recursive=False) + src = None + current_entry_node = {"children": []} + + if link and "href" in link.attrs: + src = link["href"] + + title = self._extract_nav_li_title(li_element, link, span_text) + + current_entry_node["title"] = title + current_entry_node["src"] = src + + doc_key = None + doc_idx = None + position = 0 + fragment = None + if src: + base_href, fragment = src.split("#", 1) if "#" in src else (src, None) + doc_key, doc_idx = self._find_doc_key( + base_href, doc_order, doc_order_decoded + ) + if doc_key is not None: + position = find_position_func(doc_key, fragment) + entry_data = { + "src": src, + "title": title, + "doc_href": doc_key, + "position": position, + "doc_order": doc_idx, + } + ordered_entries.append(entry_data) + current_entry_node["has_content"] = True + else: + current_entry_node["has_content"] = False + else: + current_entry_node["has_content"] = False + + for child_ol in li_element.find_all("ol", recursive=False): + for child_li in child_ol.find_all("li", recursive=False): + self._parse_html_nav_li( + child_li, + ordered_entries, + doc_order, + doc_order_decoded, + current_entry_node["children"], + find_position_func, + ) + tree_structure_list.append(current_entry_node) + + def _identify_nav_item(self): + """Identify the navigation item (HTML Nav or NCX) and its type.""" + nav_item = None + nav_type = None + + # 1. Check ITEM_NAVIGATION + nav_items = list(self.book.get_items_of_type(ebooklib.ITEM_NAVIGATION)) + + # 1.1 Support for EPUB 3 EpubNav which might be ITEM_DOCUMENT (9) but with properties=['nav'] + if not nav_items: + # Look in ITEM_DOCUMENT for items with 'nav' property + for item in self.book.get_items_of_type(ebooklib.ITEM_DOCUMENT): + if ( + hasattr(item, "get_type") + and item.get_type() == ebooklib.ITEM_DOCUMENT + ): + # Check properties - ebooklib stores opf properties in list + # Some versions use item.properties, some need checking + props = getattr(item, "properties", []) + if "nav" in props: + nav_items.append(item) + + if nav_items: + nav_item = next( + ( + item + for item in nav_items + if "nav" in item.get_name().lower() + and item.get_name().lower().endswith((".xhtml", ".html")) + ), + None, + ) or next( + ( + item + for item in nav_items + if item.get_name().lower().endswith((".xhtml", ".html")) + ), + None, + ) + if nav_item: + nav_type = "html" + + # 2. NCX in NAV + if not nav_item and nav_items: + ncx_in_nav = next( + ( + item + for item in nav_items + if item.get_name().lower().endswith(".ncx") + ), + None, + ) + if ncx_in_nav: + nav_item = ncx_in_nav + nav_type = "ncx" + + # 3. ITEM_NCX or Fallback + # If no explicit navigation item found, try to find a standard NCX file + if not nav_item: + ncx_constant = getattr(epub, "ITEM_NCX", None) + if ncx_constant is not None: + ncx_items = list(self.book.get_items_of_type(ncx_constant)) + if ncx_items: + nav_item = ncx_items[0] + nav_type = "ncx" + + # 4. Heuristic Search + # Scan documents for something that looks like a TOC if standard methods fail + if not nav_item: + for item in self.book.get_items_of_type(ebooklib.ITEM_DOCUMENT): + try: + html_content = item.get_content().decode("utf-8", errors="ignore") + if " idx_next: + docs_between = [ + spine_docs[k] + for k in range(idx_current + 1, len(spine_docs)) + ] + docs_between.extend( + [spine_docs[k] for k in range(0, idx_next)] + ) + except ValueError: + pass + + for doc_href in docs_between: + slice_html += self.doc_content.get(doc_href, "") + next_doc_html = self.doc_content.get(next_doc, "") + slice_html += next_doc_html[:next_pos] + else: + slice_html = current_doc_html[start_slice_pos:] + try: + idx_current = spine_docs.index(current_doc) + for doc_idx in range(idx_current + 1, len(spine_docs)): + slice_html += self.doc_content.get(spine_docs[doc_idx], "") + except ValueError: + pass + + if not slice_html.strip() and current_doc_html: + slice_html = current_doc_html + + if slice_html.strip(): + slice_soup = BeautifulSoup(slice_html, "html.parser") + for tag in slice_soup.find_all(["p", "div"]): + tag.append("\n\n") + + for ol in slice_soup.find_all("ol"): + start = int(ol.get("start", 1)) + for idx, li in enumerate(ol.find_all("li", recursive=False)): + number_text = f"{start + idx}) " + if li.string: + li.string.replace_with(number_text + li.string) + else: + li.insert(0, NavigableString(number_text)) + + for tag in slice_soup.find_all(["sup", "sub"]): + tag.decompose() + + text = clean_text(slice_soup.get_text()).strip() + if text: + self.content_texts[current_src] = text + self.content_lengths[current_src] = calculate_text_length(text) + else: + self.content_texts[current_src] = "" + self.content_lengths[current_src] = 0 + else: + self.content_texts[current_src] = "" + self.content_lengths[current_src] = 0 + + if ordered_nav_entries: + first_entry = ordered_nav_entries[0] + first_doc_href = first_entry["doc_href"] + first_pos = first_entry["position"] + first_doc_order = first_entry["doc_order"] + prefix_html = "" + + for doc_idx in range(first_doc_order): + if doc_idx < len(spine_docs): + intermediate_doc_href = spine_docs[doc_idx] + prefix_html += self.doc_content.get(intermediate_doc_href, "") + + first_doc_html = self.doc_content.get(first_doc_href, "") + prefix_html += first_doc_html[:first_pos] + + if prefix_html.strip(): + prefix_soup = BeautifulSoup(prefix_html, "html.parser") + for tag in prefix_soup.find_all(["sup", "sub"]): + tag.decompose() + prefix_text = clean_text(prefix_soup.get_text()).strip() + + if prefix_text: + prefix_chapter_src = "internal:prefix_content" + self.content_texts[prefix_chapter_src] = prefix_text + self.content_lengths[prefix_chapter_src] = len(prefix_text) + self.processed_nav_structure.insert( + 0, + { + "src": prefix_chapter_src, + "title": "Introduction", + "children": [], + "has_content": True, + }, + ) + + def _process_epub_content_spine_fallback(self): + """ + Process EPUB content using the spine (linear reading order) + when navigation processing fails. + """ + logging.info("Using spine fallback for EPUB processing.") + self.doc_content = {} + spine_docs = [] + for spine_item_tuple in self.book.spine: + item_id = spine_item_tuple[0] + item = self.book.get_item_with_id(item_id) + if item: + spine_docs.append(item.get_name()) + else: + logging.warning(f"Spine item with id '{item_id}' not found.") + + for item in self.book.get_items_of_type(ebooklib.ITEM_DOCUMENT): + href = item.get_name() + if href in spine_docs: + try: + html_content = item.get_content().decode("utf-8", errors="ignore") + self.doc_content[href] = html_content + except Exception: + self.doc_content[href] = "" + + self.content_texts = {} + self.content_lengths = {} + for i, doc_href in enumerate(spine_docs): + html_content = self.doc_content.get(doc_href, "") + if html_content: + soup = BeautifulSoup(html_content, "html.parser") + + # Handle ordered lists + for ol in soup.find_all("ol"): + start = int(ol.get("start", 1)) + for idx, li in enumerate(ol.find_all("li", recursive=False)): + number_text = f"{start + idx}) " + if li.string: + li.string.replace_with(number_text + li.string) + else: + li.insert(0, NavigableString(number_text)) + + # Remove sup/sub + for tag in soup.find_all(["sup", "sub"]): + tag.decompose() + + text = clean_text(soup.get_text()).strip() + if text: + self.content_texts[doc_href] = text + self.content_lengths[doc_href] = calculate_text_length(text) + + def get_chapters(self): + chapters = super().get_chapters() + if not chapters: + # Use spine order fallback if no Nav structure + if self.book: + for spine_item_tuple in self.book.spine: + item_id = spine_item_tuple[0] + item = self.book.get_item_with_id(item_id) + if item: + href = item.get_name() + if href in self.content_texts: + chapters.append((href, href)) + return chapters + + +def get_book_parser(book_path, file_type=None): + """ + Factory function to get the appropriate parser instance. + """ + book_path = os.path.normpath(os.path.abspath(book_path)) + + if not file_type: + if book_path.lower().endswith(".pdf"): + file_type = "pdf" + elif book_path.lower().endswith((".md", ".markdown")): + file_type = "markdown" + else: + file_type = "epub" + + if file_type == "pdf": + return PdfParser(book_path) + elif file_type == "markdown": + return MarkdownParser(book_path) + elif file_type == "epub": + return EpubParser(book_path) + else: + raise ValueError(f"Unsupported file type: {file_type}") diff --git a/abogen/check_cuda.py b/abogen/check_cuda.py new file mode 100644 index 0000000..e674015 --- /dev/null +++ b/abogen/check_cuda.py @@ -0,0 +1,30 @@ +import sys +import os +import platform +import ctypes +import importlib.util + +def check_cuda_with_fix(): + """ + Check if CUDA is available, with a fix for PyTorch DLL loading issue + ([WinError 1114]) on Windows. + """ + # Fix PyTorch DLL loading issue ([WinError 1114]) on Windows + try: + if platform.system() == "Windows": + spec = importlib.util.find_spec("torch") + if spec and spec.origin: + dll_path = os.path.join(os.path.dirname(spec.origin), "lib", "c10.dll") + if os.path.exists(dll_path): + ctypes.CDLL(os.path.normpath(dll_path)) + except Exception: + pass + + try: + from torch.cuda import is_available + print(is_available()) + except ImportError: + print("False") + +if __name__ == "__main__": + check_cuda_with_fix() diff --git a/abogen/chunking.py b/abogen/chunking.py new file mode 100644 index 0000000..2d6f03b --- /dev/null +++ b/abogen/chunking.py @@ -0,0 +1,275 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Dict, Iterable, Iterator, List, Literal, Optional, Tuple +from typing import Pattern + +import re + +from abogen.kokoro_text_normalization import ApostropheConfig, normalize_for_pipeline +from abogen.normalization_settings import build_apostrophe_config, get_runtime_settings + +ChunkLevel = Literal["paragraph", "sentence"] + +_SENTENCE_SPLIT_REGEX = re.compile(r"(? Dict[str, object]: + return { + "id": self.id, + "chapter_index": self.chapter_index, + "chunk_index": self.chunk_index, + "level": self.level, + "text": self.text, + "speaker_id": self.speaker_id, + "voice": self.voice, + "voice_profile": self.voice_profile, + "voice_formula": self.voice_formula, + "display_text": self.display_text, + } + + +def _iter_paragraphs(text: str) -> Iterator[str]: + for raw_segment in _PARAGRAPH_SPLIT_REGEX.split(text.strip()): + normalized = raw_segment.strip() + if normalized: + yield normalized + + +def _iter_sentences(paragraph: str) -> Iterator[Tuple[str, str]]: + if not paragraph: + return + start = 0 + for match in _SENTENCE_SPLIT_REGEX.finditer(paragraph): + end = match.end() + raw_segment = paragraph[start:end] + candidate = raw_segment.strip() + if candidate: + yield candidate, raw_segment + start = match.end() + tail_raw = paragraph[start:] + tail = tail_raw.strip() + if tail: + yield tail, tail_raw + + +def _normalize_whitespace(value: str) -> str: + return _WHITESPACE_REGEX.sub(" ", value).strip() + + +def _normalize_chunk_text(value: str) -> str: + settings = get_runtime_settings() + config = build_apostrophe_config( + settings=settings, base=_PIPELINE_APOSTROPHE_CONFIG + ) + normalized = normalize_for_pipeline(value, config=config, settings=settings) + return _normalize_whitespace(normalized) + + +def _split_sentences(paragraph: str) -> List[Tuple[str, str]]: + sentences = list(_iter_sentences(paragraph)) + if not sentences: + return [] + + merged: List[Tuple[str, str]] = [] + buffer_norm: List[str] = [] + buffer_raw: List[str] = [] + + for normalized_sentence, raw_sentence in sentences: + if buffer_norm: + buffer_norm.append(normalized_sentence) + buffer_raw.append(raw_sentence) + else: + buffer_norm = [normalized_sentence] + buffer_raw = [raw_sentence] + + if _ABBREVIATION_END_RE.search(normalized_sentence.rstrip()): + continue + + merged.append((" ".join(buffer_norm), "".join(buffer_raw))) + buffer_norm = [] + buffer_raw = [] + + if buffer_norm: + merged.append((" ".join(buffer_norm), "".join(buffer_raw))) + + return merged + + +def chunk_text( + *, + chapter_index: int, + chapter_title: str, + text: str, + level: ChunkLevel, + speaker_id: str = "narrator", + voice: Optional[str] = None, + voice_profile: Optional[str] = None, + voice_formula: Optional[str] = None, + chunk_prefix: Optional[str] = None, +) -> List[Dict[str, object]]: + """Split text into ordered chunk dictionaries.""" + + prefix = chunk_prefix or f"chap{chapter_index:04d}" + chunks: List[Dict[str, object]] = [] + + if level == "paragraph": + paragraphs = list(_iter_paragraphs(text)) or [text.strip()] + for para_index, paragraph in enumerate(paragraphs): + normalized = _normalize_whitespace(paragraph) + if not normalized: + continue + chunk_id = f"{prefix}_p{para_index:04d}" + payload = Chunk( + id=chunk_id, + chapter_index=chapter_index, + chunk_index=len(chunks), + level=level, + text=normalized, + speaker_id=speaker_id, + voice=voice, + voice_profile=voice_profile, + voice_formula=voice_formula, + ).as_dict() + payload["normalized_text"] = _normalize_chunk_text(paragraph) + payload["original_text"] = paragraph + chunks.append(payload) + _attach_display_text(text, chunks) + return chunks + + # Sentence level – flatten paragraphs into individual sentences + sentence_index = 0 + for para_index, paragraph in enumerate( + list(_iter_paragraphs(text)) or [text.strip()] + ): + normalized_para = _normalize_whitespace(paragraph) + if not normalized_para: + continue + sentence_pairs = _split_sentences(paragraph) or [(normalized_para, paragraph)] + for sent_local_index, (normalized_sentence, raw_sentence) in enumerate( + sentence_pairs + ): + normalized_sentence = _normalize_whitespace(normalized_sentence) + if not normalized_sentence: + continue + chunk_id = f"{prefix}_p{para_index:04d}_s{sent_local_index:04d}" + payload = Chunk( + id=chunk_id, + chapter_index=chapter_index, + chunk_index=sentence_index, + level=level, + text=normalized_sentence, + speaker_id=speaker_id, + voice=voice, + voice_profile=voice_profile, + voice_formula=voice_formula, + ).as_dict() + payload["normalized_text"] = _normalize_chunk_text(raw_sentence) + payload["display_text"] = raw_sentence + payload["original_text"] = raw_sentence + chunks.append(payload) + sentence_index += 1 + + _attach_display_text(text, chunks) + return chunks + + +_DISPLAY_PATTERN_CACHE: Dict[str, Pattern[str]] = {} + + +def _build_display_pattern(text: str) -> Pattern[str]: + cached = _DISPLAY_PATTERN_CACHE.get(text) + if cached is not None: + return cached + escaped = re.escape(text) + escaped = escaped.replace(r"\ ", r"\s+") + pattern = re.compile(r"(\s*" + escaped + r"\s*)", re.DOTALL) + _DISPLAY_PATTERN_CACHE[text] = pattern + return pattern + + +def _search_source_span( + source: str, normalized: str, start: int +) -> Optional[Tuple[int, int]]: + if not normalized: + return None + pattern = _build_display_pattern(normalized) + match = pattern.search(source, start) + if not match: + return None + return match.start(1), match.end(1) + + +def _attach_display_text(source: str, chunks: List[Dict[str, object]]) -> None: + if not source or not chunks: + return + cursor = 0 + for chunk in chunks: + candidate = str(chunk.get("display_text") or chunk.get("text") or "") + if not candidate: + continue + match = _search_source_span(source, candidate, cursor) + if match is None and cursor: + match = _search_source_span(source, candidate, 0) + if match is None: + chunk.setdefault("display_text", candidate) + chunk.setdefault("original_text", chunk.get("display_text") or candidate) + continue + start, end = match + chunk["display_text"] = source[start:end] + chunk["original_text"] = source[start:end] + cursor = end + + +def build_chunks_for_chapters( + chapters: Iterable[Dict[str, object]], + *, + level: ChunkLevel, + speaker_id: str = "narrator", +) -> List[Dict[str, object]]: + """Generate chunk dictionaries for a sequence of chapter payloads.""" + all_chunks: List[Dict[str, object]] = [] + for chapter_index, entry in enumerate(chapters): + if not isinstance(entry, dict): # defensive + continue + text = str(entry.get("text", "") or "").strip() + if not text: + continue + voice = entry.get("voice") + voice_profile = entry.get("voice_profile") + voice_formula = entry.get("voice_formula") + prefix = entry.get("id") or f"chap{chapter_index:04d}" + chapter_chunks = chunk_text( + chapter_index=chapter_index, + chapter_title=str(entry.get("title") or f"Chapter {chapter_index + 1}"), + text=text, + level=level, + speaker_id=speaker_id, + voice=str(voice) if voice else None, + voice_profile=str(voice_profile) if voice_profile else None, + voice_formula=str(voice_formula) if voice_formula else None, + chunk_prefix=str(prefix), + ) + all_chunks.extend(chapter_chunks) + return all_chunks diff --git a/abogen/conversion.py b/abogen/conversion.py index 2195b8a..f615349 100644 --- a/abogen/conversion.py +++ b/abogen/conversion.py @@ -1,2477 +1,16 @@ -import os -import re -import time -import hashlib # For generating unique cache filenames -from platformdirs import user_desktop_dir -from PyQt6.QtCore import QThread, pyqtSignal, Qt, QTimer -from PyQt6.QtWidgets import QCheckBox, QVBoxLayout, QDialog, QLabel, QDialogButtonBox -import soundfile as sf -from abogen.utils import ( - create_process, - get_user_cache_path, - detect_encoding, -) -from abogen.constants import ( - LANGUAGE_DESCRIPTIONS, - SAMPLE_VOICE_TEXTS, - COLORS, - CHAPTER_OPTIONS_COUNTDOWN, - SUBTITLE_FORMATS, - SUPPORTED_SOUND_FORMATS, - SUPPORTED_SUBTITLE_FORMATS, -) -from abogen.voice_formulas import get_new_voice -import abogen.hf_tracker as hf_tracker -import static_ffmpeg -import threading # for efficient waiting -import subprocess -import platform +"""Backwards-compatible re-export of conversion module. -# Configuration constants -_USER_RESPONSE_TIMEOUT = ( - 0.1 # Timeout in seconds for checking user response/cancellation +The PyQt-based implementation lives in abogen.pyqt.conversion. +The web-based implementation is in abogen.webui.conversion_runner. +""" + +from __future__ import annotations + +# Re-export PyQt conversion classes for backwards compatibility +from abogen.pyqt.conversion import ( # noqa: F401 + ConversionThread, + VoicePreviewThread, + PlayAudioThread, ) -from abogen.subtitle_utils import ( - clean_text, - parse_srt_file, - parse_vtt_file, - detect_timestamps_in_text, - parse_timestamp_text_file, - parse_ass_file, - get_sample_voice_text, - sanitize_name_for_os, - _CHAPTER_MARKER_SEARCH_PATTERN, -) - -class CountdownDialog(QDialog): - """Base dialog with auto-accept countdown functionality""" - - def __init__(self, title, countdown_seconds, parent=None): - super().__init__(parent) - self.setWindowTitle(title) - self.setMinimumWidth(350) - self.setWindowFlags( - self.windowFlags() - & ~Qt.WindowType.WindowCloseButtonHint - & ~Qt.WindowType.WindowContextHelpButtonHint - ) - - self.countdown_seconds = countdown_seconds - self.layout = QVBoxLayout(self) - self._timer = None - self._button_box = None - - def add_countdown_and_buttons(self): - """Add countdown label and OK button - call this after adding custom content""" - self.countdown_label = QLabel( - f"Auto-accepting in {self.countdown_seconds} seconds..." - ) - self.countdown_label.setStyleSheet(f"color: {COLORS['GREEN']};") - self.layout.addWidget(self.countdown_label) - - self._button_box = QDialogButtonBox(QDialogButtonBox.StandardButton.Ok) - self._button_box.accepted.connect(self.accept) - self.layout.addWidget(self._button_box) - - self._timer = QTimer(self) - self._timer.timeout.connect(self._on_timer_tick) - self._timer.start(1000) - - def _on_timer_tick(self): - self.countdown_seconds -= 1 - if self.countdown_seconds > 0: - self.countdown_label.setText( - f"Auto-accepting in {self.countdown_seconds} seconds..." - ) - else: - self._timer.stop() - self._button_box.accepted.emit() - - def closeEvent(self, event): - event.ignore() - - def keyPressEvent(self, event): - if event.key() == Qt.Key.Key_Escape: - event.ignore() - else: - super().keyPressEvent(event) - - -class ChapterOptionsDialog(CountdownDialog): - def __init__(self, chapter_count, parent=None): - super().__init__("Chapter Options", CHAPTER_OPTIONS_COUNTDOWN, parent) - - self.layout.addWidget( - QLabel(f"Detected {chapter_count} chapters in the text file.") - ) - self.layout.addWidget(QLabel("How would you like to process these chapters?")) - - self.save_separately_checkbox = QCheckBox("Save each chapter separately") - self.merge_at_end_checkbox = QCheckBox("Create a merged version at the end") - - self.save_separately_checkbox.setChecked(False) - self.merge_at_end_checkbox.setChecked(True) - - self.save_separately_checkbox.stateChanged.connect( - self.update_merge_checkbox_state - ) - - self.layout.addWidget(self.save_separately_checkbox) - self.layout.addWidget(self.merge_at_end_checkbox) - - self.add_countdown_and_buttons() - self.update_merge_checkbox_state() - - def update_merge_checkbox_state(self): - self.merge_at_end_checkbox.setEnabled(self.save_separately_checkbox.isChecked()) - - def get_options(self): - return { - "save_chapters_separately": self.save_separately_checkbox.isChecked(), - "merge_chapters_at_end": self.merge_at_end_checkbox.isChecked() - and self.merge_at_end_checkbox.isEnabled(), - } - - -class TimestampDetectionDialog(QDialog): - def __init__(self, parent=None): - super().__init__(parent) - self.setWindowTitle("Timestamps Detected") - self.setMinimumWidth(350) - self.use_timestamps_result = True - self.countdown_seconds = CHAPTER_OPTIONS_COUNTDOWN - - layout = QVBoxLayout(self) - - layout.addWidget(QLabel("This file contains timestamps in HH:MM:SS format.")) - layout.addWidget( - QLabel("Do you want to use these timestamps for precise audio timing?") - ) - - yes_label = QLabel( - "• Yes: Generate audio that matches each timestamp (subtitle mode will be ignored)" - ) - yes_label.setStyleSheet(f"color: {COLORS['BLUE_BORDER_HOVER']};") - layout.addWidget(yes_label) - - no_label = QLabel("• No: Ignore timestamps and process as regular text") - no_label.setStyleSheet(f"color: {COLORS['ORANGE']};") - layout.addWidget(no_label) - - # Countdown label - self.countdown_label = QLabel( - f"Auto-accepting in {self.countdown_seconds} seconds..." - ) - self.countdown_label.setStyleSheet(f"color: {COLORS['GREEN']};") - layout.addWidget(self.countdown_label) - - button_box = QDialogButtonBox() - yes_button = button_box.addButton("Yes", QDialogButtonBox.ButtonRole.AcceptRole) - no_button = button_box.addButton("No", QDialogButtonBox.ButtonRole.RejectRole) - - yes_button.clicked.connect(lambda: self._set_result(True)) - no_button.clicked.connect(lambda: self._set_result(False)) - - layout.addWidget(button_box) - - # Timer for countdown - self._timer = QTimer(self) - self._timer.timeout.connect(self._on_timer_tick) - self._timer.start(1000) - - def _on_timer_tick(self): - self.countdown_seconds -= 1 - if self.countdown_seconds > 0: - self.countdown_label.setText( - f"Auto-accepting in {self.countdown_seconds} seconds..." - ) - else: - self._timer.stop() - self._set_result(True) - - def _set_result(self, use_timestamps): - if self._timer: - self._timer.stop() - self.use_timestamps_result = use_timestamps - self.accept() - - def use_timestamps(self): - return self.use_timestamps_result - - -class ConversionThread(QThread): - progress_updated = pyqtSignal(int, str) # Add str for ETR - conversion_finished = pyqtSignal(object, object) # Pass output path as second arg - log_updated = pyqtSignal(object) # Updated signal for log updates - chapters_detected = pyqtSignal(int) # Signal for chapter detection - - # Punctuation constants for unified handling across languages - PUNCTUATION_SENTENCE = ".!?।。！？" - PUNCTUATION_SENTENCE_COMMA = ".!?,।。！？、，" - PUNCTUATION_COMMAS = ",，、" - - def _get_split_pattern(self, lang_code, subtitle_mode): - """ - Get the appropriate split pattern based on language and subtitle mode. - - Args: - lang_code: Language code (a, b, e, f, etc.) - subtitle_mode: Subtitle mode ("Sentence", "Sentence + Comma", "Line", etc.) - - Returns: - Split pattern string - """ - # For English, always use newline splitting only - if lang_code in ["a", "b"]: - return "\n" - - # Determine spacing pattern based on language - spacing_pattern = r"\s*" if lang_code in ["z", "j"] else r"\s+" - - # For Chinese/Japanese, when subtitle mode is Disabled or Line, prefer - # punctuation-based splitting instead of plain newline splitting. - if subtitle_mode in ("Disabled", "Line") and lang_code in ["z", "j"]: - return r"(?<=[{}]){}|\n+".format(self.PUNCTUATION_SENTENCE, spacing_pattern) - - if subtitle_mode == "Line": - return "\n" - elif subtitle_mode == "Sentence": - return r"(?<=[{}]){}|\n+".format(self.PUNCTUATION_SENTENCE, spacing_pattern) - elif subtitle_mode == "Sentence + Comma": - return r"(?<=[{}]){}|\n+".format( - self.PUNCTUATION_SENTENCE_COMMA, spacing_pattern - ) - else: - return r"\n+" # Default to line breaks - - def __init__( - self, - file_name, - lang_code, - speed, - voice, - save_option, - output_folder, - subtitle_mode, - output_format, - np_module, - kpipeline_class, - start_time, - total_char_count, - use_gpu=True, - from_queue=False, - save_base_path=None, - ): # Add use_gpu parameter - super().__init__() - self._chapter_options_event = threading.Event() - self._timestamp_response_event = threading.Event() - self.np = np_module - self.KPipeline = kpipeline_class - self.file_name = file_name - self.lang_code = lang_code - self.speed = speed - self.voice = voice - self.save_option = save_option - self.output_folder = output_folder - self.subtitle_mode = subtitle_mode - self.cancel_requested = False - self.should_cancel = False - self.process = None - self.output_format = output_format - self.from_queue = from_queue - self.start_time = start_time # Store start_time - self.total_char_count = total_char_count # Use passed total character count - self.processed_char_count = 0 # Initialize processed character count - self.display_path = None # Add variable for display path - self.save_base_path = save_base_path # Store the save base path - self.is_direct_text = ( - False # Flag to indicate if input is from textbox rather than file - ) - self.chapter_options_set = False - self.waiting_for_user_input = False - self.use_gpu = use_gpu # Store the GPU setting - self.max_subtitle_words = 50 # Default value, will be overridden from GUI - self.silence_duration = 2.0 # Default value, will be overridden from GUI - self.use_spacy_segmentation = True # Default, will be overridden from GUI - # Set split pattern based on language and subtitle mode - self.split_pattern = self._get_split_pattern(lang_code, subtitle_mode) - - def _stream_audio_in_chunks( - self, segments, process_func, progress_prefix="Processing" - ): - """ - Process audio segments in memory-efficient chunks - - Args: - segments: List of audio segments to process - process_func: Function that takes (segment_bytes, is_last) and processes a chunk - progress_prefix: Prefix for progress messages - - Returns: - Total samples processed - """ - # Calculate total size for progress reporting - total_samples = sum(len(segment) for segment in segments) - samples_processed = 0 - - self.log_updated.emit((f"\n{progress_prefix} segments...", "grey")) - - # Stream each segment individually - for i, segment in enumerate(segments): - try: - # Handle both NumPy arrays and PyTorch tensors - if hasattr(segment, "astype"): - segment_bytes = segment.astype("float32").tobytes() - else: - segment_bytes = segment.cpu().numpy().astype("float32").tobytes() - is_last = i == len(segments) - 1 - - # Update progress periodically - skip if there's only one segment - if (i % 20 == 0 or is_last) and len(segments) > 1: - progress_percent = int((samples_processed / total_samples) * 100) - self.log_updated.emit( - f"{progress_prefix} segment {i+1}/{len(segments)} ({progress_percent}% complete)" - ) - - # Process this segment - process_func(segment_bytes, is_last) - - # Update samples processed - samples_processed += len(segment) - - # Clear segment bytes from memory - del segment_bytes - except Exception as e: - self.log_updated.emit( - (f"Error processing segment {i}: {str(e)}", "red") - ) - raise - - return samples_processed - - def run(self): - print( - f"\nVoice: {self.voice}\nLanguage: {self.lang_code}\nSpeed: {self.speed}\nGPU: {self.use_gpu}\nFile: {self.file_name}\nSubtitle mode: {self.subtitle_mode}\nOutput format: {self.output_format}\nSave option: {self.save_option}\n" - ) - try: - hf_tracker.set_log_callback(lambda msg: self.log_updated.emit(msg)) - # Show configuration - self.log_updated.emit("Configuration:") - - # Determine input file and processing file - if getattr(self, "from_queue", False): - input_file = self.save_base_path or self.file_name - processing_file = self.file_name - else: - input_file = self.display_path if self.display_path else self.file_name - processing_file = self.file_name - - # Normalize paths for consistent display (fixes Windows path separator issues) - input_file = os.path.normpath(input_file) if input_file else input_file - processing_file = ( - os.path.normpath(processing_file) - if processing_file - else processing_file - ) - - self.log_updated.emit(f"- Input File: {input_file}") - if input_file != processing_file: - self.log_updated.emit(f"- Processing File: {processing_file}") - - # Use file_name for logs if from_queue, otherwise use display_path if available - if getattr(self, "from_queue", False): - base_path = ( - self.save_base_path or self.file_name - ) # Use save_base_path if available - else: - base_path = self.display_path if self.display_path else self.file_name - - # Use file size string passed from GUI - if hasattr(self, "file_size_str"): - self.log_updated.emit(f"- File size: {self.file_size_str}") - - self.log_updated.emit(f"- Total characters: {int(self.total_char_count):,}") - - self.log_updated.emit( - f"- Language: {self.lang_code} ({LANGUAGE_DESCRIPTIONS.get(self.lang_code, 'Unknown')})" - ) - self.log_updated.emit(f"- Voice: {self.voice}") - self.log_updated.emit(f"- Speed: {self.speed}") - self.log_updated.emit(f"- Subtitle mode: {self.subtitle_mode}") - self.log_updated.emit(f"- Output format: {self.output_format}") - self.log_updated.emit( - f"- Subtitle format: {next((label for value, label in SUBTITLE_FORMATS if value == getattr(self, 'subtitle_format', 'srt')), getattr(self, 'subtitle_format', 'srt'))}" - ) - self.log_updated.emit( - f"- Use spaCy for sentence segmentation: {'Yes' if getattr(self, 'use_spacy_segmentation', False) else 'No'}" - ) - self.log_updated.emit(f"- Save option: {self.save_option}") - if self.replace_single_newlines: - self.log_updated.emit(f"- Replace single newlines: Yes") - - # Check if input is a subtitle file for additional configuration - is_subtitle_input = False - if not self.is_direct_text and self.file_name: - file_ext = os.path.splitext(self.file_name)[1].lower() - if file_ext in [".srt", ".ass", ".vtt"]: - is_subtitle_input = True - - # Display subtitle-specific options if processing subtitle file - if is_subtitle_input: - if getattr(self, "use_silent_gaps", False): - self.log_updated.emit("- Use silent gaps: Yes") - speed_method = getattr(self, "subtitle_speed_method", "tts") - method_label = ( - "TTS Regeneration" - if speed_method == "tts" - else "FFmpeg Time-stretch" - ) - self.log_updated.emit(f"- Speed adjustment method: {method_label}") - - # Display save_chapters_separately flag if it's set - if hasattr(self, "save_chapters_separately"): - self.log_updated.emit( - ( - f"- Save chapters separately: {'Yes' if self.save_chapters_separately else 'No'}" - ) - ) - # Display merge_chapters_at_end flag if save_chapters_separately is True - if self.save_chapters_separately: - merge_at_end = getattr(self, "merge_chapters_at_end", True) - self.log_updated.emit( - f"- Merge chapters at the end: {'Yes' if merge_at_end else 'No'}" - ) - # Display the separate chapters format if it's set - separate_format = getattr(self, "separate_chapters_format", "wav") - self.log_updated.emit( - f"- Separate chapters format: {separate_format}" - ) - - # If merge_at_end is True, display the silence duration - if getattr(self, "merge_chapters_at_end", True): - self.log_updated.emit( - f"- Silence between chapters: {self.silence_duration} seconds" - ) - - if self.save_option == "Choose output folder": - self.log_updated.emit( - f"- Output folder: {self.output_folder or os.getcwd()}" - ) - - self.log_updated.emit(("\nInitializing TTS pipeline...", "grey")) - - # Set device based on use_gpu setting and platform - if self.use_gpu: - if platform.system() == "Darwin" and platform.processor() == "arm": - device = "mps" # Use MPS for Apple Silicon - else: - device = "cuda" # Use CUDA for other platforms - else: - device = "cpu" - - tts = self.KPipeline( - lang_code=self.lang_code, repo_id="hexgrad/Kokoro-82M", device=device - ) - - # Check if the input is a subtitle file or timestamp text file - is_subtitle_file = False - is_timestamp_text = False - if not self.is_direct_text and self.file_name: - file_ext = os.path.splitext(self.file_name)[1].lower() - if file_ext in [".srt", ".ass", ".vtt"]: - is_subtitle_file = True - self.log_updated.emit( - f"\nDetected subtitle file format: {file_ext}" - ) - elif file_ext == ".txt" and detect_timestamps_in_text(self.file_name): - is_timestamp_text = True - self.log_updated.emit( - ("\nDetected timestamps in text file", "grey") - ) - # Signal to ask user (-1 indicates timestamp detection) - self.chapters_detected.emit(-1) - # Wait for user response using event with timeout for responsive cancellation - while not self._timestamp_response_event.wait( - timeout=_USER_RESPONSE_TIMEOUT - ): - if self.cancel_requested: - self.conversion_finished.emit("Cancelled", None) - return - # Check cancellation one more time after event is set - if self.cancel_requested: - self.conversion_finished.emit("Cancelled", None) - return - if not self._timestamp_response: - is_timestamp_text = False - delattr(self, "_timestamp_response") - self._timestamp_response_event.clear() - - # Process subtitle files separately - if is_subtitle_file or is_timestamp_text: - self._process_subtitle_file(tts, base_path, is_timestamp_text) - return - - if self.is_direct_text: - text = self.file_name # Treat file_name as direct text input - else: - encoding = detect_encoding(self.file_name) - with open( - self.file_name, "r", encoding=encoding, errors="replace" - ) as file: - text = file.read() - - # Clean up text using utility function - text = clean_text(text) - - - # --- Chapter splitting logic --- - # Use pre-compiled pattern for better performance - chapter_splits = list(_CHAPTER_MARKER_SEARCH_PATTERN.finditer(text)) - chapters = [] - if chapter_splits: - # prepend Introduction for content before first marker - first_start = chapter_splits[0].start() - if first_start > 0: - intro_text = text[:first_start].strip() - if intro_text: - chapters.append(("Introduction", intro_text)) - for idx, match in enumerate(chapter_splits): - start = match.end() - end = ( - chapter_splits[idx + 1].start() - if idx + 1 < len(chapter_splits) - else len(text) - ) - chapter_name = match.group(1).strip() - chapter_text = text[start:end].strip() - chapters.append((chapter_name, chapter_text)) - else: - chapters = [("text", text)] - total_chapters = len(chapters) - - # For text files with chapters, prompt user for options if not already set - is_txt_file = not self.is_direct_text and ( - self.file_name.lower().endswith(".txt") - or (self.display_path and self.display_path.lower().endswith(".txt")) - ) - - if ( - is_txt_file - and total_chapters > 1 - and ( - not hasattr(self, "save_chapters_separately") - or not hasattr(self, "merge_chapters_at_end") - ) - and not self.chapter_options_set - ): - - # Emit signal to main thread and wait - self.chapters_detected.emit(total_chapters) - self._chapter_options_event.wait() - if self.cancel_requested: - self.conversion_finished.emit("Cancelled", None) - return - self.chapter_options_set = True - - # Log all detected chapters at the beginning - if total_chapters > 1: - chapter_list = "\n".join( - [f"{i+1}) {c[0]}" for i, c in enumerate(chapters)] - ) - self.log_updated.emit( - (f"\nDetected chapters ({total_chapters}):\n" + chapter_list) - ) - else: - self.log_updated.emit((f"\nProcessing {chapters[0][0]}...", "grey")) - - # If save_chapters_separately is enabled, find a unique suffix ONCE and use for both folder and merged file - save_chapters_separately = getattr(self, "save_chapters_separately", False) - merge_chapters_at_end = getattr(self, "merge_chapters_at_end", True) - - # Ensure merge_chapters_at_end is True if not saving chapters separately - if not save_chapters_separately: - merge_chapters_at_end = True - - chapters_out_dir = None - suffix = "" - - # Use file_name for logs if from_queue, otherwise use display_path if available - if getattr(self, "from_queue", False): - base_path = ( - self.save_base_path or self.file_name - ) # Use save_base_path if available - else: - base_path = self.display_path if self.display_path else self.file_name - - base_name = os.path.splitext(os.path.basename(base_path))[0] - # Sanitize base_name for folder/file creation based on OS - sanitized_base_name = sanitize_name_for_os(base_name, is_folder=True) - - if self.save_option == "Save to Desktop": - parent_dir = user_desktop_dir() - elif self.save_option == "Save next to input file": - parent_dir = os.path.dirname(base_path) - else: - parent_dir = self.output_folder or os.getcwd() - # Ensure the output folder exists, error if it doesn't - if not os.path.exists(parent_dir): - self.log_updated.emit( - ( - f"Output folder does not exist: {parent_dir}", - "red", - ) - ) - # Find a unique suffix for both folder and merged file, always - counter = 1 - allowed_exts = set(SUPPORTED_SOUND_FORMATS + SUPPORTED_SUBTITLE_FORMATS) - while True: - suffix = f"_{counter}" if counter > 1 else "" - chapters_out_dir_candidate = os.path.join( - parent_dir, f"{sanitized_base_name}{suffix}_chapters" - ) - # Only check for files with allowed extensions (extension without dot, case-insensitive) - # Use generator expression to avoid processing all files upfront - file_parts = ( - os.path.splitext(fname) for fname in os.listdir(parent_dir) - ) - clash = any( - name == f"{sanitized_base_name}{suffix}" - and ext[1:].lower() in allowed_exts - for name, ext in file_parts - ) - if not os.path.exists(chapters_out_dir_candidate) and not clash: - break - counter += 1 - if save_chapters_separately and total_chapters > 1: - separate_chapters_format = getattr( - self, "separate_chapters_format", "wav" - ) - chapters_out_dir = chapters_out_dir_candidate - os.makedirs(chapters_out_dir, exist_ok=True) - self.log_updated.emit( - (f"\nChapters output folder: {chapters_out_dir}", "grey") - ) - - # Prepare merged output file for incremental writing ONLY if merge_chapters_at_end is True - if merge_chapters_at_end: - out_dir = parent_dir - base_filepath_no_ext = os.path.join( - out_dir, f"{sanitized_base_name}{suffix}" - ) - merged_out_path = f"{base_filepath_no_ext}.{self.output_format}" - subtitle_entries = [] - current_time = 0.0 - rate = 24000 - subtitle_mode = self.subtitle_mode - self.etr_start_time = time.time() - self.processed_char_count = 0 - current_segment = 0 - chapters_time = [ - {"chapter": chapter[0], "start": 0.0, "end": 0.0} - for chapter in chapters - ] - # SRT numbering fix: use a global counter - merged_srt_index = 1 # SRT numbering for merged file - # Prepare output file/ffmpeg process for merged output - if self.output_format in ["wav", "mp3", "flac"]: - merged_out_file = sf.SoundFile( - merged_out_path, - "w", - samplerate=24000, - channels=1, - format=self.output_format, - ) - ffmpeg_proc = None - elif self.output_format == "m4b": - # Real-time M4B generation using FFmpeg pipe - static_ffmpeg.add_paths() - merged_out_file = None - ffmpeg_proc = None - metadata_options, cover_path = ( - self._extract_and_add_metadata_tags_to_ffmpeg_cmd() - ) - # Prepare ffmpeg command for m4b output - cmd = [ - "ffmpeg", - "-y", - "-thread_queue_size", - "32768", - "-f", - "f32le", - "-ar", - "24000", - "-ac", - "1", - "-i", - "pipe:0", - ] - if cover_path and os.path.exists(cover_path): - cmd.extend( - [ - "-i", - cover_path, - "-map", - "0:a", - "-map", - "1", - "-c:v", - "copy", - "-disposition:v", - "attached_pic", - ] - ) - cmd.extend( - [ - "-c:a", - "aac", - "-q:a", - "2", - "-movflags", - "+faststart+use_metadata_tags", - ] - ) - cmd += metadata_options - cmd.append(merged_out_path) - ffmpeg_proc = create_process(cmd, stdin=subprocess.PIPE, text=False) - elif self.output_format == "opus": - static_ffmpeg.add_paths() - cmd = [ - "ffmpeg", - "-y", - "-thread_queue_size", - "32768", - "-f", - "f32le", - "-ar", - "24000", - "-ac", - "1", - "-i", - "pipe:0", - ] - cmd.extend(["-c:a", "libopus", "-b:a", "24000"]) - cmd.append(merged_out_path) - ffmpeg_proc = create_process(cmd, stdin=subprocess.PIPE, text=False) - merged_out_file = None - else: - self.log_updated.emit( - (f"Unsupported output format: {self.output_format}", "red") - ) - self.conversion_finished.emit( - ("Audio generation failed.", "red"), None - ) - return - # Open merged subtitle file for incremental writing if needed - merged_subtitle_file = None - if self.subtitle_mode != "Disabled": - subtitle_format = getattr(self, "subtitle_format", "srt") - file_extension = "ass" if "ass" in subtitle_format else "srt" - merged_subtitle_path = ( - os.path.splitext(merged_out_path)[0] + f".{file_extension}" - ) - # Default subtitle layout flags/strings so they exist regardless - # of whether ASS-specific handling runs. This prevents runtime - # errors when non-ASS formats (like SRT) are selected. - is_centered = False - is_narrow = False - merged_subtitle_margin = "" - merged_subtitle_alignment_tag = "" - if "ass" in subtitle_format: - merged_subtitle_file = open( - merged_subtitle_path, - "w", - encoding="utf-8", - errors="replace", - ) - # Minimal ASS header - merged_subtitle_file.write("[Script Info]\n") - merged_subtitle_file.write("Title: Generated by Abogen\n") - merged_subtitle_file.write("ScriptType: v4.00+\n\n") - # Add style definitions for karaoke highlighting - if self.subtitle_mode == "Sentence + Highlighting": - merged_subtitle_file.write("[V4+ Styles]\n") - merged_subtitle_file.write( - "Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding\n" - ) - merged_subtitle_file.write( - "Style: Default,Arial,24,&H00FFFFFF,&H00808080,&H00000000,&H00404040,0,0,0,0,100,100,0,0,3,2,0,5,10,10,10,1\n\n" - ) - merged_subtitle_file.write("[Events]\n") - merged_subtitle_file.write( - "Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text\n" - ) - # Set margin/alignment for ASS - is_centered = subtitle_format in ( - "ass_centered_wide", - "ass_centered_narrow", - ) - is_narrow = subtitle_format in ( - "ass_narrow", - "ass_centered_narrow", - ) - merged_subtitle_margin = "90" if is_narrow else "" - merged_subtitle_alignment_tag = ( - f"{{\\an5}}" if is_centered else "" - ) - else: - merged_subtitle_file = open( - merged_subtitle_path, - "w", - encoding="utf-8", - errors="replace", - ) - else: - merged_subtitle_path = None - merged_subtitle_file = None - else: - # If not merging, set merged_out_file and related variables to None - merged_out_file = None - ffmpeg_proc = None - merged_out_path = None - subtitle_entries = [] - current_time = 0.0 - rate = 24000 - subtitle_mode = self.subtitle_mode - self.etr_start_time = time.time() - self.processed_char_count = 0 - current_segment = 0 - chapters_time = [ - {"chapter": chapter[0], "start": 0.0, "end": 0.0} - for chapter in chapters - ] - srt_index = 1 # SRT numbering fix for chapter-only mode - # Instead of processing the whole text, process by chapter - for chapter_idx, (chapter_name, chapter_text) in enumerate(chapters, 1): - chapter_out_path = None - chapter_out_file = None - chapter_ffmpeg_proc = None - chapter_subtitle_file = None - chapter_subtitle_path = None - if total_chapters > 1: - self.log_updated.emit( - ( - f"\nChapter {chapter_idx}/{total_chapters}: {chapter_name}", - "blue", - ) - ) - chapter_subtitle_entries = [] - chapter_current_time = 0.0 - # Set chapter start time before processing - chapter_time = chapters_time[chapter_idx - 1] - if merge_chapters_at_end: - chapter_time["start"] = current_time - - # Check if the voice is a formula and load it if necessary - if "*" in self.voice: - loaded_voice = get_new_voice(tts, self.voice, self.use_gpu) - else: - loaded_voice = self.voice - # Prepare per-chapter output file if needed - if save_chapters_separately and total_chapters > 1: - # First pass: keep alphanumeric, spaces, hyphens, and underscores - sanitized = re.sub(r"[^\w\s\-]", "", chapter_name) - # Replace multiple spaces/hyphens with single underscore - sanitized = re.sub(r"[\s\-]+", "_", sanitized).strip("_") - # Apply OS-specific sanitization - sanitized = sanitize_name_for_os(sanitized, is_folder=False) - # Limit length (leaving room for the chapter number prefix) - MAX_LEN = 80 - if len(sanitized) > MAX_LEN: - pos = sanitized[:MAX_LEN].rfind("_") - sanitized = sanitized[: pos if pos > 0 else MAX_LEN].rstrip("_") - chapter_filename = f"{chapter_idx:02d}_{sanitized}" - chapter_out_path = os.path.join( - chapters_out_dir, - f"{chapter_filename}.{separate_chapters_format}", - ) - if separate_chapters_format in ["wav", "mp3", "flac"]: - chapter_out_file = sf.SoundFile( - chapter_out_path, - "w", - samplerate=24000, - channels=1, - format=separate_chapters_format, - ) - chapter_ffmpeg_proc = None - elif separate_chapters_format == "opus": - static_ffmpeg.add_paths() - cmd = [ - "ffmpeg", - "-y", - "-thread_queue_size", - "32768", - "-f", - "f32le", - "-ar", - "24000", - "-ac", - "1", - "-i", - "pipe:0", - ] - cmd.extend(["-c:a", "libopus", "-b:a", "24000"]) - cmd.append(chapter_out_path) - chapter_ffmpeg_proc = create_process( - cmd, stdin=subprocess.PIPE, text=False - ) - chapter_out_file = None - else: - self.log_updated.emit( - ( - f"Unsupported chapter format: {separate_chapters_format}", - "red", - ) - ) - continue - # Open chapter subtitle file for incremental writing if needed - chapter_subtitle_file = None - chapter_srt_index = ( - 1 # Initialize SRT numbering for this chapter file - ) - if self.subtitle_mode != "Disabled": - subtitle_format = getattr(self, "subtitle_format", "srt") - file_extension = "ass" if "ass" in subtitle_format else "srt" - chapter_subtitle_path = os.path.join( - chapters_out_dir, f"{chapter_filename}.{file_extension}" - ) - # Ensure these variables exist even when not using ASS so - # later code can safely reference them. - is_centered = False - is_narrow = False - chapter_subtitle_margin = "" - chapter_subtitle_alignment_tag = "" - # Open the chapter subtitle file for writing for both SRT and ASS - chapter_subtitle_file = open( - chapter_subtitle_path, - "w", - encoding="utf-8", - errors="replace", - ) - if "ass" in subtitle_format: - # Minimal ASS header - chapter_subtitle_file.write("[Script Info]\n") - chapter_subtitle_file.write("Title: Generated by Abogen\n") - chapter_subtitle_file.write("ScriptType: v4.00+\n\n") - - # Add style definitions for karaoke highlighting - if self.subtitle_mode == "Sentence + Highlighting": - chapter_subtitle_file.write("[V4+ Styles]\n") - chapter_subtitle_file.write( - "Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding\n" - ) - chapter_subtitle_file.write( - "Style: Default,Arial,24,&H00FFFFFF,&H00808080,&H00000000,&H00404040,0,0,0,0,100,100,0,0,3,2,0,5,10,10,10,1\n\n" - ) - - chapter_subtitle_file.write("[Events]\n") - chapter_subtitle_file.write( - "Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text\n" - ) - is_centered = subtitle_format in ( - "ass_centered_wide", - "ass_centered_narrow", - ) - is_narrow = subtitle_format in ( - "ass_narrow", - "ass_centered_narrow", - ) - chapter_subtitle_margin = "90" if is_narrow else "" - chapter_subtitle_alignment_tag = ( - f"{{\\an5}}" if is_centered else "" - ) - else: - chapter_subtitle_file = None - else: - chapter_subtitle_path = None - chapter_subtitle_file = None - - # Determine if spaCy segmentation should be used for PRE-TTS segmentation - # Only non-English languages use spaCy for pre-segmentation - # English uses spaCy only for subtitle generation (post-TTS) - # spaCy is disabled when subtitle mode is "Disabled" or "Line" - # spaCy is also disabled when input is a subtitle file - is_subtitle_input = ( - not self.is_direct_text - and self.file_name - and os.path.splitext(self.file_name)[1].lower() - in [".srt", ".ass", ".vtt"] - ) - use_spacy = ( - getattr(self, "use_spacy_segmentation", False) - and self.subtitle_mode not in ["Disabled", "Line"] - and not is_subtitle_input - ) - spacy_sentences = None - active_split_pattern = self.split_pattern - spacing_pattern = r"\s*" if self.lang_code in ["z", "j"] else r"\s+" - - # Pre-load spaCy model for English if it will be needed for subtitle generation - if ( - use_spacy - and self.lang_code in ["a", "b"] - and self.subtitle_mode in ["Sentence", "Sentence + Comma"] - ): - from abogen.spacy_utils import get_spacy_model - - nlp = get_spacy_model( - self.lang_code, - log_callback=lambda msg: self.log_updated.emit(msg), - ) - if nlp: - self.log_updated.emit( - ( - "\nUsing spaCy for sentence segmentation (only for subtitles)...", - "grey", - ) - ) - - if use_spacy and self.lang_code not in ["a", "b"]: - # Non-English: use spaCy for pre-TTS segmentation - self.log_updated.emit( - ("\nUsing spaCy for sentence segmentation (pre-TTS)...", "grey") - ) - from abogen.spacy_utils import segment_sentences - - spacy_sentences = segment_sentences( - chapter_text, - self.lang_code, - log_callback=lambda msg: self.log_updated.emit(msg), - ) - if spacy_sentences: - self.log_updated.emit( - ( - f"\nspaCy: Text segmented into {len(spacy_sentences)} sentences...", - "grey", - ) - ) - # For Sentence + Comma mode, still split on commas within spaCy sentences - if self.subtitle_mode == "Sentence + Comma": - active_split_pattern = r"(?<=[{}]){}|\n+".format( - self.PUNCTUATION_COMMAS, spacing_pattern - ) - else: - active_split_pattern = ( - "\n" # Use newline splitting for Sentence mode - ) - else: - self.log_updated.emit( - ("\nspaCy: Fallback to default segmentation...", "grey") - ) - - # Process text - either as spaCy sentences or as single text - text_segments = spacy_sentences if spacy_sentences else [chapter_text] - - # Print active split pattern used by the TTS engine once for this batch - try: - print(f"Using split pattern: {active_split_pattern!r}") - except Exception: - # Print must never break processing - print("Using split pattern: (unprintable)") - - for text_segment in text_segments: - for result in tts( - text_segment, - voice=loaded_voice, - speed=self.speed, - split_pattern=active_split_pattern, - ): - # Print the result for debugging - # print(f"Result: {result}") - if self.cancel_requested: - if chapter_out_file: - chapter_out_file.close() - if merged_out_file: - merged_out_file.close() - self.conversion_finished.emit("Cancelled", None) - return - current_segment += 1 - grapheme_len = len(result.graphemes) - self.processed_char_count += grapheme_len - # Log progress with both character counts and the graphemes content - self.log_updated.emit( - f"\n{self.processed_char_count:,}/{self.total_char_count:,}: {result.graphemes}" - ) - - chunk_dur = len(result.audio) / rate - chunk_start = current_time - # Write audio directly to merged file ONLY if merging - if merge_chapters_at_end and merged_out_file: - merged_out_file.write(result.audio) - elif merge_chapters_at_end and ffmpeg_proc: - if hasattr(result.audio, "numpy"): - audio_bytes = ( - result.audio.numpy().astype("float32").tobytes() - ) - else: - audio_bytes = result.audio.astype("float32").tobytes() - ffmpeg_proc.stdin.write(audio_bytes) - if chapter_out_file: - chapter_out_file.write(result.audio) - elif chapter_ffmpeg_proc: - if hasattr(result.audio, "numpy"): - audio_bytes = ( - result.audio.numpy().astype("float32").tobytes() - ) - else: - audio_bytes = result.audio.astype("float32").tobytes() - chapter_ffmpeg_proc.stdin.write(audio_bytes) - # Subtitle logic - if self.subtitle_mode != "Disabled": - tokens_list = getattr(result, "tokens", []) - - # Fallback for languages without token support (non-English) - # Create a single token representing the entire segment duration - if not tokens_list and result.graphemes: - - class FakeToken: - def __init__(self, text, start, end): - self.text = text - self.start_ts = start - self.end_ts = end - self.whitespace = "" - - tokens_list = [ - FakeToken(result.graphemes, 0, chunk_dur) - ] - - tokens_with_timestamps = [] - chapter_tokens_with_timestamps = [] - - # Process every token, regardless of text or timestamps - for tok in tokens_list: - tokens_with_timestamps.append( - { - "start": chunk_start + (tok.start_ts or 0), - "end": chunk_start + (tok.end_ts or 0), - "text": tok.text, - "whitespace": tok.whitespace, - } - ) - if chapter_out_file or chapter_ffmpeg_proc: - chapter_tokens_with_timestamps.append( - { - "start": chapter_current_time - + (tok.start_ts or 0), - "end": chapter_current_time - + (tok.end_ts or 0), - "text": tok.text, - "whitespace": tok.whitespace, - } - ) - # Process tokens according to subtitle mode - # Global subtitle processing ONLY if merging - if merge_chapters_at_end: - # Incremental subtitle writing for merged output - new_entries = [] - self._process_subtitle_tokens( - tokens_with_timestamps, - new_entries, - self.max_subtitle_words, - fallback_end_time=chunk_start + chunk_dur, - ) - if merged_subtitle_file: - subtitle_format = getattr( - self, "subtitle_format", "srt" - ) - if "ass" in subtitle_format: - for start, end, text in new_entries: - start_time = self._ass_time(start) - end_time = self._ass_time(end) - # Use karaoke effect for highlighting mode - effect = ( - "karaoke" - if self.subtitle_mode - == "Sentence + Highlighting" - else "" - ) - merged_subtitle_file.write( - f"Dialogue: 0,{start_time},{end_time},Default,,{merged_subtitle_margin},{merged_subtitle_margin},0,{effect},{merged_subtitle_alignment_tag}{text}\n" - ) - else: - for entry in new_entries: - start, end, text = entry - merged_subtitle_file.write( - f"{merged_srt_index}\n{self._srt_time(start)} --> {self._srt_time(end)}\n{text}\n\n" - ) - merged_srt_index += 1 - # Per-chapter subtitle processing for both file and ffmpeg_proc - if chapter_out_file or chapter_ffmpeg_proc: - new_chapter_entries = [] - self._process_subtitle_tokens( - chapter_tokens_with_timestamps, - new_chapter_entries, - self.max_subtitle_words, - fallback_end_time=chapter_current_time + chunk_dur, - ) - if chapter_subtitle_file: - subtitle_format = getattr( - self, "subtitle_format", "srt" - ) - if "ass" in subtitle_format: - for start, end, text in new_chapter_entries: - start_time = self._ass_time(start) - end_time = self._ass_time(end) - # Use karaoke effect for highlighting mode - effect = ( - "karaoke" - if self.subtitle_mode - == "Sentence + Highlighting" - else "" - ) - chapter_subtitle_file.write( - f"Dialogue: 0,{start_time},{end_time},Default,,{chapter_subtitle_margin},{chapter_subtitle_margin},0,{effect},{chapter_subtitle_alignment_tag}{text}\n" - ) - else: - for entry in new_chapter_entries: - start, end, text = entry - chapter_subtitle_file.write( - f"{chapter_srt_index}\n{self._srt_time(start)} --> {self._srt_time(end)}\n{text}\n\n" - ) - chapter_srt_index += 1 - if merge_chapters_at_end: - current_time += chunk_dur - if chapter_out_file or chapter_ffmpeg_proc: - chapter_current_time += chunk_dur - else: - if chapter_out_file or chapter_ffmpeg_proc: - chapter_current_time += chunk_dur - # Calculate percentage based on characters processed - percent = min( - int( - self.processed_char_count / self.total_char_count * 100 - ), - 99, - ) - - # Calculate ETR based on characters processed - etr_str = "Processing..." - chars_done = self.processed_char_count - elapsed = time.time() - self.etr_start_time - - # Calculate ETR if enough data is available - if ( - chars_done > 0 and elapsed > 0.5 - ): # Check elapsed > 0.5 to avoid instability - avg_time_per_char = elapsed / chars_done - remaining = ( - self.total_char_count - self.processed_char_count - ) - if remaining > 0: - secs = avg_time_per_char * remaining - h = int(secs // 3600) - m = int((secs % 3600) // 60) - s = int(secs % 60) - etr_str = f"{h:02d}:{m:02d}:{s:02d}" - - # Update progress more frequently (after each result) - self.progress_updated.emit(percent, etr_str) - - # Add silence between chapters for merged output (except after the last chapter) - if merge_chapters_at_end and chapter_idx < total_chapters: - silence_samples = int( - self.silence_duration * 24000 - ) # Silence duration at 24,000 Hz - silence_audio = self.np.zeros(silence_samples, dtype="float32") - silence_bytes = silence_audio.tobytes() - - if merged_out_file: - merged_out_file.write(silence_audio) - elif ffmpeg_proc: - ffmpeg_proc.stdin.write(silence_bytes) - - # Update timing for the silence - current_time += self.silence_duration - if chapter_out_file or chapter_ffmpeg_proc: - chapter_current_time += self.silence_duration - - # Set chapter end time after processing - if merge_chapters_at_end: - chapter_time["end"] = current_time - # Finalize chapter file for ffmpeg formats - if chapter_out_file or chapter_ffmpeg_proc: - self.log_updated.emit(("\nProcessing chapter audio...", "grey")) - if chapter_ffmpeg_proc: - chapter_ffmpeg_proc.stdin.close() - chapter_ffmpeg_proc.wait() - if chapter_out_file: - chapter_out_file.close() - # Close chapter subtitle file if open - if chapter_subtitle_file: - chapter_subtitle_file.close() - if ( - save_chapters_separately - and total_chapters > 1 - and self.subtitle_mode != "Disabled" - and chapter_subtitle_path - ): - self.log_updated.emit( - ( - f"\nChapter {chapter_idx} saved to: {chapter_out_path}\n\nChapter subtitle saved to: {chapter_subtitle_path}", - "green", - ) - ) - elif chapter_out_path: - self.log_updated.emit( - ( - f"\nChapter {chapter_idx} saved to: {chapter_out_path}", - "green", - ) - ) - # Finalize merged output file ONLY if merging - if merge_chapters_at_end: - self.log_updated.emit(("\nFinalizing audio. Please wait...", "grey")) - if self.output_format in ["wav", "mp3", "flac"]: - merged_out_file.close() - elif self.output_format == "m4b": - ffmpeg_proc.stdin.close() - ffmpeg_proc.wait() - # Add chapters via fast post-processing - if total_chapters > 1: - chapters_info_path = f"{base_filepath_no_ext}_chapters.txt" - with open(chapters_info_path, "w", encoding="utf-8") as f: - f.write(";FFMETADATA1\n") - for chapter in chapters_time: - chapter_title = chapter["chapter"].replace("=", "\\=") - f.write(f"[CHAPTER]\n") - f.write(f"TIMEBASE=1/1000\n") - f.write(f"START={int(chapter['start']*1000)}\n") - f.write(f"END={int(chapter['end']*1000)}\n") - f.write(f"title={chapter_title}\n\n") - # Fast mux chapters into m4b (write to temp file, then replace original) - static_ffmpeg.add_paths() - orig_path = merged_out_path - root, ext = os.path.splitext(orig_path) - tmp_path = root + ".tmp" + ext - metadata_options, cover_path = ( - self._extract_and_add_metadata_tags_to_ffmpeg_cmd() - ) - cmd = [ - "ffmpeg", - "-y", - "-i", - orig_path, - "-i", - chapters_info_path, - ] - if cover_path and os.path.exists(cover_path): - cmd.extend( - [ - "-i", - cover_path, - "-map", - "0:a", - "-map", - "2", - "-c:v", - "copy", - "-disposition:v", - "attached_pic", - ] - ) - else: - cmd.extend(["-map", "0:a"]) - - cmd.extend( - [ - "-map_metadata", - "1", - "-map_chapters", - "1", - "-c:a", - "copy", - ] - ) - cmd += metadata_options - cmd.append(tmp_path) - proc = create_process(cmd) - proc.wait() - os.replace(tmp_path, orig_path) - os.remove(chapters_info_path) - elif self.output_format in ["opus"]: - ffmpeg_proc.stdin.close() - ffmpeg_proc.wait() - self.progress_updated.emit(100, "00:00:00") - # Close merged subtitle file if open - if merged_subtitle_file: - merged_subtitle_file.close() - # Subtitle and final message logic - if merge_chapters_at_end: - if self.subtitle_mode != "Disabled": - self.conversion_finished.emit( - ( - f"\nAudio saved to: {merged_out_path}\n\nSubtitle saved to: {merged_subtitle_path}", - "green", - ), - merged_out_path, - ) - else: - self.conversion_finished.emit( - (f"\nAudio saved to: {merged_out_path}", "green"), - merged_out_path, - ) - else: - # If not merging, report the folder that holds the chapter files - self.progress_updated.emit(100, "00:00:00") - chapters_dir = os.path.abspath(chapters_out_dir or parent_dir) - self.conversion_finished.emit( - (f"\nAll chapters saved to: {chapters_dir}", "green"), - chapters_dir, - ) - except Exception as e: - # Cleanup ffmpeg subprocesses on error - try: - if "ffmpeg_proc" in locals() and ffmpeg_proc: - ffmpeg_proc.stdin.close() - ffmpeg_proc.terminate() - ffmpeg_proc.wait() - except Exception: - pass - try: - if "chapter_ffmpeg_proc" in locals() and chapter_ffmpeg_proc: - chapter_ffmpeg_proc.stdin.close() - chapter_ffmpeg_proc.terminate() - chapter_ffmpeg_proc.wait() - except Exception: - pass - self.log_updated.emit((f"Error occurred: {str(e)}", "red")) - self.conversion_finished.emit(("Audio generation failed.", "red"), None) - - def _process_subtitle_file(self, tts, base_path, is_timestamp_text=False): - """Process subtitle files with precise timing and generate output subtitles.""" - try: - # Parse subtitle file - if is_timestamp_text: - subtitles = parse_timestamp_text_file(self.file_name) - else: - file_ext = os.path.splitext(self.file_name)[1].lower() - if file_ext == ".srt": - subtitles = parse_srt_file(self.file_name) - elif file_ext == ".vtt": - subtitles = parse_vtt_file(self.file_name) - else: - subtitles = parse_ass_file(self.file_name) - - if not subtitles: - self.log_updated.emit(("No valid subtitle entries found.", "red")) - self.conversion_finished.emit( - ("No subtitle entries to process.", "red"), None - ) - return - - self.log_updated.emit( - (f"\nFound {len(subtitles)} subtitle entries", "grey") - ) - - # Setup output paths - base_name = os.path.splitext(os.path.basename(base_path))[0] - sanitized_base_name = sanitize_name_for_os(base_name, is_folder=True) - parent_dir = ( - user_desktop_dir() - if self.save_option == "Save to Desktop" - else ( - os.path.dirname(base_path) - if self.save_option == "Save next to input file" - else self.output_folder or os.getcwd() - ) - ) - - if not os.path.exists(parent_dir): - self.log_updated.emit( - (f"Output folder does not exist: {parent_dir}", "red") - ) - return - - # Find unique filename - counter = 1 - allowed_exts = set(SUPPORTED_SOUND_FORMATS + SUPPORTED_SUBTITLE_FORMATS) - while True: - suffix = f"_{counter}" if counter > 1 else "" - # Use generator expression to avoid processing all files upfront - file_parts = (os.path.splitext(f) for f in os.listdir(parent_dir)) - if not any( - name == f"{sanitized_base_name}{suffix}" - and ext[1:].lower() in allowed_exts - for name, ext in file_parts - ): - break - counter += 1 - - base_filepath_no_ext = os.path.join( - parent_dir, f"{sanitized_base_name}{suffix}" - ) - merged_out_path = f"{base_filepath_no_ext}.{self.output_format}" - rate = 24000 - - # Setup audio output - merged_out_file, ffmpeg_proc = None, None - if self.output_format in ["wav", "mp3", "flac"]: - merged_out_file = sf.SoundFile( - merged_out_path, - "w", - samplerate=rate, - channels=1, - format=self.output_format, - ) - else: - static_ffmpeg.add_paths() - cmd = [ - "ffmpeg", - "-y", - "-thread_queue_size", - "32768", - "-f", - "f32le", - "-ar", - str(rate), - "-ac", - "1", - "-i", - "pipe:0", - ] - if self.output_format == "m4b": - metadata_options, cover_path = ( - self._extract_and_add_metadata_tags_to_ffmpeg_cmd() - ) - if cover_path and os.path.exists(cover_path): - cmd.extend( - [ - "-i", - cover_path, - "-map", - "0:a", - "-map", - "1", - "-c:v", - "copy", - "-disposition:v", - "attached_pic", - ] - ) - cmd.extend( - [ - "-c:a", - "aac", - "-q:a", - "2", - "-movflags", - "+faststart+use_metadata_tags", - ] - ) - cmd.extend(metadata_options) - elif self.output_format == "opus": - cmd.extend(["-c:a", "libopus", "-b:a", "24000"]) - else: - self.log_updated.emit( - (f"Unsupported output format: {self.output_format}", "red") - ) - return - cmd.append(merged_out_path) - ffmpeg_proc = create_process(cmd, stdin=subprocess.PIPE, text=False) - - # Always generate subtitles for subtitle input files - subtitle_file, subtitle_path = None, None - subtitle_format = getattr(self, "subtitle_format", "srt") - file_extension = "ass" if "ass" in subtitle_format else "srt" - subtitle_path = f"{base_filepath_no_ext}.{file_extension}" - subtitle_file = open(subtitle_path, "w", encoding="utf-8", errors="replace") - - if "ass" in subtitle_format: - # Write ASS header - subtitle_file.write( - "[Script Info]\nTitle: Generated by Abogen\nScriptType: v4.00+\n\n" - ) - if self.subtitle_mode == "Sentence + Highlighting": - subtitle_file.write( - "[V4+ Styles]\nFormat: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding\n" - ) - subtitle_file.write( - "Style: Default,Arial,24,&H00FFFFFF,&H00808080,&H00000000,&H00404040,0,0,0,0,100,100,0,0,3,2,0,5,10,10,10,1\n\n" - ) - subtitle_file.write( - "[Events]\nFormat: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text\n" - ) - - is_narrow = subtitle_format in ("ass_narrow", "ass_centered_narrow") - is_centered = subtitle_format in ( - "ass_centered_wide", - "ass_centered_narrow", - ) - margin = "90" if is_narrow else "" - alignment = "{\\an5}" if is_centered else "" - - # Load voice - loaded_voice = ( - get_new_voice(tts, self.voice, self.use_gpu) - if "*" in self.voice - else self.voice - ) - - # Calculate initial audio buffer size from timed subtitles only - max_end_time = max( - (end for _, end, _ in subtitles if end is not None), default=0 - ) - audio_buffer = self.np.zeros( - int(max_end_time * rate) + rate, dtype="float32" - ) - - # Process each subtitle and mix into buffer - self.etr_start_time = time.time() - srt_index = 1 - - for idx, (start_time, end_time, text) in enumerate(subtitles, 1): - if self.cancel_requested: - if subtitle_file: - subtitle_file.close() - self.conversion_finished.emit("Cancelled", None) - return - - # Process text and timing - replace_nl = getattr(self, "replace_single_newlines", True) - processed_text = text.replace("\n", " ") if replace_nl else text - use_gaps = getattr(self, "use_silent_gaps", False) - next_start = ( - subtitles[idx][0] - if (use_gaps and idx < len(subtitles)) - else float("inf") - ) - subtitle_duration = None if end_time is None else end_time - start_time - - h1, m1, s1 = ( - int(start_time // 3600), - int(start_time % 3600 // 60), - int(start_time % 60), - ) - ms1 = int((start_time - int(start_time)) * 1000) - is_last = ( - is_timestamp_text - or (use_gaps and idx == len(subtitles)) - or end_time is None - ) - if is_last: - time_str = ( - f"{h1:02d}:{m1:02d}:{s1:02d}" - + (f",{ms1:03d}" if ms1 > 0 else "") - + " - AUTO" - ) - else: - h2, m2, s2 = ( - int(end_time // 3600), - int(end_time % 3600 // 60), - int(end_time % 60), - ) - ms2 = int((end_time - int(end_time)) * 1000) - time_str = ( - f"{h1:02d}:{m1:02d}:{s1:02d}" - + (f",{ms1:03d}" if ms1 > 0 else "") - + " - " - + f"{h2:02d}:{m2:02d}:{s2:02d}" - + (f",{ms2:03d}" if ms2 > 0 else "") - ) - self.log_updated.emit( - f"\n[{idx}/{len(subtitles)}] {time_str}: {processed_text}" - ) - - # Generate TTS audio - tts_results = [ - r - for r in tts( - processed_text, - voice=loaded_voice, - speed=self.speed, - split_pattern=None, - ) - if not self.cancel_requested - ] - audio_chunks = [r.audio for r in tts_results] - - if self.cancel_requested: - if subtitle_file: - subtitle_file.close() - self.conversion_finished.emit("Cancelled", None) - return - - # Concatenate audio and determine duration - full_audio = ( - self.np.concatenate( - [a.numpy() if hasattr(a, "numpy") else a for a in audio_chunks] - ) - if audio_chunks - else self.np.zeros( - int((subtitle_duration or 0) * rate), dtype="float32" - ) - ) - audio_duration = len(full_audio) / rate - - # Use actual audio length for timing - if is_timestamp_text: - end_time = start_time + audio_duration - subtitle_duration = audio_duration - elif use_gaps: - end_time = min(start_time + audio_duration, next_start) - subtitle_duration = end_time - start_time - elif subtitle_duration is None: - subtitle_duration = audio_duration - end_time = start_time + audio_duration - - # Speed up if needed - speedup_threshold = ( - next_start - start_time if use_gaps else subtitle_duration - ) - if audio_duration > speedup_threshold: - speed_factor = audio_duration / speedup_threshold - - if getattr(self, "subtitle_speed_method", "tts") == "ffmpeg": - # FFmpeg time-stretch (faster processing) - self.log_updated.emit( - (f" -> FFmpeg time-stretch: {speed_factor:.2f}x", "grey") - ) - - static_ffmpeg.add_paths() - num_stages = max( - 1, - int( - self.np.ceil( - self.np.log(speed_factor) / self.np.log(2.0) - ) - ), - ) - tempo = speed_factor ** (1.0 / num_stages) - filter_str = ",".join([f"atempo={tempo:.6f}"] * num_stages) - - speed_proc = subprocess.Popen( - [ - "ffmpeg", - "-y", - "-f", - "f32le", - "-ar", - str(rate), - "-ac", - "1", - "-i", - "pipe:0", - "-filter:a", - filter_str, - "-f", - "f32le", - "-ar", - str(rate), - "-ac", - "1", - "pipe:1", - ], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - full_audio = self.np.frombuffer( - speed_proc.communicate(input=full_audio.tobytes())[0], - dtype="float32", - ) - audio_duration = len(full_audio) / rate - else: - # TTS regeneration (better quality) - new_speed = self.speed * speed_factor - self.log_updated.emit( - (f" -> Regenerating at {new_speed:.2f}x speed", "grey") - ) - - tts_results = [ - r - for r in tts( - processed_text, - voice=loaded_voice, - speed=new_speed, - split_pattern=None, - ) - if not self.cancel_requested - ] - audio_chunks = [r.audio for r in tts_results] - - full_audio = ( - self.np.concatenate( - [ - a.numpy() if hasattr(a, "numpy") else a - for a in audio_chunks - ] - ) - if audio_chunks - else self.np.zeros( - int(subtitle_duration * rate), dtype="float32" - ) - ) - audio_duration = len(full_audio) / rate - - # Adjust duration after potential speed changes - if use_gaps: - end_time = min(start_time + audio_duration, next_start) - subtitle_duration = end_time - start_time - elif subtitle_duration is None: - subtitle_duration = audio_duration - end_time = start_time + audio_duration - - # Pad or trim to subtitle duration - target_samples = int(subtitle_duration * rate) - if len(full_audio) < target_samples: - full_audio = self.np.concatenate( - [ - full_audio, - self.np.zeros( - target_samples - len(full_audio), dtype="float32" - ), - ] - ) - elif len(full_audio) > target_samples: - full_audio = full_audio[:target_samples] - - # Mix audio into buffer at the correct position (handles overlaps) - start_sample = int(start_time * rate) - end_sample = start_sample + len(full_audio) - if end_sample > len(audio_buffer): - # Extend buffer if needed - audio_buffer = self.np.concatenate( - [ - audio_buffer, - self.np.zeros( - end_sample - len(audio_buffer), dtype="float32" - ), - ] - ) - - # Mix (add) the audio - this handles overlaps by combining them - audio_buffer[start_sample:end_sample] += full_audio - - # Write subtitle - if subtitle_file: - if "ass" in subtitle_format: - effect = ( - "karaoke" - if self.subtitle_mode == "Sentence + Highlighting" - else "" - ) - ass_text = ( - processed_text - if replace_nl - else processed_text.replace("\n", "\\N") - ) - subtitle_file.write( - f"Dialogue: 0,{self._ass_time(start_time)},{self._ass_time(end_time)},Default,,{margin},{margin},0,{effect},{alignment}{ass_text}\n" - ) - else: - subtitle_file.write( - f"{srt_index}\n{self._srt_time(start_time)} --> {self._srt_time(end_time)}\n{processed_text}\n\n" - ) - srt_index += 1 - - # Update progress - percent = min(int(idx / len(subtitles) * 100), 99) - elapsed = time.time() - self.etr_start_time - etr_str = ( - "Processing..." - if elapsed <= 0.5 - else f"{int(elapsed*(len(subtitles)-idx)/idx)//3600:02d}:{(int(elapsed*(len(subtitles)-idx)/idx)%3600)//60:02d}:{int(elapsed*(len(subtitles)-idx)/idx)%60:02d}" - ) - self.progress_updated.emit(percent, etr_str) - - # Normalize audio buffer to prevent clipping from mixed overlaps - max_amplitude = self.np.abs(audio_buffer).max() - if max_amplitude > 1.0: - self.log_updated.emit( - f"\n -> Normalizing audio (peak: {max_amplitude:.2f})" - ) - audio_buffer = audio_buffer / max_amplitude - - # Write the complete audio buffer - self.log_updated.emit(("\nFinalizing audio. Please wait...", "grey")) - if merged_out_file: - merged_out_file.write(audio_buffer) - merged_out_file.close() - elif ffmpeg_proc: - ffmpeg_proc.stdin.write(audio_buffer.astype("float32").tobytes()) - ffmpeg_proc.stdin.close() - ffmpeg_proc.wait() - - if subtitle_file: - subtitle_file.close() - - self.progress_updated.emit(100, "00:00:00") - result_msg = f"\nAudio saved to: {merged_out_path}" + ( - f"\n\nSubtitle saved to: {subtitle_path}" if subtitle_path else "" - ) - self.conversion_finished.emit((result_msg, "green"), merged_out_path) - - except Exception as e: - try: - if "ffmpeg_proc" in locals() and ffmpeg_proc: - ffmpeg_proc.stdin.close() - ffmpeg_proc.terminate() - ffmpeg_proc.wait() - if "subtitle_file" in locals() and subtitle_file: - subtitle_file.close() - except: - pass - self.log_updated.emit((f"Error processing subtitle file: {str(e)}", "red")) - self.conversion_finished.emit(("Audio generation failed.", "red"), None) - - def set_chapter_options(self, options): - """Set chapter options from the dialog and resume processing""" - self.save_chapters_separately = options["save_chapters_separately"] - self.merge_chapters_at_end = options["merge_chapters_at_end"] - self.waiting_for_user_input = False - self._chapter_options_event.set() - - def set_timestamp_response(self, treat_as_subtitle): - """Set whether to treat timestamp text file as subtitle.""" - self._timestamp_response = treat_as_subtitle - self._timestamp_response_event.set() - - def _extract_and_add_metadata_tags_to_ffmpeg_cmd(self): - """Extract metadata tags from text content and add them to ffmpeg command""" - metadata_options = [] - - # Get the input text (either direct or from file) - text = "" - if self.is_direct_text: - text = self.file_name - else: - try: - encoding = detect_encoding(self.file_name) - with open( - self.file_name, "r", encoding=encoding, errors="replace" - ) as file: - text = file.read() - except Exception as e: - self.log_updated.emit( - f"Warning: Could not read file for metadata extraction: {e}" - ) - return [] - - # Extract metadata tags using regex - title_match = re.search(r"<]*)>>", text) - artist_match = re.search(r"<]*)>>", text) - album_match = re.search(r"<]*)>>", text) - year_match = re.search(r"<]*)>>", text) - album_artist_match = re.search(r"<]*)>>", text) - composer_match = re.search(r"<]*)>>", text) - genre_match = re.search(r"<]*)>>", text) - cover_match = re.search(r"<]*)>>", text) - cover_path = cover_match.group(1) if cover_match else None - - # Use display path or filename as fallback for title - - # Use file_name for logs if from_queue, otherwise use display_path if available - if getattr(self, "from_queue", False): - filename = os.path.splitext(os.path.basename(self.file_name))[0] - else: - filename = os.path.splitext( - os.path.basename( - self.display_path if self.display_path else self.file_name - ) - )[0] - - if title_match: - metadata_options.extend(["-metadata", f"title={title_match.group(1)}"]) - else: - metadata_options.extend(["-metadata", f"title={filename}"]) - - # Add artist metadata - if artist_match: - metadata_options.extend(["-metadata", f"artist={artist_match.group(1)}"]) - else: - metadata_options.extend(["-metadata", f"artist=Unknown"]) - - # Add album metadata - if album_match: - metadata_options.extend(["-metadata", f"album={album_match.group(1)}"]) - else: - metadata_options.extend(["-metadata", f"album={filename}"]) - - # Add year metadata - if year_match: - metadata_options.extend(["-metadata", f"date={year_match.group(1)}"]) - else: - # Use current year if year is not specified - import datetime - - current_year = datetime.datetime.now().year - metadata_options.extend(["-metadata", f"date={current_year}"]) - - # Add album artist metadata - if album_artist_match: - metadata_options.extend( - ["-metadata", f"album_artist={album_artist_match.group(1)}"] - ) - else: - metadata_options.extend(["-metadata", f"album_artist=Unknown"]) - - # Add composer metadata - if composer_match: - metadata_options.extend( - ["-metadata", f"composer={composer_match.group(1)}"] - ) - else: - metadata_options.extend(["-metadata", f"composer=Narrator"]) - - # Add genre metadata - if genre_match: - metadata_options.extend(["-metadata", f"genre={genre_match.group(1)}"]) - else: - metadata_options.extend(["-metadata", f"genre=Audiobook"]) - - # Add these to ffmpeg command - return metadata_options, cover_path - - def _srt_time(self, t): - """Helper function to format time for SRT files""" - h = int(t // 3600) - m = int((t % 3600) // 60) - s = int(t % 60) - ms = int((t - int(t)) * 1000) - return f"{h:02d}:{m:02d}:{s:02d},{ms:03d}" - - def _ass_time(self, t): - """Helper function to format time for ASS files""" - h = int(t // 3600) - m = int((t % 3600) // 60) - s = int(t % 60) - cs = int((t - int(t)) * 100) # Centiseconds for ASS format - return f"{h:01d}:{m:02d}:{s:02d}.{cs:02d}" - - def _process_subtitle_tokens( - self, - tokens_with_timestamps, - subtitle_entries, - max_subtitle_words, - fallback_end_time=None, - ): - """Helper function to process subtitle tokens according to the subtitle mode""" - if not tokens_with_timestamps: - return - - processed_tokens = tokens_with_timestamps # Use tokens directly - - # For English with spaCy enabled and sentence-based modes, use spaCy for sentence boundaries - # spaCy is disabled when subtitle mode is "Disabled" or "Line" - use_spacy_for_english = ( - getattr(self, "use_spacy_segmentation", False) - and self.subtitle_mode not in ["Disabled", "Line"] - and self.lang_code in ["a", "b"] - and self.subtitle_mode in ["Sentence", "Sentence + Comma"] - ) - # Use processed_tokens instead of tokens_with_timestamps for the rest of the method - if self.subtitle_mode == "Sentence + Highlighting": - # Sentence-based processing with karaoke highlighting - # Use punctuation without comma - separator = r"[{}]".format(self.PUNCTUATION_SENTENCE) - current_sentence = [] - word_count = 0 - - for token in processed_tokens: # Updated to use processed_tokens - current_sentence.append(token) - word_count += 1 - - # Split sentences based on separator or word count - if ( - re.search(separator, token["text"]) and token["whitespace"] == " " - ) or word_count >= max_subtitle_words: - if current_sentence: - # Create karaoke subtitle entry for this sentence - start_time = current_sentence[0]["start"] - end_time = current_sentence[-1]["end"] - - # Generate karaoke text with background highlighting - karaoke_text = "" - for t in current_sentence: - # Calculate duration in centiseconds - duration = ( - t["end"] - t["start"] - if t["end"] and t["start"] - else 0.5 - ) - duration_cs = int(duration * 100) - # Add karaoke effect - relies on style's SecondaryColour for highlighting - karaoke_text += f"{{\\kf{duration_cs}}}{t['text']}{t.get('whitespace', '') or ''}" - - subtitle_entries.append( - (start_time, end_time, karaoke_text.strip()) - ) - current_sentence = [] - word_count = 0 - - # Add any remaining tokens as a sentence - if current_sentence: - start_time = current_sentence[0]["start"] - end_time = current_sentence[-1]["end"] - - # Generate karaoke text for remaining tokens - karaoke_text = "" - for t in current_sentence: - duration = t["end"] - t["start"] if t["end"] and t["start"] else 0.5 - duration_cs = int(duration * 100) - karaoke_text += f"{{\\kf{duration_cs}}}{t['text']}{t.get('whitespace', '') or ''}" - subtitle_entries.append((start_time, end_time, karaoke_text.strip())) - - # Fallback for last entry - if subtitle_entries and fallback_end_time is not None: - last_entry = subtitle_entries[-1] - start, end, text = last_entry - if end is None or end <= start or end <= 0: - subtitle_entries[-1] = (start, fallback_end_time, text) - - elif self.subtitle_mode in ["Sentence", "Sentence + Comma", "Line"]: - # Check if we should use spaCy for English sentence boundaries - if use_spacy_for_english and self.subtitle_mode != "Line": - # Use spaCy for English sentence boundary detection (model already loaded) - from abogen.spacy_utils import get_spacy_model - - nlp = get_spacy_model( - self.lang_code - ) # No log_callback since model is already loaded - if nlp: - # Build full text and track character positions to token indices - full_text = "" - char_to_token = [] # Maps character index to token index - for idx, token in enumerate(processed_tokens): - start_char = len(full_text) - text_part = token["text"] + (token.get("whitespace", "") or "") - full_text += text_part - char_to_token.extend([idx] * len(text_part)) - - # Get sentence boundaries from spaCy - doc = nlp(full_text) - sentence_boundaries = [sent.end_char for sent in doc.sents] - - # For "Sentence + Comma" mode, also split on commas - if self.subtitle_mode == "Sentence + Comma": - comma_positions = [ - i + 1 for i, c in enumerate(full_text) if c == "," - ] - sentence_boundaries = sorted( - set(sentence_boundaries + comma_positions) - ) - - # Group tokens by sentence boundaries - current_sentence = [] - word_count = 0 - current_char_pos = 0 - boundary_idx = 0 - - for idx, token in enumerate(processed_tokens): - current_sentence.append(token) - word_count += 1 - text_len = len(token["text"]) + len( - token.get("whitespace", "") or "" - ) - current_char_pos += text_len - - # Check if we've hit a sentence boundary or max words - at_boundary = ( - boundary_idx < len(sentence_boundaries) - and current_char_pos >= sentence_boundaries[boundary_idx] - ) - if at_boundary or word_count >= max_subtitle_words: - if current_sentence: - start_time = current_sentence[0]["start"] - end_time = current_sentence[-1]["end"] - sentence_text = "".join( - t["text"] + (t.get("whitespace", "") or "") - for t in current_sentence - ) - subtitle_entries.append( - (start_time, end_time, sentence_text.strip()) - ) - current_sentence = [] - word_count = 0 - if at_boundary: - boundary_idx += 1 - - # Add remaining tokens - if current_sentence: - start_time = current_sentence[0]["start"] - end_time = current_sentence[-1]["end"] - sentence_text = "".join( - t["text"] + (t.get("whitespace", "") or "") - for t in current_sentence - ) - subtitle_entries.append( - (start_time, end_time, sentence_text.strip()) - ) - - # Fallback for last entry - if subtitle_entries and fallback_end_time is not None: - last_entry = subtitle_entries[-1] - start, end, text = last_entry - if end is None or end <= start or end <= 0: - subtitle_entries[-1] = (start, fallback_end_time, text) - return # Exit early, spaCy processing complete - - # Default regex-based processing (non-English or spaCy unavailable) - # Define separator pattern based on mode - if self.subtitle_mode == "Line": - separator = r"\n" - elif self.subtitle_mode == "Sentence": - # Use punctuation without comma - separator = r"[{}]".format(self.PUNCTUATION_SENTENCE) - else: # Sentence + Comma - # Use punctuation with comma - separator = r"[{}]".format(self.PUNCTUATION_SENTENCE_COMMA) - current_sentence = [] - word_count = 0 - - for token in processed_tokens: # Updated to use processed_tokens - current_sentence.append(token) - word_count += 1 - - # Split sentences based on separator or word count - if ( - re.search(separator, token["text"]) and token["whitespace"] == " " - ) or word_count >= max_subtitle_words: - if current_sentence: - # Create subtitle entry for this sentence - start_time = current_sentence[0]["start"] - end_time = current_sentence[-1]["end"] - - # Simplified text joining logic - sentence_text = "" - for t in current_sentence: - sentence_text += t["text"] + (t.get("whitespace", "") or "") - - subtitle_entries.append( - (start_time, end_time, sentence_text.strip()) - ) - current_sentence = [] - word_count = 0 - - # Add any remaining tokens as a sentence - if current_sentence: - start_time = current_sentence[0]["start"] - end_time = current_sentence[-1]["end"] - - # Simplified text joining logic - sentence_text = "" - for t in current_sentence: - sentence_text += t["text"] + (t.get("whitespace", "") or "") - subtitle_entries.append((start_time, end_time, sentence_text.strip())) - - # Fallback for last entry - if subtitle_entries and fallback_end_time is not None: - last_entry = subtitle_entries[-1] - start, end, text = last_entry - if end is None or end <= start or end <= 0: - subtitle_entries[-1] = (start, fallback_end_time, text) - - else: - # Word count-based grouping - simply count spaces and split after N spaces - try: - word_count = int(self.subtitle_mode.split()[0]) - word_count = min(word_count, max_subtitle_words) - except (ValueError, IndexError): - word_count = 1 - - current_group = [] - space_count = 0 - - for token in processed_tokens: - current_group.append(token) - - # Count spaces after tokens (in the whitespace field) - if token.get("whitespace", "") == " ": - space_count += 1 - - # Split after counting N spaces - if space_count >= word_count: - text = "".join( - t["text"] + (t.get("whitespace", "") or "") - for t in current_group - ) - subtitle_entries.append( - ( - current_group[0]["start"], - current_group[-1]["end"], - text.strip(), - ) - ) - current_group = [] - space_count = 0 - - # Add any remaining tokens - if current_group: - text = "".join( - t["text"] + (t.get("whitespace", "") or "") for t in current_group - ) - subtitle_entries.append( - (current_group[0]["start"], current_group[-1]["end"], text.strip()) - ) - - # Fallback for last entry - if subtitle_entries and fallback_end_time is not None: - last_entry = subtitle_entries[-1] - start, end, text = last_entry - if end is None or end <= start or end <= 0: - subtitle_entries[-1] = (start, fallback_end_time, text) - - def cancel(self): - self.cancel_requested = True - self.should_cancel = True - self.waiting_for_user_input = False - # Terminate subprocess if running - if self.process: - try: - self.process.terminate() - except Exception: - pass - # Terminate ffmpeg subprocesses if running - try: - if hasattr(self, "ffmpeg_proc") and self.ffmpeg_proc: - self.ffmpeg_proc.stdin.close() - self.ffmpeg_proc.terminate() - self.ffmpeg_proc.wait() - except Exception: - pass - try: - if hasattr(self, "chapter_ffmpeg_proc") and self.chapter_ffmpeg_proc: - self.chapter_ffmpeg_proc.stdin.close() - self.chapter_ffmpeg_proc.terminate() - self.chapter_ffmpeg_proc.wait() - except Exception: - pass - - -class VoicePreviewThread(QThread): - finished = pyqtSignal() - error = pyqtSignal(str) - - def __init__( - self, - np_module, - kpipeline_class, - lang_code, - voice, - speed, - use_gpu=False, - parent=None, - ): - super().__init__(parent) - self.np_module = np_module - self.kpipeline_class = kpipeline_class - self.lang_code = lang_code - self.voice = voice - self.speed = speed - self.use_gpu = use_gpu - - # Cache location for preview audio - self.cache_dir = get_user_cache_path("preview_cache") - - # Calculate cache path - self.cache_path = self._get_cache_path() - - def _get_cache_path(self): - """Generate a unique filename for the voice with its parameters""" - # For a voice formula, use a hash of the formula - if "*" in self.voice: - voice_id = ( - f"voice_formula_{hashlib.md5(self.voice.encode()).hexdigest()[:8]}" - ) - else: - voice_id = self.voice - - # Create a unique filename based on voice_id, language, and speed - filename = f"{voice_id}_{self.lang_code}_{self.speed:.2f}.wav" - return os.path.join(self.cache_dir, filename) - - def run(self): - print( - f"\nVoice: {self.voice}\nLanguage: {self.lang_code}\nSpeed: {self.speed}\nGPU: {self.use_gpu}\n" - ) - - # Generate the preview and save to cache - try: - - # Set device based on use_gpu setting and platform - if self.use_gpu: - if platform.system() == "Darwin" and platform.processor() == "arm": - device = "mps" # Use MPS for Apple Silicon - else: - device = "cuda" # Use CUDA for other platforms - else: - device = "cpu" - - tts = self.kpipeline_class( - lang_code=self.lang_code, repo_id="hexgrad/Kokoro-82M", device=device - ) - # Enable voice formula support for preview - if "*" in self.voice: - loaded_voice = get_new_voice(tts, self.voice, self.use_gpu) - else: - loaded_voice = self.voice - sample_text = get_sample_voice_text(self.lang_code) - audio_segments = [] - for result in tts( - sample_text, voice=loaded_voice, speed=self.speed, split_pattern=None - ): - audio_segments.append(result.audio) - if audio_segments: - audio = self.np_module.concatenate(audio_segments) - # Save directly to the cache path - sf.write(self.cache_path, audio, 24000) - self.temp_wav = self.cache_path - self.finished.emit() - except Exception as e: - self.error.emit(f"Voice preview error: {str(e)}") - - -class PlayAudioThread(QThread): - finished = pyqtSignal() - error = pyqtSignal(str) - - def __init__(self, wav_path, parent=None): - super().__init__(parent) - self.wav_path = wav_path - self.is_canceled = False - - def run(self): - try: - import pygame - import time as _time - - pygame.mixer.init() - pygame.mixer.music.load(self.wav_path) - pygame.mixer.music.play() - # Wait until playback is finished or canceled - while pygame.mixer.music.get_busy() and not self.is_canceled: - _time.sleep(0.2) - - # Make sure to clean up regardless of how we exited the loop - try: - pygame.mixer.music.stop() - pygame.mixer.music.unload() - pygame.mixer.quit() # Quit the mixer - except Exception: - # Ignore any errors during cleanup - pass - - self.finished.emit() - except Exception as e: - # Handle initialization errors separately to give better error messages - if "mixer not initialized" in str(e): - self.error.emit( - "Audio playback error: The audio system was not properly initialized" - ) - else: - self.error.emit(f"Audio playback error: {str(e)}") - - def stop(self): - """Safely stop playback""" - self.is_canceled = True - # Try to stop pygame if it's running, but catch all exceptions - try: - import pygame - - if pygame.mixer.get_init(): - if pygame.mixer.music.get_busy(): - pygame.mixer.music.stop() - pygame.mixer.music.unload() - except Exception: - # Ignore all errors when stopping since mixer might not be initialized - pass +__all__ = ["ConversionThread", "VoicePreviewThread", "PlayAudioThread"] diff --git a/abogen/debug_tts_samples.py b/abogen/debug_tts_samples.py new file mode 100644 index 0000000..f881101 --- /dev/null +++ b/abogen/debug_tts_samples.py @@ -0,0 +1,390 @@ +from __future__ import annotations + +import tempfile +import zipfile +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable, List, Sequence + + +MARKER_PREFIX = "[[ABOGEN-DBG:" +MARKER_SUFFIX = "]]" + + +@dataclass(frozen=True) +class DebugTTSSample: + code: str + label: str + text: str + + +DEBUG_TTS_SAMPLES: Sequence[DebugTTSSample] = ( + DebugTTSSample( + code="APOS_001", + label="Apostrophes & contractions (1)", + text="It's a beautiful day, isn't it? Let's see what we'll do.", + ), + DebugTTSSample( + code="APOS_002", + label="Apostrophes & contractions (2)", + text="I'm sure you're ready; we'd better go before it's too late.", + ), + DebugTTSSample( + code="APOS_003", + label="Apostrophes & contractions (3)", + text="He'll say it's fine, but I can't promise it'll work.", + ), + DebugTTSSample( + code="APOS_004", + label="Apostrophes & contractions (4)", + text="They've done it, and I'd agree they've earned it.", + ), + DebugTTSSample( + code="APOS_005", + label="Apostrophes & contractions (5)", + text="She's here, we're late, they're waiting, and you're right.", + ), + DebugTTSSample( + code="POS_001", + label="Plural possessives (1)", + text="The dogs' bowls were empty, but the boss's office was quiet.", + ), + DebugTTSSample( + code="POS_002", + label="Plural possessives (2)", + text="The teachers' lounge was closed during the students' exams.", + ), + DebugTTSSample( + code="POS_003", + label="Plural possessives (3)", + text="The actresses' roles changed, and the directors' notes piled up.", + ), + DebugTTSSample( + code="POS_004", + label="Plural possessives (4)", + text="The Joneses' car was parked by the neighbors' fence.", + ), + DebugTTSSample( + code="POS_005", + label="Plural possessives (5)", + text="The bosses' meeting ended before the witnesses' statements began.", + ), + DebugTTSSample( + code="NUM_001", + label="Grouped numbers (1)", + text="There are 1,234 apples, 56 oranges, and 7.89 liters of juice.", + ), + DebugTTSSample( + code="NUM_002", + label="Grouped numbers (2)", + text="The population is 10,000,000 and the area is 123.45 square miles.", + ), + DebugTTSSample( + code="NUM_003", + label="Grouped numbers (3)", + text="Set the timer for 0.5 seconds, then wait 2.0 minutes.", + ), + DebugTTSSample( + code="NUM_004", + label="Grouped numbers (4)", + text="We measured 3.1415 radians and wrote down 2,718.28 as well.", + ), + DebugTTSSample( + code="NUM_005", + label="Grouped numbers (5)", + text="The sequence is 1, 2, 3, 4, 5, and then 13.", + ), + DebugTTSSample( + code="YEAR_001", + label="Years and decades (1)", + text="In 1999, people said the '90s were over.", + ), + DebugTTSSample( + code="YEAR_002", + label="Years and decades (2)", + text="In 2001, the show premiered; by 2010 it was everywhere.", + ), + DebugTTSSample( + code="YEAR_003", + label="Years and decades (3)", + text="The 1980s were loud, and the 1970s were groovy.", + ), + DebugTTSSample( + code="YEAR_004", + label="Years and decades (4)", + text="She loved the '80s, but he preferred the '60s.", + ), + DebugTTSSample( + code="YEAR_005", + label="Years and decades (5)", + text="In 2024, we looked back at 2020 and planned for 2030.", + ), + DebugTTSSample( + code="DATE_001", + label="Dates (1)", + text="On 2023-01-01, we celebrated the new year.", + ), + DebugTTSSample( + code="DATE_002", + label="Dates (2)", + text="The deadline is 1999-12-31 at midnight.", + ), + DebugTTSSample( + code="DATE_003", + label="Dates (3)", + text="Leap day happens on 2024-02-29.", + ), + DebugTTSSample( + code="DATE_004", + label="Dates (4)", + text="Some formats look like 01/02/2003 and can be ambiguous.", + ), + DebugTTSSample( + code="DATE_005", + label="Dates (5)", + text="We met on March 5, 2020 and again on Apr. 7, 2021.", + ), + DebugTTSSample( + code="CUR_001", + label="Currency symbols (1)", + text="The price is $10.50, but it was £8.00 yesterday.", + ), + DebugTTSSample( + code="CUR_002", + label="Currency symbols (2)", + text="Tickets cost €12, and the fine was $0.99.", + ), + DebugTTSSample( + code="CUR_003", + label="Currency symbols (3)", + text="The bill was ¥500 and the refund was $-3.25.", + ), + DebugTTSSample( + code="CUR_004", + label="Currency symbols (4)", + text="He paid £1,234.56 for the instrument.", + ), + DebugTTSSample( + code="CUR_005", + label="Currency symbols (5)", + text="The subscription is $5 per month, or $50 per year.", + ), + DebugTTSSample( + code="TITLE_001", + label="Titles and abbreviations (1)", + text="Dr. Smith lives on Elm St. near the U.S. border.", + ), + DebugTTSSample( + code="TITLE_002", + label="Titles and abbreviations (2)", + text="Mr. and Mrs. Doe met Prof. Adams at 5 p.m.", + ), + DebugTTSSample( + code="TITLE_003", + label="Titles and abbreviations (3)", + text="Gen. Smith spoke to Sgt. Rivera on Main St.", + ), + DebugTTSSample( + code="TITLE_004", + label="Titles and abbreviations (4)", + text="The report came from the U.K. office, not the U.S.A. team.", + ), + DebugTTSSample( + code="TITLE_005", + label="Titles and abbreviations (5)", + text="St. John's is different from St. Louis.", + ), + DebugTTSSample( + code="PUNC_001", + label="Terminal punctuation (1)", + text="This sentence ends without punctuation", + ), + DebugTTSSample( + code="PUNC_002", + label="Terminal punctuation (2)", + text="An ellipsis is already present...", + ), + DebugTTSSample( + code="PUNC_003", + label="Terminal punctuation (3)", + text="A question without a mark", + ), + DebugTTSSample( + code="PUNC_004", + label="Terminal punctuation (4)", + text="An exclamation without a bang", + ), + DebugTTSSample( + code="PUNC_005", + label="Terminal punctuation (5)", + text='A quote ends here"', + ), + DebugTTSSample( + code="QUOTE_001", + label="ALL CAPS inside quotes (1)", + text='He shouted, "THIS IS IMPORTANT!" and then whispered, "ok."', + ), + DebugTTSSample( + code="QUOTE_002", + label="ALL CAPS inside quotes (2)", + text='She said, "NO WAY", but he replied, "maybe".', + ), + DebugTTSSample( + code="QUOTE_003", + label="ALL CAPS inside quotes (3)", + text='The sign read "DO NOT ENTER" and the note read "pls knock".', + ), + DebugTTSSample( + code="QUOTE_004", + label="ALL CAPS inside quotes (4)", + text='He muttered, "OK", then yelled, "STOP!"', + ), + DebugTTSSample( + code="QUOTE_005", + label="ALL CAPS inside quotes (5)", + text='They chanted, "USA!" and someone wrote "idk".', + ), + DebugTTSSample( + code="FOOT_001", + label="Footnote indicators (1)", + text="This is a sentence with a footnote[1] and another[12].", + ), + DebugTTSSample( + code="FOOT_002", + label="Footnote indicators (2)", + text="Some books use multiple footnotes like this[2][3] in a row.", + ), + DebugTTSSample( + code="FOOT_003", + label="Footnote indicators (3)", + text="A footnote can appear mid-sentence[4] and continue afterward.", + ), + DebugTTSSample( + code="FOOT_004", + label="Footnote indicators (4)", + text="Edge cases include [0] or very large indices like [1234].", + ), + DebugTTSSample( + code="FOOT_005", + label="Footnote indicators (5)", + text="Sometimes a footnote follows punctuation.[5] Sometimes it doesn't[6]", + ), +) + + +def marker_for(code: str) -> str: + return f"{MARKER_PREFIX}{code}{MARKER_SUFFIX}" + + +def build_debug_epub(dest_path: Path, *, title: str = "abogen debug samples") -> Path: + """Create a tiny EPUB containing all debug samples. + + The text includes stable marker codes so developers can report failures + precisely. + """ + + dest_path = Path(dest_path) + dest_path.parent.mkdir(parents=True, exist_ok=True) + + chapter_lines: List[str] = [ + '', + "", + '', + "", + f" {title}", + ' ', + "", + "", + f"

    {title}

    ", + "

    Each paragraph begins with a stable debug code marker.

    ", + ] + + for sample in DEBUG_TTS_SAMPLES: + safe_label = sample.label.replace("&", "and") + chapter_lines.append(f"

    {safe_label}

    ") + chapter_lines.append( + "

    " + + marker_for(sample.code) + + " " + + sample.text + + "

    " + ) + + chapter_lines += ["", ""] + chapter_xhtml = "\n".join(chapter_lines) + + container_xml = """ + + + + + +""" + + content_opf = """ + + + abogen-debug-samples + abogen debug samples + en + + + + + + + + + +""" + + nav_xhtml = """ + + + + Navigation + + + + + + +""" + + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + (tmp_path / "mimetype").write_text("application/epub+zip", encoding="utf-8") + meta_inf = tmp_path / "META-INF" + meta_inf.mkdir(parents=True, exist_ok=True) + (meta_inf / "container.xml").write_text(container_xml, encoding="utf-8") + oebps = tmp_path / "OEBPS" + oebps.mkdir(parents=True, exist_ok=True) + (oebps / "content.opf").write_text(content_opf, encoding="utf-8") + (oebps / "chapter.xhtml").write_text(chapter_xhtml, encoding="utf-8") + (oebps / "nav.xhtml").write_text(nav_xhtml, encoding="utf-8") + + # Per EPUB spec: mimetype must be the first entry and stored (no compression). + with zipfile.ZipFile(dest_path, "w") as zf: + zf.write( + tmp_path / "mimetype", "mimetype", compress_type=zipfile.ZIP_STORED + ) + for source in ( + meta_inf / "container.xml", + oebps / "content.opf", + oebps / "chapter.xhtml", + oebps / "nav.xhtml", + ): + arcname = str(source.relative_to(tmp_path)).replace("\\", "/") + zf.write(source, arcname, compress_type=zipfile.ZIP_DEFLATED) + + return dest_path + + +def iter_expected_codes() -> Iterable[str]: + for sample in DEBUG_TTS_SAMPLES: + yield sample.code diff --git a/abogen/entity_analysis.py b/abogen/entity_analysis.py new file mode 100644 index 0000000..4c0b561 --- /dev/null +++ b/abogen/entity_analysis.py @@ -0,0 +1,489 @@ +from __future__ import annotations + +import hashlib +import os +import re +import threading +import time +from collections import Counter +from dataclasses import dataclass, field +from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple + +try: # pragma: no cover - fallback when spaCy not available during tests + import spacy # type: ignore[import-not-found] +except Exception: # pragma: no cover - spaCy optional during runtime bootstrap + spacy = None + +_Language = Any # type: ignore[misc,assignment] +Doc = Any # type: ignore[misc,assignment] +Span = Any # type: ignore[misc,assignment] + + +_TITLE_PREFIXES = ( + "mr", + "mrs", + "ms", + "miss", + "dr", + "prof", + "sir", + "madam", + "lady", + "lord", + "capt", + "captain", + "col", + "colonel", + "maj", + "major", + "sgt", + "sergeant", + "rev", + "father", + "mother", + "brother", + "sister", +) + +_STOP_LABELS = { + "the", + "that", + "this", + "those", + "these", + "there", + "here", + "then", + "and", + "but", + "or", + "nor", + "so", + "yet", + "dr", + "mr", + "mrs", + "ms", + "miss", + "sir", + "madam", + "lady", + "lord", +} + +_EXCLUDED_NER_LABELS = { + "CARDINAL", + "DATE", + "ORDINAL", + "PERCENT", + "TIME", + "LAW", + "MONEY", + "QUANTITY", +} + +_TITLE_PATTERN = re.compile( + r"^(?:" + "|".join(re.escape(prefix) for prefix in _TITLE_PREFIXES) + r")\.?\s+", + re.IGNORECASE, +) +_POSSESSIVE_PATTERN = re.compile(r"(?:'s|’s|\u2019s)$", re.IGNORECASE) +_NON_WORD_PATTERN = re.compile(r"[^\w\s'-]+") +_MULTI_SPACE_PATTERN = re.compile(r"\s+") +_SUFFIX_PATTERN = re.compile( + r",?\s+(?:jr|sr|ii|iii|iv|v|vi|md|phd|esq|esquire|dds|dvm)\.?$", + re.IGNORECASE, +) + + +@dataclass(slots=True) +class EntityRecord: + key: Tuple[str, str] + label: str + kind: str + category: str + count: int = 0 + samples: List[Dict[str, Any]] = field(default_factory=list) + chapter_indices: set[int] = field(default_factory=set) + forms: Counter = field(default_factory=Counter) + first_position: Optional[Tuple[int, int]] = None + + def register( + self, *, chapter_index: int, position: int, text: str, sentence: Optional[str] + ) -> None: + self.count += 1 + self.chapter_indices.add(chapter_index) + self.forms[text] += 1 + if self.first_position is None: + self.first_position = (chapter_index, position) + if sentence and len(self.samples) < 5: + payload = { + "excerpt": sentence.strip(), + "chapter_index": chapter_index, + } + if payload not in self.samples: + self.samples.append(payload) + + def as_dict(self, ordinal: int) -> Dict[str, Any]: + chapter_indices = sorted(self.chapter_indices) + first_chapter = chapter_indices[0] if chapter_indices else None + return { + "id": f"{self.category}_{ordinal}", + "label": self.label, + "normalized": self.key[1], + "category": self.category, + "kind": self.kind, + "count": self.count, + "samples": list(self.samples), + "chapter_indices": chapter_indices, + "first_chapter": first_chapter, + "forms": self.forms.most_common(6), + } + + +@dataclass(slots=True) +class EntityExtractionResult: + summary: Dict[str, Any] + cache_key: str + elapsed: float + errors: List[str] + + +class EntityModelError(RuntimeError): + pass + + +_MODEL_CACHE: Dict[str, Any] = {} +_MODEL_LOCK = threading.RLock() + + +def _resolve_model_name(language: str) -> str: + override = os.environ.get("ABOGEN_SPACY_MODEL") + if override: + return override.strip() + lowered = language.strip().lower() + if lowered.startswith("en"): + return "en_core_web_sm" + return "en_core_web_sm" + + +def _load_model(language: str) -> Any: + if spacy is None: + raise EntityModelError( + "spaCy is not available. Install spaCy to enable entity extraction." + ) + + model_name = _resolve_model_name(language) + cache_key = model_name.lower() + with _MODEL_LOCK: + if cache_key in _MODEL_CACHE: + return _MODEL_CACHE[cache_key] + try: + nlp = spacy.load(model_name) # type: ignore[arg-type] + except OSError as exc: # pragma: no cover - external dependency failure + raise EntityModelError( + f"spaCy model '{model_name}' is not installed. Download it with " + "`python -m spacy download en_core_web_sm`." + ) from exc + nlp.max_length = max(nlp.max_length, 2_000_000) + _MODEL_CACHE[cache_key] = nlp + return nlp + + +def _normalize_label(text: str) -> str: + if not text: + return "" + stripped = text.strip().strip("\"'`“”’") + if not stripped: + return "" + stripped = _TITLE_PATTERN.sub("", stripped) + stripped = _SUFFIX_PATTERN.sub("", stripped) + stripped = _POSSESSIVE_PATTERN.sub("", stripped) + stripped = _NON_WORD_PATTERN.sub(" ", stripped) + stripped = _MULTI_SPACE_PATTERN.sub(" ", stripped) + stripped = stripped.strip() + if not stripped or stripped.lower() in _STOP_LABELS: + return "" + parts = stripped.split() + if not parts: + return "" + if len(parts) == 1 and len(parts[0]) <= 1: + return "" + # Normalise casing: preserve uppercase abbreviations, otherwise title case. + normalized_parts = [] + for index, part in enumerate(parts): + if part.isupper(): + normalized_parts.append(part) + elif part[:1].isupper(): + normalized_parts.append(part[:1].upper() + part[1:]) + elif index == 0: + normalized_parts.append(part[:1].upper() + part[1:]) + else: + normalized_parts.append(part) + normalized = " ".join(normalized_parts).strip() + if normalized.lower() in _STOP_LABELS: + return "" + return normalized + + +def _token_key(value: str) -> str: + return _MULTI_SPACE_PATTERN.sub(" ", value.lower().strip()).strip() + + +def _iter_named_entities(doc: Any) -> Iterable[Any]: # type: ignore[override] + for ent in getattr(doc, "ents", ()): + if ent.label_ == "": + continue + yield ent + + +def _extract_propn_tokens(doc: Any) -> Iterable[Any]: # type: ignore[override] + seen: set[Tuple[int, int]] = set() + for ent in getattr(doc, "ents", ()): # guard multi-token spans + seen.add((ent.start, ent.end)) + for token in doc: + if token.pos_ != "PROPN": + continue + span_key = (token.i, token.i + 1) + if span_key in seen: + continue + if token.is_stop: + continue + text = token.text.strip() + if not text: + continue + if token.ent_type_: + continue + yield doc[token.i : token.i + 1] + + +def _empty_result( + cache_key: str, error: Optional[str] = None +) -> EntityExtractionResult: + payload = { + "people": [], + "entities": [], + "index": {"tokens": []}, + "stats": { + "tokens": 0, + "chapters": 0, + "processed": False, + }, + "model": None, + } + errors = [error] if error else [] + return EntityExtractionResult( + summary=payload, cache_key=cache_key, elapsed=0.0, errors=errors + ) + + +def extract_entities( + chapters: Iterable[Mapping[str, Any]], + *, + language: str = "en", +) -> EntityExtractionResult: + start = time.perf_counter() + normalized_language = language or "en" + combined_hasher = hashlib.sha1() + chapter_texts: List[Tuple[int, str]] = [] + for idx, chapter in enumerate(chapters): + text = chapter.get("text") if isinstance(chapter, Mapping) else None + text_value = str(text or "") + original_index = idx + if isinstance(chapter, Mapping): + try: + original_index = int(chapter.get("index", idx)) + except (TypeError, ValueError): + original_index = idx + chapter_texts.append((original_index, text_value)) + if text_value: + combined_hasher.update(text_value.encode("utf-8", "ignore")) + combined_hasher.update(str(original_index).encode("utf-8", "ignore")) + cache_key = combined_hasher.hexdigest() + + if not chapter_texts: + return _empty_result(cache_key) + + try: + nlp = _load_model(normalized_language) + except EntityModelError as exc: + return _empty_result(cache_key, str(exc)) + + records: Dict[Tuple[str, str], EntityRecord] = {} + tokens_for_index: Dict[str, Dict[str, Any]] = {} + processed_tokens = 0 + + for chapter_index, text in chapter_texts: + trimmed = text.strip() + if not trimmed: + continue + if len(trimmed) + 1024 > nlp.max_length: + nlp.max_length = len(trimmed) + 1024 + doc = nlp(trimmed) + + def _register_span(span: Any, category_hint: Optional[str] = None) -> None: + nonlocal processed_tokens + if category_hint is None and span.label_ in _EXCLUDED_NER_LABELS: + return + cleaned = _normalize_label(span.text) + if not cleaned: + return + key = _token_key(cleaned) + if not key: + return + category = category_hint or ( + "people" if span.label_ == "PERSON" else "entities" + ) + record_key = (category, key) + record = records.get(record_key) + if record is None: + record = EntityRecord( + key=record_key, + label=cleaned, + kind=span.label_ + or ("PROPN" if category == "entities" else "PERSON"), + category=category, + ) + records[record_key] = record + sentence = ( + span.sent.text + if hasattr(span, "sent") and span.sent is not None + else None + ) + record.register( + chapter_index=chapter_index, + position=span.start, + text=span.text, + sentence=sentence, + ) + processed_tokens += 1 + index_entry = tokens_for_index.get(key) + if index_entry is None: + index_entry = { + "token": record.label, + "normalized": key, + "category": category, + "count": 0, + "samples": [], + } + tokens_for_index[key] = index_entry + index_entry["count"] += 1 + if sentence and len(index_entry["samples"]) < 3: + if sentence not in index_entry["samples"]: + index_entry["samples"].append(sentence) + + for ent in _iter_named_entities(doc): + _register_span(ent) + + for span in _extract_propn_tokens(doc): + _register_span(span, category_hint="entities") + + elapsed = time.perf_counter() - start + + people_records = [ + record for record in records.values() if record.category == "people" + ] + people_keys = {record.key[1] for record in people_records} + entity_records = [ + record + for record in records.values() + if record.category == "entities" + and record.key[1] not in people_keys + and record.kind != "PERSON" + ] + + people_records.sort(key=lambda rec: (-rec.count, rec.label)) + entity_records.sort(key=lambda rec: (-rec.count, rec.label)) + + people_payload = [ + record.as_dict(index + 1) for index, record in enumerate(people_records) + ] + entity_payload = [ + record.as_dict(index + 1) for index, record in enumerate(entity_records) + ] + + index_payload = sorted( + tokens_for_index.values(), key=lambda item: (-item["count"], item["token"]) + ) + + summary = { + "people": people_payload, + "entities": entity_payload, + "index": {"tokens": index_payload}, + "stats": { + "tokens": processed_tokens, + "chapters": len(chapter_texts), + "processed": True, + "people": len(people_payload), + "entities": len(entity_payload), + }, + "model": { + "name": getattr(nlp, "meta", {}).get("name", "unknown"), + "version": getattr(nlp, "meta", {}).get("version", "unknown"), + "lang": getattr(nlp, "meta", {}).get("lang", normalized_language), + }, + } + + return EntityExtractionResult( + summary=summary, cache_key=cache_key, elapsed=elapsed, errors=[] + ) + + +def search_tokens( + index: Mapping[str, Any], query: str, *, limit: int = 15 +) -> List[Dict[str, Any]]: + tokens = index.get("tokens") if isinstance(index, Mapping) else None + if not isinstance(tokens, list) or not query: + return [] + normalized = query.strip().lower() + if not normalized: + return tokens[:limit] + results: List[Dict[str, Any]] = [] + for entry in tokens: + token_label = str(entry.get("token", "")) + normalized_label = token_label.lower() + if normalized in normalized_label or normalized in str( + entry.get("normalized", "") + ): + results.append(entry) + if len(results) >= limit: + break + return results + + +def merge_override( + summary: Mapping[str, Any], overrides: Mapping[str, Mapping[str, Any]] +) -> Dict[str, Any]: + if not isinstance(summary, Mapping): + return {"people": [], "entities": []} + merged_summary: Dict[str, Any] = dict(summary) + for key in ("people", "entities"): + items = summary.get(key) + if not isinstance(items, list): + continue + merged_items: List[Dict[str, Any]] = [] + for entry in items: + if not isinstance(entry, Mapping): + continue + normalized = _token_key( + str(entry.get("normalized") or entry.get("label") or "") + ) + merged = dict(entry) + if normalized and normalized in overrides: + merged_override = dict(overrides[normalized]) + merged["override"] = merged_override + merged_items.append(merged) + merged_summary[key] = merged_items + return merged_summary + + +def normalize_token(token: str) -> str: + return _token_key(_normalize_label(token)) + + +def normalize_manual_override_token(token: str) -> str: + if not token: + return "" + stripped = token.strip().strip("\"'`“”’") + if not stripped: + return "" + return _MULTI_SPACE_PATTERN.sub(" ", stripped.lower()).strip() diff --git a/abogen/epub3/__init__.py b/abogen/epub3/__init__.py new file mode 100644 index 0000000..7683764 --- /dev/null +++ b/abogen/epub3/__init__.py @@ -0,0 +1,3 @@ +from .exporter import EPUB3PackageBuilder, build_epub3_package + +__all__ = ["EPUB3PackageBuilder", "build_epub3_package"] diff --git a/abogen/epub3/exporter.py b/abogen/epub3/exporter.py new file mode 100644 index 0000000..834c783 --- /dev/null +++ b/abogen/epub3/exporter.py @@ -0,0 +1,910 @@ +from __future__ import annotations + +import html +import re +import shutil +import uuid +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any, Dict, Iterable, List, Optional, Pattern, Sequence, Tuple +import zipfile + +from abogen.text_extractor import ExtractedChapter, ExtractionResult + + +@dataclass(slots=True) +class ChunkOverlay: + id: str + text: str + original_text: Optional[str] + start: Optional[float] + end: Optional[float] + speaker_id: str + voice: Optional[str] + level: Optional[str] = None + group_id: Optional[str] = None + + +@dataclass(slots=True) +class ChapterDocument: + index: int # zero-based + title: str + xhtml_name: str + smil_name: str + chunks: List[ChunkOverlay] + start: Optional[float] + end: Optional[float] + + +class EPUB3PackageBuilder: + """Constructs an EPUB 3 package with media overlays.""" + + def __init__( + self, + *, + output_path: Path, + book_id: str, + extraction: ExtractionResult, + metadata_tags: Dict[str, Any], + chapter_markers: Sequence[Dict[str, Any]], + chunk_markers: Sequence[Dict[str, Any]], + chunks: Iterable[Dict[str, Any]], + audio_path: Path, + speaker_mode: str = "single", + cover_image_path: Optional[Path] = None, + cover_image_mime: Optional[str] = None, + ) -> None: + self.output_path = output_path + self.book_id = book_id or str(uuid.uuid4()) + self.extraction = extraction + self.metadata_tags = _normalize_metadata(metadata_tags) + self.chapter_markers = list(chapter_markers or []) + self.chunk_markers = list(chunk_markers or []) + self.chunks = list(chunks or []) + self.audio_path = audio_path + self.speaker_mode = speaker_mode or "single" + self.cover_image_path = cover_image_path if cover_image_path and cover_image_path.exists() else None + self.cover_image_mime = cover_image_mime + + self._combined_metadata = _combine_metadata(extraction.metadata, self.metadata_tags) + self._title = self._combined_metadata.get("title") or self._fallback_title() + self._authors = _split_authors(self._combined_metadata) + self._language = self._determine_language() + self._publisher = self._combined_metadata.get("publisher") or "" + self._description = self._combined_metadata.get("comment") + self._duration = _calculate_total_duration(self.chunk_markers, self.chapter_markers) + self._modified = _utc_now_iso() + + def build(self) -> Path: + if not self.audio_path or not self.audio_path.exists(): + raise FileNotFoundError(f"Audio asset missing: {self.audio_path}") + + chapter_documents = self._build_chapter_documents() + + with TemporaryDirectory() as tmp_dir: + root = Path(tmp_dir) + oebps = root / "OEBPS" + text_dir = oebps / "text" + smil_dir = oebps / "smil" + audio_dir = oebps / "audio" + image_dir = oebps / "images" + stylesheet_dir = oebps / "styles" + + for directory in (oebps, text_dir, smil_dir, audio_dir, stylesheet_dir): + directory.mkdir(parents=True, exist_ok=True) + if self.cover_image_path: + image_dir.mkdir(parents=True, exist_ok=True) + + _write_mimetype(root) + _write_container_xml(root) + + audio_filename = self.audio_path.name + embedded_audio = audio_dir / audio_filename + shutil.copy2(self.audio_path, embedded_audio) + + if self.cover_image_path: + shutil.copy2(self.cover_image_path, image_dir / self.cover_image_path.name) + + stylesheet_path = stylesheet_dir / "style.css" + stylesheet_path.write_text(_DEFAULT_STYLESHEET, encoding="utf-8") + + for chapter in chapter_documents: + chapter_path = text_dir / chapter.xhtml_name + chapter_path.write_text( + self._render_chapter_xhtml(chapter), + encoding="utf-8", + ) + smil_path = smil_dir / chapter.smil_name + smil_path.write_text( + self._render_chapter_smil(chapter, f"audio/{audio_filename}"), + encoding="utf-8", + ) + + nav_path = oebps / "nav.xhtml" + nav_path.write_text(self._render_nav(chapter_documents), encoding="utf-8") + + opf_path = oebps / "content.opf" + opf_path.write_text( + self._render_opf( + chapter_documents, + audio_filename, + has_cover=self.cover_image_path is not None, + stylesheet_path=stylesheet_path.relative_to(oebps), + ), + encoding="utf-8", + ) + + self.output_path.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(self.output_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: + # Ensure mimetype is the first entry and stored without compression + mimetype_path = root / "mimetype" + info = zipfile.ZipInfo("mimetype") + info.compress_type = zipfile.ZIP_STORED + archive.writestr(info, mimetype_path.read_bytes()) + + for file_path in sorted(root.rglob("*")): + if file_path == mimetype_path or file_path.is_dir(): + continue + archive.write(file_path, file_path.relative_to(root)) + + return self.output_path + + # ------------------------------------------------------------------ + def _build_chapter_documents(self) -> List[ChapterDocument]: + chunk_lookup = _build_chunk_lookup(self.chunks) + markers_by_chapter = _group_markers_by_chapter(self.chunk_markers) + chapter_meta = {int(entry.get("index", idx + 1)) - 1: dict(entry) for idx, entry in enumerate(self.chapter_markers)} + + documents: List[ChapterDocument] = [] + for chapter_index, chapter in enumerate(self.extraction.chapters): + markers = markers_by_chapter.get(chapter_index, []) + if not markers and chunk_lookup.by_chapter.get(chapter_index): + markers = [ + { + "id": item.get("id"), + "chapter_index": chapter_index, + "chunk_index": item.get("chunk_index"), + "start": None, + "end": None, + "speaker_id": item.get("speaker_id", "narrator"), + "voice": item.get("voice"), + } + for item in chunk_lookup.by_chapter.get(chapter_index, []) + ] + + if not markers: + markers = [ + { + "id": f"chap{chapter_index:04d}_auto0000", + "chapter_index": chapter_index, + "chunk_index": 0, + "start": chapter_meta.get(chapter_index, {}).get("start"), + "end": chapter_meta.get(chapter_index, {}).get("end"), + "speaker_id": "narrator", + "voice": None, + } + ] + + overlays = self._build_overlays_for_chapter( + chapter_index, + markers, + chunk_lookup, + ) + + xhtml_name = f"chapter_{chapter_index + 1:04d}.xhtml" + smil_name = f"chapter_{chapter_index + 1:04d}.smil" + + chapter_start = chapter_meta.get(chapter_index, {}).get("start") + chapter_end = chapter_meta.get(chapter_index, {}).get("end") + + documents.append( + ChapterDocument( + index=chapter_index, + title=chapter.title or f"Chapter {chapter_index + 1}", + xhtml_name=xhtml_name, + smil_name=smil_name, + chunks=overlays, + start=chapter_start, + end=chapter_end, + ) + ) + + return documents + + def _build_overlays_for_chapter( + self, + chapter_index: int, + markers: Sequence[Dict[str, Any]], + chunk_lookup: "ChunkLookup", + ) -> List[ChunkOverlay]: + overlays: List[ChunkOverlay] = [] + used_ids: set[str] = set() + + chapter_chunks = list(chunk_lookup.by_chapter.get(chapter_index, [])) + chapter_chunks.sort(key=lambda entry: _safe_int(entry.get("chunk_index"))) + + for position, marker in enumerate(markers): + chunk_id = marker.get("id") + chunk_entry = None + if chunk_id and chunk_id in chunk_lookup.by_id: + chunk_entry = chunk_lookup.by_id[chunk_id] + else: + candidate_index = _safe_int(marker.get("chunk_index")) + chunk_entry = _find_chunk_by_index(chapter_chunks, candidate_index) + if chunk_entry is None and chapter_chunks and position < len(chapter_chunks): + chunk_entry = chapter_chunks[position] + + level = None + if chunk_entry is None: + text = self.extraction.chapters[chapter_index].text + speaker_id = str(marker.get("speaker_id") or "narrator") + voice = marker.get("voice") + else: + display_text = chunk_entry.get("display_text") + text = str(chunk_entry.get("text") or "") + speaker_id = str(chunk_entry.get("speaker_id") or marker.get("speaker_id") or "narrator") + voice = chunk_entry.get("voice") or chunk_entry.get("resolved_voice") or marker.get("voice") + level = chunk_entry.get("level") or None + if chunk_entry is None: + level = None + + normalized_id = _normalize_chunk_id(chunk_id) if chunk_id else None + if not normalized_id: + normalized_id = f"chap{chapter_index:04d}_chunk{position:04d}" + while normalized_id in used_ids: + normalized_id = f"{normalized_id}_dup" + used_ids.add(normalized_id) + + raw_group_key = chunk_entry.get("id") if chunk_entry else chunk_id + group_id = _derive_group_id(raw_group_key, level) + normalized_group_id = _normalize_chunk_id(group_id) if group_id else None + + original_text = None + if chunk_entry is not None: + original_text = chunk_entry.get("original_text") or chunk_entry.get("display_text") + + overlays.append( + ChunkOverlay( + id=normalized_id, + text=text or self.extraction.chapters[chapter_index].text, + original_text=str(original_text) if original_text is not None else None, + start=_safe_float(marker.get("start")), + end=_safe_float(marker.get("end")), + speaker_id=speaker_id, + voice=str(voice) if voice else None, + level=str(level) if level else None, + group_id=normalized_group_id, + ) + ) + + chapter_text = "" + if 0 <= chapter_index < len(self.extraction.chapters): + chapter_entry = self.extraction.chapters[chapter_index] + chapter_text = getattr(chapter_entry, "text", "") or "" + + _restore_original_chunk_text(chapter_text, overlays) + + return overlays + + def _render_chapter_xhtml(self, chapter: ChapterDocument) -> str: + language = html.escape(self._language or "en") + title = html.escape(chapter.title) + grouped_chunks = _group_chunks_for_render(chapter.chunks) + chunk_html = "\n".join( + _render_chunk_group_html(group_id, items) for group_id, items in grouped_chunks + ) + if not chunk_html: + chunk_html = "

    " + original_block = "" + if chapter.chunks: + original_text = "".join((chunk.original_text if chunk.original_text is not None else (chunk.text or "")) for chunk in chapter.chunks) + if original_text: + safe_original = html.escape(original_text) + original_block = ( + " " + ) + + return ( + "\n" + "\n" + " \n" + " {title}\n" + " \n" + " \n" + " \n" + " \n" + "
    \n" + "

    {title}

    \n" + " {chunks}\n" + "{original_block}" + "
    \n" + " \n" + "\n" + ).format( + lang=language, + title=title, + index=chapter.index + 1, + chunks=chunk_html, + original_block=("" if not original_block else f"{original_block}\n"), + ) + + def _render_chapter_smil(self, chapter: ChapterDocument, audio_href: str) -> str: + par_lines = [] + for chunk in chapter.chunks: + par_lines.append( + " \n" + " \n" + " ".format( + chunk_id=html.escape(chunk.id), + xhtml=html.escape(chapter.xhtml_name), + audio=html.escape(audio_href), + start=_format_smil_time(chunk.start), + end=_format_smil_time(chunk.end), + ) + ) + + return ( + "\n" + "\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + "{pars}\n" + " \n" + " \n" + "\n" + ).format( + title=html.escape(chapter.title), + book_id=html.escape(self.book_id), + index=chapter.index + 1, + xhtml=html.escape(chapter.xhtml_name), + pars="\n".join(par_lines) if par_lines else " ", + ) + + def _render_nav(self, chapters: Sequence[ChapterDocument]) -> str: + items = [] + for chapter in chapters: + href = f"text/{chapter.xhtml_name}" + items.append( + "
  • {title}
    • ".format( + href=html.escape(href), + title=html.escape(chapter.title), + ) + ) + + return ( + "\n" + "\n" + " \n" + " Navigation\n" + " \n" + " \n" + " \n" + " \n" + " \n" + "\n" + ).format( + lang=html.escape(self._language or "en"), + title=html.escape(self._title), + items="\n".join(items) if items else "
  • Chapter 1
    • ", + ) + + def _render_opf( + self, + chapters: Sequence[ChapterDocument], + audio_filename: str, + *, + has_cover: bool, + stylesheet_path: Path, + ) -> str: + manifest_items = [] + spine_refs = [] + for chapter in chapters: + item_id = f"chap{chapter.index + 1:04d}" + overlay_id = f"mo-{chapter.index + 1:04d}" + manifest_items.append( + " ".format( + item_id=item_id, + href=html.escape(chapter.xhtml_name), + overlay_id=overlay_id, + ) + ) + manifest_items.append( + " ".format( + overlay_id=overlay_id, + smil=html.escape(chapter.smil_name), + ) + ) + spine_refs.append(f" ") + + audio_item_id = "primary-audio" + manifest_items.append( + " ".format( + item_id=audio_item_id, + href=html.escape(audio_filename), + mime=_detect_audio_mime(audio_filename), + ) + ) + + manifest_items.append( + " " + ) + + manifest_items.append( + " ".format( + href=html.escape(str(stylesheet_path).replace("\\", "/")), + ) + ) + + if has_cover and self.cover_image_path: + cover_id = "cover-image" + manifest_items.append( + " ".format( + item_id=cover_id, + href=html.escape(self.cover_image_path.name), + mime=self.cover_image_mime or _detect_image_mime(self.cover_image_path.suffix), + ) + ) + + metadata_elements = _render_metadata_xml( + self._title, + self._authors, + self._language, + self.book_id, + duration=self._duration, + publisher=self._publisher, + description=self._description, + speaker_mode=self.speaker_mode, + modified=self._modified, + ) + + return ( + "\n" + "\n" + " \n" + "{metadata}\n" + " \n" + " \n" + "{manifest}\n" + " \n" + " \n" + "{spine}\n" + " \n" + "\n" + ).format( + metadata="\n".join(metadata_elements), + manifest="\n".join(manifest_items), + spine="\n".join(spine_refs) if spine_refs else " ", + ) + + def _fallback_title(self) -> str: + if self.extraction.chapters: + first_title = self.extraction.chapters[0].title + if first_title: + return first_title + return "Generated Audiobook" + + def _determine_language(self) -> str: + language = self._combined_metadata.get("language") + if language: + return language + return "en" + + +def build_epub3_package( + *, + output_path: Path, + book_id: str, + extraction: ExtractionResult, + metadata_tags: Dict[str, Any], + chapter_markers: Sequence[Dict[str, Any]], + chunk_markers: Sequence[Dict[str, Any]], + chunks: Iterable[Dict[str, Any]], + audio_path: Path, + speaker_mode: str = "single", + cover_image_path: Optional[Path] = None, + cover_image_mime: Optional[str] = None, +) -> Path: + builder = EPUB3PackageBuilder( + output_path=output_path, + book_id=book_id, + extraction=extraction, + metadata_tags=metadata_tags, + chapter_markers=chapter_markers, + chunk_markers=chunk_markers, + chunks=chunks, + audio_path=audio_path, + speaker_mode=speaker_mode, + cover_image_path=cover_image_path, + cover_image_mime=cover_image_mime, + ) + return builder.build() + + +# --------------------------------------------------------------------------- +# Helpers + + +@dataclass +class ChunkLookup: + by_id: Dict[str, Dict[str, Any]] + by_chapter: Dict[int, List[Dict[str, Any]]] + + +def _normalize_metadata(metadata: Optional[Dict[str, Any]]) -> Dict[str, str]: + normalized: Dict[str, str] = {} + for key, value in (metadata or {}).items(): + if value is None: + continue + normalized[str(key).lower()] = str(value) + return normalized + + +def _combine_metadata(*sources: Dict[str, Any]) -> Dict[str, str]: + combined: Dict[str, str] = {} + for source in sources: + for key, value in (source or {}).items(): + if value is None: + continue + combined[str(key).lower()] = str(value) + return combined + + +def _split_authors(metadata: Dict[str, str]) -> List[str]: + candidates = [] + for key in ("artist", "author", "authors", "album_artist", "creator"): + value = metadata.get(key) + if value: + candidates.extend(part.strip() for part in value.replace(";", ",").split(",")) + return [author for author in candidates if author] + + +def _calculate_total_duration( + chunk_markers: Sequence[Dict[str, Any]], + chapter_markers: Sequence[Dict[str, Any]], +) -> Optional[float]: + candidates: List[float] = [] + for marker in chunk_markers or []: + end_value = _safe_float(marker.get("end")) + if end_value is not None: + candidates.append(end_value) + for marker in chapter_markers or []: + end_value = _safe_float(marker.get("end")) + if end_value is not None: + candidates.append(end_value) + if not candidates: + return None + return max(candidates) + + +def _write_mimetype(root: Path) -> None: + (root / "mimetype").write_text("application/epub+zip", encoding="utf-8") + + +def _write_container_xml(root: Path) -> None: + meta_inf = root / "META-INF" + meta_inf.mkdir(parents=True, exist_ok=True) + container = meta_inf / "container.xml" + container.write_text( + ( + "\n" + "\n" + " \n" + " \n" + " \n" + "\n" + ), + encoding="utf-8", + ) + + +def _build_chunk_lookup(chunks: Iterable[Dict[str, Any]]) -> ChunkLookup: + by_id: Dict[str, Dict[str, Any]] = {} + by_chapter: Dict[int, List[Dict[str, Any]]] = {} + for entry in chunks or []: + if not isinstance(entry, dict): + continue + chunk_id = entry.get("id") + if chunk_id: + by_id[str(chunk_id)] = dict(entry) + chapter_index = _safe_int(entry.get("chapter_index")) + by_chapter.setdefault(chapter_index, []).append(dict(entry)) + return ChunkLookup(by_id=by_id, by_chapter=by_chapter) + + +def _group_markers_by_chapter(markers: Iterable[Dict[str, Any]]) -> Dict[int, List[Dict[str, Any]]]: + grouped: Dict[int, List[Dict[str, Any]]] = {} + for entry in markers or []: + if not isinstance(entry, dict): + continue + chapter_index = _safe_int(entry.get("chapter_index")) + grouped.setdefault(chapter_index, []).append(dict(entry)) + for chapter_index, items in grouped.items(): + items.sort(key=lambda payload: (_safe_int(payload.get("chunk_index")), _safe_float(payload.get("start")) or 0.0)) + return grouped + + +def _find_chunk_by_index( + chapter_chunks: Sequence[Dict[str, Any]], + chunk_index: Optional[int], +) -> Optional[Dict[str, Any]]: + if chunk_index is None: + return None + for entry in chapter_chunks: + if _safe_int(entry.get("chunk_index")) == chunk_index: + return entry + return None + + +def _normalize_chunk_id(chunk_id: Optional[Any]) -> Optional[str]: + if chunk_id is None: + return None + text = str(chunk_id).strip() + if not text: + return None + safe = "".join(ch if ch.isalnum() or ch in {"_", "-"} else "_" for ch in text) + return safe[:120] + + +def _derive_group_id(chunk_id: Optional[Any], level: Optional[Any]) -> Optional[str]: + if chunk_id is None: + return None + text = str(chunk_id).strip() + if not text: + return None + if str(level or "").lower() == "sentence": + match = re.match(r"(.+?)_s\d+(?:_.*)?$", text) + if match: + return match.group(1) + return text + + +def _group_chunks_for_render(chunks: Sequence[ChunkOverlay]) -> List[Tuple[Optional[str], List[ChunkOverlay]]]: + groups: List[Tuple[Optional[str], List[ChunkOverlay]]] = [] + current_key: Optional[str] = None + current_items: List[ChunkOverlay] = [] + + for chunk in chunks: + key = chunk.group_id or chunk.id + if current_items and key != current_key: + groups.append((current_key, current_items)) + current_items = [] + if not current_items: + current_key = key + current_items.append(chunk) + + if current_items: + groups.append((current_key, current_items)) + + return groups + + +def _render_chunk_inline(chunk: ChunkOverlay) -> str: + escaped_id = html.escape(chunk.id) + speaker_attr = f" data-speaker=\"{html.escape(chunk.speaker_id)}\"" if chunk.speaker_id else "" + voice_attr = f" data-voice=\"{html.escape(chunk.voice)}\"" if chunk.voice else "" + level_attr = f" data-level=\"{html.escape(chunk.level)}\"" if chunk.level else "" + raw_text = chunk.text or "" + escaped_text = html.escape(raw_text) + if not escaped_text: + escaped_text = " " + return ( + f"" + f"{escaped_text}" + "" + ) + + +def _render_chunk_group_html(group_id: Optional[str], chunks: Sequence[ChunkOverlay]) -> str: + if not chunks: + return "" + group_attr = f" data-group=\"{html.escape(group_id)}\"" if group_id else "" + inline_html = "".join(_render_chunk_inline(chunk) for chunk in chunks) + if not inline_html: + inline_html = " " + return f"

    {inline_html}

    " + + +def _format_smil_time(value: Optional[float]) -> str: + if value is None or value < 0: + value = 0.0 + total_ms = int(round(value * 1000)) + hours, remainder = divmod(total_ms, 3600_000) + minutes, remainder = divmod(remainder, 60_000) + seconds, milliseconds = divmod(remainder, 1000) + return f"{hours:02d}:{minutes:02d}:{seconds:02d}.{milliseconds:03d}" + + +def _safe_int(value: Any, default: int = 0) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default + + +def _safe_float(value: Any) -> Optional[float]: + if value is None: + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + +def _restore_original_chunk_text(chapter_text: str, overlays: List[ChunkOverlay]) -> None: + if not chapter_text or not overlays: + return + + cursor = 0 + for chunk in overlays: + if chunk.original_text is not None: + prepared = _prepare_display_text(chunk.original_text) + chunk.text = prepared + continue + candidate = chunk.text or "" + if not candidate: + continue + match = _search_original_span(chapter_text, candidate, cursor) + if match is None and cursor: + match = _search_original_span(chapter_text, candidate, 0) + if match is None: + if chunk.original_text is None: + chunk.original_text = chunk.text + chunk.text = _prepare_display_text(chunk.text or "") + continue + start, end = match + segment = chapter_text[start:end] + chunk.original_text = segment + chunk.text = _prepare_display_text(segment) + cursor = end + + +def _prepare_display_text(value: str) -> str: + if not value: + return "" + cleaned = re.sub(r"(?:[ \t]*\r?\n)+\Z", "", value) + return cleaned if cleaned else "" + + +def _search_original_span(source: str, normalized: str, start: int) -> Optional[Tuple[int, int]]: + if not normalized: + return None + pattern = _build_chunk_pattern(normalized) + match = pattern.search(source, start) + if not match: + return None + return match.start(1), match.end(1) + + +_CHUNK_REGEX_CACHE: Dict[str, Pattern[str]] = {} + + +def _build_chunk_pattern(text: str) -> Pattern[str]: + cached = _CHUNK_REGEX_CACHE.get(text) + if cached is not None: + return cached + escaped = re.escape(text) + escaped = escaped.replace(r"\ ", r"\s+") + pattern = re.compile(r"(\s*" + escaped + r"\s*)", re.DOTALL) + _CHUNK_REGEX_CACHE[text] = pattern + return pattern + + +def _render_metadata_xml( + title: str, + authors: Sequence[str], + language: str, + book_id: str, + *, + duration: Optional[float], + publisher: Optional[str], + description: Optional[str], + speaker_mode: Optional[str], + modified: Optional[str], +) -> List[str]: + elements = [ + f" {html.escape(book_id)}", + f" {html.escape(title)}", + f" {html.escape(language or 'en')}", + ] + + for author in authors or ["Unknown"]: + elements.append(f" {html.escape(author)}") + + if publisher: + elements.append(f" {html.escape(publisher)}") + + if description: + elements.append(f" {html.escape(description)}") + + if duration is not None: + elements.append(f" {_format_iso_duration(duration)}") + + if speaker_mode: + elements.append( + " {}".format( + html.escape(str(speaker_mode)) + ) + ) + + if modified: + elements.append(f" {html.escape(modified)}") + return elements + + +def _format_iso_duration(value: float) -> str: + total_seconds = int(value) + remainder = value - total_seconds + hours, remainder_seconds = divmod(total_seconds, 3600) + minutes, seconds = divmod(remainder_seconds, 60) + seconds_with_fraction = seconds + remainder + if seconds_with_fraction.is_integer(): + seconds_text = f"{int(seconds_with_fraction)}" + else: + seconds_text = f"{seconds_with_fraction:.3f}".rstrip("0").rstrip(".") + return f"PT{hours}H{minutes}M{seconds_text}S" + + +def _detect_audio_mime(audio_filename: str) -> str: + suffix = Path(audio_filename).suffix.lower() + return { + ".mp3": "audio/mpeg", + ".m4a": "audio/mp4", + ".m4b": "audio/mp4", + ".aac": "audio/aac", + ".wav": "audio/wav", + ".flac": "audio/flac", + ".ogg": "audio/ogg", + ".opus": "audio/ogg", + }.get(suffix, "audio/mpeg") + + +def _detect_image_mime(suffix: str) -> str: + normalized = suffix.lower() + return { + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".png": "image/png", + ".gif": "image/gif", + ".webp": "image/webp", + }.get(normalized, "image/jpeg") + + +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +_DEFAULT_STYLESHEET = """ +body { + font-family: 'Georgia', serif; + line-height: 1.6; + margin: 1.5em; +} + +h1 { + font-size: 1.5em; + margin-bottom: 0.5em; +} + +.chunk-group { + margin: 0.5em 0; +} + +.chunk-group .chunk { + white-space: pre-wrap; +} +""" diff --git a/abogen/gui.py b/abogen/gui.py index 95a779a..a023b17 100644 --- a/abogen/gui.py +++ b/abogen/gui.py @@ -1,4049 +1,11 @@ -import os -import time -import sys -import tempfile -import platform -import base64 -import re -from abogen.queue_manager_gui import QueueManager -from abogen.queued_item import QueuedItem -import abogen.hf_tracker as hf_tracker -import hashlib # Added for cache path generation -from PyQt6.QtWidgets import ( - QApplication, - QWidget, - QVBoxLayout, - QHBoxLayout, - QPushButton, - QComboBox, - QTextEdit, - QLabel, - QSlider, - QMessageBox, - QFileDialog, - QProgressBar, - QFrame, - QStyleFactory, - QInputDialog, - QFileIconProvider, - QSizePolicy, - QDialog, - QCheckBox, - QMenu, -) -from PyQt6.QtGui import QAction, QActionGroup -from PyQt6.QtCore import ( - Qt, - QUrl, - QPoint, - QFileInfo, - QThread, - pyqtSignal, - QObject, - QBuffer, - QIODevice, - QSize, - QTimer, - QEvent, - QProcess, -) -from PyQt6.QtGui import ( - QTextCursor, - QDesktopServices, - QIcon, - QPixmap, - QPainter, - QPolygon, - QColor, - QMovie, - QPalette, -) -from abogen.utils import ( - load_config, - save_config, - get_gpu_acceleration, - prevent_sleep_start, - prevent_sleep_end, - get_resource_path, - get_user_cache_path, - LoadPipelineThread, -) +"""Backwards-compatible re-export of the PyQt GUI. -from abogen.subtitle_utils import ( - clean_text, - calculate_text_length, -) - -from abogen.conversion import ConversionThread, VoicePreviewThread, PlayAudioThread -from abogen.book_handler import HandlerDialog -from abogen.constants import ( - PROGRAM_NAME, - VERSION, - GITHUB_URL, - PROGRAM_DESCRIPTION, - LANGUAGE_DESCRIPTIONS, - VOICES_INTERNAL, - SUPPORTED_LANGUAGES_FOR_SUBTITLE_GENERATION, - COLORS, - SUBTITLE_FORMATS, -) -import threading -from abogen.voice_formula_gui import VoiceFormulaDialog -from abogen.voice_profiles import load_profiles - -# Import ctypes for Windows-specific taskbar icon -if platform.system() == "Windows": - import ctypes - - -class DarkTitleBarEventFilter(QObject): - def __init__(self, is_windows, get_dark_mode_func, set_title_bar_dark_mode_func): - super().__init__() - self.is_windows = is_windows - self.get_dark_mode = get_dark_mode_func - self.set_title_bar_dark_mode = set_title_bar_dark_mode_func - - def eventFilter(self, obj, event): - if event.type() == QEvent.Type.Show: - # Only apply to QWidget windows - if isinstance(obj, QWidget) and obj.isWindow(): - if self.is_windows and self.get_dark_mode(): - self.set_title_bar_dark_mode(obj, True) - return super().eventFilter(obj, event) - - -class ShowWarningSignalEmitter(QObject): # New class to handle signal emission - show_warning_signal = pyqtSignal(str, str) - - def emit(self, title, message): - self.show_warning_signal.emit(title, message) - - -class ThreadSafeLogSignal(QObject): - log_signal = pyqtSignal(object) - - def __init__(self, parent=None): - super().__init__(parent) - - def emit_log(self, message): - self.log_signal.emit(message) - - -class IconProvider(QFileIconProvider): - def icon(self, fileInfo): - return super().icon(fileInfo) - - -LOG_COLOR_MAP = { - True: COLORS["GREEN"], - False: COLORS["RED"], - "red": COLORS["RED"], - "green": COLORS["GREEN"], - "orange": COLORS["ORANGE"], - "blue": COLORS["BLUE"], - "grey": COLORS["LIGHT_DISABLED"], - None: COLORS["LIGHT_DISABLED"], -} - - -class InputBox(QLabel): - # Define CSS styles as class constants - STYLE_DEFAULT = f"border:2px dashed #aaa; border-radius:5px; padding:20px; background:{COLORS['BLUE_BG']}; min-height:100px;" - STYLE_DEFAULT_HOVER = f"background:{COLORS['BLUE_BG_HOVER']}; border-color:{COLORS['BLUE_BORDER_HOVER']};" - - STYLE_ACTIVE = f"border:2px dashed {COLORS['GREEN']}; border-radius:5px; padding:20px; background:{COLORS['GREEN_BG']}; min-height:100px;" - STYLE_ACTIVE_HOVER = ( - f"background:{COLORS['GREEN_BG_HOVER']}; border-color:{COLORS['GREEN_BORDER']};" - ) - - STYLE_ERROR = f"border:2px dashed {COLORS['RED']}; border-radius:5px; padding:20px; background:{COLORS['RED_BG']}; min-height:100px; color:{COLORS['RED']};" - STYLE_ERROR_HOVER = ( - f"background:{COLORS['RED_BG_HOVER']}; border-color:{COLORS['RED']};" - ) - - def __init__(self, parent=None): - super().__init__(parent) - self.setAlignment(Qt.AlignmentFlag.AlignCenter) - self.setAcceptDrops(True) - self.setText( - "Drag and drop your file here or click to browse.\n(.txt, .epub, .pdf, .md, .srt, .ass, .vtt)" - ) - self.setStyleSheet( - f"QLabel {{ {self.STYLE_DEFAULT} }} QLabel:hover {{ {self.STYLE_DEFAULT_HOVER} }}" - ) - self.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding) - self.setCursor(Qt.CursorShape.PointingHandCursor) - - # Add clear button - self.clear_btn = QPushButton("✕", self) - self.clear_btn.setFixedSize(28, 28) - self.clear_btn.hide() - self.clear_btn.clicked.connect(self.clear_input) - - # Add Chapters button - self.chapters_btn = QPushButton("Chapters", self) - self.chapters_btn.setStyleSheet("QPushButton { padding: 6px 10px; }") - self.chapters_btn.hide() - self.chapters_btn.clicked.connect(self.on_chapters_clicked) - - # Add Textbox button with no padding - self.textbox_btn = QPushButton("Textbox", self) - self.textbox_btn.setStyleSheet("QPushButton { padding: 6px 10px; }") - self.textbox_btn.setToolTip("Input text directly instead of using a file") - self.textbox_btn.clicked.connect(self.on_textbox_clicked) - - # Add Edit button matching the textbox button - self.edit_btn = QPushButton("Edit", self) - self.edit_btn.setStyleSheet("QPushButton { padding: 6px 10px; }") - self.edit_btn.setToolTip("Edit the current text file") - self.edit_btn.clicked.connect(self.on_edit_clicked) - self.edit_btn.hide() - - # Add Go to folder button - self.go_to_folder_btn = QPushButton("Go to folder", self) - self.go_to_folder_btn.setStyleSheet("QPushButton { padding: 6px 10px; }") - self.go_to_folder_btn.setToolTip( - "Open the folder that contains the converted file" - ) - self.go_to_folder_btn.clicked.connect(self.on_go_to_folder_clicked) - self.go_to_folder_btn.hide() - - def resizeEvent(self, event): - super().resizeEvent(event) - margin = 12 - self.clear_btn.move(self.width() - self.clear_btn.width() - margin, margin) - self.chapters_btn.move( - margin, self.height() - self.chapters_btn.height() - margin - ) - # Position textbox button at top left - self.textbox_btn.move(margin, margin) - self.edit_btn.move(margin, margin) - # Position go to folder button at bottom right with correct margins - self.go_to_folder_btn.move( - self.width() - self.go_to_folder_btn.width() - margin, - self.height() - self.go_to_folder_btn.height() - margin, - ) - - def set_file_info(self, file_path): - # get icon without resizing using custom provider - provider = IconProvider() - qicon = provider.icon(QFileInfo(file_path)) - size = QSize(32, 32) - pixmap = qicon.pixmap(size) - # convert to base64 PNG - buffer = QBuffer() - buffer.open(QIODevice.OpenModeFlag.WriteOnly) - pixmap.save(buffer, "PNG") - img_data = base64.b64encode(buffer.data()).decode() - - size_str = self._human_readable_size(os.path.getsize(file_path)) - name = os.path.basename(file_path) - char_count = 0 - window = self.window() - cache = getattr(window, "_char_count_cache", None) - - def parse_size(size_str): - # Use regex to extract the numeric part - match = re.match(r"([\d.]+)", size_str) - if match: - return float(match.group(1)) - raise ValueError(f"Invalid size format: {size_str}") - - # Format numbers with commas - def format_num(n): - try: - if isinstance(n, str): - size = int(parse_size(n)) - return f"{size:,}" - else: - return f"{n:,}" - except Exception: - return str(n) - - doc_extensions = (".epub", ".pdf", ".md", ".markdown", ".srt", ".ass", ".vtt") - char_source_path = file_path - cached_char_count = None - - if file_path.lower().endswith(doc_extensions): - selected_file_path = getattr(window, "selected_file", None) - if selected_file_path and os.path.exists(selected_file_path): - char_source_path = selected_file_path - else: - char_source_path = None - - if cache is not None: - cached_char_count = cache.get(file_path) - if ( - cached_char_count is None - and char_source_path - and char_source_path != file_path - ): - cached_char_count = cache.get(char_source_path) - - if cached_char_count is not None: - char_count = cached_char_count - elif char_source_path: - try: - with open( - char_source_path, "r", encoding="utf-8", errors="ignore" - ) as f: - text = f.read() - cleaned_text = clean_text(text) - char_count = calculate_text_length(cleaned_text) - except Exception: - char_count = "N/A" - else: - char_count = "N/A" - - if cache is not None and isinstance(char_count, int): - cache[file_path] = char_count - if char_source_path and char_source_path != file_path: - cache[char_source_path] = char_count - - # Store numeric char_count on window - try: - window.char_count = int(char_count) - except Exception: - window.char_count = 0 - # embed icon at native size with word-wrap for the filename - self.setText( - f'
    {name}
    Size: {size_str}
    Characters: {format_num(char_count)}' - ) - # Set fixed width to force wrapping - self.setWordWrap(True) - self.setStyleSheet( - f"QLabel {{ {self.STYLE_ACTIVE} }} QLabel:hover {{ {self.STYLE_ACTIVE_HOVER} }}" - ) - self.clear_btn.show() - is_document = window.selected_file_type in ["epub", "pdf", "md", "markdown"] - self.chapters_btn.setVisible(is_document) - if is_document: - chapter_count = len(window.selected_chapters) - file_type = window.selected_file_type - # Adjust button text based on file type - if file_type == "epub" or file_type == "md" or file_type == "markdown": - self.chapters_btn.setText(f"Chapters ({chapter_count})") - else: # PDF - always use Pages - self.chapters_btn.setText(f"Pages ({chapter_count})") - - # Hide textbox and show edit only for .txt, .srt, .ass, .vtt files - self.textbox_btn.hide() - # Show edit button for txt/subtitle files directly - # Or for epub/pdf files that have generated a temp txt file - should_show_edit = file_path.lower().endswith((".txt", ".srt", ".ass", ".vtt")) - - # For epub/pdf files, show edit if we have a selected_file (temp txt) - if ( - window.selected_file_type - in ["epub", "pdf", "md", "markdown", "md", "markdown"] - and window.selected_file - ): - should_show_edit = True - - self.edit_btn.setVisible(should_show_edit) - self.go_to_folder_btn.show() - - # Disable subtitle generation for subtitle input files - is_subtitle_input = file_path.lower().endswith((".srt", ".ass", ".vtt")) - if hasattr(window, "subtitle_combo"): - window.subtitle_combo.setEnabled(not is_subtitle_input) - - # Enable add to queue button only when file is accepted (input box is green) - self.resizeEvent(None) - if hasattr(window, "btn_add_to_queue"): - window.btn_add_to_queue.setEnabled(True) - - self.chapters_btn.adjustSize() - # Reset the input_box_cleared_by_queue flag after setting file info - if hasattr(window, "input_box_cleared_by_queue"): - window.input_box_cleared_by_queue = False - - def set_error(self, message): - self.setText(message) - self.setStyleSheet( - f"QLabel {{ {self.STYLE_ERROR} }} QLabel:hover {{ {self.STYLE_ERROR_HOVER} }}" - ) - # Show textbox button in error state as well - self.textbox_btn.show() - # Disable add to queue button on error - if hasattr(self.window(), "btn_add_to_queue"): - self.window().btn_add_to_queue.setEnabled(False) - - def clear_input(self): - self.window().selected_file = None - self.window().displayed_file_path = ( - None # Reset the displayed file path when clearing input - ) - # Reset book handler attributes - self.window().save_chapters_separately = None - self.window().merge_chapters_at_end = None - self.setText( - "Drag and drop your file here or click to browse.\n(.txt, .epub, .pdf, .md, .srt, .ass, .vtt)" - ) - self.setStyleSheet( - f"QLabel {{ {self.STYLE_DEFAULT} }} QLabel:hover {{ {self.STYLE_DEFAULT_HOVER} }}" - ) - self.clear_btn.hide() - self.chapters_btn.hide() - self.chapters_btn.setText("Chapters") # Reset text - # Show textbox and hide edit when input is cleared - self.textbox_btn.show() - self.edit_btn.hide() - self.go_to_folder_btn.hide() - - # Re-enable subtitle and replace newlines controls when cleared - window = self.window() - if hasattr(window, "subtitle_combo"): - # Only enable if language supports it - current_lang = getattr(window, "selected_lang", "a") - window.subtitle_combo.setEnabled( - current_lang in SUPPORTED_LANGUAGES_FOR_SUBTITLE_GENERATION - ) - if hasattr(window, "replace_newlines_combo"): - window.replace_newlines_combo.setEnabled(True) - - # Disable add to queue button when input is cleared - if hasattr(window, "btn_add_to_queue"): - window.btn_add_to_queue.setEnabled(False) - # Reset the input_box_cleared_by_queue flag after setting file info - if hasattr(self.window(), "input_box_cleared_by_queue"): - self.window().input_box_cleared_by_queue = True - - def _human_readable_size(self, size, decimal_places=2): - for unit in ["B", "KB", "MB", "GB", "TB"]: - if size < 1024.0: - return f"{size:.{decimal_places}f} {unit}" - size /= 1024.0 - return f"{size:.{decimal_places}f} PB" - - def mousePressEvent(self, event): - if event.button() == Qt.MouseButton.LeftButton: - self.window().open_file_dialog() - - def dragEnterEvent(self, event): - if event.mimeData().hasUrls(): - urls = event.mimeData().urls() - if urls: - ext = urls[0].toLocalFile().lower() - if ( - ext.endswith(".txt") - or ext.endswith(".epub") - or ext.endswith(".pdf") - or ext.endswith((".md", ".markdown")) - or ext.endswith((".srt", ".ass", ".vtt")) - ): - event.acceptProposedAction() - # Set hover style based on current state - if self.styleSheet().find(self.STYLE_ACTIVE) != -1: - self.setStyleSheet( - f"QLabel {{ {self.STYLE_ACTIVE} }} QLabel:hover {{ {self.STYLE_ACTIVE_HOVER} }} {self.STYLE_ACTIVE_HOVER}" - ) - elif self.styleSheet().find(self.STYLE_ERROR) != -1: - self.setStyleSheet( - f"QLabel {{ {self.STYLE_ERROR} }} QLabel:hover {{ {self.STYLE_ERROR_HOVER} }} {self.STYLE_ERROR_HOVER}" - ) - else: - self.setStyleSheet( - f"QLabel {{ {self.STYLE_DEFAULT} }} QLabel:hover {{ {self.STYLE_DEFAULT_HOVER} }} {self.STYLE_DEFAULT_HOVER}" - ) - return - event.ignore() - - def dragLeaveEvent(self, event): - # Restore the style based on current state - if self.styleSheet().find(self.STYLE_ACTIVE) != -1: - self.setStyleSheet( - f"QLabel {{ {self.STYLE_ACTIVE} }} QLabel:hover {{ {self.STYLE_ACTIVE_HOVER} }}" - ) - elif self.styleSheet().find(self.STYLE_ERROR) != -1: - self.setStyleSheet( - f"QLabel {{ {self.STYLE_ERROR} }} QLabel:hover {{ {self.STYLE_ERROR_HOVER} }}" - ) - else: - self.setStyleSheet( - f"QLabel {{ {self.STYLE_DEFAULT} }} QLabel:hover {{ {self.STYLE_DEFAULT_HOVER} }}" - ) - event.accept() - - def dropEvent(self, event): - if event.mimeData().hasUrls(): - urls = event.mimeData().urls() - if not urls: - event.ignore() - return - file_path = urls[0].toLocalFile() - win = self.window() - if file_path.lower().endswith(".txt"): - win.selected_file, win.selected_file_type = file_path, "txt" - win.displayed_file_path = ( - file_path # Set the displayed file path for text files - ) - self.set_file_info(file_path) - event.acceptProposedAction() - elif ( - file_path.lower().endswith(".epub") - or file_path.lower().endswith(".pdf") - or file_path.lower().endswith((".md", ".markdown")) - or file_path.lower().endswith((".srt", ".ass", ".vtt")) - ): - # Determine file type - if file_path.lower().endswith(".epub"): - file_type = "epub" - elif file_path.lower().endswith(".pdf"): - file_type = "pdf" - elif file_path.lower().endswith((".srt", ".ass", ".vtt")): - # For subtitle files, treat them like txt files (direct processing) - win.selected_file, win.selected_file_type = file_path, "txt" - win.displayed_file_path = file_path - self.set_file_info(file_path) - event.acceptProposedAction() - return - else: - file_type = "markdown" - - # Just store the file path but don't set the file info yet - win.selected_file_type = file_type - win.selected_book_path = file_path - win.open_book_file( - file_path # This will handle the dialog and setting file info - ) - event.acceptProposedAction() - else: - self.set_error( - "Please drop a .txt, .epub, .pdf, .md, .srt, .ass, or .vtt file." - ) - event.ignore() - else: - event.ignore() - - def on_chapters_clicked(self): - win = self.window() - if ( - win.selected_file_type in ["epub", "pdf", "md", "markdown"] - and win.selected_book_path - ): - # Call open_book_file which shows the dialog and updates selected_chapters - if win.open_book_file(win.selected_book_path): - # Refresh the info label and button text after dialog closes - self.set_file_info(win.selected_book_path) - - def on_textbox_clicked(self): - self.window().open_textbox_dialog() - - def on_edit_clicked(self): - win = self.window() - # For PDFs and EPUBs, use the temporary text file - if ( - win.selected_file_type in ["epub", "pdf", "md", "markdown"] - and win.selected_file - ): - # Use the temporary .txt file that was generated - win.open_textbox_dialog(win.selected_file) - else: - # For regular txt files - win.open_textbox_dialog() - - def on_go_to_folder_clicked(self): - win = self.window() - # win.selected_file holds the path to the text that is converted. - file_to_check = win.selected_file - - # If this is a converted document (epub/pdf/markdown) that was written to the - # user's cache directory, show a menu letting the user jump to either the - # processed (cached .txt) file or the original input file (epub/pdf/md). - try: - cache_dir = get_user_cache_path() - except Exception: - cache_dir = None - - is_cached_doc = False - if ( - file_to_check - and os.path.exists(file_to_check) - and os.path.isfile(file_to_check) - and cache_dir - ): - # Consider it cached when the file is under the cache directory and is a .txt - if file_to_check.endswith(".txt") and os.path.commonpath( - [os.path.abspath(file_to_check), os.path.abspath(cache_dir)] - ) == os.path.abspath(cache_dir): - # Only treat as document-cache when original type was a document - if getattr(win, "selected_file_type", None) in [ - "epub", - "pdf", - "md", - "markdown", - ]: - is_cached_doc = True - - if is_cached_doc: - menu = QMenu(self) - act_processed = QAction("Go to processed file", self) - - def open_processed(): - folder_path = os.path.dirname(file_to_check) - QDesktopServices.openUrl(QUrl.fromLocalFile(folder_path)) - - act_processed.triggered.connect(open_processed) - menu.addAction(act_processed) - - act_input = QAction("Go to input file", self) - # Prefer displayed_file_path (original input path) then selected_book_path - input_path = getattr(win, "displayed_file_path", None) or getattr( - win, "selected_book_path", None - ) - if input_path and os.path.exists(input_path): - - def open_input(): - folder_path = os.path.dirname(input_path) - QDesktopServices.openUrl(QUrl.fromLocalFile(folder_path)) - - act_input.triggered.connect(open_input) - else: - act_input.setEnabled(False) - - menu.addAction(act_input) - # Show the menu anchored to the button - menu.exec( - self.go_to_folder_btn.mapToGlobal( - QPoint(0, self.go_to_folder_btn.height()) - ) - ) - else: - if ( - file_to_check - and os.path.exists(file_to_check) - and os.path.isfile(file_to_check) - ): - folder_path = os.path.dirname(file_to_check) - QDesktopServices.openUrl(QUrl.fromLocalFile(folder_path)) - else: - QMessageBox.warning(win, "Error", "Converted file not found.") - - -class TextboxDialog(QDialog): - def __init__(self, parent=None): - super().__init__(parent) - self.setWindowTitle("Enter Text") - self.setWindowFlags( - Qt.WindowType.Window - | Qt.WindowType.WindowCloseButtonHint - | Qt.WindowType.WindowMaximizeButtonHint - ) - self.resize(700, 500) - - layout = QVBoxLayout(self) - - # Instructions - instructions = QLabel( - "Enter or paste the text you want to convert to audio:", self - ) - layout.addWidget(instructions) - - # Text edit area - self.text_edit = QTextEdit(self) - self.text_edit.setAcceptRichText(False) - self.text_edit.setPlaceholderText("Type or paste your text here...") - layout.addWidget(self.text_edit) - - # Character count label - self.char_count_label = QLabel("Characters: 0", self) - layout.addWidget(self.char_count_label) - - # Connect text changed signal to update character count - self.text_edit.textChanged.connect(self.update_char_count) - - # Buttons - button_layout = QHBoxLayout() - - self.save_as_button = QPushButton("Save as text", self) - self.save_as_button.clicked.connect(self.save_as_text) - self.save_as_button.setToolTip("Save the current text to a file") - - self.insert_chapter_btn = QPushButton("Insert Chapter Marker", self) - self.insert_chapter_btn.setToolTip("Insert a chapter marker at the cursor") - self.insert_chapter_btn.clicked.connect(self.insert_chapter_marker) - button_layout.addWidget(self.insert_chapter_btn) - - self.cancel_button = QPushButton("Cancel", self) - self.cancel_button.clicked.connect(self.reject) - - self.ok_button = QPushButton("OK", self) - self.ok_button.setDefault(True) - self.ok_button.clicked.connect(self.handle_ok) - - button_layout.addWidget(self.save_as_button) - button_layout.addWidget(self.cancel_button) - button_layout.addWidget(self.ok_button) - layout.addLayout(button_layout) - - # Store the original text to detect changes - self.original_text = "" - - def update_char_count(self): - text = self.text_edit.toPlainText() - count = calculate_text_length(text) - self.char_count_label.setText(f"Characters: {count:,}") - - def get_text(self): - return self.text_edit.toPlainText() - - def handle_ok(self): - text = self.text_edit.toPlainText() - # Check if text is empty based on character count - if calculate_text_length(text) == 0: - QMessageBox.warning(self, "Textbox Error", "Text cannot be empty.") - return - - # If the text hasn't changed, treat as cancel - if text == self.original_text: - self.reject() - else: - # Check if we need to warn about overwriting a non-temporary file - if hasattr(self, "is_non_cache_file") and self.is_non_cache_file: - msg_box = QMessageBox(self) - msg_box.setIcon(QMessageBox.Icon.Warning) - msg_box.setWindowTitle("File Overwrite Warning") - msg_box.setText( - f"You are about to overwrite the original file:\n{self.non_cache_file_path}" - ) - msg_box.setInformativeText("Do you want to continue?") - msg_box.setStandardButtons( - QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No - ) - msg_box.setDefaultButton(QMessageBox.StandardButton.No) - - if msg_box.exec() != QMessageBox.StandardButton.Yes: - # User canceled, don't close the dialog - return - - self.accept() - - def save_as_text(self): - """Save the text content to a file chosen by the user""" - try: - text = self.text_edit.toPlainText() - if not text.strip(): - QMessageBox.warning(self, "Save Error", "There is no text to save.") - return - - # Get default filename from original file if editing - initial_path = "" - if hasattr(self, "non_cache_file_path") and self.non_cache_file_path: - initial_path = self.non_cache_file_path - - # For EPUB and PDF files, use the displayed_file_path from the main window - # This gives a better filename instead of the cache file path - main_window = self.parent() - if ( - hasattr(main_window, "displayed_file_path") - and main_window.displayed_file_path - ): - if main_window.selected_file_type in ["epub", "pdf", "md", "markdown"]: - # Use the base name of the displayed file but change extension to .txt - base_name = os.path.splitext(main_window.displayed_file_path)[0] - initial_path = base_name + ".txt" - - file_path, _ = QFileDialog.getSaveFileName( - self, "Save Text As", initial_path, "Text Files (*.txt);;All Files (*)" - ) - - if file_path: - # Add .txt extension if not specified and no other extension exists - if not os.path.splitext(file_path)[1]: - file_path += ".txt" - - with open(file_path, "w", encoding="utf-8") as f: - f.write(text) - - except Exception as e: - QMessageBox.critical(self, "Save Error", f"Could not save file:\n{e}") - - def insert_chapter_marker(self): - # Insert a fixed chapter marker without prompting - cursor = self.text_edit.textCursor() - cursor.insertText("\n<>\n") - self.text_edit.setTextCursor(cursor) - self.update_char_count() - self.text_edit.setFocus() - - -def migrate_subtitle_format(config): - """Convert old subtitle_format values to new internal keys.""" - old_to_new = { - "srt": "srt", - "ass (wide)": "ass_wide", - "ass (narrow)": "ass_narrow", - "ass (centered wide)": "ass_centered_wide", - "ass (centered narrow)": "ass_centered_narrow", - } - val = config.get("subtitle_format") - if val in old_to_new: - config["subtitle_format"] = old_to_new[val] - save_config(config) - - -class abogen(QWidget): - def __init__(self): - super().__init__() - self.config = load_config() - self.apply_theme(self.config.get("theme", "system")) - migrate_subtitle_format(self.config) - self.check_updates = self.config.get("check_updates", True) - self.save_option = self.config.get("save_option", "Save next to input file") - self.selected_output_folder = self.config.get("selected_output_folder", None) - self.selected_file = self.selected_file_type = self.selected_book_path = None - self.displayed_file_path = ( - None # Add new variable to track the displayed file path - ) - # Max log lines - self.log_window_max_lines = self.config.get("log_window_max_lines", 2000) - self.selected_chapters = set() - self.last_opened_book_path = None # Track the last opened book path - self.last_output_path = None - self.char_count = 0 - self._char_count_cache = {} - # Only one of selected_profile_name or selected_voice should be set - self.selected_profile_name = self.config.get("selected_profile_name") - self.selected_voice = None - self.selected_lang = None - self.mixed_voice_state = None - if self.selected_profile_name: - self.selected_voice = None - self.selected_lang = None - else: - self.selected_voice = self.config.get("selected_voice", "af_heart") - self.selected_lang = self.selected_voice[0] if self.selected_voice else None - self.is_converting = False - self.subtitle_mode = self.config.get("subtitle_mode", "Sentence") - self.max_subtitle_words = self.config.get( - "max_subtitle_words", 50 - ) # Default max words per subtitle - self.silence_duration = self.config.get( - "silence_duration", 2.0 - ) # Default silence duration - self.selected_format = self.config.get("selected_format", "wav") - self.separate_chapters_format = self.config.get( - "separate_chapters_format", "wav" - ) # Format for individual chapter files - self.use_gpu = self.config.get( - "use_gpu", True # Load GPU setting with default True - ) - self.replace_single_newlines = self.config.get("replace_single_newlines", True) - self.use_silent_gaps = self.config.get("use_silent_gaps", True) - self.subtitle_speed_method = self.config.get("subtitle_speed_method", "tts") - self.use_spacy_segmentation = self.config.get("use_spacy_segmentation", True) - self._pending_close_event = None - self.gpu_ok = False # Initialize GPU availability status - - # Create thread-safe logging mechanism - self.log_signal = ThreadSafeLogSignal() - self.log_signal.log_signal.connect(self._update_log_main_thread) - - # Create warning signal emitter - self.warning_signal_emitter = ShowWarningSignalEmitter() - self.warning_signal_emitter.show_warning_signal.connect( - self.show_model_download_warning - ) - hf_tracker.set_show_warning_signal_emitter(self.warning_signal_emitter) - - # Set application icon - icon_path = get_resource_path("abogen.assets", "icon.ico") - if icon_path: - self.setWindowIcon(QIcon(icon_path)) - # Set taskbar icon for Windows - if platform.system() == "Windows": - ctypes.windll.shell32.SetCurrentProcessExplicitAppUserModelID("abogen") - - # Queued items list - self.queued_items = [] - self.current_queue_index = 0 - - self.initUI() - self.speed_slider.setValue(int(self.config.get("speed", 1.00) * 100)) - self.update_speed_label() - # Set initial selection: prefer profile, else voice - idx = -1 - if self.selected_profile_name: - idx = self.voice_combo.findData(f"profile:{self.selected_profile_name}") - elif self.selected_voice: - idx = self.voice_combo.findData(self.selected_voice) - if idx >= 0: - self.voice_combo.setCurrentIndex(idx) - # If a profile is selected at startup, load voices and language - if self.selected_profile_name: - from abogen.voice_profiles import load_profiles - - entry = load_profiles().get(self.selected_profile_name, {}) - if isinstance(entry, dict): - self.mixed_voice_state = entry.get("voices", []) - self.selected_lang = entry.get("language") - else: - self.mixed_voice_state = entry - self.selected_lang = entry[0][0] if entry and entry[0] else None - if self.save_option == "Choose output folder" and self.selected_output_folder: - self.save_path_label.setText(self.selected_output_folder) - self.save_path_row_widget.show() - self.subtitle_combo.setCurrentText(self.subtitle_mode) - # Enable/disable subtitle options based on selected language (profile or voice) - enable = self.selected_lang in SUPPORTED_LANGUAGES_FOR_SUBTITLE_GENERATION - self.subtitle_combo.setEnabled(enable) - self.subtitle_format_combo.setEnabled(enable) - # loading gif for preview button - loading_gif_path = get_resource_path("abogen.assets", "loading.gif") - if loading_gif_path: - self.loading_movie = QMovie(loading_gif_path) - self.loading_movie.frameChanged.connect( - lambda: self.btn_preview.setIcon( - QIcon(self.loading_movie.currentPixmap()) - ) - ) - - # Check for updates at startup if enabled - if self.check_updates: - QTimer.singleShot(1000, self.check_for_updates_startup) - - # Set hf_tracker callbacks - hf_tracker.set_log_callback(self.update_log) - - def initUI(self): - self.setWindowTitle(f"{PROGRAM_NAME} v{VERSION}") - screen = QApplication.primaryScreen().geometry() - width, height = 500, 800 - x = (screen.width() - width) // 2 - # If desired height is larger than screen, fit to screen height - if height > screen.height() - 65: - height = screen.height() - 100 # Leave a margin for window borders - y = max((screen.height() - height) // 2, 0) - self.setGeometry(x, y, width, height) - outer_layout = QVBoxLayout() - outer_layout.setContentsMargins(15, 15, 15, 15) - container = QWidget(self) - container_layout = QVBoxLayout(container) - container_layout.setContentsMargins(0, 0, 0, 0) - container_layout.setSpacing(15) - self.input_box = InputBox(self) - container_layout.addWidget(self.input_box, 1) - # Manage queue button, start queue button - self.queue_row_widget = QWidget(self) # Make queue_row a QWidget - queue_row = QHBoxLayout(self.queue_row_widget) - queue_row.setContentsMargins(0, 0, 0, 0) - self.btn_add_to_queue = QPushButton("Add to Queue", self) - self.btn_add_to_queue.setFixedHeight(40) - self.btn_add_to_queue.setEnabled(False) - self.btn_add_to_queue.clicked.connect(self.add_to_queue) - queue_row.addWidget(self.btn_add_to_queue) - self.btn_manage_queue = QPushButton("Manage Queue", self) - self.btn_manage_queue.setFixedHeight(40) - self.btn_manage_queue.setEnabled(True) - self.btn_manage_queue.clicked.connect(self.manage_queue) - queue_row.addWidget(self.btn_manage_queue) - self.btn_clear_queue = QPushButton("Clear Queue", self) - self.btn_clear_queue.setFixedHeight(40) - self.btn_clear_queue.setEnabled(False) - self.btn_clear_queue.clicked.connect(self.clear_queue) - queue_row.addWidget(self.btn_clear_queue) - container_layout.addWidget(self.queue_row_widget) - self.log_text = QTextEdit(self) - self.log_text.setReadOnly(True) - self.log_text.setUndoRedoEnabled(False) - self.log_text.setFrameStyle(QFrame.Shape.NoFrame) - self.log_text.setStyleSheet("QTextEdit { border: none; }") - self.log_text.hide() - container_layout.addWidget(self.log_text, 1) - controls_layout = QVBoxLayout() - controls_layout.setContentsMargins(0, 10, 0, 0) - controls_layout.setSpacing(15) - # Speed controls - speed_layout = QVBoxLayout() - speed_layout.setSpacing(2) - speed_layout.addWidget(QLabel("Speed:", self)) - self.speed_slider = QSlider(Qt.Orientation.Horizontal, self) - self.speed_slider.setMinimum(10) - self.speed_slider.setMaximum(200) - self.speed_slider.setValue(100) - self.speed_slider.setTickPosition(QSlider.TickPosition.TicksBelow) - self.speed_slider.setTickInterval(5) - self.speed_slider.setSingleStep(5) - speed_layout.addWidget(self.speed_slider) - self.speed_label = QLabel("1.0", self) - speed_layout.addWidget(self.speed_label) - controls_layout.addLayout(speed_layout) - self.speed_slider.valueChanged.connect(self.update_speed_label) - # Voice selection - voice_layout = QHBoxLayout() - voice_layout.setSpacing(7) - voice_label = QLabel("Select voice:", self) - voice_layout.addWidget(voice_label) - self.voice_combo = QComboBox(self) - self.voice_combo.currentIndexChanged.connect(self.on_voice_combo_changed) - self.voice_combo.setStyleSheet( - "QComboBox { min-height: 20px; padding: 6px 12px; }" - ) - self.voice_combo.setToolTip( - "The first character represents the language:\n" - '"a" => American English\n"b" => British English\n"e" => Spanish\n"f" => French\n"h" => Hindi\n"i" => Italian\n"j" => Japanese\n"p" => Brazilian Portuguese\n"z" => Mandarin Chinese\nThe second character represents the gender:\n"m" => Male\n"f" => Female' - ) - self.voice_combo.setSizePolicy( - QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed - ) - voice_layout.addWidget(self.voice_combo) - # Voice formula button - self.btn_voice_formula_mixer = QPushButton(self) - mixer_icon_path = get_resource_path("abogen.assets", "voice_mixer.png") - self.btn_voice_formula_mixer.setIcon(QIcon(mixer_icon_path)) - self.btn_voice_formula_mixer.setToolTip("Mix and match voices") - self.btn_voice_formula_mixer.setFixedSize(40, 36) - self.btn_voice_formula_mixer.setStyleSheet("QPushButton { padding: 6px 12px; }") - self.btn_voice_formula_mixer.clicked.connect(self.show_voice_formula_dialog) - voice_layout.addWidget(self.btn_voice_formula_mixer) - - # Play/Stop icons - def make_icon(color, shape): - pix = QPixmap(20, 20) - pix.fill(Qt.GlobalColor.transparent) - p = QPainter(pix) - p.setRenderHint(QPainter.RenderHint.Antialiasing) - p.setBrush(QColor(*color)) - p.setPen(Qt.PenStyle.NoPen) - if shape == "play": - pts = [ - pix.rect().topLeft() + QPoint(4, 2), - pix.rect().bottomLeft() + QPoint(4, -2), - pix.rect().center() + QPoint(6, 0), - ] - p.drawPolygon(QPolygon(pts)) - else: - p.drawRect(5, 5, 10, 10) - p.end() - return QIcon(pix) - - self.play_icon = make_icon((40, 160, 40), "play") - self.stop_icon = make_icon((200, 60, 60), "stop") - self.btn_preview = QPushButton(self) - self.btn_preview.setIcon(self.play_icon) - self.btn_preview.setIconSize(QPixmap(20, 20).size()) - self.btn_preview.setToolTip("Preview selected voice") - self.btn_preview.setFixedSize(40, 36) - self.btn_preview.setStyleSheet("QPushButton { padding: 6px 12px; }") - self.btn_preview.clicked.connect(self.preview_voice) - voice_layout.addWidget(self.btn_preview) - self.preview_playing = False - self.play_audio_thread = None # Keep track of audio playing thread - controls_layout.addLayout(voice_layout) - - # Generate subtitles - subtitle_layout = QHBoxLayout() - subtitle_layout.setSpacing(7) - subtitle_label = QLabel("Generate subtitles:", self) - subtitle_layout.addWidget(subtitle_label) - self.subtitle_combo = QComboBox(self) - self.subtitle_combo.setToolTip( - "Choose how subtitles will be generated:\n" - "Disabled: No subtitles will be generated.\n" - "Line: Subtitles will be generated for each line.\n" - "Sentence: Subtitles will be generated for each sentence.\n" - "Sentence + Comma: Subtitles will be generated for each sentence and comma.\n" - "Sentence + Highlighting: Subtitles with word-by-word karaoke highlighting.\n" - "1+ word: Subtitles will be generated for each word(s).\n\n" - "Supported languages for subtitle generation:\n" - + "\n".join( - f'"{lang}" => {LANGUAGE_DESCRIPTIONS.get(lang, lang)}' - for lang in SUPPORTED_LANGUAGES_FOR_SUBTITLE_GENERATION - ) - ) - subtitle_options = [ - "Disabled", - "Line", - "Sentence", - "Sentence + Comma", - "Sentence + Highlighting", - ] + [f"{i} word" if i == 1 else f"{i} words" for i in range(1, 11)] - self.subtitle_combo.addItems(subtitle_options) - self.subtitle_combo.setStyleSheet( - "QComboBox { min-height: 20px; padding: 6px 12px; }" - ) - self.subtitle_combo.setSizePolicy( - QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed - ) - self.subtitle_combo.setCurrentText(self.subtitle_mode) - self.subtitle_combo.currentTextChanged.connect(self.on_subtitle_mode_changed) - subtitle_layout.addWidget(self.subtitle_combo) - controls_layout.addLayout(subtitle_layout) - - # Output voice format - format_layout = QHBoxLayout() - format_layout.setSpacing(7) - format_label = QLabel("Output voice format:", self) - format_layout.addWidget(format_label) - self.format_combo = QComboBox(self) - self.format_combo.setStyleSheet( - "QComboBox { min-height: 20px; padding: 6px 12px; }" - ) - self.format_combo.setSizePolicy( - QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed - ) - # Add items with display labels and underlying keys - for key, label in [ - ("wav", "wav"), - ("flac", "flac"), - ("mp3", "mp3"), - ("opus", "opus (best compression)"), - ("m4b", "m4b (with chapters)"), - ]: - self.format_combo.addItem(label, key) - # Initialize selection by matching saved key - idx = self.format_combo.findData(self.selected_format) - if idx >= 0: - self.format_combo.setCurrentIndex(idx) - # Map selection back to key on change - self.format_combo.currentIndexChanged.connect( - lambda i: self.on_format_changed(self.format_combo.itemData(i)) - ) - format_layout.addWidget(self.format_combo) - controls_layout.addLayout(format_layout) - - # Output subtitle format - subtitle_format_layout = QHBoxLayout() - subtitle_format_layout.setSpacing(7) - subtitle_format_label = QLabel("Output subtitle format:", self) - subtitle_format_layout.addWidget(subtitle_format_label) - self.subtitle_format_combo = QComboBox(self) - self.subtitle_format_combo.setStyleSheet( - "QComboBox { min-height: 20px; padding: 6px 12px; }" - ) - self.subtitle_format_combo.setSizePolicy( - QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed - ) - for value, text in SUBTITLE_FORMATS: - self.subtitle_format_combo.addItem(text, value) - subtitle_format = self.config.get("subtitle_format", "ass_centered_narrow") - idx = self.subtitle_format_combo.findData(subtitle_format) - if idx >= 0: - self.subtitle_format_combo.setCurrentIndex(idx) - self.subtitle_format_combo.currentIndexChanged.connect( - lambda i: self.set_subtitle_format(self.subtitle_format_combo.itemData(i)) - ) - subtitle_format_layout.addWidget(self.subtitle_format_combo) - # If subtitle mode requires highlighting, SRT is not supported. Disable SRT item - # and auto-switch to a compatible ASS format if SRT is currently selected. - try: - if ( - hasattr(self, "subtitle_mode") - and self.subtitle_mode == "Sentence + Highlighting" - ): - idx_srt = self.subtitle_format_combo.findData("srt") - if idx_srt >= 0: - item = self.subtitle_format_combo.model().item(idx_srt) - if item is not None: - item.setEnabled(False) - # If current selection is SRT, switch to centered narrow ASS - if self.subtitle_format_combo.currentData() == "srt": - new_idx = self.subtitle_format_combo.findData("ass_centered_narrow") - if new_idx >= 0: - self.subtitle_format_combo.setCurrentIndex(new_idx) - # Persist the change - self.set_subtitle_format( - self.subtitle_format_combo.itemData(new_idx) - ) - except Exception: - # Fail-safe: don't crash UI if model manipulation isn't supported on some platforms - pass - - # Enable/disable subtitle options based on selected language (profile or voice) - self.update_subtitle_options_availability() - - controls_layout.addLayout(subtitle_format_layout) - - # Replace single newlines dropdown (acts like checkbox) - replace_newlines_layout = QHBoxLayout() - replace_newlines_layout.setSpacing(7) - replace_newlines_label = QLabel("Replace single newlines:", self) - replace_newlines_layout.addWidget(replace_newlines_label) - self.replace_newlines_combo = QComboBox(self) - self.replace_newlines_combo.addItems(["Disabled", "Enabled"]) - self.replace_newlines_combo.setToolTip( - "Replace single newlines in the input text with spaces before processing." - ) - self.replace_newlines_combo.setStyleSheet( - "QComboBox { min-height: 20px; padding: 6px 12px; }" - ) - self.replace_newlines_combo.setSizePolicy( - QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed - ) - # Set initial value based on config - self.replace_newlines_combo.setCurrentIndex( - 1 if self.replace_single_newlines else 0 - ) - self.replace_newlines_combo.currentIndexChanged.connect( - lambda idx: self.toggle_replace_single_newlines(idx == 1) - ) - replace_newlines_layout.addWidget(self.replace_newlines_combo) - controls_layout.addLayout(replace_newlines_layout) - - # Save location - save_layout = QHBoxLayout() - save_layout.setSpacing(7) - save_label = QLabel("Save location:", self) - save_layout.addWidget(save_label) - self.save_combo = QComboBox(self) - save_options = [ - "Save next to input file", - "Save to Desktop", - "Choose output folder", - ] - self.save_combo.addItems(save_options) - self.save_combo.setStyleSheet( - "QComboBox { min-height: 20px; padding: 6px 12px; }" - ) - self.save_combo.setSizePolicy( - QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed - ) - self.save_combo.setCurrentText(self.save_option) - self.save_combo.currentTextChanged.connect(self.on_save_option_changed) - save_layout.addWidget(self.save_combo) - controls_layout.addLayout(save_layout) - - # Save path label - self.save_path_row_widget = QWidget(self) - save_path_row = QHBoxLayout(self.save_path_row_widget) - save_path_row.setSpacing(7) - save_path_row.setContentsMargins(0, 0, 0, 0) - selected_folder_label = QLabel("Selected folder:", self.save_path_row_widget) - save_path_row.addWidget(selected_folder_label) - self.save_path_label = QLabel("", self.save_path_row_widget) - self.save_path_label.setStyleSheet(f"QLabel {{ color: {COLORS['GREEN']}; }}") - self.save_path_label.setSizePolicy( - QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Preferred - ) - save_path_row.addWidget(self.save_path_label) - self.save_path_row_widget.hide() # Hide the whole row by default - controls_layout.addWidget(self.save_path_row_widget) - - # GPU Acceleration Checkbox with Settings button - gpu_layout = QHBoxLayout() - gpu_checkbox_layout = QVBoxLayout() - self.gpu_checkbox = QCheckBox("Use GPU Acceleration (if available)", self) - self.gpu_checkbox.setChecked(self.use_gpu) - self.gpu_checkbox.setToolTip( - "Uncheck to force using CPU even if a compatible GPU is detected." - ) - self.gpu_checkbox.stateChanged.connect(self.on_gpu_setting_changed) - gpu_checkbox_layout.addWidget(self.gpu_checkbox) - gpu_layout.addLayout(gpu_checkbox_layout) - - # Set initial enabled state for subtitle format combo - if self.subtitle_mode == "Disabled": - self.subtitle_format_combo.setEnabled(False) - else: - self.subtitle_format_combo.setEnabled(True) - - # Settings button with icon - settings_icon_path = get_resource_path("abogen.assets", "settings.svg") - self.settings_btn = QPushButton(self) - if settings_icon_path and os.path.exists(settings_icon_path): - self.settings_btn.setIcon(QIcon(settings_icon_path)) - else: - # Fallback text if icon not found - self.settings_btn.setText("⚙") - self.settings_btn.setToolTip("Settings") - self.settings_btn.setFixedSize(36, 36) - self.settings_btn.clicked.connect(self.show_settings_menu) - gpu_layout.addWidget(self.settings_btn) - - controls_layout.addLayout(gpu_layout) - - # Start button - self.btn_start = QPushButton("Start", self) - self.btn_start.setFixedHeight(60) - self.btn_start.clicked.connect(self.start_conversion) - controls_layout.addWidget(self.btn_start) - # Add controls to a container widget - self.controls_widget = QWidget() - self.controls_widget.setLayout(controls_layout) - self.controls_widget.setSizePolicy( - QSizePolicy.Policy.Preferred, QSizePolicy.Policy.Fixed - ) - container_layout.addWidget(self.controls_widget) - # Progress bar - self.progress_bar = QProgressBar(self) - self.progress_bar.setValue(0) - self.progress_bar.hide() - container_layout.addWidget(self.progress_bar) - # ETR Label - self.etr_label = QLabel("Estimated time remaining: Calculating...", self) - self.etr_label.setAlignment(Qt.AlignmentFlag.AlignCenter) - self.etr_label.hide() - container_layout.addWidget(self.etr_label) - # Cancel button - self.btn_cancel = QPushButton("Cancel", self) - self.btn_cancel.setFixedHeight(60) - self.btn_cancel.clicked.connect(self.cancel_conversion) - self.btn_cancel.hide() - container_layout.addWidget(self.btn_cancel) - # Finish buttons - self.finish_widget = QWidget() - finish_layout = QVBoxLayout() - finish_layout.setContentsMargins(0, 0, 0, 0) - finish_layout.setSpacing(10) - self.open_file_btn = None # Store reference to open file button - - # Create buttons with their functions - finish_buttons = [ - ("Open file", self.open_file, "Open the output file."), - ( - "Go to folder", - self.go_to_file, - "Open the folder containing the output file.", - ), - ("New Conversion", self.reset_ui, "Start a new conversion."), - ("Go back", self.go_back_ui, "Return to the previous screen."), - ] - - for text, func, tip in finish_buttons: - btn = QPushButton(text, self) - btn.setFixedHeight(35) - btn.setToolTip(tip) - btn.clicked.connect(func) - finish_layout.addWidget(btn) - # Identify the Open file button by its function reference - if func == self.open_file: - self.open_file_btn = btn # Save reference to the open file button - - self.finish_widget.setLayout(finish_layout) - self.finish_widget.hide() - container_layout.addWidget(self.finish_widget) - outer_layout.addWidget(container) - self.setLayout(outer_layout) - self.populate_profiles_in_voice_combo() - - # Initialize flag to track if input box was cleared by queue - self.input_box_cleared_by_queue = False - - def open_file_dialog(self): - if self.is_converting: - return - try: - file_path, _ = QFileDialog.getOpenFileName( - self, - "Select File", - "", - "Supported Files (*.txt *.epub *.pdf *.md *.srt *.ass *.vtt)", - ) - if not file_path: - return - if ( - file_path.lower().endswith(".epub") - or file_path.lower().endswith(".pdf") - or file_path.lower().endswith((".md", ".markdown")) - ): - # Determine file type - if file_path.lower().endswith(".epub"): - self.selected_file_type = "epub" - elif file_path.lower().endswith(".pdf"): - self.selected_file_type = "pdf" - else: - self.selected_file_type = "markdown" - - self.selected_book_path = file_path - # Don't set file info immediately, open_book_file will handle it after dialog is accepted - if not self.open_book_file(file_path): - return - elif file_path.lower().endswith((".srt", ".ass", ".vtt")): - # Handle subtitle files like text files - self.selected_file, self.selected_file_type = file_path, "txt" - self.displayed_file_path = file_path - self.input_box.set_file_info(file_path) - else: - self.selected_file, self.selected_file_type = file_path, "txt" - self.displayed_file_path = ( - file_path # Set the displayed file path for text files - ) - self.input_box.set_file_info(file_path) - except Exception as e: - self._show_error_message_box( - "File Dialog Error", f"Could not open file dialog:\n{e}" - ) - - def open_book_file(self, book_path): - # Clear selected chapters if this is a different book than the last one - if ( - not hasattr(self, "last_opened_book_path") - or self.last_opened_book_path != book_path - ): - self.selected_chapters = set() - self.last_opened_book_path = book_path - - # HandlerDialog uses internal caching to avoid reprocessing the same book - dialog = HandlerDialog( - book_path, - file_type=getattr(self, "selected_file_type", None), - checked_chapters=self.selected_chapters, - parent=self, - ) - dialog.setWindowModality(Qt.WindowModality.NonModal) - dialog.setModal(False) - dialog.show() # We'll handle the dialog result asynchronously - - def on_dialog_finished(result): - if result != QDialog.DialogCode.Accepted: - return False - chapters_text, all_checked_hrefs = dialog.get_selected_text() - if not all_checked_hrefs: - # Determine file type for error message - if book_path.lower().endswith(".pdf"): - file_type = "pdf" - item_type = "pages" - elif book_path.lower().endswith((".md", ".markdown")): - file_type = "markdown" - item_type = "chapters" - else: - file_type = "epub" - item_type = "chapters" - - error_msg = f"No {item_type} selected." - self._show_error_message_box(f"{file_type.upper()} Error", error_msg) - return False - self.selected_chapters = all_checked_hrefs - self.save_chapters_separately = dialog.get_save_chapters_separately() - self.merge_chapters_at_end = dialog.get_merge_chapters_at_end() - self.save_as_project = dialog.get_save_as_project() - - # Store if the PDF has bookmarks for button text display - if book_path.lower().endswith(".pdf"): - self.pdf_has_bookmarks = getattr(dialog, "has_pdf_bookmarks", False) - - cleaned_text = clean_text(chapters_text) - computed_char_count = calculate_text_length(cleaned_text) - self.char_count = computed_char_count - if isinstance(getattr(self, "_char_count_cache", None), dict): - self._char_count_cache[book_path] = computed_char_count - - # Use "abogen" prefix for cache files - # Extract base name without extension - base_name = os.path.splitext(os.path.basename(book_path))[0] - - if self.save_as_project: - # Get project directory from user - project_dir = QFileDialog.getExistingDirectory( - self, "Select Project Folder", "", QFileDialog.Option.ShowDirsOnly - ) - if not project_dir: - # User cancelled, fallback to cache - self.save_as_project = False - cache_dir = get_user_cache_path() - else: - # Create project folder structure - project_name = f"{base_name}_project" - project_dir = os.path.join(project_dir, project_name) - cache_dir = os.path.join(project_dir, "text") - os.makedirs(cache_dir, exist_ok=True) - - # Save metadata if available - meta_dir = os.path.join(project_dir, "metadata") - os.makedirs( - meta_dir, exist_ok=True - ) # Save book metadata if available - if hasattr(dialog, "book_metadata"): - meta_path = os.path.join(meta_dir, "book_info.txt") - with open(meta_path, "w", encoding="utf-8") as f: - # Clean HTML tags from metadata - title = re.sub( - r"<[^>]+>", - "", - str(dialog.book_metadata.get("title", "Unknown")), - ) - publisher = re.sub( - r"<[^>]+>", - "", - str(dialog.book_metadata.get("publisher", "Unknown")), - ) - authors = [ - re.sub(r"<[^>]+>", "", str(author)) - for author in dialog.book_metadata.get( - "authors", ["Unknown"] - ) - ] - publication_year = re.sub( - r"<[^>]+>", - "", - str( - dialog.book_metadata.get( - "publication_year", "Unknown" - ) - ), - ) - - f.write(f"Title: {title}\n") - f.write(f"Authors: {', '.join(authors)}\n") - f.write(f"Publisher: {publisher}\n") - f.write(f"Publication Year: {publication_year}\n") - if dialog.book_metadata.get("description"): - description = re.sub( - r"<[^>]+>", - "", - str(dialog.book_metadata.get("description")), - ) - f.write(f"\nDescription:\n{description}\n") - - # Save cover image if available - if dialog.book_metadata.get("cover_image"): - cover_path = os.path.join(meta_dir, "cover.png") - with open(cover_path, "wb") as f: - f.write(dialog.book_metadata["cover_image"]) - else: - cache_dir = get_user_cache_path() - - fd, tmp = tempfile.mkstemp( - prefix=f"{base_name}_", suffix=".txt", dir=cache_dir - ) - os.close(fd) - with open(tmp, "w", encoding="utf-8") as f: - f.write(chapters_text) - self.selected_file = tmp - self.selected_book_path = book_path - self.displayed_file_path = book_path - if isinstance(getattr(self, "_char_count_cache", None), dict): - self._char_count_cache[tmp] = computed_char_count - # Only set file info if dialog was accepted - self.input_box.set_file_info(book_path) - return True - - dialog.finished.connect(on_dialog_finished) - return True - - def open_textbox_dialog(self, file_path=None): - """Shows dialog for direct text input or editing and processes the entered text""" - if self.is_converting: - return - - editing = False - is_cache_file = False - # If path is explicitly provided, use it - if file_path and os.path.exists(file_path): - editing = True - edit_file = file_path - # Check if this is a cache file - is_cache_file = get_user_cache_path() in file_path - # Otherwise use selected_file if it's a txt file - elif ( - self.selected_file_type == "txt" - and self.selected_file - and os.path.exists(self.selected_file) - ): - editing = True - edit_file = self.selected_file - # Check if this is a cache file - is_cache_file = get_user_cache_path() in self.selected_file - - dialog = TextboxDialog(self) - if editing: - try: - with open(edit_file, "r", encoding="utf-8", errors="ignore") as f: - dialog.text_edit.setText(f.read()) - dialog.update_char_count() - dialog.original_text = ( - dialog.text_edit.toPlainText() - ) # Store original text - - # If editing a non-cache file, alert the user - if not is_cache_file: - dialog.is_non_cache_file = True - dialog.non_cache_file_path = edit_file - except Exception: - pass - if dialog.exec() == QDialog.DialogCode.Accepted: - text = dialog.get_text() - if not text.strip(): - self._show_error_message_box("Textbox Error", "Text cannot be empty.") - return - try: - if editing: - with open(edit_file, "w", encoding="utf-8") as f: - f.write(text) - # Update the display path to the edited file - self.displayed_file_path = edit_file - self.input_box.set_file_info(edit_file) - # Hide chapters button since we're using custom text now - self.input_box.chapters_btn.hide() - else: - cache_dir = get_user_cache_path() - fd, tmp = tempfile.mkstemp( - prefix="abogen_", suffix=".txt", dir=cache_dir - ) - os.close(fd) - with open(tmp, "w", encoding="utf-8") as f: - f.write(text) - self.selected_file = tmp - self.selected_file_type = "txt" - self.displayed_file_path = None - self.input_box.set_file_info(tmp) - # Hide chapters button since we're using custom text now - self.input_box.chapters_btn.hide() - if hasattr(self, "conversion_thread"): - self.conversion_thread.is_direct_text = True - except Exception as e: - self._show_error_message_box( - "Textbox Error", f"Could not process text input:\n{e}" - ) - - def update_speed_label(self): - s = self.speed_slider.value() / 100.0 - self.speed_label.setText(f"{s}") - self.config["speed"] = s - save_config(self.config) - - def update_subtitle_options_availability(self): - """ - Update the enabled state of subtitle options based on the selected language. - For non-English languages, only sentence-based and line-based modes are supported. - """ - # Check if current file is a subtitle file - is_subtitle_input = False - if self.selected_file and self.selected_file.lower().endswith( - (".srt", ".ass", ".vtt") - ): - is_subtitle_input = True - - if self.selected_lang not in SUPPORTED_LANGUAGES_FOR_SUBTITLE_GENERATION: - self.subtitle_combo.setEnabled(False) - self.subtitle_format_combo.setEnabled(False) - return - - # Only enable subtitle_combo if it's NOT a subtitle input - self.subtitle_combo.setEnabled(not is_subtitle_input) - self.subtitle_format_combo.setEnabled(True) - - is_english = self.selected_lang in ["a", "b"] - - # Items to keep enabled for non-English - allowed_modes = ["Disabled", "Line", "Sentence", "Sentence + Comma"] - - model = self.subtitle_combo.model() - for i in range(self.subtitle_combo.count()): - text = self.subtitle_combo.itemText(i) - item = model.item(i) - if not item: - continue - - if is_english: - item.setEnabled(True) - else: - if text in allowed_modes: - item.setEnabled(True) - else: - item.setEnabled(False) - - # If current selection is disabled, switch to a valid one - current_text = self.subtitle_combo.currentText() - current_idx = self.subtitle_combo.currentIndex() - current_item = model.item(current_idx) - - if current_item and not current_item.isEnabled(): - # Switch to "Sentence" if available, else "Disabled" - sentence_idx = self.subtitle_combo.findText("Sentence") - if sentence_idx >= 0: - self.subtitle_combo.setCurrentIndex(sentence_idx) - else: - self.subtitle_combo.setCurrentIndex(0) # Disabled - - self.subtitle_mode = self.subtitle_combo.currentText() - - def on_voice_changed(self, index): - voice = self.voice_combo.itemData(index) - self.selected_voice, self.selected_lang = voice, voice[0] - self.config["selected_voice"] = voice - save_config(self.config) - # Enable/disable subtitle options based on language - self.update_subtitle_options_availability() - - def on_voice_combo_changed(self, index): - data = self.voice_combo.itemData(index) - if isinstance(data, str) and data.startswith("profile:"): - pname = data.split(":", 1)[1] - self.selected_profile_name = pname - from abogen.voice_profiles import load_profiles - - entry = load_profiles().get(pname, {}) - # set mixed voices and language - if isinstance(entry, dict): - self.mixed_voice_state = entry.get("voices", []) - self.selected_lang = entry.get("language") - else: - self.mixed_voice_state = entry - self.selected_lang = entry[0][0] if entry and entry[0] else None - self.selected_voice = None - self.config["selected_profile_name"] = pname - self.config.pop("selected_voice", None) - save_config(self.config) - # enable subtitles based on profile language - self.update_subtitle_options_availability() - else: - self.mixed_voice_state = None - self.selected_profile_name = None - self.selected_voice, self.selected_lang = data, data[0] - self.config["selected_voice"] = data - if "selected_profile_name" in self.config: - del self.config["selected_profile_name"] - save_config(self.config) - self.update_subtitle_options_availability() - - def update_subtitle_combo_for_profile(self, profile_name): - from abogen.voice_profiles import load_profiles - - entry = load_profiles().get(profile_name, {}) - lang = entry.get("language") if isinstance(entry, dict) else None - enable = lang in SUPPORTED_LANGUAGES_FOR_SUBTITLE_GENERATION - self.subtitle_combo.setEnabled(enable) - self.subtitle_format_combo.setEnabled(enable) - - def populate_profiles_in_voice_combo(self): - # preserve current voice or profile - current = self.voice_combo.currentData() - self.voice_combo.blockSignals(True) - self.voice_combo.clear() - # re-add profiles - profile_icon = QIcon(get_resource_path("abogen.assets", "profile.png")) - for pname in load_profiles().keys(): - self.voice_combo.addItem(profile_icon, pname, f"profile:{pname}") - # re-add voices - for v in VOICES_INTERNAL: - icon = QIcon() - flag_path = get_resource_path("abogen.assets.flags", f"{v[0]}.png") - if flag_path and os.path.exists(flag_path): - icon = QIcon(flag_path) - self.voice_combo.addItem(icon, f"{v}", v) - # restore selection - idx = -1 - if self.selected_profile_name: - idx = self.voice_combo.findData(f"profile:{self.selected_profile_name}") - elif current: - idx = self.voice_combo.findData(current) - if idx >= 0: - self.voice_combo.setCurrentIndex(idx) - # Also update subtitle combo for selected profile - data = self.voice_combo.itemData(idx) - if isinstance(data, str) and data.startswith("profile:"): - pname = data.split(":", 1)[1] - self.update_subtitle_combo_for_profile(pname) - self.voice_combo.blockSignals(False) - # If no profiles exist, clear selected_profile_name from config - if not load_profiles(): - if "selected_profile_name" in self.config: - del self.config["selected_profile_name"] - save_config(self.config) - - def convert_input_box_to_log(self): - self.input_box.hide() - self.log_text.show() - self.log_text.clear() - QApplication.processEvents() - - def restore_input_box(self): - self.log_text.hide() - self.input_box.show() - - def update_log(self, message): - # Use signal-based approach for thread-safe logging - if QThread.currentThread() != QApplication.instance().thread(): - # We're in a background thread, emit signal for the main thread - self.log_signal.emit_log(message) - return - - # Direct update if already on main thread - self._update_log_main_thread(message) - - def _update_log_main_thread(self, message): - txt = self.log_text - sb = txt.verticalScrollBar() - at_bottom = sb.value() == sb.maximum() - - cursor = txt.textCursor() - cursor.movePosition(QTextCursor.MoveOperation.End) - - fmt = cursor.charFormat() - if isinstance(message, tuple): - text, spec = message - fmt.setForeground(QColor(LOG_COLOR_MAP.get(spec, COLORS["LIGHT_DISABLED"]))) - else: - text = str(message) - fmt.clearForeground() - cursor.setCharFormat(fmt) - cursor.insertText(text + "\n") - - doc = txt.document() - excess = doc.blockCount() - self.log_window_max_lines - if excess > 0: - start = doc.findBlockByNumber(0).position() - end = doc.findBlockByNumber(excess).position() - trim_cursor = QTextCursor(doc) - trim_cursor.setPosition(start) - trim_cursor.setPosition(end, QTextCursor.MoveMode.KeepAnchor) - trim_cursor.removeSelectedText() - - if at_bottom: - sb.setValue(sb.maximum()) - - def _get_queue_progress_format(self, value=None): - """Return the progress bar format string for queue mode.""" - if ( - hasattr(self, "queued_items") - and self.queued_items - and hasattr(self, "current_queue_index") - ): - N = self.current_queue_index + 1 - M = len(self.queued_items) - percent = value if value is not None else self.progress_bar.value() - return f"{percent}% ({N}/{M})" - else: - percent = value if value is not None else self.progress_bar.value() - return f"{percent}%" - - def update_progress(self, value, etr_str): # Add etr_str parameter - # Ensure progress doesn't exceed 99% - if value >= 100: - value = 99 - self.progress_bar.setValue(value) - # Show queue progress if in queue mode - if ( - hasattr(self, "queued_items") - and self.queued_items - and hasattr(self, "current_queue_index") - ): - N = self.current_queue_index + 1 - M = len(self.queued_items) - self.progress_bar.setFormat(f"{value}% ({N}/{M})") - else: - self.progress_bar.setFormat(f"{value}%") - self.etr_label.setText( - f"Estimated time remaining: {etr_str}" - ) # Update ETR label - self.etr_label.show() # Show only when estimate is ready - - # Disable cancel button if progress is >= 98% - if value >= 98: - self.btn_cancel.setEnabled(False) - - self.progress_bar.repaint() - QApplication.processEvents() - - def enable_disable_queue_buttons(self): - enabled = bool(self.queued_items) - self.btn_clear_queue.setEnabled(enabled) - # Update Manage Queue button text with count - if enabled: - self.btn_manage_queue.setText(f"Manage Queue ({len(self.queued_items)})") - self.btn_manage_queue.setStyleSheet( - f"QPushButton {{ color: {COLORS['GREEN']}; }}" - ) - else: - self.btn_manage_queue.setText("Manage Queue") - self.btn_manage_queue.setStyleSheet("") - # Change main Start button to 'Start queue' if queue has items - if enabled: - self.btn_start.setText(f"Start queue ({len(self.queued_items)})") - try: - self.btn_start.clicked.disconnect() - except Exception: - pass - self.btn_start.clicked.connect(self.start_queue) - else: - self.btn_start.setText("Start") - try: - self.btn_start.clicked.disconnect() - except Exception: - pass - self.btn_start.clicked.connect(self.start_conversion) - - def enqueue(self, item: QueuedItem): - self.queued_items.append(item) - # self.update_log((f"Enqueued: {item.file_name}", True)) - # enable start queue button, manage queue button - self.enable_disable_queue_buttons() - - def get_queue(self): - return self.queued_items - - def add_to_queue(self): - # For epub/pdf, always use the converted txt file (selected_file) - if self.selected_file_type in ["epub", "pdf", "md", "markdown"]: - file_to_queue = self.selected_file - # Use the original file path for save location - save_base_path = ( - self.displayed_file_path if self.displayed_file_path else file_to_queue - ) - else: - file_to_queue = ( - self.displayed_file_path - if self.displayed_file_path - else self.selected_file - ) - save_base_path = file_to_queue # For non-EPUB, it's the same - - if not file_to_queue: - self.input_box.set_error("Please add a file.") - return - actual_subtitle_mode = self.get_actual_subtitle_mode() - voice_formula = self.get_voice_formula() - selected_lang = self.get_selected_lang(voice_formula) - - item_queue = QueuedItem( - file_name=file_to_queue, - lang_code=selected_lang, - speed=self.speed_slider.value() / 100.0, - voice=voice_formula, - save_option=self.save_option, - output_folder=self.selected_output_folder, - subtitle_mode=actual_subtitle_mode, - output_format=self.selected_format, - total_char_count=self.char_count, - replace_single_newlines=self.replace_single_newlines, - use_silent_gaps=self.use_silent_gaps, - subtitle_speed_method=self.subtitle_speed_method, - save_base_path=save_base_path, - save_chapters_separately=getattr(self, "save_chapters_separately", None), - merge_chapters_at_end=getattr(self, "merge_chapters_at_end", None), - ) - - # Prevent adding duplicate items to the queue - for queued_item in self.queued_items: - if ( - queued_item.file_name == item_queue.file_name - and queued_item.lang_code == item_queue.lang_code - and queued_item.speed == item_queue.speed - and queued_item.voice == item_queue.voice - and queued_item.save_option == item_queue.save_option - and queued_item.output_folder == item_queue.output_folder - and queued_item.subtitle_mode == item_queue.subtitle_mode - and queued_item.output_format == item_queue.output_format - and getattr(queued_item, "replace_single_newlines", True) - == item_queue.replace_single_newlines - and getattr(queued_item, "save_base_path", None) - == item_queue.save_base_path - and getattr(queued_item, "save_chapters_separately", None) - == item_queue.save_chapters_separately - and getattr(queued_item, "merge_chapters_at_end", None) - == item_queue.merge_chapters_at_end - ): - QMessageBox.warning( - self, "Duplicate Item", "This item is already in the queue." - ) - return - - self.enqueue(item_queue) - # Clear input after adding to queue - self.input_box.clear_input() - self.input_box_cleared_by_queue = True # Set flag - self.enable_disable_queue_buttons() - - def clear_queue(self): - # Warn user if more than 1 item in the queue before clearing - if len(self.queued_items) > 1: - reply = QMessageBox.question( - self, - "Confirm Clear Queue", - f"Are you sure you want to clear {len(self.queued_items)} items from the queue?", - QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, - QMessageBox.StandardButton.No, - ) - if reply != QMessageBox.StandardButton.Yes: - return - self.queued_items = [] - self.enable_disable_queue_buttons() - - def manage_queue(self): - # show a dialog to manage the queue - dialog = QueueManager(self, self.queued_items) - if dialog.exec() == QDialog.DialogCode.Accepted: - self.queued_items = dialog.get_queue() - - # Reload config to capture the new "Override" setting - # The QueueManager writes to disk, so we must refresh our local copy - self.config = load_config() - - # re-enable/disable buttons based on queue state - self.enable_disable_queue_buttons() - - def start_queue(self): - self.current_queue_index = 0 # Start from the first item - # Set progress bar to 0% (1/M) immediately - if self.queued_items: - self.progress_bar.setValue(0) - self.progress_bar.setFormat(f"0% (1/{len(self.queued_items)})") - self.progress_bar.show() - self.start_next_queued_item() - - def start_next_queued_item(self): - if self.current_queue_index < len(self.queued_items): - queued_item = self.queued_items[self.current_queue_index] - - self.selected_file = queued_item.file_name - self.char_count = queued_item.total_char_count - - # Restore the original file path for save location (Important for EPUB/PDF) - self.displayed_file_path = ( - queued_item.save_base_path or queued_item.file_name - ) - - # Restore chapter options (Structure specific, must be preserved) - self.save_chapters_separately = getattr( - queued_item, "save_chapters_separately", None - ) - self.merge_chapters_at_end = getattr( - queued_item, "merge_chapters_at_end", None - ) - - # CHECK GLOBAL OVERRIDE SETTING - if not self.config.get("queue_override_settings", False): - self.selected_lang = queued_item.lang_code - self.speed_slider.setValue(int(queued_item.speed * 100)) - - # Load the specific voice string - self.selected_voice = queued_item.voice - # Clear complex GUI states so the specific voice string is used - self.mixed_voice_state = None - self.selected_profile_name = None - - self.save_option = queued_item.save_option - self.selected_output_folder = queued_item.output_folder - self.subtitle_mode = queued_item.subtitle_mode - self.selected_format = queued_item.output_format - self.replace_single_newlines = getattr( - queued_item, "replace_single_newlines", True - ) - self.use_silent_gaps = getattr(queued_item, "use_silent_gaps", False) - self.subtitle_speed_method = getattr( - queued_item, "subtitle_speed_method", "tts" - ) - - # This ensures that if conversion.py (or utils) reads from config/disk - # instead of using passed arguments, it sees the correct queue values. - self.config["replace_single_newlines"] = self.replace_single_newlines - self.config["subtitle_mode"] = self.subtitle_mode - self.config["selected_format"] = self.selected_format - self.config["use_silent_gaps"] = self.use_silent_gaps - self.config["subtitle_speed_method"] = self.subtitle_speed_method - - # Sync Voice/Profile in config - self.config["selected_voice"] = self.selected_voice - if "selected_profile_name" in self.config: - del self.config["selected_profile_name"] - - # Note: Speed is already synced via self.speed_slider.setValue() -> update_speed_label() - save_config(self.config) - - self.start_conversion(from_queue=True) - else: - # Queue finished, reset index - self.current_queue_index = 0 - - def queue_item_conversion_finished(self): - # Called after each conversion finishes - self.current_queue_index += 1 - if self.current_queue_index < len(self.queued_items): - self.start_next_queued_item() - else: - self.current_queue_index = 0 # Reset for next time - - def get_voice_formula(self) -> str: - if self.mixed_voice_state: - formula_components = [ - f"{name}*{weight}" for name, weight in self.mixed_voice_state - ] - return " + ".join(filter(None, formula_components)) - else: - return self.selected_voice - - def get_selected_lang(self, voice_formula) -> str: - if self.selected_profile_name: - from abogen.voice_profiles import load_profiles - - entry = load_profiles().get(self.selected_profile_name, {}) - selected_lang = entry.get("language") - else: - selected_lang = self.selected_voice[0] if self.selected_voice else None - # fallback: extract from formula if missing - if not selected_lang: - m = re.search(r"\b([a-z])", voice_formula) - selected_lang = m.group(1) if m else None - return selected_lang - - def get_actual_subtitle_mode(self) -> str: - return "Disabled" if not self.subtitle_combo.isEnabled() else self.subtitle_mode - - def start_conversion(self, from_queue=False): - if not self.selected_file: - self.input_box.set_error("Please add a file.") - return - - # Ensure we honor the currently selected save option when not running from queue - if not from_queue: - current_option = self.save_combo.currentText() - self.save_option = current_option - self.config["save_option"] = current_option - # If user is not choosing a specific folder, clear any residual folder - if current_option != "Choose output folder": - self.selected_output_folder = None - self.config["selected_output_folder"] = None - save_config(self.config) - - prevent_sleep_start() - self.is_converting = True - self.convert_input_box_to_log() - self.progress_bar.setValue(0) - # Show queue progress if in queue mode - if ( - from_queue - and hasattr(self, "queued_items") - and self.queued_items - and hasattr(self, "current_queue_index") - ): - N = self.current_queue_index + 1 - M = len(self.queued_items) - self.progress_bar.setFormat(f"0% ({N}/{M})") - else: - self.progress_bar.setFormat("%p%") # Reset format initially - self.etr_label.hide() # Hide ETR label initially - self.controls_widget.hide() - self.queue_row_widget.hide() # Hide queue row when process starts - self.progress_bar.show() - self.btn_cancel.show() - QApplication.processEvents() - self.btn_cancel.setEnabled(False) - self.start_time = time.time() - self.finish_widget.hide() - speed = self.speed_slider.value() / 100.0 - - # Get the display file path for logs - display_path = ( - self.displayed_file_path if self.displayed_file_path else self.selected_file - ) - - # Get file size string - try: - file_size_str = self.input_box._human_readable_size( - os.path.getsize(self.selected_file) - ) - except Exception: - file_size_str = "Unknown" - - # pipeline_loaded_callback remains unchanged - def pipeline_loaded_callback(np_module, kpipeline_class, error): - if error: - self.update_log((f"Error loading numpy or KPipeline: {error}", "red")) - prevent_sleep_end() - return - - self.btn_cancel.setEnabled(True) - - # Override subtitle_mode to "Disabled" if subtitle_combo is disabled - actual_subtitle_mode = self.get_actual_subtitle_mode() - - # if voice formula is not None, use the selected voice - voice_formula = self.get_voice_formula() - # determine selected language: use profile setting if profile selected, else voice code - selected_lang = self.get_selected_lang(voice_formula) - - self.conversion_thread = ConversionThread( - self.selected_file, - selected_lang, - speed, - voice_formula, - self.save_option, - self.selected_output_folder, - subtitle_mode=actual_subtitle_mode, - output_format=self.selected_format, - np_module=np_module, - kpipeline_class=kpipeline_class, - start_time=self.start_time, - total_char_count=self.char_count, - use_gpu=self.gpu_ok, - from_queue=from_queue, - save_base_path=self.displayed_file_path, # Pass the save base path (original file for EPUB) - ) # Use gpu_ok status - # Pass the displayed file path to the log_updated signal handler in ConversionThread - self.conversion_thread.display_path = display_path - # Pass the file size string - self.conversion_thread.file_size_str = file_size_str - # Pass max_subtitle_words from config - self.conversion_thread.max_subtitle_words = self.max_subtitle_words - # Pass silence_duration from config - self.conversion_thread.silence_duration = self.silence_duration - # Pass replace_single_newlines setting - self.conversion_thread.replace_single_newlines = ( - self.replace_single_newlines - ) - # Pass use_silent_gaps setting - self.conversion_thread.use_silent_gaps = self.use_silent_gaps - # Pass subtitle_speed_method setting - self.conversion_thread.subtitle_speed_method = self.subtitle_speed_method - # Pass use_spacy_segmentation setting - self.conversion_thread.use_spacy_segmentation = self.use_spacy_segmentation - # Pass separate_chapters_format setting - self.conversion_thread.separate_chapters_format = ( - self.separate_chapters_format - ) - # Pass subtitle format setting - self.conversion_thread.subtitle_format = self.config.get( - "subtitle_format", "ass_centered_narrow" - ) - # Pass chapter count for EPUB or PDF files - if self.selected_file_type in ["epub", "pdf", "md", "markdown"] and hasattr( - self, "selected_chapters" - ): - self.conversion_thread.chapter_count = len(self.selected_chapters) - # Pass save_chapters_separately flag if available - self.conversion_thread.save_chapters_separately = getattr( - self, "save_chapters_separately", False - ) - # Pass merge_chapters_at_end flag if available - self.conversion_thread.merge_chapters_at_end = getattr( - self, "merge_chapters_at_end", True - ) - self.conversion_thread.progress_updated.connect(self.update_progress) - self.conversion_thread.log_updated.connect(self.update_log) - self.conversion_thread.conversion_finished.connect( - self.on_conversion_finished - ) - - # Connect chapters_detected signal - self.conversion_thread.chapters_detected.connect( - self.show_chapter_options_dialog - ) - - self.conversion_thread.start() - QApplication.processEvents() - - # Run GPU acceleration and module loading in a background thread - def gpu_and_load(): - self.update_log("Checking GPU acceleration...") - # Pass the use_gpu setting from the checkbox - gpu_msg, gpu_ok = get_gpu_acceleration(self.gpu_checkbox.isChecked()) - # Store gpu_ok status to use when creating the conversion thread - self.gpu_ok = gpu_ok - self.update_log((gpu_msg, gpu_ok)) - self.update_log("Loading modules...") - load_thread = LoadPipelineThread(pipeline_loaded_callback) - load_thread.start() - - threading.Thread(target=gpu_and_load, daemon=True).start() - - def show_queue_summary(self): - """Show a summary dialog after queue finishes.""" - if not self.queued_items: - return - - # Check if override was active (this determines which settings were ACTUALLY used) - override_active = self.config.get("queue_override_settings", False) - - # If override is ON, capture the global settings that were used for processing - if override_active: - g_voice = self.get_voice_formula() - g_lang = self.get_selected_lang(g_voice) - g_speed = self.speed_slider.value() / 100.0 - g_sub_mode = self.get_actual_subtitle_mode() - g_format = self.selected_format - g_newlines = self.replace_single_newlines - g_silent_gaps = self.use_silent_gaps - g_speed_method = self.subtitle_speed_method - - # Build HTML summary (Default Styling) - summary_html = "" - - header_text = "Queue finished" - if override_active: - header_text += " (Global Settings Applied)" - - summary_html += ( - f"

    {header_text}

    " - f"Processed {len(self.queued_items)} items:

    " - ) - - for idx, item in enumerate(self.queued_items, 1): - # Resolve Effective Settings - if override_active: - eff_lang = g_lang - eff_voice = g_voice - eff_speed = g_speed - eff_sub_mode = g_sub_mode - eff_format = g_format - eff_newlines = g_newlines - eff_silent = g_silent_gaps - eff_method = g_speed_method - else: - eff_lang = item.lang_code - eff_voice = item.voice - eff_speed = item.speed - eff_sub_mode = item.subtitle_mode - eff_format = item.output_format - eff_newlines = getattr(item, "replace_single_newlines", True) - eff_silent = getattr(item, "use_silent_gaps", False) - eff_method = getattr(item, "subtitle_speed_method", "tts") - - # Retrieve File-Specific Data (Never Overridden) - eff_chars = item.total_char_count - eff_input = item.file_name - eff_output = getattr(item, "output_path", "Unknown") - eff_save_sep = getattr(item, "save_chapters_separately", None) - eff_merge = getattr(item, "merge_chapters_at_end", None) - - # --- Construct Display Block --- - summary_html += ( - f"{idx}) {os.path.basename(eff_input)}
    " - f"Language: {eff_lang}
    " - f"Voice: {eff_voice}
    " - f"Speed: {eff_speed}
    " - f"Characters: {eff_chars}
    " - f"Format: {eff_format}
    " - f"Subtitle Mode: {eff_sub_mode}
    " - f"Method: {eff_method}
    " - f"Silent Gaps: {eff_silent}
    " - f"Repl. Newlines: {eff_newlines}
    " - ) - - # Book/Chapter specific options - if eff_save_sep is not None: - summary_html += f"Split Chapters: {eff_save_sep}
    " - if eff_save_sep and eff_merge is not None: - summary_html += f"Merge End: {eff_merge}
    " - - summary_html += ( - f"Input: {eff_input}
    " - f"Output: {eff_output}

    " - ) - - summary_html += "" - - dialog = QDialog(self) - dialog.setWindowTitle("Queue Summary") - # Allow resizing - dialog.resize(550, 650) - - layout = QVBoxLayout(dialog) - text_edit = QTextEdit(dialog) - text_edit.setReadOnly(True) - text_edit.setHtml(summary_html) - layout.addWidget(text_edit) - - close_btn = QPushButton("Close", dialog) - close_btn.setFixedHeight(36) - close_btn.clicked.connect(dialog.accept) - layout.addWidget(close_btn) - - dialog.setLayout(layout) - dialog.setMinimumSize(400, 300) - dialog.setSizeGripEnabled(True) - dialog.exec() - - def on_conversion_finished(self, message, output_path): - prevent_sleep_end() - if message == "Cancelled": - self.etr_label.hide() # Hide ETR label - self.progress_bar.hide() - self.btn_cancel.hide() - self.is_converting = False - self.controls_widget.show() - self.finish_widget.hide() - self.restore_input_box() - display_path = ( - self.displayed_file_path - if self.displayed_file_path - else self.selected_file - ) - # Only repopulate if not cleared by queue - if not getattr(self, "input_box_cleared_by_queue", False): - if display_path and os.path.exists(display_path): - self.input_box.set_file_info(display_path) - else: - self.input_box.clear_input() - else: - self.input_box.clear_input() - return - - self.update_log(message) - if output_path: - self.last_output_path = output_path - # Store output_path in the current queued item if in queue mode - if self.queued_items and self.current_queue_index < len(self.queued_items): - self.queued_items[self.current_queue_index].output_path = output_path - - self.etr_label.hide() # Hide ETR label - self.progress_bar.setValue(100) - self.progress_bar.hide() - self.btn_cancel.hide() - self.is_converting = False - elapsed = int(time.time() - self.start_time) - h, m, s = elapsed // 3600, (elapsed % 3600) // 60, elapsed % 60 - self.update_log((f"\nTime elapsed: {h:02d}:{m:02d}:{s:02d}", "grey")) - - # Default to showing the button - show_open_file_button = True - # Check conditions to hide the button (only if flags exist for the completed conversion) - save_sep = getattr(self, "save_chapters_separately", False) - merge_end = getattr( - self, "merge_chapters_at_end", True - ) # Default to True if flag doesn't exist - if save_sep and not merge_end: - show_open_file_button = False - - if self.open_file_btn: - self.open_file_btn.setVisible(show_open_file_button) - - # Only show finish_widget if queue is done - if ( - self.current_queue_index + 1 >= len(self.queued_items) - or not self.queued_items - ): - # Queue finished, show finish screen - self.controls_widget.hide() - self.finish_widget.show() - sb = self.log_text.verticalScrollBar() - sb.setValue(sb.maximum()) - save_config(self.config) - # Show queue summary if more than one item - if len(self.queued_items) > 1: - self.show_queue_summary() - else: - # More items in queue: clear log and reload for next item - self.log_text.clear() - QApplication.processEvents() - - # Start new queued item, if we're using a queued conversion - self.queue_item_conversion_finished() - - def reset_ui(self): - try: - self.etr_label.hide() # Hide ETR label - self.progress_bar.setValue(0) - self.progress_bar.hide() - self.selected_file = self.selected_file_type = self.selected_book_path = ( - None - ) - self.selected_chapters = set() # Reset selected chapters - - # Ensure open file button is visible when resetting - if self.open_file_btn: - self.open_file_btn.show() - self.controls_widget.show() - self.queue_row_widget.show() # Show queue row on reset - self.finish_widget.hide() - self.btn_start.setText("Start") - # Disconnect only if connected, then reconnect - try: - self.btn_start.clicked.disconnect() - except TypeError: - pass # Ignore error if not connected - self.btn_start.clicked.connect(self.start_conversion) - self.enable_disable_queue_buttons() - self.restore_input_box() - self.input_box.clear_input() # Reset text and style - # Trigger the "Clear Queue" button (simulate user click) - self.btn_clear_queue.click() - except Exception as e: - self._show_error_message_box("Reset Error", f"Could not reset UI:\n{e}") - - def go_back_ui(self): - self.finish_widget.hide() - self.controls_widget.show() - self.queue_row_widget.show() # Show queue row on go back - self.progress_bar.hide() - self.restore_input_box() - self.log_text.clear() - - # Use displayed_file_path instead of selected_file for EPUBs or PDFs - display_path = ( - self.displayed_file_path if self.displayed_file_path else self.selected_file - ) - - # Only repopulate if not cleared by queue - if not getattr(self, "input_box_cleared_by_queue", False): - if display_path and os.path.exists(display_path): - self.input_box.set_file_info(display_path) - else: - self.input_box.clear_input() - else: - self.input_box.clear_input() - - # Ensure open file button is visible when going back - if self.open_file_btn: - self.open_file_btn.show() - - def on_save_option_changed(self, option): - self.save_option = option - self.config["save_option"] = option - if option == "Choose output folder": - try: - folder = QFileDialog.getExistingDirectory( - self, "Select Output Folder", "" - ) - if folder: - self.selected_output_folder = folder - self.save_path_label.setText(folder) - self.save_path_row_widget.show() - self.config["selected_output_folder"] = folder - else: - self.save_option = "Save next to input file" - self.save_combo.setCurrentText(self.save_option) - self.config["save_option"] = self.save_option - except Exception as e: - self._show_error_message_box( - "Folder Dialog Error", f"Could not open folder dialog:\n{e}" - ) - self.save_option = "Save next to input file" - self.save_combo.setCurrentText(self.save_option) - self.config["save_option"] = self.save_option - else: - self.save_path_row_widget.hide() - self.selected_output_folder = None - self.config["selected_output_folder"] = None - save_config(self.config) - - def go_to_file(self): - path = self.last_output_path - if not path: - return - try: - # Check if path is a directory (for multiple chapter files) - if os.path.isdir(path): - folder = path - else: - folder = os.path.dirname(path) - QDesktopServices.openUrl(QUrl.fromLocalFile(folder)) - except Exception as e: - self._show_error_message_box( - "Open Folder Error", f"Could not open folder:\n{e}" - ) - - def open_file(self): - path = self.last_output_path - if not path: - return - try: - # Check if path exists and is a file before opening - if os.path.exists(path): - if os.path.isdir(path): - self._show_error_message_box( - "Open File Error", - "Cannot open a directory as a file. Please use 'Go to folder' instead.", - ) - return - QDesktopServices.openUrl(QUrl.fromLocalFile(path)) - else: - self._show_error_message_box( - "Open File Error", f"File not found: {path}" - ) - except Exception as e: - self._show_error_message_box( - "Open File Error", f"Could not open file:\n{e}" - ) - - def _get_preview_cache_path(self): - """Generate the expected cache path for the current voice settings.""" - speed = self.speed_slider.value() / 100.0 - voice_to_cache = "" - lang_to_cache = "" - - if self.mixed_voice_state: - components = [f"{name}*{weight}" for name, weight in self.mixed_voice_state] - voice_formula = " + ".join(filter(None, components)) - voice_to_cache = voice_formula - if self.selected_profile_name: - from abogen.voice_profiles import load_profiles - - entry = load_profiles().get(self.selected_profile_name, {}) - lang_to_cache = entry.get("language") - else: - lang_to_cache = self.selected_lang - if not lang_to_cache and self.mixed_voice_state: - lang_to_cache = ( - self.mixed_voice_state[0][0][0] - if self.mixed_voice_state and self.mixed_voice_state[0][0] - else None - ) - elif self.selected_voice: - lang_to_cache = self.selected_voice[0] - voice_to_cache = self.selected_voice - else: # No voice or profile selected - return None - - if not lang_to_cache or not voice_to_cache: # Not enough info - return None - - cache_dir = get_user_cache_path("preview_cache") - - if "*" in voice_to_cache: # Voice formula - voice_id = ( - f"voice_formula_{hashlib.md5(voice_to_cache.encode()).hexdigest()[:8]}" - ) - else: # Single voice - voice_id = voice_to_cache - - filename = f"{voice_id}_{lang_to_cache}_{speed:.2f}.wav" - return os.path.join(cache_dir, filename) - - def preview_voice(self): - if self.preview_playing: - try: - if self.play_audio_thread and self.play_audio_thread.isRunning(): - # Call the stop method on PlayAudioThread to safely handle stopping - self.play_audio_thread.stop() - self.play_audio_thread.wait(500) # Wait a bit - except Exception as e: - print(f"Error stopping preview audio: {e}") - self._preview_cleanup() - return - - if hasattr(self, "preview_thread") and self.preview_thread.isRunning(): - return - - # Check for cache first - cached_path = self._get_preview_cache_path() - if cached_path and os.path.exists(cached_path): - print(f"Cache hit for {cached_path}") - self.btn_preview.setEnabled(False) # Disable button briefly - self.voice_combo.setEnabled(False) - self.btn_voice_formula_mixer.setEnabled(False) - self.btn_start.setEnabled(False) - - # Directly play from cache - self.preview_playing = True - self.btn_preview.setIcon(self.stop_icon) - self.btn_preview.setToolTip("Stop preview") - self.btn_preview.setEnabled(True) - - def cleanup_cached_play(): - self._preview_cleanup() - - try: - # Ensure pygame mixer is initialized for the audio thread - import pygame - - if not pygame.mixer.get_init(): - pygame.mixer.init() - - self.play_audio_thread = PlayAudioThread(cached_path) - self.play_audio_thread.finished.connect(cleanup_cached_play) - self.play_audio_thread.error.connect( - lambda msg: ( - self._show_preview_error_box(msg), - cleanup_cached_play(), - ) - ) - self.play_audio_thread.start() - except Exception as e: - self._show_error_message_box( - "Preview Error", f"Could not play cached preview audio:\n{e}" - ) - cleanup_cached_play() - return - - # If no cache hit, proceed to load pipeline and generate - self.btn_preview.setEnabled(False) - self.btn_preview.setToolTip("Loading...") - self.voice_combo.setEnabled(False) - self.btn_voice_formula_mixer.setEnabled(False) # Disable mixer button - self.btn_start.setEnabled(False) # Disable start button during preview - - # Start loading animation - ensure signal connection is always active - if hasattr(self, "loading_movie"): - # Disconnect previous connections to avoid multiple connections - try: - self.loading_movie.frameChanged.disconnect() - except TypeError: - pass # Ignore error if not connected - - # Reconnect the signal - self.loading_movie.frameChanged.connect( - lambda: self.btn_preview.setIcon( - QIcon(self.loading_movie.currentPixmap()) - ) - ) - self.loading_movie.start() - - def pipeline_loaded_callback(np_module, kpipeline_class, error): - self._on_pipeline_loaded_for_preview(np_module, kpipeline_class, error) - - load_thread = LoadPipelineThread(pipeline_loaded_callback) - load_thread.start() - - def _on_pipeline_loaded_for_preview(self, np_module, kpipeline_class, error): - # stop loading animation and restore icon on error - if error: - self.loading_movie.stop() - self._show_error_message_box( - "Loading Error", f"Error loading numpy or KPipeline: {error}" - ) - self.btn_preview.setIcon(self.play_icon) - self.btn_preview.setEnabled(True) - self.btn_preview.setToolTip("Preview selected voice") - self.voice_combo.setEnabled(True) - self.btn_voice_formula_mixer.setEnabled(True) # Re-enable mixer button - self.btn_start.setEnabled(True) # Re-enable start button on error - return - - # Support preview for voice profiles - speed = self.speed_slider.value() / 100.0 - if self.mixed_voice_state: - # Build voice formula string - components = [f"{name}*{weight}" for name, weight in self.mixed_voice_state] - voice = " + ".join(filter(None, components)) - # determine language: use profile setting, else explicit mixer selection, else fallback to first voice code - if self.selected_profile_name: - from abogen.voice_profiles import load_profiles - - entry = load_profiles().get(self.selected_profile_name, {}) - lang = entry.get("language") - else: - lang = self.selected_lang - if not lang and self.mixed_voice_state: - lang = ( - self.mixed_voice_state[0][0][0] - if self.mixed_voice_state and self.mixed_voice_state[0][0] - else None - ) - else: - lang = self.selected_voice[0] - voice = self.selected_voice - - # use same gpu/cpu logic as in conversion - gpu_msg, gpu_ok = get_gpu_acceleration(self.use_gpu) - - self.preview_thread = VoicePreviewThread( - np_module, kpipeline_class, lang, voice, speed, gpu_ok - ) - self.preview_thread.finished.connect(self._play_preview_audio) - self.preview_thread.error.connect(self._preview_error) - self.preview_thread.start() - - def _play_preview_audio(self, from_cache=True): # from_cache default is now False - # If preview_thread is the source, get temp_wav from it - if hasattr(self, "preview_thread") and not from_cache: - temp_wav = self.preview_thread.temp_wav - elif from_cache: # This case is now handled before calling _play_preview_audio - cached_path = self._get_preview_cache_path() - if cached_path and os.path.exists(cached_path): - temp_wav = cached_path - else: # Should not happen if cache check was done - self._show_error_message_box( - "Preview Error", - "Cache file expected but not found, please try again.", - ) - self._preview_cleanup() - return - else: # Should have temp_wav from preview_thread or handled by cache check - self._show_error_message_box( - "Preview Error", "Preview audio path not found." - ) - self._preview_cleanup() - return - - if not temp_wav: - if hasattr(self, "loading_movie"): - self.loading_movie.stop() - self._show_error_message_box( - "Preview Error", "Preview error: No audio generated." - ) - self._preview_cleanup() - return - - # stop loading animation, switch to stop icon - if hasattr(self, "loading_movie"): - self.loading_movie.stop() - self.preview_playing = True - self.btn_preview.setIcon(self.stop_icon) - self.btn_preview.setToolTip("Stop preview") - self.btn_preview.setEnabled(True) - - def cleanup(): - # Only remove if not from cache AND it's a temp file from VoicePreviewThread - if ( - not from_cache - and hasattr(self, "preview_thread") - and hasattr(self.preview_thread, "temp_wav") - and self.preview_thread.temp_wav == temp_wav - ): - try: - if os.path.exists( - temp_wav - ): # Ensure it exists before trying to remove - os.remove(temp_wav) - except Exception: - pass - self._preview_cleanup() - - try: - # Ensure pygame mixer is initialized for the audio thread - import pygame - - if not pygame.mixer.get_init(): - pygame.mixer.init() - - self.play_audio_thread = PlayAudioThread(temp_wav) - self.play_audio_thread.finished.connect(cleanup) - self.play_audio_thread.error.connect( - lambda msg: (self._show_preview_error_box(msg), cleanup()) - ) - self.play_audio_thread.start() - except Exception as e: - self._show_error_message_box( - "Preview Error", f"Could not play preview audio:\n{e}" - ) - cleanup() - - def _show_error_message_box(self, title, message): - box = QMessageBox(self) - box.setIcon(QMessageBox.Icon.Critical) - box.setWindowTitle(title) - box.setText(message) - copy_btn = QPushButton("Copy") - box.addButton(copy_btn, QMessageBox.ButtonRole.ActionRole) - box.addButton(QMessageBox.StandardButton.Ok) - copy_btn.clicked.connect(lambda: QApplication.clipboard().setText(message)) - box.exec() - - def _show_preview_error_box(self, msg): - self._show_error_message_box("Preview Error", f"Preview error: {msg}") - - def _preview_cleanup(self): - self.preview_playing = False - if hasattr(self, "loading_movie"): - self.loading_movie.stop() - try: - if hasattr(self, "loading_movie"): - self.loading_movie.frameChanged.disconnect() - except Exception: - pass # Ignore error if not connected - self.btn_preview.setIcon(self.play_icon) - self.btn_preview.setToolTip("Preview selected voice") - self.btn_preview.setEnabled(True) - self.voice_combo.setEnabled(True) - self.btn_voice_formula_mixer.setEnabled(True) # Re-enable mixer button - self.btn_start.setEnabled(True) - - def _preview_error(self, msg): - self._show_error_message_box("Preview Error", f"Preview error: {msg}") - self._preview_cleanup() - - def cancel_conversion(self): - if self.is_converting: - box = QMessageBox(self) - box.setIcon(QMessageBox.Icon.Warning) - box.setWindowTitle("Cancel Conversion") - box.setText( - "A conversion is currently running. Are you sure you want to cancel?" - ) - box.setStandardButtons( - QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No - ) - box.setDefaultButton(QMessageBox.StandardButton.No) - if box.exec() != QMessageBox.StandardButton.Yes: - return - try: - if ( - hasattr(self, "conversion_thread") - and self.conversion_thread.isRunning() - ): - if not hasattr(self, "_conversion_lock"): - self._conversion_lock = threading.Lock() - - def _cancel(): - with self._conversion_lock: - self.conversion_thread.cancel() # <-- Use cancel() method - self.conversion_thread.wait() - - threading.Thread(target=_cancel, daemon=True).start() - - self.is_converting = False - self.etr_label.hide() # Hide ETR label - self.progress_bar.hide() - self.btn_cancel.hide() - self.controls_widget.show() - self.queue_row_widget.show() # Show queue row on cancel - self.finish_widget.hide() - self.restore_input_box() - self.log_text.clear() - display_path = ( - self.displayed_file_path - if self.displayed_file_path - else self.selected_file - ) - # Only repopulate if not cleared by queue - if not getattr(self, "input_box_cleared_by_queue", False): - if display_path and os.path.exists(display_path): - self.input_box.set_file_info(display_path) - else: - self.input_box.clear_input() - else: - self.input_box.clear_input() - prevent_sleep_end() - except Exception as e: - self._show_error_message_box( - "Cancel Error", f"Could not cancel conversion:\n{e}" - ) - - def on_subtitle_mode_changed(self, mode): - self.subtitle_mode = mode - self.config["subtitle_mode"] = mode - save_config(self.config) - # Disable subtitle format combo if subtitles are disabled - if mode == "Disabled": - self.subtitle_format_combo.setEnabled(False) - else: - self.subtitle_format_combo.setEnabled(True) - # If highlighting mode selected, SRT is not supported. Disable SRT option and - # switch away from it if currently selected. - try: - idx_srt = self.subtitle_format_combo.findData("srt") - if mode == "Sentence + Highlighting": - if idx_srt >= 0: - item = self.subtitle_format_combo.model().item(idx_srt) - if item is not None: - item.setEnabled(False) - # If current format is SRT, switch to a compatible ASS format - if self.subtitle_format_combo.currentData() == "srt": - new_idx = self.subtitle_format_combo.findData("ass_centered_narrow") - if new_idx >= 0: - self.subtitle_format_combo.setCurrentIndex(new_idx) - self.set_subtitle_format( - self.subtitle_format_combo.itemData(new_idx) - ) - else: - # Re-enable SRT option when not in highlighting mode - if idx_srt >= 0: - item = self.subtitle_format_combo.model().item(idx_srt) - if item is not None: - item.setEnabled(True) - except Exception: - # Ignore errors interacting with model (defensive) - pass - - def on_format_changed(self, fmt): - self.selected_format = fmt - self.config["selected_format"] = fmt - save_config(self.config) - - def on_gpu_setting_changed(self, state): - self.use_gpu = state == Qt.CheckState.Checked.value - self.config["use_gpu"] = self.use_gpu - save_config(self.config) - - def cleanup_conversion_thread(self): - # Stop conversion thread - if ( - hasattr(self, "conversion_thread") - and self.conversion_thread is not None - and self.conversion_thread.isRunning() - ): - self.conversion_thread.cancel() - self.conversion_thread.wait() - - def cleanup_preview_threads(self): - # Stop preview generation thread - if ( - hasattr(self, "preview_thread") - and self.preview_thread is not None - and self.preview_thread.isRunning() - ): - self.preview_thread.terminate() - self.preview_thread.wait() - - # Stop audio playback thread - if ( - hasattr(self, "play_audio_thread") - and self.play_audio_thread is not None - and self.play_audio_thread.isRunning() - ): - self.play_audio_thread.stop() - self.play_audio_thread.wait() - - # Cleanup pygame mixer if initialized - try: - pygame = sys.modules.get("pygame") - if pygame and pygame.mixer.get_init(): - pygame.mixer.quit() - except Exception: - pass - - def closeEvent(self, event): - if self.is_converting: - box = QMessageBox(self) - box.setIcon(QMessageBox.Icon.Warning) - box.setWindowTitle("Conversion in Progress") - box.setText( - "A conversion is currently running. Are you sure you want to exit?" - ) - box.setStandardButtons( - QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No - ) - box.setDefaultButton(QMessageBox.StandardButton.No) - if box.exec() == QMessageBox.StandardButton.Yes: - self.cleanup_conversion_thread() - self.cleanup_preview_threads() - event.accept() - else: - event.ignore() - else: - self.cleanup_conversion_thread() - self.cleanup_preview_threads() - event.accept() - - def show_chapter_options_dialog(self, chapter_count): - """Show dialog to ask user about chapter processing options when chapters are detected in a .txt file""" - # Check if this is a timestamp detection (-1) or chapter detection - if chapter_count == -1: - from abogen.conversion import TimestampDetectionDialog - - dialog = TimestampDetectionDialog(parent=self) - dialog.setWindowModality(Qt.WindowModality.ApplicationModal) - - # Dialog always accepts (Yes or No), never cancels the conversion - dialog.exec() - treat_as_subtitle = dialog.use_timestamps() - if ( - hasattr(self, "conversion_thread") - and self.conversion_thread.isRunning() - ): - self.conversion_thread.set_timestamp_response(treat_as_subtitle) - return - - # Normal chapter detection - from abogen.conversion import ChapterOptionsDialog - - dialog = ChapterOptionsDialog(chapter_count, parent=self) - dialog.setWindowModality(Qt.WindowModality.ApplicationModal) - - if dialog.exec() == QDialog.DialogCode.Accepted: - options = dialog.get_options() - if ( - hasattr(self, "conversion_thread") - and self.conversion_thread.isRunning() - ): - self.conversion_thread.set_chapter_options(options) - else: - self.cancel_conversion() - - def apply_theme(self, theme): - - app = QApplication.instance() - is_windows = platform.system() == "Windows" - available_styles = [s.lower() for s in QStyleFactory.keys()] - - def is_windows_dark_mode(): - try: - import winreg - - with winreg.OpenKey( - winreg.HKEY_CURRENT_USER, - r"Software\Microsoft\Windows\CurrentVersion\Themes\Personalize", - ) as key: - value, _ = winreg.QueryValueEx(key, "AppsUseLightTheme") - return value == 0 - except Exception: - return False - - # --- Theme selection logic --- - def set_dark_palette(): - palette = QPalette() - dark_bg = QColor(COLORS["DARK_BG"]) - base_bg = QColor(COLORS["DARK_BASE"]) - alt_bg = QColor(COLORS["DARK_ALT"]) - button_bg = QColor(COLORS["DARK_BUTTON"]) - disabled_fg = QColor(COLORS["DARK_DISABLED"]) - palette.setColor(QPalette.ColorRole.Window, dark_bg) - palette.setColor(QPalette.ColorRole.WindowText, Qt.GlobalColor.white) - palette.setColor(QPalette.ColorRole.Base, base_bg) - palette.setColor(QPalette.ColorRole.AlternateBase, alt_bg) - palette.setColor(QPalette.ColorRole.ToolTipBase, dark_bg) - palette.setColor(QPalette.ColorRole.ToolTipText, Qt.GlobalColor.white) - palette.setColor(QPalette.ColorRole.Text, Qt.GlobalColor.white) - palette.setColor(QPalette.ColorRole.Button, button_bg) - palette.setColor(QPalette.ColorRole.ButtonText, Qt.GlobalColor.white) - # Disabled roles - palette.setColor( - QPalette.ColorGroup.Disabled, QPalette.ColorRole.WindowText, disabled_fg - ) - palette.setColor( - QPalette.ColorGroup.Disabled, QPalette.ColorRole.Text, disabled_fg - ) - palette.setColor( - QPalette.ColorGroup.Disabled, QPalette.ColorRole.ButtonText, disabled_fg - ) - palette.setColor( - QPalette.ColorGroup.Disabled, QPalette.ColorRole.Base, dark_bg - ) - palette.setColor( - QPalette.ColorGroup.Disabled, QPalette.ColorRole.Button, dark_bg - ) - app.setPalette(palette) - - def set_light_palette(): - palette = QPalette() - disabled_fg = QColor(COLORS["LIGHT_DISABLED"]) - palette.setColor(QPalette.ColorRole.Window, QColor(COLORS["LIGHT_BG"])) - palette.setColor(QPalette.ColorRole.WindowText, Qt.GlobalColor.black) - palette.setColor(QPalette.ColorRole.Base, Qt.GlobalColor.white) - palette.setColor(QPalette.ColorRole.AlternateBase, Qt.GlobalColor.white) - palette.setColor(QPalette.ColorRole.ToolTipBase, Qt.GlobalColor.white) - palette.setColor(QPalette.ColorRole.ToolTipText, Qt.GlobalColor.black) - palette.setColor(QPalette.ColorRole.Text, Qt.GlobalColor.black) - palette.setColor(QPalette.ColorRole.Button, Qt.GlobalColor.white) - palette.setColor(QPalette.ColorRole.ButtonText, Qt.GlobalColor.black) - # Disabled roles - palette.setColor( - QPalette.ColorGroup.Disabled, QPalette.ColorRole.WindowText, disabled_fg - ) - palette.setColor( - QPalette.ColorGroup.Disabled, QPalette.ColorRole.Text, disabled_fg - ) - palette.setColor( - QPalette.ColorGroup.Disabled, QPalette.ColorRole.ButtonText, disabled_fg - ) - palette.setColor( - QPalette.ColorGroup.Disabled, - QPalette.ColorRole.Base, - Qt.GlobalColor.white, - ) - palette.setColor( - QPalette.ColorGroup.Disabled, - QPalette.ColorRole.Button, - Qt.GlobalColor.white, - ) - app.setPalette(palette) - - # --- Dark title bar support for Windows --- - def set_title_bar_dark_mode(window, enable): - if is_windows: - try: - window.update() - DWMWA_USE_IMMERSIVE_DARK_MODE = 20 - set_window_attribute = ctypes.windll.dwmapi.DwmSetWindowAttribute - hwnd = int(window.winId()) - value = ctypes.c_int(2 if enable else 0) - set_window_attribute( - hwnd, - DWMWA_USE_IMMERSIVE_DARK_MODE, - ctypes.byref(value), - ctypes.sizeof(value), - ) - except Exception: - pass - - # Main logic - dark_mode = theme == "dark" or ( - theme == "system" and is_windows and is_windows_dark_mode() - ) - if dark_mode: - app.setStyle("Fusion") - set_dark_palette() - elif (theme == "light" or theme == "system") and is_windows: - if "windowsvista" in available_styles: - app.setStyle("windowsvista") - else: - app.setStyle("Fusion") - app.setPalette(QPalette()) - elif theme == "light": - app.setStyle("Fusion") - set_light_palette() - else: - app.setStyle("Fusion") - app.setPalette(QPalette()) - - # Always set the title bar mode according to the current theme for all top-level widgets - for widget in app.topLevelWidgets(): - set_title_bar_dark_mode(widget, dark_mode) - - # Refresh all top-level widgets - style_name = app.style().objectName() - app.setStyle(style_name) - for widget in app.topLevelWidgets(): - app.style().polish(widget) - widget.update() - - # Remove old event filter if present, then install a new one for dark title bar on new windows - if hasattr(app, "_dark_titlebar_event_filter"): - app.removeEventFilter(app._dark_titlebar_event_filter) - delattr(app, "_dark_titlebar_event_filter") - - def get_dark_mode(): - return theme == "dark" or ( - theme == "system" and is_windows and is_windows_dark_mode() - ) - - app._dark_titlebar_event_filter = DarkTitleBarEventFilter( - is_windows, get_dark_mode, set_title_bar_dark_mode - ) - app.installEventFilter(app._dark_titlebar_event_filter) - - # Save config if changed - if self.config.get("theme", "system") != theme: - self.config["theme"] = theme - save_config(self.config) - - def show_settings_menu(self): - """Show a dropdown menu for settings options.""" - menu = QMenu(self) - - theme_menu = QMenu("Theme", self) - theme_menu.setToolTip("Choose the application theme") - - theme_group = QActionGroup(self) - theme_group.setExclusive(True) - - # Theme options: (internal_value, display_text) - theme_options = [ - ("system", "System"), - ("light", "Light"), - ("dark", "Dark"), - ] - - # Get current theme from config, default to "system" - current_theme = self.config.get("theme", "system") - for value, text in theme_options: - theme_action = QAction(text, self) - theme_action.setCheckable(True) - theme_action.setChecked(current_theme == value) - theme_action.triggered.connect(lambda checked, v=value: self.apply_theme(v)) - theme_group.addAction(theme_action) - theme_menu.addAction(theme_action) - - menu.addMenu(theme_menu) - - # Add separate chapters format option - separate_chapters_format_menu = QMenu("Separate chapters audio format", self) - separate_chapters_format_menu.setToolTip( - "Choose the format for individual chapter files" - ) - - format_group = QActionGroup(self) - format_group.setExclusive(True) - - for format_option in ["wav", "flac", "mp3", "opus"]: - format_action = QAction(format_option, self) - format_action.setCheckable(True) - format_action.setChecked(self.separate_chapters_format == format_option) - format_action.triggered.connect( - lambda checked, fmt=format_option: self.set_separate_chapters_format( - fmt - ) - ) - format_group.addAction(format_action) - separate_chapters_format_menu.addAction(format_action) - - menu.addMenu(separate_chapters_format_menu) - - # Add max words per subtitle option - max_words_action = QAction("Configure max words per subtitle", self) - max_words_action.triggered.connect(self.set_max_subtitle_words) - menu.addAction(max_words_action) - - # Add silence between chapters option - silence_action = QAction("Configure silence between chapters", self) - silence_action.triggered.connect(self.set_silence_between_chapters) - menu.addAction(silence_action) - - max_lines_action = QAction("Configure max lines in log window", self) - max_lines_action.triggered.connect(self.set_max_log_lines) - menu.addAction(max_lines_action) - - # Add separator - menu.addSeparator() - - # Add shortcut to desktop (Windows or Linux) - if platform.system() == "Windows" or platform.system() == "Linux": - # Use extended label on Linux - label = ( - "Create desktop shortcut and install" - if platform.system() == "Linux" - else "Create desktop shortcut" - ) - add_shortcut_action = QAction(label, self) - add_shortcut_action.triggered.connect(self.add_shortcut_to_desktop) - menu.addAction(add_shortcut_action) - - # Add reveal config option - reveal_config_action = QAction("Open configuration directory", self) - reveal_config_action.triggered.connect(self.reveal_config_in_explorer) - menu.addAction(reveal_config_action) - - # Add open cache directory option - open_cache_action = QAction("Open cache directory", self) - open_cache_action.triggered.connect(self.open_cache_directory) - menu.addAction(open_cache_action) - - # Add clear cache files option - clear_cache_action = QAction("Clear cache files", self) - clear_cache_action.triggered.connect(self.clear_cache_files) - menu.addAction(clear_cache_action) - - # Add separator - menu.addSeparator() - - # Add use silent gaps option (for subtitle files) - self.silent_gaps_action = QAction("Use silent gaps between subtitles", self) - self.silent_gaps_action.setCheckable(True) - self.silent_gaps_action.setChecked(self.use_silent_gaps) - self.silent_gaps_action.triggered.connect( - lambda checked: self.toggle_use_silent_gaps(checked) - ) - menu.addAction(self.silent_gaps_action) - - # Subtitle speed adjustment method - speed_method_menu = menu.addMenu("Subtitle speed adjustment method") - speed_method_menu.setToolTip( - "Choose speed adjustment method:\n" - "TTS Regeneration: Better quality\n" - "FFmpeg Time-stretch: Faster processing" - ) - - speed_method_group = QActionGroup(self) - speed_method_group.setExclusive(True) - - for method, label in [ - ("tts", "TTS Regeneration (better quality)"), - ("ffmpeg", "FFmpeg Time-stretch (better speed)"), - ]: - action = QAction(label, speed_method_menu) - action.setCheckable(True) - action.setChecked(self.subtitle_speed_method == method) - action.triggered.connect( - lambda checked, m=method: self.toggle_subtitle_speed_method(m) - ) - speed_method_group.addAction(action) - speed_method_menu.addAction(action) - - self.speed_method_group = speed_method_group - - # Add separator - menu.addSeparator() - - # Add spaCy sentence segmentation option - spacy_action = QAction("Use spaCy for sentence segmentation", self) - spacy_action.setCheckable(True) - spacy_action.setChecked(self.use_spacy_segmentation) - spacy_action.triggered.connect( - lambda checked: self.toggle_spacy_segmentation(checked) - ) - menu.addAction(spacy_action) - - # Add separator - menu.addSeparator() - - # Add "Pre-download models and voices for offline use" option - predownload_action = QAction( - "Pre-download models and voices for offline use", self - ) - predownload_action.triggered.connect(self.show_predownload_dialog) - menu.addAction(predownload_action) - - # Add "Disable Kokoro's internet access" option - disable_kokoro_action = QAction("Disable Kokoro's internet access", self) - disable_kokoro_action.setCheckable(True) - disable_kokoro_action.setChecked( - self.config.get("disable_kokoro_internet", False) - ) - disable_kokoro_action.triggered.connect( - lambda checked: self.toggle_kokoro_internet_access(checked) - ) - menu.addAction(disable_kokoro_action) - - # Add check for updates option - check_updates_action = QAction("Check for updates at startup", self) - check_updates_action.setCheckable(True) - check_updates_action.setChecked(self.config.get("check_updates", True)) - check_updates_action.triggered.connect(self.toggle_check_updates) - menu.addAction(check_updates_action) - - # Add "Reset to default settings" option - reset_defaults_action = QAction("Reset to default settings", self) - reset_defaults_action.triggered.connect(self.reset_to_default_settings) - menu.addAction(reset_defaults_action) - - # Add about action - about_action = QAction("About", self) - about_action.triggered.connect(self.show_about_dialog) - menu.addAction(about_action) - - menu.exec(self.settings_btn.mapToGlobal(QPoint(0, self.settings_btn.height()))) - - def toggle_replace_single_newlines(self, enabled): - self.replace_single_newlines = enabled - self.config["replace_single_newlines"] = enabled - save_config(self.config) - - def toggle_use_silent_gaps(self, enabled): - # Show confirmation dialog with explanation - action = "enable" if enabled else "disable" - message = ( - "When enabled, allows speech to continue naturally into the silent periods between subtitles, " - "preventing unnecessary audio speed-up based on subtitle end timestamps.\n\nWhen disabled, ensures strict subtitle timing where " - f"audio ends exactly when the subtitle ends.\n\nDo you want to {action} this option?" - ) - - reply = QMessageBox.question( - self, - "Use Silent Gaps Between Subtitles", - message, - QMessageBox.StandardButton.Ok | QMessageBox.StandardButton.Cancel, - ) - - if reply == QMessageBox.StandardButton.Ok: - self.use_silent_gaps = enabled - self.config["use_silent_gaps"] = enabled - save_config(self.config) - else: - # Revert the checkbox state if cancelled - self.silent_gaps_action.setChecked(not enabled) - - def toggle_subtitle_speed_method(self, method): - self.subtitle_speed_method = method - self.config["subtitle_speed_method"] = method - save_config(self.config) - - def toggle_spacy_segmentation(self, enabled): - self.use_spacy_segmentation = enabled - self.config["use_spacy_segmentation"] = enabled - save_config(self.config) - - def restart_app(self): - - import sys - - exe = sys.executable - args = sys.argv - - # On Windows, use .exe if available - if platform.system() == "Windows": - script_path = args[0] - if not script_path.lower().endswith(".exe"): - exe_path = os.path.splitext(script_path)[0] + ".exe" - if os.path.exists(exe_path): - args[0] = exe_path - - QProcess.startDetached(exe, args) - QApplication.quit() - - def toggle_kokoro_internet_access(self, disabled): - if disabled: - message = ( - "Disabling Kokoro's internet access will block downloads of models and voices from Hugging Face Hub. " - "This can make processing faster when there is no internet connection, since no requests will be made. " - "The app needs to restart to apply this change.\n\nDo you want to continue?" - ) - else: - message = ( - "Enabling Kokoro's internet access will allow it to download models and voices from Hugging Face Hub. " - "The app needs to restart to apply this change.\n\nDo you want to continue?" - ) - reply = QMessageBox.question( - self, - "Restart Required", - message, - QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, - QMessageBox.StandardButton.No, - ) - if reply == QMessageBox.StandardButton.Yes: - self.config["disable_kokoro_internet"] = disabled - save_config(self.config) - try: - self.restart_app() - except Exception as e: - QMessageBox.critical( - self, "Restart Failed", f"Failed to restart the application:\n{e}" - ) - - def reset_to_default_settings(self): - reply = QMessageBox.question( - self, - "Reset Settings", - "This will reset all settings to their default values and restart the application.\n\nDo you want to continue?", - QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, - QMessageBox.StandardButton.No, - ) - if reply == QMessageBox.StandardButton.Yes: - from abogen.utils import get_user_config_path - - config_path = get_user_config_path() - try: - if os.path.exists(config_path): - os.remove(config_path) - self.restart_app() - except Exception as e: - QMessageBox.critical( - self, "Reset Error", f"Could not reset settings:\n{e}" - ) - - def reveal_config_in_explorer(self): - """Open the configuration file location in file explorer.""" - from abogen.utils import get_user_config_path - - try: - config_path = get_user_config_path() - # Open the directory containing the config file - QDesktopServices.openUrl(QUrl.fromLocalFile(os.path.dirname(config_path))) - except Exception as e: - QMessageBox.critical( - self, "Config Error", f"Could not open config location:\n{e}" - ) - - def open_cache_directory(self): - """Open the cache directory used by the program.""" - try: - # Get the abogen cache directory - cache_dir = get_user_cache_path() - - # Create the directory if it doesn't exist - if not os.path.exists(cache_dir): - os.makedirs(cache_dir) - - # Open the directory in file explorer - QDesktopServices.openUrl(QUrl.fromLocalFile(cache_dir)) - except Exception as e: - QMessageBox.critical( - self, "Cache Directory Error", f"Could not open cache directory:\n{e}" - ) - - def add_shortcut_to_desktop(self): - """Create a desktop shortcut to this program using PowerShell.""" - import sys - from platformdirs import user_desktop_dir - from abogen.utils import create_process - - try: - if platform.system() == "Windows": - # where to put the .lnk - desktop = user_desktop_dir() - shortcut_path = os.path.join(desktop, "abogen.lnk") - - # target exe - python_dir = os.path.dirname(sys.executable) - target = os.path.join(python_dir, "Scripts", "abogen.exe") - if not os.path.exists(target): - QMessageBox.critical( - self, - "Shortcut Error", - f"Could not find abogen.exe at:\n{target}", - ) - return - - # icon (fallback to exe if missing) - icon = get_resource_path("abogen.assets", "icon.ico") - if not icon or not os.path.exists(icon): - icon = target # Create a more direct PowerShell command - shortcut_ps = shortcut_path.replace("'", "''").replace("\\", "\\\\") - target_ps = target.replace("'", "''").replace("\\", "\\\\") - workdir_ps = ( - os.path.dirname(target).replace("'", "''").replace("\\", "\\\\") - ) - icon_ps = icon.replace("'", "''").replace("\\", "\\\\") - # Create PowerShell script as a single line with no line breaks (more reliable) - ps_cmd = f"$s=New-Object -ComObject WScript.Shell; $lnk=$s.CreateShortcut('{shortcut_ps}'); $lnk.TargetPath='{target_ps}'; $lnk.WorkingDirectory='{workdir_ps}'; $lnk.IconLocation='{icon_ps}'; $lnk.Save()" - - # Run PowerShell with the command directly - proc = create_process( - 'powershell -NoProfile -ExecutionPolicy Bypass -Command "' - + ps_cmd - + '"' - ) - proc.wait() - - if proc.returncode == 0: - QMessageBox.information( - self, - "Shortcut Created", - f"Shortcut created on desktop:\n{shortcut_path}", - ) - else: - QMessageBox.critical( - self, - "Shortcut Error", - f"PowerShell failed with exit code: {proc.returncode}", - ) - elif platform.system() == "Linux": - desktop = user_desktop_dir() - if not desktop or not os.path.isdir(desktop): - QMessageBox.critical( - self, "Shortcut Error", "Could not determine desktop directory." - ) - return - - shortcut_path = os.path.join(desktop, "abogen.desktop") - - import shutil - - found = shutil.which("abogen") - if found: - target = found - else: - local_bin = os.path.expanduser("~/.local/bin/abogen") - if os.path.exists(local_bin): - target = local_bin - else: - python_dir = os.path.dirname(sys.executable) - target = os.path.join(python_dir, "bin", "abogen") - if not os.path.exists(target): - target_fallback = os.path.join(python_dir, "abogen") - if os.path.exists(target_fallback): - target = target_fallback - else: - QMessageBox.critical( - self, - "Shortcut Error", - "Could not find abogen executable in PATH or common installation directories.", - ) - return - - icon_path = get_resource_path("abogen.assets", "icon.png") - - desktop_entry_content = f"""[Desktop Entry] -Version={VERSION} -Name={PROGRAM_NAME} -Comment={PROGRAM_DESCRIPTION} -Exec={target} -Icon={icon_path} -Terminal=false -Type=Application -Categories=AudioVideo;Audio;Utility; +The actual implementation lives in abogen.pyqt.gui. """ - with open(shortcut_path, "w", encoding="utf-8") as f: - f.write(desktop_entry_content) - os.chmod(shortcut_path, 0o755) +from __future__ import annotations - QMessageBox.information( - self, - "Shortcut Created", - f"Shortcut created on desktop:\n{shortcut_path}", - ) +from abogen.pyqt.gui import * # noqa: F401, F403 +from abogen.pyqt.gui import abogen - # Offer installation for current user under ~/.local/share/applications - reply = QMessageBox.question( - self, - "Install Application Entry", - "Install application entry for current user?", - QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, - ) - if reply == QMessageBox.StandardButton.Yes: - import shutil - - user_app_dir = os.path.expanduser("~/.local/share/applications") - os.makedirs(user_app_dir, exist_ok=True) - user_entry = os.path.join(user_app_dir, "abogen.desktop") - try: - shutil.copyfile(shortcut_path, user_entry) - os.chmod(user_entry, 0o644) - QMessageBox.information( - self, - "Installation Complete", - f"Desktop entry installed to {user_entry}", - ) - except Exception as e: - QMessageBox.warning( - self, - "Install Error", - f"Could not install entry:\n{e}", - ) - else: - QMessageBox.information( - self, - "Unsupported OS", - "Desktop shortcut creation is not supported on this operating system.", - ) - - except Exception as e: - QMessageBox.critical( - self, "Shortcut Error", f"Could not create shortcut:\n{e}" - ) - - def toggle_check_updates(self, checked): - self.config["check_updates"] = checked - save_config(self.config) - - def show_voice_formula_dialog(self): - from abogen.voice_profiles import load_profiles - - profiles = load_profiles() - initial_state = None - selected_profile = self.selected_profile_name - if selected_profile: - entry = profiles.get(selected_profile, {}) - if isinstance(entry, dict): - initial_state = entry.get("voices", []) - else: - initial_state = entry - elif self.mixed_voice_state is not None: - initial_state = self.mixed_voice_state - elif self.selected_voice: - # If a single voice is selected, default to first profile if available - if profiles: - first_profile = next(iter(profiles)) - entry = profiles[first_profile] - selected_profile = first_profile - if isinstance(entry, dict): - initial_state = entry.get("voices", []) - else: - initial_state = entry - self.selected_lang = entry[0][0] if entry and entry[0] else None - dialog = VoiceFormulaDialog( - self, initial_state=initial_state, selected_profile=selected_profile - ) - if dialog.exec() == QDialog.DialogCode.Accepted: - if dialog.current_profile: - self.selected_profile_name = dialog.current_profile - self.config["selected_profile_name"] = dialog.current_profile - if "selected_voice" in self.config: - del self.config["selected_voice"] - save_config(self.config) - self.populate_profiles_in_voice_combo() - idx = self.voice_combo.findData(f"profile:{dialog.current_profile}") - if idx >= 0: - self.voice_combo.setCurrentIndex(idx) - self.mixed_voice_state = dialog.get_selected_voices() - - def show_predownload_dialog(self): - """Show the pre-download models and voices dialog.""" - from abogen.predownload_gui import PreDownloadDialog - - dialog = PreDownloadDialog(self) - dialog.exec() - - def show_about_dialog(self): - """Show an About dialog with program information including GitHub link.""" - # Get application icon for dialog - icon = self.windowIcon() - - # Create custom dialog - dialog = QDialog(self) - dialog.setWindowTitle(f"About {PROGRAM_NAME}") - dialog.setWindowFlags( - dialog.windowFlags() & ~Qt.WindowType.WindowContextHelpButtonHint - ) - dialog.setFixedSize(400, 320) # Increased height for new button - - layout = QVBoxLayout(dialog) - layout.setSpacing(10) - - # Header with icon and title - header_layout = QHBoxLayout() - icon_label = QLabel() - if not icon.isNull(): - icon_label.setPixmap(icon.pixmap(64, 64)) - else: - # Fallback text if icon not available - icon_label.setText("📚") - icon_label.setStyleSheet("font-size: 48px;") - - header_layout.addWidget(icon_label) - - # Fix: Added style to reduce space between h1 and h3 - title_label = QLabel( - f"

    {PROGRAM_NAME} v{VERSION}

    Audiobook Generator

    " - ) - title_label.setTextFormat(Qt.TextFormat.RichText) - header_layout.addWidget(title_label, 1) - layout.addLayout(header_layout) - - # Description - desc_label = QLabel( - f"

    {PROGRAM_DESCRIPTION}

    " - "

    Visit the GitHub repository for updates, documentation, and to report issues.

    " - ) - desc_label.setTextFormat(Qt.TextFormat.RichText) - desc_label.setWordWrap(True) - layout.addWidget(desc_label) - - # GitHub link - github_btn = QPushButton("Visit GitHub Repository") - github_btn.setIcon(QIcon(get_resource_path("abogen.assets", "github.png"))) - github_btn.clicked.connect(lambda: QDesktopServices.openUrl(QUrl(GITHUB_URL))) - github_btn.setFixedHeight(32) - layout.addWidget(github_btn) - - # Check for updates button - update_btn = QPushButton("Check for updates") - update_btn.clicked.connect(self.manual_check_for_updates) - update_btn.setFixedHeight(32) - layout.addWidget(update_btn) - - # Close button - close_btn = QPushButton("Close") - close_btn.clicked.connect(dialog.accept) - close_btn.setFixedHeight(32) - layout.addWidget(close_btn) - - dialog.exec() - - def manual_check_for_updates(self): - """Manually check for updates and always show result""" - # Set a flag to always show the result message - self._show_update_check_result = True - self.check_for_updates_startup() - - def check_for_updates_startup(self): - import urllib.request - - def show_update_message(remote_version, local_version): - msg_box = QMessageBox(self) - msg_box.setIcon(QMessageBox.Icon.Information) - msg_box.setWindowTitle("Update Available") - msg_box.setText( - f"A new version of {PROGRAM_NAME} is available! ({local_version} > {remote_version})" - ) - msg_box.setInformativeText( - f"If you installed via pip, update by running:\n" - f"pip install --upgrade {PROGRAM_NAME}\n\n" - f"If you're using the Windows portable version, run 'WINDOWS_INSTALL.bat' again.\n\n" - "Alternatively, visit the GitHub repository for more information. " - "Would you like to view the changelog?" - ) - msg_box.setStandardButtons( - QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No - ) - msg_box.setDefaultButton(QMessageBox.StandardButton.Yes) - if msg_box.exec() == QMessageBox.StandardButton.Yes: - try: - QDesktopServices.openUrl(QUrl(GITHUB_URL + "/releases/latest")) - except Exception: - pass - - # Reset flag to track if we should show "no updates" message - show_result = ( - hasattr(self, "_show_update_check_result") - and self._show_update_check_result - ) - self._show_update_check_result = False - - try: - update_url = "https://raw.githubusercontent.com/denizsafak/abogen/refs/heads/main/abogen/VERSION" - with urllib.request.urlopen(update_url) as response: - remote_raw = response.read().decode().strip() - local_raw = VERSION - - # Parse version numbers - remote_version = remote_raw - local_version = local_raw - - try: - remote_num = int("".join(remote_version.split("."))) - local_num = int("".join(local_version.split("."))) - except ValueError as ve: - return - - if remote_num > local_num: - # Use QTimer to ensure UI is ready, then show update message. - QTimer.singleShot( - 1000, lambda: show_update_message(remote_version, local_version) - ) - elif show_result: - # Show "no updates" message if manually checking - QMessageBox.information( - self, - "Up to Date", - f"You are running the latest version of {PROGRAM_NAME} ({local_version}).", - ) - except Exception as e: - if show_result: - QMessageBox.warning( - self, - "Update Check Failed", - f"Could not check for updates:\n{str(e)}", - ) - pass - - def clear_cache_files(self): - """Clear cache files created by the program.""" - import glob - - try: - # Get the abogen cache directory - cache_dir = get_user_cache_path() - - # Find all .txt files and cover images in the abogen cache directory - cache_files = glob.glob(os.path.join(cache_dir, "*.txt")) - cache_files.extend(glob.glob(os.path.join(cache_dir, "cover_*.jpg"))) - - # Count the files - file_count = len(cache_files) - - # Check for preview cache files - preview_cache_dir = os.path.join(cache_dir, "preview_cache") - preview_files = [] - if os.path.exists(preview_cache_dir): - preview_pattern = os.path.join(preview_cache_dir, "*.wav") - preview_files = glob.glob(preview_pattern) - - preview_count = len(preview_files) - - if file_count == 0 and preview_count == 0: - QMessageBox.information( - self, "No Cache Files", "No cache files were found." - ) - return - - # Create a custom message box with checkbox - msg_box = QMessageBox(self) - msg_box.setIcon(QMessageBox.Icon.Question) - msg_box.setWindowTitle("Clear Cache Files") - - msg_text = f"Found {file_count} cache file{'s' if file_count != 1 else ''} in the {PROGRAM_NAME} cache folder." - if preview_count > 0: - msg_text += f"\nAlso found {preview_count} preview cache file{'s' if preview_count != 1 else ''}." - - msg_box.setText(msg_text + "\nDo you want to delete them?") - - # Add checkbox for preview cache - preview_cache_checkbox = QCheckBox("Also clean preview cache", msg_box) - preview_cache_checkbox.setChecked(False) - # Only enable checkbox if preview files exist - preview_cache_checkbox.setEnabled(preview_count > 0) - - # Add the checkbox to the layout - msg_box.setCheckBox(preview_cache_checkbox) - - # Add buttons - msg_box.setStandardButtons( - QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No - ) - msg_box.setDefaultButton(QMessageBox.StandardButton.Yes) - - if msg_box.exec() != QMessageBox.StandardButton.Yes: - return - - # Delete the text files - deleted_count = 0 - for file_path in cache_files: - try: - os.remove(file_path) - deleted_count += 1 - except Exception as e: - print(f"Error deleting {file_path}: {e}") - - # Delete preview cache files if checkbox is checked - deleted_preview_count = 0 - if preview_cache_checkbox.isChecked() and preview_count > 0: - for file_path in preview_files: - try: - os.remove(file_path) - deleted_preview_count += 1 - except Exception as e: - print(f"Error deleting preview cache {file_path}: {e}") - - # Build result message - result_msg = f"Successfully deleted {deleted_count} temporary file{'s' if deleted_count != 1 else ''}." - if preview_cache_checkbox.isChecked() and deleted_preview_count > 0: - result_msg += f"\nAlso deleted {deleted_preview_count} preview cache file{'s' if deleted_preview_count != 1 else ''}." - - # Show results - QMessageBox.information(self, "Cache Files Cleared", result_msg) - - # If currently selected file is in the cache directory, clear the UI - if ( - self.selected_file - and os.path.dirname(self.selected_file) == cache_dir - and self.selected_file.endswith(".txt") - ): - self.input_box.clear_input() - - except Exception as e: - QMessageBox.critical( - self, "Error", f"An error occurred while clearing temporary files:\n{e}" - ) - - def set_max_log_lines(self): - """Open a dialog to set the maximum lines in the log window.""" - from PyQt6.QtWidgets import QInputDialog - - value, ok = QInputDialog.getInt( - self, - "Max Lines in Log Window", - "Enter the maximum number of lines to display in the log window:", - self.log_window_max_lines, - 10, # min value - 999999999, # max value - 1, # step - ) - if ok: - self.log_window_max_lines = value - self.config["log_window_max_lines"] = value - save_config(self.config) - QMessageBox.information( - self, - "Setting Saved", - f"Maximum lines in log window set to {value}.", - ) - - def set_max_subtitle_words(self): - """Open a dialog to set the maximum words per subtitle""" - from PyQt6.QtWidgets import QInputDialog - - current_value = self.config.get("max_subtitle_words", 50) - - value, ok = QInputDialog.getInt( - self, - "Max Words Per Subtitle", - "Enter the maximum number of words per\nsubtitle (before splitting the subtitle):", - current_value, - 1, # min value - 200, # max value - 1, # step - ) - - if ok: - # Save the new value - self.max_subtitle_words = value - self.config["max_subtitle_words"] = value - save_config(self.config) - - # Show confirmation - QMessageBox.information( - self, - "Setting Saved", - f"Maximum words per subtitle set to {value}.", - ) - - def set_silence_between_chapters(self): - """Open a dialog to set the silence duration between chapters""" - - current_value = self.config.get("silence_duration", 2.0) - - dlg = QInputDialog(self) - dlg.setWindowTitle("Silence Duration (seconds)") - dlg.setLabelText( - "Enter the duration of silence\nbetween chapters (in seconds):" - ) - dlg.setInputMode(QInputDialog.InputMode.DoubleInput) - dlg.setDoubleDecimals(1) - dlg.setDoubleMinimum(0.0) - dlg.setDoubleMaximum(60.0) - dlg.setDoubleValue(current_value) - dlg.setDoubleStep(0.1) # <-- set step to 0.1 - - if dlg.exec() == QDialog.DialogCode.Accepted: - value = dlg.doubleValue() - # Round to one decimal to avoid floating-point representation noise - value = round(value, 1) - - # Save the new value - self.silence_duration = value - self.config["silence_duration"] = value - save_config(self.config) - - # Show confirmation (format with one decimal) - QMessageBox.information( - self, - "Setting Saved", - f"Silence duration between chapters set to {value:.1f} seconds.", - ) - - def set_separate_chapters_format(self, fmt): - """Set the format for separate chapters audio files.""" - self.separate_chapters_format = fmt - self.config["separate_chapters_format"] = fmt - save_config(self.config) - - def set_subtitle_format(self, fmt): - """Set the subtitle format.""" - self.config["subtitle_format"] = fmt - save_config(self.config) - - def show_model_download_warning(self, title, message): - QMessageBox.information(self, title, message) +__all__ = ["abogen"] diff --git a/abogen/heteronym_overrides.py b/abogen/heteronym_overrides.py new file mode 100644 index 0000000..08cbd07 --- /dev/null +++ b/abogen/heteronym_overrides.py @@ -0,0 +1,304 @@ +from __future__ import annotations + +import hashlib +import re +from dataclasses import dataclass +from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple + +try: # pragma: no cover - optional dependency + import spacy # type: ignore +except Exception: # pragma: no cover - spaCy may be unavailable in minimal environments + spacy = None + + +@dataclass(frozen=True) +class HeteronymVariant: + key: str + label: str + replacement_token: str + example_sentence: str + + +@dataclass(frozen=True) +class HeteronymSpec: + token: str + variants: Tuple[HeteronymVariant, HeteronymVariant] + + def default_choice_for_token(self, spacy_token: Any) -> str: + """Return the most likely variant key for this token.""" + pos = (getattr(spacy_token, "pos_", "") or "").upper() + tag = (getattr(spacy_token, "tag_", "") or "").upper() + + token_lower = self.token.casefold() + if token_lower == "wind": + # VERB => /waɪnd/, NOUN => /wɪnd/ + return "verb" if pos == "VERB" else "noun" + if token_lower == "read": + # VBD/VBN => /rɛd/ + return "past" if tag in {"VBD", "VBN"} else "present" + if token_lower == "tear": + return "verb" if pos == "VERB" else "noun" + if token_lower == "close": + return "verb" if pos == "VERB" else "adj" + if token_lower == "lead": + # Default to verb unless POS suggests noun. + return "metal" if pos == "NOUN" else "verb" + return self.variants[0].key + + +# Minimal, high-confidence starter set. +# NOTE: These replacements intentionally prioritize speech output. +# Some replacements may not be appropriate for subtitles/text exports. +_HETERONYM_SPECS: Dict[str, HeteronymSpec] = { + "wind": HeteronymSpec( + token="wind", + variants=( + HeteronymVariant( + key="noun", + label="Noun (the wind)", + replacement_token="wind", + example_sentence="Listen to the wind.", + ), + HeteronymVariant( + key="verb", + label="Verb (to wind)", + replacement_token="wynd", + example_sentence="I need to wind the watch.", + ), + ), + ), + "read": HeteronymSpec( + token="read", + variants=( + HeteronymVariant( + key="present", + label="Present (I read every day)", + replacement_token="read", + example_sentence="I read every day.", + ), + HeteronymVariant( + key="past", + label="Past (I read it yesterday)", + replacement_token="red", + example_sentence="I read it yesterday.", + ), + ), + ), + "tear": HeteronymSpec( + token="tear", + variants=( + HeteronymVariant( + key="noun", + label="Noun (a tear /crying/)", + replacement_token="tier", + example_sentence="A tear rolled down her cheek.", + ), + HeteronymVariant( + key="verb", + label="Verb (to tear /rip/)", + replacement_token="tear", + example_sentence="Please don't tear the page.", + ), + ), + ), + "close": HeteronymSpec( + token="close", + variants=( + HeteronymVariant( + key="adj", + label="Adjective (close /near/)", + replacement_token="close", + example_sentence="We are close to the station.", + ), + HeteronymVariant( + key="verb", + label="Verb (close /klohz/)", + replacement_token="cloze", + example_sentence="Please close the door.", + ), + ), + ), + "lead": HeteronymSpec( + token="lead", + variants=( + HeteronymVariant( + key="verb", + label="Verb (to lead)", + replacement_token="lead", + example_sentence="They will lead the way.", + ), + HeteronymVariant( + key="metal", + label="Noun (lead /metal/)", + replacement_token="led", + example_sentence="The pipe was made of lead.", + ), + ), + ), +} + + +def _hash_id(*parts: str) -> str: + digest = hashlib.sha1("\n".join(parts).encode("utf-8")).hexdigest() + return digest[:12] + + +_WORD_BOUNDARY_CACHE: Dict[str, re.Pattern[str]] = {} + + +def _word_boundary_pattern(token: str) -> re.Pattern[str]: + key = token.casefold() + cached = _WORD_BOUNDARY_CACHE.get(key) + if cached is not None: + return cached + escaped = re.escape(token) + pattern = re.compile( + rf"(?i)(?'s|\u2019s|\u2019)?(?!\w)" + ) + _WORD_BOUNDARY_CACHE[key] = pattern + return pattern + + +def _preserve_case(replacement: str, original: str) -> str: + if not replacement: + return replacement + if original.isupper(): + return replacement.upper() + if original[:1].isupper(): + return replacement[:1].upper() + replacement[1:] + return replacement + + +def _build_replacement_sentence( + sentence: str, token: str, replacement_token: str +) -> str: + pattern = _word_boundary_pattern(token) + + def _repl(match: re.Match[str]) -> str: + matched = match.group(0) or "" + suffix = match.group("possessive") or "" + base = matched[: len(matched) - len(suffix)] if suffix else matched + return _preserve_case(replacement_token, base) + suffix + + return pattern.sub(_repl, sentence) + + +def _load_spacy(language: str) -> Any: + if spacy is None: + return None + + # English only for now. + # Use installed small model; keep it simple. + lang = (language or "en").lower() + if lang.startswith("en"): + try: + return spacy.load("en_core_web_sm") + except Exception: + return spacy.blank("en") + return spacy.blank("xx") + + +def extract_heteronym_overrides( + chapters: Sequence[Mapping[str, Any]], + *, + language: str, + existing: Optional[Iterable[Mapping[str, Any]]] = None, +) -> List[Dict[str, Any]]: + """Extract distinct heteronym-containing sentences from chapters. + + Returns entries shaped for persistence + UI. + + Each entry contains: + - id + - token + - sentence + - options: [{key,label,replacement_token,replacement_sentence,example_sentence}] + - default_choice + - choice + """ + + lang = (language or "en").lower() + if not lang.startswith("en"): + return [] + + if spacy is None: + return [] + + nlp = _load_spacy(lang) + if nlp is None: + return [] + + previous_choices: Dict[str, str] = {} + if existing: + for item in existing: + if not isinstance(item, Mapping): + continue + entry_id = str(item.get("id") or "").strip() + choice = str(item.get("choice") or "").strip() + if entry_id and choice: + previous_choices[entry_id] = choice + + results: List[Dict[str, Any]] = [] + seen: set[tuple[str, str]] = set() + + for chapter in chapters: + if not isinstance(chapter, Mapping): + continue + text = str(chapter.get("text") or "") + if not text.strip(): + continue + + doc = nlp(text) + for sent in getattr(doc, "sents", []): + sentence = str(getattr(sent, "text", "") or "").strip() + if not sentence: + continue + + for token in sent: + token_text = str(getattr(token, "text", "") or "") + if not token_text: + continue + token_key = token_text.casefold() + spec = _HETERONYM_SPECS.get(token_key) + if not spec: + continue + + dedupe_key = (token_key, sentence) + if dedupe_key in seen: + continue + seen.add(dedupe_key) + + entry_id = _hash_id(token_key, sentence) + default_choice = spec.default_choice_for_token(token) + choice = previous_choices.get(entry_id, default_choice) + + options: List[Dict[str, Any]] = [] + for variant in spec.variants: + replacement_sentence = _build_replacement_sentence( + sentence, + token=spec.token, + replacement_token=variant.replacement_token, + ) + options.append( + { + "key": variant.key, + "label": variant.label, + "replacement_token": variant.replacement_token, + "replacement_sentence": replacement_sentence, + "example_sentence": variant.example_sentence, + } + ) + + results.append( + { + "id": entry_id, + "token": token_text, + "token_lower": token_key, + "sentence": sentence, + "options": options, + "default_choice": default_choice, + "choice": choice, + } + ) + + return results diff --git a/abogen/integrations/__init__.py b/abogen/integrations/__init__.py new file mode 100644 index 0000000..06e8198 --- /dev/null +++ b/abogen/integrations/__init__.py @@ -0,0 +1 @@ +"""Integration clients for external services.""" diff --git a/abogen/integrations/audiobookshelf.py b/abogen/integrations/audiobookshelf.py new file mode 100644 index 0000000..0538cce --- /dev/null +++ b/abogen/integrations/audiobookshelf.py @@ -0,0 +1,680 @@ +from __future__ import annotations + +import json +import logging +import math +import mimetypes +import re +from contextlib import ExitStack +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple + +import httpx + +logger = logging.getLogger(__name__) + + +class AudiobookshelfUploadError(RuntimeError): + """Raised when an upload to Audiobookshelf fails.""" + + +@dataclass(frozen=True) +class AudiobookshelfConfig: + base_url: str + api_token: str + library_id: Optional[str] = None + collection_id: Optional[str] = None + folder_id: Optional[str] = None + verify_ssl: bool = True + send_cover: bool = True + send_chapters: bool = True + send_subtitles: bool = True + timeout: float = 3600.0 + + def normalized_base_url(self) -> str: + base = (self.base_url or "").strip() + if not base: + raise ValueError("Audiobookshelf base URL is required") + normalized = base.rstrip("/") + # The web UI historically suggested including '/api' in the base URL; trim + # it here so we can safely append `/api/...` endpoints below. + if normalized.lower().endswith("/api"): + normalized = normalized[:-4] + return normalized or base + + +class AudiobookshelfClient: + """Client for the legacy Audiobookshelf multipart upload endpoint.""" + + def __init__(self, config: AudiobookshelfConfig) -> None: + if not config.api_token: + raise ValueError("Audiobookshelf API token is required") + # library_id is now optional for discovery + self._config = config + normalized = config.normalized_base_url() or "" + self._base_url = normalized.rstrip("/") or normalized + self._client_base_url = f"{self._base_url}/" + self._folder_cache: Optional[Tuple[str, str, str]] = None + + def get_libraries(self) -> List[Dict[str, Any]]: + """Fetch all libraries from the Audiobookshelf server.""" + route = self._api_path("libraries") + try: + with self._open_client() as client: + response = client.get(route) + response.raise_for_status() + data = response.json() + # data['libraries'] is a list of library objects + return data.get("libraries", []) + except httpx.HTTPError as exc: + raise AudiobookshelfUploadError(f"Failed to fetch libraries: {exc}") from exc + + def _api_path(self, suffix: str = "") -> str: + """Join the API prefix with the provided suffix without losing proxies.""" + clean_suffix = suffix.lstrip("/") + return f"api/{clean_suffix}" if clean_suffix else "api" + + def upload_audiobook( + self, + audio_path: Path, + *, + metadata: Dict[str, Any], + cover_path: Optional[Path] = None, + chapters: Optional[Iterable[Dict[str, Any]]] = None, + subtitles: Optional[Iterable[Path]] = None, + ) -> Dict[str, Any]: + if not audio_path.exists(): + raise AudiobookshelfUploadError(f"Audio path does not exist: {audio_path}") + + form_fields = self._build_upload_fields(audio_path, metadata, chapters) + file_entries = self._build_file_entries(audio_path, cover_path, subtitles) + + route = self._api_path("upload") + try: + with self._open_client() as client, ExitStack() as stack: + files_payload = self._open_file_handles(file_entries, stack) + response = client.post(route, data=form_fields, files=files_payload) + response.raise_for_status() + except httpx.HTTPStatusError as exc: + status = exc.response.status_code + detail = (exc.response.text or "").strip() + if detail: + detail = detail[:200] + message = f"Audiobookshelf upload failed with status {status}: {detail}" + else: + message = f"Audiobookshelf upload failed with status {status}" + raise AudiobookshelfUploadError( + message + ) from exc + except httpx.HTTPError as exc: + raise AudiobookshelfUploadError(f"Audiobookshelf upload failed: {exc}") from exc + + return {} + + def _open_client(self) -> httpx.Client: + headers = { + "Authorization": f"Bearer {self._config.api_token}", + "Accept": "application/json", + } + return httpx.Client( + base_url=self._client_base_url, + headers=headers, + timeout=self._config.timeout, + verify=self._config.verify_ssl, + ) + + def _build_upload_fields( + self, + audio_path: Path, + metadata: Dict[str, Any], + chapters: Optional[Iterable[Dict[str, Any]]], + ) -> Dict[str, str]: + folder_id, _, _ = self._ensure_folder() + title = self._extract_title(metadata, audio_path) + author = self._extract_author(metadata) + series = self._extract_series(metadata) + series_sequence = self._extract_series_sequence(metadata) + + fields: Dict[str, str] = { + "library": self._config.library_id, + "folder": folder_id, + "title": title, + } + if author: + fields["author"] = author + if series: + fields["series"] = series + if series_sequence: + fields["seriesSequence"] = series_sequence + if self._config.collection_id: + fields["collectionId"] = self._config.collection_id + + metadata_payload: Dict[str, Any] = metadata or {} + if chapters and self._config.send_chapters: + metadata_payload = dict(metadata_payload) + metadata_payload["chapters"] = list(chapters) + + if metadata_payload: + # Ensure authors is a list of strings in the JSON payload if it exists + if "authors" in metadata_payload: + authors_val = metadata_payload["authors"] + if isinstance(authors_val, str): + metadata_payload["authors"] = [a.strip() for a in authors_val.split(",") if a.strip()] + elif isinstance(authors_val, list): + metadata_payload["authors"] = [str(a).strip() for a in authors_val if str(a).strip()] + + try: + fields["metadata"] = json.dumps(metadata_payload, ensure_ascii=False) + except (TypeError, ValueError): + logger.debug("Failed to serialize Audiobookshelf metadata payload") + + return fields + + def _build_file_entries( + self, + audio_path: Path, + cover_path: Optional[Path], + subtitles: Optional[Iterable[Path]], + ) -> List[Tuple[str, Path]]: + entries: List[Tuple[str, Path]] = [("file0", audio_path)] + index = 1 + + if cover_path and self._config.send_cover and cover_path.exists(): + entries.append((f"file{index}", cover_path)) + index += 1 + + if subtitles and self._config.send_subtitles: + for subtitle in subtitles: + if subtitle.exists(): + entries.append((f"file{index}", subtitle)) + index += 1 + + return entries + + def _open_file_handles( + self, + entries: Sequence[Tuple[str, Path]], + stack: ExitStack, + ) -> List[Tuple[str, Tuple[str, Any, str]]]: + files: List[Tuple[str, Tuple[str, Any, str]]] = [] + for field_name, path in entries: + mime_type, _ = mimetypes.guess_type(path.name) + mime_type = mime_type or "application/octet-stream" + handle = stack.enter_context(path.open("rb")) + files.append((field_name, (path.name, handle, mime_type))) + return files + + def find_existing_items( + self, + title: str, + *, + folder_id: Optional[str] = None, + ) -> List[Mapping[str, Any]]: + normalized_title = self._normalize_title_value(title) + if not normalized_title: + return [] + + folder_hint = folder_id or self._config.folder_id + target_folders = set() + if folder_hint: + folder_token = str(folder_hint).strip().lower() + if folder_token: + target_folders.add(folder_token) + + requests = self._candidate_search_requests(title, folder_hint) + if not requests: + return [] + + matches: List[Mapping[str, Any]] = [] + + try: + with self._open_client() as client: + for route, params in requests: + try: + response = client.get(route, params=params) + except httpx.HTTPError as exc: + logger.debug("Audiobookshelf lookup failed for %s: %s", route, exc) + continue + + if response.status_code == 404: + continue + + try: + response.raise_for_status() + except httpx.HTTPStatusError as exc: + status = exc.response.status_code + if status in {401, 403}: + raise AudiobookshelfUploadError( + "Audiobookshelf authentication failed while checking for existing items." + ) from exc + logger.debug("Audiobookshelf lookup error %s for %s", status, route) + continue + + try: + payload = response.json() + except ValueError: + continue + + candidates = self._extract_candidate_items(payload) + for item in candidates: + item_title = self._normalize_item_title(item) + if not item_title or item_title != normalized_title: + continue + if target_folders: + item_folder = self._normalize_folder_id(item) + if item_folder and item_folder not in target_folders: + continue + matches.append(item) + if matches: + break + except AudiobookshelfUploadError: + raise + except Exception: + logger.debug( + "Unexpected error while checking Audiobookshelf for existing items", + exc_info=True, + ) + + return matches + + def delete_items(self, items: Iterable[Mapping[str, Any] | str]) -> None: + to_delete: List[str] = [] + for entry in items: + if isinstance(entry, Mapping): + item_id = self._extract_item_id(entry) + else: + item_id = str(entry).strip() + if item_id: + to_delete.append(item_id) + + if not to_delete: + return + + with self._open_client() as client: + for item_id in to_delete: + self._delete_single_item(client, item_id) + + def _candidate_search_requests( + self, + title: str, + folder_id: Optional[str], + ) -> List[Tuple[str, Dict[str, Any]]]: + query = (title or "").strip() + if not query: + return [] + + library_id = self._config.library_id + folder_token = (folder_id or self._config.folder_id or "").strip() + + requests: List[Tuple[str, Dict[str, Any]]] = [] + seen_routes: set[str] = set() + + def _append(route: str, params: Dict[str, Any]) -> None: + if route in seen_routes: + return + seen_routes.add(route) + requests.append((route, params)) + + if folder_token: + _append( + self._api_path(f"folders/{folder_token}/items"), + {"library": library_id, "search": query}, + ) + + _append(self._api_path(f"libraries/{library_id}/items"), {"search": query}) + _append(self._api_path("items"), {"library": library_id, "search": query}) + _append( + self._api_path("search"), + {"query": query, "library": library_id, "media": "audiobook"}, + ) + + return requests + + def _delete_single_item(self, client: httpx.Client, item_id: str) -> None: + routes = [ + self._api_path(f"items/{item_id}"), + self._api_path(f"libraries/{self._config.library_id}/items/{item_id}"), + ] + + for route in routes: + try: + response = client.delete(route) + except httpx.HTTPError as exc: + logger.debug("Audiobookshelf delete failed for %s: %s", route, exc) + continue + + if response.status_code in (200, 202, 204): + return + if response.status_code == 404: + continue + + try: + response.raise_for_status() + except httpx.HTTPStatusError as exc: + raise AudiobookshelfUploadError( + f"Failed to delete Audiobookshelf item '{item_id}': {exc}" + ) from exc + + logger.debug("Audiobookshelf item %s could not be confirmed deleted", item_id) + + def resolve_folder(self) -> Tuple[str, str, str]: + """Return the resolved folder (id, name, library name).""" + return self._ensure_folder() + + def list_folders(self) -> List[Dict[str, str]]: + """Return all folders for the configured library.""" + library_name, folders = self._load_library_metadata() + results: List[Dict[str, str]] = [] + for folder in folders: + folder_id = str(folder.get("id") or "").strip() + if not folder_id: + continue + name = self._folder_display_name(folder) + path = self._select_folder_path(folder) + results.append( + { + "id": folder_id, + "name": name, + "path": path, + "library": library_name, + } + ) + results.sort(key=lambda entry: (entry.get("path") or entry.get("name") or entry.get("id") or "").lower()) + return results + + def _ensure_folder(self) -> Tuple[str, str, str]: + if self._folder_cache: + return self._folder_cache + + identifier = (self._config.folder_id or "").strip() + if not identifier: + raise AudiobookshelfUploadError( + "Audiobookshelf folder is required; enter the folder name or ID in Settings." + ) + + identifier_norm = self._normalize_identifier(identifier) + library_name, folders = self._load_library_metadata() + + # direct ID match + for folder in folders: + folder_id = str(folder.get("id") or "").strip() + if folder_id and folder_id == identifier: + folder_name = self._folder_display_name(folder) or folder_id + self._folder_cache = (folder_id, folder_name, library_name) + return self._folder_cache + + has_path_component = "/" in identifier_norm + + for folder in folders: + folder_id = str(folder.get("id") or "").strip() + if not folder_id: + continue + folder_name = self._folder_display_name(folder) + name_norm = self._normalize_identifier(folder_name) + if name_norm and name_norm == identifier_norm: + self._folder_cache = (folder_id, folder_name or folder_id, library_name) + return self._folder_cache + + for candidate in self._folder_path_candidates(folder): + candidate_norm = self._normalize_identifier(candidate) + if not candidate_norm: + continue + if candidate_norm == identifier_norm: + self._folder_cache = (folder_id, folder_name or folder_id, library_name) + return self._folder_cache + if has_path_component and candidate_norm.endswith(identifier_norm): + self._folder_cache = (folder_id, folder_name or folder_id, library_name) + return self._folder_cache + if not has_path_component: + tail = candidate_norm.split("/")[-1] + if tail and tail == identifier_norm: + self._folder_cache = (folder_id, folder_name or folder_id, library_name) + return self._folder_cache + + raise AudiobookshelfUploadError( + f"Folder '{identifier}' was not found in library '{library_name}'. " + "Enter the folder name exactly as it appears in Audiobookshelf, a trailing path segment, or paste the folder ID." + ) + + def _load_library_metadata(self) -> Tuple[str, List[Mapping[str, Any]]]: + try: + with self._open_client() as client: + response = client.get(self._api_path(f"libraries/{self._config.library_id}")) + response.raise_for_status() + payload = response.json() + except httpx.HTTPStatusError as exc: + status = exc.response.status_code + if status == 404: + message = f"Audiobookshelf library '{self._config.library_id}' not found." + else: + detail = (exc.response.text or "").strip() + if detail: + detail = detail[:200] + message = ( + f"Failed to load Audiobookshelf library '{self._config.library_id}' " + f"(status {status}): {detail}" + ) + else: + message = ( + f"Failed to load Audiobookshelf library '{self._config.library_id}' " + f"(status {status})." + ) + raise AudiobookshelfUploadError(message) from exc + except httpx.HTTPError as exc: + raise AudiobookshelfUploadError( + f"Failed to reach Audiobookshelf library '{self._config.library_id}': {exc}" + ) from exc + + if not isinstance(payload, Mapping): + return self._config.library_id, [] + + library_name = str(payload.get("name") or payload.get("label") or self._config.library_id) + raw_folders = payload.get("libraryFolders") or payload.get("folders") or [] + folders = [entry for entry in raw_folders if isinstance(entry, Mapping)] + return library_name, folders + + @staticmethod + def _folder_path_candidates(folder: Mapping[str, Any]) -> List[str]: + candidates: List[str] = [] + for key in ("fullPath", "fullpath", "path", "folderPath", "virtualPath"): + value = folder.get(key) + if isinstance(value, str) and value.strip(): + candidates.append(value) + return candidates + + @staticmethod + def _folder_display_name(folder: Mapping[str, Any]) -> str: + name = str(folder.get("name") or folder.get("label") or "").strip() + if name: + return name + path = AudiobookshelfClient._select_folder_path(folder) + if path: + tail = path.strip("/ ") + tail = tail.split("/")[-1] if tail else "" + if tail: + return tail + return str(folder.get("id") or "").strip() + + @staticmethod + def _select_folder_path(folder: Mapping[str, Any]) -> str: + for candidate in AudiobookshelfClient._folder_path_candidates(folder): + normalized = candidate.replace("\\", "/").strip() + if normalized: + return normalized + return "" + + @staticmethod + def _normalize_identifier(value: str) -> str: + token = (value or "").strip() + token = token.replace("\\", "/") + if len(token) > 1 and token[1] == ":": + token = token[2:] + token = token.strip("/ ") + return token.lower() + + @staticmethod + def _normalize_title_value(value: Optional[str]) -> str: + if not isinstance(value, str): + return "" + normalized = re.sub(r"\s+", " ", value).strip() + return normalized.casefold() if normalized else "" + + @staticmethod + def _normalize_item_title(item: Mapping[str, Any]) -> str: + if not isinstance(item, Mapping): + return "" + for key in ("title", "name", "label"): + candidate = item.get(key) + if isinstance(candidate, str) and candidate.strip(): + return AudiobookshelfClient._normalize_title_value(candidate) + library_item = item.get("libraryItem") + if isinstance(library_item, Mapping): + return AudiobookshelfClient._normalize_item_title(library_item) + return "" + + @staticmethod + def _normalize_folder_id(item: Mapping[str, Any]) -> Optional[str]: + if not isinstance(item, Mapping): + return None + for key in ("folderId", "libraryFolderId", "folder_id", "folder"): + value = item.get(key) + if isinstance(value, str) and value.strip(): + return value.strip().lower() + if isinstance(value, (int, float)): + return str(value).strip().lower() + library_item = item.get("libraryItem") + if isinstance(library_item, Mapping): + return AudiobookshelfClient._normalize_folder_id(library_item) + return None + + @staticmethod + def _extract_item_id(item: Mapping[str, Any]) -> Optional[str]: + if not isinstance(item, Mapping): + return None + for key in ("id", "libraryItemId", "itemId"): + value = item.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + if isinstance(value, (int, float)): + return str(value).strip() + library_item = item.get("libraryItem") + if isinstance(library_item, Mapping): + return AudiobookshelfClient._extract_item_id(library_item) + return None + + @staticmethod + def _extract_candidate_items(payload: Any) -> List[Mapping[str, Any]]: + items: List[Mapping[str, Any]] = [] + seen_ids: set[str] = set() + visited: set[int] = set() + + def _visit(obj: Any) -> None: + if isinstance(obj, Mapping): + obj_id = id(obj) + if obj_id in visited: + return + visited.add(obj_id) + + title = AudiobookshelfClient._normalize_item_title(obj) + item_id = AudiobookshelfClient._extract_item_id(obj) + if title and item_id: + key = item_id.strip().lower() + if key not in seen_ids: + seen_ids.add(key) + items.append(obj) + + for value in obj.values(): + _visit(value) + + elif isinstance(obj, list): + for entry in obj: + _visit(entry) + + _visit(payload) + return items + + @staticmethod + def _extract_title(metadata: Mapping[str, Any], audio_path: Path) -> str: + title = metadata.get("title") if isinstance(metadata, Mapping) else None + candidate = str(title).strip() if isinstance(title, str) else "" + if candidate: + return candidate + return audio_path.stem or audio_path.name + + @staticmethod + def _extract_author(metadata: Mapping[str, Any]) -> str: + authors = metadata.get("authors") if isinstance(metadata, Mapping) else None + if isinstance(authors, str): + candidate = authors.strip() + return candidate + if isinstance(authors, Iterable) and not isinstance(authors, (str, Mapping)): + names = [str(entry).strip() for entry in authors if isinstance(entry, str) and entry.strip()] + if names: + # ABS expects a comma-separated string for multiple authors. + return ", ".join(names) + return "" + + @staticmethod + def _extract_series(metadata: Mapping[str, Any]) -> str: + series_name = metadata.get("seriesName") if isinstance(metadata, Mapping) else None + if isinstance(series_name, str) and series_name.strip(): + return series_name.strip() + return "" + + @staticmethod + def _extract_series_sequence(metadata: Mapping[str, Any]) -> str: + if not isinstance(metadata, Mapping): + return "" + + preferred_keys = ( + "seriesSequence", + "series_sequence", + "seriesIndex", + "series_index", + "seriesNumber", + "series_number", + "bookNumber", + "book_number", + ) + + for key in preferred_keys: + if key not in metadata: + continue + normalized = AudiobookshelfClient._normalize_series_sequence(metadata.get(key)) + if normalized: + return normalized + return "" + + @staticmethod + def _normalize_series_sequence(raw: Any) -> str: + if raw is None: + return "" + + if isinstance(raw, (int, float)): + if isinstance(raw, float) and (math.isnan(raw) or math.isinf(raw)): + return "" + text = str(raw) + else: + text = str(raw).strip() + + if not text: + return "" + + candidate = text.replace(",", ".") + match = re.search(r"\d+(?:\.\d+)?", candidate) + if not match: + return "" + + normalized = match.group(0) + if "." in normalized: + normalized = normalized.rstrip("0").rstrip(".") + if not normalized: + normalized = "0" + return normalized + + try: + return str(int(normalized)) + except ValueError: + cleaned = normalized.lstrip("0") + return cleaned or "0" diff --git a/abogen/integrations/calibre_opds.py b/abogen/integrations/calibre_opds.py new file mode 100644 index 0000000..128e2ba --- /dev/null +++ b/abogen/integrations/calibre_opds.py @@ -0,0 +1,1460 @@ +from __future__ import annotations + +import dataclasses +import html +import re +import unicodedata +import xml.etree.ElementTree as ET +from collections import deque +from dataclasses import dataclass, field +from pathlib import PurePosixPath +from typing import Any, Deque, Dict, Iterable, Iterator, List, Mapping, Optional, Set, Tuple, Union +from urllib.parse import quote, urljoin, urlparse + +import httpx + + +ATOM_NS = "http://www.w3.org/2005/Atom" +OPDS_NS = "http://opds-spec.org/2010/catalog" +DC_NS = "http://purl.org/dc/terms/" +CALIBRE_CATALOG_NS = "http://calibre.kovidgoyal.net/2009/catalog" +CALIBRE_METADATA_NS = "http://calibre.kovidgoyal.net/2009/metadata" +NS = { + "atom": ATOM_NS, + "opds": OPDS_NS, + "dc": DC_NS, + "calibre": CALIBRE_CATALOG_NS, + "calibre_md": CALIBRE_METADATA_NS, +} + + +_TAG_STRIP_RE = re.compile(r"<[^>]+>") +_SERIES_PREFIX_RE = re.compile(r"^\s*(series|books?)\s*[:\-]\s*", re.IGNORECASE) +_SERIES_NUMBER_BRACKET_RE = re.compile(r"[\[(]\s*(?:book\s*)?(\d+(?:\.\d+)?)\s*[\])]", re.IGNORECASE) +_SERIES_NUMBER_HASH_RE = re.compile(r"#\s*(\d+(?:\.\d+)?)") +_SERIES_NUMBER_BOOK_RE = re.compile(r"\bbook\s+(\d+(?:\.\d+)?)\b", re.IGNORECASE) +_SERIES_LINE_TEXT_RE = re.compile(r"^\s*series\s*[:\-]\s*(.+)$", re.IGNORECASE) +_SUMMARY_METADATA_LINE_RE = re.compile(r"^([A-Z][A-Z0-9&/\- +'\u2019]{1,40})\s*[:\-]\s*(.+)$") +_EPUB_MIME_TYPES = { + "application/epub+zip", + "application/zip", + "application/x-zip", + "application/x-zip-compressed", +} +_SUPPORTED_DOWNLOAD_MIME_TYPES = set(_EPUB_MIME_TYPES) | {"application/pdf"} +_SUPPORTED_DOWNLOAD_EXTENSIONS = {".epub", ".pdf"} +_STOP_WORDS = { + "a", "an", "the", "and", "or", "but", "if", "then", "else", "when", + "at", "by", "for", "from", "in", "into", "of", "off", "on", "onto", + "to", "with", "is", "are", "was", "were", "be", "been", "being", + "that", "this", "these", "those", "it", "its" +} + + +class CalibreOPDSError(RuntimeError): + """Raised when the Calibre OPDS client encounters an unrecoverable error.""" + + +@dataclass +class OPDSLink: + href: str + rel: Optional[str] = None + type: Optional[str] = None + title: Optional[str] = None + + def to_dict(self) -> Dict[str, Optional[str]]: + return { + "href": self.href, + "rel": self.rel, + "type": self.type, + "title": self.title, + } + + +@dataclass +class OPDSEntry: + id: str + title: str + position: Optional[int] = None + authors: List[str] = field(default_factory=list) + subtitle: Optional[str] = None + updated: Optional[str] = None + published: Optional[str] = None + summary: Optional[str] = None + download: Optional[OPDSLink] = None + alternate: Optional[OPDSLink] = None + thumbnail: Optional[OPDSLink] = None + links: List[OPDSLink] = field(default_factory=list) + series: Optional[str] = None + series_index: Optional[float] = None + tags: List[str] = field(default_factory=list) + rating: Optional[float] = None + rating_max: Optional[float] = None + + def to_dict(self) -> Dict[str, Any]: + return { + "id": self.id, + "title": self.title, + "position": self.position, + "authors": list(self.authors), + "subtitle": self.subtitle, + "updated": self.updated, + "published": self.published, + "summary": self.summary, + "download": self.download.to_dict() if self.download else None, + "alternate": self.alternate.to_dict() if self.alternate else None, + "thumbnail": self.thumbnail.to_dict() if self.thumbnail else None, + "links": [link.to_dict() for link in self.links], + "series": self.series, + "series_index": self.series_index, + "tags": list(self.tags), + "rating": self.rating, + "rating_max": self.rating_max, + } + + +@dataclass +class OPDSFeed: + id: Optional[str] + title: Optional[str] + entries: List[OPDSEntry] + links: Dict[str, OPDSLink] = field(default_factory=dict) + + def to_dict(self) -> Dict[str, Any]: + return { + "id": self.id, + "title": self.title, + "entries": [entry.to_dict() for entry in self.entries], + "links": {key: link.to_dict() for key, link in self.links.items()}, + } + + +def feed_to_dict(feed: OPDSFeed) -> Dict[str, Any]: + return feed.to_dict() + + +@dataclass +class DownloadedResource: + filename: str + mime_type: str + content: bytes + + +class CalibreOPDSClient: + """Client for interacting with a Calibre-Web OPDS catalog.""" + + def __init__( + self, + base_url: str, + *, + username: Optional[str] = None, + password: Optional[str] = None, + timeout: float = 15.0, + verify: bool = True, + ) -> None: + if not base_url: + raise ValueError("Calibre OPDS base URL is required") + normalized = base_url.strip() + if not normalized: + raise ValueError("Calibre OPDS base URL is required") + # Store the original URL without forcing a trailing slash. + # Some servers (e.g., Booklore) return 404 for URLs with trailing slashes. + self._base_url = normalized.rstrip("/") + self._auth = None + if username: + self._auth = httpx.BasicAuth(username, password or "") + self._timeout = timeout + self._verify = verify + self._headers = { + "User-Agent": "abogen-calibre-opds/1.0", + "Accept": "application/atom+xml,application/xml;q=0.9,*/*;q=0.8", + } + + @staticmethod + def _strip_html(value: Optional[str]) -> Optional[str]: + if not value: + return None + cleaned = _TAG_STRIP_RE.sub("", value) + return html.unescape(cleaned).strip() or None + + def _make_url(self, href: Optional[str]) -> str: + if not href: + return self._base_url + href = href.strip() + if href.startswith("http://") or href.startswith("https://"): + return href + if href.startswith("/"): + # Absolute path - join with origin only + parsed = urlparse(self._base_url) + return f"{parsed.scheme}://{parsed.netloc}{href}" + if href.startswith("?") or href.startswith("#"): + return f"{self._base_url}{href}" + if href.startswith("./") or href.startswith("../"): + # For relative paths, we need a trailing slash for urljoin to work correctly + base_with_slash = self._base_url if self._base_url.endswith("/") else f"{self._base_url}/" + return urljoin(base_with_slash, href) + # Relative path like "search" or "catalog?page=1" - treat as sibling + base_with_slash = self._base_url if self._base_url.endswith("/") else f"{self._base_url}/" + return urljoin(base_with_slash, href) + + def _open_client(self) -> httpx.Client: + return httpx.Client( + auth=self._auth, + headers=dict(self._headers), + timeout=self._timeout, + verify=self._verify, + ) + + def fetch_feed(self, href: Optional[str] = None, *, params: Optional[Mapping[str, Any]] = None) -> OPDSFeed: + target = self._make_url(href) + try: + with self._open_client() as client: + response = client.get(target, params=params, follow_redirects=True) + response.raise_for_status() + except httpx.HTTPStatusError as exc: # pragma: no cover - thin wrapper + raise CalibreOPDSError(f"Calibre OPDS request failed: {exc.response.status_code}") from exc + except httpx.HTTPError as exc: # pragma: no cover - thin wrapper + raise CalibreOPDSError(f"Calibre OPDS request failed: {exc}") from exc + + return self._parse_feed(response.text, base_url=target) + + def _fetch_opensearch_template(self, href: str) -> Optional[str]: + target = self._make_url(href) + try: + with self._open_client() as client: + response = client.get(target, follow_redirects=True) + response.raise_for_status() + + # Simple XML parsing to find the Url template + # We avoid full namespace handling for robustness + root = ET.fromstring(response.text) + for node in root.iter(): + if node.tag.endswith("Url"): + template = node.attrib.get("template") + if template and "{searchTerms}" in template: + mime = node.attrib.get("type", "") + # Prefer atom/xml feeds + if "atom" in mime or "xml" in mime: + return template + return None + except Exception: + return None + + def _find_best_seed_feed(self, root_feed: OPDSFeed) -> OPDSFeed: + # If the root feed already has books, use it + for entry in root_feed.entries: + if any("acquisition" in (link.rel or "") for link in entry.links): + return root_feed + + # Otherwise, look for a "By Title" or "All" navigation entry + candidates = ["title", "all", "books", "catalog"] + best_href = None + + for entry in root_feed.entries: + title_lower = (entry.title or "").lower() + if any(c in title_lower for c in candidates): + # Check if it has a navigation link + for link in entry.links: + if self._is_navigation_link(link): + best_href = link.href + # Prefer "By Title" explicitly + if "title" in title_lower: + break + if best_href and "title" in title_lower: + break + + if best_href: + try: + return self.fetch_feed(best_href) + except CalibreOPDSError: + pass + + return root_feed + + def search(self, query: str, start_href: Optional[str] = None) -> OPDSFeed: + cleaned = (query or "").strip() + if not cleaned: + return self.fetch_feed(start_href) if start_href else self.fetch_feed() + + base_feed: Optional[OPDSFeed] = None + try: + base_feed = self.fetch_feed() + except CalibreOPDSError: + pass + + # 1. Try explicit search link from feed + if base_feed: + # Check for OpenSearch description first + search_link = self._resolve_link(base_feed.links, "search") + if search_link and search_link.type == "application/opensearchdescription+xml": + template = self._fetch_opensearch_template(search_link.href) + if template: + search_url = template.replace("{searchTerms}", quote(cleaned)) + try: + feed = self.fetch_feed(search_url) + if feed.entries: + filtered = self._filter_feed_entries(feed, cleaned) + if filtered.entries: + return filtered + except CalibreOPDSError: + pass + + # Check for direct template + search_url = self._resolve_search_url(base_feed, cleaned) + if search_url: + try: + feed = self.fetch_feed(search_url) + if feed.entries: + filtered = self._filter_feed_entries(feed, cleaned) + if filtered.entries: + return filtered + except CalibreOPDSError: + pass + + # 2. Try common guesses if explicit link failed + candidates: List[Tuple[Optional[str], Optional[Mapping[str, Any]]]] = [ + ("search", {"query": cleaned}), + ("search", {"q": cleaned}), + (None, {"search": cleaned}), + ] + + last_error: Optional[Exception] = None + + for path, params in candidates: + try: + feed = self.fetch_feed(path, params=params) + if feed.entries: + # Check if the server ignored the query and returned the default feed + if base_feed and feed.title == base_feed.title: + # Compare first entry ID to see if it's the same feed + if feed.entries[0].id == base_feed.entries[0].id: + continue + + filtered = self._filter_feed_entries(feed, cleaned) + if filtered.entries: + return filtered + except CalibreOPDSError as exc: + last_error = exc + continue + + # 3. Fallback to local search (crawling) + seed_feed: Optional[OPDSFeed] = None + if start_href: + try: + seed_feed = self.fetch_feed(start_href) + except CalibreOPDSError: + pass + + if not seed_feed and base_feed: + # If we are falling back to base_feed (Root), try to find a better seed + seed_feed = self._find_best_seed_feed(base_feed) + + if not seed_feed: + try: + seed_feed = self.fetch_feed() + except CalibreOPDSError as exc: + if last_error: + raise last_error + raise exc + + # Heuristic: If the seed feed has acquisition links, use linear scan. + # Otherwise, use BFS to find content. + has_books = False + if seed_feed and seed_feed.entries: + for entry in seed_feed.entries[:5]: + for link in entry.links: + if "acquisition" in (link.rel or ""): + has_books = True + break + if has_books: + break + + if has_books: + return self._collect_search_results(seed_feed, cleaned) + else: + return self._local_search(cleaned, seed_feed=seed_feed) + + def _collect_search_results( + self, + seed_feed: OPDSFeed, + query: str, + *, + max_pages: int = 40, + ) -> OPDSFeed: + normalized = (query or "").strip() + if not normalized: + return seed_feed + seen_ids: Set[str] = set() + collected: List[OPDSEntry] = [] + for page in self._iter_paginated_feeds(seed_feed, max_pages=max_pages): + filtered = self._filter_feed_entries(page, normalized) + for entry in filtered.entries: + entry_id = (entry.id or "").strip() + if entry_id: + if entry_id in seen_ids: + continue + seen_ids.add(entry_id) + collected.append(entry) + return dataclasses.replace(seed_feed, entries=collected) + + def _iter_paginated_feeds(self, seed_feed: OPDSFeed, *, max_pages: int = 40) -> Iterator[OPDSFeed]: + yield seed_feed + next_link = self._resolve_link(seed_feed.links, "next") + visited: Set[str] = set() + pages_examined = 0 + while next_link and pages_examined < max_pages: + href = (next_link.href or "").strip() + if not href: + break + absolute = self._make_url(href) + if absolute in visited: + break + visited.add(absolute) + pages_examined += 1 + try: + page = self.fetch_feed(absolute) + except CalibreOPDSError: + break + yield page + next_link = self._resolve_link(page.links, "next") + + @staticmethod + def _merge_feed_entries(primary: OPDSFeed, secondary: OPDSFeed) -> OPDSFeed: + if primary is secondary or not secondary.entries: + return primary + seen_ids: Set[str] = set() + combined: List[OPDSEntry] = list(primary.entries) + for entry in primary.entries: + entry_id = (entry.id or "").strip() + if entry_id: + seen_ids.add(entry_id) + for entry in secondary.entries: + entry_id = (entry.id or "").strip() + if entry_id and entry_id in seen_ids: + continue + if entry_id: + seen_ids.add(entry_id) + combined.append(entry) + return dataclasses.replace(primary, entries=combined) + + def _local_search( + self, + query: str, + *, + seed_feed: Optional[OPDSFeed] = None, + max_pages: int = 40, + ) -> OPDSFeed: + normalized = (query or "").strip() + if not normalized: + return seed_feed or self.fetch_feed() + tokens = [token for token in re.split(r"\s+", normalized.lower()) if token] + if not tokens: + return seed_feed or self.fetch_feed() + + start_feed = seed_feed or self.fetch_feed() + collected: List[OPDSEntry] = [] + seen_match_ids: Set[str] = set() + + def add_matches(feed: OPDSFeed) -> None: + filtered = self._filter_feed_entries(feed, normalized) + for entry in filtered.entries: + entry_id = (entry.id or "").strip() + if entry_id: + if entry_id in seen_match_ids: + continue + seen_match_ids.add(entry_id) + collected.append(entry) + + add_matches(start_feed) + + queue: Deque[str] = deque() + queued: Set[str] = set() + visited: Set[str] = set() + + def is_navigation_link(rel_hint: Optional[str], link: OPDSLink) -> bool: + rel_candidates: List[str] = [] + if rel_hint: + rel_candidates.append(rel_hint) + if link.rel and link.rel not in rel_candidates: + rel_candidates.append(link.rel) + rel_candidates = [(rel or "").strip().lower() for rel in rel_candidates if rel] + link_type = (link.type or "").strip().lower() + if link_type and "opds-catalog" in link_type: + return True + for rel_value in rel_candidates: + if not rel_value: + continue + if "acquisition" in rel_value: + return False + if rel_value == "self": + continue + if rel_value == "next": + return True + if rel_value in {"start", "up", "down"}: + return True + if rel_value.endswith("navigation") or rel_value.endswith("collection"): + return True + if rel_value.startswith("http://opds-spec.org/"): + if rel_value.startswith("http://opds-spec.org/group") or rel_value.startswith( + "http://opds-spec.org/sort" + ): + return True + if rel_value.endswith("navigation") or rel_value.endswith("collection"): + return True + return False + + def enqueue_link(link: OPDSLink, rel_hint: Optional[str] = None) -> None: + if not is_navigation_link(rel_hint, link): + return + href = (link.href or "").strip() + if not href: + return + absolute = self._make_url(href) + if absolute in queued or absolute in visited: + return + queued.add(absolute) + queue.append(absolute) + + for rel_key, link in (start_feed.links or {}).items(): + enqueue_link(link, rel_key) + for entry in start_feed.entries: + for link in entry.links: + enqueue_link(link, link.rel) + + pages_examined = 0 + while queue and pages_examined < max_pages: + href = queue.popleft() + if href in visited: + continue + visited.add(href) + pages_examined += 1 + try: + feed = self.fetch_feed(href) + except CalibreOPDSError: + continue + add_matches(feed) + for rel_key, link in (feed.links or {}).items(): + enqueue_link(link, rel_key) + for entry in feed.entries: + for link in entry.links: + enqueue_link(link, link.rel) + if collected: + return dataclasses.replace(start_feed, entries=collected) + return dataclasses.replace(start_feed, entries=[]) + + def download(self, href: str) -> DownloadedResource: + if not href: + raise ValueError("Download link missing") + target = self._make_url(href) + try: + with self._open_client() as client: + response = client.get(target, follow_redirects=True) + response.raise_for_status() + except httpx.HTTPStatusError as exc: # pragma: no cover - thin wrapper + raise CalibreOPDSError( + f"Download failed with status {exc.response.status_code}" + ) from exc + except httpx.HTTPError as exc: # pragma: no cover - thin wrapper + raise CalibreOPDSError(f"Download failed: {exc}") from exc + + mime_type = response.headers.get("Content-Type", "application/octet-stream").split(";")[0].strip() + filename = self._deduce_filename(response, target, mime_type) + return DownloadedResource(filename=filename, mime_type=mime_type, content=response.content) + + def _deduce_filename(self, response: httpx.Response, url: str, mime_type: str) -> str: + header = response.headers.get("Content-Disposition", "") + match = re.search(r'filename="?([^";]+)"?', header) + if match: + candidate = match.group(1).strip() + if candidate: + return candidate + parsed = urlparse(url) + stem = (parsed.path or "").strip("/").split("/")[-1] + if not stem: + stem = "download" + if "." not in stem: + extension = self._extension_for_mime(mime_type) + if extension: + stem = f"{stem}{extension}" + return stem + + @staticmethod + def _extension_for_mime(mime_type: str) -> str: + normalized = mime_type.lower() + if normalized in _EPUB_MIME_TYPES: + return ".epub" + if normalized == "application/pdf": + return ".pdf" + if normalized in {"text/plain", "text/html"}: + return ".txt" + return "" + + def _parse_feed(self, xml_payload: str, *, base_url: str) -> OPDSFeed: + try: + root = ET.fromstring(xml_payload) + except ET.ParseError as exc: + raise CalibreOPDSError(f"Unable to parse OPDS feed: {exc}") from exc + + feed_id = root.findtext("atom:id", default=None, namespaces=NS) + feed_title = root.findtext("atom:title", default=None, namespaces=NS) + links_list = self._extract_links(root.findall("atom:link", NS), base_url) + links = self._links_to_dict(links_list) + parsed_entries = [self._parse_entry(node, base_url) for node in root.findall("atom:entry", NS)] + entries: List[OPDSEntry] = [] + for entry in parsed_entries: + if entry.download and self._is_supported_download(entry.download): + entries.append(entry) + continue + if self._has_navigation_link(entry): + entries.append(entry) + return OPDSFeed(id=feed_id, title=feed_title, entries=entries, links=links) + + def _parse_entry(self, node: ET.Element, base_url: str) -> OPDSEntry: + entry_id = node.findtext("atom:id", default="", namespaces=NS).strip() + title = node.findtext("atom:title", default="Untitled", namespaces=NS).strip() or "Untitled" + + subtitle = ( + node.findtext("calibre_md:subtitle", default=None, namespaces=NS) + or node.findtext("calibre:subtitle", default=None, namespaces=NS) + or node.findtext("atom:subtitle", default=None, namespaces=NS) + ) + subtitle = self._strip_html(subtitle.strip()) if subtitle else None + + position_value = self._extract_position(node) + updated = node.findtext("atom:updated", default=None, namespaces=NS) + published = ( + node.findtext("dc:date", default=None, namespaces=NS) + or node.findtext("atom:published", default=None, namespaces=NS) + ) + if published: + published = published.strip() or None + + summary_text = ( + self._extract_text(node.find("atom:summary", NS)) + or self._extract_text(node.find("atom:content", NS)) + or self._extract_text(node.find("dc:description", NS)) + ) + summary_metadata: Dict[str, str] = {} + summary_body: Optional[str] = None + if summary_text: + summary_metadata, summary_body = self._split_summary_metadata(summary_text) + cleaned_summary = self._strip_html(summary_body or summary_text) + + authors: List[str] = [] + for author_node in node.findall("atom:author", NS): + name = author_node.findtext("atom:name", default="", namespaces=NS).strip() + if name: + authors.append(name) + if not authors: + creators = node.findall("dc:creator", NS) + for creator in creators: + value = (creator.text or "").strip() + if value: + authors.append(value) + + links = node.findall("atom:link", NS) + all_links = self._extract_links(links, base_url) + link_dict = self._links_to_dict(all_links) + download_link = self._select_download_link(all_links) + alternate_link = link_dict.get("alternate") + thumb_link = link_dict.get("http://opds-spec.org/image/thumbnail") or link_dict.get( + "thumbnail" + ) + + series_name = ( + node.findtext("calibre:series", default=None, namespaces=NS) + or node.findtext("calibre_md:series", default=None, namespaces=NS) + ) + if series_name: + series_name = series_name.strip() or None + + series_index_raw = ( + node.findtext("calibre:series_index", default=None, namespaces=NS) + or node.findtext("calibre_md:series_index", default=None, namespaces=NS) + ) + series_index: Optional[float] = None + if series_index_raw is not None: + text = str(series_index_raw).strip() + if text: + try: + series_index = float(text) + except ValueError: + match = re.search(r"\d+(?:\.\d+)?", text.replace(",", ".")) + if match: + try: + series_index = float(match.group(0)) + except ValueError: + series_index = None + + if series_name is None or series_index is None: + category_series_name, category_series_index = self._extract_series_from_categories( + node.findall("atom:category", NS), + authors=authors, + ) + if series_name is None and category_series_name: + series_name = category_series_name + if series_index is None and category_series_index is not None: + series_index = category_series_index + + if (series_name is None or series_index is None) and summary_text: + text_series_name, text_series_index = self._extract_series_from_text(summary_text) + if series_name is None and text_series_name: + series_name = text_series_name + if series_index is None and text_series_index is not None: + series_index = text_series_index + + tags_value = summary_metadata.get("TAGS") + tags = self._parse_tags(tags_value) if tags_value else [] + rating_value = summary_metadata.get("RATING") + rating, rating_max = self._parse_rating(rating_value) if rating_value else (None, None) + + return OPDSEntry( + id=entry_id or title, + title=title, + position=position_value, + authors=authors, + subtitle=subtitle, + updated=updated, + published=published, + summary=cleaned_summary, + download=download_link, + alternate=alternate_link, + thumbnail=thumb_link, + links=all_links, + series=series_name, + series_index=series_index, + tags=tags, + rating=rating, + rating_max=rating_max, + ) + + def _extract_series_from_categories( + self, + category_nodes: List[ET.Element], + *, + authors: Optional[List[str]] = None, + ) -> tuple[Optional[str], Optional[float]]: + name: Optional[str] = None + index: Optional[float] = None + author_set = {str(author).strip().casefold() for author in (authors or []) if str(author).strip()} + for category in category_nodes: + scheme = (category.attrib.get("scheme") or "").strip().lower() + label = (category.attrib.get("label") or "").strip() + term = (category.attrib.get("term") or "").strip() + values: List[str] = [] + if label: + values.append(label) + if term and term not in values: + values.append(term) + + # Be conservative: category schemes are often URLs and can contain unrelated substrings. + # Also, some catalog feeds incorrectly include author names in series-like categories. + is_series_hint = self._is_series_scheme(scheme) or any("series" in value.lower() for value in values if value) + if not is_series_hint: + continue + + for value in values: + if not value: + continue + candidate_name, candidate_index = self._parse_series_value(value) + if candidate_name and candidate_name.casefold() in author_set: + # Guardrail: avoid mapping the author name into series. + continue + if candidate_name and not name: + name = candidate_name + if candidate_index is not None and index is None: + index = candidate_index + if name and index is not None: + return name, index + return name, index + + @staticmethod + def _is_series_scheme(scheme: str) -> bool: + cleaned = (scheme or "").strip().lower() + if not cleaned: + return False + if "author" in cleaned: + return False + return bool(re.search(r"(^|[/#:\-])series([/#:\-]|$)", cleaned)) + + def _parse_series_value(self, value: str) -> tuple[Optional[str], Optional[float]]: + cleaned = re.sub(r"\s+", " ", value or "").strip() + if not cleaned: + return None, None + cleaned = _SERIES_PREFIX_RE.sub("", cleaned) + working = cleaned + number: Optional[float] = None + + bracket_match = _SERIES_NUMBER_BRACKET_RE.search(working) + if bracket_match: + number = self._coerce_series_index(bracket_match.group(1)) + start, end = bracket_match.span() + working = (working[:start] + working[end:]).strip() + + if number is None: + hash_match = _SERIES_NUMBER_HASH_RE.search(working) + if hash_match: + number = self._coerce_series_index(hash_match.group(1)) + start, end = hash_match.span() + working = (working[:start] + working[end:]).strip() + + if number is None: + book_match = _SERIES_NUMBER_BOOK_RE.search(working) + if book_match: + number = self._coerce_series_index(book_match.group(1)) + start, end = book_match.span() + working = (working[:start] + working[end:]).strip() + + name = working.strip(" -–—,:") + name = re.sub(r"\s+", " ", name).strip() + if not name: + name = None + return name, number + + @staticmethod + def _extract_text(node: Optional[ET.Element]) -> Optional[str]: + if node is None: + return None + # Prefer itertext to capture nested XHTML content + parts = list(node.itertext()) + if not parts: + return (node.text or "").strip() or None + combined = "".join(parts).strip() + return combined or None + + def _extract_series_from_text(self, text: str) -> tuple[Optional[str], Optional[float]]: + for line in text.splitlines(): + match = _SERIES_LINE_TEXT_RE.match(line) + if not match: + continue + candidate = match.group(1).strip() + if not candidate: + continue + name, number = self._parse_series_value(candidate) + if name or number is not None: + return name, number + return None, None + + def _split_summary_metadata(self, text: Optional[str]) -> tuple[Dict[str, str], Optional[str]]: + metadata: Dict[str, str] = {} + if text is None: + return metadata, None + lines = text.splitlines() + index = 0 + total = len(lines) + while index < total and not lines[index].strip(): + index += 1 + while index < total: + stripped = lines[index].strip() + if not stripped: + break + match = _SUMMARY_METADATA_LINE_RE.match(stripped) + if not match: + break + key = match.group(1).strip().upper() + value = match.group(2).strip() + if key and value: + metadata[key] = value + index += 1 + remainder = "\n".join(lines[index:]).strip() + return metadata, (remainder or None) + + @staticmethod + def _parse_tags(value: str) -> List[str]: + if not value: + return [] + tokens = re.split(r"[;,\n]\s*", value) + cleaned: List[str] = [] + seen: set[str] = set() + for token in tokens: + entry = token.strip() + if not entry: + continue + key = entry.casefold() + if key in seen: + continue + seen.add(key) + cleaned.append(entry) + return cleaned + + @staticmethod + def _parse_rating(value: str) -> tuple[Optional[float], Optional[float]]: + if not value: + return None, None + text = value.strip() + if not text: + return None, None + stars = text.count("★") + half = 0.5 if "½" in text else 0.0 + if stars or half: + rating = stars + half + return (rating if rating > 0 else None, 5.0) + match = re.search(r"\d+(?:\.\d+)?", text.replace(",", ".")) + if match: + try: + rating_value = float(match.group(0)) + except ValueError: + return None, None + return rating_value, 5.0 + return None, None + + @staticmethod + def _coerce_series_index(value: str) -> Optional[float]: + text = value.strip().replace(",", ".") + if not text: + return None + try: + return float(text) + except ValueError: + return None + + def _extract_position(self, node: ET.Element) -> Optional[int]: + candidates = [ + node.findtext("opds:position", default=None, namespaces=NS), + node.findtext("opds:groupPosition", default=None, namespaces=NS), + node.findtext("opds:order", default=None, namespaces=NS), + node.findtext("dc:identifier", default=None, namespaces=NS), + ] + for value in candidates: + if value is None: + continue + text = str(value).strip() + if not text: + continue + try: + return int(float(text)) + except (TypeError, ValueError): + continue + return None + + def _extract_links(self, link_nodes: List[ET.Element], base_url: str) -> List[OPDSLink]: + links: List[OPDSLink] = [] + for link in link_nodes: + href = link.attrib.get("href") + if not href: + continue + rel = link.attrib.get("rel") + link_type = link.attrib.get("type") + title = link.attrib.get("title") + base_for_join = base_url or self._base_url + absolute_href = urljoin(base_for_join, href) + links.append(OPDSLink(href=absolute_href, rel=rel, type=link_type, title=title)) + return links + + def _links_to_dict(self, links: List[OPDSLink]) -> Dict[str, OPDSLink]: + results: Dict[str, OPDSLink] = {} + for entry in links: + key = entry.rel or entry.href + if not key: + continue + + # Prioritize search links with template parameters + if key == "search" and key in results: + existing = results[key] + if "{searchTerms}" in (existing.href or ""): + continue + if "{searchTerms}" in (entry.href or ""): + results[key] = entry + continue + + results[key] = entry + return results + + @staticmethod + def _is_supported_download(link: OPDSLink) -> bool: + mime = (link.type or "").split(";")[0].strip().lower() + if mime in _SUPPORTED_DOWNLOAD_MIME_TYPES: + return True + href = (link.href or "").strip() + if not href: + return False + parsed_path = urlparse(href).path or "" + extension = PurePosixPath(parsed_path).suffix.lower() + return extension in _SUPPORTED_DOWNLOAD_EXTENSIONS + + @staticmethod + def _select_download_link(links: Mapping[str, OPDSLink] | Iterable[OPDSLink]) -> Optional[OPDSLink]: + if isinstance(links, Mapping): + iterable: List[OPDSLink] = list(links.values()) + else: + iterable = list(links) + supported = [link for link in iterable if CalibreOPDSClient._is_supported_download(link)] + best: Optional[OPDSLink] = None + for link in supported: + rel = (link.rel or "").lower() + if "acquisition" not in rel: + continue + mime = (link.type or "").lower() + if mime in _EPUB_MIME_TYPES: + return link + if best is None: + best = link + if best: + return best + if supported: + return supported[0] + # No valid acquisition-style link exposed + return None + + @staticmethod + def _resolve_link(links: Optional[Mapping[str, OPDSLink]], rel: str) -> Optional[OPDSLink]: + if not links: + return None + if rel in links: + return links[rel] + rel_lower = rel.lower() + for key, link in links.items(): + key_lower = (key or "").strip().lower() + if key_lower == rel_lower or key_lower.endswith(rel_lower): + return link + return None + + @staticmethod + def _is_navigation_link(link: OPDSLink) -> bool: + href = (link.href or "").strip() + if not href: + return False + rel = (link.rel or "").strip().lower() + link_type = (link.type or "").strip().lower() + if "acquisition" in rel: + return False + if rel == "self": + return False + if "opds-catalog" in link_type: + return True + if rel.endswith("navigation") or rel.endswith("collection"): + return True + if rel.startswith("http://opds-spec.org/sort") or rel.startswith("http://opds-spec.org/group"): + return True + return False + + @staticmethod + def _has_navigation_link(entry: OPDSEntry) -> bool: + return any(CalibreOPDSClient._is_navigation_link(link) for link in entry.links) + + @staticmethod + def _browse_mode_for_title(title: Optional[str]) -> str: + if not title: + return "generic" + lowered = title.lower() + if "author" in lowered: + return "author" + if "series" in lowered: + return "series" + if "title" in lowered or "book" in lowered: + return "title" + return "generic" + + @staticmethod + def _strip_leading_article(text: str) -> str: + working = text.strip() + lowered = working.lower() + for article in ("the ", "a ", "an "): + if lowered.startswith(article): + return working[len(article):].strip() + return working + + @staticmethod + def _alphabet_source(entry: OPDSEntry, mode: str) -> str: + if mode == "author" and entry.authors: + candidate = entry.authors[0] or "" + if "," in candidate: + return candidate.split(",", 1)[0].strip() + parts = candidate.split() + if len(parts) > 1: + return parts[-1].strip() + return candidate.strip() + if mode == "series" and entry.series: + return entry.series.strip() + if entry.title: + return entry.title.strip() + if entry.series: + return entry.series.strip() + for link in entry.links: + if link.title: + return link.title.strip() + return "" + + @staticmethod + def _normalize_text(text: str) -> str: + if not text: + return "" + # Normalize unicode characters to their base form (e.g. é -> e) + normalized = unicodedata.normalize('NFKD', text).encode('ASCII', 'ignore').decode('utf-8') + return normalized.lower().strip() + + @staticmethod + def _alphabet_letter_for_entry(entry: OPDSEntry, mode: str) -> Optional[str]: + source = CalibreOPDSClient._alphabet_source(entry, mode) + if not source: + return None + if mode == "title": + source = CalibreOPDSClient._strip_leading_article(source) + + # Normalize to handle accents (É -> E) + normalized_source = unicodedata.normalize('NFKD', source).encode('ASCII', 'ignore').decode('utf-8') + + cleaned = re.sub(r"^[^0-9A-Za-z]+", "", normalized_source) + if not cleaned: + return "#" + initial = cleaned[0] + if initial.isalpha(): + return initial.upper() + if initial.isdigit(): + return "#" + return "#" + + @staticmethod + def _entry_matches_query(entry: OPDSEntry, tokens: List[str]) -> bool: + if not tokens: + return True + search_fragments: List[str] = [] + + if entry.title: + search_fragments.append(CalibreOPDSClient._normalize_text(entry.title)) + if entry.series: + search_fragments.append(CalibreOPDSClient._normalize_text(entry.series)) + + for author in entry.authors: + cleaned = (author or "").strip() + if not cleaned: + continue + normalized_author = CalibreOPDSClient._normalize_text(cleaned) + search_fragments.append(normalized_author) + for part in re.split(r"[\s,]+", normalized_author): + part = part.strip() + if part: + search_fragments.append(part) + + if not search_fragments: + return False + + # Check if all tokens match at least one fragment + # Tokens are already normalized in _filter_feed_entries + return all(any(token in fragment for fragment in search_fragments) for token in tokens) + + def _filter_feed_entries(self, feed: OPDSFeed, query: str) -> OPDSFeed: + normalized_query = CalibreOPDSClient._normalize_text(query) + if not normalized_query: + return feed + tokens = [token for token in re.split(r"\s+", normalized_query) if token] + if not tokens: + return feed + + scored_entries = [] + for entry in feed.entries: + if not self._entry_matches_query(entry, tokens): + continue + score = self._calculate_match_score(entry, tokens) + # Require a minimum score to avoid weak matches (e.g. single word in summary) + if score >= 10: + scored_entries.append((score, entry)) + + # Sort by score descending + scored_entries.sort(key=lambda x: x[0], reverse=True) + + filtered = [e for s, e in scored_entries] + return dataclasses.replace(feed, entries=filtered) + + def _estimate_letter_position(self, letter: str) -> float: + """Estimate the relative position (0.0-1.0) of a letter in an alphabetical list.""" + if letter == "#": + return 0.0 + if not letter or not letter.isalpha(): + return 0.0 + + # Approximate cumulative distribution of starting letters in English book titles + # This is a heuristic to jump closer to the target + weights = { + 'A': 0.00, 'B': 0.08, 'C': 0.15, 'D': 0.22, 'E': 0.28, + 'F': 0.33, 'G': 0.38, 'H': 0.43, 'I': 0.49, 'J': 0.53, + 'K': 0.55, 'L': 0.58, 'M': 0.63, 'N': 0.68, 'O': 0.71, + 'P': 0.75, 'Q': 0.80, 'R': 0.81, 'S': 0.85, 'T': 0.92, + 'U': 0.97, 'V': 0.98, 'W': 0.99, 'X': 0.995, 'Y': 0.997, 'Z': 0.999 + } + return weights.get(letter.upper(), 0.0) + + def _attempt_smart_jump(self, feed: OPDSFeed, letter: str) -> Optional[OPDSFeed]: + """ + Attempt to jump to a page closer to the target letter by analyzing pagination links. + Returns a new OPDSFeed if a jump was successful, or None. + """ + first_link = self._resolve_link(feed.links, "first") + last_link = self._resolve_link(feed.links, "last") + next_link = self._resolve_link(feed.links, "next") + + if not (first_link and last_link and next_link): + return None + + # Try to extract offsets from URLs to determine page size and total items + # Common Calibre pattern: .../offset/0, .../offset/50 + def extract_offset(href: str) -> Optional[int]: + match = re.search(r"/(\d+)/?$", href) + if match: + return int(match.group(1)) + # Try query param + parsed = urlparse(href) + qs = dict(pair.split('=') for pair in parsed.query.split('&') if '=' in pair) + if 'offset' in qs: + try: + return int(qs['offset']) + except ValueError: + pass + return None + + start_offset = extract_offset(first_link.href) + next_offset = extract_offset(next_link.href) + last_offset = extract_offset(last_link.href) + + if start_offset is None or next_offset is None or last_offset is None: + return None + + page_size = next_offset - start_offset + if page_size <= 0: + return None + + # Estimate total items (last_offset is the start of the last page) + # We assume the last page is roughly half full for estimation + total_items = last_offset + (page_size // 2) + + target_ratio = self._estimate_letter_position(letter) + # Aim slightly early (subtract 1-2 pages worth) to ensure we don't miss the start + target_offset = int(total_items * target_ratio) + target_offset = max(0, target_offset - (page_size * 2)) + + # Round to nearest page boundary + target_offset = (target_offset // page_size) * page_size + + # If the jump is too small (e.g. we are already near the start), don't bother + if target_offset < (page_size * 3): + return None + + # Construct the new URL + # We assume the URL structure is consistent and we can just replace the offset + # This is risky but works for standard Calibre OPDS + base_href = first_link.href + if str(start_offset) in base_href: + # Path based replacement + # Replace the last occurrence of the offset + parts = base_href.rsplit(str(start_offset), 1) + if len(parts) == 2: + new_href = f"{parts[0]}{target_offset}{parts[1]}" + try: + return self.fetch_feed(new_href) + except Exception: + return None + + return None + + def browse_letter( + self, + letter: str, + *, + start_href: Optional[str] = None, + max_pages: int = 40, + ) -> OPDSFeed: + normalized = (letter or "").strip() + if not normalized: + return self.fetch_feed(start_href) + key = normalized.upper() + if key in {"ALL", "*"}: + return self.fetch_feed(start_href) + if key in {"0-9", "NUMERIC"}: + key = "#" + if len(key) > 1: + key = key[0] + if key != "#" and not key.isalpha(): + key = "#" + base_feed = self.fetch_feed(start_href) + + # Ensure we start from the beginning of the feed if possible + first_link = self._resolve_link(base_feed.links, "first") or self._resolve_link(base_feed.links, "start") + if first_link and first_link.href: + try: + # Only switch if the href is different to avoid redundant fetch + if not start_href or first_link.href != start_href: + base_feed = self.fetch_feed(first_link.href) + except CalibreOPDSError: + pass + + mode = self._browse_mode_for_title(base_feed.title) + + def letter_matches(entry: OPDSEntry, active_mode: str) -> bool: + letter_value = self._alphabet_letter_for_entry(entry, active_mode) + if not letter_value: + return False + if key == "#": + return letter_value == "#" + return letter_value == key + + collected: List[OPDSEntry] = [] + seen_ids: Set[str] = set() + letter_href: Optional[str] = None + + def add_entry(entry: OPDSEntry) -> None: + entry_id = (entry.id or "").strip() + if entry_id: + if entry_id in seen_ids: + return + seen_ids.add(entry_id) + collected.append(entry) + + # Check the first page for navigation links before attempting any jumps + # This handles the case where "By Title" has "A", "B", "C" folders + has_nav_links = False + for entry in base_feed.entries: + if self._has_navigation_link(entry): + has_nav_links = True + if letter_matches(entry, mode): + for link in entry.links: + if self._is_navigation_link(link): + href = (link.href or "").strip() + if href: + letter_href = href + break + if letter_href: + break + + # If we didn't find a direct link, and it looks like a flat list (no nav links matching criteria), + # try to jump closer to the target letter if we are in a sorted mode + if not letter_href and mode in {"title", "author", "series"}: + jump_feed = self._attempt_smart_jump(base_feed, key) + if jump_feed: + base_feed = jump_feed + + for page in self._iter_paginated_feeds(base_feed, max_pages=max_pages): + for entry in page.entries: + if not letter_matches(entry, mode): + continue + if self._has_navigation_link(entry): + if letter_href is None: + for link in entry.links: + if self._is_navigation_link(link): + href = (link.href or "").strip() + if href: + letter_href = href + break + else: + add_entry(entry) + + letter_feed: Optional[OPDSFeed] = None + if letter_href: + try: + letter_feed = self.fetch_feed(letter_href) + except CalibreOPDSError: + letter_feed = None + else: + letter_mode = self._browse_mode_for_title(letter_feed.title) + for page in self._iter_paginated_feeds(letter_feed, max_pages=max_pages): + for entry in page.entries: + if not letter_matches(entry, letter_mode): + continue + if self._has_navigation_link(entry): + continue + add_entry(entry) + + template = letter_feed or base_feed + if collected: + return dataclasses.replace(template, entries=collected) + return dataclasses.replace(template, entries=[]) + + def _resolve_search_url(self, feed: OPDSFeed, query: str) -> Optional[str]: + link = self._resolve_link(feed.links, "search") + if not link: + link = self._resolve_link(feed.links, "http://opds-spec.org/search") + + if not link or not link.href: + return None + + href = link.href.strip() + if "{searchTerms}" in href: + return href.replace("{searchTerms}", quote(query)) + + return href + + def _calculate_match_score(self, entry: OPDSEntry, tokens: List[str]) -> int: + if not tokens: + return 0 + + score = 0 + + # Prepare normalized text + title = self._normalize_text(entry.title) + authors = [self._normalize_text(a) for a in entry.authors] + series = self._normalize_text(entry.series) if entry.series else "" + summary = self._normalize_text(entry.summary) if entry.summary else "" + tags = [self._normalize_text(t) for t in entry.tags] + + # 1. Exact/Phrase matches + query_phrase = " ".join(tokens) + if query_phrase == title: + score += 1000 + elif query_phrase in title: + score += 500 + + for author in authors: + if query_phrase in author: + score += 300 + + if query_phrase in series: + score += 200 + + for tag in tags: + if query_phrase == tag: + score += 100 + elif query_phrase in tag: + score += 50 + + # 2. Token matches + # Filter out stop words unless the query is only stop words + significant_tokens = [t for t in tokens if t not in _STOP_WORDS] + if not significant_tokens: + significant_tokens = tokens + + for token in significant_tokens: + token_score = 0 + # Use regex for word boundary matching + # Escape token to handle special chars + token_regex = r"\b" + re.escape(token) + r"\b" + + # Title: High weight + if re.search(token_regex, title): + token_score = max(token_score, 50) + elif token in title: + token_score = max(token_score, 5) + + # Author: Medium-High weight + for author in authors: + if re.search(token_regex, author): + token_score = max(token_score, 40) + elif token in author: + token_score = max(token_score, 5) + + # Series: Medium weight + if token in series: + if re.search(token_regex, series): + token_score = max(token_score, 30) + else: + token_score = max(token_score, 5) + + # Tags: Medium weight + for tag in tags: + if re.search(token_regex, tag): + token_score = max(token_score, 30) + elif token in tag: + token_score = max(token_score, 5) + + # Summary: Low weight + if token in summary: + if re.search(token_regex, summary): + # Only add if not found elsewhere? Or just add small amount? + if token_score == 0: + token_score = 15 + else: + token_score += 5 # Small boost if also in description + elif token_score == 0: + token_score = 2 # Very low for substring in summary + + score += token_score + + return score diff --git a/abogen/kokoro_text_normalization.py b/abogen/kokoro_text_normalization.py new file mode 100644 index 0000000..17665ad --- /dev/null +++ b/abogen/kokoro_text_normalization.py @@ -0,0 +1,2378 @@ +from __future__ import annotations + +import json +import re +import unicodedata +import os +import locale +from fractions import Fraction +from dataclasses import dataclass, field +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterable, + List, + Mapping, + Optional, + Sequence, + Tuple, +) +import logging + +logger = logging.getLogger(__name__) + +try: # pragma: no cover - optional dependency guard + from num2words import num2words +except ImportError: + num2words = None + logger.warning( + "num2words library not found. Number normalization will be disabled." + ) +except Exception as e: # pragma: no cover - graceful degradation + num2words = None + logger.error(f"Failed to import num2words: {e}") + +HAS_NUM2WORDS = num2words is not None + +if TYPE_CHECKING: # pragma: no cover - type checking only + from abogen.llm_client import LLMCompletion + +from abogen.spacy_contraction_resolver import resolve_ambiguous_contractions + +# ---------- Contraction Category Defaults ---------- + +CONTRACTION_CATEGORY_DEFAULTS: Dict[str, bool] = { + "contraction_aux_be": True, + "contraction_aux_have": True, + "contraction_modal_will": True, + "contraction_modal_would": True, + "contraction_negation_not": True, + "contraction_let_us": True, +} + +# ---------- Configuration Dataclass ---------- + + +@dataclass +class ApostropheConfig: + contraction_mode: str = "expand" # expand|collapse|keep + possessive_mode: str = "keep" # keep|collapse + plural_possessive_mode: str = "collapse" # keep|collapse + irregular_possessive_mode: str = ( + "keep" # keep|expand (expand just means keep or add hints; modify if needed) + ) + sibilant_possessive_mode: str = "mark" # keep|mark|approx + fantasy_mode: str = "keep" # keep|mark|collapse_internal + acronym_possessive_mode: str = "keep" # keep|collapse_add_s + decades_mode: str = "expand" # keep|expand + leading_elision_mode: str = "expand" # keep|expand + ambiguous_past_modal_mode: str = ( + "contextual" # keep|expand_prefer_would|expand_prefer_had|contextual + ) + add_phoneme_hints: bool = True # Whether to emit markers like ‹IZ› + fantasy_marker: str = "‹FAP›" # Marker inserted if fantasy_mode == mark + sibilant_iz_marker: str = "‹IZ›" # Marker for /ɪz/ insertion + joiner: str = "" # Replacement used when collapsing internal apostrophes + lowercase_for_matching: bool = ( + True # Normalize to lower for rule matching (not output) + ) + protect_cultural_names: bool = True # Always keep O'Brien, D'Angelo, etc. + convert_numbers: bool = True # Convert grouped numbers such as 12,500 to words + convert_currency: bool = True # Convert currency symbols to words + remove_footnotes: bool = True # Remove footnote indicators + number_lang: str = "en" # num2words language code + year_pronunciation_mode: str = "american" # off|american (extend if needed) + contraction_categories: Dict[str, bool] = field( + default_factory=lambda: dict(CONTRACTION_CATEGORY_DEFAULTS) + ) + + def is_contraction_enabled(self, category: str) -> bool: + return self.contraction_categories.get(category, True) + + +# ---------- Dictionaries / Patterns ---------- + +# Common contraction expansions (type + expansion words) +CONTRACTION_LEXICON: Dict[str, Tuple[str, Tuple[str, ...]]] = { + "let's": ("contraction_let_us", ("let", "us")), + "can't": ("contraction_negation_not", ("can", "not")), + "won't": ("contraction_negation_not", ("will", "not")), + "don't": ("contraction_negation_not", ("do", "not")), + "doesn't": ("contraction_negation_not", ("does", "not")), + "didn't": ("contraction_negation_not", ("did", "not")), + "isn't": ("contraction_negation_not", ("is", "not")), + "aren't": ("contraction_negation_not", ("are", "not")), + "wasn't": ("contraction_negation_not", ("was", "not")), + "weren't": ("contraction_negation_not", ("were", "not")), + "haven't": ("contraction_negation_not", ("have", "not")), + "hasn't": ("contraction_negation_not", ("has", "not")), + "hadn't": ("contraction_negation_not", ("had", "not")), + "couldn't": ("contraction_negation_not", ("could", "not")), + "shouldn't": ("contraction_negation_not", ("should", "not")), + "wouldn't": ("contraction_negation_not", ("would", "not")), + "mustn't": ("contraction_negation_not", ("must", "not")), + "mightn't": ("contraction_negation_not", ("might", "not")), + "shan't": ("contraction_negation_not", ("shall", "not")), +} + +SUFFIX_CONTRACTION_RULES: Tuple[Tuple[str, str, str], ...] = ( + ("'ll", "will", "contraction_modal_will"), + ("'re", "are", "contraction_aux_be"), + ("'ve", "have", "contraction_aux_have"), +) + +SUFFIX_CONTRACTION_BASES: Dict[str, Tuple[str, ...]] = { + "'m": ("i",), +} + +# For ambiguous 'd and 's we handle separately +_NUMBER_WITH_GROUP_RE = re.compile(r"(?{_NUMBER_CORE_PATTERN})(?P\s*[{_NUMBER_RANGE_CLASS}]\s*)(?P{_NUMBER_CORE_PATTERN})(?![\w{_NUMBER_RANGE_CLASS}/])" +) +_NUMBER_SPACE_RANGE_RE = re.compile( + rf"(?{_NUMBER_CORE_PATTERN})(?P\s+)(?P{_NUMBER_CORE_PATTERN})(?![\w{_NUMBER_RANGE_CLASS}/])" +) +_FRACTION_SLASHES = "/⁄" +_FRACTION_SLASH_CLASS = re.escape(_FRACTION_SLASHES) +_FRACTION_RE = re.compile( + rf"(?-?\d+)\s*[{_FRACTION_SLASH_CLASS}]\s*(?P-?\d+)(?![\w{_FRACTION_SLASH_CLASS}])" +) + +_CURRENCY_RE = re.compile( + r"(?P[$£€¥])\s*(?P\d{1,3}(?:,\d{3})*(?:\.\d+)?)(?:\s+(?Phundred|thousand|million|billion|trillion|quadrillion))?(?!\d)", + re.IGNORECASE, +) + +_URL_RE = re.compile( + r"(https?://)?(www\.)?(?P[a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)+)(/[^\s]*)?" +) +_FOOTNOTE_RE = re.compile(r"([a-zA-Z]+)(\d+)") +_BRACKET_FOOTNOTE_RE = re.compile(r"\[\d+\]") + +_ISO_DATE_RE = re.compile( + r"\b(?P\d{4})[/-](?P\d{1,2})[/-](?P\d{1,2})\b" +) +_MDY_DATE_RE = re.compile( + r"\b(?PJan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\.?\s+" + r"(?P\d{1,2})(?:st|nd|rd|th)?\s*,\s*(?P\d{4})\b", + re.IGNORECASE, +) + +_TIME_RE = re.compile( + r"\b(?P\d{1,2})(?::(?P\d{2}))?\s*(?Pa\.?m\.?|p\.?m\.?)\b", + re.IGNORECASE, +) + +_ADDRESS_ABBR_RE = re.compile( + r"(?P\b\w+\s+)(?PSt|Rd|Ave|Blvd|Ln|Dr)\.(?=\s*(?:,|\.|!|\?|$))" +) + +_MONTH_MAP = { + "jan": "January", + "january": "January", + "feb": "February", + "february": "February", + "mar": "March", + "march": "March", + "apr": "April", + "april": "April", + "may": "May", + "jun": "June", + "june": "June", + "jul": "July", + "july": "July", + "aug": "August", + "august": "August", + "sep": "September", + "sept": "September", + "september": "September", + "oct": "October", + "october": "October", + "nov": "November", + "november": "November", + "dec": "December", + "december": "December", +} + + +def _is_us_locale() -> bool: + for key in ("LC_ALL", "LC_TIME", "LANG"): + value = os.environ.get(key) + if value and "en_US" in value: + return True + try: + loc = locale.getlocale(locale.LC_TIME) + if loc and loc[0] and "en_US" in loc[0]: + return True + except Exception: + pass + return False + + +def _year_to_words_american(value: int, language: str) -> str: + if language.lower().startswith("en") and 2000 <= value <= 2099: + if value == 2000: + return "two thousand" + if 2001 <= value <= 2009: + tail = _DIGIT_WORDS[value % 10] + return f"two thousand {tail}" + + first_two = value // 100 + last_two = value % 100 + first_words = _int_to_words(first_two, language) or "twenty" + if last_two == 0: + return f"{first_words} hundred" + if last_two < 10: + return f"{first_words} oh {_DIGIT_WORDS[last_two]}" + last_words = _int_to_words(last_two, language) + return f"{first_words} {last_words or last_two}" + + words = _int_to_words(value, language) + return words or str(value) + + +def _normalize_dates(text: str, language: str) -> str: + us = _is_us_locale() + + def _format_iso(match: re.Match[str]) -> str: + year = int(match.group("year")) + month = int(match.group("month")) + day = int(match.group("day")) + if not (1 <= month <= 12 and 1 <= day <= 31): + return match.group(0) + month_name = [ + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", + ][month - 1] + ordinal = _int_to_ordinal_words(day, language) or str(day) + year_words = _year_to_words_american(year, language) + return ( + f"{month_name} {ordinal}, {year_words}" + if us + else f"{ordinal} {month_name} {year_words}" + ) + + def _format_mdy(match: re.Match[str]) -> str: + month_raw = str(match.group("month") or "").strip().lower().rstrip(".") + month_name = _MONTH_MAP.get(month_raw) + if not month_name: + return match.group(0) + day = int(match.group("day")) + year = int(match.group("year")) + ordinal = _int_to_ordinal_words(day, language) or str(day) + year_words = _year_to_words_american(year, language) + return ( + f"{month_name} {ordinal}, {year_words}" + if us + else f"{ordinal} {month_name} {year_words}" + ) + + out = _ISO_DATE_RE.sub(_format_iso, text) + out = _MDY_DATE_RE.sub(_format_mdy, out) + return out + + +def _normalize_times(text: str) -> str: + def _replace(match: re.Match[str]) -> str: + hour = match.group("hour") + minute = match.group("minute") + meridian = match.group("meridian").lower().replace(".", "") + if minute: + return f"{hour}:{minute} {meridian}" + return f"{hour} {meridian}" + + return _TIME_RE.sub(_replace, text) + + +def _normalize_address_abbreviations(text: str) -> str: + mapping = { + "st": "street", + "rd": "road", + "ave": "avenue", + "blvd": "boulevard", + "ln": "lane", + "dr": "drive", + } + + def _replace(match: re.Match[str]) -> str: + abbr = match.group("abbr") + full = mapping.get(abbr.lower()) + if not full: + return match.group(0) + return match.group("prefix") + _match_casing(abbr, full) + + return _ADDRESS_ABBR_RE.sub(_replace, text) + + +def _normalize_internet_slang(text: str) -> str: + mapping = { + "pls": "please", + "plz": "please", + } + + def _replace(match: re.Match[str]) -> str: + token = match.group(0) + replacement = mapping.get(token.lower()) + if not replacement: + return token + return _match_casing(token, replacement) + + return re.sub(r"\b(?:pls|plz)\b", _replace, text, flags=re.IGNORECASE) + + +_DECIMAL_NUMBER_RE = re.compile( + rf"(?-?(?:\d{{1,3}}(?:,\d{{3}})+|\d+)\.(?P\d+))(?![\w{_NUMBER_RANGE_CLASS}/])" +) +_PLAIN_NUMBER_RE = re.compile( + rf"(?{_NUMBER_CORE_PATTERN})(?![\w{_NUMBER_RANGE_CLASS}/])" +) + +_DIGIT_WORDS = ( + "zero", + "one", + "two", + "three", + "four", + "five", + "six", + "seven", + "eight", + "nine", +) + + +def _int_to_words(value: int, language: str) -> Optional[str]: + """Convert integer to spelled-out words using configured language.""" + if num2words is None: + return None + + try: + words = num2words(abs(value), lang=language) + except Exception: # pragma: no cover - unsupported locale + return None + + if value < 0: + return f"minus {words}" + return words + + +def _int_to_ordinal_words(value: int, language: str) -> Optional[str]: + if num2words is not None: + try: + return num2words(value, lang=language, ordinal=True) + except Exception: # pragma: no cover - unsupported locale + return None + + if language.lower().startswith("en"): + ordinals = { + 1: "first", + 2: "second", + 3: "third", + 4: "fourth", + 5: "fifth", + 6: "sixth", + 7: "seventh", + 8: "eighth", + 9: "ninth", + 10: "tenth", + 11: "eleventh", + 12: "twelfth", + 13: "thirteenth", + 14: "fourteenth", + 15: "fifteenth", + 16: "sixteenth", + 17: "seventeenth", + 18: "eighteenth", + 19: "nineteenth", + 20: "twentieth", + 21: "twenty-first", + 22: "twenty-second", + 23: "twenty-third", + 24: "twenty-fourth", + 25: "twenty-fifth", + 26: "twenty-sixth", + 27: "twenty-seventh", + 28: "twenty-eighth", + 29: "twenty-ninth", + 30: "thirtieth", + 31: "thirty-first", + } + return ordinals.get(int(value)) + + return None + + +def _pluralize_fraction_word(base: str) -> str: + if base == "half": + return "halves" + if base == "calf": # defensive; unlikely but keeps pattern predictable + return "calves" + if base.endswith("f"): + return base[:-1] + "ves" + if base.endswith("fe"): + return base[:-2] + "ves" + return base + "s" + + +def _fraction_denominator_word( + denominator: int, numerator: int, language: str +) -> Optional[str]: + """Return spoken form for fraction denominator respecting plurality.""" + if denominator == 0: + return None + + numerator_abs = abs(numerator) + if denominator == 1: + return "" + if denominator == 2: + return "half" if numerator_abs == 1 else "halves" + if denominator == 4: + return "quarter" if numerator_abs == 1 else "quarters" + + base = _int_to_ordinal_words(denominator, language) + if base is None: + return None + if numerator_abs == 1: + return base + return _pluralize_fraction_word(base) + + +def _format_fraction_words( + numerator: int, denominator: int, language: str +) -> Optional[str]: + """Return spoken representation of a simple fraction.""" + if denominator == 0: + return None + + fraction = Fraction(numerator, denominator) + num = fraction.numerator + den = fraction.denominator + + if abs(den) > 100: + return None + + numerator_words = _int_to_words(abs(num), language) + if numerator_words is None: + return None + + denom_word = _fraction_denominator_word(den, num, language) + if denom_word is None: + return None + + if denom_word: + if num < 0: + numerator_words = f"minus {numerator_words}" + return f"{numerator_words} {denom_word}".strip() + + # If denominator collapses to 1, just speak the integer value. + spoken = _int_to_words(num, language) + return spoken + + +def _replace_number_range(match: re.Match[str], language: str) -> str: + left_raw = match.group("left") + right_raw = match.group("right") + left = _coerce_int_token(left_raw) + right = _coerce_int_token(right_raw) + if left is None or right is None: + return match.group(0) + + left_words = _int_to_words(left, language) + right_words = _int_to_words(right, language) + if not left_words or not right_words: + return match.group(0) + + return f"{left_words} to {right_words}" + + +def _replace_space_separated_range(match: re.Match[str], language: str) -> str: + left_raw = match.group("left") + right_raw = match.group("right") + left = _coerce_int_token(left_raw) + right = _coerce_int_token(right_raw) + if left is None or right is None: + return match.group(0) + + left_words = _int_to_words(left, language) + right_words = _int_to_words(right, language) + if not left_words or not right_words: + return match.group(0) + + return f"{left_words} to {right_words}" + + +def _replace_fraction(match: re.Match[str], language: str) -> str: + numerator_raw = match.group("numerator") + denominator_raw = match.group("denominator") + try: + numerator = int(numerator_raw) + denominator = int(denominator_raw) + except ValueError: + return match.group(0) + + spoken = _format_fraction_words(numerator, denominator, language) + if not spoken: + return match.group(0) + return spoken + + +def _coerce_int_token(token: str) -> Optional[int]: + if token is None: + return None + cleaned = token.replace(",", "").strip() + if not cleaned or cleaned in {"-", "+"}: + return None + try: + return int(cleaned) + except ValueError: + return None + + +AMBIGUOUS_D_BASES = {"i", "you", "he", "she", "we", "they"} +AMBIGUOUS_S_BASES = { + "it", + "that", + "what", + "where", + "who", + "when", + "how", + "there", + "here", + "he", + "she", + "we", + "they", + "you", +} + + +def _is_ambiguous_d(token: str) -> bool: + low = token.lower() + return low.endswith("'d") and low[:-2] in AMBIGUOUS_D_BASES + + +def _is_ambiguous_s(token: str) -> bool: + low = token.lower() + return low.endswith("'s") and low[:-2] in AMBIGUOUS_S_BASES + + +# Irregular possessives that are not formed by simple + 's logic +IRREGULAR_POSSESSIVES = { + "children's": "children's", + "men's": "men's", + "women's": "women's", + "people's": "people's", + "geese's": "geese's", + "mouse's": "mouse's", # singular irregular +} + +SIBILANT_END_RE = re.compile(r"(?:[sxz]|(?:ch|sh))$", re.IGNORECASE) + +DECADE_RE = re.compile(r"^'\d0s$", re.IGNORECASE) # '90s, '80s +LEADING_ELISION = { + "'tis": "it is", + "'twas": "it was", + "'cause": "because", + "'em": "them", + "'round": "around", + "'til": "until", +} + +CULTURAL_NAME_PATTERNS = [ + re.compile(r"^O'[A-Z][a-z]+$"), + re.compile(r"^D'[A-Z][a-z]+$"), + re.compile(r"^L'[A-Za-z].*$"), + re.compile(r"^Mc[A-Z].*$"), # not apostrophe, but often relevant (kept anyway) +] + +ACRONYM_POSSESSIVE_RE = re.compile(r"^[A-Z]{2,}'s$") + +INTERNAL_APOSTROPHE_RE = re.compile(r"[A-Za-z]'.+[A-Za-z]") # apostrophe not at edge + +# Capture contiguous runs of Unicode letters/digits/apostrophes/hyphens, otherwise fall back to +# single-character tokens (punctuation, symbols, etc.). +WORD_TOKEN_RE = re.compile( + r"[0-9A-Za-z'’\u00C0-\u1FFF\u2C00-\uD7FF\-]+|[^0-9A-Za-z\s]", + re.UNICODE, +) + +APOSTROPHE_CHARS = "’`´ꞌʼ" + +TERMINAL_PUNCTUATION = {".", "?", "!", "…", ";", ":"} +CLOSING_PUNCTUATION = "\"'”’)]}»›" +ELLIPSIS_SUFFIXES = ("...", "…") +_LINE_SPLIT_RE = re.compile(r"(\n+)") + +TITLE_ABBREVIATIONS = { + "mr": "mister", + "mrs": "missus", + "ms": "miz", + "dr": "doctor", + "prof": "professor", + "rev": "reverend", + "gen": "general", + "sgt": "sergeant", +} + +SUFFIX_ABBREVIATIONS = { + "jr": "junior", + "sr": "senior", +} + +_TITLE_PATTERN = re.compile( + r"\b(?P" + + "|".join(sorted(TITLE_ABBREVIATIONS.keys(), key=len, reverse=True)) + + r")\.", + re.IGNORECASE, +) +_SUFFIX_PATTERN = re.compile( + r"\b(?P" + + "|".join(sorted(SUFFIX_ABBREVIATIONS.keys(), key=len, reverse=True)) + + r")\.", + re.IGNORECASE, +) + +# ---------- Utility Functions ---------- + + +def normalize_unicode_apostrophes(text: str) -> str: + text = unicodedata.normalize("NFKC", text) + for ch in APOSTROPHE_CHARS: + text = text.replace(ch, "'") + return text + + +def tokenize(text: str) -> List[str]: + # Simple tokenization preserving punctuation tokens + return WORD_TOKEN_RE.findall(text) + + +def tokenize_with_spans(text: str) -> List[Tuple[str, int, int]]: + return [ + (match.group(0), match.start(), match.end()) + for match in WORD_TOKEN_RE.finditer(text) + ] + + +def _cleanup_spacing(text: str) -> str: + if not text: + return text + + for marker in ("\ufeff", "\u200b", "\u200c", "\u200d", "\u2060"): + text = text.replace(marker, "") + + # Collapse spaces before closing punctuation. + text = re.sub(r"\s+([,.;:!?%])", r"\1", text) + text = re.sub(r"\s+([’\"”»›)\]\}])", r"\1", text) + + # Remove spaces directly after opening punctuation/quotes. + text = re.sub(r"([«‹“‘\"'(\[\{])\s+", r"\1", text) + + # Ensure spaces exist after sentence punctuation when followed by a word/quote. + text = re.sub(r"([,.;:!?%])(?![\s”'\"’»›)])", r"\1 ", text) + text = re.sub(r"([”\"’])(?![\s.,;:!?\"”’»›)])", r"\1 ", text) + + # Tighten hyphen/em dash spacing between word characters. + text = re.sub(r"(?<=\w)\s*([-–—])\s*(?=\w)", r"\1", text) + + # Normalize multiple spaces. + text = re.sub(r"\s{2,}", " ", text) + return text.strip() + + +_ROMAN_VALUE_MAP = { + "I": 1, + "V": 5, + "X": 10, + "L": 50, + "C": 100, + "D": 500, + "M": 1000, +} + +_ROMAN_COMPOSE_ORDER = [ + (1000, "M"), + (900, "CM"), + (500, "D"), + (400, "CD"), + (100, "C"), + (90, "XC"), + (50, "L"), + (40, "XL"), + (10, "X"), + (9, "IX"), + (5, "V"), + (4, "IV"), + (1, "I"), +] + +_ROMAN_PREFIX_RE = re.compile( + r"^(?P[IVXLCDM]+)(?P[\s\.:,;\-–—]*)", re.IGNORECASE +) + +_ROMAN_TOKEN_RE = re.compile(r"^[IVXLCDM]+$") +_ROMAN_CARDINAL_CONTEXTS = { + "act", + "appendix", + "article", + "battle", + "book", + "campaign", + "chapter", + "episode", + "film", + "final", + "fantasy", + "game", + "installment", + "lesson", + "level", + "mission", + "movement", + "opus", + "operation", + "page", + "part", + "phase", + "psalm", + "round", + "scene", + "season", + "section", + "series", + "song", + "super", + "bowl", + "stage", + "step", + "track", + "volume", + "war", + "world", + "world war", +} +_ROMAN_NAME_TITLES = { + "baron", + "baroness", + "captain", + "cardinal", + "count", + "countess", + "duchess", + "duke", + "emperor", + "empress", + "general", + "governor", + "king", + "lord", + "lady", + "major", + "pope", + "president", + "prince", + "princess", + "queen", + "saint", + "sir", +} +_ROMAN_NAME_CONNECTORS = { + "de", + "del", + "della", + "der", + "di", + "dos", + "la", + "le", + "of", + "the", + "van", + "von", +} +_ROMAN_BREAK_TOKENS = { + ",", + ".", + "!", + "?", + ";", + ":", + "(", + ")", + "[", + "]", + "{", + "}", + "—", + "–", + "-", + "'", + '"', +} + +_ROMAN_CONTEXT_PASSTHROUGH = {"-", "–", "—", ":"} +_ROMAN_CONTEXT_COMPOUND_RE = re.compile( + r"^(?P[A-Za-z]+)(?P[-–—:])(?P[IVXLCDM]+)$", + re.IGNORECASE, +) + + +def _roman_to_int(token: str) -> Optional[int]: + if not token: + return None + total = 0 + prev = 0 + token_upper = token.upper() + for char in reversed(token_upper): + value = _ROMAN_VALUE_MAP.get(char) + if value is None: + return None + if value < prev: + total -= value + else: + total += value + prev = value + if total <= 0: + return None + if _int_to_roman(total) != token_upper: + return None + return total + + +def _int_to_roman(value: int) -> str: + parts: List[str] = [] + remaining = value + for amount, symbol in _ROMAN_COMPOSE_ORDER: + while remaining >= amount: + parts.append(symbol) + remaining -= amount + return "".join(parts) + + +def _is_titlecase_token(token: str) -> bool: + cleaned = token.replace("'", "").replace("-", "") + if not cleaned: + return False + if not cleaned[0].isalpha() or not cleaned[0].isupper(): + return False + tail = cleaned[1:] + return not tail or tail.islower() + + +def _token_is_cardinal_context(token: str) -> bool: + return token.lower() in _ROMAN_CARDINAL_CONTEXTS + + +def _has_cardinal_leading_context( + tokens: Sequence[Tuple[str, int, int]], index: int +) -> bool: + j = index - 1 + while j >= 0: + token, *_ = tokens[j] + stripped = token.strip() + if not stripped: + j -= 1 + continue + lowered = stripped.lower() + if lowered in _ROMAN_CONTEXT_PASSTHROUGH: + j -= 1 + continue + if lowered in _ROMAN_BREAK_TOKENS: + return False + cleaned = lowered.strip("()[]{}\"'.,;!?") + if cleaned in _ROMAN_CARDINAL_CONTEXTS: + return True + if cleaned: + return False + j -= 1 + return False + + +def _should_render_ordinal( + tokens: Sequence[Tuple[str, int, int]], + index: int, + value: int, +) -> bool: + # Treat trailing roman numerals in name-like sequences as ordinals while + # leaving enumerated headings or series labels as cardinals. + if value <= 0: + return False + if index <= 0: + return False + + uppercase_count = 0 + title_count = 0 + j = index - 1 + while j >= 0: + token, *_ = tokens[j] + lowered = token.lower() + + if lowered in _ROMAN_CARDINAL_CONTEXTS: + return False + if lowered in _ROMAN_BREAK_TOKENS or token.isdigit(): + break + if lowered in _ROMAN_NAME_CONNECTORS: + j -= 1 + continue + if _is_titlecase_token(token): + uppercase_count += 1 + if lowered in _ROMAN_NAME_TITLES: + title_count += 1 + j -= 1 + continue + break + + if not uppercase_count: + return False + + if title_count: + return value <= 50 + + if uppercase_count >= 2: + return value <= 20 + + return False + + +def _normalize_roman_numerals(text: str, language: str) -> str: + if not text: + return text + + tokens = tokenize_with_spans(text) + if not tokens: + return text + + parts: List[str] = [] + cursor = 0 + + for index, (token, start, end) in enumerate(tokens): + parts.append(text[cursor:start]) + replacement = token + + compound_match = _ROMAN_CONTEXT_COMPOUND_RE.match(token) + if compound_match: + context_word = compound_match.group("context") + separator = compound_match.group("sep") + roman_part = compound_match.group("roman") + numeric_value = _roman_to_int(roman_part.upper()) + if ( + numeric_value is not None + and numeric_value <= 200 + and context_word.lower() in _ROMAN_CARDINAL_CONTEXTS + ): + words = _int_to_words(numeric_value, language) + if words: + if separator == ":": + replacement = f"{context_word}: {words}" + else: + replacement = f"{context_word} {words}" + else: + candidate = token.upper() + is_roman = _ROMAN_TOKEN_RE.match(candidate) + if is_roman: + numeric_value = _roman_to_int(candidate) + if numeric_value is not None: + convert = False + if len(token) >= 2: + if token.isupper(): + convert = True + elif numeric_value <= 200 and _has_cardinal_leading_context( + tokens, index + ): + convert = True + elif len(token) == 1: + # Only convert single letters if context is strong + if _has_cardinal_leading_context(tokens, index): + convert = True + + if convert: + if _should_render_ordinal(tokens, index, numeric_value): + ordinal = _int_to_ordinal_words(numeric_value, language) + if ordinal: + replacement = f"the {ordinal}" + else: + words = _int_to_words(numeric_value, language) + if words: + replacement = words + + parts.append(replacement) + cursor = end + + parts.append(text[cursor:]) + return "".join(parts) + + +_ACRONYM_ALLOWLIST = { + "AI", + "API", + "CPU", + "DIY", + "GPU", + "HTML", + "HTTP", + "HTTPS", + "ID", + "JSON", + "MP3", + "MP4", + "M4B", + "NASA", + "OCR", + "PDF", + "SQL", + "TV", + "TTS", + "UK", + "UN", + "UFO", + "OK", + "URL", + "USA", + "US", + "VR", +} +_ROMAN_NUMERAL_LETTERS = frozenset("IVXLCDM") +_CAPS_WORD_PATTERN = re.compile(r"[A-Z][A-Z0-9'\u2019-]*") +_WORD_PATTERN = re.compile(r"[A-Za-z][A-Za-z0-9'\u2019-]*") +_QUOTE_PAIRS = { + '"': '"', + "“": "”", + "„": "“", + "«": "»", + "‹": "›", +} + + +def _should_preserve_caps_word(word: str) -> bool: + letters = "".join(ch for ch in word if ch.isalpha()) + if not letters: + return False + base = letters + if word.endswith(("'S", "’S")) and len(letters) > 1: + base = letters[:-1] + upper_base = base.upper() + if upper_base in _ACRONYM_ALLOWLIST: + return True + if ( + all(ch in _ROMAN_NUMERAL_LETTERS for ch in letters.upper()) + and len(letters) <= 7 + ): + return True + return False + + +def _should_normalize_caps_segment(segment: str) -> bool: + letters = [ch for ch in segment if ch.isalpha()] + if not letters: + return False + if any(ch.islower() for ch in letters): + return False + if len(letters) <= 1: + return False + if not any(ch.isspace() for ch in segment) and len(letters) <= 4: + return False + return True + + +def _normalize_caps_segment(segment: str) -> str: + if not segment: + return segment + + preserve: Dict[str, str] = {} + for match in _CAPS_WORD_PATTERN.finditer(segment): + word = match.group(0) + if _should_preserve_caps_word(word): + preserve[word.lower()] = word + + lowered = segment.lower() + result_chars: List[str] = [] + capitalize_next = True + for char in lowered: + if capitalize_next and char.isalpha(): + result_chars.append(char.upper()) + capitalize_next = False + else: + result_chars.append(char) + if char.isalpha(): + capitalize_next = False + if char in ".!?": + capitalize_next = True + elif char in "\n": + capitalize_next = True + + def _restore(match: re.Match[str]) -> str: + token = match.group(0) + lookup = preserve.get(token.lower()) + if lookup: + return lookup + lower = token.lower() + if lower == "i": + return "I" + if lower.startswith("i'") or lower.startswith("i\u2019"): + return "I" + token[1:] + return token + + return _WORD_PATTERN.sub(_restore, "".join(result_chars)) + + +def _normalize_all_caps_quotes(text: str) -> str: + if not text: + return text + + builder: List[str] = [] + index = 0 + length = len(text) + + while index < length: + char = text[index] + closing = _QUOTE_PAIRS.get(char) + if not closing: + builder.append(char) + index += 1 + continue + + cursor = index + 1 + while cursor < length and text[cursor] != closing: + cursor += 1 + + if cursor >= length: + builder.append(text[index:]) + break + + body = text[index + 1 : cursor] + if _should_normalize_caps_segment(body): + normalized = _normalize_caps_segment(body) + builder.append(char + normalized + closing) + else: + builder.append(text[index : cursor + 1]) + + index = cursor + 1 + + if index < length: + builder.append(text[index:]) + + return "".join(builder) + + +def normalize_roman_numeral_titles( + titles: Sequence[str], + *, + threshold: float = 0.5, +) -> List[str]: + if not titles: + return [] + + normalized: List[str] = [] + matches: List[Tuple[int, str, int, str, str]] = [] + non_empty = 0 + + for index, raw in enumerate(titles): + title = "" if raw is None else str(raw) + stripped = title.lstrip() + leading_ws = title[: len(title) - len(stripped)] + if not stripped: + normalized.append(title) + continue + + non_empty += 1 + match = _ROMAN_PREFIX_RE.match(stripped) + if not match: + normalized.append(title) + continue + + roman_token = match.group("roman") + separator = match.group("sep") or "" + rest = stripped[match.end() :] + + if not separator and rest and rest[:1].isalnum(): + normalized.append(title) + continue + + numeric_value = _roman_to_int(roman_token) + if numeric_value is None: + normalized.append(title) + continue + + matches.append((index, leading_ws, numeric_value, separator, rest)) + normalized.append(title) + + if not matches or non_empty == 0: + return list(normalized) + + if len(matches) <= non_empty * threshold: + return list(normalized) + + output = list(normalized) + for idx, leading_ws, value, separator, rest in matches: + new_title = f"{leading_ws}{value}" + if separator: + new_title += separator + elif rest and not rest[0].isspace() and rest[0] not in ".-–—:;,": + new_title += " " + new_title += rest + output[idx] = new_title + + return output + + +def _match_casing(template: str, replacement: str) -> str: + if template.isupper(): + return replacement.upper() + if template[:1].isupper() and template[1:].islower(): + return replacement.capitalize() + if template[:1].isupper(): + # Mixed case (e.g., Mc), fall back to title case + return replacement.capitalize() + return replacement + + +def expand_titles_and_suffixes(text: str) -> str: + def _replace(match: re.Match[str], mapping: dict[str, str]) -> str: + abbr = match.group("abbr") + lookup = mapping.get(abbr.lower()) + if not lookup: + return match.group(0) + return _match_casing(abbr, lookup) + + text = _TITLE_PATTERN.sub(lambda m: _replace(m, TITLE_ABBREVIATIONS), text) + text = _SUFFIX_PATTERN.sub(lambda m: _replace(m, SUFFIX_ABBREVIATIONS), text) + return text + + +def ensure_terminal_punctuation(text: str) -> str: + def _amend(segment: str) -> str: + if not segment or not segment.strip(): + return segment + + stripped = segment.rstrip() + trailing_ws = segment[len(stripped) :] + + match = re.match(rf"^(.*?)([{re.escape(CLOSING_PUNCTUATION)}]*)$", stripped) + if not match: + return segment + + body, closers = match.groups() + if not body: + return segment + + normalized_body = body.rstrip() + trailing_body_ws = body[len(normalized_body) :] + + if normalized_body.endswith(ELLIPSIS_SUFFIXES): + return normalized_body + trailing_body_ws + closers + trailing_ws + + last_char = normalized_body[-1] + if last_char in TERMINAL_PUNCTUATION: + return normalized_body + trailing_body_ws + closers + trailing_ws + + return normalized_body + "." + trailing_body_ws + closers + trailing_ws + + parts = _LINE_SPLIT_RE.split(text) + amended: List[str] = [] + for part in parts: + if not part: + continue + if part.startswith("\n"): + amended.append(part) + else: + amended.append(_amend(part)) + if not parts: + return _amend(text) + return "".join(amended) + + +def is_cultural_name(token: str, cfg: ApostropheConfig) -> bool: + if not cfg.protect_cultural_names: + return False + for pat in CULTURAL_NAME_PATTERNS: + if pat.match(token): + return True + return False + + +def _case_preserving_words(original: str, words: Sequence[str]) -> str: + if not words: + return "" + if original.isupper(): + return " ".join(word.upper() for word in words) + + if original[:1].isupper(): + adjusted = [words[0].capitalize()] + if len(words) > 1: + adjusted.extend(words[1:]) + return " ".join(adjusted) + + return " ".join(words) + + +def _apply_contraction_policy( + token: str, + *, + category: str, + cfg: ApostropheConfig, + expand: Callable[[], str], + collapse: Optional[str] = None, +) -> str: + mode = cfg.contraction_mode + if mode == "collapse": + return collapse if collapse is not None else token.replace("'", "") + if mode != "expand": + return token + if not cfg.is_contraction_enabled(category): + return token + return expand() + + +def _assemble_contraction_expansion( + base_text: str, surface_text: str, expansion_word: str +) -> str: + if not expansion_word: + return base_text + + if surface_text.isupper() and expansion_word.isalpha(): + adjusted = expansion_word.upper() + elif len(surface_text) > 2 and surface_text[:-2].istitle() and expansion_word: + adjusted = expansion_word.lower() + else: + adjusted = expansion_word + + return f"{base_text} {adjusted}".strip() + + +def _classify_ambiguous_d(token: str, cfg: ApostropheConfig) -> Tuple[str, str]: + base = token[:-2] + collapse_value = base + "d" + + if cfg.contraction_mode == "collapse": + return "contraction_modal_would", collapse_value + if cfg.contraction_mode != "expand": + return "contraction_modal_would", token + + mode = cfg.ambiguous_past_modal_mode + if mode == "expand_prefer_had": + candidates = [ + ("contraction_aux_have", "had"), + ("contraction_modal_would", "would"), + ] + elif mode == "expand_prefer_would": + candidates = [ + ("contraction_modal_would", "would"), + ("contraction_aux_have", "had"), + ] + else: # contextual + candidates = [ + ("contraction_modal_would", "would"), + ("contraction_aux_have", "had"), + ] + + for category, word in candidates: + if not cfg.is_contraction_enabled(category): + continue + expanded = _assemble_contraction_expansion(base, token, word) + return category, expanded + + # If every category is disabled, leave the token as-is but report default category + return candidates[0][0], token + + +def _classify_ambiguous_s(token: str, cfg: ApostropheConfig) -> Tuple[str, str]: + base = token[:-2] + + if cfg.contraction_mode == "collapse": + return "contraction_aux_be", base + "s" + if cfg.contraction_mode != "expand": + return "contraction_aux_be", token + + candidates = [ + ("contraction_aux_be", "is"), + ("contraction_aux_have", "has"), + ] + + for category, word in candidates: + if not cfg.is_contraction_enabled(category): + continue + expanded = _assemble_contraction_expansion(base, token, word) + return category, expanded + + return candidates[0][0], token + + +def classify_token(token: str, cfg: ApostropheConfig) -> Tuple[str, str]: + """ + Classify apostrophe usage and propose normalized form. + Returns (category, normalized_token_or_same). + Categories include: contraction_* variants, plural_possessive, irregular_possessive, + sibilant_possessive, singular_possessive, acronym_possessive, decade, leading_elision, + fantasy_internal, cultural_name, other. + """ + if "'" not in token: + return "other", token + + low = token.lower() + + # 1. Decades + if DECADE_RE.match(token): + if cfg.decades_mode == "expand": + decade_digit = token[1] + decade_map = { + "0": "two thousands", + "1": "tens", + "2": "twenties", + "3": "thirties", + "4": "forties", + "5": "fifties", + "6": "sixties", + "7": "seventies", + "8": "eighties", + "9": "nineties", + } + spoken = decade_map.get(decade_digit) + if spoken: + return "decade", spoken + return "decade", token + return "decade", token + + # 2. Leading elision + if low in LEADING_ELISION: + if cfg.leading_elision_mode == "expand": + return "leading_elision", LEADING_ELISION[low] + return "leading_elision", token + + # 3. Ambiguous 'd contractions + if _is_ambiguous_d(token): + return _classify_ambiguous_d(token, cfg) + + # 4. Ambiguous 's contractions + if _is_ambiguous_s(token): + return _classify_ambiguous_s(token, cfg) + + # 5. Lexicon-based contractions + lex_entry = CONTRACTION_LEXICON.get(low) + if lex_entry is not None: + category, words = lex_entry + + def _expand() -> str: + return _case_preserving_words(token, words) + + collapse_value = token.replace("'", "") + normalized = _apply_contraction_policy( + token, category=category, cfg=cfg, expand=_expand, collapse=collapse_value + ) + return category, normalized + + # 6. Suffix contractions ('m handled separately) + if low.endswith("'m") and low[:-2] in SUFFIX_CONTRACTION_BASES.get( + "'m", () + ): # pronoun I'm + + def _expand_m() -> str: + base = token[:-2] + return _assemble_contraction_expansion(base, token, "am") + + normalized = _apply_contraction_policy( + token, + category="contraction_aux_be", + cfg=cfg, + expand=_expand_m, + collapse=token.replace("'", ""), + ) + return "contraction_aux_be", normalized + + for suffix, append_word, category in SUFFIX_CONTRACTION_RULES: + if low.endswith(suffix) and len(token) > len(suffix): + base = token[: -len(suffix)] + + def _expand_suffix() -> str: + return _assemble_contraction_expansion(base, token, append_word) + + normalized = _apply_contraction_policy( + token, + category=category, + cfg=cfg, + expand=_expand_suffix, + collapse=token.replace("'", ""), + ) + return category, normalized + + # 7. Irregular possessives (keep or expand logic) + if low in IRREGULAR_POSSESSIVES: + if cfg.irregular_possessive_mode == "keep": + return "irregular_possessive", token + return "irregular_possessive", token + + # 8. Plural possessive pattern dogs' + if re.match(r"^[A-Za-z0-9]+s'$", token): + if cfg.plural_possessive_mode == "collapse": + return "plural_possessive", token[:-1] + return "plural_possessive", token + + # 9. Acronym possessive NASA's + if ACRONYM_POSSESSIVE_RE.match(token): + if cfg.acronym_possessive_mode == "collapse_add_s": + return "acronym_possessive", token.replace("'", "") + return "acronym_possessive", token + + # 10. Sibilant singular possessive boss's, church's + if low.endswith("'s"): + base = token[:-2] + if SIBILANT_END_RE.search(base): + if cfg.sibilant_possessive_mode == "keep": + return "sibilant_possessive", token + if cfg.sibilant_possessive_mode == "approx": + return "sibilant_possessive", base + "es" + if cfg.sibilant_possessive_mode == "mark": + normalized = base + normalized += cfg.sibilant_iz_marker if cfg.add_phoneme_hints else "es" + return "sibilant_possessive", normalized + + # 11. Generic singular possessive (\w+'s) + if re.match(r"^[A-Za-z0-9]+'s$", token): + if cfg.possessive_mode == "collapse": + return "singular_possessive", token.replace("'", "") + return "singular_possessive", token + + # 12. Cultural names or fantasy internal + if is_cultural_name(token, cfg): + return "cultural_name", token + + if INTERNAL_APOSTROPHE_RE.search(token): + if cfg.fantasy_mode == "keep": + return "fantasy_internal", token + if cfg.fantasy_mode == "mark": + out = token + (cfg.fantasy_marker if cfg.add_phoneme_hints else "") + return "fantasy_internal", out + if cfg.fantasy_mode == "collapse_internal": + inner = re.sub(r"(?<=\w)'+(?=\w)", cfg.joiner, token) + return "fantasy_internal", inner + + if cfg.fantasy_mode == "collapse_internal": + return "other", token.replace("'", cfg.joiner) + return "other", token + + +def normalize_apostrophes( + text: str, cfg: ApostropheConfig | None = None +) -> Tuple[str, List[Tuple[str, str, str]]]: + """ + Normalize apostrophes per config. + Returns normalized text AND a list of (original_token, category, normalized_token) + so you can debug or post-process (e.g., apply phoneme replacement for ‹IZ›). + """ + if cfg is None: + cfg = ApostropheConfig() + + text = normalize_unicode_apostrophes(text) + text = _normalize_grouped_numbers(text, cfg) + token_entries = tokenize_with_spans(text) + + use_contextual_s = cfg.contraction_mode == "expand" + use_contextual_d = ( + cfg.contraction_mode == "expand" + and cfg.ambiguous_past_modal_mode == "contextual" + ) + + need_contextual = False + if (use_contextual_s or use_contextual_d) and token_entries: + for token_value, _, _ in token_entries: + if use_contextual_s and _is_ambiguous_s(token_value): + need_contextual = True + break + if use_contextual_d and _is_ambiguous_d(token_value): + need_contextual = True + break + + contextual_resolutions = ( + resolve_ambiguous_contractions(text) if need_contextual else {} + ) + + results: List[Tuple[str, str, str]] = [] + normalized_tokens: List[str] = [] + + for tok, start, end in token_entries: + category, norm = classify_token(tok, cfg) + + resolution = ( + contextual_resolutions.get((start, end)) if contextual_resolutions else None + ) + if resolution is not None and cfg.contraction_mode == "expand": + if cfg.is_contraction_enabled(resolution.category): + category = resolution.category + norm = resolution.expansion + else: + norm = tok + + results.append((tok, category, norm)) + normalized_tokens.append(norm) + + filtered = [token for token in normalized_tokens if token] + normalized_text = _cleanup_spacing(" ".join(filtered)) + return normalized_text, results + + +def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str: + if not text: + return text + + language = (cfg.number_lang or "en").strip() or "en" + + def _year_mode() -> str: + mode = (cfg.year_pronunciation_mode or "").strip().lower() + if mode in {"", "none", "off", "disabled"}: + return "off" + if mode not in {"american"}: + return "off" + return mode + + year_mode = _year_mode() + + def _format_year_tail(value: int, *, allow_oh: bool = True) -> Optional[str]: + if value == 0: + return "" + if value < 10: + if allow_oh: + return f"oh {_DIGIT_WORDS[value]}" + return _DIGIT_WORDS[value] + words = _int_to_words(value, language) + if not words: + return None + return words + + def _format_year_like(token: str, value: int) -> Optional[str]: + if year_mode == "off" or num2words is None: + return None + if len(token) != 4 or not token.isdigit(): + return None + if value < 1000 or value > 9999: + return None + style = year_mode + + def _words(value_to_convert: int) -> Optional[str]: + words = _int_to_words(value_to_convert, language) + return words + + if style == "american": + # Special handling for 2000-2009 to use "two thousand X" + if 2000 <= value <= 2009: + words = _words(value) + if words: + return words.replace(" and ", " ") + return None + + # US-style: 1100-1999 are often spoken as "X hundred Y". + if 1100 <= value <= 1999: + hundreds = value // 100 + remainder = value % 100 + prefix = _words(hundreds) + if not prefix: + return None + if remainder == 0: + return f"{prefix} hundred" + tail = _format_year_tail(remainder, allow_oh=True) + if tail is None: + return None + return f"{prefix} hundred {tail}".strip() + + if value % 1000 == 0: + thousands = value // 1000 + thousands_words = _words(thousands) + if thousands_words: + return f"{thousands_words} thousand" + return None + + first_two = value // 100 + last_two = value % 100 + + prefix = _words(first_two) + if not prefix: + return None + + if last_two == 0: + return f"{prefix} hundred" + + if last_two < 10: + # Use "oh X" format (e.g. "nineteen oh five") + return f"{prefix} oh {_DIGIT_WORDS[last_two]}" + + tail = _format_year_tail(last_two) + if tail: + return f"{prefix} {tail}" + return prefix + + return None + + def _replace_grouped(match: re.Match[str]) -> str: + token = match.group(1) + value = _coerce_int_token(token) + if value is None: + cleaned = token.replace(",", "") + return cleaned + if num2words is None: + return str(value) + words = _int_to_words(value, language) + return words or str(value) + + def _replace_plain(match: re.Match[str]) -> str: + token = match.group("number") + if "," in token: + return token.replace(",", "") + + start, end = match.span() + source = match.string + before = source[start - 1] if start > 0 else "" + after = source[end] if end < len(source) else "" + + if before == "/" or after == "/": + return token + + if after == ".": + next_char = source[end + 1] if end + 1 < len(source) else "" + if next_char.isdigit(): + return token + + if before == ".": + prev_char = source[start - 2] if start >= 2 else "" + if prev_char.isdigit() or start == 1: + return token + + value = _coerce_int_token(token) + if value is None: + return token + + # Check context for "address" vs year markers to avoid converting house numbers to years + window_start = max(0, start - 60) + window_end = min(len(source), end + 60) + context = source[window_start:window_end].lower() + + # Check for "address" or "addresses" as a whole word + has_address = bool(re.search(r"\baddress(es)?\b", context)) + + # Check for year markers as whole words + has_year_marker = bool( + re.search(r"\b(bc|ad|bce|ce|b\.c\.|a\.d\.|b\.c\.e\.|c\.e\.)\b", context) + ) + + should_try_year = True + if has_address and not has_year_marker: + should_try_year = False + + if should_try_year: + year_like = _format_year_like(token, value) + if year_like: + return year_like + + if num2words is None: + return str(value) + words = _int_to_words(value, language) + return words or str(value) + + def _replace_decimal(match: re.Match[str]) -> str: + token = match.group("number") + fraction_part = match.group("fraction") + start, end = match.span() + source = match.string + + if end < len(source) and source[end] == ".": + next_char = source[end + 1] if end + 1 < len(source) else "" + if next_char.isdigit(): + return token + + is_negative = token.startswith("-") + core = token[1:] if is_negative else token + if "." not in core: + return token + + integer_part, _, _ = core.partition(".") + if not integer_part or not fraction_part: + return token + + integer_value = _coerce_int_token(integer_part.replace(",", "")) + if integer_value is None: + return token + + trimmed_fraction = fraction_part.rstrip("0") + integer_words = _int_to_words(integer_value, language) + + if not trimmed_fraction: + if integer_words is None: + return token + spoken = integer_words + return f"minus {spoken}" if is_negative else spoken + + if integer_words is None: + fallback_core = core.replace(".", " point ") + return f"minus {fallback_core}" if is_negative else fallback_core + + digit_words: List[str] = [] + for digit in trimmed_fraction: + if not digit.isdigit(): + return token + digit_words.append(_DIGIT_WORDS[int(digit)]) + + spoken = f"{integer_words} point {' '.join(digit_words)}" + return f"minus {spoken}" if is_negative else spoken + + def _replace_currency(match: re.Match[str]) -> str: + if num2words is None: + return match.group(0) + + symbol = match.group("symbol") + amount_str = match.group("amount").replace(",", "") + magnitude = match.group("magnitude") + + try: + amount = float(amount_str) + except ValueError: + return match.group(0) + + if magnitude: + # Magnitude case: $2.5 million -> two point five million dollars + if "." in amount_str: + integer_part, fraction_part = amount_str.split(".", 1) + integer_val = int(integer_part) + integer_words = _int_to_words(integer_val, language) + + # Spell out fraction digits + digit_words = [] + for digit in fraction_part: + if digit.isdigit(): + digit_words.append(_DIGIT_WORDS[int(digit)]) + + amount_spoken = f"{integer_words} point {' '.join(digit_words)}" + else: + amount_spoken = _int_to_words(int(amount), language) + + currency_names = { + "$": "dollars", + "£": "pounds", + "€": "euros", + "¥": "yen", + } + currency_name = currency_names.get(symbol, "dollars") + + return f"{amount_spoken} {magnitude} {currency_name}" + + # Handle $0.99 -> ninety-nine cents (avoid "zero dollars and..."). + if amount_str.startswith("0") and amount < 1.0: + dollars_part, dot, fraction = amount_str.partition(".") + if dot and fraction: + cents_str = (fraction + "00")[:2] + try: + cents_value = int(cents_str) + except ValueError: + cents_value = 0 + if cents_value > 0: + cents_words = _int_to_words(cents_value, language) or str( + cents_value + ) + subunit = { + "$": "cent", + "€": "cent", + "£": "penny", + "¥": "yen", + }.get(symbol, "cent") + if symbol == "£": + subunit = "pence" if cents_value != 1 else "penny" + else: + subunit = ( + (subunit + "s") + if cents_value != 1 and subunit not in {"pence", "yen"} + else subunit + ) + return f"{cents_words} {subunit}".strip() + + currency_map = { + "$": "USD", + "£": "GBP", + "€": "EUR", + "¥": "JPY", + } + currency_code = currency_map.get(symbol, "USD") + + try: + # Always use float to avoid num2words treating int as cents (if that's what it does) + # or to ensure consistent behavior. + words = num2words( + amount, to="currency", currency=currency_code, lang=language + ) + + # Remove "zero cents" if present + # Patterns: ", zero cents", " and zero cents" + words = words.replace(", zero cents", "").replace(" and zero cents", "") + return words + except Exception: + return match.group(0) + + def _replace_url(match: re.Match[str]) -> str: + domain = match.group("domain") + + # Avoid matching dotted abbreviations/acronyms without an explicit URL prefix. + has_prefix = match.group(1) or match.group(2) + if not has_prefix: + parts = [p for p in domain.split(".") if p] + if parts and all(p.isalpha() and len(p) <= 2 for p in parts): + return match.group(0) + + # Avoid matching numbers like 1.05 or 12.34.56 + # If the domain consists only of digits and dots, ignore it (unless it has http/www prefix) + has_prefix = match.group(1) or match.group(2) + if not has_prefix and all(c.isdigit() or c == "." or c == "-" for c in domain): + # Check if it really looks like a number (e.g. 1.05) + # If it has multiple dots like 1.2.3.4 it might be an IP, which we might want to speak as dot? + # But 1.05 is definitely a number. + # Let's be safe: if it looks like a float, skip. + try: + float(domain) + return match.group(0) + except ValueError: + pass + + if domain.startswith("www."): + domain = domain[4:] + spoken = domain.replace(".", " dot ") + return spoken + + def _remove_footnote(match: re.Match[str]) -> str: + return match.group(1) + + normalized = text + + # Apply URL replacement first (independent of number conversion). + normalized = _URL_RE.sub(_replace_url, normalized) + + if getattr(cfg, "remove_footnotes", False): + normalized = _FOOTNOTE_RE.sub(_remove_footnote, normalized) + normalized = _BRACKET_FOOTNOTE_RE.sub("", normalized) + + if cfg.convert_currency: + normalized = _CURRENCY_RE.sub(_replace_currency, normalized) + + if cfg.convert_numbers: + normalized = _NUMBER_RANGE_RE.sub( + lambda m: _replace_number_range(m, language), normalized + ) + normalized = _NUMBER_SPACE_RANGE_RE.sub( + lambda m: _replace_space_separated_range(m, language), normalized + ) + normalized = _FRACTION_RE.sub( + lambda m: _replace_fraction(m, language), normalized + ) + normalized = _DECIMAL_NUMBER_RE.sub(_replace_decimal, normalized) + normalized = _NUMBER_WITH_GROUP_RE.sub(_replace_grouped, normalized) + normalized = _PLAIN_NUMBER_RE.sub(_replace_plain, normalized) + normalized = _normalize_roman_numerals(normalized, language) + + return normalized + + +def _normalize_dotted_acronyms(text: str) -> str: + def _replace(match: re.Match[str]) -> str: + value = match.group(0) + compact = value.replace(".", "") + return compact + + return re.sub(r"\b(?:[A-Z]\.){1,}[A-Z]\.?(?=\W|$)", _replace, text) + + +# ---------- Optional phoneme hint post-processing ---------- + + +def apply_phoneme_hints(text: str, iz_marker="‹IZ›") -> str: + """ + Replace markers with an orthographic sequence that + your phonemizer will reliably convert to /ɪz/. + """ + return text.replace(iz_marker, " iz") + + +DEFAULT_APOSTROPHE_CONFIG = ApostropheConfig() + + +_MUSTACHE_PATTERN = re.compile(r"{{\s*([a-zA-Z0-9_]+)\s*}}") +_LLM_SYSTEM_PROMPT = ( + "You assist with audiobook preparation. Review the sentence, identify any apostrophes or " + "contractions that should be expanded for clarity, and respond by calling the " + "apply_regex_replacements tool. Each replacement must target a single token, include a precise " + "regex pattern, and provide the exact replacement text. If no changes are required, call the tool " + "with an empty replacements list. Do not rewrite the sentence directly." +) + +_LLM_REGEX_TOOL_NAME = "apply_regex_replacements" +_LLM_REGEX_TOOL = { + "type": "function", + "function": { + "name": _LLM_REGEX_TOOL_NAME, + "description": ( + "Return regex substitutions to normalize apostrophes or contractions in the provided sentence." + ), + "parameters": { + "type": "object", + "properties": { + "replacements": { + "description": "Ordered substitutions to apply to the sentence.", + "type": "array", + "items": { + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "Regular expression that matches the token to replace.", + }, + "replacement": { + "type": "string", + "description": "Replacement text for the match.", + }, + "flags": { + "type": "array", + "items": {"type": "string"}, + "description": "Optional re flags such as IGNORECASE.", + }, + "count": { + "type": "integer", + "description": "Optional maximum number of replacements (default all).", + }, + "reason": { + "type": "string", + "description": "Short explanation of why the replacement is needed.", + }, + }, + "required": ["pattern", "replacement"], + }, + } + }, + "required": ["replacements"], + }, + }, +} +_LLM_REGEX_TOOL_CHOICE = { + "type": "function", + "function": {"name": _LLM_REGEX_TOOL_NAME}, +} +_LLM_ALLOWED_REGEX_FLAGS = { + "IGNORECASE": re.IGNORECASE, + "MULTILINE": re.MULTILINE, + "DOTALL": re.DOTALL, +} + + +def _render_mustache(template: str, context: Mapping[str, str]) -> str: + if not template: + return "" + + def _replace(match: re.Match[str]) -> str: + key = match.group(1) + return context.get(key, "") + + return _MUSTACHE_PATTERN.sub(_replace, template) + + +_SENTENCE_CAPTURE_RE = re.compile(r"[^.!?]+[.!?]+|[^.!?]+$", re.MULTILINE) + + +def _split_sentences_for_llm(text: str) -> List[str]: + sentences = [ + segment.strip() for segment in _SENTENCE_CAPTURE_RE.findall(text or "") + ] + return [segment for segment in sentences if segment] + + +def _normalize_with_llm( + text: str, + *, + settings: Mapping[str, Any], + config: ApostropheConfig, +) -> str: + from abogen.normalization_settings import ( + build_llm_configuration, + DEFAULT_LLM_PROMPT, + ) + from abogen.llm_client import generate_completion, LLMClientError + + llm_config = build_llm_configuration(settings) + if not llm_config.is_configured(): + raise LLMClientError("LLM configuration is incomplete") + + prompt_template = str(settings.get("llm_prompt") or DEFAULT_LLM_PROMPT) + lines = text.splitlines(keepends=True) + if not lines: + return text + + normalized_lines: List[str] = [] + for raw_line in lines: + newline = "" + if raw_line.endswith(("\r", "\n")): + stripped_newline = raw_line.rstrip("\r\n") + newline = raw_line[len(stripped_newline) :] + line_body = stripped_newline + else: + line_body = raw_line + + if not line_body.strip(): + normalized_lines.append(line_body + newline) + continue + + leading_ws = line_body[: len(line_body) - len(line_body.lstrip())] + trailing_ws = line_body[len(line_body.rstrip()) :] + core = line_body[len(leading_ws) : len(line_body) - len(trailing_ws)] + + sentences = _split_sentences_for_llm(core) + if not sentences: + normalized_lines.append(line_body + newline) + continue + + paragraph_context = core + rewritten_sentences: List[str] = [] + for sentence in sentences: + prompt_context = { + "text": sentence, + "sentence": sentence, + "paragraph": paragraph_context, + } + prompt = _render_mustache(prompt_template, prompt_context) + completion = generate_completion( + llm_config, + system_message=_LLM_SYSTEM_PROMPT, + user_message=prompt, + tools=[_LLM_REGEX_TOOL], + tool_choice=_LLM_REGEX_TOOL_CHOICE, + ) + rewritten_sentences.append( + _apply_llm_regex_replacements(sentence, completion) + ) + + normalized_core = " ".join(filter(None, rewritten_sentences)) or core + + rebuilt = f"{leading_ws}{normalized_core}{trailing_ws}{newline}" + normalized_lines.append(rebuilt) + + result = "".join(normalized_lines) + return result if result else text + + +def _apply_llm_regex_replacements(sentence: str, completion: "LLMCompletion") -> str: + replacements = _extract_llm_replacements(completion) + if not replacements: + return sentence + + updated = sentence + for spec in replacements: + updated = _apply_single_regex_replacement(updated, spec) + return updated + + +def _extract_llm_replacements(completion: "LLMCompletion") -> List[Dict[str, Any]]: + if completion is None: + return [] + + for call in getattr(completion, "tool_calls", ()): # type: ignore[attr-defined] + if getattr(call, "name", None) != _LLM_REGEX_TOOL_NAME: + continue + payload = _safe_load_json(getattr(call, "arguments", None)) + replacements = _coerce_replacement_list(payload) + if replacements: + return replacements + + if getattr(completion, "content", None): + payload = _safe_load_json(completion.content) + replacements = _coerce_replacement_list(payload) + if replacements: + return replacements + + return [] + + +def _safe_load_json(raw: Optional[str]) -> Any: + if not raw: + return None + try: + return json.loads(raw) + except json.JSONDecodeError: + return None + + +def _coerce_replacement_list(raw: Any) -> List[Dict[str, Any]]: + if isinstance(raw, Mapping): + candidates = raw.get("replacements") + else: + candidates = raw + + if not isinstance(candidates, list): + return [] + + replacements: List[Dict[str, Any]] = [] + for item in candidates: + if not isinstance(item, Mapping): + continue + pattern = str(item.get("pattern") or "").strip() + if not pattern: + continue + replacement = str(item.get("replacement") or "") + entry: Dict[str, Any] = {"pattern": pattern, "replacement": replacement} + + flags = _normalize_flag_field(item.get("flags")) + if flags: + entry["flags"] = flags + + count = item.get("count") + if isinstance(count, int) and count >= 0: + entry["count"] = count + + replacements.append(entry) + + return replacements + + +def _normalize_flag_field(raw: Any) -> List[str]: + if not raw: + return [] + + if isinstance(raw, str): + raw_iterable: Iterable[Any] = [raw] + elif isinstance(raw, Iterable) and not isinstance(raw, (bytes, str, Mapping)): + raw_iterable = raw + else: + return [] + + normalized: List[str] = [] + seen: set[str] = set() + for value in raw_iterable: + candidate = str(value or "").strip().upper() + if ( + not candidate + or candidate not in _LLM_ALLOWED_REGEX_FLAGS + or candidate in seen + ): + continue + seen.add(candidate) + normalized.append(candidate) + return normalized + + +def _apply_single_regex_replacement(text: str, spec: Mapping[str, Any]) -> str: + pattern = str(spec.get("pattern") or "") + replacement = str(spec.get("replacement") or "") + if not pattern: + return text + + flags_value = 0 + flag_names = spec.get("flags") + if isinstance(flag_names, str): + flag_iterable: Iterable[Any] = [flag_names] + elif isinstance(flag_names, Iterable) and not isinstance( + flag_names, (bytes, str, Mapping) + ): + flag_iterable = flag_names + else: + flag_iterable = [] + + for flag_name in flag_iterable: + lookup = str(flag_name or "").strip().upper() + flags_value |= _LLM_ALLOWED_REGEX_FLAGS.get(lookup, 0) + + count = spec.get("count") + count_value = count if isinstance(count, int) and count >= 0 else 0 + + try: + return re.sub(pattern, replacement, text, count=count_value, flags=flags_value) + except re.error: + return text + + +def normalize_for_pipeline( + text: str, + *, + config: Optional[ApostropheConfig] = None, + settings: Optional[Mapping[str, Any]] = None, +) -> str: + """Normalize text for the synthesis pipeline with runtime settings.""" + + from abogen.normalization_settings import ( + build_apostrophe_config, + get_runtime_settings, + ) + from abogen.llm_client import LLMClientError + + runtime_settings = settings or get_runtime_settings() + base_config = config or DEFAULT_APOSTROPHE_CONFIG + cfg = build_apostrophe_config(settings=runtime_settings, base=base_config) + + mode = str(runtime_settings.get("normalization_apostrophe_mode", "spacy")).lower() + normalized = text + + # Pre-normalization that must happen before number/url parsing. + if runtime_settings.get("normalization_numbers", True): + normalized = _normalize_dates(normalized, cfg.number_lang) + normalized = _normalize_times(normalized) + normalized = _normalize_dotted_acronyms(normalized) + if runtime_settings.get("normalization_titles", True): + normalized = _normalize_address_abbreviations(normalized) + if runtime_settings.get("normalization_internet_slang", False): + normalized = _normalize_internet_slang(normalized) + + if mode == "off": + normalized = normalize_unicode_apostrophes(normalized) + if ( + cfg.convert_numbers + or cfg.convert_currency + or getattr(cfg, "remove_footnotes", False) + ): + normalized = _normalize_grouped_numbers(normalized, cfg) + normalized = _cleanup_spacing(normalized) + elif mode == "llm": + try: + normalized = _normalize_with_llm( + normalized, settings=runtime_settings, config=cfg + ) + except LLMClientError: + raise + if ( + cfg.convert_numbers + or cfg.convert_currency + or getattr(cfg, "remove_footnotes", False) + ): + normalized = _normalize_grouped_numbers(normalized, cfg) + normalized = _cleanup_spacing(normalized) + else: + normalized, _ = normalize_apostrophes(normalized, cfg) + + if runtime_settings.get("normalization_titles", True): + normalized = expand_titles_and_suffixes(normalized) + if runtime_settings.get("normalization_terminal", True): + normalized = ensure_terminal_punctuation(normalized) + if runtime_settings.get("normalization_caps_quotes", True): + normalized = _normalize_all_caps_quotes(normalized) + + if cfg.add_phoneme_hints: + normalized = apply_phoneme_hints(normalized, iz_marker=cfg.sibilant_iz_marker) + + return normalized + + +# ---------- Example Usage ---------- + +if __name__ == "__main__": + sample = "Bob's boss's chair. The dogs' collars. It's cold. Ta'veren and Sha'hal. O'Brien's code in the '90s. Boss's orders." + config = ApostropheConfig() + norm_text, details = normalize_apostrophes(sample, config) + norm_text = apply_phoneme_hints(norm_text) + print("Original:", sample) + print("Normalized:", norm_text) + for orig, cat, norm in details: + print(f"{orig:15} -> {norm:15} [{cat}]") diff --git a/abogen/llm_client.py b/abogen/llm_client.py new file mode 100644 index 0000000..2937fa3 --- /dev/null +++ b/abogen/llm_client.py @@ -0,0 +1,210 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple +from urllib import error, parse, request + + +class LLMClientError(RuntimeError): + """Raised when an LLM request fails.""" + + +@dataclass(frozen=True) +class LLMConfiguration: + base_url: str + api_key: str + model: str + timeout: float = 30.0 + + def is_configured(self) -> bool: + return bool(self.base_url.strip() and self.model.strip()) + + +@dataclass(frozen=True) +class LLMToolCall: + name: str + arguments: str + + +@dataclass(frozen=True) +class LLMCompletion: + content: Optional[str] + tool_calls: Tuple[LLMToolCall, ...] + + +_DEFAULT_HEADERS = { + "Content-Type": "application/json", + "Accept": "application/json", +} + + +def _normalized_base_url(base_url: str) -> str: + trimmed = (base_url or "").strip() + if not trimmed: + raise LLMClientError("LLM base URL is required") + if not trimmed.endswith("/"): + trimmed += "/" + return trimmed + + +def _build_url(base_url: str, path: str) -> str: + normalized = _normalized_base_url(base_url) + trimmed_path = path.lstrip("/") + parsed = parse.urlparse(normalized) + if parsed.path.rstrip("/").lower().endswith("/v1") and trimmed_path.startswith( + "v1/" + ): + trimmed_path = trimmed_path[len("v1/") :] + return parse.urljoin(normalized, trimmed_path) + + +def _build_headers(api_key: str) -> Dict[str, str]: + headers = dict(_DEFAULT_HEADERS) + token = (api_key or "").strip() + if token and token.lower() != "ollama": + headers["Authorization"] = f"Bearer {token}" + return headers + + +def _perform_request( + method: str, + url: str, + *, + headers: Optional[Mapping[str, str]] = None, + payload: Optional[Mapping[str, Any]] = None, + timeout: float = 30.0, +) -> Any: + data_bytes: Optional[bytes] = None + if payload is not None: + data_bytes = json.dumps(payload).encode("utf-8") + request_headers = dict(headers or {}) + req = request.Request( + url, data=data_bytes, headers=request_headers, method=method.upper() + ) + try: + with request.urlopen(req, timeout=timeout) as response: + body = response.read() + except error.HTTPError as exc: # pragma: no cover - defensive network guard + message = exc.read().decode("utf-8", "ignore") if exc.fp else exc.reason + raise LLMClientError(f"LLM request failed ({exc.code}): {message}") from exc + except error.URLError as exc: # pragma: no cover - defensive network guard + raise LLMClientError(f"LLM request failed: {exc.reason}") from exc + except Exception as exc: # pragma: no cover - defensive network guard + raise LLMClientError("LLM request failed") from exc + + if not body: + return None + try: + return json.loads(body.decode("utf-8")) + except json.JSONDecodeError as exc: + raise LLMClientError("LLM response was not valid JSON") from exc + + +def list_models(configuration: LLMConfiguration) -> List[Dict[str, str]]: + if not configuration.is_configured() and not configuration.base_url.strip(): + raise LLMClientError("LLM configuration is incomplete") + url = _build_url(configuration.base_url, "v1/models") + headers = _build_headers(configuration.api_key) + payload = _perform_request( + "GET", url, headers=headers, timeout=configuration.timeout + ) + if not isinstance(payload, Mapping): + raise LLMClientError("Unexpected response when listing models") + data = payload.get("data") + if not isinstance(data, list): + return [] + models: List[Dict[str, str]] = [] + for entry in data: + if not isinstance(entry, Mapping): + continue + identifier = str(entry.get("id") or "").strip() + if not identifier: + continue + description = str(entry.get("name") or entry.get("description") or identifier) + models.append({"id": identifier, "label": description}) + return models + + +def generate_completion( + configuration: LLMConfiguration, + *, + system_message: str, + user_message: str, + temperature: float = 0.2, + max_tokens: Optional[int] = None, + tools: Optional[Sequence[Mapping[str, Any]]] = None, + tool_choice: Optional[Mapping[str, Any]] = None, + response_format: Optional[Mapping[str, Any]] = None, +) -> LLMCompletion: + if not configuration.is_configured(): + raise LLMClientError("LLM configuration is incomplete") + + url = _build_url(configuration.base_url, "v1/chat/completions") + headers = _build_headers(configuration.api_key) + payload: Dict[str, Any] = { + "model": configuration.model, + "messages": [ + {"role": "system", "content": system_message}, + {"role": "user", "content": user_message}, + ], + "temperature": temperature, + } + if max_tokens is not None: + payload["max_tokens"] = max_tokens + if tools: + payload["tools"] = list(tools) + if tool_choice: + payload["tool_choice"] = dict(tool_choice) + if response_format: + payload["response_format"] = dict(response_format) + + response = _perform_request( + "POST", url, headers=headers, payload=payload, timeout=configuration.timeout + ) + if not isinstance(response, Mapping): + raise LLMClientError("Unexpected response from LLM") + choices = response.get("choices") + if not isinstance(choices, list) or not choices: + raise LLMClientError("LLM response did not include choices") + first = choices[0] + if not isinstance(first, Mapping): + raise LLMClientError("LLM response choice was invalid") + message = first.get("message") + content: Optional[str] = None + tool_calls: List[LLMToolCall] = [] + if isinstance(message, Mapping): + content = message.get("content") + if isinstance(content, str): + stripped = content.strip() + if stripped: + content = stripped + else: + content = None + tool_call_entries = message.get("tool_calls") + if isinstance(tool_call_entries, list): + for entry in tool_call_entries: + if not isinstance(entry, Mapping): + continue + fn = entry.get("function") + if not isinstance(fn, Mapping): + continue + name = str(fn.get("name") or "").strip() + if not name: + continue + args = fn.get("arguments", "") + if isinstance(args, (dict, list)): + arguments = json.dumps(args) + else: + arguments = str(args) + tool_calls.append(LLMToolCall(name=name, arguments=arguments)) + if content: + return LLMCompletion(content=content, tool_calls=tuple(tool_calls)) + text = first.get("text") + if isinstance(text, str): + stripped = text.strip() + if stripped: + content = stripped + if content or tool_calls: + return LLMCompletion(content=content, tool_calls=tuple(tool_calls)) + raise LLMClientError("LLM response did not include text content") diff --git a/abogen/main.py b/abogen/main.py index ad80b20..90a55e1 100644 --- a/abogen/main.py +++ b/abogen/main.py @@ -1,116 +1,36 @@ -import os -import sys -import platform +"""Backwards-compatible entry point that now launches the web UI.""" + +from __future__ import annotations + import atexit +import os +import platform import signal -from abogen.utils import get_resource_path, load_config, prevent_sleep_end +import sys +from abogen.utils import load_config, prevent_sleep_end +from abogen.webui.app import main as _run_web_ui -# Fix PyTorch DLL loading issue ([WinError 1114]) on Windows before importing PyQt6 -if platform.system() == "Windows": - import ctypes - from importlib.util import find_spec - - try: - if ( - (spec := find_spec("torch")) - and spec.origin - and os.path.exists( - dll_path := os.path.join(os.path.dirname(spec.origin), "lib", "c10.dll") - ) - ): - ctypes.CDLL(os.path.normpath(dll_path)) - except Exception: - pass - - -# Qt platform plugin detection (fixes #59) -try: - from PyQt6.QtCore import QLibraryInfo - - # Get the path to the plugins directory - plugins = QLibraryInfo.path(QLibraryInfo.LibraryPath.PluginsPath) - - # Normalize path to use the OS-native separators and absolute path - platform_dir = os.path.normpath(os.path.join(plugins, "platforms")) - - # Ensure we work with an absolute path for clarity - platform_dir = os.path.abspath(platform_dir) - - if os.path.isdir(platform_dir): - os.environ["QT_QPA_PLATFORM_PLUGIN_PATH"] = platform_dir - print("QT_QPA_PLATFORM_PLUGIN_PATH set to:", platform_dir) - else: - print("PyQt6 platform plugins not found at", platform_dir) -except ImportError: - print("PyQt6 not installed.") - - -# Pre-load "libxcb-cursor" on Linux (fixes #101) -if platform.system() == "Linux": - arch = platform.machine().lower() - lib_filename = {"x86_64": "libxcb-cursor-amd64.so.0", "amd64": "libxcb-cursor-amd64.so.0", "aarch64": "libxcb-cursor-arm64.so.0", "arm64": "libxcb-cursor-arm64.so.0"}.get(arch) - if lib_filename: - import ctypes - try: - # Try to load the system libxcb-cursor.so.0 first - ctypes.CDLL('libxcb-cursor.so.0', mode=ctypes.RTLD_GLOBAL) - except OSError: - # System lib not available, load the bundled version - lib_path = get_resource_path('abogen.libs', lib_filename) - if lib_path: - try: - ctypes.CDLL(lib_path, mode=ctypes.RTLD_GLOBAL) - except OSError: - # If it fails (e.g. wrong glibc version on very old systems), - # we simply ignore it and hope the system has the library. - pass - - -# Set application ID for Windows taskbar icon -if platform.system() == "Windows": - try: - from abogen.constants import PROGRAM_NAME, VERSION - import ctypes - - app_id = f"{PROGRAM_NAME}.{VERSION}" - ctypes.windll.shell32.SetCurrentProcessExplicitAppUserModelID(app_id) - except Exception as e: - print("Warning: failed to set AppUserModelID:", e) - -from PyQt6.QtWidgets import QApplication -from PyQt6.QtGui import QIcon -from PyQt6.QtCore import ( - QLibraryInfo, - qInstallMessageHandler, - QtMsgType, -) - -# Add the directory to Python path -sys.path.insert(0, os.path.join(os.path.dirname(__file__))) - -# Set Hugging Face Hub environment variables -os.environ["HF_HUB_DISABLE_TELEMETRY"] = "1" # Disable Hugging Face telemetry -os.environ["HF_HUB_ETAG_TIMEOUT"] = "10" # Metadata request timeout (seconds) -os.environ["HF_HUB_DOWNLOAD_TIMEOUT"] = "10" # File download timeout (seconds) -os.environ["HF_HUB_DISABLE_SYMLINKS_WARNING"] = "1" # Disable symlinks warning +# Configure Hugging Face Hub behaviour (mirrors legacy GUI defaults). +os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1") +os.environ.setdefault("HF_HUB_ETAG_TIMEOUT", "10") +os.environ.setdefault("HF_HUB_DOWNLOAD_TIMEOUT", "10") +os.environ.setdefault("HF_HUB_DISABLE_SYMLINKS_WARNING", "1") if load_config().get("disable_kokoro_internet", False): - print("INFO: Kokoro's internet access is disabled.") - os.environ["HF_HUB_OFFLINE"] = "1" # Disable Hugging Face Hub internet access + os.environ["HF_HUB_OFFLINE"] = "1" -from abogen.gui import abogen -from abogen.constants import PROGRAM_NAME, VERSION +# Prefer faster ROCm tuning defaults when available. +os.environ.setdefault("MIOPEN_FIND_MODE", "FAST") +os.environ.setdefault("MIOPEN_CONV_PRECISE_ROCM_TUNING", "0") -# Set environment variables for AMD ROCm -os.environ["MIOPEN_FIND_MODE"] = "FAST" -os.environ["MIOPEN_CONV_PRECISE_ROCM_TUNING"] = "0" +# Enable MPS GPU acceleration on Apple Silicon. +if platform.system() == "Darwin" and platform.processor() == "arm": + os.environ.setdefault("PYTORCH_ENABLE_MPS_FALLBACK", "1") -# Reset sleep states atexit.register(prevent_sleep_end) -# Also handle signals (Ctrl+C, kill, etc.) -def _cleanup_sleep(signum, frame): +def _cleanup_sleep(signum, _frame): prevent_sleep_end() sys.exit(0) @@ -118,70 +38,12 @@ def _cleanup_sleep(signum, frame): signal.signal(signal.SIGINT, _cleanup_sleep) signal.signal(signal.SIGTERM, _cleanup_sleep) -# Ensure sys.stdout and sys.stderr are valid in GUI mode -if sys.stdout is None: - sys.stdout = open(os.devnull, "w") -if sys.stderr is None: - sys.stderr = open(os.devnull, "w") -# Enable MPS GPU acceleration on Mac Apple Silicon -if platform.system() == "Darwin" and platform.processor() == "arm": - os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1" +def main() -> None: + """Launch the Flask-based web UI.""" + + _run_web_ui() -# Custom message handler to filter out specific Qt warnings -def qt_message_handler(mode, context, message): - # In PyQt6, the mode is an enum, so we compare with the enum members - if "Wayland does not support QWindow::requestActivate()" in message: - return # Suppress this specific message - if "setGrabPopup called with a parent, QtWaylandClient" in message: - return - - if mode == QtMsgType.QtWarningMsg: - print(f"Qt Warning: {message}") - elif mode == QtMsgType.QtCriticalMsg: - print(f"Qt Critical: {message}") - elif mode == QtMsgType.QtFatalMsg: - print(f"Qt Fatal: {message}") - elif mode == QtMsgType.QtInfoMsg: - print(f"Qt Info: {message}") - - -# Install the custom message handler -qInstallMessageHandler(qt_message_handler) - -# Handle Wayland on Linux GNOME -if platform.system() == "Linux": - xdg_session = os.environ.get("XDG_SESSION_TYPE", "").lower() - desktop = os.environ.get("XDG_CURRENT_DESKTOP", "").lower() - if ( - "gnome" in desktop - and xdg_session == "wayland" - and "QT_QPA_PLATFORM" not in os.environ - ): - os.environ["QT_QPA_PLATFORM"] = "wayland" - - -def main(): - """Main entry point for console usage.""" - app = QApplication(sys.argv) - - # Set application icon using get_resource_path from utils - icon_path = get_resource_path("abogen.assets", "icon.ico") - if icon_path: - app.setWindowIcon(QIcon(icon_path)) - - # Set the .desktop name on Linux - if platform.system() == "Linux": - try: - app.setDesktopFileName("abogen") - except AttributeError: - pass - - ex = abogen() - ex.show() - sys.exit(app.exec()) - - -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover - manual execution hook main() diff --git a/abogen/normalization_settings.py b/abogen/normalization_settings.py new file mode 100644 index 0000000..4242265 --- /dev/null +++ b/abogen/normalization_settings.py @@ -0,0 +1,246 @@ +from __future__ import annotations + +import os +from dataclasses import replace +from functools import lru_cache +from typing import Any, Dict, Mapping, Optional + +from abogen.kokoro_text_normalization import ( + ApostropheConfig, + CONTRACTION_CATEGORY_DEFAULTS, +) +from abogen.llm_client import LLMConfiguration +from abogen.utils import load_config + +DEFAULT_LLM_PROMPT = ( + "You are assisting with audiobook preparation. Analyze the sentence and identify any apostrophes or " + "contractions that should be expanded for clarity. Call the apply_regex_replacements tool with precise " + "regex substitutions for only the words that need adjustment. If no changes are required, return an empty list.\n" + "Sentence: {{ sentence }}" +) + +_LEGACY_REWRITE_ONLY_PROMPT = ( + "You are assisting with audiobook preparation. Rewrite the provided sentence so apostrophes and " + "contractions are unambiguous for text-to-speech. Respond with only the rewritten sentence.\n" + "Sentence: {{ sentence }}\n" + "Context: {{ paragraph }}" +) + +_SETTINGS_DEFAULTS: Dict[str, Any] = { + "llm_base_url": "", + "llm_api_key": "", + "llm_model": "", + "llm_timeout": 30.0, + "llm_prompt": DEFAULT_LLM_PROMPT, + "llm_context_mode": "sentence", + "normalization_numbers": True, + "normalization_numbers_year_style": "american", + "normalization_currency": True, + "normalization_footnotes": True, + "normalization_titles": True, + "normalization_terminal": True, + "normalization_phoneme_hints": True, + "normalization_caps_quotes": True, + "normalization_internet_slang": False, + "normalization_apostrophes_contractions": True, + "normalization_apostrophes_plural_possessives": True, + "normalization_apostrophes_sibilant_possessives": True, + "normalization_apostrophes_decades": True, + "normalization_apostrophes_leading_elisions": True, + "normalization_apostrophe_mode": "spacy", + "normalization_contraction_aux_be": True, + "normalization_contraction_aux_have": True, + "normalization_contraction_modal_will": True, + "normalization_contraction_modal_would": True, + "normalization_contraction_negation_not": True, + "normalization_contraction_let_us": True, +} + +_CONTRACTION_SETTING_MAP: Dict[str, str] = { + "normalization_contraction_aux_be": "contraction_aux_be", + "normalization_contraction_aux_have": "contraction_aux_have", + "normalization_contraction_modal_will": "contraction_modal_will", + "normalization_contraction_modal_would": "contraction_modal_would", + "normalization_contraction_negation_not": "contraction_negation_not", + "normalization_contraction_let_us": "contraction_let_us", +} + +_ENVIRONMENT_KEYS: Dict[str, str] = { + "llm_base_url": "ABOGEN_LLM_BASE_URL", + "llm_api_key": "ABOGEN_LLM_API_KEY", + "llm_model": "ABOGEN_LLM_MODEL", + "llm_timeout": "ABOGEN_LLM_TIMEOUT", + "llm_prompt": "ABOGEN_LLM_PROMPT", + "llm_context_mode": "ABOGEN_LLM_CONTEXT_MODE", +} + +NORMALIZATION_SAMPLE_TEXTS: Dict[str, str] = { + "apostrophes": "I've heard the captain'll arrive by dusk, but they'd said the same yesterday.", + "numbers": "The ledger listed 1,204 outstanding debts totaling $57,890.", + "titles": "Dr. Smith met Mr. O'Leary outside St. John's Church on Jan. 4th.", + "punctuation": "Meet me at the docks tonight We'll decide then", # missing punctuation +} + + +@lru_cache(maxsize=1) +def _environment_defaults() -> Dict[str, Any]: + overrides: Dict[str, Any] = {} + for key, env_var in _ENVIRONMENT_KEYS.items(): + default = _SETTINGS_DEFAULTS.get(key) + if default is None: + continue + value = os.environ.get(env_var) + if value is None or value == "": + continue + if isinstance(default, bool): + overrides[key] = _coerce_bool(value, default) + elif isinstance(default, float): + overrides[key] = _coerce_float(value, float(default)) + else: + overrides[key] = value + return overrides + + +def environment_llm_defaults() -> Dict[str, Any]: + defaults = dict(_environment_defaults()) + if defaults: + _apply_llm_migrations(defaults) + return defaults + + +def _coerce_bool(value: Any, default: bool) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, str): + lowered = value.strip().lower() + if lowered in {"1", "true", "yes", "on"}: + return True + if lowered in {"0", "false", "no", "off"}: + return False + return default + + +def _coerce_float(value: Any, default: float) -> float: + try: + return float(value) + except (TypeError, ValueError): + return default + + +def _apply_llm_migrations(settings: Dict[str, Any]) -> None: + prompt_value = str(settings.get("llm_prompt") or "") + if prompt_value.strip() == _LEGACY_REWRITE_ONLY_PROMPT.strip(): + settings["llm_prompt"] = DEFAULT_LLM_PROMPT + + context_mode = str(settings.get("llm_context_mode") or "").strip().lower() + if context_mode != "sentence": + settings["llm_context_mode"] = "sentence" + + +def _extract_settings(source: Mapping[str, Any]) -> Dict[str, Any]: + env_defaults = _environment_defaults() + extracted: Dict[str, Any] = {} + for key, default in _SETTINGS_DEFAULTS.items(): + if key in source: + raw_value = source.get(key) + elif key in env_defaults: + raw_value = env_defaults[key] + else: + raw_value = default + if isinstance(default, bool): + extracted[key] = _coerce_bool(raw_value, default) + elif isinstance(default, float): + extracted[key] = _coerce_float(raw_value, default) + else: + extracted[key] = ( + str(raw_value or "") if isinstance(default, str) else raw_value + ) + _apply_llm_migrations(extracted) + return extracted + + +@lru_cache(maxsize=1) +def _cached_settings() -> Dict[str, Any]: + config = load_config() or {} + return _extract_settings(config) + + +def get_runtime_settings() -> Dict[str, Any]: + return dict(_cached_settings()) + + +def clear_cached_settings() -> None: + _cached_settings.cache_clear() + + +def build_apostrophe_config( + *, + settings: Mapping[str, Any], + base: Optional[ApostropheConfig] = None, +) -> ApostropheConfig: + config = replace(base or ApostropheConfig()) + config.convert_numbers = bool(settings.get("normalization_numbers", True)) + config.convert_currency = bool(settings.get("normalization_currency", True)) + config.remove_footnotes = bool(settings.get("normalization_footnotes", True)) + config.year_pronunciation_mode = ( + str(settings.get("normalization_numbers_year_style", "american") or "") + .strip() + .lower() + ) + config.add_phoneme_hints = bool(settings.get("normalization_phoneme_hints", True)) + config.contraction_mode = ( + "expand" + if settings.get("normalization_apostrophes_contractions", True) + else "keep" + ) + config.plural_possessive_mode = ( + "collapse" + if settings.get("normalization_apostrophes_plural_possessives", True) + else "keep" + ) + config.sibilant_possessive_mode = ( + "mark" + if settings.get("normalization_apostrophes_sibilant_possessives", True) + else "keep" + ) + config.decades_mode = ( + "expand" if settings.get("normalization_apostrophes_decades", True) else "keep" + ) + config.leading_elision_mode = ( + "expand" + if settings.get("normalization_apostrophes_leading_elisions", True) + else "keep" + ) + config.ambiguous_past_modal_mode = ( + "contextual" if config.contraction_mode == "expand" else "keep" + ) + category_flags = dict(CONTRACTION_CATEGORY_DEFAULTS) + for setting_key, category in _CONTRACTION_SETTING_MAP.items(): + default_value = bool(_SETTINGS_DEFAULTS.get(setting_key, True)) + raw_value = settings.get(setting_key, default_value) + category_flags[category] = _coerce_bool(raw_value, default_value) + config.contraction_categories = category_flags + return config + + +def build_llm_configuration(settings: Mapping[str, Any]) -> LLMConfiguration: + return LLMConfiguration( + base_url=str(settings.get("llm_base_url") or ""), + api_key=str(settings.get("llm_api_key") or ""), + model=str(settings.get("llm_model") or ""), + timeout=_coerce_float( + settings.get("llm_timeout"), float(_SETTINGS_DEFAULTS["llm_timeout"]) + ), + ) + + +def apply_overrides( + base: Mapping[str, Any], overrides: Mapping[str, Any] +) -> Dict[str, Any]: + merged: Dict[str, Any] = dict(base) + for key, value in overrides.items(): + if key not in _SETTINGS_DEFAULTS: + continue + merged[key] = value + _apply_llm_migrations(merged) + return merged diff --git a/abogen/pronunciation_store.py b/abogen/pronunciation_store.py new file mode 100644 index 0000000..5a5e7fb --- /dev/null +++ b/abogen/pronunciation_store.py @@ -0,0 +1,256 @@ +from __future__ import annotations + +import json +import sqlite3 +import shutil +import threading +import time +import uuid +from pathlib import Path +from typing import Any, Dict, Iterable, List, Mapping, Optional + +from .entity_analysis import normalize_token +from .utils import get_internal_cache_path, get_user_settings_dir + +_DB_LOCK = threading.RLock() +_SCHEMA_VERSION = 1 + + +def _store_path() -> Path: + try: + base_dir = Path(get_user_settings_dir()) + except ModuleNotFoundError: + base_dir = Path(get_internal_cache_path("pronunciations")) + target = base_dir / "overrides.json" + target.parent.mkdir(parents=True, exist_ok=True) + return target + + +def _migrate_legacy_sqlite(target_json_path: Path) -> None: + try: + base_dir = Path(get_user_settings_dir()) + except ModuleNotFoundError: + base_dir = Path(get_internal_cache_path("pronunciations")) + + sqlite_path = base_dir / "pronunciations.db" + if not sqlite_path.exists(): + return + + try: + conn = sqlite3.connect(sqlite_path) + conn.row_factory = sqlite3.Row + + # Check if table exists + cursor = conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name='overrides'" + ) + if not cursor.fetchone(): + conn.close() + return + + cursor = conn.execute("SELECT * FROM overrides") + rows = cursor.fetchall() + + data = {"version": _SCHEMA_VERSION, "overrides": {}} + + for row in rows: + lang = row["language"] + if lang not in data["overrides"]: + data["overrides"][lang] = {} + + entry = { + "id": str(row["id"]), + "normalized": row["normalized"], + "token": row["token"], + "language": row["language"], + "pronunciation": row["pronunciation"], + "voice": row["voice"], + "notes": row["notes"], + "context": row["context"], + "usage_count": row["usage_count"], + "created_at": row["created_at"], + "updated_at": row["updated_at"], + } + data["overrides"][lang][row["normalized"]] = entry + + conn.close() + + # Save to JSON + with open(target_json_path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + # Rename old DB + sqlite_path.rename(sqlite_path.with_suffix(".db.bak")) + + except Exception: + pass + + +def _load_db() -> Dict[str, Any]: + path = _store_path() + if not path.exists(): + _migrate_legacy_sqlite(path) + if not path.exists(): + return {"version": _SCHEMA_VERSION, "overrides": {}} + try: + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + except (json.JSONDecodeError, OSError): + return {"version": _SCHEMA_VERSION, "overrides": {}} + + +def _save_db(data: Dict[str, Any]) -> None: + path = _store_path() + # Atomic write + temp_path = path.with_suffix(".tmp") + with open(temp_path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + shutil.move(str(temp_path), str(path)) + + +def load_overrides(language: str, tokens: Iterable[str]) -> Dict[str, Dict[str, Any]]: + normalized_tokens = {normalize_token(token) for token in tokens if token} + if not normalized_tokens: + return {} + + with _DB_LOCK: + db = _load_db() + lang_overrides = db.get("overrides", {}).get(language, {}) + + results: Dict[str, Dict[str, Any]] = {} + for normalized in normalized_tokens: + if normalized in lang_overrides: + results[normalized] = lang_overrides[normalized] + return results + + +def search_overrides( + language: str, query: str, *, limit: int = 15 +) -> List[Dict[str, Any]]: + if not query: + return [] + + query = query.lower() + with _DB_LOCK: + db = _load_db() + lang_overrides = db.get("overrides", {}).get(language, {}) + + matches = [] + for entry in lang_overrides.values(): + if query in entry["normalized"] or query in entry["token"].lower(): + matches.append(entry) + + # Sort by usage count desc, then updated_at desc + matches.sort( + key=lambda x: (x.get("usage_count", 0), x.get("updated_at", 0)), + reverse=True, + ) + return matches[:limit] + + +def save_override( + *, + language: str, + token: str, + pronunciation: Optional[str] = None, + voice: Optional[str] = None, + notes: Optional[str] = None, + context: Optional[str] = None, +) -> Dict[str, Any]: + normalized = normalize_token(token) + if not normalized: + raise ValueError("Provide a token to override") + + timestamp = time.time() + with _DB_LOCK: + db = _load_db() + overrides = db.setdefault("overrides", {}) + lang_overrides = overrides.setdefault(language, {}) + + existing = lang_overrides.get(normalized) + + if existing: + entry = existing + entry["token"] = token + entry["pronunciation"] = pronunciation + entry["voice"] = voice + entry["notes"] = notes + entry["context"] = context + entry["updated_at"] = timestamp + else: + entry = { + "id": str(uuid.uuid4()), + "normalized": normalized, + "token": token, + "language": language, + "pronunciation": pronunciation, + "voice": voice, + "notes": notes, + "context": context, + "usage_count": 0, + "created_at": timestamp, + "updated_at": timestamp, + } + lang_overrides[normalized] = entry + + _save_db(db) + return entry + + +def delete_override(*, language: str, token: str) -> None: + normalized = normalize_token(token) + if not normalized: + return + + with _DB_LOCK: + db = _load_db() + lang_overrides = db.get("overrides", {}).get(language, {}) + + if normalized in lang_overrides: + del lang_overrides[normalized] + _save_db(db) + + +def all_overrides(language: str) -> List[Dict[str, Any]]: + with _DB_LOCK: + db = _load_db() + lang_overrides = db.get("overrides", {}).get(language, {}) + + results = list(lang_overrides.values()) + results.sort(key=lambda x: x.get("updated_at", 0), reverse=True) + return results + + +def increment_usage(*, language: str, token: str, amount: int = 1) -> None: + normalized = normalize_token(token) + if not normalized: + return + + with _DB_LOCK: + db = _load_db() + lang_overrides = db.get("overrides", {}).get(language, {}) + + if normalized in lang_overrides: + entry = lang_overrides[normalized] + entry["usage_count"] = entry.get("usage_count", 0) + amount + entry["updated_at"] = time.time() + _save_db(db) + + +def get_override_stats(language: str) -> Dict[str, int]: + with _DB_LOCK: + db = _load_db() + lang_overrides = db.get("overrides", {}).get(language, {}) + + total = len(lang_overrides) + with_pronunciation = sum( + 1 for x in lang_overrides.values() if x.get("pronunciation") + ) + with_voice = sum(1 for x in lang_overrides.values() if x.get("voice")) + + return { + "total": total, + "filtered": total, + "with_pronunciation": with_pronunciation, + "with_voice": with_voice, + } diff --git a/abogen/pyqt/__init__.py b/abogen/pyqt/__init__.py new file mode 100644 index 0000000..cc5cb14 --- /dev/null +++ b/abogen/pyqt/__init__.py @@ -0,0 +1,7 @@ +"""PyQt6 Desktop GUI for abogen. + +This package contains the traditional PyQt6-based desktop interface. +For the web-based interface, see abogen.webui. +""" + +from __future__ import annotations diff --git a/abogen/book_handler.py b/abogen/pyqt/book_handler.py similarity index 50% rename from abogen/book_handler.py rename to abogen/pyqt/book_handler.py index b8073c1..fc1d806 100644 --- a/abogen/book_handler.py +++ b/abogen/pyqt/book_handler.py @@ -1,8 +1,5 @@ import re -import ebooklib import base64 -import fitz # PyMuPDF for PDF support -from ebooklib import epub from bs4 import BeautifulSoup, NavigableString from PyQt6.QtGui import QMovie from PyQt6.QtWidgets import ( @@ -31,6 +28,7 @@ from abogen.utils import ( detect_encoding, get_resource_path, ) +from abogen.book_parser import get_book_parser from abogen.subtitle_utils import ( clean_text, @@ -38,9 +36,8 @@ from abogen.subtitle_utils import ( ) import os -import logging # Add logging +import logging import urllib.parse -import markdown import textwrap # Setup logging @@ -48,13 +45,6 @@ logging.basicConfig( level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" ) -# Pre-compile frequently used regex patterns for better performance -_BRACKETED_NUMBERS_PATTERN = re.compile(r"\[\s*\d+\s*\]") -_STANDALONE_PAGE_NUMBERS_PATTERN = re.compile(r"^\s*\d+\s*$", re.MULTILINE) -_PAGE_NUMBERS_AT_END_PATTERN = re.compile(r"\s+\d+\s*$", re.MULTILINE) -_PAGE_NUMBERS_WITH_DASH_PATTERN = re.compile( - r"\s+[-–—]\s*\d+\s*[-–—]?\s*$", re.MULTILINE -) _HTML_TAG_PATTERN = re.compile(r"<[^>]+>") _LEADING_DASH_PATTERN = re.compile(r"^\s*[-–—]\s*") _LEADING_SIMPLE_DASH_PATTERN = re.compile(r"^\s*-\s*") @@ -106,23 +96,22 @@ class HandlerDialog(QDialog): # Normalize path book_path = os.path.normpath(os.path.abspath(book_path)) - - # Determine file type if not explicitly provided - if file_type: - self.file_type = file_type - elif book_path.lower().endswith(".pdf"): - self.file_type = "pdf" - elif book_path.lower().endswith((".md", ".markdown")): - self.file_type = "markdown" - else: - self.file_type = "epub" self.book_path = book_path + # Initialize Parser + try: + # Factory handles file type detection if file_type is None + self.parser = get_book_parser(book_path, file_type=file_type) + # Parser loads automatically in init now + except Exception as e: + logging.error(f"Failed to initialize parser for {book_path}: {e}") + raise + # Extract book name from file path book_name = os.path.splitext(os.path.basename(book_path))[0] # Set window title based on file type and book name - item_type = "Chapters" if self.file_type in ["epub", "markdown"] else "Pages" + item_type = "Chapters" if self.parser.file_type in ["epub", "markdown"] else "Pages" self.setWindowTitle(f"Select {item_type} - {book_name}") self.resize(1200, 900) self._block_signals = False # Flag to prevent recursive signals @@ -138,57 +127,8 @@ class HandlerDialog(QDialog): self.merge_chapters_at_end = HandlerDialog._merge_chapters_at_end self.save_as_project = HandlerDialog._save_as_project - # Load the book based on file type - try: - if self.file_type == "epub": - self.book = epub.read_epub(book_path) - elif self.file_type == "markdown": - self.book = None # Markdown doesn't use ebooklib - else: - self.book = None - except KeyError as e: - logging.error( - f"EPUB file is missing a referenced file: {e}. Skipping missing file." - ) - # Try to patch ebooklib to skip missing files (monkey-patch read_file) - import types - - orig_read_file = None - try: - from ebooklib import epub as _epub_module - - reader_class = _epub_module.EpubReader - orig_read_file = reader_class.read_file - - def safe_read_file(self, name): - try: - return orig_read_file(self, name) - except KeyError: - logging.warning( - f"Missing file in EPUB: {name}. Returning empty bytes." - ) - return b"" - - reader_class.read_file = safe_read_file - self.book = epub.read_epub(book_path) - reader_class.read_file = orig_read_file # Restore - except Exception as patch_e: - logging.error(f"Failed to patch ebooklib for missing files: {patch_e}") - raise e - self.pdf_doc = fitz.open(book_path) if self.file_type == "pdf" else None - self.markdown_text = None - if self.file_type == "markdown": - try: - encoding = detect_encoding(book_path) - with open(book_path, "r", encoding=encoding, errors="replace") as f: - self.markdown_text = f.read() - except Exception as e: - logging.error(f"Error reading markdown file: {e}") - self.markdown_text = "" - self.markdown_toc = [] # For storing parsed markdown TOC - - # Extract book metadata - self.book_metadata = self._extract_book_metadata() + # Initialize metadata dict; will be populated in _preprocess_content by the background loader + self.book_metadata = {} # Initialize UI elements that are used in other methods self.save_chapters_checkbox = None @@ -207,6 +147,8 @@ class HandlerDialog(QDialog): # For storing content and lengths (will be filled by background loader) self.content_texts = {} self.content_lengths = {} + # Also maintain refs for structure + self.processed_nav_structure = [] # Add a placeholder "Information" item so the tree isn't empty immediately info_item = QTreeWidgetItem(self.treeWidget, ["Information"]) @@ -396,787 +338,51 @@ class HandlerDialog(QDialog): cfg = load_config() replace_single_newlines = cfg.get("replace_single_newlines", True) - cache_key = (self.book_path, mod_time, self.file_type, replace_single_newlines) + cache_key = (self.book_path, mod_time, self.parser.file_type, replace_single_newlines) # Check if content is already cached if cache_key in HandlerDialog._content_cache: cached_data = HandlerDialog._content_cache[cache_key] self.content_texts = cached_data["content_texts"] self.content_lengths = cached_data["content_lengths"] - if "doc_content" in cached_data: - self.doc_content = cached_data["doc_content"] - if "markdown_toc" in cached_data: - self.markdown_toc = cached_data["markdown_toc"] + if "processed_nav_structure" in cached_data: + self.processed_nav_structure = cached_data["processed_nav_structure"] + if "book_metadata" in cached_data: + self.book_metadata = cached_data["book_metadata"] + + # Apply to parser so it stays in sync if used elsewhere + self.parser.content_texts = self.content_texts + self.parser.content_lengths = self.content_lengths + self.parser.processed_nav_structure = self.processed_nav_structure + self.parser.book_metadata = self.book_metadata + logging.info(f"Using cached content for {os.path.basename(self.book_path)}") return # Process content if not cached - if self.file_type == "epub": - try: - self._process_epub_content_nav() # Use the new navigation-based method - except Exception as e: - logging.error( - f"Error processing EPUB with navigation: {e}. Falling back to TOC/spine.", - exc_info=True, - ) - # Fallback to a simpler spine-based processing if nav fails - self._process_epub_content_spine_fallback() - elif self.file_type == "markdown": - self._preprocess_markdown_content() - else: - self._preprocess_pdf_content() + try: + self.parser.process_content(replace_single_newlines=replace_single_newlines) + self.content_texts = self.parser.content_texts + self.content_lengths = self.parser.content_lengths + self.processed_nav_structure = self.parser.processed_nav_structure + self.book_metadata = self.parser.get_metadata() + except Exception as e: + logging.error(f"Error processing content: {e}", exc_info=True) + # Handle empty/failure case + self.content_texts = {} + self.content_lengths = {} # Cache the processed content cache_data = { "content_texts": self.content_texts, "content_lengths": self.content_lengths, + "processed_nav_structure": self.processed_nav_structure, + "book_metadata": self.book_metadata, } - if hasattr(self, "doc_content"): - cache_data["doc_content"] = self.doc_content - if hasattr(self, "markdown_toc"): - cache_data["markdown_toc"] = self.markdown_toc HandlerDialog._content_cache[cache_key] = cache_data logging.info(f"Cached content for {os.path.basename(self.book_path)}") - def _preprocess_pdf_content(self): - """Pre-process all page contents from PDF document""" - for page_num in range(len(self.pdf_doc)): - text = clean_text(self.pdf_doc[page_num].get_text()) - # Remove bracketed numbers, page numbers, etc. using pre-compiled patterns - # Combine all regex operations for better performance - text = _BRACKETED_NUMBERS_PATTERN.sub("", text) - text = _STANDALONE_PAGE_NUMBERS_PATTERN.sub("", text) - text = _PAGE_NUMBERS_AT_END_PATTERN.sub("", text) - text = _PAGE_NUMBERS_WITH_DASH_PATTERN.sub("", text) - - page_id = f"page_{page_num + 1}" - self.content_texts[page_id] = text - self.content_lengths[page_id] = calculate_text_length(text) - - def _preprocess_markdown_content(self): - if not self.markdown_text: - return - - # Generate TOC from the original (dedented) markdown BEFORE cleaning, - # so header ids/anchors are preserved for reliable position detection. - original_text = textwrap.dedent(self.markdown_text) - md = markdown.Markdown(extensions=["toc", "fenced_code"]) - html = md.convert(original_text) - self.markdown_toc = md.toc_tokens - - # Use cleaned text for stored content/length calculations - cleaned_full_text = clean_text(original_text) - - soup = BeautifulSoup(html, "html.parser") - self.content_texts = {} - self.content_lengths = {} - - if not self.markdown_toc: - chapter_id = "markdown_content" - self.content_texts[chapter_id] = cleaned_full_text - self.content_lengths[chapter_id] = calculate_text_length(cleaned_full_text) - return - - all_headers = [] - - def flatten_toc(toc_list): - for header in toc_list: - all_headers.append(header) - if header.get("children"): - flatten_toc(header["children"]) - - flatten_toc(self.markdown_toc) - - header_positions = [] - for header in all_headers: - header_id = header["id"] - id_pattern = f'id="{header_id}"' - pos = html.find(id_pattern) - if pos != -1: - tag_start = html.rfind("<", 0, pos) - header_positions.append( - {"id": header_id, "start": tag_start, "name": header["name"]} - ) - header_positions.sort(key=lambda x: x["start"]) - - for i, header_pos in enumerate(header_positions): - header_id = header_pos["id"] - header_name = header_pos["name"] - content_start = header_pos["start"] - content_end = ( - header_positions[i + 1]["start"] - if i + 1 < len(header_positions) - else len(html) - ) - section_html = html[content_start:content_end] - section_soup = BeautifulSoup(section_html, "html.parser") - header_tag = section_soup.find(attrs={"id": header_id}) - if header_tag: - header_tag.decompose() - # Clean section text for storage/lengths - section_text = clean_text(section_soup.get_text()).strip() - chapter_id = header_id - if section_text: - full_content = f"{header_name}\n\n{section_text}" - self.content_texts[chapter_id] = full_content - self.content_lengths[chapter_id] = calculate_text_length(full_content) - else: - self.content_texts[chapter_id] = header_name - self.content_lengths[chapter_id] = calculate_text_length(header_name) - - def _process_epub_content_spine_fallback(self): - """Fallback EPUB processing based purely on spine order.""" - logging.info("Using spine fallback for EPUB processing.") - self.doc_content = {} - spine_docs = [] - for spine_item_tuple in self.book.spine: - item_id = spine_item_tuple[0] - item = self.book.get_item_with_id(item_id) - if item: - spine_docs.append(item.get_name()) - else: - logging.warning(f"Spine item with id '{item_id}' not found.") - - # Cache content - for item in self.book.get_items_of_type(ebooklib.ITEM_DOCUMENT): - href = item.get_name() - if href in spine_docs: - try: - html_content = item.get_content().decode("utf-8", errors="ignore") - self.doc_content[href] = html_content - except Exception as e: - logging.error(f"Error decoding content for {href}: {e}") - self.doc_content[href] = "" - - # Create a simple TOC based on spine order - synthetic_toc = [] - self.content_texts = {} - self.content_lengths = {} - for i, doc_href in enumerate(spine_docs): - html_content = self.doc_content.get(doc_href, "") - if html_content: - soup = BeautifulSoup(html_content, "html.parser") - - # Handle ordered lists by prepending numbers to list items - for ol in soup.find_all("ol"): - # Get start attribute or default to 1 - start = int(ol.get("start", 1)) - for i, li in enumerate(ol.find_all("li", recursive=False)): - # Insert the number at the beginning of the list item - number_text = f"{start + i}) " - if li.string: - li.string.replace_with(number_text + li.string) - else: - li.insert(0, NavigableString(number_text)) - - # Remove sup and sub tags - for tag in soup.find_all(["sup", "sub"]): - tag.decompose() - - text = clean_text(soup.get_text()).strip() - if text: - self.content_texts[doc_href] = text - self.content_lengths[doc_href] = calculate_text_length(text) - - title = None - if soup.title and soup.title.string: - title = soup.title.string.strip() - elif (h1 := soup.find("h1")) and h1.get_text(strip=True): - title = h1.get_text(strip=True) - - if not title: - title = f"Untitled Chapter {i + 1}" - synthetic_toc.append( - (epub.Link(doc_href, title, doc_href), []) - ) # Wrap in tuple and empty list for compatibility - - # Replace book.toc with the synthetic one if it was empty or fallback was triggered - if not self.book.toc or not hasattr( - self, "processed_nav_structure" - ): # Check if nav processing failed - self.book.toc = synthetic_toc - logging.info(f"Generated synthetic TOC with {len(synthetic_toc)} entries.") - - def _process_epub_content_nav(self): - """ - Process EPUB content using ITEM_NAVIGATION (NAV HTML) or ITEM_NCX. - Globally orders navigation entries and slices content between them. - """ - logging.info( - "Attempting to process EPUB using navigation document (NAV/NCX)..." - ) - nav_item = None - nav_type = None - - # 1. Check ITEM_NAVIGATION for actual NAV HTML (.xhtml/.html) - nav_items = list(self.book.get_items_of_type(ebooklib.ITEM_NAVIGATION)) - if nav_items: - # Prefer files explicitly named 'nav.xhtml' or similar - preferred_nav = next( - ( - item - for item in nav_items - if "nav" in item.get_name().lower() - and item.get_name().lower().endswith((".xhtml", ".html")) - ), - None, - ) - if preferred_nav: - nav_item = preferred_nav - nav_type = "html" - logging.info(f"Found preferred NAV HTML item: {nav_item.get_name()}") - else: - # Check if any ITEM_NAVIGATION is actually HTML - html_nav = next( - ( - item - for item in nav_items - if item.get_name().lower().endswith((".xhtml", ".html")) - ), - None, - ) - if html_nav: - nav_item = html_nav - nav_type = "html" - logging.info( - f"Found NAV HTML item in ITEM_NAVIGATION: {html_nav.get_name()}" - ) - - # 2. If no NAV HTML found via ITEM_NAVIGATION, check if ITEM_NAVIGATION points to NCX - if not nav_item and nav_items: - ncx_in_nav = next( - ( - item - for item in nav_items - if item.get_name().lower().endswith(".ncx") - ), - None, - ) - if ncx_in_nav: - nav_item = ncx_in_nav - nav_type = "ncx" - logging.info( - f"Found NCX item via ITEM_NAVIGATION: {ncx_in_nav.get_name()}" - ) - - # 3. If still no nav_item, check for NCX or fallback to NAV HTML in all ITEM_DOCUMENTs - ncx_constant = getattr(epub, "ITEM_NCX", None) - if not nav_item and ncx_constant is not None: - ncx_items = list(self.book.get_items_of_type(ncx_constant)) - if ncx_items: - nav_item = ncx_items[0] - nav_type = "ncx" - logging.info(f"Found NCX item via ITEM_NCX: {nav_item.get_name()}") - # Fallback: search all ITEM_DOCUMENTs for a NAV HTML with