mirror of
https://github.com/denizsafak/abogen.git
synced 2026-07-18 13:40:27 +02:00
fix: Enhance author handling in metadata payload and add currency conversion support in normalization
This commit is contained in:
@@ -156,6 +156,14 @@ class AudiobookshelfClient:
|
|||||||
metadata_payload["chapters"] = list(chapters)
|
metadata_payload["chapters"] = list(chapters)
|
||||||
|
|
||||||
if metadata_payload:
|
if metadata_payload:
|
||||||
|
# Ensure authors is a list of strings in the JSON payload if it exists
|
||||||
|
if "authors" in metadata_payload:
|
||||||
|
authors_val = metadata_payload["authors"]
|
||||||
|
if isinstance(authors_val, str):
|
||||||
|
metadata_payload["authors"] = [a.strip() for a in authors_val.split(",") if a.strip()]
|
||||||
|
elif isinstance(authors_val, list):
|
||||||
|
metadata_payload["authors"] = [str(a).strip() for a in authors_val if str(a).strip()]
|
||||||
|
|
||||||
try:
|
try:
|
||||||
fields["metadata"] = json.dumps(metadata_payload, ensure_ascii=False)
|
fields["metadata"] = json.dumps(metadata_payload, ensure_ascii=False)
|
||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
@@ -603,6 +611,7 @@ class AudiobookshelfClient:
|
|||||||
if isinstance(authors, Iterable) and not isinstance(authors, (str, Mapping)):
|
if isinstance(authors, Iterable) and not isinstance(authors, (str, Mapping)):
|
||||||
names = [str(entry).strip() for entry in authors if isinstance(entry, str) and entry.strip()]
|
names = [str(entry).strip() for entry in authors if isinstance(entry, str) and entry.strip()]
|
||||||
if names:
|
if names:
|
||||||
|
# ABS expects a comma-separated string for multiple authors.
|
||||||
return ", ".join(names)
|
return ", ".join(names)
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
|
|||||||
@@ -6,10 +6,20 @@ import unicodedata
|
|||||||
from fractions import Fraction
|
from fractions import Fraction
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple
|
from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple
|
||||||
|
import logging
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
try: # pragma: no cover - optional dependency guard
|
try: # pragma: no cover - optional dependency guard
|
||||||
from num2words import num2words
|
from num2words import num2words
|
||||||
except Exception: # pragma: no cover - graceful degradation
|
except ImportError:
|
||||||
num2words = None # type: ignore
|
num2words = None
|
||||||
|
logger.warning("num2words library not found. Number normalization will be disabled.")
|
||||||
|
except Exception as e: # pragma: no cover - graceful degradation
|
||||||
|
num2words = None
|
||||||
|
logger.error(f"Failed to import num2words: {e}")
|
||||||
|
|
||||||
|
HAS_NUM2WORDS = num2words is not None
|
||||||
|
|
||||||
if TYPE_CHECKING: # pragma: no cover - type checking only
|
if TYPE_CHECKING: # pragma: no cover - type checking only
|
||||||
from abogen.llm_client import LLMCompletion
|
from abogen.llm_client import LLMCompletion
|
||||||
@@ -48,6 +58,7 @@ class ApostropheConfig:
|
|||||||
lowercase_for_matching: bool = True # Normalize to lower for rule matching (not output)
|
lowercase_for_matching: bool = True # Normalize to lower for rule matching (not output)
|
||||||
protect_cultural_names: bool = True # Always keep O'Brien, D'Angelo, etc.
|
protect_cultural_names: bool = True # Always keep O'Brien, D'Angelo, etc.
|
||||||
convert_numbers: bool = True # Convert grouped numbers such as 12,500 to words
|
convert_numbers: bool = True # Convert grouped numbers such as 12,500 to words
|
||||||
|
convert_currency: bool = True # Convert currency symbols to words
|
||||||
number_lang: str = "en" # num2words language code
|
number_lang: str = "en" # num2words language code
|
||||||
year_pronunciation_mode: str = "american" # off|american (extend if needed)
|
year_pronunciation_mode: str = "american" # off|american (extend if needed)
|
||||||
contraction_categories: Dict[str, bool] = field(default_factory=lambda: dict(CONTRACTION_CATEGORY_DEFAULTS))
|
contraction_categories: Dict[str, bool] = field(default_factory=lambda: dict(CONTRACTION_CATEGORY_DEFAULTS))
|
||||||
@@ -107,6 +118,11 @@ _FRACTION_SLASH_CLASS = re.escape(_FRACTION_SLASHES)
|
|||||||
_FRACTION_RE = re.compile(
|
_FRACTION_RE = re.compile(
|
||||||
rf"(?<!\w)(?P<numerator>-?\d+)\s*[{_FRACTION_SLASH_CLASS}]\s*(?P<denominator>-?\d+)(?![\w{_FRACTION_SLASH_CLASS}])"
|
rf"(?<!\w)(?P<numerator>-?\d+)\s*[{_FRACTION_SLASH_CLASS}]\s*(?P<denominator>-?\d+)(?![\w{_FRACTION_SLASH_CLASS}])"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
_CURRENCY_RE = re.compile(
|
||||||
|
r"(?P<symbol>[$£€¥])\s*(?P<amount>\d{1,3}(?:,\d{3})*(?:\.\d{1,2})?)(?!\d)"
|
||||||
|
)
|
||||||
|
|
||||||
_DECIMAL_NUMBER_RE = re.compile(
|
_DECIMAL_NUMBER_RE = re.compile(
|
||||||
rf"(?<![\w{_NUMBER_RANGE_CLASS}/])(?P<number>-?(?:\d{{1,3}}(?:,\d{{3}})+|\d+)\.(?P<fraction>\d+))(?![\w{_NUMBER_RANGE_CLASS}/])"
|
rf"(?<![\w{_NUMBER_RANGE_CLASS}/])(?P<number>-?(?:\d{{1,3}}(?:,\d{{3}})+|\d+)\.(?P<fraction>\d+))(?![\w{_NUMBER_RANGE_CLASS}/])"
|
||||||
)
|
)
|
||||||
@@ -1478,7 +1494,34 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
|
|||||||
spoken = f"{integer_words} point {' '.join(digit_words)}"
|
spoken = f"{integer_words} point {' '.join(digit_words)}"
|
||||||
return f"minus {spoken}" if is_negative else spoken
|
return f"minus {spoken}" if is_negative else spoken
|
||||||
|
|
||||||
|
def _replace_currency(match: re.Match[str]) -> str:
|
||||||
|
if num2words is None:
|
||||||
|
return match.group(0)
|
||||||
|
|
||||||
|
symbol = match.group("symbol")
|
||||||
|
amount_str = match.group("amount").replace(",", "")
|
||||||
|
try:
|
||||||
|
amount = float(amount_str)
|
||||||
|
except ValueError:
|
||||||
|
return match.group(0)
|
||||||
|
|
||||||
|
currency_map = {
|
||||||
|
"$": "USD",
|
||||||
|
"£": "GBP",
|
||||||
|
"€": "EUR",
|
||||||
|
"¥": "JPY",
|
||||||
|
}
|
||||||
|
currency_code = currency_map.get(symbol, "USD")
|
||||||
|
|
||||||
|
try:
|
||||||
|
words = num2words(amount, to="currency", currency=currency_code, lang=language)
|
||||||
|
return words
|
||||||
|
except Exception:
|
||||||
|
return match.group(0)
|
||||||
|
|
||||||
normalized = text
|
normalized = text
|
||||||
|
if cfg.convert_currency:
|
||||||
|
normalized = _CURRENCY_RE.sub(_replace_currency, normalized)
|
||||||
normalized = _NUMBER_RANGE_RE.sub(lambda m: _replace_number_range(m, language), normalized)
|
normalized = _NUMBER_RANGE_RE.sub(lambda m: _replace_number_range(m, language), normalized)
|
||||||
normalized = _NUMBER_SPACE_RANGE_RE.sub(lambda m: _replace_space_separated_range(m, language), normalized)
|
normalized = _NUMBER_SPACE_RANGE_RE.sub(lambda m: _replace_space_separated_range(m, language), normalized)
|
||||||
normalized = _FRACTION_RE.sub(lambda m: _replace_fraction(m, language), normalized)
|
normalized = _FRACTION_RE.sub(lambda m: _replace_fraction(m, language), normalized)
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ _SETTINGS_DEFAULTS: Dict[str, Any] = {
|
|||||||
"llm_context_mode": "sentence",
|
"llm_context_mode": "sentence",
|
||||||
"normalization_numbers": True,
|
"normalization_numbers": True,
|
||||||
"normalization_numbers_year_style": "american",
|
"normalization_numbers_year_style": "american",
|
||||||
|
"normalization_currency": True,
|
||||||
"normalization_titles": True,
|
"normalization_titles": True,
|
||||||
"normalization_terminal": True,
|
"normalization_terminal": True,
|
||||||
"normalization_phoneme_hints": True,
|
"normalization_phoneme_hints": True,
|
||||||
@@ -172,6 +173,7 @@ def build_apostrophe_config(
|
|||||||
) -> ApostropheConfig:
|
) -> ApostropheConfig:
|
||||||
config = replace(base or ApostropheConfig())
|
config = replace(base or ApostropheConfig())
|
||||||
config.convert_numbers = bool(settings.get("normalization_numbers", True))
|
config.convert_numbers = bool(settings.get("normalization_numbers", True))
|
||||||
|
config.convert_currency = bool(settings.get("normalization_currency", True))
|
||||||
config.year_pronunciation_mode = str(settings.get("normalization_numbers_year_style", "american") or "").strip().lower()
|
config.year_pronunciation_mode = str(settings.get("normalization_numbers_year_style", "american") or "").strip().lower()
|
||||||
config.add_phoneme_hints = bool(settings.get("normalization_phoneme_hints", True))
|
config.add_phoneme_hints = bool(settings.get("normalization_phoneme_hints", True))
|
||||||
config.contraction_mode = "expand" if settings.get("normalization_apostrophes_contractions", True) else "keep"
|
config.contraction_mode = "expand" if settings.get("normalization_apostrophes_contractions", True) else "keep"
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ import static_ffmpeg
|
|||||||
|
|
||||||
from abogen.constants import VOICES_INTERNAL
|
from abogen.constants import VOICES_INTERNAL
|
||||||
from abogen.epub3.exporter import build_epub3_package
|
from abogen.epub3.exporter import build_epub3_package
|
||||||
from abogen.kokoro_text_normalization import ApostropheConfig, normalize_for_pipeline
|
from abogen.kokoro_text_normalization import ApostropheConfig, normalize_for_pipeline, HAS_NUM2WORDS
|
||||||
from abogen.normalization_settings import (
|
from abogen.normalization_settings import (
|
||||||
build_apostrophe_config,
|
build_apostrophe_config,
|
||||||
build_llm_configuration,
|
build_llm_configuration,
|
||||||
@@ -125,7 +125,30 @@ def _simplify_heading_text(text: str) -> str:
|
|||||||
def _headings_equivalent(left: str, right: str) -> bool:
|
def _headings_equivalent(left: str, right: str) -> bool:
|
||||||
simple_left = _simplify_heading_text(left)
|
simple_left = _simplify_heading_text(left)
|
||||||
simple_right = _simplify_heading_text(right)
|
simple_right = _simplify_heading_text(right)
|
||||||
return bool(simple_left and simple_left == simple_right)
|
if not simple_left or not simple_right:
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Exact match
|
||||||
|
if simple_left == simple_right:
|
||||||
|
return True
|
||||||
|
|
||||||
|
# Check if one is a prefix of the other (e.g. "Chapter 2" vs "Chapter 2: The Return")
|
||||||
|
# But be careful not to match "Chapter 1" with "Chapter 10"
|
||||||
|
# _simplify_heading_text removes "chapter" prefix, so we are comparing "2" vs "2thereturn"
|
||||||
|
|
||||||
|
# If left is "2" and right is "2thereturn", left is prefix of right.
|
||||||
|
if simple_right.startswith(simple_left):
|
||||||
|
return True
|
||||||
|
|
||||||
|
# If left is "2thereturn" and right is "2", right is prefix of left.
|
||||||
|
if simple_left.startswith(simple_right):
|
||||||
|
return True
|
||||||
|
|
||||||
|
# Also check if the line is contained in the heading if it's long enough
|
||||||
|
if len(simple_left) > 5 and simple_left in simple_right:
|
||||||
|
return True
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
def _format_spoken_chapter_title(title: str, index: int, apply_prefix: bool) -> str:
|
def _format_spoken_chapter_title(title: str, index: int, apply_prefix: bool) -> str:
|
||||||
@@ -1247,6 +1270,15 @@ def run_conversion_job(job: Job) -> None:
|
|||||||
settings=normalization_settings,
|
settings=normalization_settings,
|
||||||
base=_APOSTROPHE_CONFIG,
|
base=_APOSTROPHE_CONFIG,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if apostrophe_config.convert_numbers and not HAS_NUM2WORDS:
|
||||||
|
job.add_log(
|
||||||
|
"Number normalization is enabled but 'num2words' library is not available. "
|
||||||
|
"Numbers (including years) will NOT be converted to words. "
|
||||||
|
"Please install 'num2words' to enable this feature.",
|
||||||
|
level="warning"
|
||||||
|
)
|
||||||
|
|
||||||
apostrophe_mode = str(normalization_settings.get("normalization_apostrophe_mode", "spacy")).lower()
|
apostrophe_mode = str(normalization_settings.get("normalization_apostrophe_mode", "spacy")).lower()
|
||||||
if apostrophe_mode == "llm":
|
if apostrophe_mode == "llm":
|
||||||
llm_config = build_llm_configuration(normalization_settings)
|
llm_config = build_llm_configuration(normalization_settings)
|
||||||
|
|||||||
Reference in New Issue
Block a user