Reformat using black

This commit is contained in:
Deniz Şafak
2026-01-09 01:36:14 +03:00
parent 3c800df450
commit 5ae153f841
48 changed files with 1530 additions and 894 deletions
+32 -17
View File
@@ -18,7 +18,9 @@ from abogen.subtitle_utils import clean_text, calculate_text_length
_BRACKETED_NUMBERS_PATTERN = re.compile(r"\[\s*\d+\s*\]") _BRACKETED_NUMBERS_PATTERN = re.compile(r"\[\s*\d+\s*\]")
_STANDALONE_PAGE_NUMBERS_PATTERN = re.compile(r"^\s*\d+\s*$", re.MULTILINE) _STANDALONE_PAGE_NUMBERS_PATTERN = re.compile(r"^\s*\d+\s*$", re.MULTILINE)
_PAGE_NUMBERS_AT_END_PATTERN = re.compile(r"\s+\d+\s*$", re.MULTILINE) _PAGE_NUMBERS_AT_END_PATTERN = re.compile(r"\s+\d+\s*$", re.MULTILINE)
_PAGE_NUMBERS_WITH_DASH_PATTERN = re.compile(r"\s+[-–—]\s*\d+\s*[-–—]?\s*$", re.MULTILINE) _PAGE_NUMBERS_WITH_DASH_PATTERN = re.compile(
r"\s+[-–—]\s*\d+\s*[-–—]?\s*$", re.MULTILINE
)
class BaseBookParser(ABC): class BaseBookParser(ABC):
@@ -259,7 +261,7 @@ class MarkdownParser(BaseBookParser):
def get_chapters(self): def get_chapters(self):
chapters = super().get_chapters() chapters = super().get_chapters()
if not chapters and "markdown_content" in self.content_texts: if not chapters and "markdown_content" in self.content_texts:
chapters.append(("markdown_content", "Content")) chapters.append(("markdown_content", "Content"))
return chapters return chapters
@@ -290,7 +292,9 @@ class EpubParser(BaseBookParser):
try: try:
return orig_read_file(self, name) return orig_read_file(self, name)
except KeyError: except KeyError:
logging.warning(f"Missing file in EPUB: {name}. Returning empty bytes.") logging.warning(
f"Missing file in EPUB: {name}. Returning empty bytes."
)
return b"" return b""
reader_class.read_file = safe_read_file reader_class.read_file = safe_read_file
@@ -428,7 +432,9 @@ class EpubParser(BaseBookParser):
if src: if src:
base_href, fragment = src.split("#", 1) if "#" in src else (src, None) base_href, fragment = src.split("#", 1) if "#" in src else (src, None)
doc_key, doc_idx = self._find_doc_key(base_href, doc_order, doc_order_decoded) doc_key, doc_idx = self._find_doc_key(
base_href, doc_order, doc_order_decoded
)
if not doc_key: if not doc_key:
current_entry_node["has_content"] = False current_entry_node["has_content"] = False
else: else:
@@ -458,7 +464,8 @@ class EpubParser(BaseBookParser):
) )
if title and ( if title and (
current_entry_node.get("has_content", False) or current_entry_node["children"] current_entry_node.get("has_content", False)
or current_entry_node["children"]
): ):
tree_structure_list.append(current_entry_node) tree_structure_list.append(current_entry_node)
@@ -484,7 +491,7 @@ class EpubParser(BaseBookParser):
# Second fallback: if we have a span but title is still empty, try span text again # Second fallback: if we have a span but title is still empty, try span text again
# (covered by logic above mostly, but mirroring original logic's intense fallback) # (covered by logic above mostly, but mirroring original logic's intense fallback)
if (not title.strip() or title == "Untitled Section") and span_text: if (not title.strip() or title == "Untitled Section") and span_text:
title = span_text.get_text(strip=True) or title title = span_text.get_text(strip=True) or title
return title return title
@@ -521,7 +528,9 @@ class EpubParser(BaseBookParser):
fragment = None fragment = None
if src: if src:
base_href, fragment = src.split("#", 1) if "#" in src else (src, None) base_href, fragment = src.split("#", 1) if "#" in src else (src, None)
doc_key, doc_idx = self._find_doc_key(base_href, doc_order, doc_order_decoded) doc_key, doc_idx = self._find_doc_key(
base_href, doc_order, doc_order_decoded
)
if doc_key is not None: if doc_key is not None:
position = find_position_func(doc_key, fragment) position = find_position_func(doc_key, fragment)
entry_data = { entry_data = {
@@ -560,14 +569,17 @@ class EpubParser(BaseBookParser):
# 1.1 Support for EPUB 3 EpubNav which might be ITEM_DOCUMENT (9) but with properties=['nav'] # 1.1 Support for EPUB 3 EpubNav which might be ITEM_DOCUMENT (9) but with properties=['nav']
if not nav_items: if not nav_items:
# Look in ITEM_DOCUMENT for items with 'nav' property # Look in ITEM_DOCUMENT for items with 'nav' property
for item in self.book.get_items_of_type(ebooklib.ITEM_DOCUMENT): for item in self.book.get_items_of_type(ebooklib.ITEM_DOCUMENT):
if hasattr(item, 'get_type') and item.get_type() == ebooklib.ITEM_DOCUMENT: if (
# Check properties - ebooklib stores opf properties in list hasattr(item, "get_type")
# Some versions use item.properties, some need checking and item.get_type() == ebooklib.ITEM_DOCUMENT
props = getattr(item, 'properties', []) ):
if 'nav' in props: # Check properties - ebooklib stores opf properties in list
nav_items.append(item) # Some versions use item.properties, some need checking
props = getattr(item, "properties", [])
if "nav" in props:
nav_items.append(item)
if nav_items: if nav_items:
nav_item = next( nav_item = next(
@@ -592,7 +604,11 @@ class EpubParser(BaseBookParser):
# 2. NCX in NAV # 2. NCX in NAV
if not nav_item and nav_items: if not nav_item and nav_items:
ncx_in_nav = next( ncx_in_nav = next(
(item for item in nav_items if item.get_name().lower().endswith(".ncx")), (
item
for item in nav_items
if item.get_name().lower().endswith(".ncx")
),
None, None,
) )
if ncx_in_nav: if ncx_in_nav:
@@ -627,7 +643,6 @@ class EpubParser(BaseBookParser):
return nav_item, nav_type return nav_item, nav_type
def _execute_nav_parsing_logic(self, nav_item, nav_type): def _execute_nav_parsing_logic(self, nav_item, nav_type):
"""Parse the identified navigation item and slice content accordingly.""" """Parse the identified navigation item and slice content accordingly."""
+12 -4
View File
@@ -80,7 +80,9 @@ def _normalize_whitespace(value: str) -> str:
def _normalize_chunk_text(value: str) -> str: def _normalize_chunk_text(value: str) -> str:
settings = get_runtime_settings() settings = get_runtime_settings()
config = build_apostrophe_config(settings=settings, base=_PIPELINE_APOSTROPHE_CONFIG) config = build_apostrophe_config(
settings=settings, base=_PIPELINE_APOSTROPHE_CONFIG
)
normalized = normalize_for_pipeline(value, config=config, settings=settings) normalized = normalize_for_pipeline(value, config=config, settings=settings)
return _normalize_whitespace(normalized) return _normalize_whitespace(normalized)
@@ -158,12 +160,16 @@ def chunk_text(
# Sentence level flatten paragraphs into individual sentences # Sentence level flatten paragraphs into individual sentences
sentence_index = 0 sentence_index = 0
for para_index, paragraph in enumerate(list(_iter_paragraphs(text)) or [text.strip()]): for para_index, paragraph in enumerate(
list(_iter_paragraphs(text)) or [text.strip()]
):
normalized_para = _normalize_whitespace(paragraph) normalized_para = _normalize_whitespace(paragraph)
if not normalized_para: if not normalized_para:
continue continue
sentence_pairs = _split_sentences(paragraph) or [(normalized_para, paragraph)] sentence_pairs = _split_sentences(paragraph) or [(normalized_para, paragraph)]
for sent_local_index, (normalized_sentence, raw_sentence) in enumerate(sentence_pairs): for sent_local_index, (normalized_sentence, raw_sentence) in enumerate(
sentence_pairs
):
normalized_sentence = _normalize_whitespace(normalized_sentence) normalized_sentence = _normalize_whitespace(normalized_sentence)
if not normalized_sentence: if not normalized_sentence:
continue continue
@@ -203,7 +209,9 @@ def _build_display_pattern(text: str) -> Pattern[str]:
return pattern return pattern
def _search_source_span(source: str, normalized: str, start: int) -> Optional[Tuple[int, int]]: def _search_source_span(
source: str, normalized: str, start: int
) -> Optional[Tuple[int, int]]:
if not normalized: if not normalized:
return None return None
pattern = _build_display_pattern(normalized) pattern = _build_display_pattern(normalized)
+17 -6
View File
@@ -287,12 +287,12 @@ def build_debug_epub(dest_path: Path, *, title: str = "abogen debug samples") ->
dest_path.parent.mkdir(parents=True, exist_ok=True) dest_path.parent.mkdir(parents=True, exist_ok=True)
chapter_lines: List[str] = [ chapter_lines: List[str] = [
"<?xml version=\"1.0\" encoding=\"utf-8\"?>", '<?xml version="1.0" encoding="utf-8"?>',
"<!DOCTYPE html>", "<!DOCTYPE html>",
"<html xmlns=\"http://www.w3.org/1999/xhtml\">", '<html xmlns="http://www.w3.org/1999/xhtml">',
"<head>", "<head>",
f" <title>{title}</title>", f" <title>{title}</title>",
" <meta charset=\"utf-8\" />", ' <meta charset="utf-8" />',
"</head>", "</head>",
"<body>", "<body>",
f" <h1>{title}</h1>", f" <h1>{title}</h1>",
@@ -303,7 +303,11 @@ def build_debug_epub(dest_path: Path, *, title: str = "abogen debug samples") ->
safe_label = sample.label.replace("&", "and") safe_label = sample.label.replace("&", "and")
chapter_lines.append(f" <h2>{safe_label}</h2>") chapter_lines.append(f" <h2>{safe_label}</h2>")
chapter_lines.append( chapter_lines.append(
" <p><strong>" + marker_for(sample.code) + "</strong> " + sample.text + "</p>" " <p><strong>"
+ marker_for(sample.code)
+ "</strong> "
+ sample.text
+ "</p>"
) )
chapter_lines += ["</body>", "</html>"] chapter_lines += ["</body>", "</html>"]
@@ -366,8 +370,15 @@ def build_debug_epub(dest_path: Path, *, title: str = "abogen debug samples") ->
# Per EPUB spec: mimetype must be the first entry and stored (no compression). # Per EPUB spec: mimetype must be the first entry and stored (no compression).
with zipfile.ZipFile(dest_path, "w") as zf: with zipfile.ZipFile(dest_path, "w") as zf:
zf.write(tmp_path / "mimetype", "mimetype", compress_type=zipfile.ZIP_STORED) zf.write(
for source in (meta_inf / "container.xml", oebps / "content.opf", oebps / "chapter.xhtml", oebps / "nav.xhtml"): tmp_path / "mimetype", "mimetype", compress_type=zipfile.ZIP_STORED
)
for source in (
meta_inf / "container.xml",
oebps / "content.opf",
oebps / "chapter.xhtml",
oebps / "nav.xhtml",
):
arcname = str(source.relative_to(tmp_path)).replace("\\", "/") arcname = str(source.relative_to(tmp_path)).replace("\\", "/")
zf.write(source, arcname, compress_type=zipfile.ZIP_DEFLATED) zf.write(source, arcname, compress_type=zipfile.ZIP_DEFLATED)
+53 -17
View File
@@ -82,7 +82,10 @@ _EXCLUDED_NER_LABELS = {
"QUANTITY", "QUANTITY",
} }
_TITLE_PATTERN = re.compile(r"^(?:" + "|".join(re.escape(prefix) for prefix in _TITLE_PREFIXES) + r")\.?\s+", re.IGNORECASE) _TITLE_PATTERN = re.compile(
r"^(?:" + "|".join(re.escape(prefix) for prefix in _TITLE_PREFIXES) + r")\.?\s+",
re.IGNORECASE,
)
_POSSESSIVE_PATTERN = re.compile(r"(?:'s|s|\u2019s)$", re.IGNORECASE) _POSSESSIVE_PATTERN = re.compile(r"(?:'s|s|\u2019s)$", re.IGNORECASE)
_NON_WORD_PATTERN = re.compile(r"[^\w\s'-]+") _NON_WORD_PATTERN = re.compile(r"[^\w\s'-]+")
_MULTI_SPACE_PATTERN = re.compile(r"\s+") _MULTI_SPACE_PATTERN = re.compile(r"\s+")
@@ -104,7 +107,9 @@ class EntityRecord:
forms: Counter = field(default_factory=Counter) forms: Counter = field(default_factory=Counter)
first_position: Optional[Tuple[int, int]] = None first_position: Optional[Tuple[int, int]] = None
def register(self, *, chapter_index: int, position: int, text: str, sentence: Optional[str]) -> None: def register(
self, *, chapter_index: int, position: int, text: str, sentence: Optional[str]
) -> None:
self.count += 1 self.count += 1
self.chapter_indices.add(chapter_index) self.chapter_indices.add(chapter_index)
self.forms[text] += 1 self.forms[text] += 1
@@ -163,7 +168,9 @@ def _resolve_model_name(language: str) -> str:
def _load_model(language: str) -> Any: def _load_model(language: str) -> Any:
if spacy is None: if spacy is None:
raise EntityModelError("spaCy is not available. Install spaCy to enable entity extraction.") raise EntityModelError(
"spaCy is not available. Install spaCy to enable entity extraction."
)
model_name = _resolve_model_name(language) model_name = _resolve_model_name(language)
cache_key = model_name.lower() cache_key = model_name.lower()
@@ -249,7 +256,9 @@ def _extract_propn_tokens(doc: Any) -> Iterable[Any]: # type: ignore[override]
yield doc[token.i : token.i + 1] yield doc[token.i : token.i + 1]
def _empty_result(cache_key: str, error: Optional[str] = None) -> EntityExtractionResult: def _empty_result(
cache_key: str, error: Optional[str] = None
) -> EntityExtractionResult:
payload = { payload = {
"people": [], "people": [],
"entities": [], "entities": [],
@@ -262,7 +271,9 @@ def _empty_result(cache_key: str, error: Optional[str] = None) -> EntityExtracti
"model": None, "model": None,
} }
errors = [error] if error else [] errors = [error] if error else []
return EntityExtractionResult(summary=payload, cache_key=cache_key, elapsed=0.0, errors=errors) return EntityExtractionResult(
summary=payload, cache_key=cache_key, elapsed=0.0, errors=errors
)
def extract_entities( def extract_entities(
@@ -319,18 +330,25 @@ def extract_entities(
key = _token_key(cleaned) key = _token_key(cleaned)
if not key: if not key:
return return
category = category_hint or ("people" if span.label_ == "PERSON" else "entities") category = category_hint or (
"people" if span.label_ == "PERSON" else "entities"
)
record_key = (category, key) record_key = (category, key)
record = records.get(record_key) record = records.get(record_key)
if record is None: if record is None:
record = EntityRecord( record = EntityRecord(
key=record_key, key=record_key,
label=cleaned, label=cleaned,
kind=span.label_ or ("PROPN" if category == "entities" else "PERSON"), kind=span.label_
or ("PROPN" if category == "entities" else "PERSON"),
category=category, category=category,
) )
records[record_key] = record records[record_key] = record
sentence = span.sent.text if hasattr(span, "sent") and span.sent is not None else None sentence = (
span.sent.text
if hasattr(span, "sent") and span.sent is not None
else None
)
record.register( record.register(
chapter_index=chapter_index, chapter_index=chapter_index,
position=span.start, position=span.start,
@@ -361,7 +379,9 @@ def extract_entities(
elapsed = time.perf_counter() - start elapsed = time.perf_counter() - start
people_records = [record for record in records.values() if record.category == "people"] people_records = [
record for record in records.values() if record.category == "people"
]
people_keys = {record.key[1] for record in people_records} people_keys = {record.key[1] for record in people_records}
entity_records = [ entity_records = [
record record
@@ -374,10 +394,16 @@ def extract_entities(
people_records.sort(key=lambda rec: (-rec.count, rec.label)) people_records.sort(key=lambda rec: (-rec.count, rec.label))
entity_records.sort(key=lambda rec: (-rec.count, rec.label)) entity_records.sort(key=lambda rec: (-rec.count, rec.label))
people_payload = [record.as_dict(index + 1) for index, record in enumerate(people_records)] people_payload = [
entity_payload = [record.as_dict(index + 1) for index, record in enumerate(entity_records)] record.as_dict(index + 1) for index, record in enumerate(people_records)
]
entity_payload = [
record.as_dict(index + 1) for index, record in enumerate(entity_records)
]
index_payload = sorted(tokens_for_index.values(), key=lambda item: (-item["count"], item["token"])) index_payload = sorted(
tokens_for_index.values(), key=lambda item: (-item["count"], item["token"])
)
summary = { summary = {
"people": people_payload, "people": people_payload,
@@ -397,10 +423,14 @@ def extract_entities(
}, },
} }
return EntityExtractionResult(summary=summary, cache_key=cache_key, elapsed=elapsed, errors=[]) return EntityExtractionResult(
summary=summary, cache_key=cache_key, elapsed=elapsed, errors=[]
)
def search_tokens(index: Mapping[str, Any], query: str, *, limit: int = 15) -> List[Dict[str, Any]]: def search_tokens(
index: Mapping[str, Any], query: str, *, limit: int = 15
) -> List[Dict[str, Any]]:
tokens = index.get("tokens") if isinstance(index, Mapping) else None tokens = index.get("tokens") if isinstance(index, Mapping) else None
if not isinstance(tokens, list) or not query: if not isinstance(tokens, list) or not query:
return [] return []
@@ -411,14 +441,18 @@ def search_tokens(index: Mapping[str, Any], query: str, *, limit: int = 15) -> L
for entry in tokens: for entry in tokens:
token_label = str(entry.get("token", "")) token_label = str(entry.get("token", ""))
normalized_label = token_label.lower() normalized_label = token_label.lower()
if normalized in normalized_label or normalized in str(entry.get("normalized", "")): if normalized in normalized_label or normalized in str(
entry.get("normalized", "")
):
results.append(entry) results.append(entry)
if len(results) >= limit: if len(results) >= limit:
break break
return results return results
def merge_override(summary: Mapping[str, Any], overrides: Mapping[str, Mapping[str, Any]]) -> Dict[str, Any]: def merge_override(
summary: Mapping[str, Any], overrides: Mapping[str, Mapping[str, Any]]
) -> Dict[str, Any]:
if not isinstance(summary, Mapping): if not isinstance(summary, Mapping):
return {"people": [], "entities": []} return {"people": [], "entities": []}
merged_summary: Dict[str, Any] = dict(summary) merged_summary: Dict[str, Any] = dict(summary)
@@ -430,7 +464,9 @@ def merge_override(summary: Mapping[str, Any], overrides: Mapping[str, Mapping[s
for entry in items: for entry in items:
if not isinstance(entry, Mapping): if not isinstance(entry, Mapping):
continue continue
normalized = _token_key(str(entry.get("normalized") or entry.get("label") or "")) normalized = _token_key(
str(entry.get("normalized") or entry.get("label") or "")
)
merged = dict(entry) merged = dict(entry)
if normalized and normalized in overrides: if normalized and normalized in overrides:
merged_override = dict(overrides[normalized]) merged_override = dict(overrides[normalized])
+9 -3
View File
@@ -152,7 +152,9 @@ def _word_boundary_pattern(token: str) -> re.Pattern[str]:
if cached is not None: if cached is not None:
return cached return cached
escaped = re.escape(token) escaped = re.escape(token)
pattern = re.compile(rf"(?i)(?<!\w){escaped}(?P<possessive>'s|\u2019s|\u2019)?(?!\w)") pattern = re.compile(
rf"(?i)(?<!\w){escaped}(?P<possessive>'s|\u2019s|\u2019)?(?!\w)"
)
_WORD_BOUNDARY_CACHE[key] = pattern _WORD_BOUNDARY_CACHE[key] = pattern
return pattern return pattern
@@ -167,7 +169,9 @@ def _preserve_case(replacement: str, original: str) -> str:
return replacement return replacement
def _build_replacement_sentence(sentence: str, token: str, replacement_token: str) -> str: def _build_replacement_sentence(
sentence: str, token: str, replacement_token: str
) -> str:
pattern = _word_boundary_pattern(token) pattern = _word_boundary_pattern(token)
def _repl(match: re.Match[str]) -> str: def _repl(match: re.Match[str]) -> str:
@@ -271,7 +275,9 @@ def extract_heteronym_overrides(
options: List[Dict[str, Any]] = [] options: List[Dict[str, Any]] = []
for variant in spec.variants: for variant in spec.variants:
replacement_sentence = _build_replacement_sentence( replacement_sentence = _build_replacement_sentence(
sentence, token=spec.token, replacement_token=variant.replacement_token sentence,
token=spec.token,
replacement_token=variant.replacement_token,
) )
options.append( options.append(
{ {
+235 -97
View File
@@ -7,7 +7,18 @@ import os
import locale import locale
from fractions import Fraction from fractions import Fraction
from dataclasses import dataclass, field from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple from typing import (
TYPE_CHECKING,
Any,
Callable,
Dict,
Iterable,
List,
Mapping,
Optional,
Sequence,
Tuple,
)
import logging import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -16,7 +27,9 @@ try: # pragma: no cover - optional dependency guard
from num2words import num2words from num2words import num2words
except ImportError: except ImportError:
num2words = None num2words = None
logger.warning("num2words library not found. Number normalization will be disabled.") logger.warning(
"num2words library not found. Number normalization will be disabled."
)
except Exception as e: # pragma: no cover - graceful degradation except Exception as e: # pragma: no cover - graceful degradation
num2words = None num2words = None
logger.error(f"Failed to import num2words: {e}") logger.error(f"Failed to import num2words: {e}")
@@ -41,34 +54,44 @@ CONTRACTION_CATEGORY_DEFAULTS: Dict[str, bool] = {
# ---------- Configuration Dataclass ---------- # ---------- Configuration Dataclass ----------
@dataclass @dataclass
class ApostropheConfig: class ApostropheConfig:
contraction_mode: str = "expand" # expand|collapse|keep contraction_mode: str = "expand" # expand|collapse|keep
possessive_mode: str = "keep" # keep|collapse possessive_mode: str = "keep" # keep|collapse
plural_possessive_mode: str = "collapse" # keep|collapse plural_possessive_mode: str = "collapse" # keep|collapse
irregular_possessive_mode: str = "keep" # keep|expand (expand just means keep or add hints; modify if needed) irregular_possessive_mode: str = (
sibilant_possessive_mode: str = "mark" # keep|mark|approx "keep" # keep|expand (expand just means keep or add hints; modify if needed)
fantasy_mode: str = "keep" # keep|mark|collapse_internal )
acronym_possessive_mode: str = "keep" # keep|collapse_add_s sibilant_possessive_mode: str = "mark" # keep|mark|approx
decades_mode: str = "expand" # keep|expand fantasy_mode: str = "keep" # keep|mark|collapse_internal
leading_elision_mode: str = "expand" # keep|expand acronym_possessive_mode: str = "keep" # keep|collapse_add_s
ambiguous_past_modal_mode: str = "contextual" # keep|expand_prefer_would|expand_prefer_had|contextual decades_mode: str = "expand" # keep|expand
add_phoneme_hints: bool = True # Whether to emit markers like IZ leading_elision_mode: str = "expand" # keep|expand
fantasy_marker: str = "FAP" # Marker inserted if fantasy_mode == mark ambiguous_past_modal_mode: str = (
sibilant_iz_marker: str = "IZ" # Marker for /ɪz/ insertion "contextual" # keep|expand_prefer_would|expand_prefer_had|contextual
joiner: str = "" # Replacement used when collapsing internal apostrophes )
lowercase_for_matching: bool = True # Normalize to lower for rule matching (not output) add_phoneme_hints: bool = True # Whether to emit markers like IZ
protect_cultural_names: bool = True # Always keep O'Brien, D'Angelo, etc. fantasy_marker: str = "FAP" # Marker inserted if fantasy_mode == mark
convert_numbers: bool = True # Convert grouped numbers such as 12,500 to words sibilant_iz_marker: str = "IZ" # Marker for /ɪz/ insertion
convert_currency: bool = True # Convert currency symbols to words joiner: str = "" # Replacement used when collapsing internal apostrophes
remove_footnotes: bool = True # Remove footnote indicators lowercase_for_matching: bool = (
number_lang: str = "en" # num2words language code True # Normalize to lower for rule matching (not output)
year_pronunciation_mode: str = "american" # off|american (extend if needed) )
contraction_categories: Dict[str, bool] = field(default_factory=lambda: dict(CONTRACTION_CATEGORY_DEFAULTS)) protect_cultural_names: bool = True # Always keep O'Brien, D'Angelo, etc.
convert_numbers: bool = True # Convert grouped numbers such as 12,500 to words
convert_currency: bool = True # Convert currency symbols to words
remove_footnotes: bool = True # Remove footnote indicators
number_lang: str = "en" # num2words language code
year_pronunciation_mode: str = "american" # off|american (extend if needed)
contraction_categories: Dict[str, bool] = field(
default_factory=lambda: dict(CONTRACTION_CATEGORY_DEFAULTS)
)
def is_contraction_enabled(self, category: str) -> bool: def is_contraction_enabled(self, category: str) -> bool:
return self.contraction_categories.get(category, True) return self.contraction_categories.get(category, True)
# ---------- Dictionaries / Patterns ---------- # ---------- Dictionaries / Patterns ----------
# Common contraction expansions (type + expansion words) # Common contraction expansions (type + expansion words)
@@ -124,14 +147,18 @@ _FRACTION_RE = re.compile(
_CURRENCY_RE = re.compile( _CURRENCY_RE = re.compile(
r"(?P<symbol>[$£€¥])\s*(?P<amount>\d{1,3}(?:,\d{3})*(?:\.\d+)?)(?:\s+(?P<magnitude>hundred|thousand|million|billion|trillion|quadrillion))?(?!\d)", r"(?P<symbol>[$£€¥])\s*(?P<amount>\d{1,3}(?:,\d{3})*(?:\.\d+)?)(?:\s+(?P<magnitude>hundred|thousand|million|billion|trillion|quadrillion))?(?!\d)",
re.IGNORECASE re.IGNORECASE,
) )
_URL_RE = re.compile(r"(https?://)?(www\.)?(?P<domain>[a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)+)(/[^\s]*)?") _URL_RE = re.compile(
r"(https?://)?(www\.)?(?P<domain>[a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)+)(/[^\s]*)?"
)
_FOOTNOTE_RE = re.compile(r"([a-zA-Z]+)(\d+)") _FOOTNOTE_RE = re.compile(r"([a-zA-Z]+)(\d+)")
_BRACKET_FOOTNOTE_RE = re.compile(r"\[\d+\]") _BRACKET_FOOTNOTE_RE = re.compile(r"\[\d+\]")
_ISO_DATE_RE = re.compile(r"\b(?P<year>\d{4})[/-](?P<month>\d{1,2})[/-](?P<day>\d{1,2})\b") _ISO_DATE_RE = re.compile(
r"\b(?P<year>\d{4})[/-](?P<month>\d{1,2})[/-](?P<day>\d{1,2})\b"
)
_MDY_DATE_RE = re.compile( _MDY_DATE_RE = re.compile(
r"\b(?P<month>Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\.?\s+" r"\b(?P<month>Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\.?\s+"
r"(?P<day>\d{1,2})(?:st|nd|rd|th)?\s*,\s*(?P<year>\d{4})\b", r"(?P<day>\d{1,2})(?:st|nd|rd|th)?\s*,\s*(?P<year>\d{4})\b",
@@ -143,7 +170,9 @@ _TIME_RE = re.compile(
re.IGNORECASE, re.IGNORECASE,
) )
_ADDRESS_ABBR_RE = re.compile(r"(?P<prefix>\b\w+\s+)(?P<abbr>St|Rd|Ave|Blvd|Ln|Dr)\.(?=\s*(?:,|\.|!|\?|$))") _ADDRESS_ABBR_RE = re.compile(
r"(?P<prefix>\b\w+\s+)(?P<abbr>St|Rd|Ave|Blvd|Ln|Dr)\.(?=\s*(?:,|\.|!|\?|$))"
)
_MONTH_MAP = { _MONTH_MAP = {
"jan": "January", "jan": "January",
@@ -218,25 +247,27 @@ def _normalize_dates(text: str, language: str) -> str:
day = int(match.group("day")) day = int(match.group("day"))
if not (1 <= month <= 12 and 1 <= day <= 31): if not (1 <= month <= 12 and 1 <= day <= 31):
return match.group(0) return match.group(0)
month_name = ( month_name = [
[ "January",
"January", "February",
"February", "March",
"March", "April",
"April", "May",
"May", "June",
"June", "July",
"July", "August",
"August", "September",
"September", "October",
"October", "November",
"November", "December",
"December", ][month - 1]
][month - 1]
)
ordinal = _int_to_ordinal_words(day, language) or str(day) ordinal = _int_to_ordinal_words(day, language) or str(day)
year_words = _year_to_words_american(year, language) year_words = _year_to_words_american(year, language)
return f"{month_name} {ordinal}, {year_words}" if us else f"{ordinal} {month_name} {year_words}" return (
f"{month_name} {ordinal}, {year_words}"
if us
else f"{ordinal} {month_name} {year_words}"
)
def _format_mdy(match: re.Match[str]) -> str: def _format_mdy(match: re.Match[str]) -> str:
month_raw = str(match.group("month") or "").strip().lower().rstrip(".") month_raw = str(match.group("month") or "").strip().lower().rstrip(".")
@@ -247,7 +278,11 @@ def _normalize_dates(text: str, language: str) -> str:
year = int(match.group("year")) year = int(match.group("year"))
ordinal = _int_to_ordinal_words(day, language) or str(day) ordinal = _int_to_ordinal_words(day, language) or str(day)
year_words = _year_to_words_american(year, language) year_words = _year_to_words_american(year, language)
return f"{month_name} {ordinal}, {year_words}" if us else f"{ordinal} {month_name} {year_words}" return (
f"{month_name} {ordinal}, {year_words}"
if us
else f"{ordinal} {month_name} {year_words}"
)
out = _ISO_DATE_RE.sub(_format_iso, text) out = _ISO_DATE_RE.sub(_format_iso, text)
out = _MDY_DATE_RE.sub(_format_mdy, out) out = _MDY_DATE_RE.sub(_format_mdy, out)
@@ -301,6 +336,7 @@ def _normalize_internet_slang(text: str) -> str:
return re.sub(r"\b(?:pls|plz)\b", _replace, text, flags=re.IGNORECASE) return re.sub(r"\b(?:pls|plz)\b", _replace, text, flags=re.IGNORECASE)
_DECIMAL_NUMBER_RE = re.compile( _DECIMAL_NUMBER_RE = re.compile(
rf"(?<![\w{_NUMBER_RANGE_CLASS}/])(?P<number>-?(?:\d{{1,3}}(?:,\d{{3}})+|\d+)\.(?P<fraction>\d+))(?![\w{_NUMBER_RANGE_CLASS}/])" rf"(?<![\w{_NUMBER_RANGE_CLASS}/])(?P<number>-?(?:\d{{1,3}}(?:,\d{{3}})+|\d+)\.(?P<fraction>\d+))(?![\w{_NUMBER_RANGE_CLASS}/])"
) )
@@ -308,7 +344,18 @@ _PLAIN_NUMBER_RE = re.compile(
rf"(?<![\w{_NUMBER_RANGE_CLASS}/])(?P<number>{_NUMBER_CORE_PATTERN})(?![\w{_NUMBER_RANGE_CLASS}/])" rf"(?<![\w{_NUMBER_RANGE_CLASS}/])(?P<number>{_NUMBER_CORE_PATTERN})(?![\w{_NUMBER_RANGE_CLASS}/])"
) )
_DIGIT_WORDS = ("zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine") _DIGIT_WORDS = (
"zero",
"one",
"two",
"three",
"four",
"five",
"six",
"seven",
"eight",
"nine",
)
def _int_to_words(value: int, language: str) -> Optional[str]: def _int_to_words(value: int, language: str) -> Optional[str]:
@@ -384,7 +431,9 @@ def _pluralize_fraction_word(base: str) -> str:
return base + "s" return base + "s"
def _fraction_denominator_word(denominator: int, numerator: int, language: str) -> Optional[str]: def _fraction_denominator_word(
denominator: int, numerator: int, language: str
) -> Optional[str]:
"""Return spoken form for fraction denominator respecting plurality.""" """Return spoken form for fraction denominator respecting plurality."""
if denominator == 0: if denominator == 0:
return None return None
@@ -405,7 +454,9 @@ def _fraction_denominator_word(denominator: int, numerator: int, language: str)
return _pluralize_fraction_word(base) return _pluralize_fraction_word(base)
def _format_fraction_words(numerator: int, denominator: int, language: str) -> Optional[str]: def _format_fraction_words(
numerator: int, denominator: int, language: str
) -> Optional[str]:
"""Return spoken representation of a simple fraction.""" """Return spoken representation of a simple fraction."""
if denominator == 0: if denominator == 0:
return None return None
@@ -492,7 +543,9 @@ def _coerce_int_token(token: str) -> Optional[int]:
return int(cleaned) return int(cleaned)
except ValueError: except ValueError:
return None return None
AMBIGUOUS_D_BASES = {"i","you","he","she","we","they"}
AMBIGUOUS_D_BASES = {"i", "you", "he", "she", "we", "they"}
AMBIGUOUS_S_BASES = { AMBIGUOUS_S_BASES = {
"it", "it",
"that", "that",
@@ -520,6 +573,7 @@ def _is_ambiguous_s(token: str) -> bool:
low = token.lower() low = token.lower()
return low.endswith("'s") and low[:-2] in AMBIGUOUS_S_BASES return low.endswith("'s") and low[:-2] in AMBIGUOUS_S_BASES
# Irregular possessives that are not formed by simple + 's logic # Irregular possessives that are not formed by simple + 's logic
IRREGULAR_POSSESSIVES = { IRREGULAR_POSSESSIVES = {
"children's": "children's", "children's": "children's",
@@ -527,12 +581,12 @@ IRREGULAR_POSSESSIVES = {
"women's": "women's", "women's": "women's",
"people's": "people's", "people's": "people's",
"geese's": "geese's", "geese's": "geese's",
"mouse's": "mouse's", # singular irregular "mouse's": "mouse's", # singular irregular
} }
SIBILANT_END_RE = re.compile(r"(?:[sxz]|(?:ch|sh))$", re.IGNORECASE) SIBILANT_END_RE = re.compile(r"(?:[sxz]|(?:ch|sh))$", re.IGNORECASE)
DECADE_RE = re.compile(r"^'\d0s$", re.IGNORECASE) # '90s, '80s DECADE_RE = re.compile(r"^'\d0s$", re.IGNORECASE) # '90s, '80s
LEADING_ELISION = { LEADING_ELISION = {
"'tis": "it is", "'tis": "it is",
"'twas": "it was", "'twas": "it was",
@@ -546,7 +600,7 @@ CULTURAL_NAME_PATTERNS = [
re.compile(r"^O'[A-Z][a-z]+$"), re.compile(r"^O'[A-Z][a-z]+$"),
re.compile(r"^D'[A-Z][a-z]+$"), re.compile(r"^D'[A-Z][a-z]+$"),
re.compile(r"^L'[A-Za-z].*$"), re.compile(r"^L'[A-Za-z].*$"),
re.compile(r"^Mc[A-Z].*$"), # not apostrophe, but often relevant (kept anyway) re.compile(r"^Mc[A-Z].*$"), # not apostrophe, but often relevant (kept anyway)
] ]
ACRONYM_POSSESSIVE_RE = re.compile(r"^[A-Z]{2,}'s$") ACRONYM_POSSESSIVE_RE = re.compile(r"^[A-Z]{2,}'s$")
@@ -563,7 +617,7 @@ WORD_TOKEN_RE = re.compile(
APOSTROPHE_CHARS = "`´ꞌʼ" APOSTROPHE_CHARS = "`´ꞌʼ"
TERMINAL_PUNCTUATION = {".", "?", "!", "", ";", ":"} TERMINAL_PUNCTUATION = {".", "?", "!", "", ";", ":"}
CLOSING_PUNCTUATION = '"\'”’)]}»›' CLOSING_PUNCTUATION = "\"'”’)]}»›"
ELLIPSIS_SUFFIXES = ("...", "") ELLIPSIS_SUFFIXES = ("...", "")
_LINE_SPLIT_RE = re.compile(r"(\n+)") _LINE_SPLIT_RE = re.compile(r"(\n+)")
@@ -584,29 +638,38 @@ SUFFIX_ABBREVIATIONS = {
} }
_TITLE_PATTERN = re.compile( _TITLE_PATTERN = re.compile(
r"\b(?P<abbr>" + "|".join(sorted(TITLE_ABBREVIATIONS.keys(), key=len, reverse=True)) + r")\.", r"\b(?P<abbr>"
+ "|".join(sorted(TITLE_ABBREVIATIONS.keys(), key=len, reverse=True))
+ r")\.",
re.IGNORECASE, re.IGNORECASE,
) )
_SUFFIX_PATTERN = re.compile( _SUFFIX_PATTERN = re.compile(
r"\b(?P<abbr>" + "|".join(sorted(SUFFIX_ABBREVIATIONS.keys(), key=len, reverse=True)) + r")\.", r"\b(?P<abbr>"
+ "|".join(sorted(SUFFIX_ABBREVIATIONS.keys(), key=len, reverse=True))
+ r")\.",
re.IGNORECASE, re.IGNORECASE,
) )
# ---------- Utility Functions ---------- # ---------- Utility Functions ----------
def normalize_unicode_apostrophes(text: str) -> str: def normalize_unicode_apostrophes(text: str) -> str:
text = unicodedata.normalize("NFKC", text) text = unicodedata.normalize("NFKC", text)
for ch in APOSTROPHE_CHARS: for ch in APOSTROPHE_CHARS:
text = text.replace(ch, "'") text = text.replace(ch, "'")
return text return text
def tokenize(text: str) -> List[str]: def tokenize(text: str) -> List[str]:
# Simple tokenization preserving punctuation tokens # Simple tokenization preserving punctuation tokens
return WORD_TOKEN_RE.findall(text) return WORD_TOKEN_RE.findall(text)
def tokenize_with_spans(text: str) -> List[Tuple[str, int, int]]: def tokenize_with_spans(text: str) -> List[Tuple[str, int, int]]:
return [(match.group(0), match.start(), match.end()) for match in WORD_TOKEN_RE.finditer(text)] return [
(match.group(0), match.start(), match.end())
for match in WORD_TOKEN_RE.finditer(text)
]
def _cleanup_spacing(text: str) -> str: def _cleanup_spacing(text: str) -> str:
@@ -661,7 +724,9 @@ _ROMAN_COMPOSE_ORDER = [
(1, "I"), (1, "I"),
] ]
_ROMAN_PREFIX_RE = re.compile(r"^(?P<roman>[IVXLCDM]+)(?P<sep>[\s\.:,;\-–—]*)", re.IGNORECASE) _ROMAN_PREFIX_RE = re.compile(
r"^(?P<roman>[IVXLCDM]+)(?P<sep>[\s\.:,;\-–—]*)", re.IGNORECASE
)
_ROMAN_TOKEN_RE = re.compile(r"^[IVXLCDM]+$") _ROMAN_TOKEN_RE = re.compile(r"^[IVXLCDM]+$")
_ROMAN_CARDINAL_CONTEXTS = { _ROMAN_CARDINAL_CONTEXTS = {
@@ -816,7 +881,9 @@ def _token_is_cardinal_context(token: str) -> bool:
return token.lower() in _ROMAN_CARDINAL_CONTEXTS return token.lower() in _ROMAN_CARDINAL_CONTEXTS
def _has_cardinal_leading_context(tokens: Sequence[Tuple[str, int, int]], index: int) -> bool: def _has_cardinal_leading_context(
tokens: Sequence[Tuple[str, int, int]], index: int
) -> bool:
j = index - 1 j = index - 1
while j >= 0: while j >= 0:
token, *_ = tokens[j] token, *_ = tokens[j]
@@ -927,7 +994,9 @@ def _normalize_roman_numerals(text: str, language: str) -> str:
if len(token) >= 2: if len(token) >= 2:
if token.isupper(): if token.isupper():
convert = True convert = True
elif numeric_value <= 200 and _has_cardinal_leading_context(tokens, index): elif numeric_value <= 200 and _has_cardinal_leading_context(
tokens, index
):
convert = True convert = True
elif len(token) == 1: elif len(token) == 1:
# Only convert single letters if context is strong # Only convert single letters if context is strong
@@ -1002,7 +1071,10 @@ def _should_preserve_caps_word(word: str) -> bool:
upper_base = base.upper() upper_base = base.upper()
if upper_base in _ACRONYM_ALLOWLIST: if upper_base in _ACRONYM_ALLOWLIST:
return True return True
if all(ch in _ROMAN_NUMERAL_LETTERS for ch in letters.upper()) and len(letters) <= 7: if (
all(ch in _ROMAN_NUMERAL_LETTERS for ch in letters.upper())
and len(letters) <= 7
):
return True return True
return False return False
@@ -1128,7 +1200,7 @@ def normalize_roman_numeral_titles(
roman_token = match.group("roman") roman_token = match.group("roman")
separator = match.group("sep") or "" separator = match.group("sep") or ""
rest = stripped[match.end():] rest = stripped[match.end() :]
if not separator and rest and rest[:1].isalnum(): if not separator and rest and rest[:1].isalnum():
normalized.append(title) normalized.append(title)
@@ -1269,7 +1341,9 @@ def _apply_contraction_policy(
return expand() return expand()
def _assemble_contraction_expansion(base_text: str, surface_text: str, expansion_word: str) -> str: def _assemble_contraction_expansion(
base_text: str, surface_text: str, expansion_word: str
) -> str:
if not expansion_word: if not expansion_word:
return base_text return base_text
@@ -1340,6 +1414,7 @@ def _classify_ambiguous_s(token: str, cfg: ApostropheConfig) -> Tuple[str, str]:
return candidates[0][0], token return candidates[0][0], token
def classify_token(token: str, cfg: ApostropheConfig) -> Tuple[str, str]: def classify_token(token: str, cfg: ApostropheConfig) -> Tuple[str, str]:
""" """
Classify apostrophe usage and propose normalized form. Classify apostrophe usage and propose normalized form.
@@ -1398,11 +1473,15 @@ def classify_token(token: str, cfg: ApostropheConfig) -> Tuple[str, str]:
return _case_preserving_words(token, words) return _case_preserving_words(token, words)
collapse_value = token.replace("'", "") collapse_value = token.replace("'", "")
normalized = _apply_contraction_policy(token, category=category, cfg=cfg, expand=_expand, collapse=collapse_value) normalized = _apply_contraction_policy(
token, category=category, cfg=cfg, expand=_expand, collapse=collapse_value
)
return category, normalized return category, normalized
# 6. Suffix contractions ('m handled separately) # 6. Suffix contractions ('m handled separately)
if low.endswith("'m") and low[:-2] in SUFFIX_CONTRACTION_BASES.get("'m", ()): # pronoun I'm if low.endswith("'m") and low[:-2] in SUFFIX_CONTRACTION_BASES.get(
"'m", ()
): # pronoun I'm
def _expand_m() -> str: def _expand_m() -> str:
base = token[:-2] base = token[:-2]
@@ -1488,7 +1567,10 @@ def classify_token(token: str, cfg: ApostropheConfig) -> Tuple[str, str]:
return "other", token.replace("'", cfg.joiner) return "other", token.replace("'", cfg.joiner)
return "other", token return "other", token
def normalize_apostrophes(text: str, cfg: ApostropheConfig | None = None) -> Tuple[str, List[Tuple[str,str,str]]]:
def normalize_apostrophes(
text: str, cfg: ApostropheConfig | None = None
) -> Tuple[str, List[Tuple[str, str, str]]]:
""" """
Normalize apostrophes per config. Normalize apostrophes per config.
Returns normalized text AND a list of (original_token, category, normalized_token) Returns normalized text AND a list of (original_token, category, normalized_token)
@@ -1502,7 +1584,10 @@ def normalize_apostrophes(text: str, cfg: ApostropheConfig | None = None) -> Tup
token_entries = tokenize_with_spans(text) token_entries = tokenize_with_spans(text)
use_contextual_s = cfg.contraction_mode == "expand" use_contextual_s = cfg.contraction_mode == "expand"
use_contextual_d = cfg.contraction_mode == "expand" and cfg.ambiguous_past_modal_mode == "contextual" use_contextual_d = (
cfg.contraction_mode == "expand"
and cfg.ambiguous_past_modal_mode == "contextual"
)
need_contextual = False need_contextual = False
if (use_contextual_s or use_contextual_d) and token_entries: if (use_contextual_s or use_contextual_d) and token_entries:
@@ -1514,7 +1599,9 @@ def normalize_apostrophes(text: str, cfg: ApostropheConfig | None = None) -> Tup
need_contextual = True need_contextual = True
break break
contextual_resolutions = resolve_ambiguous_contractions(text) if need_contextual else {} contextual_resolutions = (
resolve_ambiguous_contractions(text) if need_contextual else {}
)
results: List[Tuple[str, str, str]] = [] results: List[Tuple[str, str, str]] = []
normalized_tokens: List[str] = [] normalized_tokens: List[str] = []
@@ -1522,7 +1609,9 @@ def normalize_apostrophes(text: str, cfg: ApostropheConfig | None = None) -> Tup
for tok, start, end in token_entries: for tok, start, end in token_entries:
category, norm = classify_token(tok, cfg) category, norm = classify_token(tok, cfg)
resolution = contextual_resolutions.get((start, end)) if contextual_resolutions else None resolution = (
contextual_resolutions.get((start, end)) if contextual_resolutions else None
)
if resolution is not None and cfg.contraction_mode == "expand": if resolution is not None and cfg.contraction_mode == "expand":
if cfg.is_contraction_enabled(resolution.category): if cfg.is_contraction_enabled(resolution.category):
category = resolution.category category = resolution.category
@@ -1537,6 +1626,7 @@ def normalize_apostrophes(text: str, cfg: ApostropheConfig | None = None) -> Tup
normalized_text = _cleanup_spacing(" ".join(filtered)) normalized_text = _cleanup_spacing(" ".join(filtered))
return normalized_text, results return normalized_text, results
def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str: def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
if not text: if not text:
return text return text
@@ -1675,7 +1765,9 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
has_address = bool(re.search(r"\baddress(es)?\b", context)) has_address = bool(re.search(r"\baddress(es)?\b", context))
# Check for year markers as whole words # Check for year markers as whole words
has_year_marker = bool(re.search(r"\b(bc|ad|bce|ce|b\.c\.|a\.d\.|b\.c\.e\.|c\.e\.)\b", context)) has_year_marker = bool(
re.search(r"\b(bc|ad|bce|ce|b\.c\.|a\.d\.|b\.c\.e\.|c\.e\.)\b", context)
)
should_try_year = True should_try_year = True
if has_address and not has_year_marker: if has_address and not has_year_marker:
@@ -1787,7 +1879,9 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
except ValueError: except ValueError:
cents_value = 0 cents_value = 0
if cents_value > 0: if cents_value > 0:
cents_words = _int_to_words(cents_value, language) or str(cents_value) cents_words = _int_to_words(cents_value, language) or str(
cents_value
)
subunit = { subunit = {
"$": "cent", "$": "cent",
"": "cent", "": "cent",
@@ -1797,7 +1891,11 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
if symbol == "£": if symbol == "£":
subunit = "pence" if cents_value != 1 else "penny" subunit = "pence" if cents_value != 1 else "penny"
else: else:
subunit = (subunit + "s") if cents_value != 1 and subunit not in {"pence", "yen"} else subunit subunit = (
(subunit + "s")
if cents_value != 1 and subunit not in {"pence", "yen"}
else subunit
)
return f"{cents_words} {subunit}".strip() return f"{cents_words} {subunit}".strip()
currency_map = { currency_map = {
@@ -1811,7 +1909,9 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
try: try:
# Always use float to avoid num2words treating int as cents (if that's what it does) # Always use float to avoid num2words treating int as cents (if that's what it does)
# or to ensure consistent behavior. # or to ensure consistent behavior.
words = num2words(amount, to="currency", currency=currency_code, lang=language) words = num2words(
amount, to="currency", currency=currency_code, lang=language
)
# Remove "zero cents" if present # Remove "zero cents" if present
# Patterns: ", zero cents", " and zero cents" # Patterns: ", zero cents", " and zero cents"
@@ -1833,16 +1933,16 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
# Avoid matching numbers like 1.05 or 12.34.56 # Avoid matching numbers like 1.05 or 12.34.56
# If the domain consists only of digits and dots, ignore it (unless it has http/www prefix) # If the domain consists only of digits and dots, ignore it (unless it has http/www prefix)
has_prefix = match.group(1) or match.group(2) has_prefix = match.group(1) or match.group(2)
if not has_prefix and all(c.isdigit() or c == '.' or c == '-' for c in domain): if not has_prefix and all(c.isdigit() or c == "." or c == "-" for c in domain):
# Check if it really looks like a number (e.g. 1.05) # Check if it really looks like a number (e.g. 1.05)
# If it has multiple dots like 1.2.3.4 it might be an IP, which we might want to speak as dot? # If it has multiple dots like 1.2.3.4 it might be an IP, which we might want to speak as dot?
# But 1.05 is definitely a number. # But 1.05 is definitely a number.
# Let's be safe: if it looks like a float, skip. # Let's be safe: if it looks like a float, skip.
try: try:
float(domain) float(domain)
return match.group(0) return match.group(0)
except ValueError: except ValueError:
pass pass
if domain.startswith("www."): if domain.startswith("www."):
domain = domain[4:] domain = domain[4:]
@@ -1865,15 +1965,23 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
normalized = _CURRENCY_RE.sub(_replace_currency, normalized) normalized = _CURRENCY_RE.sub(_replace_currency, normalized)
if cfg.convert_numbers: if cfg.convert_numbers:
normalized = _NUMBER_RANGE_RE.sub(lambda m: _replace_number_range(m, language), normalized) normalized = _NUMBER_RANGE_RE.sub(
normalized = _NUMBER_SPACE_RANGE_RE.sub(lambda m: _replace_space_separated_range(m, language), normalized) lambda m: _replace_number_range(m, language), normalized
normalized = _FRACTION_RE.sub(lambda m: _replace_fraction(m, language), normalized) )
normalized = _NUMBER_SPACE_RANGE_RE.sub(
lambda m: _replace_space_separated_range(m, language), normalized
)
normalized = _FRACTION_RE.sub(
lambda m: _replace_fraction(m, language), normalized
)
normalized = _DECIMAL_NUMBER_RE.sub(_replace_decimal, normalized) normalized = _DECIMAL_NUMBER_RE.sub(_replace_decimal, normalized)
normalized = _NUMBER_WITH_GROUP_RE.sub(_replace_grouped, normalized) normalized = _NUMBER_WITH_GROUP_RE.sub(_replace_grouped, normalized)
normalized = _PLAIN_NUMBER_RE.sub(_replace_plain, normalized) normalized = _PLAIN_NUMBER_RE.sub(_replace_plain, normalized)
normalized = _normalize_roman_numerals(normalized, language) normalized = _normalize_roman_numerals(normalized, language)
return normalized return normalized
def _normalize_dotted_acronyms(text: str) -> str: def _normalize_dotted_acronyms(text: str) -> str:
def _replace(match: re.Match[str]) -> str: def _replace(match: re.Match[str]) -> str:
value = match.group(0) value = match.group(0)
@@ -1882,8 +1990,10 @@ def _normalize_dotted_acronyms(text: str) -> str:
return re.sub(r"\b(?:[A-Z]\.){1,}[A-Z]\.?(?=\W|$)", _replace, text) return re.sub(r"\b(?:[A-Z]\.){1,}[A-Z]\.?(?=\W|$)", _replace, text)
# ---------- Optional phoneme hint post-processing ---------- # ---------- Optional phoneme hint post-processing ----------
def apply_phoneme_hints(text: str, iz_marker="IZ") -> str: def apply_phoneme_hints(text: str, iz_marker="IZ") -> str:
""" """
Replace markers with an orthographic sequence that Replace markers with an orthographic sequence that
@@ -1951,7 +2061,10 @@ _LLM_REGEX_TOOL = {
}, },
}, },
} }
_LLM_REGEX_TOOL_CHOICE = {"type": "function", "function": {"name": _LLM_REGEX_TOOL_NAME}} _LLM_REGEX_TOOL_CHOICE = {
"type": "function",
"function": {"name": _LLM_REGEX_TOOL_NAME},
}
_LLM_ALLOWED_REGEX_FLAGS = { _LLM_ALLOWED_REGEX_FLAGS = {
"IGNORECASE": re.IGNORECASE, "IGNORECASE": re.IGNORECASE,
"MULTILINE": re.MULTILINE, "MULTILINE": re.MULTILINE,
@@ -1974,7 +2087,9 @@ _SENTENCE_CAPTURE_RE = re.compile(r"[^.!?]+[.!?]+|[^.!?]+$", re.MULTILINE)
def _split_sentences_for_llm(text: str) -> List[str]: def _split_sentences_for_llm(text: str) -> List[str]:
sentences = [segment.strip() for segment in _SENTENCE_CAPTURE_RE.findall(text or "")] sentences = [
segment.strip() for segment in _SENTENCE_CAPTURE_RE.findall(text or "")
]
return [segment for segment in sentences if segment] return [segment for segment in sentences if segment]
@@ -1984,7 +2099,10 @@ def _normalize_with_llm(
settings: Mapping[str, Any], settings: Mapping[str, Any],
config: ApostropheConfig, config: ApostropheConfig,
) -> str: ) -> str:
from abogen.normalization_settings import build_llm_configuration, DEFAULT_LLM_PROMPT from abogen.normalization_settings import (
build_llm_configuration,
DEFAULT_LLM_PROMPT,
)
from abogen.llm_client import generate_completion, LLMClientError from abogen.llm_client import generate_completion, LLMClientError
llm_config = build_llm_configuration(settings) llm_config = build_llm_configuration(settings)
@@ -2001,7 +2119,7 @@ def _normalize_with_llm(
newline = "" newline = ""
if raw_line.endswith(("\r", "\n")): if raw_line.endswith(("\r", "\n")):
stripped_newline = raw_line.rstrip("\r\n") stripped_newline = raw_line.rstrip("\r\n")
newline = raw_line[len(stripped_newline):] newline = raw_line[len(stripped_newline) :]
line_body = stripped_newline line_body = stripped_newline
else: else:
line_body = raw_line line_body = raw_line
@@ -2011,7 +2129,7 @@ def _normalize_with_llm(
continue continue
leading_ws = line_body[: len(line_body) - len(line_body.lstrip())] leading_ws = line_body[: len(line_body) - len(line_body.lstrip())]
trailing_ws = line_body[len(line_body.rstrip()):] trailing_ws = line_body[len(line_body.rstrip()) :]
core = line_body[len(leading_ws) : len(line_body) - len(trailing_ws)] core = line_body[len(leading_ws) : len(line_body) - len(trailing_ws)]
sentences = _split_sentences_for_llm(core) sentences = _split_sentences_for_llm(core)
@@ -2136,7 +2254,11 @@ def _normalize_flag_field(raw: Any) -> List[str]:
seen: set[str] = set() seen: set[str] = set()
for value in raw_iterable: for value in raw_iterable:
candidate = str(value or "").strip().upper() candidate = str(value or "").strip().upper()
if not candidate or candidate not in _LLM_ALLOWED_REGEX_FLAGS or candidate in seen: if (
not candidate
or candidate not in _LLM_ALLOWED_REGEX_FLAGS
or candidate in seen
):
continue continue
seen.add(candidate) seen.add(candidate)
normalized.append(candidate) normalized.append(candidate)
@@ -2153,7 +2275,9 @@ def _apply_single_regex_replacement(text: str, spec: Mapping[str, Any]) -> str:
flag_names = spec.get("flags") flag_names = spec.get("flags")
if isinstance(flag_names, str): if isinstance(flag_names, str):
flag_iterable: Iterable[Any] = [flag_names] flag_iterable: Iterable[Any] = [flag_names]
elif isinstance(flag_names, Iterable) and not isinstance(flag_names, (bytes, str, Mapping)): elif isinstance(flag_names, Iterable) and not isinstance(
flag_names, (bytes, str, Mapping)
):
flag_iterable = flag_names flag_iterable = flag_names
else: else:
flag_iterable = [] flag_iterable = []
@@ -2179,7 +2303,10 @@ def normalize_for_pipeline(
) -> str: ) -> str:
"""Normalize text for the synthesis pipeline with runtime settings.""" """Normalize text for the synthesis pipeline with runtime settings."""
from abogen.normalization_settings import build_apostrophe_config, get_runtime_settings from abogen.normalization_settings import (
build_apostrophe_config,
get_runtime_settings,
)
from abogen.llm_client import LLMClientError from abogen.llm_client import LLMClientError
runtime_settings = settings or get_runtime_settings() runtime_settings = settings or get_runtime_settings()
@@ -2201,15 +2328,25 @@ def normalize_for_pipeline(
if mode == "off": if mode == "off":
normalized = normalize_unicode_apostrophes(normalized) normalized = normalize_unicode_apostrophes(normalized)
if cfg.convert_numbers or cfg.convert_currency or getattr(cfg, "remove_footnotes", False): if (
cfg.convert_numbers
or cfg.convert_currency
or getattr(cfg, "remove_footnotes", False)
):
normalized = _normalize_grouped_numbers(normalized, cfg) normalized = _normalize_grouped_numbers(normalized, cfg)
normalized = _cleanup_spacing(normalized) normalized = _cleanup_spacing(normalized)
elif mode == "llm": elif mode == "llm":
try: try:
normalized = _normalize_with_llm(normalized, settings=runtime_settings, config=cfg) normalized = _normalize_with_llm(
normalized, settings=runtime_settings, config=cfg
)
except LLMClientError: except LLMClientError:
raise raise
if cfg.convert_numbers or cfg.convert_currency or getattr(cfg, "remove_footnotes", False): if (
cfg.convert_numbers
or cfg.convert_currency
or getattr(cfg, "remove_footnotes", False)
):
normalized = _normalize_grouped_numbers(normalized, cfg) normalized = _normalize_grouped_numbers(normalized, cfg)
normalized = _cleanup_spacing(normalized) normalized = _cleanup_spacing(normalized)
else: else:
@@ -2227,6 +2364,7 @@ def normalize_for_pipeline(
return normalized return normalized
# ---------- Example Usage ---------- # ---------- Example Usage ----------
if __name__ == "__main__": if __name__ == "__main__":
+13 -5
View File
@@ -52,8 +52,10 @@ def _build_url(base_url: str, path: str) -> str:
normalized = _normalized_base_url(base_url) normalized = _normalized_base_url(base_url)
trimmed_path = path.lstrip("/") trimmed_path = path.lstrip("/")
parsed = parse.urlparse(normalized) parsed = parse.urlparse(normalized)
if parsed.path.rstrip("/").lower().endswith("/v1") and trimmed_path.startswith("v1/"): if parsed.path.rstrip("/").lower().endswith("/v1") and trimmed_path.startswith(
trimmed_path = trimmed_path[len("v1/"):] "v1/"
):
trimmed_path = trimmed_path[len("v1/") :]
return parse.urljoin(normalized, trimmed_path) return parse.urljoin(normalized, trimmed_path)
@@ -77,7 +79,9 @@ def _perform_request(
if payload is not None: if payload is not None:
data_bytes = json.dumps(payload).encode("utf-8") data_bytes = json.dumps(payload).encode("utf-8")
request_headers = dict(headers or {}) request_headers = dict(headers or {})
req = request.Request(url, data=data_bytes, headers=request_headers, method=method.upper()) req = request.Request(
url, data=data_bytes, headers=request_headers, method=method.upper()
)
try: try:
with request.urlopen(req, timeout=timeout) as response: with request.urlopen(req, timeout=timeout) as response:
body = response.read() body = response.read()
@@ -102,7 +106,9 @@ def list_models(configuration: LLMConfiguration) -> List[Dict[str, str]]:
raise LLMClientError("LLM configuration is incomplete") raise LLMClientError("LLM configuration is incomplete")
url = _build_url(configuration.base_url, "v1/models") url = _build_url(configuration.base_url, "v1/models")
headers = _build_headers(configuration.api_key) headers = _build_headers(configuration.api_key)
payload = _perform_request("GET", url, headers=headers, timeout=configuration.timeout) payload = _perform_request(
"GET", url, headers=headers, timeout=configuration.timeout
)
if not isinstance(payload, Mapping): if not isinstance(payload, Mapping):
raise LLMClientError("Unexpected response when listing models") raise LLMClientError("Unexpected response when listing models")
data = payload.get("data") data = payload.get("data")
@@ -153,7 +159,9 @@ def generate_completion(
if response_format: if response_format:
payload["response_format"] = dict(response_format) payload["response_format"] = dict(response_format)
response = _perform_request("POST", url, headers=headers, payload=payload, timeout=configuration.timeout) response = _perform_request(
"POST", url, headers=headers, payload=payload, timeout=configuration.timeout
)
if not isinstance(response, Mapping): if not isinstance(response, Mapping):
raise LLMClientError("Unexpected response from LLM") raise LLMClientError("Unexpected response from LLM")
choices = response.get("choices") choices = response.get("choices")
+38 -11
View File
@@ -5,7 +5,10 @@ from dataclasses import replace
from functools import lru_cache from functools import lru_cache
from typing import Any, Dict, Mapping, Optional from typing import Any, Dict, Mapping, Optional
from abogen.kokoro_text_normalization import ApostropheConfig, CONTRACTION_CATEGORY_DEFAULTS from abogen.kokoro_text_normalization import (
ApostropheConfig,
CONTRACTION_CATEGORY_DEFAULTS,
)
from abogen.llm_client import LLMConfiguration from abogen.llm_client import LLMConfiguration
from abogen.utils import load_config from abogen.utils import load_config
@@ -149,7 +152,9 @@ def _extract_settings(source: Mapping[str, Any]) -> Dict[str, Any]:
elif isinstance(default, float): elif isinstance(default, float):
extracted[key] = _coerce_float(raw_value, default) extracted[key] = _coerce_float(raw_value, default)
else: else:
extracted[key] = str(raw_value or "") if isinstance(default, str) else raw_value extracted[key] = (
str(raw_value or "") if isinstance(default, str) else raw_value
)
_apply_llm_migrations(extracted) _apply_llm_migrations(extracted)
return extracted return extracted
@@ -177,20 +182,38 @@ def build_apostrophe_config(
config.convert_numbers = bool(settings.get("normalization_numbers", True)) config.convert_numbers = bool(settings.get("normalization_numbers", True))
config.convert_currency = bool(settings.get("normalization_currency", True)) config.convert_currency = bool(settings.get("normalization_currency", True))
config.remove_footnotes = bool(settings.get("normalization_footnotes", True)) config.remove_footnotes = bool(settings.get("normalization_footnotes", True))
config.year_pronunciation_mode = str(settings.get("normalization_numbers_year_style", "american") or "").strip().lower() config.year_pronunciation_mode = (
str(settings.get("normalization_numbers_year_style", "american") or "")
.strip()
.lower()
)
config.add_phoneme_hints = bool(settings.get("normalization_phoneme_hints", True)) config.add_phoneme_hints = bool(settings.get("normalization_phoneme_hints", True))
config.contraction_mode = "expand" if settings.get("normalization_apostrophes_contractions", True) else "keep" config.contraction_mode = (
"expand"
if settings.get("normalization_apostrophes_contractions", True)
else "keep"
)
config.plural_possessive_mode = ( config.plural_possessive_mode = (
"collapse" if settings.get("normalization_apostrophes_plural_possessives", True) else "keep" "collapse"
if settings.get("normalization_apostrophes_plural_possessives", True)
else "keep"
) )
config.sibilant_possessive_mode = ( config.sibilant_possessive_mode = (
"mark" if settings.get("normalization_apostrophes_sibilant_possessives", True) else "keep" "mark"
if settings.get("normalization_apostrophes_sibilant_possessives", True)
else "keep"
)
config.decades_mode = (
"expand" if settings.get("normalization_apostrophes_decades", True) else "keep"
) )
config.decades_mode = "expand" if settings.get("normalization_apostrophes_decades", True) else "keep"
config.leading_elision_mode = ( config.leading_elision_mode = (
"expand" if settings.get("normalization_apostrophes_leading_elisions", True) else "keep" "expand"
if settings.get("normalization_apostrophes_leading_elisions", True)
else "keep"
)
config.ambiguous_past_modal_mode = (
"contextual" if config.contraction_mode == "expand" else "keep"
) )
config.ambiguous_past_modal_mode = "contextual" if config.contraction_mode == "expand" else "keep"
category_flags = dict(CONTRACTION_CATEGORY_DEFAULTS) category_flags = dict(CONTRACTION_CATEGORY_DEFAULTS)
for setting_key, category in _CONTRACTION_SETTING_MAP.items(): for setting_key, category in _CONTRACTION_SETTING_MAP.items():
default_value = bool(_SETTINGS_DEFAULTS.get(setting_key, True)) default_value = bool(_SETTINGS_DEFAULTS.get(setting_key, True))
@@ -205,11 +228,15 @@ def build_llm_configuration(settings: Mapping[str, Any]) -> LLMConfiguration:
base_url=str(settings.get("llm_base_url") or ""), base_url=str(settings.get("llm_base_url") or ""),
api_key=str(settings.get("llm_api_key") or ""), api_key=str(settings.get("llm_api_key") or ""),
model=str(settings.get("llm_model") or ""), model=str(settings.get("llm_model") or ""),
timeout=_coerce_float(settings.get("llm_timeout"), float(_SETTINGS_DEFAULTS["llm_timeout"])), timeout=_coerce_float(
settings.get("llm_timeout"), float(_SETTINGS_DEFAULTS["llm_timeout"])
),
) )
def apply_overrides(base: Mapping[str, Any], overrides: Mapping[str, Any]) -> Dict[str, Any]: def apply_overrides(
base: Mapping[str, Any], overrides: Mapping[str, Any]
) -> Dict[str, Any]:
merged: Dict[str, Any] = dict(base) merged: Dict[str, Any] = dict(base)
for key, value in overrides.items(): for key, value in overrides.items():
if key not in _SETTINGS_DEFAULTS: if key not in _SETTINGS_DEFAULTS:
+13 -4
View File
@@ -41,7 +41,9 @@ def _migrate_legacy_sqlite(target_json_path: Path) -> None:
conn.row_factory = sqlite3.Row conn.row_factory = sqlite3.Row
# Check if table exists # Check if table exists
cursor = conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='overrides'") cursor = conn.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name='overrides'"
)
if not cursor.fetchone(): if not cursor.fetchone():
conn.close() conn.close()
return return
@@ -122,7 +124,9 @@ def load_overrides(language: str, tokens: Iterable[str]) -> Dict[str, Dict[str,
return results return results
def search_overrides(language: str, query: str, *, limit: int = 15) -> List[Dict[str, Any]]: def search_overrides(
language: str, query: str, *, limit: int = 15
) -> List[Dict[str, Any]]:
if not query: if not query:
return [] return []
@@ -137,7 +141,10 @@ def search_overrides(language: str, query: str, *, limit: int = 15) -> List[Dict
matches.append(entry) matches.append(entry)
# Sort by usage count desc, then updated_at desc # Sort by usage count desc, then updated_at desc
matches.sort(key=lambda x: (x.get("usage_count", 0), x.get("updated_at", 0)), reverse=True) matches.sort(
key=lambda x: (x.get("usage_count", 0), x.get("updated_at", 0)),
reverse=True,
)
return matches[:limit] return matches[:limit]
@@ -236,7 +243,9 @@ def get_override_stats(language: str) -> Dict[str, int]:
lang_overrides = db.get("overrides", {}).get(language, {}) lang_overrides = db.get("overrides", {}).get(language, {})
total = len(lang_overrides) total = len(lang_overrides)
with_pronunciation = sum(1 for x in lang_overrides.values() if x.get("pronunciation")) with_pronunciation = sum(
1 for x in lang_overrides.values() if x.get("pronunciation")
)
with_voice = sum(1 for x in lang_overrides.values() if x.get("voice")) with_voice = sum(1 for x in lang_overrides.values() if x.get("voice"))
return { return {
+12 -5
View File
@@ -49,7 +49,9 @@ def _load_spacy_model(model: str = _DEFAULT_MODEL) -> Optional[Language]:
return nlp return nlp
def resolve_ambiguous_contractions(text: str, *, model: Optional[str] = None) -> Dict[Tuple[int, int], ContractionResolution]: def resolve_ambiguous_contractions(
text: str, *, model: Optional[str] = None
) -> Dict[Tuple[int, int], ContractionResolution]:
"""Use spaCy to disambiguate ambiguous contractions in *text*. """Use spaCy to disambiguate ambiguous contractions in *text*.
Returns a mapping from (start, end) spans to their resolved expansion. Returns a mapping from (start, end) spans to their resolved expansion.
@@ -80,7 +82,9 @@ def resolve_ambiguous_contractions(text: str, *, model: Optional[str] = None) ->
return resolutions return resolutions
def _resolution(prev: Token, token: Token, expansion_word: str, category: str, lemma_hint: str) -> Optional[ContractionResolution]: def _resolution(
prev: Token, token: Token, expansion_word: str, category: str, lemma_hint: str
) -> Optional[ContractionResolution]:
if token is None or prev is None: if token is None or prev is None:
return None return None
@@ -186,10 +190,14 @@ def _resolve_apostrophe_d(token: Token) -> Optional[ContractionResolution]:
next_lemma = "" next_lemma = ""
if next_tag == "VB": if next_tag == "VB":
return _resolution(prev, token, "would", "contraction_modal_would", lemma or "will") return _resolution(
prev, token, "would", "contraction_modal_would", lemma or "will"
)
if token.tag_ == "MD" or lemma in {"will", "would", "shall"}: if token.tag_ == "MD" or lemma in {"will", "would", "shall"}:
return _resolution(prev, token, "would", "contraction_modal_would", lemma or "will") return _resolution(
prev, token, "would", "contraction_modal_would", lemma or "will"
)
if next_lemma in {"been", "gone", "had", "better"} or next_tag in {"VBN", "VBD"}: if next_lemma in {"been", "gone", "had", "better"} or next_tag in {"VBN", "VBD"}:
return _resolution(prev, token, "had", "contraction_aux_have", "have") return _resolution(prev, token, "had", "contraction_aux_have", "have")
@@ -255,4 +263,3 @@ def _context_prefers_had(token: Token) -> bool:
if next_lemma == "better": if next_lemma == "better":
return True return True
return False return False
+67 -20
View File
@@ -164,10 +164,16 @@ class SpeakerGuess:
sample_excerpt: Optional[str] = None, sample_excerpt: Optional[str] = None,
) -> None: ) -> None:
self.count += 1 self.count += 1
if _CONFIDENCE_RANK.get(confidence, 0) > _CONFIDENCE_RANK.get(self.confidence, 0): if _CONFIDENCE_RANK.get(confidence, 0) > _CONFIDENCE_RANK.get(
self.confidence, 0
):
self.confidence = confidence self.confidence = confidence
excerpt = sample_excerpt if sample_excerpt is not None else _build_excerpt(text, quote) excerpt = (
sample_excerpt
if sample_excerpt is not None
else _build_excerpt(text, quote)
)
gender_hint = _format_gender_hint(male_votes, female_votes) gender_hint = _format_gender_hint(male_votes, female_votes)
if excerpt: if excerpt:
payload = {"excerpt": excerpt, "gender_hint": gender_hint} payload = {"excerpt": excerpt, "gender_hint": gender_hint}
@@ -180,9 +186,13 @@ class SpeakerGuess:
self.male_votes += male_votes self.male_votes += male_votes
if female_votes: if female_votes:
self.female_votes += female_votes self.female_votes += female_votes
self.detected_gender = _derive_gender(self.male_votes, self.female_votes, self.detected_gender) self.detected_gender = _derive_gender(
self.male_votes, self.female_votes, self.detected_gender
)
if self.gender in {"unknown", "male", "female"}: if self.gender in {"unknown", "male", "female"}:
self.gender = _derive_gender(self.male_votes, self.female_votes, self.gender) self.gender = _derive_gender(
self.male_votes, self.female_votes, self.gender
)
def as_dict(self) -> Dict[str, Any]: def as_dict(self) -> Dict[str, Any]:
return { return {
@@ -211,7 +221,10 @@ class SpeakerAnalysis:
"version": self.version, "version": self.version,
"narrator": self.narrator, "narrator": self.narrator,
"assignments": dict(self.assignments), "assignments": dict(self.assignments),
"speakers": {speaker_id: guess.as_dict() for speaker_id, guess in self.speakers.items()}, "speakers": {
speaker_id: guess.as_dict()
for speaker_id, guess in self.speakers.items()
},
"suppressed": list(self.suppressed), "suppressed": list(self.suppressed),
"stats": dict(self.stats), "stats": dict(self.stats),
} }
@@ -226,7 +239,9 @@ def analyze_speakers(
) -> SpeakerAnalysis: ) -> SpeakerAnalysis:
narrator_id = "narrator" narrator_id = "narrator"
speaker_guesses: Dict[str, SpeakerGuess] = { speaker_guesses: Dict[str, SpeakerGuess] = {
narrator_id: SpeakerGuess(speaker_id=narrator_id, label="Narrator", confidence="low") narrator_id: SpeakerGuess(
speaker_id=narrator_id, label="Narrator", confidence="low"
)
} }
label_index: Dict[str, str] = {"Narrator": narrator_id} label_index: Dict[str, str] = {"Narrator": narrator_id}
assignments: Dict[str, str] = {} assignments: Dict[str, str] = {}
@@ -265,21 +280,31 @@ def analyze_speakers(
if record_id is None: if record_id is None:
record_id = _dedupe_slug(_slugify(label), speaker_guesses) record_id = _dedupe_slug(_slugify(label), speaker_guesses)
label_index[label] = record_id label_index[label] = record_id
speaker_guesses[record_id] = SpeakerGuess(speaker_id=record_id, label=label) speaker_guesses[record_id] = SpeakerGuess(
speaker_id=record_id, label=label
)
guess = speaker_guesses[record_id] guess = speaker_guesses[record_id]
assignments[chunk_id] = record_id assignments[chunk_id] = record_id
unique_speakers.add(record_id) unique_speakers.add(record_id)
if record_id != narrator_id and record_id != speaker_id and speaker_id == last_explicit: if (
record_id != narrator_id
and record_id != speaker_id
and speaker_id == last_explicit
):
last_explicit = record_id last_explicit = record_id
sample_excerpt = None sample_excerpt = None
if record_id != narrator_id: if record_id != narrator_id:
sample_excerpt = _select_sample_excerpt(ordered_chunks, index, guess.label, quote, confidence) sample_excerpt = _select_sample_excerpt(
ordered_chunks, index, guess.label, quote, confidence
)
male_votes, female_votes = _count_gender_votes(text, guess.label) male_votes, female_votes = _count_gender_votes(text, guess.label)
guess.register_occurrence(confidence, text, quote, male_votes, female_votes, sample_excerpt) guess.register_occurrence(
confidence, text, quote, male_votes, female_votes, sample_excerpt
)
active_speakers = [sid for sid in speaker_guesses if sid != narrator_id] active_speakers = [sid for sid in speaker_guesses if sid != narrator_id]
# Apply minimum occurrence threshold. # Apply minimum occurrence threshold.
@@ -302,7 +327,9 @@ def analyze_speakers(
active_speakers = active_speakers[:max_speakers] active_speakers = active_speakers[:max_speakers]
narrator_guess = speaker_guesses[narrator_id] narrator_guess = speaker_guesses[narrator_id]
narrator_guess.count = sum(1 for value in assignments.values() if value == narrator_id) narrator_guess.count = sum(
1 for value in assignments.values() if value == narrator_id
)
narrator_guess.confidence = "low" narrator_guess.confidence = "low"
stats = { stats = {
@@ -322,7 +349,9 @@ def analyze_speakers(
) )
def _infer_chunk_speaker(text: str, last_explicit: Optional[str]) -> Tuple[Optional[str], str, Optional[str]]: def _infer_chunk_speaker(
text: str, last_explicit: Optional[str]
) -> Tuple[Optional[str], str, Optional[str]]:
normalized = text.strip() normalized = text.strip()
if not normalized: if not normalized:
return None, "low", None return None, "low", None
@@ -376,7 +405,9 @@ def _match_name_near_quote(before: str, after: str) -> Optional[str]:
if _looks_like_name(name): if _looks_like_name(name):
return name return name
match = re.search(rf"({_NAME_PATTERN})\s*,?\s*{_VERB_PATTERN}", leading, flags=re.IGNORECASE) match = re.search(
rf"({_NAME_PATTERN})\s*,?\s*{_VERB_PATTERN}", leading, flags=re.IGNORECASE
)
if match: if match:
name = match.group(1) name = match.group(1)
if _looks_like_name(name): if _looks_like_name(name):
@@ -466,8 +497,14 @@ def _count_gender_votes(text: str, label: Optional[str]) -> Tuple[int, int]:
if windows: if windows:
mapped: List[Tuple[int, int]] = [] mapped: List[Tuple[int, int]] = []
for start, end in windows: for start, end in windows:
start_idx = min(len(search_text) - 1, int(start * len(search_text) / max(len(ascii_text), 1))) start_idx = min(
end_idx = min(len(search_text), int(end * len(search_text) / max(len(ascii_text), 1))) len(search_text) - 1,
int(start * len(search_text) / max(len(ascii_text), 1)),
)
end_idx = min(
len(search_text),
int(end * len(search_text) / max(len(ascii_text), 1)),
)
mapped.append((start_idx, end_idx)) mapped.append((start_idx, end_idx))
windows = mapped windows = mapped
else: else:
@@ -485,7 +522,9 @@ def _count_gender_votes(text: str, label: Optional[str]) -> Tuple[int, int]:
except IndexError: except IndexError:
content_start, content_end = match.span() content_start, content_end = match.span()
if content_start < content_end: if content_start < content_end:
quote_spans.append((content_start, content_end, search_text[content_start:content_end])) quote_spans.append(
(content_start, content_end, search_text[content_start:content_end])
)
normalized_label = _normalize_candidate_name(label) if label else None normalized_label = _normalize_candidate_name(label) if label else None
normalized_label_lower = normalized_label.lower() if normalized_label else None normalized_label_lower = normalized_label.lower() if normalized_label else None
@@ -511,7 +550,11 @@ def _count_gender_votes(text: str, label: Optional[str]) -> Tuple[int, int]:
name_matches = list(re.finditer(_NAME_PATTERN, tail)) name_matches = list(re.finditer(_NAME_PATTERN, tail))
if name_matches: if name_matches:
last_name = _normalize_candidate_name(name_matches[-1].group(0)) last_name = _normalize_candidate_name(name_matches[-1].group(0))
if normalized_label_lower and last_name and last_name.lower() == normalized_label_lower: if (
normalized_label_lower
and last_name
and last_name.lower() == normalized_label_lower
):
return 0.6 return 0.6
return 0.05 return 0.05
if re.search(r"[.!?]\s", prefix): if re.search(r"[.!?]\s", prefix):
@@ -607,8 +650,12 @@ def _contains_dialogue_attribution(label: str, text: str, quote: Optional[str])
if not label or not text: if not label or not text:
return False return False
escaped_label = re.escape(label) escaped_label = re.escape(label)
direct_pattern = re.compile(rf"\b{escaped_label}\b\s+(?:{_VERB_PATTERN})\b", re.IGNORECASE) direct_pattern = re.compile(
reverse_pattern = re.compile(rf"(?:{_VERB_PATTERN})\s+\b{escaped_label}\b", re.IGNORECASE) rf"\b{escaped_label}\b\s+(?:{_VERB_PATTERN})\b", re.IGNORECASE
)
reverse_pattern = re.compile(
rf"(?:{_VERB_PATTERN})\s+\b{escaped_label}\b", re.IGNORECASE
)
colon_pattern = re.compile(rf"^\s*{escaped_label}\s*:\s*", re.IGNORECASE) colon_pattern = re.compile(rf"^\s*{escaped_label}\s*:\s*", re.IGNORECASE)
if colon_pattern.search(text): if colon_pattern.search(text):
@@ -679,7 +726,7 @@ def _format_gender_hint(male_votes: int, female_votes: int) -> str:
def _normalize_candidate_name(raw: str) -> Optional[str]: def _normalize_candidate_name(raw: str) -> Optional[str]:
if not raw: if not raw:
return None return None
cleaned = raw.strip().strip('"“”\'.,:;!') cleaned = raw.strip().strip("\"“”'.,:;!")
cleaned = re.sub(r"\s+", " ", cleaned).strip() cleaned = re.sub(r"\s+", " ", cleaned).strip()
if not cleaned: if not cleaned:
return None return None
+5 -1
View File
@@ -135,7 +135,11 @@ def _sanitize_speaker(entry: Dict[str, Any]) -> Dict[str, Any]:
normalized_langs.append(code.lower()) normalized_langs.append(code.lower())
resolved_voice = entry.get("resolved_voice") or voice_formula or voice resolved_voice = entry.get("resolved_voice") or voice_formula or voice
resolved_label = label or entry.get("id") or "" resolved_label = label or entry.get("id") or ""
slug = entry.get("id") if isinstance(entry.get("id"), str) else slugify_label(resolved_label) slug = (
entry.get("id")
if isinstance(entry.get("id"), str)
else slugify_label(resolved_label)
)
return { return {
"id": slug, "id": slug,
"label": resolved_label, "label": resolved_label,
+2
View File
@@ -37,6 +37,7 @@ def clean_subtitle_text(text):
text = _CHAPTER_MARKER_PATTERN.sub("", text) text = _CHAPTER_MARKER_PATTERN.sub("", text)
return text.strip() return text.strip()
def calculate_text_length(text): def calculate_text_length(text):
# Use pre-compiled patterns for better performance # Use pre-compiled patterns for better performance
# Ignore chapter markers and metadata patterns in a single pass # Ignore chapter markers and metadata patterns in a single pass
@@ -48,6 +49,7 @@ def calculate_text_length(text):
char_count = len(text) char_count = len(text)
return char_count return char_count
def clean_text(text, *args, **kwargs): def clean_text(text, *args, **kwargs):
# Remove metadata tags first # Remove metadata tags first
text = _METADATA_TAG_PATTERN.sub("", text) text = _METADATA_TAG_PATTERN.sub("", text)
+112 -35
View File
@@ -110,13 +110,19 @@ def _extract_from_string(raw: str, default_title: str) -> ExtractionResult:
normalized_tags = _normalize_metadata_keys(raw_metadata) normalized_tags = _normalize_metadata_keys(raw_metadata)
chapter_count = len(chapters) chapter_count = len(chapters)
artist_value = normalized_tags.get("artist") artist_value = normalized_tags.get("artist")
authors = [name.strip() for name in artist_value.split(",") if name.strip()] if artist_value else [] authors = (
[name.strip() for name in artist_value.split(",") if name.strip()]
if artist_value
else []
)
metadata_source = MetadataSource( metadata_source = MetadataSource(
title=normalized_tags.get("title") or default_title, title=normalized_tags.get("title") or default_title,
authors=authors, authors=authors,
publication_year=normalized_tags.get("year"), publication_year=normalized_tags.get("year"),
) )
metadata = _build_metadata_payload(metadata_source, chapter_count, "text", default_title) metadata = _build_metadata_payload(
metadata_source, chapter_count, "text", default_title
)
metadata.update(normalized_tags) metadata.update(normalized_tags)
if not chapters: if not chapters:
chapters = [ExtractedChapter(title=default_title, text="")] chapters = [ExtractedChapter(title=default_title, text="")]
@@ -148,9 +154,11 @@ def _split_chapters(content: str, default_title: str) -> List[ExtractedChapter]:
current_title = default_title current_title = default_title
for match in matches: for match in matches:
segment = content[last_index:match.start()] segment = content[last_index : match.start()]
if segment.strip(): if segment.strip():
chapters.append(ExtractedChapter(title=current_title, text=clean_text(segment))) chapters.append(
ExtractedChapter(title=current_title, text=clean_text(segment))
)
current_title = match.group(1).strip() or default_title current_title = match.group(1).strip() or default_title
last_index = match.end() last_index = match.end()
@@ -271,19 +279,27 @@ def _extract_markdown(path: Path) -> ExtractionResult:
raw = path.read_text(encoding=encoding, errors="replace") raw = path.read_text(encoding=encoding, errors="replace")
metadata_source, chapters = _parse_markdown(raw, path.stem) metadata_source, chapters = _parse_markdown(raw, path.stem)
if not chapters: if not chapters:
chapters = [ExtractedChapter(title=metadata_source.title or path.stem, text=clean_text(raw))] chapters = [
metadata = _build_metadata_payload(metadata_source, len(chapters), "markdown", path.stem) ExtractedChapter(
title=metadata_source.title or path.stem, text=clean_text(raw)
)
]
metadata = _build_metadata_payload(
metadata_source, len(chapters), "markdown", path.stem
)
return ExtractionResult(chapters=chapters, metadata=metadata) return ExtractionResult(chapters=chapters, metadata=metadata)
def _parse_markdown(raw: str, default_title: str) -> Tuple[MetadataSource, List[ExtractedChapter]]: def _parse_markdown(
raw: str, default_title: str
) -> Tuple[MetadataSource, List[ExtractedChapter]]:
metadata = MetadataSource() metadata = MetadataSource()
text = textwrap.dedent(raw) text = textwrap.dedent(raw)
frontmatter_match = re.match(r"^---\s*\n(.*?)\n---\s*\n", text, re.DOTALL) frontmatter_match = re.match(r"^---\s*\n(.*?)\n---\s*\n", text, re.DOTALL)
if frontmatter_match: if frontmatter_match:
frontmatter = frontmatter_match.group(1) frontmatter = frontmatter_match.group(1)
_parse_markdown_frontmatter(frontmatter, metadata) _parse_markdown_frontmatter(frontmatter, metadata)
text_body = text[frontmatter_match.end():] text_body = text[frontmatter_match.end() :]
else: else:
text_body = text text_body = text
@@ -324,7 +340,11 @@ def _parse_markdown(raw: str, default_title: str) -> Tuple[MetadataSource, List[
chapters: List[ExtractedChapter] = [] chapters: List[ExtractedChapter] = []
for index, (header_id, start, name) in enumerate(header_positions): for index, (header_id, start, name) in enumerate(header_positions):
end = header_positions[index + 1][1] if index + 1 < len(header_positions) else len(html) end = (
header_positions[index + 1][1]
if index + 1 < len(header_positions)
else len(html)
)
section_html = html[start:end] section_html = html[start:end]
section_soup = BeautifulSoup(section_html, "html.parser") section_soup = BeautifulSoup(section_html, "html.parser")
header_tag = section_soup.find(attrs={"id": header_id}) header_tag = section_soup.find(attrs={"id": header_id})
@@ -336,7 +356,14 @@ def _parse_markdown(raw: str, default_title: str) -> Tuple[MetadataSource, List[
chapters.append(ExtractedChapter(title=name.strip(), text=section_text)) chapters.append(ExtractedChapter(title=name.strip(), text=section_text))
if not metadata.title: if not metadata.title:
first_h1 = next((header for header in headers if header.get("level") == 1 and header.get("name")), None) first_h1 = next(
(
header
for header in headers
if header.get("level") == 1 and header.get("name")
),
None,
)
if first_h1: if first_h1:
metadata.title = str(first_h1["name"]) metadata.title = str(first_h1["name"])
@@ -344,21 +371,27 @@ def _parse_markdown(raw: str, default_title: str) -> Tuple[MetadataSource, List[
def _parse_markdown_frontmatter(frontmatter: str, metadata: MetadataSource) -> None: def _parse_markdown_frontmatter(frontmatter: str, metadata: MetadataSource) -> None:
title_match = re.search(r"^title:\s*(.+)$", frontmatter, re.MULTILINE | re.IGNORECASE) title_match = re.search(
r"^title:\s*(.+)$", frontmatter, re.MULTILINE | re.IGNORECASE
)
if title_match: if title_match:
metadata.title = title_match.group(1).strip().strip('"\'') metadata.title = title_match.group(1).strip().strip("\"'")
author_match = re.search(r"^author:\s*(.+)$", frontmatter, re.MULTILINE | re.IGNORECASE) author_match = re.search(
r"^author:\s*(.+)$", frontmatter, re.MULTILINE | re.IGNORECASE
)
if author_match: if author_match:
metadata.authors = [author_match.group(1).strip().strip('"\'')] metadata.authors = [author_match.group(1).strip().strip("\"'")]
desc_match = re.search(r"^description:\s*(.+)$", frontmatter, re.MULTILINE | re.IGNORECASE) desc_match = re.search(
r"^description:\s*(.+)$", frontmatter, re.MULTILINE | re.IGNORECASE
)
if desc_match: if desc_match:
metadata.description = desc_match.group(1).strip().strip('"\'') metadata.description = desc_match.group(1).strip().strip("\"'")
date_match = re.search(r"^date:\s*(.+)$", frontmatter, re.MULTILINE | re.IGNORECASE) date_match = re.search(r"^date:\s*(.+)$", frontmatter, re.MULTILINE | re.IGNORECASE)
if date_match: if date_match:
date_str = date_match.group(1).strip().strip('\"\'') date_str = date_match.group(1).strip().strip("\"'")
year_match = re.search(r"\b(19|20)\d{2}\b", date_str) year_match = re.search(r"\b(19|20)\d{2}\b", date_str)
if year_match: if year_match:
metadata.publication_year = year_match.group(0) metadata.publication_year = year_match.group(0)
@@ -390,7 +423,9 @@ class EpubExtractor:
chapters = self._process_spine_fallback() chapters = self._process_spine_fallback()
if not chapters: if not chapters:
chapters = [ExtractedChapter(title=self.path.stem, text="")] chapters = [ExtractedChapter(title=self.path.stem, text="")]
metadata = _build_metadata_payload(metadata_source, len(chapters), "epub", self.path.stem) metadata = _build_metadata_payload(
metadata_source, len(chapters), "epub", self.path.stem
)
metadata.setdefault("chapter_count", str(len(chapters))) metadata.setdefault("chapter_count", str(len(chapters)))
if metadata_source.series: if metadata_source.series:
series_text = str(metadata_source.series).strip() series_text = str(metadata_source.series).strip()
@@ -424,7 +459,9 @@ class EpubExtractor:
try: try:
author_items = self.book.get_metadata("DC", "creator") author_items = self.book.get_metadata("DC", "creator")
if author_items: if author_items:
metadata.authors = [author[0] for author in author_items if author and author[0]] metadata.authors = [
author[0] for author in author_items if author and author[0]
]
except Exception as exc: except Exception as exc:
logger.debug("Failed to extract EPUB author metadata: %s", exc) logger.debug("Failed to extract EPUB author metadata: %s", exc)
@@ -447,7 +484,9 @@ class EpubExtractor:
if date_items: if date_items:
date_str = date_items[0][0] date_str = date_items[0][0]
year_match = re.search(r"\b(19|20)\d{2}\b", date_str) year_match = re.search(r"\b(19|20)\d{2}\b", date_str)
metadata.publication_year = year_match.group(0) if year_match else date_str metadata.publication_year = (
year_match.group(0) if year_match else date_str
)
except Exception as exc: except Exception as exc:
logger.debug("Failed to extract EPUB publication year metadata: %s", exc) logger.debug("Failed to extract EPUB publication year metadata: %s", exc)
@@ -482,7 +521,16 @@ class EpubExtractor:
if name in {"calibre:series", "series"} and series_name is None: if name in {"calibre:series", "series"} and series_name is None:
series_name = candidate_text series_name = candidate_text
continue continue
if name in {"calibre:series_index", "calibre:seriesindex", "series_index", "seriesindex"} and series_index is None: if (
name
in {
"calibre:series_index",
"calibre:seriesindex",
"series_index",
"seriesindex",
}
and series_index is None
):
series_index = candidate_text series_index = candidate_text
continue continue
@@ -533,7 +581,9 @@ class EpubExtractor:
self.spine_docs = self._build_spine_docs() self.spine_docs = self._build_spine_docs()
doc_order = {href: index for index, href in enumerate(self.spine_docs)} doc_order = {href: index for index, href in enumerate(self.spine_docs)}
doc_order_decoded = {urllib.parse.unquote(href): index for href, index in doc_order.items()} doc_order_decoded = {
urllib.parse.unquote(href): index for href, index in doc_order.items()
}
nav_targets = self._collect_nav_targets(nav_soup, nav_type) nav_targets = self._collect_nav_targets(nav_soup, nav_type)
self._cache_relevant_documents(doc_order, nav_targets) self._cache_relevant_documents(doc_order, nav_targets)
@@ -544,7 +594,9 @@ class EpubExtractor:
if not nav_map: if not nav_map:
raise ValueError("NCX navigation missing <navMap>") raise ValueError("NCX navigation missing <navMap>")
for nav_point in nav_map.find_all("navPoint", recursive=False): for nav_point in nav_map.find_all("navPoint", recursive=False):
self._parse_ncx_navpoint(nav_point, ordered_entries, doc_order, doc_order_decoded) self._parse_ncx_navpoint(
nav_point, ordered_entries, doc_order, doc_order_decoded
)
else: else:
toc_nav = nav_soup.find("nav", attrs={"epub:type": "toc"}) toc_nav = nav_soup.find("nav", attrs={"epub:type": "toc"})
if toc_nav is None: if toc_nav is None:
@@ -558,7 +610,9 @@ class EpubExtractor:
if top_ol is None: if top_ol is None:
raise ValueError("TOC navigation missing <ol>") raise ValueError("TOC navigation missing <ol>")
for li in top_ol.find_all("li", recursive=False): for li in top_ol.find_all("li", recursive=False):
self._parse_html_nav_li(li, ordered_entries, doc_order, doc_order_decoded) self._parse_html_nav_li(
li, ordered_entries, doc_order, doc_order_decoded
)
if not ordered_entries: if not ordered_entries:
raise ValueError("No navigation entries found") raise ValueError("No navigation entries found")
@@ -591,7 +645,9 @@ class EpubExtractor:
text = self._html_to_text(html_content) text = self._html_to_text(html_content)
if not text: if not text:
continue continue
title = self._resolve_document_title(html_content, fallback=f"Untitled Chapter {index + 1}") title = self._resolve_document_title(
html_content, fallback=f"Untitled Chapter {index + 1}"
)
chapters.append(ExtractedChapter(title=title, text=text)) chapters.append(ExtractedChapter(title=title, text=text))
return chapters return chapters
@@ -605,7 +661,8 @@ class EpubExtractor:
( (
item item
for item in nav_items for item in nav_items
if "nav" in item.get_name().lower() and item.get_name().lower().endswith((".xhtml", ".html")) if "nav" in item.get_name().lower()
and item.get_name().lower().endswith((".xhtml", ".html"))
), ),
None, None,
) )
@@ -627,7 +684,11 @@ class EpubExtractor:
if not nav_item and nav_items: if not nav_item and nav_items:
ncx_candidate = next( ncx_candidate = next(
(item for item in nav_items if item.get_name().lower().endswith(".ncx")), (
item
for item in nav_items
if item.get_name().lower().endswith(".ncx")
),
None, None,
) )
if ncx_candidate: if ncx_candidate:
@@ -682,7 +743,9 @@ class EpubExtractor:
targets.append(href_value.split("#", 1)[0]) targets.append(href_value.split("#", 1)[0])
return targets return targets
def _cache_relevant_documents(self, doc_order: Dict[str, int], nav_targets: List[str]) -> None: def _cache_relevant_documents(
self, doc_order: Dict[str, int], nav_targets: List[str]
) -> None:
needed: set[str] = set(doc_order.keys()) needed: set[str] = set(doc_order.keys())
for target in nav_targets: for target in nav_targets:
needed.add(target) needed.add(target)
@@ -718,7 +781,9 @@ class EpubExtractor:
if src: if src:
base_href, fragment = src.split("#", 1) if "#" in src else (src, None) base_href, fragment = src.split("#", 1) if "#" in src else (src, None)
doc_key, doc_idx = self._find_doc_key(base_href, doc_order, doc_order_decoded) doc_key, doc_idx = self._find_doc_key(
base_href, doc_order, doc_order_decoded
)
if doc_key is not None and doc_idx is not None: if doc_key is not None and doc_idx is not None:
position = self._find_position_robust(doc_key, fragment) position = self._find_position_robust(doc_key, fragment)
ordered_entries.append( ordered_entries.append(
@@ -738,7 +803,9 @@ class EpubExtractor:
) )
for child_navpoint in nav_point.find_all("navPoint", recursive=False): for child_navpoint in nav_point.find_all("navPoint", recursive=False):
self._parse_ncx_navpoint(child_navpoint, ordered_entries, doc_order, doc_order_decoded) self._parse_ncx_navpoint(
child_navpoint, ordered_entries, doc_order, doc_order_decoded
)
def _parse_html_nav_li( def _parse_html_nav_li(
self, self,
@@ -767,7 +834,9 @@ class EpubExtractor:
if src: if src:
base_href, fragment = src.split("#", 1) if "#" in src else (src, None) base_href, fragment = src.split("#", 1) if "#" in src else (src, None)
doc_key, doc_idx = self._find_doc_key(base_href, doc_order, doc_order_decoded) doc_key, doc_idx = self._find_doc_key(
base_href, doc_order, doc_order_decoded
)
if doc_key is not None and doc_idx is not None: if doc_key is not None and doc_idx is not None:
position = self._find_position_robust(doc_key, fragment) position = self._find_position_robust(doc_key, fragment)
ordered_entries.append( ordered_entries.append(
@@ -788,7 +857,9 @@ class EpubExtractor:
for child_ol in li_element.find_all("ol", recursive=False): for child_ol in li_element.find_all("ol", recursive=False):
for child_li in child_ol.find_all("li", recursive=False): for child_li in child_ol.find_all("li", recursive=False):
self._parse_html_nav_li(child_li, ordered_entries, doc_order, doc_order_decoded) self._parse_html_nav_li(
child_li, ordered_entries, doc_order, doc_order_decoded
)
def _find_doc_key( def _find_doc_key(
self, self,
@@ -825,7 +896,9 @@ class EpubExtractor:
if pos != -1: if pos != -1:
return pos return pos
except Exception: except Exception:
logger.debug("BeautifulSoup failed to locate id '%s' in %s", fragment_id, doc_href) logger.debug(
"BeautifulSoup failed to locate id '%s' in %s", fragment_id, doc_href
)
safe_fragment_id = re.escape(fragment_id) safe_fragment_id = re.escape(fragment_id)
id_name_pattern = re.compile( id_name_pattern = re.compile(
@@ -844,13 +917,17 @@ class EpubExtractor:
tag_start = html_content.rfind("<", 0, pos) tag_start = html_content.rfind("<", 0, pos)
return tag_start if tag_start != -1 else pos return tag_start if tag_start != -1 else pos
logger.warning("Anchor '%s' not found in %s. Defaulting to start.", fragment_id, doc_href) logger.warning(
"Anchor '%s' not found in %s. Defaulting to start.", fragment_id, doc_href
)
return 0 return 0
def _slice_entries(self, ordered_entries: List[NavEntry]) -> List[ExtractedChapter]: def _slice_entries(self, ordered_entries: List[NavEntry]) -> List[ExtractedChapter]:
chapters: List[ExtractedChapter] = [] chapters: List[ExtractedChapter] = []
for index, entry in enumerate(ordered_entries): for index, entry in enumerate(ordered_entries):
next_entry = ordered_entries[index + 1] if index + 1 < len(ordered_entries) else None next_entry = (
ordered_entries[index + 1] if index + 1 < len(ordered_entries) else None
)
slice_html = self._slice_entry(entry, next_entry) slice_html = self._slice_entry(entry, next_entry)
text = self._html_to_text(slice_html) text = self._html_to_text(slice_html)
if not text: if not text:
+17 -5
View File
@@ -47,7 +47,9 @@ def _resample_linear(audio: np.ndarray, src_rate: int, dst_rate: int) -> np.ndar
return np.interp(x_new, x_old, audio).astype("float32", copy=False) return np.interp(x_new, x_old, audio).astype("float32", copy=False)
def _split_text(text: str, *, split_pattern: Optional[str], max_chunk_length: int) -> list[str]: def _split_text(
text: str, *, split_pattern: Optional[str], max_chunk_length: int
) -> list[str]:
stripped = (text or "").strip() stripped = (text or "").strip()
if not stripped: if not stripped:
return [] return []
@@ -81,13 +83,17 @@ def _split_text(text: str, *, split_pattern: Optional[str], max_chunk_length: in
return result return result
_UNSUPPORTED_CHARS_RE = re.compile(r"unsupported character\(s\):\s*(\[[^\]]*\])", re.IGNORECASE) _UNSUPPORTED_CHARS_RE = re.compile(
r"unsupported character\(s\):\s*(\[[^\]]*\])", re.IGNORECASE
)
def _parse_unsupported_characters(error: BaseException) -> list[str]: def _parse_unsupported_characters(error: BaseException) -> list[str]:
"""Best-effort extraction of unsupported characters from SuperTonic errors.""" """Best-effort extraction of unsupported characters from SuperTonic errors."""
message = " ".join(str(part) for part in getattr(error, "args", ()) if part is not None) or str(error) message = " ".join(
str(part) for part in getattr(error, "args", ()) if part is not None
) or str(error)
match = _UNSUPPORTED_CHARS_RE.search(message) match = _UNSUPPORTED_CHARS_RE.search(message)
if not match: if not match:
return [] return []
@@ -127,6 +133,7 @@ def _configure_supertonic_gpu() -> None:
"""Patch supertonic's config to enable GPU acceleration if available.""" """Patch supertonic's config to enable GPU acceleration if available."""
try: try:
import onnxruntime as ort import onnxruntime as ort
available = ort.get_available_providers() available = ort.get_available_providers()
# Use CUDA if available, skip TensorRT (requires extra libs not always present) # Use CUDA if available, skip TensorRT (requires extra libs not always present)
@@ -141,6 +148,7 @@ def _configure_supertonic_gpu() -> None:
# We must patch both because loader imports the value at module load time # We must patch both because loader imports the value at module load time
import supertonic.config as supertonic_config import supertonic.config as supertonic_config
import supertonic.loader as supertonic_loader import supertonic.loader as supertonic_loader
supertonic_config.DEFAULT_ONNX_PROVIDERS = providers supertonic_config.DEFAULT_ONNX_PROVIDERS = providers
supertonic_loader.DEFAULT_ONNX_PROVIDERS = providers supertonic_loader.DEFAULT_ONNX_PROVIDERS = providers
logger.info("Supertonic ONNX providers configured: %s", providers) logger.info("Supertonic ONNX providers configured: %s", providers)
@@ -191,7 +199,9 @@ class SupertonicPipeline:
speed_value = max(0.7, min(2.0, speed_value)) speed_value = max(0.7, min(2.0, speed_value))
style = self._tts.get_voice_style(voice_name=voice_name) style = self._tts.get_voice_style(voice_name=voice_name)
chunks = _split_text(text, split_pattern=split_pattern, max_chunk_length=self.max_chunk_length) chunks = _split_text(
text, split_pattern=split_pattern, max_chunk_length=self.max_chunk_length
)
for chunk in chunks: for chunk in chunks:
chunk_to_speak = chunk chunk_to_speak = chunk
removed: set[str] = set() removed: set[str] = set()
@@ -217,7 +227,9 @@ class SupertonicPipeline:
raise raise
removed.update(unsupported) removed.update(unsupported)
sanitized = _remove_unsupported_characters(chunk_to_speak, unsupported).strip() sanitized = _remove_unsupported_characters(
chunk_to_speak, unsupported
).strip()
# If we didn't change anything, don't loop forever. # If we didn't change anything, don't loop forever.
if sanitized == chunk_to_speak.strip(): if sanitized == chunk_to_speak.strip():
+17 -7
View File
@@ -14,6 +14,7 @@ from functools import lru_cache
from dotenv import load_dotenv, find_dotenv from dotenv import load_dotenv, find_dotenv
def _load_environment() -> None: def _load_environment() -> None:
explicit_path = os.environ.get("ABOGEN_ENV_FILE") explicit_path = os.environ.get("ABOGEN_ENV_FILE")
if explicit_path: if explicit_path:
@@ -56,6 +57,7 @@ def detect_encoding(file_path):
encoding = detected_encoding if detected_encoding else "utf-8" encoding = detected_encoding if detected_encoding else "utf-8"
return encoding.lower() return encoding.lower()
def get_resource_path(package, resource): def get_resource_path(package, resource):
""" """
Get the path to a resource file, with fallback to local file system. Get the path to a resource file, with fallback to local file system.
@@ -146,7 +148,9 @@ def get_user_settings_dir():
if os.path.exists(legacy_dir): if os.path.exists(legacy_dir):
return ensure_directory(legacy_dir) return ensure_directory(legacy_dir)
config_dir = user_config_dir("abogen", appauthor=False, roaming=True, ensure_exists=True) config_dir = user_config_dir(
"abogen", appauthor=False, roaming=True, ensure_exists=True
)
return ensure_directory(config_dir) return ensure_directory(config_dir)
@@ -250,7 +254,9 @@ def get_user_cache_root():
def get_internal_cache_root(): def get_internal_cache_root():
root = os.environ.get("ABOGEN_INTERNAL_CACHE_ROOT") or os.environ.get("XDG_CACHE_HOME") root = os.environ.get("ABOGEN_INTERNAL_CACHE_ROOT") or os.environ.get(
"XDG_CACHE_HOME"
)
if root: if root:
return ensure_directory(root) return ensure_directory(root)
home_dir = os.environ.get("HOME") or os.path.join("/tmp", "abogen-home") home_dir = os.environ.get("HOME") or os.path.join("/tmp", "abogen-home")
@@ -274,9 +280,8 @@ def get_user_cache_path(folder=None):
@lru_cache(maxsize=1) @lru_cache(maxsize=1)
def get_user_output_root(): def get_user_output_root():
override = ( override = os.environ.get("ABOGEN_OUTPUT_DIR") or os.environ.get(
os.environ.get("ABOGEN_OUTPUT_DIR") "ABOGEN_OUTPUT_ROOT"
or os.environ.get("ABOGEN_OUTPUT_ROOT")
) )
if override: if override:
return ensure_directory(override) return ensure_directory(override)
@@ -290,7 +295,10 @@ def get_user_output_path(folder=None):
return base return base
_sleep_procs: Dict[str, Optional[subprocess.Popen[str]]] = {"Darwin": None, "Linux": None} # Store sleep prevention processes _sleep_procs: Dict[str, Optional[subprocess.Popen[str]]] = {
"Darwin": None,
"Linux": None,
} # Store sleep prevention processes
def clean_text(text, *args, **kwargs): def clean_text(text, *args, **kwargs):
@@ -328,7 +336,9 @@ def create_process(cmd, stdin=None, text=True, capture_output=False):
# Determine shell usage: use shell only for string commands # Determine shell usage: use shell only for string commands
use_shell = isinstance(cmd, str) use_shell = isinstance(cmd, str)
if use_shell: if use_shell:
logger.warning("Security Warning: create_process called with string command. Prefer using a list of arguments to avoid shell injection risks.") logger.warning(
"Security Warning: create_process called with string command. Prefer using a list of arguments to avoid shell injection risks."
)
kwargs = { kwargs = {
"shell": use_shell, "shell": use_shell,
+2
View File
@@ -12,9 +12,11 @@ except Exception: # pragma: no cover - import fallback
LocalEntryNotFoundError = None # type: ignore[assignment] LocalEntryNotFoundError = None # type: ignore[assignment]
if LocalEntryNotFoundError is None: # pragma: no cover - fallback for tests if LocalEntryNotFoundError is None: # pragma: no cover - fallback for tests
class LocalEntryNotFoundError(Exception): class LocalEntryNotFoundError(Exception):
pass pass
from abogen.constants import VOICES_INTERNAL from abogen.constants import VOICES_INTERNAL
_CACHE_LOCK = threading.Lock() _CACHE_LOCK = threading.Lock()
+11 -3
View File
@@ -110,9 +110,17 @@ def normalize_profile_entry(entry: Any) -> Dict[str, Any]:
return { return {
"provider": "supertonic", "provider": "supertonic",
"language": language, "language": language,
"voice": _normalize_supertonic_voice(entry.get("voice") or entry.get("voice_name") or entry.get("name")), "voice": _normalize_supertonic_voice(
"total_steps": _coerce_supertonic_steps(entry.get("total_steps") or entry.get("supertonic_total_steps") or entry.get("quality")), entry.get("voice") or entry.get("voice_name") or entry.get("name")
"speed": _coerce_supertonic_speed(entry.get("speed") or entry.get("supertonic_speed")), ),
"total_steps": _coerce_supertonic_steps(
entry.get("total_steps")
or entry.get("supertonic_total_steps")
or entry.get("quality")
),
"speed": _coerce_supertonic_speed(
entry.get("speed") or entry.get("supertonic_speed")
),
} }
voices = _normalize_voice_entries(entry.get("voices", [])) voices = _normalize_voice_entries(entry.get("voices", []))
+1
View File
@@ -31,6 +31,7 @@ def main():
# Check if build module is installed, install if not # Check if build module is installed, install if not
# Temporarily remove script_dir from sys.path to avoid importing local build.py # Temporarily remove script_dir from sys.path to avoid importing local build.py
import sys import sys
original_path = sys.path[:] original_path = sys.path[:]
try: try:
sys.path = [p for p in sys.path if os.path.abspath(p) != script_dir] sys.path = [p for p in sys.path if os.path.abspath(p) != script_dir]
+3 -1
View File
@@ -10,7 +10,9 @@ import sys
from types import ModuleType from types import ModuleType
def _soundfile_write_stub(file_obj, data, samplerate, format="WAV", **_kwargs): # pragma: no cover - stub def _soundfile_write_stub(
file_obj, data, samplerate, format="WAV", **_kwargs
): # pragma: no cover - stub
"""Minimal stand-in for soundfile.write used in tests. """Minimal stand-in for soundfile.write used in tests.
The real library streams waveform data to disk. Our tests don't exercise The real library streams waveform data to disk. Our tests don't exercise
+4 -1
View File
@@ -2,7 +2,10 @@ from __future__ import annotations
import json import json
from abogen.integrations.audiobookshelf import AudiobookshelfClient, AudiobookshelfConfig from abogen.integrations.audiobookshelf import (
AudiobookshelfClient,
AudiobookshelfConfig,
)
def test_upload_fields_include_series_sequence(tmp_path): def test_upload_fields_include_series_sequence(tmp_path):
+8 -3
View File
@@ -6,7 +6,7 @@ import time
from PyQt6.QtWidgets import QApplication from PyQt6.QtWidgets import QApplication
# Ensure we can import the module # Ensure we can import the module
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from abogen.book_handler import HandlerDialog from abogen.book_handler import HandlerDialog
from ebooklib import epub from ebooklib import epub
@@ -14,6 +14,7 @@ from ebooklib import epub
# We need a QApplication instance for QWriter/QDialog # We need a QApplication instance for QWriter/QDialog
app = QApplication(sys.argv) app = QApplication(sys.argv)
class TestBookHandlerRegression(unittest.TestCase): class TestBookHandlerRegression(unittest.TestCase):
def setUp(self): def setUp(self):
@@ -63,10 +64,13 @@ class TestBookHandlerRegression(unittest.TestCase):
# We can check if content_texts is populated # We can check if content_texts is populated
if dialog.content_texts: if dialog.content_texts:
break break
app.processEvents() # Process Qt events to let thread signals propagate app.processEvents() # Process Qt events to let thread signals propagate
time.sleep(0.1) time.sleep(0.1)
self.assertTrue(len(dialog.content_texts) > 0, "HandlerDialog failed to process content in time") self.assertTrue(
len(dialog.content_texts) > 0,
"HandlerDialog failed to process content in time",
)
# Validate content similar to what we expect # Validate content similar to what we expect
# intro.xhtml should be there # intro.xhtml should be there
@@ -80,5 +84,6 @@ class TestBookHandlerRegression(unittest.TestCase):
# Cleanup # Cleanup
dialog.close() dialog.close()
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
+5 -3
View File
@@ -6,10 +6,11 @@ import fitz # PyMuPDF
from ebooklib import epub from ebooklib import epub
# Ensure we can import the module # Ensure we can import the module
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from abogen.book_parser import get_book_parser, PdfParser, EpubParser, MarkdownParser from abogen.book_parser import get_book_parser, PdfParser, EpubParser, MarkdownParser
class TestBookParser(unittest.TestCase): class TestBookParser(unittest.TestCase):
def setUp(self): def setUp(self):
@@ -38,7 +39,7 @@ class TestBookParser(unittest.TestCase):
page1.insert_text((50, 50), "Page 1 content") page1.insert_text((50, 50), "Page 1 content")
# Add pattern to be cleaned # Add pattern to be cleaned
page1.insert_text((50, 100), "[12]") page1.insert_text((50, 100), "[12]")
page1.insert_text((50, 200), "1") # Page number at bottom page1.insert_text((50, 200), "1") # Page number at bottom
# Page 2 # Page 2
page2 = doc.new_page() page2 = doc.new_page()
@@ -156,7 +157,7 @@ class TestBookParser(unittest.TestCase):
def test_find_position_robust_logic(self): def test_find_position_robust_logic(self):
"""Unit test for _find_position_robust on EpubParser.""" """Unit test for _find_position_robust on EpubParser."""
parser = EpubParser(self.sample_epub_path) # Instantiate directly parser = EpubParser(self.sample_epub_path) # Instantiate directly
html = '<html><body><p>Start</p><h1 id="target">Heading</h1><p>End</p></body></html>' html = '<html><body><p>Start</p><h1 id="target">Heading</h1><p>End</p></body></html>'
parser.doc_content["dummy.html"] = html parser.doc_content["dummy.html"] = html
@@ -206,5 +207,6 @@ class TestBookParser(unittest.TestCase):
md_parser = MarkdownParser(self.sample_md_path) md_parser = MarkdownParser(self.sample_md_path)
self.assertEqual(md_parser.file_type, "markdown") self.assertEqual(md_parser.file_type, "markdown")
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
+345 -314
View File
@@ -1,4 +1,10 @@
from abogen.integrations.calibre_opds import CalibreOPDSClient, OPDSEntry, OPDSFeed, OPDSLink, feed_to_dict from abogen.integrations.calibre_opds import (
CalibreOPDSClient,
OPDSEntry,
OPDSFeed,
OPDSLink,
feed_to_dict,
)
def test_calibre_opds_feed_exposes_series_metadata() -> None: def test_calibre_opds_feed_exposes_series_metadata() -> None:
@@ -158,25 +164,33 @@ This is the detailed summary text.</summary>
def test_calibre_opds_relative_urls_keep_catalog_prefix() -> None: def test_calibre_opds_relative_urls_keep_catalog_prefix() -> None:
client = CalibreOPDSClient("http://example.com/opds/") client = CalibreOPDSClient("http://example.com/opds/")
assert client._make_url("search") == "http://example.com/opds/search" assert client._make_url("search") == "http://example.com/opds/search"
assert client._make_url("books/sample.epub") == "http://example.com/opds/books/sample.epub" assert (
assert client._make_url("/cover/1") == "http://example.com/cover/1" client._make_url("books/sample.epub")
assert client._make_url("?page=2") == "http://example.com/opds?page=2" == "http://example.com/opds/books/sample.epub"
)
assert client._make_url("/cover/1") == "http://example.com/cover/1"
assert client._make_url("?page=2") == "http://example.com/opds?page=2"
def test_calibre_opds_base_url_without_trailing_slash() -> None: def test_calibre_opds_base_url_without_trailing_slash() -> None:
"""Ensure the client works with base URLs that don't have trailing slashes.""" """Ensure the client works with base URLs that don't have trailing slashes."""
client = CalibreOPDSClient("http://example.com/api/v1/opds") client = CalibreOPDSClient("http://example.com/api/v1/opds")
# Base URL should be stored without trailing slash # Base URL should be stored without trailing slash
assert client._base_url == "http://example.com/api/v1/opds" assert client._base_url == "http://example.com/api/v1/opds"
# Relative paths should resolve as siblings to the base URL # Relative paths should resolve as siblings to the base URL
assert client._make_url("catalog") == "http://example.com/api/v1/opds/catalog" assert client._make_url("catalog") == "http://example.com/api/v1/opds/catalog"
assert client._make_url("search?q=test") == "http://example.com/api/v1/opds/search?q=test" assert (
assert client._make_url("/api/v1/opds/books") == "http://example.com/api/v1/opds/books" client._make_url("search?q=test")
assert client._make_url("?page=2") == "http://example.com/api/v1/opds?page=2" == "http://example.com/api/v1/opds/search?q=test"
)
assert (
client._make_url("/api/v1/opds/books") == "http://example.com/api/v1/opds/books"
)
assert client._make_url("?page=2") == "http://example.com/api/v1/opds?page=2"
def test_calibre_opds_filters_out_unsupported_formats() -> None: def test_calibre_opds_filters_out_unsupported_formats() -> None:
@@ -253,365 +267,382 @@ def test_calibre_opds_navigation_entries_without_download_are_preserved() -> Non
def test_calibre_opds_search_filters_by_title_and_author() -> None: def test_calibre_opds_search_filters_by_title_and_author() -> None:
client = CalibreOPDSClient("http://example.com/catalog") client = CalibreOPDSClient("http://example.com/catalog")
feed = OPDSFeed( feed = OPDSFeed(
id="catalog", id="catalog",
title="Catalog", title="Catalog",
entries=[ entries=[
OPDSEntry(id="1", title="The Long Journey", authors=["Alice Smith"]), OPDSEntry(id="1", title="The Long Journey", authors=["Alice Smith"]),
OPDSEntry(id="2", title="Hidden Worlds", authors=["Bob Johnson"]), OPDSEntry(id="2", title="Hidden Worlds", authors=["Bob Johnson"]),
OPDSEntry(id="3", title="Side Stories", authors=["Cara Nguyen"], series="Journey Tales"), OPDSEntry(
], id="3",
) title="Side Stories",
authors=["Cara Nguyen"],
series="Journey Tales",
),
],
)
filtered = client._filter_feed_entries(feed, "journey alice") filtered = client._filter_feed_entries(feed, "journey alice")
assert [entry.id for entry in filtered.entries] == ["1"] assert [entry.id for entry in filtered.entries] == ["1"]
filtered = client._filter_feed_entries(feed, "bob") filtered = client._filter_feed_entries(feed, "bob")
assert [entry.id for entry in filtered.entries] == ["2"] assert [entry.id for entry in filtered.entries] == ["2"]
filtered = client._filter_feed_entries(feed, "journey tales") filtered = client._filter_feed_entries(feed, "journey tales")
assert [entry.id for entry in filtered.entries] == ["3"] assert [entry.id for entry in filtered.entries] == ["3"]
filtered = client._filter_feed_entries(feed, "missing") filtered = client._filter_feed_entries(feed, "missing")
assert filtered.entries == [] assert filtered.entries == []
def test_calibre_opds_local_search_follows_next(monkeypatch) -> None: def test_calibre_opds_local_search_follows_next(monkeypatch) -> None:
client = CalibreOPDSClient("http://example.com/catalog") client = CalibreOPDSClient("http://example.com/catalog")
page_one = OPDSFeed( page_one = OPDSFeed(
id="catalog", id="catalog",
title="Catalog", title="Catalog",
entries=[OPDSEntry(id="1", title="Unrelated", authors=["Alice Smith"])], entries=[OPDSEntry(id="1", title="Unrelated", authors=["Alice Smith"])],
links={"next": OPDSLink(href="http://example.com/catalog?page=2", rel="next")}, links={"next": OPDSLink(href="http://example.com/catalog?page=2", rel="next")},
) )
page_two = OPDSFeed( page_two = OPDSFeed(
id="catalog", id="catalog",
title="Catalog", title="Catalog",
entries=[OPDSEntry(id="2", title="The Journey Continues", authors=["Bob Johnson"])], entries=[
links={}, OPDSEntry(id="2", title="The Journey Continues", authors=["Bob Johnson"])
) ],
links={},
)
def fake_fetch(href=None, params=None): def fake_fetch(href=None, params=None):
if href == "http://example.com/catalog?page=2": if href == "http://example.com/catalog?page=2":
return page_two return page_two
return page_one return page_one
monkeypatch.setattr(client, "fetch_feed", fake_fetch) monkeypatch.setattr(client, "fetch_feed", fake_fetch)
result = client._local_search("journey", seed_feed=page_one) result = client._local_search("journey", seed_feed=page_one)
assert [entry.id for entry in result.entries] == ["2"] assert [entry.id for entry in result.entries] == ["2"]
def test_calibre_opds_local_search_traverses_navigation(monkeypatch) -> None: def test_calibre_opds_local_search_traverses_navigation(monkeypatch) -> None:
client = CalibreOPDSClient("http://example.com/catalog") client = CalibreOPDSClient("http://example.com/catalog")
root_feed = OPDSFeed( root_feed = OPDSFeed(
id="catalog", id="catalog",
title="Catalog", title="Catalog",
entries=[ entries=[
OPDSEntry( OPDSEntry(
id="nav-authors", id="nav-authors",
title="Browse Authors", title="Browse Authors",
links=[ links=[
OPDSLink( OPDSLink(
href="http://example.com/catalog/authors", href="http://example.com/catalog/authors",
rel="http://opds-spec.org/navigation", rel="http://opds-spec.org/navigation",
type="application/atom+xml;profile=opds-catalog", type="application/atom+xml;profile=opds-catalog",
) )
],
)
], ],
) links={},
], )
links={}, authors_feed = OPDSFeed(
) id="authors",
authors_feed = OPDSFeed( title="Authors",
id="authors", entries=[
title="Authors", OPDSEntry(
entries=[ id="book-42",
OPDSEntry(id="book-42", title="The Count of Monte Cristo", authors=["Alexandre Dumas"]) title="The Count of Monte Cristo",
], authors=["Alexandre Dumas"],
links={}, )
) ],
links={},
)
def fake_fetch(href=None, params=None): def fake_fetch(href=None, params=None):
if href == "http://example.com/catalog/authors": if href == "http://example.com/catalog/authors":
return authors_feed return authors_feed
return root_feed return root_feed
monkeypatch.setattr(client, "fetch_feed", fake_fetch) monkeypatch.setattr(client, "fetch_feed", fake_fetch)
result = client._local_search("monte cristo", seed_feed=root_feed) result = client._local_search("monte cristo", seed_feed=root_feed)
assert [entry.id for entry in result.entries] == ["book-42"] assert [entry.id for entry in result.entries] == ["book-42"]
def test_calibre_opds_search_falls_back_to_local_search(monkeypatch) -> None: def test_calibre_opds_search_falls_back_to_local_search(monkeypatch) -> None:
client = CalibreOPDSClient("http://example.com/catalog") client = CalibreOPDSClient("http://example.com/catalog")
search_page = OPDSFeed( search_page = OPDSFeed(
id="catalog", id="catalog",
title="Catalog", title="Catalog",
entries=[OPDSEntry(id="1", title="Unrelated", authors=["Alice Smith"])], entries=[OPDSEntry(id="1", title="Unrelated", authors=["Alice Smith"])],
links={"next": OPDSLink(href="http://example.com/catalog?page=2", rel="next")}, links={"next": OPDSLink(href="http://example.com/catalog?page=2", rel="next")},
) )
next_page = OPDSFeed( next_page = OPDSFeed(
id="catalog", id="catalog",
title="Catalog", title="Catalog",
entries=[OPDSEntry(id="2", title="Journey in Space", authors=["Cara Nguyen"])], entries=[OPDSEntry(id="2", title="Journey in Space", authors=["Cara Nguyen"])],
links={}, links={},
) )
def fake_fetch(path=None, params=None): def fake_fetch(path=None, params=None):
if path == "search": if path == "search":
return search_page return search_page
if path == "http://example.com/catalog?page=2": if path == "http://example.com/catalog?page=2":
return next_page return next_page
return search_page return search_page
monkeypatch.setattr(client, "fetch_feed", fake_fetch) monkeypatch.setattr(client, "fetch_feed", fake_fetch)
result = client.search("journey") result = client.search("journey")
assert [entry.id for entry in result.entries] == ["2"] assert [entry.id for entry in result.entries] == ["2"]
def test_calibre_opds_search_collects_next_page_results(monkeypatch) -> None: def test_calibre_opds_search_collects_next_page_results(monkeypatch) -> None:
client = CalibreOPDSClient("http://example.com/catalog") client = CalibreOPDSClient("http://example.com/catalog")
first_page = OPDSFeed( first_page = OPDSFeed(
id="catalog", id="catalog",
title="Catalog", title="Catalog",
entries=[OPDSEntry(id="1", title="Ryan's Adventure")], entries=[OPDSEntry(id="1", title="Ryan's Adventure")],
links={"next": OPDSLink(href="http://example.com/catalog?page=2", rel="next")}, links={"next": OPDSLink(href="http://example.com/catalog?page=2", rel="next")},
) )
second_page = OPDSFeed( second_page = OPDSFeed(
id="catalog", id="catalog",
title="Catalog", title="Catalog",
entries=[OPDSEntry(id="2", title="Return of Ryan")], entries=[OPDSEntry(id="2", title="Return of Ryan")],
links={}, links={},
) )
def fake_fetch(path=None, params=None): def fake_fetch(path=None, params=None):
if path == "search": if path == "search":
return first_page return first_page
if path == "http://example.com/catalog?page=2": if path == "http://example.com/catalog?page=2":
return second_page return second_page
if path is None and params is None: if path is None and params is None:
return first_page return first_page
return first_page return first_page
monkeypatch.setattr(client, "fetch_feed", fake_fetch) monkeypatch.setattr(client, "fetch_feed", fake_fetch)
result = client.search("ryan") result = client.search("ryan")
assert [entry.id for entry in result.entries] == ["1", "2"] assert [entry.id for entry in result.entries] == ["1", "2"]
def test_calibre_opds_search_supplements_with_local_navigation(monkeypatch) -> None: def test_calibre_opds_search_supplements_with_local_navigation(monkeypatch) -> None:
client = CalibreOPDSClient("http://example.com/catalog") client = CalibreOPDSClient("http://example.com/catalog")
search_feed = OPDSFeed( search_feed = OPDSFeed(
id="catalog", id="catalog",
title="Catalog", title="Catalog",
entries=[ entries=[
OPDSEntry(id="book-1", title="Ryan's First Mission"), OPDSEntry(id="book-1", title="Ryan's First Mission"),
OPDSEntry( OPDSEntry(
id="nav-authors", id="nav-authors",
title="Browse Authors", title="Browse Authors",
links=[ links=[
OPDSLink( OPDSLink(
href="http://example.com/catalog/authors", href="http://example.com/catalog/authors",
rel="http://opds-spec.org/navigation", rel="http://opds-spec.org/navigation",
type="application/atom+xml;profile=opds-catalog", type="application/atom+xml;profile=opds-catalog",
) )
],
),
], ],
), links={},
], )
links={}, authors_feed = OPDSFeed(
) id="authors",
authors_feed = OPDSFeed( title="Authors",
id="authors", entries=[OPDSEntry(id="book-2", title="Chronicles of Ryan")],
title="Authors", links={},
entries=[OPDSEntry(id="book-2", title="Chronicles of Ryan")], )
links={},
)
def fake_fetch(path=None, params=None): def fake_fetch(path=None, params=None):
if path == "search": if path == "search":
return search_feed return search_feed
if path == "http://example.com/catalog/authors": if path == "http://example.com/catalog/authors":
return authors_feed return authors_feed
if path is None and params is None: if path is None and params is None:
return search_feed return search_feed
return search_feed return search_feed
monkeypatch.setattr(client, "fetch_feed", fake_fetch) monkeypatch.setattr(client, "fetch_feed", fake_fetch)
result = client.search("ryan") result = client.search("ryan")
assert [entry.id for entry in result.entries] == ["book-1", "book-2"] assert [entry.id for entry in result.entries] == ["book-1", "book-2"]
def test_calibre_opds_browse_letter_traverses_next(monkeypatch) -> None: def test_calibre_opds_browse_letter_traverses_next(monkeypatch) -> None:
client = CalibreOPDSClient("http://example.com/catalog") client = CalibreOPDSClient("http://example.com/catalog")
root_feed = OPDSFeed( root_feed = OPDSFeed(
id="catalog", id="catalog",
title="Browse Authors", title="Browse Authors",
entries=[ entries=[
OPDSEntry( OPDSEntry(
id="nav-a", id="nav-a",
title="A", title="A",
links=[ links=[
OPDSLink( OPDSLink(
href="http://example.com/catalog/authors/a", href="http://example.com/catalog/authors/a",
rel="http://opds-spec.org/navigation", rel="http://opds-spec.org/navigation",
type="application/atom+xml;profile=opds-catalog", type="application/atom+xml;profile=opds-catalog",
) )
],
)
], ],
) links={"next": OPDSLink(href="http://example.com/catalog?page=2", rel="next")},
], )
links={"next": OPDSLink(href="http://example.com/catalog?page=2", rel="next")}, page_two = OPDSFeed(
) id="catalog",
page_two = OPDSFeed( title="Browse Authors",
id="catalog", entries=[
title="Browse Authors", OPDSEntry(
entries=[ id="nav-c",
OPDSEntry( title="C",
id="nav-c", links=[
title="C", OPDSLink(
links=[ href="http://example.com/catalog/authors/c",
OPDSLink( rel="http://opds-spec.org/navigation",
href="http://example.com/catalog/authors/c", type="application/atom+xml;profile=opds-catalog",
rel="http://opds-spec.org/navigation", )
type="application/atom+xml;profile=opds-catalog", ],
) )
], ],
) links={},
], )
links={}, letter_feed = OPDSFeed(
) id="authors-c",
letter_feed = OPDSFeed( title="Authors starting with C",
id="authors-c", entries=[OPDSEntry(id="author-1", title="Clarke, Arthur C.")],
title="Authors starting with C", links={},
entries=[OPDSEntry(id="author-1", title="Clarke, Arthur C.")], )
links={},
)
def fake_fetch(href=None, params=None): def fake_fetch(href=None, params=None):
if not href: if not href:
return root_feed return root_feed
if href == "http://example.com/catalog?page=2": if href == "http://example.com/catalog?page=2":
return page_two return page_two
if href == "http://example.com/catalog/authors/c": if href == "http://example.com/catalog/authors/c":
return letter_feed return letter_feed
return root_feed return root_feed
monkeypatch.setattr(client, "fetch_feed", fake_fetch) monkeypatch.setattr(client, "fetch_feed", fake_fetch)
result = client.browse_letter("C") result = client.browse_letter("C")
assert [entry.id for entry in result.entries] == ["author-1"] assert [entry.id for entry in result.entries] == ["author-1"]
def test_calibre_opds_browse_letter_filters_when_missing_navigation(monkeypatch) -> None: def test_calibre_opds_browse_letter_filters_when_missing_navigation(
client = CalibreOPDSClient("http://example.com/catalog") monkeypatch,
titles_feed = OPDSFeed( ) -> None:
id="catalog", client = CalibreOPDSClient("http://example.com/catalog")
title="Browse Titles", titles_feed = OPDSFeed(
entries=[ id="catalog",
OPDSEntry(id="book-1", title="The Moon is a Harsh Mistress"), title="Browse Titles",
OPDSEntry(id="book-2", title="Another Story"), entries=[
], OPDSEntry(id="book-1", title="The Moon is a Harsh Mistress"),
links={}, OPDSEntry(id="book-2", title="Another Story"),
) ],
links={},
)
def fake_fetch(href=None, params=None): def fake_fetch(href=None, params=None):
return titles_feed return titles_feed
monkeypatch.setattr(client, "fetch_feed", fake_fetch) monkeypatch.setattr(client, "fetch_feed", fake_fetch)
result = client.browse_letter("M") result = client.browse_letter("M")
assert [entry.id for entry in result.entries] == ["book-1"] assert [entry.id for entry in result.entries] == ["book-1"]
def test_calibre_opds_browse_letter_collects_paginated_entries(monkeypatch) -> None: def test_calibre_opds_browse_letter_collects_paginated_entries(monkeypatch) -> None:
client = CalibreOPDSClient("http://example.com/catalog") client = CalibreOPDSClient("http://example.com/catalog")
first_page = OPDSFeed( first_page = OPDSFeed(
id="catalog", id="catalog",
title="Browse Titles", title="Browse Titles",
entries=[ entries=[
OPDSEntry(id="book-1", title="Ryan's First Adventure"), OPDSEntry(id="book-1", title="Ryan's First Adventure"),
OPDSEntry(id="book-2", title="Another Tale"), OPDSEntry(id="book-2", title="Another Tale"),
], ],
links={"next": OPDSLink(href="http://example.com/catalog?page=2", rel="next")}, links={"next": OPDSLink(href="http://example.com/catalog?page=2", rel="next")},
) )
second_page = OPDSFeed( second_page = OPDSFeed(
id="catalog", id="catalog",
title="Browse Titles", title="Browse Titles",
entries=[OPDSEntry(id="book-3", title="Return of Ryan")], entries=[OPDSEntry(id="book-3", title="Return of Ryan")],
links={}, links={},
) )
def fake_fetch(href=None, params=None): def fake_fetch(href=None, params=None):
if not href: if not href:
return first_page return first_page
if href == "http://example.com/catalog?page=2": if href == "http://example.com/catalog?page=2":
return second_page return second_page
return first_page return first_page
monkeypatch.setattr(client, "fetch_feed", fake_fetch) monkeypatch.setattr(client, "fetch_feed", fake_fetch)
result = client.browse_letter("R") result = client.browse_letter("R")
assert [entry.id for entry in result.entries] == ["book-1", "book-3"] assert [entry.id for entry in result.entries] == ["book-1", "book-3"]
def test_calibre_opds_browse_letter_collects_paginated_navigation(monkeypatch) -> None: def test_calibre_opds_browse_letter_collects_paginated_navigation(monkeypatch) -> None:
client = CalibreOPDSClient("http://example.com/catalog") client = CalibreOPDSClient("http://example.com/catalog")
root_feed = OPDSFeed( root_feed = OPDSFeed(
id="catalog", id="catalog",
title="Browse Authors", title="Browse Authors",
entries=[ entries=[
OPDSEntry( OPDSEntry(
id="nav-a", id="nav-a",
title="A", title="A",
links=[ links=[
OPDSLink( OPDSLink(
href="http://example.com/catalog/authors/a", href="http://example.com/catalog/authors/a",
rel="http://opds-spec.org/navigation", rel="http://opds-spec.org/navigation",
type="application/atom+xml;profile=opds-catalog", type="application/atom+xml;profile=opds-catalog",
) )
],
),
OPDSEntry(
id="nav-r",
title="R",
links=[
OPDSLink(
href="http://example.com/catalog/authors/r",
rel="http://opds-spec.org/navigation",
type="application/atom+xml;profile=opds-catalog",
)
],
),
], ],
), links={},
OPDSEntry( )
id="nav-r", letter_feed = OPDSFeed(
title="R", id="authors-r",
links=[ title="Authors — R",
OPDSLink( entries=[
href="http://example.com/catalog/authors/r", OPDSEntry(id="author-1", title="Ryan, Alice"),
rel="http://opds-spec.org/navigation",
type="application/atom+xml;profile=opds-catalog",
)
], ],
), links={
], "next": OPDSLink(
links={}, href="http://example.com/catalog/authors/r?page=2", rel="next"
) )
letter_feed = OPDSFeed( },
id="authors-r", )
title="Authors — R", letter_page_two = OPDSFeed(
entries=[ id="authors-r",
OPDSEntry(id="author-1", title="Ryan, Alice"), title="Authors — R",
], entries=[OPDSEntry(id="author-2", title="Ryan, Bob")],
links={"next": OPDSLink(href="http://example.com/catalog/authors/r?page=2", rel="next")}, links={},
) )
letter_page_two = OPDSFeed(
id="authors-r",
title="Authors — R",
entries=[OPDSEntry(id="author-2", title="Ryan, Bob")],
links={},
)
def fake_fetch(href=None, params=None): def fake_fetch(href=None, params=None):
if not href: if not href:
return root_feed return root_feed
if href == "http://example.com/catalog/authors/r": if href == "http://example.com/catalog/authors/r":
return letter_feed return letter_feed
if href == "http://example.com/catalog/authors/r?page=2": if href == "http://example.com/catalog/authors/r?page=2":
return letter_page_two return letter_page_two
return root_feed return root_feed
monkeypatch.setattr(client, "fetch_feed", fake_fetch) monkeypatch.setattr(client, "fetch_feed", fake_fetch)
result = client.browse_letter("R") result = client.browse_letter("R")
assert [entry.id for entry in result.entries] == ["author-1", "author-2"] assert [entry.id for entry in result.entries] == ["author-1", "author-2"]
+17 -5
View File
@@ -43,7 +43,11 @@ def _install_dependency_stubs() -> None:
setattr(numpy_stub, "float32", "float32") setattr(numpy_stub, "float32", "float32")
setattr(numpy_stub, "array", lambda data, dtype=None: data) setattr(numpy_stub, "array", lambda data, dtype=None: data)
setattr(numpy_stub, "asarray", lambda data, dtype=None: data) setattr(numpy_stub, "asarray", lambda data, dtype=None: data)
setattr(numpy_stub, "concatenate", lambda seq, axis=0: sum((list(item) for item in seq), [])) setattr(
numpy_stub,
"concatenate",
lambda seq, axis=0: sum((list(item) for item in seq), []),
)
sys.modules["numpy"] = numpy_stub sys.modules["numpy"] = numpy_stub
if "soundfile" not in sys.modules: if "soundfile" not in sys.modules:
@@ -117,7 +121,9 @@ def test_apply_chapter_overrides_with_custom_text() -> None:
{"index": 1, "enabled": False}, {"index": 1, "enabled": False},
] ]
selected, metadata, diagnostics = _apply_chapter_overrides(_sample_chapters(), overrides) selected, metadata, diagnostics = _apply_chapter_overrides(
_sample_chapters(), overrides
)
assert len(selected) == 1 assert len(selected) == 1
assert selected[0].title == "Intro" assert selected[0].title == "Intro"
@@ -132,7 +138,9 @@ def test_apply_chapter_overrides_uses_original_content_when_text_missing() -> No
{"index": 1, "enabled": True}, {"index": 1, "enabled": True},
] ]
selected, metadata, diagnostics = _apply_chapter_overrides(_sample_chapters(), overrides) selected, metadata, diagnostics = _apply_chapter_overrides(
_sample_chapters(), overrides
)
assert len(selected) == 1 assert len(selected) == 1
assert selected[0].title == "Chapter 2" assert selected[0].title == "Chapter 2"
@@ -152,7 +160,9 @@ def test_apply_chapter_overrides_collects_metadata_updates() -> None:
} }
] ]
selected, metadata, diagnostics = _apply_chapter_overrides(_sample_chapters(), overrides) selected, metadata, diagnostics = _apply_chapter_overrides(
_sample_chapters(), overrides
)
assert len(selected) == 1 assert len(selected) == 1
assert metadata == {"artist": "Test Author", "year": "2024"} assert metadata == {"artist": "Test Author", "year": "2024"}
@@ -164,7 +174,9 @@ def test_apply_chapter_overrides_reports_diagnostics_for_invalid_payload() -> No
{"enabled": True, "title": "Missing"}, {"enabled": True, "title": "Missing"},
] ]
selected, metadata, diagnostics = _apply_chapter_overrides(_sample_chapters(), overrides) selected, metadata, diagnostics = _apply_chapter_overrides(
_sample_chapters(), overrides
)
assert selected == [] assert selected == []
assert metadata == {} assert metadata == {}
+3 -1
View File
@@ -27,7 +27,9 @@ def test_chunk_voice_spec_prefers_chunk_overrides() -> None:
def test_chunk_voice_spec_falls_back_to_speaker_voice() -> None: def test_chunk_voice_spec_falls_back_to_speaker_voice() -> None:
job = SimpleNamespace(voice="base_voice", speakers={"narrator": {"voice": "speaker_voice"}}) job = SimpleNamespace(
voice="base_voice", speakers={"narrator": {"voice": "speaker_voice"}}
)
chunk = {"speaker_id": "narrator"} chunk = {"speaker_id": "narrator"}
assert _chunk_voice_spec(job, chunk, "fallback") == "speaker_voice" assert _chunk_voice_spec(job, chunk, "fallback") == "speaker_voice"
+10 -3
View File
@@ -74,7 +74,9 @@ def test_format_spoken_chapter_title_adds_prefix() -> None:
def test_format_spoken_chapter_title_respects_existing_prefix() -> None: def test_format_spoken_chapter_title_respects_existing_prefix() -> None:
assert _format_spoken_chapter_title("Chapter 2: Story", 2, True) == "Chapter 2: Story" assert (
_format_spoken_chapter_title("Chapter 2: Story", 2, True) == "Chapter 2: Story"
)
def test_format_spoken_chapter_title_handles_empty_title() -> None: def test_format_spoken_chapter_title_handles_empty_title() -> None:
@@ -82,7 +84,10 @@ def test_format_spoken_chapter_title_handles_empty_title() -> None:
def test_format_spoken_chapter_title_trims_delimiters() -> None: def test_format_spoken_chapter_title_trims_delimiters() -> None:
assert _format_spoken_chapter_title("7 - Into the Wild", 7, True) == "Chapter 7. Into the Wild" assert (
_format_spoken_chapter_title("7 - Into the Wild", 7, True)
== "Chapter 7. Into the Wild"
)
def test_headings_equivalent_ignores_case_and_prefix() -> None: def test_headings_equivalent_ignores_case_and_prefix() -> None:
@@ -90,7 +95,9 @@ def test_headings_equivalent_ignores_case_and_prefix() -> None:
def test_strip_duplicate_heading_line_removes_first_match() -> None: def test_strip_duplicate_heading_line_removes_first_match() -> None:
text, removed = _strip_duplicate_heading_line("Chapter 3: Intro\nBody text", "Chapter 3: Intro") text, removed = _strip_duplicate_heading_line(
"Chapter 3: Intro\nBody text", "Chapter 3: Intro"
)
assert removed is True assert removed is True
assert text.strip() == "Body text" assert text.strip() == "Body text"
+8 -8
View File
@@ -14,14 +14,14 @@ def _sample_job(formula: str) -> Job:
return cast( return cast(
Job, Job,
SimpleNamespace( SimpleNamespace(
voice="__custom_mix", voice="__custom_mix",
speakers={ speakers={
"narrator": { "narrator": {
"resolved_voice": formula, "resolved_voice": formula,
} }
}, },
chapters=[], chapters=[],
chunks=[{}], chunks=[{}],
), ),
) )
+11 -4
View File
@@ -1,13 +1,19 @@
import pytest import pytest
from abogen.kokoro_text_normalization import _normalize_grouped_numbers, ApostropheConfig from abogen.kokoro_text_normalization import (
_normalize_grouped_numbers,
ApostropheConfig,
)
@pytest.fixture @pytest.fixture
def cfg(): def cfg():
return ApostropheConfig(convert_numbers=True, year_pronunciation_mode="american") return ApostropheConfig(convert_numbers=True, year_pronunciation_mode="american")
def normalize(text, config): def normalize(text, config):
return _normalize_grouped_numbers(text, config) return _normalize_grouped_numbers(text, config)
class TestDateNormalization: class TestDateNormalization:
def test_standard_years(self, cfg): def test_standard_years(self, cfg):
@@ -18,7 +24,9 @@ class TestDateNormalization:
# 2023 -> twenty twenty-three # 2023 -> twenty twenty-three
assert "twenty twenty-three" in normalize("It is currently 2023.", cfg) assert "twenty twenty-three" in normalize("It is currently 2023.", cfg)
# 1905 -> nineteen hundred oh five # 1905 -> nineteen hundred oh five
assert "nineteen hundred oh five" in normalize("In 1905, Einstein published.", cfg) assert "nineteen hundred oh five" in normalize(
"In 1905, Einstein published.", cfg
)
def test_future_years(self, cfg): def test_future_years(self, cfg):
# 3400 -> thirty-four hundred # 3400 -> thirty-four hundred
@@ -46,7 +54,7 @@ class TestDateNormalization:
assert "one thousand" in res or "nineteen hundred" in res assert "one thousand" in res or "nineteen hundred" in res
res = normalize("Please send it to the address: 3400 North Blvd.", cfg) res = normalize("Please send it to the address: 3400 North Blvd.", cfg)
assert "thirty-four hundred" not in res # Should not be year style assert "thirty-four hundred" not in res # Should not be year style
assert "three thousand" in res or "thirty-four hundred" in res assert "three thousand" in res or "thirty-four hundred" in res
# Wait, "thirty-four hundred" IS how you say 3400 in num2words sometimes? # Wait, "thirty-four hundred" IS how you say 3400 in num2words sometimes?
# num2words(3400) -> "three thousand, four hundred" usually. # num2words(3400) -> "three thousand, four hundred" usually.
@@ -113,4 +121,3 @@ class TestDateNormalization:
res = normalize("The addresses are 1925 and 1926.", cfg) res = normalize("The addresses are 1925 and 1926.", cfg)
# Expectation: should probably be numbers, not years. # Expectation: should probably be numbers, not years.
assert "nineteen twenty-five" not in res assert "nineteen twenty-five" not in res
+30 -9
View File
@@ -4,7 +4,12 @@ from pathlib import Path
import numpy as np import numpy as np
import pytest import pytest
from abogen.debug_tts_samples import DEBUG_TTS_SAMPLES, MARKER_PREFIX, MARKER_SUFFIX, iter_expected_codes from abogen.debug_tts_samples import (
DEBUG_TTS_SAMPLES,
MARKER_PREFIX,
MARKER_SUFFIX,
iter_expected_codes,
)
from abogen.kokoro_text_normalization import HAS_NUM2WORDS, normalize_for_pipeline from abogen.kokoro_text_normalization import HAS_NUM2WORDS, normalize_for_pipeline
from abogen.normalization_settings import build_apostrophe_config from abogen.normalization_settings import build_apostrophe_config
from abogen.text_extractor import extract_from_path from abogen.text_extractor import extract_from_path
@@ -16,7 +21,9 @@ def test_debug_epub_contains_all_codes():
assert epub_path.exists() assert epub_path.exists()
extraction = extract_from_path(epub_path) extraction = extract_from_path(epub_path)
combined = extraction.combined_text or "\n\n".join((c.text or "") for c in extraction.chapters) combined = extraction.combined_text or "\n\n".join(
(c.text or "") for c in extraction.chapters
)
for code in iter_expected_codes(): for code in iter_expected_codes():
marker = f"{MARKER_PREFIX}{code}{MARKER_SUFFIX}" marker = f"{MARKER_PREFIX}{code}{MARKER_SUFFIX}"
@@ -32,7 +39,9 @@ def test_debug_samples_normalize_smoke():
runtime = dict(settings) runtime = dict(settings)
normalized = { normalized = {
sample.code: normalize_for_pipeline(sample.text, config=apostrophe, settings=runtime) sample.code: normalize_for_pipeline(
sample.text, config=apostrophe, settings=runtime
)
for sample in DEBUG_TTS_SAMPLES for sample in DEBUG_TTS_SAMPLES
} }
@@ -69,7 +78,9 @@ def test_settings_debug_route_writes_manifest(tmp_path, monkeypatch):
audio[::100] = 0.1 audio[::100] = 0.1
yield _Seg(audio) yield _Seg(audio)
monkeypatch.setattr(runner, "_load_pipeline", lambda language, use_gpu: DummyPipeline()) monkeypatch.setattr(
runner, "_load_pipeline", lambda language, use_gpu: DummyPipeline()
)
app = create_app( app = create_app(
{ {
@@ -87,14 +98,18 @@ def test_settings_debug_route_writes_manifest(tmp_path, monkeypatch):
assert "/settings/debug/" in location assert "/settings/debug/" in location
# Extract run id from /settings/debug/<run_id> # Extract run id from /settings/debug/<run_id>
run_id = location.rsplit("/settings/debug/", 1)[1].split("?", 1)[0].split("#", 1)[0] run_id = (
location.rsplit("/settings/debug/", 1)[1].split("?", 1)[0].split("#", 1)[0]
)
manifest_path = tmp_path / "debug" / run_id / "manifest.json" manifest_path = tmp_path / "debug" / run_id / "manifest.json"
assert manifest_path.exists() assert manifest_path.exists()
manifest = json.loads(manifest_path.read_text(encoding="utf-8")) manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
filenames = {item["filename"] for item in manifest.get("artifacts", [])} filenames = {item["filename"] for item in manifest.get("artifacts", [])}
assert "overall.wav" in filenames assert "overall.wav" in filenames
assert any(name.startswith("case_") and name.endswith(".wav") for name in filenames) assert any(
name.startswith("case_") and name.endswith(".wav") for name in filenames
)
def test_debug_samples_have_minimum_per_category(): def test_debug_samples_have_minimum_per_category():
@@ -125,7 +140,9 @@ def test_debug_runner_resolves_profile_voice_before_pipeline(tmp_path, monkeypat
from abogen.webui import debug_tts_runner as runner from abogen.webui import debug_tts_runner as runner
# Stub voice setting resolution so we don't depend on the user's profile file. # Stub voice setting resolution so we don't depend on the user's profile file.
monkeypatch.setattr(runner, "_resolve_voice_setting", lambda value: ("af_heart", "AM HQ Alt", None)) monkeypatch.setattr(
runner, "_resolve_voice_setting", lambda value: ("af_heart", "AM HQ Alt", None)
)
calls = [] calls = []
@@ -139,7 +156,9 @@ def test_debug_runner_resolves_profile_voice_before_pipeline(tmp_path, monkeypat
audio = np.zeros(int(0.05 * runner.SAMPLE_RATE), dtype="float32") audio = np.zeros(int(0.05 * runner.SAMPLE_RATE), dtype="float32")
yield _Seg(audio) yield _Seg(audio)
monkeypatch.setattr(runner, "_load_pipeline", lambda language, use_gpu: DummyPipeline()) monkeypatch.setattr(
runner, "_load_pipeline", lambda language, use_gpu: DummyPipeline()
)
settings = { settings = {
"language": "en", "language": "en",
@@ -152,4 +171,6 @@ def test_debug_runner_resolves_profile_voice_before_pipeline(tmp_path, monkeypat
assert manifest.get("run_id") assert manifest.get("run_id")
assert calls assert calls
# Must not pass through the profile:* string. # Must not pass through the profile:* string.
assert all(isinstance(v, str) and not v.lower().startswith("profile:") for v in calls) assert all(
isinstance(v, str) and not v.lower().startswith("profile:") for v in calls
)
+15 -11
View File
@@ -5,10 +5,11 @@ import sys
from ebooklib import epub from ebooklib import epub
# Ensure import path # Ensure import path
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from abogen.book_parser import get_book_parser from abogen.book_parser import get_book_parser
class TestEpubContentSlicing(unittest.TestCase): class TestEpubContentSlicing(unittest.TestCase):
""" """
Tests for the complex content slicing logic in _execute_nav_parsing_logic. Tests for the complex content slicing logic in _execute_nav_parsing_logic.
@@ -71,18 +72,19 @@ class TestEpubContentSlicing(unittest.TestCase):
# OPF Patching to valid crash # OPF Patching to valid crash
import zipfile import zipfile
patched = False patched = False
with zipfile.ZipFile(self.epub_path, 'r') as zin: with zipfile.ZipFile(self.epub_path, "r") as zin:
opf_content = zin.read('EPUB/content.opf').decode('utf-8') opf_content = zin.read("EPUB/content.opf").decode("utf-8")
if 'toc="ncx"' in opf_content: if 'toc="ncx"' in opf_content:
opf_content = opf_content.replace('toc="ncx"', '') opf_content = opf_content.replace('toc="ncx"', "")
patched = True patched = True
if patched: if patched:
TEMP_EPUB = self.epub_path + ".temp" TEMP_EPUB = self.epub_path + ".temp"
with zipfile.ZipFile(TEMP_EPUB, 'w') as zout: with zipfile.ZipFile(TEMP_EPUB, "w") as zout:
for item in zin.infolist(): for item in zin.infolist():
if item.filename == 'EPUB/content.opf': if item.filename == "EPUB/content.opf":
zout.writestr(item, opf_content) zout.writestr(item, opf_content)
else: else:
zout.writestr(item, zin.read(item.filename)) zout.writestr(item, zin.read(item.filename))
@@ -159,17 +161,18 @@ class TestEpubContentSlicing(unittest.TestCase):
# Patch # Patch
import zipfile import zipfile
patched = False patched = False
with zipfile.ZipFile(self.epub_path, 'r') as zin: with zipfile.ZipFile(self.epub_path, "r") as zin:
opf_content = zin.read('EPUB/content.opf').decode('utf-8') opf_content = zin.read("EPUB/content.opf").decode("utf-8")
if 'toc="ncx"' in opf_content: if 'toc="ncx"' in opf_content:
opf_content = opf_content.replace('toc="ncx"', '') opf_content = opf_content.replace('toc="ncx"', "")
patched = True patched = True
if patched: if patched:
TEMP_EPUB = self.epub_path + ".temp" TEMP_EPUB = self.epub_path + ".temp"
with zipfile.ZipFile(TEMP_EPUB, 'w') as zout: with zipfile.ZipFile(TEMP_EPUB, "w") as zout:
for item in zin.infolist(): for item in zin.infolist():
if item.filename == 'EPUB/content.opf': if item.filename == "EPUB/content.opf":
zout.writestr(item, opf_content) zout.writestr(item, opf_content)
else: else:
zout.writestr(item, zin.read(item.filename)) zout.writestr(item, zin.read(item.filename))
@@ -195,5 +198,6 @@ class TestEpubContentSlicing(unittest.TestCase):
self.assertIn("3) Item C", text2) self.assertIn("3) Item C", text2)
self.assertIn("4) Item D", text2) self.assertIn("4) Item D", text2)
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
+35 -9
View File
@@ -37,8 +37,20 @@ def test_build_epub3_package_creates_expected_structure(tmp_path) -> None:
}, },
] ]
chunk_markers = [ chunk_markers = [
{"id": "chap0000_p0000", "chapter_index": 0, "chunk_index": 0, "start": 0.0, "end": 1.2}, {
{"id": "chap0001_p0000", "chapter_index": 1, "chunk_index": 0, "start": 1.2, "end": 2.4}, "id": "chap0000_p0000",
"chapter_index": 0,
"chunk_index": 0,
"start": 0.0,
"end": 1.2,
},
{
"id": "chap0001_p0000",
"chapter_index": 1,
"chunk_index": 0,
"start": 1.2,
"end": 2.4,
},
] ]
chapter_markers = [ chapter_markers = [
{"index": 1, "title": "Chapter 1", "start": 0.0, "end": 1.2}, {"index": 1, "title": "Chapter 1", "start": 0.0, "end": 1.2},
@@ -76,7 +88,7 @@ def test_build_epub3_package_creates_expected_structure(tmp_path) -> None:
chapter_doc = archive.read("OEBPS/text/chapter_0001.xhtml").decode("utf-8") chapter_doc = archive.read("OEBPS/text/chapter_0001.xhtml").decode("utf-8")
assert "Hello world." in chapter_doc assert "Hello world." in chapter_doc
smil_doc = archive.read("OEBPS/smil/chapter_0001.smil").decode("utf-8") smil_doc = archive.read("OEBPS/smil/chapter_0001.smil").decode("utf-8")
assert "clipBegin=\"00:00:00.000\"" in smil_doc assert 'clipBegin="00:00:00.000"' in smil_doc
opf_doc = archive.read("OEBPS/content.opf").decode("utf-8") opf_doc = archive.read("OEBPS/content.opf").decode("utf-8")
assert "media-overlay" in opf_doc assert "media-overlay" in opf_doc
assert "media:duration" in opf_doc assert "media:duration" in opf_doc
@@ -145,7 +157,13 @@ def test_epub3_preserves_original_whitespace(tmp_path) -> None:
] ]
chunk_markers = [ chunk_markers = [
{"id": chunk["id"], "chapter_index": 0, "chunk_index": chunk["chunk_index"], "start": None, "end": None} {
"id": chunk["id"],
"chapter_index": 0,
"chunk_index": chunk["chunk_index"],
"start": None,
"end": None,
}
for chunk in chunks for chunk in chunks
] ]
@@ -174,7 +192,9 @@ def test_epub3_preserves_original_whitespace(tmp_path) -> None:
assert "Second line" in chunk_section assert "Second line" in chunk_section
assert "Third paragraph." in chunk_section assert "Third paragraph." in chunk_section
match = re.search(r"<pre class=\"chapter-original\"[^>]*>(.*?)</pre>", chapter_doc, re.DOTALL) match = re.search(
r"<pre class=\"chapter-original\"[^>]*>(.*?)</pre>", chapter_doc, re.DOTALL
)
assert match is not None assert match is not None
original_text = html.unescape(match.group(1)) original_text = html.unescape(match.group(1))
assert "Second line\n\nThird paragraph." in original_text assert "Second line\n\nThird paragraph." in original_text
@@ -219,7 +239,13 @@ def test_epub3_sentence_chunks_render_as_paragraphs(tmp_path) -> None:
] ]
chunk_markers = [ chunk_markers = [
{"id": chunk["id"], "chapter_index": 0, "chunk_index": chunk["chunk_index"], "start": None, "end": None} {
"id": chunk["id"],
"chapter_index": 0,
"chunk_index": chunk["chunk_index"],
"start": None,
"end": None,
}
for chunk in chunks for chunk in chunks
] ]
@@ -244,11 +270,11 @@ def test_epub3_sentence_chunks_render_as_paragraphs(tmp_path) -> None:
assert '<div class="chunk"' not in chapter_doc assert '<div class="chunk"' not in chapter_doc
assert chapter_doc.count('<p class="chunk-group"') == 2 assert chapter_doc.count('<p class="chunk-group"') == 2
assert 'First sentence.' in chapter_doc assert "First sentence." in chapter_doc
assert 'Second sentence in same paragraph.' in chapter_doc assert "Second sentence in same paragraph." in chapter_doc
first_paragraph_start = chapter_doc.find('<p class="chunk-group"') first_paragraph_start = chapter_doc.find('<p class="chunk-group"')
first_paragraph_end = chapter_doc.find('</p>', first_paragraph_start) first_paragraph_end = chapter_doc.find("</p>", first_paragraph_start)
first_paragraph = chapter_doc[first_paragraph_start:first_paragraph_end] first_paragraph = chapter_doc[first_paragraph_start:first_paragraph_end]
assert "First sentence." in first_paragraph assert "First sentence." in first_paragraph
assert "Second sentence in same paragraph." in first_paragraph assert "Second sentence in same paragraph." in first_paragraph
+14 -8
View File
@@ -5,10 +5,11 @@ import sys
from ebooklib import epub from ebooklib import epub
# Ensure import path # Ensure import path
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from abogen.book_parser import get_book_parser from abogen.book_parser import get_book_parser
class TestEpubHeuristicNav(unittest.TestCase): class TestEpubHeuristicNav(unittest.TestCase):
""" """
Tests for the heuristic fallback in _identify_nav_item (Step 4), Tests for the heuristic fallback in _identify_nav_item (Step 4),
@@ -66,26 +67,27 @@ class TestEpubHeuristicNav(unittest.TestCase):
# 5. Patch OPF to ensure ebooklib didn't sneakily add ITEM_NAVIGATION or toc="ncx" # 5. Patch OPF to ensure ebooklib didn't sneakily add ITEM_NAVIGATION or toc="ncx"
import zipfile import zipfile
patched = False patched = False
with zipfile.ZipFile(self.epub_path, 'r') as zin: with zipfile.ZipFile(self.epub_path, "r") as zin:
opf_content = zin.read('EPUB/content.opf').decode('utf-8') opf_content = zin.read("EPUB/content.opf").decode("utf-8")
# Remove toc="ncx" attribute if present (causes crash if no NCX) # Remove toc="ncx" attribute if present (causes crash if no NCX)
if 'toc="ncx"' in opf_content: if 'toc="ncx"' in opf_content:
opf_content = opf_content.replace('toc="ncx"', '') opf_content = opf_content.replace('toc="ncx"', "")
patched = True patched = True
# Ideally we'd verify properties="nav" isn't there, but EpubHtml shouldn't add it. # Ideally we'd verify properties="nav" isn't there, but EpubHtml shouldn't add it.
# If ebooklib added it, we might need to strip it to force heuristic. # If ebooklib added it, we might need to strip it to force heuristic.
if 'properties="nav"' in opf_content: if 'properties="nav"' in opf_content:
opf_content = opf_content.replace('properties="nav"', '') opf_content = opf_content.replace('properties="nav"', "")
patched = True patched = True
if patched: if patched:
TEMP_EPUB = self.epub_path + ".temp" TEMP_EPUB = self.epub_path + ".temp"
with zipfile.ZipFile(TEMP_EPUB, 'w') as zout: with zipfile.ZipFile(TEMP_EPUB, "w") as zout:
for item in zin.infolist(): for item in zin.infolist():
if item.filename == 'EPUB/content.opf': if item.filename == "EPUB/content.opf":
zout.writestr(item, opf_content) zout.writestr(item, opf_content)
else: else:
zout.writestr(item, zin.read(item.filename)) zout.writestr(item, zin.read(item.filename))
@@ -96,9 +98,12 @@ class TestEpubHeuristicNav(unittest.TestCase):
# 6. Verify our setup: Ensure NO ITEM_NAVIGATION exists # 6. Verify our setup: Ensure NO ITEM_NAVIGATION exists
# We can inspect using ebooklib again # We can inspect using ebooklib again
import ebooklib import ebooklib
check_book = epub.read_epub(self.epub_path) check_book = epub.read_epub(self.epub_path)
nav_items = list(check_book.get_items_of_type(ebooklib.ITEM_NAVIGATION)) nav_items = list(check_book.get_items_of_type(ebooklib.ITEM_NAVIGATION))
self.assertEqual(len(nav_items), 0, "Setup failed: explicit navigation item found!") self.assertEqual(
len(nav_items), 0, "Setup failed: explicit navigation item found!"
)
# 7. Run Parser # 7. Run Parser
parser = get_book_parser(self.epub_path) parser = get_book_parser(self.epub_path)
@@ -113,5 +118,6 @@ class TestEpubHeuristicNav(unittest.TestCase):
# Also verify we hit the "html" type in identification # Also verify we hit the "html" type in identification
# We can't easily check private variables, but success implies it worked. # We can't easily check private variables, but success implies it worked.
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
+15 -12
View File
@@ -5,10 +5,11 @@ import sys
from ebooklib import epub from ebooklib import epub
# Ensure import path # Ensure import path
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from abogen.book_parser import get_book_parser from abogen.book_parser import get_book_parser
class TestEpubHtmlNavParsing(unittest.TestCase): class TestEpubHtmlNavParsing(unittest.TestCase):
""" """
Tests for EPUB 3 HTML5 Navigation Document parsing logic (_parse_html_nav_li). Tests for EPUB 3 HTML5 Navigation Document parsing logic (_parse_html_nav_li).
@@ -57,15 +58,15 @@ class TestEpubHtmlNavParsing(unittest.TestCase):
# because we intentionally excluded the legacy NCX file. # because we intentionally excluded the legacy NCX file.
import zipfile import zipfile
with zipfile.ZipFile(self.epub_path, 'r') as zin: with zipfile.ZipFile(self.epub_path, "r") as zin:
opf_content = zin.read('EPUB/content.opf').decode('utf-8') opf_content = zin.read("EPUB/content.opf").decode("utf-8")
opf_content = opf_content.replace('toc="ncx"', '') opf_content = opf_content.replace('toc="ncx"', "")
# Repack # Repack
TEMP_EPUB = self.epub_path + ".temp" TEMP_EPUB = self.epub_path + ".temp"
with zipfile.ZipFile(TEMP_EPUB, 'w') as zout: with zipfile.ZipFile(TEMP_EPUB, "w") as zout:
for item in zin.infolist(): for item in zin.infolist():
if item.filename == 'EPUB/content.opf': if item.filename == "EPUB/content.opf":
zout.writestr(item, opf_content) zout.writestr(item, opf_content)
else: else:
zout.writestr(item, zin.read(item.filename)) zout.writestr(item, zin.read(item.filename))
@@ -155,13 +156,14 @@ class TestEpubHtmlNavParsing(unittest.TestCase):
# Check internal structure # Check internal structure
# Find the node named "Part I" in the processed structure # Find the node named "Part I" in the processed structure
root_node = next(node for node in parser.processed_nav_structure if node['title'] == "Part I") root_node = next(
node for node in parser.processed_nav_structure if node["title"] == "Part I"
self.assertEqual(root_node['title'], "Part I") )
self.assertFalse(root_node['has_content'])
self.assertEqual(len(root_node['children']), 1)
self.assertEqual(root_node['children'][0]['title'], "Chapter 1")
self.assertEqual(root_node["title"], "Part I")
self.assertFalse(root_node["has_content"])
self.assertEqual(len(root_node["children"]), 1)
self.assertEqual(root_node["children"][0]["title"], "Chapter 1")
def test_identify_nav_item(self): def test_identify_nav_item(self):
"""Test the _identify_nav_item method specifically.""" """Test the _identify_nav_item method specifically."""
@@ -180,5 +182,6 @@ class TestEpubHtmlNavParsing(unittest.TestCase):
self.assertIsNotNone(nav_item) self.assertIsNotNone(nav_item)
self.assertTrue("nav.xhtml" in nav_item.get_name()) self.assertTrue("nav.xhtml" in nav_item.get_name())
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
@@ -7,10 +7,11 @@ import logging
from ebooklib import epub from ebooklib import epub
# Ensure import path # Ensure import path
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from abogen.book_parser import get_book_parser from abogen.book_parser import get_book_parser
class TestEpubMissingFileErrorHandling(unittest.TestCase): class TestEpubMissingFileErrorHandling(unittest.TestCase):
""" """
Tests for robust error handling and recovery in the book parser. Tests for robust error handling and recovery in the book parser.
@@ -58,8 +59,8 @@ class TestEpubMissingFileErrorHandling(unittest.TestCase):
epub.write_epub(temp_path, book) epub.write_epub(temp_path, book)
# 3. Physically remove 'ghost.xhtml' from the ZIP # 3. Physically remove 'ghost.xhtml' from the ZIP
with zipfile.ZipFile(temp_path, 'r') as zin: with zipfile.ZipFile(temp_path, "r") as zin:
with zipfile.ZipFile(self.broken_epub_path, 'w') as zout: with zipfile.ZipFile(self.broken_epub_path, "w") as zout:
for item in zin.infolist(): for item in zin.infolist():
# Copy everything EXCEPT the ghost file # Copy everything EXCEPT the ghost file
# Note: ebooklib might put files in OEPS/ or EPUB/ folders depending on version, # Note: ebooklib might put files in OEPS/ or EPUB/ folders depending on version,
@@ -96,5 +97,6 @@ class TestEpubMissingFileErrorHandling(unittest.TestCase):
except Exception as e: except Exception as e:
self.fail(f"Parser raised unexpected exception: {e}") self.fail(f"Parser raised unexpected exception: {e}")
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
+5 -5
View File
@@ -5,10 +5,11 @@ import sys
from ebooklib import epub from ebooklib import epub
# Ensure we can import the module # Ensure we can import the module
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from abogen.book_parser import get_book_parser, EpubParser from abogen.book_parser import get_book_parser, EpubParser
class TestEpubNcxParsing(unittest.TestCase): class TestEpubNcxParsing(unittest.TestCase):
""" """
Focused tests for NCX navigation scenarios, ensuring legacy/compatibility Focused tests for NCX navigation scenarios, ensuring legacy/compatibility
@@ -63,7 +64,7 @@ class TestEpubNcxParsing(unittest.TestCase):
# 1. Setup Data # 1. Setup Data
chapters_data = [ chapters_data = [
("Chapter 1", "This is the first chapter."), ("Chapter 1", "This is the first chapter."),
("Chapter 2", "This is the second chapter.") ("Chapter 2", "This is the second chapter."),
] ]
self._create_ncx_only_epub(chapters_data) self._create_ncx_only_epub(chapters_data)
@@ -116,9 +117,7 @@ class TestEpubNcxParsing(unittest.TestCase):
link_sect = epub.Link("main.xhtml#sect1", "Section 1", "sect1") link_sect = epub.Link("main.xhtml#sect1", "Section 1", "sect1")
# Structure: Intro -> Section 1 (as child) # Structure: Intro -> Section 1 (as child)
book.toc = ( book.toc = ((link_root, (link_sect,)),)
(link_root, (link_sect, )),
)
book.add_item(epub.EpubNcx()) book.add_item(epub.EpubNcx())
book.spine = ["nav", c1] book.spine = ["nav", c1]
@@ -136,5 +135,6 @@ class TestEpubNcxParsing(unittest.TestCase):
self.assertIn("Introduction", titles) self.assertIn("Introduction", titles)
self.assertIn("Section 1", titles) self.assertIn("Section 1", titles)
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
+11 -8
View File
@@ -7,10 +7,11 @@ import ebooklib
from unittest.mock import MagicMock from unittest.mock import MagicMock
# Ensure import path # Ensure import path
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from abogen.book_parser import get_book_parser from abogen.book_parser import get_book_parser
class TestEpubStandardNav(unittest.TestCase): class TestEpubStandardNav(unittest.TestCase):
""" """
Tests for the standard ITEM_NAVIGATION discovery in _identify_nav_item. Tests for the standard ITEM_NAVIGATION discovery in _identify_nav_item.
@@ -51,16 +52,17 @@ class TestEpubStandardNav(unittest.TestCase):
# We manually remove this attribute to ensure the test EPUB is valid and readable. # We manually remove this attribute to ensure the test EPUB is valid and readable.
# TODO - find real world examples of EPUB 3 files that use HTML Nav # TODO - find real world examples of EPUB 3 files that use HTML Nav
import zipfile import zipfile
patched = False patched = False
with zipfile.ZipFile(self.epub_path, 'r') as zin: with zipfile.ZipFile(self.epub_path, "r") as zin:
opf_content = zin.read('EPUB/content.opf').decode('utf-8') opf_content = zin.read("EPUB/content.opf").decode("utf-8")
if 'toc="ncx"' in opf_content: if 'toc="ncx"' in opf_content:
opf_content = opf_content.replace('toc="ncx"', '') opf_content = opf_content.replace('toc="ncx"', "")
patched = True patched = True
TEMP_EPUB = self.epub_path + ".temp" TEMP_EPUB = self.epub_path + ".temp"
with zipfile.ZipFile(TEMP_EPUB, 'w') as zout: with zipfile.ZipFile(TEMP_EPUB, "w") as zout:
for item in zin.infolist(): for item in zin.infolist():
if item.filename == 'EPUB/content.opf': if item.filename == "EPUB/content.opf":
zout.writestr(item, opf_content) zout.writestr(item, opf_content)
else: else:
zout.writestr(item, zin.read(item.filename)) zout.writestr(item, zin.read(item.filename))
@@ -110,7 +112,7 @@ class TestEpubStandardNav(unittest.TestCase):
# We use a real EpubNav object to ensure structural correctness. # We use a real EpubNav object to ensure structural correctness.
proper_nav = epub.EpubNav(uid=original_nav.id, file_name=original_nav.file_name) proper_nav = epub.EpubNav(uid=original_nav.id, file_name=original_nav.file_name)
proper_nav.content = original_nav.content proper_nav.content = original_nav.content
proper_nav.properties = ['nav'] proper_nav.properties = ["nav"]
# Swap it into the book items list # Swap it into the book items list
try: try:
@@ -124,7 +126,8 @@ class TestEpubStandardNav(unittest.TestCase):
self.assertEqual(nav_type, "html") self.assertEqual(nav_type, "html")
self.assertEqual(nav_item.get_name(), "nav.xhtml") self.assertEqual(nav_item.get_name(), "nav.xhtml")
# Check that we actually found the one with properties # Check that we actually found the one with properties
self.assertEqual(getattr(nav_item, 'properties', []), ['nav']) self.assertEqual(getattr(nav_item, "properties", []), ["nav"])
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
+15 -5
View File
@@ -30,16 +30,22 @@ def _sample_job(tmp_path: Path) -> Job:
) )
def test_prepare_project_layout_uses_timestamped_folder(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: def test_prepare_project_layout_uses_timestamped_folder(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
job = _sample_job(tmp_path) job = _sample_job(tmp_path)
monkeypatch.setattr( monkeypatch.setattr(
"abogen.webui.conversion_runner._output_timestamp_token", "abogen.webui.conversion_runner._output_timestamp_token",
lambda: "20250101-120000", lambda: "20250101-120000",
) )
project_root, audio_dir, subtitle_dir, metadata_dir = _prepare_project_layout(job, tmp_path) project_root, audio_dir, subtitle_dir, metadata_dir = _prepare_project_layout(
job, tmp_path
)
assert project_root.name.startswith("20250101-120000_Sample_Title"), project_root.name assert project_root.name.startswith(
"20250101-120000_Sample_Title"
), project_root.name
assert audio_dir == project_root assert audio_dir == project_root
assert subtitle_dir == project_root assert subtitle_dir == project_root
assert metadata_dir is None assert metadata_dir is None
@@ -48,7 +54,9 @@ def test_prepare_project_layout_uses_timestamped_folder(monkeypatch: pytest.Monk
assert output_path == project_root / "Sample_Title.mp3" assert output_path == project_root / "Sample_Title.mp3"
def test_prepare_project_layout_creates_project_subdirs(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: def test_prepare_project_layout_creates_project_subdirs(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
job = _sample_job(tmp_path) job = _sample_job(tmp_path)
job.save_as_project = True job.save_as_project = True
monkeypatch.setattr( monkeypatch.setattr(
@@ -56,7 +64,9 @@ def test_prepare_project_layout_creates_project_subdirs(monkeypatch: pytest.Monk
lambda: "20250101-120500", lambda: "20250101-120500",
) )
project_root, audio_dir, subtitle_dir, metadata_dir = _prepare_project_layout(job, tmp_path) project_root, audio_dir, subtitle_dir, metadata_dir = _prepare_project_layout(
job, tmp_path
)
assert audio_dir == project_root / "audio" assert audio_dir == project_root / "audio"
assert subtitle_dir == project_root / "subtitles" assert subtitle_dir == project_root / "subtitles"
+8 -4
View File
@@ -102,7 +102,9 @@ def test_resolve_voice_setting_handles_profile_reference():
} }
} }
voice, profile_name, language = resolve_voice_setting("profile:Blend", profiles=profiles) voice, profile_name, language = resolve_voice_setting(
"profile:Blend", profiles=profiles
)
assert voice == "af_nova*0.5+am_liam*0.5" assert voice == "af_nova*0.5+am_liam*0.5"
assert profile_name == "Blend" assert profile_name == "Blend"
@@ -112,9 +114,11 @@ def test_resolve_voice_setting_handles_profile_reference():
def test_apply_prepare_form_updates_closing_outro_flag(): def test_apply_prepare_form_updates_closing_outro_flag():
pending = _make_pending_job() pending = _make_pending_job()
pending.read_closing_outro = True pending.read_closing_outro = True
form = MultiDict({ form = MultiDict(
"read_closing_outro": "false", {
}) "read_closing_outro": "false",
}
)
apply_prepare_form(pending, form) apply_prepare_form(pending, form)
@@ -13,7 +13,9 @@ def test_preview_applies_manual_override_before_normalization(monkeypatch):
def normalize_for_pipeline(text): def normalize_for_pipeline(text):
return text return text
monkeypatch.setitem(__import__("sys").modules, "abogen.kokoro_text_normalization", _Norm) monkeypatch.setitem(
__import__("sys").modules, "abogen.kokoro_text_normalization", _Norm
)
# And stub the kokoro pipeline path so generate_preview_audio won't proceed. # And stub the kokoro pipeline path so generate_preview_audio won't proceed.
# We'll instead validate by calling the override logic through generate_preview_audio # We'll instead validate by calling the override logic through generate_preview_audio
@@ -28,7 +30,11 @@ def test_preview_applies_manual_override_before_normalization(monkeypatch):
captured["text"] = text captured["text"] = text
return iter(()) return iter(())
monkeypatch.setitem(__import__("sys").modules, "abogen.tts_supertonic", type("M", (), {"SupertonicPipeline": DummyPipeline})) monkeypatch.setitem(
__import__("sys").modules,
"abogen.tts_supertonic",
type("M", (), {"SupertonicPipeline": DummyPipeline}),
)
try: try:
preview.generate_preview_audio( preview.generate_preview_audio(
+12 -1
View File
@@ -1,8 +1,12 @@
import pytest import pytest
from unittest.mock import patch from unittest.mock import patch
from abogen.kokoro_text_normalization import normalize_for_pipeline, DEFAULT_APOSTROPHE_CONFIG from abogen.kokoro_text_normalization import (
normalize_for_pipeline,
DEFAULT_APOSTROPHE_CONFIG,
)
from abogen.normalization_settings import build_apostrophe_config, _SETTINGS_DEFAULTS from abogen.normalization_settings import build_apostrophe_config, _SETTINGS_DEFAULTS
def normalize(text, overrides=None): def normalize(text, overrides=None):
settings = dict(_SETTINGS_DEFAULTS) settings = dict(_SETTINGS_DEFAULTS)
if overrides: if overrides:
@@ -11,6 +15,7 @@ def normalize(text, overrides=None):
config = build_apostrophe_config(settings=settings, base=DEFAULT_APOSTROPHE_CONFIG) config = build_apostrophe_config(settings=settings, base=DEFAULT_APOSTROPHE_CONFIG)
return normalize_for_pipeline(text, config=config, settings=settings) return normalize_for_pipeline(text, config=config, settings=settings)
def test_year_pronunciation(): def test_year_pronunciation():
# 1925 -> Nineteen Hundred Twenty Five # 1925 -> Nineteen Hundred Twenty Five
normalized = normalize("1925") normalized = normalize("1925")
@@ -24,6 +29,7 @@ def test_year_pronunciation():
assert "twenty twenty" in normalized.lower() assert "twenty twenty" in normalized.lower()
assert "five" in normalized.lower() assert "five" in normalized.lower()
def test_currency_pronunciation(): def test_currency_pronunciation():
# $1.00 -> One dollar (no zero cents) # $1.00 -> One dollar (no zero cents)
normalized = normalize("$1.00") normalized = normalize("$1.00")
@@ -37,6 +43,7 @@ def test_currency_pronunciation():
assert "one dollar" in normalized.lower() assert "one dollar" in normalized.lower()
assert "five cents" in normalized.lower() assert "five cents" in normalized.lower()
def test_url_pronunciation(): def test_url_pronunciation():
# https://www.amazon.com -> amazon dot com # https://www.amazon.com -> amazon dot com
normalized = normalize("https://www.amazon.com") normalized = normalize("https://www.amazon.com")
@@ -50,6 +57,7 @@ def test_url_pronunciation():
print(f"www.google.com -> {normalized}") print(f"www.google.com -> {normalized}")
assert "google dot com" in normalized.lower() assert "google dot com" in normalized.lower()
def test_roman_numerals_world_war(): def test_roman_numerals_world_war():
# World War I -> World War One # World War I -> World War One
normalized = normalize("World War I") normalized = normalize("World War I")
@@ -61,6 +69,7 @@ def test_roman_numerals_world_war():
print(f"World War II -> {normalized}") print(f"World War II -> {normalized}")
assert "world war two" in normalized.lower() assert "world war two" in normalized.lower()
def test_footnote_removal(): def test_footnote_removal():
# Bob is awesome1. -> Bob is awesome. # Bob is awesome1. -> Bob is awesome.
normalized = normalize("Bob is awesome1.") normalized = normalize("Bob is awesome1.")
@@ -74,8 +83,10 @@ def test_footnote_removal():
assert "citation needed." in normalized.lower() assert "citation needed." in normalized.lower()
assert "[1]" not in normalized assert "[1]" not in normalized
def test_manual_override_normalization(): def test_manual_override_normalization():
from abogen.entity_analysis import normalize_manual_override_token from abogen.entity_analysis import normalize_manual_override_token
assert normalize_manual_override_token("The") == "the" assert normalize_manual_override_token("The") == "the"
assert normalize_manual_override_token(" A ") == "a" assert normalize_manual_override_token(" A ") == "a"
assert normalize_manual_override_token("word") == "word" assert normalize_manual_override_token("word") == "word"
+12 -2
View File
@@ -2,7 +2,13 @@ from __future__ import annotations
import io import io
import time import time
from abogen.webui.service import Job, JobStatus, build_service, _JOB_LOGGER, build_audiobookshelf_metadata from abogen.webui.service import (
Job,
JobStatus,
build_service,
_JOB_LOGGER,
build_audiobookshelf_metadata,
)
def test_service_processes_job(tmp_path): def test_service_processes_job(tmp_path):
@@ -47,7 +53,11 @@ def test_service_processes_job(tmp_path):
) )
deadline = time.time() + 5 deadline = time.time() + 5
while time.time() < deadline and job.status not in {JobStatus.COMPLETED, JobStatus.FAILED, JobStatus.CANCELLED}: while time.time() < deadline and job.status not in {
JobStatus.COMPLETED,
JobStatus.FAILED,
JobStatus.CANCELLED,
}:
time.sleep(0.05) time.sleep(0.05)
service.shutdown() service.shutdown()
+10 -7
View File
@@ -24,9 +24,12 @@ def _chunk(text: str, idx: int) -> dict:
def test_analyze_speakers_infers_gender_from_pronouns(): def test_analyze_speakers_infers_gender_from_pronouns():
chunks = [ chunks = [
_chunk("\"Greetings,\" said John. He adjusted his hat as he smiled.", 0), _chunk('"Greetings," said John. He adjusted his hat as he smiled.', 0),
_chunk("\"Hello,\" said Mary. She straightened her dress as she introduced herself.", 1), _chunk(
_chunk("\"Nice to meet you,\" said Alex.", 2), '"Hello," said Mary. She straightened her dress as she introduced herself.',
1,
),
_chunk('"Nice to meet you," said Alex.', 2),
] ]
analysis = analyze_speakers(_chapters(), chunks, threshold=1, max_speakers=0) analysis = analyze_speakers(_chapters(), chunks, threshold=1, max_speakers=0)
@@ -63,8 +66,8 @@ def test_analyze_speakers_ignores_leading_stopwords():
def test_analyze_speakers_applies_threshold_suppression(): def test_analyze_speakers_applies_threshold_suppression():
chunks = [ chunks = [
_chunk("\"Hello there,\" said Narrator.", 0), _chunk('"Hello there," said Narrator.', 0),
_chunk("\"It is lying,\" said Green.", 1), _chunk('"It is lying," said Green.', 1),
] ]
analysis = analyze_speakers(_chapters(), chunks, threshold=3, max_speakers=0) analysis = analyze_speakers(_chapters(), chunks, threshold=3, max_speakers=0)
@@ -78,7 +81,7 @@ def test_analyze_speakers_applies_threshold_suppression():
def test_sample_excerpt_includes_context_paragraphs(): def test_sample_excerpt_includes_context_paragraphs():
chunks = [ chunks = [
_chunk("The hallway was quiet as footsteps approached.", 0), _chunk("The hallway was quiet as footsteps approached.", 0),
_chunk('\"Open the door,\" said John as he reached for the handle.', 1), _chunk('"Open the door," said John as he reached for the handle.', 1),
_chunk("Mary watched him closely, unsure of his intent.", 2), _chunk("Mary watched him closely, unsure of his intent.", 2),
] ]
@@ -89,5 +92,5 @@ def test_sample_excerpt_includes_context_paragraphs():
assert john.sample_quotes, "Expected John to have at least one sample quote" assert john.sample_quotes, "Expected John to have at least one sample quote"
excerpt = john.sample_quotes[0]["excerpt"] excerpt = john.sample_quotes[0]["excerpt"]
assert "The hallway was quiet" in excerpt assert "The hallway was quiet" in excerpt
assert "\"Open the door,\" said John" in excerpt assert '"Open the door," said John' in excerpt
assert "Mary watched him closely" in excerpt assert "Mary watched him closely" in excerpt
+9 -3
View File
@@ -6,7 +6,9 @@ from abogen.text_extractor import extract_from_path
from abogen.utils import calculate_text_length from abogen.utils import calculate_text_length
ASSET = Path("test_assets/alexandre-dumas_the-count-of-monte-cristo_chapman-and-hall.epub") ASSET = Path(
"test_assets/alexandre-dumas_the-count-of-monte-cristo_chapman-and-hall.epub"
)
def test_epub_character_counts_align_with_calculated_total(): def test_epub_character_counts_align_with_calculated_total():
@@ -37,8 +39,12 @@ def test_epub_series_metadata_extracted_from_opf_meta(tmp_path):
book.add_author("Example Author") book.add_author("Example Author")
# Calibre-style series metadata # Calibre-style series metadata
book.add_metadata("OPF", "meta", "", {"name": "calibre:series", "content": "Example Saga"}) book.add_metadata(
book.add_metadata("OPF", "meta", "", {"name": "calibre:series_index", "content": "2"}) "OPF", "meta", "", {"name": "calibre:series", "content": "Example Saga"}
)
book.add_metadata(
"OPF", "meta", "", {"name": "calibre:series_index", "content": "2"}
)
chapter = epub.EpubHtml(title="Chapter 1", file_name="chap_01.xhtml", lang="en") chapter = epub.EpubHtml(title="Chapter 1", file_name="chap_01.xhtml", lang="en")
chapter.content = "<h1>Chapter 1</h1><p>Hello</p>" chapter.content = "<h1>Chapter 1</h1><p>Hello</p>"
+28 -10
View File
@@ -16,14 +16,22 @@ from abogen.normalization_settings import (
from abogen.spacy_contraction_resolver import resolve_ambiguous_contractions from abogen.spacy_contraction_resolver import resolve_ambiguous_contractions
SPACY_RESOLVER_AVAILABLE = bool(resolve_ambiguous_contractions("It's been a long time.")) SPACY_RESOLVER_AVAILABLE = bool(
resolve_ambiguous_contractions("It's been a long time.")
)
def _normalize_text(text: str, *, normalization_overrides: dict[str, object] | None = None) -> str: def _normalize_text(
text: str, *, normalization_overrides: dict[str, object] | None = None
) -> str:
runtime_settings = get_runtime_settings() runtime_settings = get_runtime_settings()
if normalization_overrides: if normalization_overrides:
runtime_settings = apply_normalization_overrides(runtime_settings, normalization_overrides) runtime_settings = apply_normalization_overrides(
config = build_apostrophe_config(settings=runtime_settings, base=DEFAULT_APOSTROPHE_CONFIG) runtime_settings, normalization_overrides
)
config = build_apostrophe_config(
settings=runtime_settings, base=DEFAULT_APOSTROPHE_CONFIG
)
return normalize_for_pipeline(text, config=config, settings=runtime_settings) return normalize_for_pipeline(text, config=config, settings=runtime_settings)
@@ -202,7 +210,9 @@ def test_contractions_can_be_kept_when_override_disabled() -> None:
def test_sibilant_possessives_remain_when_marking_disabled() -> None: def test_sibilant_possessives_remain_when_marking_disabled() -> None:
normalized = _normalize_text( normalized = _normalize_text(
"The boss's chair wobbled.", "The boss's chair wobbled.",
normalization_overrides={"normalization_apostrophes_sibilant_possessives": False}, normalization_overrides={
"normalization_apostrophes_sibilant_possessives": False
},
) )
assert "boss's" in normalized assert "boss's" in normalized
assert "boss iz" not in normalized.lower() assert "boss iz" not in normalized.lower()
@@ -276,10 +286,13 @@ def test_spacy_disambiguates_she_would() -> None:
@pytest.mark.skipif(not SPACY_RESOLVER_AVAILABLE, reason="spaCy model unavailable") @pytest.mark.skipif(not SPACY_RESOLVER_AVAILABLE, reason="spaCy model unavailable")
def test_sample_sentence_handles_complex_contractions() -> None: def test_sample_sentence_handles_complex_contractions() -> None:
sample = "I've heard the captain'll arrive by dusk, but they'd said the same yesterday." sample = (
"I've heard the captain'll arrive by dusk, but they'd said the same yesterday."
)
normalized = _normalize_text(sample) normalized = _normalize_text(sample)
assert ( assert (
"I have heard the captain will arrive by dusk, but they had said the same yesterday." == normalized "I have heard the captain will arrive by dusk, but they had said the same yesterday."
== normalized
) )
@@ -316,9 +329,12 @@ def mock_settings():
"normalization_footnotes": True, "normalization_footnotes": True,
"normalization_numbers_year_style": "american", "normalization_numbers_year_style": "american",
} }
with patch("tests.test_text_normalization.get_runtime_settings", return_value=defaults): with patch(
"tests.test_text_normalization.get_runtime_settings", return_value=defaults
):
yield yield
def test_currency_magnitude(): def test_currency_magnitude():
cases = [ cases = [
("$2 million", "two million dollars"), ("$2 million", "two million dollars"),
@@ -335,9 +351,11 @@ def test_currency_magnitude():
settings = { settings = {
"normalization_numbers": True, "normalization_numbers": True,
"normalization_currency": True, "normalization_currency": True,
"normalization_apostrophe_mode": "spacy" "normalization_apostrophe_mode": "spacy",
} }
for input_text, expected in cases: for input_text, expected in cases:
normalized = _normalize_text(input_text, normalization_overrides=settings) normalized = _normalize_text(input_text, normalization_overrides=settings)
assert expected.lower() in normalized.lower(), f"Failed for {input_text}: got '{normalized}'" assert (
expected.lower() in normalized.lower()
), f"Failed for {input_text}: got '{normalized}'"
+5 -1
View File
@@ -4,7 +4,11 @@ from typing import cast
import pytest import pytest
from abogen.constants import VOICES_INTERNAL from abogen.constants import VOICES_INTERNAL
from abogen.voice_cache import LocalEntryNotFoundError, _CACHED_VOICES, ensure_voice_assets from abogen.voice_cache import (
LocalEntryNotFoundError,
_CACHED_VOICES,
ensure_voice_assets,
)
from abogen.webui.conversion_runner import _collect_required_voice_ids from abogen.webui.conversion_runner import _collect_required_voice_ids
from abogen.webui.service import Job from abogen.webui.service import Job