From 60ba01557e78dfe3756a8c74d70e856b2ba9927e Mon Sep 17 00:00:00 2001 From: Artem Akymenko Date: Tue, 14 Jul 2026 10:17:20 +0000 Subject: [PATCH] refactor: extract chapter title processing into domain/chapter_titles.py - Extract simplify_heading_text, headings_equivalent, strip_duplicate_heading_line - Extract normalize_caps_word, normalize_chapter_opening_caps - Extract format_spoken_chapter_title - Remove _HEADING_SANITIZE_RE, _HEADING_NUMBER_PREFIX_RE, _ACRONYM_ALLOWLIST, _ROMAN_NUMERAL_CHARS, _CAPS_WORD_RE from conversion_runner.py - Add tests/test_chapter_titles.py with 31 tests - All tests match old behavior --- abogen/domain/chapter_titles.py | 172 +++++++++++++++++++++++++ abogen/webui/conversion_runner.py | 207 ++---------------------------- tests/test_chapter_titles.py | 144 +++++++++++++++++++++ 3 files changed, 324 insertions(+), 199 deletions(-) create mode 100644 abogen/domain/chapter_titles.py create mode 100644 tests/test_chapter_titles.py diff --git a/abogen/domain/chapter_titles.py b/abogen/domain/chapter_titles.py new file mode 100644 index 0000000..1d39288 --- /dev/null +++ b/abogen/domain/chapter_titles.py @@ -0,0 +1,172 @@ +from __future__ import annotations + +import re +from typing import List, Tuple + + +_HEADING_SANITIZE_RE = re.compile(r"[^a-z0-9]+") +_HEADING_NUMBER_PREFIX_RE = re.compile( + r"^\s*(?P(?:\d+|[ivxlcdm]+))(?P(?:[\s.:;-].*)?)$", + re.IGNORECASE, +) +_ACRONYM_ALLOWLIST = { + "AI", "API", "CPU", "DIY", "GPU", "HTML", "HTTP", "HTTPS", "ID", + "JSON", "MP3", "MP4", "M4B", "NASA", "OCR", "PDF", "SQL", "TV", + "TTS", "UK", "UN", "UFO", "OK", "URL", "USA", "US", "VR", +} +_ROMAN_NUMERAL_CHARS = frozenset("IVXLCDM") +_CAPS_WORD_RE = re.compile(r"[A-Z][A-Z0-9'\u2019-]*") + + +def simplify_heading_text(text: str) -> str: + raw = str(text or "").strip().lower() + if not raw: + return "" + simplified = _HEADING_SANITIZE_RE.sub("", raw) + if simplified.startswith("chapter"): + simplified = simplified[7:] + return simplified + + +def headings_equivalent(left: str, right: str) -> bool: + simple_left = simplify_heading_text(left) + simple_right = simplify_heading_text(right) + if not simple_left or not simple_right: + return False + if simple_left == simple_right: + return True + if simple_right.startswith(simple_left): + return True + if simple_left.startswith(simple_right): + return True + if len(simple_left) > 5 and simple_left in simple_right: + return True + return False + + +def strip_duplicate_heading_line(text: str, heading: str) -> Tuple[str, bool]: + source_text = str(text or "") + if not source_text: + return source_text, False + normalized_heading = simplify_heading_text(heading) + if not normalized_heading: + return source_text, False + lines = source_text.splitlines() + new_lines: List[str] = [] + removed = False + for line in lines: + stripped = line.strip() + if not removed and stripped: + if headings_equivalent(stripped, heading): + removed = True + continue + new_lines.append(line) + if not removed: + return source_text, False + while new_lines and not new_lines[0].strip(): + new_lines.pop(0) + return "\n".join(new_lines), True + + +def normalize_caps_word(word: str) -> str: + upper = word.upper() + letters = [char for char in upper if char.isalpha()] + if not letters: + return word + if upper in _ACRONYM_ALLOWLIST: + return word + if len(letters) <= 1: + return word + if all(char in _ROMAN_NUMERAL_CHARS for char in letters) and len(letters) <= 7: + return word + + parts = re.split(r"(['\-\u2019])", word) + normalized_parts: List[str] = [] + for part in parts: + if part in {"'", "-", "\u2019"}: + normalized_parts.append(part) + continue + if not part: + continue + normalized_parts.append(part[0].upper() + part[1:].lower()) + return "".join(normalized_parts) or word + + +def normalize_chapter_opening_caps(text: str) -> Tuple[str, bool]: + if not text: + return text, False + + leading_len = len(text) - len(text.lstrip()) + leading = text[:leading_len] + working = text[leading_len:] + if not working: + return text, False + + builder: List[str] = [] + pos = 0 + changed = False + + while pos < len(working): + char = working[pos] + if char in "\r\n": + builder.append(working[pos:]) + pos = len(working) + break + if char.isspace(): + builder.append(char) + pos += 1 + continue + if char.islower(): + builder.append(working[pos:]) + pos = len(working) + break + if not char.isalpha(): + builder.append(char) + pos += 1 + continue + + match = _CAPS_WORD_RE.match(working, pos) + if not match: + builder.append(char) + pos += 1 + continue + + word = match.group(0) + if any(ch.islower() for ch in word): + builder.append(working[pos:]) + pos = len(working) + break + + normalized = normalize_caps_word(word) + if normalized != word: + changed = True + builder.append(normalized) + pos = match.end() + + if pos < len(working): + builder.append(working[pos:]) + + if not changed: + return text, False + + return leading + "".join(builder), True + + +def format_spoken_chapter_title(title: str, index: int, apply_prefix: bool) -> str: + base = str(title or "").strip() + if not base: + return f"Chapter {index}" if apply_prefix else "" + if not apply_prefix: + return base + lowered = base.lower() + if lowered.startswith("chapter") and (len(lowered) == 7 or not lowered[7].isalpha()): + return base + match = _HEADING_NUMBER_PREFIX_RE.match(base) + if match: + number = match.group("number") or "" + suffix = match.group("suffix") or "" + cleaned_suffix = suffix.lstrip(" .,:;-_ \t\u2013\u2014\u00b7\u2022") + if cleaned_suffix: + return f"Chapter {number}. {cleaned_suffix}" + return f"Chapter {number}" + return base diff --git a/abogen/webui/conversion_runner.py b/abogen/webui/conversion_runner.py index 4508f32..c211297 100644 --- a/abogen/webui/conversion_runner.py +++ b/abogen/webui/conversion_runner.py @@ -46,6 +46,14 @@ from abogen.voice_profiles import load_profiles, normalize_profile_entry from abogen.pronunciation_store import increment_usage from abogen.llm_client import LLMClientError from abogen.infrastructure.subtitle_writer import create_subtitle_writer, SubtitleMode +from abogen.domain.chapter_titles import ( + simplify_heading_text as _simplify_heading_text, + headings_equivalent as _headings_equivalent, + strip_duplicate_heading_line as _strip_duplicate_heading_line, + normalize_caps_word as _normalize_caps_word, + normalize_chapter_opening_caps as _normalize_chapter_opening_caps, + format_spoken_chapter_title as _format_spoken_chapter_title, +) from .service import Job, JobStatus @@ -148,208 +156,9 @@ def _coerce_truthy(value: Any, default: bool = True) -> bool: return bool(value) -_HEADING_SANITIZE_RE = re.compile(r"[^a-z0-9]+") -_HEADING_NUMBER_PREFIX_RE = re.compile(r"^\s*(?P(?:\d+|[ivxlcdm]+))(?P(?:[\s.:;-].*)?)$", re.IGNORECASE) -_ACRONYM_ALLOWLIST = { - "AI", - "API", - "CPU", - "DIY", - "GPU", - "HTML", - "HTTP", - "HTTPS", - "ID", - "JSON", - "MP3", - "MP4", - "M4B", - "NASA", - "OCR", - "PDF", - "SQL", - "TV", - "TTS", - "UK", - "UN", - "UFO", - "OK", - "URL", - "USA", - "US", - "VR", -} -_ROMAN_NUMERAL_CHARS = frozenset("IVXLCDM") -_CAPS_WORD_RE = re.compile(r"[A-Z][A-Z0-9'\u2019-]*") _OUTPUT_SANITIZE_RE = re.compile(r"[^\w\-_.]+") -def _simplify_heading_text(text: str) -> str: - raw = str(text or "").strip().lower() - if not raw: - return "" - simplified = _HEADING_SANITIZE_RE.sub("", raw) - if simplified.startswith("chapter"): - simplified = simplified[7:] - return simplified - - -def _headings_equivalent(left: str, right: str) -> bool: - simple_left = _simplify_heading_text(left) - simple_right = _simplify_heading_text(right) - if not simple_left or not simple_right: - return False - - # Exact match - if simple_left == simple_right: - return True - - # Check if one is a prefix of the other (e.g. "Chapter 2" vs "Chapter 2: The Return") - # But be careful not to match "Chapter 1" with "Chapter 10" - # _simplify_heading_text removes "chapter" prefix, so we are comparing "2" vs "2thereturn" - - # If left is "2" and right is "2thereturn", left is prefix of right. - if simple_right.startswith(simple_left): - return True - - # If left is "2thereturn" and right is "2", right is prefix of left. - if simple_left.startswith(simple_right): - return True - - # Also check if the line is contained in the heading if it's long enough - if len(simple_left) > 5 and simple_left in simple_right: - return True - - return False - - -def _format_spoken_chapter_title(title: str, index: int, apply_prefix: bool) -> str: - base = str(title or "").strip() - if not base: - return f"Chapter {index}" if apply_prefix else "" - if not apply_prefix: - return base - lowered = base.lower() - if lowered.startswith("chapter") and (len(lowered) == 7 or not lowered[7].isalpha()): - return base - match = _HEADING_NUMBER_PREFIX_RE.match(base) - if match: - number = match.group("number") or "" - suffix = match.group("suffix") or "" - cleaned_suffix = suffix.lstrip(" .,:;-_\t\u2013\u2014\u00b7\u2022") - if cleaned_suffix: - return f"Chapter {number}. {cleaned_suffix}" - return f"Chapter {number}" - return base - - -def _strip_duplicate_heading_line(text: str, heading: str) -> tuple[str, bool]: - source_text = str(text or "") - if not source_text: - return source_text, False - normalized_heading = _simplify_heading_text(heading) - if not normalized_heading: - return source_text, False - lines = source_text.splitlines() - new_lines: List[str] = [] - removed = False - for line in lines: - stripped = line.strip() - if not removed and stripped: - if _headings_equivalent(stripped, heading): - removed = True - continue - new_lines.append(line) - if not removed: - return source_text, False - while new_lines and not new_lines[0].strip(): - new_lines.pop(0) - return "\n".join(new_lines), True - - -def _normalize_caps_word(word: str) -> str: - upper = word.upper() - letters = [char for char in upper if char.isalpha()] - if not letters: - return word - if upper in _ACRONYM_ALLOWLIST: - return word - if len(letters) <= 1: - return word - if all(char in _ROMAN_NUMERAL_CHARS for char in letters) and len(letters) <= 7: - return word - - parts = re.split(r"(['\-\u2019])", word) - normalized_parts: List[str] = [] - for part in parts: - if part in {"'", "-", "\u2019"}: - normalized_parts.append(part) - continue - if not part: - continue - normalized_parts.append(part[0].upper() + part[1:].lower()) - return "".join(normalized_parts) or word - - -def _normalize_chapter_opening_caps(text: str) -> tuple[str, bool]: - if not text: - return text, False - - leading_len = len(text) - len(text.lstrip()) - leading = text[:leading_len] - working = text[leading_len:] - if not working: - return text, False - - builder: List[str] = [] - pos = 0 - changed = False - - while pos < len(working): - char = working[pos] - if char in "\r\n": - builder.append(working[pos:]) - pos = len(working) - break - if char.isspace(): - builder.append(char) - pos += 1 - continue - if char.islower(): - builder.append(working[pos:]) - pos = len(working) - break - if not char.isalpha(): - builder.append(char) - pos += 1 - continue - - match = _CAPS_WORD_RE.match(working, pos) - if not match: - builder.append(char) - pos += 1 - continue - - word = match.group(0) - if any(ch.islower() for ch in word): - builder.append(working[pos:]) - pos = len(working) - break - - normalized = _normalize_caps_word(word) - if normalized != word: - changed = True - builder.append(normalized) - pos = match.end() - - if pos < len(working): - builder.append(working[pos:]) - - if not changed: - return text, False - - return leading + "".join(builder), True - def _normalize_metadata_map(values: Optional[Mapping[str, Any]]) -> Dict[str, str]: normalized: Dict[str, str] = {} if not values: diff --git a/tests/test_chapter_titles.py b/tests/test_chapter_titles.py new file mode 100644 index 0000000..a0abe69 --- /dev/null +++ b/tests/test_chapter_titles.py @@ -0,0 +1,144 @@ +"""Tests for domain/chapter_titles.py.""" +import sys +import os +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +from abogen.domain.chapter_titles import ( + simplify_heading_text, + headings_equivalent, + strip_duplicate_heading_line, + normalize_caps_word, + normalize_chapter_opening_caps, + format_spoken_chapter_title, +) + + +class TestSimplifyHeadingText: + def test_empty(self): + assert simplify_heading_text("") == "" + + def test_none(self): + assert simplify_heading_text(None) == "" + + def test_chapter_prefix_removed(self): + assert simplify_heading_text("Chapter 1") == "1" + + def test_lowercase(self): + assert simplify_heading_text("Chapter 1: The Beginning") == "1thebeginning" + + def test_strips_special_chars(self): + result = simplify_heading_text("Ch. 1") + assert "1" in result + assert "." not in result + + def test_no_chapter_prefix(self): + result = simplify_heading_text("Part 2") + assert "part" in result + + +class TestHeadingsEquivalent: + def test_exact_match(self): + assert headings_equivalent("Chapter 1", "Chapter 1") + + def test_prefix_match(self): + assert headings_equivalent("Chapter 2", "Chapter 2: The Return") + + def test_reverse_prefix(self): + assert headings_equivalent("Chapter 2: The Return", "Chapter 2") + + def test_different_numbers(self): + assert not headings_equivalent("Chapter 1", "Chapter 2") + + def test_empty(self): + assert not headings_equivalent("", "Chapter 1") + + def test_long_containment(self): + assert headings_equivalent("Introduction", "Introduction to Everything") + + +class TestStripDuplicateHeadingLine: + def test_removes_heading(self): + text = "Chapter 1\n\nSome text here" + result, removed = strip_duplicate_heading_line(text, "Chapter 1") + assert removed is True + assert "Chapter 1" not in result + assert "Some text here" in result + + def test_no_heading(self): + text = "Just some text" + result, removed = strip_duplicate_heading_line(text, "Chapter 1") + assert removed is False + assert result == text + + def test_empty_text(self): + result, removed = strip_duplicate_heading_line("", "Chapter 1") + assert removed is False + + def test_strips_leading_empty_lines(self): + text = "Chapter 1\n\n\n\nText" + result, removed = strip_duplicate_heading_line(text, "Chapter 1") + assert removed is True + assert result.startswith("Text") + + +class TestNormalizeCapsWord: + def test_acronym_kept(self): + assert normalize_caps_word("TTS") == "TTS" + + def test_single_letter_kept(self): + assert normalize_caps_word("A") == "A" + + def test_roman_numeral_kept(self): + assert normalize_caps_word("IV") == "IV" + + def test_all_caps_converted(self): + result = normalize_caps_word("HELLO") + assert result == "Hello" + + def test_with_hyphen(self): + result = normalize_caps_word("WELL-KNOWN") + assert result == "Well-Known" + + +class TestNormalizeChapterOpeningCaps: + def test_all_caps_words(self): + text = "THIS IS A TEST" + result, changed = normalize_chapter_opening_caps(text) + assert changed is True + assert result == "This Is A Test" + + def test_already_normal(self): + text = "This is normal" + result, changed = normalize_chapter_opening_caps(text) + assert changed is False + + def test_empty(self): + result, changed = normalize_chapter_opening_caps("") + assert changed is False + + def test_mixed(self): + text = "HELLO world" + result, changed = normalize_chapter_opening_caps(text) + assert changed is True + + +class TestFormatSpokenChapterTitle: + def test_empty_no_prefix(self): + assert format_spoken_chapter_title("", 1, False) == "" + + def test_empty_with_prefix(self): + assert format_spoken_chapter_title("", 1, True) == "Chapter 1" + + def test_no_prefix_returns_base(self): + assert format_spoken_chapter_title("My Chapter", 1, False) == "My Chapter" + + def test_already_has_chapter(self): + assert format_spoken_chapter_title("Chapter 5", 1, True) == "Chapter 5" + + def test_number_prefix(self): + result = format_spoken_chapter_title("3. The End", 1, True) + assert result == "Chapter 3. The End" + + def test_number_only(self): + result = format_spoken_chapter_title("7", 1, True) + assert result == "Chapter 7"