feat: Integrate roman numeral normalization in chapter titles and enhance related tests

abogen/kokoro_text_normalization.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 import re
 import unicodedata
 from dataclasses import dataclass
-from typing import List, Tuple, Iterable, Callable, Optional
+from typing import Callable, Iterable, List, Optional, Sequence, Tuple
 
 # ---------- Configuration Dataclass ----------
 
@@ -116,7 +116,12 @@ ACRONYM_POSSESSIVE_RE = re.compile(r"^[A-Z]{2,}'s$")
 
 INTERNAL_APOSTROPHE_RE = re.compile(r"[A-Za-z]'.+[A-Za-z]")  # apostrophe not at edge
 
-WORD_TOKEN_RE = re.compile(r"[A-Za-z0-9'’]+|[^A-Za-z0-9\s]")
+# Capture contiguous runs of Unicode letters/digits/apostrophes/hyphens, otherwise fall back to
+# single-character tokens (punctuation, symbols, etc.).
+WORD_TOKEN_RE = re.compile(
+    r"[0-9A-Za-z'’\u00C0-\u1FFF\u2C00-\uD7FF\-]+|[^0-9A-Za-z\s]",
+    re.UNICODE,
+)
 
 APOSTROPHE_CHARS = "’`´ꞌʼ"
 
@@ -161,6 +166,154 @@ def tokenize(text: str) -> List[str]:
     return WORD_TOKEN_RE.findall(text)
 
 
+def _cleanup_spacing(text: str) -> str:
+    if not text:
+        return text
+
+    for marker in ("\ufeff", "\u200b", "\u200c", "\u200d", "\u2060"):
+        text = text.replace(marker, "")
+
+    # Collapse spaces before closing punctuation.
+    text = re.sub(r"\s+([,.;:!?%])", r"\1", text)
+    text = re.sub(r"\s+([’\"”»›)\]\}])", r"\1", text)
+
+    # Remove spaces directly after opening punctuation/quotes.
+    text = re.sub(r"([«‹“‘\"'(\[\{])\s+", r"\1", text)
+
+    # Ensure spaces exist after sentence punctuation when followed by a word/quote.
+    text = re.sub(r"([,.;:!?%])(?![\s”'\"’»›)])", r"\1 ", text)
+    text = re.sub(r"([”\"’])(?![\s.,;:!?\"”’»›)])", r"\1 ", text)
+
+    # Tighten hyphen/em dash spacing between word characters.
+    text = re.sub(r"(?<=\w)\s*([-–—])\s*(?=\w)", r"\1", text)
+
+    # Normalize multiple spaces.
+    text = re.sub(r"\s{2,}", " ", text)
+    return text.strip()
+
+
+_ROMAN_VALUE_MAP = {
+    "I": 1,
+    "V": 5,
+    "X": 10,
+    "L": 50,
+    "C": 100,
+    "D": 500,
+    "M": 1000,
+}
+
+_ROMAN_COMPOSE_ORDER = [
+    (1000, "M"),
+    (900, "CM"),
+    (500, "D"),
+    (400, "CD"),
+    (100, "C"),
+    (90, "XC"),
+    (50, "L"),
+    (40, "XL"),
+    (10, "X"),
+    (9, "IX"),
+    (5, "V"),
+    (4, "IV"),
+    (1, "I"),
+]
+
+_ROMAN_PREFIX_RE = re.compile(r"^(?P<roman>[IVXLCDM]+)(?P<sep>[\s\.:,;\-–—]*)", re.IGNORECASE)
+
+
+def _roman_to_int(token: str) -> Optional[int]:
+    if not token:
+        return None
+    total = 0
+    prev = 0
+    token_upper = token.upper()
+    for char in reversed(token_upper):
+        value = _ROMAN_VALUE_MAP.get(char)
+        if value is None:
+            return None
+        if value < prev:
+            total -= value
+        else:
+            total += value
+            prev = value
+    if total <= 0:
+        return None
+    if _int_to_roman(total) != token_upper:
+        return None
+    return total
+
+
+def _int_to_roman(value: int) -> str:
+    parts: List[str] = []
+    remaining = value
+    for amount, symbol in _ROMAN_COMPOSE_ORDER:
+        while remaining >= amount:
+            parts.append(symbol)
+            remaining -= amount
+    return "".join(parts)
+
+
+def normalize_roman_numeral_titles(
+    titles: Sequence[str],
+    *,
+    threshold: float = 0.5,
+) -> List[str]:
+    if not titles:
+        return []
+
+    normalized: List[str] = []
+    matches: List[Tuple[int, str, int, str, str]] = []
+    non_empty = 0
+
+    for index, raw in enumerate(titles):
+        title = "" if raw is None else str(raw)
+        stripped = title.lstrip()
+        leading_ws = title[: len(title) - len(stripped)]
+        if not stripped:
+            normalized.append(title)
+            continue
+
+        non_empty += 1
+        match = _ROMAN_PREFIX_RE.match(stripped)
+        if not match:
+            normalized.append(title)
+            continue
+
+        roman_token = match.group("roman")
+        separator = match.group("sep") or ""
+        rest = stripped[match.end():]
+
+        if not separator and rest and rest[:1].isalnum():
+            normalized.append(title)
+            continue
+
+        numeric_value = _roman_to_int(roman_token)
+        if numeric_value is None:
+            normalized.append(title)
+            continue
+
+        matches.append((index, leading_ws, numeric_value, separator, rest))
+        normalized.append(title)
+
+    if not matches or non_empty == 0:
+        return list(normalized)
+
+    if len(matches) <= non_empty * threshold:
+        return list(normalized)
+
+    output = list(normalized)
+    for idx, leading_ws, value, separator, rest in matches:
+        new_title = f"{leading_ws}{value}"
+        if separator:
+            new_title += separator
+        elif rest and not rest[0].isspace() and rest[0] not in ".-–—:;,":
+            new_title += " "
+        new_title += rest
+        output[idx] = new_title
+
+    return output
+
+
 def _match_casing(template: str, replacement: str) -> str:
     if template.isupper():
         return replacement.upper()
@@ -385,26 +538,8 @@ def normalize_apostrophes(text: str, cfg: ApostropheConfig | None = None) -> Tup
         results.append((tok, category, norm))
         normalized_tokens.append(norm)
 
-    # Simple rejoin heuristic:
+    filtered = [token for token in normalized_tokens if token]
-    # If token is purely punctuation, attach without extra space.
+    normalized_text = _cleanup_spacing(" ".join(filtered))
-    out_parts = []
-    for i, (orig, cat, norm) in enumerate(results):
-        if i == 0:
-            out_parts.append(norm)
-            continue
-        prev = results[i-1][2]
-        if re.match(r"^[.,;:!?)]$", norm):
-            # Attach to previous
-            out_parts[-1] = out_parts[-1] + norm
-        elif re.match(r"^[(]$", norm):
-            out_parts.append(norm)
-        else:
-            # Normal separation
-            if not (re.match(r"^[.,;:!?)]$", prev) or prev.endswith("—")):
-                out_parts.append(" " + norm)
-            else:
-                out_parts.append(norm)
-    normalized_text = "".join(out_parts)
     return normalized_text, results
 
 # ---------- Optional phoneme hint post-processing ----------

abogen/web/routes.py

@@ -41,6 +41,7 @@ from abogen.constants import (
     VOICES_INTERNAL,
 )
 from abogen.chunking import ChunkLevel, build_chunks_for_chapters
+from abogen.kokoro_text_normalization import normalize_roman_numeral_titles
 from abogen.utils import (
     calculate_text_length,
     clean_text,
@@ -2049,6 +2050,13 @@ def enqueue_job() -> ResponseReturnValue:
 
     cover_path, cover_mime = _persist_cover_image(extraction, stored_path)
 
+    if extraction.chapters:
+        original_titles = [chapter.title for chapter in extraction.chapters]
+        normalized_titles = normalize_roman_numeral_titles(original_titles)
+        if normalized_titles != original_titles:
+            for chapter, new_title in zip(extraction.chapters, normalized_titles):
+                chapter.title = new_title
+
     metadata_tags = extraction.metadata or {}
     total_chars = extraction.total_characters or calculate_text_length(extraction.combined_text)
     total_chapter_count = len(extraction.chapters)

tests/test_text_normalization.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from abogen.kokoro_text_normalization import normalize_roman_numeral_titles
 from abogen.web.conversion_runner import _normalize_for_pipeline
 
 
@@ -27,3 +28,38 @@ def test_terminal_punctuation_respects_closing_quotes():
     normalized = _normalize_for_pipeline('"Chapter 1"')
     compact = normalized.replace(" ", "")
     assert compact.endswith('."')
+
+
+def test_normalization_preserves_spacing_around_quotes_and_hyphen():
+    sample = "“Still,” said Château-Renaud, “Dr. d’Avrigny, who attends my mother, declares he is in despair about it."
+    normalized = _normalize_for_pipeline(sample)
+
+    assert normalized.startswith(
+        "“Still,” said Château-Renaud, “Doctor d'Avrigny, who attends my mother, declares he is in despair about it."
+    )
+    assert "  " not in normalized
+    assert "Château-Renaud" in normalized
+    assert "Doctor d'Avrigny" in normalized
+
+
+def test_normalize_roman_titles_converts_when_majority() -> None:
+    titles = ["I: Opening", "II: Rising Action", "III: Climax"]
+    normalized = normalize_roman_numeral_titles(titles)
+
+    assert normalized == ["1: Opening", "2: Rising Action", "3: Climax"]
+
+
+def test_normalize_roman_titles_skips_when_not_majority() -> None:
+    titles = ["Preface", "I: Opening", "Acknowledgements"]
+    normalized = normalize_roman_numeral_titles(titles)
+
+    assert normalized == titles
+
+
+def test_normalize_roman_titles_preserves_separators() -> None:
+    titles = ["  IV.  The Trial", "V - The Verdict", "VI\nAftermath"]
+    normalized = normalize_roman_numeral_titles(titles)
+
+    assert normalized[0] == "  4.  The Trial"
+    assert normalized[1] == "5 - The Verdict"
+    assert normalized[2].startswith("6\nAftermath")