Dymas/abogen
1
0
Fork 0
mirror of https://github.com/denizsafak/abogen.git synced 2026-07-18 13:40:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: Implement normalize chapter opening caps feature and update related settings

Browse Source
This commit is contained in:
JB
2025-10-27 15:58:34 -07:00
parent 9d35e39e89
commit 6e536c6e3b
11 changed files with 333 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
abogen/web/conversion_runner.py
+151 -2
View File
@@ -75,6 +75,37 @@ def _coerce_truthy(value: Any, default: bool = True) -> bool:
_HEADING_SANITIZE_RE = re.compile(r"[^a-z0-9]+")
_HEADING_NUMBER_PREFIX_RE = re.compile(r"^\s*(?P<number>(?:\d+|[ivxlcdm]+))(?P<suffix>(?:[\s.:;-].*)?)$", re.IGNORECASE)
_ACRONYM_ALLOWLIST = {
"AI",
"API",
"CPU",
"DIY",
"GPU",
"HTML",
"HTTP",
"HTTPS",
"ID",
"JSON",
"MP3",
"MP4",
"M4B",
"NASA",
"OCR",
"PDF",
"SQL",
"TV",
"TTS",
"UK",
"UN",
"UFO",
"OK",
"URL",
"USA",
"US",
"VR",
}
_ROMAN_NUMERAL_CHARS = frozenset("IVXLCDM")
_CAPS_WORD_RE = re.compile(r"[A-Z][A-Z0-9'\u2019-]*")
def _simplify_heading_text(text: str) -> str:
@@ -136,6 +167,90 @@ def _strip_duplicate_heading_line(text: str, heading: str) -> tuple[str, bool]:
new_lines.pop(0)
return "\n".join(new_lines), True
def _normalize_caps_word(word: str) -> str:
upper = word.upper()
letters = [char for char in upper if char.isalpha()]
if not letters:
return word
if upper in _ACRONYM_ALLOWLIST:
return word
if len(letters) <= 1:
return word
if all(char in _ROMAN_NUMERAL_CHARS for char in letters) and len(letters) <= 7:
return word
parts = re.split(r"(['\-\u2019])", word)
normalized_parts: List[str] = []
for part in parts:
if part in {"'", "-", "\u2019"}:
normalized_parts.append(part)
continue
if not part:
continue
normalized_parts.append(part[0].upper() + part[1:].lower())
return "".join(normalized_parts) or word
def _normalize_chapter_opening_caps(text: str) -> tuple[str, bool]:
if not text:
return text, False
leading_len = len(text) - len(text.lstrip())
leading = text[:leading_len]
working = text[leading_len:]
if not working:
return text, False
builder: List[str] = []
pos = 0
changed = False
while pos < len(working):
char = working[pos]
if char in "\r\n":
builder.append(working[pos:])
pos = len(working)
break
if char.isspace():
builder.append(char)
pos += 1
continue
if char.islower():
builder.append(working[pos:])
pos = len(working)
break
if not char.isalpha():
builder.append(char)
pos += 1
continue
match = _CAPS_WORD_RE.match(working, pos)
if not match:
builder.append(char)
pos += 1
continue
word = match.group(0)
if any(ch.islower() for ch in word):
builder.append(working[pos:])
pos = len(working)
break
normalized = _normalize_caps_word(word)
if normalized != word:
changed = True
builder.append(normalized)
pos = match.end()
if pos < len(working):
builder.append(working[pos:])
if not changed:
return text, False
return leading + "".join(builder), True
def _normalize_metadata_map(values: Optional[Mapping[str, Any]]) -> Dict[str, str]:
normalized: Dict[str, str] = {}
if not values:
@@ -1247,6 +1362,7 @@ def run_conversion_job(job: Job) -> None:
heading_text = spoken_title or raw_title
chapter_display_title = heading_text or f"Chapter {idx}"
job.add_log(f"Processing chapter {idx}/{total_chapters}: {chapter_display_title}")
normalize_opening_caps = bool(getattr(job, "normalize_chapter_opening_caps", True))
chapter_start_time = current_time
chapter_override = (
@@ -1323,6 +1439,8 @@ def run_conversion_job(job: Job) -> None:
chunks_for_chapter = chunk_groups.get(idx - 1, []) if chunk_groups else []
body_segments = 0
pending_heading_strip = remove_heading_from_body
opening_caps_pending = normalize_opening_caps
opening_caps_logged = False
if chunks_for_chapter:
job.add_log(
f"Emitting {len(chunks_for_chapter)} {job.chunk_level} chunks for chapter {idx}.",
@@ -1337,15 +1455,34 @@ def run_conversion_job(job: Job) -> None:
if not chunk_text:
continue
mutated_entry = False
if pending_heading_strip and heading_text:
chunk_text, removed_heading = _strip_duplicate_heading_line(chunk_text, heading_text)
if removed_heading:
pending_heading_strip = False
chunk_entry = dict(chunk_entry)
chunk_entry["normalized_text"] = chunk_text
mutated_entry = True
if not chunk_text.strip():
continue
if opening_caps_pending and chunk_text:
normalized_text, normalized_changed = _normalize_chapter_opening_caps(chunk_text)
if normalized_changed:
if not mutated_entry:
chunk_entry = dict(chunk_entry)
mutated_entry = True
chunk_entry["normalized_text"] = normalized_text
chunk_text = normalized_text
if not opening_caps_logged:
job.add_log(
f"Normalized uppercase chapter opening for chapter {idx}.",
level="debug",
)
opening_caps_logged = True
if chunk_text.strip():
opening_caps_pending = False
chunk_voice_spec = _chunk_voice_spec(
job,
chunk_entry,
@@ -1396,11 +1533,23 @@ def run_conversion_job(job: Job) -> None:
if body_segments == 0:
chapter_body_start = current_time
chapter_text = chapter.text
chapter_text = str(chapter.text or "")
if pending_heading_strip and heading_text:
chapter_text, removed_heading = _strip_duplicate_heading_line(chapter_text, heading_text)
if removed_heading:
pending_heading_strip = False
if opening_caps_pending and chapter_text:
normalized_body, normalized_changed = _normalize_chapter_opening_caps(chapter_text)
if normalized_changed:
chapter_text = normalized_body
if not opening_caps_logged:
job.add_log(
f"Normalized uppercase chapter opening for chapter {idx}.",
level="debug",
)
opening_caps_logged = True
if str(chapter_text or "").strip():
opening_caps_pending = False
emitted = emit_text(
chapter_text,
voice_choice=voice_choice,
@@ -1418,7 +1567,7 @@ def run_conversion_job(job: Job) -> None:
"speaker_id": "narrator",
"voice": chapter_voice_spec,
"level": job.chunk_level,
"characters": len(chapter.text or ""),
"characters": len(chapter_text or ""),
}
)
elif chunks_for_chapter:
abogen/web/routes.py
+43
View File
@@ -1543,6 +1543,22 @@ def _apply_prepare_form(
elif hasattr(form, "__contains__") and "read_title_intro" in form:
pending.read_title_intro = False
caps_values: List[str] = []
if callable(getter):
raw_caps_values = getter("normalize_chapter_opening_caps")
if raw_caps_values:
caps_values = list(cast(Iterable[str], raw_caps_values))
else:
raw_caps = form.get("normalize_chapter_opening_caps")
if raw_caps is not None:
caps_values = [raw_caps]
if caps_values:
pending.normalize_chapter_opening_caps = _coerce_bool(
caps_values[-1], getattr(pending, "normalize_chapter_opening_caps", True)
)
elif hasattr(form, "__contains__") and "normalize_chapter_opening_caps" in form:
pending.normalize_chapter_opening_caps = False
overrides: List[Dict[str, Any]] = []
selected_total = 0
@@ -1669,6 +1685,28 @@ def _apply_book_step_form(
else:
pending.read_title_intro = intro_default
caps_default = (
pending.normalize_chapter_opening_caps
if isinstance(getattr(pending, "normalize_chapter_opening_caps", None), bool)
else bool(settings.get("normalize_chapter_opening_caps", True))
)
caps_values: List[str] = []
getter = getattr(form, "getlist", None)
if callable(getter):
raw_caps_values = getter("normalize_chapter_opening_caps")
if raw_caps_values:
caps_values = list(cast(Iterable[str], raw_caps_values))
else:
raw_caps_flag = form.get("normalize_chapter_opening_caps")
if raw_caps_flag is not None:
caps_values = [raw_caps_flag]
if caps_values:
pending.normalize_chapter_opening_caps = _coerce_bool(caps_values[-1], caps_default)
elif hasattr(form, "__contains__") and "normalize_chapter_opening_caps" in form:
pending.normalize_chapter_opening_caps = False
else:
pending.normalize_chapter_opening_caps = caps_default
speed_value = form.get("speed")
if speed_value is not None:
try:
@@ -1918,6 +1956,7 @@ BOOLEAN_SETTINGS = {
"enable_entity_recognition",
"read_title_intro",
"auto_prefix_chapter_titles",
"normalize_chapter_opening_caps",
"normalization_numbers",
"normalization_titles",
"normalization_terminal",
@@ -1948,6 +1987,7 @@ def _settings_defaults() -> Dict[str, Any]:
"silence_between_chapters": 2.0,
"chapter_intro_delay": 0.5,
"read_title_intro": False,
"normalize_chapter_opening_caps": True,
"max_subtitle_words": 50,
"chunk_level": "paragraph",
"enable_entity_recognition": True,
@@ -3411,6 +3451,7 @@ def enqueue_job() -> ResponseReturnValue:
silence_between_chapters = settings["silence_between_chapters"]
chapter_intro_delay = settings["chapter_intro_delay"]
read_title_intro = settings["read_title_intro"]
normalize_chapter_opening_caps = settings["normalize_chapter_opening_caps"]
max_subtitle_words = settings["max_subtitle_words"]
auto_prefix_chapter_titles = settings["auto_prefix_chapter_titles"]
@@ -3479,6 +3520,7 @@ def enqueue_job() -> ResponseReturnValue:
cover_image_mime=cover_mime,
chapter_intro_delay=chapter_intro_delay,
read_title_intro=bool(read_title_intro),
normalize_chapter_opening_caps=bool(normalize_chapter_opening_caps),
auto_prefix_chapter_titles=bool(auto_prefix_chapter_titles),
chunk_level=chunk_level_value,
speaker_mode=speaker_mode_value,
@@ -3760,6 +3802,7 @@ def finalize_job(pending_id: str) -> ResponseReturnValue:
cover_image_mime=pending.cover_image_mime,
chapter_intro_delay=pending.chapter_intro_delay,
read_title_intro=pending.read_title_intro,
normalize_chapter_opening_caps=getattr(pending, "normalize_chapter_opening_caps", True),
auto_prefix_chapter_titles=getattr(pending, "auto_prefix_chapter_titles", True),
chunk_level=pending.chunk_level,
chunks=processed_chunks,
abogen/web/service.py
+11 -2
View File
@@ -24,7 +24,7 @@ def _create_set_event() -> threading.Event:
return event
STATE_VERSION = 7
STATE_VERSION = 8
_JOB_LOGGER = logging.getLogger("abogen.jobs")
@@ -107,6 +107,7 @@ class Job:
chapter_intro_delay: float = 0.5
read_title_intro: bool = False
auto_prefix_chapter_titles: bool = True
normalize_chapter_opening_caps: bool = True
status: JobStatus = JobStatus.PENDING
started_at: Optional[float] = None
finished_at: Optional[float] = None
@@ -175,6 +176,7 @@ class Job:
"chapter_intro_delay": self.chapter_intro_delay,
"read_title_intro": getattr(self, "read_title_intro", False),
"auto_prefix_chapter_titles": getattr(self, "auto_prefix_chapter_titles", True),
"normalize_chapter_opening_caps": getattr(self, "normalize_chapter_opening_caps", True),
},
"metadata_tags": dict(self.metadata_tags),
"chapters": [
@@ -239,6 +241,7 @@ class PendingJob:
chapter_intro_delay: float = 0.5
read_title_intro: bool = False
auto_prefix_chapter_titles: bool = True
normalize_chapter_opening_caps: bool = True
chunk_level: str = "paragraph"
chunks: List[Dict[str, Any]] = field(default_factory=list)
speakers: Dict[str, Any] = field(default_factory=dict)
@@ -320,6 +323,7 @@ class ConversionService:
chapter_intro_delay: float = 0.5,
read_title_intro: bool = False,
auto_prefix_chapter_titles: bool = True,
normalize_chapter_opening_caps: bool = True,
chunk_level: str = "paragraph",
chunks: Optional[Iterable[Any]] = None,
speakers: Optional[Mapping[str, Any]] = None,
@@ -368,6 +372,7 @@ class ConversionService:
chapter_intro_delay=chapter_intro_delay,
read_title_intro=bool(read_title_intro),
auto_prefix_chapter_titles=bool(auto_prefix_chapter_titles),
normalize_chapter_opening_caps=bool(normalize_chapter_opening_caps),
chunk_level=chunk_level,
chunks=normalized_chunks,
speakers=dict(speakers or {}),
@@ -524,6 +529,7 @@ class ConversionService:
chapter_intro_delay=job.chapter_intro_delay,
read_title_intro=job.read_title_intro,
auto_prefix_chapter_titles=job.auto_prefix_chapter_titles,
normalize_chapter_opening_caps=job.normalize_chapter_opening_caps,
chunk_level=job.chunk_level,
chunks=job.chunks,
speakers=job.speakers,
@@ -751,6 +757,7 @@ class ConversionService:
"chapter_intro_delay": job.chapter_intro_delay,
"read_title_intro": job.read_title_intro,
"auto_prefix_chapter_titles": job.auto_prefix_chapter_titles,
"normalize_chapter_opening_caps": job.normalize_chapter_opening_caps,
"chunk_level": job.chunk_level,
"chunks": [dict(entry) for entry in job.chunks],
"speakers": dict(job.speakers),
@@ -844,6 +851,7 @@ class ConversionService:
chapter_intro_delay=float(payload.get("chapter_intro_delay", 0.5)),
read_title_intro=bool(payload.get("read_title_intro", False)),
auto_prefix_chapter_titles=bool(payload.get("auto_prefix_chapter_titles", True)),
normalize_chapter_opening_caps=bool(payload.get("normalize_chapter_opening_caps", True)),
)
job.status = JobStatus(payload.get("status", job.status.value))
job.started_at = payload.get("started_at")
@@ -898,7 +906,8 @@ class ConversionService:
except Exception:
return
if payload.get("version") != STATE_VERSION:
version = int(payload.get("version", 0) or 0)
if version not in {STATE_VERSION, STATE_VERSION - 1}:
return
jobs_payload = payload.get("jobs", [])
abogen/web/templates/job_detail.html
+1
View File
@@ -25,6 +25,7 @@
<li><strong>Silence between chapters:</strong> {{ '%.1f'|format(job.silence_between_chapters) }}s</li>
<li><strong>Chapter intro delay:</strong> {{ '%.1f'|format(job.chapter_intro_delay) }}s</li>
<li><strong>Title intro:</strong> {{ 'Yes' if job.read_title_intro else 'No' }}</li>
<li><strong>Normalize chapter openings:</strong> {{ 'Yes' if job.normalize_chapter_opening_caps else 'No' }}</li>
<li><strong>Prefix chapter titles:</strong> {{ 'Yes' if job.auto_prefix_chapter_titles else 'No' }}</li>
<li><strong>Max words per subtitle:</strong> {{ job.max_subtitle_words }}</li>
<li><strong>Project folder:</strong> {{ 'Yes' if job.save_as_project else 'No' }}</li>
abogen/web/templates/partials/new_job_step_book.html
+18
View File
@@ -65,6 +65,16 @@
{% set read_title_intro = settings_dict.get('read_title_intro', False) %}
{% endif %}
{% endif %}
{% set normalize_caps_value = form_values.get('normalize_chapter_opening_caps') if form_values else None %}
{% if normalize_caps_value is not none %}
{% set normalize_chapter_opening_caps = ((normalize_caps_value|string)|lower) in ['true', '1', 'yes', 'on'] %}
{% else %}
{% if pending is not none %}
{% set normalize_chapter_opening_caps = pending.normalize_chapter_opening_caps %}
{% else %}
{% set normalize_chapter_opening_caps = settings_dict.get('normalize_chapter_opening_caps', True) %}
{% endif %}
{% endif %}
{% set selected_config = form_values.get('speaker_config') if form_values else None %}
{% if selected_config is none %}
{% if pending and pending.applied_speaker_config %}
@@ -283,6 +293,14 @@
</label>
<p class="hint">When enabled, the narrator speaks the book title and author list before chapter one begins.</p>
</div>
<div class="field field--stack">
<label class="toggle-pill">
<input type="hidden" name="normalize_chapter_opening_caps" value="false">
<input type="checkbox" name="normalize_chapter_opening_caps" value="true" {% if normalize_chapter_opening_caps %}checked{% endif %} {{ 'disabled' if readonly else '' }}>
<span>Normalize ALL CAPS chapter openings</span>
</label>
<p class="hint">Converts opening sentences written in uppercase to sentence case while keeping acronyms like AI intact.</p>
</div>
</div>
</section>
</div>
abogen/web/templates/partials/new_job_step_chapters.html
+1
View File
@@ -13,6 +13,7 @@
<input type="hidden" name="speaker_analysis_threshold" value="{{ pending.speaker_analysis_threshold }}">
<input type="hidden" name="chapter_intro_delay" value="{{ '%.2f'|format(pending.chapter_intro_delay) }}">
<input type="hidden" name="read_title_intro" value="{{ 'true' if pending.read_title_intro else 'false' }}">
<input type="hidden" name="normalize_chapter_opening_caps" value="{{ 'true' if pending.normalize_chapter_opening_caps else 'false' }}">
{% if pending.generate_epub3 %}
<input type="hidden" name="generate_epub3" value="true">
{% endif %}
abogen/web/templates/partials/new_job_step_entities.html
+1
View File
@@ -26,6 +26,7 @@
<input type="hidden" name="speaker_analysis_threshold" value="{{ pending.speaker_analysis_threshold }}">
<input type="hidden" name="chapter_intro_delay" value="{{ '%.2f'|format(pending.chapter_intro_delay) }}">
<input type="hidden" name="read_title_intro" value="{{ 'true' if pending.read_title_intro else 'false' }}">
<input type="hidden" name="normalize_chapter_opening_caps" value="{{ 'true' if pending.normalize_chapter_opening_caps else 'false' }}">
{% if pending.generate_epub3 %}
<input type="hidden" name="generate_epub3" value="true">
{% endif %}
abogen/web/templates/prepare_chapters.html
+1
View File
@@ -53,6 +53,7 @@
<input type="hidden" name="speaker_analysis_threshold" value="{{ pending.speaker_analysis_threshold }}">
<input type="hidden" name="chapter_intro_delay" value="{{ '%.2f'|format(pending.chapter_intro_delay) }}">
<input type="hidden" name="read_title_intro" value="{{ 'true' if pending.read_title_intro else 'false' }}">
<input type="hidden" name="normalize_chapter_opening_caps" value="{{ 'true' if pending.normalize_chapter_opening_caps else 'false' }}">
{% if pending.generate_epub3 %}
<input type="hidden" name="generate_epub3" value="true">
{% endif %}
abogen/web/templates/settings.html
+7
View File
@@ -130,6 +130,13 @@
</label>
<p class="hint">When enabled, the narrator speaks the title, optional subtitle, and author names before chapter one.</p>
</div>
<div class="field field--stack">
<label class="toggle-pill">
<input type="checkbox" name="normalize_chapter_opening_caps" value="true" {% if settings.normalize_chapter_opening_caps %}checked{% endif %}>
<span>Normalize ALL CAPS Chapter Openings</span>
</label>
<p class="hint">Converts screaming uppercase openings to sentence case while preserving acronyms.</p>
</div>
<div class="field field--stack">
<label class="toggle-pill">
<input type="checkbox" name="auto_prefix_chapter_titles" value="true" {% if settings.auto_prefix_chapter_titles %}checked{% endif %}>
tests/test_conversion_chapter_titles.py
+88
View File
@@ -1,6 +1,70 @@
import sys
import types
if "soundfile" not in sys.modules:
soundfile_stub = types.ModuleType("soundfile")
class _SoundFileStub: # pragma: no cover - placeholder to satisfy imports
def __init__(self, *args: object, **kwargs: object) -> None:
raise RuntimeError("soundfile is not installed in the test environment")
soundfile_stub.SoundFile = _SoundFileStub # type: ignore[attr-defined]
sys.modules["soundfile"] = soundfile_stub
if "static_ffmpeg" not in sys.modules:
sys.modules["static_ffmpeg"] = types.ModuleType("static_ffmpeg")
if "ebooklib" not in sys.modules:
ebooklib_stub = types.ModuleType("ebooklib")
ebooklib_epub_stub = types.ModuleType("ebooklib.epub")
ebooklib_stub.epub = ebooklib_epub_stub # type: ignore[attr-defined]
sys.modules["ebooklib"] = ebooklib_stub
sys.modules["ebooklib.epub"] = ebooklib_epub_stub
if "fitz" not in sys.modules:
sys.modules["fitz"] = types.ModuleType("fitz")
if "markdown" not in sys.modules:
markdown_stub = types.ModuleType("markdown")
class _MarkdownStub:
def __init__(self, *args: object, **kwargs: object) -> None:
self.toc_tokens = []
def convert(self, text: str) -> str:
return text
markdown_stub.Markdown = _MarkdownStub # type: ignore[attr-defined]
sys.modules["markdown"] = markdown_stub
if "bs4" not in sys.modules:
bs4_stub = types.ModuleType("bs4")
class _BeautifulSoupStub:
def __init__(self, *args: object, **kwargs: object) -> None:
self._text = ""
def find(self, *args: object, **kwargs: object) -> None:
return None
def get_text(self) -> str:
return self._text
def decompose(self) -> None: # pragma: no cover - compatibility shim
return None
class _NavigableStringStub(str):
pass
bs4_stub.BeautifulSoup = _BeautifulSoupStub # type: ignore[attr-defined]
bs4_stub.NavigableString = _NavigableStringStub # type: ignore[attr-defined]
sys.modules["bs4"] = bs4_stub
from abogen.web.conversion_runner import (
_format_spoken_chapter_title,
_headings_equivalent,
_normalize_chapter_opening_caps,
_strip_duplicate_heading_line,
)
@@ -29,3 +93,27 @@ def test_strip_duplicate_heading_line_removes_first_match() -> None:
text, removed = _strip_duplicate_heading_line("Chapter 3: Intro\nBody text", "Chapter 3: Intro")
assert removed is True
assert text.strip() == "Body text"
def test_normalize_chapter_opening_caps_basic_title() -> None:
normalized, changed = _normalize_chapter_opening_caps("ALL CAPS TITLE")
assert normalized == "All Caps Title"
assert changed is True
def test_normalize_chapter_opening_caps_respects_acronyms() -> None:
normalized, changed = _normalize_chapter_opening_caps("NASA MISSION LOG")
assert normalized == "NASA Mission Log"
assert changed is True
def test_normalize_chapter_opening_caps_handles_roman_numerals() -> None:
normalized, changed = _normalize_chapter_opening_caps("IV. THE RETURN")
assert normalized == "IV. The Return"
assert changed is True
def test_normalize_chapter_opening_caps_keeps_mixed_case() -> None:
normalized, changed = _normalize_chapter_opening_caps("Already Mixed Case")
assert normalized == "Already Mixed Case"
assert changed is False
tests/test_prepare_form.py
+1
View File
@@ -33,6 +33,7 @@ def _make_pending_job() -> PendingJob:
chapters=[],
created_at=0.0,
read_title_intro=False,
normalize_chapter_opening_caps=True,
)