refactor: heading transforms + dedup moved to shared layer

- conversion_planner.py: caps normalization in _build_chapters()
- conversion_executor.py: heading dedup + state machine via headings_equivalent()
- Fixed seg_start_time → chapter_body_start bug in executor
- Removed getattr fallback defaults in both adapters
- Added 4 tests for caps normalization and heading dedup
- Updated ARCHITECTURE_REFACTOR_PLAN.md with deferred WebUI cleanup
This commit is contained in:
Artem Akymenko
2026-07-26 11:41:59 +03:00
parent f516cf1985
commit 0b953d48e8
6 changed files with 180 additions and 4 deletions
+33 -2
View File
@@ -32,6 +32,10 @@ from abogen.domain.conversion_engine import (
) )
from abogen.domain.enums import OutputFormat, SubtitleMode from abogen.domain.enums import OutputFormat, SubtitleMode
from abogen.domain.normalization import TTSContext from abogen.domain.normalization import TTSContext
from abogen.domain.chapter_titles import (
apply_chapter_text_transforms,
headings_equivalent as _headings_equivalent,
)
from abogen.domain.output_paths import sanitize_filename_for_chapter from abogen.domain.output_paths import sanitize_filename_for_chapter
from abogen.infrastructure.subtitle_writer import make_subtitle_writer from abogen.infrastructure.subtitle_writer import make_subtitle_writer
@@ -249,6 +253,7 @@ def execute_conversion(
) )
# Process heading # Process heading
heading_text = ""
if chapter.title: if chapter.title:
heading_text = _format_heading(chapter.title, chapter_idx, request) heading_text = _format_heading(chapter.title, chapter_idx, request)
if heading_text: if heading_text:
@@ -269,11 +274,37 @@ def execute_conversion(
stats=stats, stats=stats,
) )
# Heading dedup: check if first line of body matches heading
pending_heading_strip = False
if heading_text and chapter.body_text:
first_line = next(
(line.strip() for line in chapter.body_text.splitlines() if line.strip()),
"",
)
if first_line and _headings_equivalent(first_line, heading_text):
pending_heading_strip = True
# Process body segments # Process body segments
chapter_chunk_markers: List[Dict[str, Any]] = [] chapter_chunk_markers: List[Dict[str, Any]] = []
chapter_body_start = stats.current_time
for seg_idx, segment in enumerate(chapter.segments): for seg_idx, segment in enumerate(chapter.segments):
check_cancelled() check_cancelled()
# Apply heading dedup to first segment (consume-once)
seg_text = segment.text
if pending_heading_strip and seg_text.strip():
seg_text, heading_removed, _ = apply_chapter_text_transforms(
seg_text,
heading_text=heading_text,
raw_title=chapter.title,
strip_heading=True,
normalize_caps=False,
)
if heading_removed:
pending_heading_strip = False
if not seg_text.strip():
continue
# Resolve segment voice (may differ from chapter voice) # Resolve segment voice (may differ from chapter voice)
if segment.voice_spec != chapter.voice_spec: if segment.voice_spec != chapter.voice_spec:
seg_provider, seg_voice, seg_speed, seg_steps = _resolve_voice( seg_provider, seg_voice, seg_speed, seg_steps = _resolve_voice(
@@ -288,7 +319,7 @@ def execute_conversion(
seg_start_time = stats.current_time seg_start_time = stats.current_time
local_segments, accumulated_tokens = synthesize_text( local_segments, accumulated_tokens = synthesize_text(
text=segment.text, text=seg_text,
params=synth, params=synth,
backend=seg_backend, backend=seg_backend,
voice=seg_voice, voice=seg_voice,
@@ -355,7 +386,7 @@ def execute_conversion(
result.chapter_markers.append({ result.chapter_markers.append({
"chapter_index": chapter_idx - 1, "chapter_index": chapter_idx - 1,
"title": chapter.title, "title": chapter.title,
"start": stats.current_time - (stats.current_time - seg_start_time) if chapter.segments else stats.current_time, "start": chapter_body_start,
"end": stats.current_time, "end": stats.current_time,
}) })
+6
View File
@@ -210,9 +210,15 @@ def _build_chapters(
selected_chapters: List[Tuple[str, str, str]], request: ConversionRequest selected_chapters: List[Tuple[str, str, str]], request: ConversionRequest
) -> List[ChapterPlan]: ) -> List[ChapterPlan]:
"""Build ChapterPlan with SegmentPlan for each chapter.""" """Build ChapterPlan with SegmentPlan for each chapter."""
from abogen.domain.chapter_titles import normalize_chapter_opening_caps
chapters = [] chapters = []
for idx, (title, body_text, default_voice) in enumerate(selected_chapters, 1): for idx, (title, body_text, default_voice) in enumerate(selected_chapters, 1):
# Apply caps normalization to body text if enabled
if request.normalize_chapter_opening_caps and body_text:
body_text, _ = normalize_chapter_opening_caps(body_text)
# Build segments for this chapter (idx is 1-based, chunks use 0-based) # Build segments for this chapter (idx is 1-based, chunks use 0-based)
segments = _build_segments(body_text, default_voice, request, chapter_index=idx - 1) segments = _build_segments(body_text, default_voice, request, chapter_index=idx - 1)
+1 -1
View File
@@ -112,7 +112,7 @@ def build_conversion_request_from_thread(thread: Any) -> ConversionRequest:
read_title_intro=getattr(thread, "read_title_intro", False), read_title_intro=getattr(thread, "read_title_intro", False),
read_closing_outro=getattr(thread, "read_closing_outro", True), read_closing_outro=getattr(thread, "read_closing_outro", True),
auto_prefix_chapter_titles=getattr(thread, "auto_prefix_chapter_titles", True), auto_prefix_chapter_titles=getattr(thread, "auto_prefix_chapter_titles", True),
normalize_chapter_opening_caps=getattr(thread, "normalize_chapter_opening_caps", False), normalize_chapter_opening_caps=thread.normalize_chapter_opening_caps,
# Metadata # Metadata
metadata_tags=getattr(thread, "metadata_tags", {}) or {}, metadata_tags=getattr(thread, "metadata_tags", {}) or {},
# Artifacts # Artifacts
+1 -1
View File
@@ -474,7 +474,7 @@ def run_conversion_job(job: Job) -> None:
heading_text = spoken_title or raw_title heading_text = spoken_title or raw_title
chapter_display_title = heading_text or f"Chapter {idx}" chapter_display_title = heading_text or f"Chapter {idx}"
job.add_log(f"Processing chapter {idx}/{total_chapters}: {chapter_display_title}") job.add_log(f"Processing chapter {idx}/{total_chapters}: {chapter_display_title}")
normalize_opening_caps = bool(getattr(job, "normalize_chapter_opening_caps", True)) normalize_opening_caps = bool(job.normalize_chapter_opening_caps)
chapter_start_time = current_time chapter_start_time = current_time
chapter_override = ( chapter_override = (
+111
View File
@@ -511,3 +511,114 @@ class TestExecuteConversion:
assert result.metadata["title"] == "Test Book" assert result.metadata["title"] == "Test Book"
assert result.metadata["author"] == "Author" assert result.metadata["author"] == "Author"
class TestHeadingDedup:
"""Tests for heading dedup in executor."""
def test_heading_dedup_strips_matching_first_line(self):
"""When first segment matches heading, it should be stripped."""
with tempfile.TemporaryDirectory() as tmpdir:
req = ConversionRequest(
direct_text="Hello",
voice="M1",
auto_prefix_chapter_titles=True,
)
# Simulate: heading = "Chapter 1", first segment = "Chapter 1: The Beginning"
# headings_equivalent should match these
plan = ConversionPlan(
request=req,
metadata={},
chapters=[
ChapterPlan(
index=1,
title="Chapter 1",
original_title="Chapter 1",
body_text="Chapter 1: The Beginning\nBody text here",
segments=[
SegmentPlan(
text="Chapter 1: The Beginning",
voice_spec="M1",
kind="body",
source="chapter",
),
SegmentPlan(
text="Body text here",
voice_spec="M1",
kind="body",
source="chapter",
),
],
voice_spec="M1",
)
],
output_layout=OutputLayout(
parent_dir=Path(tmpdir),
audio_dir=Path(tmpdir),
),
)
events = FakeEvents()
pipeline = FakePipelineProvider()
resolver = FakeVoiceResolver()
tts_context = TTSContext()
result = execute_conversion(
plan, events, pipeline, resolver, tts_context
)
# The executor should have logged the heading
log_messages = [m for m, _ in events.logs if "Title:" in m]
assert len(log_messages) >= 1
def test_heading_dedup_no_match_preserves_all(self):
"""When first segment doesn't match heading, nothing is stripped."""
with tempfile.TemporaryDirectory() as tmpdir:
req = ConversionRequest(
direct_text="Hello",
voice="M1",
auto_prefix_chapter_titles=True,
)
plan = ConversionPlan(
request=req,
metadata={},
chapters=[
ChapterPlan(
index=1,
title="Chapter 1",
original_title="Chapter 1",
body_text="Completely different text\nMore text",
segments=[
SegmentPlan(
text="Completely different text",
voice_spec="M1",
kind="body",
source="chapter",
),
SegmentPlan(
text="More text",
voice_spec="M1",
kind="body",
source="chapter",
),
],
voice_spec="M1",
)
],
output_layout=OutputLayout(
parent_dir=Path(tmpdir),
audio_dir=Path(tmpdir),
),
)
events = FakeEvents()
pipeline = FakePipelineProvider()
resolver = FakeVoiceResolver()
tts_context = TTSContext()
result = execute_conversion(
plan, events, pipeline, resolver, tts_context
)
# Both segments should be synthesized (heading + 2 body segments)
assert result.total_segments >= 2
+28
View File
@@ -641,3 +641,31 @@ class TestFeatureParity:
if output_format.lower() == "m4b": if output_format.lower() == "m4b":
merge_chapters_at_end = True merge_chapters_at_end = True
assert merge_chapters_at_end is True assert merge_chapters_at_end is True
class TestCapsNormalization:
"""Tests for caps normalization in planner."""
def test_caps_normalization_applied_when_enabled(self):
"""When normalize_chapter_opening_caps=True, body text is normalized."""
req = ConversionRequest(
direct_text="<<CHAPTER_MARKER:Chapter 1>>\nALL CAPS OPENING TEXT here",
voice="M1",
normalize_chapter_opening_caps=True,
)
plan = build_conversion_plan(req)
body = plan.chapters[0].body_text
# ALL CAPS should be normalized to Title Case
assert body != "ALL CAPS OPENING TEXT here"
assert "ALL CAPS" not in body
def test_caps_normalization_skipped_when_disabled(self):
"""When normalize_chapter_opening_caps=False, body text is unchanged."""
req = ConversionRequest(
direct_text="<<CHAPTER_MARKER:Chapter 1>>\nALL CAPS OPENING TEXT here",
voice="M1",
normalize_chapter_opening_caps=False,
)
plan = build_conversion_plan(req)
body = plan.chapters[0].body_text
assert "ALL CAPS OPENING TEXT" in body