mirror of
https://github.com/denizsafak/abogen.git
synced 2026-09-20 11:40:57 +02:00
refactor: heading transforms + dedup moved to shared layer
- conversion_planner.py: caps normalization in _build_chapters() - conversion_executor.py: heading dedup + state machine via headings_equivalent() - Fixed seg_start_time → chapter_body_start bug in executor - Removed getattr fallback defaults in both adapters - Added 4 tests for caps normalization and heading dedup - Updated ARCHITECTURE_REFACTOR_PLAN.md with deferred WebUI cleanup
This commit is contained in:
@@ -32,6 +32,10 @@ from abogen.domain.conversion_engine import (
|
|||||||
)
|
)
|
||||||
from abogen.domain.enums import OutputFormat, SubtitleMode
|
from abogen.domain.enums import OutputFormat, SubtitleMode
|
||||||
from abogen.domain.normalization import TTSContext
|
from abogen.domain.normalization import TTSContext
|
||||||
|
from abogen.domain.chapter_titles import (
|
||||||
|
apply_chapter_text_transforms,
|
||||||
|
headings_equivalent as _headings_equivalent,
|
||||||
|
)
|
||||||
from abogen.domain.output_paths import sanitize_filename_for_chapter
|
from abogen.domain.output_paths import sanitize_filename_for_chapter
|
||||||
from abogen.infrastructure.subtitle_writer import make_subtitle_writer
|
from abogen.infrastructure.subtitle_writer import make_subtitle_writer
|
||||||
|
|
||||||
@@ -249,6 +253,7 @@ def execute_conversion(
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Process heading
|
# Process heading
|
||||||
|
heading_text = ""
|
||||||
if chapter.title:
|
if chapter.title:
|
||||||
heading_text = _format_heading(chapter.title, chapter_idx, request)
|
heading_text = _format_heading(chapter.title, chapter_idx, request)
|
||||||
if heading_text:
|
if heading_text:
|
||||||
@@ -269,11 +274,37 @@ def execute_conversion(
|
|||||||
stats=stats,
|
stats=stats,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Heading dedup: check if first line of body matches heading
|
||||||
|
pending_heading_strip = False
|
||||||
|
if heading_text and chapter.body_text:
|
||||||
|
first_line = next(
|
||||||
|
(line.strip() for line in chapter.body_text.splitlines() if line.strip()),
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
if first_line and _headings_equivalent(first_line, heading_text):
|
||||||
|
pending_heading_strip = True
|
||||||
|
|
||||||
# Process body segments
|
# Process body segments
|
||||||
chapter_chunk_markers: List[Dict[str, Any]] = []
|
chapter_chunk_markers: List[Dict[str, Any]] = []
|
||||||
|
chapter_body_start = stats.current_time
|
||||||
for seg_idx, segment in enumerate(chapter.segments):
|
for seg_idx, segment in enumerate(chapter.segments):
|
||||||
check_cancelled()
|
check_cancelled()
|
||||||
|
|
||||||
|
# Apply heading dedup to first segment (consume-once)
|
||||||
|
seg_text = segment.text
|
||||||
|
if pending_heading_strip and seg_text.strip():
|
||||||
|
seg_text, heading_removed, _ = apply_chapter_text_transforms(
|
||||||
|
seg_text,
|
||||||
|
heading_text=heading_text,
|
||||||
|
raw_title=chapter.title,
|
||||||
|
strip_heading=True,
|
||||||
|
normalize_caps=False,
|
||||||
|
)
|
||||||
|
if heading_removed:
|
||||||
|
pending_heading_strip = False
|
||||||
|
if not seg_text.strip():
|
||||||
|
continue
|
||||||
|
|
||||||
# Resolve segment voice (may differ from chapter voice)
|
# Resolve segment voice (may differ from chapter voice)
|
||||||
if segment.voice_spec != chapter.voice_spec:
|
if segment.voice_spec != chapter.voice_spec:
|
||||||
seg_provider, seg_voice, seg_speed, seg_steps = _resolve_voice(
|
seg_provider, seg_voice, seg_speed, seg_steps = _resolve_voice(
|
||||||
@@ -288,7 +319,7 @@ def execute_conversion(
|
|||||||
|
|
||||||
seg_start_time = stats.current_time
|
seg_start_time = stats.current_time
|
||||||
local_segments, accumulated_tokens = synthesize_text(
|
local_segments, accumulated_tokens = synthesize_text(
|
||||||
text=segment.text,
|
text=seg_text,
|
||||||
params=synth,
|
params=synth,
|
||||||
backend=seg_backend,
|
backend=seg_backend,
|
||||||
voice=seg_voice,
|
voice=seg_voice,
|
||||||
@@ -355,7 +386,7 @@ def execute_conversion(
|
|||||||
result.chapter_markers.append({
|
result.chapter_markers.append({
|
||||||
"chapter_index": chapter_idx - 1,
|
"chapter_index": chapter_idx - 1,
|
||||||
"title": chapter.title,
|
"title": chapter.title,
|
||||||
"start": stats.current_time - (stats.current_time - seg_start_time) if chapter.segments else stats.current_time,
|
"start": chapter_body_start,
|
||||||
"end": stats.current_time,
|
"end": stats.current_time,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|||||||
@@ -210,9 +210,15 @@ def _build_chapters(
|
|||||||
selected_chapters: List[Tuple[str, str, str]], request: ConversionRequest
|
selected_chapters: List[Tuple[str, str, str]], request: ConversionRequest
|
||||||
) -> List[ChapterPlan]:
|
) -> List[ChapterPlan]:
|
||||||
"""Build ChapterPlan with SegmentPlan for each chapter."""
|
"""Build ChapterPlan with SegmentPlan for each chapter."""
|
||||||
|
from abogen.domain.chapter_titles import normalize_chapter_opening_caps
|
||||||
|
|
||||||
chapters = []
|
chapters = []
|
||||||
|
|
||||||
for idx, (title, body_text, default_voice) in enumerate(selected_chapters, 1):
|
for idx, (title, body_text, default_voice) in enumerate(selected_chapters, 1):
|
||||||
|
# Apply caps normalization to body text if enabled
|
||||||
|
if request.normalize_chapter_opening_caps and body_text:
|
||||||
|
body_text, _ = normalize_chapter_opening_caps(body_text)
|
||||||
|
|
||||||
# Build segments for this chapter (idx is 1-based, chunks use 0-based)
|
# Build segments for this chapter (idx is 1-based, chunks use 0-based)
|
||||||
segments = _build_segments(body_text, default_voice, request, chapter_index=idx - 1)
|
segments = _build_segments(body_text, default_voice, request, chapter_index=idx - 1)
|
||||||
|
|
||||||
|
|||||||
@@ -112,7 +112,7 @@ def build_conversion_request_from_thread(thread: Any) -> ConversionRequest:
|
|||||||
read_title_intro=getattr(thread, "read_title_intro", False),
|
read_title_intro=getattr(thread, "read_title_intro", False),
|
||||||
read_closing_outro=getattr(thread, "read_closing_outro", True),
|
read_closing_outro=getattr(thread, "read_closing_outro", True),
|
||||||
auto_prefix_chapter_titles=getattr(thread, "auto_prefix_chapter_titles", True),
|
auto_prefix_chapter_titles=getattr(thread, "auto_prefix_chapter_titles", True),
|
||||||
normalize_chapter_opening_caps=getattr(thread, "normalize_chapter_opening_caps", False),
|
normalize_chapter_opening_caps=thread.normalize_chapter_opening_caps,
|
||||||
# Metadata
|
# Metadata
|
||||||
metadata_tags=getattr(thread, "metadata_tags", {}) or {},
|
metadata_tags=getattr(thread, "metadata_tags", {}) or {},
|
||||||
# Artifacts
|
# Artifacts
|
||||||
|
|||||||
@@ -474,7 +474,7 @@ def run_conversion_job(job: Job) -> None:
|
|||||||
heading_text = spoken_title or raw_title
|
heading_text = spoken_title or raw_title
|
||||||
chapter_display_title = heading_text or f"Chapter {idx}"
|
chapter_display_title = heading_text or f"Chapter {idx}"
|
||||||
job.add_log(f"Processing chapter {idx}/{total_chapters}: {chapter_display_title}")
|
job.add_log(f"Processing chapter {idx}/{total_chapters}: {chapter_display_title}")
|
||||||
normalize_opening_caps = bool(getattr(job, "normalize_chapter_opening_caps", True))
|
normalize_opening_caps = bool(job.normalize_chapter_opening_caps)
|
||||||
|
|
||||||
chapter_start_time = current_time
|
chapter_start_time = current_time
|
||||||
chapter_override = (
|
chapter_override = (
|
||||||
|
|||||||
@@ -511,3 +511,114 @@ class TestExecuteConversion:
|
|||||||
|
|
||||||
assert result.metadata["title"] == "Test Book"
|
assert result.metadata["title"] == "Test Book"
|
||||||
assert result.metadata["author"] == "Author"
|
assert result.metadata["author"] == "Author"
|
||||||
|
|
||||||
|
|
||||||
|
class TestHeadingDedup:
|
||||||
|
"""Tests for heading dedup in executor."""
|
||||||
|
|
||||||
|
def test_heading_dedup_strips_matching_first_line(self):
|
||||||
|
"""When first segment matches heading, it should be stripped."""
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
req = ConversionRequest(
|
||||||
|
direct_text="Hello",
|
||||||
|
voice="M1",
|
||||||
|
auto_prefix_chapter_titles=True,
|
||||||
|
)
|
||||||
|
# Simulate: heading = "Chapter 1", first segment = "Chapter 1: The Beginning"
|
||||||
|
# headings_equivalent should match these
|
||||||
|
plan = ConversionPlan(
|
||||||
|
request=req,
|
||||||
|
metadata={},
|
||||||
|
chapters=[
|
||||||
|
ChapterPlan(
|
||||||
|
index=1,
|
||||||
|
title="Chapter 1",
|
||||||
|
original_title="Chapter 1",
|
||||||
|
body_text="Chapter 1: The Beginning\nBody text here",
|
||||||
|
segments=[
|
||||||
|
SegmentPlan(
|
||||||
|
text="Chapter 1: The Beginning",
|
||||||
|
voice_spec="M1",
|
||||||
|
kind="body",
|
||||||
|
source="chapter",
|
||||||
|
),
|
||||||
|
SegmentPlan(
|
||||||
|
text="Body text here",
|
||||||
|
voice_spec="M1",
|
||||||
|
kind="body",
|
||||||
|
source="chapter",
|
||||||
|
),
|
||||||
|
],
|
||||||
|
voice_spec="M1",
|
||||||
|
)
|
||||||
|
],
|
||||||
|
output_layout=OutputLayout(
|
||||||
|
parent_dir=Path(tmpdir),
|
||||||
|
audio_dir=Path(tmpdir),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
events = FakeEvents()
|
||||||
|
pipeline = FakePipelineProvider()
|
||||||
|
resolver = FakeVoiceResolver()
|
||||||
|
tts_context = TTSContext()
|
||||||
|
|
||||||
|
result = execute_conversion(
|
||||||
|
plan, events, pipeline, resolver, tts_context
|
||||||
|
)
|
||||||
|
|
||||||
|
# The executor should have logged the heading
|
||||||
|
log_messages = [m for m, _ in events.logs if "Title:" in m]
|
||||||
|
assert len(log_messages) >= 1
|
||||||
|
|
||||||
|
def test_heading_dedup_no_match_preserves_all(self):
|
||||||
|
"""When first segment doesn't match heading, nothing is stripped."""
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
req = ConversionRequest(
|
||||||
|
direct_text="Hello",
|
||||||
|
voice="M1",
|
||||||
|
auto_prefix_chapter_titles=True,
|
||||||
|
)
|
||||||
|
plan = ConversionPlan(
|
||||||
|
request=req,
|
||||||
|
metadata={},
|
||||||
|
chapters=[
|
||||||
|
ChapterPlan(
|
||||||
|
index=1,
|
||||||
|
title="Chapter 1",
|
||||||
|
original_title="Chapter 1",
|
||||||
|
body_text="Completely different text\nMore text",
|
||||||
|
segments=[
|
||||||
|
SegmentPlan(
|
||||||
|
text="Completely different text",
|
||||||
|
voice_spec="M1",
|
||||||
|
kind="body",
|
||||||
|
source="chapter",
|
||||||
|
),
|
||||||
|
SegmentPlan(
|
||||||
|
text="More text",
|
||||||
|
voice_spec="M1",
|
||||||
|
kind="body",
|
||||||
|
source="chapter",
|
||||||
|
),
|
||||||
|
],
|
||||||
|
voice_spec="M1",
|
||||||
|
)
|
||||||
|
],
|
||||||
|
output_layout=OutputLayout(
|
||||||
|
parent_dir=Path(tmpdir),
|
||||||
|
audio_dir=Path(tmpdir),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
events = FakeEvents()
|
||||||
|
pipeline = FakePipelineProvider()
|
||||||
|
resolver = FakeVoiceResolver()
|
||||||
|
tts_context = TTSContext()
|
||||||
|
|
||||||
|
result = execute_conversion(
|
||||||
|
plan, events, pipeline, resolver, tts_context
|
||||||
|
)
|
||||||
|
|
||||||
|
# Both segments should be synthesized (heading + 2 body segments)
|
||||||
|
assert result.total_segments >= 2
|
||||||
|
|||||||
@@ -641,3 +641,31 @@ class TestFeatureParity:
|
|||||||
if output_format.lower() == "m4b":
|
if output_format.lower() == "m4b":
|
||||||
merge_chapters_at_end = True
|
merge_chapters_at_end = True
|
||||||
assert merge_chapters_at_end is True
|
assert merge_chapters_at_end is True
|
||||||
|
|
||||||
|
|
||||||
|
class TestCapsNormalization:
|
||||||
|
"""Tests for caps normalization in planner."""
|
||||||
|
|
||||||
|
def test_caps_normalization_applied_when_enabled(self):
|
||||||
|
"""When normalize_chapter_opening_caps=True, body text is normalized."""
|
||||||
|
req = ConversionRequest(
|
||||||
|
direct_text="<<CHAPTER_MARKER:Chapter 1>>\nALL CAPS OPENING TEXT here",
|
||||||
|
voice="M1",
|
||||||
|
normalize_chapter_opening_caps=True,
|
||||||
|
)
|
||||||
|
plan = build_conversion_plan(req)
|
||||||
|
body = plan.chapters[0].body_text
|
||||||
|
# ALL CAPS should be normalized to Title Case
|
||||||
|
assert body != "ALL CAPS OPENING TEXT here"
|
||||||
|
assert "ALL CAPS" not in body
|
||||||
|
|
||||||
|
def test_caps_normalization_skipped_when_disabled(self):
|
||||||
|
"""When normalize_chapter_opening_caps=False, body text is unchanged."""
|
||||||
|
req = ConversionRequest(
|
||||||
|
direct_text="<<CHAPTER_MARKER:Chapter 1>>\nALL CAPS OPENING TEXT here",
|
||||||
|
voice="M1",
|
||||||
|
normalize_chapter_opening_caps=False,
|
||||||
|
)
|
||||||
|
plan = build_conversion_plan(req)
|
||||||
|
body = plan.chapters[0].body_text
|
||||||
|
assert "ALL CAPS OPENING TEXT" in body
|
||||||
|
|||||||
Reference in New Issue
Block a user