feat: Enhance text normalization and chunking logic to preserve original whitespace and handle abbreviations

2026-07-18 13:40:27 +02:00 · 2025-10-10 08:38:00 -07:00
parent 258c3549f7
commit 3a91e79cb6
9 changed files with 251 additions and 71 deletions
@@ -104,4 +104,68 @@ def test_build_epub3_package_handles_missing_markers(tmp_path) -> None:
        nav_doc = archive.read("OEBPS/nav.xhtml").decode("utf-8")
        assert "Chapter 1" in nav_doc
        chapter_doc = archive.read("OEBPS/text/chapter_0001.xhtml").decode("utf-8")
-        assert "Hello world." in chapter_doc
+        assert "Hello world." in chapter_doc
+
+
+def test_epub3_preserves_original_whitespace(tmp_path) -> None:
+    extraction = ExtractionResult(
+        chapters=[
+            ExtractedChapter(
+                title="Intro",
+                text="Line one  with  double spaces.\nSecond line\n\nThird paragraph.",
+            )
+        ],
+        metadata={"title": "Sample", "artist": "Author", "language": "en"},
+    )
+
+    chunks = [
+        {
+            "id": "chap0000_p0000",
+            "chapter_index": 0,
+            "chunk_index": 0,
+            "text": "Line one with double spaces.",
+            "speaker_id": "narrator",
+        },
+        {
+            "id": "chap0000_p0001",
+            "chapter_index": 0,
+            "chunk_index": 1,
+            "text": "Second line",
+            "speaker_id": "narrator",
+        },
+        {
+            "id": "chap0000_p0002",
+            "chapter_index": 0,
+            "chunk_index": 2,
+            "text": "Third paragraph.",
+            "speaker_id": "narrator",
+        },
+    ]
+
+    chunk_markers = [
+        {"id": chunk["id"], "chapter_index": 0, "chunk_index": chunk["chunk_index"], "start": None, "end": None}
+        for chunk in chunks
+    ]
+
+    metadata_tags = {"title": "Sample", "artist": "Author", "language": "en"}
+    audio_path = tmp_path / "audio.mp3"
+    audio_path.write_bytes(b"ID3 audio")
+    output_path = tmp_path / "output.epub"
+
+    build_epub3_package(
+        output_path=output_path,
+        book_id="job-whitespace",
+        extraction=extraction,
+        metadata_tags=metadata_tags,
+        chapter_markers=[],
+        chunk_markers=chunk_markers,
+        chunks=chunks,
+        audio_path=audio_path,
+        speaker_mode="single",
+    )
+
+    with zipfile.ZipFile(output_path) as archive:
+        chapter_doc = archive.read("OEBPS/text/chapter_0001.xhtml").decode("utf-8")
+    assert "Line one  with  double spaces." in chapter_doc
+    normalized = chapter_doc.replace("        ", "")
+    assert "Second line\n\nThird paragraph." in normalized