fix(normalization): stop unclosed quotes from duplicating the text tail

An opening quote without a matching closing mark made
_normalize_all_caps_quotes append the remainder twice: the unmatched
branch appends text[index:] and breaks without advancing index, so the
tail block below the loop appends the same remainder again. Because
emit_text normalizes every chunk before synthesis, the duplicate was
spoken aloud and written into the caption cues (issue #196). Sentence
and paragraph chunking routinely produce unclosed marks, so this hit
every quotation spanning multiple sentences.

Consume the remainder before breaking, and add regression tests that
cover the unmatched-remainder cases across quote styles plus a pair
that must still get caps normalization before an unclosed tail.

🤖 Generated with Codebuff
Co-Authored-By: Codebuff <noreply@codebuff.com>
This commit is contained in:
Chirag Jangir committed 2026-10-13 21:29:07 +05:30
1 parent 315840309f
commit 08b2477e1e
2 files changed
+34

No files matched your search

+5
View File
@@ -1180,7 +1180,12 @@ def _normalize_all_caps_quotes(text: str) -> str:
cursor += 1
if cursor >= length:
# Unmatched opening mark: the remainder has no closing mark to
# delimit a passage, so emit it verbatim and consume it. Not
# advancing `index` here would make the tail block below append
# the same remainder a second time (#196).
builder.append(text[index:])
index = length
break
body = text[index + 1 : cursor]
+29
View File
@@ -179,6 +179,35 @@ def test_caps_quote_normalization_respects_override() -> None:
assert '"KEEP SHOUTING."' in cleaned
@pytest.mark.parametrize(
("text", "expected"),
[
# Unmatched opening marks: the tail has no closing mark to delimit a
# passage, so it must be emitted once, verbatim (#196).
("“Hello.", "“Hello."),
("«Bonjour.", "«Bonjour."),
("He said “hi”. Then “bye.", "He said “hi”. Then “bye."),
# `_cleanup_spacing` drops the space after the mark; the tail still
# must appear exactly once.
("A stray “ mark, and words.", "A stray “mark, and words."),
],
)
def test_unclosed_quote_tail_is_not_duplicated(text: str, expected: str) -> None:
# Regression test for #196: an unclosed quote used to duplicate the whole
# remainder of the text, and the duplicate was synthesized into the
# audiobook and written into the caption cues.
assert _normalize_text(text) == expected
def test_balanced_quote_before_unclosed_tail_is_still_normalized() -> None:
# A closed pair earlier in the text keeps its caps normalization, while
# the unclosed remainder after it is emitted once.
normalized = _normalize_text("“THE NASA TEAM ARRIVED.” Then “THE CREW waited.")
assert normalized == (
"“The NASA team arrived.” Then “THE CREW waited."
)
def test_recent_years_split_twenty_style() -> None:
normalized = _normalize_text("In 2025 we planned ahead")
folded = normalized.lower().replace("-", " ")