mirror of
https://github.com/denizsafak/abogen.git
synced 2026-11-19 01:00:54 +01:00
fix(normalization): stop unclosed quotes from duplicating the text tail
An opening quote without a matching closing mark made _normalize_all_caps_quotes append the remainder twice: the unmatched branch appends text[index:] and breaks without advancing index, so the tail block below the loop appends the same remainder again. Because emit_text normalizes every chunk before synthesis, the duplicate was spoken aloud and written into the caption cues (issue #196). Sentence and paragraph chunking routinely produce unclosed marks, so this hit every quotation spanning multiple sentences. Consume the remainder before breaking, and add regression tests that cover the unmatched-remainder cases across quote styles plus a pair that must still get caps normalization before an unclosed tail. 🤖 Generated with Codebuff Co-Authored-By: Codebuff <noreply@codebuff.com>
This commit is contained in:
1 parent
315840309f
commit
08b2477e1e
2 files changed
+34
No files matched your search
@@ -1180,7 +1180,12 @@ def _normalize_all_caps_quotes(text: str) -> str:
|
||||
cursor += 1
|
||||
|
||||
if cursor >= length:
|
||||
# Unmatched opening mark: the remainder has no closing mark to
|
||||
# delimit a passage, so emit it verbatim and consume it. Not
|
||||
# advancing `index` here would make the tail block below append
|
||||
# the same remainder a second time (#196).
|
||||
builder.append(text[index:])
|
||||
index = length
|
||||
break
|
||||
|
||||
body = text[index + 1 : cursor]
|
||||
|
||||
@@ -179,6 +179,35 @@ def test_caps_quote_normalization_respects_override() -> None:
|
||||
assert '"KEEP SHOUTING."' in cleaned
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "expected"),
|
||||
[
|
||||
# Unmatched opening marks: the tail has no closing mark to delimit a
|
||||
# passage, so it must be emitted once, verbatim (#196).
|
||||
("“Hello.", "“Hello."),
|
||||
("«Bonjour.", "«Bonjour."),
|
||||
("He said “hi”. Then “bye.", "He said “hi”. Then “bye."),
|
||||
# `_cleanup_spacing` drops the space after the mark; the tail still
|
||||
# must appear exactly once.
|
||||
("A stray “ mark, and words.", "A stray “mark, and words."),
|
||||
],
|
||||
)
|
||||
def test_unclosed_quote_tail_is_not_duplicated(text: str, expected: str) -> None:
|
||||
# Regression test for #196: an unclosed quote used to duplicate the whole
|
||||
# remainder of the text, and the duplicate was synthesized into the
|
||||
# audiobook and written into the caption cues.
|
||||
assert _normalize_text(text) == expected
|
||||
|
||||
|
||||
def test_balanced_quote_before_unclosed_tail_is_still_normalized() -> None:
|
||||
# A closed pair earlier in the text keeps its caps normalization, while
|
||||
# the unclosed remainder after it is emitted once.
|
||||
normalized = _normalize_text("“THE NASA TEAM ARRIVED.” Then “THE CREW waited.")
|
||||
assert normalized == (
|
||||
"“The NASA team arrived.” Then “THE CREW waited."
|
||||
)
|
||||
|
||||
|
||||
def test_recent_years_split_twenty_style() -> None:
|
||||
normalized = _normalize_text("In 2025 we planned ahead")
|
||||
folded = normalized.lower().replace("-", " ")
|
||||
|
||||
Reference in new issue
Block a user