Merge pull request #212 from chiragjangiir/fix/unclosed-quote-duplication

fix(normalization): stop unclosed quotes from duplicating the text tail
This commit is contained in:
Deniz Şafak authored and GitHub committed 2026-10-15 07:23:37 -07:00
commit 9deda55296
2 files changed
+34

No files matched your search

+5
View File
@@ -1180,7 +1180,12 @@ def _normalize_all_caps_quotes(text: str) -> str:
cursor += 1
if cursor >= length:
# Unmatched opening mark: the remainder has no closing mark to
# delimit a passage, so emit it verbatim and consume it. Not
# advancing `index` here would make the tail block below append
# the same remainder a second time (#196).
builder.append(text[index:])
index = length
break
body = text[index + 1 : cursor]
+29
View File
@@ -179,6 +179,35 @@ def test_caps_quote_normalization_respects_override() -> None:
assert '"KEEP SHOUTING."' in cleaned
@pytest.mark.parametrize(
("text", "expected"),
[
# Unmatched opening marks: the tail has no closing mark to delimit a
# passage, so it must be emitted once, verbatim (#196).
("“Hello.", "“Hello."),
("«Bonjour.", "«Bonjour."),
("He said “hi”. Then “bye.", "He said “hi”. Then “bye."),
# `_cleanup_spacing` drops the space after the mark; the tail still
# must appear exactly once.
("A stray “ mark, and words.", "A stray “mark, and words."),
],
)
def test_unclosed_quote_tail_is_not_duplicated(text: str, expected: str) -> None:
# Regression test for #196: an unclosed quote used to duplicate the whole
# remainder of the text, and the duplicate was synthesized into the
# audiobook and written into the caption cues.
assert _normalize_text(text) == expected
def test_balanced_quote_before_unclosed_tail_is_still_normalized() -> None:
# A closed pair earlier in the text keeps its caps normalization, while
# the unclosed remainder after it is emitted once.
normalized = _normalize_text("“THE NASA TEAM ARRIVED.” Then “THE CREW waited.")
assert normalized == (
"“The NASA team arrived.” Then “THE CREW waited."
)
def test_recent_years_split_twenty_style() -> None:
normalized = _normalize_text("In 2025 we planned ahead")
folded = normalized.lower().replace("-", " ")