feat: Enhance text normalization with support for internet slang expansion, currency formatting, and date handling; update debug WAVs interface and settings

This commit is contained in:
JB
2025-12-15 17:27:08 -08:00
parent 0afaaf561b
commit 015435adb6
10 changed files with 472 additions and 74 deletions
+1 -1
View File
@@ -182,7 +182,7 @@ DEBUG_TTS_SAMPLES: Sequence[DebugTTSSample] = (
DebugTTSSample( DebugTTSSample(
code="TITLE_003", code="TITLE_003",
label="Titles and abbreviations (3)", label="Titles and abbreviations (3)",
text="Gen. Lee spoke to Sgt. Rivera on Main St.", text="Gen. Smith spoke to Sgt. Rivera on Main St.",
), ),
DebugTTSSample( DebugTTSSample(
code="TITLE_004", code="TITLE_004",
+12 -1
View File
@@ -5,7 +5,10 @@ import re
from dataclasses import dataclass from dataclasses import dataclass
from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple
import spacy try: # pragma: no cover - optional dependency
import spacy # type: ignore
except Exception: # pragma: no cover - spaCy may be unavailable in minimal environments
spacy = None
@dataclass(frozen=True) @dataclass(frozen=True)
@@ -177,6 +180,9 @@ def _build_replacement_sentence(sentence: str, token: str, replacement_token: st
def _load_spacy(language: str) -> Any: def _load_spacy(language: str) -> Any:
if spacy is None:
return None
# English only for now. # English only for now.
# Use installed small model; keep it simple. # Use installed small model; keep it simple.
lang = (language or "en").lower() lang = (language or "en").lower()
@@ -211,7 +217,12 @@ def extract_heteronym_overrides(
if not lang.startswith("en"): if not lang.startswith("en"):
return [] return []
if spacy is None:
return []
nlp = _load_spacy(lang) nlp = _load_spacy(lang)
if nlp is None:
return []
previous_choices: Dict[str, str] = {} previous_choices: Dict[str, str] = {}
if existing: if existing:
+313 -23
View File
@@ -3,6 +3,8 @@ from __future__ import annotations
import json import json
import re import re
import unicodedata import unicodedata
import os
import locale
from fractions import Fraction from fractions import Fraction
from dataclasses import dataclass, field from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple
@@ -129,6 +131,176 @@ _URL_RE = re.compile(r"(https?://)?(www\.)?(?P<domain>[a-zA-Z0-9-]+(\.[a-zA-Z0-9
_FOOTNOTE_RE = re.compile(r"([a-zA-Z]+)(\d+)") _FOOTNOTE_RE = re.compile(r"([a-zA-Z]+)(\d+)")
_BRACKET_FOOTNOTE_RE = re.compile(r"\[\d+\]") _BRACKET_FOOTNOTE_RE = re.compile(r"\[\d+\]")
_ISO_DATE_RE = re.compile(r"\b(?P<year>\d{4})[/-](?P<month>\d{1,2})[/-](?P<day>\d{1,2})\b")
_MDY_DATE_RE = re.compile(
r"\b(?P<month>Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\.?\s+"
r"(?P<day>\d{1,2})(?:st|nd|rd|th)?\s*,\s*(?P<year>\d{4})\b",
re.IGNORECASE,
)
_TIME_RE = re.compile(
r"\b(?P<hour>\d{1,2})(?::(?P<minute>\d{2}))?\s*(?P<meridian>a\.?m\.?|p\.?m\.?)\b",
re.IGNORECASE,
)
_ADDRESS_ABBR_RE = re.compile(r"(?P<prefix>\b\w+\s+)(?P<abbr>St|Rd|Ave|Blvd|Ln|Dr)\.(?=\s*(?:,|\.|!|\?|$))")
_MONTH_MAP = {
"jan": "January",
"january": "January",
"feb": "February",
"february": "February",
"mar": "March",
"march": "March",
"apr": "April",
"april": "April",
"may": "May",
"jun": "June",
"june": "June",
"jul": "July",
"july": "July",
"aug": "August",
"august": "August",
"sep": "September",
"sept": "September",
"september": "September",
"oct": "October",
"october": "October",
"nov": "November",
"november": "November",
"dec": "December",
"december": "December",
}
def _is_us_locale() -> bool:
for key in ("LC_ALL", "LC_TIME", "LANG"):
value = os.environ.get(key)
if value and "en_US" in value:
return True
try:
loc = locale.getlocale(locale.LC_TIME)
if loc and loc[0] and "en_US" in loc[0]:
return True
except Exception:
pass
return False
def _year_to_words_american(value: int, language: str) -> str:
if language.lower().startswith("en") and 2000 <= value <= 2099:
if value == 2000:
return "two thousand"
if 2001 <= value <= 2009:
tail = _DIGIT_WORDS[value % 10]
return f"two thousand {tail}"
first_two = value // 100
last_two = value % 100
first_words = _int_to_words(first_two, language) or "twenty"
if last_two == 0:
return f"{first_words} hundred"
if last_two < 10:
return f"{first_words} oh {_DIGIT_WORDS[last_two]}"
last_words = _int_to_words(last_two, language)
return f"{first_words} {last_words or last_two}"
words = _int_to_words(value, language)
return words or str(value)
def _normalize_dates(text: str, language: str) -> str:
us = _is_us_locale()
def _format_iso(match: re.Match[str]) -> str:
year = int(match.group("year"))
month = int(match.group("month"))
day = int(match.group("day"))
if not (1 <= month <= 12 and 1 <= day <= 31):
return match.group(0)
month_name = (
[
"January",
"February",
"March",
"April",
"May",
"June",
"July",
"August",
"September",
"October",
"November",
"December",
][month - 1]
)
ordinal = _int_to_ordinal_words(day, language) or str(day)
year_words = _year_to_words_american(year, language)
return f"{month_name} {ordinal}, {year_words}" if us else f"{ordinal} {month_name} {year_words}"
def _format_mdy(match: re.Match[str]) -> str:
month_raw = str(match.group("month") or "").strip().lower().rstrip(".")
month_name = _MONTH_MAP.get(month_raw)
if not month_name:
return match.group(0)
day = int(match.group("day"))
year = int(match.group("year"))
ordinal = _int_to_ordinal_words(day, language) or str(day)
year_words = _year_to_words_american(year, language)
return f"{month_name} {ordinal}, {year_words}" if us else f"{ordinal} {month_name} {year_words}"
out = _ISO_DATE_RE.sub(_format_iso, text)
out = _MDY_DATE_RE.sub(_format_mdy, out)
return out
def _normalize_times(text: str) -> str:
def _replace(match: re.Match[str]) -> str:
hour = match.group("hour")
minute = match.group("minute")
meridian = match.group("meridian").lower().replace(".", "")
if minute:
return f"{hour}:{minute} {meridian}"
return f"{hour} {meridian}"
return _TIME_RE.sub(_replace, text)
def _normalize_address_abbreviations(text: str) -> str:
mapping = {
"st": "street",
"rd": "road",
"ave": "avenue",
"blvd": "boulevard",
"ln": "lane",
"dr": "drive",
}
def _replace(match: re.Match[str]) -> str:
abbr = match.group("abbr")
full = mapping.get(abbr.lower())
if not full:
return match.group(0)
return match.group("prefix") + _match_casing(abbr, full)
return _ADDRESS_ABBR_RE.sub(_replace, text)
def _normalize_internet_slang(text: str) -> str:
mapping = {
"pls": "please",
"plz": "please",
}
def _replace(match: re.Match[str]) -> str:
token = match.group(0)
replacement = mapping.get(token.lower())
if not replacement:
return token
return _match_casing(token, replacement)
return re.sub(r"\b(?:pls|plz)\b", _replace, text, flags=re.IGNORECASE)
_DECIMAL_NUMBER_RE = re.compile( _DECIMAL_NUMBER_RE = re.compile(
rf"(?<![\w{_NUMBER_RANGE_CLASS}/])(?P<number>-?(?:\d{{1,3}}(?:,\d{{3}})+|\d+)\.(?P<fraction>\d+))(?![\w{_NUMBER_RANGE_CLASS}/])" rf"(?<![\w{_NUMBER_RANGE_CLASS}/])(?P<number>-?(?:\d{{1,3}}(?:,\d{{3}})+|\d+)\.(?P<fraction>\d+))(?![\w{_NUMBER_RANGE_CLASS}/])"
) )
@@ -155,13 +327,49 @@ def _int_to_words(value: int, language: str) -> Optional[str]:
def _int_to_ordinal_words(value: int, language: str) -> Optional[str]: def _int_to_ordinal_words(value: int, language: str) -> Optional[str]:
if num2words is None: if num2words is not None:
return None try:
return num2words(value, lang=language, ordinal=True)
except Exception: # pragma: no cover - unsupported locale
return None
try: if language.lower().startswith("en"):
return num2words(value, lang=language, ordinal=True) ordinals = {
except Exception: # pragma: no cover - unsupported locale 1: "first",
return None 2: "second",
3: "third",
4: "fourth",
5: "fifth",
6: "sixth",
7: "seventh",
8: "eighth",
9: "ninth",
10: "tenth",
11: "eleventh",
12: "twelfth",
13: "thirteenth",
14: "fourteenth",
15: "fifteenth",
16: "sixteenth",
17: "seventeenth",
18: "eighteenth",
19: "nineteenth",
20: "twentieth",
21: "twenty-first",
22: "twenty-second",
23: "twenty-third",
24: "twenty-fourth",
25: "twenty-fifth",
26: "twenty-sixth",
27: "twenty-seventh",
28: "twenty-eighth",
29: "twenty-ninth",
30: "thirtieth",
31: "thirty-first",
}
return ordinals.get(int(value))
return None
def _pluralize_fraction_word(base: str) -> str: def _pluralize_fraction_word(base: str) -> str:
@@ -366,6 +574,8 @@ TITLE_ABBREVIATIONS = {
"dr": "doctor", "dr": "doctor",
"prof": "professor", "prof": "professor",
"rev": "reverend", "rev": "reverend",
"gen": "general",
"sgt": "sergeant",
} }
SUFFIX_ABBREVIATIONS = { SUFFIX_ABBREVIATIONS = {
@@ -1146,7 +1356,23 @@ def classify_token(token: str, cfg: ApostropheConfig) -> Tuple[str, str]:
# 1. Decades # 1. Decades
if DECADE_RE.match(token): if DECADE_RE.match(token):
if cfg.decades_mode == "expand": if cfg.decades_mode == "expand":
return "decade", f"19{token[2:4]}s" decade_digit = token[1]
decade_map = {
"0": "two thousands",
"1": "tens",
"2": "twenties",
"3": "thirties",
"4": "forties",
"5": "fifties",
"6": "sixties",
"7": "seventies",
"8": "eighties",
"9": "nineties",
}
spoken = decade_map.get(decade_digit)
if spoken:
return "decade", spoken
return "decade", token
return "decade", token return "decade", token
# 2. Leading elision # 2. Leading elision
@@ -1312,7 +1538,7 @@ def normalize_apostrophes(text: str, cfg: ApostropheConfig | None = None) -> Tup
return normalized_text, results return normalized_text, results
def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str: def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
if not text or not cfg.convert_numbers: if not text:
return text return text
language = (cfg.number_lang or "en").strip() or "en" language = (cfg.number_lang or "en").strip() or "en"
@@ -1360,6 +1586,20 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
return words.replace(" and ", " ") return words.replace(" and ", " ")
return None return None
# 1200s are commonly spoken as "twelve hundred".
if 1200 <= value < 1300:
hundreds = value // 100
remainder = value % 100
prefix = _words(hundreds)
if not prefix:
return None
if remainder == 0:
return f"{prefix} hundred"
tail = _format_year_tail(remainder, allow_oh=True)
if tail is None:
return None
return f"{prefix} hundred {tail}".strip()
if value % 1000 == 0: if value % 1000 == 0:
thousands = value // 1000 thousands = value // 1000
thousands_words = _words(thousands) thousands_words = _words(thousands)
@@ -1537,6 +1777,29 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
return f"{amount_spoken} {magnitude} {currency_name}" return f"{amount_spoken} {magnitude} {currency_name}"
# Handle $0.99 -> ninety-nine cents (avoid "zero dollars and...").
if amount_str.startswith("0") and amount < 1.0:
dollars_part, dot, fraction = amount_str.partition(".")
if dot and fraction:
cents_str = (fraction + "00")[:2]
try:
cents_value = int(cents_str)
except ValueError:
cents_value = 0
if cents_value > 0:
cents_words = _int_to_words(cents_value, language) or str(cents_value)
subunit = {
"$": "cent",
"": "cent",
"£": "penny",
"¥": "yen",
}.get(symbol, "cent")
if symbol == "£":
subunit = "pence" if cents_value != 1 else "penny"
else:
subunit = (subunit + "s") if cents_value != 1 and subunit not in {"pence", "yen"} else subunit
return f"{cents_words} {subunit}".strip()
currency_map = { currency_map = {
"$": "USD", "$": "USD",
"£": "GBP", "£": "GBP",
@@ -1559,6 +1822,13 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
def _replace_url(match: re.Match[str]) -> str: def _replace_url(match: re.Match[str]) -> str:
domain = match.group("domain") domain = match.group("domain")
# Avoid matching dotted abbreviations/acronyms without an explicit URL prefix.
has_prefix = match.group(1) or match.group(2)
if not has_prefix:
parts = [p for p in domain.split(".") if p]
if parts and all(p.isalpha() and len(p) <= 2 for p in parts):
return match.group(0)
# Avoid matching numbers like 1.05 or 12.34.56 # Avoid matching numbers like 1.05 or 12.34.56
# If the domain consists only of digits and dots, ignore it (unless it has http/www prefix) # If the domain consists only of digits and dots, ignore it (unless it has http/www prefix)
@@ -1583,24 +1853,34 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
return match.group(1) return match.group(1)
normalized = text normalized = text
# Apply URL replacement first # Apply URL replacement first (independent of number conversion).
normalized = _URL_RE.sub(_replace_url, normalized) normalized = _URL_RE.sub(_replace_url, normalized)
if getattr(cfg, "remove_footnotes", False): if getattr(cfg, "remove_footnotes", False):
normalized = _FOOTNOTE_RE.sub(_remove_footnote, normalized) normalized = _FOOTNOTE_RE.sub(_remove_footnote, normalized)
normalized = _BRACKET_FOOTNOTE_RE.sub("", normalized) normalized = _BRACKET_FOOTNOTE_RE.sub("", normalized)
if cfg.convert_currency: if cfg.convert_currency:
normalized = _CURRENCY_RE.sub(_replace_currency, normalized) normalized = _CURRENCY_RE.sub(_replace_currency, normalized)
normalized = _NUMBER_RANGE_RE.sub(lambda m: _replace_number_range(m, language), normalized)
normalized = _NUMBER_SPACE_RANGE_RE.sub(lambda m: _replace_space_separated_range(m, language), normalized) if cfg.convert_numbers:
normalized = _FRACTION_RE.sub(lambda m: _replace_fraction(m, language), normalized) normalized = _NUMBER_RANGE_RE.sub(lambda m: _replace_number_range(m, language), normalized)
normalized = _DECIMAL_NUMBER_RE.sub(_replace_decimal, normalized) normalized = _NUMBER_SPACE_RANGE_RE.sub(lambda m: _replace_space_separated_range(m, language), normalized)
normalized = _NUMBER_WITH_GROUP_RE.sub(_replace_grouped, normalized) normalized = _FRACTION_RE.sub(lambda m: _replace_fraction(m, language), normalized)
normalized = _PLAIN_NUMBER_RE.sub(_replace_plain, normalized) normalized = _DECIMAL_NUMBER_RE.sub(_replace_decimal, normalized)
normalized = _normalize_roman_numerals(normalized, language) normalized = _NUMBER_WITH_GROUP_RE.sub(_replace_grouped, normalized)
normalized = _PLAIN_NUMBER_RE.sub(_replace_plain, normalized)
normalized = _normalize_roman_numerals(normalized, language)
return normalized return normalized
def _normalize_dotted_acronyms(text: str) -> str:
def _replace(match: re.Match[str]) -> str:
value = match.group(0)
compact = value.replace(".", "")
return compact
return re.sub(r"\b(?:[A-Z]\.){1,}[A-Z]\.?(?=\W|$)", _replace, text)
# ---------- Optional phoneme hint post-processing ---------- # ---------- Optional phoneme hint post-processing ----------
@@ -1909,21 +2189,31 @@ def normalize_for_pipeline(
mode = str(runtime_settings.get("normalization_apostrophe_mode", "spacy")).lower() mode = str(runtime_settings.get("normalization_apostrophe_mode", "spacy")).lower()
normalized = text normalized = text
# Pre-normalization that must happen before number/url parsing.
if runtime_settings.get("normalization_numbers", True):
normalized = _normalize_dates(normalized, cfg.number_lang)
normalized = _normalize_times(normalized)
normalized = _normalize_dotted_acronyms(normalized)
if runtime_settings.get("normalization_titles", True):
normalized = _normalize_address_abbreviations(normalized)
if runtime_settings.get("normalization_internet_slang", False):
normalized = _normalize_internet_slang(normalized)
if mode == "off": if mode == "off":
normalized = normalize_unicode_apostrophes(text) normalized = normalize_unicode_apostrophes(normalized)
if cfg.convert_numbers: if cfg.convert_numbers or cfg.convert_currency or getattr(cfg, "remove_footnotes", False):
normalized = _normalize_grouped_numbers(normalized, cfg) normalized = _normalize_grouped_numbers(normalized, cfg)
normalized = _cleanup_spacing(normalized) normalized = _cleanup_spacing(normalized)
elif mode == "llm": elif mode == "llm":
try: try:
normalized = _normalize_with_llm(text, settings=runtime_settings, config=cfg) normalized = _normalize_with_llm(normalized, settings=runtime_settings, config=cfg)
except LLMClientError: except LLMClientError:
raise raise
if cfg.convert_numbers: if cfg.convert_numbers or cfg.convert_currency or getattr(cfg, "remove_footnotes", False):
normalized = _normalize_grouped_numbers(normalized, cfg) normalized = _normalize_grouped_numbers(normalized, cfg)
normalized = _cleanup_spacing(normalized) normalized = _cleanup_spacing(normalized)
else: else:
normalized, _ = normalize_apostrophes(text, cfg) normalized, _ = normalize_apostrophes(normalized, cfg)
if runtime_settings.get("normalization_titles", True): if runtime_settings.get("normalization_titles", True):
normalized = expand_titles_and_suffixes(normalized) normalized = expand_titles_and_suffixes(normalized)
+1
View File
@@ -38,6 +38,7 @@ _SETTINGS_DEFAULTS: Dict[str, Any] = {
"normalization_terminal": True, "normalization_terminal": True,
"normalization_phoneme_hints": True, "normalization_phoneme_hints": True,
"normalization_caps_quotes": True, "normalization_caps_quotes": True,
"normalization_internet_slang": False,
"normalization_apostrophes_contractions": True, "normalization_apostrophes_contractions": True,
"normalization_apostrophes_plural_possessives": True, "normalization_apostrophes_plural_possessives": True,
"normalization_apostrophes_sibilant_possessives": True, "normalization_apostrophes_sibilant_possessives": True,
+43 -8
View File
@@ -26,6 +26,7 @@ class DebugWavArtifact:
label: str label: str
filename: str filename: str
code: Optional[str] = None code: Optional[str] = None
text: Optional[str] = None
def _resolve_voice_setting(value: str) -> tuple[str, Optional[str], Optional[str]]: def _resolve_voice_setting(value: str) -> tuple[str, Optional[str], Optional[str]]:
@@ -65,6 +66,21 @@ def _extract_cases_from_text(text: str) -> List[Tuple[str, str]]:
return cases return cases
def _spoken_id(code: str) -> str:
# Make IDs pronounceable and stable (avoid reading as a word).
out: List[str] = []
for ch in str(code or ""):
if ch == "_":
out.append(" ")
elif ch.isalnum():
out.append(ch)
else:
out.append(" ")
# Add spaces between alnum to encourage letter-by-letter reading.
spaced = " ".join("".join(out).split())
return spaced
def run_debug_tts_wavs( def run_debug_tts_wavs(
*, *,
output_root: Path, output_root: Path,
@@ -96,6 +112,14 @@ def run_debug_tts_wavs(
combined_text = extraction.combined_text or "\n\n".join((c.text or "") for c in extraction.chapters) combined_text = extraction.combined_text or "\n\n".join((c.text or "") for c in extraction.chapters)
cases = _extract_cases_from_text(combined_text) cases = _extract_cases_from_text(combined_text)
# Prefer the canonical sample catalog for text (EPUB extraction may include headings).
try:
from abogen.debug_tts_samples import DEBUG_TTS_SAMPLES
sample_text_by_code = {sample.code: sample.text for sample in DEBUG_TTS_SAMPLES}
except Exception:
sample_text_by_code = {}
expected = list(iter_expected_codes()) expected = list(iter_expected_codes())
found_codes = {code for code, _ in cases} found_codes = {code for code, _ in cases}
missing = [code for code in expected if code not in found_codes] missing = [code for code in expected if code not in found_codes]
@@ -162,11 +186,15 @@ def run_debug_tts_wavs(
overall_path = run_dir / "overall.wav" overall_path = run_dir / "overall.wav"
overall_audio: List[np.ndarray] = [] overall_audio: List[np.ndarray] = []
def synth(text: str) -> np.ndarray: def synth(text: str, *, apply_normalization: bool = True) -> np.ndarray:
normalized = normalize_for_pipeline( normalized = (
text, normalize_for_pipeline(
config=apostrophe_config, text,
settings=normalization_settings, config=apostrophe_config,
settings=normalization_settings,
)
if apply_normalization
else str(text or "")
) )
parts: List[np.ndarray] = [] parts: List[np.ndarray] = []
for segment in pipeline( for segment in pipeline(
@@ -182,19 +210,26 @@ def run_debug_tts_wavs(
return np.zeros(0, dtype="float32") return np.zeros(0, dtype="float32")
return np.concatenate(parts).astype("float32", copy=False) return np.concatenate(parts).astype("float32", copy=False)
pause_1s = np.zeros(int(1.0 * SAMPLE_RATE), dtype="float32")
between_cases = np.zeros(int(0.35 * SAMPLE_RATE), dtype="float32")
# Per sample # Per sample
for code, snippet in cases: for code, snippet in cases:
snippet = sample_text_by_code.get(code, snippet)
if not snippet: if not snippet:
continue continue
audio = synth(snippet) id_audio = synth(_spoken_id(code), apply_normalization=False)
text_audio = synth(snippet, apply_normalization=True)
audio = np.concatenate([id_audio, pause_1s, text_audio]).astype("float32", copy=False)
filename = f"case_{code}.wav" filename = f"case_{code}.wav"
path = run_dir / filename path = run_dir / filename
# Write float32 PCM WAV. # Write float32 PCM WAV.
import soundfile as sf import soundfile as sf
sf.write(path, audio, SAMPLE_RATE, subtype="FLOAT") sf.write(path, audio, SAMPLE_RATE, subtype="FLOAT")
artifacts.append(DebugWavArtifact(label=f"{code}", filename=filename, code=code)) artifacts.append(DebugWavArtifact(label=f"{code}", filename=filename, code=code, text=snippet))
overall_audio.append(audio) overall_audio.append(audio)
overall_audio.append(between_cases)
# Overall # Overall
if overall_audio: if overall_audio:
@@ -204,7 +239,7 @@ def run_debug_tts_wavs(
import soundfile as sf import soundfile as sf
sf.write(overall_path, combined, SAMPLE_RATE, subtype="FLOAT") sf.write(overall_path, combined, SAMPLE_RATE, subtype="FLOAT")
artifacts.insert(0, DebugWavArtifact(label="Overall", filename="overall.wav", code=None)) artifacts.insert(0, DebugWavArtifact(label="Overall", filename="overall.wav", code=None, text=None))
manifest = { manifest = {
"run_id": run_id, "run_id": run_id,
+20 -3
View File
@@ -80,9 +80,17 @@ def update_settings() -> ResponseReturnValue:
maximum=25, maximum=25,
) )
def _extract_checkbox(name: str, default: bool) -> bool:
values = form.getlist(name) if hasattr(form, "getlist") else []
if values:
return coerce_bool(values[-1], default)
if hasattr(form, "__contains__") and name in form:
return False
return default
# Normalization settings # Normalization settings
for key in _NORMALIZATION_BOOLEAN_KEYS: for key in _NORMALIZATION_BOOLEAN_KEYS:
current[key] = coerce_bool(form.get(key), False) current[key] = _extract_checkbox(key, bool(current.get(key, True)))
for key in _NORMALIZATION_STRING_KEYS: for key in _NORMALIZATION_STRING_KEYS:
current[key] = (form.get(key) or "").strip() current[key] = (form.get(key) or "").strip()
@@ -236,7 +244,8 @@ def download_debug_wav(run_id: str, filename: str) -> ResponseReturnValue:
safe_name = (filename or "").strip() safe_name = (filename or "").strip()
if not safe_run or not safe_name or "/" in safe_name or "\\" in safe_name: if not safe_run or not safe_name or "/" in safe_name or "\\" in safe_name:
abort(404) abort(404)
if not safe_name.lower().endswith(".wav") and safe_name != "manifest.json": is_wav = safe_name.lower().endswith(".wav")
if not is_wav and safe_name != "manifest.json":
abort(404) abort(404)
root = Path(current_app.config.get("OUTPUT_FOLDER") or get_user_output_path("web")) root = Path(current_app.config.get("OUTPUT_FOLDER") or get_user_output_path("web"))
@@ -247,4 +256,12 @@ def download_debug_wav(run_id: str, filename: str) -> ResponseReturnValue:
expected_dir = (root / "debug" / safe_run).resolve() expected_dir = (root / "debug" / safe_run).resolve()
if expected_dir not in path.parents: if expected_dir not in path.parents:
abort(404) abort(404)
return send_file(path, as_attachment=True, download_name=path.name) wants_download = str(request.args.get("download") or "").strip().lower() in {"1", "true", "yes"}
mimetype = "audio/wav" if is_wav else "application/json"
# Inline playback should work for WAVs; allow explicit downloads via ?download=1.
return send_file(
path,
mimetype=mimetype,
as_attachment=wants_download,
download_name=path.name,
)
+10 -2
View File
@@ -47,6 +47,9 @@ _NORMALIZATION_BOOLEAN_KEYS = {
"normalization_terminal", "normalization_terminal",
"normalization_phoneme_hints", "normalization_phoneme_hints",
"normalization_caps_quotes", "normalization_caps_quotes",
"normalization_currency",
"normalization_footnotes",
"normalization_internet_slang",
"normalization_apostrophes_contractions", "normalization_apostrophes_contractions",
"normalization_apostrophes_plural_possessives", "normalization_apostrophes_plural_possessives",
"normalization_apostrophes_sibilant_possessives", "normalization_apostrophes_sibilant_possessives",
@@ -58,8 +61,6 @@ _NORMALIZATION_BOOLEAN_KEYS = {
"normalization_contraction_modal_would", "normalization_contraction_modal_would",
"normalization_contraction_negation_not", "normalization_contraction_negation_not",
"normalization_contraction_let_us", "normalization_contraction_let_us",
"normalization_currency",
"normalization_footnotes",
} }
_NORMALIZATION_STRING_KEYS = { _NORMALIZATION_STRING_KEYS = {
@@ -84,6 +85,9 @@ BOOLEAN_SETTINGS = {
"normalization_terminal", "normalization_terminal",
"normalization_phoneme_hints", "normalization_phoneme_hints",
"normalization_caps_quotes", "normalization_caps_quotes",
"normalization_currency",
"normalization_footnotes",
"normalization_internet_slang",
"normalization_apostrophes_contractions", "normalization_apostrophes_contractions",
"normalization_apostrophes_plural_possessives", "normalization_apostrophes_plural_possessives",
"normalization_apostrophes_sibilant_possessives", "normalization_apostrophes_sibilant_possessives",
@@ -107,6 +111,7 @@ _NORMALIZATION_GROUPS = [
{"key": "normalization_numbers", "label": "Convert grouped numbers to words"}, {"key": "normalization_numbers", "label": "Convert grouped numbers to words"},
{"key": "normalization_currency", "label": "Convert currency symbols ($10 → ten dollars)"}, {"key": "normalization_currency", "label": "Convert currency symbols ($10 → ten dollars)"},
{"key": "normalization_titles", "label": "Expand titles and suffixes (Dr., St., Jr., …)"}, {"key": "normalization_titles", "label": "Expand titles and suffixes (Dr., St., Jr., …)"},
{"key": "normalization_internet_slang", "label": "Expand internet slang (pls → please)"},
{"key": "normalization_footnotes", "label": "Remove footnote indicators ([1], [2])"}, {"key": "normalization_footnotes", "label": "Remove footnote indicators ([1], [2])"},
{"key": "normalization_terminal", "label": "Ensure sentences end with terminal punctuation"}, {"key": "normalization_terminal", "label": "Ensure sentences end with terminal punctuation"},
{"key": "normalization_caps_quotes", "label": "Convert ALL CAPS dialogue inside quotes"}, {"key": "normalization_caps_quotes", "label": "Convert ALL CAPS dialogue inside quotes"},
@@ -195,10 +200,13 @@ def settings_defaults() -> Dict[str, Any]:
"llm_prompt": llm_env_defaults.get("llm_prompt", DEFAULT_LLM_PROMPT), "llm_prompt": llm_env_defaults.get("llm_prompt", DEFAULT_LLM_PROMPT),
"llm_context_mode": llm_env_defaults.get("llm_context_mode", "sentence"), "llm_context_mode": llm_env_defaults.get("llm_context_mode", "sentence"),
"normalization_numbers": True, "normalization_numbers": True,
"normalization_currency": True,
"normalization_footnotes": True,
"normalization_titles": True, "normalization_titles": True,
"normalization_terminal": True, "normalization_terminal": True,
"normalization_phoneme_hints": True, "normalization_phoneme_hints": True,
"normalization_caps_quotes": True, "normalization_caps_quotes": True,
"normalization_internet_slang": False,
"normalization_apostrophes_contractions": True, "normalization_apostrophes_contractions": True,
"normalization_apostrophes_plural_possessives": True, "normalization_apostrophes_plural_possessives": True,
"normalization_apostrophes_sibilant_possessives": True, "normalization_apostrophes_sibilant_possessives": True,
+10 -34
View File
@@ -7,10 +7,7 @@
<h1 class="card__title">Debug WAVs</h1> <h1 class="card__title">Debug WAVs</h1>
<p class="tag">Run ID: <code>{{ run_id }}</code></p> <p class="tag">Run ID: <code>{{ run_id }}</code></p>
<div class="field field--stack"> <p class="hint">Each clip reads: ID, one second pause, then the reference text.</p>
<audio data-role="debug-audio" controls preload="none" style="width: 100%;"></audio>
<p class="hint">Click an ID to play its WAV. “Overall” is the concatenation of all samples.</p>
</div>
{% if artifacts %} {% if artifacts %}
<div class="field field--wide"> <div class="field field--wide">
@@ -18,15 +15,15 @@
<ul> <ul>
{% for item in artifacts %} {% for item in artifacts %}
<li> <li>
<button <div class="field field--stack" style="margin: 0;">
type="button" <div>
class="button button--ghost" <strong>{{ item.label }}</strong>
data-role="debug-play" {% if item.text %}
data-audio-src="{{ item.url }}" <span class="muted">{{ item.text }}</span>
> {% endif %}
{{ item.label }} </div>
</button> <audio controls preload="none" src="{{ item.url }}"></audio>
<a class="muted" href="{{ item.url }}" download>{{ item.filename }}</a> </div>
</li> </li>
{% endfor %} {% endfor %}
</ul> </ul>
@@ -37,25 +34,4 @@
<a href="{{ url_for('settings.settings_page', _anchor='debug') }}" class="button button--ghost">Back to Settings</a> <a href="{{ url_for('settings.settings_page', _anchor='debug') }}" class="button button--ghost">Back to Settings</a>
</div> </div>
</section> </section>
<script>
(function () {
const audio = document.querySelector('[data-role="debug-audio"]');
const buttons = Array.from(document.querySelectorAll('[data-role="debug-play"]'));
if (!audio || !buttons.length) {
return;
}
buttons.forEach((button) => {
button.addEventListener('click', () => {
const src = button.dataset.audioSrc;
if (!src) return;
if (audio.src !== src) {
audio.src = src;
}
audio.play();
});
});
})();
</script>
{% endblock %} {% endblock %}
+28 -2
View File
@@ -1,9 +1,13 @@
import importlib import importlib
import sys import sys
import os
import pytest
# Ensure real optional dependencies are imported before tests that install stubs # Ensure real optional dependencies are imported before tests that install stubs
# so that available packages (like ebooklib, bs4) aren't replaced with dummy modules. # so that available packages (like ebooklib, bs4, numpy) aren't replaced with dummy modules.
for module_name in ("ebooklib", "bs4"): for module_name in ("ebooklib", "bs4", "numpy"):
if module_name not in sys.modules: if module_name not in sys.modules:
try: try:
importlib.import_module(module_name) importlib.import_module(module_name)
@@ -11,3 +15,25 @@ for module_name in ("ebooklib", "bs4"):
# On environments without the optional dependency, downstream tests # On environments without the optional dependency, downstream tests
# will install lightweight stubs as needed. # will install lightweight stubs as needed.
pass pass
@pytest.fixture(autouse=True, scope="session")
def _isolate_settings_dir(tmp_path_factory: pytest.TempPathFactory):
settings_dir = tmp_path_factory.mktemp("abogen-settings")
os.environ["ABOGEN_SETTINGS_DIR"] = str(settings_dir)
try:
from abogen.utils import get_user_settings_dir
get_user_settings_dir.cache_clear()
except Exception:
pass
try:
from abogen.normalization_settings import clear_cached_settings
clear_cached_settings()
except Exception:
pass
yield
+34
View File
@@ -215,6 +215,40 @@ def test_decades_can_skip_expansion_when_disabled() -> None:
assert "'90s" in normalized assert "'90s" in normalized
def test_abbreviated_decades_expand_to_spoken_form() -> None:
normalized = _normalize_text("She loved music from the '80s.")
assert "eighties" in normalized.lower()
def test_currency_under_one_dollar_uses_cents() -> None:
normalized = _normalize_text("It cost $0.99.")
folded = normalized.lower().replace("-", " ")
assert "zero dollars" not in folded
assert "cents" in folded
def test_iso_dates_use_locale_order_and_ordinals(monkeypatch) -> None:
monkeypatch.setenv("LC_TIME", "en_US.UTF-8")
normalized = _normalize_text("The date is 2025/12/15.")
folded = normalized.lower().replace("-", " ")
assert "december" in folded
assert "fifteenth" in folded
def test_times_and_acronyms_do_not_say_dot() -> None:
normalized = _normalize_text("Meet at 5 p.m. near the U.S.A. border.")
folded = normalized.lower()
assert " dot " not in folded
def test_internet_slang_expansion_is_configurable() -> None:
normalized = _normalize_text(
"pls knock before entering.",
normalization_overrides={"normalization_internet_slang": True},
)
assert "please" in normalized.lower()
@pytest.mark.skipif(not SPACY_RESOLVER_AVAILABLE, reason="spaCy model unavailable") @pytest.mark.skipif(not SPACY_RESOLVER_AVAILABLE, reason="spaCy model unavailable")
def test_spacy_disambiguates_it_has_from_context() -> None: def test_spacy_disambiguates_it_has_from_context() -> None:
normalized = _normalize_text("It's been a long time.") normalized = _normalize_text("It's been a long time.")