mirror of
https://github.com/denizsafak/abogen.git
synced 2026-07-18 13:40:27 +02:00
feat: Enhance text normalization with support for internet slang expansion, currency formatting, and date handling; update debug WAVs interface and settings
This commit is contained in:
@@ -182,7 +182,7 @@ DEBUG_TTS_SAMPLES: Sequence[DebugTTSSample] = (
|
||||
DebugTTSSample(
|
||||
code="TITLE_003",
|
||||
label="Titles and abbreviations (3)",
|
||||
text="Gen. Lee spoke to Sgt. Rivera on Main St.",
|
||||
text="Gen. Smith spoke to Sgt. Rivera on Main St.",
|
||||
),
|
||||
DebugTTSSample(
|
||||
code="TITLE_004",
|
||||
|
||||
@@ -5,7 +5,10 @@ import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple
|
||||
|
||||
import spacy
|
||||
try: # pragma: no cover - optional dependency
|
||||
import spacy # type: ignore
|
||||
except Exception: # pragma: no cover - spaCy may be unavailable in minimal environments
|
||||
spacy = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -177,6 +180,9 @@ def _build_replacement_sentence(sentence: str, token: str, replacement_token: st
|
||||
|
||||
|
||||
def _load_spacy(language: str) -> Any:
|
||||
if spacy is None:
|
||||
return None
|
||||
|
||||
# English only for now.
|
||||
# Use installed small model; keep it simple.
|
||||
lang = (language or "en").lower()
|
||||
@@ -211,7 +217,12 @@ def extract_heteronym_overrides(
|
||||
if not lang.startswith("en"):
|
||||
return []
|
||||
|
||||
if spacy is None:
|
||||
return []
|
||||
|
||||
nlp = _load_spacy(lang)
|
||||
if nlp is None:
|
||||
return []
|
||||
|
||||
previous_choices: Dict[str, str] = {}
|
||||
if existing:
|
||||
|
||||
@@ -3,6 +3,8 @@ from __future__ import annotations
|
||||
import json
|
||||
import re
|
||||
import unicodedata
|
||||
import os
|
||||
import locale
|
||||
from fractions import Fraction
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple
|
||||
@@ -129,6 +131,176 @@ _URL_RE = re.compile(r"(https?://)?(www\.)?(?P<domain>[a-zA-Z0-9-]+(\.[a-zA-Z0-9
|
||||
_FOOTNOTE_RE = re.compile(r"([a-zA-Z]+)(\d+)")
|
||||
_BRACKET_FOOTNOTE_RE = re.compile(r"\[\d+\]")
|
||||
|
||||
_ISO_DATE_RE = re.compile(r"\b(?P<year>\d{4})[/-](?P<month>\d{1,2})[/-](?P<day>\d{1,2})\b")
|
||||
_MDY_DATE_RE = re.compile(
|
||||
r"\b(?P<month>Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\.?\s+"
|
||||
r"(?P<day>\d{1,2})(?:st|nd|rd|th)?\s*,\s*(?P<year>\d{4})\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_TIME_RE = re.compile(
|
||||
r"\b(?P<hour>\d{1,2})(?::(?P<minute>\d{2}))?\s*(?P<meridian>a\.?m\.?|p\.?m\.?)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_ADDRESS_ABBR_RE = re.compile(r"(?P<prefix>\b\w+\s+)(?P<abbr>St|Rd|Ave|Blvd|Ln|Dr)\.(?=\s*(?:,|\.|!|\?|$))")
|
||||
|
||||
_MONTH_MAP = {
|
||||
"jan": "January",
|
||||
"january": "January",
|
||||
"feb": "February",
|
||||
"february": "February",
|
||||
"mar": "March",
|
||||
"march": "March",
|
||||
"apr": "April",
|
||||
"april": "April",
|
||||
"may": "May",
|
||||
"jun": "June",
|
||||
"june": "June",
|
||||
"jul": "July",
|
||||
"july": "July",
|
||||
"aug": "August",
|
||||
"august": "August",
|
||||
"sep": "September",
|
||||
"sept": "September",
|
||||
"september": "September",
|
||||
"oct": "October",
|
||||
"october": "October",
|
||||
"nov": "November",
|
||||
"november": "November",
|
||||
"dec": "December",
|
||||
"december": "December",
|
||||
}
|
||||
|
||||
|
||||
def _is_us_locale() -> bool:
|
||||
for key in ("LC_ALL", "LC_TIME", "LANG"):
|
||||
value = os.environ.get(key)
|
||||
if value and "en_US" in value:
|
||||
return True
|
||||
try:
|
||||
loc = locale.getlocale(locale.LC_TIME)
|
||||
if loc and loc[0] and "en_US" in loc[0]:
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def _year_to_words_american(value: int, language: str) -> str:
|
||||
if language.lower().startswith("en") and 2000 <= value <= 2099:
|
||||
if value == 2000:
|
||||
return "two thousand"
|
||||
if 2001 <= value <= 2009:
|
||||
tail = _DIGIT_WORDS[value % 10]
|
||||
return f"two thousand {tail}"
|
||||
|
||||
first_two = value // 100
|
||||
last_two = value % 100
|
||||
first_words = _int_to_words(first_two, language) or "twenty"
|
||||
if last_two == 0:
|
||||
return f"{first_words} hundred"
|
||||
if last_two < 10:
|
||||
return f"{first_words} oh {_DIGIT_WORDS[last_two]}"
|
||||
last_words = _int_to_words(last_two, language)
|
||||
return f"{first_words} {last_words or last_two}"
|
||||
|
||||
words = _int_to_words(value, language)
|
||||
return words or str(value)
|
||||
|
||||
|
||||
def _normalize_dates(text: str, language: str) -> str:
|
||||
us = _is_us_locale()
|
||||
|
||||
def _format_iso(match: re.Match[str]) -> str:
|
||||
year = int(match.group("year"))
|
||||
month = int(match.group("month"))
|
||||
day = int(match.group("day"))
|
||||
if not (1 <= month <= 12 and 1 <= day <= 31):
|
||||
return match.group(0)
|
||||
month_name = (
|
||||
[
|
||||
"January",
|
||||
"February",
|
||||
"March",
|
||||
"April",
|
||||
"May",
|
||||
"June",
|
||||
"July",
|
||||
"August",
|
||||
"September",
|
||||
"October",
|
||||
"November",
|
||||
"December",
|
||||
][month - 1]
|
||||
)
|
||||
ordinal = _int_to_ordinal_words(day, language) or str(day)
|
||||
year_words = _year_to_words_american(year, language)
|
||||
return f"{month_name} {ordinal}, {year_words}" if us else f"{ordinal} {month_name} {year_words}"
|
||||
|
||||
def _format_mdy(match: re.Match[str]) -> str:
|
||||
month_raw = str(match.group("month") or "").strip().lower().rstrip(".")
|
||||
month_name = _MONTH_MAP.get(month_raw)
|
||||
if not month_name:
|
||||
return match.group(0)
|
||||
day = int(match.group("day"))
|
||||
year = int(match.group("year"))
|
||||
ordinal = _int_to_ordinal_words(day, language) or str(day)
|
||||
year_words = _year_to_words_american(year, language)
|
||||
return f"{month_name} {ordinal}, {year_words}" if us else f"{ordinal} {month_name} {year_words}"
|
||||
|
||||
out = _ISO_DATE_RE.sub(_format_iso, text)
|
||||
out = _MDY_DATE_RE.sub(_format_mdy, out)
|
||||
return out
|
||||
|
||||
|
||||
def _normalize_times(text: str) -> str:
|
||||
def _replace(match: re.Match[str]) -> str:
|
||||
hour = match.group("hour")
|
||||
minute = match.group("minute")
|
||||
meridian = match.group("meridian").lower().replace(".", "")
|
||||
if minute:
|
||||
return f"{hour}:{minute} {meridian}"
|
||||
return f"{hour} {meridian}"
|
||||
|
||||
return _TIME_RE.sub(_replace, text)
|
||||
|
||||
|
||||
def _normalize_address_abbreviations(text: str) -> str:
|
||||
mapping = {
|
||||
"st": "street",
|
||||
"rd": "road",
|
||||
"ave": "avenue",
|
||||
"blvd": "boulevard",
|
||||
"ln": "lane",
|
||||
"dr": "drive",
|
||||
}
|
||||
|
||||
def _replace(match: re.Match[str]) -> str:
|
||||
abbr = match.group("abbr")
|
||||
full = mapping.get(abbr.lower())
|
||||
if not full:
|
||||
return match.group(0)
|
||||
return match.group("prefix") + _match_casing(abbr, full)
|
||||
|
||||
return _ADDRESS_ABBR_RE.sub(_replace, text)
|
||||
|
||||
|
||||
def _normalize_internet_slang(text: str) -> str:
|
||||
mapping = {
|
||||
"pls": "please",
|
||||
"plz": "please",
|
||||
}
|
||||
|
||||
def _replace(match: re.Match[str]) -> str:
|
||||
token = match.group(0)
|
||||
replacement = mapping.get(token.lower())
|
||||
if not replacement:
|
||||
return token
|
||||
return _match_casing(token, replacement)
|
||||
|
||||
return re.sub(r"\b(?:pls|plz)\b", _replace, text, flags=re.IGNORECASE)
|
||||
|
||||
_DECIMAL_NUMBER_RE = re.compile(
|
||||
rf"(?<![\w{_NUMBER_RANGE_CLASS}/])(?P<number>-?(?:\d{{1,3}}(?:,\d{{3}})+|\d+)\.(?P<fraction>\d+))(?![\w{_NUMBER_RANGE_CLASS}/])"
|
||||
)
|
||||
@@ -155,13 +327,49 @@ def _int_to_words(value: int, language: str) -> Optional[str]:
|
||||
|
||||
|
||||
def _int_to_ordinal_words(value: int, language: str) -> Optional[str]:
|
||||
if num2words is None:
|
||||
return None
|
||||
if num2words is not None:
|
||||
try:
|
||||
return num2words(value, lang=language, ordinal=True)
|
||||
except Exception: # pragma: no cover - unsupported locale
|
||||
return None
|
||||
|
||||
try:
|
||||
return num2words(value, lang=language, ordinal=True)
|
||||
except Exception: # pragma: no cover - unsupported locale
|
||||
return None
|
||||
if language.lower().startswith("en"):
|
||||
ordinals = {
|
||||
1: "first",
|
||||
2: "second",
|
||||
3: "third",
|
||||
4: "fourth",
|
||||
5: "fifth",
|
||||
6: "sixth",
|
||||
7: "seventh",
|
||||
8: "eighth",
|
||||
9: "ninth",
|
||||
10: "tenth",
|
||||
11: "eleventh",
|
||||
12: "twelfth",
|
||||
13: "thirteenth",
|
||||
14: "fourteenth",
|
||||
15: "fifteenth",
|
||||
16: "sixteenth",
|
||||
17: "seventeenth",
|
||||
18: "eighteenth",
|
||||
19: "nineteenth",
|
||||
20: "twentieth",
|
||||
21: "twenty-first",
|
||||
22: "twenty-second",
|
||||
23: "twenty-third",
|
||||
24: "twenty-fourth",
|
||||
25: "twenty-fifth",
|
||||
26: "twenty-sixth",
|
||||
27: "twenty-seventh",
|
||||
28: "twenty-eighth",
|
||||
29: "twenty-ninth",
|
||||
30: "thirtieth",
|
||||
31: "thirty-first",
|
||||
}
|
||||
return ordinals.get(int(value))
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _pluralize_fraction_word(base: str) -> str:
|
||||
@@ -366,6 +574,8 @@ TITLE_ABBREVIATIONS = {
|
||||
"dr": "doctor",
|
||||
"prof": "professor",
|
||||
"rev": "reverend",
|
||||
"gen": "general",
|
||||
"sgt": "sergeant",
|
||||
}
|
||||
|
||||
SUFFIX_ABBREVIATIONS = {
|
||||
@@ -1146,7 +1356,23 @@ def classify_token(token: str, cfg: ApostropheConfig) -> Tuple[str, str]:
|
||||
# 1. Decades
|
||||
if DECADE_RE.match(token):
|
||||
if cfg.decades_mode == "expand":
|
||||
return "decade", f"19{token[2:4]}s"
|
||||
decade_digit = token[1]
|
||||
decade_map = {
|
||||
"0": "two thousands",
|
||||
"1": "tens",
|
||||
"2": "twenties",
|
||||
"3": "thirties",
|
||||
"4": "forties",
|
||||
"5": "fifties",
|
||||
"6": "sixties",
|
||||
"7": "seventies",
|
||||
"8": "eighties",
|
||||
"9": "nineties",
|
||||
}
|
||||
spoken = decade_map.get(decade_digit)
|
||||
if spoken:
|
||||
return "decade", spoken
|
||||
return "decade", token
|
||||
return "decade", token
|
||||
|
||||
# 2. Leading elision
|
||||
@@ -1312,7 +1538,7 @@ def normalize_apostrophes(text: str, cfg: ApostropheConfig | None = None) -> Tup
|
||||
return normalized_text, results
|
||||
|
||||
def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
|
||||
if not text or not cfg.convert_numbers:
|
||||
if not text:
|
||||
return text
|
||||
|
||||
language = (cfg.number_lang or "en").strip() or "en"
|
||||
@@ -1360,6 +1586,20 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
|
||||
return words.replace(" and ", " ")
|
||||
return None
|
||||
|
||||
# 1200s are commonly spoken as "twelve hundred".
|
||||
if 1200 <= value < 1300:
|
||||
hundreds = value // 100
|
||||
remainder = value % 100
|
||||
prefix = _words(hundreds)
|
||||
if not prefix:
|
||||
return None
|
||||
if remainder == 0:
|
||||
return f"{prefix} hundred"
|
||||
tail = _format_year_tail(remainder, allow_oh=True)
|
||||
if tail is None:
|
||||
return None
|
||||
return f"{prefix} hundred {tail}".strip()
|
||||
|
||||
if value % 1000 == 0:
|
||||
thousands = value // 1000
|
||||
thousands_words = _words(thousands)
|
||||
@@ -1537,6 +1777,29 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
|
||||
|
||||
return f"{amount_spoken} {magnitude} {currency_name}"
|
||||
|
||||
# Handle $0.99 -> ninety-nine cents (avoid "zero dollars and...").
|
||||
if amount_str.startswith("0") and amount < 1.0:
|
||||
dollars_part, dot, fraction = amount_str.partition(".")
|
||||
if dot and fraction:
|
||||
cents_str = (fraction + "00")[:2]
|
||||
try:
|
||||
cents_value = int(cents_str)
|
||||
except ValueError:
|
||||
cents_value = 0
|
||||
if cents_value > 0:
|
||||
cents_words = _int_to_words(cents_value, language) or str(cents_value)
|
||||
subunit = {
|
||||
"$": "cent",
|
||||
"€": "cent",
|
||||
"£": "penny",
|
||||
"¥": "yen",
|
||||
}.get(symbol, "cent")
|
||||
if symbol == "£":
|
||||
subunit = "pence" if cents_value != 1 else "penny"
|
||||
else:
|
||||
subunit = (subunit + "s") if cents_value != 1 and subunit not in {"pence", "yen"} else subunit
|
||||
return f"{cents_words} {subunit}".strip()
|
||||
|
||||
currency_map = {
|
||||
"$": "USD",
|
||||
"£": "GBP",
|
||||
@@ -1559,6 +1822,13 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
|
||||
|
||||
def _replace_url(match: re.Match[str]) -> str:
|
||||
domain = match.group("domain")
|
||||
|
||||
# Avoid matching dotted abbreviations/acronyms without an explicit URL prefix.
|
||||
has_prefix = match.group(1) or match.group(2)
|
||||
if not has_prefix:
|
||||
parts = [p for p in domain.split(".") if p]
|
||||
if parts and all(p.isalpha() and len(p) <= 2 for p in parts):
|
||||
return match.group(0)
|
||||
|
||||
# Avoid matching numbers like 1.05 or 12.34.56
|
||||
# If the domain consists only of digits and dots, ignore it (unless it has http/www prefix)
|
||||
@@ -1583,24 +1853,34 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
|
||||
return match.group(1)
|
||||
|
||||
normalized = text
|
||||
|
||||
# Apply URL replacement first
|
||||
|
||||
# Apply URL replacement first (independent of number conversion).
|
||||
normalized = _URL_RE.sub(_replace_url, normalized)
|
||||
|
||||
|
||||
if getattr(cfg, "remove_footnotes", False):
|
||||
normalized = _FOOTNOTE_RE.sub(_remove_footnote, normalized)
|
||||
normalized = _BRACKET_FOOTNOTE_RE.sub("", normalized)
|
||||
|
||||
if cfg.convert_currency:
|
||||
normalized = _CURRENCY_RE.sub(_replace_currency, normalized)
|
||||
normalized = _NUMBER_RANGE_RE.sub(lambda m: _replace_number_range(m, language), normalized)
|
||||
normalized = _NUMBER_SPACE_RANGE_RE.sub(lambda m: _replace_space_separated_range(m, language), normalized)
|
||||
normalized = _FRACTION_RE.sub(lambda m: _replace_fraction(m, language), normalized)
|
||||
normalized = _DECIMAL_NUMBER_RE.sub(_replace_decimal, normalized)
|
||||
normalized = _NUMBER_WITH_GROUP_RE.sub(_replace_grouped, normalized)
|
||||
normalized = _PLAIN_NUMBER_RE.sub(_replace_plain, normalized)
|
||||
normalized = _normalize_roman_numerals(normalized, language)
|
||||
|
||||
if cfg.convert_numbers:
|
||||
normalized = _NUMBER_RANGE_RE.sub(lambda m: _replace_number_range(m, language), normalized)
|
||||
normalized = _NUMBER_SPACE_RANGE_RE.sub(lambda m: _replace_space_separated_range(m, language), normalized)
|
||||
normalized = _FRACTION_RE.sub(lambda m: _replace_fraction(m, language), normalized)
|
||||
normalized = _DECIMAL_NUMBER_RE.sub(_replace_decimal, normalized)
|
||||
normalized = _NUMBER_WITH_GROUP_RE.sub(_replace_grouped, normalized)
|
||||
normalized = _PLAIN_NUMBER_RE.sub(_replace_plain, normalized)
|
||||
normalized = _normalize_roman_numerals(normalized, language)
|
||||
|
||||
return normalized
|
||||
def _normalize_dotted_acronyms(text: str) -> str:
|
||||
def _replace(match: re.Match[str]) -> str:
|
||||
value = match.group(0)
|
||||
compact = value.replace(".", "")
|
||||
return compact
|
||||
|
||||
return re.sub(r"\b(?:[A-Z]\.){1,}[A-Z]\.?(?=\W|$)", _replace, text)
|
||||
|
||||
# ---------- Optional phoneme hint post-processing ----------
|
||||
|
||||
@@ -1909,21 +2189,31 @@ def normalize_for_pipeline(
|
||||
mode = str(runtime_settings.get("normalization_apostrophe_mode", "spacy")).lower()
|
||||
normalized = text
|
||||
|
||||
# Pre-normalization that must happen before number/url parsing.
|
||||
if runtime_settings.get("normalization_numbers", True):
|
||||
normalized = _normalize_dates(normalized, cfg.number_lang)
|
||||
normalized = _normalize_times(normalized)
|
||||
normalized = _normalize_dotted_acronyms(normalized)
|
||||
if runtime_settings.get("normalization_titles", True):
|
||||
normalized = _normalize_address_abbreviations(normalized)
|
||||
if runtime_settings.get("normalization_internet_slang", False):
|
||||
normalized = _normalize_internet_slang(normalized)
|
||||
|
||||
if mode == "off":
|
||||
normalized = normalize_unicode_apostrophes(text)
|
||||
if cfg.convert_numbers:
|
||||
normalized = normalize_unicode_apostrophes(normalized)
|
||||
if cfg.convert_numbers or cfg.convert_currency or getattr(cfg, "remove_footnotes", False):
|
||||
normalized = _normalize_grouped_numbers(normalized, cfg)
|
||||
normalized = _cleanup_spacing(normalized)
|
||||
elif mode == "llm":
|
||||
try:
|
||||
normalized = _normalize_with_llm(text, settings=runtime_settings, config=cfg)
|
||||
normalized = _normalize_with_llm(normalized, settings=runtime_settings, config=cfg)
|
||||
except LLMClientError:
|
||||
raise
|
||||
if cfg.convert_numbers:
|
||||
if cfg.convert_numbers or cfg.convert_currency or getattr(cfg, "remove_footnotes", False):
|
||||
normalized = _normalize_grouped_numbers(normalized, cfg)
|
||||
normalized = _cleanup_spacing(normalized)
|
||||
else:
|
||||
normalized, _ = normalize_apostrophes(text, cfg)
|
||||
normalized, _ = normalize_apostrophes(normalized, cfg)
|
||||
|
||||
if runtime_settings.get("normalization_titles", True):
|
||||
normalized = expand_titles_and_suffixes(normalized)
|
||||
|
||||
@@ -38,6 +38,7 @@ _SETTINGS_DEFAULTS: Dict[str, Any] = {
|
||||
"normalization_terminal": True,
|
||||
"normalization_phoneme_hints": True,
|
||||
"normalization_caps_quotes": True,
|
||||
"normalization_internet_slang": False,
|
||||
"normalization_apostrophes_contractions": True,
|
||||
"normalization_apostrophes_plural_possessives": True,
|
||||
"normalization_apostrophes_sibilant_possessives": True,
|
||||
|
||||
@@ -26,6 +26,7 @@ class DebugWavArtifact:
|
||||
label: str
|
||||
filename: str
|
||||
code: Optional[str] = None
|
||||
text: Optional[str] = None
|
||||
|
||||
|
||||
def _resolve_voice_setting(value: str) -> tuple[str, Optional[str], Optional[str]]:
|
||||
@@ -65,6 +66,21 @@ def _extract_cases_from_text(text: str) -> List[Tuple[str, str]]:
|
||||
return cases
|
||||
|
||||
|
||||
def _spoken_id(code: str) -> str:
|
||||
# Make IDs pronounceable and stable (avoid reading as a word).
|
||||
out: List[str] = []
|
||||
for ch in str(code or ""):
|
||||
if ch == "_":
|
||||
out.append(" ")
|
||||
elif ch.isalnum():
|
||||
out.append(ch)
|
||||
else:
|
||||
out.append(" ")
|
||||
# Add spaces between alnum to encourage letter-by-letter reading.
|
||||
spaced = " ".join("".join(out).split())
|
||||
return spaced
|
||||
|
||||
|
||||
def run_debug_tts_wavs(
|
||||
*,
|
||||
output_root: Path,
|
||||
@@ -96,6 +112,14 @@ def run_debug_tts_wavs(
|
||||
combined_text = extraction.combined_text or "\n\n".join((c.text or "") for c in extraction.chapters)
|
||||
cases = _extract_cases_from_text(combined_text)
|
||||
|
||||
# Prefer the canonical sample catalog for text (EPUB extraction may include headings).
|
||||
try:
|
||||
from abogen.debug_tts_samples import DEBUG_TTS_SAMPLES
|
||||
|
||||
sample_text_by_code = {sample.code: sample.text for sample in DEBUG_TTS_SAMPLES}
|
||||
except Exception:
|
||||
sample_text_by_code = {}
|
||||
|
||||
expected = list(iter_expected_codes())
|
||||
found_codes = {code for code, _ in cases}
|
||||
missing = [code for code in expected if code not in found_codes]
|
||||
@@ -162,11 +186,15 @@ def run_debug_tts_wavs(
|
||||
overall_path = run_dir / "overall.wav"
|
||||
overall_audio: List[np.ndarray] = []
|
||||
|
||||
def synth(text: str) -> np.ndarray:
|
||||
normalized = normalize_for_pipeline(
|
||||
text,
|
||||
config=apostrophe_config,
|
||||
settings=normalization_settings,
|
||||
def synth(text: str, *, apply_normalization: bool = True) -> np.ndarray:
|
||||
normalized = (
|
||||
normalize_for_pipeline(
|
||||
text,
|
||||
config=apostrophe_config,
|
||||
settings=normalization_settings,
|
||||
)
|
||||
if apply_normalization
|
||||
else str(text or "")
|
||||
)
|
||||
parts: List[np.ndarray] = []
|
||||
for segment in pipeline(
|
||||
@@ -182,19 +210,26 @@ def run_debug_tts_wavs(
|
||||
return np.zeros(0, dtype="float32")
|
||||
return np.concatenate(parts).astype("float32", copy=False)
|
||||
|
||||
pause_1s = np.zeros(int(1.0 * SAMPLE_RATE), dtype="float32")
|
||||
between_cases = np.zeros(int(0.35 * SAMPLE_RATE), dtype="float32")
|
||||
|
||||
# Per sample
|
||||
for code, snippet in cases:
|
||||
snippet = sample_text_by_code.get(code, snippet)
|
||||
if not snippet:
|
||||
continue
|
||||
audio = synth(snippet)
|
||||
id_audio = synth(_spoken_id(code), apply_normalization=False)
|
||||
text_audio = synth(snippet, apply_normalization=True)
|
||||
audio = np.concatenate([id_audio, pause_1s, text_audio]).astype("float32", copy=False)
|
||||
filename = f"case_{code}.wav"
|
||||
path = run_dir / filename
|
||||
# Write float32 PCM WAV.
|
||||
import soundfile as sf
|
||||
|
||||
sf.write(path, audio, SAMPLE_RATE, subtype="FLOAT")
|
||||
artifacts.append(DebugWavArtifact(label=f"{code}", filename=filename, code=code))
|
||||
artifacts.append(DebugWavArtifact(label=f"{code}", filename=filename, code=code, text=snippet))
|
||||
overall_audio.append(audio)
|
||||
overall_audio.append(between_cases)
|
||||
|
||||
# Overall
|
||||
if overall_audio:
|
||||
@@ -204,7 +239,7 @@ def run_debug_tts_wavs(
|
||||
import soundfile as sf
|
||||
|
||||
sf.write(overall_path, combined, SAMPLE_RATE, subtype="FLOAT")
|
||||
artifacts.insert(0, DebugWavArtifact(label="Overall", filename="overall.wav", code=None))
|
||||
artifacts.insert(0, DebugWavArtifact(label="Overall", filename="overall.wav", code=None, text=None))
|
||||
|
||||
manifest = {
|
||||
"run_id": run_id,
|
||||
|
||||
@@ -80,9 +80,17 @@ def update_settings() -> ResponseReturnValue:
|
||||
maximum=25,
|
||||
)
|
||||
|
||||
def _extract_checkbox(name: str, default: bool) -> bool:
|
||||
values = form.getlist(name) if hasattr(form, "getlist") else []
|
||||
if values:
|
||||
return coerce_bool(values[-1], default)
|
||||
if hasattr(form, "__contains__") and name in form:
|
||||
return False
|
||||
return default
|
||||
|
||||
# Normalization settings
|
||||
for key in _NORMALIZATION_BOOLEAN_KEYS:
|
||||
current[key] = coerce_bool(form.get(key), False)
|
||||
current[key] = _extract_checkbox(key, bool(current.get(key, True)))
|
||||
for key in _NORMALIZATION_STRING_KEYS:
|
||||
current[key] = (form.get(key) or "").strip()
|
||||
|
||||
@@ -236,7 +244,8 @@ def download_debug_wav(run_id: str, filename: str) -> ResponseReturnValue:
|
||||
safe_name = (filename or "").strip()
|
||||
if not safe_run or not safe_name or "/" in safe_name or "\\" in safe_name:
|
||||
abort(404)
|
||||
if not safe_name.lower().endswith(".wav") and safe_name != "manifest.json":
|
||||
is_wav = safe_name.lower().endswith(".wav")
|
||||
if not is_wav and safe_name != "manifest.json":
|
||||
abort(404)
|
||||
|
||||
root = Path(current_app.config.get("OUTPUT_FOLDER") or get_user_output_path("web"))
|
||||
@@ -247,4 +256,12 @@ def download_debug_wav(run_id: str, filename: str) -> ResponseReturnValue:
|
||||
expected_dir = (root / "debug" / safe_run).resolve()
|
||||
if expected_dir not in path.parents:
|
||||
abort(404)
|
||||
return send_file(path, as_attachment=True, download_name=path.name)
|
||||
wants_download = str(request.args.get("download") or "").strip().lower() in {"1", "true", "yes"}
|
||||
mimetype = "audio/wav" if is_wav else "application/json"
|
||||
# Inline playback should work for WAVs; allow explicit downloads via ?download=1.
|
||||
return send_file(
|
||||
path,
|
||||
mimetype=mimetype,
|
||||
as_attachment=wants_download,
|
||||
download_name=path.name,
|
||||
)
|
||||
|
||||
@@ -47,6 +47,9 @@ _NORMALIZATION_BOOLEAN_KEYS = {
|
||||
"normalization_terminal",
|
||||
"normalization_phoneme_hints",
|
||||
"normalization_caps_quotes",
|
||||
"normalization_currency",
|
||||
"normalization_footnotes",
|
||||
"normalization_internet_slang",
|
||||
"normalization_apostrophes_contractions",
|
||||
"normalization_apostrophes_plural_possessives",
|
||||
"normalization_apostrophes_sibilant_possessives",
|
||||
@@ -58,8 +61,6 @@ _NORMALIZATION_BOOLEAN_KEYS = {
|
||||
"normalization_contraction_modal_would",
|
||||
"normalization_contraction_negation_not",
|
||||
"normalization_contraction_let_us",
|
||||
"normalization_currency",
|
||||
"normalization_footnotes",
|
||||
}
|
||||
|
||||
_NORMALIZATION_STRING_KEYS = {
|
||||
@@ -84,6 +85,9 @@ BOOLEAN_SETTINGS = {
|
||||
"normalization_terminal",
|
||||
"normalization_phoneme_hints",
|
||||
"normalization_caps_quotes",
|
||||
"normalization_currency",
|
||||
"normalization_footnotes",
|
||||
"normalization_internet_slang",
|
||||
"normalization_apostrophes_contractions",
|
||||
"normalization_apostrophes_plural_possessives",
|
||||
"normalization_apostrophes_sibilant_possessives",
|
||||
@@ -107,6 +111,7 @@ _NORMALIZATION_GROUPS = [
|
||||
{"key": "normalization_numbers", "label": "Convert grouped numbers to words"},
|
||||
{"key": "normalization_currency", "label": "Convert currency symbols ($10 → ten dollars)"},
|
||||
{"key": "normalization_titles", "label": "Expand titles and suffixes (Dr., St., Jr., …)"},
|
||||
{"key": "normalization_internet_slang", "label": "Expand internet slang (pls → please)"},
|
||||
{"key": "normalization_footnotes", "label": "Remove footnote indicators ([1], [2])"},
|
||||
{"key": "normalization_terminal", "label": "Ensure sentences end with terminal punctuation"},
|
||||
{"key": "normalization_caps_quotes", "label": "Convert ALL CAPS dialogue inside quotes"},
|
||||
@@ -195,10 +200,13 @@ def settings_defaults() -> Dict[str, Any]:
|
||||
"llm_prompt": llm_env_defaults.get("llm_prompt", DEFAULT_LLM_PROMPT),
|
||||
"llm_context_mode": llm_env_defaults.get("llm_context_mode", "sentence"),
|
||||
"normalization_numbers": True,
|
||||
"normalization_currency": True,
|
||||
"normalization_footnotes": True,
|
||||
"normalization_titles": True,
|
||||
"normalization_terminal": True,
|
||||
"normalization_phoneme_hints": True,
|
||||
"normalization_caps_quotes": True,
|
||||
"normalization_internet_slang": False,
|
||||
"normalization_apostrophes_contractions": True,
|
||||
"normalization_apostrophes_plural_possessives": True,
|
||||
"normalization_apostrophes_sibilant_possessives": True,
|
||||
|
||||
@@ -7,10 +7,7 @@
|
||||
<h1 class="card__title">Debug WAVs</h1>
|
||||
<p class="tag">Run ID: <code>{{ run_id }}</code></p>
|
||||
|
||||
<div class="field field--stack">
|
||||
<audio data-role="debug-audio" controls preload="none" style="width: 100%;"></audio>
|
||||
<p class="hint">Click an ID to play its WAV. “Overall” is the concatenation of all samples.</p>
|
||||
</div>
|
||||
<p class="hint">Each clip reads: ID, one second pause, then the reference text.</p>
|
||||
|
||||
{% if artifacts %}
|
||||
<div class="field field--wide">
|
||||
@@ -18,15 +15,15 @@
|
||||
<ul>
|
||||
{% for item in artifacts %}
|
||||
<li>
|
||||
<button
|
||||
type="button"
|
||||
class="button button--ghost"
|
||||
data-role="debug-play"
|
||||
data-audio-src="{{ item.url }}"
|
||||
>
|
||||
{{ item.label }}
|
||||
</button>
|
||||
<a class="muted" href="{{ item.url }}" download>{{ item.filename }}</a>
|
||||
<div class="field field--stack" style="margin: 0;">
|
||||
<div>
|
||||
<strong>{{ item.label }}</strong>
|
||||
{% if item.text %}
|
||||
— <span class="muted">{{ item.text }}</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
<audio controls preload="none" src="{{ item.url }}"></audio>
|
||||
</div>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
@@ -37,25 +34,4 @@
|
||||
<a href="{{ url_for('settings.settings_page', _anchor='debug') }}" class="button button--ghost">Back to Settings</a>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<script>
|
||||
(function () {
|
||||
const audio = document.querySelector('[data-role="debug-audio"]');
|
||||
const buttons = Array.from(document.querySelectorAll('[data-role="debug-play"]'));
|
||||
if (!audio || !buttons.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
buttons.forEach((button) => {
|
||||
button.addEventListener('click', () => {
|
||||
const src = button.dataset.audioSrc;
|
||||
if (!src) return;
|
||||
if (audio.src !== src) {
|
||||
audio.src = src;
|
||||
}
|
||||
audio.play();
|
||||
});
|
||||
});
|
||||
})();
|
||||
</script>
|
||||
{% endblock %}
|
||||
|
||||
Reference in New Issue
Block a user