mirror of
https://github.com/denizsafak/abogen.git
synced 2026-07-18 13:40:27 +02:00
feat: Enhance text normalization with support for internet slang expansion, currency formatting, and date handling; update debug WAVs interface and settings
This commit is contained in:
@@ -182,7 +182,7 @@ DEBUG_TTS_SAMPLES: Sequence[DebugTTSSample] = (
|
|||||||
DebugTTSSample(
|
DebugTTSSample(
|
||||||
code="TITLE_003",
|
code="TITLE_003",
|
||||||
label="Titles and abbreviations (3)",
|
label="Titles and abbreviations (3)",
|
||||||
text="Gen. Lee spoke to Sgt. Rivera on Main St.",
|
text="Gen. Smith spoke to Sgt. Rivera on Main St.",
|
||||||
),
|
),
|
||||||
DebugTTSSample(
|
DebugTTSSample(
|
||||||
code="TITLE_004",
|
code="TITLE_004",
|
||||||
|
|||||||
@@ -5,7 +5,10 @@ import re
|
|||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple
|
from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple
|
||||||
|
|
||||||
import spacy
|
try: # pragma: no cover - optional dependency
|
||||||
|
import spacy # type: ignore
|
||||||
|
except Exception: # pragma: no cover - spaCy may be unavailable in minimal environments
|
||||||
|
spacy = None
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
@@ -177,6 +180,9 @@ def _build_replacement_sentence(sentence: str, token: str, replacement_token: st
|
|||||||
|
|
||||||
|
|
||||||
def _load_spacy(language: str) -> Any:
|
def _load_spacy(language: str) -> Any:
|
||||||
|
if spacy is None:
|
||||||
|
return None
|
||||||
|
|
||||||
# English only for now.
|
# English only for now.
|
||||||
# Use installed small model; keep it simple.
|
# Use installed small model; keep it simple.
|
||||||
lang = (language or "en").lower()
|
lang = (language or "en").lower()
|
||||||
@@ -211,7 +217,12 @@ def extract_heteronym_overrides(
|
|||||||
if not lang.startswith("en"):
|
if not lang.startswith("en"):
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
if spacy is None:
|
||||||
|
return []
|
||||||
|
|
||||||
nlp = _load_spacy(lang)
|
nlp = _load_spacy(lang)
|
||||||
|
if nlp is None:
|
||||||
|
return []
|
||||||
|
|
||||||
previous_choices: Dict[str, str] = {}
|
previous_choices: Dict[str, str] = {}
|
||||||
if existing:
|
if existing:
|
||||||
|
|||||||
@@ -3,6 +3,8 @@ from __future__ import annotations
|
|||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
import unicodedata
|
import unicodedata
|
||||||
|
import os
|
||||||
|
import locale
|
||||||
from fractions import Fraction
|
from fractions import Fraction
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple
|
from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple
|
||||||
@@ -129,6 +131,176 @@ _URL_RE = re.compile(r"(https?://)?(www\.)?(?P<domain>[a-zA-Z0-9-]+(\.[a-zA-Z0-9
|
|||||||
_FOOTNOTE_RE = re.compile(r"([a-zA-Z]+)(\d+)")
|
_FOOTNOTE_RE = re.compile(r"([a-zA-Z]+)(\d+)")
|
||||||
_BRACKET_FOOTNOTE_RE = re.compile(r"\[\d+\]")
|
_BRACKET_FOOTNOTE_RE = re.compile(r"\[\d+\]")
|
||||||
|
|
||||||
|
_ISO_DATE_RE = re.compile(r"\b(?P<year>\d{4})[/-](?P<month>\d{1,2})[/-](?P<day>\d{1,2})\b")
|
||||||
|
_MDY_DATE_RE = re.compile(
|
||||||
|
r"\b(?P<month>Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\.?\s+"
|
||||||
|
r"(?P<day>\d{1,2})(?:st|nd|rd|th)?\s*,\s*(?P<year>\d{4})\b",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
|
||||||
|
_TIME_RE = re.compile(
|
||||||
|
r"\b(?P<hour>\d{1,2})(?::(?P<minute>\d{2}))?\s*(?P<meridian>a\.?m\.?|p\.?m\.?)\b",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
|
||||||
|
_ADDRESS_ABBR_RE = re.compile(r"(?P<prefix>\b\w+\s+)(?P<abbr>St|Rd|Ave|Blvd|Ln|Dr)\.(?=\s*(?:,|\.|!|\?|$))")
|
||||||
|
|
||||||
|
_MONTH_MAP = {
|
||||||
|
"jan": "January",
|
||||||
|
"january": "January",
|
||||||
|
"feb": "February",
|
||||||
|
"february": "February",
|
||||||
|
"mar": "March",
|
||||||
|
"march": "March",
|
||||||
|
"apr": "April",
|
||||||
|
"april": "April",
|
||||||
|
"may": "May",
|
||||||
|
"jun": "June",
|
||||||
|
"june": "June",
|
||||||
|
"jul": "July",
|
||||||
|
"july": "July",
|
||||||
|
"aug": "August",
|
||||||
|
"august": "August",
|
||||||
|
"sep": "September",
|
||||||
|
"sept": "September",
|
||||||
|
"september": "September",
|
||||||
|
"oct": "October",
|
||||||
|
"october": "October",
|
||||||
|
"nov": "November",
|
||||||
|
"november": "November",
|
||||||
|
"dec": "December",
|
||||||
|
"december": "December",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _is_us_locale() -> bool:
|
||||||
|
for key in ("LC_ALL", "LC_TIME", "LANG"):
|
||||||
|
value = os.environ.get(key)
|
||||||
|
if value and "en_US" in value:
|
||||||
|
return True
|
||||||
|
try:
|
||||||
|
loc = locale.getlocale(locale.LC_TIME)
|
||||||
|
if loc and loc[0] and "en_US" in loc[0]:
|
||||||
|
return True
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _year_to_words_american(value: int, language: str) -> str:
|
||||||
|
if language.lower().startswith("en") and 2000 <= value <= 2099:
|
||||||
|
if value == 2000:
|
||||||
|
return "two thousand"
|
||||||
|
if 2001 <= value <= 2009:
|
||||||
|
tail = _DIGIT_WORDS[value % 10]
|
||||||
|
return f"two thousand {tail}"
|
||||||
|
|
||||||
|
first_two = value // 100
|
||||||
|
last_two = value % 100
|
||||||
|
first_words = _int_to_words(first_two, language) or "twenty"
|
||||||
|
if last_two == 0:
|
||||||
|
return f"{first_words} hundred"
|
||||||
|
if last_two < 10:
|
||||||
|
return f"{first_words} oh {_DIGIT_WORDS[last_two]}"
|
||||||
|
last_words = _int_to_words(last_two, language)
|
||||||
|
return f"{first_words} {last_words or last_two}"
|
||||||
|
|
||||||
|
words = _int_to_words(value, language)
|
||||||
|
return words or str(value)
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_dates(text: str, language: str) -> str:
|
||||||
|
us = _is_us_locale()
|
||||||
|
|
||||||
|
def _format_iso(match: re.Match[str]) -> str:
|
||||||
|
year = int(match.group("year"))
|
||||||
|
month = int(match.group("month"))
|
||||||
|
day = int(match.group("day"))
|
||||||
|
if not (1 <= month <= 12 and 1 <= day <= 31):
|
||||||
|
return match.group(0)
|
||||||
|
month_name = (
|
||||||
|
[
|
||||||
|
"January",
|
||||||
|
"February",
|
||||||
|
"March",
|
||||||
|
"April",
|
||||||
|
"May",
|
||||||
|
"June",
|
||||||
|
"July",
|
||||||
|
"August",
|
||||||
|
"September",
|
||||||
|
"October",
|
||||||
|
"November",
|
||||||
|
"December",
|
||||||
|
][month - 1]
|
||||||
|
)
|
||||||
|
ordinal = _int_to_ordinal_words(day, language) or str(day)
|
||||||
|
year_words = _year_to_words_american(year, language)
|
||||||
|
return f"{month_name} {ordinal}, {year_words}" if us else f"{ordinal} {month_name} {year_words}"
|
||||||
|
|
||||||
|
def _format_mdy(match: re.Match[str]) -> str:
|
||||||
|
month_raw = str(match.group("month") or "").strip().lower().rstrip(".")
|
||||||
|
month_name = _MONTH_MAP.get(month_raw)
|
||||||
|
if not month_name:
|
||||||
|
return match.group(0)
|
||||||
|
day = int(match.group("day"))
|
||||||
|
year = int(match.group("year"))
|
||||||
|
ordinal = _int_to_ordinal_words(day, language) or str(day)
|
||||||
|
year_words = _year_to_words_american(year, language)
|
||||||
|
return f"{month_name} {ordinal}, {year_words}" if us else f"{ordinal} {month_name} {year_words}"
|
||||||
|
|
||||||
|
out = _ISO_DATE_RE.sub(_format_iso, text)
|
||||||
|
out = _MDY_DATE_RE.sub(_format_mdy, out)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_times(text: str) -> str:
|
||||||
|
def _replace(match: re.Match[str]) -> str:
|
||||||
|
hour = match.group("hour")
|
||||||
|
minute = match.group("minute")
|
||||||
|
meridian = match.group("meridian").lower().replace(".", "")
|
||||||
|
if minute:
|
||||||
|
return f"{hour}:{minute} {meridian}"
|
||||||
|
return f"{hour} {meridian}"
|
||||||
|
|
||||||
|
return _TIME_RE.sub(_replace, text)
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_address_abbreviations(text: str) -> str:
|
||||||
|
mapping = {
|
||||||
|
"st": "street",
|
||||||
|
"rd": "road",
|
||||||
|
"ave": "avenue",
|
||||||
|
"blvd": "boulevard",
|
||||||
|
"ln": "lane",
|
||||||
|
"dr": "drive",
|
||||||
|
}
|
||||||
|
|
||||||
|
def _replace(match: re.Match[str]) -> str:
|
||||||
|
abbr = match.group("abbr")
|
||||||
|
full = mapping.get(abbr.lower())
|
||||||
|
if not full:
|
||||||
|
return match.group(0)
|
||||||
|
return match.group("prefix") + _match_casing(abbr, full)
|
||||||
|
|
||||||
|
return _ADDRESS_ABBR_RE.sub(_replace, text)
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_internet_slang(text: str) -> str:
|
||||||
|
mapping = {
|
||||||
|
"pls": "please",
|
||||||
|
"plz": "please",
|
||||||
|
}
|
||||||
|
|
||||||
|
def _replace(match: re.Match[str]) -> str:
|
||||||
|
token = match.group(0)
|
||||||
|
replacement = mapping.get(token.lower())
|
||||||
|
if not replacement:
|
||||||
|
return token
|
||||||
|
return _match_casing(token, replacement)
|
||||||
|
|
||||||
|
return re.sub(r"\b(?:pls|plz)\b", _replace, text, flags=re.IGNORECASE)
|
||||||
|
|
||||||
_DECIMAL_NUMBER_RE = re.compile(
|
_DECIMAL_NUMBER_RE = re.compile(
|
||||||
rf"(?<![\w{_NUMBER_RANGE_CLASS}/])(?P<number>-?(?:\d{{1,3}}(?:,\d{{3}})+|\d+)\.(?P<fraction>\d+))(?![\w{_NUMBER_RANGE_CLASS}/])"
|
rf"(?<![\w{_NUMBER_RANGE_CLASS}/])(?P<number>-?(?:\d{{1,3}}(?:,\d{{3}})+|\d+)\.(?P<fraction>\d+))(?![\w{_NUMBER_RANGE_CLASS}/])"
|
||||||
)
|
)
|
||||||
@@ -155,14 +327,50 @@ def _int_to_words(value: int, language: str) -> Optional[str]:
|
|||||||
|
|
||||||
|
|
||||||
def _int_to_ordinal_words(value: int, language: str) -> Optional[str]:
|
def _int_to_ordinal_words(value: int, language: str) -> Optional[str]:
|
||||||
if num2words is None:
|
if num2words is not None:
|
||||||
return None
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
return num2words(value, lang=language, ordinal=True)
|
return num2words(value, lang=language, ordinal=True)
|
||||||
except Exception: # pragma: no cover - unsupported locale
|
except Exception: # pragma: no cover - unsupported locale
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
if language.lower().startswith("en"):
|
||||||
|
ordinals = {
|
||||||
|
1: "first",
|
||||||
|
2: "second",
|
||||||
|
3: "third",
|
||||||
|
4: "fourth",
|
||||||
|
5: "fifth",
|
||||||
|
6: "sixth",
|
||||||
|
7: "seventh",
|
||||||
|
8: "eighth",
|
||||||
|
9: "ninth",
|
||||||
|
10: "tenth",
|
||||||
|
11: "eleventh",
|
||||||
|
12: "twelfth",
|
||||||
|
13: "thirteenth",
|
||||||
|
14: "fourteenth",
|
||||||
|
15: "fifteenth",
|
||||||
|
16: "sixteenth",
|
||||||
|
17: "seventeenth",
|
||||||
|
18: "eighteenth",
|
||||||
|
19: "nineteenth",
|
||||||
|
20: "twentieth",
|
||||||
|
21: "twenty-first",
|
||||||
|
22: "twenty-second",
|
||||||
|
23: "twenty-third",
|
||||||
|
24: "twenty-fourth",
|
||||||
|
25: "twenty-fifth",
|
||||||
|
26: "twenty-sixth",
|
||||||
|
27: "twenty-seventh",
|
||||||
|
28: "twenty-eighth",
|
||||||
|
29: "twenty-ninth",
|
||||||
|
30: "thirtieth",
|
||||||
|
31: "thirty-first",
|
||||||
|
}
|
||||||
|
return ordinals.get(int(value))
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _pluralize_fraction_word(base: str) -> str:
|
def _pluralize_fraction_word(base: str) -> str:
|
||||||
if base == "half":
|
if base == "half":
|
||||||
@@ -366,6 +574,8 @@ TITLE_ABBREVIATIONS = {
|
|||||||
"dr": "doctor",
|
"dr": "doctor",
|
||||||
"prof": "professor",
|
"prof": "professor",
|
||||||
"rev": "reverend",
|
"rev": "reverend",
|
||||||
|
"gen": "general",
|
||||||
|
"sgt": "sergeant",
|
||||||
}
|
}
|
||||||
|
|
||||||
SUFFIX_ABBREVIATIONS = {
|
SUFFIX_ABBREVIATIONS = {
|
||||||
@@ -1146,7 +1356,23 @@ def classify_token(token: str, cfg: ApostropheConfig) -> Tuple[str, str]:
|
|||||||
# 1. Decades
|
# 1. Decades
|
||||||
if DECADE_RE.match(token):
|
if DECADE_RE.match(token):
|
||||||
if cfg.decades_mode == "expand":
|
if cfg.decades_mode == "expand":
|
||||||
return "decade", f"19{token[2:4]}s"
|
decade_digit = token[1]
|
||||||
|
decade_map = {
|
||||||
|
"0": "two thousands",
|
||||||
|
"1": "tens",
|
||||||
|
"2": "twenties",
|
||||||
|
"3": "thirties",
|
||||||
|
"4": "forties",
|
||||||
|
"5": "fifties",
|
||||||
|
"6": "sixties",
|
||||||
|
"7": "seventies",
|
||||||
|
"8": "eighties",
|
||||||
|
"9": "nineties",
|
||||||
|
}
|
||||||
|
spoken = decade_map.get(decade_digit)
|
||||||
|
if spoken:
|
||||||
|
return "decade", spoken
|
||||||
|
return "decade", token
|
||||||
return "decade", token
|
return "decade", token
|
||||||
|
|
||||||
# 2. Leading elision
|
# 2. Leading elision
|
||||||
@@ -1312,7 +1538,7 @@ def normalize_apostrophes(text: str, cfg: ApostropheConfig | None = None) -> Tup
|
|||||||
return normalized_text, results
|
return normalized_text, results
|
||||||
|
|
||||||
def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
|
def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
|
||||||
if not text or not cfg.convert_numbers:
|
if not text:
|
||||||
return text
|
return text
|
||||||
|
|
||||||
language = (cfg.number_lang or "en").strip() or "en"
|
language = (cfg.number_lang or "en").strip() or "en"
|
||||||
@@ -1360,6 +1586,20 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
|
|||||||
return words.replace(" and ", " ")
|
return words.replace(" and ", " ")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
# 1200s are commonly spoken as "twelve hundred".
|
||||||
|
if 1200 <= value < 1300:
|
||||||
|
hundreds = value // 100
|
||||||
|
remainder = value % 100
|
||||||
|
prefix = _words(hundreds)
|
||||||
|
if not prefix:
|
||||||
|
return None
|
||||||
|
if remainder == 0:
|
||||||
|
return f"{prefix} hundred"
|
||||||
|
tail = _format_year_tail(remainder, allow_oh=True)
|
||||||
|
if tail is None:
|
||||||
|
return None
|
||||||
|
return f"{prefix} hundred {tail}".strip()
|
||||||
|
|
||||||
if value % 1000 == 0:
|
if value % 1000 == 0:
|
||||||
thousands = value // 1000
|
thousands = value // 1000
|
||||||
thousands_words = _words(thousands)
|
thousands_words = _words(thousands)
|
||||||
@@ -1537,6 +1777,29 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
|
|||||||
|
|
||||||
return f"{amount_spoken} {magnitude} {currency_name}"
|
return f"{amount_spoken} {magnitude} {currency_name}"
|
||||||
|
|
||||||
|
# Handle $0.99 -> ninety-nine cents (avoid "zero dollars and...").
|
||||||
|
if amount_str.startswith("0") and amount < 1.0:
|
||||||
|
dollars_part, dot, fraction = amount_str.partition(".")
|
||||||
|
if dot and fraction:
|
||||||
|
cents_str = (fraction + "00")[:2]
|
||||||
|
try:
|
||||||
|
cents_value = int(cents_str)
|
||||||
|
except ValueError:
|
||||||
|
cents_value = 0
|
||||||
|
if cents_value > 0:
|
||||||
|
cents_words = _int_to_words(cents_value, language) or str(cents_value)
|
||||||
|
subunit = {
|
||||||
|
"$": "cent",
|
||||||
|
"€": "cent",
|
||||||
|
"£": "penny",
|
||||||
|
"¥": "yen",
|
||||||
|
}.get(symbol, "cent")
|
||||||
|
if symbol == "£":
|
||||||
|
subunit = "pence" if cents_value != 1 else "penny"
|
||||||
|
else:
|
||||||
|
subunit = (subunit + "s") if cents_value != 1 and subunit not in {"pence", "yen"} else subunit
|
||||||
|
return f"{cents_words} {subunit}".strip()
|
||||||
|
|
||||||
currency_map = {
|
currency_map = {
|
||||||
"$": "USD",
|
"$": "USD",
|
||||||
"£": "GBP",
|
"£": "GBP",
|
||||||
@@ -1560,6 +1823,13 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
|
|||||||
def _replace_url(match: re.Match[str]) -> str:
|
def _replace_url(match: re.Match[str]) -> str:
|
||||||
domain = match.group("domain")
|
domain = match.group("domain")
|
||||||
|
|
||||||
|
# Avoid matching dotted abbreviations/acronyms without an explicit URL prefix.
|
||||||
|
has_prefix = match.group(1) or match.group(2)
|
||||||
|
if not has_prefix:
|
||||||
|
parts = [p for p in domain.split(".") if p]
|
||||||
|
if parts and all(p.isalpha() and len(p) <= 2 for p in parts):
|
||||||
|
return match.group(0)
|
||||||
|
|
||||||
# Avoid matching numbers like 1.05 or 12.34.56
|
# Avoid matching numbers like 1.05 or 12.34.56
|
||||||
# If the domain consists only of digits and dots, ignore it (unless it has http/www prefix)
|
# If the domain consists only of digits and dots, ignore it (unless it has http/www prefix)
|
||||||
has_prefix = match.group(1) or match.group(2)
|
has_prefix = match.group(1) or match.group(2)
|
||||||
@@ -1584,7 +1854,7 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
|
|||||||
|
|
||||||
normalized = text
|
normalized = text
|
||||||
|
|
||||||
# Apply URL replacement first
|
# Apply URL replacement first (independent of number conversion).
|
||||||
normalized = _URL_RE.sub(_replace_url, normalized)
|
normalized = _URL_RE.sub(_replace_url, normalized)
|
||||||
|
|
||||||
if getattr(cfg, "remove_footnotes", False):
|
if getattr(cfg, "remove_footnotes", False):
|
||||||
@@ -1593,6 +1863,8 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
|
|||||||
|
|
||||||
if cfg.convert_currency:
|
if cfg.convert_currency:
|
||||||
normalized = _CURRENCY_RE.sub(_replace_currency, normalized)
|
normalized = _CURRENCY_RE.sub(_replace_currency, normalized)
|
||||||
|
|
||||||
|
if cfg.convert_numbers:
|
||||||
normalized = _NUMBER_RANGE_RE.sub(lambda m: _replace_number_range(m, language), normalized)
|
normalized = _NUMBER_RANGE_RE.sub(lambda m: _replace_number_range(m, language), normalized)
|
||||||
normalized = _NUMBER_SPACE_RANGE_RE.sub(lambda m: _replace_space_separated_range(m, language), normalized)
|
normalized = _NUMBER_SPACE_RANGE_RE.sub(lambda m: _replace_space_separated_range(m, language), normalized)
|
||||||
normalized = _FRACTION_RE.sub(lambda m: _replace_fraction(m, language), normalized)
|
normalized = _FRACTION_RE.sub(lambda m: _replace_fraction(m, language), normalized)
|
||||||
@@ -1600,7 +1872,15 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
|
|||||||
normalized = _NUMBER_WITH_GROUP_RE.sub(_replace_grouped, normalized)
|
normalized = _NUMBER_WITH_GROUP_RE.sub(_replace_grouped, normalized)
|
||||||
normalized = _PLAIN_NUMBER_RE.sub(_replace_plain, normalized)
|
normalized = _PLAIN_NUMBER_RE.sub(_replace_plain, normalized)
|
||||||
normalized = _normalize_roman_numerals(normalized, language)
|
normalized = _normalize_roman_numerals(normalized, language)
|
||||||
|
|
||||||
return normalized
|
return normalized
|
||||||
|
def _normalize_dotted_acronyms(text: str) -> str:
|
||||||
|
def _replace(match: re.Match[str]) -> str:
|
||||||
|
value = match.group(0)
|
||||||
|
compact = value.replace(".", "")
|
||||||
|
return compact
|
||||||
|
|
||||||
|
return re.sub(r"\b(?:[A-Z]\.){1,}[A-Z]\.?(?=\W|$)", _replace, text)
|
||||||
|
|
||||||
# ---------- Optional phoneme hint post-processing ----------
|
# ---------- Optional phoneme hint post-processing ----------
|
||||||
|
|
||||||
@@ -1909,21 +2189,31 @@ def normalize_for_pipeline(
|
|||||||
mode = str(runtime_settings.get("normalization_apostrophe_mode", "spacy")).lower()
|
mode = str(runtime_settings.get("normalization_apostrophe_mode", "spacy")).lower()
|
||||||
normalized = text
|
normalized = text
|
||||||
|
|
||||||
|
# Pre-normalization that must happen before number/url parsing.
|
||||||
|
if runtime_settings.get("normalization_numbers", True):
|
||||||
|
normalized = _normalize_dates(normalized, cfg.number_lang)
|
||||||
|
normalized = _normalize_times(normalized)
|
||||||
|
normalized = _normalize_dotted_acronyms(normalized)
|
||||||
|
if runtime_settings.get("normalization_titles", True):
|
||||||
|
normalized = _normalize_address_abbreviations(normalized)
|
||||||
|
if runtime_settings.get("normalization_internet_slang", False):
|
||||||
|
normalized = _normalize_internet_slang(normalized)
|
||||||
|
|
||||||
if mode == "off":
|
if mode == "off":
|
||||||
normalized = normalize_unicode_apostrophes(text)
|
normalized = normalize_unicode_apostrophes(normalized)
|
||||||
if cfg.convert_numbers:
|
if cfg.convert_numbers or cfg.convert_currency or getattr(cfg, "remove_footnotes", False):
|
||||||
normalized = _normalize_grouped_numbers(normalized, cfg)
|
normalized = _normalize_grouped_numbers(normalized, cfg)
|
||||||
normalized = _cleanup_spacing(normalized)
|
normalized = _cleanup_spacing(normalized)
|
||||||
elif mode == "llm":
|
elif mode == "llm":
|
||||||
try:
|
try:
|
||||||
normalized = _normalize_with_llm(text, settings=runtime_settings, config=cfg)
|
normalized = _normalize_with_llm(normalized, settings=runtime_settings, config=cfg)
|
||||||
except LLMClientError:
|
except LLMClientError:
|
||||||
raise
|
raise
|
||||||
if cfg.convert_numbers:
|
if cfg.convert_numbers or cfg.convert_currency or getattr(cfg, "remove_footnotes", False):
|
||||||
normalized = _normalize_grouped_numbers(normalized, cfg)
|
normalized = _normalize_grouped_numbers(normalized, cfg)
|
||||||
normalized = _cleanup_spacing(normalized)
|
normalized = _cleanup_spacing(normalized)
|
||||||
else:
|
else:
|
||||||
normalized, _ = normalize_apostrophes(text, cfg)
|
normalized, _ = normalize_apostrophes(normalized, cfg)
|
||||||
|
|
||||||
if runtime_settings.get("normalization_titles", True):
|
if runtime_settings.get("normalization_titles", True):
|
||||||
normalized = expand_titles_and_suffixes(normalized)
|
normalized = expand_titles_and_suffixes(normalized)
|
||||||
|
|||||||
@@ -38,6 +38,7 @@ _SETTINGS_DEFAULTS: Dict[str, Any] = {
|
|||||||
"normalization_terminal": True,
|
"normalization_terminal": True,
|
||||||
"normalization_phoneme_hints": True,
|
"normalization_phoneme_hints": True,
|
||||||
"normalization_caps_quotes": True,
|
"normalization_caps_quotes": True,
|
||||||
|
"normalization_internet_slang": False,
|
||||||
"normalization_apostrophes_contractions": True,
|
"normalization_apostrophes_contractions": True,
|
||||||
"normalization_apostrophes_plural_possessives": True,
|
"normalization_apostrophes_plural_possessives": True,
|
||||||
"normalization_apostrophes_sibilant_possessives": True,
|
"normalization_apostrophes_sibilant_possessives": True,
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ class DebugWavArtifact:
|
|||||||
label: str
|
label: str
|
||||||
filename: str
|
filename: str
|
||||||
code: Optional[str] = None
|
code: Optional[str] = None
|
||||||
|
text: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
def _resolve_voice_setting(value: str) -> tuple[str, Optional[str], Optional[str]]:
|
def _resolve_voice_setting(value: str) -> tuple[str, Optional[str], Optional[str]]:
|
||||||
@@ -65,6 +66,21 @@ def _extract_cases_from_text(text: str) -> List[Tuple[str, str]]:
|
|||||||
return cases
|
return cases
|
||||||
|
|
||||||
|
|
||||||
|
def _spoken_id(code: str) -> str:
|
||||||
|
# Make IDs pronounceable and stable (avoid reading as a word).
|
||||||
|
out: List[str] = []
|
||||||
|
for ch in str(code or ""):
|
||||||
|
if ch == "_":
|
||||||
|
out.append(" ")
|
||||||
|
elif ch.isalnum():
|
||||||
|
out.append(ch)
|
||||||
|
else:
|
||||||
|
out.append(" ")
|
||||||
|
# Add spaces between alnum to encourage letter-by-letter reading.
|
||||||
|
spaced = " ".join("".join(out).split())
|
||||||
|
return spaced
|
||||||
|
|
||||||
|
|
||||||
def run_debug_tts_wavs(
|
def run_debug_tts_wavs(
|
||||||
*,
|
*,
|
||||||
output_root: Path,
|
output_root: Path,
|
||||||
@@ -96,6 +112,14 @@ def run_debug_tts_wavs(
|
|||||||
combined_text = extraction.combined_text or "\n\n".join((c.text or "") for c in extraction.chapters)
|
combined_text = extraction.combined_text or "\n\n".join((c.text or "") for c in extraction.chapters)
|
||||||
cases = _extract_cases_from_text(combined_text)
|
cases = _extract_cases_from_text(combined_text)
|
||||||
|
|
||||||
|
# Prefer the canonical sample catalog for text (EPUB extraction may include headings).
|
||||||
|
try:
|
||||||
|
from abogen.debug_tts_samples import DEBUG_TTS_SAMPLES
|
||||||
|
|
||||||
|
sample_text_by_code = {sample.code: sample.text for sample in DEBUG_TTS_SAMPLES}
|
||||||
|
except Exception:
|
||||||
|
sample_text_by_code = {}
|
||||||
|
|
||||||
expected = list(iter_expected_codes())
|
expected = list(iter_expected_codes())
|
||||||
found_codes = {code for code, _ in cases}
|
found_codes = {code for code, _ in cases}
|
||||||
missing = [code for code in expected if code not in found_codes]
|
missing = [code for code in expected if code not in found_codes]
|
||||||
@@ -162,12 +186,16 @@ def run_debug_tts_wavs(
|
|||||||
overall_path = run_dir / "overall.wav"
|
overall_path = run_dir / "overall.wav"
|
||||||
overall_audio: List[np.ndarray] = []
|
overall_audio: List[np.ndarray] = []
|
||||||
|
|
||||||
def synth(text: str) -> np.ndarray:
|
def synth(text: str, *, apply_normalization: bool = True) -> np.ndarray:
|
||||||
normalized = normalize_for_pipeline(
|
normalized = (
|
||||||
|
normalize_for_pipeline(
|
||||||
text,
|
text,
|
||||||
config=apostrophe_config,
|
config=apostrophe_config,
|
||||||
settings=normalization_settings,
|
settings=normalization_settings,
|
||||||
)
|
)
|
||||||
|
if apply_normalization
|
||||||
|
else str(text or "")
|
||||||
|
)
|
||||||
parts: List[np.ndarray] = []
|
parts: List[np.ndarray] = []
|
||||||
for segment in pipeline(
|
for segment in pipeline(
|
||||||
normalized,
|
normalized,
|
||||||
@@ -182,19 +210,26 @@ def run_debug_tts_wavs(
|
|||||||
return np.zeros(0, dtype="float32")
|
return np.zeros(0, dtype="float32")
|
||||||
return np.concatenate(parts).astype("float32", copy=False)
|
return np.concatenate(parts).astype("float32", copy=False)
|
||||||
|
|
||||||
|
pause_1s = np.zeros(int(1.0 * SAMPLE_RATE), dtype="float32")
|
||||||
|
between_cases = np.zeros(int(0.35 * SAMPLE_RATE), dtype="float32")
|
||||||
|
|
||||||
# Per sample
|
# Per sample
|
||||||
for code, snippet in cases:
|
for code, snippet in cases:
|
||||||
|
snippet = sample_text_by_code.get(code, snippet)
|
||||||
if not snippet:
|
if not snippet:
|
||||||
continue
|
continue
|
||||||
audio = synth(snippet)
|
id_audio = synth(_spoken_id(code), apply_normalization=False)
|
||||||
|
text_audio = synth(snippet, apply_normalization=True)
|
||||||
|
audio = np.concatenate([id_audio, pause_1s, text_audio]).astype("float32", copy=False)
|
||||||
filename = f"case_{code}.wav"
|
filename = f"case_{code}.wav"
|
||||||
path = run_dir / filename
|
path = run_dir / filename
|
||||||
# Write float32 PCM WAV.
|
# Write float32 PCM WAV.
|
||||||
import soundfile as sf
|
import soundfile as sf
|
||||||
|
|
||||||
sf.write(path, audio, SAMPLE_RATE, subtype="FLOAT")
|
sf.write(path, audio, SAMPLE_RATE, subtype="FLOAT")
|
||||||
artifacts.append(DebugWavArtifact(label=f"{code}", filename=filename, code=code))
|
artifacts.append(DebugWavArtifact(label=f"{code}", filename=filename, code=code, text=snippet))
|
||||||
overall_audio.append(audio)
|
overall_audio.append(audio)
|
||||||
|
overall_audio.append(between_cases)
|
||||||
|
|
||||||
# Overall
|
# Overall
|
||||||
if overall_audio:
|
if overall_audio:
|
||||||
@@ -204,7 +239,7 @@ def run_debug_tts_wavs(
|
|||||||
import soundfile as sf
|
import soundfile as sf
|
||||||
|
|
||||||
sf.write(overall_path, combined, SAMPLE_RATE, subtype="FLOAT")
|
sf.write(overall_path, combined, SAMPLE_RATE, subtype="FLOAT")
|
||||||
artifacts.insert(0, DebugWavArtifact(label="Overall", filename="overall.wav", code=None))
|
artifacts.insert(0, DebugWavArtifact(label="Overall", filename="overall.wav", code=None, text=None))
|
||||||
|
|
||||||
manifest = {
|
manifest = {
|
||||||
"run_id": run_id,
|
"run_id": run_id,
|
||||||
|
|||||||
@@ -80,9 +80,17 @@ def update_settings() -> ResponseReturnValue:
|
|||||||
maximum=25,
|
maximum=25,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _extract_checkbox(name: str, default: bool) -> bool:
|
||||||
|
values = form.getlist(name) if hasattr(form, "getlist") else []
|
||||||
|
if values:
|
||||||
|
return coerce_bool(values[-1], default)
|
||||||
|
if hasattr(form, "__contains__") and name in form:
|
||||||
|
return False
|
||||||
|
return default
|
||||||
|
|
||||||
# Normalization settings
|
# Normalization settings
|
||||||
for key in _NORMALIZATION_BOOLEAN_KEYS:
|
for key in _NORMALIZATION_BOOLEAN_KEYS:
|
||||||
current[key] = coerce_bool(form.get(key), False)
|
current[key] = _extract_checkbox(key, bool(current.get(key, True)))
|
||||||
for key in _NORMALIZATION_STRING_KEYS:
|
for key in _NORMALIZATION_STRING_KEYS:
|
||||||
current[key] = (form.get(key) or "").strip()
|
current[key] = (form.get(key) or "").strip()
|
||||||
|
|
||||||
@@ -236,7 +244,8 @@ def download_debug_wav(run_id: str, filename: str) -> ResponseReturnValue:
|
|||||||
safe_name = (filename or "").strip()
|
safe_name = (filename or "").strip()
|
||||||
if not safe_run or not safe_name or "/" in safe_name or "\\" in safe_name:
|
if not safe_run or not safe_name or "/" in safe_name or "\\" in safe_name:
|
||||||
abort(404)
|
abort(404)
|
||||||
if not safe_name.lower().endswith(".wav") and safe_name != "manifest.json":
|
is_wav = safe_name.lower().endswith(".wav")
|
||||||
|
if not is_wav and safe_name != "manifest.json":
|
||||||
abort(404)
|
abort(404)
|
||||||
|
|
||||||
root = Path(current_app.config.get("OUTPUT_FOLDER") or get_user_output_path("web"))
|
root = Path(current_app.config.get("OUTPUT_FOLDER") or get_user_output_path("web"))
|
||||||
@@ -247,4 +256,12 @@ def download_debug_wav(run_id: str, filename: str) -> ResponseReturnValue:
|
|||||||
expected_dir = (root / "debug" / safe_run).resolve()
|
expected_dir = (root / "debug" / safe_run).resolve()
|
||||||
if expected_dir not in path.parents:
|
if expected_dir not in path.parents:
|
||||||
abort(404)
|
abort(404)
|
||||||
return send_file(path, as_attachment=True, download_name=path.name)
|
wants_download = str(request.args.get("download") or "").strip().lower() in {"1", "true", "yes"}
|
||||||
|
mimetype = "audio/wav" if is_wav else "application/json"
|
||||||
|
# Inline playback should work for WAVs; allow explicit downloads via ?download=1.
|
||||||
|
return send_file(
|
||||||
|
path,
|
||||||
|
mimetype=mimetype,
|
||||||
|
as_attachment=wants_download,
|
||||||
|
download_name=path.name,
|
||||||
|
)
|
||||||
|
|||||||
@@ -47,6 +47,9 @@ _NORMALIZATION_BOOLEAN_KEYS = {
|
|||||||
"normalization_terminal",
|
"normalization_terminal",
|
||||||
"normalization_phoneme_hints",
|
"normalization_phoneme_hints",
|
||||||
"normalization_caps_quotes",
|
"normalization_caps_quotes",
|
||||||
|
"normalization_currency",
|
||||||
|
"normalization_footnotes",
|
||||||
|
"normalization_internet_slang",
|
||||||
"normalization_apostrophes_contractions",
|
"normalization_apostrophes_contractions",
|
||||||
"normalization_apostrophes_plural_possessives",
|
"normalization_apostrophes_plural_possessives",
|
||||||
"normalization_apostrophes_sibilant_possessives",
|
"normalization_apostrophes_sibilant_possessives",
|
||||||
@@ -58,8 +61,6 @@ _NORMALIZATION_BOOLEAN_KEYS = {
|
|||||||
"normalization_contraction_modal_would",
|
"normalization_contraction_modal_would",
|
||||||
"normalization_contraction_negation_not",
|
"normalization_contraction_negation_not",
|
||||||
"normalization_contraction_let_us",
|
"normalization_contraction_let_us",
|
||||||
"normalization_currency",
|
|
||||||
"normalization_footnotes",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
_NORMALIZATION_STRING_KEYS = {
|
_NORMALIZATION_STRING_KEYS = {
|
||||||
@@ -84,6 +85,9 @@ BOOLEAN_SETTINGS = {
|
|||||||
"normalization_terminal",
|
"normalization_terminal",
|
||||||
"normalization_phoneme_hints",
|
"normalization_phoneme_hints",
|
||||||
"normalization_caps_quotes",
|
"normalization_caps_quotes",
|
||||||
|
"normalization_currency",
|
||||||
|
"normalization_footnotes",
|
||||||
|
"normalization_internet_slang",
|
||||||
"normalization_apostrophes_contractions",
|
"normalization_apostrophes_contractions",
|
||||||
"normalization_apostrophes_plural_possessives",
|
"normalization_apostrophes_plural_possessives",
|
||||||
"normalization_apostrophes_sibilant_possessives",
|
"normalization_apostrophes_sibilant_possessives",
|
||||||
@@ -107,6 +111,7 @@ _NORMALIZATION_GROUPS = [
|
|||||||
{"key": "normalization_numbers", "label": "Convert grouped numbers to words"},
|
{"key": "normalization_numbers", "label": "Convert grouped numbers to words"},
|
||||||
{"key": "normalization_currency", "label": "Convert currency symbols ($10 → ten dollars)"},
|
{"key": "normalization_currency", "label": "Convert currency symbols ($10 → ten dollars)"},
|
||||||
{"key": "normalization_titles", "label": "Expand titles and suffixes (Dr., St., Jr., …)"},
|
{"key": "normalization_titles", "label": "Expand titles and suffixes (Dr., St., Jr., …)"},
|
||||||
|
{"key": "normalization_internet_slang", "label": "Expand internet slang (pls → please)"},
|
||||||
{"key": "normalization_footnotes", "label": "Remove footnote indicators ([1], [2])"},
|
{"key": "normalization_footnotes", "label": "Remove footnote indicators ([1], [2])"},
|
||||||
{"key": "normalization_terminal", "label": "Ensure sentences end with terminal punctuation"},
|
{"key": "normalization_terminal", "label": "Ensure sentences end with terminal punctuation"},
|
||||||
{"key": "normalization_caps_quotes", "label": "Convert ALL CAPS dialogue inside quotes"},
|
{"key": "normalization_caps_quotes", "label": "Convert ALL CAPS dialogue inside quotes"},
|
||||||
@@ -195,10 +200,13 @@ def settings_defaults() -> Dict[str, Any]:
|
|||||||
"llm_prompt": llm_env_defaults.get("llm_prompt", DEFAULT_LLM_PROMPT),
|
"llm_prompt": llm_env_defaults.get("llm_prompt", DEFAULT_LLM_PROMPT),
|
||||||
"llm_context_mode": llm_env_defaults.get("llm_context_mode", "sentence"),
|
"llm_context_mode": llm_env_defaults.get("llm_context_mode", "sentence"),
|
||||||
"normalization_numbers": True,
|
"normalization_numbers": True,
|
||||||
|
"normalization_currency": True,
|
||||||
|
"normalization_footnotes": True,
|
||||||
"normalization_titles": True,
|
"normalization_titles": True,
|
||||||
"normalization_terminal": True,
|
"normalization_terminal": True,
|
||||||
"normalization_phoneme_hints": True,
|
"normalization_phoneme_hints": True,
|
||||||
"normalization_caps_quotes": True,
|
"normalization_caps_quotes": True,
|
||||||
|
"normalization_internet_slang": False,
|
||||||
"normalization_apostrophes_contractions": True,
|
"normalization_apostrophes_contractions": True,
|
||||||
"normalization_apostrophes_plural_possessives": True,
|
"normalization_apostrophes_plural_possessives": True,
|
||||||
"normalization_apostrophes_sibilant_possessives": True,
|
"normalization_apostrophes_sibilant_possessives": True,
|
||||||
|
|||||||
@@ -7,10 +7,7 @@
|
|||||||
<h1 class="card__title">Debug WAVs</h1>
|
<h1 class="card__title">Debug WAVs</h1>
|
||||||
<p class="tag">Run ID: <code>{{ run_id }}</code></p>
|
<p class="tag">Run ID: <code>{{ run_id }}</code></p>
|
||||||
|
|
||||||
<div class="field field--stack">
|
<p class="hint">Each clip reads: ID, one second pause, then the reference text.</p>
|
||||||
<audio data-role="debug-audio" controls preload="none" style="width: 100%;"></audio>
|
|
||||||
<p class="hint">Click an ID to play its WAV. “Overall” is the concatenation of all samples.</p>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{% if artifacts %}
|
{% if artifacts %}
|
||||||
<div class="field field--wide">
|
<div class="field field--wide">
|
||||||
@@ -18,15 +15,15 @@
|
|||||||
<ul>
|
<ul>
|
||||||
{% for item in artifacts %}
|
{% for item in artifacts %}
|
||||||
<li>
|
<li>
|
||||||
<button
|
<div class="field field--stack" style="margin: 0;">
|
||||||
type="button"
|
<div>
|
||||||
class="button button--ghost"
|
<strong>{{ item.label }}</strong>
|
||||||
data-role="debug-play"
|
{% if item.text %}
|
||||||
data-audio-src="{{ item.url }}"
|
— <span class="muted">{{ item.text }}</span>
|
||||||
>
|
{% endif %}
|
||||||
{{ item.label }}
|
</div>
|
||||||
</button>
|
<audio controls preload="none" src="{{ item.url }}"></audio>
|
||||||
<a class="muted" href="{{ item.url }}" download>{{ item.filename }}</a>
|
</div>
|
||||||
</li>
|
</li>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
</ul>
|
</ul>
|
||||||
@@ -37,25 +34,4 @@
|
|||||||
<a href="{{ url_for('settings.settings_page', _anchor='debug') }}" class="button button--ghost">Back to Settings</a>
|
<a href="{{ url_for('settings.settings_page', _anchor='debug') }}" class="button button--ghost">Back to Settings</a>
|
||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
<script>
|
|
||||||
(function () {
|
|
||||||
const audio = document.querySelector('[data-role="debug-audio"]');
|
|
||||||
const buttons = Array.from(document.querySelectorAll('[data-role="debug-play"]'));
|
|
||||||
if (!audio || !buttons.length) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
buttons.forEach((button) => {
|
|
||||||
button.addEventListener('click', () => {
|
|
||||||
const src = button.dataset.audioSrc;
|
|
||||||
if (!src) return;
|
|
||||||
if (audio.src !== src) {
|
|
||||||
audio.src = src;
|
|
||||||
}
|
|
||||||
audio.play();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
})();
|
|
||||||
</script>
|
|
||||||
{% endblock %}
|
{% endblock %}
|
||||||
|
|||||||
+28
-2
@@ -1,9 +1,13 @@
|
|||||||
import importlib
|
import importlib
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
# Ensure real optional dependencies are imported before tests that install stubs
|
# Ensure real optional dependencies are imported before tests that install stubs
|
||||||
# so that available packages (like ebooklib, bs4) aren't replaced with dummy modules.
|
# so that available packages (like ebooklib, bs4, numpy) aren't replaced with dummy modules.
|
||||||
for module_name in ("ebooklib", "bs4"):
|
for module_name in ("ebooklib", "bs4", "numpy"):
|
||||||
if module_name not in sys.modules:
|
if module_name not in sys.modules:
|
||||||
try:
|
try:
|
||||||
importlib.import_module(module_name)
|
importlib.import_module(module_name)
|
||||||
@@ -11,3 +15,25 @@ for module_name in ("ebooklib", "bs4"):
|
|||||||
# On environments without the optional dependency, downstream tests
|
# On environments without the optional dependency, downstream tests
|
||||||
# will install lightweight stubs as needed.
|
# will install lightweight stubs as needed.
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True, scope="session")
|
||||||
|
def _isolate_settings_dir(tmp_path_factory: pytest.TempPathFactory):
|
||||||
|
settings_dir = tmp_path_factory.mktemp("abogen-settings")
|
||||||
|
os.environ["ABOGEN_SETTINGS_DIR"] = str(settings_dir)
|
||||||
|
|
||||||
|
try:
|
||||||
|
from abogen.utils import get_user_settings_dir
|
||||||
|
|
||||||
|
get_user_settings_dir.cache_clear()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
from abogen.normalization_settings import clear_cached_settings
|
||||||
|
|
||||||
|
clear_cached_settings()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
yield
|
||||||
|
|||||||
@@ -215,6 +215,40 @@ def test_decades_can_skip_expansion_when_disabled() -> None:
|
|||||||
assert "'90s" in normalized
|
assert "'90s" in normalized
|
||||||
|
|
||||||
|
|
||||||
|
def test_abbreviated_decades_expand_to_spoken_form() -> None:
|
||||||
|
normalized = _normalize_text("She loved music from the '80s.")
|
||||||
|
assert "eighties" in normalized.lower()
|
||||||
|
|
||||||
|
|
||||||
|
def test_currency_under_one_dollar_uses_cents() -> None:
|
||||||
|
normalized = _normalize_text("It cost $0.99.")
|
||||||
|
folded = normalized.lower().replace("-", " ")
|
||||||
|
assert "zero dollars" not in folded
|
||||||
|
assert "cents" in folded
|
||||||
|
|
||||||
|
|
||||||
|
def test_iso_dates_use_locale_order_and_ordinals(monkeypatch) -> None:
|
||||||
|
monkeypatch.setenv("LC_TIME", "en_US.UTF-8")
|
||||||
|
normalized = _normalize_text("The date is 2025/12/15.")
|
||||||
|
folded = normalized.lower().replace("-", " ")
|
||||||
|
assert "december" in folded
|
||||||
|
assert "fifteenth" in folded
|
||||||
|
|
||||||
|
|
||||||
|
def test_times_and_acronyms_do_not_say_dot() -> None:
|
||||||
|
normalized = _normalize_text("Meet at 5 p.m. near the U.S.A. border.")
|
||||||
|
folded = normalized.lower()
|
||||||
|
assert " dot " not in folded
|
||||||
|
|
||||||
|
|
||||||
|
def test_internet_slang_expansion_is_configurable() -> None:
|
||||||
|
normalized = _normalize_text(
|
||||||
|
"pls knock before entering.",
|
||||||
|
normalization_overrides={"normalization_internet_slang": True},
|
||||||
|
)
|
||||||
|
assert "please" in normalized.lower()
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skipif(not SPACY_RESOLVER_AVAILABLE, reason="spaCy model unavailable")
|
@pytest.mark.skipif(not SPACY_RESOLVER_AVAILABLE, reason="spaCy model unavailable")
|
||||||
def test_spacy_disambiguates_it_has_from_context() -> None:
|
def test_spacy_disambiguates_it_has_from_context() -> None:
|
||||||
normalized = _normalize_text("It's been a long time.")
|
normalized = _normalize_text("It's been a long time.")
|
||||||
|
|||||||
Reference in New Issue
Block a user