mirror of
https://github.com/denizsafak/abogen.git
synced 2026-07-18 13:40:27 +02:00
feat: Update year normalization logic to reflect US-style pronunciation for years 1100-1999; adjust related tests for consistency
This commit is contained in:
@@ -1586,8 +1586,8 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
|
|||||||
return words.replace(" and ", " ")
|
return words.replace(" and ", " ")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# 1200s are commonly spoken as "twelve hundred".
|
# US-style: 1100-1999 are often spoken as "X hundred Y".
|
||||||
if 1200 <= value < 1300:
|
if 1100 <= value <= 1999:
|
||||||
hundreds = value // 100
|
hundreds = value // 100
|
||||||
remainder = value % 100
|
remainder = value % 100
|
||||||
prefix = _words(hundreds)
|
prefix = _words(hundreds)
|
||||||
|
|||||||
@@ -92,7 +92,8 @@ def update_settings() -> ResponseReturnValue:
|
|||||||
for key in _NORMALIZATION_BOOLEAN_KEYS:
|
for key in _NORMALIZATION_BOOLEAN_KEYS:
|
||||||
current[key] = _extract_checkbox(key, bool(current.get(key, True)))
|
current[key] = _extract_checkbox(key, bool(current.get(key, True)))
|
||||||
for key in _NORMALIZATION_STRING_KEYS:
|
for key in _NORMALIZATION_STRING_KEYS:
|
||||||
current[key] = (form.get(key) or "").strip()
|
if hasattr(form, "__contains__") and key in form:
|
||||||
|
current[key] = (form.get(key) or "").strip()
|
||||||
|
|
||||||
# Integrations
|
# Integrations
|
||||||
current["integrations"] = current.get("integrations", {})
|
current["integrations"] = current.get("integrations", {})
|
||||||
|
|||||||
@@ -316,6 +316,12 @@ def normalize_setting_value(key: str, value: Any, defaults: Dict[str, Any]) -> A
|
|||||||
if normalized_mode in {"off", "spacy", "llm"}:
|
if normalized_mode in {"off", "spacy", "llm"}:
|
||||||
return normalized_mode
|
return normalized_mode
|
||||||
return defaults[key]
|
return defaults[key]
|
||||||
|
if key == "normalization_numbers_year_style":
|
||||||
|
if isinstance(value, str):
|
||||||
|
normalized_style = value.strip().lower()
|
||||||
|
if normalized_style in {"american", "off"}:
|
||||||
|
return normalized_style
|
||||||
|
return defaults[key]
|
||||||
if key == "llm_context_mode":
|
if key == "llm_context_mode":
|
||||||
if isinstance(value, str):
|
if isinstance(value, str):
|
||||||
normalized_scope = value.strip().lower()
|
normalized_scope = value.strip().lower()
|
||||||
|
|||||||
@@ -11,14 +11,14 @@ def normalize(text, config):
|
|||||||
class TestDateNormalization:
|
class TestDateNormalization:
|
||||||
|
|
||||||
def test_standard_years(self, cfg):
|
def test_standard_years(self, cfg):
|
||||||
# 1990 -> nineteen ninety
|
# 1990 -> nineteen hundred ninety
|
||||||
assert "nineteen ninety" in normalize("In 1990, the web was born.", cfg)
|
assert "nineteen hundred ninety" in normalize("In 1990, the web was born.", cfg)
|
||||||
# 1066 -> ten sixty-six
|
# 1066 -> ten sixty-six
|
||||||
assert "ten sixty-six" in normalize("The battle was in 1066.", cfg)
|
assert "ten sixty-six" in normalize("The battle was in 1066.", cfg)
|
||||||
# 2023 -> twenty twenty-three
|
# 2023 -> twenty twenty-three
|
||||||
assert "twenty twenty-three" in normalize("It is currently 2023.", cfg)
|
assert "twenty twenty-three" in normalize("It is currently 2023.", cfg)
|
||||||
# 1905 -> nineteen oh five
|
# 1905 -> nineteen hundred oh five
|
||||||
assert "nineteen oh five" in normalize("In 1905, Einstein published.", cfg)
|
assert "nineteen hundred oh five" in normalize("In 1905, Einstein published.", cfg)
|
||||||
|
|
||||||
def test_future_years(self, cfg):
|
def test_future_years(self, cfg):
|
||||||
# 3400 -> thirty-four hundred
|
# 3400 -> thirty-four hundred
|
||||||
@@ -62,14 +62,14 @@ class TestDateNormalization:
|
|||||||
|
|
||||||
def test_ambiguous_numbers(self, cfg):
|
def test_ambiguous_numbers(self, cfg):
|
||||||
# Just a number, no "address", no markers. Should default to year if 4 digits 1000-9999
|
# Just a number, no "address", no markers. Should default to year if 4 digits 1000-9999
|
||||||
assert "nineteen fifty" in normalize("I have 1950 apples.", cfg)
|
assert "nineteen hundred fifty" in normalize("I have 1950 apples.", cfg)
|
||||||
# This is a known limitation/feature: it aggressively identifies years.
|
# This is a known limitation/feature: it aggressively identifies years.
|
||||||
|
|
||||||
def test_specific_user_examples(self, cfg):
|
def test_specific_user_examples(self, cfg):
|
||||||
# 1021
|
# 1021
|
||||||
assert "ten twenty-one" in normalize("1021", cfg)
|
assert "ten twenty-one" in normalize("1021", cfg)
|
||||||
# 1925
|
# 1925
|
||||||
assert "nineteen twenty-five" in normalize("1925", cfg)
|
assert "nineteen hundred" in normalize("1925", cfg)
|
||||||
# 3400
|
# 3400
|
||||||
assert "thirty-four hundred" in normalize("3400", cfg)
|
assert "thirty-four hundred" in normalize("3400", cfg)
|
||||||
|
|
||||||
@@ -88,13 +88,13 @@ class TestDateNormalization:
|
|||||||
# "address" is far away (> 60 chars). Should be year.
|
# "address" is far away (> 60 chars). Should be year.
|
||||||
padding = "x" * 70
|
padding = "x" * 70
|
||||||
text = f"address {padding} 1999"
|
text = f"address {padding} 1999"
|
||||||
assert "nineteen ninety-nine" in normalize(text, cfg)
|
assert "nineteen hundred ninety-nine" in normalize(text, cfg)
|
||||||
|
|
||||||
# "address" is close (< 60 chars). Should be number.
|
# "address" is close (< 60 chars). Should be number.
|
||||||
padding = "x" * 10
|
padding = "x" * 10
|
||||||
text = f"address {padding} 1999"
|
text = f"address {padding} 1999"
|
||||||
res = normalize(text, cfg)
|
res = normalize(text, cfg)
|
||||||
assert "nineteen ninety-nine" not in res
|
assert "nineteen hundred ninety-nine" not in res
|
||||||
assert "one thousand" in res
|
assert "one thousand" in res
|
||||||
|
|
||||||
def test_2000s(self, cfg):
|
def test_2000s(self, cfg):
|
||||||
|
|||||||
@@ -12,11 +12,10 @@ def normalize(text, overrides=None):
|
|||||||
return normalize_for_pipeline(text, config=config, settings=settings)
|
return normalize_for_pipeline(text, config=config, settings=settings)
|
||||||
|
|
||||||
def test_year_pronunciation():
|
def test_year_pronunciation():
|
||||||
# 1925 -> Nineteen Twenty Five
|
# 1925 -> Nineteen Hundred Twenty Five
|
||||||
normalized = normalize("1925")
|
normalized = normalize("1925")
|
||||||
print(f"1925 -> {normalized}")
|
print(f"1925 -> {normalized}")
|
||||||
# num2words might output "nineteen twenty-five"
|
assert "nineteen hundred" in normalized.lower()
|
||||||
assert "nineteen twenty" in normalized.lower()
|
|
||||||
assert "five" in normalized.lower()
|
assert "five" in normalized.lower()
|
||||||
|
|
||||||
# 2025 -> Twenty Twenty Five
|
# 2025 -> Twenty Twenty Five
|
||||||
|
|||||||
@@ -121,7 +121,8 @@ def test_space_separated_numbers_become_ranges() -> None:
|
|||||||
def test_year_like_numbers_use_common_pronunciation() -> None:
|
def test_year_like_numbers_use_common_pronunciation() -> None:
|
||||||
normalized = _normalize_text("In 1924 the journey began")
|
normalized = _normalize_text("In 1924 the journey began")
|
||||||
folded = normalized.lower().replace("-", " ")
|
folded = normalized.lower().replace("-", " ")
|
||||||
assert "nineteen twenty four" in folded
|
assert "nineteen hundred" in folded
|
||||||
|
assert "twenty four" in folded
|
||||||
|
|
||||||
|
|
||||||
def test_early_century_years_use_hundred_format() -> None:
|
def test_early_century_years_use_hundred_format() -> None:
|
||||||
|
|||||||
Reference in New Issue
Block a user