From e7f6f0221d1741f2ca5554d7b9630866b7c94062 Mon Sep 17 00:00:00 2001 From: JB Date: Thu, 30 Oct 2025 06:25:08 -0700 Subject: [PATCH] feat: Implement apostrophe normalization settings and overrides in the UI and backend --- abogen/normalization_settings.py | 16 ++++++ abogen/web/conversion_runner.py | 18 +++++++ abogen/web/routes.py | 51 +++++++++++++++++++ abogen/web/service.py | 8 +++ abogen/web/static/settings.js | 5 ++ .../templates/partials/new_job_step_book.html | 43 ++++++++++++++++ abogen/web/templates/settings.html | 21 ++++++++ tests/test_prepare_form.py | 1 + tests/test_text_normalization.py | 25 +++++++++ 9 files changed, 188 insertions(+) diff --git a/abogen/normalization_settings.py b/abogen/normalization_settings.py index 2f300fa..5524100 100644 --- a/abogen/normalization_settings.py +++ b/abogen/normalization_settings.py @@ -34,6 +34,11 @@ _SETTINGS_DEFAULTS: Dict[str, Any] = { "normalization_titles": True, "normalization_terminal": True, "normalization_phoneme_hints": True, + "normalization_apostrophes_contractions": True, + "normalization_apostrophes_plural_possessives": True, + "normalization_apostrophes_sibilant_possessives": True, + "normalization_apostrophes_decades": True, + "normalization_apostrophes_leading_elisions": True, "normalization_apostrophe_mode": "spacy", } @@ -151,6 +156,17 @@ def build_apostrophe_config( config = replace(base or ApostropheConfig()) config.convert_numbers = bool(settings.get("normalization_numbers", True)) config.add_phoneme_hints = bool(settings.get("normalization_phoneme_hints", True)) + config.contraction_mode = "expand" if settings.get("normalization_apostrophes_contractions", True) else "keep" + config.plural_possessive_mode = ( + "collapse" if settings.get("normalization_apostrophes_plural_possessives", True) else "keep" + ) + config.sibilant_possessive_mode = ( + "mark" if settings.get("normalization_apostrophes_sibilant_possessives", True) else "keep" + ) + config.decades_mode = "expand" if settings.get("normalization_apostrophes_decades", True) else "keep" + config.leading_elision_mode = ( + "expand" if settings.get("normalization_apostrophes_leading_elisions", True) else "keep" + ) return config diff --git a/abogen/web/conversion_runner.py b/abogen/web/conversion_runner.py index d0e5d29..a7497fc 100644 --- a/abogen/web/conversion_runner.py +++ b/abogen/web/conversion_runner.py @@ -25,6 +25,7 @@ from abogen.normalization_settings import ( build_apostrophe_config, build_llm_configuration, get_runtime_settings, + apply_overrides as apply_normalization_overrides, ) from abogen.entity_analysis import normalize_token as normalize_entity_token from abogen.text_extractor import ExtractedChapter, extract_from_path @@ -669,6 +670,20 @@ def _merge_metadata( _APOSTROPHE_CONFIG = ApostropheConfig() +def _normalize_for_pipeline( + text: str, + *, + normalization_overrides: Optional[Mapping[str, Any]] = None, +) -> str: + """Normalize text for tests or utilities with optional overrides.""" + + runtime_settings = get_runtime_settings() + if normalization_overrides: + runtime_settings = apply_normalization_overrides(runtime_settings, normalization_overrides) + apostrophe_config = build_apostrophe_config(settings=runtime_settings, base=_APOSTROPHE_CONFIG) + return normalize_for_pipeline(text, config=apostrophe_config, settings=runtime_settings) + + def _compile_pronunciation_rules( overrides: Optional[Iterable[Mapping[str, Any]]], ) -> List[Dict[str, Any]]: @@ -1101,6 +1116,9 @@ def run_conversion_job(job: Job) -> None: canceller = _make_canceller(job) normalization_settings = get_runtime_settings() + job_overrides = getattr(job, "normalization_overrides", None) + if job_overrides: + normalization_settings = apply_normalization_overrides(normalization_settings, job_overrides) apostrophe_config = build_apostrophe_config( settings=normalization_settings, base=_APOSTROPHE_CONFIG, diff --git a/abogen/web/routes.py b/abogen/web/routes.py index 99c561b..9d5ccd5 100644 --- a/abogen/web/routes.py +++ b/abogen/web/routes.py @@ -1723,6 +1723,30 @@ def _apply_book_step_form( else: pending.normalize_chapter_opening_caps = caps_default + def _extract_checkbox(name: str, default: bool) -> bool: + values: List[str] = [] + getter = getattr(form, "getlist", None) + if callable(getter): + raw_values = getter(name) + if raw_values: + values = list(cast(Iterable[str], raw_values)) + else: + raw_flag = form.get(name) + if raw_flag is not None: + values = [raw_flag] + if values: + return _coerce_bool(values[-1], default) + if hasattr(form, "__contains__") and name in form: + return False + return default + + overrides_existing = getattr(pending, "normalization_overrides", None) + overrides: Dict[str, bool] = dict(overrides_existing or {}) + for key in _APOSTROPHE_OVERRIDE_KEYS: + default_toggle = overrides.get(key, bool(settings.get(key, True))) + overrides[key] = _extract_checkbox(key, default_toggle) + pending.normalization_overrides = overrides + speed_value = form.get("speed") if speed_value is not None: try: @@ -1977,11 +2001,24 @@ BOOLEAN_SETTINGS = { "normalization_titles", "normalization_terminal", "normalization_phoneme_hints", + "normalization_apostrophes_contractions", + "normalization_apostrophes_plural_possessives", + "normalization_apostrophes_sibilant_possessives", + "normalization_apostrophes_decades", + "normalization_apostrophes_leading_elisions", } FLOAT_SETTINGS = {"silence_between_chapters", "chapter_intro_delay", "llm_timeout"} INT_SETTINGS = {"max_subtitle_words", "speaker_analysis_threshold"} +_APOSTROPHE_OVERRIDE_KEYS = ( + "normalization_apostrophes_contractions", + "normalization_apostrophes_plural_possessives", + "normalization_apostrophes_sibilant_possessives", + "normalization_apostrophes_decades", + "normalization_apostrophes_leading_elisions", +) + def _integration_defaults() -> Dict[str, Dict[str, Any]]: return { @@ -2047,6 +2084,11 @@ def _settings_defaults() -> Dict[str, Any]: "normalization_titles": True, "normalization_terminal": True, "normalization_phoneme_hints": True, + "normalization_apostrophes_contractions": True, + "normalization_apostrophes_plural_possessives": True, + "normalization_apostrophes_sibilant_possessives": True, + "normalization_apostrophes_decades": True, + "normalization_apostrophes_leading_elisions": True, "normalization_apostrophe_mode": "spacy", } @@ -3125,6 +3167,11 @@ def api_normalization_preview() -> ResponseReturnValue: "normalization_titles", "normalization_terminal", "normalization_phoneme_hints", + "normalization_apostrophes_contractions", + "normalization_apostrophes_plural_possessives", + "normalization_apostrophes_sibilant_possessives", + "normalization_apostrophes_decades", + "normalization_apostrophes_leading_elisions", ) for key in boolean_keys: if key in normalization_payload: @@ -4076,6 +4123,9 @@ def _build_pending_job_from_extraction( max_subtitle_words=max_subtitle_words, metadata_tags=metadata_tags, chapters=chapters_payload, + normalization_overrides={ + key: bool(settings.get(key, True)) for key in _APOSTROPHE_OVERRIDE_KEYS + }, created_at=time.time(), cover_image_path=cover_path, cover_image_mime=cover_mime, @@ -4455,6 +4505,7 @@ def finalize_job(pending_id: str) -> ResponseReturnValue: entity_summary=pending.entity_summary, manual_overrides=pending.manual_overrides, pronunciation_overrides=pending.pronunciation_overrides, + normalization_overrides=pending.normalization_overrides, ) if config_languages: diff --git a/abogen/web/service.py b/abogen/web/service.py index 0794450..20c0773 100644 --- a/abogen/web/service.py +++ b/abogen/web/service.py @@ -148,6 +148,7 @@ class Job: entity_summary: Dict[str, Any] = field(default_factory=dict) manual_overrides: List[Dict[str, Any]] = field(default_factory=list) pronunciation_overrides: List[Dict[str, Any]] = field(default_factory=list) + normalization_overrides: Dict[str, Any] = field(default_factory=dict) def add_log(self, message: str, level: str = "info") -> None: entry = JobLog(timestamp=time.time(), message=message, level=level) @@ -216,6 +217,7 @@ class Job: "entity_summary": dict(self.entity_summary), "manual_overrides": [dict(entry) for entry in self.manual_overrides], "pronunciation_overrides": [dict(entry) for entry in self.pronunciation_overrides], + "normalization_overrides": dict(self.normalization_overrides), } @@ -431,6 +433,7 @@ class PendingJob: max_subtitle_words: int metadata_tags: Dict[str, Any] chapters: List[Dict[str, Any]] + normalization_overrides: Dict[str, Any] created_at: float cover_image_path: Optional[Path] = None cover_image_mime: Optional[str] = None @@ -531,6 +534,7 @@ class ConversionService: entity_summary: Optional[Mapping[str, Any]] = None, manual_overrides: Optional[Iterable[Mapping[str, Any]]] = None, pronunciation_overrides: Optional[Iterable[Mapping[str, Any]]] = None, + normalization_overrides: Optional[Mapping[str, Any]] = None, ) -> Job: job_id = uuid.uuid4().hex normalized_metadata = self._normalize_metadata_tags(metadata_tags) @@ -580,6 +584,7 @@ class ConversionService: entity_summary=dict(entity_summary or {}), manual_overrides=[dict(entry) for entry in manual_overrides] if manual_overrides else [], pronunciation_overrides=[dict(entry) for entry in pronunciation_overrides] if pronunciation_overrides else [], + normalization_overrides=dict(normalization_overrides or {}), ) with self._lock: self._jobs[job_id] = job @@ -737,6 +742,7 @@ class ConversionService: entity_summary=job.entity_summary, manual_overrides=job.manual_overrides, pronunciation_overrides=job.pronunciation_overrides, + normalization_overrides=job.normalization_overrides, ) new_job.speaker_voice_languages = list(job.speaker_voice_languages) @@ -1039,6 +1045,7 @@ class ConversionService: "entity_summary": dict(job.entity_summary), "manual_overrides": [dict(entry) for entry in job.manual_overrides], "pronunciation_overrides": [dict(entry) for entry in job.pronunciation_overrides], + "normalization_overrides": dict(job.normalization_overrides), } def _persist_state(self) -> None: @@ -1164,6 +1171,7 @@ class ConversionService: job.pronunciation_overrides = [ dict(entry) for entry in payload.get("pronunciation_overrides", []) if isinstance(entry, Mapping) ] + job.normalization_overrides = dict(payload.get("normalization_overrides", {}) or {}) job.pause_event.set() return job diff --git a/abogen/web/static/settings.js b/abogen/web/static/settings.js index f7631ed..c65f4e3 100644 --- a/abogen/web/static/settings.js +++ b/abogen/web/static/settings.js @@ -264,6 +264,11 @@ function collectNormalizationSettings() { normalization_titles: Boolean(form.querySelector('input[name="normalization_titles"]')?.checked), normalization_terminal: Boolean(form.querySelector('input[name="normalization_terminal"]')?.checked), normalization_phoneme_hints: Boolean(form.querySelector('input[name="normalization_phoneme_hints"]')?.checked), + normalization_apostrophes_contractions: Boolean(form.querySelector('input[name="normalization_apostrophes_contractions"]')?.checked), + normalization_apostrophes_plural_possessives: Boolean(form.querySelector('input[name="normalization_apostrophes_plural_possessives"]')?.checked), + normalization_apostrophes_sibilant_possessives: Boolean(form.querySelector('input[name="normalization_apostrophes_sibilant_possessives"]')?.checked), + normalization_apostrophes_decades: Boolean(form.querySelector('input[name="normalization_apostrophes_decades"]')?.checked), + normalization_apostrophes_leading_elisions: Boolean(form.querySelector('input[name="normalization_apostrophes_leading_elisions"]')?.checked), normalization_apostrophe_mode: form.querySelector('input[name="normalization_apostrophe_mode"]:checked')?.value || 'spacy', }; return normalization; diff --git a/abogen/web/templates/partials/new_job_step_book.html b/abogen/web/templates/partials/new_job_step_book.html index aa30fed..b10013b 100644 --- a/abogen/web/templates/partials/new_job_step_book.html +++ b/abogen/web/templates/partials/new_job_step_book.html @@ -112,6 +112,34 @@ {% else %} {% set narrator_voice = settings_dict.get('default_voice', options.voices[0] if options.voices else '') %} {% endif %} +{% set normalization_overrides = pending.normalization_overrides if pending and pending.normalization_overrides else {} %} +{% set apostrophe_toggles = [ + { + 'name': 'normalization_apostrophes_contractions', + 'label': "Expand contractions (it's -> it is)", + 'value': normalization_overrides.get('normalization_apostrophes_contractions', settings_dict.get('normalization_apostrophes_contractions', True)), + }, + { + 'name': 'normalization_apostrophes_plural_possessives', + 'label': "Collapse plural possessives (dogs' -> dogs)", + 'value': normalization_overrides.get('normalization_apostrophes_plural_possessives', settings_dict.get('normalization_apostrophes_plural_possessives', True)), + }, + { + 'name': 'normalization_apostrophes_sibilant_possessives', + 'label': "Mark sibilant possessives (boss's -> boss + IZ marker)", + 'value': normalization_overrides.get('normalization_apostrophes_sibilant_possessives', settings_dict.get('normalization_apostrophes_sibilant_possessives', True)), + }, + { + 'name': 'normalization_apostrophes_decades', + 'label': "Expand decades ('90s -> 1990s)", + 'value': normalization_overrides.get('normalization_apostrophes_decades', settings_dict.get('normalization_apostrophes_decades', True)), + }, + { + 'name': 'normalization_apostrophes_leading_elisions', + 'label': "Expand leading elisions ('tis -> it is)", + 'value': normalization_overrides.get('normalization_apostrophes_leading_elisions', settings_dict.get('normalization_apostrophes_leading_elisions', True)), + }, +] %} {% set voice_formula_value = '' %} {% set profile_value = narrator_profile if narrator_profile else '__standard' %} {% if profile_value == '__formula' %} @@ -253,6 +281,21 @@ +
+

Text normalization

+
+ {% for toggle in apostrophe_toggles %} +
+ +
+ {% endfor %} +
+
+

Entities & casting

diff --git a/abogen/web/templates/settings.html b/abogen/web/templates/settings.html index afe9dc9..193ec6f 100644 --- a/abogen/web/templates/settings.html +++ b/abogen/web/templates/settings.html @@ -269,6 +269,27 @@ Add phoneme hints for possessives + Apostrophes + + + + +
Apostrophe strategy diff --git a/tests/test_prepare_form.py b/tests/test_prepare_form.py index 4df8fa8..25cbfb2 100644 --- a/tests/test_prepare_form.py +++ b/tests/test_prepare_form.py @@ -31,6 +31,7 @@ def _make_pending_job() -> PendingJob: max_subtitle_words=50, metadata_tags={}, chapters=[], + normalization_overrides={}, created_at=0.0, read_title_intro=False, normalize_chapter_opening_caps=True, diff --git a/tests/test_text_normalization.py b/tests/test_text_normalization.py index cf7dab5..76ac32e 100644 --- a/tests/test_text_normalization.py +++ b/tests/test_text_normalization.py @@ -78,3 +78,28 @@ def test_numeric_ranges_are_spoken_with_to() -> None: def test_simple_fractions_are_spoken() -> None: normalized = _normalize_for_pipeline("Add 1/2 cup of sugar") assert "one half" in normalized.lower() + + +def test_contractions_can_be_kept_when_override_disabled() -> None: + normalized = _normalize_for_pipeline( + "It's a good day.", + normalization_overrides={"normalization_apostrophes_contractions": False}, + ) + assert "It's" in normalized + + +def test_sibilant_possessives_remain_when_marking_disabled() -> None: + normalized = _normalize_for_pipeline( + "The boss's chair wobbled.", + normalization_overrides={"normalization_apostrophes_sibilant_possessives": False}, + ) + assert "boss's" in normalized + assert "boss iz" not in normalized.lower() + + +def test_decades_can_skip_expansion_when_disabled() -> None: + normalized = _normalize_for_pipeline( + "Classic hits from the '90s filled the hall.", + normalization_overrides={"normalization_apostrophes_decades": False}, + ) + assert "'90s" in normalized