From ccf5a11222c82c7b6a7d6c8e41aaf978d1342a78 Mon Sep 17 00:00:00 2001 From: JB Date: Sat, 11 Oct 2025 11:14:52 -0700 Subject: [PATCH] Enhance speaker analysis and web templates - Added gender inference improvements in speaker_analysis.py, including handling of titles and diacritics. - Updated analyze_speakers function to include sample excerpts with context paragraphs. - Modified routes.py to skip suppressed speakers in the speaker roster. - Enhanced prepare.js to manage speaker samples and pronunciation previews more effectively. - Refined prepare_chapters.html and prepare_speakers.html templates for better navigation and user experience. - Added tests for speaker analysis to ensure proper handling of stopwords and threshold suppression. --- abogen/speaker_analysis.py | 329 ++++++++++++++++++--- abogen/web/routes.py | 7 +- abogen/web/static/prepare.js | 162 ++++++++-- abogen/web/templates/prepare_chapters.html | 119 ++++---- abogen/web/templates/prepare_speakers.html | 88 +++--- tests/test_speaker_analysis.py | 49 +++ 6 files changed, 592 insertions(+), 162 deletions(-) diff --git a/abogen/speaker_analysis.py b/abogen/speaker_analysis.py index 287f3e7..40a7d66 100644 --- a/abogen/speaker_analysis.py +++ b/abogen/speaker_analysis.py @@ -4,6 +4,8 @@ import re from dataclasses import dataclass, field from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple +import unicodedata + _DIALOGUE_VERBS = ( "said", "asked", @@ -71,6 +73,73 @@ _PRONOUN_LABELS = { _CONFIDENCE_RANK = {"low": 1, "medium": 2, "high": 3} +_FEMALE_TITLE_HINTS = ( + "madame", + "mme", + "madam", + "mrs", + "miss", + "ms", + "lady", + "countess", + "baroness", + "princess", + "queen", + "mademoiselle", +) + +_MALE_TITLE_HINTS = ( + "monsieur", + "m.", + "mr", + "sir", + "lord", + "count", + "baron", + "prince", + "king", + "abbé", + "abbe", +) + +_MALE_TOKEN_WEIGHTS = { + "he": 1.0, + "him": 0.6, + "his": 0.75, + "himself": 1.0, +} + +_FEMALE_TOKEN_WEIGHTS = { + "she": 1.0, + "her": 0.4, + "hers": 0.75, + "herself": 1.0, +} + +_STOP_LABELS = { + "and", + "but", + "then", + "though", + "meanwhile", + "therefore", + "after", + "before", + "when", + "while", + "because", + "as", + "yet", + "nor", + "so", + "thus", + "suddenly", + "eventually", + "finally", + "until", + "unless", +} + @dataclass(slots=True) class SpeakerGuess: @@ -92,12 +161,13 @@ class SpeakerGuess: quote: Optional[str], male_votes: int, female_votes: int, + sample_excerpt: Optional[str] = None, ) -> None: self.count += 1 if _CONFIDENCE_RANK.get(confidence, 0) > _CONFIDENCE_RANK.get(self.confidence, 0): self.confidence = confidence - excerpt = _build_excerpt(text, quote) + excerpt = sample_excerpt if sample_excerpt is not None else _build_excerpt(text, quote) gender_hint = _format_gender_hint(male_votes, female_votes) if excerpt: payload = {"excerpt": excerpt, "gender_hint": gender_hint} @@ -173,9 +243,9 @@ def analyze_speakers( explicit_assignments = 0 unique_speakers: set[str] = set() - for chunk in ordered_chunks: + for index, chunk in enumerate(ordered_chunks): chunk_id = str(chunk.get("id") or "") - text = str(chunk.get("normalized_text") or chunk.get("text") or "") + text = _get_chunk_text(chunk) speaker_id, confidence, quote = _infer_chunk_speaker(text, last_explicit) if speaker_id is None: speaker_id = last_explicit or narrator_id @@ -184,10 +254,6 @@ def analyze_speakers( if speaker_id != narrator_id: last_explicit = speaker_id explicit_assignments += 1 - assignments[chunk_id] = speaker_id - unique_speakers.add(speaker_id) - - male_votes, female_votes = _count_gender_votes(text) if speaker_id in speaker_guesses: record_id = speaker_id @@ -201,12 +267,19 @@ def analyze_speakers( label_index[label] = record_id speaker_guesses[record_id] = SpeakerGuess(speaker_id=record_id, label=label) guess = speaker_guesses[record_id] - guess.register_occurrence(confidence, text, quote, male_votes, female_votes) - if record_id != speaker_id: - # Maintain mapping to canonical ID in assignments. - assignments[chunk_id] = record_id - if speaker_id == last_explicit: - last_explicit = record_id + assignments[chunk_id] = record_id + unique_speakers.add(record_id) + + if record_id != narrator_id and record_id != speaker_id and speaker_id == last_explicit: + last_explicit = record_id + + sample_excerpt = None + if record_id != narrator_id: + sample_excerpt = _select_sample_excerpt(ordered_chunks, index, guess.label, quote, confidence) + + male_votes, female_votes = _count_gender_votes(text, guess.label) + + guess.register_occurrence(confidence, text, quote, male_votes, female_votes, sample_excerpt) active_speakers = [sid for sid in speaker_guesses if sid != narrator_id] # Apply minimum occurrence threshold. @@ -367,31 +440,125 @@ def _reassign(assignments: Dict[str, str], old: str, new: str) -> None: assignments[key] = new -def _count_gender_votes(text: str) -> Tuple[int, int]: +def _strip_diacritics(value: str) -> str: + normalized = unicodedata.normalize("NFKD", value) + return "".join(char for char in normalized if not unicodedata.combining(char)) + + +def _count_gender_votes(text: str, label: Optional[str]) -> Tuple[int, int]: if not text: return 0, 0 - male_votes = 0.0 - for token in _MALE_PRONOUN_PATTERN.findall(text): - lowered = token.lower() - if lowered in {"he", "himself"}: - male_votes += 1.0 - elif lowered == "his": - male_votes += 0.75 - else: # him - male_votes += 0.6 + search_text = text + windows: List[Tuple[int, int]] = [] + degrade_factor = 1.0 - female_votes = 0.0 - for token in _FEMALE_PRONOUN_PATTERN.findall(text): - lowered = token.lower() - if lowered in {"she", "herself"}: - female_votes += 1.0 - elif lowered == "hers": - female_votes += 0.75 - else: # her - female_votes += 0.4 + if label: + pattern = re.compile(re.escape(label), re.IGNORECASE) + matches = list(pattern.finditer(search_text)) + if not matches: + alt_label = _strip_diacritics(label) + if alt_label and alt_label != label: + ascii_text = _strip_diacritics(search_text) + pattern_alt = re.compile(re.escape(alt_label), re.IGNORECASE) + windows = [match.span() for match in pattern_alt.finditer(ascii_text)] + # Map spans back roughly using proportional index + if windows: + mapped: List[Tuple[int, int]] = [] + for start, end in windows: + start_idx = min(len(search_text) - 1, int(start * len(search_text) / max(len(ascii_text), 1))) + end_idx = min(len(search_text), int(end * len(search_text) / max(len(ascii_text), 1))) + mapped.append((start_idx, end_idx)) + windows = mapped + else: + windows = [match.span() for match in matches] - return int(round(male_votes)), int(round(female_votes)) + if not windows: + windows = [(0, len(search_text))] + degrade_factor = 0.25 + + radius = 60 + quote_spans: List[Tuple[int, int, str]] = [] + for match in _QUOTE_PATTERN.finditer(search_text): + try: + content_start, content_end = match.span(1) + except IndexError: + content_start, content_end = match.span() + if content_start < content_end: + quote_spans.append((content_start, content_end, search_text[content_start:content_end])) + + normalized_label = _normalize_candidate_name(label) if label else None + normalized_label_lower = normalized_label.lower() if normalized_label else None + + def _window_weight(position: int) -> float: + for start, end in windows: + if position < start - radius or position > end + radius: + continue + if position >= end: + return 1.0 + if position <= start: + return 0.2 + return 1.0 + return 0.0 + + def _quote_weight(position: int) -> float: + for start, end, content in quote_spans: + if position < start or position >= end: + continue + local_index = position - start + prefix = content[:local_index] + tail = prefix[-80:] + name_matches = list(re.finditer(_NAME_PATTERN, tail)) + if name_matches: + last_name = _normalize_candidate_name(name_matches[-1].group(0)) + if normalized_label_lower and last_name and last_name.lower() == normalized_label_lower: + return 0.6 + return 0.05 + if re.search(r"[.!?]\s", prefix): + return 0.2 + if prefix.strip(): + return 0.15 + return 0.1 + return 1.0 + + male_score = 0.0 + for match in _MALE_PRONOUN_PATTERN.finditer(search_text): + base_weight = _window_weight(match.start()) + if not base_weight: + continue + quote_modifier = _quote_weight(match.start()) + weight = base_weight * quote_modifier + if not weight: + continue + token = match.group(0).lower() + male_score += _MALE_TOKEN_WEIGHTS.get(token, 0.6) * weight + + female_score = 0.0 + for match in _FEMALE_PRONOUN_PATTERN.finditer(search_text): + base_weight = _window_weight(match.start()) + if not base_weight: + continue + quote_modifier = _quote_weight(match.start()) + weight = base_weight * quote_modifier + if not weight: + continue + if quote_modifier >= 0.95: + weight = max(weight, 0.4) + token = match.group(0).lower() + female_score += _FEMALE_TOKEN_WEIGHTS.get(token, 0.4) * weight + + for start, end in windows: + span_start = max(0, start - 40) + span_end = min(len(search_text), end + 40) + span_text = search_text[span_start:span_end].lower() + if any(title in span_text for title in _FEMALE_TITLE_HINTS): + female_score += 2.5 + if any(title in span_text for title in _MALE_TITLE_HINTS): + male_score += 2.5 + + male_votes = int(round(male_score * degrade_factor)) + female_votes = int(round(female_score * degrade_factor)) + return male_votes, female_votes def _derive_gender(male_votes: int, female_votes: int, current: str) -> str: @@ -411,6 +578,70 @@ def _derive_gender(male_votes: int, female_votes: int, current: str) -> str: return "unknown" +def _get_chunk_text(chunk: Dict[str, Any]) -> str: + if not isinstance(chunk, dict): + return "" + value = chunk.get("normalized_text") or chunk.get("text") or "" + return str(value) + + +def _trim_paragraph(paragraph: str, limit: int = 600) -> str: + normalized = (paragraph or "").strip() + if not normalized: + return "" + if len(normalized) <= limit: + return normalized + return normalized[: limit - 1].rstrip() + "…" + + +def _compose_context_excerpt(before: str, current: str, after: str) -> str: + segments = [] + for value in (before, current, after): + trimmed = _trim_paragraph(value) + if trimmed: + segments.append(trimmed) + return "\n\n".join(segments) + + +def _contains_dialogue_attribution(label: str, text: str, quote: Optional[str]) -> bool: + if not label or not text: + return False + escaped_label = re.escape(label) + direct_pattern = re.compile(rf"\b{escaped_label}\b\s+(?:{_VERB_PATTERN})\b", re.IGNORECASE) + reverse_pattern = re.compile(rf"(?:{_VERB_PATTERN})\s+\b{escaped_label}\b", re.IGNORECASE) + colon_pattern = re.compile(rf"^\s*{escaped_label}\s*:\s*", re.IGNORECASE) + + if colon_pattern.search(text): + return True + if direct_pattern.search(text) or reverse_pattern.search(text): + return True + if quote: + before, after = _split_around_quote(text, quote) + if direct_pattern.search(before) or reverse_pattern.search(after): + return True + return False + + +def _select_sample_excerpt( + chunks: Sequence[Dict[str, Any]], + index: int, + label: str, + quote: Optional[str], + confidence: str, +) -> Optional[str]: + if confidence != "high" or not label: + return None + if index < 0 or index >= len(chunks): + return None + current = _get_chunk_text(chunks[index]) + if not current or not _contains_dialogue_attribution(label, current, quote): + return None + previous = _get_chunk_text(chunks[index - 1]) if index > 0 else "" + following = _get_chunk_text(chunks[index + 1]) if index + 1 < len(chunks) else "" + excerpt = _compose_context_excerpt(previous, current, following) + return excerpt or None + + def _build_excerpt(text: str, quote: Optional[str]) -> str: normalized = (text or "").strip() if not normalized: @@ -452,7 +683,33 @@ def _normalize_candidate_name(raw: str) -> Optional[str]: cleaned = re.sub(r"\s+", " ", cleaned).strip() if not cleaned: return None - lowered = cleaned.lower() - if lowered in _PRONOUN_LABELS: + parts = cleaned.split() + filtered: List[str] = [] + for part in parts: + if not part: + continue + if not filtered and part.lower() in _STOP_LABELS: + continue + filtered.append(part) + while filtered and filtered[-1].lower() in _STOP_LABELS: + filtered.pop() + if not filtered: return None - return cleaned \ No newline at end of file + if all(part.lower() in _STOP_LABELS for part in filtered): + return None + contiguous: List[str] = [] + for part in filtered: + if part and part[0].isupper(): + contiguous.append(part) + else: + break + if contiguous: + candidate = " ".join(contiguous) + else: + candidate = "" + if not candidate: + return None + lowered = candidate.lower() + if lowered in _PRONOUN_LABELS or lowered in _STOP_LABELS: + return None + return candidate \ No newline at end of file diff --git a/abogen/web/routes.py b/abogen/web/routes.py index 95c109e..318022e 100644 --- a/abogen/web/routes.py +++ b/abogen/web/routes.py @@ -501,6 +501,7 @@ def _build_speaker_roster( payload = speakers.get(speaker_id, {}) if speaker_id == "narrator": continue + if isinstance(payload, Mapping) and payload.get("suppressed"): continue previous = existing_map.get(speaker_id) roster[speaker_id] = { @@ -864,7 +865,11 @@ def _prepare_speaker_metadata( ordered_ids = [ sid for sid, meta in sorted( - ((sid, meta) for sid, meta in speakers_payload.items() if sid != "narrator"), + ( + (sid, meta) + for sid, meta in speakers_payload.items() + if sid != "narrator" and isinstance(meta, Mapping) and not meta.get("suppressed") + ), key=lambda item: item[1].get("count", 0), reverse=True, ) diff --git a/abogen/web/static/prepare.js b/abogen/web/static/prepare.js index d73892c..b4ff76c 100644 --- a/abogen/web/static/prepare.js +++ b/abogen/web/static/prepare.js @@ -18,6 +18,113 @@ document.addEventListener("DOMContentLoaded", () => { const languageMap = parseJSONScript("voice-language-map") || {}; const voiceCatalogMap = new Map(voiceCatalog.map((voice) => [voice.id, voice])); + const sampleIndexState = new WeakMap(); + + const readSpeakerSamples = (speakerItem) => { + if (!speakerItem) return []; + const template = speakerItem.querySelector('template[data-role="speaker-samples"]'); + if (!template) return []; + let parsed = []; + try { + const raw = template.innerHTML || "[]"; + const data = JSON.parse(raw); + if (Array.isArray(data)) { + parsed = data; + } + } catch (error) { + console.warn("Unable to parse speaker samples", error); + return []; + } + + const seen = new Set(); + const normalised = []; + for (const entry of parsed) { + let excerpt = ""; + let genderHint = ""; + if (typeof entry === "string") { + excerpt = entry; + } else if (entry && typeof entry === "object") { + excerpt = String(entry.excerpt || ""); + genderHint = typeof entry.gender_hint === "string" ? entry.gender_hint : ""; + } + const key = excerpt.trim(); + if (!key || seen.has(key)) { + continue; + } + seen.add(key); + normalised.push({ excerpt: key, genderHint }); + } + return normalised; + }; + + const getPronunciationText = (container) => { + if (!container) return ""; + const input = container.querySelector('[data-role="speaker-pronunciation"]'); + const raw = input?.value?.trim(); + if (raw) { + return raw; + } + return (container.dataset.defaultPronunciation || "").trim(); + }; + + const syncPronunciationPreview = (container) => { + if (!container) return; + const text = getPronunciationText(container); + const previewButtons = container.querySelectorAll('[data-role="speaker-preview"][data-preview-source]'); + previewButtons.forEach((button) => { + const source = button.dataset.previewSource; + if (["pronunciation", "generated", "mix"].includes(source)) { + button.dataset.previewText = text; + } + }); + }; + + const setSpeakerSample = (speakerItem, index) => { + if (!speakerItem) return; + const samples = readSpeakerSamples(speakerItem); + if (!samples.length) return; + const maxIndex = samples.length; + const normalisedIndex = ((index % maxIndex) + maxIndex) % maxIndex; + sampleIndexState.set(speakerItem, normalisedIndex); + const sample = samples[normalisedIndex]; + const article = speakerItem.querySelector('[data-role="speaker-sample"]'); + if (!article) return; + const textNode = article.querySelector('[data-role="sample-text"]'); + const hintNode = article.querySelector('[data-role="sample-hint"]'); + if (textNode) { + textNode.textContent = sample.excerpt; + } + if (hintNode) { + if (sample.genderHint) { + hintNode.hidden = false; + hintNode.textContent = sample.genderHint; + } else { + hintNode.hidden = true; + hintNode.textContent = ""; + } + } + const previewButton = article.querySelector('[data-role="speaker-preview"][data-preview-source="sample"]'); + if (previewButton) { + previewButton.dataset.previewText = sample.excerpt; + } + const voiceBrowserButton = article.querySelector('[data-role="open-voice-browser"]'); + if (voiceBrowserButton) { + voiceBrowserButton.dataset.sampleIndex = String(normalisedIndex); + } + }; + + const initialiseSpeakerItem = (speakerItem) => { + syncPronunciationPreview(speakerItem); + const samples = readSpeakerSamples(speakerItem); + if (samples.length) { + setSpeakerSample(speakerItem, 0); + const nextButton = speakerItem.querySelector('[data-role="speaker-next-sample"]'); + if (nextButton) { + nextButton.disabled = samples.length <= 1; + } + } + }; + const formatCustomMixLabel = (formula) => { if (!formula) return "Custom mix"; const segments = formula @@ -229,6 +336,9 @@ document.addEventListener("DOMContentLoaded", () => { updatePreviewVoice(select); }); + const speakerItems = Array.from(form.querySelectorAll(".speaker-list__item")); + speakerItems.forEach((item) => initialiseSpeakerItem(item)); + const activeStepInput = form.querySelector('[data-role="active-step-input"]'); const analysisButtons = Array.from(form.querySelectorAll('[data-role="submit-speaker-analysis"]')); analysisButtons.forEach((button) => { @@ -690,7 +800,7 @@ document.addEventListener("DOMContentLoaded", () => { if (!voiceModal) return; modalState.speakerItem = speakerItem; const select = speakerItem.querySelector('[data-role="speaker-voice"]'); - const previewTrigger = speakerItem.querySelector('[data-role="speaker-preview"]'); + const previewTrigger = speakerItem.querySelector('[data-role="speaker-preview"][data-preview-source="pronunciation"]'); const formulaInput = speakerItem.querySelector('[data-role="speaker-formula"]'); modalState.defaultVoice = select?.dataset.defaultVoice || previewTrigger?.dataset.voice || ""; modalState.mix = formulaInput?.value ? parseFormula(formulaInput.value) : new Map(); @@ -698,36 +808,32 @@ document.addEventListener("DOMContentLoaded", () => { modalState.mix.set(select.value, 1); } modalState.mix = normaliseMix(modalState.mix); + modalState.previewSettings = { language: previewTrigger?.dataset.language || "a", speed: previewTrigger?.dataset.speed || "1", useGpu: previewTrigger?.dataset.useGpu || "true", }; - const samplesTemplate = speakerItem.querySelector('template[data-role="speaker-samples"]'); - let samples = []; - if (samplesTemplate) { - try { - const raw = samplesTemplate.innerHTML || "[]"; - const parsed = JSON.parse(raw); - if (Array.isArray(parsed)) { - samples = parsed - .map((entry) => (typeof entry === "string" ? entry : entry?.excerpt)) - .filter((value) => typeof value === "string" && value.trim().length); - } - } catch (error) { - console.warn("Unable to parse speaker samples", error); + const samples = readSpeakerSamples(speakerItem); + let excerpts = samples.map((sample) => sample.excerpt); + const storedIndex = sampleIndexState.get(speakerItem) || 0; + let effectiveIndex = Number.isFinite(sampleIndex) ? sampleIndex : 0; + if (!Number.isFinite(effectiveIndex) || effectiveIndex < 0 || effectiveIndex >= excerpts.length) { + effectiveIndex = storedIndex; + } + if (excerpts.length && effectiveIndex > 0 && effectiveIndex < excerpts.length) { + const [selected] = excerpts.splice(effectiveIndex, 1); + excerpts.unshift(selected); + } + if (!excerpts.length) { + const sampleButton = speakerItem.querySelector('[data-role="speaker-preview"][data-preview-source="sample"]'); + const previewText = sampleButton?.dataset.previewText?.trim(); + if (previewText) { + excerpts = [previewText]; } } - if (previewTrigger?.dataset.previewText) { - samples.unshift(previewTrigger.dataset.previewText); - } - const uniqueSamples = Array.from(new Set(samples)); - if (sampleIndex > 0 && sampleIndex < uniqueSamples.length) { - const [picked] = uniqueSamples.splice(sampleIndex, 1); - uniqueSamples.unshift(picked); - } - modalState.samples = uniqueSamples; + modalState.samples = Array.from(new Set(excerpts)); modalState.recommended = new Set( Array.from(speakerItem.querySelectorAll('[data-role="recommended-voice"]')).map((btn) => btn.dataset.voice).filter(Boolean) ); @@ -899,6 +1005,16 @@ document.addEventListener("DOMContentLoaded", () => { return; } + const nextSampleButton = event.target.closest('[data-role="speaker-next-sample"]'); + if (nextSampleButton) { + event.preventDefault(); + const container = nextSampleButton.closest(".speaker-list__item"); + if (!container) return; + const currentIndex = sampleIndexState.get(container) || 0; + setSpeakerSample(container, currentIndex + 1); + return; + } + const clearMixButton = event.target.closest('[data-role="clear-mix"]'); if (clearMixButton) { event.preventDefault(); diff --git a/abogen/web/templates/prepare_chapters.html b/abogen/web/templates/prepare_chapters.html index ec84cb1..b451816 100644 --- a/abogen/web/templates/prepare_chapters.html +++ b/abogen/web/templates/prepare_chapters.html @@ -8,26 +8,30 @@
-
-

Step 2 of 3

-

Select chapters

-

Choose which chapters to convert and tweak conversion options. We'll analyse speakers automatically when you continue.

-
-
-
- +
+
+ + {% endif %} + +

Select chapters

+

Choose which chapters to convert. We'll analyse speakers automatically when you continue.

+
+

{{ pending.original_filename }}

@@ -35,8 +39,18 @@ action="{{ url_for('web.finalize_job', pending_id=pending.id) }}" class="prepare-form" id="prepare-form" + data-speaker-mode="{{ pending.speaker_mode }}" data-analyze-url="{{ url_for('web.analyze_pending_job', pending_id=pending.id) }}"> +
{% if error %} @@ -62,23 +76,32 @@ {% elif chapter.voice_formula %} {% set selected_option = 'formula' %} {% endif %} -
-
-
- -
+ {% set raw_excerpt = (chapter.text or '') | replace('\r', ' ') | replace('\n', ' ') %} + {% set excerpt = raw_excerpt[:240] %} +
+
+ +

+ {{ excerpt }}{% if raw_excerpt|length > 240 %}…{% endif %} +

-
+
- Preview text + Preview full text 
{{ chapter.text[:2000] }}{% if chapter.text|length > 2000 %}…{% endif %}
@@ -100,7 +123,13 @@ {% endif %} - +
- -
-

Conversion defaults

-
-
- - -

Paragraphs work well for long-form narration; sentences give finer subtitle sync.

-
-
- - -
-
-
-
- - -

Only speakers that appear at least this many times will keep unique voices in multi-speaker mode.

-
-
- - -

Set to 0 to disable the pause after speaking each chapter title.

-
-
- -