From 832e2c5197fe7d520742c8ef89e0be9b06435564 Mon Sep 17 00:00:00 2001 From: Artem Akymenko Date: Thu, 16 Jul 2026 08:13:37 +0000 Subject: [PATCH] refactor(domain): extract chapter classification heuristics from form.py Moved supplement_score, should_preselect_chapter, and ensure_at_least_one_chapter_enabled to domain/chapter_classification.py. Also moved coerce_bool from settings.py to common.py to break circular import introduced in previous commit. 1025 tests pass. --- abogen/domain/chapter_classification.py | 131 ++++++++++++++++++++++++ abogen/webui/routes/utils/form.py | 109 +------------------- tests/test_chapter_classification.py | 97 ++++++++++++++++++ 3 files changed, 233 insertions(+), 104 deletions(-) create mode 100644 abogen/domain/chapter_classification.py create mode 100644 tests/test_chapter_classification.py diff --git a/abogen/domain/chapter_classification.py b/abogen/domain/chapter_classification.py new file mode 100644 index 0000000..f0b9959 --- /dev/null +++ b/abogen/domain/chapter_classification.py @@ -0,0 +1,131 @@ +"""Heuristics for classifying chapters as content vs. supplements. + +A 'supplement' is any non-story material that a listener would typically +skip: title page, copyright, table of contents, acknowledgements, etc. +The scoring functions return a float; higher ⇒ more likely to be a +supplement. ``should_preselect_chapter`` turns that score into a +boolean suitable for a web form default. +""" + +from __future__ import annotations + +import re +from typing import Any, Dict, List, Tuple + +# Compiled once at module load – these are immutable. + +_SUPPLEMENT_TITLE_PATTERNS: List[Tuple[re.Pattern[str], float]] = [ + (re.compile(r"\btitle\s+page\b"), 3.0), + (re.compile(r"\bcopyright\b"), 2.4), + (re.compile(r"\btable\s+of\s+contents\b"), 2.8), + (re.compile(r"\bcontents\b"), 2.0), + (re.compile(r"\backnowledg(e)?ments?\b"), 2.0), + (re.compile(r"\bdedication\b"), 2.0), + (re.compile(r"\babout\s+the\s+author(s)?\b"), 2.4), + (re.compile(r"\balso\s+by\b"), 2.0), + (re.compile(r"\bpraise\s+for\b"), 2.0), + (re.compile(r"\bcolophon\b"), 2.2), + (re.compile(r"\bpublication\s+data\b"), 2.2), + (re.compile(r"\btranscriber'?s?\s+note\b"), 2.2), + (re.compile(r"\bglossary\b"), 2.2), + (re.compile(r"\bindex\b"), 2.0), + (re.compile(r"\bbibliograph(y|ies)\b"), 2.0), + (re.compile(r"\breferences\b"), 1.8), + (re.compile(r"\bappendix\b"), 1.9), +] + +_CONTENT_TITLE_PATTERNS: List[re.Pattern[str]] = [ + re.compile(r"\bchapter\b"), + re.compile(r"\bbook\b"), + re.compile(r"\bpart\b"), + re.compile(r"\bsection\b"), + re.compile(r"\bscene\b"), + re.compile(r"\bprologue\b"), + re.compile(r"\bepilogue\b"), + re.compile(r"\bintroduction\b"), + re.compile(r"\bstory\b"), +] + +_SUPPLEMENT_TEXT_KEYWORDS: List[Tuple[str, float]] = [ + ("copyright", 1.2), + ("all rights reserved", 1.1), + ("isbn", 0.9), + ("library of congress", 1.0), + ("table of contents", 1.0), + ("dedicated to", 0.8), + ("acknowledg", 0.8), + ("printed in", 0.6), + ("permission", 0.6), + ("publisher", 0.5), + ("praise for", 0.9), + ("also by", 0.9), + ("glossary", 0.8), + ("index", 0.8), + ("newsletter", 3.2), + ("mailing list", 2.6), + ("sign-up", 2.2), +] + + +def supplement_score(title: str, text: str, index: int) -> float: + """Return a score indicating how likely *title*/*text* is a supplement. + + Higher values ⇒ more likely to be non-story material (title page, + copyright, acknowledgements, etc.). + """ + normalized_title = (title or "").lower() + score = 0.0 + + for pattern, weight in _SUPPLEMENT_TITLE_PATTERNS: + if pattern.search(normalized_title): + score += weight + + for pattern in _CONTENT_TITLE_PATTERNS: + if pattern.search(normalized_title): + score -= 2.0 + + stripped_text = (text or "").strip() + length = len(stripped_text) + if length <= 150: + score += 0.9 + elif length <= 400: + score += 0.6 + elif length <= 800: + score += 0.35 + + lowercase_text = stripped_text.lower() + for keyword, weight in _SUPPLEMENT_TEXT_KEYWORDS: + if keyword in lowercase_text: + score += weight + + if index == 0 and score > 0: + score += 0.25 + + return score + + +def should_preselect_chapter( + title: str, + text: str, + index: int, + total_count: int, +) -> bool: + """Return True if the chapter should be *enabled* by default in the form. + + A single chapter is always preselected. For multi-chapter books, the + chapter is preselected when its supplement score is below 1.9. + """ + if total_count <= 1: + return True + score = supplement_score(title, text, index) + return score < 1.9 + + +def ensure_at_least_one_chapter_enabled(chapters: List[Dict[str, Any]]) -> None: + """Mutate *chapters* in-place so that at least one has ``enabled=True``.""" + if not chapters: + return + if any(chapter.get("enabled") for chapter in chapters): + return + best_index = max(range(len(chapters)), key=lambda idx: chapters[idx].get("characters", 0)) + chapters[best_index]["enabled"] = True diff --git a/abogen/webui/routes/utils/form.py b/abogen/webui/routes/utils/form.py index c1a81f7..9301c55 100644 --- a/abogen/webui/routes/utils/form.py +++ b/abogen/webui/routes/utils/form.py @@ -1,10 +1,14 @@ -import re import time import uuid from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple, cast from flask import request, render_template, jsonify from flask.typing import ResponseReturnValue +from abogen.domain.chapter_classification import ( + supplement_score, + should_preselect_chapter, + ensure_at_least_one_chapter_enabled, +) from abogen.webui.service import PendingJob, JobStatus from abogen.webui.routes.utils.service import get_service from abogen.tts_plugin.utils import is_plugin_registered @@ -66,109 +70,6 @@ _WIZARD_STEP_META = { }, } -_SUPPLEMENT_TITLE_PATTERNS: List[tuple[re.Pattern[str], float]] = [ - (re.compile(r"\btitle\s+page\b"), 3.0), - (re.compile(r"\bcopyright\b"), 2.4), - (re.compile(r"\btable\s+of\s+contents\b"), 2.8), - (re.compile(r"\bcontents\b"), 2.0), - (re.compile(r"\backnowledg(e)?ments?\b"), 2.0), - (re.compile(r"\bdedication\b"), 2.0), - (re.compile(r"\babout\s+the\s+author(s)?\b"), 2.4), - (re.compile(r"\balso\s+by\b"), 2.0), - (re.compile(r"\bpraise\s+for\b"), 2.0), - (re.compile(r"\bcolophon\b"), 2.2), - (re.compile(r"\bpublication\s+data\b"), 2.2), - (re.compile(r"\btranscriber'?s?\s+note\b"), 2.2), - (re.compile(r"\bglossary\b"), 2.0), - (re.compile(r"\bindex\b"), 2.0), - (re.compile(r"\bbibliograph(y|ies)\b"), 2.0), - (re.compile(r"\breferences\b"), 1.8), - (re.compile(r"\bappendix\b"), 1.9), -] - -_CONTENT_TITLE_PATTERNS: List[re.Pattern[str]] = [ - re.compile(r"\bchapter\b"), - re.compile(r"\bbook\b"), - re.compile(r"\bpart\b"), - re.compile(r"\bsection\b"), - re.compile(r"\bscene\b"), - re.compile(r"\bprologue\b"), - re.compile(r"\bepilogue\b"), - re.compile(r"\bintroduction\b"), - re.compile(r"\bstory\b"), -] - -_SUPPLEMENT_TEXT_KEYWORDS: List[tuple[str, float]] = [ - ("copyright", 1.2), - ("all rights reserved", 1.1), - ("isbn", 0.9), - ("library of congress", 1.0), - ("table of contents", 1.0), - ("dedicated to", 0.8), - ("acknowledg", 0.8), - ("printed in", 0.6), - ("permission", 0.6), - ("publisher", 0.5), - ("praise for", 0.9), - ("also by", 0.9), - ("glossary", 0.8), - ("index", 0.8), - ("newsletter", 3.2), - ("mailing list", 2.6), - ("sign-up", 2.2), -] - -def supplement_score(title: str, text: str, index: int) -> float: - normalized_title = (title or "").lower() - score = 0.0 - - for pattern, weight in _SUPPLEMENT_TITLE_PATTERNS: - if pattern.search(normalized_title): - score += weight - - for pattern in _CONTENT_TITLE_PATTERNS: - if pattern.search(normalized_title): - score -= 2.0 - - stripped_text = (text or "").strip() - length = len(stripped_text) - if length <= 150: - score += 0.9 - elif length <= 400: - score += 0.6 - elif length <= 800: - score += 0.35 - - lowercase_text = stripped_text.lower() - for keyword, weight in _SUPPLEMENT_TEXT_KEYWORDS: - if keyword in lowercase_text: - score += weight - - if index == 0 and score > 0: - score += 0.25 - - return score - - -def should_preselect_chapter( - title: str, - text: str, - index: int, - total_count: int, -) -> bool: - if total_count <= 1: - return True - score = supplement_score(title, text, index) - return score < 1.9 - - -def ensure_at_least_one_chapter_enabled(chapters: List[Dict[str, Any]]) -> None: - if not chapters: - return - if any(chapter.get("enabled") for chapter in chapters): - return - best_index = max(range(len(chapters)), key=lambda idx: chapters[idx].get("characters", 0)) - chapters[best_index]["enabled"] = True def apply_prepare_form( pending: PendingJob, form: Mapping[str, Any] diff --git a/tests/test_chapter_classification.py b/tests/test_chapter_classification.py new file mode 100644 index 0000000..77e3c46 --- /dev/null +++ b/tests/test_chapter_classification.py @@ -0,0 +1,97 @@ +"""Tests for domain/chapter_classification.py.""" + +from abogen.domain.chapter_classification import ( + supplement_score, + should_preselect_chapter, + ensure_at_least_one_chapter_enabled, +) + + +class TestSupplementScore: + def test_title_page_high_score(self): + score = supplement_score("Title Page", "", 0) + assert score > 3.0 + + def test_chapter_title_negative_score(self): + score = supplement_score("Chapter 1", "", 0) + assert score < 0 + + def test_copyright_high_score(self): + score = supplement_score("Copyright", "All rights reserved.", 0) + assert score > 2.0 + + def test_short_text_adds_score(self): + score = supplement_score("Some Title", "Short text.", 0) + assert score > 0.5 + + def test_long_text_low_score_contribution(self): + score = supplement_score("Some Title", "word " * 200, 0) + assert score < 0.5 + + def test_index_zero_bonus(self): + score_title = supplement_score("Dedication", "For my family.", 0) + score_other = supplement_score("Dedication", "For my family.", 3) + assert score_title > score_other + + def test_empty_title_and_text(self): + score = supplement_score("", "", 5) + assert score == 0.9 # short text bonus only (len("") ≤ 150) + + def test_newsletter_keyword_high_score(self): + score = supplement_score("Subscribe", "Join our newsletter today", 0) + assert score > 3.0 + + def test_acknowledgments_pattern(self): + score = supplement_score("Acknowledgements", "", 0) + assert score > 2.0 + + def test_glossary_in_title(self): + score = supplement_score("Glossary", "", 0) + assert score > 2.0 + + +class TestShouldPreselectChapter: + def test_single_chapter_always_preselected(self): + assert should_preselect_chapter("Anything", "", 0, 1) is True + + def test_chapter_preselected_when_low_score(self): + assert should_preselect_chapter("Chapter 1", "The story begins.", 0, 10) is True + + def test_title_page_not_preselected(self): + assert should_preselect_chapter("Title Page", "", 0, 10) is False + + def test_copyright_not_preselected(self): + assert should_preselect_chapter("Copyright", "All rights reserved.", 0, 10) is False + + def test_toc_not_preselected(self): + assert should_preselect_chapter("Table of Contents", "", 0, 10) is False + + +class TestEnsureAtLeastOneChapterEnabled: + def test_empty_list(self): + chapters = [] + ensure_at_least_one_chapter_enabled(chapters) + assert chapters == [] + + def test_already_has_enabled(self): + chapters = [ + {"title": "Ch1", "enabled": False}, + {"title": "Ch2", "enabled": True}, + ] + ensure_at_least_one_chapter_enabled(chapters) + assert chapters[1]["enabled"] is True + assert chapters[0]["enabled"] is False + + def test_none_enabled_picks_longest(self): + chapters = [ + {"title": "Ch1", "enabled": False, "characters": 100}, + {"title": "Ch2", "enabled": False, "characters": 500}, + {"title": "Ch3", "enabled": False, "characters": 200}, + ] + ensure_at_least_one_chapter_enabled(chapters) + assert chapters[1]["enabled"] is True + + def test_single_chapter_gets_enabled(self): + chapters = [{"title": "Only", "enabled": False}] + ensure_at_least_one_chapter_enabled(chapters) + assert chapters[0]["enabled"] is True