From 783dbcf8f24f6b2eb2aa39938b3735891f595fd1 Mon Sep 17 00:00:00 2001 From: JB Date: Tue, 7 Oct 2025 14:19:49 -0700 Subject: [PATCH] feat: Add supplemental section detection for improved content selection in HandlerDialog --- abogen/book_handler.py | 107 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 105 insertions(+), 2 deletions(-) diff --git a/abogen/book_handler.py b/abogen/book_handler.py index 73079bc..6ae8368 100644 --- a/abogen/book_handler.py +++ b/abogen/book_handler.py @@ -29,6 +29,77 @@ import urllib.parse import markdown import textwrap + +SUPPLEMENTAL_TITLE_PATTERNS = [ + re.compile(pattern, re.IGNORECASE) + for pattern in [ + r"\\btable\\s+of\\s+contents\\b", + r"^contents$", + r"\\babout\\s+the\\s+author(s)?\\b", + r"\\babout\\s+the\\s+illustrator\\b", + r"\\babout\\s+the\\s+translator\\b", + r"\\btitle\\s+page\\b", + r"\\bcopyright\\b", + r"\\bcolophon\\b", + r"\\bfront\\s*matter\\b", + r"\\bback\\s*matter\\b", + r"\\bindex\\b", + r"\\bglossary\\b", + r"\\bbibliograph", + r"\\backnowledg(e)?ments?\\b", + r"\\bpublication\\s+data\\b", + ] +] + +SUPPLEMENTAL_TEXT_PATTERNS = [ + re.compile(pattern, re.IGNORECASE) + for pattern in [ + r"all rights reserved", + r"no part of (this )?book may be reproduced", + r"isbn", + r"library of congress", + r"printed in", + r"cover design", + r"for more information", + r"this is a work of fiction", + r"copyright", + r"table of contents", + ] +] + + +def is_supplemental_section(title: str | None, text: str | None) -> bool: + """Return True if the title/content look like non-narrative front/back matter.""" + + title = (title or "").strip() + if title: + for pattern in SUPPLEMENTAL_TITLE_PATTERNS: + if pattern.search(title): + return True + + if not text: + return False + + snippet = text[:4000] + for pattern in SUPPLEMENTAL_TEXT_PATTERNS: + if pattern.search(snippet): + return True + + lines = [line.strip() for line in snippet.splitlines() if line.strip()] + if not lines: + return False + + preview = lines[:25] + short_lines = sum(1 for line in preview if len(line.split()) <= 4) + dotted_lines = sum(1 for line in preview if re.search(r"\.{3,}", line)) + numbered_lines = sum(1 for line in preview if re.search(r"\b\d{1,4}\b", line)) + + # Table-of-contents heuristic: many short entries with leaders or numbers. + if preview and short_lines / len(preview) >= 0.6 and (dotted_lines >= 2 or numbered_lines >= 4): + return True + + return False + # Setup logging logging.basicConfig( level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" @@ -1541,6 +1612,12 @@ class HandlerDialog(QDialog): self._block_signals = False self._update_checked_set_from_tree() + def _should_skip_auto_select(self, title, identifier): + content = None + if identifier: + content = self.content_texts.get(identifier, "") + return is_supplemental_section(title, content) + def _run_epub_auto_check(self): iterator = QTreeWidgetItemIterator(self.treeWidget) while iterator.value(): @@ -1550,6 +1627,12 @@ class HandlerDialog(QDialog): continue src = item.data(0, Qt.UserRole) + title = item.text(0) or "" + + if self._should_skip_auto_select(title, src): + item.setCheckState(0, Qt.Unchecked) + iterator += 1 + continue has_significant_content = src and self.content_lengths.get(src, 0) > 1000 is_parent = item.childCount() > 0 @@ -1561,12 +1644,16 @@ class HandlerDialog(QDialog): child = item.child(i) if child.flags() & Qt.ItemIsUserCheckable: child_src = child.data(0, Qt.UserRole) + child_title = child.text(0) or "" child_has_content = ( child_src and self.content_lengths.get(child_src, 0) > 0 ) child_is_parent = child.childCount() > 0 if child_has_content or child_is_parent: - child.setCheckState(0, Qt.Checked) + if self._should_skip_auto_select(child_title, child_src): + child.setCheckState(0, Qt.Unchecked) + else: + child.setCheckState(0, Qt.Checked) else: item.setCheckState(0, Qt.Unchecked) @@ -1582,6 +1669,12 @@ class HandlerDialog(QDialog): continue identifier = item.data(0, Qt.UserRole) + title = item.text(0) or "" + + if self._should_skip_auto_select(title, identifier): + item.setCheckState(0, Qt.Unchecked) + iterator += 1 + continue # Select chapters with content > 500 characters or parent items has_significant_content = identifier and self.content_lengths.get(identifier, 0) > 500 @@ -1595,12 +1688,16 @@ class HandlerDialog(QDialog): child = item.child(i) if child.flags() & Qt.ItemIsUserCheckable: child_identifier = child.data(0, Qt.UserRole) + child_title = child.text(0) or "" child_has_content = ( child_identifier and self.content_lengths.get(child_identifier, 0) > 0 ) child_is_parent = child.childCount() > 0 if child_has_content or child_is_parent: - child.setCheckState(0, Qt.Checked) + if self._should_skip_auto_select(child_title, child_identifier): + child.setCheckState(0, Qt.Unchecked) + else: + child.setCheckState(0, Qt.Checked) else: item.setCheckState(0, Qt.Unchecked) @@ -1629,6 +1726,12 @@ class HandlerDialog(QDialog): iterator += 1 continue + title = item.text(0) or "" + if self._should_skip_auto_select(title, identifier): + item.setCheckState(0, Qt.Unchecked) + iterator += 1 + continue + if ( not identifier.startswith("page_") or self.content_lengths.get(identifier, 0) > 0