From e2c451ba115e5b14d08d92f9f89777a7e1040d17 Mon Sep 17 00:00:00 2001 From: MarkMad <13027760+MarkMad@users.noreply.github.com> Date: Sat, 24 Oct 2026 15:27:17 +1100 Subject: [PATCH] perf(epub): speed up opening large EPUBs Opening a large EPUB spent most of its time in two linear scans repeated once per navigation entry: - _find_doc_key scanned every spine document for a basename match. - spine_docs.index() / _docs_between looked up spine positions by list scan. Build a basename lookup once per doc_order mapping and a {document: first spine index} map, in both the PyQt parser (book_parser.py) and the WebUI extractor (text_extractor.py). Exact and URL-decoded matches are still tried first, and spine positions keep list.index() first-occurrence semantics. On a 41 MB EPUB with 7,520 chapters the PyQt parser drops from ~147 s to ~19 s and the WebUI extractor from ~30 s to ~19 s, with byte-identical extracted text. Partially addresses #209 (suggestions 1 and 2). Co-Authored-By: Claude Sonnet 5.5 --- abogen/book_parser.py | 45 +++++++++++++-------- abogen/text_extractor.py | 51 ++++++++++++++++++----- tests/test_epub_doc_lookup.py | 76 +++++++++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 26 deletions(-) create mode 100644 tests/test_epub_doc_lookup.py diff --git a/abogen/book_parser.py b/abogen/book_parser.py index 19ca04e..ebfeda2 100644 --- a/abogen/book_parser.py +++ b/abogen/book_parser.py @@ -1,3 +1,4 @@ +import itertools import os import re import logging @@ -475,19 +476,26 @@ class EpubParser(BaseBookParser): return metadata def _find_doc_key(self, base_href, doc_order, doc_order_decoded): - candidates = [ - base_href, - urllib.parse.unquote(base_href), - ] - base_name = os.path.basename(base_href).lower() - for k in list(doc_order.keys()) + list(doc_order_decoded.keys()): - if os.path.basename(k).lower() == base_name: - candidates.append(k) - for candidate in candidates: + for candidate in (base_href, urllib.parse.unquote(base_href)): if candidate in doc_order: return candidate, doc_order[candidate] - elif candidate in doc_order_decoded: + if candidate in doc_order_decoded: return candidate, doc_order_decoded[candidate] + + # Fall back to a basename match, using a lookup built once per + # doc_order mapping instead of scanning every document per call. + cache = getattr(self, "_basename_cache", None) + if cache is None or cache[0] is not doc_order or cache[1] is not doc_order_decoded: + by_name = {} + for k in itertools.chain(doc_order, doc_order_decoded): + by_name.setdefault(os.path.basename(k).lower(), k) + cache = (doc_order, doc_order_decoded, by_name) + self._basename_cache = cache + key = cache[2].get(os.path.basename(base_href).lower()) + if key is not None: + if key in doc_order: + return key, doc_order[key] + return key, doc_order_decoded[key] return None, None def _find_position_robust(self, doc_href, fragment_id): @@ -796,6 +804,10 @@ class EpubParser(BaseBookParser): if item: spine_docs.append(item.get_name()) doc_order = {href: i for i, href in enumerate(spine_docs)} + # First-occurrence position per doc, equivalent to list.index() in O(1). + spine_pos = {} + for i, href in enumerate(spine_docs): + spine_pos.setdefault(href, i) doc_order_decoded = { urllib.parse.unquote(href): i for href, i in doc_order.items() } @@ -881,8 +893,8 @@ class EpubParser(BaseBookParser): slice_html = current_doc_html[start_slice_pos:] docs_between = [] try: - idx_current = spine_docs.index(current_doc) - idx_next = spine_docs.index(next_doc) + idx_current = spine_pos[current_doc] + idx_next = spine_pos[next_doc] if idx_current < idx_next: docs_between = [ spine_docs[k] for k in range(idx_current + 1, idx_next) @@ -895,7 +907,7 @@ class EpubParser(BaseBookParser): docs_between.extend( [spine_docs[k] for k in range(0, idx_next)] ) - except ValueError: + except KeyError: pass for doc_href in docs_between: @@ -905,10 +917,10 @@ class EpubParser(BaseBookParser): else: slice_html = current_doc_html[start_slice_pos:] try: - idx_current = spine_docs.index(current_doc) + idx_current = spine_pos[current_doc] for doc_idx in range(idx_current + 1, len(spine_docs)): slice_html += self.doc_content.get(spine_docs[doc_idx], "") - except ValueError: + except KeyError: pass if not slice_html.strip() and current_doc_html: @@ -997,9 +1009,10 @@ class EpubParser(BaseBookParser): else: logging.warning(f"Spine item with id '{item_id}' not found.") + spine_set = set(spine_docs) for item in self.book.get_items_of_type(ebooklib.ITEM_DOCUMENT): href = item.get_name() - if href in spine_docs: + if href in spine_set: try: html_content = item.get_content().decode("utf-8", errors="ignore") self.doc_content[href] = html_content diff --git a/abogen/text_extractor.py b/abogen/text_extractor.py index b47c349..878a246 100644 --- a/abogen/text_extractor.py +++ b/abogen/text_extractor.py @@ -1,6 +1,7 @@ from __future__ import annotations import datetime +import itertools import logging import mimetypes import re @@ -409,6 +410,21 @@ class EpubExtractor: self.book = epub.read_epub(str(path)) self.doc_content: Dict[str, str] = {} self.spine_docs: List[str] = [] + self._spine_pos: Dict[str, int] = {} + self._spine_pos_src: Optional[List[str]] = None + + def _spine_index(self, href: str) -> int: + # First-occurrence index, equivalent to list.index() but O(1). + if self._spine_pos_src is not self.spine_docs: + pos: Dict[str, int] = {} + for i, doc in enumerate(self.spine_docs): + pos.setdefault(doc, i) + self._spine_pos = pos + self._spine_pos_src = self.spine_docs + try: + return self._spine_pos[href] + except KeyError: + raise ValueError(href) from None def extract(self) -> ExtractionResult: metadata_source = self._collect_metadata() @@ -626,11 +642,12 @@ class EpubExtractor: def _process_spine_fallback(self) -> List[ExtractedChapter]: chapters: List[ExtractedChapter] = [] self.spine_docs = self._build_spine_docs() + spine_set = set(self.spine_docs) self.doc_content = {} for item in self.book.get_items_of_type(ebooklib.ITEM_DOCUMENT): href = item.get_name() - if href not in self.spine_docs: + if href not in spine_set: continue try: html_content = item.get_content().decode("utf-8", errors="ignore") @@ -868,16 +885,30 @@ class EpubExtractor: doc_order: Dict[str, int], doc_order_decoded: Dict[str, int], ) -> Tuple[Optional[str], Optional[int]]: - candidates = {base_href, urllib.parse.unquote(base_href)} - base_name = urllib.parse.unquote(base_href).split("/")[-1].lower() - for key in list(doc_order.keys()) + list(doc_order_decoded.keys()): - if key.split("/")[-1].lower() == base_name: - candidates.add(key) - for candidate in candidates: + for candidate in (base_href, urllib.parse.unquote(base_href)): if candidate in doc_order: return candidate, doc_order[candidate] if candidate in doc_order_decoded: return candidate, doc_order_decoded[candidate] + + # Basename fallback via a lookup built once per doc_order mapping. + cache = getattr(self, "_basename_cache", None) + if ( + cache is None + or cache[0] is not doc_order + or cache[1] is not doc_order_decoded + ): + by_name: Dict[str, str] = {} + for key in itertools.chain(doc_order, doc_order_decoded): + by_name.setdefault(key.split("/")[-1].lower(), key) + cache = (doc_order, doc_order_decoded, by_name) + self._basename_cache = cache + base_name = urllib.parse.unquote(base_href).split("/")[-1].lower() + key = cache[2].get(base_name) + if key is not None: + if key in doc_order: + return key, doc_order[key] + return key, doc_order_decoded[key] return None, None def _find_position_robust(self, doc_href: str, fragment_id: Optional[str]) -> int: @@ -973,7 +1004,7 @@ class EpubExtractor: def _docs_between(self, current_doc: str, next_doc: Optional[str]) -> List[str]: docs: List[str] = [] try: - current_idx = self.spine_docs.index(current_doc) + current_idx = self._spine_index(current_doc) except ValueError: return docs @@ -982,7 +1013,7 @@ class EpubExtractor: return docs try: - next_idx = self.spine_docs.index(next_doc) + next_idx = self._spine_index(next_doc) except ValueError: return docs @@ -1008,7 +1039,7 @@ class EpubExtractor: prefix_html = "" try: - first_idx = self.spine_docs.index(first_doc) + first_idx = self._spine_index(first_doc) except ValueError: first_idx = -1 diff --git a/tests/test_epub_doc_lookup.py b/tests/test_epub_doc_lookup.py new file mode 100644 index 0000000..720bb54 --- /dev/null +++ b/tests/test_epub_doc_lookup.py @@ -0,0 +1,76 @@ +"""Regression tests for #209 (very slow opening of large EPUBs). + +`_find_doc_key` must resolve documents via a prebuilt lookup instead of +scanning every spine document per navigation entry, and spine positions must +be O(1) with first-occurrence semantics like ``list.index``. +""" + +import os +import sys +import time +from pathlib import Path + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +from abogen.book_parser import EpubParser +from abogen.text_extractor import EpubExtractor + + +def _orders(hrefs): + import urllib.parse + + order = {h: i for i, h in enumerate(hrefs)} + decoded = {urllib.parse.unquote(h): i for h, i in order.items()} + return order, decoded + + +def _parser_finders(): + qt = EpubParser.__new__(EpubParser) + web = EpubExtractor.__new__(EpubExtractor) + return [qt._find_doc_key, web._find_doc_key] + + +def test_find_doc_key_exact_decoded_and_basename(): + order, decoded = _orders(["OEBPS/ch%201.xhtml", "OEBPS/ch2.xhtml"]) + for find in _parser_finders(): + assert find("OEBPS/ch2.xhtml", order, decoded) == ("OEBPS/ch2.xhtml", 1) + assert find("OEBPS/ch%201.xhtml", order, decoded) == ("OEBPS/ch%201.xhtml", 0) + # Different directory prefix resolves by case-insensitive basename. + assert find("../Text/CH2.xhtml", order, decoded) == ("OEBPS/ch2.xhtml", 1) + assert find("missing.xhtml", order, decoded) == (None, None) + + +def test_find_doc_key_rebuilds_for_new_mappings(): + for find in _parser_finders(): + order, decoded = _orders(["a/one.xhtml"]) + assert find("x/one.xhtml", order, decoded)[0] == "a/one.xhtml" + order2, decoded2 = _orders(["b/two.xhtml"]) + assert find("x/one.xhtml", order2, decoded2) == (None, None) + assert find("x/two.xhtml", order2, decoded2)[0] == "b/two.xhtml" + + +def test_find_doc_key_scales_with_many_documents(): + hrefs = [f"OEBPS/Text/chapter_{i:05d}.xhtml" for i in range(8000)] + order, decoded = _orders(hrefs) + for find in _parser_finders(): + start = time.perf_counter() + for i in range(8000): + assert find(f"../Text/chapter_{i:05d}.xhtml", order, decoded)[1] == i + # The old per-call scan took minutes at this size. + assert time.perf_counter() - start < 5 + + +def test_spine_index_first_occurrence_and_missing(): + web = EpubExtractor.__new__(EpubExtractor) + web._spine_pos_src = None + web.spine_docs = ["a", "b", "a"] + assert web._spine_index("a") == 0 + assert web._spine_index("b") == 1 + try: + web._spine_index("zzz") + except ValueError: + pass + else: + raise AssertionError("expected ValueError") + web.spine_docs = ["z", "a"] # reassigned list invalidates the cache + assert web._spine_index("a") == 1