perf(epub): speed up opening large EPUBs

Opening a large EPUB spent most of its time in two linear scans repeated
once per navigation entry:

- _find_doc_key scanned every spine document for a basename match.
- spine_docs.index() / _docs_between looked up spine positions by list scan.

Build a basename lookup once per doc_order mapping and a
{document: first spine index} map, in both the PyQt parser (book_parser.py)
and the WebUI extractor (text_extractor.py). Exact and URL-decoded matches
are still tried first, and spine positions keep list.index() first-occurrence
semantics.

On a 41 MB EPUB with 7,520 chapters the PyQt parser drops from ~147 s to
~19 s and the WebUI extractor from ~30 s to ~19 s, with byte-identical
extracted text.

Partially addresses #209 (suggestions 1 and 2).

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
MarkMadandClaude Sonnet 5.5 committed 2026-10-24 15:27:17 +11:00
1 parent 654ec6d6fd
commit e2c451ba11
3 files changed
+146 -26

No files matched your search

+29 -16
View File
@@ -1,3 +1,4 @@
import itertools
import os
import re
import logging
@@ -475,19 +476,26 @@ class EpubParser(BaseBookParser):
return metadata
def _find_doc_key(self, base_href, doc_order, doc_order_decoded):
candidates = [
base_href,
urllib.parse.unquote(base_href),
]
base_name = os.path.basename(base_href).lower()
for k in list(doc_order.keys()) + list(doc_order_decoded.keys()):
if os.path.basename(k).lower() == base_name:
candidates.append(k)
for candidate in candidates:
for candidate in (base_href, urllib.parse.unquote(base_href)):
if candidate in doc_order:
return candidate, doc_order[candidate]
elif candidate in doc_order_decoded:
if candidate in doc_order_decoded:
return candidate, doc_order_decoded[candidate]
# Fall back to a basename match, using a lookup built once per
# doc_order mapping instead of scanning every document per call.
cache = getattr(self, "_basename_cache", None)
if cache is None or cache[0] is not doc_order or cache[1] is not doc_order_decoded:
by_name = {}
for k in itertools.chain(doc_order, doc_order_decoded):
by_name.setdefault(os.path.basename(k).lower(), k)
cache = (doc_order, doc_order_decoded, by_name)
self._basename_cache = cache
key = cache[2].get(os.path.basename(base_href).lower())
if key is not None:
if key in doc_order:
return key, doc_order[key]
return key, doc_order_decoded[key]
return None, None
def _find_position_robust(self, doc_href, fragment_id):
@@ -796,6 +804,10 @@ class EpubParser(BaseBookParser):
if item:
spine_docs.append(item.get_name())
doc_order = {href: i for i, href in enumerate(spine_docs)}
# First-occurrence position per doc, equivalent to list.index() in O(1).
spine_pos = {}
for i, href in enumerate(spine_docs):
spine_pos.setdefault(href, i)
doc_order_decoded = {
urllib.parse.unquote(href): i for href, i in doc_order.items()
}
@@ -881,8 +893,8 @@ class EpubParser(BaseBookParser):
slice_html = current_doc_html[start_slice_pos:]
docs_between = []
try:
idx_current = spine_docs.index(current_doc)
idx_next = spine_docs.index(next_doc)
idx_current = spine_pos[current_doc]
idx_next = spine_pos[next_doc]
if idx_current < idx_next:
docs_between = [
spine_docs[k] for k in range(idx_current + 1, idx_next)
@@ -895,7 +907,7 @@ class EpubParser(BaseBookParser):
docs_between.extend(
[spine_docs[k] for k in range(0, idx_next)]
)
except ValueError:
except KeyError:
pass
for doc_href in docs_between:
@@ -905,10 +917,10 @@ class EpubParser(BaseBookParser):
else:
slice_html = current_doc_html[start_slice_pos:]
try:
idx_current = spine_docs.index(current_doc)
idx_current = spine_pos[current_doc]
for doc_idx in range(idx_current + 1, len(spine_docs)):
slice_html += self.doc_content.get(spine_docs[doc_idx], "")
except ValueError:
except KeyError:
pass
if not slice_html.strip() and current_doc_html:
@@ -997,9 +1009,10 @@ class EpubParser(BaseBookParser):
else:
logging.warning(f"Spine item with id '{item_id}' not found.")
spine_set = set(spine_docs)
for item in self.book.get_items_of_type(ebooklib.ITEM_DOCUMENT):
href = item.get_name()
if href in spine_docs:
if href in spine_set:
try:
html_content = item.get_content().decode("utf-8", errors="ignore")
self.doc_content[href] = html_content
+41 -10
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
import datetime
import itertools
import logging
import mimetypes
import re
@@ -409,6 +410,21 @@ class EpubExtractor:
self.book = epub.read_epub(str(path))
self.doc_content: Dict[str, str] = {}
self.spine_docs: List[str] = []
self._spine_pos: Dict[str, int] = {}
self._spine_pos_src: Optional[List[str]] = None
def _spine_index(self, href: str) -> int:
# First-occurrence index, equivalent to list.index() but O(1).
if self._spine_pos_src is not self.spine_docs:
pos: Dict[str, int] = {}
for i, doc in enumerate(self.spine_docs):
pos.setdefault(doc, i)
self._spine_pos = pos
self._spine_pos_src = self.spine_docs
try:
return self._spine_pos[href]
except KeyError:
raise ValueError(href) from None
def extract(self) -> ExtractionResult:
metadata_source = self._collect_metadata()
@@ -626,11 +642,12 @@ class EpubExtractor:
def _process_spine_fallback(self) -> List[ExtractedChapter]:
chapters: List[ExtractedChapter] = []
self.spine_docs = self._build_spine_docs()
spine_set = set(self.spine_docs)
self.doc_content = {}
for item in self.book.get_items_of_type(ebooklib.ITEM_DOCUMENT):
href = item.get_name()
if href not in self.spine_docs:
if href not in spine_set:
continue
try:
html_content = item.get_content().decode("utf-8", errors="ignore")
@@ -868,16 +885,30 @@ class EpubExtractor:
doc_order: Dict[str, int],
doc_order_decoded: Dict[str, int],
) -> Tuple[Optional[str], Optional[int]]:
candidates = {base_href, urllib.parse.unquote(base_href)}
base_name = urllib.parse.unquote(base_href).split("/")[-1].lower()
for key in list(doc_order.keys()) + list(doc_order_decoded.keys()):
if key.split("/")[-1].lower() == base_name:
candidates.add(key)
for candidate in candidates:
for candidate in (base_href, urllib.parse.unquote(base_href)):
if candidate in doc_order:
return candidate, doc_order[candidate]
if candidate in doc_order_decoded:
return candidate, doc_order_decoded[candidate]
# Basename fallback via a lookup built once per doc_order mapping.
cache = getattr(self, "_basename_cache", None)
if (
cache is None
or cache[0] is not doc_order
or cache[1] is not doc_order_decoded
):
by_name: Dict[str, str] = {}
for key in itertools.chain(doc_order, doc_order_decoded):
by_name.setdefault(key.split("/")[-1].lower(), key)
cache = (doc_order, doc_order_decoded, by_name)
self._basename_cache = cache
base_name = urllib.parse.unquote(base_href).split("/")[-1].lower()
key = cache[2].get(base_name)
if key is not None:
if key in doc_order:
return key, doc_order[key]
return key, doc_order_decoded[key]
return None, None
def _find_position_robust(self, doc_href: str, fragment_id: Optional[str]) -> int:
@@ -973,7 +1004,7 @@ class EpubExtractor:
def _docs_between(self, current_doc: str, next_doc: Optional[str]) -> List[str]:
docs: List[str] = []
try:
current_idx = self.spine_docs.index(current_doc)
current_idx = self._spine_index(current_doc)
except ValueError:
return docs
@@ -982,7 +1013,7 @@ class EpubExtractor:
return docs
try:
next_idx = self.spine_docs.index(next_doc)
next_idx = self._spine_index(next_doc)
except ValueError:
return docs
@@ -1008,7 +1039,7 @@ class EpubExtractor:
prefix_html = ""
try:
first_idx = self.spine_docs.index(first_doc)
first_idx = self._spine_index(first_doc)
except ValueError:
first_idx = -1
+76
View File
@@ -0,0 +1,76 @@
"""Regression tests for #209 (very slow opening of large EPUBs).
`_find_doc_key` must resolve documents via a prebuilt lookup instead of
scanning every spine document per navigation entry, and spine positions must
be O(1) with first-occurrence semantics like ``list.index``.
"""
import os
import sys
import time
from pathlib import Path
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from abogen.book_parser import EpubParser
from abogen.text_extractor import EpubExtractor
def _orders(hrefs):
import urllib.parse
order = {h: i for i, h in enumerate(hrefs)}
decoded = {urllib.parse.unquote(h): i for h, i in order.items()}
return order, decoded
def _parser_finders():
qt = EpubParser.__new__(EpubParser)
web = EpubExtractor.__new__(EpubExtractor)
return [qt._find_doc_key, web._find_doc_key]
def test_find_doc_key_exact_decoded_and_basename():
order, decoded = _orders(["OEBPS/ch%201.xhtml", "OEBPS/ch2.xhtml"])
for find in _parser_finders():
assert find("OEBPS/ch2.xhtml", order, decoded) == ("OEBPS/ch2.xhtml", 1)
assert find("OEBPS/ch%201.xhtml", order, decoded) == ("OEBPS/ch%201.xhtml", 0)
# Different directory prefix resolves by case-insensitive basename.
assert find("../Text/CH2.xhtml", order, decoded) == ("OEBPS/ch2.xhtml", 1)
assert find("missing.xhtml", order, decoded) == (None, None)
def test_find_doc_key_rebuilds_for_new_mappings():
for find in _parser_finders():
order, decoded = _orders(["a/one.xhtml"])
assert find("x/one.xhtml", order, decoded)[0] == "a/one.xhtml"
order2, decoded2 = _orders(["b/two.xhtml"])
assert find("x/one.xhtml", order2, decoded2) == (None, None)
assert find("x/two.xhtml", order2, decoded2)[0] == "b/two.xhtml"
def test_find_doc_key_scales_with_many_documents():
hrefs = [f"OEBPS/Text/chapter_{i:05d}.xhtml" for i in range(8000)]
order, decoded = _orders(hrefs)
for find in _parser_finders():
start = time.perf_counter()
for i in range(8000):
assert find(f"../Text/chapter_{i:05d}.xhtml", order, decoded)[1] == i
# The old per-call scan took minutes at this size.
assert time.perf_counter() - start < 5
def test_spine_index_first_occurrence_and_missing():
web = EpubExtractor.__new__(EpubExtractor)
web._spine_pos_src = None
web.spine_docs = ["a", "b", "a"]
assert web._spine_index("a") == 0
assert web._spine_index("b") == 1
try:
web._spine_index("zzz")
except ValueError:
pass
else:
raise AssertionError("expected ValueError")
web.spine_docs = ["z", "a"] # reassigned list invalidates the cache
assert web._spine_index("a") == 1