mirror of
https://github.com/denizsafak/abogen.git
synced 2026-11-19 01:00:54 +01:00
perf(epub): speed up opening large EPUBs
Opening a large EPUB spent most of its time in two linear scans repeated
once per navigation entry:
- _find_doc_key scanned every spine document for a basename match.
- spine_docs.index() / _docs_between looked up spine positions by list scan.
Build a basename lookup once per doc_order mapping and a
{document: first spine index} map, in both the PyQt parser (book_parser.py)
and the WebUI extractor (text_extractor.py). Exact and URL-decoded matches
are still tried first, and spine positions keep list.index() first-occurrence
semantics.
On a 41 MB EPUB with 7,520 chapters the PyQt parser drops from ~147 s to
~19 s and the WebUI extractor from ~30 s to ~19 s, with byte-identical
extracted text.
Partially addresses #209 (suggestions 1 and 2).
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
654ec6d6fd
commit
e2c451ba11
3 files changed
+146
-26
No files matched your search
+29
-16
@@ -1,3 +1,4 @@
|
||||
import itertools
|
||||
import os
|
||||
import re
|
||||
import logging
|
||||
@@ -475,19 +476,26 @@ class EpubParser(BaseBookParser):
|
||||
return metadata
|
||||
|
||||
def _find_doc_key(self, base_href, doc_order, doc_order_decoded):
|
||||
candidates = [
|
||||
base_href,
|
||||
urllib.parse.unquote(base_href),
|
||||
]
|
||||
base_name = os.path.basename(base_href).lower()
|
||||
for k in list(doc_order.keys()) + list(doc_order_decoded.keys()):
|
||||
if os.path.basename(k).lower() == base_name:
|
||||
candidates.append(k)
|
||||
for candidate in candidates:
|
||||
for candidate in (base_href, urllib.parse.unquote(base_href)):
|
||||
if candidate in doc_order:
|
||||
return candidate, doc_order[candidate]
|
||||
elif candidate in doc_order_decoded:
|
||||
if candidate in doc_order_decoded:
|
||||
return candidate, doc_order_decoded[candidate]
|
||||
|
||||
# Fall back to a basename match, using a lookup built once per
|
||||
# doc_order mapping instead of scanning every document per call.
|
||||
cache = getattr(self, "_basename_cache", None)
|
||||
if cache is None or cache[0] is not doc_order or cache[1] is not doc_order_decoded:
|
||||
by_name = {}
|
||||
for k in itertools.chain(doc_order, doc_order_decoded):
|
||||
by_name.setdefault(os.path.basename(k).lower(), k)
|
||||
cache = (doc_order, doc_order_decoded, by_name)
|
||||
self._basename_cache = cache
|
||||
key = cache[2].get(os.path.basename(base_href).lower())
|
||||
if key is not None:
|
||||
if key in doc_order:
|
||||
return key, doc_order[key]
|
||||
return key, doc_order_decoded[key]
|
||||
return None, None
|
||||
|
||||
def _find_position_robust(self, doc_href, fragment_id):
|
||||
@@ -796,6 +804,10 @@ class EpubParser(BaseBookParser):
|
||||
if item:
|
||||
spine_docs.append(item.get_name())
|
||||
doc_order = {href: i for i, href in enumerate(spine_docs)}
|
||||
# First-occurrence position per doc, equivalent to list.index() in O(1).
|
||||
spine_pos = {}
|
||||
for i, href in enumerate(spine_docs):
|
||||
spine_pos.setdefault(href, i)
|
||||
doc_order_decoded = {
|
||||
urllib.parse.unquote(href): i for href, i in doc_order.items()
|
||||
}
|
||||
@@ -881,8 +893,8 @@ class EpubParser(BaseBookParser):
|
||||
slice_html = current_doc_html[start_slice_pos:]
|
||||
docs_between = []
|
||||
try:
|
||||
idx_current = spine_docs.index(current_doc)
|
||||
idx_next = spine_docs.index(next_doc)
|
||||
idx_current = spine_pos[current_doc]
|
||||
idx_next = spine_pos[next_doc]
|
||||
if idx_current < idx_next:
|
||||
docs_between = [
|
||||
spine_docs[k] for k in range(idx_current + 1, idx_next)
|
||||
@@ -895,7 +907,7 @@ class EpubParser(BaseBookParser):
|
||||
docs_between.extend(
|
||||
[spine_docs[k] for k in range(0, idx_next)]
|
||||
)
|
||||
except ValueError:
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
for doc_href in docs_between:
|
||||
@@ -905,10 +917,10 @@ class EpubParser(BaseBookParser):
|
||||
else:
|
||||
slice_html = current_doc_html[start_slice_pos:]
|
||||
try:
|
||||
idx_current = spine_docs.index(current_doc)
|
||||
idx_current = spine_pos[current_doc]
|
||||
for doc_idx in range(idx_current + 1, len(spine_docs)):
|
||||
slice_html += self.doc_content.get(spine_docs[doc_idx], "")
|
||||
except ValueError:
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
if not slice_html.strip() and current_doc_html:
|
||||
@@ -997,9 +1009,10 @@ class EpubParser(BaseBookParser):
|
||||
else:
|
||||
logging.warning(f"Spine item with id '{item_id}' not found.")
|
||||
|
||||
spine_set = set(spine_docs)
|
||||
for item in self.book.get_items_of_type(ebooklib.ITEM_DOCUMENT):
|
||||
href = item.get_name()
|
||||
if href in spine_docs:
|
||||
if href in spine_set:
|
||||
try:
|
||||
html_content = item.get_content().decode("utf-8", errors="ignore")
|
||||
self.doc_content[href] = html_content
|
||||
|
||||
+41
-10
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import itertools
|
||||
import logging
|
||||
import mimetypes
|
||||
import re
|
||||
@@ -409,6 +410,21 @@ class EpubExtractor:
|
||||
self.book = epub.read_epub(str(path))
|
||||
self.doc_content: Dict[str, str] = {}
|
||||
self.spine_docs: List[str] = []
|
||||
self._spine_pos: Dict[str, int] = {}
|
||||
self._spine_pos_src: Optional[List[str]] = None
|
||||
|
||||
def _spine_index(self, href: str) -> int:
|
||||
# First-occurrence index, equivalent to list.index() but O(1).
|
||||
if self._spine_pos_src is not self.spine_docs:
|
||||
pos: Dict[str, int] = {}
|
||||
for i, doc in enumerate(self.spine_docs):
|
||||
pos.setdefault(doc, i)
|
||||
self._spine_pos = pos
|
||||
self._spine_pos_src = self.spine_docs
|
||||
try:
|
||||
return self._spine_pos[href]
|
||||
except KeyError:
|
||||
raise ValueError(href) from None
|
||||
|
||||
def extract(self) -> ExtractionResult:
|
||||
metadata_source = self._collect_metadata()
|
||||
@@ -626,11 +642,12 @@ class EpubExtractor:
|
||||
def _process_spine_fallback(self) -> List[ExtractedChapter]:
|
||||
chapters: List[ExtractedChapter] = []
|
||||
self.spine_docs = self._build_spine_docs()
|
||||
spine_set = set(self.spine_docs)
|
||||
self.doc_content = {}
|
||||
|
||||
for item in self.book.get_items_of_type(ebooklib.ITEM_DOCUMENT):
|
||||
href = item.get_name()
|
||||
if href not in self.spine_docs:
|
||||
if href not in spine_set:
|
||||
continue
|
||||
try:
|
||||
html_content = item.get_content().decode("utf-8", errors="ignore")
|
||||
@@ -868,16 +885,30 @@ class EpubExtractor:
|
||||
doc_order: Dict[str, int],
|
||||
doc_order_decoded: Dict[str, int],
|
||||
) -> Tuple[Optional[str], Optional[int]]:
|
||||
candidates = {base_href, urllib.parse.unquote(base_href)}
|
||||
base_name = urllib.parse.unquote(base_href).split("/")[-1].lower()
|
||||
for key in list(doc_order.keys()) + list(doc_order_decoded.keys()):
|
||||
if key.split("/")[-1].lower() == base_name:
|
||||
candidates.add(key)
|
||||
for candidate in candidates:
|
||||
for candidate in (base_href, urllib.parse.unquote(base_href)):
|
||||
if candidate in doc_order:
|
||||
return candidate, doc_order[candidate]
|
||||
if candidate in doc_order_decoded:
|
||||
return candidate, doc_order_decoded[candidate]
|
||||
|
||||
# Basename fallback via a lookup built once per doc_order mapping.
|
||||
cache = getattr(self, "_basename_cache", None)
|
||||
if (
|
||||
cache is None
|
||||
or cache[0] is not doc_order
|
||||
or cache[1] is not doc_order_decoded
|
||||
):
|
||||
by_name: Dict[str, str] = {}
|
||||
for key in itertools.chain(doc_order, doc_order_decoded):
|
||||
by_name.setdefault(key.split("/")[-1].lower(), key)
|
||||
cache = (doc_order, doc_order_decoded, by_name)
|
||||
self._basename_cache = cache
|
||||
base_name = urllib.parse.unquote(base_href).split("/")[-1].lower()
|
||||
key = cache[2].get(base_name)
|
||||
if key is not None:
|
||||
if key in doc_order:
|
||||
return key, doc_order[key]
|
||||
return key, doc_order_decoded[key]
|
||||
return None, None
|
||||
|
||||
def _find_position_robust(self, doc_href: str, fragment_id: Optional[str]) -> int:
|
||||
@@ -973,7 +1004,7 @@ class EpubExtractor:
|
||||
def _docs_between(self, current_doc: str, next_doc: Optional[str]) -> List[str]:
|
||||
docs: List[str] = []
|
||||
try:
|
||||
current_idx = self.spine_docs.index(current_doc)
|
||||
current_idx = self._spine_index(current_doc)
|
||||
except ValueError:
|
||||
return docs
|
||||
|
||||
@@ -982,7 +1013,7 @@ class EpubExtractor:
|
||||
return docs
|
||||
|
||||
try:
|
||||
next_idx = self.spine_docs.index(next_doc)
|
||||
next_idx = self._spine_index(next_doc)
|
||||
except ValueError:
|
||||
return docs
|
||||
|
||||
@@ -1008,7 +1039,7 @@ class EpubExtractor:
|
||||
|
||||
prefix_html = ""
|
||||
try:
|
||||
first_idx = self.spine_docs.index(first_doc)
|
||||
first_idx = self._spine_index(first_doc)
|
||||
except ValueError:
|
||||
first_idx = -1
|
||||
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Regression tests for #209 (very slow opening of large EPUBs).
|
||||
|
||||
`_find_doc_key` must resolve documents via a prebuilt lookup instead of
|
||||
scanning every spine document per navigation entry, and spine positions must
|
||||
be O(1) with first-occurrence semantics like ``list.index``.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
||||
|
||||
from abogen.book_parser import EpubParser
|
||||
from abogen.text_extractor import EpubExtractor
|
||||
|
||||
|
||||
def _orders(hrefs):
|
||||
import urllib.parse
|
||||
|
||||
order = {h: i for i, h in enumerate(hrefs)}
|
||||
decoded = {urllib.parse.unquote(h): i for h, i in order.items()}
|
||||
return order, decoded
|
||||
|
||||
|
||||
def _parser_finders():
|
||||
qt = EpubParser.__new__(EpubParser)
|
||||
web = EpubExtractor.__new__(EpubExtractor)
|
||||
return [qt._find_doc_key, web._find_doc_key]
|
||||
|
||||
|
||||
def test_find_doc_key_exact_decoded_and_basename():
|
||||
order, decoded = _orders(["OEBPS/ch%201.xhtml", "OEBPS/ch2.xhtml"])
|
||||
for find in _parser_finders():
|
||||
assert find("OEBPS/ch2.xhtml", order, decoded) == ("OEBPS/ch2.xhtml", 1)
|
||||
assert find("OEBPS/ch%201.xhtml", order, decoded) == ("OEBPS/ch%201.xhtml", 0)
|
||||
# Different directory prefix resolves by case-insensitive basename.
|
||||
assert find("../Text/CH2.xhtml", order, decoded) == ("OEBPS/ch2.xhtml", 1)
|
||||
assert find("missing.xhtml", order, decoded) == (None, None)
|
||||
|
||||
|
||||
def test_find_doc_key_rebuilds_for_new_mappings():
|
||||
for find in _parser_finders():
|
||||
order, decoded = _orders(["a/one.xhtml"])
|
||||
assert find("x/one.xhtml", order, decoded)[0] == "a/one.xhtml"
|
||||
order2, decoded2 = _orders(["b/two.xhtml"])
|
||||
assert find("x/one.xhtml", order2, decoded2) == (None, None)
|
||||
assert find("x/two.xhtml", order2, decoded2)[0] == "b/two.xhtml"
|
||||
|
||||
|
||||
def test_find_doc_key_scales_with_many_documents():
|
||||
hrefs = [f"OEBPS/Text/chapter_{i:05d}.xhtml" for i in range(8000)]
|
||||
order, decoded = _orders(hrefs)
|
||||
for find in _parser_finders():
|
||||
start = time.perf_counter()
|
||||
for i in range(8000):
|
||||
assert find(f"../Text/chapter_{i:05d}.xhtml", order, decoded)[1] == i
|
||||
# The old per-call scan took minutes at this size.
|
||||
assert time.perf_counter() - start < 5
|
||||
|
||||
|
||||
def test_spine_index_first_occurrence_and_missing():
|
||||
web = EpubExtractor.__new__(EpubExtractor)
|
||||
web._spine_pos_src = None
|
||||
web.spine_docs = ["a", "b", "a"]
|
||||
assert web._spine_index("a") == 0
|
||||
assert web._spine_index("b") == 1
|
||||
try:
|
||||
web._spine_index("zzz")
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError("expected ValueError")
|
||||
web.spine_docs = ["z", "a"] # reassigned list invalidates the cache
|
||||
assert web._spine_index("a") == 1
|
||||
Reference in new issue
Block a user