mirror of
https://github.com/denizsafak/abogen.git
synced 2026-07-18 13:40:27 +02:00
feat: Enhance series metadata handling in Calibre OPDS integration and Epub extraction
This commit is contained in:
@@ -129,6 +129,10 @@ class OPDSFeed:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def feed_to_dict(feed: OPDSFeed) -> Dict[str, Any]:
|
||||||
|
return feed.to_dict()
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class DownloadedResource:
|
class DownloadedResource:
|
||||||
filename: str
|
filename: str
|
||||||
@@ -677,7 +681,8 @@ class CalibreOPDSClient:
|
|||||||
|
|
||||||
if series_name is None or series_index is None:
|
if series_name is None or series_index is None:
|
||||||
category_series_name, category_series_index = self._extract_series_from_categories(
|
category_series_name, category_series_index = self._extract_series_from_categories(
|
||||||
node.findall("atom:category", NS)
|
node.findall("atom:category", NS),
|
||||||
|
authors=authors,
|
||||||
)
|
)
|
||||||
if series_name is None and category_series_name:
|
if series_name is None and category_series_name:
|
||||||
series_name = category_series_name
|
series_name = category_series_name
|
||||||
@@ -716,9 +721,15 @@ class CalibreOPDSClient:
|
|||||||
rating_max=rating_max,
|
rating_max=rating_max,
|
||||||
)
|
)
|
||||||
|
|
||||||
def _extract_series_from_categories(self, category_nodes: List[ET.Element]) -> tuple[Optional[str], Optional[float]]:
|
def _extract_series_from_categories(
|
||||||
|
self,
|
||||||
|
category_nodes: List[ET.Element],
|
||||||
|
*,
|
||||||
|
authors: Optional[List[str]] = None,
|
||||||
|
) -> tuple[Optional[str], Optional[float]]:
|
||||||
name: Optional[str] = None
|
name: Optional[str] = None
|
||||||
index: Optional[float] = None
|
index: Optional[float] = None
|
||||||
|
author_set = {str(author).strip().casefold() for author in (authors or []) if str(author).strip()}
|
||||||
for category in category_nodes:
|
for category in category_nodes:
|
||||||
scheme = (category.attrib.get("scheme") or "").strip().lower()
|
scheme = (category.attrib.get("scheme") or "").strip().lower()
|
||||||
label = (category.attrib.get("label") or "").strip()
|
label = (category.attrib.get("label") or "").strip()
|
||||||
@@ -729,7 +740,9 @@ class CalibreOPDSClient:
|
|||||||
if term and term not in values:
|
if term and term not in values:
|
||||||
values.append(term)
|
values.append(term)
|
||||||
|
|
||||||
is_series_hint = "series" in scheme or any("series" in value.lower() for value in values if value)
|
# Be conservative: category schemes are often URLs and can contain unrelated substrings.
|
||||||
|
# Also, some catalog feeds incorrectly include author names in series-like categories.
|
||||||
|
is_series_hint = self._is_series_scheme(scheme) or any("series" in value.lower() for value in values if value)
|
||||||
if not is_series_hint:
|
if not is_series_hint:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -737,6 +750,9 @@ class CalibreOPDSClient:
|
|||||||
if not value:
|
if not value:
|
||||||
continue
|
continue
|
||||||
candidate_name, candidate_index = self._parse_series_value(value)
|
candidate_name, candidate_index = self._parse_series_value(value)
|
||||||
|
if candidate_name and candidate_name.casefold() in author_set:
|
||||||
|
# Guardrail: avoid mapping the author name into series.
|
||||||
|
continue
|
||||||
if candidate_name and not name:
|
if candidate_name and not name:
|
||||||
name = candidate_name
|
name = candidate_name
|
||||||
if candidate_index is not None and index is None:
|
if candidate_index is not None and index is None:
|
||||||
@@ -745,6 +761,15 @@ class CalibreOPDSClient:
|
|||||||
return name, index
|
return name, index
|
||||||
return name, index
|
return name, index
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _is_series_scheme(scheme: str) -> bool:
|
||||||
|
cleaned = (scheme or "").strip().lower()
|
||||||
|
if not cleaned:
|
||||||
|
return False
|
||||||
|
if "author" in cleaned:
|
||||||
|
return False
|
||||||
|
return bool(re.search(r"(^|[/#:\-])series([/#:\-]|$)", cleaned))
|
||||||
|
|
||||||
def _parse_series_value(self, value: str) -> tuple[Optional[str], Optional[float]]:
|
def _parse_series_value(self, value: str) -> tuple[Optional[str], Optional[float]]:
|
||||||
cleaned = re.sub(r"\s+", " ", value or "").strip()
|
cleaned = re.sub(r"\s+", " ", value or "").strip()
|
||||||
if not cleaned:
|
if not cleaned:
|
||||||
@@ -1111,6 +1136,8 @@ class CalibreOPDSClient:
|
|||||||
|
|
||||||
scored_entries = []
|
scored_entries = []
|
||||||
for entry in feed.entries:
|
for entry in feed.entries:
|
||||||
|
if not self._entry_matches_query(entry, tokens):
|
||||||
|
continue
|
||||||
score = self._calculate_match_score(entry, tokens)
|
score = self._calculate_match_score(entry, tokens)
|
||||||
# Require a minimum score to avoid weak matches (e.g. single word in summary)
|
# Require a minimum score to avoid weak matches (e.g. single word in summary)
|
||||||
if score >= 10:
|
if score >= 10:
|
||||||
|
|||||||
@@ -72,6 +72,8 @@ class MetadataSource:
|
|||||||
publisher: Optional[str] = None
|
publisher: Optional[str] = None
|
||||||
publication_year: Optional[str] = None
|
publication_year: Optional[str] = None
|
||||||
language: Optional[str] = None
|
language: Optional[str] = None
|
||||||
|
series: Optional[str] = None
|
||||||
|
series_index: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -390,6 +392,18 @@ class EpubExtractor:
|
|||||||
chapters = [ExtractedChapter(title=self.path.stem, text="")]
|
chapters = [ExtractedChapter(title=self.path.stem, text="")]
|
||||||
metadata = _build_metadata_payload(metadata_source, len(chapters), "epub", self.path.stem)
|
metadata = _build_metadata_payload(metadata_source, len(chapters), "epub", self.path.stem)
|
||||||
metadata.setdefault("chapter_count", str(len(chapters)))
|
metadata.setdefault("chapter_count", str(len(chapters)))
|
||||||
|
if metadata_source.series:
|
||||||
|
series_text = str(metadata_source.series).strip()
|
||||||
|
if series_text:
|
||||||
|
metadata.setdefault("series", series_text)
|
||||||
|
metadata.setdefault("series_name", series_text)
|
||||||
|
metadata.setdefault("seriesname", series_text)
|
||||||
|
if metadata_source.series_index:
|
||||||
|
idx_text = str(metadata_source.series_index).strip()
|
||||||
|
if idx_text:
|
||||||
|
metadata.setdefault("series_index", idx_text)
|
||||||
|
metadata.setdefault("series_sequence", idx_text)
|
||||||
|
metadata.setdefault("book_number", idx_text)
|
||||||
cover_image, cover_mime = self._extract_cover()
|
cover_image, cover_mime = self._extract_cover()
|
||||||
return ExtractionResult(
|
return ExtractionResult(
|
||||||
chapters=chapters,
|
chapters=chapters,
|
||||||
@@ -444,6 +458,41 @@ class EpubExtractor:
|
|||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.debug("Failed to extract EPUB language metadata: %s", exc)
|
logger.debug("Failed to extract EPUB language metadata: %s", exc)
|
||||||
|
|
||||||
|
# Series metadata (best-effort). Common sources:
|
||||||
|
# - Calibre embeds OPF meta tags: <meta name="calibre:series" content="..." />
|
||||||
|
# - EPUB3 collections via: <meta property="belongs-to-collection">...</meta>
|
||||||
|
try:
|
||||||
|
meta_items = self.book.get_metadata("OPF", "meta")
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug("Failed to extract EPUB OPF meta tags: %s", exc)
|
||||||
|
meta_items = []
|
||||||
|
|
||||||
|
series_name: Optional[str] = None
|
||||||
|
series_index: Optional[str] = None
|
||||||
|
for value, attrs in meta_items or []:
|
||||||
|
attrs_dict = attrs or {}
|
||||||
|
name = str(attrs_dict.get("name") or "").strip().casefold()
|
||||||
|
prop = str(attrs_dict.get("property") or "").strip().casefold()
|
||||||
|
content = attrs_dict.get("content")
|
||||||
|
candidate = content if content is not None else value
|
||||||
|
candidate_text = str(candidate or "").strip()
|
||||||
|
if not candidate_text:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if name in {"calibre:series", "series"} and series_name is None:
|
||||||
|
series_name = candidate_text
|
||||||
|
continue
|
||||||
|
if name in {"calibre:series_index", "calibre:seriesindex", "series_index", "seriesindex"} and series_index is None:
|
||||||
|
series_index = candidate_text
|
||||||
|
continue
|
||||||
|
|
||||||
|
if prop.endswith("belongs-to-collection") and series_name is None:
|
||||||
|
series_name = candidate_text
|
||||||
|
continue
|
||||||
|
|
||||||
|
metadata.series = series_name
|
||||||
|
metadata.series_index = series_index
|
||||||
|
|
||||||
return metadata
|
return metadata
|
||||||
|
|
||||||
def _extract_cover(self) -> Tuple[Optional[bytes], Optional[str]]:
|
def _extract_cover(self) -> Tuple[Optional[bytes], Optional[str]]:
|
||||||
|
|||||||
@@ -1780,11 +1780,30 @@ def run_conversion_job(job: Job) -> None:
|
|||||||
auto_prefix_titles = getattr(job, "auto_prefix_chapter_titles", True)
|
auto_prefix_titles = getattr(job, "auto_prefix_chapter_titles", True)
|
||||||
read_title_intro = getattr(job, "read_title_intro", False)
|
read_title_intro = getattr(job, "read_title_intro", False)
|
||||||
book_intro_text = ""
|
book_intro_text = ""
|
||||||
|
intro_provider: Optional[str] = None
|
||||||
|
intro_voice_choice: Any = None
|
||||||
|
intro_speed: Optional[float] = None
|
||||||
|
intro_steps: Optional[int] = None
|
||||||
if read_title_intro:
|
if read_title_intro:
|
||||||
book_intro_text = _build_title_intro_text(job.metadata_tags, job.original_filename)
|
book_intro_text = _build_title_intro_text(job.metadata_tags, job.original_filename)
|
||||||
if book_intro_text:
|
if book_intro_text:
|
||||||
preview = book_intro_text if len(book_intro_text) <= 120 else f"{book_intro_text[:117]}…"
|
preview = book_intro_text if len(book_intro_text) <= 120 else f"{book_intro_text[:117]}…"
|
||||||
job.add_log(f"Title intro enabled: {preview}", level="debug")
|
job.add_log(f"Title intro enabled: {preview}", level="debug")
|
||||||
|
|
||||||
|
intro_voice_spec = base_voice_spec or job.voice
|
||||||
|
if intro_voice_spec == "__custom_mix":
|
||||||
|
intro_voice_spec = base_voice_spec or ""
|
||||||
|
if not intro_voice_spec:
|
||||||
|
fallback_key = next(iter(voice_cache.keys()), "")
|
||||||
|
if fallback_key and fallback_key != "__custom_mix":
|
||||||
|
intro_voice_spec = fallback_key.split(":", 1)[-1]
|
||||||
|
if not intro_voice_spec and VOICES_INTERNAL:
|
||||||
|
intro_voice_spec = VOICES_INTERNAL[0]
|
||||||
|
|
||||||
|
if intro_voice_spec:
|
||||||
|
intro_provider, _, intro_voice_choice, intro_speed, intro_steps = resolve_voice_choice(
|
||||||
|
intro_voice_spec
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
job.add_log("Title intro enabled but no usable metadata was found.", level="debug")
|
job.add_log("Title intro enabled but no usable metadata was found.", level="debug")
|
||||||
intro_emitted = False
|
intro_emitted = False
|
||||||
@@ -1958,14 +1977,18 @@ def run_conversion_job(job: Job) -> None:
|
|||||||
remove_heading_from_body = True
|
remove_heading_from_body = True
|
||||||
|
|
||||||
if not intro_emitted and book_intro_text:
|
if not intro_emitted and book_intro_text:
|
||||||
|
intro_use_provider = intro_provider or chapter_provider
|
||||||
|
intro_use_voice_choice = intro_voice_choice if intro_voice_choice is not None else voice_choice
|
||||||
|
intro_use_speed = intro_speed if intro_speed is not None else chapter_speed
|
||||||
|
intro_use_steps = intro_steps if intro_steps is not None else chapter_steps
|
||||||
intro_segments = emit_text(
|
intro_segments = emit_text(
|
||||||
book_intro_text,
|
book_intro_text,
|
||||||
voice_choice=voice_choice,
|
voice_choice=intro_use_voice_choice,
|
||||||
chapter_sink=chapter_sink,
|
chapter_sink=chapter_sink,
|
||||||
preview_prefix="Book intro",
|
preview_prefix="Book intro",
|
||||||
tts_provider=chapter_provider,
|
tts_provider=intro_use_provider,
|
||||||
speed_override=chapter_speed,
|
speed_override=intro_use_speed,
|
||||||
supertonic_steps_override=chapter_steps,
|
supertonic_steps_override=intro_use_steps,
|
||||||
)
|
)
|
||||||
intro_emitted = True
|
intro_emitted = True
|
||||||
if intro_segments > 0 and job.chapter_intro_delay > 0:
|
if intro_segments > 0 and job.chapter_intro_delay > 0:
|
||||||
|
|||||||
@@ -89,6 +89,40 @@ def test_calibre_opds_feed_extracts_series_from_categories() -> None:
|
|||||||
assert entry.series_index == 5.0
|
assert entry.series_index == 5.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_calibre_opds_does_not_map_author_into_series_from_categories() -> None:
|
||||||
|
client = CalibreOPDSClient("http://example.com/catalog")
|
||||||
|
xml_payload = """<?xml version=\"1.0\" encoding=\"UTF-8\"?>
|
||||||
|
<feed xmlns=\"http://www.w3.org/2005/Atom\"
|
||||||
|
xmlns:dc=\"http://purl.org/dc/terms/\"
|
||||||
|
xmlns:calibre=\"http://calibre.kovidgoyal.net/2009/catalog\">
|
||||||
|
<id>catalog</id>
|
||||||
|
<title>Example Catalog</title>
|
||||||
|
<entry>
|
||||||
|
<id>book-author-series-bug</id>
|
||||||
|
<title>Sample Book</title>
|
||||||
|
<author>
|
||||||
|
<name>Alexandre Dumas</name>
|
||||||
|
</author>
|
||||||
|
<category
|
||||||
|
scheme=\"http://calibre.kovidgoyal.net/2009/series\"
|
||||||
|
term=\"Books: Alexandre Dumas\"
|
||||||
|
label=\"Books: Alexandre Dumas\" />
|
||||||
|
<link rel=\"http://opds-spec.org/acquisition\"
|
||||||
|
href=\"books/sample.epub\"
|
||||||
|
type=\"application/epub+zip\" />
|
||||||
|
</entry>
|
||||||
|
</feed>
|
||||||
|
"""
|
||||||
|
|
||||||
|
feed = client._parse_feed(xml_payload, base_url="http://example.com/catalog")
|
||||||
|
assert feed.entries
|
||||||
|
entry = feed.entries[0]
|
||||||
|
|
||||||
|
assert entry.authors == ["Alexandre Dumas"]
|
||||||
|
assert entry.series is None
|
||||||
|
assert entry.series_index is None
|
||||||
|
|
||||||
|
|
||||||
def test_calibre_opds_extracts_tags_and_rating_from_summary() -> None:
|
def test_calibre_opds_extracts_tags_and_rating_from_summary() -> None:
|
||||||
client = CalibreOPDSClient("http://example.com/catalog")
|
client = CalibreOPDSClient("http://example.com/catalog")
|
||||||
xml_payload = """<?xml version=\"1.0\" encoding=\"UTF-8\"?>
|
xml_payload = """<?xml version=\"1.0\" encoding=\"UTF-8\"?>
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from ebooklib import epub
|
||||||
|
|
||||||
from abogen.text_extractor import extract_from_path
|
from abogen.text_extractor import extract_from_path
|
||||||
from abogen.utils import calculate_text_length
|
from abogen.utils import calculate_text_length
|
||||||
|
|
||||||
@@ -25,3 +27,30 @@ def test_epub_metadata_composer_matches_artist():
|
|||||||
assert composer
|
assert composer
|
||||||
assert composer == artist
|
assert composer == artist
|
||||||
assert composer != "Narrator"
|
assert composer != "Narrator"
|
||||||
|
|
||||||
|
|
||||||
|
def test_epub_series_metadata_extracted_from_opf_meta(tmp_path):
|
||||||
|
book = epub.EpubBook()
|
||||||
|
book.set_identifier("id")
|
||||||
|
book.set_title("Example Title")
|
||||||
|
book.set_language("en")
|
||||||
|
book.add_author("Example Author")
|
||||||
|
|
||||||
|
# Calibre-style series metadata
|
||||||
|
book.add_metadata("OPF", "meta", "", {"name": "calibre:series", "content": "Example Saga"})
|
||||||
|
book.add_metadata("OPF", "meta", "", {"name": "calibre:series_index", "content": "2"})
|
||||||
|
|
||||||
|
chapter = epub.EpubHtml(title="Chapter 1", file_name="chap_01.xhtml", lang="en")
|
||||||
|
chapter.content = "<h1>Chapter 1</h1><p>Hello</p>"
|
||||||
|
book.add_item(chapter)
|
||||||
|
book.spine = ["nav", chapter]
|
||||||
|
book.add_item(epub.EpubNcx())
|
||||||
|
book.add_item(epub.EpubNav())
|
||||||
|
|
||||||
|
path = tmp_path / "example.epub"
|
||||||
|
epub.write_epub(str(path), book)
|
||||||
|
|
||||||
|
result = extract_from_path(path)
|
||||||
|
|
||||||
|
assert result.metadata.get("series") == "Example Saga"
|
||||||
|
assert result.metadata.get("series_index") == "2"
|
||||||
|
|||||||
Reference in New Issue
Block a user