From d238524cbb2839f63f88b7b4a13dc4af84a45684 Mon Sep 17 00:00:00 2001 From: JB Date: Sat, 1 Nov 2025 07:27:56 -0700 Subject: [PATCH] feat: Enhance Calibre OPDS integration to extract and handle tags, ratings, and publication dates from metadata --- abogen/integrations/calibre_opds.py | 136 +++++++++++++++++++++++++++- abogen/web/routes.py | 52 +++++++++++ abogen/web/service.py | 16 ++++ abogen/web/static/find_books.js | 45 +++++++++ tests/test_calibre_opds.py | 34 +++++++ 5 files changed, 278 insertions(+), 5 deletions(-) diff --git a/abogen/integrations/calibre_opds.py b/abogen/integrations/calibre_opds.py index 508819d..2d264d3 100644 --- a/abogen/integrations/calibre_opds.py +++ b/abogen/integrations/calibre_opds.py @@ -30,6 +30,8 @@ _SERIES_PREFIX_RE = re.compile(r"^\s*(series|books?)\s*[:\-]\s*", re.IGNORECASE) _SERIES_NUMBER_BRACKET_RE = re.compile(r"[\[(]\s*(?:book\s*)?(\d+(?:\.\d+)?)\s*[\])]", re.IGNORECASE) _SERIES_NUMBER_HASH_RE = re.compile(r"#\s*(\d+(?:\.\d+)?)") _SERIES_NUMBER_BOOK_RE = re.compile(r"\bbook\s+(\d+(?:\.\d+)?)\b", re.IGNORECASE) +_SERIES_LINE_TEXT_RE = re.compile(r"^\s*series\s*[:\-]\s*(.+)$", re.IGNORECASE) +_SUMMARY_METADATA_LINE_RE = re.compile(r"^([A-Z][A-Z0-9&/\- +'\u2019]{1,40})\s*[:\-]\s*(.+)$") _EPUB_MIME_TYPES = { "application/epub+zip", "application/zip", @@ -65,6 +67,7 @@ class OPDSEntry: position: Optional[int] = None authors: List[str] = field(default_factory=list) updated: Optional[str] = None + published: Optional[str] = None summary: Optional[str] = None download: Optional[OPDSLink] = None alternate: Optional[OPDSLink] = None @@ -72,6 +75,9 @@ class OPDSEntry: links: List[OPDSLink] = field(default_factory=list) series: Optional[str] = None series_index: Optional[float] = None + tags: List[str] = field(default_factory=list) + rating: Optional[float] = None + rating_max: Optional[float] = None def to_dict(self) -> Dict[str, Any]: return { @@ -80,6 +86,7 @@ class OPDSEntry: "position": self.position, "authors": list(self.authors), "updated": self.updated, + "published": self.published, "summary": self.summary, "download": self.download.to_dict() if self.download else None, "alternate": self.alternate.to_dict() if self.alternate else None, @@ -87,6 +94,9 @@ class OPDSEntry: "links": [link.to_dict() for link in self.links], "series": self.series, "series_index": self.series_index, + "tags": list(self.tags), + "rating": self.rating, + "rating_max": self.rating_max, } @@ -263,12 +273,24 @@ class CalibreOPDSClient: title = node.findtext("atom:title", default="Untitled", namespaces=NS).strip() or "Untitled" position_value = self._extract_position(node) updated = node.findtext("atom:updated", default=None, namespaces=NS) - summary = node.findtext("atom:summary", default=None, namespaces=NS) or node.findtext( - "atom:content", default=None, namespaces=NS + published = ( + node.findtext("dc:date", default=None, namespaces=NS) + or node.findtext("atom:published", default=None, namespaces=NS) ) - dc_summary = node.findtext("dc:description", default=None, namespaces=NS) - summary = summary or dc_summary - cleaned_summary = self._strip_html(summary) + if published: + published = published.strip() or None + + summary_text = ( + self._extract_text(node.find("atom:summary", NS)) + or self._extract_text(node.find("atom:content", NS)) + or self._extract_text(node.find("dc:description", NS)) + ) + summary_metadata: Dict[str, str] = {} + summary_body: Optional[str] = None + if summary_text: + summary_metadata, summary_body = self._split_summary_metadata(summary_text) + cleaned_summary = self._strip_html(summary_body or summary_text) + authors: List[str] = [] for author_node in node.findall("atom:author", NS): name = author_node.findtext("atom:name", default="", namespaces=NS).strip() @@ -322,12 +344,26 @@ class CalibreOPDSClient: series_name = category_series_name if series_index is None and category_series_index is not None: series_index = category_series_index + + if (series_name is None or series_index is None) and summary_text: + text_series_name, text_series_index = self._extract_series_from_text(summary_text) + if series_name is None and text_series_name: + series_name = text_series_name + if series_index is None and text_series_index is not None: + series_index = text_series_index + + tags_value = summary_metadata.get("TAGS") + tags = self._parse_tags(tags_value) if tags_value else [] + rating_value = summary_metadata.get("RATING") + rating, rating_max = self._parse_rating(rating_value) if rating_value else (None, None) + return OPDSEntry( id=entry_id or title, title=title, position=position_value, authors=authors, updated=updated, + published=published, summary=cleaned_summary, download=download_link, alternate=alternate_link, @@ -335,6 +371,9 @@ class CalibreOPDSClient: links=list(parsed_links.values()), series=series_name, series_index=series_index, + tags=tags, + rating=rating, + rating_max=rating_max, ) def _extract_series_from_categories(self, category_nodes: List[ET.Element]) -> tuple[Optional[str], Optional[float]]: @@ -400,6 +439,93 @@ class CalibreOPDSClient: name = None return name, number + @staticmethod + def _extract_text(node: Optional[ET.Element]) -> Optional[str]: + if node is None: + return None + # Prefer itertext to capture nested XHTML content + parts = list(node.itertext()) + if not parts: + return (node.text or "").strip() or None + combined = "".join(parts).strip() + return combined or None + + def _extract_series_from_text(self, text: str) -> tuple[Optional[str], Optional[float]]: + for line in text.splitlines(): + match = _SERIES_LINE_TEXT_RE.match(line) + if not match: + continue + candidate = match.group(1).strip() + if not candidate: + continue + name, number = self._parse_series_value(candidate) + if name or number is not None: + return name, number + return None, None + + def _split_summary_metadata(self, text: Optional[str]) -> tuple[Dict[str, str], Optional[str]]: + metadata: Dict[str, str] = {} + if text is None: + return metadata, None + lines = text.splitlines() + index = 0 + total = len(lines) + while index < total and not lines[index].strip(): + index += 1 + while index < total: + stripped = lines[index].strip() + if not stripped: + break + match = _SUMMARY_METADATA_LINE_RE.match(stripped) + if not match: + break + key = match.group(1).strip().upper() + value = match.group(2).strip() + if key and value: + metadata[key] = value + index += 1 + remainder = "\n".join(lines[index:]).strip() + return metadata, (remainder or None) + + @staticmethod + def _parse_tags(value: str) -> List[str]: + if not value: + return [] + tokens = re.split(r"[;,\n]\s*", value) + cleaned: List[str] = [] + seen: set[str] = set() + for token in tokens: + entry = token.strip() + if not entry: + continue + key = entry.casefold() + if key in seen: + continue + seen.add(key) + cleaned.append(entry) + return cleaned + + @staticmethod + def _parse_rating(value: str) -> tuple[Optional[float], Optional[float]]: + if not value: + return None, None + text = value.strip() + if not text: + return None, None + stars = text.count("★") + half = 0.5 if "½" in text else 0.0 + if stars or half: + rating = stars + half + return (rating if rating > 0 else None, 5.0) + match = re.search(r"\d+(?:\.\d+)?", text.replace(",", ".")) + if match: + try: + rating_value = float(match.group(0)) + except ValueError: + return None, None + return rating_value, 5.0 + return None, None + @staticmethod def _coerce_series_index(value: str) -> Optional[float]: text = value.strip().replace(",", ".") diff --git a/abogen/web/routes.py b/abogen/web/routes.py index a574368..fccf30b 100644 --- a/abogen/web/routes.py +++ b/abogen/web/routes.py @@ -2922,6 +2922,16 @@ def calibre_opds_import() -> ResponseReturnValue: metadata_payload = data.get("metadata") if isinstance(data, Mapping) else None metadata_overrides: Dict[str, Any] = {} if isinstance(metadata_payload, Mapping): + def _stringify_metadata_value(value: Any) -> str: + if value is None: + return "" + if isinstance(value, (list, tuple, set)): + parts = [str(item).strip() for item in value if item is not None] + parts = [part for part in parts if part] + return ", ".join(parts) + text = str(value).strip() + return text + raw_series = metadata_payload.get("series") or metadata_payload.get("series_name") series_name = str(raw_series or "").strip() if series_name: @@ -2940,6 +2950,48 @@ def calibre_opds_import() -> ResponseReturnValue: metadata_overrides.setdefault("series_position", series_index_text) metadata_overrides.setdefault("series_sequence", series_index_text) metadata_overrides.setdefault("book_number", series_index_text) + tags_value = metadata_payload.get("tags") or metadata_payload.get("keywords") + if tags_value: + tags_text = _stringify_metadata_value(tags_value) + if tags_text: + metadata_overrides.setdefault("tags", tags_text) + metadata_overrides.setdefault("keywords", tags_text) + metadata_overrides.setdefault("genre", tags_text) + description_value = metadata_payload.get("description") or metadata_payload.get("summary") + if description_value: + description_text = _stringify_metadata_value(description_value) + if description_text: + metadata_overrides.setdefault("description", description_text) + metadata_overrides.setdefault("summary", description_text) + published_value = metadata_payload.get("published") or metadata_payload.get("publication_date") + if published_value: + published_text = _stringify_metadata_value(published_value) + if published_text: + metadata_overrides.setdefault("published", published_text) + metadata_overrides.setdefault("publication_date", published_text) + publication_year = metadata_payload.get("publication_year") or metadata_payload.get("year") + if publication_year: + year_text = _stringify_metadata_value(publication_year) + if year_text: + metadata_overrides.setdefault("publication_year", year_text) + metadata_overrides.setdefault("year", year_text) + rating_value = metadata_payload.get("rating") + if rating_value is not None: + rating_text = _stringify_metadata_value(rating_value) + if rating_text: + metadata_overrides.setdefault("rating", rating_text) + rating_max = metadata_payload.get("rating_max") + if rating_max is not None: + rating_max_text = _stringify_metadata_value(rating_max) + if rating_max_text: + metadata_overrides.setdefault("rating_max", rating_max_text) + for key, value in metadata_payload.items(): + if value is None: + continue + text_value = _stringify_metadata_value(value) + if not text_value: + continue + metadata_overrides.setdefault(str(key), text_value) stored_settings = _stored_integration_config("calibre_opds") if not stored_settings or not _coerce_bool(stored_settings.get("enabled"), False): diff --git a/abogen/web/service.py b/abogen/web/service.py index 3ba92a7..779412e 100644 --- a/abogen/web/service.py +++ b/abogen/web/service.py @@ -355,6 +355,22 @@ def _build_audiobookshelf_metadata(job: Job) -> Dict[str, Any]: "seriesSequence": series_sequence, "isbn": _first_nonempty(tags.get("isbn"), tags.get("asin")), } + published_date = _first_nonempty(tags.get("published"), tags.get("publication_date"), tags.get("date")) + if published_date: + data["publishedDate"] = published_date + + rating_text = _first_nonempty(tags.get("rating"), tags.get("my_rating")) + if rating_text: + try: + data["rating"] = float(str(rating_text).strip()) + except ValueError: + pass + rating_max_text = _first_nonempty(tags.get("rating_max"), tags.get("rating_scale")) + if rating_max_text: + try: + data["ratingMax"] = float(str(rating_max_text).strip()) + except ValueError: + pass # Remove empty values cleaned: Dict[str, Any] = {} for key, value in data.items(): diff --git a/abogen/web/static/find_books.js b/abogen/web/static/find_books.js index 9903535..23f0e1f 100644 --- a/abogen/web/static/find_books.js +++ b/abogen/web/static/find_books.js @@ -314,6 +314,21 @@ if (modal && browser) { header.appendChild(seriesMeta); } + if (entry.rating !== null && entry.rating !== undefined && entry.rating !== '') { + const ratingMeta = document.createElement('p'); + ratingMeta.className = 'opds-browser__meta'; + const ratingMax = entry.rating_max ?? entry.ratingMax ?? 5; + ratingMeta.textContent = `Rating: ${entry.rating}${ratingMax ? ` / ${ratingMax}` : ''}`; + header.appendChild(ratingMeta); + } + + if (Array.isArray(entry.tags) && entry.tags.length > 0) { + const tagsMeta = document.createElement('p'); + tagsMeta.className = 'opds-browser__meta'; + tagsMeta.textContent = `Tags: ${entry.tags.join(', ')}`; + header.appendChild(tagsMeta); + } + item.appendChild(header); const summarySource = entry.summary || entry?.alternate?.title || entry?.download?.title || ''; @@ -436,6 +451,36 @@ if (modal && browser) { metadata.series_position = seriesIndex; metadata.book_number = seriesIndex; } + if (Array.isArray(entry.tags) && entry.tags.length > 0) { + const tagsText = entry.tags.join(', '); + metadata.tags = tagsText; + metadata.keywords = tagsText; + metadata.genre = tagsText; + } + if (typeof entry.summary === 'string' && entry.summary.trim()) { + metadata.description = entry.summary; + metadata.summary = entry.summary; + } + if (entry.rating !== null && entry.rating !== undefined && entry.rating !== '') { + metadata.rating = String(entry.rating); + } + if (entry.rating_max !== null && entry.rating_max !== undefined && entry.rating_max !== '') { + metadata.rating_max = String(entry.rating_max); + } + if (entry.published) { + metadata.published = entry.published; + metadata.publication_date = entry.published; + try { + const publishedDate = new Date(entry.published); + if (!Number.isNaN(publishedDate.getTime())) { + const year = String(publishedDate.getUTCFullYear()); + metadata.publication_year = year; + metadata.year = year; + } + } catch (error) { + // Ignore invalid date parsing issues + } + } if (Object.keys(metadata).length > 0) { requestPayload.metadata = metadata; } diff --git a/tests/test_calibre_opds.py b/tests/test_calibre_opds.py index 573086f..06d8e43 100644 --- a/tests/test_calibre_opds.py +++ b/tests/test_calibre_opds.py @@ -61,3 +61,37 @@ def test_calibre_opds_feed_extracts_series_from_categories() -> None: assert entry.series == "The Murderbot Diaries" assert entry.series_index == 5.0 + + +def test_calibre_opds_extracts_tags_and_rating_from_summary() -> None: + client = CalibreOPDSClient("http://example.com/catalog") + xml_payload = """ + + catalog + Example Catalog + + book-3 + Summary Sample + 2024-01-15T00:00:00+00:00 + RATING: ★★★½ +TAGS: Science Fiction; Adventure +SERIES: Saga [3] +This is the detailed summary text. + + + + """ + + feed = client._parse_feed(xml_payload, base_url="http://example.com/catalog") + entry = feed.entries[0] + + assert entry.series == "Saga" + assert entry.series_index == 3.0 + assert entry.tags == ["Science Fiction", "Adventure"] + assert entry.rating == 3.5 + assert entry.rating_max == 5.0 + assert entry.summary == "This is the detailed summary text." + assert entry.published == "2024-01-15T00:00:00+00:00"