feat: Implement existing item lookup and overwrite handling for Audiobookshelf uploads

This commit is contained in:
JB
2025-11-02 06:25:14 -08:00
parent af90da0b99
commit 72b2ada9ac
8 changed files with 511 additions and 58 deletions
+235
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import json
import logging
import mimetypes
import re
from contextlib import ExitStack
from dataclasses import dataclass
from pathlib import Path
@@ -180,6 +181,159 @@ class AudiobookshelfClient:
files.append((field_name, (path.name, handle, mime_type)))
return files
def find_existing_items(
self,
title: str,
*,
folder_id: Optional[str] = None,
) -> List[Mapping[str, Any]]:
normalized_title = self._normalize_title_value(title)
if not normalized_title:
return []
folder_hint = folder_id or self._config.folder_id
target_folders = set()
if folder_hint:
folder_token = str(folder_hint).strip().lower()
if folder_token:
target_folders.add(folder_token)
requests = self._candidate_search_requests(title, folder_hint)
if not requests:
return []
matches: List[Mapping[str, Any]] = []
try:
with self._open_client() as client:
for route, params in requests:
try:
response = client.get(route, params=params)
except httpx.HTTPError as exc:
logger.debug("Audiobookshelf lookup failed for %s: %s", route, exc)
continue
if response.status_code == 404:
continue
try:
response.raise_for_status()
except httpx.HTTPStatusError as exc:
status = exc.response.status_code
if status in {401, 403}:
raise AudiobookshelfUploadError(
"Audiobookshelf authentication failed while checking for existing items."
) from exc
logger.debug("Audiobookshelf lookup error %s for %s", status, route)
continue
try:
payload = response.json()
except ValueError:
continue
candidates = self._extract_candidate_items(payload)
for item in candidates:
item_title = self._normalize_item_title(item)
if not item_title or item_title != normalized_title:
continue
if target_folders:
item_folder = self._normalize_folder_id(item)
if item_folder and item_folder not in target_folders:
continue
matches.append(item)
if matches:
break
except AudiobookshelfUploadError:
raise
except Exception:
logger.debug(
"Unexpected error while checking Audiobookshelf for existing items",
exc_info=True,
)
return matches
def delete_items(self, items: Iterable[Mapping[str, Any] | str]) -> None:
to_delete: List[str] = []
for entry in items:
if isinstance(entry, Mapping):
item_id = self._extract_item_id(entry)
else:
item_id = str(entry).strip()
if item_id:
to_delete.append(item_id)
if not to_delete:
return
with self._open_client() as client:
for item_id in to_delete:
self._delete_single_item(client, item_id)
def _candidate_search_requests(
self,
title: str,
folder_id: Optional[str],
) -> List[Tuple[str, Dict[str, Any]]]:
query = (title or "").strip()
if not query:
return []
library_id = self._config.library_id
folder_token = (folder_id or self._config.folder_id or "").strip()
requests: List[Tuple[str, Dict[str, Any]]] = []
seen_routes: set[str] = set()
def _append(route: str, params: Dict[str, Any]) -> None:
if route in seen_routes:
return
seen_routes.add(route)
requests.append((route, params))
if folder_token:
_append(
self._api_path(f"folders/{folder_token}/items"),
{"library": library_id, "search": query},
)
_append(self._api_path(f"libraries/{library_id}/items"), {"search": query})
_append(self._api_path("items"), {"library": library_id, "search": query})
_append(
self._api_path("search"),
{"query": query, "library": library_id, "media": "audiobook"},
)
return requests
def _delete_single_item(self, client: httpx.Client, item_id: str) -> None:
routes = [
self._api_path(f"items/{item_id}"),
self._api_path(f"libraries/{self._config.library_id}/items/{item_id}"),
]
for route in routes:
try:
response = client.delete(route)
except httpx.HTTPError as exc:
logger.debug("Audiobookshelf delete failed for %s: %s", route, exc)
continue
if response.status_code in (200, 202, 204):
return
if response.status_code == 404:
continue
try:
response.raise_for_status()
except httpx.HTTPStatusError as exc:
raise AudiobookshelfUploadError(
f"Failed to delete Audiobookshelf item '{item_id}': {exc}"
) from exc
logger.debug("Audiobookshelf item %s could not be confirmed deleted", item_id)
def resolve_folder(self) -> Tuple[str, str, str]:
"""Return the resolved folder (id, name, library name)."""
return self._ensure_folder()
@@ -335,6 +489,87 @@ class AudiobookshelfClient:
token = token.strip("/ ")
return token.lower()
@staticmethod
def _normalize_title_value(value: Optional[str]) -> str:
if not isinstance(value, str):
return ""
normalized = re.sub(r"\s+", " ", value).strip()
return normalized.casefold() if normalized else ""
@staticmethod
def _normalize_item_title(item: Mapping[str, Any]) -> str:
if not isinstance(item, Mapping):
return ""
for key in ("title", "name", "label"):
candidate = item.get(key)
if isinstance(candidate, str) and candidate.strip():
return AudiobookshelfClient._normalize_title_value(candidate)
library_item = item.get("libraryItem")
if isinstance(library_item, Mapping):
return AudiobookshelfClient._normalize_item_title(library_item)
return ""
@staticmethod
def _normalize_folder_id(item: Mapping[str, Any]) -> Optional[str]:
if not isinstance(item, Mapping):
return None
for key in ("folderId", "libraryFolderId", "folder_id", "folder"):
value = item.get(key)
if isinstance(value, str) and value.strip():
return value.strip().lower()
if isinstance(value, (int, float)):
return str(value).strip().lower()
library_item = item.get("libraryItem")
if isinstance(library_item, Mapping):
return AudiobookshelfClient._normalize_folder_id(library_item)
return None
@staticmethod
def _extract_item_id(item: Mapping[str, Any]) -> Optional[str]:
if not isinstance(item, Mapping):
return None
for key in ("id", "libraryItemId", "itemId"):
value = item.get(key)
if isinstance(value, str) and value.strip():
return value.strip()
if isinstance(value, (int, float)):
return str(value).strip()
library_item = item.get("libraryItem")
if isinstance(library_item, Mapping):
return AudiobookshelfClient._extract_item_id(library_item)
return None
@staticmethod
def _extract_candidate_items(payload: Any) -> List[Mapping[str, Any]]:
items: List[Mapping[str, Any]] = []
seen_ids: set[str] = set()
visited: set[int] = set()
def _visit(obj: Any) -> None:
if isinstance(obj, Mapping):
obj_id = id(obj)
if obj_id in visited:
return
visited.add(obj_id)
title = AudiobookshelfClient._normalize_item_title(obj)
item_id = AudiobookshelfClient._extract_item_id(obj)
if title and item_id:
key = item_id.strip().lower()
if key not in seen_ids:
seen_ids.add(key)
items.append(obj)
for value in obj.values():
_visit(value)
elif isinstance(obj, list):
for entry in obj:
_visit(entry)
_visit(payload)
return items
@staticmethod
def _extract_title(metadata: Mapping[str, Any], audio_path: Path) -> str:
title = metadata.get("title") if isinstance(metadata, Mapping) else None
+78 -34
View File
@@ -93,15 +93,22 @@ SUFFIX_CONTRACTION_BASES: Dict[str, Tuple[str, ...]] = {
_NUMBER_WITH_GROUP_RE = re.compile(r"(?<![\w\d])(-?\d{1,3}(?:,\d{3})+)(?![\w\d])")
_NUMBER_RANGE_SEPARATORS = "-‐‑–—−"
_NUMBER_RANGE_CLASS = re.escape(_NUMBER_RANGE_SEPARATORS)
_NUMBER_CORE_PATTERN = r"-?(?:\d{1,3}(?:,\d{3})+|\d+)"
_WIDE_RANGE_SEPARATORS = {"", ""}
_NUMBER_RANGE_RE = re.compile(
rf"(?<!\w)(?P<left>-?\d+)(?P<sep>\s*[{_NUMBER_RANGE_CLASS}]\s*)(?P<right>-?\d+)(?![\w{_NUMBER_RANGE_CLASS}/])"
rf"(?<!\w)(?P<left>{_NUMBER_CORE_PATTERN})(?P<sep>\s*[{_NUMBER_RANGE_CLASS}]\s*)(?P<right>{_NUMBER_CORE_PATTERN})(?![\w{_NUMBER_RANGE_CLASS}/])"
)
_NUMBER_SPACE_RANGE_RE = re.compile(
rf"(?<![\w{_NUMBER_RANGE_CLASS}/])(?P<left>{_NUMBER_CORE_PATTERN})(?P<gap>\s+)(?P<right>{_NUMBER_CORE_PATTERN})(?![\w{_NUMBER_RANGE_CLASS}/])"
)
_FRACTION_SLASHES = "/"
_FRACTION_SLASH_CLASS = re.escape(_FRACTION_SLASHES)
_FRACTION_RE = re.compile(
rf"(?<!\w)(?P<numerator>-?\d+)\s*[{_FRACTION_SLASH_CLASS}]\s*(?P<denominator>-?\d+)(?![\w{_FRACTION_SLASH_CLASS}])"
)
_PLAIN_NUMBER_RE = re.compile(
rf"(?<![\w{_NUMBER_RANGE_CLASS}/])(?P<number>{_NUMBER_CORE_PATTERN})(?![\w{_NUMBER_RANGE_CLASS}/])"
)
def _int_to_words(value: int, language: str) -> Optional[str]:
@@ -195,21 +202,25 @@ def _format_fraction_words(numerator: int, denominator: int, language: str) -> O
def _replace_number_range(match: re.Match[str], language: str) -> str:
left_raw = match.group("left")
right_raw = match.group("right")
separator_text = match.group("sep") or ""
separator_char = next((ch for ch in separator_text if ch in _NUMBER_RANGE_SEPARATORS), "-")
has_whitespace = any(ch.isspace() for ch in separator_text)
left_digits = len(left_raw.lstrip("-"))
right_digits = len(right_raw.lstrip("-"))
if (left_digits >= 4 or right_digits >= 4) and separator_char not in _WIDE_RANGE_SEPARATORS and not has_whitespace:
left = _coerce_int_token(left_raw)
right = _coerce_int_token(right_raw)
if left is None or right is None:
return match.group(0)
if {left_digits, right_digits} == {3, 4} and separator_char not in _WIDE_RANGE_SEPARATORS and not has_whitespace:
left_words = _int_to_words(left, language)
right_words = _int_to_words(right, language)
if not left_words or not right_words:
return match.group(0)
try:
left = int(left_raw)
right = int(right_raw)
except ValueError:
return f"{left_words} to {right_words}"
def _replace_space_separated_range(match: re.Match[str], language: str) -> str:
left_raw = match.group("left")
right_raw = match.group("right")
left = _coerce_int_token(left_raw)
right = _coerce_int_token(right_raw)
if left is None or right is None:
return match.group(0)
left_words = _int_to_words(left, language)
@@ -233,6 +244,18 @@ def _replace_fraction(match: re.Match[str], language: str) -> str:
if not spoken:
return match.group(0)
return spoken
def _coerce_int_token(token: str) -> Optional[int]:
if token is None:
return None
cleaned = token.replace(",", "").strip()
if not cleaned or cleaned in {"-", "+"}:
return None
try:
return int(cleaned)
except ValueError:
return None
AMBIGUOUS_D_BASES = {"i","you","he","she","we","they"}
AMBIGUOUS_S_BASES = {
"it",
@@ -862,33 +885,54 @@ def _normalize_grouped_numbers(text: str, cfg: ApostropheConfig) -> str:
language = (cfg.number_lang or "en").strip() or "en"
def _replace(match: re.Match[str]) -> str:
def _replace_grouped(match: re.Match[str]) -> str:
token = match.group(1)
value = _coerce_int_token(token)
if value is None:
cleaned = token.replace(",", "")
if not cleaned:
return token
negative = cleaned.startswith("-")
cleaned_digits = cleaned[1:] if negative else cleaned
if not cleaned_digits.isdigit():
return cleaned_digits if not negative else f"-{cleaned_digits}"
if num2words is None:
return ("-" if negative else "") + cleaned_digits
try:
value = int(cleaned)
except ValueError:
return cleaned
words = _int_to_words(value, language)
if not words:
if num2words is None:
return str(value)
return words
words = _int_to_words(value, language)
return words or str(value)
normalized = _NUMBER_WITH_GROUP_RE.sub(_replace, text)
def _replace_plain(match: re.Match[str]) -> str:
token = match.group("number")
if "," in token:
return token.replace(",", "")
start, end = match.span()
source = match.string
before = source[start - 1] if start > 0 else ""
after = source[end] if end < len(source) else ""
if before == "/" or after == "/":
return token
if after == ".":
next_char = source[end + 1] if end + 1 < len(source) else ""
if next_char.isdigit():
return token
if before == ".":
prev_char = source[start - 2] if start >= 2 else ""
if prev_char.isdigit() or start == 1:
return token
value = _coerce_int_token(token)
if value is None:
return token
if num2words is None:
return str(value)
words = _int_to_words(value, language)
return words or str(value)
normalized = text
normalized = _NUMBER_RANGE_RE.sub(lambda m: _replace_number_range(m, language), normalized)
normalized = _NUMBER_SPACE_RANGE_RE.sub(lambda m: _replace_space_separated_range(m, language), normalized)
normalized = _FRACTION_RE.sub(lambda m: _replace_fraction(m, language), normalized)
normalized = _NUMBER_WITH_GROUP_RE.sub(_replace_grouped, normalized)
normalized = _PLAIN_NUMBER_RE.sub(_replace_plain, normalized)
return normalized
# ---------- Optional phoneme hint post-processing ----------
+22 -22
View File
@@ -8,6 +8,7 @@ import subprocess
import sys
import tempfile
import traceback
from datetime import datetime
from collections import defaultdict
from contextlib import ExitStack
from dataclasses import dataclass
@@ -107,6 +108,7 @@ _ACRONYM_ALLOWLIST = {
}
_ROMAN_NUMERAL_CHARS = frozenset("IVXLCDM")
_CAPS_WORD_RE = re.compile(r"[A-Z][A-Z0-9'\u2019-]*")
_OUTPUT_SANITIZE_RE = re.compile(r"[^\w\-_.]+")
def _simplify_heading_text(text: str) -> str:
@@ -1844,7 +1846,7 @@ def run_conversion_job(job: Job) -> None:
if audio_asset:
try:
epub_root = project_root if job.save_as_project else base_output_dir
epub_root = project_root
epub_output_path = _build_output_path(epub_root, job.original_filename, "epub")
job.add_log("Generating EPUB 3 package with synchronized narration…")
epub_path = build_epub3_package(
@@ -1989,21 +1991,19 @@ def _prepare_output_dir(job: Job) -> Path:
def _build_output_path(directory: Path, original_name: str, extension: str) -> Path:
base_name = Path(original_name).stem
sanitized = re.sub(r"[^\w\-_.]+", "_", base_name).strip("_") or "output"
candidate = directory / f"{sanitized}.{extension}"
counter = 1
while candidate.exists():
candidate = directory / f"{sanitized}_{counter}.{extension}"
counter += 1
return candidate
sanitized = _sanitize_output_stem(original_name)
directory.mkdir(parents=True, exist_ok=True)
return directory / f"{sanitized}.{extension}"
def _prepare_project_layout(job: Job, base_dir: Path) -> tuple[Path, Path, Path, Optional[Path]]:
base_dir.mkdir(parents=True, exist_ok=True)
stem = Path(job.original_filename).stem
sanitized = _sanitize_output_stem(job.original_filename)
folder_name = f"{_output_timestamp_token()}_{sanitized}"
project_root = base_dir / folder_name
project_root.mkdir(parents=True, exist_ok=True)
if job.save_as_project:
project_root = _ensure_unique_directory(base_dir, f"{stem}_project")
audio_dir = project_root / "audio"
subtitle_dir = project_root / "subtitles"
metadata_dir = project_root / "metadata"
@@ -2011,17 +2011,7 @@ def _prepare_project_layout(job: Job, base_dir: Path) -> tuple[Path, Path, Path,
directory.mkdir(parents=True, exist_ok=True)
return project_root, audio_dir, subtitle_dir, metadata_dir
return base_dir, base_dir, base_dir, None
def _ensure_unique_directory(parent: Path, name: str) -> Path:
candidate = parent / name
counter = 1
while candidate.exists():
candidate = parent / f"{name}_{counter}"
counter += 1
candidate.mkdir(parents=True, exist_ok=True)
return candidate
return project_root, project_root, project_root, None
def _apply_newline_policy(chapters: List[ExtractedChapter], replace_single_newlines: bool) -> None:
@@ -2039,6 +2029,16 @@ def _slugify(title: str, index: int) -> str:
return sanitized[:80]
def _sanitize_output_stem(name: str) -> str:
base = Path(name or "").stem
sanitized = _OUTPUT_SANITIZE_RE.sub("_", base).strip("_")
return sanitized or "output"
def _output_timestamp_token() -> str:
return datetime.now().strftime("%Y%m%d-%H%M%S")
def _open_audio_sink(
path: Path,
job: Job,
+47 -1
View File
@@ -5064,10 +5064,56 @@ def send_job_to_audiobookshelf(job_id: str) -> ResponseReturnValue:
subtitles = _existing_paths(job.result.subtitle_paths) if config.send_subtitles else None
chapters = _load_audiobookshelf_chapters(job) if config.send_chapters else None
metadata = _build_audiobookshelf_metadata(job)
display_title = metadata.get("title") or audio_path.stem
overwrite_requested = request.form.get("overwrite") == "true" or request.args.get("overwrite") == "true"
try:
client = AudiobookshelfClient(config)
except ValueError as exc: # pragma: no cover - defensive guard
job.add_log(f"Audiobookshelf configuration error: {exc}", level="error")
service._persist_state()
return _panel_response()
try:
existing_items = client.find_existing_items(display_title, folder_id=config.folder_id)
except AudiobookshelfUploadError as exc:
job.add_log(f"Audiobookshelf lookup failed: {exc}", level="error")
service._persist_state()
return _panel_response()
if existing_items and not overwrite_requested:
job.add_log(
f"Audiobookshelf already contains '{display_title}'. Awaiting overwrite confirmation.",
level="warning",
)
service._persist_state()
if request.headers.get("HX-Request"):
detail = {
"jobId": job.id,
"title": display_title,
"url": url_for("web.send_job_to_audiobookshelf", job_id=job.id),
"target": request.headers.get("HX-Target") or "#jobs-panel",
"message": f'Audiobookshelf already contains "{display_title}". Overwrite?',
}
headers = {"HX-Trigger": json.dumps({"audiobookshelf-overwrite-prompt": detail})}
return Response("", status=204, headers=headers)
return _panel_response()
if existing_items and overwrite_requested:
try:
client.delete_items(existing_items)
except AudiobookshelfUploadError as exc:
job.add_log(f"Audiobookshelf overwrite aborted: {exc}", level="error")
service._persist_state()
return _panel_response()
else:
job.add_log(
f"Removed {len(existing_items)} existing Audiobookshelf item(s) prior to overwrite.",
level="info",
)
job.add_log("Audiobookshelf upload triggered manually.", level="info")
try:
client = AudiobookshelfClient(config)
client.upload_audiobook(
audio_path,
metadata=metadata,
+15
View File
@@ -999,6 +999,21 @@ class ConversionService:
metadata = _build_audiobookshelf_metadata(job)
client = AudiobookshelfClient(config)
display_title = metadata.get("title") or audio_path.stem
try:
existing_items = client.find_existing_items(display_title, folder_id=config.folder_id)
except AudiobookshelfUploadError as exc:
job.add_log(f"Audiobookshelf lookup failed: {exc}", level="error")
return
if existing_items:
job.add_log(
f"Audiobookshelf upload skipped: '{display_title}' already exists in the configured folder.",
level="warning",
)
return
client.upload_audiobook(
audio_path,
metadata=metadata,
+34
View File
@@ -1,7 +1,41 @@
import { initReaderUI } from "./reader.js";
const queueState = (window.AbogenQueueState = window.AbogenQueueState || {
boundOverwritePrompt: false,
});
const handleOverwritePrompt = (event) => {
const detail = event?.detail || {};
const title = detail.title || "this item";
const message = detail.message || `Audiobookshelf already has "${title}". Overwrite?`;
if (!window.confirm(message)) {
return;
}
const url = detail.url;
if (!url || typeof htmx === "undefined") {
return;
}
const target = detail.target || "#jobs-panel";
const values = { overwrite: "true" };
if (detail.values && typeof detail.values === "object") {
Object.assign(values, detail.values);
}
htmx.ajax("POST", url, {
target,
swap: "innerHTML",
values,
});
};
const initQueuePage = () => {
initReaderUI();
if (!queueState.boundOverwritePrompt) {
queueState.boundOverwritePrompt = true;
document.addEventListener("audiobookshelf-overwrite-prompt", handleOverwritePrompt);
}
};
if (document.readyState === "loading") {
+69
View File
@@ -0,0 +1,69 @@
from __future__ import annotations
import time
from pathlib import Path
import pytest
from abogen.web.conversion_runner import _build_output_path, _prepare_project_layout
from abogen.web.service import Job
def _sample_job(tmp_path: Path) -> Job:
source = tmp_path / "sample.txt"
source.write_text("example", encoding="utf-8")
return Job(
id="job-1",
original_filename="Sample Title.txt",
stored_path=source,
language="en",
voice="af_alloy",
speed=1.0,
use_gpu=False,
subtitle_mode="Sentence",
output_format="mp3",
save_mode="Use default save location",
output_folder=tmp_path,
replace_single_newlines=False,
subtitle_format="srt",
created_at=time.time(),
)
def test_prepare_project_layout_uses_timestamped_folder(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
job = _sample_job(tmp_path)
monkeypatch.setattr(
"abogen.web.conversion_runner._output_timestamp_token",
lambda: "20250101-120000",
)
project_root, audio_dir, subtitle_dir, metadata_dir = _prepare_project_layout(job, tmp_path)
assert project_root.name.startswith("20250101-120000_sample_title"), project_root.name
assert audio_dir == project_root
assert subtitle_dir == project_root
assert metadata_dir is None
output_path = _build_output_path(audio_dir, job.original_filename, "mp3")
assert output_path == project_root / "Sample Title.mp3"
def test_prepare_project_layout_creates_project_subdirs(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
job = _sample_job(tmp_path)
job.save_as_project = True
monkeypatch.setattr(
"abogen.web.conversion_runner._output_timestamp_token",
lambda: "20250101-120500",
)
project_root, audio_dir, subtitle_dir, metadata_dir = _prepare_project_layout(job, tmp_path)
assert audio_dir == project_root / "audio"
assert subtitle_dir == project_root / "subtitles"
assert metadata_dir == project_root / "metadata"
assert audio_dir.is_dir()
assert subtitle_dir.is_dir()
assert metadata_dir is not None and metadata_dir.is_dir()
output_path = _build_output_path(audio_dir, job.original_filename, "wav")
assert output_path == audio_dir / "Sample Title.wav"
+10
View File
@@ -86,6 +86,16 @@ def test_simple_fractions_are_spoken() -> None:
assert "one half" in normalized.lower()
def test_plain_numbers_are_spelled_out() -> None:
normalized = _normalize_for_pipeline("He rolled a 42.")
assert "forty-two" in normalized.lower()
def test_space_separated_numbers_become_ranges() -> None:
normalized = _normalize_for_pipeline("Read pages 12 14 tonight.")
assert "pages twelve to fourteen" in normalized.lower()
def test_contractions_can_be_kept_when_override_disabled() -> None:
normalized = _normalize_for_pipeline(
"It's a good day.",