Fix queue worker failure handling and thumbnail fallbacks

2026-05-16 11:02:18 +02:00
parent 0194476b33
commit 92762b07d5
7 changed files with 181 additions and 17 deletions
@@ -1,5 +1,12 @@
 # Changelog

+## 0.29.1 - 2026-05-16
+
+- Hardened download queue startup so a failed `ani-cli` process launch now marks the job as failed instead of crashing the background queue worker thread.
+- Cleaned up per-job staging directories after canceled and failed download runs, preventing partial artifacts from piling up under `.ani-cli-web/staging/`.
+- Fixed thumbnail refresh fallback flow so stale stored AnimeSchedule routes and AniDB IDs now continue on to title-based lookups instead of aborting the whole cover refresh attempt early.
+- Expanded regression coverage for queue startup failures, failed-download staging cleanup, and stale thumbnail source fallback behavior.
+
 ## 0.29.0 - 2026-05-16

 - Fixed watchlist thumbnail retry backoff so failed automatic cover lookups now record a retry timestamp and stop hammering AnimeSchedule or AniDB on every later warm-up pass.
@@ -2,7 +2,7 @@

 A local web UI for a system-wide `ani-cli` install.

-Current version: `0.29.0`
+Current version: `0.29.1`

 ## Project Layout

@@ -85,6 +85,8 @@ The app currently includes:
 - Search-page queue polling also guards against stale out-of-order refresh responses now.
 - Queue log writes are batched during active downloads to reduce SQLite churn on noisy jobs.
 - Queue persistence now stores core job metadata and logs in structured SQLite columns instead of treating the whole job as one large JSON blob.
+- Queue worker startup failures now degrade into a normal failed job state instead of crashing the background queue thread.
+- Failed and canceled downloads now clean their project-local staging directories automatically so partial files do not accumulate under `.ani-cli-web/staging/`.
 - Failed thumbnail refresh attempts now record a retry timestamp, so unresolved covers back off instead of immediately requerying upstream sources on every later warm-up.
 - Download-to-watchlist fallback matching now keeps season and part identity intact while still tolerating equivalent season notation such as `2nd Season` vs `Season 2`.
 - Project-local runtime state under `.ani-cli-web/`.
@@ -214,6 +216,7 @@ Current thumbnail behavior:
 - If a show disappears while a thumbnail warm-up batch is running, that entry is skipped cleanly instead of failing the whole batch request.
 - Thumbnail image tags only hit the local route for already-cached files, so opening the watchlist no longer triggers a per-card remote lookup burst.
 - AnimeSchedule is tried first for cover lookup.
+- Stale stored AnimeSchedule routes and AniDB IDs now fall back to fresh title-based lookups instead of aborting the whole thumbnail refresh attempt.
 - If a season-labeled title fails as-is, lookup retries with simplified base-title queries.
 - AniDB is used as a fallback when AnimeSchedule does not resolve a cover.
 - AniDB fallback reuses the cached parsed title list directly to keep repeated lookups lighter.
@@ -1 +1 @@
-0.29.0
+0.29.1
@@ -1592,8 +1592,6 @@ class WatchlistStore:
                        cover = fetch_animeschedule_cover_by_route(stored_route, fallback_title=stored_schedule_title)
                    except Exception:
                        debug_log("thumbnail.ensure.route_failed", source="AnimeSchedule", show_id=show_id, route=stored_route)
-                        if not force:
-                            raise
                if cover is None:
                    try:
                        debug_log("thumbnail.ensure.try_title", source="AnimeSchedule", show_id=show_id, title=item["title"])
@@ -1609,8 +1607,6 @@ class WatchlistStore:
                        cover = fetch_anidb_cover_by_aid(stored_aid, fallback_title=stored_title)
                    except Exception:
                        debug_log("thumbnail.ensure.route_failed", source="AniDB", show_id=show_id, aid=stored_aid)
-                        if not force:
-                            raise
                if cover is None:
                    debug_log("thumbnail.ensure.try_title", source="AniDB", show_id=show_id, title=item["title"])
                    cover = fetch_anidb_cover(item["title"])
@@ -16,7 +16,7 @@ from uuid import uuid4
 PROJECT_ROOT = Path(__file__).resolve().parent
 ANI_CLI = os.environ.get("ANI_CLI_BIN") or shutil.which("ani-cli") or "ani-cli"
 APP_NAME = "ani-cli-web"
-VERSION = "0.29.0"
+VERSION = "0.29.1"
 ALLANIME_BASE = "allanime.day"
 ALLANIME_API = f"https://api.{ALLANIME_BASE}"
 ALLANIME_REFERER = "https://allmanga.to"
@@ -5,6 +5,7 @@
 import concurrent.futures
 import json
 import os
+import shutil
 import signal
 import sqlite3
 import subprocess
@@ -740,6 +741,33 @@ class DownloadQueue:
            if should_flush:
                self._save_job_locked(job)

+    def _cleanup_staging_dir(self, job):
+        staging_dir = job_staging_dir(job)
+        if not staging_dir.exists():
+            return False
+        try:
+            shutil.rmtree(staging_dir)
+            debug_log("queue.cleanup.staging_removed", job_id=job.get("id"), staging_dir=staging_dir)
+            return True
+        except OSError as exc:
+            debug_log("queue.cleanup.staging_failed", job_id=job.get("id"), staging_dir=staging_dir, error=exc)
+            return False
+
+    def _fail_job_startup(self, job, exc):
+        message = f"Could not start download: {exc}"
+        with self.lock:
+            job["status"] = "failed"
+            job["exit_code"] = 1
+            job["pid"] = None
+            job["finished_at"] = now_iso()
+            job["updated_at"] = job["finished_at"]
+            job.setdefault("log", []).append(message)
+            self.current_process = None
+            self.current_job_id = None
+            self.current_job = None
+            self._save_job_locked(job)
+        self._cleanup_staging_dir(job)
+
    def _run(self):
        while not self.stop_event.is_set():
            job = self._next_pending()
@@ -781,6 +809,7 @@ class DownloadQueue:
            job["_dirty_log_lines"] = 0
            self._save_job_locked(job)

+        try:
            process = subprocess.Popen(
                command,
                cwd=str(PROJECT_ROOT),
@@ -791,6 +820,9 @@ class DownloadQueue:
                bufsize=1,
                start_new_session=True,
            )
+        except Exception as exc:
+            self._fail_job_startup(job, exc)
+            return
        with self.lock:
            self.current_process = process
            self.current_job_id = job["id"]
@@ -805,6 +837,7 @@ class DownloadQueue:
        finalize_error = None
        moved_files = []
        watchlist_sync_error = None
+        cleanup_staging = False
        if exit_code == 0 and not job.get("cancel_requested"):
            try:
                moved_files = finalize_library_files(job)
@@ -841,7 +874,12 @@ class DownloadQueue:
            else:
                job["status"] = "failed"
                job.setdefault("log", []).append(f"Download failed with exit code {exit_code}.")
+                cleanup_staging = True
+            if canceled:
+                cleanup_staging = True
            self.current_process = None
            self.current_job_id = None
            self.current_job = None
            self._save_job_locked(job)
+        if cleanup_staging:
+            self._cleanup_staging_dir(job)
@@ -225,6 +225,48 @@ class DownloadQueueCancelTests(unittest.TestCase):
        killpg.assert_called_once_with(4321, queue_jobs.signal.SIGTERM)


+class DownloadQueueWorkerFailureTests(unittest.TestCase):
+    def setUp(self):
+        queue = APP.DownloadQueue(lambda: {"mode": "sub", "quality": "best", "download_dir": "/tmp/example"}, start_worker=False)
+        with queue._connect() as conn:
+            conn.execute("DELETE FROM jobs")
+
+    def test_run_job_marks_startup_failure_and_cleans_staging(self):
+        queue = APP.DownloadQueue(lambda: {"mode": "sub", "quality": "best", "download_dir": "/tmp/example"}, start_worker=False)
+        job = queue.add({"query": "Queue Show", "title": "Queue Show", "season": "1", "episodes": "1"})
+        staging_dir = queue_jobs.job_staging_dir(job)
+
+        with mock.patch.object(queue_jobs.subprocess, "Popen", side_effect=FileNotFoundError("ani-cli missing")):
+            queue._run_job(job)
+
+        stored = queue._find(job["id"])
+        self.assertEqual(stored["status"], "failed")
+        self.assertEqual(stored["exit_code"], 1)
+        self.assertIn("Could not start download", stored["log"][-1])
+        self.assertFalse(staging_dir.exists())
+
+    def test_failed_download_cleans_staging_dir(self):
+        queue = APP.DownloadQueue(lambda: {"mode": "sub", "quality": "best", "download_dir": "/tmp/example"}, start_worker=False)
+        job = queue.add({"query": "Queue Show", "title": "Queue Show", "season": "1", "episodes": "1"})
+        staging_dir = queue_jobs.job_staging_dir(job)
+        staging_dir.mkdir(parents=True, exist_ok=True)
+        (staging_dir / "partial.mp4").write_text("partial", encoding="utf-8")
+
+        class FakeProc:
+            pid = 4321
+            stdout = []
+
+            def wait(self):
+                return 1
+
+        with mock.patch.object(queue_jobs.subprocess, "Popen", return_value=FakeProc()):
+            queue._run_job(job)
+
+        stored = queue._find(job["id"])
+        self.assertEqual(stored["status"], "failed")
+        self.assertFalse(staging_dir.exists())
+
+
 class WatchlistRefreshAllTests(unittest.TestCase):
    def test_refresh_all_iterates_every_show_id(self):
        store = object.__new__(APP.WatchlistStore)
@@ -475,6 +517,84 @@ class ThumbnailEventRecoveryTests(unittest.TestCase):

        self.assertIsNone(second)

+    def test_ensure_thumbnail_falls_back_from_stale_animeschedule_route(self):
+        now = APP.now_iso()
+        item = {
+            "show_id": "show-thumb-2",
+            "title": "Route Fallback Show",
+            "category": "watching",
+            "downloaded": False,
+            "status": "tracked",
+            "status_message": "Waiting for thumbnail.",
+            "created_at": now,
+            "updated_at": now,
+            "last_checked": None,
+            "sub_count": 0,
+            "dub_count": 0,
+            "expected_count": 0,
+            "airing_status": None,
+            "sub_latest_episode": None,
+            "dub_latest_episode": None,
+            "animeschedule_route": "stale-route",
+            "animeschedule_title": "Route Fallback Show",
+            "anidb_aid": None,
+            "anidb_title": None,
+            "thumbnail_path": None,
+            "thumbnail_checked_at": None,
+        }
+        with APP.WATCHLIST.lock, APP.WATCHLIST._connect() as conn:
+            APP.WATCHLIST._upsert_conn(conn, item)
+
+        with mock.patch.object(APP, "fetch_animeschedule_cover_by_route", side_effect=RuntimeError("stale route")), mock.patch.object(
+            APP, "fetch_animeschedule_cover", return_value={"route": "fresh-route", "title": "Route Fallback Show", "url": "https://example.com/cover.jpg"}
+        ) as title_lookup, mock.patch.object(APP, "cache_thumbnail_image", return_value="show-thumb-2.jpg"):
+            result = APP.WATCHLIST.ensure_thumbnail("show-thumb-2")
+
+        title_lookup.assert_called_once_with("Route Fallback Show")
+        self.assertEqual(result, APP.THUMBNAIL_ROOT / "show-thumb-2.jpg")
+        refreshed = APP.WATCHLIST.get("show-thumb-2")
+        self.assertEqual(refreshed["animeschedule_route"], "fresh-route")
+
+    def test_ensure_thumbnail_falls_back_from_stale_anidb_aid(self):
+        now = APP.now_iso()
+        item = {
+            "show_id": "show-thumb-3",
+            "title": "AniDB Fallback Show",
+            "category": "watching",
+            "downloaded": False,
+            "status": "tracked",
+            "status_message": "Waiting for thumbnail.",
+            "created_at": now,
+            "updated_at": now,
+            "last_checked": None,
+            "sub_count": 0,
+            "dub_count": 0,
+            "expected_count": 0,
+            "airing_status": None,
+            "sub_latest_episode": None,
+            "dub_latest_episode": None,
+            "animeschedule_route": None,
+            "animeschedule_title": None,
+            "anidb_aid": "12345",
+            "anidb_title": "AniDB Fallback Show",
+            "thumbnail_path": None,
+            "thumbnail_checked_at": None,
+        }
+        with APP.WATCHLIST.lock, APP.WATCHLIST._connect() as conn:
+            APP.WATCHLIST._upsert_conn(conn, item)
+
+        with mock.patch.object(APP, "fetch_animeschedule_cover", side_effect=RuntimeError("animeschedule failed")), mock.patch.object(
+            APP, "fetch_anidb_cover_by_aid", side_effect=RuntimeError("stale aid")
+        ), mock.patch.object(
+            APP, "fetch_anidb_cover", return_value={"aid": "67890", "title": "AniDB Fallback Show", "url": "https://example.com/cover.jpg"}
+        ) as title_lookup, mock.patch.object(APP, "cache_thumbnail_image", return_value="show-thumb-3.jpg"):
+            result = APP.WATCHLIST.ensure_thumbnail("show-thumb-3")
+
+        title_lookup.assert_called_once_with("AniDB Fallback Show")
+        self.assertEqual(result, APP.THUMBNAIL_ROOT / "show-thumb-3.jpg")
+        refreshed = APP.WATCHLIST.get("show-thumb-3")
+        self.assertEqual(refreshed["anidb_aid"], "67890")
+

 class WatchlistCompletionTests(unittest.TestCase):
    def setUp(self):
@@ -1 +1 @@
 .29.0
 .29.1