Pace bulk watchlist refresh requests

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 0.32.0 - 2026-05-17
+
+- Changed bulk `Refresh All` processing to pace requests with a configurable delay between shows, reducing the chance of flooding upstream episode sources on large watchlists.
+- Added a Config page setting for the per-show bulk refresh delay and expanded regression coverage for the new config field and pacing behavior.
+
 ## 0.31.1 - 2026-05-17
 
 - Added stdout lifecycle logging for scheduled watchlist refresh jobs so container logs now show when a scheduled run is queued, starts, finishes successfully, finishes with errors, gets interrupted, or fails.

README.md

@@ -2,7 +2,7 @@
 
 A local web UI for a system-wide `ani-cli` install.
 
-Current version: `0.31.1`
+Current version: `0.32.0`
 
 ## Project Layout
 
@@ -156,7 +156,7 @@ The app currently includes:
 - Per-show watchlist refresh queueing now also deduplicates in-flight refreshes, so the same show is not fetched twice back-to-back while one refresh is already running.
 - The watchlist page now silently refreshes itself while queued per-show refresh work is still being processed, so queued entries settle into their updated state without a manual reload.
 - The watchlist page now also avoids re-triggering the same unresolved thumbnail warm-up request on every silent poll cycle while a show's thumbnail state is unchanged.
-- Background `Refresh All` execution with status polling, parallel per-show refresh work, and no inline thumbnail downloads during the bulk job.
+- Background `Refresh All` execution with status polling, no inline thumbnail downloads during the bulk job, and a configurable delay between shows to reduce upstream request bursts.
 - Optional scheduled background watchlist refreshes with Config page controls for enable/disable and refresh period.
 - Scheduled background refresh runs now log queue, start, and completion status to stdout so container logs show whether they finished successfully, with errors, or failed.
 - Search and Watchlist polling now use a shared serial browser polling helper so slow responses do not cause overlapping poll requests to pile up in the background.
@@ -206,6 +206,7 @@ Use it to:
 3. Choose the default search quality used to prefill the Search page.
 4. Enable or disable periodic watchlist refresh checks.
 5. Set the refresh period in minutes for that scheduled watchlist job.
+6. Set the delay in seconds between shows during bulk watchlist refresh runs.
 
 Those saved defaults are written into the project-local `config.json` and loaded automatically when the Search page opens.
 
@@ -278,7 +279,7 @@ You can add entries in two ways:
 Watchlist features:
 
 1. `Refresh` updates one tracked show.
-2. `Refresh All` runs in the background, rejects duplicate queueing while a refresh is already pending or running, preserves interrupted job status across restarts, and refreshes shows in parallel without doing inline thumbnail downloads for every entry.
+2. `Refresh All` runs in the background, rejects duplicate queueing while a refresh is already pending or running, preserves interrupted job status across restarts, and refreshes shows one by one with a configurable delay without doing inline thumbnail downloads for every entry.
 3. `Open` jumps back to the Search page with the title pre-filled.
 4. Clicking the anime title opens the AllManga source page at `bangumi/<show_id>` in a new tab.
 5. `Remove` deletes the watchlist entry and clears its cached poster file.
@@ -392,7 +393,7 @@ Environment variables:
 
 Saved defaults from the UI are written into the project-local `config.json`.
 
-The saved config now also includes `watchlist_auto_refresh_enabled` and `watchlist_auto_refresh_minutes`, which control the periodic background watchlist refresh schedule.
+The saved config now also includes `watchlist_auto_refresh_enabled`, `watchlist_auto_refresh_minutes`, and `watchlist_refresh_delay_seconds`, which control the periodic background watchlist refresh schedule and the per-show pacing used by bulk refresh jobs.
 
 ## Notes
 

VERSION

@@ -1 +1 @@
-0.31.1
+0.32.0

app.py

@@ -1814,7 +1814,11 @@ def build_runtime(start_workers=None):
         job_prepare_fn=prepare_download_job,
         start_worker=worker_state,
     )
-    watchlist_refresh = WatchlistRefreshJobs(watchlist, start_worker=worker_state)
+    watchlist_refresh = WatchlistRefreshJobs(
+        watchlist,
+        config_getter=lambda: get_config_snapshot(config),
+        start_worker=worker_state,
+    )
     watchlist_auto_refresh = WatchlistAutoRefreshScheduler(
         config_getter=lambda: get_config_snapshot(config),
         refresh_start_fn=watchlist_refresh.start,

app_support.py

@@ -16,7 +16,7 @@ from uuid import uuid4
 PROJECT_ROOT = Path(__file__).resolve().parent
 ANI_CLI = os.environ.get("ANI_CLI_BIN") or shutil.which("ani-cli") or "ani-cli"
 APP_NAME = "ani-cli-web"
-VERSION = "0.31.1"
+VERSION = "0.32.0"
 ALLANIME_BASE = "allanime.day"
 ALLANIME_API = f"https://api.{ALLANIME_BASE}"
 ALLANIME_REFERER = "https://allmanga.to"
@@ -45,6 +45,9 @@ WORKER_SHUTDOWN_TIMEOUT_SECONDS = 10
 WATCHLIST_AUTO_REFRESH_DEFAULT_MINUTES = 60
 WATCHLIST_AUTO_REFRESH_MIN_MINUTES = 1
 WATCHLIST_AUTO_REFRESH_MAX_MINUTES = 7 * 24 * 60
+WATCHLIST_REFRESH_DELAY_DEFAULT_SECONDS = 5
+WATCHLIST_REFRESH_DELAY_MIN_SECONDS = 0
+WATCHLIST_REFRESH_DELAY_MAX_SECONDS = 300
 
 
 def env_flag(name, default=False):
@@ -119,6 +122,7 @@ DEFAULT_CONFIG = {
     "quality": os.environ.get("ANI_CLI_QUALITY", "best"),
     "watchlist_auto_refresh_enabled": False,
     "watchlist_auto_refresh_minutes": WATCHLIST_AUTO_REFRESH_DEFAULT_MINUTES,
+    "watchlist_refresh_delay_seconds": WATCHLIST_REFRESH_DELAY_DEFAULT_SECONDS,
 }
 
 
@@ -261,6 +265,7 @@ def normalize_config(data):
     download_dir = str(config.get("download_dir") or DEFAULT_CONFIG["download_dir"])
     auto_refresh_enabled = config.get("watchlist_auto_refresh_enabled")
     auto_refresh_minutes = config.get("watchlist_auto_refresh_minutes")
+    refresh_delay_seconds = config.get("watchlist_refresh_delay_seconds")
 
     if isinstance(auto_refresh_enabled, str):
         auto_refresh_enabled = auto_refresh_enabled.strip().lower() in {"1", "true", "yes", "on"}
@@ -275,12 +280,21 @@ def normalize_config(data):
         WATCHLIST_AUTO_REFRESH_MIN_MINUTES,
         min(WATCHLIST_AUTO_REFRESH_MAX_MINUTES, auto_refresh_minutes),
     )
+    try:
+        refresh_delay_seconds = int(str(refresh_delay_seconds).strip() or WATCHLIST_REFRESH_DELAY_DEFAULT_SECONDS)
+    except (TypeError, ValueError):
+        refresh_delay_seconds = WATCHLIST_REFRESH_DELAY_DEFAULT_SECONDS
+    refresh_delay_seconds = max(
+        WATCHLIST_REFRESH_DELAY_MIN_SECONDS,
+        min(WATCHLIST_REFRESH_DELAY_MAX_SECONDS, refresh_delay_seconds),
+    )
 
     config["mode"] = mode if mode in MODE_CHOICES else "sub"
     config["quality"] = quality if quality in QUALITY_CHOICES else "best"
     config["download_dir"] = str(Path(download_dir).expanduser())
     config["watchlist_auto_refresh_enabled"] = auto_refresh_enabled
     config["watchlist_auto_refresh_minutes"] = auto_refresh_minutes
+    config["watchlist_refresh_delay_seconds"] = refresh_delay_seconds
     return config
 
 

config_page.py

@@ -323,8 +323,11 @@ CONFIG_HTML = r"""<!doctype html>
           <label>Refresh period (minutes)
             <input id="watchlistAutoRefreshMinutes" type="number" min="1" max="10080" step="1" value="60">
           </label>
+          <label>Delay between series (seconds)
+            <input id="watchlistRefreshDelaySeconds" type="number" min="0" max="300" step="1" value="5">
+          </label>
         </div>
-        <div class="field-hint">When enabled, the server periodically starts the existing background “Refresh All” job using this interval.</div>
+        <div class="field-hint">When enabled, the server periodically starts the existing background “Refresh All” job using this interval. Bulk refreshes now wait this many seconds between shows to avoid flooding upstream sources.</div>
       </section>
 
       <section class="deps" id="deps"></section>
@@ -338,7 +341,8 @@ CONFIG_HTML = r"""<!doctype html>
         quality: "best",
         download_dir: "",
         watchlist_auto_refresh_enabled: false,
-        watchlist_auto_refresh_minutes: 60
+        watchlist_auto_refresh_minutes: 60,
+        watchlist_refresh_delay_seconds: 5
       }
     };
 
@@ -356,6 +360,7 @@ CONFIG_HTML = r"""<!doctype html>
       $("configQuality").value = state.config.quality;
       $("watchlistAutoRefreshEnabled").value = String(Boolean(state.config.watchlist_auto_refresh_enabled));
       $("watchlistAutoRefreshMinutes").value = state.config.watchlist_auto_refresh_minutes;
+      $("watchlistRefreshDelaySeconds").value = state.config.watchlist_refresh_delay_seconds;
     }
 
     async function saveConfig() {
@@ -367,7 +372,8 @@ CONFIG_HTML = r"""<!doctype html>
             mode: $("configMode").value,
             quality: $("configQuality").value,
             watchlist_auto_refresh_enabled: $("watchlistAutoRefreshEnabled").value === "true",
-            watchlist_auto_refresh_minutes: $("watchlistAutoRefreshMinutes").value
+            watchlist_auto_refresh_minutes: $("watchlistAutoRefreshMinutes").value,
+            watchlist_refresh_delay_seconds: $("watchlistRefreshDelaySeconds").value
           })
         });
         state.config = data;
@@ -376,6 +382,7 @@ CONFIG_HTML = r"""<!doctype html>
         $("configQuality").value = data.quality;
         $("watchlistAutoRefreshEnabled").value = String(Boolean(data.watchlist_auto_refresh_enabled));
         $("watchlistAutoRefreshMinutes").value = data.watchlist_auto_refresh_minutes;
+        $("watchlistRefreshDelaySeconds").value = data.watchlist_refresh_delay_seconds;
         setNotice("Defaults saved.");
       } catch (error) {
         setNotice(error.message);

queue_jobs.py

@@ -2,7 +2,6 @@
 
 """Queue and background job services for ani-cli-web."""
 
-import concurrent.futures
 import json
 import os
 import shutil
@@ -33,7 +32,6 @@ from app_support import (
 
 LOG_PERSIST_INTERVAL_SECONDS = 0.75
 LOG_PERSIST_LINE_BATCH = 8
-WATCHLIST_REFRESH_MAX_WORKERS = 4
 JOB_STRUCTURED_COLUMNS = (
     "show_id",
     "title",
@@ -59,8 +57,9 @@ JOB_EXTRA_PAYLOAD_COLUMN = "payload"
 
 
 class WatchlistRefreshJobs:
-    def __init__(self, store, start_worker=True):
+    def __init__(self, store, config_getter=None, start_worker=True):
         self.store = store
+        self.config_getter = config_getter or (lambda: {})
         self.lock = threading.RLock()
         self.wakeup = threading.Event()
         self.stop_event = threading.Event()
@@ -153,6 +152,14 @@ class WatchlistRefreshJobs:
             self._upsert_job_conn(conn, job)
             self._trim_history_conn(conn)
 
+    def _refresh_delay_seconds(self):
+        config = self.config_getter() or {}
+        try:
+            delay = int(str(config.get("watchlist_refresh_delay_seconds", 5)).strip() or "5")
+        except (TypeError, ValueError):
+            delay = 5
+        return max(0, min(300, delay))
+
     def _load_jobs(self):
         with self.lock, self._connect() as conn:
             rows = conn.execute(
@@ -257,6 +264,7 @@ class WatchlistRefreshJobs:
         source_name = str(job.get("source") or "manual").strip().lower() or "manual"
         try:
             show_ids = self.store.all_show_ids()
+            delay_seconds = self._refresh_delay_seconds()
             started_at = now_iso()
             with self.lock:
                 job["status"] = "running"
@@ -271,13 +279,14 @@ class WatchlistRefreshJobs:
             if source_name == "scheduled":
                 self._stdout_log(f"Scheduled refresh started (job={job['id']}, total={len(show_ids)}).")
 
-            def refresh_one(show_id):
+            for index, show_id in enumerate(show_ids, start=1):
+                if self.stop_event.is_set():
+                    interrupted = True
+                    break
                 title = show_id
                 errored = False
                 skipped = False
                 error_message = ""
-                if self.stop_event.is_set():
-                    return {"title": title, "errored": False, "skipped": True, "error": ""}
                 try:
                     item = self.store.refresh(show_id, include_thumbnail=False)
                     if item is None:
@@ -290,34 +299,22 @@ class WatchlistRefreshJobs:
                     errored = True
                     error_message = str(exc)
                     debug_log("watchlist.refresh_all.item_error", show_id=show_id, error=exc)
-                return {"title": title, "errored": errored, "skipped": skipped, "error": error_message}
-
-            max_workers = max(1, min(WATCHLIST_REFRESH_MAX_WORKERS, len(show_ids) or 1))
-            with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
                 with self.lock:
-                    self.current_executor = executor
-                futures = [executor.submit(refresh_one, show_id) for show_id in show_ids]
-                for index, future in enumerate(concurrent.futures.as_completed(futures), start=1):
-                    if self.stop_event.is_set():
+                    job["completed"] = index
+                    job["errors"] += 1 if errored else 0
+                    job["current_title"] = title
+                    job["updated_at"] = now_iso()
+                    if errored and error_message:
+                        job["last_error"] = error_message
+                    if skipped:
+                        job["message"] = f"Refreshing {index}/{len(show_ids)}: removed entry"
+                    else:
+                        job["message"] = f"Refreshing {index}/{len(show_ids)}: {title}"
+                    self._save_job_locked(job)
+                if index < len(show_ids) and delay_seconds > 0:
+                    if self.stop_event.wait(delay_seconds):
                         interrupted = True
-                        executor.shutdown(wait=False, cancel_futures=True)
                         break
-                    result = future.result()
-                    title = result["title"]
-                    with self.lock:
-                        job["completed"] = index
-                        job["errors"] += 1 if result["errored"] else 0
-                        job["current_title"] = title
-                        job["updated_at"] = now_iso()
-                        if result["errored"] and result["error"]:
-                            job["last_error"] = result["error"]
-                        if result["skipped"]:
-                            job["message"] = f"Refreshing {index}/{len(show_ids)}: removed entry"
-                        else:
-                            job["message"] = f"Refreshing {index}/{len(show_ids)}: {title}"
-                        self._save_job_locked(job)
-                with self.lock:
-                    self.current_executor = None
 
             finished_at = now_iso()
             with self.lock:

test_app.py

@@ -1074,6 +1074,7 @@ class ConfigSnapshotTests(unittest.TestCase):
             "download_dir": "/tmp/example",
             "watchlist_auto_refresh_enabled": False,
             "watchlist_auto_refresh_minutes": 60,
+            "watchlist_refresh_delay_seconds": 5,
         }
         try:
             snapshot = APP.get_config_snapshot()
@@ -1086,16 +1087,19 @@ class ConfigSnapshotTests(unittest.TestCase):
         config = APP.normalize_config({})
         self.assertFalse(config["watchlist_auto_refresh_enabled"])
         self.assertEqual(config["watchlist_auto_refresh_minutes"], 60)
+        self.assertEqual(config["watchlist_refresh_delay_seconds"], 5)
 
     def test_normalize_config_parses_watchlist_schedule_fields(self):
         config = APP.normalize_config(
             {
                 "watchlist_auto_refresh_enabled": "true",
                 "watchlist_auto_refresh_minutes": "15",
+                "watchlist_refresh_delay_seconds": "7",
             }
         )
         self.assertTrue(config["watchlist_auto_refresh_enabled"])
         self.assertEqual(config["watchlist_auto_refresh_minutes"], 15)
+        self.assertEqual(config["watchlist_refresh_delay_seconds"], 7)
 
 
 class WatchlistAutoRefreshSchedulerTests(unittest.TestCase):
@@ -1112,6 +1116,28 @@ class WatchlistAutoRefreshSchedulerTests(unittest.TestCase):
         self.assertEqual(scheduler.tick(now_monotonic=700.0), 600)
         start.assert_called_once_with(source="scheduled")
 
+    def test_refresh_job_waits_between_series_using_configured_delay(self):
+        class FakeStore:
+            def all_show_ids(self):
+                return ["show-a", "show-b", "show-c"]
+
+            def refresh(self, show_id, include_thumbnail=True):
+                return {"title": show_id, "status": "updated"}
+
+        service = APP.WatchlistRefreshJobs(
+            FakeStore(),
+            config_getter=lambda: {"watchlist_refresh_delay_seconds": 5},
+            start_worker=False,
+        )
+        job = service.start()["job"]
+        job = service._next_pending()
+
+        with mock.patch.object(service.stop_event, "wait", return_value=False) as wait_mock:
+            service._run_job(job)
+
+        self.assertEqual(wait_mock.call_count, 2)
+        wait_mock.assert_called_with(5)
+
     def test_tick_resets_timer_when_schedule_changes(self):
         config = {
             "watchlist_auto_refresh_enabled": True,
@@ -1196,6 +1222,7 @@ class StartupBehaviorTests(unittest.TestCase):
                 "download_dir": "/tmp/old",
                 "watchlist_auto_refresh_enabled": False,
                 "watchlist_auto_refresh_minutes": 60,
+                "watchlist_refresh_delay_seconds": 5,
             }
             APP.WATCHLIST_AUTO_REFRESH = mock.Mock()
             saved = APP.save_runtime_config(
@@ -1205,6 +1232,7 @@ class StartupBehaviorTests(unittest.TestCase):
                     "download_dir": "/tmp/new",
                     "watchlist_auto_refresh_enabled": True,
                     "watchlist_auto_refresh_minutes": 15,
+                    "watchlist_refresh_delay_seconds": 8,
                 }
             )
             self.assertEqual(saved["mode"], "dub")
@@ -1212,6 +1240,7 @@ class StartupBehaviorTests(unittest.TestCase):
             self.assertEqual(APP.CONFIG["quality"], "720")
             self.assertTrue(APP.CONFIG["watchlist_auto_refresh_enabled"])
             self.assertEqual(APP.CONFIG["watchlist_auto_refresh_minutes"], 15)
+            self.assertEqual(APP.CONFIG["watchlist_refresh_delay_seconds"], 8)
             APP.WATCHLIST_AUTO_REFRESH.notify_config_changed.assert_called_once_with()
         finally:
             APP.CONFIG = original
@@ -1247,12 +1276,13 @@ class HandlerRouteTests(unittest.TestCase):
             APP.initialize_runtime(start_workers=False)
 
     def test_config_post_accepts_watchlist_schedule_fields(self):
-        body = b'{"download_dir":"/tmp/example","mode":"sub","quality":"best","watchlist_auto_refresh_enabled":true,"watchlist_auto_refresh_minutes":"25"}'
+        body = b'{"download_dir":"/tmp/example","mode":"sub","quality":"best","watchlist_auto_refresh_enabled":true,"watchlist_auto_refresh_minutes":"25","watchlist_refresh_delay_seconds":"9"}'
         handler = DummyHandler("/api/config", body=body, content_length=len(body))
         APP.Handler.do_POST(handler)
         self.assertEqual(handler.json_status, HTTPStatus.OK)
         self.assertTrue(handler.json_payload["watchlist_auto_refresh_enabled"])
         self.assertEqual(handler.json_payload["watchlist_auto_refresh_minutes"], 25)
+        self.assertEqual(handler.json_payload["watchlist_refresh_delay_seconds"], 9)
 
     def test_body_json_rejects_oversized_request(self):
         handler = DummyHandler("/api/watchlist/upload-thumbnail", body=b"{}", content_length=APP.MAX_JSON_BODY_BYTES + 1)