diff --git a/CHANGELOG.md b/CHANGELOG.md index dcedbbb..ad61965 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 0.32.0 - 2026-05-17 + +- Changed bulk `Refresh All` processing to pace requests with a configurable delay between shows, reducing the chance of flooding upstream episode sources on large watchlists. +- Added a Config page setting for the per-show bulk refresh delay and expanded regression coverage for the new config field and pacing behavior. + ## 0.31.1 - 2026-05-17 - Added stdout lifecycle logging for scheduled watchlist refresh jobs so container logs now show when a scheduled run is queued, starts, finishes successfully, finishes with errors, gets interrupted, or fails. diff --git a/README.md b/README.md index fce902d..361ac17 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ A local web UI for a system-wide `ani-cli` install. -Current version: `0.31.1` +Current version: `0.32.0` ## Project Layout @@ -156,7 +156,7 @@ The app currently includes: - Per-show watchlist refresh queueing now also deduplicates in-flight refreshes, so the same show is not fetched twice back-to-back while one refresh is already running. - The watchlist page now silently refreshes itself while queued per-show refresh work is still being processed, so queued entries settle into their updated state without a manual reload. - The watchlist page now also avoids re-triggering the same unresolved thumbnail warm-up request on every silent poll cycle while a show's thumbnail state is unchanged. -- Background `Refresh All` execution with status polling, parallel per-show refresh work, and no inline thumbnail downloads during the bulk job. +- Background `Refresh All` execution with status polling, no inline thumbnail downloads during the bulk job, and a configurable delay between shows to reduce upstream request bursts. - Optional scheduled background watchlist refreshes with Config page controls for enable/disable and refresh period. - Scheduled background refresh runs now log queue, start, and completion status to stdout so container logs show whether they finished successfully, with errors, or failed. - Search and Watchlist polling now use a shared serial browser polling helper so slow responses do not cause overlapping poll requests to pile up in the background. @@ -206,6 +206,7 @@ Use it to: 3. Choose the default search quality used to prefill the Search page. 4. Enable or disable periodic watchlist refresh checks. 5. Set the refresh period in minutes for that scheduled watchlist job. +6. Set the delay in seconds between shows during bulk watchlist refresh runs. Those saved defaults are written into the project-local `config.json` and loaded automatically when the Search page opens. @@ -278,7 +279,7 @@ You can add entries in two ways: Watchlist features: 1. `Refresh` updates one tracked show. -2. `Refresh All` runs in the background, rejects duplicate queueing while a refresh is already pending or running, preserves interrupted job status across restarts, and refreshes shows in parallel without doing inline thumbnail downloads for every entry. +2. `Refresh All` runs in the background, rejects duplicate queueing while a refresh is already pending or running, preserves interrupted job status across restarts, and refreshes shows one by one with a configurable delay without doing inline thumbnail downloads for every entry. 3. `Open` jumps back to the Search page with the title pre-filled. 4. Clicking the anime title opens the AllManga source page at `bangumi/` in a new tab. 5. `Remove` deletes the watchlist entry and clears its cached poster file. @@ -392,7 +393,7 @@ Environment variables: Saved defaults from the UI are written into the project-local `config.json`. -The saved config now also includes `watchlist_auto_refresh_enabled` and `watchlist_auto_refresh_minutes`, which control the periodic background watchlist refresh schedule. +The saved config now also includes `watchlist_auto_refresh_enabled`, `watchlist_auto_refresh_minutes`, and `watchlist_refresh_delay_seconds`, which control the periodic background watchlist refresh schedule and the per-show pacing used by bulk refresh jobs. ## Notes diff --git a/VERSION b/VERSION index f176c94..9eb2aa3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.31.1 +0.32.0 diff --git a/app.py b/app.py index 52597d4..ac06be2 100644 --- a/app.py +++ b/app.py @@ -1814,7 +1814,11 @@ def build_runtime(start_workers=None): job_prepare_fn=prepare_download_job, start_worker=worker_state, ) - watchlist_refresh = WatchlistRefreshJobs(watchlist, start_worker=worker_state) + watchlist_refresh = WatchlistRefreshJobs( + watchlist, + config_getter=lambda: get_config_snapshot(config), + start_worker=worker_state, + ) watchlist_auto_refresh = WatchlistAutoRefreshScheduler( config_getter=lambda: get_config_snapshot(config), refresh_start_fn=watchlist_refresh.start, diff --git a/app_support.py b/app_support.py index 75d3556..4482bbc 100644 --- a/app_support.py +++ b/app_support.py @@ -16,7 +16,7 @@ from uuid import uuid4 PROJECT_ROOT = Path(__file__).resolve().parent ANI_CLI = os.environ.get("ANI_CLI_BIN") or shutil.which("ani-cli") or "ani-cli" APP_NAME = "ani-cli-web" -VERSION = "0.31.1" +VERSION = "0.32.0" ALLANIME_BASE = "allanime.day" ALLANIME_API = f"https://api.{ALLANIME_BASE}" ALLANIME_REFERER = "https://allmanga.to" @@ -45,6 +45,9 @@ WORKER_SHUTDOWN_TIMEOUT_SECONDS = 10 WATCHLIST_AUTO_REFRESH_DEFAULT_MINUTES = 60 WATCHLIST_AUTO_REFRESH_MIN_MINUTES = 1 WATCHLIST_AUTO_REFRESH_MAX_MINUTES = 7 * 24 * 60 +WATCHLIST_REFRESH_DELAY_DEFAULT_SECONDS = 5 +WATCHLIST_REFRESH_DELAY_MIN_SECONDS = 0 +WATCHLIST_REFRESH_DELAY_MAX_SECONDS = 300 def env_flag(name, default=False): @@ -119,6 +122,7 @@ DEFAULT_CONFIG = { "quality": os.environ.get("ANI_CLI_QUALITY", "best"), "watchlist_auto_refresh_enabled": False, "watchlist_auto_refresh_minutes": WATCHLIST_AUTO_REFRESH_DEFAULT_MINUTES, + "watchlist_refresh_delay_seconds": WATCHLIST_REFRESH_DELAY_DEFAULT_SECONDS, } @@ -261,6 +265,7 @@ def normalize_config(data): download_dir = str(config.get("download_dir") or DEFAULT_CONFIG["download_dir"]) auto_refresh_enabled = config.get("watchlist_auto_refresh_enabled") auto_refresh_minutes = config.get("watchlist_auto_refresh_minutes") + refresh_delay_seconds = config.get("watchlist_refresh_delay_seconds") if isinstance(auto_refresh_enabled, str): auto_refresh_enabled = auto_refresh_enabled.strip().lower() in {"1", "true", "yes", "on"} @@ -275,12 +280,21 @@ def normalize_config(data): WATCHLIST_AUTO_REFRESH_MIN_MINUTES, min(WATCHLIST_AUTO_REFRESH_MAX_MINUTES, auto_refresh_minutes), ) + try: + refresh_delay_seconds = int(str(refresh_delay_seconds).strip() or WATCHLIST_REFRESH_DELAY_DEFAULT_SECONDS) + except (TypeError, ValueError): + refresh_delay_seconds = WATCHLIST_REFRESH_DELAY_DEFAULT_SECONDS + refresh_delay_seconds = max( + WATCHLIST_REFRESH_DELAY_MIN_SECONDS, + min(WATCHLIST_REFRESH_DELAY_MAX_SECONDS, refresh_delay_seconds), + ) config["mode"] = mode if mode in MODE_CHOICES else "sub" config["quality"] = quality if quality in QUALITY_CHOICES else "best" config["download_dir"] = str(Path(download_dir).expanduser()) config["watchlist_auto_refresh_enabled"] = auto_refresh_enabled config["watchlist_auto_refresh_minutes"] = auto_refresh_minutes + config["watchlist_refresh_delay_seconds"] = refresh_delay_seconds return config diff --git a/config_page.py b/config_page.py index 67639a7..609aef6 100644 --- a/config_page.py +++ b/config_page.py @@ -323,8 +323,11 @@ CONFIG_HTML = r""" + -
When enabled, the server periodically starts the existing background “Refresh All” job using this interval.
+
When enabled, the server periodically starts the existing background “Refresh All” job using this interval. Bulk refreshes now wait this many seconds between shows to avoid flooding upstream sources.
@@ -338,7 +341,8 @@ CONFIG_HTML = r""" quality: "best", download_dir: "", watchlist_auto_refresh_enabled: false, - watchlist_auto_refresh_minutes: 60 + watchlist_auto_refresh_minutes: 60, + watchlist_refresh_delay_seconds: 5 } }; @@ -356,6 +360,7 @@ CONFIG_HTML = r""" $("configQuality").value = state.config.quality; $("watchlistAutoRefreshEnabled").value = String(Boolean(state.config.watchlist_auto_refresh_enabled)); $("watchlistAutoRefreshMinutes").value = state.config.watchlist_auto_refresh_minutes; + $("watchlistRefreshDelaySeconds").value = state.config.watchlist_refresh_delay_seconds; } async function saveConfig() { @@ -367,7 +372,8 @@ CONFIG_HTML = r""" mode: $("configMode").value, quality: $("configQuality").value, watchlist_auto_refresh_enabled: $("watchlistAutoRefreshEnabled").value === "true", - watchlist_auto_refresh_minutes: $("watchlistAutoRefreshMinutes").value + watchlist_auto_refresh_minutes: $("watchlistAutoRefreshMinutes").value, + watchlist_refresh_delay_seconds: $("watchlistRefreshDelaySeconds").value }) }); state.config = data; @@ -376,6 +382,7 @@ CONFIG_HTML = r""" $("configQuality").value = data.quality; $("watchlistAutoRefreshEnabled").value = String(Boolean(data.watchlist_auto_refresh_enabled)); $("watchlistAutoRefreshMinutes").value = data.watchlist_auto_refresh_minutes; + $("watchlistRefreshDelaySeconds").value = data.watchlist_refresh_delay_seconds; setNotice("Defaults saved."); } catch (error) { setNotice(error.message); diff --git a/queue_jobs.py b/queue_jobs.py index 8448494..e89c95d 100644 --- a/queue_jobs.py +++ b/queue_jobs.py @@ -2,7 +2,6 @@ """Queue and background job services for ani-cli-web.""" -import concurrent.futures import json import os import shutil @@ -33,7 +32,6 @@ from app_support import ( LOG_PERSIST_INTERVAL_SECONDS = 0.75 LOG_PERSIST_LINE_BATCH = 8 -WATCHLIST_REFRESH_MAX_WORKERS = 4 JOB_STRUCTURED_COLUMNS = ( "show_id", "title", @@ -59,8 +57,9 @@ JOB_EXTRA_PAYLOAD_COLUMN = "payload" class WatchlistRefreshJobs: - def __init__(self, store, start_worker=True): + def __init__(self, store, config_getter=None, start_worker=True): self.store = store + self.config_getter = config_getter or (lambda: {}) self.lock = threading.RLock() self.wakeup = threading.Event() self.stop_event = threading.Event() @@ -153,6 +152,14 @@ class WatchlistRefreshJobs: self._upsert_job_conn(conn, job) self._trim_history_conn(conn) + def _refresh_delay_seconds(self): + config = self.config_getter() or {} + try: + delay = int(str(config.get("watchlist_refresh_delay_seconds", 5)).strip() or "5") + except (TypeError, ValueError): + delay = 5 + return max(0, min(300, delay)) + def _load_jobs(self): with self.lock, self._connect() as conn: rows = conn.execute( @@ -257,6 +264,7 @@ class WatchlistRefreshJobs: source_name = str(job.get("source") or "manual").strip().lower() or "manual" try: show_ids = self.store.all_show_ids() + delay_seconds = self._refresh_delay_seconds() started_at = now_iso() with self.lock: job["status"] = "running" @@ -271,13 +279,14 @@ class WatchlistRefreshJobs: if source_name == "scheduled": self._stdout_log(f"Scheduled refresh started (job={job['id']}, total={len(show_ids)}).") - def refresh_one(show_id): + for index, show_id in enumerate(show_ids, start=1): + if self.stop_event.is_set(): + interrupted = True + break title = show_id errored = False skipped = False error_message = "" - if self.stop_event.is_set(): - return {"title": title, "errored": False, "skipped": True, "error": ""} try: item = self.store.refresh(show_id, include_thumbnail=False) if item is None: @@ -290,34 +299,22 @@ class WatchlistRefreshJobs: errored = True error_message = str(exc) debug_log("watchlist.refresh_all.item_error", show_id=show_id, error=exc) - return {"title": title, "errored": errored, "skipped": skipped, "error": error_message} - - max_workers = max(1, min(WATCHLIST_REFRESH_MAX_WORKERS, len(show_ids) or 1)) - with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: with self.lock: - self.current_executor = executor - futures = [executor.submit(refresh_one, show_id) for show_id in show_ids] - for index, future in enumerate(concurrent.futures.as_completed(futures), start=1): - if self.stop_event.is_set(): + job["completed"] = index + job["errors"] += 1 if errored else 0 + job["current_title"] = title + job["updated_at"] = now_iso() + if errored and error_message: + job["last_error"] = error_message + if skipped: + job["message"] = f"Refreshing {index}/{len(show_ids)}: removed entry" + else: + job["message"] = f"Refreshing {index}/{len(show_ids)}: {title}" + self._save_job_locked(job) + if index < len(show_ids) and delay_seconds > 0: + if self.stop_event.wait(delay_seconds): interrupted = True - executor.shutdown(wait=False, cancel_futures=True) break - result = future.result() - title = result["title"] - with self.lock: - job["completed"] = index - job["errors"] += 1 if result["errored"] else 0 - job["current_title"] = title - job["updated_at"] = now_iso() - if result["errored"] and result["error"]: - job["last_error"] = result["error"] - if result["skipped"]: - job["message"] = f"Refreshing {index}/{len(show_ids)}: removed entry" - else: - job["message"] = f"Refreshing {index}/{len(show_ids)}: {title}" - self._save_job_locked(job) - with self.lock: - self.current_executor = None finished_at = now_iso() with self.lock: diff --git a/test_app.py b/test_app.py index 308e02f..a926a12 100644 --- a/test_app.py +++ b/test_app.py @@ -1074,6 +1074,7 @@ class ConfigSnapshotTests(unittest.TestCase): "download_dir": "/tmp/example", "watchlist_auto_refresh_enabled": False, "watchlist_auto_refresh_minutes": 60, + "watchlist_refresh_delay_seconds": 5, } try: snapshot = APP.get_config_snapshot() @@ -1086,16 +1087,19 @@ class ConfigSnapshotTests(unittest.TestCase): config = APP.normalize_config({}) self.assertFalse(config["watchlist_auto_refresh_enabled"]) self.assertEqual(config["watchlist_auto_refresh_minutes"], 60) + self.assertEqual(config["watchlist_refresh_delay_seconds"], 5) def test_normalize_config_parses_watchlist_schedule_fields(self): config = APP.normalize_config( { "watchlist_auto_refresh_enabled": "true", "watchlist_auto_refresh_minutes": "15", + "watchlist_refresh_delay_seconds": "7", } ) self.assertTrue(config["watchlist_auto_refresh_enabled"]) self.assertEqual(config["watchlist_auto_refresh_minutes"], 15) + self.assertEqual(config["watchlist_refresh_delay_seconds"], 7) class WatchlistAutoRefreshSchedulerTests(unittest.TestCase): @@ -1112,6 +1116,28 @@ class WatchlistAutoRefreshSchedulerTests(unittest.TestCase): self.assertEqual(scheduler.tick(now_monotonic=700.0), 600) start.assert_called_once_with(source="scheduled") + def test_refresh_job_waits_between_series_using_configured_delay(self): + class FakeStore: + def all_show_ids(self): + return ["show-a", "show-b", "show-c"] + + def refresh(self, show_id, include_thumbnail=True): + return {"title": show_id, "status": "updated"} + + service = APP.WatchlistRefreshJobs( + FakeStore(), + config_getter=lambda: {"watchlist_refresh_delay_seconds": 5}, + start_worker=False, + ) + job = service.start()["job"] + job = service._next_pending() + + with mock.patch.object(service.stop_event, "wait", return_value=False) as wait_mock: + service._run_job(job) + + self.assertEqual(wait_mock.call_count, 2) + wait_mock.assert_called_with(5) + def test_tick_resets_timer_when_schedule_changes(self): config = { "watchlist_auto_refresh_enabled": True, @@ -1196,6 +1222,7 @@ class StartupBehaviorTests(unittest.TestCase): "download_dir": "/tmp/old", "watchlist_auto_refresh_enabled": False, "watchlist_auto_refresh_minutes": 60, + "watchlist_refresh_delay_seconds": 5, } APP.WATCHLIST_AUTO_REFRESH = mock.Mock() saved = APP.save_runtime_config( @@ -1205,6 +1232,7 @@ class StartupBehaviorTests(unittest.TestCase): "download_dir": "/tmp/new", "watchlist_auto_refresh_enabled": True, "watchlist_auto_refresh_minutes": 15, + "watchlist_refresh_delay_seconds": 8, } ) self.assertEqual(saved["mode"], "dub") @@ -1212,6 +1240,7 @@ class StartupBehaviorTests(unittest.TestCase): self.assertEqual(APP.CONFIG["quality"], "720") self.assertTrue(APP.CONFIG["watchlist_auto_refresh_enabled"]) self.assertEqual(APP.CONFIG["watchlist_auto_refresh_minutes"], 15) + self.assertEqual(APP.CONFIG["watchlist_refresh_delay_seconds"], 8) APP.WATCHLIST_AUTO_REFRESH.notify_config_changed.assert_called_once_with() finally: APP.CONFIG = original @@ -1247,12 +1276,13 @@ class HandlerRouteTests(unittest.TestCase): APP.initialize_runtime(start_workers=False) def test_config_post_accepts_watchlist_schedule_fields(self): - body = b'{"download_dir":"/tmp/example","mode":"sub","quality":"best","watchlist_auto_refresh_enabled":true,"watchlist_auto_refresh_minutes":"25"}' + body = b'{"download_dir":"/tmp/example","mode":"sub","quality":"best","watchlist_auto_refresh_enabled":true,"watchlist_auto_refresh_minutes":"25","watchlist_refresh_delay_seconds":"9"}' handler = DummyHandler("/api/config", body=body, content_length=len(body)) APP.Handler.do_POST(handler) self.assertEqual(handler.json_status, HTTPStatus.OK) self.assertTrue(handler.json_payload["watchlist_auto_refresh_enabled"]) self.assertEqual(handler.json_payload["watchlist_auto_refresh_minutes"], 25) + self.assertEqual(handler.json_payload["watchlist_refresh_delay_seconds"], 9) def test_body_json_rejects_oversized_request(self): handler = DummyHandler("/api/watchlist/upload-thumbnail", body=b"{}", content_length=APP.MAX_JSON_BODY_BYTES + 1)