From a9549cf882ceacd989ecbf0e3e7749283f4e5ef9 Mon Sep 17 00:00:00 2001 From: Dymas Date: Sat, 5 Sep 2026 21:13:41 +0200 Subject: [PATCH] Route disguised HLS segments natively --- CHANGELOG.md | 5 +++++ README.md | 4 ++-- VERSION | 2 +- provider_downloader.py | 18 ++++++++++++++++-- test_app.py | 12 ++++++++++++ 5 files changed, 36 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 835a722..9e12ac3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 0.52.22 - 2026-09-05 + +- Routed SnapCDN and disguised HLS media segments through the native segment downloader to avoid ffmpeg hammering provider CDNs into `HTTP 429` failures. +- Added `KAIZOKU_SEGMENT_DOWNLOAD_DELAY` for optional pacing between native HLS segment requests. + ## 0.52.21 - 2026-09-05 - Changed provider downloads to try alternate same-mode server sources for an episode before falling back to another provider. diff --git a/README.md b/README.md index 1fc579f..124fb77 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ Useful environment variables: - `KAIZOKU_DOWNLOAD_DIR=/downloads` for the default library output path. - `KAIZOKU_REMOTE_PATH_ROOTS=/downloads,/jellyfin/tv,/jellyfin/movies` to limit remote filesystem browsing; comma-separated and platform path separators are accepted. - `KAIZOKU_MODE=sub`, `KAIZOKU_QUALITY=best`, and `KAIZOKU_DEBUG=1` for runtime defaults and diagnostics. -- `KAIZOKU_SEGMENT_RETRIES=8`, `KAIZOKU_SEGMENT_RETRY_DELAY=1.25`, and `KAIZOKU_SEGMENT_RETRY_MAX_DELAY=15` to tune retry/backoff behavior for native HLS segment downloads. +- `KAIZOKU_SEGMENT_RETRIES=8`, `KAIZOKU_SEGMENT_RETRY_DELAY=1.25`, `KAIZOKU_SEGMENT_RETRY_MAX_DELAY=15`, and `KAIZOKU_SEGMENT_DOWNLOAD_DELAY=0` to tune retry/backoff and optional pacing behavior for native HLS segment downloads. ## Docker @@ -100,7 +100,7 @@ Keep `./.kaizoku` mounted for production instances. That directory contains the ## Download Flow -Kaizoku stores provider-backed show IDs as `provider:id`, for example `anikoto:some-show-slug`. Queue jobs resolve the episode source through `providers/bridge.js`, then `provider_downloader.py` downloads the media with `ffmpeg` into a staging directory. If a provider returns multiple servers for the requested sub or dub mode, Kaizoku tries the same-mode server sources in quality order before falling back to another provider. It does not use sub sources for dub downloads, or dub sources for sub downloads. If a provider returns a master HLS playlist, Kaizoku selects the highest-bandwidth variant before starting `ffmpeg`. When a media playlist uses extensionless CDN segments, Kaizoku skips direct ffmpeg and uses a StrawVerse-style segment downloader: it fetches the media playlist, downloads and concatenates segments itself, strips short PNG wrappers when present, retries temporary HTTP failures such as `429 Too Many Requests`, then remuxes the local transport stream to MP4. Direct ffmpeg attempts also have a timeout guard so stalled HLS inputs can fall back cleanly. Each episode is written as a temporary `.mp4.part` file and moved into the downloads library after that episode succeeds, so already-finished episodes from a larger batch survive if a later episode fails. Retrying that failed queue job requests only the remaining episodes while keeping the original episode range for display and watchlist sync. Finalization preserves episode numbers from staged `SxxEyy` or `Episode yy` filenames before applying configured season/episode offsets. +Kaizoku stores provider-backed show IDs as `provider:id`, for example `anikoto:some-show-slug`. Queue jobs resolve the episode source through `providers/bridge.js`, then `provider_downloader.py` downloads the media with `ffmpeg` into a staging directory. If a provider returns multiple servers for the requested sub or dub mode, Kaizoku tries the same-mode server sources in quality order before falling back to another provider. It does not use sub sources for dub downloads, or dub sources for sub downloads. If a provider returns a master HLS playlist, Kaizoku selects the highest-bandwidth variant before starting `ffmpeg`. When a media playlist uses extensionless, SnapCDN, or disguised CDN segments, Kaizoku skips direct ffmpeg and uses a StrawVerse-style segment downloader: it fetches the media playlist, downloads and concatenates segments itself, strips short PNG wrappers when present, retries temporary HTTP failures such as `429 Too Many Requests`, optionally paces requests with `KAIZOKU_SEGMENT_DOWNLOAD_DELAY`, then remuxes the local transport stream to MP4. Direct ffmpeg attempts also have a timeout guard so stalled HLS inputs can fall back cleanly. Each episode is written as a temporary `.mp4.part` file and moved into the downloads library after that episode succeeds, so already-finished episodes from a larger batch survive if a later episode fails. Retrying that failed queue job requests only the remaining episodes while keeping the original episode range for display and watchlist sync. Finalization preserves episode numbers from staged `SxxEyy` or `Episode yy` filenames before applying configured season/episode offsets. If the primary provider cannot list, resolve, or download a requested episode, Kaizoku searches the same title on the remaining providers and tries the matching episode there. Existing finalization code moves staged files into the configured library layout, preserving data already present in production download folders. diff --git a/VERSION b/VERSION index a06aa74..2d236ed 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.52.21 +0.52.22 diff --git a/provider_downloader.py b/provider_downloader.py index 79551dd..65b6d8f 100755 --- a/provider_downloader.py +++ b/provider_downloader.py @@ -275,6 +275,10 @@ def fetch_bytes_with_retries(url, headers=None, timeout=30, retries=None, retry_ time.sleep(delay) +def segment_download_delay_seconds(): + return float_env("KAIZOKU_SEGMENT_DOWNLOAD_DELAY", 0.0, minimum=0.0) + + def fetch_text(url, headers=None, timeout=30): return fetch_bytes(url, headers=headers, timeout=timeout).decode("utf-8", errors="replace") @@ -391,13 +395,20 @@ def parse_hls_segments(playlist_url, playlist_text): def hls_segments_need_native_download(segments): + disguised_segment_extensions = {".jpg", ".jpeg", ".png", ".webp", ".ico", ".css", ".js", ".html", ".txt"} for segment in segments or []: - path = urlparse(segment.get("url") or "").path + parsed = urlparse(segment.get("url") or "") + path = parsed.path name = path.rsplit("/", 1)[-1] + suffix = Path(name).suffix.lower() if "." not in name: return True if "/ad-site-i18n/" in path: return True + if parsed.hostname and parsed.hostname.endswith("snapcdn.top"): + return True + if suffix in disguised_segment_extensions: + return True return False @@ -448,8 +459,11 @@ def download_hls_segments(stream, input_url, target, partial, episode_number=Non percent=0, ) last_percent = -1 + segment_delay = segment_download_delay_seconds() with ts_file.open("wb") as joined: for index, segment in enumerate(segments, start=1): + if segment_delay and index > 1: + time.sleep(segment_delay) data = fetch_bytes_with_retries( segment["url"], headers=stream.get("headers") or {}, @@ -566,7 +580,7 @@ def download_episode(stream, target, episode_number=None, episode_index=None, ep media_url, playlist = hls_media_playlist(stream, input_url) segments = parse_hls_segments(media_url, playlist) if hls_segments_need_native_download(segments): - message = "HLS playlist uses extensionless provider segments; skipping direct ffmpeg." + message = "HLS playlist uses provider segments that need native download; skipping direct ffmpeg." print(message, flush=True) emit_progress( phase="preflight", diff --git a/test_app.py b/test_app.py index 955f2b9..4ae4b4b 100644 --- a/test_app.py +++ b/test_app.py @@ -4534,6 +4534,18 @@ seg-3.ts ) ) + def test_hls_segments_need_native_download_for_snapcdn_disguised_segments(self): + self.assertTrue( + provider_downloader.hls_segments_need_native_download( + [{"url": "https://shard-102.snapcdn.top/anime/show/episode/seg-f1-00176.jpg"}] + ) + ) + self.assertTrue( + provider_downloader.hls_segments_need_native_download( + [{"url": "https://cdn.example/video/seg-f1-00170.js"}] + ) + ) + def test_fetch_bytes_with_retries_recovers_from_segment_rate_limit(self): error = urllib.error.HTTPError( "https://cdn.example/video/raw-segment",