Skip ffmpeg for extensionless HLS segments

2026-08-09 12:46:19 +02:00
parent 3018e7aa66
commit 499271c3b8
5 changed files with 62 additions and 9 deletions
@@ -1,5 +1,10 @@
 # Changelog

+## 0.51.7 - 2026-08-09
+
+- Changed provider HLS downloads to preflight media playlists and skip direct ffmpeg when extensionless CDN segment URLs are detected.
+- Added a timeout guard around direct ffmpeg attempts so stalled HLS inputs can fall back instead of hanging the queue worker.
+
 ## 0.51.6 - 2026-08-09

 - Added a StrawVerse-style HLS segment downloader fallback for providers whose playlists contain extensionless CDN segment URLs that ffmpeg rejects.
@@ -71,7 +71,7 @@ The Docker image installs Python, Node.js, npm, and `ffmpeg`, then runs `npm ins

 ## Download Flow

-Kaizoku stores provider-backed show IDs as `provider:id`, for example `anikoto:some-show-slug`. Queue jobs resolve the episode source through `providers/bridge.js`, then `provider_downloader.py` downloads the media with `ffmpeg` into a staging directory. If a provider returns a master HLS playlist, Kaizoku selects the highest-bandwidth variant before starting `ffmpeg`. When ffmpeg rejects provider HLS playlists with extensionless CDN segments, Kaizoku falls back to a StrawVerse-style segment downloader: it fetches the media playlist, downloads and concatenates segments itself, strips short PNG wrappers when present, then remuxes the local transport stream to MP4. Each episode is written as a temporary `.mp4.part` file and moved into place only after the download succeeds, so failed fallback attempts do not leave broken final MP4 files behind. If the primary provider cannot list, resolve, or download a requested episode, Kaizoku searches the same title on the remaining providers and tries the matching episode there. Existing finalization code moves staged files into the configured library layout, preserving data already present in production download folders.
+Kaizoku stores provider-backed show IDs as `provider:id`, for example `anikoto:some-show-slug`. Queue jobs resolve the episode source through `providers/bridge.js`, then `provider_downloader.py` downloads the media with `ffmpeg` into a staging directory. If a provider returns a master HLS playlist, Kaizoku selects the highest-bandwidth variant before starting `ffmpeg`. When a media playlist uses extensionless CDN segments, Kaizoku skips direct ffmpeg and uses a StrawVerse-style segment downloader: it fetches the media playlist, downloads and concatenates segments itself, strips short PNG wrappers when present, then remuxes the local transport stream to MP4. Direct ffmpeg attempts also have a timeout guard so stalled HLS inputs can fall back cleanly. Each episode is written as a temporary `.mp4.part` file and moved into place only after the download succeeds, so failed fallback attempts do not leave broken final MP4 files behind. If the primary provider cannot list, resolve, or download a requested episode, Kaizoku searches the same title on the remaining providers and tries the matching episode there. Existing finalization code moves staged files into the configured library layout, preserving data already present in production download folders.

 ## Data Safety

@@ -1 +1 @@
-0.51.6
+0.51.7
@@ -247,6 +247,14 @@ def resolve_hls_input_url(stream):
    return best_hls_variant_url(url, data)


+def hls_media_playlist(stream, input_url):
+    playlist = fetch_text(input_url, headers=stream.get("headers") or {})
+    if "#EXT-X-STREAM-INF" in playlist:
+        input_url = best_hls_variant_url(input_url, playlist)
+        playlist = fetch_text(input_url, headers=stream.get("headers") or {})
+    return input_url, playlist
+
+
 def ffmpeg_base_command(stream, input_url):
    cmd = [
        "ffmpeg",
@@ -320,6 +328,17 @@ def parse_hls_segments(playlist_url, playlist_text):
    return segments


+def hls_segments_need_native_download(segments):
+    for segment in segments or []:
+        path = urlparse(segment.get("url") or "").path
+        name = path.rsplit("/", 1)[-1]
+        if "." not in name:
+            return True
+        if "/ad-site-i18n/" in path:
+            return True
+    return False
+
+
 def decrypt_aes128_segment(data, key, iv_value):
    openssl = shutil.which("openssl")
    if not openssl:
@@ -343,10 +362,7 @@ def decrypt_aes128_segment(data, key, iv_value):


 def download_hls_segments(stream, input_url, target, partial):
-    playlist = fetch_text(input_url, headers=stream.get("headers") or {})
-    if "#EXT-X-STREAM-INF" in playlist:
-        input_url = best_hls_variant_url(input_url, playlist)
-        playlist = fetch_text(input_url, headers=stream.get("headers") or {})
+    input_url, playlist = hls_media_playlist(stream, input_url)
    segments = parse_hls_segments(input_url, playlist)
    if not segments:
        return 1
@@ -357,7 +373,7 @@ def download_hls_segments(stream, input_url, target, partial):
    segment_dir.mkdir(parents=True, exist_ok=True)
    key_cache = {}
    try:
-        print(f"Direct ffmpeg HLS failed; downloading {len(segments)} playlist segments...")
+        print(f"Downloading {len(segments)} HLS playlist segments directly...")
        with ts_file.open("wb") as joined:
            for index, segment in enumerate(segments, start=1):
                data = fetch_bytes(segment["url"], headers=stream.get("headers") or {}, timeout=60)
@@ -392,7 +408,7 @@ def download_hls_segments(stream, input_url, target, partial):
            "mp4",
            str(partial),
        ]
-        code = subprocess.call(cmd)
+        code = run_ffmpeg(cmd, timeout=300)
        if code == 0 and partial.exists() and partial.stat().st_size > 0:
            partial.replace(target)
            return 0
@@ -405,6 +421,14 @@ def download_hls_segments(stream, input_url, target, partial):
        shutil.rmtree(segment_dir, ignore_errors=True)


+def run_ffmpeg(cmd, timeout=180):
+    try:
+        return subprocess.run(cmd, check=False, timeout=timeout).returncode
+    except subprocess.TimeoutExpired:
+        print(f"ffmpeg timed out after {timeout}s; trying fallback.", file=sys.stderr)
+        return 124
+
+
 def download_subtitles(subtitles, output_base):
    saved = []
    english = [
@@ -430,6 +454,18 @@ def download_episode(stream, target):
            path.unlink()
        except FileNotFoundError:
            pass
+    if stream.get("isM3U8") or ".m3u8" in str(input_url):
+        try:
+            media_url, playlist = hls_media_playlist(stream, input_url)
+            segments = parse_hls_segments(media_url, playlist)
+            if hls_segments_need_native_download(segments):
+                print("HLS playlist uses extensionless provider segments; skipping direct ffmpeg.")
+                code = download_hls_segments(stream, media_url, target, partial)
+                if code == 0:
+                    return 0
+                return code
+        except Exception as exc:
+            print(f"Native HLS preflight failed: {exc}", file=sys.stderr)
    attempts = [
        [
            *ffmpeg_base_command(stream, input_url),
@@ -468,7 +504,7 @@ def download_episode(stream, target):
            partial.unlink()
        except FileNotFoundError:
            pass
-        code = subprocess.call(cmd)
+        code = run_ffmpeg(cmd)
        if code == 0 and partial.exists() and partial.stat().st_size > 0:
            partial.replace(target)
            return 0
@@ -4305,6 +4305,18 @@ seg-3.ts

        self.assertEqual(provider_downloader.strip_png_header(payload), b"VIDEO")

+    def test_hls_segments_need_native_download_for_extensionless_provider_urls(self):
+        self.assertTrue(
+            provider_downloader.hls_segments_need_native_download(
+                [{"url": "https://p16-ad-sg.ibyteimg.com/obj/ad-site-i18n/202606025d0da6f666bd3d7b4e7ab879"}]
+            )
+        )
+        self.assertFalse(
+            provider_downloader.hls_segments_need_native_download(
+                [{"url": "https://cdn.example/video/segment-1.ts"}]
+            )
+        )
+

 if __name__ == "__main__":
    unittest.main()
@@ -1 +1 @@
 .51.6
 .51.7