Add HLS segment download fallback

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 0.51.6 - 2026-08-09
+
+- Added a StrawVerse-style HLS segment downloader fallback for providers whose playlists contain extensionless CDN segment URLs that ffmpeg rejects.
+- Added support for stripping short PNG wrappers from downloaded HLS segment payloads before concatenation.
+- Added AES-128 HLS segment decryption through `openssl` when encrypted media playlists are encountered.
+
 ## 0.51.5 - 2026-08-09
 
 - Fixed provider downloader episode matching so episode `10` no longer collides with episode `1` when resolving requested ranges.

README.md

@@ -71,7 +71,7 @@ The Docker image installs Python, Node.js, npm, and `ffmpeg`, then runs `npm ins
 
 ## Download Flow
 
-Kaizoku stores provider-backed show IDs as `provider:id`, for example `anikoto:some-show-slug`. Queue jobs resolve the episode source through `providers/bridge.js`, then `provider_downloader.py` downloads the media with `ffmpeg` into a staging directory. If a provider returns a master HLS playlist, Kaizoku selects the highest-bandwidth variant before starting `ffmpeg`. Each episode is written as a temporary `.mp4.part` file and moved into place only after the download succeeds, so failed fallback attempts do not leave broken final MP4 files behind. If the primary provider cannot list, resolve, or download a requested episode, Kaizoku searches the same title on the remaining providers and tries the matching episode there. Existing finalization code moves staged files into the configured library layout, preserving data already present in production download folders.
+Kaizoku stores provider-backed show IDs as `provider:id`, for example `anikoto:some-show-slug`. Queue jobs resolve the episode source through `providers/bridge.js`, then `provider_downloader.py` downloads the media with `ffmpeg` into a staging directory. If a provider returns a master HLS playlist, Kaizoku selects the highest-bandwidth variant before starting `ffmpeg`. When ffmpeg rejects provider HLS playlists with extensionless CDN segments, Kaizoku falls back to a StrawVerse-style segment downloader: it fetches the media playlist, downloads and concatenates segments itself, strips short PNG wrappers when present, then remuxes the local transport stream to MP4. Each episode is written as a temporary `.mp4.part` file and moved into place only after the download succeeds, so failed fallback attempts do not leave broken final MP4 files behind. If the primary provider cannot list, resolve, or download a requested episode, Kaizoku searches the same title on the remaining providers and tries the matching episode there. Existing finalization code moves staged files into the configured library layout, preserving data already present in production download folders.
 
 ## Data Safety
 

VERSION

@@ -1 +1 @@
-0.51.5
+0.51.6

provider_downloader.py

@@ -6,9 +6,10 @@ import argparse
 import json
 import os
 import re
+import shutil
 import subprocess
 import sys
-from urllib.parse import urljoin
+from urllib.parse import urljoin, urlparse
 import urllib.request
 from pathlib import Path
 
@@ -184,6 +185,38 @@ def ffmpeg_headers(headers):
     return "\r\n".join(pairs) + ("\r\n" if pairs else "")
 
 
+def request_headers(headers=None, referer=None):
+    merged = {
+        "User-Agent": "Mozilla/5.0",
+        "Accept": "*/*",
+        "Accept-Language": "en-US,en;q=0.9",
+        "Cache-Control": "no-cache",
+        "Pragma": "no-cache",
+    }
+    merged.update(headers or {})
+    if referer and not (merged.get("Referer") or merged.get("referer")):
+        merged["Referer"] = referer
+    final_referer = merged.get("Referer") or merged.get("referer")
+    if final_referer and not (merged.get("Origin") or merged.get("origin")):
+        try:
+            parsed = urlparse(final_referer)
+            if parsed.scheme and parsed.netloc:
+                merged["Origin"] = f"{parsed.scheme}://{parsed.netloc}"
+        except Exception:
+            pass
+    return merged
+
+
+def fetch_bytes(url, headers=None, timeout=30):
+    request = urllib.request.Request(url, headers=request_headers(headers))
+    with urllib.request.urlopen(request, timeout=timeout) as response:
+        return response.read()
+
+
+def fetch_text(url, headers=None, timeout=30):
+    return fetch_bytes(url, headers=headers, timeout=timeout).decode("utf-8", errors="replace")
+
+
 def best_hls_variant_url(master_url, playlist_text):
     variants = []
     pending_bandwidth = 0
@@ -208,12 +241,8 @@ def resolve_hls_input_url(stream):
     url = stream.get("url")
     if not stream.get("isM3U8") and ".m3u8" not in str(url):
         return url
-    headers = {"User-Agent": "Mozilla/5.0", **(stream.get("headers") or {})}
+    data = fetch_text(url, headers=stream.get("headers") or {})
-    request = urllib.request.Request(url, headers=headers)
+    if "#EXT-X-STREAM-INF" not in data:
-    with urllib.request.urlopen(request, timeout=30) as response:
-        content_type = response.headers.get("Content-Type", "")
-        data = response.read(2_000_000).decode("utf-8", errors="replace")
-    if "#EXT-X-STREAM-INF" not in data and "mpegurl" not in content_type.lower():
         return url
     return best_hls_variant_url(url, data)
 
@@ -229,6 +258,8 @@ def ffmpeg_base_command(stream, input_url):
         "file,http,https,tcp,tls,crypto",
         "-allowed_extensions",
         "ALL",
+        "-allowed_segment_extensions",
+        "ALL",
     ]
     headers = ffmpeg_headers(stream.get("headers") or {})
     if headers:
@@ -240,6 +271,140 @@ def ffmpeg_base_command(stream, input_url):
     return cmd
 
 
+def strip_png_header(data):
+    png_header = b"\x89PNG\r\n\x1a\n"
+    if not data.startswith(png_header):
+        return data
+    iend_offset = data.find(b"IEND")
+    if iend_offset != -1 and iend_offset < 1024:
+        return data[iend_offset + 8:]
+    return data
+
+
+def parse_hls_segments(playlist_url, playlist_text):
+    segments = []
+    key_url = None
+    iv = None
+    media_sequence = 1
+    segment_index = 0
+    for raw_line in str(playlist_text or "").splitlines():
+        line = raw_line.strip()
+        if not line:
+            continue
+        if line.startswith("#EXT-X-MEDIA-SEQUENCE:"):
+            try:
+                media_sequence = int(line.split(":", 1)[1])
+            except ValueError:
+                media_sequence = 1
+            continue
+        if line.startswith("#EXT-X-KEY:"):
+            attrs = {
+                match.group(1): match.group(2) if match.group(2) is not None else match.group(3)
+                for match in re.finditer(r'([A-Z0-9_-]+)=(?:"([^"]*)"|([^,]*))', line.split(":", 1)[1])
+            }
+            if str(attrs.get("METHOD") or "").upper() == "AES-128":
+                key_url = urljoin(playlist_url, attrs.get("URI") or "")
+                iv = attrs.get("IV")
+            else:
+                key_url = None
+                iv = None
+            continue
+        if line.startswith("#"):
+            continue
+        segment = {"url": urljoin(playlist_url, line)}
+        if key_url:
+            segment["key_url"] = key_url
+            segment["iv"] = iv or str(media_sequence + segment_index)
+        segments.append(segment)
+        segment_index += 1
+    return segments
+
+
+def decrypt_aes128_segment(data, key, iv_value):
+    openssl = shutil.which("openssl")
+    if not openssl:
+        raise RuntimeError("Encrypted HLS segment requires openssl, but openssl is not available.")
+    iv = bytearray(16)
+    text = str(iv_value or "").strip()
+    if text.lower().startswith("0x"):
+        raw = bytes.fromhex(text[2:])
+        iv[:len(raw[:16])] = raw[:16]
+    else:
+        iv[-4:] = int(text or "0").to_bytes(4, "big")
+    proc = subprocess.run(
+        [openssl, "enc", "-d", "-aes-128-cbc", "-K", key.hex(), "-iv", bytes(iv).hex()],
+        input=data,
+        capture_output=True,
+        check=False,
+    )
+    if proc.returncode != 0:
+        raise RuntimeError((proc.stderr or b"openssl AES-128 decrypt failed").decode("utf-8", errors="replace").strip())
+    return proc.stdout
+
+
+def download_hls_segments(stream, input_url, target, partial):
+    playlist = fetch_text(input_url, headers=stream.get("headers") or {})
+    if "#EXT-X-STREAM-INF" in playlist:
+        input_url = best_hls_variant_url(input_url, playlist)
+        playlist = fetch_text(input_url, headers=stream.get("headers") or {})
+    segments = parse_hls_segments(input_url, playlist)
+    if not segments:
+        return 1
+    ts_file = target.with_suffix(target.suffix + ".ts.part")
+    segment_dir = target.parent / f".segments_{target.stem}"
+    if segment_dir.exists():
+        shutil.rmtree(segment_dir)
+    segment_dir.mkdir(parents=True, exist_ok=True)
+    key_cache = {}
+    try:
+        print(f"Direct ffmpeg HLS failed; downloading {len(segments)} playlist segments...")
+        with ts_file.open("wb") as joined:
+            for index, segment in enumerate(segments, start=1):
+                data = fetch_bytes(segment["url"], headers=stream.get("headers") or {}, timeout=60)
+                data = strip_png_header(data)
+                key_url = segment.get("key_url")
+                if key_url:
+                    if key_url not in key_cache:
+                        key_cache[key_url] = fetch_bytes(key_url, headers=stream.get("headers") or {}, timeout=30)
+                    data = decrypt_aes128_segment(data, key_cache[key_url], segment.get("iv"))
+                if not data:
+                    raise RuntimeError(f"Segment {index} was empty.")
+                joined.write(data)
+                if index == 1 or index == len(segments) or index % 25 == 0:
+                    print(f"Downloaded segment {index}/{len(segments)}")
+        cmd = [
+            "ffmpeg",
+            "-hide_banner",
+            "-loglevel",
+            "info",
+            "-y",
+            "-f",
+            "mpegts",
+            "-i",
+            str(ts_file),
+            "-c",
+            "copy",
+            "-bsf:a",
+            "aac_adtstoasc",
+            "-movflags",
+            "+faststart",
+            "-f",
+            "mp4",
+            str(partial),
+        ]
+        code = subprocess.call(cmd)
+        if code == 0 and partial.exists() and partial.stat().st_size > 0:
+            partial.replace(target)
+            return 0
+        return code
+    finally:
+        try:
+            ts_file.unlink()
+        except FileNotFoundError:
+            pass
+        shutil.rmtree(segment_dir, ignore_errors=True)
+
+
 def download_subtitles(subtitles, output_base):
     saved = []
     english = [
@@ -250,7 +415,7 @@ def download_subtitles(subtitles, output_base):
         url = sub["url"]
         suffix = ".srt" if ".srt" in url.lower() else ".vtt"
         target = output_base.with_suffix(f".eng{suffix}" if index == 1 else f".eng-{index}{suffix}")
-        request = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0", "Referer": sub.get("referer") or ""})
+        request = urllib.request.Request(url, headers=request_headers(referer=sub.get("referer") or ""))
         with urllib.request.urlopen(request, timeout=30) as response:
             target.write_bytes(response.read())
         saved.append(target)
@@ -307,6 +472,14 @@ def download_episode(stream, target):
         if code == 0 and partial.exists() and partial.stat().st_size > 0:
             partial.replace(target)
             return 0
+    if stream.get("isM3U8") or ".m3u8" in str(input_url):
+        try:
+            code = download_hls_segments(stream, input_url, target, partial)
+            if code == 0:
+                return 0
+        except Exception as exc:
+            print(f"Segment downloader failed: {exc}", file=sys.stderr)
+            code = 1
     try:
         partial.unlink()
     except FileNotFoundError:

test_app.py

@@ -4275,6 +4275,36 @@ mid/index.m3u8
             "https://cdn.example/show/high/index.m3u8",
         )
 
+    def test_parse_hls_segments_resolves_plain_and_encrypted_segments(self):
+        playlist = """#EXTM3U
+#EXT-X-MEDIA-SEQUENCE:7
+#EXTINF:4.0,
+seg-1
+#EXT-X-KEY:METHOD=AES-128,URI="keys/key.bin"
+#EXTINF:4.0,
+https://cdn.example/raw-segment
+#EXTINF:4.0,
+seg-3.ts
+"""
+
+        segments = provider_downloader.parse_hls_segments("https://cdn.example/path/index.m3u8", playlist)
+
+        self.assertEqual(segments[0], {"url": "https://cdn.example/path/seg-1"})
+        self.assertEqual(
+            segments[1],
+            {
+                "url": "https://cdn.example/raw-segment",
+                "key_url": "https://cdn.example/path/keys/key.bin",
+                "iv": "8",
+            },
+        )
+        self.assertEqual(segments[2]["iv"], "9")
+
+    def test_strip_png_header_removes_short_wrapper(self):
+        payload = b"\x89PNG\r\n\x1a\njunkIENDabcdVIDEO"
+
+        self.assertEqual(provider_downloader.strip_png_header(payload), b"VIDEO")
+
 
 if __name__ == "__main__":
     unittest.main()