diff --git a/CHANGELOG.md b/CHANGELOG.md index 90d3543..e6f4819 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 0.51.6 - 2026-08-09 + +- Added a StrawVerse-style HLS segment downloader fallback for providers whose playlists contain extensionless CDN segment URLs that ffmpeg rejects. +- Added support for stripping short PNG wrappers from downloaded HLS segment payloads before concatenation. +- Added AES-128 HLS segment decryption through `openssl` when encrypted media playlists are encountered. + ## 0.51.5 - 2026-08-09 - Fixed provider downloader episode matching so episode `10` no longer collides with episode `1` when resolving requested ranges. diff --git a/README.md b/README.md index e3cab58..3df9d55 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ The Docker image installs Python, Node.js, npm, and `ffmpeg`, then runs `npm ins ## Download Flow -Kaizoku stores provider-backed show IDs as `provider:id`, for example `anikoto:some-show-slug`. Queue jobs resolve the episode source through `providers/bridge.js`, then `provider_downloader.py` downloads the media with `ffmpeg` into a staging directory. If a provider returns a master HLS playlist, Kaizoku selects the highest-bandwidth variant before starting `ffmpeg`. Each episode is written as a temporary `.mp4.part` file and moved into place only after the download succeeds, so failed fallback attempts do not leave broken final MP4 files behind. If the primary provider cannot list, resolve, or download a requested episode, Kaizoku searches the same title on the remaining providers and tries the matching episode there. Existing finalization code moves staged files into the configured library layout, preserving data already present in production download folders. +Kaizoku stores provider-backed show IDs as `provider:id`, for example `anikoto:some-show-slug`. Queue jobs resolve the episode source through `providers/bridge.js`, then `provider_downloader.py` downloads the media with `ffmpeg` into a staging directory. If a provider returns a master HLS playlist, Kaizoku selects the highest-bandwidth variant before starting `ffmpeg`. When ffmpeg rejects provider HLS playlists with extensionless CDN segments, Kaizoku falls back to a StrawVerse-style segment downloader: it fetches the media playlist, downloads and concatenates segments itself, strips short PNG wrappers when present, then remuxes the local transport stream to MP4. Each episode is written as a temporary `.mp4.part` file and moved into place only after the download succeeds, so failed fallback attempts do not leave broken final MP4 files behind. If the primary provider cannot list, resolve, or download a requested episode, Kaizoku searches the same title on the remaining providers and tries the matching episode there. Existing finalization code moves staged files into the configured library layout, preserving data already present in production download folders. ## Data Safety diff --git a/VERSION b/VERSION index 74aa6c8..6114cad 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.51.5 +0.51.6 diff --git a/provider_downloader.py b/provider_downloader.py index bd4073f..ca3ea0c 100755 --- a/provider_downloader.py +++ b/provider_downloader.py @@ -6,9 +6,10 @@ import argparse import json import os import re +import shutil import subprocess import sys -from urllib.parse import urljoin +from urllib.parse import urljoin, urlparse import urllib.request from pathlib import Path @@ -184,6 +185,38 @@ def ffmpeg_headers(headers): return "\r\n".join(pairs) + ("\r\n" if pairs else "") +def request_headers(headers=None, referer=None): + merged = { + "User-Agent": "Mozilla/5.0", + "Accept": "*/*", + "Accept-Language": "en-US,en;q=0.9", + "Cache-Control": "no-cache", + "Pragma": "no-cache", + } + merged.update(headers or {}) + if referer and not (merged.get("Referer") or merged.get("referer")): + merged["Referer"] = referer + final_referer = merged.get("Referer") or merged.get("referer") + if final_referer and not (merged.get("Origin") or merged.get("origin")): + try: + parsed = urlparse(final_referer) + if parsed.scheme and parsed.netloc: + merged["Origin"] = f"{parsed.scheme}://{parsed.netloc}" + except Exception: + pass + return merged + + +def fetch_bytes(url, headers=None, timeout=30): + request = urllib.request.Request(url, headers=request_headers(headers)) + with urllib.request.urlopen(request, timeout=timeout) as response: + return response.read() + + +def fetch_text(url, headers=None, timeout=30): + return fetch_bytes(url, headers=headers, timeout=timeout).decode("utf-8", errors="replace") + + def best_hls_variant_url(master_url, playlist_text): variants = [] pending_bandwidth = 0 @@ -208,12 +241,8 @@ def resolve_hls_input_url(stream): url = stream.get("url") if not stream.get("isM3U8") and ".m3u8" not in str(url): return url - headers = {"User-Agent": "Mozilla/5.0", **(stream.get("headers") or {})} - request = urllib.request.Request(url, headers=headers) - with urllib.request.urlopen(request, timeout=30) as response: - content_type = response.headers.get("Content-Type", "") - data = response.read(2_000_000).decode("utf-8", errors="replace") - if "#EXT-X-STREAM-INF" not in data and "mpegurl" not in content_type.lower(): + data = fetch_text(url, headers=stream.get("headers") or {}) + if "#EXT-X-STREAM-INF" not in data: return url return best_hls_variant_url(url, data) @@ -229,6 +258,8 @@ def ffmpeg_base_command(stream, input_url): "file,http,https,tcp,tls,crypto", "-allowed_extensions", "ALL", + "-allowed_segment_extensions", + "ALL", ] headers = ffmpeg_headers(stream.get("headers") or {}) if headers: @@ -240,6 +271,140 @@ def ffmpeg_base_command(stream, input_url): return cmd +def strip_png_header(data): + png_header = b"\x89PNG\r\n\x1a\n" + if not data.startswith(png_header): + return data + iend_offset = data.find(b"IEND") + if iend_offset != -1 and iend_offset < 1024: + return data[iend_offset + 8:] + return data + + +def parse_hls_segments(playlist_url, playlist_text): + segments = [] + key_url = None + iv = None + media_sequence = 1 + segment_index = 0 + for raw_line in str(playlist_text or "").splitlines(): + line = raw_line.strip() + if not line: + continue + if line.startswith("#EXT-X-MEDIA-SEQUENCE:"): + try: + media_sequence = int(line.split(":", 1)[1]) + except ValueError: + media_sequence = 1 + continue + if line.startswith("#EXT-X-KEY:"): + attrs = { + match.group(1): match.group(2) if match.group(2) is not None else match.group(3) + for match in re.finditer(r'([A-Z0-9_-]+)=(?:"([^"]*)"|([^,]*))', line.split(":", 1)[1]) + } + if str(attrs.get("METHOD") or "").upper() == "AES-128": + key_url = urljoin(playlist_url, attrs.get("URI") or "") + iv = attrs.get("IV") + else: + key_url = None + iv = None + continue + if line.startswith("#"): + continue + segment = {"url": urljoin(playlist_url, line)} + if key_url: + segment["key_url"] = key_url + segment["iv"] = iv or str(media_sequence + segment_index) + segments.append(segment) + segment_index += 1 + return segments + + +def decrypt_aes128_segment(data, key, iv_value): + openssl = shutil.which("openssl") + if not openssl: + raise RuntimeError("Encrypted HLS segment requires openssl, but openssl is not available.") + iv = bytearray(16) + text = str(iv_value or "").strip() + if text.lower().startswith("0x"): + raw = bytes.fromhex(text[2:]) + iv[:len(raw[:16])] = raw[:16] + else: + iv[-4:] = int(text or "0").to_bytes(4, "big") + proc = subprocess.run( + [openssl, "enc", "-d", "-aes-128-cbc", "-K", key.hex(), "-iv", bytes(iv).hex()], + input=data, + capture_output=True, + check=False, + ) + if proc.returncode != 0: + raise RuntimeError((proc.stderr or b"openssl AES-128 decrypt failed").decode("utf-8", errors="replace").strip()) + return proc.stdout + + +def download_hls_segments(stream, input_url, target, partial): + playlist = fetch_text(input_url, headers=stream.get("headers") or {}) + if "#EXT-X-STREAM-INF" in playlist: + input_url = best_hls_variant_url(input_url, playlist) + playlist = fetch_text(input_url, headers=stream.get("headers") or {}) + segments = parse_hls_segments(input_url, playlist) + if not segments: + return 1 + ts_file = target.with_suffix(target.suffix + ".ts.part") + segment_dir = target.parent / f".segments_{target.stem}" + if segment_dir.exists(): + shutil.rmtree(segment_dir) + segment_dir.mkdir(parents=True, exist_ok=True) + key_cache = {} + try: + print(f"Direct ffmpeg HLS failed; downloading {len(segments)} playlist segments...") + with ts_file.open("wb") as joined: + for index, segment in enumerate(segments, start=1): + data = fetch_bytes(segment["url"], headers=stream.get("headers") or {}, timeout=60) + data = strip_png_header(data) + key_url = segment.get("key_url") + if key_url: + if key_url not in key_cache: + key_cache[key_url] = fetch_bytes(key_url, headers=stream.get("headers") or {}, timeout=30) + data = decrypt_aes128_segment(data, key_cache[key_url], segment.get("iv")) + if not data: + raise RuntimeError(f"Segment {index} was empty.") + joined.write(data) + if index == 1 or index == len(segments) or index % 25 == 0: + print(f"Downloaded segment {index}/{len(segments)}") + cmd = [ + "ffmpeg", + "-hide_banner", + "-loglevel", + "info", + "-y", + "-f", + "mpegts", + "-i", + str(ts_file), + "-c", + "copy", + "-bsf:a", + "aac_adtstoasc", + "-movflags", + "+faststart", + "-f", + "mp4", + str(partial), + ] + code = subprocess.call(cmd) + if code == 0 and partial.exists() and partial.stat().st_size > 0: + partial.replace(target) + return 0 + return code + finally: + try: + ts_file.unlink() + except FileNotFoundError: + pass + shutil.rmtree(segment_dir, ignore_errors=True) + + def download_subtitles(subtitles, output_base): saved = [] english = [ @@ -250,7 +415,7 @@ def download_subtitles(subtitles, output_base): url = sub["url"] suffix = ".srt" if ".srt" in url.lower() else ".vtt" target = output_base.with_suffix(f".eng{suffix}" if index == 1 else f".eng-{index}{suffix}") - request = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0", "Referer": sub.get("referer") or ""}) + request = urllib.request.Request(url, headers=request_headers(referer=sub.get("referer") or "")) with urllib.request.urlopen(request, timeout=30) as response: target.write_bytes(response.read()) saved.append(target) @@ -307,6 +472,14 @@ def download_episode(stream, target): if code == 0 and partial.exists() and partial.stat().st_size > 0: partial.replace(target) return 0 + if stream.get("isM3U8") or ".m3u8" in str(input_url): + try: + code = download_hls_segments(stream, input_url, target, partial) + if code == 0: + return 0 + except Exception as exc: + print(f"Segment downloader failed: {exc}", file=sys.stderr) + code = 1 try: partial.unlink() except FileNotFoundError: diff --git a/test_app.py b/test_app.py index b640e93..92247bc 100644 --- a/test_app.py +++ b/test_app.py @@ -4275,6 +4275,36 @@ mid/index.m3u8 "https://cdn.example/show/high/index.m3u8", ) + def test_parse_hls_segments_resolves_plain_and_encrypted_segments(self): + playlist = """#EXTM3U +#EXT-X-MEDIA-SEQUENCE:7 +#EXTINF:4.0, +seg-1 +#EXT-X-KEY:METHOD=AES-128,URI="keys/key.bin" +#EXTINF:4.0, +https://cdn.example/raw-segment +#EXTINF:4.0, +seg-3.ts +""" + + segments = provider_downloader.parse_hls_segments("https://cdn.example/path/index.m3u8", playlist) + + self.assertEqual(segments[0], {"url": "https://cdn.example/path/seg-1"}) + self.assertEqual( + segments[1], + { + "url": "https://cdn.example/raw-segment", + "key_url": "https://cdn.example/path/keys/key.bin", + "iv": "8", + }, + ) + self.assertEqual(segments[2]["iv"], "9") + + def test_strip_png_header_removes_short_wrapper(self): + payload = b"\x89PNG\r\n\x1a\njunkIENDabcdVIDEO" + + self.assertEqual(provider_downloader.strip_png_header(payload), b"VIDEO") + if __name__ == "__main__": unittest.main()