Dymas/kaizoku
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add HLS segment download fallback

Browse Source
This commit is contained in:
Codex
2026-08-09 12:40:11 +02:00
parent 54d559d73c
commit 3018e7aa66
5 changed files with 219 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+6
View File
@@ -1,5 +1,11 @@
# Changelog
## 0.51.6 - 2026-08-09
- Added a StrawVerse-style HLS segment downloader fallback for providers whose playlists contain extensionless CDN segment URLs that ffmpeg rejects.
- Added support for stripping short PNG wrappers from downloaded HLS segment payloads before concatenation.
- Added AES-128 HLS segment decryption through `openssl` when encrypted media playlists are encountered.
## 0.51.5 - 2026-08-09
- Fixed provider downloader episode matching so episode `10` no longer collides with episode `1` when resolving requested ranges.
README.md
+1 -1
View File
@@ -71,7 +71,7 @@ The Docker image installs Python, Node.js, npm, and `ffmpeg`, then runs `npm ins
## Download Flow
Kaizoku stores provider-backed show IDs as `provider:id`, for example `anikoto:some-show-slug`. Queue jobs resolve the episode source through `providers/bridge.js`, then `provider_downloader.py` downloads the media with `ffmpeg` into a staging directory. If a provider returns a master HLS playlist, Kaizoku selects the highest-bandwidth variant before starting `ffmpeg`. Each episode is written as a temporary `.mp4.part` file and moved into place only after the download succeeds, so failed fallback attempts do not leave broken final MP4 files behind. If the primary provider cannot list, resolve, or download a requested episode, Kaizoku searches the same title on the remaining providers and tries the matching episode there. Existing finalization code moves staged files into the configured library layout, preserving data already present in production download folders.
Kaizoku stores provider-backed show IDs as `provider:id`, for example `anikoto:some-show-slug`. Queue jobs resolve the episode source through `providers/bridge.js`, then `provider_downloader.py` downloads the media with `ffmpeg` into a staging directory. If a provider returns a master HLS playlist, Kaizoku selects the highest-bandwidth variant before starting `ffmpeg`. When ffmpeg rejects provider HLS playlists with extensionless CDN segments, Kaizoku falls back to a StrawVerse-style segment downloader: it fetches the media playlist, downloads and concatenates segments itself, strips short PNG wrappers when present, then remuxes the local transport stream to MP4. Each episode is written as a temporary `.mp4.part` file and moved into place only after the download succeeds, so failed fallback attempts do not leave broken final MP4 files behind. If the primary provider cannot list, resolve, or download a requested episode, Kaizoku searches the same title on the remaining providers and tries the matching episode there. Existing finalization code moves staged files into the configured library layout, preserving data already present in production download folders.
## Data Safety
VERSION
+1 -1
View File
@@ -1 +1 @@
0.51.5
0.51.6
provider_downloader.py
+181 -8
View File
@@ -6,9 +6,10 @@ import argparse
import json
import os
import re
import shutil
import subprocess
import sys
from urllib.parse import urljoin
from urllib.parse import urljoin, urlparse
import urllib.request
from pathlib import Path
@@ -184,6 +185,38 @@ def ffmpeg_headers(headers):
return "\r\n".join(pairs) + ("\r\n" if pairs else "")
def request_headers(headers=None, referer=None):
merged = {
"User-Agent": "Mozilla/5.0",
"Accept": "*/*",
"Accept-Language": "en-US,en;q=0.9",
"Cache-Control": "no-cache",
"Pragma": "no-cache",
}
merged.update(headers or {})
if referer and not (merged.get("Referer") or merged.get("referer")):
merged["Referer"] = referer
final_referer = merged.get("Referer") or merged.get("referer")
if final_referer and not (merged.get("Origin") or merged.get("origin")):
try:
parsed = urlparse(final_referer)
if parsed.scheme and parsed.netloc:
merged["Origin"] = f"{parsed.scheme}://{parsed.netloc}"
except Exception:
pass
return merged
def fetch_bytes(url, headers=None, timeout=30):
request = urllib.request.Request(url, headers=request_headers(headers))
with urllib.request.urlopen(request, timeout=timeout) as response:
return response.read()
def fetch_text(url, headers=None, timeout=30):
return fetch_bytes(url, headers=headers, timeout=timeout).decode("utf-8", errors="replace")
def best_hls_variant_url(master_url, playlist_text):
variants = []
pending_bandwidth = 0
@@ -208,12 +241,8 @@ def resolve_hls_input_url(stream):
url = stream.get("url")
if not stream.get("isM3U8") and ".m3u8" not in str(url):
return url
headers = {"User-Agent": "Mozilla/5.0", **(stream.get("headers") or {})}
request = urllib.request.Request(url, headers=headers)
with urllib.request.urlopen(request, timeout=30) as response:
content_type = response.headers.get("Content-Type", "")
data = response.read(2_000_000).decode("utf-8", errors="replace")
if "#EXT-X-STREAM-INF" not in data and "mpegurl" not in content_type.lower():
data = fetch_text(url, headers=stream.get("headers") or {})
if "#EXT-X-STREAM-INF" not in data:
return url
return best_hls_variant_url(url, data)
@@ -229,6 +258,8 @@ def ffmpeg_base_command(stream, input_url):
"file,http,https,tcp,tls,crypto",
"-allowed_extensions",
"ALL",
"-allowed_segment_extensions",
"ALL",
]
headers = ffmpeg_headers(stream.get("headers") or {})
if headers:
@@ -240,6 +271,140 @@ def ffmpeg_base_command(stream, input_url):
return cmd
def strip_png_header(data):
png_header = b"\x89PNG\r\n\x1a\n"
if not data.startswith(png_header):
return data
iend_offset = data.find(b"IEND")
if iend_offset != -1 and iend_offset < 1024:
return data[iend_offset + 8:]
return data
def parse_hls_segments(playlist_url, playlist_text):
segments = []
key_url = None
iv = None
media_sequence = 1
segment_index = 0
for raw_line in str(playlist_text or "").splitlines():
line = raw_line.strip()
if not line:
continue
if line.startswith("#EXT-X-MEDIA-SEQUENCE:"):
try:
media_sequence = int(line.split(":", 1)[1])
except ValueError:
media_sequence = 1
continue
if line.startswith("#EXT-X-KEY:"):
attrs = {
match.group(1): match.group(2) if match.group(2) is not None else match.group(3)
for match in re.finditer(r'([A-Z0-9_-]+)=(?:"([^"]*)"|([^,]*))', line.split(":", 1)[1])
}
if str(attrs.get("METHOD") or "").upper() == "AES-128":
key_url = urljoin(playlist_url, attrs.get("URI") or "")
iv = attrs.get("IV")
else:
key_url = None
iv = None
continue
if line.startswith("#"):
continue
segment = {"url": urljoin(playlist_url, line)}
if key_url:
segment["key_url"] = key_url
segment["iv"] = iv or str(media_sequence + segment_index)
segments.append(segment)
segment_index += 1
return segments
def decrypt_aes128_segment(data, key, iv_value):
openssl = shutil.which("openssl")
if not openssl:
raise RuntimeError("Encrypted HLS segment requires openssl, but openssl is not available.")
iv = bytearray(16)
text = str(iv_value or "").strip()
if text.lower().startswith("0x"):
raw = bytes.fromhex(text[2:])
iv[:len(raw[:16])] = raw[:16]
else:
iv[-4:] = int(text or "0").to_bytes(4, "big")
proc = subprocess.run(
[openssl, "enc", "-d", "-aes-128-cbc", "-K", key.hex(), "-iv", bytes(iv).hex()],
input=data,
capture_output=True,
check=False,
)
if proc.returncode != 0:
raise RuntimeError((proc.stderr or b"openssl AES-128 decrypt failed").decode("utf-8", errors="replace").strip())
return proc.stdout
def download_hls_segments(stream, input_url, target, partial):
playlist = fetch_text(input_url, headers=stream.get("headers") or {})
if "#EXT-X-STREAM-INF" in playlist:
input_url = best_hls_variant_url(input_url, playlist)
playlist = fetch_text(input_url, headers=stream.get("headers") or {})
segments = parse_hls_segments(input_url, playlist)
if not segments:
return 1
ts_file = target.with_suffix(target.suffix + ".ts.part")
segment_dir = target.parent / f".segments_{target.stem}"
if segment_dir.exists():
shutil.rmtree(segment_dir)
segment_dir.mkdir(parents=True, exist_ok=True)
key_cache = {}
try:
print(f"Direct ffmpeg HLS failed; downloading {len(segments)} playlist segments...")
with ts_file.open("wb") as joined:
for index, segment in enumerate(segments, start=1):
data = fetch_bytes(segment["url"], headers=stream.get("headers") or {}, timeout=60)
data = strip_png_header(data)
key_url = segment.get("key_url")
if key_url:
if key_url not in key_cache:
key_cache[key_url] = fetch_bytes(key_url, headers=stream.get("headers") or {}, timeout=30)
data = decrypt_aes128_segment(data, key_cache[key_url], segment.get("iv"))
if not data:
raise RuntimeError(f"Segment {index} was empty.")
joined.write(data)
if index == 1 or index == len(segments) or index % 25 == 0:
print(f"Downloaded segment {index}/{len(segments)}")
cmd = [
"ffmpeg",
"-hide_banner",
"-loglevel",
"info",
"-y",
"-f",
"mpegts",
"-i",
str(ts_file),
"-c",
"copy",
"-bsf:a",
"aac_adtstoasc",
"-movflags",
"+faststart",
"-f",
"mp4",
str(partial),
]
code = subprocess.call(cmd)
if code == 0 and partial.exists() and partial.stat().st_size > 0:
partial.replace(target)
return 0
return code
finally:
try:
ts_file.unlink()
except FileNotFoundError:
pass
shutil.rmtree(segment_dir, ignore_errors=True)
def download_subtitles(subtitles, output_base):
saved = []
english = [
@@ -250,7 +415,7 @@ def download_subtitles(subtitles, output_base):
url = sub["url"]
suffix = ".srt" if ".srt" in url.lower() else ".vtt"
target = output_base.with_suffix(f".eng{suffix}" if index == 1 else f".eng-{index}{suffix}")
request = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0", "Referer": sub.get("referer") or ""})
request = urllib.request.Request(url, headers=request_headers(referer=sub.get("referer") or ""))
with urllib.request.urlopen(request, timeout=30) as response:
target.write_bytes(response.read())
saved.append(target)
@@ -307,6 +472,14 @@ def download_episode(stream, target):
if code == 0 and partial.exists() and partial.stat().st_size > 0:
partial.replace(target)
return 0
if stream.get("isM3U8") or ".m3u8" in str(input_url):
try:
code = download_hls_segments(stream, input_url, target, partial)
if code == 0:
return 0
except Exception as exc:
print(f"Segment downloader failed: {exc}", file=sys.stderr)
code = 1
try:
partial.unlink()
except FileNotFoundError:
test_app.py
+30
View File
@@ -4275,6 +4275,36 @@ mid/index.m3u8
"https://cdn.example/show/high/index.m3u8",
)
def test_parse_hls_segments_resolves_plain_and_encrypted_segments(self):
playlist = """#EXTM3U
#EXT-X-MEDIA-SEQUENCE:7
#EXTINF:4.0,
seg-1
#EXT-X-KEY:METHOD=AES-128,URI="keys/key.bin"
#EXTINF:4.0,
https://cdn.example/raw-segment
#EXTINF:4.0,
seg-3.ts
"""
segments = provider_downloader.parse_hls_segments("https://cdn.example/path/index.m3u8", playlist)
self.assertEqual(segments[0], {"url": "https://cdn.example/path/seg-1"})
self.assertEqual(
segments[1],
{
"url": "https://cdn.example/raw-segment",
"key_url": "https://cdn.example/path/keys/key.bin",
"iv": "8",
},
)
self.assertEqual(segments[2]["iv"], "9")
def test_strip_png_header_removes_short_wrapper(self):
payload = b"\x89PNG\r\n\x1a\njunkIENDabcdVIDEO"
self.assertEqual(provider_downloader.strip_png_header(payload), b"VIDEO")
if __name__ == "__main__":
unittest.main()