Add HLS segment download fallback
This commit is contained in:
Codex
+181
-8
@@ -6,9 +6,10 @@ import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from urllib.parse import urljoin
|
||||
from urllib.parse import urljoin, urlparse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
@@ -184,6 +185,38 @@ def ffmpeg_headers(headers):
|
||||
return "\r\n".join(pairs) + ("\r\n" if pairs else "")
|
||||
|
||||
|
||||
def request_headers(headers=None, referer=None):
|
||||
merged = {
|
||||
"User-Agent": "Mozilla/5.0",
|
||||
"Accept": "*/*",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"Cache-Control": "no-cache",
|
||||
"Pragma": "no-cache",
|
||||
}
|
||||
merged.update(headers or {})
|
||||
if referer and not (merged.get("Referer") or merged.get("referer")):
|
||||
merged["Referer"] = referer
|
||||
final_referer = merged.get("Referer") or merged.get("referer")
|
||||
if final_referer and not (merged.get("Origin") or merged.get("origin")):
|
||||
try:
|
||||
parsed = urlparse(final_referer)
|
||||
if parsed.scheme and parsed.netloc:
|
||||
merged["Origin"] = f"{parsed.scheme}://{parsed.netloc}"
|
||||
except Exception:
|
||||
pass
|
||||
return merged
|
||||
|
||||
|
||||
def fetch_bytes(url, headers=None, timeout=30):
|
||||
request = urllib.request.Request(url, headers=request_headers(headers))
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
return response.read()
|
||||
|
||||
|
||||
def fetch_text(url, headers=None, timeout=30):
|
||||
return fetch_bytes(url, headers=headers, timeout=timeout).decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def best_hls_variant_url(master_url, playlist_text):
|
||||
variants = []
|
||||
pending_bandwidth = 0
|
||||
@@ -208,12 +241,8 @@ def resolve_hls_input_url(stream):
|
||||
url = stream.get("url")
|
||||
if not stream.get("isM3U8") and ".m3u8" not in str(url):
|
||||
return url
|
||||
headers = {"User-Agent": "Mozilla/5.0", **(stream.get("headers") or {})}
|
||||
request = urllib.request.Request(url, headers=headers)
|
||||
with urllib.request.urlopen(request, timeout=30) as response:
|
||||
content_type = response.headers.get("Content-Type", "")
|
||||
data = response.read(2_000_000).decode("utf-8", errors="replace")
|
||||
if "#EXT-X-STREAM-INF" not in data and "mpegurl" not in content_type.lower():
|
||||
data = fetch_text(url, headers=stream.get("headers") or {})
|
||||
if "#EXT-X-STREAM-INF" not in data:
|
||||
return url
|
||||
return best_hls_variant_url(url, data)
|
||||
|
||||
@@ -229,6 +258,8 @@ def ffmpeg_base_command(stream, input_url):
|
||||
"file,http,https,tcp,tls,crypto",
|
||||
"-allowed_extensions",
|
||||
"ALL",
|
||||
"-allowed_segment_extensions",
|
||||
"ALL",
|
||||
]
|
||||
headers = ffmpeg_headers(stream.get("headers") or {})
|
||||
if headers:
|
||||
@@ -240,6 +271,140 @@ def ffmpeg_base_command(stream, input_url):
|
||||
return cmd
|
||||
|
||||
|
||||
def strip_png_header(data):
|
||||
png_header = b"\x89PNG\r\n\x1a\n"
|
||||
if not data.startswith(png_header):
|
||||
return data
|
||||
iend_offset = data.find(b"IEND")
|
||||
if iend_offset != -1 and iend_offset < 1024:
|
||||
return data[iend_offset + 8:]
|
||||
return data
|
||||
|
||||
|
||||
def parse_hls_segments(playlist_url, playlist_text):
|
||||
segments = []
|
||||
key_url = None
|
||||
iv = None
|
||||
media_sequence = 1
|
||||
segment_index = 0
|
||||
for raw_line in str(playlist_text or "").splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("#EXT-X-MEDIA-SEQUENCE:"):
|
||||
try:
|
||||
media_sequence = int(line.split(":", 1)[1])
|
||||
except ValueError:
|
||||
media_sequence = 1
|
||||
continue
|
||||
if line.startswith("#EXT-X-KEY:"):
|
||||
attrs = {
|
||||
match.group(1): match.group(2) if match.group(2) is not None else match.group(3)
|
||||
for match in re.finditer(r'([A-Z0-9_-]+)=(?:"([^"]*)"|([^,]*))', line.split(":", 1)[1])
|
||||
}
|
||||
if str(attrs.get("METHOD") or "").upper() == "AES-128":
|
||||
key_url = urljoin(playlist_url, attrs.get("URI") or "")
|
||||
iv = attrs.get("IV")
|
||||
else:
|
||||
key_url = None
|
||||
iv = None
|
||||
continue
|
||||
if line.startswith("#"):
|
||||
continue
|
||||
segment = {"url": urljoin(playlist_url, line)}
|
||||
if key_url:
|
||||
segment["key_url"] = key_url
|
||||
segment["iv"] = iv or str(media_sequence + segment_index)
|
||||
segments.append(segment)
|
||||
segment_index += 1
|
||||
return segments
|
||||
|
||||
|
||||
def decrypt_aes128_segment(data, key, iv_value):
|
||||
openssl = shutil.which("openssl")
|
||||
if not openssl:
|
||||
raise RuntimeError("Encrypted HLS segment requires openssl, but openssl is not available.")
|
||||
iv = bytearray(16)
|
||||
text = str(iv_value or "").strip()
|
||||
if text.lower().startswith("0x"):
|
||||
raw = bytes.fromhex(text[2:])
|
||||
iv[:len(raw[:16])] = raw[:16]
|
||||
else:
|
||||
iv[-4:] = int(text or "0").to_bytes(4, "big")
|
||||
proc = subprocess.run(
|
||||
[openssl, "enc", "-d", "-aes-128-cbc", "-K", key.hex(), "-iv", bytes(iv).hex()],
|
||||
input=data,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError((proc.stderr or b"openssl AES-128 decrypt failed").decode("utf-8", errors="replace").strip())
|
||||
return proc.stdout
|
||||
|
||||
|
||||
def download_hls_segments(stream, input_url, target, partial):
|
||||
playlist = fetch_text(input_url, headers=stream.get("headers") or {})
|
||||
if "#EXT-X-STREAM-INF" in playlist:
|
||||
input_url = best_hls_variant_url(input_url, playlist)
|
||||
playlist = fetch_text(input_url, headers=stream.get("headers") or {})
|
||||
segments = parse_hls_segments(input_url, playlist)
|
||||
if not segments:
|
||||
return 1
|
||||
ts_file = target.with_suffix(target.suffix + ".ts.part")
|
||||
segment_dir = target.parent / f".segments_{target.stem}"
|
||||
if segment_dir.exists():
|
||||
shutil.rmtree(segment_dir)
|
||||
segment_dir.mkdir(parents=True, exist_ok=True)
|
||||
key_cache = {}
|
||||
try:
|
||||
print(f"Direct ffmpeg HLS failed; downloading {len(segments)} playlist segments...")
|
||||
with ts_file.open("wb") as joined:
|
||||
for index, segment in enumerate(segments, start=1):
|
||||
data = fetch_bytes(segment["url"], headers=stream.get("headers") or {}, timeout=60)
|
||||
data = strip_png_header(data)
|
||||
key_url = segment.get("key_url")
|
||||
if key_url:
|
||||
if key_url not in key_cache:
|
||||
key_cache[key_url] = fetch_bytes(key_url, headers=stream.get("headers") or {}, timeout=30)
|
||||
data = decrypt_aes128_segment(data, key_cache[key_url], segment.get("iv"))
|
||||
if not data:
|
||||
raise RuntimeError(f"Segment {index} was empty.")
|
||||
joined.write(data)
|
||||
if index == 1 or index == len(segments) or index % 25 == 0:
|
||||
print(f"Downloaded segment {index}/{len(segments)}")
|
||||
cmd = [
|
||||
"ffmpeg",
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"info",
|
||||
"-y",
|
||||
"-f",
|
||||
"mpegts",
|
||||
"-i",
|
||||
str(ts_file),
|
||||
"-c",
|
||||
"copy",
|
||||
"-bsf:a",
|
||||
"aac_adtstoasc",
|
||||
"-movflags",
|
||||
"+faststart",
|
||||
"-f",
|
||||
"mp4",
|
||||
str(partial),
|
||||
]
|
||||
code = subprocess.call(cmd)
|
||||
if code == 0 and partial.exists() and partial.stat().st_size > 0:
|
||||
partial.replace(target)
|
||||
return 0
|
||||
return code
|
||||
finally:
|
||||
try:
|
||||
ts_file.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
shutil.rmtree(segment_dir, ignore_errors=True)
|
||||
|
||||
|
||||
def download_subtitles(subtitles, output_base):
|
||||
saved = []
|
||||
english = [
|
||||
@@ -250,7 +415,7 @@ def download_subtitles(subtitles, output_base):
|
||||
url = sub["url"]
|
||||
suffix = ".srt" if ".srt" in url.lower() else ".vtt"
|
||||
target = output_base.with_suffix(f".eng{suffix}" if index == 1 else f".eng-{index}{suffix}")
|
||||
request = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0", "Referer": sub.get("referer") or ""})
|
||||
request = urllib.request.Request(url, headers=request_headers(referer=sub.get("referer") or ""))
|
||||
with urllib.request.urlopen(request, timeout=30) as response:
|
||||
target.write_bytes(response.read())
|
||||
saved.append(target)
|
||||
@@ -307,6 +472,14 @@ def download_episode(stream, target):
|
||||
if code == 0 and partial.exists() and partial.stat().st_size > 0:
|
||||
partial.replace(target)
|
||||
return 0
|
||||
if stream.get("isM3U8") or ".m3u8" in str(input_url):
|
||||
try:
|
||||
code = download_hls_segments(stream, input_url, target, partial)
|
||||
if code == 0:
|
||||
return 0
|
||||
except Exception as exc:
|
||||
print(f"Segment downloader failed: {exc}", file=sys.stderr)
|
||||
code = 1
|
||||
try:
|
||||
partial.unlink()
|
||||
except FileNotFoundError:
|
||||
|
||||
Reference in New Issue
Block a user