Fix external subtitle downloads and library sidecar finalization
This commit is contained in:
1 parent
2b08837e13
commit
d97801bf9d
7 files changed
+179
-18
No files matched your search
@@ -1,5 +1,12 @@
|
||||
# Changelog
|
||||
|
||||
## 0.53.13 - 2026-10-07
|
||||
|
||||
- Fixed external subtitle requests to use the media downloader's cookies, HTTP fallback, retries, and provider/track headers. Failed English tracks now fall through to another English track, and unavailable tracks are logged instead of labeling an unrelated language as English.
|
||||
- Preserved provider-level subtitle tracks when a resolved stream supplies an empty list, and forwarded subtitle-specific referers from the bridge.
|
||||
- Fixed subtitle language tags being parsed as episode numbers during incremental finalization. English subtitle sidecars now move with their video, retain language/flags and season/episode offsets, and share collision-safe filenames without replacing existing library files.
|
||||
- Checked bundled anime providers against upstream; no updates were available. External subtitles still depend on tracks exposed by the selected source.
|
||||
|
||||
## 0.53.12 - 2026-10-07
|
||||
|
||||
- Fixed cross-provider download fallback selecting an earlier season, another part, or a loosely related title when a requested episode was missing or failed. Fallback now requires a unique normalized title match that retains season, part, and cour labels, while accepting equivalent `Season 2` / `2nd Season` notation.
|
||||
|
||||
@@ -119,6 +119,8 @@ Kaizoku stores provider-backed show IDs as `provider:id`, for example `anikoto:s
|
||||
|
||||
Provider server aliases that resolve to the same URL and request headers are downloaded only once, avoiding duplicate CDN requests and unnecessary rate limiting.
|
||||
|
||||
When the selected source exposes English external subtitle tracks, Kaizoku saves the first successfully fetched track alongside the video as `.eng.vtt`, `.eng.srt`, `.eng.ass`, or `.eng.ssa`. Subtitle requests use provider/track headers, session cookies, retries, and the media HTTP fallback. Subtitles move into the library with each completed video and keep matching filenames after custom naming, season/episode offsets, or filename collisions. Subtitle failures do not fail an otherwise completed video; the queue log reports failed tracks or that no English external track was exposed. Burned-in subtitles cannot be saved as a separate file, and Kaizoku does not extract embedded or HLS-playlist-only subtitle tracks. The bundled AnimePahe adapter currently exposes no external subtitle tracks. These changes apply to new downloads and do not backfill subtitles for existing videos.
|
||||
|
||||
If the primary provider cannot list, resolve, or download a requested episode, Kaizoku searches the selected provider's title on the remaining providers and tries the matching episode there. A custom library name is used only for output naming. Fallback requires a unique normalized title match, retaining season, part, and cour labels; equivalent notation such as `Season 2` and `2nd Season` is accepted. Shared words or a matching title prefix are insufficient. Ambiguous results, different seasons, and titles that cannot be verified (including alternate translations) are skipped. If no matching source succeeds, the job fails while preserving completed episodes. This prevents unavailable later-season episodes from being silently replaced with season 1 episodes; it does not identify or repair previously downloaded incorrect files. Existing finalization code moves staged files into the configured library layout, preserving data already present in production download folders.
|
||||
|
||||
## Data Safety
|
||||
|
||||
+38
-6
@@ -55,6 +55,7 @@ PROVIDER_CHOICES = ("anikoto", "anineko", "pahe")
|
||||
DOWNLOAD_METHOD_CHOICES = ("kaizoku",)
|
||||
MEDIA_TYPE_CHOICES = {"tv", "movie"}
|
||||
VIDEO_FILE_SUFFIXES = frozenset({".mp4", ".mkv", ".avi", ".mov", ".webm", ".m4v", ".ts"})
|
||||
SUBTITLE_FILE_SUFFIXES = frozenset({".srt", ".vtt", ".ass", ".ssa", ".sub", ".idx"})
|
||||
WATCHLIST_CATEGORY_LABELS = {
|
||||
"watching": "Watching",
|
||||
"planned": "Planned",
|
||||
@@ -1133,11 +1134,20 @@ def apply_episode_offset(value, offset):
|
||||
return f"{int(normalized_offset, 10) + number - 1}{suffix}"
|
||||
|
||||
|
||||
def subtitle_filename_parts(path):
|
||||
"""Separate downloaded English subtitle tags from the media basename."""
|
||||
if path.suffix.lower() not in SUBTITLE_FILE_SUFFIXES:
|
||||
return path.stem, ""
|
||||
match = re.search(r"(?:\.(?:(?:en|eng)(?:-[a-z0-9]+)?|forced|sdh|cc|default))+$", path.stem, re.IGNORECASE)
|
||||
return (path.stem[:match.start()], match.group()) if match else (path.stem, "")
|
||||
|
||||
|
||||
def extract_episode_from_filename(path):
|
||||
match = re.search(r"\bS[0-9]+E([0-9][0-9A-Za-z.\-]*)\b", path.stem, re.IGNORECASE)
|
||||
stem, _tags = subtitle_filename_parts(path)
|
||||
match = re.search(r"\bS[0-9]+E([0-9][0-9A-Za-z.\-]*)\b", stem, re.IGNORECASE)
|
||||
if match:
|
||||
return match.group(1)
|
||||
match = re.search(r"\bEpisode\s+([0-9][0-9A-Za-z.\-]*)\s*$", path.stem)
|
||||
match = re.search(r"\bEpisode\s+([0-9][0-9A-Za-z.\-]*)\s*$", stem)
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
@@ -1204,9 +1214,15 @@ def finalize_library_files(job, episode=None, cleanup_staging=True, allow_empty=
|
||||
|
||||
moved = []
|
||||
fallback_episode = 1
|
||||
for source in files:
|
||||
# Move each video and its subtitles together so collision suffixes and
|
||||
# custom library names remain identical for players such as Jellyfin.
|
||||
processed = set()
|
||||
for source in sorted(files, key=lambda path: path.suffix.lower() not in VIDEO_FILE_SUFFIXES):
|
||||
if source in processed:
|
||||
continue
|
||||
_stem, tags = subtitle_filename_parts(source)
|
||||
if media_type == "movie":
|
||||
final_name = f"{library_name}{source.suffix}"
|
||||
final_name = f"{library_name}{tags}{source.suffix}"
|
||||
else:
|
||||
ep = None
|
||||
if ep is None:
|
||||
@@ -1215,10 +1231,26 @@ def finalize_library_files(job, episode=None, cleanup_staging=True, allow_empty=
|
||||
ep = str(fallback_episode)
|
||||
fallback_episode += 1
|
||||
ep = apply_episode_offset(ep, episode_offset)
|
||||
final_name = f"{library_name} - S{season}E{episode_label(ep)}{source.suffix}"
|
||||
destination = unique_destination(target_dir / final_name)
|
||||
final_name = f"{library_name} - S{season}E{episode_label(ep)}{tags}{source.suffix}"
|
||||
companions = []
|
||||
if source.suffix.lower() in VIDEO_FILE_SUFFIXES:
|
||||
companions = [path for path in files if path not in processed and path != source
|
||||
and path.parent == source.parent and path.suffix.lower() in SUBTITLE_FILE_SUFFIXES
|
||||
and subtitle_filename_parts(path)[0] == source.stem]
|
||||
base = target_dir / final_name
|
||||
destination = base
|
||||
counter = 2
|
||||
while destination.exists() or any(destination.with_name(destination.stem + subtitle_filename_parts(path)[1] + path.suffix).exists() for path in companions):
|
||||
destination = base.with_name(f"{base.stem} ({counter}){base.suffix}")
|
||||
counter += 1
|
||||
shutil.move(str(source), str(destination))
|
||||
moved.append(str(destination))
|
||||
processed.add(source)
|
||||
for companion in companions:
|
||||
subtitle_target = destination.with_name(destination.stem + subtitle_filename_parts(companion)[1] + companion.suffix)
|
||||
shutil.move(str(companion), str(subtitle_target))
|
||||
moved.append(str(subtitle_target))
|
||||
processed.add(companion)
|
||||
|
||||
if cleanup_staging:
|
||||
shutil.rmtree(staging_dir, ignore_errors=True)
|
||||
|
||||
+29
-10
@@ -712,20 +712,39 @@ def run_ffmpeg(cmd, timeout=180):
|
||||
return 124
|
||||
|
||||
|
||||
def download_subtitles(subtitles, output_base):
|
||||
def download_subtitles(subtitles, output_base, headers=None):
|
||||
saved = []
|
||||
english = [
|
||||
sub for sub in subtitles or []
|
||||
if sub.get("url") and re.search(r"eng|english", str(sub.get("lang") or ""), re.I)
|
||||
] or [sub for sub in subtitles or [] if sub.get("url")]
|
||||
for index, sub in enumerate(english[:1], start=1):
|
||||
if sub.get("url") and re.match(r"^(?:en|eng|english)(?:$|[\s_\-(])", str(sub.get("lang") or "").strip(), re.I)
|
||||
]
|
||||
if not english:
|
||||
print("Subtitle skipped: provider exposed no English external subtitle track.", flush=True)
|
||||
for sub in english:
|
||||
url = sub["url"]
|
||||
suffix = ".srt" if ".srt" in url.lower() else ".vtt"
|
||||
target = output_base.with_suffix(f".eng{suffix}" if index == 1 else f".eng-{index}{suffix}")
|
||||
request = urllib.request.Request(url, headers=request_headers(referer=sub.get("referer") or ""))
|
||||
with urllib.request.urlopen(request, timeout=30) as response:
|
||||
target.write_bytes(response.read())
|
||||
suffix = Path(urlparse(url).path).suffix.lower()
|
||||
if suffix not in {".srt", ".vtt", ".ass", ".ssa"}:
|
||||
suffix = ".vtt"
|
||||
target = output_base.with_suffix(f".eng{suffix}")
|
||||
subtitle_headers = dict(headers or {})
|
||||
subtitle_headers.update(sub.get("headers") or {})
|
||||
if sub.get("referer"):
|
||||
subtitle_headers = {key: value for key, value in subtitle_headers.items() if key.lower() != "referer"}
|
||||
subtitle_headers["Referer"] = sub["referer"]
|
||||
partial = target.with_suffix(target.suffix + ".part")
|
||||
try:
|
||||
data = fetch_bytes_with_retries(url, headers=subtitle_headers, retry_label="Subtitle")
|
||||
if not data.strip():
|
||||
raise RuntimeError("Empty subtitle response")
|
||||
partial.write_bytes(data)
|
||||
partial.replace(target)
|
||||
except Exception as exc:
|
||||
print(f"Subtitle track failed: {concise_error(exc)}", file=sys.stderr, flush=True)
|
||||
continue
|
||||
finally:
|
||||
partial.unlink(missing_ok=True)
|
||||
saved.append(target)
|
||||
break
|
||||
return saved
|
||||
|
||||
|
||||
@@ -987,7 +1006,7 @@ def main():
|
||||
print(str(last_error), file=sys.stderr)
|
||||
continue
|
||||
try:
|
||||
for subtitle_path in download_subtitles(stream.get("subtitles") or [], target):
|
||||
for subtitle_path in download_subtitles(stream.get("subtitles") or [], target, headers=stream.get("headers")):
|
||||
print(f"Saved subtitle: {subtitle_path.name}")
|
||||
except Exception as exc:
|
||||
print(f"Subtitle download skipped: {exc}")
|
||||
|
||||
+7
-1
@@ -130,7 +130,13 @@ async function resolveAll(provider, episodeId, mode, quality) {
|
||||
quality: resolved.quality || source.quality || source.name || "auto",
|
||||
type: sourceType,
|
||||
headers,
|
||||
subtitles: resolved.subtitles || sourcesPayload.subtitles || [],
|
||||
subtitles: (resolved.subtitles?.length ? resolved.subtitles : sourcesPayload.subtitles || []).map((subtitle) => {
|
||||
let referer = subtitle.referer || subtitle.headers?.Referer || subtitle.headers?.referer || "";
|
||||
try {
|
||||
referer ||= dynamicReferers.get(new URL(subtitle.url).hostname) || "";
|
||||
} catch (_) {}
|
||||
return { ...subtitle, referer: referer || headers.Referer || fallbackReferer };
|
||||
}),
|
||||
server: source.name || resolved.name || resolved.quality || "server",
|
||||
});
|
||||
}
|
||||
|
||||
+95
@@ -4999,6 +4999,101 @@ class AniDbCacheTests(unittest.TestCase):
|
||||
APP.ANIDB_TITLES_CACHE.update(original_cache)
|
||||
|
||||
|
||||
class SubtitleTests(unittest.TestCase):
|
||||
def test_bridge_preserves_payload_tracks_and_subtitle_referer(self):
|
||||
script = r"""
|
||||
const Module = require("module");
|
||||
const originalLoad = Module._load;
|
||||
const provider = {
|
||||
fetchEpisodeSources: async () => ({
|
||||
sources: [{ url: "https://video.example/video.mp4", type: "sub", subtitles: [], headers: { Referer: "https://video-player.example/" } }],
|
||||
subtitles: [
|
||||
{ url: "https://subs.example/en.vtt", lang: "en" },
|
||||
{ url: "https://subs.example/alternate.vtt", lang: "eng", headers: { Referer: "https://explicit.example/" } },
|
||||
],
|
||||
}),
|
||||
};
|
||||
Module._load = function(request, parent, isMain) {
|
||||
if (String(request).endsWith("/providers/extensions/Anime/anikoto.js")) {
|
||||
global.setDynamicReferer("subs.example", "https://subtitle-player.example/");
|
||||
return provider;
|
||||
}
|
||||
return originalLoad.call(this, request, parent, isMain);
|
||||
};
|
||||
process.argv = ["node", "bridge.js", "resolve-all", "anikoto", "ep1", "sub", "best"];
|
||||
require("./providers/bridge.js");
|
||||
"""
|
||||
proc = subprocess.run(["node", "-e", script], cwd=ROOT, text=True, capture_output=True)
|
||||
self.assertEqual(proc.returncode, 0, proc.stderr)
|
||||
tracks = json.loads(proc.stdout)["sources"][0]["subtitles"]
|
||||
self.assertEqual(tracks[0], {"url": "https://subs.example/en.vtt", "lang": "en", "referer": "https://subtitle-player.example/"})
|
||||
self.assertEqual(tracks[1]["referer"], "https://explicit.example/")
|
||||
|
||||
def test_external_subtitle_fetch_uses_media_client_and_tries_next_english_track(self):
|
||||
tracks = [
|
||||
{"url": "https://cdn.example/bn.vtt", "lang": "Bengali"},
|
||||
{"url": "https://cdn.example/broken.vtt", "lang": "en-US"},
|
||||
{"url": "https://cdn.example/en.srt?token=a", "lang": "English", "referer": "https://subtitle-player.example/", "headers": {"X-Token": "subtitle"}},
|
||||
]
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
output = Path(directory) / "Show - S01E06.mp4"
|
||||
with mock.patch.object(provider_downloader, "fetch_bytes_with_retries", side_effect=[RuntimeError("blocked"), b"subtitle"]) as fetch:
|
||||
saved = provider_downloader.download_subtitles(tracks, output, headers={"Referer": "https://video-player.example/", "Origin": "https://player.example"})
|
||||
self.assertEqual(saved, [output.with_suffix(".eng.srt")])
|
||||
self.assertEqual(saved[0].read_bytes(), b"subtitle")
|
||||
self.assertEqual(fetch.call_count, 2)
|
||||
self.assertEqual(fetch.call_args_list[0].args[0], tracks[1]["url"])
|
||||
self.assertEqual(fetch.call_args.kwargs["headers"], {"Referer": "https://subtitle-player.example/", "Origin": "https://player.example", "X-Token": "subtitle"})
|
||||
self.assertFalse(list(Path(directory).glob("*.part")))
|
||||
|
||||
def test_no_english_track_does_not_mislabel_another_language(self):
|
||||
with mock.patch.object(provider_downloader, "fetch_bytes_with_retries") as fetch, mock.patch("sys.stdout", new_callable=io.StringIO) as log:
|
||||
saved = provider_downloader.download_subtitles([{"url": "https://cdn.example/fr.vtt", "lang": "French"}], Path("Show.mp4"))
|
||||
self.assertEqual(saved, [])
|
||||
fetch.assert_not_called()
|
||||
self.assertIn("no English external subtitle track", log.getvalue())
|
||||
|
||||
def test_failed_subtitle_fetch_preserves_existing_file(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
output = Path(directory) / "Show.mp4"
|
||||
existing = output.with_suffix(".eng.vtt")
|
||||
existing.write_bytes(b"existing subtitle")
|
||||
with mock.patch.object(provider_downloader, "fetch_bytes_with_retries", return_value=b""):
|
||||
self.assertEqual(provider_downloader.download_subtitles([{"url": "https://cdn.example/en.vtt", "lang": "eng"}], output), [])
|
||||
self.assertEqual(existing.read_bytes(), b"existing subtitle")
|
||||
|
||||
def test_finalizer_keeps_subtitles_with_video_and_preserves_existing_library_files(self):
|
||||
for media_type in ("tv", "movie"):
|
||||
for collision in ("none", "video", "subtitle"):
|
||||
with self.subTest(media_type=media_type, collision=collision), tempfile.TemporaryDirectory() as directory:
|
||||
job = app_support.build_job(
|
||||
{"query": "Provider Show", "anime_name": "Library Show", "media_type": media_type,
|
||||
"season": "2", "episode_offset": "13", "episodes": "6-7", "download_dir": directory},
|
||||
{"mode": "sub", "quality": "best", "download_dir": directory},
|
||||
)
|
||||
staging = app_support.job_staging_dir(job)
|
||||
staging.mkdir(parents=True)
|
||||
(staging / "Provider Show - S01E06.mp4").write_bytes(b"video")
|
||||
(staging / "Provider Show - S01E06.eng.vtt").write_bytes(b"english")
|
||||
(staging / "Provider Show - S01E06.en.forced.srt").write_bytes(b"forced")
|
||||
pending = staging / "Provider Show - S01E07.eng.vtt"
|
||||
pending.write_bytes(b"pending")
|
||||
output = app_support.job_output_dir(job)
|
||||
output.mkdir(parents=True)
|
||||
stem = "Library Show - S02E18" if media_type == "tv" else "Library Show"
|
||||
existing = output / (stem + (".mp4" if collision == "video" else ".eng.vtt"))
|
||||
if collision != "none":
|
||||
existing.write_bytes(b"existing")
|
||||
moved = app_support.finalize_library_files(job, episode=6, cleanup_staging=False)
|
||||
expected_stem = stem + (" (2)" if collision != "none" else "")
|
||||
self.assertEqual({Path(path).name for path in moved}, {expected_stem + suffix for suffix in (".mp4", ".eng.vtt", ".en.forced.srt")})
|
||||
self.assertEqual((output / (expected_stem + ".eng.vtt")).read_bytes(), b"english")
|
||||
self.assertTrue(pending.exists())
|
||||
if collision != "none":
|
||||
self.assertEqual(existing.read_bytes(), b"existing")
|
||||
app_support.shutil.rmtree(staging)
|
||||
|
||||
|
||||
class ProviderDownloaderTests(unittest.TestCase):
|
||||
def test_fallback_requires_unique_matching_season_and_part(self):
|
||||
title = "Trapped in a Dating Sim The World of Otome Games Is Tough for Mob"
|
||||
|
||||
Reference in new issue
Block a user