Dymas/ani-cli-web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix thumbnail backoff and watchlist fallback matching

Browse Source
This commit is contained in:
Dymas
2026-05-16 10:25:24 +02:00
parent 6b1e0ce7f0
commit d12afb1284
8 changed files with 438 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app.py
+87 -102
View File
@@ -9,7 +9,6 @@ import shutil
import sqlite3
import threading
import time
import unicodedata
import urllib.error
import urllib.request
import xml.etree.ElementTree as ET
@@ -45,6 +44,8 @@ from app_support import (
WATCHLIST_CATEGORY_CHOICES,
WATCHLIST_CATEGORY_LABELS,
WATCHLIST_JSON_PATH,
WATCHLIST_REFRESH_REQUEST_TIMEOUT_SECONDS,
WORKER_SHUTDOWN_TIMEOUT_SECONDS,
background_workers_enabled,
client_address_is_local,
debug_log,
@@ -57,6 +58,14 @@ from app_support import (
)
from http_handler import HandlerContext, build_handler_class, server_host, server_port
from queue_jobs import DownloadQueue, WatchlistRefreshJobs
from title_matching import (
base_title_variants,
normalize_title_key,
normalize_identity_title_key,
title_lookup_queries,
title_match_variants,
)
from watchlist_identity import resolve_job_watchlist_identity as resolve_job_watchlist_identity_impl
from web_templates import CONFIG_HTML, INDEX_HTML, PAGE_LINKS, WATCHLIST_HTML, render_page_links, render_sidebar_brand
RUNTIME_LOCK = threading.RLock()
@@ -75,7 +84,7 @@ DOWNLOAD_QUEUE = None
WATCHLIST_REFRESH = None
def graph_request(query, variables):
def graph_request(query, variables, timeout=25):
payload = json.dumps({"variables": variables, "query": query}).encode("utf-8")
request = urllib.request.Request(
f"{ALLANIME_API}/api",
@@ -88,7 +97,7 @@ def graph_request(query, variables):
method="POST",
)
try:
with urllib.request.urlopen(request, timeout=25) as response:
with urllib.request.urlopen(request, timeout=timeout) as response:
raw = response.read().decode("utf-8")
except urllib.error.URLError as exc:
raise RuntimeError(f"Could not reach AllAnime: {exc}") from exc
@@ -201,64 +210,7 @@ def decode_image_data_url(data_url):
return content_type, content
def normalize_title_key(value):
text = unicodedata.normalize("NFKD", str(value or ""))
text = "".join(char for char in text if not unicodedata.combining(char))
text = text.lower()
text = re.sub(r"\((19|20)\d{2}\)", " ", text)
text = re.sub(r"[^a-z0-9]+", " ", text)
return re.sub(r"\s+", " ", text).strip()
def base_title_variants(value):
text = str(value or "").strip()
variants = []
if text:
variants.append(text)
patterns = [
r"\s+\((19|20)\d{2}\)\s*$",
r"\s+season\s+\d+\s*$",
r"\s+part\s+\d+\s*$",
r"\s+cour\s+\d+\s*$",
r"\s+\d+(st|nd|rd|th)\s+season\s*$",
]
changed = True
current = text
while current and changed:
changed = False
for pattern in patterns:
trimmed = re.sub(pattern, "", current, flags=re.IGNORECASE).strip(" -:")
if trimmed and trimmed != current:
current = trimmed
if current not in variants:
variants.append(current)
changed = True
return variants
def title_match_variants(value):
variants = set()
for raw_value in base_title_variants(value):
normalized = normalize_title_key(raw_value)
if normalized:
variants.add(normalized)
stripped = re.sub(r"\b(19|20)\d{2}\b", " ", normalized)
stripped = re.sub(r"\s+", " ", stripped).strip()
if stripped:
variants.add(stripped)
return variants
def title_lookup_queries(value):
queries = []
for raw_value in base_title_variants(value):
text = str(raw_value or "").strip()
if text and text not in queries:
queries.append(text)
return queries
def animeschedule_request(path, params=None):
def animeschedule_request(path, params=None, timeout=25):
params = params or {}
query = "&".join(f"{quote(str(key), safe='')}={quote(str(value), safe='')}" for key, value in params.items())
url = f"{ANIMESCHEDULE_API}{path}"
@@ -274,7 +226,7 @@ def animeschedule_request(path, params=None):
method="GET",
)
try:
with urllib.request.urlopen(request, timeout=25) as response:
with urllib.request.urlopen(request, timeout=timeout) as response:
raw = response.read().decode("utf-8")
debug_log(
"animeschedule.response",
@@ -474,8 +426,8 @@ def find_animeschedule_match(title):
return best
def fetch_animeschedule_anime_by_route(route, fallback_title=""):
payload = animeschedule_request(f"/anime/{quote(str(route).strip(), safe='')}")
def fetch_animeschedule_anime_by_route(route, fallback_title="", timeout=25):
payload = animeschedule_request(f"/anime/{quote(str(route).strip(), safe='')}", timeout=timeout)
entry = normalize_animeschedule_entry(payload)
if not entry:
raise RuntimeError("AnimeSchedule anime response was missing required fields")
@@ -484,16 +436,16 @@ def fetch_animeschedule_anime_by_route(route, fallback_title=""):
return entry
def resolve_animeschedule_anime(title, route=""):
def resolve_animeschedule_anime(title, route="", timeout=25):
stored_route = str(route or "").strip()
if stored_route:
try:
return fetch_animeschedule_anime_by_route(stored_route, fallback_title=title)
return fetch_animeschedule_anime_by_route(stored_route, fallback_title=title, timeout=timeout)
except Exception:
debug_log("animeschedule.route_refresh_failed", route=stored_route, title=title)
best = find_animeschedule_match(title)
return fetch_animeschedule_anime_by_route(best["route"], fallback_title=best["title"])
return fetch_animeschedule_anime_by_route(best["route"], fallback_title=best["title"], timeout=timeout)
def fetch_animeschedule_cover(title):
@@ -864,9 +816,9 @@ def numeric_episode_key(value):
return key
def fetch_show_episode_snapshot(show_id):
def fetch_show_episode_snapshot(show_id, timeout=25):
query = "query ($showId: String!) { show(_id: $showId) { _id name availableEpisodesDetail } }"
data = graph_request(query, {"showId": show_id})
data = graph_request(query, {"showId": show_id}, timeout=timeout)
show = data.get("show") or {}
detail = show.get("availableEpisodesDetail") or {}
episodes = {}
@@ -948,9 +900,12 @@ class WatchlistStore:
self.refresh_wakeup.set()
worker = self.refresh_worker
if wait and worker is not None and worker.is_alive():
worker.join(timeout=2.5)
worker.join(timeout=WORKER_SHUTDOWN_TIMEOUT_SECONDS)
return worker is None or not worker.is_alive()
def _refresh_request_timeout(self):
return WATCHLIST_REFRESH_REQUEST_TIMEOUT_SECONDS
def _next_pending_refresh(self):
with self.lock:
if not self.refresh_pending:
@@ -1256,7 +1211,6 @@ class WatchlistStore:
def list(self, page=1, per_page=30, category=None):
page = max(1, int(page or 1))
per_page = min(60, max(1, int(per_page or 30)))
offset = (page - 1) * per_page
active_category = normalize_watchlist_category(category) if category else None
with self.lock, self._connect() as conn:
overall_total = conn.execute("SELECT COUNT(*) AS count FROM watchlist").fetchone()["count"]
@@ -1279,6 +1233,9 @@ class WatchlistStore:
"SELECT category, COUNT(*) AS count FROM watchlist GROUP BY category"
).fetchall()
queued_total = conn.execute("SELECT COUNT(*) AS count FROM watchlist WHERE status = 'queued'").fetchone()["count"]
pages = max(1, (total + per_page - 1) // per_page)
page = min(page, pages)
offset = (page - 1) * per_page
if active_category:
rows = conn.execute(
"SELECT * FROM watchlist WHERE category = ? ORDER BY title COLLATE NOCASE ASC LIMIT ? OFFSET ?",
@@ -1290,14 +1247,13 @@ class WatchlistStore:
(per_page, offset),
).fetchall()
items = [self._row_to_item(row) for row in rows]
pages = max(1, (total + per_page - 1) // per_page)
category_counts = {key: 0 for key in WATCHLIST_CATEGORY_CHOICES}
for row in category_rows:
category_counts[normalize_watchlist_category(row["category"])] = int(row["count"] or 0)
return {
"items": items,
"total": total,
"page": min(page, pages),
"page": page,
"pages": pages,
"per_page": per_page,
"active_category": active_category,
@@ -1368,17 +1324,26 @@ class WatchlistStore:
existing = self.get(show_id)
normalized_show_id = existing["show_id"]
now = now_iso()
request_timeout = self._refresh_request_timeout()
try:
snapshot = fetch_show_episode_snapshot(normalized_show_id)
snapshot = fetch_show_episode_snapshot(normalized_show_id, timeout=request_timeout)
if self.refresh_stop_event.is_set():
return None
if not self.has_show(normalized_show_id):
return None
sub_episodes = snapshot["episodes"]["sub"]
dub_episodes = snapshot["episodes"]["dub"]
schedule = None
try:
schedule = resolve_animeschedule_anime(snapshot["title"] or existing["title"], route=existing.get("animeschedule_route"))
schedule = resolve_animeschedule_anime(
snapshot["title"] or existing["title"],
route=existing.get("animeschedule_route"),
timeout=request_timeout,
)
except Exception:
schedule = None
if self.refresh_stop_event.is_set():
return None
if not self.has_show(normalized_show_id):
return None
expected_count = parse_nonnegative_int((schedule or {}).get("episodes")) or int(existing.get("expected_count") or 0)
@@ -1405,6 +1370,8 @@ class WatchlistStore:
normalized_show_id,
)
except Exception as exc:
if self.refresh_stop_event.is_set():
return None
if not self.has_show(normalized_show_id):
return None
payload = (
@@ -1425,6 +1392,8 @@ class WatchlistStore:
)
with self.lock, self._connect() as conn:
if self.refresh_stop_event.is_set():
return None
cursor = conn.execute(
"""
UPDATE watchlist
@@ -1448,6 +1417,8 @@ class WatchlistStore:
)
if cursor.rowcount < 1:
return None
if self.refresh_stop_event.is_set():
return None
if include_thumbnail and self.has_show(normalized_show_id):
try:
self.ensure_thumbnail(normalized_show_id)
@@ -1653,7 +1624,6 @@ class WatchlistStore:
except Exception as exc:
debug_log("thumbnail.ensure.failed", show_id=show_id, error=exc)
cached_name = None
checked_at = item.get("thumbnail_checked_at")
with self.lock, self._connect() as conn:
conn.execute(
@@ -1760,39 +1730,47 @@ def upload_watchlist_thumbnail(payload):
def resolve_job_watchlist_identity(job, config=None):
show_id = str(job.get("show_id") or "").strip()
title = str(job.get("title") or job.get("anime_name") or "").strip()
if show_id:
return show_id, title
try:
index = max(1, int(job.get("result_index") or 1))
except (TypeError, ValueError):
return "", title
try:
active_config = config or CONFIG or DEFAULT_CONFIG
results = search_anime(job.get("query") or title, job.get("mode") or active_config["mode"])
except Exception:
return "", title
if index > len(results):
return "", title
result = results[index - 1]
return str(result.get("id") or "").strip(), str(result.get("title") or title).strip()
active_config = config or CONFIG or DEFAULT_CONFIG
return resolve_job_watchlist_identity_impl(
job,
search_fn=search_anime,
normalize_title_key_fn=normalize_identity_title_key,
base_title_variants_fn=base_title_variants,
default_mode=active_config["mode"],
)
def sync_downloaded_job_to_watchlist(job):
ensure_runtime()
show_id, title = resolve_job_watchlist_identity(job)
show_id, title, reason = resolve_job_watchlist_identity_impl(
job,
search_fn=search_anime,
normalize_title_key_fn=normalize_identity_title_key,
base_title_variants_fn=base_title_variants,
default_mode=(CONFIG or DEFAULT_CONFIG)["mode"],
return_reason=True,
)
if not show_id:
raise ValueError("Could not resolve a watchlist show_id for the completed download.")
debug_log("watchlist.sync.resolve_failed", job=job, reason=reason)
raise ValueError(f"Could not resolve a watchlist show_id for the completed download: {reason}.")
return WATCHLIST.mark_downloaded(show_id, title=title)
def prepare_download_job(job):
config = ensure_runtime()["config"]
show_id, title = resolve_job_watchlist_identity(job, config=config)
show_id, title, reason = resolve_job_watchlist_identity_impl(
job,
search_fn=search_anime,
normalize_title_key_fn=normalize_identity_title_key,
base_title_variants_fn=base_title_variants,
default_mode=config["mode"],
return_reason=True,
)
if show_id:
job["show_id"] = show_id
if title:
job["title"] = title
if not show_id and reason:
debug_log("watchlist.prepare_job.resolve_skipped", job=job, reason=reason)
return job
@@ -1878,23 +1856,29 @@ def shutdown_runtime(wait=False, cancel_active_downloads=None):
services = (WATCHLIST, DOWNLOAD_QUEUE, WATCHLIST_REFRESH)
watchlist, download_queue, watchlist_refresh = services
should_cancel_downloads = bool(wait) if cancel_active_downloads is None else bool(cancel_active_downloads)
stopped = True
if watchlist is not None:
watchlist.shutdown(wait=wait)
stopped = watchlist.shutdown(wait=wait) and stopped
if download_queue is not None:
download_queue.shutdown(wait=wait, cancel_active=should_cancel_downloads)
stopped = download_queue.shutdown(wait=wait, cancel_active=should_cancel_downloads) and stopped
if watchlist_refresh is not None:
watchlist_refresh.shutdown(wait=wait)
return True
stopped = watchlist_refresh.shutdown(wait=wait) and stopped
return stopped
def reset_runtime(wait=False, cancel_active_downloads=None):
global CONFIG, WATCHLIST, DOWNLOAD_QUEUE, WATCHLIST_REFRESH
shutdown_runtime(wait=wait, cancel_active_downloads=cancel_active_downloads)
stopped = shutdown_runtime(wait=wait, cancel_active_downloads=cancel_active_downloads)
if not stopped:
if wait:
raise RuntimeError("Runtime shutdown did not complete cleanly.")
return False
with RUNTIME_LOCK:
CONFIG = None
WATCHLIST = None
DOWNLOAD_QUEUE = None
WATCHLIST_REFRESH = None
return True
Handler = build_handler_class(
@@ -1944,7 +1928,8 @@ def main():
except KeyboardInterrupt:
print("\nShutting down server.")
finally:
shutdown_runtime(wait=True, cancel_active_downloads=True)
if not shutdown_runtime(wait=True, cancel_active_downloads=True):
print("Warning: background workers did not stop cleanly.")
server.server_close()