diff --git a/CHANGELOG.md b/CHANGELOG.md index 5abdbf5..7b2a577 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,105 @@ # Changelog +## 0.26.0 - 2026-05-15 + +- Made controlled runtime shutdown stricter by canceling active download workers during blocking teardown paths such as app exit and runtime reset, reducing the chance of orphaned background work outliving the server. +- Hardened watchlist thumbnail warm-up batches so shows removed mid-request are skipped cleanly instead of bubbling a stale lookup back as a route error. +- Added shared serial browser polling helpers and switched the Search and Watchlist pages to use them, preventing overlapping queue and watchlist poll requests from stacking up when the server responds slowly. +- Improved background refresh diagnostics by logging per-show worker failures and preserving the last bulk-refresh item error in refresh-job state. +- Expanded regression coverage for deterministic download-queue shutdown, removed-entry thumbnail warm-up handling, shutdown-on-exit behavior, and shared serial polling usage in page templates. + +## 0.25.0 - 2026-05-15 + +- Stopped the watchlist page from repeatedly rewarming the same unresolved thumbnails during silent queued-refresh polling by caching per-show warm-up attempts in the browser until thumbnail state changes. +- Made background `Refresh All` lighter and faster by skipping inline thumbnail downloads during bulk refreshes and refreshing multiple watchlist shows in parallel. +- Taught per-show refreshes to bail out quietly when a show disappears mid-refresh, avoiding stale watchlist writes and extra thumbnail work after removal. +- Woke waiting thumbnail resolvers immediately when a show is removed, so stale in-flight cover waits no longer sit on the timeout path. +- Expanded regression coverage for thumbnail warm-up deduplication, bulk refresh thumbnail skipping, thumbnail wait cleanup, and removed-entry handling during background refresh jobs. + +## 0.24.0 - 2026-05-15 + +- Moved single-item watchlist refresh requests onto the existing per-show background refresh queue instead of doing episode, AnimeSchedule, and thumbnail work synchronously in the request handler. +- Added queued-item counts to watchlist listing responses and taught the watchlist page to silently refresh itself while queued per-show refresh work is still draining. +- Cleared removed shows out of the in-memory per-show refresh queue and in-flight tracking sets so deleted entries no longer consume stale queued refresh work. +- Expanded regression coverage for queued watchlist counts, queued single-item refreshes, queued refresh removal cleanup, and watchlist-page queued polling behavior. + +## 0.23.0 - 2026-05-15 + +- Fixed the per-show watchlist refresh queue so a show already being refreshed is no longer re-enqueued while its refresh is still in flight. +- Added regression coverage for in-flight watchlist refresh deduplication and cleanup of the in-flight marker after refresh completion. + +## 0.22.0 - 2026-05-15 + +- Added explicit runtime shutdown and reset hooks so shared globals and background workers can be torn down cleanly between in-process runs and tests. +- Fixed deferred worker startup to keep respecting `ANI_CLI_WEB_DISABLE_WORKER`, preventing runtime-backed handler calls from silently enabling background workers in worker-disabled environments. +- Split download-queue persistence into structured SQLite columns plus a separate log column, leaving JSON payload storage only for extra fields instead of rewriting the whole job record on routine state changes. +- Reduced server-side exception noise by only printing full tracebacks in debug mode. +- Centralized the shared browser `api()` fetch helper used by the Search, Config, and Watchlist pages. +- Expanded regression coverage for runtime reset, worker-disable behavior, structured queue persistence, and the quieter exception path. + +## 0.21.0 - 2026-05-15 + +- Persisted the per-show watchlist refresh queue across restarts by rebuilding pending refresh work from watchlist rows still marked `queued` during runtime startup. +- Preserved queued refresh ordering during restart recovery so older queued watchlist items resume before newer ones. +- Added regression coverage for queued watchlist refresh restoration after restart. + +## 0.20.0 - 2026-05-15 + +- Replaced the HTTP handler's old mutable string-key service locator with an explicit configured handler context, making route dependencies clearer and surfacing wiring issues at startup time instead of during later requests. +- Moved watchlist add and successful-download sync off the hot request path by queueing per-show background refresh work instead of doing episode, AnimeSchedule, and thumbnail fetches synchronously before the request returns. +- Ensured runtime-backed HTTP requests start background workers when allowed, while still respecting `ANI_CLI_WEB_DISABLE_WORKER` in tests and intentionally worker-free runs. +- Expanded regression coverage for the new non-blocking watchlist add/download-sync flow. + +## 0.19.0 - 2026-05-15 + +- Guarded Search-page queue polling against stale out-of-order responses so an older refresh result no longer overwrites a newer queue state. +- Added `get_cached_anidb_titles()` and switched AniDB lookup to reuse the cached parsed title list directly instead of rebuilding a duplicate list on every fallback lookup. +- Expanded regression coverage for the queue freshness guard and AniDB cache reuse path. + +## 0.18.0 - 2026-05-15 + +- Removed the eager runtime bootstrap from `main()`, so server startup no longer forces migrations, DB setup, worker startup, and config writes before the first runtime-backed request. +- Batched queue log persistence so running downloads no longer rewrite the entire job payload into SQLite on every single output line. +- Added regression coverage for lazy startup and queue log batching. + +## 0.17.0 - 2026-05-15 + +- Tightened the remote-access guard so loopback reverse-proxy traffic with forwarded external client IPs no longer bypasses the non-local token requirement. +- Stopped mutating the shared runtime config dict in place; queue creation now reads a stable config snapshot and config saves replace the runtime config atomically. +- Reduced cheap-route startup overhead by serving page shells, `/api/version`, `/api/dependencies`, and `/api/config` without forcing a full runtime bootstrap first. +- Added request-order guards on the Search and Watchlist pages so slower stale responses no longer overwrite newer searches, result selections, or category/page switches. +- Added AniDB title lookup caching and indexed candidate narrowing, while preserving a full-scan fallback when narrowed candidates miss. +- Expanded regression coverage for forwarded-client access checks and config snapshot behavior. + +## 0.16.0 - 2026-05-15 + +- Hardened non-local access so `ANI_CLI_WEB_ALLOW_REMOTE=1` now also requires `ANI_CLI_WEB_REMOTE_TOKEN`, accepted through `Authorization: Bearer ...` or `X-AniCli-Web-Token`, before remote clients can use the web UI or JSON APIs. +- Removed import-time state migration side effects from `app_support.py`; legacy state migration now runs during runtime bootstrap instead of plain module import. +- Narrowed watchlist persistence updates so refreshes, category changes, and download-sync writes stop rewriting the full row and are less likely to clobber unrelated fields. +- Resolve missing queue `show_id` values when jobs are queued, making completed-download watchlist sync stable even if upstream search ordering changes later. +- Continued the backend split by moving queue workers into `queue_jobs.py`, HTTP routing into `http_handler.py`, and page bodies into `search_page.py`, `config_page.py`, and `watchlist_page.py`, while keeping `app.py` and `web_templates.py` as stable aggregation entrypoints. +- Hardened the shared frontend `api()` helpers so page scripts handle non-JSON error responses more gracefully and only send JSON content headers when a request body is present. +- Expanded regression coverage for the new remote token gate and queue identity enrichment path. + +## 0.15.0 - 2026-05-15 + +- Started the backend/module split by moving shared runtime, state-path, config, and queue/file helper logic out of `app.py` into `app_support.py`. +- Split shared page-shell helpers out of `web_templates.py` into `template_helpers.py`, so page templates no longer carry their own navigation/branding helpers inline. +- Kept `app.py` and `web_templates.py` as stable aggregation points so the current tests and imports continue to work while the rest of the modularization can happen in smaller follow-up steps. + +## 0.14.0 - 2026-05-15 + +- Added watchlist categories with separate `Watching`, `Planned`, `Finished`, and `Dropped` tabs, category-aware add forms, and in-card category editing for existing entries. +- Migrated existing watchlist rows into the `Watching` category by default so older tracked shows keep working without manual cleanup. +- Added finished-state check badges: finished anime that were downloaded now show a green check, while anime moved into `Finished` manually show a gray check. +- Automatically sync successful downloads into the watchlist by flagging tracked shows as downloaded and creating a new `Finished` entry when the show was not already tracked. + +## 0.13.0 - 2026-05-15 + +- Added AnimeSchedule-backed watchlist metadata refresh so tracked shows now store the expected total episode count and upstream airing status alongside the live AllAnime sub and dub availability counts. +- Marked watchlist episode pills with `current / total` progress when the total is known, and highlight `Sub` or `Dub` in green when that release track has fully finished airing and is ready for download. +- Added compact watchlist status tags such as `Finished`, `Sub ready`, and `Dub ready`, plus regression coverage for the new completion-state persistence and messaging. + ## 0.12.0 - 2026-05-13 - Deferred runtime initialization until the server starts or a runtime-backed API/helper is first used, removing the earlier worker/thread startup and state writes from plain module import. diff --git a/README.md b/README.md index adcc296..697b0a7 100644 --- a/README.md +++ b/README.md @@ -2,14 +2,21 @@ A local web UI for a system-wide `ani-cli` install. -Current version: `0.12.0` +Current version: `0.26.0` ## Project Layout ```text ani-cli-web/ - app.py Web server, runtime wiring, request handlers, and CLI entrypoint. - web_templates.py Shared Search, Config, and Watchlist page markup plus page-shell helpers. + app.py Public entrypoint, runtime wiring, watchlist domain logic, and CLI startup. + app_support.py Shared runtime/state/config helpers plus queue and library file utilities. + queue_jobs.py Download queue and background watchlist-refresh workers. + http_handler.py HTTP route handling, request parsing, and remote-access guard. + web_templates.py Shared Search, Config, and Watchlist page markup aggregation. + search_page.py Search page template. + config_page.py Config page template. + watchlist_page.py Watchlist page template. + template_helpers.py Shared page-shell navigation and sidebar brand helpers. ani-cli-web Launcher script. test_app.py Regression tests with isolated temp-state setup. VERSION Project version. @@ -53,11 +60,26 @@ The app currently includes: - A dedicated Config page for saved default folder, mode, and quality settings. - A paged SQLite-backed download queue with retry, cancel, remove, retry-failed, and clear-finished actions. - A paged SQLite-backed watchlist with summary cards, per-show refresh, bulk refresh, source links, and thumbnail tools. -- Background `Refresh All` execution with status polling instead of a long blocking HTTP request. +- Watchlist category tabs for `Watching`, `Planned`, `Finished`, and `Dropped`, with per-anime category editing. +- AnimeSchedule-backed watchlist completion tracking that compares live sub/dub availability with the expected total episode count when known. +- Automatic watchlist sync after successful downloads, including downloaded-vs-manual finished markers. +- Per-show background watchlist refresh queueing so adding a show or syncing a finished download does not block the request on upstream episode, schedule, and thumbnail lookups. +- Queued per-show watchlist refresh work now survives app restarts and resumes from rows still marked `queued`. +- Per-show watchlist refresh queueing now also deduplicates in-flight refreshes, so the same show is not fetched twice back-to-back while one refresh is already running. +- The watchlist page now silently refreshes itself while queued per-show refresh work is still being processed, so queued entries settle into their updated state without a manual reload. +- The watchlist page now also avoids re-triggering the same unresolved thumbnail warm-up request on every silent poll cycle while a show's thumbnail state is unchanged. +- Background `Refresh All` execution with status polling, parallel per-show refresh work, and no inline thumbnail downloads during the bulk job. +- Search and Watchlist polling now use a shared serial browser polling helper so slow responses do not cause overlapping poll requests to pile up in the background. +- Controlled shutdown and runtime reset now cancel active download workers when waiting for teardown, reducing the chance of orphaned background downloads outliving the web process. - Persistent watchlist refresh history that survives restarts and marks interrupted bulk refresh jobs clearly. - Local thumbnail caching plus manual thumbnail upload when automatic cover lookup fails. - A refreshed glassy dark UI inspired by the Tsuki frontend design language. -- Shared page templates moved out of `app.py` plus broader regression tests for higher-risk queue, watchlist, and handler flows. +- Shared page templates and backend services split into dedicated modules while keeping `app.py` and `web_templates.py` as stable public aggregation points. +- Broader regression coverage for higher-risk queue, watchlist, remote-access, and handler flows. +- Search and watchlist requests guard against stale out-of-order async responses during rapid repeated actions. +- Search-page queue polling also guards against stale out-of-order refresh responses now. +- Queue log writes are batched during active downloads to reduce SQLite churn on noisy jobs. +- Queue persistence now stores core job metadata and logs in structured SQLite columns instead of treating the whole job as one large JSON blob. - Project-local runtime state under `.ani-cli-web/`. ## Search Page @@ -97,14 +119,24 @@ If you intentionally want LAN or remote access, set: ```sh ANI_CLI_WEB_ALLOW_REMOTE=1 +ANI_CLI_WEB_REMOTE_TOKEN=choose-a-long-random-token ``` -This keeps the queue, config, and watchlist mutation APIs safer by default when the app is rebound away from localhost. +Non-local clients must then send either: + +- `Authorization: Bearer ` +- `X-AniCli-Web-Token: ` + +When the app is placed behind a loopback reverse proxy, forwarded external client IP headers such as `X-Forwarded-For` are also treated as non-local, so proxied internet or LAN traffic still needs the token. + +This keeps the queue, config, and watchlist APIs safer when the app is intentionally exposed beyond localhost. ## Download Queue The queue is stored in SQLite and shown 10 jobs at a time. +Core queue fields and logs are now stored in structured SQLite columns, while JSON payload storage is kept only for extra fields that do not have dedicated columns yet. + Queue actions currently available: - `Retry` for finished or failed jobs. @@ -121,22 +153,43 @@ ANI_CLI_DOWNLOAD_DIR/anime_name/Season 01/anime_name - S01E01.mp4 ## Watchlist -The watchlist stores one row per show in SQLite and keeps separate `sub` and `dub` episode counts. +The watchlist stores one row per show in SQLite, keeps separate `sub` and `dub` episode counts, and assigns each show to one of four categories: + +- `Watching` +- `Planned` +- `Finished` +- `Dropped` You can add entries in two ways: -1. From the Search page with `Add to watchlist`. -2. Manually on `/watchlist` with a title and AllAnime `show_id`. +1. From the Search page with `Add to watchlist`, choosing the category first. +2. Manually on `/watchlist` with a title, AllAnime `show_id`, and category. Watchlist features: 1. `Refresh` updates one tracked show. -2. `Refresh All` runs in the background, rejects duplicate queueing while a refresh is already pending or running, and preserves interrupted job status across restarts. +2. `Refresh All` runs in the background, rejects duplicate queueing while a refresh is already pending or running, preserves interrupted job status across restarts, and refreshes shows in parallel without doing inline thumbnail downloads for every entry. 3. `Open` jumps back to the Search page with the title pre-filled. 4. Clicking the anime title opens the AllManga source page at `bangumi/` in a new tab. 5. `Remove` deletes the watchlist entry and clears its cached poster file. 6. Poster cards use a compact multi-column grid with pagination at 30 cards per page. 7. Summary cards show tracked shows plus total `sub` and `dub` episode counts. +8. Episode pills show `current / total` progress when AnimeSchedule exposes the planned episode count. +9. `Sub` and `Dub` pills turn green and gain `Sub ready` or `Dub ready` tags when that release track appears fully finished and ready for download. +10. Each category has its own tab on the watchlist page. +11. Every card includes an inline category selector so you can move tracked anime between tabs at any time. +12. Finished shows display a small check badge: green means the anime was downloaded through the app, gray means it was marked finished manually. + +### Download Sync + +Successful downloads now feed back into the watchlist automatically. + +Current behavior: + +- If the anime is already on the watchlist, a successful download marks it as downloaded and keeps its current category. +- If the anime is not yet on the watchlist, `ani-cli-web` creates a new watchlist entry in `Finished` after the download succeeds. +- Those add and download-sync paths now queue a background refresh for episode counts, AnimeSchedule metadata, and thumbnails instead of waiting for all upstream fetches before returning. +- Existing watchlist entries from older versions are migrated into the `Watching` category automatically. ### Thumbnail Behavior @@ -147,10 +200,13 @@ Current thumbnail behavior: - Cached thumbnails render from the local project cache when available. - The page adds a fresh nonce to thumbnail URLs to avoid stale browser-cached misses. - Missing visible covers are warmed up in small batches instead of one request per card all at once. +- The page only retries an unresolved cover warm-up once per thumbnail state change during the current browser session, so silent watchlist polling does not keep hammering the same missing poster. +- If a show disappears while a thumbnail warm-up batch is running, that entry is skipped cleanly instead of failing the whole batch request. - Thumbnail image tags only hit the local route for already-cached files, so opening the watchlist no longer triggers a per-card remote lookup burst. - AnimeSchedule is tried first for cover lookup. - If a season-labeled title fails as-is, lookup retries with simplified base-title queries. - AniDB is used as a fallback when AnimeSchedule does not resolve a cover. +- AniDB fallback reuses the cached parsed title list directly to keep repeated lookups lighter. - Failed fetch attempts no longer refresh the retry window, so a bad lookup does not get stuck in an immediate repeat-404 state. Per-card thumbnail tools: @@ -161,6 +217,22 @@ Per-card thumbnail tools: Manual uploads are saved into the same local thumbnail cache used by automatic cover downloads. +### Completion Detection + +Watchlist refresh still uses AllAnime, the same upstream source that `ani-cli` uses, to count which `sub` and `dub` episodes are actually available right now. + +To decide whether a release track looks complete, `ani-cli-web` also checks AnimeSchedule for: + +- the anime's expected total episode count +- the upstream airing status such as `Finished` or `Ongoing` + +When AnimeSchedule reports a known total and `Finished`, the watchlist compares that total with the live AllAnime `sub` and `dub` counts: + +- `Sub` becomes ready when available sub episodes reach the expected total. +- `Dub` becomes ready when available dub episodes reach the expected total. + +If AnimeSchedule does not know the total yet, the watchlist keeps showing the plain available-count value without a completion highlight. + ## Runtime State All runtime state is stored under: @@ -193,6 +265,7 @@ Environment variables: - `ANI_CLI_WEB_HOST`: server host, default `127.0.0.1`. - `ANI_CLI_WEB_PORT`: server port, default `8421`. - `ANI_CLI_WEB_ALLOW_REMOTE`: allow non-loopback clients when set to `1`, `true`, `yes`, or `on`. +- `ANI_CLI_WEB_REMOTE_TOKEN`: required shared secret for non-loopback clients when remote access is enabled. - `ANI_CLI_WEB_DEBUG`: enable debug logging when set to `1`, `true`, `yes`, or `on`. - `ANI_CLI_WEB_STATE_ROOT`: override the project-local state directory path; useful for isolated test runs or custom storage locations. @@ -201,6 +274,9 @@ Saved defaults from the UI are written into the project-local `config.json`. ## Notes - Importing `app.py` no longer boots the runtime worker threads automatically; runtime setup is deferred until startup or first use. +- Importing `app_support.py` no longer migrates or creates runtime state on its own; migrations now happen during runtime bootstrap. +- Cheap routes such as the page shells, `/api/version`, `/api/dependencies`, and `/api/config` do not need to boot the full runtime before responding. +- The CLI startup path itself also stays lazy now; runtime bootstrap happens when the first runtime-backed request arrives. - Watchlist and queue data are stored in the same project-local SQLite database file, `state.sqlite3`. - Runtime folders such as `.ani-cli-web/`, `downloads/`, and Python bytecode cache directories at any depth are ignored by git. - Focused regression tests live in `test_app.py`, run against an isolated temporary state root, and can be started with `python3 -m unittest test_app.py`. diff --git a/VERSION b/VERSION index ac454c6..4e8f395 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.12.0 +0.26.0 diff --git a/app.py b/app.py index 94e5057..be82954 100644 --- a/app.py +++ b/app.py @@ -1,15 +1,12 @@ #!/usr/bin/env python3 import base64 import gzip -import ipaddress import json import mimetypes import os import re import shutil -import signal import sqlite3 -import subprocess import threading import time import unicodedata @@ -18,45 +15,52 @@ import urllib.request import xml.etree.ElementTree as ET from binascii import Error as BinasciiError from datetime import datetime, timezone -from http import HTTPStatus -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from http.server import ThreadingHTTPServer from pathlib import Path import sys -from urllib.parse import parse_qs, quote, unquote, urljoin, urlparse -from uuid import uuid4 +from urllib.parse import quote, urljoin, urlparse +import app_support +from app_support import ( + AGENT, + ALLANIME_API, + ALLANIME_REFERER, + ALLMANGA_SITE_BASE, + ANIDB_ANIME_PAGE, + ANIDB_TITLES_CACHE, + ANIDB_TITLES_PATH, + ANIDB_TITLES_URL, + ANIMESCHEDULE_API, + ANIMESCHEDULE_IMAGE_BASE, + APP_NAME, + CONFIG_PATH, + DEFAULT_CONFIG, + LEGACY_THUMBNAIL_ROOT, + MAX_JSON_BODY_BYTES, + MODE_CHOICES, + STATE_DB_PATH, + THUMBNAIL_RETRY_SECONDS, + THUMBNAIL_ROOT, + VERSION, + WATCHLIST_CATEGORY_CHOICES, + WATCHLIST_CATEGORY_LABELS, + WATCHLIST_JSON_PATH, + background_workers_enabled, + client_address_is_local, + debug_log, + load_json, + migrate_legacy_state_storage, + normalize_config, + now_iso, + sanitize_path_component, + write_json, +) +from http_handler import HandlerContext, build_handler_class, server_host, server_port +from queue_jobs import DownloadQueue, WatchlistRefreshJobs from web_templates import CONFIG_HTML, INDEX_HTML, PAGE_LINKS, WATCHLIST_HTML, render_page_links, render_sidebar_brand - -PROJECT_ROOT = Path(__file__).resolve().parent -ANI_CLI = os.environ.get("ANI_CLI_BIN") or shutil.which("ani-cli") or "ani-cli" -APP_NAME = "ani-cli-web" -VERSION = "0.12.0" -ALLANIME_BASE = "allanime.day" -ALLANIME_API = f"https://api.{ALLANIME_BASE}" -ALLANIME_REFERER = "https://allmanga.to" -ALLMANGA_SITE_BASE = "https://allmanga.to" -ANIMESCHEDULE_API = "https://animeschedule.net/api/v3" -ANIMESCHEDULE_IMAGE_BASE = "https://img.animeschedule.net/production/assets/public/img/" -ANIDB_TITLES_URL = "https://anidb.net/api/anime-titles.xml.gz" -ANIDB_ANIME_PAGE = "https://anidb.net/anime/{aid}" -AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/121.0" -CLIENT_DISCONNECT_ERRORS = (BrokenPipeError, ConnectionResetError, ConnectionAbortedError) -DEBUG_MODE = False RUNTIME_LOCK = threading.RLock() -QUALITY_CHOICES = {"best", "1080", "1080p", "720", "720p", "480", "480p", "360", "360p", "worst"} -MODE_CHOICES = {"sub", "dub"} -MAX_LOG_LINES = 240 -MAX_JSON_BODY_BYTES = 12 * 1024 * 1024 - - -def env_flag(name, default=False): - raw = os.environ.get(name) - if raw is None: - return default - return str(raw).strip().lower() in {"1", "true", "yes", "on"} - class HttpError(Exception): def __init__(self, status, message): @@ -65,200 +69,6 @@ class HttpError(Exception): self.message = message -def debug_enabled(): - return DEBUG_MODE or env_flag("ANI_CLI_WEB_DEBUG") - - -def remote_access_allowed(): - return env_flag("ANI_CLI_WEB_ALLOW_REMOTE") - - -def background_workers_enabled(): - return not env_flag("ANI_CLI_WEB_DISABLE_WORKER") - - -def client_address_is_local(host): - text = str(host or "").strip() - if not text: - return False - if text == "localhost": - return True - try: - return ipaddress.ip_address(text).is_loopback - except ValueError: - return False - - -def debug_log(event, **fields): - if not debug_enabled(): - return - parts = [f"[debug] {event}"] - for key, value in fields.items(): - text = str(value) - if len(text) > 240: - text = text[:237] + "..." - parts.append(f"{key}={text}") - print(" | ".join(parts), flush=True) - - -def default_download_dir(): - if os.environ.get("ANI_CLI_DOWNLOAD_DIR"): - return os.environ["ANI_CLI_DOWNLOAD_DIR"] - home_downloads = Path.home() / "Downloads" / "Anime" - if os.access(home_downloads.parent if home_downloads.parent.exists() else Path.home(), os.W_OK): - return str(home_downloads) - return str(PROJECT_ROOT / "downloads") - - -DEFAULT_CONFIG = { - "download_dir": default_download_dir(), - "mode": os.environ.get("ANI_CLI_MODE", "sub"), - "quality": os.environ.get("ANI_CLI_QUALITY", "best"), -} - - -def now_iso(): - return datetime.now(timezone.utc).isoformat() - - -def project_app_root(): - configured = str(os.environ.get("ANI_CLI_WEB_STATE_ROOT") or "").strip() - if configured: - return Path(configured).expanduser() - return PROJECT_ROOT / ".ani-cli-web" - - -PROJECT_APP_ROOT = project_app_root() - - -def ensure_project_app_root(): - PROJECT_APP_ROOT.mkdir(parents=True, exist_ok=True) - return PROJECT_APP_ROOT - - -def legacy_state_root(): - return Path(os.environ.get("XDG_STATE_HOME", Path.home() / ".local" / "state")) / APP_NAME - - -def legacy_config_root(): - return Path(os.environ.get("XDG_CONFIG_HOME", Path.home() / ".config")) / APP_NAME - - -def project_state_path(*parts): - return PROJECT_APP_ROOT.joinpath(*parts) - - -def project_state_dir(*parts): - return PROJECT_APP_ROOT.joinpath(*parts) - - -def migrate_legacy_file(project_path, legacy_path, label): - if project_path.exists() or not legacy_path.exists(): - return project_path - project_path.parent.mkdir(parents=True, exist_ok=True) - try: - shutil.move(str(legacy_path), str(project_path)) - debug_log("state.migrated", label=label, from_path=legacy_path, to_path=project_path) - except OSError as exc: - debug_log("state.migrate_failed", label=label, from_path=legacy_path, to_path=project_path, error=exc) - return project_path - - -def migrate_legacy_dir(project_dir, legacy_dir, label): - if not legacy_dir.exists(): - return project_dir - project_dir.mkdir(parents=True, exist_ok=True) - moved = 0 - for child in legacy_dir.iterdir(): - target = project_dir / child.name - if target.exists(): - if child.is_dir() and target.is_dir(): - migrate_legacy_dir(target, child, f"{label}/{child.name}") - continue - try: - shutil.move(str(child), str(target)) - moved += 1 - except OSError as exc: - debug_log("state.migrate_failed", label=label, from_path=child, to_path=target, error=exc) - try: - legacy_dir.rmdir() - except OSError: - pass - if moved: - debug_log("state.migrated", label=label, from_path=legacy_dir, to_path=project_dir, entries=moved) - return project_dir - - -LEGACY_STATE_ROOT = legacy_state_root() -LEGACY_CONFIG_ROOT = legacy_config_root() -LEGACY_CONFIG_PATH = LEGACY_CONFIG_ROOT / "config.json" -LEGACY_QUEUE_PATH = LEGACY_STATE_ROOT / "queue.json" -LEGACY_QUEUE_DB_PATH = LEGACY_STATE_ROOT / "queue.sqlite3" -LEGACY_STAGING_ROOT = LEGACY_STATE_ROOT / "staging" -LEGACY_WATCHLIST_JSON_PATH = LEGACY_STATE_ROOT / "watchlist.json" -LEGACY_THUMBNAIL_ROOT = LEGACY_STATE_ROOT / "thumbnails" -LEGACY_ANIDB_TITLES_PATH = LEGACY_STATE_ROOT / "anidb-anime-titles.xml.gz" - -CONFIG_PATH = project_state_path("config.json") -QUEUE_PATH = project_state_path("queue.json") -STATE_DB_PATH = project_state_path("state.sqlite3") -STAGING_ROOT = project_state_dir("staging") -WATCHLIST_JSON_PATH = project_state_path("watchlist.json") -THUMBNAIL_ROOT = project_state_dir("thumbnails") -THUMBNAIL_RETRY_SECONDS = 15 * 60 -ANIDB_TITLES_PATH = project_state_path("anidb-anime-titles.xml.gz") -ANIDB_TITLES_CACHE = {"mtime": None, "entries": None} - - -def migrate_legacy_state_storage(): - ensure_project_app_root() - migrate_legacy_file(CONFIG_PATH, LEGACY_CONFIG_PATH, "config.json") - migrate_legacy_file(QUEUE_PATH, LEGACY_QUEUE_PATH, "queue.json") - migrate_legacy_file(STATE_DB_PATH, PROJECT_APP_ROOT / "queue.sqlite3", "queue.sqlite3") - migrate_legacy_file(STATE_DB_PATH, LEGACY_QUEUE_DB_PATH, "queue.sqlite3") - migrate_legacy_file(WATCHLIST_JSON_PATH, LEGACY_WATCHLIST_JSON_PATH, "watchlist.json") - migrate_legacy_file(ANIDB_TITLES_PATH, LEGACY_ANIDB_TITLES_PATH, "anidb-anime-titles.xml.gz") - migrate_legacy_dir(STAGING_ROOT, LEGACY_STAGING_ROOT, "staging") - migrate_legacy_dir(THUMBNAIL_ROOT, LEGACY_THUMBNAIL_ROOT, "thumbnails") - - -migrate_legacy_state_storage() - - -def load_json(path, fallback): - try: - with path.open("r", encoding="utf-8") as handle: - data = json.load(handle) - return data if isinstance(data, type(fallback)) else fallback - except (FileNotFoundError, json.JSONDecodeError, OSError): - return fallback - - -def write_json(path, data): - path.parent.mkdir(parents=True, exist_ok=True) - tmp = path.with_suffix(path.suffix + ".tmp") - with tmp.open("w", encoding="utf-8") as handle: - json.dump(data, handle, indent=2, sort_keys=True) - tmp.replace(path) - - -def normalize_config(data): - config = dict(DEFAULT_CONFIG) - if isinstance(data, dict): - config.update({key: data[key] for key in DEFAULT_CONFIG if key in data}) - - mode = str(config.get("mode", "sub")).lower() - quality = str(config.get("quality", "best")).lower() - if quality.endswith("p") and quality[:-1].isdigit(): - quality = quality[:-1] - download_dir = str(config.get("download_dir") or DEFAULT_CONFIG["download_dir"]) - - config["mode"] = mode if mode in MODE_CHOICES else "sub" - config["quality"] = quality if quality in QUALITY_CHOICES else "best" - config["download_dir"] = str(Path(download_dir).expanduser()) - return config - - CONFIG = None WATCHLIST = None DOWNLOAD_QUEUE = None @@ -486,24 +296,64 @@ def animeschedule_request(path, params=None): raise RuntimeError("AnimeSchedule returned unreadable JSON") from exc +def parse_nonnegative_int(value, default=0): + try: + number = int(value) + except (TypeError, ValueError): + return default + return number if number >= 0 else default + + +def normalize_animeschedule_status(value): + text = str(value or "").strip() + return text.title() if text else "" + + +def normalize_watchlist_category(value, default="watching"): + text = str(value or "").strip().lower() + aliases = { + "watching": "watching", + "planned": "planned", + "planed": "planned", + "finished": "finished", + "dropped": "dropped", + } + return aliases.get(text, default) + + +def watchlist_category_label(value): + return WATCHLIST_CATEGORY_LABELS[normalize_watchlist_category(value)] + + +def normalize_animeschedule_entry(node): + if not isinstance(node, dict): + return None + route = str(node.get("route") or "").strip() + image_route = str(node.get("imageVersionRoute") or "").strip() + title = str(node.get("title") or "").strip() + names = node.get("names") if isinstance(node.get("names"), dict) else {} + if not route or not (title or names): + return None + return { + "route": route, + "title": title, + "imageVersionRoute": image_route, + "names": names, + "status": normalize_animeschedule_status(node.get("status")), + "episodes": parse_nonnegative_int(node.get("episodes")), + "subPremier": str(node.get("subPremier") or "").strip() or None, + "dubPremier": str(node.get("dubPremier") or "").strip() or None, + } + + def collect_animeschedule_entries(payload): entries = [] def walk(node): if isinstance(node, dict): - route = str(node.get("route") or "").strip() - image_route = str(node.get("imageVersionRoute") or "").strip() - title = str(node.get("title") or "").strip() - names = node.get("names") if isinstance(node.get("names"), dict) else {} - if route and (title or names): - entries.append( - { - "route": route, - "title": title, - "imageVersionRoute": image_route, - "names": names, - } - ) + entry = normalize_animeschedule_entry(node) + if entry: + entries.append(entry) for value in node.values(): walk(value) elif isinstance(node, list): @@ -558,27 +408,25 @@ def score_animeschedule_match(query_variants, entry): def fetch_animeschedule_cover_by_route(route, fallback_title=""): debug_log("thumbnail.source.route_lookup", source="AnimeSchedule", route=route, fallback_title=fallback_title) - payload = animeschedule_request(f"/anime/{quote(str(route).strip(), safe='')}") - if not isinstance(payload, dict): - raise RuntimeError("AnimeSchedule anime response was not an object") - image_route = str(payload.get("imageVersionRoute") or "").strip() + entry = fetch_animeschedule_anime_by_route(route, fallback_title=fallback_title) + image_route = str(entry.get("imageVersionRoute") or "").strip() if not image_route: raise RuntimeError("AnimeSchedule anime did not provide an image route") debug_log( "thumbnail.source.route_match", source="AnimeSchedule", - route=str(payload.get("route") or route).strip(), + route=entry["route"], image_route=image_route, - title=str(payload.get("title") or fallback_title or "").strip(), + title=entry["title"], ) return { - "route": str(payload.get("route") or route).strip(), - "title": str(payload.get("title") or fallback_title or "").strip(), + "route": entry["route"], + "title": entry["title"], "url": urljoin(ANIMESCHEDULE_IMAGE_BASE, image_route), } -def fetch_animeschedule_cover(title): +def find_animeschedule_match(title): query_variants = title_match_variants(title) if not query_variants: raise RuntimeError("Missing title for AnimeSchedule lookup") @@ -623,6 +471,33 @@ def fetch_animeschedule_cover(title): score=best["score"], candidates=candidate_count, ) + return best + + +def fetch_animeschedule_anime_by_route(route, fallback_title=""): + payload = animeschedule_request(f"/anime/{quote(str(route).strip(), safe='')}") + entry = normalize_animeschedule_entry(payload) + if not entry: + raise RuntimeError("AnimeSchedule anime response was missing required fields") + if fallback_title and not entry.get("title"): + entry["title"] = str(fallback_title).strip() + return entry + + +def resolve_animeschedule_anime(title, route=""): + stored_route = str(route or "").strip() + if stored_route: + try: + return fetch_animeschedule_anime_by_route(stored_route, fallback_title=title) + except Exception: + debug_log("animeschedule.route_refresh_failed", route=stored_route, title=title) + + best = find_animeschedule_match(title) + return fetch_animeschedule_anime_by_route(best["route"], fallback_title=best["title"]) + + +def fetch_animeschedule_cover(title): + best = find_animeschedule_match(title) if best["imageVersionRoute"]: return { "route": best["route"], @@ -691,6 +566,8 @@ def iter_anidb_titles(): return entries = [] + title_index = {} + token_index = {} with gzip.open(path, "rb") as handle: context = ET.iterparse(handle, events=("end",)) for _, elem in context: @@ -704,23 +581,42 @@ def iter_anidb_titles(): title = str(title_elem.text or "").strip() if not title: continue - entries.append( - { - "aid": aid, - "title": title, - "type": str(title_elem.attrib.get("type") or "").strip().lower(), - "lang": str(title_elem.attrib.get("{http://www.w3.org/XML/1998/namespace}lang") or "").strip().lower(), - } - ) + normalized = normalize_title_key(title) + entry = { + "aid": aid, + "title": title, + "type": str(title_elem.attrib.get("type") or "").strip().lower(), + "lang": str(title_elem.attrib.get("{http://www.w3.org/XML/1998/namespace}lang") or "").strip().lower(), + "normalized_title": normalized, + } + index = len(entries) + entries.append(entry) + if normalized: + title_index.setdefault(normalized, []).append(index) + for token in set(normalized.split()): + if len(token) >= 3: + token_index.setdefault(token, set()).add(index) elem.clear() ANIDB_TITLES_CACHE["mtime"] = mtime ANIDB_TITLES_CACHE["entries"] = entries + ANIDB_TITLES_CACHE["title_index"] = title_index + ANIDB_TITLES_CACHE["token_index"] = {token: sorted(indices) for token, indices in token_index.items()} + ANIDB_TITLES_CACHE["match_cache"] = {} for entry in entries: yield entry +def get_cached_anidb_titles(): + path = ensure_anidb_titles_dump() + mtime = path.stat().st_mtime if path.exists() else None + if ANIDB_TITLES_CACHE["entries"] is None or ANIDB_TITLES_CACHE["mtime"] != mtime: + for _entry in iter_anidb_titles(): + pass + return ANIDB_TITLES_CACHE["entries"] or [] + + def score_anidb_title_match(query_variants, candidate): - normalized = normalize_title_key(candidate["title"]) + normalized = candidate.get("normalized_title") or normalize_title_key(candidate["title"]) if not normalized: return None score = None @@ -752,21 +648,46 @@ def resolve_anidb_aid(title): if not query_variants: raise RuntimeError("Missing title for AniDB lookup") - best = None - candidate_count = 0 - for candidate in iter_anidb_titles(): - candidate_count += 1 - score = score_anidb_title_match(query_variants, candidate) - if score is None: - continue - if best is None or score > best["score"]: - best = { - "aid": candidate["aid"], - "title": candidate["title"], - "score": score, - } + cache_key = tuple(sorted(query_variants)) + cached = ANIDB_TITLES_CACHE["match_cache"].get(cache_key) + if cached is not None: + if cached: + return dict(cached) + raise RuntimeError("AniDB title lookup did not find a matching anime") + + entries = get_cached_anidb_titles() + title_index = ANIDB_TITLES_CACHE.get("title_index") or {} + token_index = ANIDB_TITLES_CACHE.get("token_index") or {} + candidate_indices = set() + for variant in query_variants: + candidate_indices.update(title_index.get(variant, [])) + for token in variant.split(): + if len(token) >= 3: + candidate_indices.update(token_index.get(token, [])) + candidates = [entries[index] for index in sorted(candidate_indices)] if candidate_indices else entries + + def pick_best(pool, current_best=None): + best_match = current_best + for candidate in pool: + score = score_anidb_title_match(query_variants, candidate) + if score is None: + continue + if best_match is None or score > best_match["score"]: + best_match = { + "aid": candidate["aid"], + "title": candidate["title"], + "score": score, + } + return best_match + + best = pick_best(candidates) + candidate_count = len(candidates) + if best is None and candidate_indices: + best = pick_best(entries) + candidate_count = len(entries) if not best: debug_log("thumbnail.source.no_match", source="AniDB", title=title, candidates=candidate_count) + ANIDB_TITLES_CACHE["match_cache"][cache_key] = None raise RuntimeError("AniDB title lookup did not find a matching anime") debug_log( "thumbnail.source.best_match", @@ -777,6 +698,7 @@ def resolve_anidb_aid(title): score=best["score"], candidates=candidate_count, ) + ANIDB_TITLES_CACHE["match_cache"][cache_key] = dict(best) return best @@ -965,18 +887,164 @@ def episode_list(show_id, mode): return fetch_show_episode_snapshot(show_id)["episodes"][normalized_mode] +def format_episode_progress(available_count, expected_count): + available = parse_nonnegative_int(available_count) + expected = parse_nonnegative_int(expected_count) + if expected > 0: + return f"{available}/{expected}" + return str(available) + + +def mode_is_complete(available_count, expected_count, airing_status): + expected = parse_nonnegative_int(expected_count) + available = parse_nonnegative_int(available_count) + return expected > 0 and normalize_animeschedule_status(airing_status) == "Finished" and available >= expected + + +def watchlist_status_message(sub_count, dub_count, expected_count=0, airing_status=""): + parts = [ + f"Sub: {format_episode_progress(sub_count, expected_count)} available", + f"Dub: {format_episode_progress(dub_count, expected_count)} available", + ] + status = normalize_animeschedule_status(airing_status) + if status: + parts.append(f"AnimeSchedule: {status}") + if mode_is_complete(sub_count, expected_count, status): + parts.append("Sub ready") + if mode_is_complete(dub_count, expected_count, status): + parts.append("Dub ready") + return ". ".join(parts) + "." + + class WatchlistStore: - def __init__(self): + def __init__(self, start_worker=True): self.lock = threading.RLock() self.thumbnail_events = {} + self.refresh_pending = [] + self.refresh_pending_set = set() + self.refresh_inflight_set = set() + self.refresh_wakeup = threading.Event() + self.refresh_stop_event = threading.Event() + self.refresh_worker = None + self.worker_enabled = bool(start_worker) self._init_db() self._migrate_legacy_json() + self._restore_queued_refreshes() + if start_worker: + self.ensure_worker() + + def ensure_worker(self): + if not self.worker_enabled: + return + with self.lock: + if self.refresh_worker is not None and self.refresh_worker.is_alive(): + return + self.refresh_stop_event.clear() + self.refresh_worker = threading.Thread(target=self._refresh_loop, daemon=True) + self.refresh_worker.start() + + def shutdown(self, wait=False): + self.refresh_stop_event.set() + self.refresh_wakeup.set() + worker = self.refresh_worker + if wait and worker is not None and worker.is_alive(): + worker.join(timeout=2.5) + return worker is None or not worker.is_alive() + + def _next_pending_refresh(self): + with self.lock: + if not self.refresh_pending: + return None + show_id = self.refresh_pending.pop(0) + self.refresh_pending_set.discard(show_id) + self.refresh_inflight_set.add(show_id) + return show_id + + def _enqueue_refresh_locked(self, show_id): + normalized_show_id = str(show_id or "").strip() + if ( + not normalized_show_id + or normalized_show_id in self.refresh_pending_set + or normalized_show_id in self.refresh_inflight_set + ): + return False + self.refresh_pending.append(normalized_show_id) + self.refresh_pending_set.add(normalized_show_id) + return True + + def _drop_refresh_locked(self, show_id): + normalized_show_id = str(show_id or "").strip() + if not normalized_show_id: + return False + removed = False + if normalized_show_id in self.refresh_pending_set: + self.refresh_pending = [value for value in self.refresh_pending if value != normalized_show_id] + self.refresh_pending_set.discard(normalized_show_id) + removed = True + if normalized_show_id in self.refresh_inflight_set: + self.refresh_inflight_set.discard(normalized_show_id) + removed = True + return removed + + def _restore_queued_refreshes(self): + with self.lock, self._connect() as conn: + rows = conn.execute( + """ + SELECT show_id + FROM watchlist + WHERE status = 'queued' + ORDER BY updated_at ASC, created_at ASC, title COLLATE NOCASE ASC + """ + ).fetchall() + for row in rows: + self._enqueue_refresh_locked(str(row["show_id"]).strip()) + if self.refresh_pending: + self.refresh_wakeup.set() + + def _refresh_loop(self): + while not self.refresh_stop_event.is_set(): + show_id = self._next_pending_refresh() + if not show_id: + self.refresh_wakeup.wait(2) + self.refresh_wakeup.clear() + continue + try: + self.refresh(show_id) + except Exception as exc: + debug_log("watchlist.refresh_worker.error", show_id=show_id, error=exc) + finally: + with self.lock: + self.refresh_inflight_set.discard(show_id) + + def schedule_refresh(self, show_id): + normalized_show_id = str(show_id or "").strip() + if not normalized_show_id: + raise ValueError("Missing show_id.") + self.ensure_worker() + with self.lock: + self._enqueue_refresh_locked(normalized_show_id) + self.refresh_wakeup.set() + return self.get(normalized_show_id) def _connect(self): conn = sqlite3.connect(STATE_DB_PATH, timeout=30) conn.row_factory = sqlite3.Row return conn + def _exists_conn(self, conn, show_id): + row = conn.execute( + "SELECT 1 FROM watchlist WHERE show_id = ? LIMIT 1", + (str(show_id or "").strip(),), + ).fetchone() + return bool(row) + + def has_show(self, show_id): + normalized_show_id = str(show_id or "").strip() + if not normalized_show_id: + return False + with self.lock, self._connect() as conn: + return self._exists_conn(conn, normalized_show_id) + def _init_db(self): STATE_DB_PATH.parent.mkdir(parents=True, exist_ok=True) with self._connect() as conn: @@ -985,6 +1053,8 @@ class WatchlistStore: CREATE TABLE IF NOT EXISTS watchlist ( show_id TEXT PRIMARY KEY, title TEXT NOT NULL, + category TEXT NOT NULL DEFAULT 'watching', + downloaded INTEGER NOT NULL DEFAULT 0, status TEXT NOT NULL, status_message TEXT NOT NULL, created_at TEXT NOT NULL, @@ -992,6 +1062,8 @@ class WatchlistStore: last_checked TEXT, sub_count INTEGER NOT NULL DEFAULT 0, dub_count INTEGER NOT NULL DEFAULT 0, + expected_count INTEGER NOT NULL DEFAULT 0, + airing_status TEXT, sub_latest_episode TEXT, dub_latest_episode TEXT, animeschedule_route TEXT, @@ -1004,10 +1076,18 @@ class WatchlistStore: """ ) columns = {row["name"] for row in conn.execute("PRAGMA table_info(watchlist)").fetchall()} + if "category" not in columns: + conn.execute("ALTER TABLE watchlist ADD COLUMN category TEXT NOT NULL DEFAULT 'watching'") + if "downloaded" not in columns: + conn.execute("ALTER TABLE watchlist ADD COLUMN downloaded INTEGER NOT NULL DEFAULT 0") if "thumbnail_path" not in columns: conn.execute("ALTER TABLE watchlist ADD COLUMN thumbnail_path TEXT") if "thumbnail_checked_at" not in columns: conn.execute("ALTER TABLE watchlist ADD COLUMN thumbnail_checked_at TEXT") + if "expected_count" not in columns: + conn.execute("ALTER TABLE watchlist ADD COLUMN expected_count INTEGER NOT NULL DEFAULT 0") + if "airing_status" not in columns: + conn.execute("ALTER TABLE watchlist ADD COLUMN airing_status TEXT") if "anidb_aid" not in columns: conn.execute("ALTER TABLE watchlist ADD COLUMN anidb_aid TEXT") if "anidb_title" not in columns: @@ -1016,14 +1096,30 @@ class WatchlistStore: conn.execute("ALTER TABLE watchlist ADD COLUMN animeschedule_route TEXT") if "animeschedule_title" not in columns: conn.execute("ALTER TABLE watchlist ADD COLUMN animeschedule_title TEXT") + conn.execute( + "UPDATE watchlist SET category = 'watching' WHERE category IS NULL OR TRIM(category) = ''" + ) conn.execute("CREATE INDEX IF NOT EXISTS idx_watchlist_title ON watchlist(title)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_watchlist_category ON watchlist(category)") conn.execute("CREATE INDEX IF NOT EXISTS idx_watchlist_updated_at ON watchlist(updated_at DESC)") def _row_to_item(self, row): item = dict(row) + item["category"] = normalize_watchlist_category(item.get("category")) + item["category_label"] = watchlist_category_label(item["category"]) + item["downloaded"] = bool(int(item.get("downloaded") or 0)) item["sub_count"] = int(item.get("sub_count") or 0) item["dub_count"] = int(item.get("dub_count") or 0) + item["expected_count"] = int(item.get("expected_count") or 0) + item["airing_status"] = normalize_animeschedule_status(item.get("airing_status")) item["total_count"] = item["sub_count"] + item["dub_count"] + item["sub_progress"] = format_episode_progress(item["sub_count"], item["expected_count"]) + item["dub_progress"] = format_episode_progress(item["dub_count"], item["expected_count"]) + item["sub_complete"] = mode_is_complete(item["sub_count"], item["expected_count"], item["airing_status"]) + item["dub_complete"] = mode_is_complete(item["dub_count"], item["expected_count"], item["airing_status"]) + item["finished_badge"] = ( + "downloaded" if item["category"] == "finished" and item["downloaded"] else "manual" if item["category"] == "finished" else "" + ) item["source_url"] = anime_source_url(item.get("show_id"), item.get("title")) thumb_path = thumbnail_file_path(item.get("thumbnail_path")) item["thumbnail_ready"] = bool(thumb_path and thumb_path.exists()) @@ -1044,19 +1140,24 @@ class WatchlistStore: conn.execute( """ INSERT INTO watchlist ( - show_id, title, status, status_message, created_at, updated_at, - last_checked, sub_count, dub_count, sub_latest_episode, dub_latest_episode, + show_id, title, category, downloaded, status, status_message, created_at, updated_at, + last_checked, sub_count, dub_count, expected_count, airing_status, + sub_latest_episode, dub_latest_episode, animeschedule_route, animeschedule_title, anidb_aid, anidb_title, thumbnail_path, thumbnail_checked_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(show_id) DO UPDATE SET title = excluded.title, + category = excluded.category, + downloaded = excluded.downloaded, status = excluded.status, status_message = excluded.status_message, updated_at = excluded.updated_at, last_checked = excluded.last_checked, sub_count = excluded.sub_count, dub_count = excluded.dub_count, + expected_count = excluded.expected_count, + airing_status = excluded.airing_status, sub_latest_episode = excluded.sub_latest_episode, dub_latest_episode = excluded.dub_latest_episode, animeschedule_route = excluded.animeschedule_route, @@ -1069,6 +1170,8 @@ class WatchlistStore: ( item["show_id"], item["title"], + normalize_watchlist_category(item.get("category")), + 1 if item.get("downloaded") else 0, item["status"], item["status_message"], item["created_at"], @@ -1076,6 +1179,8 @@ class WatchlistStore: item["last_checked"], int(item.get("sub_count") or 0), int(item.get("dub_count") or 0), + int(item.get("expected_count") or 0), + item.get("airing_status"), item.get("sub_latest_episode"), item.get("dub_latest_episode"), item.get("animeschedule_route"), @@ -1110,6 +1215,8 @@ class WatchlistStore: { "show_id": show_id, "title": str(value.get("title") or "Unknown Anime").strip() or "Unknown Anime", + "category": "watching", + "downloaded": False, "status": str(value.get("status") or "tracked"), "status_message": str(value.get("status_message") or "Migrated from legacy watchlist."), "created_at": str(value.get("added_at") or now_iso()), @@ -1117,6 +1224,8 @@ class WatchlistStore: "last_checked": value.get("last_checked"), "sub_count": 0, "dub_count": 0, + "expected_count": 0, + "airing_status": None, "sub_latest_episode": None, "dub_latest_episode": None, "animeschedule_route": None, @@ -1144,12 +1253,20 @@ class WatchlistStore: self._upsert_conn(conn, item) WATCHLIST_JSON_PATH.rename(WATCHLIST_JSON_PATH.with_suffix(".json.migrated")) - def list(self, page=1, per_page=30): + def list(self, page=1, per_page=30, category=None): page = max(1, int(page or 1)) per_page = min(60, max(1, int(per_page or 30))) offset = (page - 1) * per_page + active_category = normalize_watchlist_category(category) if category else None with self.lock, self._connect() as conn: - total = conn.execute("SELECT COUNT(*) AS count FROM watchlist").fetchone()["count"] + overall_total = conn.execute("SELECT COUNT(*) AS count FROM watchlist").fetchone()["count"] + if active_category: + total = conn.execute( + "SELECT COUNT(*) AS count FROM watchlist WHERE category = ?", + (active_category,), + ).fetchone()["count"] + else: + total = conn.execute("SELECT COUNT(*) AS count FROM watchlist").fetchone()["count"] totals = conn.execute( """ SELECT @@ -1158,20 +1275,37 @@ class WatchlistStore: FROM watchlist """ ).fetchone() - rows = conn.execute( - "SELECT * FROM watchlist ORDER BY title COLLATE NOCASE ASC LIMIT ? OFFSET ?", - (per_page, offset), + category_rows = conn.execute( + "SELECT category, COUNT(*) AS count FROM watchlist GROUP BY category" ).fetchall() + queued_total = conn.execute("SELECT COUNT(*) AS count FROM watchlist WHERE status = 'queued'").fetchone()["count"] + if active_category: + rows = conn.execute( + "SELECT * FROM watchlist WHERE category = ? ORDER BY title COLLATE NOCASE ASC LIMIT ? OFFSET ?", + (active_category, per_page, offset), + ).fetchall() + else: + rows = conn.execute( + "SELECT * FROM watchlist ORDER BY title COLLATE NOCASE ASC LIMIT ? OFFSET ?", + (per_page, offset), + ).fetchall() items = [self._row_to_item(row) for row in rows] pages = max(1, (total + per_page - 1) // per_page) + category_counts = {key: 0 for key in WATCHLIST_CATEGORY_CHOICES} + for row in category_rows: + category_counts[normalize_watchlist_category(row["category"])] = int(row["count"] or 0) return { "items": items, "total": total, "page": min(page, pages), "pages": pages, "per_page": per_page, + "active_category": active_category, + "active_category_label": watchlist_category_label(active_category) if active_category else "", + "categories": category_counts, + "queued_count": int(queued_total or 0), "summary": { - "shows": int(total), + "shows": int(overall_total or 0), "sub": int(totals["sub_total"] or 0), "dub": int(totals["dub_total"] or 0), }, @@ -1205,13 +1339,17 @@ class WatchlistStore: item = { "show_id": show_id, "title": title, - "status": "tracked", - "status_message": "Waiting for the first manual refresh.", + "category": normalize_watchlist_category(payload.get("category")), + "downloaded": False, + "status": "queued", + "status_message": "Queued initial watchlist refresh.", "created_at": now, "updated_at": now, "last_checked": None, "sub_count": 0, "dub_count": 0, + "expected_count": 0, + "airing_status": None, "sub_latest_episode": None, "dub_latest_episode": None, "animeschedule_route": None, @@ -1223,68 +1361,177 @@ class WatchlistStore: } self._upsert_conn(conn, item) - refreshed = self.refresh(show_id) - return {"message": f"Added {refreshed['title']} to the watchlist.", "item": refreshed, "created": True} + queued = self.schedule_refresh(show_id) + return {"message": f"Added {queued['title']} to the watchlist. Refresh queued.", "item": queued, "created": True} - def refresh(self, show_id): + def refresh(self, show_id, include_thumbnail=True): existing = self.get(show_id) + normalized_show_id = existing["show_id"] now = now_iso() try: - snapshot = fetch_show_episode_snapshot(show_id) + snapshot = fetch_show_episode_snapshot(normalized_show_id) + if not self.has_show(normalized_show_id): + return None sub_episodes = snapshot["episodes"]["sub"] dub_episodes = snapshot["episodes"]["dub"] - item = { - "show_id": existing["show_id"], - "title": snapshot["title"] or existing["title"], - "status": "updated", - "status_message": f"Sub: {len(sub_episodes)} episodes available. Dub: {len(dub_episodes)} episodes available.", - "created_at": existing["created_at"], - "updated_at": now, - "last_checked": now, - "sub_count": len(sub_episodes), - "dub_count": len(dub_episodes), - "sub_latest_episode": sub_episodes[-1] if sub_episodes else None, - "dub_latest_episode": dub_episodes[-1] if dub_episodes else None, - "animeschedule_route": existing.get("animeschedule_route"), - "animeschedule_title": existing.get("animeschedule_title"), - "anidb_aid": existing.get("anidb_aid"), - "anidb_title": existing.get("anidb_title"), - "thumbnail_path": existing.get("thumbnail_path"), - "thumbnail_checked_at": existing.get("thumbnail_checked_at"), - } + schedule = None + try: + schedule = resolve_animeschedule_anime(snapshot["title"] or existing["title"], route=existing.get("animeschedule_route")) + except Exception: + schedule = None + if not self.has_show(normalized_show_id): + return None + expected_count = parse_nonnegative_int((schedule or {}).get("episodes")) or int(existing.get("expected_count") or 0) + airing_status = normalize_animeschedule_status((schedule or {}).get("status") or existing.get("airing_status")) + payload = ( + snapshot["title"] or existing["title"], + "updated", + watchlist_status_message( + len(sub_episodes), + len(dub_episodes), + expected_count=expected_count, + airing_status=airing_status, + ), + now, + now, + len(sub_episodes), + len(dub_episodes), + expected_count, + airing_status, + sub_episodes[-1] if sub_episodes else None, + dub_episodes[-1] if dub_episodes else None, + (schedule or {}).get("route") or existing.get("animeschedule_route"), + (schedule or {}).get("title") or existing.get("animeschedule_title"), + normalized_show_id, + ) except Exception as exc: - item = { - "show_id": existing["show_id"], - "title": existing["title"], - "status": "error", - "status_message": f"Could not refresh episodes: {exc}", - "created_at": existing["created_at"], - "updated_at": now, - "last_checked": now, - "sub_count": existing["sub_count"], - "dub_count": existing["dub_count"], - "sub_latest_episode": existing.get("sub_latest_episode"), - "dub_latest_episode": existing.get("dub_latest_episode"), - "animeschedule_route": existing.get("animeschedule_route"), - "animeschedule_title": existing.get("animeschedule_title"), - "anidb_aid": existing.get("anidb_aid"), - "anidb_title": existing.get("anidb_title"), - "thumbnail_path": existing.get("thumbnail_path"), - "thumbnail_checked_at": existing.get("thumbnail_checked_at"), - } + if not self.has_show(normalized_show_id): + return None + payload = ( + existing["title"], + "error", + f"Could not refresh episodes: {exc}", + now, + now, + existing["sub_count"], + existing["dub_count"], + existing.get("expected_count"), + existing.get("airing_status"), + existing.get("sub_latest_episode"), + existing.get("dub_latest_episode"), + existing.get("animeschedule_route"), + existing.get("animeschedule_title"), + normalized_show_id, + ) with self.lock, self._connect() as conn: - self._upsert_conn(conn, item) - try: - self.ensure_thumbnail(show_id) - except Exception: - pass - return self.get(show_id) + cursor = conn.execute( + """ + UPDATE watchlist + SET + title = ?, + status = ?, + status_message = ?, + updated_at = ?, + last_checked = ?, + sub_count = ?, + dub_count = ?, + expected_count = ?, + airing_status = ?, + sub_latest_episode = ?, + dub_latest_episode = ?, + animeschedule_route = ?, + animeschedule_title = ? + WHERE show_id = ? + """, + payload, + ) + if cursor.rowcount < 1: + return None + if include_thumbnail and self.has_show(normalized_show_id): + try: + self.ensure_thumbnail(normalized_show_id) + except Exception: + pass + return self.get(normalized_show_id) def refresh_all(self): - refreshed = [self.refresh(show_id) for show_id in self.all_show_ids()] + refreshed = [] + for show_id in self.all_show_ids(): + item = self.refresh(show_id, include_thumbnail=False) + if item is not None: + refreshed.append(item) return {"count": len(refreshed), "items": refreshed, "message": f"Refreshed {len(refreshed)} watchlist entries."} + def update_category(self, show_id, category): + existing = self.get(show_id) + now = now_iso() + with self.lock, self._connect() as conn: + conn.execute( + "UPDATE watchlist SET category = ?, updated_at = ? WHERE show_id = ?", + (normalize_watchlist_category(category), now, existing["show_id"]), + ) + return self.get(show_id) + + def queue_refresh(self, show_id, status_message="Queued watchlist refresh."): + existing = self.get(show_id) + now = now_iso() + with self.lock, self._connect() as conn: + conn.execute( + "UPDATE watchlist SET status = 'queued', status_message = ?, updated_at = ? WHERE show_id = ?", + (str(status_message or "Queued watchlist refresh.").strip(), now, existing["show_id"]), + ) + return self.schedule_refresh(existing["show_id"]) + + def mark_downloaded(self, show_id, title=""): + normalized_show_id = str(show_id or "").strip() + if not normalized_show_id: + raise ValueError("Missing show_id for watchlist download sync.") + + created = False + try: + existing = self.get(normalized_show_id) + except KeyError: + created = True + now = now_iso() + existing = { + "show_id": normalized_show_id, + "title": str(title or "Unknown Anime").strip() or "Unknown Anime", + "category": "finished", + "downloaded": True, + "status": "queued", + "status_message": "Queued refresh after successful download.", + "created_at": now, + "updated_at": now, + "last_checked": None, + "sub_count": 0, + "dub_count": 0, + "expected_count": 0, + "airing_status": None, + "sub_latest_episode": None, + "dub_latest_episode": None, + "animeschedule_route": None, + "animeschedule_title": None, + "anidb_aid": None, + "anidb_title": None, + "thumbnail_path": None, + "thumbnail_checked_at": None, + } + with self.lock, self._connect() as conn: + self._upsert_conn(conn, existing) + else: + with self.lock, self._connect() as conn: + conn.execute( + """ + UPDATE watchlist + SET downloaded = 1, status = 'queued', status_message = ?, updated_at = ? + WHERE show_id = ? + """, + ("Queued refresh after successful download.", now_iso(), normalized_show_id), + ) + self.schedule_refresh(normalized_show_id) + return self.get(normalized_show_id) + def remove(self, show_id): existing = self.get(show_id) thumb_path = thumbnail_file_path(existing.get("thumbnail_path")) @@ -1292,7 +1539,10 @@ class WatchlistStore: cursor = conn.execute("DELETE FROM watchlist WHERE show_id = ?", (str(show_id).strip(),)) if cursor.rowcount < 1: raise KeyError("Anime not found in watchlist.") - self.thumbnail_events.pop(str(show_id).strip(), None) + event = self.thumbnail_events.pop(str(show_id).strip(), None) + if event is not None: + event.set() + self._drop_refresh_locked(show_id) if thumb_path and thumb_path.exists(): thumb_path.unlink(missing_ok=True) return {"ok": True, "message": f"Removed {existing['title']} from the watchlist.", "item": existing} @@ -1350,6 +1600,9 @@ class WatchlistStore: if event: wait_event.wait(timeout=20) + if not self.has_show(show_id): + debug_log("thumbnail.ensure.waited_removed", show_id=show_id) + return None refreshed = self.get(show_id) refreshed_path = thumbnail_file_path(refreshed.get("thumbnail_path")) debug_log("thumbnail.ensure.waited_for_inflight", show_id=show_id, resolved_path=refreshed_path) @@ -1432,10 +1685,10 @@ class WatchlistStore: return thumbnail_file_path(cached_name) finally: if owns_event: + wait_event.set() with self.lock: inflight = self.thumbnail_events.get(show_id) if inflight is wait_event: - inflight.set() self.thumbnail_events.pop(show_id, None) def ensure_thumbnails(self, show_ids, limit=6, force=False): @@ -1446,7 +1699,10 @@ class WatchlistStore: normalized.append(text) updated = [] for show_id in normalized[: max(1, min(int(limit or 6), 10))]: - item = self.get(show_id) + try: + item = self.get(show_id) + except KeyError: + continue if not force and item.get("thumbnail_ready"): updated.append(item) continue @@ -1454,7 +1710,10 @@ class WatchlistStore: self.ensure_thumbnail(show_id, force=force) except Exception: pass - updated.append(self.get(show_id)) + try: + updated.append(self.get(show_id)) + except KeyError: + continue return {"items": updated} @@ -1463,15 +1722,21 @@ def add_to_watchlist(payload): return WATCHLIST.add(payload) -def get_watchlist(page=1, per_page=30): +def get_watchlist(page=1, per_page=30, category=None): ensure_runtime() - return WATCHLIST.list(page=page, per_page=per_page) + return WATCHLIST.list(page=page, per_page=per_page, category=category) def update_watchlist_status(show_id, _mode=None): ensure_runtime() - item = WATCHLIST.refresh(show_id) - return {"message": f"Updated {item['title']}.", "item": item} + item = WATCHLIST.queue_refresh(show_id) + return {"message": f"Queued refresh for {item['title']}.", "item": item} + + +def update_watchlist_category(show_id, category): + ensure_runtime() + item = WATCHLIST.update_category(show_id, category) + return {"message": f"Moved {item['title']} to {item['category_label']}.", "item": item} def update_all_watchlist_statuses(): @@ -1494,732 +1759,81 @@ def upload_watchlist_thumbnail(payload): return WATCHLIST.set_manual_thumbnail(payload["show_id"], payload.get("filename"), payload.get("data_url")) -class WatchlistRefreshJobs: - def __init__(self, store, start_worker=True): - self.store = store - self.lock = threading.RLock() - self.wakeup = threading.Event() - self.pending = [] - self.jobs = [] - self.current_job_id = None - self._init_db() - self._restore_interrupted_jobs() - self._load_jobs() - self.worker = None - if start_worker: - self.ensure_worker() - - def _connect(self): - conn = sqlite3.connect(STATE_DB_PATH, timeout=30) - conn.row_factory = sqlite3.Row - return conn - - def ensure_worker(self): - with self.lock: - if self.worker is not None and self.worker.is_alive(): - return - self.worker = threading.Thread(target=self._run, daemon=True) - self.worker.start() - - def _init_db(self): - STATE_DB_PATH.parent.mkdir(parents=True, exist_ok=True) - with self._connect() as conn: - conn.execute( - """ - CREATE TABLE IF NOT EXISTS watchlist_refresh_jobs ( - id TEXT PRIMARY KEY, - status TEXT NOT NULL, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL, - payload TEXT NOT NULL - ) - """ - ) - conn.execute("CREATE INDEX IF NOT EXISTS idx_watchlist_refresh_created_at ON watchlist_refresh_jobs(created_at)") - conn.execute("CREATE INDEX IF NOT EXISTS idx_watchlist_refresh_status ON watchlist_refresh_jobs(status)") - - def _job_from_row(self, row): - payload = row["payload"] if isinstance(row, sqlite3.Row) else row[0] - return json.loads(payload) - - def _upsert_job_conn(self, conn, job): - conn.execute( - """ - INSERT INTO watchlist_refresh_jobs (id, status, created_at, updated_at, payload) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT(id) DO UPDATE SET - status = excluded.status, - updated_at = excluded.updated_at, - payload = excluded.payload - """, - ( - job["id"], - job["status"], - job["created_at"], - job["updated_at"], - json.dumps(job, sort_keys=True), - ), - ) - - def _trim_history_conn(self, conn): - rows = conn.execute( - "SELECT id FROM watchlist_refresh_jobs ORDER BY created_at DESC LIMIT -1 OFFSET 10" - ).fetchall() - if rows: - conn.executemany("DELETE FROM watchlist_refresh_jobs WHERE id = ?", [(row["id"],) for row in rows]) - - def _save_job_locked(self, job): - with self._connect() as conn: - self._upsert_job_conn(conn, job) - self._trim_history_conn(conn) - - def _load_jobs(self): - with self.lock, self._connect() as conn: - rows = conn.execute( - "SELECT payload FROM watchlist_refresh_jobs ORDER BY created_at DESC LIMIT 10" - ).fetchall() - self.jobs = [self._job_from_row(row) for row in rows] - - def _restore_interrupted_jobs(self): - with self.lock, self._connect() as conn: - rows = conn.execute( - "SELECT payload FROM watchlist_refresh_jobs WHERE status IN ('pending', 'running')" - ).fetchall() - for row in rows: - job = self._job_from_row(row) - job["status"] = "interrupted" - job["finished_at"] = now_iso() - job["updated_at"] = job["finished_at"] - job["current_title"] = None - job["message"] = "Interrupted by web app restart." - self._upsert_job_conn(conn, job) - self._trim_history_conn(conn) - - def _copy_job(self, job): - return json.loads(json.dumps(job)) - - def _find_locked(self, job_id): - for job in self.jobs: - if job["id"] == job_id: - return job - return None - - def status(self): - with self.lock: - job = next((candidate for candidate in self.jobs if candidate["status"] in {"pending", "running"}), None) - if job is None: - job = self.jobs[0] if self.jobs else None - active = bool(job and job["status"] in {"pending", "running"}) - return {"job": self._copy_job(job) if job else None, "running": active} - - def start(self): - with self.lock: - job = next((candidate for candidate in self.jobs if candidate["status"] in {"pending", "running"}), None) - if job is not None: - return { - "message": "Watchlist refresh already queued." if job["status"] == "pending" else "Watchlist refresh already running.", - "job": self._copy_job(job) if job else None, - "created": False, - } - now = now_iso() - job = { - "id": uuid4().hex, - "status": "pending", - "created_at": now, - "updated_at": now, - "started_at": None, - "finished_at": None, - "total": 0, - "completed": 0, - "errors": 0, - "current_title": None, - "message": "Queued watchlist refresh.", - } - self.jobs.insert(0, job) - self.jobs = self.jobs[:10] - self._save_job_locked(job) - self.pending.append(job["id"]) - self.wakeup.set() - return {"message": "Watchlist refresh started.", "job": self._copy_job(job), "created": True} - - def _next_pending(self): - with self.lock: - while self.pending: - job_id = self.pending.pop(0) - job = self._find_locked(job_id) - if job is not None: - self.current_job_id = job_id - return job - return None - - def _run(self): - while True: - job = self._next_pending() - if not job: - self.wakeup.wait(2) - self.wakeup.clear() - continue - self._run_job(job) - - def _run_job(self, job): - try: - show_ids = self.store.all_show_ids() - started_at = now_iso() - with self.lock: - job["status"] = "running" - job["started_at"] = started_at - job["updated_at"] = started_at - job["total"] = len(show_ids) - job["completed"] = 0 - job["errors"] = 0 - job["current_title"] = None - job["message"] = f"Refreshing 0/{len(show_ids)} watchlist entries." - self._save_job_locked(job) - - for index, show_id in enumerate(show_ids, start=1): - title = show_id - errored = False - try: - item = self.store.refresh(show_id) - title = item.get("title") or title - errored = item.get("status") == "error" - except Exception: - errored = True - with self.lock: - job["completed"] = index - job["errors"] += 1 if errored else 0 - job["current_title"] = title - job["updated_at"] = now_iso() - job["message"] = f"Refreshing {index}/{len(show_ids)}: {title}" - self._save_job_locked(job) - - finished_at = now_iso() - with self.lock: - job["status"] = "done" - job["finished_at"] = finished_at - job["updated_at"] = finished_at - job["current_title"] = None - if job["errors"]: - job["message"] = ( - f"Refreshed {job['completed']} watchlist entries with {job['errors']} refresh errors." - ) - else: - job["message"] = f"Refreshed {job['completed']} watchlist entries." - self._save_job_locked(job) - except Exception as exc: - finished_at = now_iso() - with self.lock: - job["status"] = "failed" - job["finished_at"] = finished_at - job["updated_at"] = finished_at - job["current_title"] = None - job["message"] = f"Watchlist refresh failed: {exc}" - self._save_job_locked(job) - finally: - with self.lock: - self.current_job_id = None - - -class DownloadQueue: - def __init__(self, start_worker=True): - self.lock = threading.RLock() - self.wakeup = threading.Event() - self.current_process = None - self.current_job_id = None - self.current_job = None - self._init_db() - self._migrate_json_queue() - self._restore_interrupted_jobs() - self.worker = None - if start_worker: - self.ensure_worker() - - def _connect(self): - conn = sqlite3.connect(STATE_DB_PATH, timeout=30) - conn.row_factory = sqlite3.Row - return conn - - def ensure_worker(self): - with self.lock: - if self.worker is not None and self.worker.is_alive(): - return - self.worker = threading.Thread(target=self._run, daemon=True) - self.worker.start() - - def _init_db(self): - STATE_DB_PATH.parent.mkdir(parents=True, exist_ok=True) - with self._connect() as conn: - conn.execute( - """ - CREATE TABLE IF NOT EXISTS jobs ( - id TEXT PRIMARY KEY, - status TEXT NOT NULL, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL, - payload TEXT NOT NULL - ) - """ - ) - conn.execute("CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status)") - conn.execute("CREATE INDEX IF NOT EXISTS idx_jobs_created_at ON jobs(created_at)") - - def _migrate_json_queue(self): - if not QUEUE_PATH.exists(): - return - jobs = load_json(QUEUE_PATH, []) - if not jobs: - QUEUE_PATH.rename(QUEUE_PATH.with_suffix(".json.migrated")) - return - with self.lock, self._connect() as conn: - existing = conn.execute("SELECT COUNT(*) AS count FROM jobs").fetchone()["count"] - if existing: - return - for job in jobs: - if isinstance(job, dict) and job.get("id"): - self._upsert_job_conn(conn, job) - QUEUE_PATH.rename(QUEUE_PATH.with_suffix(".json.migrated")) - - def _restore_interrupted_jobs(self): - with self.lock, self._connect() as conn: - rows = conn.execute("SELECT payload FROM jobs WHERE status = 'running'").fetchall() - for row in rows: - job = self._job_from_row(row) - job["status"] = "pending" - job["pid"] = None - job.setdefault("log", []).append("Restarted after web app restart.") - self._upsert_job_conn(conn, job) - - def _job_from_row(self, row): - payload = row["payload"] if isinstance(row, sqlite3.Row) else row[0] - return json.loads(payload) - - def _upsert_job_conn(self, conn, job): - clean = dict(job) - clean.pop("process", None) - conn.execute( - """ - INSERT INTO jobs (id, status, created_at, updated_at, payload) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT(id) DO UPDATE SET - status = excluded.status, - updated_at = excluded.updated_at, - payload = excluded.payload - """, - ( - clean["id"], - clean.get("status", "pending"), - clean.get("created_at") or now_iso(), - clean.get("updated_at") or now_iso(), - json.dumps(clean, sort_keys=True), - ), - ) - - def _save_job_locked(self, job): - with self._connect() as conn: - self._upsert_job_conn(conn, job) - - def list(self, page=1, per_page=10): - page = max(1, int(page or 1)) - per_page = min(50, max(1, int(per_page or 10))) - offset = (page - 1) * per_page - with self.lock, self._connect() as conn: - total = conn.execute("SELECT COUNT(*) AS count FROM jobs").fetchone()["count"] - rows = conn.execute( - "SELECT payload FROM jobs ORDER BY created_at DESC LIMIT ? OFFSET ?", - (per_page, offset), - ).fetchall() - pages = max(1, (total + per_page - 1) // per_page) - return { - "jobs": [self._job_from_row(row) for row in rows], - "page": page, - "per_page": per_page, - "total": total, - "pages": pages, - } - - def add(self, payload): - job = build_job(payload) - with self.lock: - self._save_job_locked(job) - self.wakeup.set() - return job - - def retry(self, job_id): - with self.lock: - job = self._find(job_id) - if job["status"] == "running": - raise ValueError("Running jobs cannot be retried") - job["status"] = "pending" - job["exit_code"] = None - job["pid"] = None - job["started_at"] = None - job["finished_at"] = None - job["updated_at"] = now_iso() - job["log"] = [] - self._save_job_locked(job) - self.wakeup.set() - return job - - def retry_all_failed(self): - count = 0 - with self.lock, self._connect() as conn: - rows = conn.execute("SELECT payload FROM jobs WHERE status = 'failed'").fetchall() - for row in rows: - job = self._job_from_row(row) - job["status"] = "pending" - job["exit_code"] = None - job["pid"] = None - job["started_at"] = None - job["finished_at"] = None - job["updated_at"] = now_iso() - job["log"] = [] - self._upsert_job_conn(conn, job) - count += 1 - if count: - self.wakeup.set() - return {"ok": True, "count": count} - - def cancel(self, job_id): - with self.lock: - job = self._find(job_id) - if job["status"] == "pending": - job["status"] = "canceled" - job["updated_at"] = now_iso() - self._save_job_locked(job) - return job - if job["status"] != "running": - return job - if self.current_job_id == job_id and self.current_process: - try: - os.killpg(os.getpgid(self.current_process.pid), signal.SIGTERM) - except OSError: - self.current_process.terminate() - updated_at = now_iso() - job["cancel_requested"] = True - job["updated_at"] = updated_at - if self.current_job is not None and self.current_job.get("id") == job_id: - self.current_job["cancel_requested"] = True - self.current_job["updated_at"] = updated_at - self._save_job_locked(job) - return job - raise ValueError("Could not find the running process") - - def remove(self, job_id): - with self.lock: - job = self._find(job_id) - if job["status"] == "running": - raise ValueError("Running jobs must be canceled before removing") - with self._connect() as conn: - conn.execute("DELETE FROM jobs WHERE id = ?", (job_id,)) - return {"ok": True} - - def clear_finished(self): - with self.lock, self._connect() as conn: - cursor = conn.execute("DELETE FROM jobs WHERE status IN ('done', 'canceled')") - return {"ok": True, "count": cursor.rowcount} - - def _find(self, job_id): - with self._connect() as conn: - row = conn.execute("SELECT payload FROM jobs WHERE id = ?", (job_id,)).fetchone() - if row: - return self._job_from_row(row) - raise KeyError("Job not found") - - def _next_pending(self): - with self.lock: - with self._connect() as conn: - row = conn.execute( - "SELECT payload FROM jobs WHERE status = 'pending' ORDER BY created_at ASC LIMIT 1" - ).fetchone() - if row: - return self._job_from_row(row) - return None - - def _append_log(self, job, line): - text = strip_control(line).strip() - if not text: - return - with self.lock: - job.setdefault("log", []).append(text) - job["log"] = job["log"][-MAX_LOG_LINES:] - job["updated_at"] = now_iso() - self._save_job_locked(job) - - def _run(self): - while True: - job = self._next_pending() - if not job: - self.wakeup.wait(2) - self.wakeup.clear() - continue - self._run_job(job) - - def _run_job(self, job): - command = command_for_job(job) - staging_dir = job_staging_dir(job) - target_dir = job_output_dir(job) - env = os.environ.copy() - env.update( - { - "ANI_CLI_DOWNLOAD_DIR": str(staging_dir), - "ANI_CLI_MODE": job["mode"], - "ANI_CLI_QUALITY": job["quality"], - "TERM": env.get("TERM", "xterm-256color"), - } - ) - staging_dir.mkdir(parents=True, exist_ok=True) - target_dir.mkdir(parents=True, exist_ok=True) - - with self.lock: - job["status"] = "running" - job["started_at"] = now_iso() - job["updated_at"] = job["started_at"] - job["command"] = printable_command(command) - job["staging_dir"] = str(staging_dir) - job["target_dir"] = str(target_dir) - job["log"] = [ - f"Starting: {job['command']}", - f"Staging in: {job['staging_dir']}", - f"Final folder: {job['target_dir']}", - ] - job["cancel_requested"] = False - self._save_job_locked(job) - - process = subprocess.Popen( - command, - cwd=str(PROJECT_ROOT), - env=env, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - bufsize=1, - start_new_session=True, - ) - with self.lock: - self.current_process = process - self.current_job_id = job["id"] - self.current_job = job - job["pid"] = process.pid - self._save_job_locked(job) - - assert process.stdout is not None - for line in process.stdout: - self._append_log(job, line) - exit_code = process.wait() - finalize_error = None - moved_files = [] - if exit_code == 0 and not job.get("cancel_requested"): - try: - moved_files = finalize_library_files(job) - except Exception as exc: - finalize_error = exc - - with self.lock: - canceled = job.get("cancel_requested") - job["exit_code"] = 1 if finalize_error else exit_code - job["pid"] = None - job["finished_at"] = now_iso() - job["updated_at"] = job["finished_at"] - if canceled: - job["status"] = "canceled" - job.setdefault("log", []).append("Canceled.") - elif finalize_error: - job["status"] = "failed" - job.setdefault("log", []).append(f"Library rename failed: {finalize_error}") - elif exit_code == 0: - job["status"] = "done" - for path in moved_files: - job.setdefault("log", []).append(f"Saved: {path}") - job.setdefault("log", []).append("Download completed.") - else: - job["status"] = "failed" - job.setdefault("log", []).append(f"Download failed with exit code {exit_code}.") - self.current_process = None - self.current_job_id = None - self.current_job = None - self._save_job_locked(job) - - -def strip_control(value): - return re.sub(r"\x1b\[[0-9;?]*[A-Za-z]", "", str(value)).replace("\r", "\n") - - -def printable_command(command): - return " ".join(sh_quote(part) for part in command) - - -def sh_quote(value): - if re.match(r"^[A-Za-z0-9_./:=+-]+$", value): - return value - return "'" + value.replace("'", "'\"'\"'") + "'" - - -def validate_episode_spec(value): - text = str(value or "").strip() - if not text: - raise ValueError("Choose at least one episode") - if not re.match(r"^[0-9.\-\s]+$", text): - raise ValueError("Episodes can contain numbers, dots, spaces, and ranges") - return re.sub(r"\s+", " ", text) - - -def sanitize_path_component(value, fallback="Anime"): - text = str(value or "").strip() - text = re.sub(r'[\\/:*?"<>|]+', "-", text) - text = re.sub(r"\s+", " ", text).strip(" .") - return text or fallback - - -def normalize_season(value): - text = str(value or "1").strip() - if not re.match(r"^[0-9]+$", text): - raise ValueError("Season must be a positive number") - number = int(text, 10) - if number < 1: - raise ValueError("Season must be a positive number") - return str(number) - - -def job_output_dir(job): - anime_name = sanitize_path_component(job.get("anime_name") or job.get("title")) - return Path(job["download_dir"]).expanduser() / anime_name / f"Season {season_label(job)}" - - -def job_staging_dir(job): - return STAGING_ROOT / job["id"] - - -def season_label(job): - return f"{int(job.get('season') or 1):02d}" - - -def episode_label(value): - text = str(value or "").strip() - match = re.match(r"^0*([0-9]+)(.*)$", text) - if match: - number = int(match.group(1) or "0", 10) - suffix = re.sub(r"[^0-9A-Za-z.]+", "-", match.group(2)).strip("-") - return f"{number:02d}{suffix}" - return re.sub(r"[^0-9A-Za-z.]+", "-", text).strip("-") or "00" - - -def extract_episode_from_filename(path): - match = re.search(r"\bEpisode\s+([0-9][0-9A-Za-z.\-]*)\s*$", path.stem) - return match.group(1) if match else None - - -def unique_destination(path): - if not path.exists(): - return path - counter = 2 - while True: - candidate = path.with_name(f"{path.stem} ({counter}){path.suffix}") - if not candidate.exists(): - return candidate - counter += 1 - - -def finalize_library_files(job): - staging_dir = job_staging_dir(job) - target_dir = job_output_dir(job) - library_name = sanitize_path_component(job.get("anime_name") or job.get("title")) - season = season_label(job) - target_dir.mkdir(parents=True, exist_ok=True) - files = sorted( - [path for path in staging_dir.iterdir() if path.is_file()], - key=lambda path: (path.stat().st_mtime, path.name), - ) - if not files: - raise RuntimeError("No downloaded files were found in the staging folder") - - moved = [] - fallback_episode = 1 - for source in files: - ep = extract_episode_from_filename(source) - if ep is None: - ep = str(fallback_episode) - fallback_episode += 1 - final_name = f"{library_name} - S{season}E{episode_label(ep)}{source.suffix}" - destination = unique_destination(target_dir / final_name) - shutil.move(str(source), str(destination)) - moved.append(str(destination)) - +def resolve_job_watchlist_identity(job, config=None): + show_id = str(job.get("show_id") or "").strip() + title = str(job.get("title") or job.get("anime_name") or "").strip() + if show_id: + return show_id, title try: - staging_dir.rmdir() - except OSError: - pass - return moved - - -def build_job(payload): - if not isinstance(payload, dict): - raise ValueError("Expected a JSON object") - query = str(payload.get("query") or payload.get("title") or "").strip() - title = str(payload.get("title") or query).strip() - anime_name = sanitize_path_component(payload.get("anime_name") or title) - if not query: - raise ValueError("Missing search query") + index = max(1, int(job.get("result_index") or 1)) + except (TypeError, ValueError): + return "", title try: - index = int(payload.get("index", 1)) - except (TypeError, ValueError) as exc: - raise ValueError("Invalid result index") from exc - if index < 1: - raise ValueError("Result index must be positive") + active_config = config or CONFIG or DEFAULT_CONFIG + results = search_anime(job.get("query") or title, job.get("mode") or active_config["mode"]) + except Exception: + return "", title + if index > len(results): + return "", title + result = results[index - 1] + return str(result.get("id") or "").strip(), str(result.get("title") or title).strip() - mode = str(payload.get("mode") or CONFIG["mode"]).lower() - quality = str(payload.get("quality") or CONFIG["quality"]).lower() - download_dir = str(payload.get("download_dir") or CONFIG["download_dir"]).strip() - if mode not in MODE_CHOICES: - raise ValueError("Mode must be sub or dub") - if quality not in QUALITY_CHOICES: - raise ValueError("Unknown quality") +def sync_downloaded_job_to_watchlist(job): + ensure_runtime() + show_id, title = resolve_job_watchlist_identity(job) + if not show_id: + raise ValueError("Could not resolve a watchlist show_id for the completed download.") + return WATCHLIST.mark_downloaded(show_id, title=title) + +def prepare_download_job(job): + config = ensure_runtime()["config"] + show_id, title = resolve_job_watchlist_identity(job, config=config) + if show_id: + job["show_id"] = show_id + if title: + job["title"] = title + return job + + +def get_config_snapshot(fallback=None): + with RUNTIME_LOCK: + source = CONFIG if CONFIG is not None else fallback + if source is not None: + return dict(source) + return normalize_config(load_json(CONFIG_PATH, DEFAULT_CONFIG)) + + +def save_runtime_config(config): + normalized = normalize_config(config) + with RUNTIME_LOCK: + global CONFIG + CONFIG = dict(normalized) + write_json(CONFIG_PATH, CONFIG) + return dict(CONFIG) + + +def runtime_state(): return { - "id": uuid4().hex, - "title": title, - "anime_name": anime_name, - "season": normalize_season(payload.get("season")), - "query": query, - "result_index": index, - "mode": mode, - "quality": quality, - "episodes": validate_episode_spec(payload.get("episodes")), - "download_dir": str(Path(download_dir).expanduser()), - "status": "pending", - "created_at": now_iso(), - "updated_at": now_iso(), - "started_at": None, - "finished_at": None, - "exit_code": None, - "pid": None, - "log": [], + "config": CONFIG, + "watchlist": WATCHLIST, + "download_queue": DOWNLOAD_QUEUE, + "watchlist_refresh": WATCHLIST_REFRESH, } -def command_for_job(job): - command = [ - ANI_CLI, - "-d", - "-q", - job["quality"], - "-e", - job["episodes"], - "-S", - str(job["result_index"]), - ] - if job["mode"] == "dub": - command.append("--dub") - command.append(job["query"]) - return command - - def build_runtime(start_workers=None): migrate_legacy_state_storage() - worker_state = background_workers_enabled() if start_workers is None else bool(start_workers) + worker_state = background_workers_enabled() if start_workers is None else bool(start_workers) and background_workers_enabled() config = normalize_config(load_json(CONFIG_PATH, DEFAULT_CONFIG)) write_json(CONFIG_PATH, config) - watchlist = WatchlistStore() - download_queue = DownloadQueue(start_worker=worker_state) + watchlist = WatchlistStore(start_worker=worker_state) + download_queue = DownloadQueue( + config_getter=lambda: get_config_snapshot(config), + watchlist_sync_fn=sync_downloaded_job_to_watchlist, + job_prepare_fn=prepare_download_job, + start_worker=worker_state, + ) watchlist_refresh = WatchlistRefreshJobs(watchlist, start_worker=worker_state) return { "config": config, @@ -2232,16 +1846,14 @@ def build_runtime(start_workers=None): def initialize_runtime(start_workers=None): global CONFIG, WATCHLIST, DOWNLOAD_QUEUE, WATCHLIST_REFRESH with RUNTIME_LOCK: + workers_allowed = background_workers_enabled() if CONFIG is not None and WATCHLIST is not None and DOWNLOAD_QUEUE is not None and WATCHLIST_REFRESH is not None: - if start_workers: + if start_workers and workers_allowed: + WATCHLIST.worker_enabled = True + WATCHLIST.ensure_worker() DOWNLOAD_QUEUE.ensure_worker() WATCHLIST_REFRESH.ensure_worker() - return { - "config": CONFIG, - "watchlist": WATCHLIST, - "download_queue": DOWNLOAD_QUEUE, - "watchlist_refresh": WATCHLIST_REFRESH, - } + return runtime_state() runtime = build_runtime(start_workers=start_workers) CONFIG = runtime["config"] WATCHLIST = runtime["watchlist"] @@ -2253,257 +1865,68 @@ def initialize_runtime(start_workers=None): def ensure_runtime(start_workers=False): if CONFIG is None or WATCHLIST is None or DOWNLOAD_QUEUE is None or WATCHLIST_REFRESH is None: return initialize_runtime(start_workers=start_workers) - return { - "config": CONFIG, - "watchlist": WATCHLIST, - "download_queue": DOWNLOAD_QUEUE, - "watchlist_refresh": WATCHLIST_REFRESH, - } + if start_workers: + return initialize_runtime(start_workers=True) + return runtime_state() -def dependency_status(): - checks = ["curl", "sed", "grep", "openssl", "fzf", "aria2c", "ffmpeg", "yt-dlp"] - result = {name: bool(shutil.which(name)) for name in checks} - result["ani-cli"] = bool(shutil.which(ANI_CLI) or (Path(ANI_CLI).exists() and os.access(ANI_CLI, os.X_OK))) - return result +def shutdown_runtime(wait=False, cancel_active_downloads=None): + services = None + with RUNTIME_LOCK: + if CONFIG is None and WATCHLIST is None and DOWNLOAD_QUEUE is None and WATCHLIST_REFRESH is None: + return False + services = (WATCHLIST, DOWNLOAD_QUEUE, WATCHLIST_REFRESH) + watchlist, download_queue, watchlist_refresh = services + should_cancel_downloads = bool(wait) if cancel_active_downloads is None else bool(cancel_active_downloads) + if watchlist is not None: + watchlist.shutdown(wait=wait) + if download_queue is not None: + download_queue.shutdown(wait=wait, cancel_active=should_cancel_downloads) + if watchlist_refresh is not None: + watchlist_refresh.shutdown(wait=wait) + return True -class Handler(BaseHTTPRequestHandler): - server_version = "AniCliWeb/1.0" - - def ensure_client_access(self): - if remote_access_allowed(): - return - client_host = self.client_address[0] if self.client_address else "" - if client_address_is_local(client_host): - return - raise PermissionError( - "Remote access is disabled. Set ANI_CLI_WEB_ALLOW_REMOTE=1 to allow non-local clients." - ) - - def do_GET(self): - parsed = urlparse(self.path) - try: - self.ensure_client_access() - ensure_runtime() - if parsed.path == "/": - self.html(INDEX_HTML) - elif parsed.path == "/config": - self.html(CONFIG_HTML) - elif parsed.path == "/watchlist": - self.html(WATCHLIST_HTML) - elif parsed.path == "/api/config": - self.json(CONFIG) - elif parsed.path == "/api/version": - self.json({"name": APP_NAME, "version": VERSION}) - elif parsed.path == "/api/dependencies": - self.json(dependency_status()) - elif parsed.path == "/api/search": - params = parse_qs(parsed.query) - query = (params.get("q") or [""])[0].strip() - mode = (params.get("mode") or [CONFIG["mode"]])[0].lower() - if not query: - raise ValueError("Search query is required") - if mode not in MODE_CHOICES: - raise ValueError("Mode must be sub or dub") - self.json({"results": search_anime(query, mode)}) - elif parsed.path.startswith("/api/anime/") and parsed.path.endswith("/episodes"): - show_id = unquote(parsed.path[len("/api/anime/") : -len("/episodes")]) - params = parse_qs(parsed.query) - mode = (params.get("mode") or [CONFIG["mode"]])[0].lower() - if mode not in MODE_CHOICES: - raise ValueError("Mode must be sub or dub") - self.json({"episodes": episode_list(show_id, mode)}) - elif parsed.path == "/api/queue": - params = parse_qs(parsed.query) - page = (params.get("page") or ["1"])[0] - per_page = (params.get("per_page") or ["10"])[0] - self.json(DOWNLOAD_QUEUE.list(page=page, per_page=per_page)) - elif parsed.path.startswith("/api/watchlist/thumb/"): - show_id = unquote(parsed.path[len("/api/watchlist/thumb/") :]).strip() - debug_log("thumbnail.route.request", show_id=show_id, path=parsed.path, query=parsed.query) - item = WATCHLIST.get(show_id) - path = thumbnail_file_path(item.get("thumbnail_path")) - if not path or not path.exists(): - debug_log("thumbnail.route.miss", show_id=show_id, resolved_path=path) - self.error(HTTPStatus.NOT_FOUND, "Thumbnail not found") - return - debug_log("thumbnail.route.hit", show_id=show_id, resolved_path=path) - self.file(path) - elif parsed.path in {"/api/watchlist", "/api/watchlist/get"}: - params = parse_qs(parsed.query) - page = (params.get("page") or ["1"])[0] - per_page = (params.get("per_page") or ["30"])[0] - return self.json(get_watchlist(page=page, per_page=per_page)) - elif parsed.path == "/api/watchlist/refresh-status": - self.json(get_watchlist_refresh_status()) - else: - self.error(HTTPStatus.NOT_FOUND, "Not found") - except Exception as exc: - self.exception(exc) - - def do_POST(self): - parsed = urlparse(self.path) - try: - self.ensure_client_access() - ensure_runtime() - if parsed.path == "/api/config": - self.update_config(self.body_json()) - elif parsed.path == "/api/queue": - self.json(DOWNLOAD_QUEUE.add(self.body_json()), HTTPStatus.CREATED) - elif parsed.path == "/api/queue/retry-failed": - self.json(DOWNLOAD_QUEUE.retry_all_failed()) - elif parsed.path == "/api/queue/clear-finished": - self.json(DOWNLOAD_QUEUE.clear_finished()) - elif parsed.path.startswith("/api/queue/"): - parts = parsed.path.strip("/").split("/") - if len(parts) != 4: - self.error(HTTPStatus.NOT_FOUND, "Not found") - return - _, _, job_id, action = parts - actions = { - "retry": DOWNLOAD_QUEUE.retry, - "cancel": DOWNLOAD_QUEUE.cancel, - "remove": DOWNLOAD_QUEUE.remove, - } - if action not in actions: - self.error(HTTPStatus.NOT_FOUND, "Not found") - return - self.json(actions[action](job_id)) - elif parsed.path in {"/api/watchlist", "/api/watchlist/add"}: - self.json(add_to_watchlist(self.body_json()), HTTPStatus.CREATED) - elif parsed.path == "/api/watchlist/ensure-thumbnails": - payload = self.body_json() - self.json( - WATCHLIST.ensure_thumbnails( - payload.get("show_ids") or [], - limit=payload.get("limit", 6), - force=bool(payload.get("force")), - ) - ) - elif parsed.path == "/api/watchlist/update-status": - payload = self.body_json() - self.json(update_watchlist_status(payload["show_id"], payload.get("mode"))) - elif parsed.path == "/api/watchlist/update-all": - result = update_all_watchlist_statuses() - status = HTTPStatus.ACCEPTED if result.get("created") else HTTPStatus.OK - self.json(result, status) - elif parsed.path == "/api/watchlist/remove": - payload = self.body_json() - self.json(remove_from_watchlist(payload["show_id"])) - elif parsed.path == "/api/watchlist/upload-thumbnail": - payload = self.body_json() - self.json(upload_watchlist_thumbnail(payload)) - else: - self.error(HTTPStatus.NOT_FOUND, "Not found") - except Exception as exc: - self.exception(exc) - - def update_config(self, payload): - global CONFIG - config = normalize_config(payload) - Path(config["download_dir"]).expanduser().mkdir(parents=True, exist_ok=True) - CONFIG = config - write_json(CONFIG_PATH, CONFIG) - self.json(CONFIG) - - def body_json(self): - try: - length = int(self.headers.get("Content-Length", "0") or "0") - except ValueError as exc: - raise ValueError("Invalid Content-Length header") from exc - if length < 0: - raise ValueError("Invalid Content-Length header") - if length > MAX_JSON_BODY_BYTES: - raise HttpError(HTTPStatus.REQUEST_ENTITY_TOO_LARGE, "Request body too large.") - raw_bytes = self.rfile.read(length) if length else b"{}" - if len(raw_bytes) > MAX_JSON_BODY_BYTES: - raise HttpError(HTTPStatus.REQUEST_ENTITY_TOO_LARGE, "Request body too large.") - try: - raw = raw_bytes.decode("utf-8") - except UnicodeDecodeError as exc: - raise ValueError("Request body must be valid UTF-8 JSON.") from exc - try: - return json.loads(raw) - except json.JSONDecodeError as exc: - raise ValueError("Invalid JSON body") from exc - - def html(self, content): - data = content.encode("utf-8") - self.write_response_bytes(data, HTTPStatus.OK, {"Content-Type": "text/html; charset=utf-8"}) - - def file(self, path): - payload = Path(path).read_bytes() - content_type = mimetypes.guess_type(str(path))[0] or "application/octet-stream" - debug_log("file.serve", path=path, content_type=content_type, bytes=len(payload)) - self.write_response_bytes( - payload, - HTTPStatus.OK, - { - "Content-Type": content_type, - "Cache-Control": "public, max-age=86400", - }, - ) - - def json(self, payload, status=HTTPStatus.OK): - data = json.dumps(payload).encode("utf-8") - self.write_response_bytes(data, status, {"Content-Type": "application/json"}) - - def write_response_bytes(self, payload, status, headers): - try: - self.send_response(status) - for key, value in headers.items(): - self.send_header(key, value) - self.send_header("Content-Length", str(len(payload))) - self.end_headers() - self.wfile.write(payload) - except CLIENT_DISCONNECT_ERRORS: - return - - def error(self, status, message): - try: - self.json({"error": message}, status) - except CLIENT_DISCONNECT_ERRORS: - return - - def exception(self, exc): - if isinstance(exc, CLIENT_DISCONNECT_ERRORS): - return - if isinstance(exc, HttpError): - self.error(exc.status, exc.message) - elif isinstance(exc, KeyError): - self.error(HTTPStatus.NOT_FOUND, str(exc).strip("'")) - elif isinstance(exc, PermissionError): - self.error(HTTPStatus.FORBIDDEN, str(exc)) - elif isinstance(exc, ValueError): - self.error(HTTPStatus.BAD_REQUEST, str(exc)) - else: - self.error(HTTPStatus.INTERNAL_SERVER_ERROR, str(exc)) - - def log_message(self, fmt, *args): - print(f"{self.address_string()} - {fmt % args}") - -def server_host(): - return os.environ.get("ANI_CLI_WEB_HOST", "127.0.0.1").strip() or "127.0.0.1" +def reset_runtime(wait=False, cancel_active_downloads=None): + global CONFIG, WATCHLIST, DOWNLOAD_QUEUE, WATCHLIST_REFRESH + shutdown_runtime(wait=wait, cancel_active_downloads=cancel_active_downloads) + with RUNTIME_LOCK: + CONFIG = None + WATCHLIST = None + DOWNLOAD_QUEUE = None + WATCHLIST_REFRESH = None -def server_port(): - raw = str(os.environ.get("ANI_CLI_WEB_PORT", "8421")).strip() or "8421" - try: - port = int(raw, 10) - except ValueError as exc: - raise ValueError("ANI_CLI_WEB_PORT must be a valid integer") from exc - if not 1 <= port <= 65535: - raise ValueError("ANI_CLI_WEB_PORT must be between 1 and 65535") - return port +Handler = build_handler_class( + HandlerContext( + add_to_watchlist=add_to_watchlist, + config_html=CONFIG_HTML, + ensure_runtime=ensure_runtime, + episode_list=episode_list, + get_config_snapshot=get_config_snapshot, + get_watchlist=get_watchlist, + get_watchlist_refresh_status=get_watchlist_refresh_status, + http_error=HttpError, + index_html=INDEX_HTML, + remove_from_watchlist=remove_from_watchlist, + runtime_state=runtime_state, + save_runtime_config=save_runtime_config, + search_anime=search_anime, + thumbnail_file_path=thumbnail_file_path, + update_all_watchlist_statuses=update_all_watchlist_statuses, + update_watchlist_category=update_watchlist_category, + update_watchlist_status=update_watchlist_status, + upload_watchlist_thumbnail=upload_watchlist_thumbnail, + watchlist_html=WATCHLIST_HTML, + ) +) def parse_runtime_flags(argv): - global DEBUG_MODE remaining = [] for arg in argv: if arg in {"--debug", "---debug"}: - DEBUG_MODE = True + app_support.DEBUG_MODE = True continue remaining.append(arg) return remaining @@ -2511,7 +1934,6 @@ def parse_runtime_flags(argv): def main(): parse_runtime_flags(sys.argv[1:]) - initialize_runtime(start_workers=True) host = server_host() port = server_port() server = ThreadingHTTPServer((host, port), Handler) @@ -2522,6 +1944,7 @@ def main(): except KeyboardInterrupt: print("\nShutting down server.") finally: + shutdown_runtime(wait=True, cancel_active_downloads=True) server.server_close() diff --git a/app_support.py b/app_support.py new file mode 100644 index 0000000..37f7753 --- /dev/null +++ b/app_support.py @@ -0,0 +1,433 @@ +#!/usr/bin/env python3 + +"""Shared constants and helper utilities for ani-cli-web.""" + +import hmac +import ipaddress +import json +import os +import re +import shutil +from datetime import datetime, timezone +from pathlib import Path +from uuid import uuid4 + + +PROJECT_ROOT = Path(__file__).resolve().parent +ANI_CLI = os.environ.get("ANI_CLI_BIN") or shutil.which("ani-cli") or "ani-cli" +APP_NAME = "ani-cli-web" +VERSION = "0.26.0" +ALLANIME_BASE = "allanime.day" +ALLANIME_API = f"https://api.{ALLANIME_BASE}" +ALLANIME_REFERER = "https://allmanga.to" +ALLMANGA_SITE_BASE = "https://allmanga.to" +ANIMESCHEDULE_API = "https://animeschedule.net/api/v3" +ANIMESCHEDULE_IMAGE_BASE = "https://img.animeschedule.net/production/assets/public/img/" +ANIDB_TITLES_URL = "https://anidb.net/api/anime-titles.xml.gz" +ANIDB_ANIME_PAGE = "https://anidb.net/anime/{aid}" +AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/121.0" +CLIENT_DISCONNECT_ERRORS = (BrokenPipeError, ConnectionResetError, ConnectionAbortedError) +DEBUG_MODE = False + +QUALITY_CHOICES = {"best", "1080", "1080p", "720", "720p", "480", "480p", "360", "360p", "worst"} +MODE_CHOICES = {"sub", "dub"} +WATCHLIST_CATEGORY_LABELS = { + "watching": "Watching", + "planned": "Planned", + "finished": "Finished", + "dropped": "Dropped", +} +WATCHLIST_CATEGORY_CHOICES = tuple(WATCHLIST_CATEGORY_LABELS.keys()) +MAX_LOG_LINES = 240 +MAX_JSON_BODY_BYTES = 12 * 1024 * 1024 + + +def env_flag(name, default=False): + raw = os.environ.get(name) + if raw is None: + return default + return str(raw).strip().lower() in {"1", "true", "yes", "on"} + + +def debug_enabled(): + return DEBUG_MODE or env_flag("ANI_CLI_WEB_DEBUG") + + +def remote_access_allowed(): + return env_flag("ANI_CLI_WEB_ALLOW_REMOTE") + + +def remote_access_token(): + return str(os.environ.get("ANI_CLI_WEB_REMOTE_TOKEN") or "").strip() + + +def remote_access_token_configured(): + return bool(remote_access_token()) + + +def remote_access_token_matches(value): + provided = str(value or "").strip() + expected = remote_access_token() + return bool(expected and provided and hmac.compare_digest(provided, expected)) + + +def background_workers_enabled(): + return not env_flag("ANI_CLI_WEB_DISABLE_WORKER") + + +def client_address_is_local(host): + text = str(host or "").strip() + if not text: + return False + if text == "localhost": + return True + try: + return ipaddress.ip_address(text).is_loopback + except ValueError: + return False + + +def debug_log(event, **fields): + if not debug_enabled(): + return + parts = [f"[debug] {event}"] + for key, value in fields.items(): + text = str(value) + if len(text) > 240: + text = text[:237] + "..." + parts.append(f"{key}={text}") + print(" | ".join(parts), flush=True) + + +def default_download_dir(): + if os.environ.get("ANI_CLI_DOWNLOAD_DIR"): + return os.environ["ANI_CLI_DOWNLOAD_DIR"] + home_downloads = Path.home() / "Downloads" / "Anime" + if os.access(home_downloads.parent if home_downloads.parent.exists() else Path.home(), os.W_OK): + return str(home_downloads) + return str(PROJECT_ROOT / "downloads") + + +DEFAULT_CONFIG = { + "download_dir": default_download_dir(), + "mode": os.environ.get("ANI_CLI_MODE", "sub"), + "quality": os.environ.get("ANI_CLI_QUALITY", "best"), +} + + +def now_iso(): + return datetime.now(timezone.utc).isoformat() + + +def project_app_root(): + configured = str(os.environ.get("ANI_CLI_WEB_STATE_ROOT") or "").strip() + if configured: + return Path(configured).expanduser() + return PROJECT_ROOT / ".ani-cli-web" + + +PROJECT_APP_ROOT = project_app_root() + + +def ensure_project_app_root(): + PROJECT_APP_ROOT.mkdir(parents=True, exist_ok=True) + return PROJECT_APP_ROOT + + +def legacy_state_root(): + return Path(os.environ.get("XDG_STATE_HOME", Path.home() / ".local" / "state")) / APP_NAME + + +def legacy_config_root(): + return Path(os.environ.get("XDG_CONFIG_HOME", Path.home() / ".config")) / APP_NAME + + +def project_state_path(*parts): + return PROJECT_APP_ROOT.joinpath(*parts) + + +def project_state_dir(*parts): + return PROJECT_APP_ROOT.joinpath(*parts) + + +def migrate_legacy_file(project_path, legacy_path, label): + if project_path.exists() or not legacy_path.exists(): + return project_path + project_path.parent.mkdir(parents=True, exist_ok=True) + try: + shutil.move(str(legacy_path), str(project_path)) + debug_log("state.migrated", label=label, from_path=legacy_path, to_path=project_path) + except OSError as exc: + debug_log("state.migrate_failed", label=label, from_path=legacy_path, to_path=project_path, error=exc) + return project_path + + +def migrate_legacy_dir(project_dir, legacy_dir, label): + if not legacy_dir.exists(): + return project_dir + project_dir.mkdir(parents=True, exist_ok=True) + moved = 0 + for child in legacy_dir.iterdir(): + target = project_dir / child.name + if target.exists(): + if child.is_dir() and target.is_dir(): + migrate_legacy_dir(target, child, f"{label}/{child.name}") + continue + try: + shutil.move(str(child), str(target)) + moved += 1 + except OSError as exc: + debug_log("state.migrate_failed", label=label, from_path=child, to_path=target, error=exc) + try: + legacy_dir.rmdir() + except OSError: + pass + if moved: + debug_log("state.migrated", label=label, from_path=legacy_dir, to_path=project_dir, entries=moved) + return project_dir + + +LEGACY_STATE_ROOT = legacy_state_root() +LEGACY_CONFIG_ROOT = legacy_config_root() +LEGACY_CONFIG_PATH = LEGACY_CONFIG_ROOT / "config.json" +LEGACY_QUEUE_PATH = LEGACY_STATE_ROOT / "queue.json" +LEGACY_QUEUE_DB_PATH = LEGACY_STATE_ROOT / "queue.sqlite3" +LEGACY_STAGING_ROOT = LEGACY_STATE_ROOT / "staging" +LEGACY_WATCHLIST_JSON_PATH = LEGACY_STATE_ROOT / "watchlist.json" +LEGACY_THUMBNAIL_ROOT = LEGACY_STATE_ROOT / "thumbnails" +LEGACY_ANIDB_TITLES_PATH = LEGACY_STATE_ROOT / "anidb-anime-titles.xml.gz" + +CONFIG_PATH = project_state_path("config.json") +QUEUE_PATH = project_state_path("queue.json") +STATE_DB_PATH = project_state_path("state.sqlite3") +STAGING_ROOT = project_state_dir("staging") +WATCHLIST_JSON_PATH = project_state_path("watchlist.json") +THUMBNAIL_ROOT = project_state_dir("thumbnails") +THUMBNAIL_RETRY_SECONDS = 15 * 60 +ANIDB_TITLES_PATH = project_state_path("anidb-anime-titles.xml.gz") +ANIDB_TITLES_CACHE = { + "mtime": None, + "entries": None, + "title_index": None, + "token_index": None, + "match_cache": {}, +} + + +def migrate_legacy_state_storage(): + ensure_project_app_root() + migrate_legacy_file(CONFIG_PATH, LEGACY_CONFIG_PATH, "config.json") + migrate_legacy_file(QUEUE_PATH, LEGACY_QUEUE_PATH, "queue.json") + migrate_legacy_file(STATE_DB_PATH, PROJECT_APP_ROOT / "queue.sqlite3", "queue.sqlite3") + migrate_legacy_file(STATE_DB_PATH, LEGACY_QUEUE_DB_PATH, "queue.sqlite3") + migrate_legacy_file(WATCHLIST_JSON_PATH, LEGACY_WATCHLIST_JSON_PATH, "watchlist.json") + migrate_legacy_file(ANIDB_TITLES_PATH, LEGACY_ANIDB_TITLES_PATH, "anidb-anime-titles.xml.gz") + migrate_legacy_dir(STAGING_ROOT, LEGACY_STAGING_ROOT, "staging") + migrate_legacy_dir(THUMBNAIL_ROOT, LEGACY_THUMBNAIL_ROOT, "thumbnails") + +def load_json(path, fallback): + try: + with path.open("r", encoding="utf-8") as handle: + data = json.load(handle) + return data if isinstance(data, type(fallback)) else fallback + except (FileNotFoundError, json.JSONDecodeError, OSError): + return fallback + + +def write_json(path, data): + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(path.suffix + ".tmp") + with tmp.open("w", encoding="utf-8") as handle: + json.dump(data, handle, indent=2, sort_keys=True) + tmp.replace(path) + + +def normalize_config(data): + config = dict(DEFAULT_CONFIG) + if isinstance(data, dict): + config.update({key: data[key] for key in DEFAULT_CONFIG if key in data}) + + mode = str(config.get("mode", "sub")).lower() + quality = str(config.get("quality", "best")).lower() + if quality.endswith("p") and quality[:-1].isdigit(): + quality = quality[:-1] + download_dir = str(config.get("download_dir") or DEFAULT_CONFIG["download_dir"]) + + config["mode"] = mode if mode in MODE_CHOICES else "sub" + config["quality"] = quality if quality in QUALITY_CHOICES else "best" + config["download_dir"] = str(Path(download_dir).expanduser()) + return config + + +def strip_control(value): + return re.sub(r"\x1b\[[0-9;?]*[A-Za-z]", "", str(value)).replace("\r", "\n") + + +def printable_command(command): + return " ".join(sh_quote(part) for part in command) + + +def sh_quote(value): + if re.match(r"^[A-Za-z0-9_./:=+-]+$", value): + return value + return "'" + value.replace("'", "'\"'\"'") + "'" + + +def validate_episode_spec(value): + text = str(value or "").strip() + if not text: + raise ValueError("Choose at least one episode") + if not re.match(r"^[0-9.\-\s]+$", text): + raise ValueError("Episodes can contain numbers, dots, spaces, and ranges") + return re.sub(r"\s+", " ", text) + + +def sanitize_path_component(value, fallback="Anime"): + text = str(value or "").strip() + text = re.sub(r'[\\/:*?"<>|]+', "-", text) + text = re.sub(r"\s+", " ", text).strip(" .") + return text or fallback + + +def normalize_season(value): + text = str(value or "1").strip() + if not re.match(r"^[0-9]+$", text): + raise ValueError("Season must be a positive number") + number = int(text, 10) + if number < 1: + raise ValueError("Season must be a positive number") + return str(number) + + +def job_output_dir(job): + anime_name = sanitize_path_component(job.get("anime_name") or job.get("title")) + return Path(job["download_dir"]).expanduser() / anime_name / f"Season {season_label(job)}" + + +def job_staging_dir(job): + return STAGING_ROOT / job["id"] + + +def season_label(job): + return f"{int(job.get('season') or 1):02d}" + + +def episode_label(value): + text = str(value or "").strip() + match = re.match(r"^0*([0-9]+)(.*)$", text) + if match: + number = int(match.group(1) or "0", 10) + suffix = re.sub(r"[^0-9A-Za-z.]+", "-", match.group(2)).strip("-") + return f"{number:02d}{suffix}" + return re.sub(r"[^0-9A-Za-z.]+", "-", text).strip("-") or "00" + + +def extract_episode_from_filename(path): + match = re.search(r"\bEpisode\s+([0-9][0-9A-Za-z.\-]*)\s*$", path.stem) + return match.group(1) if match else None + + +def unique_destination(path): + if not path.exists(): + return path + counter = 2 + while True: + candidate = path.with_name(f"{path.stem} ({counter}){path.suffix}") + if not candidate.exists(): + return candidate + counter += 1 + + +def finalize_library_files(job): + staging_dir = job_staging_dir(job) + target_dir = job_output_dir(job) + library_name = sanitize_path_component(job.get("anime_name") or job.get("title")) + season = season_label(job) + target_dir.mkdir(parents=True, exist_ok=True) + files = sorted( + [path for path in staging_dir.iterdir() if path.is_file()], + key=lambda path: (path.stat().st_mtime, path.name), + ) + if not files: + raise RuntimeError("No downloaded files were found in the staging folder") + + moved = [] + fallback_episode = 1 + for source in files: + ep = extract_episode_from_filename(source) + if ep is None: + ep = str(fallback_episode) + fallback_episode += 1 + final_name = f"{library_name} - S{season}E{episode_label(ep)}{source.suffix}" + destination = unique_destination(target_dir / final_name) + shutil.move(str(source), str(destination)) + moved.append(str(destination)) + + try: + staging_dir.rmdir() + except OSError: + pass + return moved + + +def build_job(payload, config): + if not isinstance(payload, dict): + raise ValueError("Expected a JSON object") + query = str(payload.get("query") or payload.get("title") or "").strip() + title = str(payload.get("title") or query).strip() + anime_name = sanitize_path_component(payload.get("anime_name") or title) + if not query: + raise ValueError("Missing search query") + try: + index = int(payload.get("index", 1)) + except (TypeError, ValueError) as exc: + raise ValueError("Invalid result index") from exc + if index < 1: + raise ValueError("Result index must be positive") + + mode = str(payload.get("mode") or config["mode"]).lower() + quality = str(payload.get("quality") or config["quality"]).lower() + download_dir = str(payload.get("download_dir") or config["download_dir"]).strip() + if mode not in MODE_CHOICES: + raise ValueError("Mode must be sub or dub") + if quality not in QUALITY_CHOICES: + raise ValueError("Unknown quality") + + return { + "id": uuid4().hex, + "show_id": str(payload.get("show_id") or "").strip(), + "title": title, + "anime_name": anime_name, + "season": normalize_season(payload.get("season")), + "query": query, + "result_index": index, + "mode": mode, + "quality": quality, + "episodes": validate_episode_spec(payload.get("episodes")), + "download_dir": str(Path(download_dir).expanduser()), + "status": "pending", + "created_at": now_iso(), + "updated_at": now_iso(), + "started_at": None, + "finished_at": None, + "exit_code": None, + "pid": None, + "log": [], + } + + +def command_for_job(job): + command = [ + ANI_CLI, + "-d", + "-q", + job["quality"], + "-e", + job["episodes"], + "-S", + str(job["result_index"]), + ] + if job["mode"] == "dub": + command.append("--dub") + command.append(job["query"]) + return command diff --git a/config_page.py b/config_page.py new file mode 100644 index 0000000..cad72b2 --- /dev/null +++ b/config_page.py @@ -0,0 +1,386 @@ +#!/usr/bin/env python3 + +"""Config page template for ani-cli-web.""" + +from template_helpers import BROWSER_API_HELPER, render_page_links, render_sidebar_brand + + +CONFIG_HTML = r""" + + + + + ani-cli web - config + + + +
+ + +
+
+
+
+

Defaults

+

Manage the folder, mode, and quality that the Search page starts with.

+
+ +
+
+
+ +
+
+ + +
+
Tip: these are the saved defaults only. The active search form can still be adjusted per download.
+
+ +
+
+
+ + + + +""" diff --git a/http_handler.py b/http_handler.py new file mode 100644 index 0000000..24d8702 --- /dev/null +++ b/http_handler.py @@ -0,0 +1,383 @@ +#!/usr/bin/env python3 + +"""HTTP handler and route wiring for ani-cli-web.""" + +import json +import mimetypes +import os +import shutil +import traceback +from dataclasses import dataclass +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler +from pathlib import Path +from urllib.parse import parse_qs, unquote, urlparse + +from app_support import ( + ANI_CLI, + APP_NAME, + CLIENT_DISCONNECT_ERRORS, + MAX_JSON_BODY_BYTES, + MODE_CHOICES, + VERSION, + client_address_is_local, + debug_enabled, + debug_log, + normalize_config, + remote_access_allowed, + remote_access_token_configured, + remote_access_token_matches, +) + +@dataclass(frozen=True) +class HandlerContext: + add_to_watchlist: object + config_html: str + ensure_runtime: object + episode_list: object + get_config_snapshot: object + get_watchlist: object + get_watchlist_refresh_status: object + http_error: object + index_html: str + remove_from_watchlist: object + runtime_state: object + save_runtime_config: object + search_anime: object + thumbnail_file_path: object + update_all_watchlist_statuses: object + update_watchlist_category: object + update_watchlist_status: object + upload_watchlist_thumbnail: object + watchlist_html: str + + +def build_handler_class(context): + class ConfiguredHandler(Handler): + handler_context = context + + return ConfiguredHandler + + +def dependency_status(): + checks = ["curl", "sed", "grep", "openssl", "fzf", "aria2c", "ffmpeg", "yt-dlp"] + result = {name: bool(shutil.which(name)) for name in checks} + result["ani-cli"] = bool(shutil.which(ANI_CLI) or (Path(ANI_CLI).exists() and os.access(ANI_CLI, os.X_OK))) + return result + + +class Handler(BaseHTTPRequestHandler): + server_version = "AniCliWeb/1.0" + + def _context(self): + context = getattr(self, "handler_context", None) or getattr(type(self), "handler_context", None) + if context is None: + raise RuntimeError("Handler context is not configured.") + return context + + def _runtime(self): + context = Handler._context(self) + context.ensure_runtime(start_workers=True) + return context.runtime_state() + + def _effective_client_host(self): + peer_host = self.client_address[0] if self.client_address else "" + if not client_address_is_local(peer_host): + return peer_host + forwarded = ( + self.headers.get("X-Forwarded-For") + or self.headers.get("x-forwarded-for") + or self.headers.get("X-Real-IP") + or self.headers.get("x-real-ip") + or "" + ) + if forwarded: + first = str(forwarded).split(",", 1)[0].strip().strip("[]") + if first: + return first + forwarded_header = str(self.headers.get("Forwarded") or self.headers.get("forwarded") or "").strip() + if forwarded_header: + for part in forwarded_header.split(";"): + key, _, value = part.partition("=") + if key.strip().lower() != "for": + continue + candidate = value.strip().strip('"') + if candidate.startswith("[") and "]" in candidate: + candidate = candidate[1 : candidate.index("]")] + elif ":" in candidate and candidate.count(":") == 1: + candidate = candidate.split(":", 1)[0] + candidate = candidate.strip() + if candidate: + return candidate + return peer_host + + def _remote_token(self): + return ( + self.headers.get("X-AniCli-Web-Token") + or self.headers.get("x-ani-cli-web-token") + or Handler._bearer_token(self) + or "" + ) + + def _bearer_token(self): + header = str(self.headers.get("Authorization") or "").strip() + if not header.startswith("Bearer "): + return "" + return header[len("Bearer ") :].strip() + + def ensure_client_access(self): + client_host = Handler._effective_client_host(self) + if client_address_is_local(client_host): + return + if not remote_access_allowed(): + raise PermissionError( + "Remote access is disabled. Set ANI_CLI_WEB_ALLOW_REMOTE=1 to allow non-local clients." + ) + if not remote_access_token_configured(): + raise PermissionError( + "Remote access requires ANI_CLI_WEB_REMOTE_TOKEN for non-local clients." + ) + if not remote_access_token_matches(Handler._remote_token(self)): + raise PermissionError("Remote access token missing or invalid.") + + def do_GET(self): + parsed = urlparse(self.path) + try: + self.ensure_client_access() + if parsed.path == "/": + self.html(Handler._context(self).index_html) + elif parsed.path == "/config": + self.html(Handler._context(self).config_html) + elif parsed.path == "/watchlist": + self.html(Handler._context(self).watchlist_html) + elif parsed.path == "/api/config": + self.json(Handler._context(self).get_config_snapshot()) + elif parsed.path == "/api/version": + self.json({"name": APP_NAME, "version": VERSION}) + elif parsed.path == "/api/dependencies": + self.json(dependency_status()) + elif parsed.path == "/api/search": + runtime = Handler._runtime(self) + config = runtime["config"] + params = parse_qs(parsed.query) + query = (params.get("q") or [""])[0].strip() + mode = (params.get("mode") or [config["mode"]])[0].lower() + if not query: + raise ValueError("Search query is required") + if mode not in MODE_CHOICES: + raise ValueError("Mode must be sub or dub") + self.json({"results": Handler._context(self).search_anime(query, mode)}) + elif parsed.path.startswith("/api/anime/") and parsed.path.endswith("/episodes"): + runtime = Handler._runtime(self) + config = runtime["config"] + show_id = unquote(parsed.path[len("/api/anime/") : -len("/episodes")]) + params = parse_qs(parsed.query) + mode = (params.get("mode") or [config["mode"]])[0].lower() + if mode not in MODE_CHOICES: + raise ValueError("Mode must be sub or dub") + self.json({"episodes": Handler._context(self).episode_list(show_id, mode)}) + elif parsed.path == "/api/queue": + runtime = Handler._runtime(self) + params = parse_qs(parsed.query) + page = (params.get("page") or ["1"])[0] + per_page = (params.get("per_page") or ["10"])[0] + self.json(runtime["download_queue"].list(page=page, per_page=per_page)) + elif parsed.path.startswith("/api/watchlist/thumb/"): + runtime = Handler._runtime(self) + show_id = unquote(parsed.path[len("/api/watchlist/thumb/") :]).strip() + debug_log("thumbnail.route.request", show_id=show_id, path=parsed.path, query=parsed.query) + item = runtime["watchlist"].get(show_id) + path = Handler._context(self).thumbnail_file_path(item.get("thumbnail_path")) + if not path or not path.exists(): + debug_log("thumbnail.route.miss", show_id=show_id, resolved_path=path) + self.error(HTTPStatus.NOT_FOUND, "Thumbnail not found") + return + debug_log("thumbnail.route.hit", show_id=show_id, resolved_path=path) + self.file(path) + elif parsed.path in {"/api/watchlist", "/api/watchlist/get"}: + Handler._runtime(self) + params = parse_qs(parsed.query) + page = (params.get("page") or ["1"])[0] + per_page = (params.get("per_page") or ["30"])[0] + category = (params.get("category") or [""])[0].strip().lower() or None + self.json(Handler._context(self).get_watchlist(page=page, per_page=per_page, category=category)) + elif parsed.path == "/api/watchlist/refresh-status": + Handler._runtime(self) + self.json(Handler._context(self).get_watchlist_refresh_status()) + else: + self.error(HTTPStatus.NOT_FOUND, "Not found") + except Exception as exc: + self.exception(exc) + + def do_POST(self): + parsed = urlparse(self.path) + try: + self.ensure_client_access() + if parsed.path == "/api/config": + self.update_config(self.body_json()) + elif parsed.path == "/api/queue": + runtime = Handler._runtime(self) + self.json(runtime["download_queue"].add(self.body_json()), HTTPStatus.CREATED) + elif parsed.path == "/api/queue/retry-failed": + runtime = Handler._runtime(self) + self.json(runtime["download_queue"].retry_all_failed()) + elif parsed.path == "/api/queue/clear-finished": + runtime = Handler._runtime(self) + self.json(runtime["download_queue"].clear_finished()) + elif parsed.path.startswith("/api/queue/"): + runtime = Handler._runtime(self) + parts = parsed.path.strip("/").split("/") + if len(parts) != 4: + self.error(HTTPStatus.NOT_FOUND, "Not found") + return + _, _, job_id, action = parts + actions = { + "retry": runtime["download_queue"].retry, + "cancel": runtime["download_queue"].cancel, + "remove": runtime["download_queue"].remove, + } + if action not in actions: + self.error(HTTPStatus.NOT_FOUND, "Not found") + return + self.json(actions[action](job_id)) + elif parsed.path in {"/api/watchlist", "/api/watchlist/add"}: + Handler._runtime(self) + self.json(Handler._context(self).add_to_watchlist(self.body_json()), HTTPStatus.CREATED) + elif parsed.path == "/api/watchlist/ensure-thumbnails": + runtime = Handler._runtime(self) + payload = self.body_json() + self.json( + runtime["watchlist"].ensure_thumbnails( + payload.get("show_ids") or [], + limit=payload.get("limit", 6), + force=bool(payload.get("force")), + ) + ) + elif parsed.path == "/api/watchlist/update-status": + Handler._runtime(self) + payload = self.body_json() + self.json(Handler._context(self).update_watchlist_status(payload["show_id"], payload.get("mode"))) + elif parsed.path == "/api/watchlist/update-category": + Handler._runtime(self) + payload = self.body_json() + self.json(Handler._context(self).update_watchlist_category(payload["show_id"], payload.get("category"))) + elif parsed.path == "/api/watchlist/update-all": + Handler._runtime(self) + result = Handler._context(self).update_all_watchlist_statuses() + status = HTTPStatus.ACCEPTED if result.get("created") else HTTPStatus.OK + self.json(result, status) + elif parsed.path == "/api/watchlist/remove": + Handler._runtime(self) + payload = self.body_json() + self.json(Handler._context(self).remove_from_watchlist(payload["show_id"])) + elif parsed.path == "/api/watchlist/upload-thumbnail": + Handler._runtime(self) + payload = self.body_json() + self.json(Handler._context(self).upload_watchlist_thumbnail(payload)) + else: + self.error(HTTPStatus.NOT_FOUND, "Not found") + except Exception as exc: + self.exception(exc) + + def update_config(self, payload): + config = normalize_config(payload) + Path(config["download_dir"]).expanduser().mkdir(parents=True, exist_ok=True) + context = Handler._context(self) + context.save_runtime_config(config) + self.json(context.get_config_snapshot()) + + def body_json(self): + try: + length = int(self.headers.get("Content-Length", "0") or "0") + except ValueError as exc: + raise ValueError("Invalid Content-Length header") from exc + if length < 0: + raise ValueError("Invalid Content-Length header") + if length > MAX_JSON_BODY_BYTES: + raise Handler._context(self).http_error(HTTPStatus.REQUEST_ENTITY_TOO_LARGE, "Request body too large.") + raw_bytes = self.rfile.read(length) if length else b"{}" + if len(raw_bytes) > MAX_JSON_BODY_BYTES: + raise Handler._context(self).http_error(HTTPStatus.REQUEST_ENTITY_TOO_LARGE, "Request body too large.") + try: + raw = raw_bytes.decode("utf-8") + except UnicodeDecodeError as exc: + raise ValueError("Request body must be valid UTF-8 JSON.") from exc + try: + return json.loads(raw) + except json.JSONDecodeError as exc: + raise ValueError("Invalid JSON body") from exc + + def html(self, content): + data = content.encode("utf-8") + self.write_response_bytes(data, HTTPStatus.OK, {"Content-Type": "text/html; charset=utf-8"}) + + def file(self, path): + payload = Path(path).read_bytes() + content_type = mimetypes.guess_type(str(path))[0] or "application/octet-stream" + debug_log("file.serve", path=path, content_type=content_type, bytes=len(payload)) + self.write_response_bytes( + payload, + HTTPStatus.OK, + { + "Content-Type": content_type, + "Cache-Control": "public, max-age=86400", + }, + ) + + def json(self, payload, status=HTTPStatus.OK): + data = json.dumps(payload).encode("utf-8") + self.write_response_bytes(data, status, {"Content-Type": "application/json"}) + + def write_response_bytes(self, payload, status, headers): + try: + self.send_response(status) + for key, value in headers.items(): + self.send_header(key, value) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + except CLIENT_DISCONNECT_ERRORS: + return + + def error(self, status, message): + try: + self.json({"error": message}, status) + except CLIENT_DISCONNECT_ERRORS: + return + + def exception(self, exc): + if isinstance(exc, CLIENT_DISCONNECT_ERRORS): + return + if isinstance(exc, Handler._context(self).http_error): + self.error(exc.status, exc.message) + elif isinstance(exc, KeyError): + self.error(HTTPStatus.NOT_FOUND, str(exc).strip("'")) + elif isinstance(exc, PermissionError): + self.error(HTTPStatus.FORBIDDEN, str(exc)) + elif isinstance(exc, ValueError): + self.error(HTTPStatus.BAD_REQUEST, str(exc)) + else: + if debug_enabled(): + debug_log("handler.exception", path=getattr(self, "path", ""), error=exc) + traceback.print_exc() + else: + debug_log("handler.exception", path=getattr(self, "path", ""), error=exc) + self.error(HTTPStatus.INTERNAL_SERVER_ERROR, "Internal server error.") + + def log_message(self, fmt, *args): + print(f"{self.address_string()} - {fmt % args}") + + +def server_host(): + return os.environ.get("ANI_CLI_WEB_HOST", "127.0.0.1").strip() or "127.0.0.1" + + +def server_port(): + raw = str(os.environ.get("ANI_CLI_WEB_PORT", "8421")).strip() or "8421" + try: + port = int(raw, 10) + except ValueError as exc: + raise ValueError("ANI_CLI_WEB_PORT must be a valid integer") from exc + if not 1 <= port <= 65535: + raise ValueError("ANI_CLI_WEB_PORT must be between 1 and 65535") + return port diff --git a/queue_jobs.py b/queue_jobs.py new file mode 100644 index 0000000..6bcff3f --- /dev/null +++ b/queue_jobs.py @@ -0,0 +1,840 @@ +#!/usr/bin/env python3 + +"""Queue and background job services for ani-cli-web.""" + +import concurrent.futures +import json +import os +import signal +import sqlite3 +import subprocess +import threading +import time + +from app_support import ( + MAX_LOG_LINES, + PROJECT_ROOT, + QUEUE_PATH, + STATE_DB_PATH, + build_job, + command_for_job, + finalize_library_files, + job_output_dir, + job_staging_dir, + load_json, + now_iso, + printable_command, + strip_control, + debug_log, +) + + +LOG_PERSIST_INTERVAL_SECONDS = 0.75 +LOG_PERSIST_LINE_BATCH = 8 +WATCHLIST_REFRESH_MAX_WORKERS = 4 +WORKER_JOIN_TIMEOUT_SECONDS = 2.5 +JOB_STRUCTURED_COLUMNS = ( + "show_id", + "title", + "anime_name", + "season", + "query", + "result_index", + "mode", + "quality", + "episodes", + "download_dir", + "started_at", + "finished_at", + "exit_code", + "pid", + "command", + "staging_dir", + "target_dir", + "cancel_requested", +) +JOB_LOG_COLUMN = "log_payload" +JOB_EXTRA_PAYLOAD_COLUMN = "payload" + + +class WatchlistRefreshJobs: + def __init__(self, store, start_worker=True): + self.store = store + self.lock = threading.RLock() + self.wakeup = threading.Event() + self.stop_event = threading.Event() + self.pending = [] + self.jobs = [] + self.current_job_id = None + self.current_executor = None + self._init_db() + self._restore_interrupted_jobs() + self._load_jobs() + self.worker = None + if start_worker: + self.ensure_worker() + + def _connect(self): + conn = sqlite3.connect(STATE_DB_PATH, timeout=30) + conn.row_factory = sqlite3.Row + return conn + + def ensure_worker(self): + with self.lock: + if self.worker is not None and self.worker.is_alive(): + return + self.stop_event.clear() + self.worker = threading.Thread(target=self._run, daemon=True) + self.worker.start() + + def shutdown(self, wait=False): + self.stop_event.set() + executor = None + with self.lock: + executor = self.current_executor + if executor is not None: + executor.shutdown(wait=False, cancel_futures=True) + self.wakeup.set() + worker = self.worker + if wait and worker is not None and worker.is_alive(): + worker.join(timeout=WORKER_JOIN_TIMEOUT_SECONDS) + return worker is None or not worker.is_alive() + + def _init_db(self): + STATE_DB_PATH.parent.mkdir(parents=True, exist_ok=True) + with self._connect() as conn: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS watchlist_refresh_jobs ( + id TEXT PRIMARY KEY, + status TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + payload TEXT NOT NULL + ) + """ + ) + conn.execute("CREATE INDEX IF NOT EXISTS idx_watchlist_refresh_created_at ON watchlist_refresh_jobs(created_at)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_watchlist_refresh_status ON watchlist_refresh_jobs(status)") + + def _job_from_row(self, row): + payload = row["payload"] if isinstance(row, sqlite3.Row) else row[0] + return json.loads(payload) + + def _upsert_job_conn(self, conn, job): + conn.execute( + """ + INSERT INTO watchlist_refresh_jobs (id, status, created_at, updated_at, payload) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + status = excluded.status, + updated_at = excluded.updated_at, + payload = excluded.payload + """, + ( + job["id"], + job["status"], + job["created_at"], + job["updated_at"], + json.dumps(job, sort_keys=True), + ), + ) + + def _trim_history_conn(self, conn): + rows = conn.execute( + "SELECT id FROM watchlist_refresh_jobs ORDER BY created_at DESC LIMIT -1 OFFSET 10" + ).fetchall() + if rows: + conn.executemany("DELETE FROM watchlist_refresh_jobs WHERE id = ?", [(row["id"],) for row in rows]) + + def _save_job_locked(self, job): + with self._connect() as conn: + self._upsert_job_conn(conn, job) + self._trim_history_conn(conn) + + def _load_jobs(self): + with self.lock, self._connect() as conn: + rows = conn.execute( + "SELECT payload FROM watchlist_refresh_jobs ORDER BY created_at DESC LIMIT 10" + ).fetchall() + self.jobs = [self._job_from_row(row) for row in rows] + + def _restore_interrupted_jobs(self): + with self.lock, self._connect() as conn: + rows = conn.execute( + "SELECT payload FROM watchlist_refresh_jobs WHERE status IN ('pending', 'running')" + ).fetchall() + for row in rows: + job = self._job_from_row(row) + job["status"] = "interrupted" + job["finished_at"] = now_iso() + job["updated_at"] = job["finished_at"] + job["current_title"] = None + job["message"] = "Interrupted by web app restart." + self._upsert_job_conn(conn, job) + self._trim_history_conn(conn) + + def _copy_job(self, job): + return json.loads(json.dumps(job)) + + def _find_locked(self, job_id): + for job in self.jobs: + if job["id"] == job_id: + return job + return None + + def status(self): + with self.lock: + job = next((candidate for candidate in self.jobs if candidate["status"] in {"pending", "running"}), None) + if job is None: + job = self.jobs[0] if self.jobs else None + active = bool(job and job["status"] in {"pending", "running"}) + return {"job": self._copy_job(job) if job else None, "running": active} + + def start(self): + with self.lock: + job = next((candidate for candidate in self.jobs if candidate["status"] in {"pending", "running"}), None) + if job is not None: + return { + "message": "Watchlist refresh already queued." if job["status"] == "pending" else "Watchlist refresh already running.", + "job": self._copy_job(job) if job else None, + "created": False, + } + now = now_iso() + job = { + "id": os.urandom(16).hex(), + "status": "pending", + "created_at": now, + "updated_at": now, + "started_at": None, + "finished_at": None, + "total": 0, + "completed": 0, + "errors": 0, + "current_title": None, + "message": "Queued watchlist refresh.", + } + self.jobs.insert(0, job) + self.jobs = self.jobs[:10] + self._save_job_locked(job) + self.pending.append(job["id"]) + self.wakeup.set() + return {"message": "Watchlist refresh started.", "job": self._copy_job(job), "created": True} + + def _next_pending(self): + with self.lock: + while self.pending: + job_id = self.pending.pop(0) + job = self._find_locked(job_id) + if job is not None: + self.current_job_id = job_id + return job + return None + + def _run(self): + while not self.stop_event.is_set(): + job = self._next_pending() + if not job: + self.wakeup.wait(2) + self.wakeup.clear() + continue + self._run_job(job) + + def _run_job(self, job): + try: + show_ids = self.store.all_show_ids() + started_at = now_iso() + with self.lock: + job["status"] = "running" + job["started_at"] = started_at + job["updated_at"] = started_at + job["total"] = len(show_ids) + job["completed"] = 0 + job["errors"] = 0 + job["current_title"] = None + job["message"] = f"Refreshing 0/{len(show_ids)} watchlist entries." + self._save_job_locked(job) + + def refresh_one(show_id): + title = show_id + errored = False + skipped = False + error_message = "" + if self.stop_event.is_set(): + return {"title": title, "errored": False, "skipped": True, "error": ""} + try: + item = self.store.refresh(show_id, include_thumbnail=False) + if item is None: + skipped = True + else: + title = item.get("title") or title + errored = item.get("status") == "error" + error_message = str(item.get("status_message") or "").strip() if errored else "" + except Exception as exc: + errored = True + error_message = str(exc) + debug_log("watchlist.refresh_all.item_error", show_id=show_id, error=exc) + return {"title": title, "errored": errored, "skipped": skipped, "error": error_message} + + max_workers = max(1, min(WATCHLIST_REFRESH_MAX_WORKERS, len(show_ids) or 1)) + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + with self.lock: + self.current_executor = executor + futures = [executor.submit(refresh_one, show_id) for show_id in show_ids] + for index, future in enumerate(concurrent.futures.as_completed(futures), start=1): + result = future.result() + title = result["title"] + with self.lock: + job["completed"] = index + job["errors"] += 1 if result["errored"] else 0 + job["current_title"] = title + job["updated_at"] = now_iso() + if result["errored"] and result["error"]: + job["last_error"] = result["error"] + if result["skipped"]: + job["message"] = f"Refreshing {index}/{len(show_ids)}: removed entry" + else: + job["message"] = f"Refreshing {index}/{len(show_ids)}: {title}" + self._save_job_locked(job) + with self.lock: + self.current_executor = None + + finished_at = now_iso() + with self.lock: + job["status"] = "done" + job["finished_at"] = finished_at + job["updated_at"] = finished_at + job["current_title"] = None + if job["errors"]: + job["message"] = ( + f"Refreshed {job['completed']} watchlist entries with {job['errors']} refresh errors." + ) + else: + job["message"] = f"Refreshed {job['completed']} watchlist entries." + self._save_job_locked(job) + except Exception as exc: + finished_at = now_iso() + with self.lock: + job["status"] = "failed" + job["finished_at"] = finished_at + job["updated_at"] = finished_at + job["current_title"] = None + job["message"] = f"Watchlist refresh failed: {exc}" + self._save_job_locked(job) + finally: + with self.lock: + self.current_executor = None + self.current_job_id = None + + +class DownloadQueue: + def __init__(self, config_getter, watchlist_sync_fn=None, job_prepare_fn=None, start_worker=True): + self.config_getter = config_getter + self.watchlist_sync_fn = watchlist_sync_fn + self.job_prepare_fn = job_prepare_fn + self.lock = threading.RLock() + self.wakeup = threading.Event() + self.stop_event = threading.Event() + self.current_process = None + self.current_job_id = None + self.current_job = None + self._init_db() + self._migrate_json_queue() + self._restore_interrupted_jobs() + self.worker = None + if start_worker: + self.ensure_worker() + + def _connect(self): + conn = sqlite3.connect(STATE_DB_PATH, timeout=30) + conn.row_factory = sqlite3.Row + return conn + + def ensure_worker(self): + with self.lock: + if self.worker is not None and self.worker.is_alive(): + return + self.stop_event.clear() + self.worker = threading.Thread(target=self._run, daemon=True) + self.worker.start() + + def shutdown(self, wait=False, cancel_active=False): + should_cancel = bool(cancel_active or wait) + self.stop_event.set() + if should_cancel: + self._signal_active_process(signal.SIGTERM) + self.wakeup.set() + worker = self.worker + if wait and worker is not None and worker.is_alive(): + worker.join(timeout=WORKER_JOIN_TIMEOUT_SECONDS) + if worker.is_alive(): + self._signal_active_process(signal.SIGKILL) + worker.join(timeout=WORKER_JOIN_TIMEOUT_SECONDS) + return worker is None or not worker.is_alive() + + def _signal_active_process(self, signum): + with self.lock: + process = self.current_process + job = self.current_job + if process is None: + return False + if job is not None: + job["cancel_requested"] = True + try: + os.killpg(os.getpgid(process.pid), signum) + return True + except OSError: + try: + if signum == signal.SIGKILL: + process.kill() + else: + process.terminate() + return True + except OSError: + return False + + def _init_db(self): + STATE_DB_PATH.parent.mkdir(parents=True, exist_ok=True) + with self._connect() as conn: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS jobs ( + id TEXT PRIMARY KEY, + status TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + show_id TEXT, + title TEXT, + anime_name TEXT, + season TEXT, + query TEXT, + result_index INTEGER, + mode TEXT, + quality TEXT, + episodes TEXT, + download_dir TEXT, + started_at TEXT, + finished_at TEXT, + exit_code INTEGER, + pid INTEGER, + command TEXT, + staging_dir TEXT, + target_dir TEXT, + cancel_requested INTEGER NOT NULL DEFAULT 0, + log_payload TEXT NOT NULL DEFAULT '[]', + payload TEXT NOT NULL DEFAULT '{}' + ) + """ + ) + columns = {row["name"] for row in conn.execute("PRAGMA table_info(jobs)").fetchall()} + alter_statements = ( + ("show_id", "ALTER TABLE jobs ADD COLUMN show_id TEXT"), + ("title", "ALTER TABLE jobs ADD COLUMN title TEXT"), + ("anime_name", "ALTER TABLE jobs ADD COLUMN anime_name TEXT"), + ("season", "ALTER TABLE jobs ADD COLUMN season TEXT"), + ("query", "ALTER TABLE jobs ADD COLUMN query TEXT"), + ("result_index", "ALTER TABLE jobs ADD COLUMN result_index INTEGER"), + ("mode", "ALTER TABLE jobs ADD COLUMN mode TEXT"), + ("quality", "ALTER TABLE jobs ADD COLUMN quality TEXT"), + ("episodes", "ALTER TABLE jobs ADD COLUMN episodes TEXT"), + ("download_dir", "ALTER TABLE jobs ADD COLUMN download_dir TEXT"), + ("started_at", "ALTER TABLE jobs ADD COLUMN started_at TEXT"), + ("finished_at", "ALTER TABLE jobs ADD COLUMN finished_at TEXT"), + ("exit_code", "ALTER TABLE jobs ADD COLUMN exit_code INTEGER"), + ("pid", "ALTER TABLE jobs ADD COLUMN pid INTEGER"), + ("command", "ALTER TABLE jobs ADD COLUMN command TEXT"), + ("staging_dir", "ALTER TABLE jobs ADD COLUMN staging_dir TEXT"), + ("target_dir", "ALTER TABLE jobs ADD COLUMN target_dir TEXT"), + ("cancel_requested", "ALTER TABLE jobs ADD COLUMN cancel_requested INTEGER NOT NULL DEFAULT 0"), + (JOB_LOG_COLUMN, "ALTER TABLE jobs ADD COLUMN log_payload TEXT NOT NULL DEFAULT '[]'"), + ) + for column, statement in alter_statements: + if column not in columns: + conn.execute(statement) + conn.execute("CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_jobs_created_at ON jobs(created_at)") + + def _migrate_json_queue(self): + if not QUEUE_PATH.exists(): + return + jobs = load_json(QUEUE_PATH, []) + if not jobs: + QUEUE_PATH.rename(QUEUE_PATH.with_suffix(".json.migrated")) + return + with self.lock, self._connect() as conn: + existing = conn.execute("SELECT COUNT(*) AS count FROM jobs").fetchone()["count"] + if existing: + return + for job in jobs: + if isinstance(job, dict) and job.get("id"): + self._upsert_job_conn(conn, job) + QUEUE_PATH.rename(QUEUE_PATH.with_suffix(".json.migrated")) + + def _restore_interrupted_jobs(self): + with self.lock, self._connect() as conn: + rows = conn.execute("SELECT * FROM jobs WHERE status = 'running'").fetchall() + for row in rows: + job = self._job_from_row(row) + job["status"] = "pending" + job["pid"] = None + job.setdefault("log", []).append("Restarted after web app restart.") + self._upsert_job_conn(conn, job) + + def _job_from_row(self, row): + if not isinstance(row, sqlite3.Row): + return json.loads(row[0]) + raw = dict(row) + payload = raw.get(JOB_EXTRA_PAYLOAD_COLUMN) or "{}" + try: + job = json.loads(payload) + except json.JSONDecodeError: + job = {} + if not isinstance(job, dict): + job = {} + log_payload = raw.get(JOB_LOG_COLUMN) or "[]" + try: + log_lines = json.loads(log_payload) + except json.JSONDecodeError: + log_lines = [] + if not isinstance(log_lines, list): + log_lines = [] + for key in ("id", "status", "created_at", "updated_at"): + value = raw.get(key) + if value is not None: + job[key] = value + for key in JOB_STRUCTURED_COLUMNS: + value = raw.get(key) + if value is None: + continue + if key == "cancel_requested": + job[key] = bool(int(value or 0)) + else: + job[key] = value + job["log"] = [str(line) for line in log_lines][-MAX_LOG_LINES:] + return job + + def _extra_job_payload(self, job): + clean = dict(job) + clean.pop("process", None) + clean.pop("_last_persist_monotonic", None) + clean.pop("_dirty_log_lines", None) + clean.pop("log", None) + for key in ("id", "status", "created_at", "updated_at"): + clean.pop(key, None) + for key in JOB_STRUCTURED_COLUMNS: + clean.pop(key, None) + return clean + + def _upsert_job_conn(self, conn, job): + log_lines = [str(line) for line in job.get("log", [])][-MAX_LOG_LINES:] + extra_payload = self._extra_job_payload(job) + conn.execute( + """ + INSERT INTO jobs ( + id, status, created_at, updated_at, + show_id, title, anime_name, season, query, result_index, mode, quality, episodes, download_dir, + started_at, finished_at, exit_code, pid, command, staging_dir, target_dir, cancel_requested, + log_payload, payload + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + status = excluded.status, + updated_at = excluded.updated_at, + show_id = excluded.show_id, + title = excluded.title, + anime_name = excluded.anime_name, + season = excluded.season, + query = excluded.query, + result_index = excluded.result_index, + mode = excluded.mode, + quality = excluded.quality, + episodes = excluded.episodes, + download_dir = excluded.download_dir, + started_at = excluded.started_at, + finished_at = excluded.finished_at, + exit_code = excluded.exit_code, + pid = excluded.pid, + command = excluded.command, + staging_dir = excluded.staging_dir, + target_dir = excluded.target_dir, + cancel_requested = excluded.cancel_requested, + log_payload = excluded.log_payload, + payload = excluded.payload + """, + ( + job["id"], + job.get("status", "pending"), + job.get("created_at") or now_iso(), + job.get("updated_at") or now_iso(), + job.get("show_id"), + job.get("title"), + job.get("anime_name"), + job.get("season"), + job.get("query"), + job.get("result_index"), + job.get("mode"), + job.get("quality"), + job.get("episodes"), + job.get("download_dir"), + job.get("started_at"), + job.get("finished_at"), + job.get("exit_code"), + job.get("pid"), + job.get("command"), + job.get("staging_dir"), + job.get("target_dir"), + 1 if job.get("cancel_requested") else 0, + json.dumps(log_lines), + json.dumps(extra_payload, sort_keys=True), + ), + ) + + def _save_job_locked(self, job): + with self._connect() as conn: + self._upsert_job_conn(conn, job) + job["_last_persist_monotonic"] = time.monotonic() + job["_dirty_log_lines"] = 0 + + def list(self, page=1, per_page=10): + page = max(1, int(page or 1)) + per_page = min(50, max(1, int(per_page or 10))) + offset = (page - 1) * per_page + with self.lock, self._connect() as conn: + total = conn.execute("SELECT COUNT(*) AS count FROM jobs").fetchone()["count"] + rows = conn.execute( + "SELECT * FROM jobs ORDER BY created_at DESC LIMIT ? OFFSET ?", + (per_page, offset), + ).fetchall() + pages = max(1, (total + per_page - 1) // per_page) + return { + "jobs": [self._job_from_row(row) for row in rows], + "page": page, + "per_page": per_page, + "total": total, + "pages": pages, + } + + def add(self, payload): + job = build_job(payload, self.config_getter()) + if self.job_prepare_fn is not None: + prepared = self.job_prepare_fn(dict(job)) + if prepared is not None: + job = prepared + with self.lock: + self._save_job_locked(job) + self.wakeup.set() + return job + + def retry(self, job_id): + with self.lock: + job = self._find(job_id) + if job["status"] == "running": + raise ValueError("Running jobs cannot be retried") + job["status"] = "pending" + job["exit_code"] = None + job["pid"] = None + job["started_at"] = None + job["finished_at"] = None + job["updated_at"] = now_iso() + job["log"] = [] + self._save_job_locked(job) + self.wakeup.set() + return job + + def retry_all_failed(self): + count = 0 + with self.lock, self._connect() as conn: + rows = conn.execute("SELECT * FROM jobs WHERE status = 'failed'").fetchall() + for row in rows: + job = self._job_from_row(row) + job["status"] = "pending" + job["exit_code"] = None + job["pid"] = None + job["started_at"] = None + job["finished_at"] = None + job["updated_at"] = now_iso() + job["log"] = [] + self._upsert_job_conn(conn, job) + count += 1 + if count: + self.wakeup.set() + return {"ok": True, "count": count} + + def cancel(self, job_id): + with self.lock: + job = self._find(job_id) + if job["status"] == "pending": + job["status"] = "canceled" + job["updated_at"] = now_iso() + self._save_job_locked(job) + return job + if job["status"] != "running": + return job + if self.current_job_id == job_id and self.current_process: + try: + os.killpg(os.getpgid(self.current_process.pid), signal.SIGTERM) + except OSError: + self.current_process.terminate() + updated_at = now_iso() + job["cancel_requested"] = True + job["updated_at"] = updated_at + if self.current_job is not None and self.current_job.get("id") == job_id: + self.current_job["cancel_requested"] = True + self.current_job["updated_at"] = updated_at + self._save_job_locked(job) + return job + raise ValueError("Could not find the running process") + + def remove(self, job_id): + with self.lock: + job = self._find(job_id) + if job["status"] == "running": + raise ValueError("Running jobs must be canceled before removing") + with self._connect() as conn: + conn.execute("DELETE FROM jobs WHERE id = ?", (job_id,)) + return {"ok": True} + + def clear_finished(self): + with self.lock, self._connect() as conn: + cursor = conn.execute("DELETE FROM jobs WHERE status IN ('done', 'canceled')") + return {"ok": True, "count": cursor.rowcount} + + def _find(self, job_id): + with self._connect() as conn: + row = conn.execute("SELECT * FROM jobs WHERE id = ?", (job_id,)).fetchone() + if row: + return self._job_from_row(row) + raise KeyError("Job not found") + + def _next_pending(self): + with self.lock: + with self._connect() as conn: + row = conn.execute( + "SELECT * FROM jobs WHERE status = 'pending' ORDER BY created_at ASC LIMIT 1" + ).fetchone() + if row: + return self._job_from_row(row) + return None + + def _append_log(self, job, line): + text = strip_control(line).strip() + if not text: + return + with self.lock: + job.setdefault("log", []).append(text) + job["log"] = job["log"][-MAX_LOG_LINES:] + job["updated_at"] = now_iso() + job["_dirty_log_lines"] = int(job.get("_dirty_log_lines") or 0) + 1 + last_persist = float(job.get("_last_persist_monotonic") or 0.0) + should_flush = ( + job["_dirty_log_lines"] >= LOG_PERSIST_LINE_BATCH + or (time.monotonic() - last_persist) >= LOG_PERSIST_INTERVAL_SECONDS + ) + if should_flush: + self._save_job_locked(job) + + def _run(self): + while not self.stop_event.is_set(): + job = self._next_pending() + if not job: + self.wakeup.wait(2) + self.wakeup.clear() + continue + self._run_job(job) + + def _run_job(self, job): + command = command_for_job(job) + staging_dir = job_staging_dir(job) + target_dir = job_output_dir(job) + env = os.environ.copy() + env.update( + { + "ANI_CLI_DOWNLOAD_DIR": str(staging_dir), + "ANI_CLI_MODE": job["mode"], + "ANI_CLI_QUALITY": job["quality"], + "TERM": env.get("TERM", "xterm-256color"), + } + ) + staging_dir.mkdir(parents=True, exist_ok=True) + target_dir.mkdir(parents=True, exist_ok=True) + + with self.lock: + job["status"] = "running" + job["started_at"] = now_iso() + job["updated_at"] = job["started_at"] + job["command"] = printable_command(command) + job["staging_dir"] = str(staging_dir) + job["target_dir"] = str(target_dir) + job["log"] = [ + f"Starting: {job['command']}", + f"Staging in: {job['staging_dir']}", + f"Final folder: {job['target_dir']}", + ] + job["cancel_requested"] = False + job["_dirty_log_lines"] = 0 + self._save_job_locked(job) + + process = subprocess.Popen( + command, + cwd=str(PROJECT_ROOT), + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + start_new_session=True, + ) + with self.lock: + self.current_process = process + self.current_job_id = job["id"] + self.current_job = job + job["pid"] = process.pid + self._save_job_locked(job) + + assert process.stdout is not None + for line in process.stdout: + self._append_log(job, line) + exit_code = process.wait() + finalize_error = None + moved_files = [] + watchlist_sync_error = None + if exit_code == 0 and not job.get("cancel_requested"): + try: + moved_files = finalize_library_files(job) + except Exception as exc: + finalize_error = exc + else: + if self.watchlist_sync_fn is not None: + try: + self.watchlist_sync_fn(job) + except Exception as exc: + watchlist_sync_error = exc + + with self.lock: + canceled = job.get("cancel_requested") + job["exit_code"] = 1 if finalize_error else exit_code + job["pid"] = None + job["finished_at"] = now_iso() + job["updated_at"] = job["finished_at"] + if canceled: + job["status"] = "canceled" + job.setdefault("log", []).append("Canceled.") + elif finalize_error: + job["status"] = "failed" + job.setdefault("log", []).append(f"Library rename failed: {finalize_error}") + elif exit_code == 0: + job["status"] = "done" + for path in moved_files: + job.setdefault("log", []).append(f"Saved: {path}") + job.setdefault("log", []).append("Download completed.") + if watchlist_sync_error: + job.setdefault("log", []).append(f"Watchlist sync failed: {watchlist_sync_error}") + elif self.watchlist_sync_fn is not None: + job.setdefault("log", []).append("Watchlist download state synced.") + else: + job["status"] = "failed" + job.setdefault("log", []).append(f"Download failed with exit code {exit_code}.") + self.current_process = None + self.current_job_id = None + self.current_job = None + self._save_job_locked(job) diff --git a/search_page.py b/search_page.py new file mode 100644 index 0000000..255c78b --- /dev/null +++ b/search_page.py @@ -0,0 +1,778 @@ +#!/usr/bin/env python3 + +"""Search page template for ani-cli-web.""" + +from template_helpers import BROWSER_API_HELPER, BROWSER_POLL_HELPER, render_page_links, render_sidebar_brand + + +INDEX_HTML = r""" + + + + + ani-cli web + + + +
+ + +
+
+
+
+

Select a result

+

Episodes will appear here.

+
+
+ + + +
+
+
+ + +
+
+ + +
+
+
+
+ + + +
+
+
+ +
+
+

Download queue

+
+ + +
+
+
+
+
+
+
+ + + + +""" diff --git a/template_helpers.py b/template_helpers.py new file mode 100644 index 0000000..f06faf0 --- /dev/null +++ b/template_helpers.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 + +"""Shared page-shell helpers for inline HTML templates.""" + +PAGE_LINKS = ( + ("search", "Search", "/"), + ("config", "Config", "/config"), + ("watchlist", "Watchlist", "/watchlist"), +) + + +BROWSER_API_HELPER = r""" + async function api(path, options = {}) { + const headers = new Headers(options.headers || {}); + if (options.body && !headers.has("Content-Type")) { + headers.set("Content-Type", "application/json"); + } + const response = await fetch(path, { + ...options, + headers + }); + const raw = await response.text(); + let data = {}; + if (raw) { + try { + data = JSON.parse(raw); + } catch (_error) { + data = { error: raw }; + } + } + if (!response.ok) throw new Error(data.error || response.statusText || "Request failed"); + return data; + } +""" + + +BROWSER_POLL_HELPER = r""" + function startSerialPoll(callback, intervalMs) { + let inFlight = false; + return window.setInterval(async () => { + if (inFlight) return; + inFlight = true; + try { + await callback(); + } finally { + inFlight = false; + } + }, intervalMs); + } +""" + + +def render_sidebar_brand(note, badge, badge_id=""): + badge_attr = f' id="{badge_id}"' if badge_id else "" + return ( + '
\n' + '
\n' + '

ani-cli web

\n' + f"

{note}

\n" + "
\n" + f" {badge}\n" + "
\n" + ) + + +def render_page_links(active_page): + links = [] + for key, label, href in PAGE_LINKS: + class_name = "page-link active" if key == active_page else "page-link" + links.append(f' {label}') + return "
\n" + "\n".join(links) + "\n
\n" diff --git a/test_app.py b/test_app.py index 6acb910..f2c486b 100644 --- a/test_app.py +++ b/test_app.py @@ -18,6 +18,9 @@ TEMP_STATE = tempfile.TemporaryDirectory() if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) +import queue_jobs +import http_handler + os.environ.setdefault("ANI_CLI_WEB_DISABLE_WORKER", "1") os.environ["ANI_CLI_WEB_STATE_ROOT"] = TEMP_STATE.name @@ -29,12 +32,16 @@ IMPORT_RUNTIME_WAS_DEFERRED = ( APP.CONFIG is None and APP.WATCHLIST is None and APP.DOWNLOAD_QUEUE is None and APP.WATCHLIST_REFRESH is None ) APP.initialize_runtime(start_workers=False) +APP.reset_runtime(wait=True) +APP.initialize_runtime(start_workers=False) +unittest.addModuleCleanup(lambda: APP.reset_runtime(wait=True)) class DummyHandler: def __init__(self, path, body=b"{}", content_length=None): self.path = path self.client_address = ("127.0.0.1", 8421) + self.handler_context = APP.Handler.handler_context self.headers = {} if content_length is not None: self.headers["Content-Length"] = str(content_length) @@ -82,8 +89,46 @@ class NetworkGuardTests(unittest.TestCase): self.assertFalse(APP.client_address_is_local("192.168.1.25")) self.assertFalse(APP.client_address_is_local("10.0.0.8")) + def test_remote_access_requires_token_for_non_local_clients(self): + handler = DummyHandler("/api/config") + handler.client_address = ("192.168.1.25", 8421) + + with mock.patch.dict(os.environ, {"ANI_CLI_WEB_ALLOW_REMOTE": "1"}, clear=False): + with self.assertRaises(PermissionError) as ctx: + APP.Handler.ensure_client_access(handler) + + self.assertIn("REMOTE_TOKEN", str(ctx.exception)) + + def test_remote_access_accepts_valid_token_for_non_local_clients(self): + handler = DummyHandler("/api/config") + handler.client_address = ("192.168.1.25", 8421) + handler.headers["Authorization"] = "Bearer secret-token" + + with mock.patch.dict( + os.environ, + {"ANI_CLI_WEB_ALLOW_REMOTE": "1", "ANI_CLI_WEB_REMOTE_TOKEN": "secret-token"}, + clear=False, + ): + APP.Handler.ensure_client_access(handler) + + def test_loopback_proxy_with_forwarded_remote_ip_requires_token(self): + handler = DummyHandler("/api/config") + handler.client_address = ("127.0.0.1", 8421) + handler.headers["X-Forwarded-For"] = "203.0.113.10" + + with mock.patch.dict(os.environ, {"ANI_CLI_WEB_ALLOW_REMOTE": "1"}, clear=False): + with self.assertRaises(PermissionError) as ctx: + APP.Handler.ensure_client_access(handler) + + self.assertIn("REMOTE_TOKEN", str(ctx.exception)) + class DownloadQueueCancelTests(unittest.TestCase): + def setUp(self): + queue = APP.DownloadQueue(lambda: {"mode": "sub", "quality": "best", "download_dir": "/tmp/example"}, start_worker=False) + with queue._connect() as conn: + conn.execute("DELETE FROM jobs") + def test_cancel_updates_inflight_job_state(self): queue = object.__new__(APP.DownloadQueue) queue.lock = threading.RLock() @@ -104,19 +149,94 @@ class DownloadQueueCancelTests(unittest.TestCase): self.assertEqual(saved[-1]["id"], "job-1") self.assertTrue(saved[-1]["cancel_requested"]) + def test_append_log_batches_persistence(self): + queue = object.__new__(APP.DownloadQueue) + queue.lock = threading.RLock() + saved = [] + + def fake_save(job): + saved.append(list(job.get("log", []))) + job["_dirty_log_lines"] = 0 + job["_last_persist_monotonic"] = 0.0 + + queue._save_job_locked = fake_save + job = {"log": [], "updated_at": APP.now_iso()} + + with mock.patch.object(queue_jobs.time, "monotonic", return_value=0.0): + for index in range(queue_jobs.LOG_PERSIST_LINE_BATCH - 1): + APP.DownloadQueue._append_log(queue, job, f"line {index}") + + self.assertEqual(saved, []) + + APP.DownloadQueue._append_log(queue, job, "line flush") + + self.assertEqual(len(saved), 1) + self.assertEqual(saved[0][-1], "line flush") + + def test_save_job_splits_log_and_extra_payload_columns(self): + queue = APP.DownloadQueue(lambda: {"mode": "sub", "quality": "best", "download_dir": "/tmp/example"}, start_worker=False) + job = { + "id": "job-structured", + "show_id": "show-1", + "title": "Structured Show", + "anime_name": "Structured Show", + "season": "1", + "query": "Structured Show", + "result_index": 1, + "mode": "sub", + "quality": "best", + "episodes": "1-2", + "download_dir": "/tmp/example", + "status": "pending", + "created_at": APP.now_iso(), + "updated_at": APP.now_iso(), + "log": ["line 1", "line 2"], + "custom_flag": True, + } + + with queue.lock: + queue._save_job_locked(job) + + with queue._connect() as conn: + row = conn.execute("SELECT * FROM jobs WHERE id = ?", (job["id"],)).fetchone() + + self.assertEqual(row["title"], "Structured Show") + self.assertEqual(row["log_payload"], '["line 1", "line 2"]') + self.assertEqual(APP.json.loads(row["payload"]), {"custom_flag": True}) + restored = queue._job_from_row(row) + self.assertEqual(restored["log"], ["line 1", "line 2"]) + self.assertTrue(restored["custom_flag"]) + + def test_shutdown_wait_cancels_active_process_for_deterministic_teardown(self): + queue = object.__new__(APP.DownloadQueue) + queue.lock = threading.RLock() + queue.stop_event = threading.Event() + queue.wakeup = threading.Event() + queue.current_process = mock.Mock(pid=4321) + queue.current_job = {"id": "job-1", "cancel_requested": False} + queue.worker = mock.Mock() + queue.worker.is_alive.side_effect = [True, False, False] + + with mock.patch.object(APP.os, "getpgid", return_value=4321), mock.patch.object(APP.os, "killpg") as killpg: + stopped = APP.DownloadQueue.shutdown(queue, wait=True, cancel_active=False) + + self.assertTrue(stopped) + self.assertTrue(queue.current_job["cancel_requested"]) + killpg.assert_called_once_with(4321, queue_jobs.signal.SIGTERM) + class WatchlistRefreshAllTests(unittest.TestCase): def test_refresh_all_iterates_every_show_id(self): store = object.__new__(APP.WatchlistStore) calls = [] store.all_show_ids = lambda: ["show-a", "show-b", "show-c"] - store.refresh = lambda show_id: calls.append(show_id) or {"show_id": show_id} + store.refresh = lambda show_id, include_thumbnail=True: calls.append((show_id, include_thumbnail)) or {"show_id": show_id} result = APP.WatchlistStore.refresh_all(store) - self.assertEqual(calls, ["show-a", "show-b", "show-c"]) + self.assertEqual(calls, [("show-a", False), ("show-b", False), ("show-c", False)]) self.assertEqual(result["count"], 3) - self.assertEqual([item["show_id"] for item in result["items"]], calls) + self.assertEqual([item["show_id"] for item in result["items"]], ["show-a", "show-b", "show-c"]) class WatchlistRefreshJobTests(unittest.TestCase): @@ -129,7 +249,8 @@ class WatchlistRefreshJobTests(unittest.TestCase): def all_show_ids(self): return ["show-a", "show-b"] - def refresh(self, show_id): + def refresh(self, show_id, include_thumbnail=True): + self.last_include_thumbnail = include_thumbnail if show_id == "show-a": return {"title": "Show A", "status": "updated"} return {"title": "Show B", "status": "error"} @@ -147,13 +268,41 @@ class WatchlistRefreshJobTests(unittest.TestCase): self.assertEqual(status["status"], "done") self.assertEqual(status["completed"], 2) self.assertEqual(status["errors"], 1) + self.assertFalse(service.store.last_include_thumbnail) + + def test_refresh_job_skips_removed_items_without_thumbnail_warmup(self): + calls = [] + + class FakeStore: + def all_show_ids(self): + return ["show-a", "show-b"] + + def refresh(self, show_id, include_thumbnail=True): + calls.append((show_id, include_thumbnail)) + if show_id == "show-a": + return None + return {"title": "Show B", "status": "updated"} + + service = APP.WatchlistRefreshJobs(FakeStore(), start_worker=False) + started = service.start() + self.assertTrue(started["created"]) + + job = service._next_pending() + self.assertIsNotNone(job) + service._run_job(job) + + status = service.status()["job"] + self.assertEqual(status["status"], "done") + self.assertEqual(status["completed"], 2) + self.assertEqual(status["errors"], 0) + self.assertCountEqual(calls, [("show-a", False), ("show-b", False)]) def test_refresh_start_reuses_pending_job(self): class FakeStore: def all_show_ids(self): return [] - def refresh(self, _show_id): + def refresh(self, _show_id, include_thumbnail=True): return {"title": "Unused", "status": "updated"} service = APP.WatchlistRefreshJobs(FakeStore(), start_worker=False) @@ -170,7 +319,7 @@ class WatchlistRefreshJobTests(unittest.TestCase): def all_show_ids(self): return [] - def refresh(self, _show_id): + def refresh(self, _show_id, include_thumbnail=True): return {"title": "Unused", "status": "updated"} service = APP.WatchlistRefreshJobs(FakeStore(), start_worker=False) @@ -184,6 +333,44 @@ class WatchlistRefreshJobTests(unittest.TestCase): class ThumbnailEventRecoveryTests(unittest.TestCase): + def test_ensure_thumbnails_skips_show_removed_during_batch(self): + store = object.__new__(APP.WatchlistStore) + item = { + "show_id": "show-1", + "thumbnail_ready": False, + } + store.get = mock.Mock(side_effect=[item, KeyError("Anime not found in watchlist.")]) + store.ensure_thumbnail = mock.Mock(return_value=None) + + result = APP.WatchlistStore.ensure_thumbnails(store, ["show-1"]) + + self.assertEqual(result, {"items": []}) + store.ensure_thumbnail.assert_called_once_with("show-1", force=False) + + def test_ensure_thumbnail_waiter_returns_none_after_show_is_removed(self): + store = object.__new__(APP.WatchlistStore) + store.lock = threading.RLock() + inflight = threading.Event() + inflight.set() + store.thumbnail_events = {"show-1": inflight} + store.has_show = lambda _show_id: False + store.get = mock.Mock(return_value={ + "show_id": "show-1", + "title": "Show One", + "thumbnail_path": None, + "thumbnail_checked_at": None, + "thumbnail_ready": False, + "animeschedule_route": None, + "animeschedule_title": None, + "anidb_aid": None, + "anidb_title": None, + }) + + result = APP.WatchlistStore.ensure_thumbnail(store, "show-1") + + self.assertIsNone(result) + self.assertEqual(store.get.call_count, 1) + def test_ensure_thumbnail_clears_inflight_event_after_update_failure(self): store = object.__new__(APP.WatchlistStore) store.lock = threading.RLock() @@ -221,7 +408,384 @@ class ThumbnailEventRecoveryTests(unittest.TestCase): self.assertEqual(store.thumbnail_events, {}) +class WatchlistCompletionTests(unittest.TestCase): + def setUp(self): + with APP.WATCHLIST._connect() as conn: + conn.execute("DELETE FROM watchlist") + + def seed_watchlist_item(self, **overrides): + now = APP.now_iso() + item = { + "show_id": "show-1", + "title": "Example Show", + "category": "watching", + "downloaded": False, + "status": "tracked", + "status_message": "Waiting for the first manual refresh.", + "created_at": now, + "updated_at": now, + "last_checked": None, + "sub_count": 0, + "dub_count": 0, + "expected_count": 0, + "airing_status": None, + "sub_latest_episode": None, + "dub_latest_episode": None, + "animeschedule_route": None, + "animeschedule_title": None, + "anidb_aid": None, + "anidb_title": None, + "thumbnail_path": None, + "thumbnail_checked_at": None, + } + item.update(overrides) + with APP.WATCHLIST.lock, APP.WATCHLIST._connect() as conn: + APP.WATCHLIST._upsert_conn(conn, item) + + def test_watchlist_status_message_marks_ready_modes(self): + message = APP.watchlist_status_message(12, 10, expected_count=12, airing_status="finished") + self.assertIn("Sub: 12/12 available.", message) + self.assertIn("Dub: 10/12 available.", message) + self.assertIn("AnimeSchedule: Finished.", message) + self.assertIn("Sub ready.", message) + self.assertNotIn("Dub ready.", message) + + def test_refresh_persists_expected_episode_total_and_ready_flags(self): + self.seed_watchlist_item() + + snapshot = { + "show_id": "show-1", + "title": "Example Show", + "episodes": { + "sub": [str(index) for index in range(1, 13)], + "dub": [str(index) for index in range(1, 13)], + }, + } + schedule = { + "route": "example-show", + "title": "Example Show", + "episodes": 12, + "status": "Finished", + } + + with mock.patch.object(APP, "fetch_show_episode_snapshot", return_value=snapshot), mock.patch.object( + APP, "resolve_animeschedule_anime", return_value=schedule + ), mock.patch.object(APP.WatchlistStore, "ensure_thumbnail", return_value=None): + item = APP.WATCHLIST.refresh("show-1") + + self.assertEqual(item["expected_count"], 12) + self.assertEqual(item["airing_status"], "Finished") + self.assertTrue(item["sub_complete"]) + self.assertTrue(item["dub_complete"]) + self.assertEqual(item["sub_progress"], "12/12") + self.assertEqual(item["dub_progress"], "12/12") + self.assertEqual(item["animeschedule_route"], "example-show") + self.assertIn("Sub ready.", item["status_message"]) + self.assertIn("Dub ready.", item["status_message"]) + + def test_list_filters_by_category_and_reports_category_counts(self): + self.seed_watchlist_item(show_id="show-1", title="Watching Show", category="watching", status="queued") + self.seed_watchlist_item(show_id="show-2", title="Finished Show", category="finished", downloaded=True) + + listing = APP.WATCHLIST.list(category="watching") + + self.assertEqual(listing["active_category"], "watching") + self.assertEqual(listing["summary"]["shows"], 2) + self.assertEqual(listing["categories"]["watching"], 1) + self.assertEqual(listing["categories"]["finished"], 1) + self.assertEqual(listing["queued_count"], 1) + self.assertEqual([item["show_id"] for item in listing["items"]], ["show-1"]) + + def test_queue_refresh_marks_item_queued_without_sync_refresh(self): + self.seed_watchlist_item(show_id="show-11", title="Refresh Show", category="watching", status="updated") + + with mock.patch.object(APP.WATCHLIST, "schedule_refresh", side_effect=lambda show_id: APP.WATCHLIST.get(show_id)) as schedule_refresh, mock.patch.object( + APP.WATCHLIST, "refresh", side_effect=AssertionError("queue_refresh should not refresh synchronously") + ): + item = APP.WATCHLIST.queue_refresh("show-11", status_message="Queued from test.") + + schedule_refresh.assert_called_once_with("show-11") + self.assertEqual(item["status"], "queued") + self.assertEqual(item["status_message"], "Queued from test.") + + def test_mark_downloaded_keeps_existing_category_and_sets_download_flag(self): + self.seed_watchlist_item(show_id="show-9", title="Queue Show", category="planned", downloaded=False) + + with mock.patch.object(APP.WATCHLIST, "schedule_refresh", side_effect=lambda show_id: APP.WATCHLIST.get(show_id)), mock.patch.object( + APP.WATCHLIST, "refresh", side_effect=AssertionError("mark_downloaded should not refresh synchronously") + ): + item = APP.WATCHLIST.mark_downloaded("show-9", title="Queue Show") + + self.assertEqual(item["category"], "planned") + self.assertTrue(item["downloaded"]) + self.assertEqual(item["finished_badge"], "") + self.assertEqual(item["status"], "queued") + + def test_mark_downloaded_creates_missing_show_in_finished_category(self): + with mock.patch.object(APP.WATCHLIST, "schedule_refresh", side_effect=lambda show_id: APP.WATCHLIST.get(show_id)), mock.patch.object( + APP.WATCHLIST, "refresh", side_effect=AssertionError("mark_downloaded should not refresh synchronously") + ): + item = APP.WATCHLIST.mark_downloaded("show-10", title="Downloaded Show") + + self.assertEqual(item["category"], "finished") + self.assertTrue(item["downloaded"]) + self.assertEqual(item["finished_badge"], "downloaded") + self.assertEqual(item["status"], "queued") + + def test_add_queues_refresh_without_sync_refresh(self): + with mock.patch.object(APP.WATCHLIST, "schedule_refresh", side_effect=lambda show_id: APP.WATCHLIST.get(show_id)) as schedule_refresh, mock.patch.object( + APP.WATCHLIST, "refresh", side_effect=AssertionError("add should not refresh synchronously") + ): + result = APP.WATCHLIST.add({"show_id": "show-22", "title": "Queued Show", "category": "watching"}) + + schedule_refresh.assert_called_once_with("show-22") + self.assertTrue(result["created"]) + self.assertEqual(result["item"]["status"], "queued") + self.assertIn("Refresh queued", result["message"]) + + def test_sync_downloaded_job_to_watchlist_resolves_missing_show_id_from_search(self): + job = {"query": "Queue Show", "mode": "sub", "result_index": 1, "title": "Queue Show"} + + with mock.patch.object(APP, "search_anime", return_value=[{"id": "show-42", "title": "Queue Show"}]), mock.patch.object( + APP.WATCHLIST, "mark_downloaded", return_value={"show_id": "show-42"} + ) as mark_downloaded: + APP.sync_downloaded_job_to_watchlist(job) + + mark_downloaded.assert_called_once_with("show-42", title="Queue Show") + + def test_prepare_download_job_persists_resolved_show_id(self): + job = {"query": "Queue Show", "mode": "sub", "result_index": 1, "title": "Queue Show", "show_id": ""} + + with mock.patch.object(APP, "search_anime", return_value=[{"id": "show-42", "title": "Queue Show"}]): + prepared = APP.prepare_download_job(dict(job)) + + self.assertEqual(prepared["show_id"], "show-42") + self.assertEqual(prepared["title"], "Queue Show") + + +class WatchlistQueuedRefreshRecoveryTests(unittest.TestCase): + def setUp(self): + with APP.WATCHLIST._connect() as conn: + conn.execute("DELETE FROM watchlist") + + def seed_watchlist_item(self, **overrides): + now = APP.now_iso() + item = { + "show_id": "show-1", + "title": "Example Show", + "category": "watching", + "downloaded": False, + "status": "tracked", + "status_message": "Waiting for the first manual refresh.", + "created_at": now, + "updated_at": now, + "last_checked": None, + "sub_count": 0, + "dub_count": 0, + "expected_count": 0, + "airing_status": None, + "sub_latest_episode": None, + "dub_latest_episode": None, + "animeschedule_route": None, + "animeschedule_title": None, + "anidb_aid": None, + "anidb_title": None, + "thumbnail_path": None, + "thumbnail_checked_at": None, + } + item.update(overrides) + with APP.WATCHLIST.lock, APP.WATCHLIST._connect() as conn: + APP.WATCHLIST._upsert_conn(conn, item) + + def test_restart_restores_queued_watchlist_refreshes(self): + self.seed_watchlist_item( + show_id="show-b", + title="Beta Show", + status="queued", + status_message="Queued refresh", + updated_at="2026-05-15T10:00:00+00:00", + ) + self.seed_watchlist_item( + show_id="show-a", + title="Alpha Show", + status="queued", + status_message="Queued refresh", + updated_at="2026-05-15T09:00:00+00:00", + ) + self.seed_watchlist_item( + show_id="show-c", + title="Gamma Show", + status="updated", + status_message="Updated", + updated_at="2026-05-15T11:00:00+00:00", + ) + + restored = APP.WatchlistStore(start_worker=False) + + self.assertEqual(restored.refresh_pending, ["show-a", "show-b"]) + self.assertEqual(restored.refresh_pending_set, {"show-a", "show-b"}) + + +class WatchlistRefreshQueueTests(unittest.TestCase): + def test_schedule_refresh_skips_show_already_in_flight(self): + store = object.__new__(APP.WatchlistStore) + store.lock = threading.RLock() + store.thumbnail_events = {} + store.refresh_pending = [] + store.refresh_pending_set = set() + store.refresh_inflight_set = {"show-1"} + store.refresh_wakeup = threading.Event() + store.refresh_stop_event = threading.Event() + store.refresh_worker = None + store.worker_enabled = False + store.ensure_worker = lambda: None + store.get = lambda show_id: {"show_id": show_id, "status": "queued"} + + item = APP.WatchlistStore.schedule_refresh(store, "show-1") + + self.assertEqual(item["show_id"], "show-1") + self.assertEqual(store.refresh_pending, []) + self.assertEqual(store.refresh_pending_set, set()) + self.assertEqual(store.refresh_inflight_set, {"show-1"}) + + def test_refresh_loop_clears_inflight_marker_after_refresh(self): + store = object.__new__(APP.WatchlistStore) + store.lock = threading.RLock() + store.thumbnail_events = {} + store.refresh_pending = ["show-2"] + store.refresh_pending_set = {"show-2"} + store.refresh_inflight_set = set() + store.refresh_wakeup = threading.Event() + store.refresh_stop_event = threading.Event() + store.refresh_worker = None + store.worker_enabled = False + calls = [] + + def fake_refresh(show_id): + calls.append(show_id) + store.refresh_stop_event.set() + + store.refresh = fake_refresh + + APP.WatchlistStore._refresh_loop(store) + + self.assertEqual(calls, ["show-2"]) + self.assertEqual(store.refresh_inflight_set, set()) + self.assertEqual(store.refresh_pending, []) + self.assertEqual(store.refresh_pending_set, set()) + + def test_remove_drops_pending_and_inflight_refresh_state(self): + store = object.__new__(APP.WatchlistStore) + store.lock = threading.RLock() + thumbnail_event = threading.Event() + store.thumbnail_events = {"show-3": thumbnail_event} + store.refresh_pending = ["show-3", "show-4"] + store.refresh_pending_set = {"show-3", "show-4"} + store.refresh_inflight_set = {"show-3"} + store._connect = APP.WATCHLIST._connect + store.get = lambda show_id: {"show_id": show_id, "title": "Queued Show", "thumbnail_path": None} + + class DummyCursor: + rowcount = 1 + + class DummyConn: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def execute(self, _sql, _params): + return DummyCursor() + + store._connect = lambda: DummyConn() + + result = APP.WatchlistStore.remove(store, "show-3") + + self.assertEqual(result["item"]["show_id"], "show-3") + self.assertEqual(store.refresh_pending, ["show-4"]) + self.assertEqual(store.refresh_pending_set, {"show-4"}) + self.assertEqual(store.refresh_inflight_set, set()) + self.assertTrue(thumbnail_event.is_set()) + self.assertNotIn("show-3", store.thumbnail_events) + + +class ConfigSnapshotTests(unittest.TestCase): + def test_get_config_snapshot_returns_copy(self): + original = APP.CONFIG + APP.CONFIG = {"mode": "sub", "quality": "best", "download_dir": "/tmp/example"} + try: + snapshot = APP.get_config_snapshot() + snapshot["mode"] = "dub" + self.assertEqual(APP.CONFIG["mode"], "sub") + finally: + APP.CONFIG = original + + +class StartupBehaviorTests(unittest.TestCase): + def test_main_does_not_eagerly_initialize_runtime(self): + server = mock.Mock() + server.serve_forever.side_effect = KeyboardInterrupt() + + with mock.patch.object(APP, "initialize_runtime") as initialize_runtime, mock.patch.object( + APP, "ThreadingHTTPServer", return_value=server + ), mock.patch.object(APP, "shutdown_runtime") as shutdown_runtime: + APP.main() + + initialize_runtime.assert_not_called() + shutdown_runtime.assert_called_once_with(wait=True, cancel_active_downloads=True) + server.serve_forever.assert_called_once() + server.server_close.assert_called_once() + + def test_reset_runtime_wait_defaults_to_cancel_active_downloads(self): + APP.initialize_runtime(start_workers=False) + try: + with mock.patch.object(APP, "shutdown_runtime") as shutdown_runtime: + APP.reset_runtime(wait=True) + + shutdown_runtime.assert_called_once_with(wait=True, cancel_active_downloads=None) + self.assertIsNone(APP.CONFIG) + finally: + APP.initialize_runtime(start_workers=False) + + def test_save_runtime_config_replaces_runtime_config(self): + original = APP.CONFIG + try: + APP.CONFIG = {"mode": "sub", "quality": "best", "download_dir": "/tmp/old"} + saved = APP.save_runtime_config({"mode": "dub", "quality": "720", "download_dir": "/tmp/new"}) + self.assertEqual(saved["mode"], "dub") + self.assertEqual(APP.CONFIG["mode"], "dub") + self.assertEqual(APP.CONFIG["quality"], "720") + finally: + APP.CONFIG = original + + def test_reset_runtime_clears_runtime_services(self): + APP.initialize_runtime(start_workers=False) + APP.reset_runtime(wait=True) + try: + self.assertIsNone(APP.CONFIG) + self.assertIsNone(APP.WATCHLIST) + self.assertIsNone(APP.DOWNLOAD_QUEUE) + self.assertIsNone(APP.WATCHLIST_REFRESH) + finally: + APP.initialize_runtime(start_workers=False) + + class HandlerRouteTests(unittest.TestCase): + def test_runtime_backed_route_respects_worker_disable_flag(self): + handler = DummyHandler("/api/watchlist/refresh-status") + APP.reset_runtime(wait=True) + try: + with mock.patch.dict(os.environ, {"ANI_CLI_WEB_DISABLE_WORKER": "1"}, clear=False): + APP.Handler.do_GET(handler) + self.assertFalse(APP.WATCHLIST.worker_enabled) + self.assertIsNone(APP.WATCHLIST.refresh_worker) + self.assertIsNone(APP.DOWNLOAD_QUEUE.worker) + self.assertIsNone(APP.WATCHLIST_REFRESH.worker) + finally: + APP.reset_runtime(wait=True) + APP.initialize_runtime(start_workers=False) + def test_body_json_rejects_oversized_request(self): handler = DummyHandler("/api/watchlist/upload-thumbnail", body=b"{}", content_length=APP.MAX_JSON_BODY_BYTES + 1) with self.assertRaises(APP.HttpError) as ctx: @@ -265,6 +829,27 @@ class HandlerRouteTests(unittest.TestCase): self.assertEqual(handler.json_payload, payload) self.assertEqual(handler.json_status, HTTPStatus.ACCEPTED) + def test_update_status_post_queues_refresh(self): + body = b'{"show_id":"show-1"}' + handler = DummyHandler("/api/watchlist/update-status", body=body, content_length=len(body)) + queued_item = {"show_id": "show-1", "title": "Show 1", "status": "queued"} + with mock.patch.object(APP.WATCHLIST, "queue_refresh", return_value=queued_item): + APP.Handler.do_POST(handler) + self.assertEqual(handler.json_payload, {"message": "Queued refresh for Show 1.", "item": queued_item}) + self.assertEqual(handler.json_status, HTTPStatus.OK) + + def test_generic_handler_exception_skips_traceback_when_debug_disabled(self): + handler = DummyHandler("/api/config") + with mock.patch.object(http_handler, "debug_enabled", return_value=False), mock.patch.object( + http_handler, "debug_log" + ) as debug_log, mock.patch.object(http_handler.traceback, "print_exc") as print_exc: + APP.Handler.exception(handler, RuntimeError("boom")) + + print_exc.assert_not_called() + debug_log.assert_called_once() + self.assertEqual(handler.error_status, HTTPStatus.INTERNAL_SERVER_ERROR) + self.assertEqual(handler.error_message, "Internal server error.") + class TemplateHelperTests(unittest.TestCase): def test_page_links_marks_active_page(self): @@ -277,6 +862,55 @@ class TemplateHelperTests(unittest.TestCase): self.assertIn('id="serverBadge"', markup) self.assertIn("Shared note", markup) + def test_search_page_queue_loader_guards_against_stale_responses(self): + self.assertIn("queueRequestId", APP.INDEX_HTML) + self.assertIn("if (requestId !== state.queueRequestId) return data;", APP.INDEX_HTML) + + def test_pages_share_browser_api_helper(self): + self.assertIn("async function api(path, options = {})", APP.INDEX_HTML) + self.assertIn("async function api(path, options = {})", APP.CONFIG_HTML) + self.assertIn("async function api(path, options = {})", APP.WATCHLIST_HTML) + + def test_pages_share_serial_polling_helper(self): + self.assertIn("function startSerialPoll(callback, intervalMs)", APP.INDEX_HTML) + self.assertIn("function startSerialPoll(callback, intervalMs)", APP.WATCHLIST_HTML) + self.assertIn("startSerialPoll(() => loadQueue(), 2500)", APP.INDEX_HTML) + self.assertIn("startSerialPoll(async () => {", APP.WATCHLIST_HTML) + + def test_watchlist_page_polls_when_queued_items_exist(self): + self.assertIn("queuedCount: 0", APP.WATCHLIST_HTML) + self.assertIn("else if (state.queuedCount > 0)", APP.WATCHLIST_HTML) + self.assertIn("fetchWatchlist(state.watchlistPage, { silent: true })", APP.WATCHLIST_HTML) + + def test_watchlist_thumbnail_warmup_skips_repeated_attempts(self): + self.assertIn("thumbnailWarmupPending: new Set()", APP.WATCHLIST_HTML) + self.assertIn("function thumbnailWarmupToken(item)", APP.WATCHLIST_HTML) + self.assertIn("if (state.thumbnailWarmupTokens[item.show_id] === token) continue;", APP.WATCHLIST_HTML) + self.assertIn("state.thumbnailWarmupPending.delete(showId);", APP.WATCHLIST_HTML) + + +class AniDbCacheTests(unittest.TestCase): + def test_get_cached_anidb_titles_reuses_cached_entries(self): + original_cache = dict(APP.ANIDB_TITLES_CACHE) + try: + cached_entries = [{"aid": "1", "title": "Example", "normalized_title": "example"}] + APP.ANIDB_TITLES_CACHE.update( + { + "mtime": 123.0, + "entries": cached_entries, + "title_index": {"example": [0]}, + "token_index": {"example": [0]}, + "match_cache": {}, + } + ) + with mock.patch.object(APP, "ensure_anidb_titles_dump") as ensure_dump: + ensure_dump.return_value = mock.Mock(exists=lambda: True, stat=lambda: mock.Mock(st_mtime=123.0)) + result = APP.get_cached_anidb_titles() + self.assertIs(result, cached_entries) + finally: + APP.ANIDB_TITLES_CACHE.clear() + APP.ANIDB_TITLES_CACHE.update(original_cache) + if __name__ == "__main__": unittest.main() diff --git a/watchlist_page.py b/watchlist_page.py new file mode 100644 index 0000000..9e27880 --- /dev/null +++ b/watchlist_page.py @@ -0,0 +1,1208 @@ +#!/usr/bin/env python3 + +"""Watchlist page template for ani-cli-web.""" + +from template_helpers import BROWSER_API_HELPER, BROWSER_POLL_HELPER, render_page_links, render_sidebar_brand + + +WATCHLIST_HTML = r""" + + + + + ani-cli web - watchlist + + + +
+ + +
+
+
+
+

Tracked shows

+

Compact cards with local AnimeSchedule cover caching when artwork is available.

+
+ +
+
+
+
+ 0 + Tracked shows +
+
+ 0 + Sub episodes +
+
+ 0 + Dub episodes +
+
+
+ +
+
+
+

Watchlist library

+

Compact poster grid for faster scanning.

+

+
+
+
+
+
+
+
+
+ + + + +""" diff --git a/web_templates.py b/web_templates.py index dc9e3e3..f5b74f8 100644 --- a/web_templates.py +++ b/web_templates.py @@ -1,2122 +1,17 @@ #!/usr/bin/env python3 -"""Shared inline page templates for ani-cli-web.""" - -PAGE_LINKS = ( - ("search", "Search", "/"), - ("config", "Config", "/config"), - ("watchlist", "Watchlist", "/watchlist"), -) - - -def render_sidebar_brand(note, badge, badge_id=""): - badge_attr = f' id="{badge_id}"' if badge_id else "" - return ( - '
\n' - '
\n' - '

ani-cli web

\n' - f"

{note}

\n" - "
\n" - f" {badge}\n" - "
\n" - ) - - -def render_page_links(active_page): - links = [] - for key, label, href in PAGE_LINKS: - class_name = "page-link active" if key == active_page else "page-link" - links.append(f' {label}') - return "
\n" + "\n".join(links) + "\n
\n" - - -INDEX_HTML = r""" - - - - - ani-cli web - - - -
- - -
-
-
-
-

Select a result

-

Episodes will appear here.

-
-
- - -
-
-
- - -
-
- - -
-
-
-
- - - -
-
-
- -
-
-

Download queue

-
- - -
-
-
-
-
-
-
- - - - -""" - - -CONFIG_HTML = r""" - - - - - ani-cli web - config - - - -
- - -
-
-
-
-

Defaults

-

Manage the folder, mode, and quality that the Search page starts with.

-
- -
-
-
- -
-
- - -
-
Tip: these are the saved defaults only. The active search form can still be adjusted per download.
-
- -
-
-
- - - - -""" - - -WATCHLIST_HTML = r""" - - - - - ani-cli web - watchlist - - - -
- - -
-
-
-
-

Tracked shows

-

Compact cards with local AnimeSchedule cover caching when artwork is available.

-
- -
-
-
-
- 0 - Tracked shows -
-
- 0 - Sub episodes -
-
- 0 - Dub episodes -
-
-
- -
-
-
-

Watchlist library

-

Compact poster grid for faster scanning.

-

-
-
-
-
-
-
-
- - - - -""" +"""Aggregated template exports for ani-cli-web.""" + +from config_page import CONFIG_HTML +from search_page import INDEX_HTML +from template_helpers import PAGE_LINKS, render_page_links, render_sidebar_brand +from watchlist_page import WATCHLIST_HTML + +__all__ = [ + "CONFIG_HTML", + "INDEX_HTML", + "PAGE_LINKS", + "WATCHLIST_HTML", + "render_page_links", + "render_sidebar_brand", +]