Tighten worker shutdown and poll helper cleanup
This commit is contained in:
+14
-7
@@ -16,8 +16,10 @@ from app_support import (
|
|||||||
PROJECT_ROOT,
|
PROJECT_ROOT,
|
||||||
QUEUE_PATH,
|
QUEUE_PATH,
|
||||||
STATE_DB_PATH,
|
STATE_DB_PATH,
|
||||||
|
WORKER_SHUTDOWN_TIMEOUT_SECONDS,
|
||||||
build_job,
|
build_job,
|
||||||
command_for_job,
|
command_for_job,
|
||||||
|
debug_log,
|
||||||
finalize_library_files,
|
finalize_library_files,
|
||||||
job_output_dir,
|
job_output_dir,
|
||||||
job_staging_dir,
|
job_staging_dir,
|
||||||
@@ -25,14 +27,12 @@ from app_support import (
|
|||||||
now_iso,
|
now_iso,
|
||||||
printable_command,
|
printable_command,
|
||||||
strip_control,
|
strip_control,
|
||||||
debug_log,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
LOG_PERSIST_INTERVAL_SECONDS = 0.75
|
LOG_PERSIST_INTERVAL_SECONDS = 0.75
|
||||||
LOG_PERSIST_LINE_BATCH = 8
|
LOG_PERSIST_LINE_BATCH = 8
|
||||||
WATCHLIST_REFRESH_MAX_WORKERS = 4
|
WATCHLIST_REFRESH_MAX_WORKERS = 4
|
||||||
WORKER_JOIN_TIMEOUT_SECONDS = 2.5
|
|
||||||
JOB_STRUCTURED_COLUMNS = (
|
JOB_STRUCTURED_COLUMNS = (
|
||||||
"show_id",
|
"show_id",
|
||||||
"title",
|
"title",
|
||||||
@@ -97,7 +97,7 @@ class WatchlistRefreshJobs:
|
|||||||
self.wakeup.set()
|
self.wakeup.set()
|
||||||
worker = self.worker
|
worker = self.worker
|
||||||
if wait and worker is not None and worker.is_alive():
|
if wait and worker is not None and worker.is_alive():
|
||||||
worker.join(timeout=WORKER_JOIN_TIMEOUT_SECONDS)
|
worker.join(timeout=WORKER_SHUTDOWN_TIMEOUT_SECONDS)
|
||||||
return worker is None or not worker.is_alive()
|
return worker is None or not worker.is_alive()
|
||||||
|
|
||||||
def _init_db(self):
|
def _init_db(self):
|
||||||
@@ -241,6 +241,7 @@ class WatchlistRefreshJobs:
|
|||||||
self._run_job(job)
|
self._run_job(job)
|
||||||
|
|
||||||
def _run_job(self, job):
|
def _run_job(self, job):
|
||||||
|
interrupted = False
|
||||||
try:
|
try:
|
||||||
show_ids = self.store.all_show_ids()
|
show_ids = self.store.all_show_ids()
|
||||||
started_at = now_iso()
|
started_at = now_iso()
|
||||||
@@ -282,6 +283,10 @@ class WatchlistRefreshJobs:
|
|||||||
self.current_executor = executor
|
self.current_executor = executor
|
||||||
futures = [executor.submit(refresh_one, show_id) for show_id in show_ids]
|
futures = [executor.submit(refresh_one, show_id) for show_id in show_ids]
|
||||||
for index, future in enumerate(concurrent.futures.as_completed(futures), start=1):
|
for index, future in enumerate(concurrent.futures.as_completed(futures), start=1):
|
||||||
|
if self.stop_event.is_set():
|
||||||
|
interrupted = True
|
||||||
|
executor.shutdown(wait=False, cancel_futures=True)
|
||||||
|
break
|
||||||
result = future.result()
|
result = future.result()
|
||||||
title = result["title"]
|
title = result["title"]
|
||||||
with self.lock:
|
with self.lock:
|
||||||
@@ -301,11 +306,13 @@ class WatchlistRefreshJobs:
|
|||||||
|
|
||||||
finished_at = now_iso()
|
finished_at = now_iso()
|
||||||
with self.lock:
|
with self.lock:
|
||||||
job["status"] = "done"
|
job["status"] = "interrupted" if interrupted else "done"
|
||||||
job["finished_at"] = finished_at
|
job["finished_at"] = finished_at
|
||||||
job["updated_at"] = finished_at
|
job["updated_at"] = finished_at
|
||||||
job["current_title"] = None
|
job["current_title"] = None
|
||||||
if job["errors"]:
|
if interrupted:
|
||||||
|
job["message"] = f"Watchlist refresh interrupted after {job['completed']} entries."
|
||||||
|
elif job["errors"]:
|
||||||
job["message"] = (
|
job["message"] = (
|
||||||
f"Refreshed {job['completed']} watchlist entries with {job['errors']} refresh errors."
|
f"Refreshed {job['completed']} watchlist entries with {job['errors']} refresh errors."
|
||||||
)
|
)
|
||||||
@@ -366,10 +373,10 @@ class DownloadQueue:
|
|||||||
self.wakeup.set()
|
self.wakeup.set()
|
||||||
worker = self.worker
|
worker = self.worker
|
||||||
if wait and worker is not None and worker.is_alive():
|
if wait and worker is not None and worker.is_alive():
|
||||||
worker.join(timeout=WORKER_JOIN_TIMEOUT_SECONDS)
|
worker.join(timeout=WORKER_SHUTDOWN_TIMEOUT_SECONDS)
|
||||||
if worker.is_alive():
|
if worker.is_alive():
|
||||||
self._signal_active_process(signal.SIGKILL)
|
self._signal_active_process(signal.SIGKILL)
|
||||||
worker.join(timeout=WORKER_JOIN_TIMEOUT_SECONDS)
|
worker.join(timeout=WORKER_SHUTDOWN_TIMEOUT_SECONDS)
|
||||||
return worker is None or not worker.is_alive()
|
return worker is None or not worker.is_alive()
|
||||||
|
|
||||||
def _signal_active_process(self, signum):
|
def _signal_active_process(self, signum):
|
||||||
|
|||||||
+20
-3
@@ -37,15 +37,32 @@ BROWSER_API_HELPER = r"""
|
|||||||
BROWSER_POLL_HELPER = r"""
|
BROWSER_POLL_HELPER = r"""
|
||||||
function startSerialPoll(callback, intervalMs) {
|
function startSerialPoll(callback, intervalMs) {
|
||||||
let inFlight = false;
|
let inFlight = false;
|
||||||
return window.setInterval(async () => {
|
let active = true;
|
||||||
if (inFlight) return;
|
let timer = null;
|
||||||
|
const stop = () => {
|
||||||
|
active = false;
|
||||||
|
if (timer !== null) {
|
||||||
|
window.clearTimeout(timer);
|
||||||
|
timer = null;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
const tick = async () => {
|
||||||
|
if (!active || inFlight) {
|
||||||
|
if (active) timer = window.setTimeout(tick, intervalMs);
|
||||||
|
return;
|
||||||
|
}
|
||||||
inFlight = true;
|
inFlight = true;
|
||||||
try {
|
try {
|
||||||
await callback();
|
await callback();
|
||||||
} finally {
|
} finally {
|
||||||
inFlight = false;
|
inFlight = false;
|
||||||
|
if (active) timer = window.setTimeout(tick, intervalMs);
|
||||||
}
|
}
|
||||||
}, intervalMs);
|
};
|
||||||
|
window.addEventListener("pagehide", stop, { once: true });
|
||||||
|
window.addEventListener("beforeunload", stop, { once: true });
|
||||||
|
timer = window.setTimeout(tick, intervalMs);
|
||||||
|
return { stop };
|
||||||
}
|
}
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user