124 lines
5.0 KiB
Python
124 lines
5.0 KiB
Python
"""Scheduled, downloadable ZIP snapshots of the watchlist database export."""
|
|
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import zipfile
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
|
|
BACKUP_NAME = re.compile(r"^kaizoku-watchlist-\d{8}T\d{6}Z(?:-\d+)?\.zip$")
|
|
ARCHIVE_MEMBER = "watchlist.json"
|
|
|
|
|
|
class WatchlistBackupScheduler:
|
|
def __init__(self, export_fn, import_fn, config_getter, backup_dir=None, start_worker=True):
|
|
self.export_fn = export_fn
|
|
self.import_fn = import_fn
|
|
self.config_getter = config_getter
|
|
self.backup_dir = Path(backup_dir or os.environ.get("KAIZOKU_BACKUP_DIR") or (Path(__file__).resolve().parent / "backup"))
|
|
self.lock = threading.RLock()
|
|
self.wakeup = threading.Event()
|
|
self.stop_event = threading.Event()
|
|
self.worker = None
|
|
self.worker_enabled = bool(start_worker)
|
|
self.last_error = ""
|
|
if start_worker:
|
|
self.ensure_worker()
|
|
|
|
def ensure_worker(self):
|
|
if not self.worker_enabled:
|
|
return
|
|
with self.lock:
|
|
if self.worker and self.worker.is_alive():
|
|
return
|
|
self.stop_event.clear()
|
|
self.worker = threading.Thread(target=self._loop, name="watchlist-backup", daemon=True)
|
|
self.worker.start()
|
|
|
|
def shutdown(self, wait=False):
|
|
self.stop_event.set()
|
|
self.wakeup.set()
|
|
if wait and self.worker and self.worker.is_alive():
|
|
self.worker.join(timeout=10)
|
|
return not (self.worker and self.worker.is_alive())
|
|
|
|
def notify_config_changed(self):
|
|
self.wakeup.set()
|
|
|
|
def _paths(self):
|
|
if not self.backup_dir.exists():
|
|
return []
|
|
return sorted((path for path in self.backup_dir.iterdir() if path.is_file() and BACKUP_NAME.fullmatch(path.name)), key=lambda path: (path.stat().st_mtime, path.name), reverse=True)
|
|
|
|
def list(self):
|
|
with self.lock:
|
|
return {"backups": [{"name": path.name, "size": path.stat().st_size, "created_at": datetime.fromtimestamp(path.stat().st_mtime, timezone.utc).isoformat()} for path in self._paths()], "directory": str(self.backup_dir), "error": self.last_error}
|
|
|
|
def _path(self, name):
|
|
if not isinstance(name, str) or not BACKUP_NAME.fullmatch(name):
|
|
raise ValueError("Invalid backup filename.")
|
|
path = self.backup_dir / name
|
|
if not path.is_file() or path.is_symlink():
|
|
raise ValueError("Backup not found.")
|
|
return path
|
|
|
|
def download(self, name):
|
|
with self.lock:
|
|
return self._path(name).read_bytes()
|
|
|
|
def create(self):
|
|
with self.lock:
|
|
self.backup_dir.mkdir(parents=True, exist_ok=True)
|
|
snapshot = self.export_fn()
|
|
stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
|
name = f"kaizoku-watchlist-{stamp}.zip"
|
|
counter = 1
|
|
while (self.backup_dir / name).exists():
|
|
name = f"kaizoku-watchlist-{stamp}-{counter}.zip"
|
|
counter += 1
|
|
descriptor, temporary = tempfile.mkstemp(prefix=".watchlist-", suffix=".tmp", dir=self.backup_dir)
|
|
try:
|
|
with os.fdopen(descriptor, "wb") as output:
|
|
with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
|
archive.writestr(ARCHIVE_MEMBER, json.dumps(snapshot, indent=2, sort_keys=True))
|
|
os.replace(temporary, self.backup_dir / name)
|
|
finally:
|
|
if os.path.exists(temporary):
|
|
os.unlink(temporary)
|
|
keep = int(self.config_getter().get("watchlist_backup_keep", 14))
|
|
for stale in self._paths()[max(1, keep):]:
|
|
stale.unlink()
|
|
self.last_error = ""
|
|
return {"name": name, "message": "Watchlist backup created."}
|
|
|
|
def restore(self, name):
|
|
with self.lock:
|
|
payload = self._path(name).read_bytes()
|
|
with zipfile.ZipFile(io.BytesIO(payload)) as archive:
|
|
members = archive.infolist()
|
|
if len(members) != 1 or members[0].filename != ARCHIVE_MEMBER or members[0].file_size > 50_000_000:
|
|
raise ValueError("Backup archive must contain one watchlist.json file under 50 MB.")
|
|
snapshot = json.loads(archive.read(ARCHIVE_MEMBER))
|
|
return self.import_fn(snapshot)
|
|
|
|
def _loop(self):
|
|
while not self.stop_event.is_set():
|
|
config = self.config_getter()
|
|
if config.get("watchlist_backup_enabled", True):
|
|
try:
|
|
with self.lock:
|
|
paths = self._paths()
|
|
due = not paths or time.time() - paths[0].stat().st_mtime >= int(config.get("watchlist_backup_minutes", 1440)) * 60
|
|
if due:
|
|
self.create()
|
|
except Exception as exc:
|
|
self.last_error = str(exc)
|
|
self.wakeup.wait(timeout=60)
|
|
self.wakeup.clear()
|