191 lines
10 KiB
Python
191 lines
10 KiB
Python
"""Durable FIFO queue. Trusted plugin workers own execution and renew leases."""
|
||
from contextlib import contextmanager
|
||
import json
|
||
import secrets
|
||
import time
|
||
from web_framework.config import IDENTIFIER
|
||
|
||
STATES = ('queued', 'running', 'completed', 'failed', 'cancelled')
|
||
|
||
|
||
def bounded_text(value, label, maximum):
|
||
if not isinstance(value, str) or not value.strip() or len(value) > maximum:
|
||
raise ValueError(f'{label} must contain 1–{maximum} characters.')
|
||
return value.strip()
|
||
|
||
|
||
class Queue:
|
||
"""Acquire via get_plugins().service('queues', 'queue') in an app context.
|
||
|
||
Each mutation uses the control store's SQLite transaction. Claim tokens are
|
||
private capabilities: a stale worker cannot update a reclaimed/cancelled job.
|
||
"""
|
||
def __init__(self, app):
|
||
self.app = app
|
||
self.state = app.extensions['state']
|
||
with self.state.db() as db:
|
||
db.executescript('''
|
||
CREATE TABLE IF NOT EXISTS queue_jobs (
|
||
sequence INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
id TEXT UNIQUE NOT NULL, source TEXT NOT NULL, title TEXT NOT NULL,
|
||
payload BLOB NOT NULL, status TEXT NOT NULL DEFAULT 'queued',
|
||
completed INTEGER NOT NULL DEFAULT 0, total INTEGER NOT NULL DEFAULT 100,
|
||
created REAL NOT NULL, updated REAL NOT NULL,
|
||
token TEXT, lease_until REAL, attempts INTEGER NOT NULL DEFAULT 0);
|
||
CREATE INDEX IF NOT EXISTS queue_ready ON queue_jobs(status, source, sequence);
|
||
CREATE TABLE IF NOT EXISTS queue_logs (
|
||
sequence INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
job_id TEXT NOT NULL REFERENCES queue_jobs(id) ON DELETE CASCADE,
|
||
stamp REAL NOT NULL, level TEXT NOT NULL, message TEXT NOT NULL);
|
||
CREATE INDEX IF NOT EXISTS queue_log_lookup ON queue_logs(job_id, sequence);
|
||
''')
|
||
|
||
@contextmanager
|
||
def _db(self):
|
||
if 'queues' not in self.app.extensions['plugins'].enabled():
|
||
raise ValueError('The Queues plugin is disabled.')
|
||
with self.state.db() as db:
|
||
yield db
|
||
|
||
def _public(self, row, payload=False):
|
||
item = {key: row[key] for key in ('id', 'source', 'title', 'status', 'completed', 'total', 'created', 'updated', 'attempts')}
|
||
item['progress'] = round(100 * item['completed'] / item['total'], 1)
|
||
if payload:
|
||
item['payload'] = json.loads(self.state.cipher.decrypt(row['payload']))
|
||
return item
|
||
|
||
def _event(self, db, job_id, message, level='info'):
|
||
message = bounded_text(message, 'Log message', 4000)
|
||
if level not in ('info', 'warning', 'error'):
|
||
raise ValueError('Choose info, warning, or error.')
|
||
db.execute('INSERT INTO queue_logs(job_id,stamp,level,message) VALUES(?,?,?,?)', (job_id, time.time(), level, message))
|
||
db.execute('DELETE FROM queue_logs WHERE job_id=? AND sequence NOT IN (SELECT sequence FROM queue_logs WHERE job_id=? ORDER BY sequence DESC LIMIT 500)', (job_id, job_id))
|
||
|
||
def _owned(self, db, job_id, token):
|
||
row = db.execute('SELECT * FROM queue_jobs WHERE id=?', (job_id,)).fetchone()
|
||
if not row:
|
||
raise ValueError('Job not found.')
|
||
if row['status'] != 'running' or not isinstance(token, str) or not row['token'] or not secrets.compare_digest(row['token'], token) or row['lease_until'] <= time.time():
|
||
raise ValueError('Job claim expired or is no longer active.')
|
||
return row
|
||
|
||
def enqueue(self, source, title, payload=None, *, total=100):
|
||
"""Submit JSON data under the producing plugin's slug; return its job ID."""
|
||
if not isinstance(source, str) or not IDENTIFIER.fullmatch(source):
|
||
raise ValueError('Source must be a plugin slug.')
|
||
title = bounded_text(title, 'Title', 160)
|
||
self._counts(0, total)
|
||
raw = json.dumps(payload, allow_nan=False).encode()
|
||
if len(raw) > 65536:
|
||
raise ValueError('Payload exceeds 64 KiB.')
|
||
job_id, now = secrets.token_hex(16), time.time()
|
||
with self._db() as db:
|
||
db.execute('INSERT INTO queue_jobs(id,source,title,payload,total,created,updated) VALUES(?,?,?,?,?,?,?)',
|
||
(job_id, source, title, self.state.cipher.encrypt(raw), total, now, now))
|
||
self._event(db, job_id, 'Job queued.')
|
||
return job_id
|
||
|
||
def claim(self, source, *, lease_seconds=300):
|
||
"""Atomically claim the oldest queued/expired job for a source, or None.
|
||
|
||
Returned token must accompany worker updates. Expired jobs are delivered
|
||
again; handlers must be idempotent because execution is at least once.
|
||
"""
|
||
self._lease(lease_seconds)
|
||
now = time.time()
|
||
with self._db() as db:
|
||
row = db.execute("SELECT * FROM queue_jobs WHERE source=? AND (status='queued' OR (status='running' AND lease_until<=?)) ORDER BY sequence LIMIT 1", (source, now)).fetchone()
|
||
if row is None:
|
||
return None
|
||
token = secrets.token_hex(32)
|
||
db.execute("UPDATE queue_jobs SET status='running',token=?,lease_until=?,updated=?,attempts=attempts+1 WHERE id=?", (token, now + lease_seconds, now, row['id']))
|
||
self._event(db, row['id'], 'Expired claim recovered.' if row['status'] == 'running' else 'Job started.')
|
||
fresh = db.execute('SELECT * FROM queue_jobs WHERE id=?', (row['id'],)).fetchone()
|
||
return {**self._public(fresh, payload=True), 'token': token}
|
||
|
||
@staticmethod
|
||
def _lease(seconds):
|
||
if type(seconds) is not int or not 10 <= seconds <= 86400:
|
||
raise ValueError('Lease must be 10–86400 whole seconds.')
|
||
|
||
@staticmethod
|
||
def _counts(completed, total):
|
||
if type(completed) is not int or type(total) is not int or not 1 <= total <= 2147483647 or not 0 <= completed <= total:
|
||
raise ValueError('Progress must be whole numbers with 0 <= completed <= total and total > 0.')
|
||
|
||
def progress(self, job_id, token, completed, *, total=None, message=None, lease_seconds=300):
|
||
"""Report work and renew the lease; optional message appears in job logs."""
|
||
self._lease(lease_seconds)
|
||
with self._db() as db:
|
||
row = self._owned(db, job_id, token)
|
||
total = row['total'] if total is None else total
|
||
self._counts(completed, total)
|
||
now = time.time()
|
||
db.execute('UPDATE queue_jobs SET completed=?,total=?,updated=?,lease_until=? WHERE id=?', (completed, total, now, now + lease_seconds, job_id))
|
||
if message is not None:
|
||
self._event(db, job_id, message)
|
||
|
||
def log(self, job_id, token, message, *, level='info'):
|
||
"""Append a bounded log entry; use progress() to renew the claim."""
|
||
with self._db() as db:
|
||
self._owned(db, job_id, token)
|
||
self._event(db, job_id, message, level)
|
||
|
||
def _finish(self, job_id, token, status, message):
|
||
with self._db() as db:
|
||
self._owned(db, job_id, token)
|
||
db.execute('UPDATE queue_jobs SET status=?,completed=CASE WHEN ?=\'completed\' THEN total ELSE completed END,updated=?,token=NULL,lease_until=NULL WHERE id=?', (status, status, time.time(), job_id))
|
||
self._event(db, job_id, message, 'error' if status == 'failed' else 'info')
|
||
|
||
def complete(self, job_id, token, message='Job completed.'):
|
||
"""Mark success and set progress to 100%."""
|
||
self._finish(job_id, token, 'completed', message)
|
||
|
||
def fail(self, job_id, token, message='Job failed.'):
|
||
"""Mark failure, preserving the last reported progress."""
|
||
self._finish(job_id, token, 'failed', message)
|
||
|
||
def cancel(self, job_id):
|
||
"""Cancel queued/running work and invalidate its claim cooperatively."""
|
||
with self._db() as db:
|
||
changed = db.execute("UPDATE queue_jobs SET status='cancelled',token=NULL,lease_until=NULL,updated=? WHERE id=? AND status IN ('queued','running')", (time.time(), job_id)).rowcount
|
||
if not changed:
|
||
raise ValueError('Only queued or running jobs can be cancelled.')
|
||
self._event(db, job_id, 'Job cancelled; worker claim revoked.', 'warning')
|
||
|
||
def retry(self, job_id):
|
||
"""Requeue failed/cancelled jobs with fresh progress; preserve history."""
|
||
with self._db() as db:
|
||
changed = db.execute("UPDATE queue_jobs SET status='queued',completed=0,updated=? WHERE id=? AND status IN ('failed','cancelled')", (time.time(), job_id)).rowcount
|
||
if not changed:
|
||
raise ValueError('Only failed or cancelled jobs can be retried.')
|
||
self._event(db, job_id, 'Job queued for retry.')
|
||
|
||
def jobs(self, *, status=None, source=None, limit=100, offset=0):
|
||
"""Return paginated metadata, newest first; never expose payloads/tokens."""
|
||
if status is not None and status not in STATES:
|
||
raise ValueError('Unknown queue status.')
|
||
if type(limit) is not int or not 1 <= limit <= 200 or type(offset) is not int or offset < 0:
|
||
raise ValueError('Invalid pagination.')
|
||
clauses, parameters = [], []
|
||
for key, value in (('status', status), ('source', source)):
|
||
if value is not None:
|
||
clauses.append(key + '=?')
|
||
parameters.append(value)
|
||
where = ' WHERE ' + ' AND '.join(clauses) if clauses else ''
|
||
with self._db() as db:
|
||
return [self._public(row) for row in db.execute('SELECT * FROM queue_jobs' + where + ' ORDER BY sequence DESC LIMIT ? OFFSET ?', (*parameters, limit, offset))]
|
||
|
||
def get(self, job_id):
|
||
"""Read metadata; raises ValueError for a missing job."""
|
||
with self._db() as db:
|
||
row = db.execute('SELECT * FROM queue_jobs WHERE id=?', (job_id,)).fetchone()
|
||
if row is None:
|
||
raise ValueError('Job not found.')
|
||
return self._public(row)
|
||
|
||
def logs(self, job_id):
|
||
"""Return the latest 500 entries in chronological order."""
|
||
with self._db() as db:
|
||
return [dict(row) for row in db.execute('SELECT stamp,level,message FROM queue_logs WHERE job_id=? ORDER BY sequence', (job_id,))]
|