Files
web-framework-resources/plugins/queues/connector.py
T

191 lines
10 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Durable FIFO queue. Trusted plugin workers own execution and renew leases."""
from contextlib import contextmanager
import json
import secrets
import time
from web_framework.config import IDENTIFIER
STATES = ('queued', 'running', 'completed', 'failed', 'cancelled')
def bounded_text(value, label, maximum):
if not isinstance(value, str) or not value.strip() or len(value) > maximum:
raise ValueError(f'{label} must contain 1–{maximum} characters.')
return value.strip()
class Queue:
"""Acquire via get_plugins().service('queues', 'queue') in an app context.
Each mutation uses the control store's SQLite transaction. Claim tokens are
private capabilities: a stale worker cannot update a reclaimed/cancelled job.
"""
def __init__(self, app):
self.app = app
self.state = app.extensions['state']
with self.state.db() as db:
db.executescript('''
CREATE TABLE IF NOT EXISTS queue_jobs (
sequence INTEGER PRIMARY KEY AUTOINCREMENT,
id TEXT UNIQUE NOT NULL, source TEXT NOT NULL, title TEXT NOT NULL,
payload BLOB NOT NULL, status TEXT NOT NULL DEFAULT 'queued',
completed INTEGER NOT NULL DEFAULT 0, total INTEGER NOT NULL DEFAULT 100,
created REAL NOT NULL, updated REAL NOT NULL,
token TEXT, lease_until REAL, attempts INTEGER NOT NULL DEFAULT 0);
CREATE INDEX IF NOT EXISTS queue_ready ON queue_jobs(status, source, sequence);
CREATE TABLE IF NOT EXISTS queue_logs (
sequence INTEGER PRIMARY KEY AUTOINCREMENT,
job_id TEXT NOT NULL REFERENCES queue_jobs(id) ON DELETE CASCADE,
stamp REAL NOT NULL, level TEXT NOT NULL, message TEXT NOT NULL);
CREATE INDEX IF NOT EXISTS queue_log_lookup ON queue_logs(job_id, sequence);
''')
@contextmanager
def _db(self):
if 'queues' not in self.app.extensions['plugins'].enabled():
raise ValueError('The Queues plugin is disabled.')
with self.state.db() as db:
yield db
def _public(self, row, payload=False):
item = {key: row[key] for key in ('id', 'source', 'title', 'status', 'completed', 'total', 'created', 'updated', 'attempts')}
item['progress'] = round(100 * item['completed'] / item['total'], 1)
if payload:
item['payload'] = json.loads(self.state.cipher.decrypt(row['payload']))
return item
def _event(self, db, job_id, message, level='info'):
message = bounded_text(message, 'Log message', 4000)
if level not in ('info', 'warning', 'error'):
raise ValueError('Choose info, warning, or error.')
db.execute('INSERT INTO queue_logs(job_id,stamp,level,message) VALUES(?,?,?,?)', (job_id, time.time(), level, message))
db.execute('DELETE FROM queue_logs WHERE job_id=? AND sequence NOT IN (SELECT sequence FROM queue_logs WHERE job_id=? ORDER BY sequence DESC LIMIT 500)', (job_id, job_id))
def _owned(self, db, job_id, token):
row = db.execute('SELECT * FROM queue_jobs WHERE id=?', (job_id,)).fetchone()
if not row:
raise ValueError('Job not found.')
if row['status'] != 'running' or not isinstance(token, str) or not row['token'] or not secrets.compare_digest(row['token'], token) or row['lease_until'] <= time.time():
raise ValueError('Job claim expired or is no longer active.')
return row
def enqueue(self, source, title, payload=None, *, total=100):
"""Submit JSON data under the producing plugin's slug; return its job ID."""
if not isinstance(source, str) or not IDENTIFIER.fullmatch(source):
raise ValueError('Source must be a plugin slug.')
title = bounded_text(title, 'Title', 160)
self._counts(0, total)
raw = json.dumps(payload, allow_nan=False).encode()
if len(raw) > 65536:
raise ValueError('Payload exceeds 64 KiB.')
job_id, now = secrets.token_hex(16), time.time()
with self._db() as db:
db.execute('INSERT INTO queue_jobs(id,source,title,payload,total,created,updated) VALUES(?,?,?,?,?,?,?)',
(job_id, source, title, self.state.cipher.encrypt(raw), total, now, now))
self._event(db, job_id, 'Job queued.')
return job_id
def claim(self, source, *, lease_seconds=300):
"""Atomically claim the oldest queued/expired job for a source, or None.
Returned token must accompany worker updates. Expired jobs are delivered
again; handlers must be idempotent because execution is at least once.
"""
self._lease(lease_seconds)
now = time.time()
with self._db() as db:
row = db.execute("SELECT * FROM queue_jobs WHERE source=? AND (status='queued' OR (status='running' AND lease_until<=?)) ORDER BY sequence LIMIT 1", (source, now)).fetchone()
if row is None:
return None
token = secrets.token_hex(32)
db.execute("UPDATE queue_jobs SET status='running',token=?,lease_until=?,updated=?,attempts=attempts+1 WHERE id=?", (token, now + lease_seconds, now, row['id']))
self._event(db, row['id'], 'Expired claim recovered.' if row['status'] == 'running' else 'Job started.')
fresh = db.execute('SELECT * FROM queue_jobs WHERE id=?', (row['id'],)).fetchone()
return {**self._public(fresh, payload=True), 'token': token}
@staticmethod
def _lease(seconds):
if type(seconds) is not int or not 10 <= seconds <= 86400:
raise ValueError('Lease must be 10–86400 whole seconds.')
@staticmethod
def _counts(completed, total):
if type(completed) is not int or type(total) is not int or not 1 <= total <= 2147483647 or not 0 <= completed <= total:
raise ValueError('Progress must be whole numbers with 0 <= completed <= total and total > 0.')
def progress(self, job_id, token, completed, *, total=None, message=None, lease_seconds=300):
"""Report work and renew the lease; optional message appears in job logs."""
self._lease(lease_seconds)
with self._db() as db:
row = self._owned(db, job_id, token)
total = row['total'] if total is None else total
self._counts(completed, total)
now = time.time()
db.execute('UPDATE queue_jobs SET completed=?,total=?,updated=?,lease_until=? WHERE id=?', (completed, total, now, now + lease_seconds, job_id))
if message is not None:
self._event(db, job_id, message)
def log(self, job_id, token, message, *, level='info'):
"""Append a bounded log entry; use progress() to renew the claim."""
with self._db() as db:
self._owned(db, job_id, token)
self._event(db, job_id, message, level)
def _finish(self, job_id, token, status, message):
with self._db() as db:
self._owned(db, job_id, token)
db.execute('UPDATE queue_jobs SET status=?,completed=CASE WHEN ?=\'completed\' THEN total ELSE completed END,updated=?,token=NULL,lease_until=NULL WHERE id=?', (status, status, time.time(), job_id))
self._event(db, job_id, message, 'error' if status == 'failed' else 'info')
def complete(self, job_id, token, message='Job completed.'):
"""Mark success and set progress to 100%."""
self._finish(job_id, token, 'completed', message)
def fail(self, job_id, token, message='Job failed.'):
"""Mark failure, preserving the last reported progress."""
self._finish(job_id, token, 'failed', message)
def cancel(self, job_id):
"""Cancel queued/running work and invalidate its claim cooperatively."""
with self._db() as db:
changed = db.execute("UPDATE queue_jobs SET status='cancelled',token=NULL,lease_until=NULL,updated=? WHERE id=? AND status IN ('queued','running')", (time.time(), job_id)).rowcount
if not changed:
raise ValueError('Only queued or running jobs can be cancelled.')
self._event(db, job_id, 'Job cancelled; worker claim revoked.', 'warning')
def retry(self, job_id):
"""Requeue failed/cancelled jobs with fresh progress; preserve history."""
with self._db() as db:
changed = db.execute("UPDATE queue_jobs SET status='queued',completed=0,updated=? WHERE id=? AND status IN ('failed','cancelled')", (time.time(), job_id)).rowcount
if not changed:
raise ValueError('Only failed or cancelled jobs can be retried.')
self._event(db, job_id, 'Job queued for retry.')
def jobs(self, *, status=None, source=None, limit=100, offset=0):
"""Return paginated metadata, newest first; never expose payloads/tokens."""
if status is not None and status not in STATES:
raise ValueError('Unknown queue status.')
if type(limit) is not int or not 1 <= limit <= 200 or type(offset) is not int or offset < 0:
raise ValueError('Invalid pagination.')
clauses, parameters = [], []
for key, value in (('status', status), ('source', source)):
if value is not None:
clauses.append(key + '=?')
parameters.append(value)
where = ' WHERE ' + ' AND '.join(clauses) if clauses else ''
with self._db() as db:
return [self._public(row) for row in db.execute('SELECT * FROM queue_jobs' + where + ' ORDER BY sequence DESC LIMIT ? OFFSET ?', (*parameters, limit, offset))]
def get(self, job_id):
"""Read metadata; raises ValueError for a missing job."""
with self._db() as db:
row = db.execute('SELECT * FROM queue_jobs WHERE id=?', (job_id,)).fetchone()
if row is None:
raise ValueError('Job not found.')
return self._public(row)
def logs(self, job_id):
"""Return the latest 500 entries in chronological order."""
with self._db() as db:
return [dict(row) for row in db.execute('SELECT stamp,level,message FROM queue_logs WHERE job_id=? ORDER BY sequence', (job_id,))]