Files
web-framework-resources/plugins/queues/tests/test_queue.py
T

147 lines
6.3 KiB
Python

"""Connector, durable worker claims, and browser access integration."""
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import pytest
from web_framework import create_app
@pytest.fixture
def app(tmp_path, monkeypatch):
monkeypatch.setenv('WEB_FRAMEWORK_DATA_DIR', str(tmp_path / 'state'))
app = create_app(overrides={
'runtime': {'plugin_dir': str(Path(__file__).resolve().parents[2]), 'theme_dir': str(tmp_path / 'themes')},
'plugins': {'enabled': ['queues']},
'resources': {'check_updates_on_startup': False},
})
app.testing = True
return app
@pytest.fixture
def queue(app):
return app.extensions['plugins'].service('queues', 'queue')
def authenticate(app):
client = app.test_client()
password = (app.extensions['state'].directory / 'initial-admin-password').read_text().strip()
client.get('/login')
with client.session_transaction() as session:
token = session['csrf']
client.post('/login', data={'username': 'admin', 'password': password, 'csrf_token': token})
client.post('/account', data={'current_password': password, 'new_password': 'A-secure-password!42', 'confirm_password': 'A-secure-password!42', 'csrf_token': csrf(client)})
return client
def csrf(client):
with client.session_transaction() as session:
return session['csrf']
def test_lifecycle_payload_progress_logs_and_restart(app, queue):
job_id = queue.enqueue('reports', '<script>Report</script>', {'rows': [1, 2]}, total=2)
claimed = queue.claim('reports')
assert claimed['id'] == job_id and claimed['payload'] == {'rows': [1, 2]}
assert claimed['attempts'] == 1
assert queue.claim('reports') is None
queue.progress(job_id, claimed['token'], 1, message='<script>half done</script>')
assert queue.get(job_id)['progress'] == 50
queue.complete(job_id, claimed['token'])
assert queue.get(job_id)['status'] == 'completed'
assert queue.get(job_id)['progress'] == 100
assert len(queue.logs(job_id)) == 4
restarted = create_app(overrides=app.extensions['workspace']['config'])
restored = restarted.extensions['plugins'].service('queues', 'queue')
assert restored.get(job_id)['status'] == 'completed'
assert restored.logs(job_id) == queue.logs(job_id)
with pytest.raises(ValueError):
queue.progress(job_id, claimed['token'], 0)
with pytest.raises(ValueError):
queue.retry(job_id)
client = authenticate(app)
response = client.get('/plugins/queues/jobs/' + job_id)
assert b'&lt;script&gt;' in response.data and b'<script>half' not in response.data
metadata = client.get('/plugins/queues/api/jobs').json['jobs'][0]
assert 'token' not in metadata and 'payload' not in metadata
assert b'value="2" max="2"' in response.data
def test_fifo_concurrent_claims_and_sources(queue):
first = queue.enqueue('one', 'First')
second = queue.enqueue('one', 'Second')
other = queue.enqueue('two', 'Other')
with ThreadPoolExecutor(max_workers=4) as pool:
jobs = list(pool.map(lambda _: queue.claim('one'), range(4)))
assert {job['id'] for job in jobs if job} == {first, second}
assert len([job for job in jobs if job]) == 2
assert queue.claim('two')['id'] == other
assert queue.jobs(source='one', limit=1, offset=1)[0]['id'] == first
def test_lease_expiry_recovery_and_stale_worker(queue):
job_id = queue.enqueue('worker', 'Recover me')
original = queue.claim('worker')
with queue.state.db() as db:
db.execute('UPDATE queue_jobs SET lease_until=0 WHERE id=?', (job_id,))
with pytest.raises(ValueError, match='expired'):
queue.complete(job_id, original['token'])
recovered = queue.claim('worker')
assert recovered['attempts'] == 2 and recovered['token'] != original['token']
with pytest.raises(ValueError):
queue.fail(job_id, original['token'])
queue.fail(job_id, recovered['token'], 'Worker failed.')
queue.retry(job_id)
fresh = queue.claim('worker')
queue.cancel(job_id)
with pytest.raises(ValueError):
queue.complete(job_id, fresh['token'])
assert queue.get(job_id)['status'] == 'cancelled'
def test_permissions_csrf_filtering_and_disabled_handle(app, queue):
job_id = queue.enqueue('worker', 'Pending')
anonymous = app.test_client()
assert anonymous.get('/plugins/queues/').status_code == 302
client = authenticate(app)
assert client.get('/plugins/queues/').status_code == 200
assert client.get('/plugins/queues/?status=invalid').status_code == 400
assert client.get('/plugins/queues/?page=-1').status_code == 400
assert client.get('/plugins/queues/jobs/missing').status_code == 404
endpoint = f'/plugins/queues/jobs/{job_id}/cancel'
assert client.post(endpoint).status_code == 400
with queue.state.db() as db:
db.execute("UPDATE users SET role='viewer'")
assert client.get('/plugins/queues/').status_code == 200
assert client.post(endpoint, data={'csrf_token': csrf(client)}).status_code == 403
queue.state.set('plugins.enabled', [])
assert client.get('/plugins/queues/').status_code == 404
with pytest.raises(ValueError, match='disabled'):
queue.enqueue('worker', 'Must not run')
with pytest.raises(ValueError, match='disabled'):
app.extensions['plugins'].service('queues', 'queue')
@pytest.mark.parametrize('completed,total', [(-1, 10), (11, 10), (0, 0), (True, 10), (1, 1.2)])
def test_invalid_progress_is_atomic(queue, completed, total):
job_id = queue.enqueue('worker', 'Validate')
job = queue.claim('worker')
with pytest.raises(ValueError):
queue.progress(job_id, job['token'], completed, total=total)
assert queue.get(job_id)['completed'] == 0
def test_logs_bounded_payload_encrypted_and_failed_update_rollback(queue):
job_id = queue.enqueue('worker', 'Secret payload', {'secret': 'payload-private-marker'})
assert b'payload-private-marker' not in queue.state.path.read_bytes()
job = queue.claim('worker')
with pytest.raises(ValueError):
queue.progress(job_id, job['token'], 50, message='')
assert queue.get(job_id)['completed'] == 0
with queue._db() as db:
for i in range(505):
queue._event(db, job_id, str(i))
logs = queue.logs(job_id)
assert len(logs) == 500 and logs[0]['message'] == '5'
with pytest.raises(ValueError):
queue.enqueue('worker', 'Too large', 'x' * 65537)