Files
kaizoku/test_ai_subtitles.py

777 lines
43 KiB
Python

"""Subtitle safety and recovery tests use disposable libraries and a fake model."""
import json
import tempfile
import threading
import unittest
from pathlib import Path
from unittest.mock import patch, Mock, MagicMock
from ai_subtitles import SubtitleDocument, SubtitleService, language_for, output_path, validate_text, validate_translation_quality, parse_batch, protect_markup, restore_markup, InlineFormattingError, MARKUP, ollama_translate, openai_translate, available_models
SRT = b'1\r\n00:00:01,000 --> 00:00:02,500\r\nHello there.\r\n\r\n2\r\n00:00:03,000 --> 00:00:04,000\r\nGoodbye!\r\n'
VTT = b'WEBVTT\nKind: captions\n\nNOTE kept\nmetadata --> untouched\n\nSTYLE\n::cue {color: lime}\n\ncue-one\n00:01.000 --> 00:02.000 align:start\n<b>Hello!</b>\n\n'
class DocumentTests(unittest.TestCase):
def test_provider_models_and_openai_translation_keep_source_structure(self):
def response(body):
handle = MagicMock()
handle.__enter__.return_value.read.return_value = json.dumps(body).encode()
return handle
with patch('ai_subtitles.urllib.request.urlopen', return_value=response({'models':[{'name':'translategemma:4b'}]})) as fetch:
self.assertEqual(available_models({'ai_subtitle_ollama_url':'http://localhost:11434'}, 'ollama'), ['translategemma:4b'])
self.assertEqual(fetch.call_args.args[0].full_url, 'http://localhost:11434/api/tags')
with patch('ai_subtitles.urllib.request.urlopen', return_value=response({'data':[{'id':'gpt-4.1-mini'}]})) as fetch:
self.assertEqual(available_models({'ai_subtitle_openai_url':'https://api.openai.com/v1','ai_subtitle_openai_api_key':'secret'}, 'openai'), ['gpt-4.1-mini'])
self.assertEqual(fetch.call_args.args[0].get_header('Authorization'), 'Bearer secret')
with patch('ai_subtitles.urllib.request.urlopen', return_value=response({'output':[{'content':[{'type':'output_text','text':'Ahoj!'}]}]})) as fetch:
self.assertEqual(openai_translate(['<b>Hello!</b>'], [], '', url='https://api.openai.com/v1', model='gpt-4.1-mini', api_key='secret'), ['<b>Ahoj!</b>'])
payload = json.loads(fetch.call_args.args[0].data)
self.assertNotIn('00:00:', payload['input'])
self.assertNotIn('secret', payload['input'])
def test_srt_preserves_ids_timing_crlf_and_bom(self):
doc = SubtitleDocument(b'\xef\xbb\xbf' + SRT)
result = doc.render(['Ahoj.', 'Sbohem!'])
self.assertEqual(result, b'\xef\xbb\xbf' + SRT.replace(b'Hello there.', b'Ahoj.').replace(b'Goodbye!', b'Sbohem!'))
def test_vtt_preserves_header_notes_styles_identifiers_and_settings(self):
doc = SubtitleDocument(VTT)
self.assertEqual(doc.render(['<b>Ahoj!</b>']), VTT.replace(b'Hello!', b'Ahoj!'))
def test_bad_translations_cannot_create_or_delete_cues(self):
for value in ['', 'x\n\ny', '00:01 --> 00:02', None, 'x\r\ny', '<i>Ahoj</i>', '<b></b>']:
with self.subTest(value=value), self.assertRaises(ValueError):
validate_text('<b>Hello</b>', value)
with self.assertRaises(ValueError):
SubtitleDocument(SRT).render(['Ahoj'])
def test_malformed_subtitles_are_rejected(self):
for value in [b'', b'1\n00:01 --> 00:02\nHi\n', b'1\n00:00:01,000 --> 00:00:02,000\n', b'\xff']:
with self.subTest(value=value), self.assertRaises(ValueError):
SubtitleDocument(value)
def test_languages_and_output_names(self):
for code in ['cs', 'ces', 'cze', 'cz']:
self.assertEqual(language_for(Path(f'Show.{code}.srt'), '')[0], 'cs')
self.assertEqual(language_for(Path('Show.eng.forced.vtt'), '')[0], 'en')
self.assertEqual(language_for(Path('Show.srt'), 'Hello!')[0], 'und')
self.assertEqual(language_for(Path('Show.eng.srt'), 'Proč jsem tady? Tohle není dobře.')[0], 'cs')
self.assertEqual(output_path(Path('Show.eng.forced.vtt')), Path('Show.forced.cs.vtt'))
self.assertEqual(output_path(Path('Show.en-US.srt')), Path('Show.cs.srt'))
def test_batch_markers_reject_missing_duplicate_reordered_and_commentary(self):
self.assertEqual(parse_batch('[c0] Ahoj.\n[c1] Sbohem!', 2), ['Ahoj.', 'Sbohem!'])
for response in ['[c0] Ahoj.', '[c0] A\n[c0] B', '[c1] A\n[c0] B',
'Here you go:\n[c0] A\n[c1] B', '[c0] A [c1] B', '[c0] A\n[c1] B\n[c2] C']:
with self.subTest(response=response), self.assertRaises(ValueError):
parse_batch(response, 2)
def test_gemma4_chat_separates_instructions_and_rejects_instruction_echo(self):
import io
sources = ['ONE', 'ONE TWO', 'ONE TWO THREE', 'ONE TWO THREE FOUR']
bad = 'Profesní český (en) koupíř pro česky (cs).'
def model(request, timeout):
self.assertEqual(request.full_url, 'http://localhost:11434/api/chat')
payload = json.loads(request.data)
self.assertEqual(payload['model'], 'gemma4:e4b')
self.assertFalse(payload['think'])
self.assertEqual([item['role'] for item in payload['messages']], ['system', 'user'])
self.assertNotIn('ONE', payload['messages'][0]['content'])
self.assertIn('[c0] ONE', payload['messages'][1]['content'])
return io.BytesIO(json.dumps({'message': {'content': '\n'.join(f'[c{i}] {bad}' for i in range(4))}}).encode())
with patch('ai_subtitles.urllib.request.urlopen', side_effect=model):
with self.assertRaisesRegex(ValueError, 'echoed translation instructions'):
ollama_translate(sources, [], '', url='http://localhost:11434', model='gemma4:e4b')
with self.assertRaisesRegex(ValueError, 'repeated the same response'):
validate_translation_quality(sources, ['Dlouhá nesmyslná stejná odpověď.'] * 4)
def test_gemma4_chat_accepts_distinct_dialogue(self):
import io
with patch('ai_subtitles.urllib.request.urlopen', return_value=io.BytesIO(json.dumps({
'message': {'content': '[c0] JEDEN\n[c1] JEDEN DVA'}}).encode())):
self.assertEqual(ollama_translate(['ONE', 'ONE TWO'], [], '', url='http://localhost:11434',
model='gemma4:e4b'), ['JEDEN', 'JEDEN DVA'])
def test_subtitle_finetune_uses_current_source_and_prior_context(self):
import io
requests=[]
replies=iter(['Ahoj.', 'Jak se máš?'])
def model(request, timeout):
self.assertEqual(request.full_url, 'http://localhost:11434/api/chat')
payload=json.loads(request.data)
requests.append(payload)
return io.BytesIO(json.dumps({'message': {'content': next(replies)}, 'done_reason':'stop'}).encode())
name='huggingface.co/17slever17/translate-gemma-4-sub-e4b-GGUF:Q4_K_XL'
with patch('ai_subtitles.urllib.request.urlopen', side_effect=model):
result=ollama_translate(['Hello.', 'How are you?'], [], 'Ichigo stays Ichigo.',
url='http://localhost:11434', model=name)
self.assertEqual(result, ['Ahoj.', 'Jak se máš?'])
self.assertEqual(len(requests),2)
self.assertIn('TASK: Translate English subtitles into Czech.',requests[0]['messages'][0]['content'])
self.assertIn('Ichigo stays Ichigo.',requests[0]['messages'][0]['content'])
self.assertEqual(requests[0]['messages'][1]['content'],'[CURRENT_SOURCE]\nHello.')
self.assertIn('[PREVIOUS_SOURCE]\nHello.',requests[1]['messages'][1]['content'])
self.assertIn('[PREVIOUS_TRANSLATION]\nAhoj.',requests[1]['messages'][1]['content'])
self.assertTrue(all(not request['messages'][1]['content'].startswith('[c') for request in requests))
def test_subtitle_finetune_rejects_instruction_echo(self):
import io
name='huggingface.co/17slever17/translate-gemma-4-sub-e4b-GGUF:Q4_K_XL'
with patch('ai_subtitles.urllib.request.urlopen', return_value=io.BytesIO(json.dumps({
'message': {'content': 'Professional English (en) to Czech (cs) translator.'}}).encode())):
with self.assertRaisesRegex(ValueError,'echoed translation instructions'):
ollama_translate(['Hello.'], [], '', url='http://localhost:11434', model=name)
def test_markup_is_restored_without_model_editing(self):
text='<b>Hello <i>there</i> friend.</b>'
plain, formatting=protect_markup(text)
self.assertEqual(plain, 'Hello __TAG0__there__TAG1__ friend.')
self.assertEqual(restore_markup('Ahoj __TAG0__tam__TAG1__ příteli.', formatting),
'<b>Ahoj <i>tam</i> příteli.</b>')
with self.assertRaises(ValueError): restore_markup('Ahoj.', formatting)
def test_user_samples_round_trip_without_structural_changes(self):
paths = list((Path(__file__).parent / '_tmp_').rglob('*'))
subtitles = [p for p in paths if p.suffix.lower() in {'.srt', '.vtt'}]
if not subtitles:
self.skipTest('Optional development samples are not present.')
for path in subtitles:
with self.subTest(path=path):
raw = path.read_bytes()
doc = SubtitleDocument(raw)
self.assertEqual(doc.render([c[2].replace('\r\n', '\n') for c in doc.cues]), raw)
class ServiceTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name)
self.tv, self.movies = self.root / 'tv', self.root / 'movies'
self.tv.mkdir(); self.movies.mkdir()
self.config = {'jellyfin_tv_dir': str(self.tv), 'jellyfin_movie_dir': str(self.movies)}
self.calls = []
def translate(texts, context, notes, shorten=False):
self.calls.append((texts, context, notes, shorten))
return ['Ahoj.' for _ in texts]
self.service = SubtitleService(self.root / 'state' / 'ai.sqlite3', lambda: self.config, translator=translate)
self.addCleanup(lambda: self.service.shutdown(wait=True))
def file(self, name='Show/Season 01/Show - S01E01.eng.srt', content=SRT, root=None):
path = (root or self.tv) / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(content)
return path
def scan_queue(self, selection=None):
self.service.scan()
result = self.service.queue(selection or {'kind': 'tv', 'title': 'Show'})
return result
def job(self):
with self.service.db() as db:
return dict(db.execute('SELECT * FROM jobs ORDER BY id LIMIT 1').fetchone())
def process(self):
self.service.process(self.job()['id'])
if self.service.scanner:
self.service.scanner.join(5)
def test_catalog_groups_movies_seasons_and_skips_czech(self):
self.file()
self.file('Show/Season 02/Show - S02E01.cze.vtt', VTT)
self.file('Movie/Movie.en.srt', root=self.movies)
self.service.scan()
rows = self.service.snapshot()['subtitles']
self.assertEqual({(r['kind'], r['title'], r['season'], r['language']) for r in rows},
{('tv','Show','Season 01','en'),('tv','Show','Season 02','cs'),('movie','Movie','','en')})
self.assertEqual(self.service.queue({'kind':'tv','title':'Show'})['queued'],1)
self.assertEqual(self.service.queue({'kind':'tv','title':'Show','season':'Season 02'})['queued'],0)
self.assertEqual(self.service.queue({'kind':'movie','title':'Movie'})['queued'],1)
def test_completes_sidecar_without_modifying_source_or_other_state(self):
path = self.file()
unrelated = self.root / 'state' / 'watchlist.json'
unrelated.write_text('preserve me')
self.scan_queue(); self.process()
self.assertEqual(self.job()['status'], 'completed')
self.assertEqual(path.read_bytes(), SRT)
self.assertEqual(output_path(path).read_bytes(), SRT.replace(b'Hello there.',b'Ahoj.').replace(b'Goodbye!',b'Ahoj.'))
self.assertEqual(unrelated.read_text(),'preserve me')
self.assertTrue(any(r['language']=='cs' for r in self.service.snapshot()['subtitles']))
def test_existing_czech_in_other_format_is_never_overwritten(self):
path = self.file()
other = self.file('Show/Season 01/Show - S01E01.cs.vtt', VTT)
self.assertEqual(self.scan_queue()['queued'],0)
self.assertEqual(other.read_bytes(),VTT)
self.assertFalse(output_path(path).exists())
def test_czech_ass_sidecar_is_cataloged_and_prevents_translation(self):
path = self.file()
self.file('Show/Season 01/Show - S01E01.cz.ass', b'[Script Info]\nTitle: Czech\n')
self.service.scan()
rows = self.service.snapshot()['subtitles']
self.assertEqual(len(rows), 2)
self.assertTrue(any(row['language'] == 'cs' and row['filename'].endswith('.ass') for row in rows))
self.assertEqual(self.service.queue({'id': next(row['id'] for row in rows if row['path'] == str(path))})['queued'], 0)
def test_queued_job_retains_openai_provider_and_model(self):
self.config.update({'ai_subtitle_backend':'openai', 'ai_subtitle_openai_url':'https://api.openai.com/v1',
'ai_subtitle_openai_model':'gpt-4.1-mini', 'ai_subtitle_openai_api_key':'secret'})
self.file()
self.scan_queue()
job = self.job()
self.assertEqual((job['backend'], job['endpoint'], job['model']),
('openai', 'https://api.openai.com/v1', 'gpt-4.1-mini'))
with self.service.db() as db:
self.assertNotIn('secret', json.dumps(dict(db.execute('SELECT * FROM jobs WHERE id=?', (job['id'],)).fetchone())))
def test_retries_malformed_output_and_retains_progress(self):
path = self.file()
self.scan_queue()
self.service.translator = lambda *args, **kwargs: ['']
with patch.object(self.service.stop,'wait',return_value=False):
self.process()
self.assertEqual(self.job()['status'],'failed')
self.assertIn('3 attempts',self.job()['error'])
self.assertFalse(output_path(path).exists())
self.assertEqual(self.job()['progress'],0)
def test_existing_instruction_echo_checkpoint_requires_review_correction(self):
path = self.file()
self.scan_queue()
bad = 'Profesní český (en) koupíř pro česky (cs).'
with self.service.db() as db:
db.execute('INSERT INTO translations VALUES(1,0,?,?)', ('Hello there.', bad))
db.execute('UPDATE jobs SET progress=1 WHERE id=1')
self.process()
self.assertEqual(self.job()['status'], 'failed')
self.assertIn('Saved cue 1 needs correction in Review', self.job()['error'])
self.assertFalse(output_path(path).exists())
self.assertEqual(self.service.review(1)['cues'][0]['czech'], bad)
self.service.edit_cue(1, 1, 'Ahoj.', bad)
self.service.queue({'id': 1})
self.process()
self.assertEqual(self.job()['status'], 'completed')
self.assertEqual(SubtitleDocument(output_path(path).read_bytes()).cues[0][2], 'Ahoj.')
def test_resume_reuses_batches_and_series_notes(self):
source = ''.join(f'{i}\n00:00:01,000 --> 00:00:02,000\nHello number {i}.\n\n' for i in range(1,18)).encode()
path = self.file(content=source)
self.service.save_notes('tv','Show','Asta stays Asta.')
self.scan_queue()
calls = []
def fail_second(texts, context, notes, shorten=False):
calls.append(texts)
if len(calls)>1:
raise ValueError('offline')
return ['Ahoj.' for t in texts]
self.service.translator=fail_second
with patch.object(self.service.stop,'wait',return_value=False):
self.process()
self.assertEqual(self.job()['progress'],8)
self.assertFalse(output_path(path).exists())
resumed = SubtitleService(self.service.db_path,lambda:self.config,translator=lambda texts,context,notes,shorten=False: ['Nazdar.' for t in texts])
self.addCleanup(lambda:resumed.shutdown(wait=True))
resumed.queue({'kind':'tv','title':'Show'})
resumed.process(self.job()['id'])
if resumed.scanner: resumed.scanner.join(5)
self.assertEqual(self.job()['status'],'completed')
out = SubtitleDocument(output_path(path).read_bytes())
self.assertEqual([c[2] for c in out.cues],['Ahoj.']*8+['Nazdar.']*9)
self.assertEqual(self.job()['glossary'],'Asta stays Asta.')
def test_subtitle_finetune_saves_each_cue_for_resume(self):
path=self.file()
self.config['ai_subtitle_ollama_model']='huggingface.co/17slever17/translate-gemma-4-sub-e4b-GGUF:Q4_K_XL'
self.scan_queue()
calls=[]
def translate(texts, context, notes, shorten=False):
calls.append((texts,context))
if len(calls)>1:
raise ValueError('offline')
return ['Ahoj.']
self.service.translator=translate
with patch.object(self.service.stop,'wait',return_value=False):
self.process()
self.assertEqual(self.job()['status'],'failed')
self.assertEqual(self.job()['progress'],1)
self.assertEqual(calls[0][0],['Hello there.'])
self.assertEqual(calls[1][0],['Goodbye!'])
self.assertFalse(output_path(path).exists())
def test_bad_batches_fall_back_to_individual_cues_with_context(self):
self.file();self.scan_queue()
calls=[]
def single_only(texts,context,notes,shorten=False):
calls.append((texts,context))
if len(texts)>1: raise ValueError('Model merged cues')
return ['Ahoj.']
self.service.translator=single_only
with patch.object(self.service.stop,'wait',return_value=False):self.process()
self.assertEqual(self.job()['status'],'completed')
self.assertEqual(calls[-1][1][-1], {'source':'Hello there.','target':'Ahoj.'})
def test_startup_recovers_running_but_leaves_paused_jobs(self):
self.file();self.scan_queue()
with self.service.db() as db: db.execute("UPDATE jobs SET status='running'")
recovered=SubtitleService(self.service.db_path,lambda:self.config)
self.assertEqual(self.job()['status'],'pending')
recovered.pause(self.job()['id'])
SubtitleService(self.service.db_path,lambda:self.config)
self.assertEqual(self.job()['status'],'paused')
def test_source_change_blocks_output_and_retains_original_progress(self):
path=self.file();self.scan_queue()
path.write_bytes(SRT.replace(b'Goodbye!',b'Changed!'))
self.process()
self.assertEqual(self.job()['status'],'failed')
self.assertIn('Source changed',self.job()['error'])
self.assertFalse(output_path(path).exists())
def test_symlinks_and_changed_library_roots_cannot_escape(self):
outside=self.root/'outside.srt';outside.write_bytes(SRT)
link=self.tv/'outside.eng.srt';link.symlink_to(outside)
self.file();self.scan_queue()
self.assertEqual(len(self.service.snapshot()['subtitles']),1)
self.config['jellyfin_tv_dir']=str(self.movies)
self.process()
self.assertEqual(self.job()['status'],'failed')
self.assertIn('outside',self.job()['error'])
def test_pause_during_request_saves_no_output(self):
path=self.file();self.scan_queue()
def pause(texts,*args,**kwargs):
self.service.pause(self.job()['id'])
return ['Ahoj.' for _ in texts]
self.service.translator=pause
self.process()
self.assertEqual(self.job()['status'],'paused')
self.assertFalse(output_path(path).exists())
def test_language_confirmation_and_duplicate_queue(self):
self.file('Show/Season 01/Show - S01E01.srt')
self.service.scan(); row=self.service.snapshot()['subtitles'][0]
self.assertEqual(row['language'],'und')
self.assertEqual(self.service.queue({'id':row['id']})['queued'],0)
self.service.set_language(row['id'],'en')
self.assertEqual(self.service.queue({'id':row['id']})['queued'],1)
self.assertEqual(self.service.queue({'id':row['id']})['queued'],0)
with self.assertRaises(ValueError): self.service.set_language(row['id'],'cs')
self.service.scan()
self.assertEqual(self.service.snapshot()['subtitles'][0]['language_source'],'manual')
def test_long_translations_are_shortened_or_flagged(self):
self.file();self.scan_queue()
self.service.translator=lambda texts,*args,**kwargs:['Velmi dlouhá česká věta ' * 5 for _ in texts]
self.process()
self.assertEqual(self.job()['status'],'completed')
self.assertEqual(json.loads(self.job()['warnings']),[1,2])
def test_unavailable_mount_keeps_catalog_and_changed_roots_mark_it_missing(self):
self.file();self.service.scan()
self.tv.rename(self.root/'offline')
self.service.scan()
self.assertTrue(self.service.snapshot()['subtitles'][0]['present'])
self.assertTrue(self.service.snapshot()['scan_errors'])
self.config['jellyfin_tv_dir']=str(self.movies)
self.service.scan()
self.assertFalse(self.service.snapshot()['subtitles'][0]['present'])
def test_season_and_series_queue_requests_are_grouped_with_logs(self):
self.file()
self.file('Show/Season 01/Show - S01E02.eng.srt')
self.file('Show/Season 02/Show - S02E01.eng.srt')
self.service.scan()
self.service.queue({'kind':'tv','title':'Show','season':'Season 01'})
groups=self.service.list_queue_jobs()
self.assertEqual(len(groups),1)
self.assertEqual(groups[0]['total'],2)
self.assertIn('Season 01',groups[0]['title'])
self.assertEqual(len(groups[0]['log']),2)
self.process()
group=self.service.get_queue_job(groups[0]['id'])
self.assertEqual(group['completed'],1)
self.assertEqual(group['progress']['percent'],50)
self.assertTrue(any('Saved 2 / 2' in line for line in group['log']))
self.service.queue_action(group['id'],'pause')
self.assertEqual(self.service.get_queue_job(group['id'])['status'],'paused')
self.service.queue_action(group['id'],'retry')
self.assertEqual(self.service.get_queue_job(group['id'])['status'],'pending')
self.assertEqual(len(self.service.list_queue_jobs()),1)
def test_stop_then_remove_group_keeps_checkpoints_and_completed_sidecars(self):
first=self.file()
self.file('Show/Season 01/Show - S01E02.eng.srt')
self.service.scan()
self.service.queue({'kind':'tv','title':'Show','season':'Season 01'})
group=self.service.list_queue_jobs()[0]['id']
with self.assertRaisesRegex(ValueError,'Stop'):
self.service.queue_action(group,'remove')
self.service.process(self.job()['id'])
self.assertTrue(output_path(first).exists())
with self.service.db() as db:
other = db.execute("SELECT id FROM jobs WHERE id<>?", (self.job()['id'],)).fetchone()[0]
db.execute('INSERT INTO translations VALUES(?,?,?,?)', (other,0,'Hello there.','Bad partial cue.'))
db.execute('UPDATE jobs SET progress=1 WHERE id=?', (other,))
self.service.queue_action(group,'stop')
stopped=self.service.get_queue_job(group)
self.assertEqual(stopped['status'],'canceled')
self.assertEqual(stopped['completed'],1)
self.assertTrue(any(entry['status']=='canceled' for entry in stopped['episode_jobs']))
self.service.queue_action(group,'remove')
self.assertEqual(self.service.list_queue_jobs(),[])
self.assertTrue(output_path(first).exists())
with self.service.db() as db:
self.assertEqual(db.execute('SELECT COUNT(*) FROM translations').fetchone()[0],2)
self.assertEqual(db.execute('SELECT progress,status FROM jobs WHERE id=?',(other,)).fetchone()[:],(0,'cleared'))
def test_remove_failed_job_clears_progress_and_next_queue_uses_current_model(self):
path=self.file()
self.config['ai_subtitle_ollama_model']='old-model'
self.scan_queue()
first_group=self.service.list_queue_jobs()[0]['id']
with self.service.db() as db:
db.execute('INSERT INTO translations VALUES(1,0,?,?)', ('Hello there.','Wrong Czech.'))
db.execute("UPDATE jobs SET status='failed',progress=1,error='bad output' WHERE id=1")
self.config['ai_subtitle_ollama_model']='new-model'
self.service.queue_action(first_group,'remove')
self.assertEqual(self.service.list_queue_jobs(),[])
self.assertEqual((self.job()['status'],self.job()['progress']),('cleared',0))
with self.service.db() as db:
self.assertEqual(db.execute('SELECT COUNT(*) FROM translations WHERE job_id=1').fetchone()[0],0)
self.assertEqual(path.read_bytes(),SRT)
self.service.queue({'id':1})
self.assertEqual(self.job()['model'],'new-model')
self.assertEqual(self.job()['progress'],0)
self.assertNotEqual(self.service.list_queue_jobs()[0]['id'],first_group)
def test_older_hidden_removed_job_is_fresh_on_next_queue(self):
self.file();self.config['ai_subtitle_ollama_model']='old-model';self.scan_queue()
with self.service.db() as db:
db.execute('INSERT INTO translations VALUES(1,0,?,?)', ('Hello there.','Wrong Czech.'))
db.execute("UPDATE jobs SET status='failed',progress=1 WHERE id=1")
db.execute('UPDATE batches SET hidden=1')
self.config['ai_subtitle_ollama_model']='new-model'
self.service.queue({'id':1})
self.assertEqual((self.job()['model'],self.job()['progress']),('new-model',0))
with self.service.db() as db:
self.assertEqual(db.execute('SELECT COUNT(*) FROM translations WHERE job_id=1').fetchone()[0],0)
def test_clear_canceled_file_keeps_source_and_allows_new_model(self):
path=self.file()
self.config['ai_subtitle_ollama_model']='old-model';self.scan_queue()
with self.service.db() as db:
db.execute('INSERT INTO translations VALUES(1,0,?,?)', ('Hello there.','Wrong Czech.'))
db.execute("UPDATE jobs SET status='canceled',progress=1 WHERE id=1")
self.service.clear_file(1)
self.assertEqual(self.service.snapshot()['subtitles'][0]['status'],'cleared')
self.assertEqual(self.service.snapshot()['subtitles'][0]['review_id'],None)
self.assertEqual(self.service.list_queue_jobs(),[])
self.assertEqual(path.read_bytes(),SRT)
self.config['ai_subtitle_ollama_model']='new-model'
self.service.queue({'id':1})
self.assertEqual((self.job()['model'],self.job()['progress']),('new-model',0))
self.assertNotIn('Wrong Czech.',str(self.service.review(1)))
def test_direct_resume_keeps_original_model(self):
self.file();self.config['ai_subtitle_ollama_model']='old-model';self.scan_queue()
with self.service.db() as db:
db.execute("UPDATE jobs SET status='failed' WHERE id=1")
self.config['ai_subtitle_ollama_model']='new-model'
self.service.queue({'id':1})
self.assertEqual(self.job()['model'],'old-model')
def test_stop_running_job_discards_inflight_model_result_and_can_retry(self):
path=self.file()
self.scan_queue()
group=self.service.list_queue_jobs()[0]['id']
entered=threading.Event();release=threading.Event()
def delayed(texts,*args,**kwargs):
entered.set()
self.assertTrue(release.wait(5))
return ['Ahoj.' for _ in texts]
self.service.translator=delayed
worker=threading.Thread(target=self.service.process,args=(self.job()['id'],))
worker.start()
self.assertTrue(entered.wait(5))
self.service.queue_action(group,'stop')
self.assertEqual(self.service.get_queue_job(group)['status'],'canceled')
release.set();worker.join(5)
self.assertFalse(worker.is_alive())
self.assertFalse(output_path(path).exists())
self.assertEqual(self.job()['progress'],0)
self.service.queue_action(group,'retry')
self.assertEqual(self.service.get_queue_job(group)['status'],'pending')
self.service.queue_action(group,'stop')
self.service.queue_action(group,'remove')
def test_restart_file_discards_saved_cues_and_translates_from_start(self):
path=self.file()
self.config['ai_subtitle_ollama_model']='old-model'
self.scan_queue()
with self.service.db() as db:
db.execute('INSERT INTO translations VALUES(1,0,?,?)', ('Hello there.', 'Špatně.'))
db.execute("UPDATE jobs SET status='canceled',progress=1,warnings='[1]' WHERE id=1")
self.config['ai_subtitle_ollama_model']='new-model'
with patch.object(self.service, 'ensure_worker'):
result=self.service.restart_file(1)
self.assertIn('cue 1',result['message'])
self.assertEqual(self.job()['status'],'pending')
self.assertEqual(self.job()['progress'],0)
self.assertEqual(self.job()['warnings'],'[]')
self.assertEqual(self.job()['model'],'new-model')
with self.service.db() as db:
self.assertEqual(db.execute('SELECT COUNT(*) FROM translations WHERE job_id=1').fetchone()[0],0)
self.process()
self.assertEqual(self.job()['status'],'completed')
self.assertEqual(SubtitleDocument(output_path(path).read_bytes()).cues[0][2],'Ahoj.')
self.assertEqual(len(self.calls),1)
def test_restart_during_inflight_request_cannot_save_old_response(self):
path=self.file()
self.scan_queue()
entered=threading.Event();release=threading.Event()
def delayed(texts,*args,**kwargs):
entered.set()
self.assertTrue(release.wait(5))
return ['Starý překlad.' for _ in texts]
self.service.translator=delayed
worker=threading.Thread(target=self.service.process,args=(1,))
worker.start()
self.assertTrue(entered.wait(5))
self.service.queue_action(self.service.list_queue_jobs()[0]['id'],'stop')
with patch.object(self.service, 'ensure_worker'):
self.service.restart_file(1)
release.set();worker.join(5)
self.assertFalse(worker.is_alive())
self.assertEqual(self.job()['status'],'pending')
self.assertEqual(self.job()['progress'],0)
self.assertFalse(output_path(path).exists())
with self.service.db() as db:
self.assertEqual(db.execute('SELECT COUNT(*) FROM translations').fetchone()[0],0)
self.service.translator=lambda texts,*args,**kwargs: ['Nový překlad.' for _ in texts]
self.process()
self.assertEqual(SubtitleDocument(output_path(path).read_bytes()).cues[0][2],'Nový překlad.')
def test_single_file_jobs_and_existing_jobs_migrate_without_progress_loss(self):
self.file();self.scan_queue();self.process()
with self.service.db() as db:
saved=list(db.execute('SELECT * FROM translations'))
db.execute('DELETE FROM batch_jobs');db.execute('DELETE FROM batches')
db.execute('ALTER TABLE subtitles DROP COLUMN hidden')
reopened=SubtitleService(self.service.db_path,lambda:self.config)
jobs=reopened.list_queue_jobs()
self.assertEqual(len(jobs),1)
self.assertEqual(jobs[0]['status'],'done')
self.assertEqual(jobs[0]['total'],1)
with reopened.db() as db:self.assertEqual(len(list(db.execute('SELECT * FROM translations'))),len(saved))
reopened.queue_action(jobs[0]['id'],'remove')
self.assertEqual(reopened.list_queue_jobs(),[])
self.assertEqual(self.job()['status'],'completed')
def test_missing_cleanup_preserves_history_and_reappearance_restores_entry(self):
path=self.file();self.scan_queue();self.process()
with self.assertRaises(ValueError):self.service.remove_missing(1)
path.unlink();self.service.scan();self.service.remove_missing(1)
self.assertFalse(any(r['id']==1 for r in self.service.snapshot()['subtitles']))
self.assertEqual(self.job()['status'],'completed')
self.assertEqual(len(self.service.list_queue_jobs()),1)
self.service.scan()
self.assertFalse(any(r['id']==1 for r in self.service.snapshot()['subtitles']))
path.write_bytes(SRT);self.service.scan()
self.assertTrue(any(r['id']==1 for r in self.service.snapshot()['subtitles']))
def test_invalid_series_request_cannot_leave_ungrouped_pending_files(self):
old=self.file('Show/Season 01/Show - S01E02.eng.srt')
self.scan_queue();self.service.pause(self.job()['id'])
old.write_bytes(SRT.replace(b'Hello there.',b'Changed source.'))
self.file('Show/Season 01/Show - S01E01.eng.srt');self.service.scan()
with self.assertRaises(ValueError):self.service.queue({'kind':'tv','title':'Show'})
with self.service.db() as db:
self.assertEqual(db.execute('SELECT count(*) FROM jobs').fetchone()[0],1)
self.assertEqual(len(self.service.list_queue_jobs()),1)
self.assertEqual(self.job()['status'],'paused')
def test_missing_cleanup_blocks_active_jobs_and_existing_files(self):
path=self.file();self.scan_queue();path.unlink();self.service.scan()
with self.assertRaises(ValueError):self.service.remove_missing(1)
self.service.pause(self.job()['id'])
path.write_bytes(SRT)
with self.assertRaises(ValueError):self.service.remove_missing(1)
path.unlink();self.service.remove_missing(1)
self.assertEqual(self.service.snapshot()['subtitles'],[])
def test_review_reads_completed_files_and_rejects_changed_source_or_root(self):
path=self.file();self.scan_queue();self.process()
review=self.service.review(1)
self.assertEqual(review['cues'][0]['english'],'Hello there.')
self.assertEqual(review['cues'][0]['czech'],'Ahoj.')
self.assertEqual(review['cues'][0]['timing'],review['cues'][0]['translated_timing'])
self.assertFalse(review['saved_preview'])
self.assertEqual({r['review_id'] for r in self.service.snapshot()['subtitles']},{1})
path.write_bytes(SRT.replace(b'Hello there.',b'Changed.'))
with self.assertRaises(ValueError):self.service.review(1)
path.write_bytes(SRT)
self.config['jellyfin_tv_dir']=str(self.movies)
with self.assertRaises(ValueError):self.service.review(1)
def test_review_shows_saved_partial_progress_without_publishing(self):
self.file();self.scan_queue()
with self.service.db() as db:
db.execute('INSERT INTO translations VALUES(1,0,?,?)',('Hello there.','Ahoj.'))
review=self.service.review(1)
self.assertTrue(review['saved_preview'])
self.assertEqual([c['czech'] for c in review['cues']],['Ahoj.',''])
def test_manual_review_edit_updates_sidecar_and_checkpoint_without_changing_timing(self):
source=self.file();self.scan_queue();self.process()
target=output_path(source)
before=self.service.review(1)
result=self.service.edit_cue(1,1,'Nazdar, kamaráde.','Ahoj.',before['output_fingerprint'])
self.assertEqual(result['review']['cues'][0]['czech'],'Nazdar, kamaráde.')
self.assertEqual(source.read_bytes(),SRT)
self.assertEqual(target.read_bytes(),SRT.replace(b'Hello there.', 'Nazdar, kamaráde.'.encode()).replace(b'Goodbye!',b'Ahoj.'))
with self.service.db() as db:
saved=db.execute('SELECT target FROM translations WHERE job_id=1 AND cue=0').fetchone()[0]
self.assertEqual(saved,'Nazdar, kamaráde.')
with self.assertRaisesRegex(ValueError,'changed since you opened'):
self.service.edit_cue(1,1,'Jiný text.','Ahoj.',before['output_fingerprint'])
def test_manual_review_edit_rejects_invalid_text_and_external_changes(self):
source=self.file('Show/Season 01/Show - S01E01.eng.vtt',VTT)
self.service.translator=lambda texts,*args,**kwargs:['<b>Ahoj!</b>' for _ in texts]
self.scan_queue();self.process()
before=self.service.review(1)
for bad in ('', '<i>Ahoj!</i>', 'Ahoj!\n\nDalší', '00:01.000 --> 00:02.000'):
with self.subTest(bad=bad), self.assertRaises(ValueError):
self.service.edit_cue(1,1,bad,'<b>Ahoj!</b>',before['output_fingerprint'])
target=output_path(source)
target.write_bytes(target.read_bytes().replace(b'Ahoj!',b'Nazdar!'))
with self.assertRaisesRegex(ValueError,'changed since you opened'):
self.service.edit_cue(1,1,'<b>Zdravím!</b>','<b>Ahoj!</b>',before['output_fingerprint'])
def test_manual_review_edit_works_on_paused_checkpoint_but_not_active_job(self):
self.file();self.scan_queue()
with self.service.db() as db:
db.execute('INSERT INTO translations VALUES(1,0,?,?)',('Hello there.','Ahoj.'))
active=self.service.review(1)
self.assertFalse(active['cues'][0]['editable'])
with self.assertRaisesRegex(ValueError,'saved cues'):
self.service.edit_cue(1,1,'Nazdar.','Ahoj.')
self.service.pause(1)
paused=self.service.review(1)
self.assertTrue(paused['cues'][0]['editable'])
self.assertFalse(paused['cues'][1]['editable'])
self.service.edit_cue(1,1,'Nazdar.','Ahoj.')
self.assertEqual(self.service.review(1)['cues'][0]['czech'],'Nazdar.')
self.assertFalse(output_path(self.tv/'Show/Season 01/Show - S01E01.eng.srt').exists())
self.service.queue({'id':1});self.process()
self.assertEqual(self.service.review(1)['cues'][0]['czech'],'Nazdar.')
def test_inline_marker_failure_uses_plain_spans_and_preserves_all_formatting(self):
source = '<b>He is <i>our captain</i>.</b>\n<v Bob>Goodbye!</v>{\\an8}'
raw = ('WEBVTT\n\nvoice-cue\n00:01.000 --> 00:04.000 align:start\n' + source + '\n\n').encode()
path = self.file('Show/Season 01/Show - S01E01.eng.vtt', raw)
self.scan_queue()
seen = []
def translate(texts, context, notes, shorten=False):
seen.append((texts, context))
if any(MARKUP.search(text) for text in texts):
raise InlineFormattingError('Model changed inline formatting markers.')
return [{'He is':'On je', 'our captain':'náš kapitán', 'Goodbye!':'Sbohem!'}[text] for text in texts]
self.service.translator = translate
with patch.object(self.service.stop, 'wait', return_value=False):self.process()
self.assertEqual(self.job()['status'], 'completed')
result = SubtitleDocument(output_path(path).read_bytes())
self.assertEqual(result.cues[0][2], '<b>On je <i>náš kapitán</i>.</b>\n<v Bob>Sbohem!</v>{\\an8}')
self.assertEqual(result.render([source]), raw)
self.assertEqual(path.read_bytes(), raw)
plain_calls = [(texts,context) for texts,context in seen if not any(MARKUP.search(t) for t in texts)]
self.assertEqual([texts for texts,_ in plain_calls], [['He is'],['our captain'],['Goodbye!']])
self.assertEqual(plain_calls[0][1], [])
self.assertEqual(plain_calls[1][1][-1], {'source':'He is','target':'On je'})
self.assertEqual(plain_calls[2][1][-1], {'source':'our captain','target':'náš kapitán'})
self.assertTrue(any('formatting held in Python' in line for line in self.service.list_queue_jobs()[0]['log']))
def test_real_ollama_adapter_recovers_dropped_markers(self):
import io
raw = b'1\n00:00:01,000 --> 00:00:04,000\nHello <i>my friend</i>!\n'
path = self.file(content=raw);self.scan_queue()
self.service.translator = ollama_translate
requests = []
def model(request, timeout):
payload = json.loads(request.data)
dialogue = payload['prompt'].rsplit('\n\n\n',1)[-1]
requests.append(dialogue)
text = 'Ahoj kamaráde!' if '__TAG' in dialogue else {'Hello':'Ahoj','my friend':'kamaráde'}[dialogue]
return io.BytesIO(json.dumps({'response':text,'done_reason':'stop'}).encode())
with patch('ai_subtitles.urllib.request.urlopen',side_effect=model), patch.object(self.service.stop,'wait',return_value=False):
self.process()
self.assertEqual(self.job()['status'],'completed')
self.assertEqual(requests[-2:],['Hello','my friend'])
self.assertEqual(SubtitleDocument(output_path(path).read_bytes()).cues[0][2],'Ahoj <i>kamaráde</i>!')
def test_marker_fallback_resumes_after_96_saved_cues(self):
texts = [f'Original {i}' for i in range(1,97)] + ['Hello <i>friend</i>!']
raw = ''.join(f'{i}\n00:00:01,000 --> 00:00:02,000\n{text}\n\n' for i,text in enumerate(texts,1)).encode()
path = self.file(content=raw);self.scan_queue()
with self.service.db() as db:
db.executemany('INSERT INTO translations VALUES(1,?,?,?)',[(i,texts[i],'Saved '+str(i)) for i in range(96)])
db.execute('UPDATE jobs SET progress=96')
calls=[]
def translate(texts, *args, **kwargs):
calls.extend(texts)
if any(MARKUP.search(t) for t in texts):raise InlineFormattingError('Lost marker')
return [{'Hello':'Ahoj','friend':'příteli'}[t] for t in texts]
self.service.translator=translate
with patch.object(self.service.stop,'wait',return_value=False):self.process()
self.assertEqual(self.job()['status'],'completed')
self.assertFalse(any(t.startswith('Original') for t in calls))
result=SubtitleDocument(output_path(path).read_bytes())
self.assertEqual([c[2] for c in result.cues[:96]],['Saved '+str(i) for i in range(96)])
self.assertEqual(result.cues[96][2],'Ahoj <i>příteli</i>!')
self.assertEqual(result.render(texts),raw)
def test_marker_fallback_still_rejects_invalid_plain_translation(self):
path=self.file(content=b'1\n00:00:01,000 --> 00:00:02,000\nHello <i>friend</i>!\n')
self.scan_queue()
def broken(texts,*args,**kwargs):
if any(MARKUP.search(t) for t in texts):raise InlineFormattingError('Lost marker')
return ['' for t in texts]
self.service.translator=broken
with patch.object(self.service.stop,'wait',return_value=False):self.process()
self.assertEqual(self.job()['status'],'failed')
self.assertIn('Empty translation',self.job()['error'])
self.assertFalse(output_path(path).exists())
def test_marker_fallback_respects_pause_between_spans(self):
path=self.file(content=b'1\n00:00:01,000 --> 00:00:02,000\nHello <i>friend</i>!\n')
self.scan_queue()
def pause(texts,*args,**kwargs):
if any(MARKUP.search(t) for t in texts):raise InlineFormattingError('Lost marker')
self.service.pause(1)
return ['Ahoj']
self.service.translator=pause
with patch.object(self.service.stop,'wait',return_value=False):self.process()
self.assertEqual(self.job()['status'],'paused')
self.assertFalse(output_path(path).exists())
def test_publication_race_does_not_replace_existing_file(self):
path=self.file();self.scan_queue()
target=output_path(path)
import os
link=os.link
def race(src,dst):
target.write_bytes(b'existing translation')
return link(src,dst)
with patch('ai_subtitles.os.link',side_effect=race):self.process()
self.assertEqual(target.read_bytes(),b'existing translation')
self.assertEqual(self.job()['status'],'failed')
self.assertFalse(list(path.parent.glob('.kaizoku-subtitle-*')))
if __name__=='__main__': unittest.main()