Files
web-framework-resources/plugins/ai-subtitles/document.py
T

157 lines
7.1 KiB
Python

"""Subtitle document and formatting helpers adapted from Dymas/kaizoku b20ac193.
Original author Dymas; implementation retained with general-language processing.
"""
import re
TIMING = re.compile(r"^(?:\d{2,}:)?\d{2}:\d{2}[.,]\d{3}\s+-->\s+(?:\d{2,}:)?\d{2}:\d{2}[.,]\d{3}(?:\s+.*)?$")
MARKUP = re.compile(r"<[^>\n]+>|\{\\[^}\n]+\}")
LANGUAGES = {'en': 'en', 'eng': 'en', 'cs': 'cs', 'ces': 'cs', 'cze': 'cs', 'ja': 'ja', 'de': 'de'}
class SubtitleDocument:
"""Keep every non-dialogue line verbatim, including VTT metadata and cue IDs."""
def __init__(self, data):
if len(data) > 10 * 1024 * 1024:
raise ValueError('Subtitle exceeds the 10 MiB limit.')
self.bom = data.startswith(b'\xef\xbb\xbf')
try:
self.text = data.decode('utf-8-sig')
except UnicodeDecodeError as exc:
raise ValueError('Convert this subtitle to UTF-8 before translating.') from exc
self.lines = self.text.splitlines(keepends=True)
self.cues = []
start = 0
while start < len(self.lines):
end = start
while end < len(self.lines) and self.lines[end].strip():
end += 1
block = self.lines[start:end]
# NOTE/STYLE/REGION blocks may contain arrows; they are not dialogue.
if block and not re.match(r'^(WEBVTT|NOTE|STYLE|REGION)(?:\s|$)', block[0]):
times = [i for i, line in enumerate(block) if TIMING.fullmatch(line.strip())]
if times:
if len(times) != 1 or times[0] > 1 or times[0] == len(block) - 1:
raise ValueError('Malformed subtitle cue.')
body_start = start + times[0] + 1
self.cues.append((body_start, end, ''.join(self.lines[body_start:end]).rstrip('\r\n').replace('\r\n', '\n')))
elif any('-->' in line for line in block):
raise ValueError('Malformed subtitle timestamp.')
start = end + 1
if not self.cues:
raise ValueError('No supported subtitle cues found.')
def render(self, translations):
if len(translations) != len(self.cues):
raise ValueError('Missing translated cues.')
lines = list(self.lines)
for (start, end, source), translated in reversed(list(zip(self.cues, translations))):
validate_text(source, translated)
newline = '\r\n' if self.lines[start].endswith('\r\n') else '\n'
ending = newline if self.lines[end - 1].endswith('\n') else ''
lines[start:end] = [translated.replace('\r\n', '\n').replace('\n', newline) + ending]
return (b'\xef\xbb\xbf' if self.bom else b'') + ''.join(lines).encode('utf-8')
def validate_text(source, translated):
if not isinstance(translated, str) or not translated.strip():
raise ValueError('Empty translation.')
if MARKUP.sub('', source).strip() and not MARKUP.sub('', translated).strip():
raise ValueError('Empty translated dialogue.')
if '-->' in translated or '\r' in translated or '\x00' in translated or re.search(r'\n\s*\n', translated):
raise ValueError('Malformed translated dialogue.')
if MARKUP.findall(source) != MARKUP.findall(translated):
raise ValueError('Subtitle formatting was changed.')
if len(translated) > max(500, len(source) * 6):
raise ValueError('Translation is unexpectedly long.')
def language_for(path, text):
# Only suffix tokens: a show named "English" must not set its track language.
tokens = re.split(r'[.\[\] _-]+', path.stem.casefold())
labelled = next((LANGUAGES[token] for token in reversed(tokens[-3:]) if token in LANGUAGES), None)
words = re.findall(r'[^\W\d_]+', MARKUP.sub('', text).casefold())
cs = sum(w in {'jsem', 'jsi', 'jste', 'není', 'proč', 'že', 'něco', 'ještě', 'můžu', 'tady', 'tohle', 'dobře'} for w in words)
en = sum(w in {'the', 'you', 'what', 'this', 'that', 'have', 'with', 'your', 'are', 'was', 'not'} for w in words)
# Protect Czech dialogue even when an imported track is mislabeled English.
if labelled in {None, 'en'} and cs >= 4 and cs > en:
return 'cs', 'text estimate'
if labelled:
return labelled, 'filename'
if en >= 8 and en > cs * 3:
return 'en', 'text estimate'
return 'und', 'unknown'
def output_path(path):
parts = path.stem.split('.')
parts = [p for i, p in enumerate(parts)
if i == 0 or p.casefold().split('-')[0] not in LANGUAGES]
return path.with_name('.'.join(parts) + '.cs' + path.suffix.lower())
def track_key(path):
return output_path(path).stem.removesuffix('.cs').casefold()
def length_warning(text):
plain = MARKUP.sub('', text)
return len(plain.replace('\n', ' ')) > 84 or any(len(line) > 42 for line in plain.splitlines()) or len(plain.splitlines()) > 2
def protect_markup(text):
"""Keep outer styling in Python; mask inline tags that must stay within dialogue."""
prefix, suffix = '', ''
while (match := MARKUP.match(text)):
prefix += match[0]
text = text[match.end():]
while text.endswith('>') or text.endswith('}'):
matches = list(MARKUP.finditer(text))
if not matches or matches[-1].end() != len(text):
break
match = matches[-1]
suffix = match[0] + suffix
text = text[:match.start()]
tags = []
def mask(match):
tags.append(match[0])
return f'__TAG{len(tags)-1}__'
return MARKUP.sub(mask, text), (prefix, suffix, tags)
class InlineFormattingError(ValueError):
"""The model lost or changed a protected inline subtitle tag."""
def validate_translation_quality(sources, targets):
"""Reject clear instruction echoes and repeated non-dialogue before saving cues."""
for source, target in zip(sources, targets):
# Language labels belong to our prompt, not to a subtitle response.
labels = re.findall(r'\((?:en|cs)\)', target, re.I)
if len(set(label.lower() for label in labels)) == 2 and not re.search(r'\((?:en|cs)\)', source, re.I):
raise ValueError('Model echoed translation instructions instead of translating dialogue.')
if len(sources) >= 3 and len(set(sources)) > 1:
counts = {}
for source, target in zip(sources, targets):
if len(target.strip()) >= 24:
counts.setdefault(target.strip().casefold(), set()).add(source)
if any(len(originals) >= 3 for originals in counts.values()):
raise ValueError('Model repeated the same response for different dialogue cues.')
def translation_has_quality_error(source, target):
try:
validate_translation_quality([source], [target])
return False
except ValueError:
return True
def restore_markup(text, formatting):
prefix, suffix, tags = formatting
expected = [f'__TAG{i}__' for i in range(len(tags))]
if re.findall(r'__TAG\d+__', text) != expected:
raise InlineFormattingError('Model changed inline formatting markers.')
for marker, tag in zip(expected, tags):
text = text.replace(marker, tag)
return prefix + text + suffix