157 lines
7.1 KiB
Python
157 lines
7.1 KiB
Python
"""Subtitle document and formatting helpers adapted from Dymas/kaizoku b20ac193.
|
|
Original author Dymas; implementation retained with general-language processing.
|
|
"""
|
|
import re
|
|
TIMING = re.compile(r"^(?:\d{2,}:)?\d{2}:\d{2}[.,]\d{3}\s+-->\s+(?:\d{2,}:)?\d{2}:\d{2}[.,]\d{3}(?:\s+.*)?$")
|
|
MARKUP = re.compile(r"<[^>\n]+>|\{\\[^}\n]+\}")
|
|
LANGUAGES = {'en': 'en', 'eng': 'en', 'cs': 'cs', 'ces': 'cs', 'cze': 'cs', 'ja': 'ja', 'de': 'de'}
|
|
class SubtitleDocument:
|
|
"""Keep every non-dialogue line verbatim, including VTT metadata and cue IDs."""
|
|
|
|
def __init__(self, data):
|
|
if len(data) > 10 * 1024 * 1024:
|
|
raise ValueError('Subtitle exceeds the 10 MiB limit.')
|
|
self.bom = data.startswith(b'\xef\xbb\xbf')
|
|
try:
|
|
self.text = data.decode('utf-8-sig')
|
|
except UnicodeDecodeError as exc:
|
|
raise ValueError('Convert this subtitle to UTF-8 before translating.') from exc
|
|
self.lines = self.text.splitlines(keepends=True)
|
|
self.cues = []
|
|
start = 0
|
|
while start < len(self.lines):
|
|
end = start
|
|
while end < len(self.lines) and self.lines[end].strip():
|
|
end += 1
|
|
block = self.lines[start:end]
|
|
# NOTE/STYLE/REGION blocks may contain arrows; they are not dialogue.
|
|
if block and not re.match(r'^(WEBVTT|NOTE|STYLE|REGION)(?:\s|$)', block[0]):
|
|
times = [i for i, line in enumerate(block) if TIMING.fullmatch(line.strip())]
|
|
if times:
|
|
if len(times) != 1 or times[0] > 1 or times[0] == len(block) - 1:
|
|
raise ValueError('Malformed subtitle cue.')
|
|
body_start = start + times[0] + 1
|
|
self.cues.append((body_start, end, ''.join(self.lines[body_start:end]).rstrip('\r\n').replace('\r\n', '\n')))
|
|
elif any('-->' in line for line in block):
|
|
raise ValueError('Malformed subtitle timestamp.')
|
|
start = end + 1
|
|
if not self.cues:
|
|
raise ValueError('No supported subtitle cues found.')
|
|
|
|
def render(self, translations):
|
|
if len(translations) != len(self.cues):
|
|
raise ValueError('Missing translated cues.')
|
|
lines = list(self.lines)
|
|
for (start, end, source), translated in reversed(list(zip(self.cues, translations))):
|
|
validate_text(source, translated)
|
|
newline = '\r\n' if self.lines[start].endswith('\r\n') else '\n'
|
|
ending = newline if self.lines[end - 1].endswith('\n') else ''
|
|
lines[start:end] = [translated.replace('\r\n', '\n').replace('\n', newline) + ending]
|
|
return (b'\xef\xbb\xbf' if self.bom else b'') + ''.join(lines).encode('utf-8')
|
|
|
|
|
|
def validate_text(source, translated):
|
|
if not isinstance(translated, str) or not translated.strip():
|
|
raise ValueError('Empty translation.')
|
|
if MARKUP.sub('', source).strip() and not MARKUP.sub('', translated).strip():
|
|
raise ValueError('Empty translated dialogue.')
|
|
if '-->' in translated or '\r' in translated or '\x00' in translated or re.search(r'\n\s*\n', translated):
|
|
raise ValueError('Malformed translated dialogue.')
|
|
if MARKUP.findall(source) != MARKUP.findall(translated):
|
|
raise ValueError('Subtitle formatting was changed.')
|
|
if len(translated) > max(500, len(source) * 6):
|
|
raise ValueError('Translation is unexpectedly long.')
|
|
|
|
|
|
def language_for(path, text):
|
|
# Only suffix tokens: a show named "English" must not set its track language.
|
|
tokens = re.split(r'[.\[\] _-]+', path.stem.casefold())
|
|
labelled = next((LANGUAGES[token] for token in reversed(tokens[-3:]) if token in LANGUAGES), None)
|
|
words = re.findall(r'[^\W\d_]+', MARKUP.sub('', text).casefold())
|
|
cs = sum(w in {'jsem', 'jsi', 'jste', 'není', 'proč', 'že', 'něco', 'ještě', 'můžu', 'tady', 'tohle', 'dobře'} for w in words)
|
|
en = sum(w in {'the', 'you', 'what', 'this', 'that', 'have', 'with', 'your', 'are', 'was', 'not'} for w in words)
|
|
# Protect Czech dialogue even when an imported track is mislabeled English.
|
|
if labelled in {None, 'en'} and cs >= 4 and cs > en:
|
|
return 'cs', 'text estimate'
|
|
if labelled:
|
|
return labelled, 'filename'
|
|
if en >= 8 and en > cs * 3:
|
|
return 'en', 'text estimate'
|
|
return 'und', 'unknown'
|
|
|
|
|
|
def output_path(path):
|
|
parts = path.stem.split('.')
|
|
parts = [p for i, p in enumerate(parts)
|
|
if i == 0 or p.casefold().split('-')[0] not in LANGUAGES]
|
|
return path.with_name('.'.join(parts) + '.cs' + path.suffix.lower())
|
|
|
|
|
|
def track_key(path):
|
|
return output_path(path).stem.removesuffix('.cs').casefold()
|
|
|
|
|
|
def length_warning(text):
|
|
plain = MARKUP.sub('', text)
|
|
return len(plain.replace('\n', ' ')) > 84 or any(len(line) > 42 for line in plain.splitlines()) or len(plain.splitlines()) > 2
|
|
|
|
|
|
def protect_markup(text):
|
|
"""Keep outer styling in Python; mask inline tags that must stay within dialogue."""
|
|
prefix, suffix = '', ''
|
|
while (match := MARKUP.match(text)):
|
|
prefix += match[0]
|
|
text = text[match.end():]
|
|
while text.endswith('>') or text.endswith('}'):
|
|
matches = list(MARKUP.finditer(text))
|
|
if not matches or matches[-1].end() != len(text):
|
|
break
|
|
match = matches[-1]
|
|
suffix = match[0] + suffix
|
|
text = text[:match.start()]
|
|
tags = []
|
|
def mask(match):
|
|
tags.append(match[0])
|
|
return f'__TAG{len(tags)-1}__'
|
|
return MARKUP.sub(mask, text), (prefix, suffix, tags)
|
|
|
|
|
|
class InlineFormattingError(ValueError):
|
|
"""The model lost or changed a protected inline subtitle tag."""
|
|
|
|
|
|
def validate_translation_quality(sources, targets):
|
|
"""Reject clear instruction echoes and repeated non-dialogue before saving cues."""
|
|
for source, target in zip(sources, targets):
|
|
# Language labels belong to our prompt, not to a subtitle response.
|
|
labels = re.findall(r'\((?:en|cs)\)', target, re.I)
|
|
if len(set(label.lower() for label in labels)) == 2 and not re.search(r'\((?:en|cs)\)', source, re.I):
|
|
raise ValueError('Model echoed translation instructions instead of translating dialogue.')
|
|
if len(sources) >= 3 and len(set(sources)) > 1:
|
|
counts = {}
|
|
for source, target in zip(sources, targets):
|
|
if len(target.strip()) >= 24:
|
|
counts.setdefault(target.strip().casefold(), set()).add(source)
|
|
if any(len(originals) >= 3 for originals in counts.values()):
|
|
raise ValueError('Model repeated the same response for different dialogue cues.')
|
|
|
|
|
|
def translation_has_quality_error(source, target):
|
|
try:
|
|
validate_translation_quality([source], [target])
|
|
return False
|
|
except ValueError:
|
|
return True
|
|
|
|
|
|
def restore_markup(text, formatting):
|
|
prefix, suffix, tags = formatting
|
|
expected = [f'__TAG{i}__' for i in range(len(tags))]
|
|
if re.findall(r'__TAG\d+__', text) != expected:
|
|
raise InlineFormattingError('Model changed inline formatting markers.')
|
|
for marker, tag in zip(expected, tags):
|
|
text = text.replace(marker, tag)
|
|
return prefix + text + suffix
|
|
|
|
|