diff --git a/.travis.yml b/.travis.yml index 7dbc7fa3..3e0cdc72 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,9 @@ language: python python: - - 3.4 - - 3.5 + - 2.7 + - 3.4 + - 3.5 script: python setup.py test diff --git a/MANIFEST.in b/MANIFEST.in index f7dbd719..865bdc7e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,5 @@ include pycaption/english.pickle +include pycaption/*.ttf +include pycaption/*.otf recursive-exclude tests * -include README.rst +include README.rst requirements.txt diff --git a/pycaption/NotoSansArabic-Regular.ttf b/pycaption/NotoSansArabic-Regular.ttf new file mode 100644 index 00000000..79359c46 Binary files /dev/null and b/pycaption/NotoSansArabic-Regular.ttf differ diff --git a/pycaption/NotoSansDisplay-Regular-Note-Math.ttf b/pycaption/NotoSansDisplay-Regular-Note-Math.ttf new file mode 100644 index 00000000..13793a15 Binary files /dev/null and b/pycaption/NotoSansDisplay-Regular-Note-Math.ttf differ diff --git a/pycaption/NotoSansDisplay-RegularAndArabic.ttf b/pycaption/NotoSansDisplay-RegularAndArabic.ttf new file mode 100644 index 00000000..fb5d8cee Binary files /dev/null and b/pycaption/NotoSansDisplay-RegularAndArabic.ttf differ diff --git a/pycaption/NotoSansJP+Math-Regular.ttf b/pycaption/NotoSansJP+Math-Regular.ttf new file mode 100644 index 00000000..3c29cffe Binary files /dev/null and b/pycaption/NotoSansJP+Math-Regular.ttf differ diff --git a/pycaption/NotoSansKR+Math-Regular.ttf b/pycaption/NotoSansKR+Math-Regular.ttf new file mode 100644 index 00000000..0bae5a8e Binary files /dev/null and b/pycaption/NotoSansKR+Math-Regular.ttf differ diff --git a/pycaption/NotoSansSC+Math-Regular.ttf b/pycaption/NotoSansSC+Math-Regular.ttf new file mode 100644 index 00000000..7cc1d9f1 Binary files /dev/null and b/pycaption/NotoSansSC+Math-Regular.ttf differ diff --git a/pycaption/NotoSansTC+Math-Regular.ttf b/pycaption/NotoSansTC+Math-Regular.ttf new file mode 100644 index 00000000..1c72e41a Binary files /dev/null and b/pycaption/NotoSansTC+Math-Regular.ttf differ diff --git a/pycaption/__init__.py b/pycaption/__init__.py index 784a3ec0..7866b3ee 100644 --- a/pycaption/__init__.py +++ b/pycaption/__init__.py @@ -1,7 +1,10 @@ +import bs4 + from .base import ( CaptionConverter, CaptionNode, Caption, CaptionList, CaptionSet) from .dfxp import DFXPWriter, DFXPReader from .sami import SAMIReader, SAMIWriter +from .scenarist import ScenaristDVDWriter from .srt import SRTReader, SRTWriter from .scc import SCCReader, SCCWriter from .webvtt import WebVTTReader, WebVTTWriter @@ -14,7 +17,7 @@ 'SAMIReader', 'SAMIWriter', 'SRTReader', 'SRTWriter', 'SCCReader', 'SCCWriter', 'WebVTTReader', 'WebVTTWriter', 'CaptionReadError', 'CaptionReadNoCaptions', 'CaptionReadSyntaxError', - 'detect_format', 'CaptionNode', 'Caption', 'CaptionList', 'CaptionSet' + 'detect_format', 'CaptionNode', 'Caption', 'CaptionList', 'CaptionSet', 'ScenaristDVDWriter' ] SUPPORTED_READERS = ( diff --git a/pycaption/base.py b/pycaption/base.py index b5e30d2a..7d83e758 100644 --- a/pycaption/base.py +++ b/pycaption/base.py @@ -1,6 +1,5 @@ from datetime import timedelta from numbers import Number -from six import text_type from .exceptions import CaptionReadError, CaptionReadTimingError @@ -232,7 +231,7 @@ def get_text_for_node(node): def _format_timestamp(self, value, msec_separator=None): datetime_value = timedelta(milliseconds=(int(value / 1000))) - str_value = text_type(datetime_value)[:11] + str_value = str(datetime_value)[:11] if not datetime_value.microseconds: str_value += '.000' diff --git a/pycaption/dfxp/__init__.py b/pycaption/dfxp/__init__.py index 0a6ea04f..7ce255d3 100644 --- a/pycaption/dfxp/__init__.py +++ b/pycaption/dfxp/__init__.py @@ -1,2 +1,2 @@ -from .base import * -from .extras import SinglePositioningDFXPWriter, LegacyDFXPWriter +from .base import * # noqa: F401, F403 +from .extras import SinglePositioningDFXPWriter, LegacyDFXPWriter # noqa: F401 \ No newline at end of file diff --git a/pycaption/dfxp/base.py b/pycaption/dfxp/base.py index e1045715..63ad7477 100644 --- a/pycaption/dfxp/base.py +++ b/pycaption/dfxp/base.py @@ -1,16 +1,21 @@ import re from copy import deepcopy from xml.sax.saxutils import escape + from bs4 import BeautifulSoup, NavigableString from ..base import ( BaseReader, BaseWriter, CaptionSet, CaptionList, Caption, CaptionNode, - DEFAULT_LANGUAGE_CODE) + DEFAULT_LANGUAGE_CODE, +) from ..exceptions import ( - CaptionReadNoCaptions, CaptionReadSyntaxError, InvalidInputError) + CaptionReadNoCaptions, CaptionReadSyntaxError, InvalidInputError, + CaptionReadTimingError, +) from ..geometry import ( Point, Stretch, UnitEnum, Padding, VerticalAlignmentEnum, - HorizontalAlignmentEnum, Alignment, Layout) + HorizontalAlignmentEnum, Alignment, Layout, +) from ..utils import is_leaf __all__ = [ @@ -37,17 +42,42 @@ DFXP_DEFAULT_REGION = Layout( alignment=Alignment( - HorizontalAlignmentEnum.CENTER, VerticalAlignmentEnum.BOTTOM) + HorizontalAlignmentEnum.START, VerticalAlignmentEnum.BOTTOM) ) DFXP_DEFAULT_STYLE_ID = 'default' DFXP_DEFAULT_REGION_ID = 'bottom' +CLOCK_TIME_PATTERN = ( + r'(?P(?P\d+):(?P\d{2}):(?P\d{2})' + r'(:(?P\d{2})|\.(?P\d+))?)' +) +OFFSET_TIME_PATTERN = (r'(?P(?P\d+(\.\d+)?)' + r'(?Ph|m|s|ms|f|t))') +TIME_EXPRESSION_PATTERN = re.compile( + fr'^({CLOCK_TIME_PATTERN}|{OFFSET_TIME_PATTERN})$') + +MICROSECONDS_PER_UNIT = { + "hours": 3600000000, + "minutes": 60000000, + "seconds": 1000000, + "milliseconds": 1000 +} -class DFXPReader(BaseReader): +DFXP_DEFAULT_LANGUAGE_CODE = "en" + +class DFXPReader(BaseReader): def __init__(self, *args, **kw): - super(DFXPReader, self).__init__(*args, **kw) + super().__init__(*args, **kw) + """ This is to support positioning attributes directly on the paragraph + elements. According to the documentation, these attributes shouldn't be + taken into account: + https://www.w3.org/TR/ttml1/#style-attribute-origin + This attribute (origin, extent) may be specified by any element type + that permits use of attributes in the TT Style Namespace; however, + this attribute applies as a style property only to those element types + indicated in the following table.""" self.read_invalid_positioning = ( kw.get('read_invalid_positioning', False)) self.nodes = [] @@ -59,20 +89,24 @@ def detect(self, content): return False def read(self, content): - if type(content) != str: + if not isinstance(content, str): raise InvalidInputError('The content is not a unicode string.') dfxp_document = self._get_dfxp_parser_class()( - content, read_invalid_positioning=self.read_invalid_positioning) + content, read_invalid_positioning= + self.read_invalid_positioning) caption_dict = {} style_dict = {} + default_language = dfxp_document.tt.attrs.get('xml:lang', + DEFAULT_LANGUAGE_CODE) + # Each div represents all the captions for a single language. for div in dfxp_document.find_all('div'): - lang = div.attrs.get('xml:lang', DEFAULT_LANGUAGE_CODE) + lang = div.attrs.get('xml:lang', default_language) - caption_dict[lang] = self._translate_div(div) + caption_dict[lang] = self._convert_div_to_caption_list(div) for style in dfxp_document.find_all('style'): id_ = style.attrs.get('xml:id') or style.attrs.get('id') @@ -80,9 +114,8 @@ def read(self, content): # Don't create document styles for those styles that are # descendants of tags. See link: # http://www.w3.org/TR/ttaf1-dfxp/#styling-vocabulary-style - if 'region' not in [ - parent_.name for parent_ in style.parents]: - style_dict[id_] = self._translate_style(style) + if 'region' not in [parent_.name for parent_ in style.parents]: + style_dict[id_] = self._convert_style(style) caption_set = CaptionSet(caption_dict, styles=style_dict) @@ -93,21 +126,21 @@ def read(self, content): @staticmethod def _get_dfxp_parser_class(): - """Hook method for providing a custom DFXP parser - """ + """Hook method for providing a custom DFXP parser""" return LayoutAwareDFXPParser - def _translate_div(self, div): + def _convert_div_to_caption_list(self, div): return CaptionList( - [self._translate_p_tag(p_tag) for p_tag in div.find_all('p')], + [self._convert_p_tag_to_caption(p_tag) + for p_tag in div.find_all('p') if p_tag.get_text().strip()], div.layout_info ) - def _translate_p_tag(self, p_tag): - start, end = self._find_times(p_tag) + def _convert_p_tag_to_caption(self, p_tag): + start, end = self._find_and_convert_times(p_tag) self.nodes = [] - self._translate_tag(p_tag) - styles = self._translate_style(p_tag) + self._convert_tag_to_node(p_tag) + styles = self._convert_style(p_tag) if len(self.nodes) > 0: return Caption( @@ -115,55 +148,78 @@ def _translate_p_tag(self, p_tag): layout_info=p_tag.layout_info) return None - def _find_times(self, p_tag): - start = self._translate_time(p_tag['begin']) - - try: - end = self._translate_time(p_tag['end']) - except KeyError: - dur = self._translate_time(p_tag['dur']) + def _find_and_convert_times(self, p_tag): + begin = p_tag.get('begin') + if not begin: + raise CaptionReadTimingError( + f'Missing begin time on line {p_tag}.') + + end = p_tag.get('end') + dur = p_tag.get('dur') + if not end and not dur: + raise CaptionReadTimingError( + f'Missing end time or duration on line {p_tag}.') + + start = self._convert_timestamp_to_microseconds(begin) + if end: + end = self._convert_timestamp_to_microseconds(p_tag['end']) + else: + dur = self._convert_timestamp_to_microseconds(p_tag['dur']) end = start + dur return start, end - def _translate_time(self, stamp): - if stamp[-1].isdigit(): - timesplit = stamp.split(':') - if '.' not in timesplit[2]: - timesplit[2] += '.000' - secsplit = timesplit[2].split('.') - if len(timesplit) > 3: - secsplit.append((int(timesplit[3]) / 30) * 100) - while len(secsplit[1]) < 3: - secsplit[1] += '0' - microseconds = (int(timesplit[0]) * 3600000000 + - int(timesplit[1]) * 60000000 + - int(secsplit[0]) * 1000000 + - int(secsplit[1]) * 1000) - return microseconds + def _convert_timestamp_to_microseconds(self, stamp): + match = TIME_EXPRESSION_PATTERN.search(stamp) + if not match: + raise CaptionReadTimingError( + f'Invalid timestamp: {stamp}. Accepted formats: hh:mm:ss / ' + 'hh:mm:ss:ff / hh:mm:ss.sub-frames / time_count h|m|s|ms|f.') + if match.group('clock_time'): + return self._convert_clock_time_to_microseconds(match) else: - # Must be offset-time - m = re.search('^([0-9.]+)([a-z]+)$', stamp) - value = float(m.group(1)) - metric = m.group(2) - if metric == "h": - microseconds = value * 60 * 60 * 1000000 - elif metric == "m": - microseconds = value * 60 * 1000000 - elif metric == "s": - microseconds = value * 1000000 - elif metric == "ms": - microseconds = value * 1000 - else: - raise InvalidInputError("Unsupported offset-time metric " + metric) - - return int(microseconds) + return self._convert_time_count_to_microseconds(match) - def _translate_tag(self, tag): + @staticmethod + def _convert_clock_time_to_microseconds(clock_time_match): + microseconds = int(clock_time_match.group('hours')) * \ + MICROSECONDS_PER_UNIT["hours"] + microseconds += int(clock_time_match.group('minutes')) * \ + MICROSECONDS_PER_UNIT["minutes"] + microseconds += int(clock_time_match.group('seconds')) * \ + MICROSECONDS_PER_UNIT["seconds"] + if clock_time_match.group('sub_frames'): + microseconds += int(clock_time_match.group('sub_frames').ljust( + 3, '0')) * MICROSECONDS_PER_UNIT["milliseconds"] + elif clock_time_match.group('frames'): + microseconds += int(clock_time_match.group('frames')) / 30 * \ + MICROSECONDS_PER_UNIT["seconds"] + return int(microseconds) + + @staticmethod + def _convert_time_count_to_microseconds(time_count_match): + value = float(time_count_match.group('time_count')) + metric = time_count_match.group("metric") + if metric == "h": + microseconds = value * MICROSECONDS_PER_UNIT["hours"] + elif metric == "m": + microseconds = value * MICROSECONDS_PER_UNIT["minutes"] + elif metric == "s": + microseconds = value * MICROSECONDS_PER_UNIT["seconds"] + elif metric == "ms": + microseconds = value * MICROSECONDS_PER_UNIT["milliseconds"] + elif metric == "f": + microseconds = value / 30 * MICROSECONDS_PER_UNIT["seconds"] + elif metric == "t": + raise NotImplementedError("The tick metric for time count is " + "not currently implemented.") + return int(microseconds) + + def _convert_tag_to_node(self, tag): # convert text if isinstance(tag, NavigableString): # strips indentation whitespace only - pattern = re.compile("^(?:[\n\r]+\s*)?(.+)") + pattern = re.compile("^(?:[\n\r]+\\s*)?(.+)") result = pattern.search(tag) if result: # Escaping/unescaping xml entities is the responsibility of the @@ -182,15 +238,15 @@ def _translate_tag(self, tag): # convert italics elif tag.name == 'span': # convert span - self._translate_span(tag) + self._convert_span_to_nodes(tag) else: # recursively call function for any children elements for a in tag.contents: - self._translate_tag(a) + self._convert_tag_to_node(a) - def _translate_span(self, tag): + def _convert_span_to_nodes(self, tag): # convert tag attributes - args = self._translate_style(tag) + args = self._convert_style(tag) # only include span tag if attributes returned # TODO - this is an obvious very old bug. args will be a dictionary. # but since nobody complained, I'll leave it like that. @@ -204,7 +260,7 @@ def _translate_span(self, tag): # recursively call function for any children elements for a in tag.contents: - self._translate_tag(a) + self._convert_tag_to_node(a) node = CaptionNode.create_style( False, args, layout_info=tag.layout_info) node.start = False @@ -212,10 +268,10 @@ def _translate_span(self, tag): self.nodes.append(node) else: for a in tag.contents: - self._translate_tag(a) + self._convert_tag_to_node(a) # convert style from DFXP - def _translate_style(self, tag): + def _convert_style(self, tag): """Converts the attributes of an XML node to a dictionary. This is a deprecated method of handling styling/ layout information, and overlaps (in partially known ways)with the newer way of doing stuff. @@ -239,7 +295,8 @@ def _translate_style(self, tag): attrs['italics'] = True elif arg.lower() == "tts:fontweight" and dfxp_attrs[arg] == "bold": attrs['bold'] = True - elif arg.lower() == "tts:textdecoration" and "underline" in dfxp_attrs[arg].strip().split(" "): + elif (arg.lower() == "tts:textdecoration" + and "underline" in dfxp_attrs[arg].strip().split(" ")): attrs['underline'] = True elif arg.lower() == "tts:textalign": attrs['text-align'] = dfxp_attrs[arg] @@ -259,22 +316,25 @@ def __init__(self, *args, **kwargs): self.p_style = False self.open_span = False self.region_creator = None - super(DFXPWriter, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def write(self, caption_set, force=''): """Converts a CaptionSet into an equivalent corresponding DFXP file :type caption_set: pycaption.base.CaptionSet - :param force: only use this language, if available in the caption_set + :param force: only use this language, if available in the + caption_set - :rtype: unicode + :rtype: str """ dfxp = BeautifulSoup(DFXP_BASE_MARKUP, 'lxml-xml') - dfxp.find('tt')['xml:lang'] = "en" langs = caption_set.get_languages() if force in langs: langs = [force] + dfxp.find('tt')['xml:lang'] = force + else: + dfxp.find('tt')['xml:lang'] = DFXP_DEFAULT_LANGUAGE_CODE caption_set = deepcopy(caption_set) @@ -296,14 +356,15 @@ def write(self, caption_set, force=''): dfxp = self._recreate_styling_tag( DFXP_DEFAULT_STYLE_ID, DFXP_DEFAULT_STYLE, dfxp) - self.region_creator = self._get_region_creator_class()(dfxp, caption_set) + self.region_creator = self._get_region_creator_class()( + dfxp, caption_set) self.region_creator.create_document_regions() body = dfxp.find('body') for lang in langs: div = dfxp.new_tag('div') - div['xml:lang'] = str(lang) + div['xml:lang'] = lang self._assign_positioning_data(div, lang, caption_set) for caption in caption_set.get_captions(lang): @@ -324,8 +385,7 @@ def write(self, caption_set, force=''): @staticmethod def _get_region_creator_class(): - """Hook method for providing a custom RegionCreator - """ + """Hook method for providing a custom RegionCreator""" return RegionCreator def _assign_positioning_data(self, tag, lang, caption_set=None, @@ -333,7 +393,7 @@ def _assign_positioning_data(self, tag, lang, caption_set=None, """Modifies the current tag, assigning it the 'region' attribute. :param tag: the BeautifulSoup tag to be modified - :type lang: unicode + :type lang: str :param lang: the caption language :type caption_set: CaptionSet :param caption_set: The CaptionSet parent @@ -407,18 +467,17 @@ def _recreate_span(self, line, node, dfxp, caption_set=None, caption=None, content_with_style = _recreate_style(node.content, dfxp) for style, value in list(content_with_style.items()): - styles += ' %s="%s"' % (style, value) + styles += f' {style}="{value}"' if node.layout_info: region_id, region_attribs = ( self.region_creator.get_positioning_info( lang, caption_set, caption, node )) - styles += ' region="{region_id}"'.format( - region_id=region_id) + styles += f' region="{region_id}"' if self.write_inline_positioning: styles += ' ' + ' '.join( [ - '{key}="{val}"'.format(key=k_, val=v_) + f'{k_}="{v_}"' for k_, v_ in list(region_attribs.items()) ] ) @@ -426,7 +485,7 @@ def _recreate_span(self, line, node, dfxp, caption_set=None, caption=None, if styles: if self.open_span: line = line.rstrip() + ' ' - line += '' % styles + line += f'' self.open_span = True elif self.open_span: @@ -440,7 +499,7 @@ def _encode(self, s): Escapes XML 1.0 illegal or discouraged characters For details see: - http://www.w3.org/TR/2008/REC-xml-20081126/#dt-chardata - :type s: unicode + :type s: str :param s: The content of a text node """ return escape(s) @@ -487,8 +546,8 @@ def __init__(self, markup="", features="html.parser", builder=None, dependency and (2) BeautifulSoup says it's the slowest option. :type read_invalid_positioning: bool - :param read_invalid_positioning: if True, will try to also look for - layout info on every element itself (even if the docs explicitly + :param read_invalid_positioning: if True, will try to also look + for layout info on every element itself (even if the docs explicitly call for ignoring attributes, when incorrectly placed) @@ -499,7 +558,7 @@ def __init__(self, markup="", features="html.parser", builder=None, # Work around for lack of ''' support in html.parser markup = markup.replace("'", "'") - super(LayoutAwareDFXPParser, self).__init__( + super().__init__( markup, features, builder, parse_only, from_encoding, **kwargs) self.read_invalid_positioning = read_invalid_positioning @@ -534,8 +593,7 @@ def _pre_order_visit(self, element, inherit_from=None): @staticmethod def _get_region_from_ancestors(element): - """Try to get the region ID from the nearest ancestor that has it - """ + """Try to get the region ID from the nearest ancestor that has it""" region_id = None parent = element.parent while parent: @@ -603,7 +661,7 @@ def _extract_positioning_information(self, region_id, element): :param region_id: the id of the region to which the element is associated - :type region_id: unicode + :type region_id: str :param element: BeautifulSoup Tag or NavigableString; this only comes into action (at the moment) if the :rtype: Layout @@ -613,8 +671,7 @@ def _extract_positioning_information(self, region_id, element): if region_id is not None: region_tag = self.find('region', {'xml:id': region_id}) - region_scraper = ( - self._get_layout_info_scraper_class()(self, region_tag)) + region_scraper = self._get_layout_info_scraper_class()(self, region_tag) layout_info = region_scraper.scrape_positioning_info( element, self.read_invalid_positioning @@ -629,21 +686,20 @@ def _extract_positioning_information(self, region_id, element): @staticmethod def _get_layout_info_scraper_class(): - """Hook method for getting an implementation of a LayoutInfoScraper. - """ + """Hook method for getting an implementation of a LayoutInfoScraper.""" return LayoutInfoScraper @staticmethod def _get_layout_class(): - """Hook method for providing the Layout class to use - """ + """Hook method for providing the Layout class to use""" return Layout -class LayoutInfoScraper(object): +class LayoutInfoScraper: """Encapsulates the methods for determining the layout information about an element (with the element's region playing an important role). """ + def __init__(self, document, region=None): """ :param document: the BeautifulSoup document instance, of which `region` @@ -706,9 +762,8 @@ def _get_style_sources(cls, styling_section, element): 'style', {'xml:id': referenced_style_id} ) - referenced_styles = ( - cls._get_style_reference_chain( - referenced_style, styling_section) + referenced_styles = cls._get_style_reference_chain( + referenced_style, styling_section ) return nested_styles + referenced_styles @@ -745,8 +800,7 @@ def _get_style_reference_chain(cls, style, styling_tag): elif len(referenced_styles) > 1: raise CaptionReadSyntaxError( "Invalid caption file. " - "More than 1 style with 'xml:id': {id}" - .format(id=reference) + f"More than 1 style with 'xml:id': {reference}" ) return result @@ -797,16 +851,16 @@ def scrape_positioning_info(self, element=None, even_invalid=False): text_align_source = None text_align = ( - self._find_attribute(text_align_source, 'tts:textAlign') - or _create_external_horizontal_alignment( - DFXP_DEFAULT_REGION.alignment.horizontal - ) + self._find_attribute(text_align_source, 'tts:textAlign') + or _create_external_horizontal_alignment( + DFXP_DEFAULT_REGION.alignment.horizontal + ) ) display_align = ( - self._find_attribute(usable_elem, 'tts:displayAlign') - or _create_external_vertical_alignment( - DFXP_DEFAULT_REGION.alignment.vertical - ) + self._find_attribute(usable_elem, 'tts:displayAlign') + or _create_external_vertical_alignment( + DFXP_DEFAULT_REGION.alignment.vertical + ) ) alignment = _create_internal_alignment(text_align, display_align) @@ -817,7 +871,7 @@ def _find_attribute_on_element_or_styles(self, attribute_name, element, """Look up the given attribute on the element, and all the styles referenced by it. - :type attribute_name: unicode + :type attribute_name: str :param element: BeautifulSoup Tag or NavigableString :param factory: a function, to apply to the xml attribute :param ignore: a list of values to ignore @@ -847,16 +901,16 @@ def _find_attribute(self, element, attribute_name, factory=lambda x: x, parents and all their styles (and referenced styles). :param element: BeautifulSoup Tag or NavigableString - :type attribute_name: unicode + :type attribute_name: str :param attribute_name: the name of the attribute to resolve - :type attribute_name: unicode + :type attribute_name: str :param factory: callable to transform the xml attribute into something :param ignore: iterable of values to ignore (will return None if the xml attribute is in that list) :param ignorecase: if True, the attribute will be searched in lowercase too :type ignorecase: bool - :rtype: unicode + :rtype: str :raises CaptionSyntaxError: """ value = None @@ -912,7 +966,7 @@ def _find_root_extent(self): return extent -class RegionCreator(object): +class RegionCreator: """Creates the DFXP regions, and knows how retrieve them, for assigning region IDs to every element @@ -931,6 +985,7 @@ class RegionCreator(object): *: NULL means LayoutAwareBeautifulParser.NO_POSITIONING_INFO """ + def __init__(self, dfxp, caption_set): """ :type dfxp: BeautifulSoup @@ -996,9 +1051,8 @@ def _create_unique_regions(unique_layouts, dfxp, id_factory): for region_spec in unique_layouts: if ( - region_spec.origin or region_spec.extent or - region_spec.padding or region_spec.alignment): - + region_spec.origin or region_spec.extent + or region_spec.padding or region_spec.alignment): new_region = dfxp.new_tag('region') new_id = id_factory() new_region['xml:id'] = new_id @@ -1032,9 +1086,9 @@ def create_document_regions(self): def _get_new_id(self, prefix='r'): """Return new, unique ids (use an internal counter). - :type prefix: unicode + :type prefix: str """ - new_id = str((prefix or '') + str(self._id_seed)) + new_id = f'{prefix}{self._id_seed}' self._id_seed += 1 return new_id @@ -1053,13 +1107,13 @@ def get_positioning_info(
tags mean the caption is None and caption_node is None.

tags mean the caption_node is None - :type lang: unicode + :type lang: str :param lang: the language of the current caption element :type caption_set: CaptionSet :type caption: Caption :type caption_node: CaptionNode :rtype: tuple - :return: (unicode, dict) + :return: (str, dict) """ # More intelligent people would have used an elem.parent.parent..parent # pattern, but pycaption is not yet structured for this. 3 params @@ -1092,8 +1146,7 @@ def get_positioning_info( return region_id, positioning_attributes def cleanup_regions(self): - """Remove the unused regions from the output file - """ + """Remove the unused regions from the output file""" layout_tag = self._dfxp.find('layout') if not layout_tag: return @@ -1144,8 +1197,8 @@ def _create_internal_alignment(text_align, display_align): "before", "center" and "after", with the default of "before". These refer to top/bottom positioning. - :type text_align: unicode - :type display_align: unicode + :type text_align: str + :type display_align: str :rtype: Alignment """ if not (text_align or display_align): @@ -1159,8 +1212,8 @@ def _create_external_horizontal_alignment(horizontal_component): """From an internal horizontal alignment value, create a value to be used in the dfxp output file. - :type horizontal_component: unicode - :rtype: unicode + :type horizontal_component: str + :rtype: str """ result = None @@ -1182,8 +1235,8 @@ def _create_external_vertical_alignment(vertical_component): """Given an alignment value used in the internal representation of the caption, return a value usable in the actual dfxp output file. - :type vertical_component: unicode - :rtype: unicode + :type vertical_component: str + :rtype: str """ result = None @@ -1273,10 +1326,7 @@ def _convert_layout_to_attributes(layout): """ result = {} if not layout: - # TODO - change this to actually use the DFXP_DEFAULT_REGION - result['tts:textAlign'] = HorizontalAlignmentEnum.CENTER - result['tts:displayAlign'] = VerticalAlignmentEnum.BOTTOM - return result + return _create_external_alignment(DFXP_DEFAULT_REGION.alignment) if layout.origin: result['tts:origin'] = layout.origin.to_xml_attribute() @@ -1289,6 +1339,8 @@ def _convert_layout_to_attributes(layout): if layout.alignment: result.update(_create_external_alignment(layout.alignment)) + else: + result.update(_create_external_alignment(DFXP_DEFAULT_REGION.alignment)) return result @@ -1297,10 +1349,48 @@ class _OrderedSet(list): """Quick implementation of a set that tracks the order. If this is a performance bottleneck, replace it with some other implementation. """ + def add(self, p_object): if p_object not in self: - super(_OrderedSet, self).append(p_object) + super().append(p_object) def discard(self, value): if value in self: - super(_OrderedSet, self).remove(value) + super().remove(value) + + +class DFXPReaderNewLineFix(DFXPReader): + def _translate_tag(self, tag): + # convert text + if isinstance(tag, NavigableString): + # strips indentation whitespace only + pattern = re.compile(r"^(?:[\n\r]+\s*)?(.+)", re.MULTILINE) + matches = list(pattern.finditer(tag)) + + # result = pattern.search(tag) + while matches: + m = matches.pop(0) + # Escaping/unescaping xml entities is the responsibility of the + # xml parser used by BeautifulSoup in its initialization. The + # content of the tag variable at this point should be a plain + # unicode string with xml entities already converted to unicode + # characters. + tag_text = m.groups()[0].strip() + node = CaptionNode.create_text(tag_text, layout_info=tag.layout_info) + self.nodes.append(node) + if matches: + self.nodes.append(CaptionNode.create_break(layout_info=tag.layout_info)) + # convert line breaks + elif tag.name == "br": + self.nodes.append(CaptionNode.create_break(layout_info=tag.layout_info)) + # convert italics + elif tag.name == "span": + # convert span + self._translate_span(tag) + elif tag.name == "p" and not tag.contents: + node = CaptionNode.create_text("", layout_info=tag.layout_info) + self.nodes.append(node) + else: + # recursively call function for any children elements + for a in tag.contents: + self._translate_tag(a) \ No newline at end of file diff --git a/pycaption/dfxp/extras.py b/pycaption/dfxp/extras.py index 48855559..5bfee8c5 100644 --- a/pycaption/dfxp/extras.py +++ b/pycaption/dfxp/extras.py @@ -2,13 +2,13 @@ # in a lot of cases, but since the transformations on them could be quite # complex, the deepcopy method is good enough sometimes. from copy import deepcopy +from xml.sax.saxutils import escape + +from bs4 import BeautifulSoup from .base import DFXPWriter, DFXP_DEFAULT_REGION from ..base import BaseWriter, CaptionNode, merge_concurrent_captions -from xml.sax.saxutils import escape -from bs4 import BeautifulSoup - LEGACY_DFXP_BASE_MARKUP = ''' @@ -36,11 +36,12 @@ class SinglePositioningDFXPWriter(DFXPWriter): - """A dfxp writer, that ignores all positioning, using a single provided value + """ + A dfxp writer, that ignores all positioning, using a single provided value """ def __init__(self, default_positioning=DFXP_DEFAULT_REGION, *args, **kwargs): - super(SinglePositioningDFXPWriter, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.default_positioning = default_positioning def write(self, captions_set, force=''): @@ -48,12 +49,12 @@ def write(self, captions_set, force=''): :type captions_set: pycaption.base.CaptionSet :param force: only write this language, if available in the CaptionSet - :rtype: unicode + :rtype: str """ captions_set = self._create_single_positioning_caption_set( captions_set, self.default_positioning) - return super(SinglePositioningDFXPWriter, self).write(captions_set, force) # noqa + return super().write(captions_set, force) # noqa @staticmethod def _create_single_positioning_caption_set(caption_set, positioning): @@ -123,12 +124,13 @@ def write(self, caption_set, force=''): for lang in langs: div = dfxp.new_tag('div') - div['xml:lang'] = '%s' % lang + div['xml:lang'] = lang for caption in caption_set.get_captions(lang): if caption.style: caption_style = caption.style - caption_style.update({'region': LEGACY_DFXP_DEFAULT_REGION_ID}) + caption_style.update( + {'region': LEGACY_DFXP_DEFAULT_REGION_ID}) else: caption_style = {'class': LEGACY_DFXP_DEFAULT_STYLE_ID, 'region': LEGACY_DFXP_DEFAULT_REGION_ID} @@ -209,12 +211,12 @@ def _recreate_span(self, line, node, dfxp): content_with_style = self._recreate_style(node.content, dfxp) for style, value in list(content_with_style.items()): - styles += ' %s="%s"' % (style, value) + styles += f' {style}="{value}"' if styles: if self.open_span: line = line.rstrip() + ' ' - line += '' % styles + line += f'' self.open_span = True elif self.open_span: @@ -245,4 +247,4 @@ def _recreate_style(self, content, dfxp): if 'display-align' in content: dfxp_style['tts:displayAlign'] = content['display-align'] - return dfxp_style + return dfxp_style \ No newline at end of file diff --git a/pycaption/geometry.py b/pycaption/geometry.py index a39e9b1b..115c6d9d 100644 --- a/pycaption/geometry.py +++ b/pycaption/geometry.py @@ -7,7 +7,6 @@ responsible for the recalculation should return a new object with the necessary modifications. """ -import six from enum import Enum from .exceptions import RelativizationError @@ -129,7 +128,7 @@ def from_xml_attribute(cls, attribute): :type attribute: unicode """ - horizontal, vertical = six.text_type(attribute).split(' ') + horizontal, vertical = str(attribute).split(' ') horizontal = Size.from_string(horizontal) vertical = Size.from_string(vertical) @@ -404,7 +403,6 @@ def to_xml_attribute(self, **kwargs): x=self.x.to_xml_attribute(), y=self.y.to_xml_attribute()) -@six.python_2_unicode_compatible class Size(object): """Ties together a number with a unit, to represent a size. @@ -562,7 +560,7 @@ def __str__(self): def to_xml_attribute(self, **kwargs): """Returns a unicode representation of this object, as an xml attribute """ - return six.text_type(self) + return str(self) def serialized(self): """Returns the "useful" values of this object""" @@ -628,7 +626,7 @@ def from_xml_attribute(cls, attribute): :param attribute: a string like object, representing a dfxp attr. value :return: a Padding object """ - values_list = six.text_type(attribute).split(' ') + values_list = str(attribute).split(' ') sizes = [] for value in values_list: diff --git a/pycaption/mergefonts.ff b/pycaption/mergefonts.ff new file mode 100644 index 00000000..79390f83 --- /dev/null +++ b/pycaption/mergefonts.ff @@ -0,0 +1,13 @@ +#!/usr/local/bin/fontforge +Open($1) +SelectAll() +Generate("1.ttf") +Close() +Open($2) +SelectAll() +Generate("2.ttf") +Close() +Open("1.ttf") +MergeFonts("2.ttf") +Generate($3) +Close() diff --git a/pycaption/pl_stt.py b/pycaption/pl_stt.py new file mode 100644 index 00000000..55211abb --- /dev/null +++ b/pycaption/pl_stt.py @@ -0,0 +1,133 @@ +from striprtf import striprtf +from .base import BaseReader, CaptionSet, CaptionList, Caption, CaptionNode +from .exceptions import CaptionReadNoCaptions, InvalidInputError + +import re + + +class PLSTTReader(BaseReader): + RE_HEADER = re.compile(r"\[header](.*?)\[/header]", re.DOTALL | re.IGNORECASE) + RE_BODY = re.compile(r"\[body](.*?)\[/body]", re.DOTALL | re.IGNORECASE) + + RE_SUBS_SPLIT = r"^\[\d+]$\n" + RE_HTML = re.compile(r"\[[^>]+]") + + def _get_header(self, content) -> dict: + header_match = self.RE_HEADER.search(content) + if header_match is None: + return {} + + header_lines = header_match.group(1).strip().splitlines() + if not header_lines: + return {} + + ret_headers = {k: v for k, v in [l.split("=") for l in header_lines]} + return ret_headers + + def _get_body(self, content) -> str: + body_match = self.RE_BODY.search(content) + if body_match is None: + return "" + return body_match.group(1).strip() + + def _parse_sub(self, sub): + sub_split = sub.split("\n") + + sub_start = sub_split[0].strip() + sub_end = sub_split[1].strip() + sub_text = [l.strip() for l in sub_split[2:]] + + return sub_start, sub_end, sub_text + + def detect(self, content): + if content.startswith(u'{\\rtf1'): + content = striprtf.rtf_to_text(content) + if self._get_header(content) and self._get_body(content): + return True + else: + return False + + def _guess_framerate(self, nonempty_splits): + # Try to guess the framerate by taking the highest frame number encountered across all start and end times. + # Once found, add 1 to it to get the best guess framerate, clamp it to 24fps as the minimum framerate and return it. + frame_nums = [] + for sub in nonempty_splits: + sub_start, sub_end, sub_text = self._parse_sub(sub) + frame_nums.append(int(sub_start.split(":")[-1])) + frame_nums.append(int(sub_end.split(":")[-1])) + + frame_nums = sorted(list(set(frame_nums)), reverse=True) + return float(max(24, frame_nums[0] + 1)) + + def read(self, content, lang="en-US"): + if type(content) != str: + raise InvalidInputError("The content is not a unicode string.") + if content.startswith(u'{\\rtf1'): + content = striprtf.rtf_to_text(content) + + try: + header = self._get_header(content) + if not header: + raise InvalidInputError("Invalid or missing header.") + except: + raise InvalidInputError("Invalid or missing header.") + + framerate = None + try: + framerate = float(header.get("TIME_FRAME_RATE")) + except: + framerate = None + + body = self._get_body(content) + + captions = CaptionList() + all_splits = re.split(PLSTTReader.RE_SUBS_SPLIT, body, flags=re.MULTILINE) + nonempty_splits = [split.strip() for split in all_splits if split and split.strip()] + if framerate is None: + framerate = self._guess_framerate(nonempty_splits) + + for sub in nonempty_splits: + sub_start, sub_end, sub_text = self._parse_sub(sub) + start = self._srttomicro(sub_start, framerate) + end = self._srttomicro(sub_end, framerate) + + nodes = [] + for line in sub_text: + # TODO: handle formatting and positioning tags + line = line.replace("[TOP]", "").replace("[BOTTOM]", "").replace("[CENTER]", "") + line = line.replace("[I]", "").replace("[/I]", "") + nodes.append(CaptionNode.create_text(line)) + nodes.append(CaptionNode.create_break()) + + if nodes: + # remove the last break + nodes.pop() + + if nodes: + captions.append(Caption(start, end, nodes)) + + if not captions: + raise CaptionReadNoCaptions("No captions found in file.") + return CaptionSet({lang: captions}) + + def _srttomicro(self, stamp, framerate): + h, m, s, frame = stamp.split(":") + + frame = int(frame) + split_second = frame / framerate + microseconds = int(h) * 3600000000 + int(m) * 60000000 + int(s) * 1000000 + split_second * 1000000 + return microseconds + + def _find_text_line(self, start_line, lines): + end_line = start_line + + found = False + while end_line < len(lines): + if lines[end_line].strip() == "": + found = True + elif found is True: + end_line -= 1 + break + end_line += 1 + + return end_line + 1 diff --git a/pycaption/sami.py b/pycaption/sami.py index 4911e771..a0f7e937 100644 --- a/pycaption/sami.py +++ b/pycaption/sami.py @@ -36,11 +36,9 @@ """ import re -import six from logging import FATAL from collections import deque from copy import deepcopy -from future.backports.html.parser import HTMLParseError from html.parser import HTMLParser from html.entities import name2codepoint @@ -85,7 +83,7 @@ def detect(self, content): return False def read(self, content): - if type(content) != six.text_type: + if type(content) != str: raise InvalidInputError('The content is not a unicode string.') content, doc_styles, doc_langs = ( @@ -552,10 +550,10 @@ def _recreate_style_block(self, target, rules, layout_info): if layout_info and layout_info.padding: rules.update({ - 'margin-top': six.text_type(layout_info.padding.before), - 'margin-right': six.text_type(layout_info.padding.end), - 'margin-bottom': six.text_type(layout_info.padding.after), - 'margin-left': six.text_type(layout_info.padding.start), + 'margin-top': str(layout_info.padding.before), + 'margin-right': str(layout_info.padding.end), + 'margin-bottom': str(layout_info.padding.after), + 'margin-left': str(layout_info.padding.start), }) for attr, value in sorted(self._recreate_style(rules).items()): @@ -751,7 +749,7 @@ def feed(self, data): data = data.replace(';>', '>') try: HTMLParser.feed(self, data) - except HTMLParseError as e: + except AttributeError as e: raise CaptionReadSyntaxError(e) # close any tags that remain in the queue diff --git a/pycaption/scc/__init__.py b/pycaption/scc/__init__.py index f109885b..6da0303f 100644 --- a/pycaption/scc/__init__.py +++ b/pycaption/scc/__init__.py @@ -1,5 +1,4 @@ #!/usr/bin/python -# -*- coding: utf-8 -*- """ 3 types of SCC captions: Roll-Up @@ -24,7 +23,6 @@ 97a1, 97a2, 9723 - [TO] move 1, 2 or 3 columns - Tab Over command - this moves the positioning 1, 2, or 3 columns to the right - - Nothing regarding this is implemented. 942f - [EOC] - display the buffer on the screen - End Of Caption ... - [PAC] - Preamble address code (can set positioning and style) @@ -40,11 +38,14 @@ the the commands don't have to necessarily be on the same row. 1. 94ae [ENM] (erase non displayed memory) - 2. 9420 [RCL] (resume caption loading => this command here means we're using Pop-On captions) + 2. 9420 [RCL] (resume caption loading => this command here means we're using + Pop-On captions) 2.1? [ENM] - if step 0 was skipped? - 3. [PAC] Positioning/ styling command (can position on columns divisible by 4) + 3. [PAC] Positioning/ styling command + (can position on columns divisible by 4) The control chars is called Preamble Address Code [PAC]. - 4. If positioning needs to be on columns not divisible by 4, use a [TO] command + 4. If positioning needs to be on columns not divisible by 4, use a [TO] + command 5. text 6. 942c [EDM] - optionally, erase the currently displayed caption 7. 942f [EOC] display the caption @@ -77,13 +78,12 @@ just carried over when implementing positioning. """ -import re import math +import re import textwrap +from collections import deque, OrderedDict from copy import deepcopy -import six - from pycaption.base import ( BaseReader, BaseWriter, CaptionSet, CaptionNode, ) @@ -93,14 +93,16 @@ MICROSECONDS_PER_CODEWORD, CHARACTER_TO_CODE, SPECIAL_OR_EXTENDED_CHAR_TO_CODE, PAC_BYTES_TO_POSITIONING_MAP, PAC_HIGH_BYTE_BY_ROW, PAC_LOW_BYTE_BY_ROW_RESTRICTED, + PAC_TAB_OFFSET_COMMANDS, ) -from .specialized_collections import ( +from .specialized_collections import ( # noqa: F401 TimingCorrectingCaptionList, NotifyingDict, CaptionCreator, - InstructionNodeCreator) + InstructionNodeCreator, PopOnCue, +) from .state_machines import DefaultProvidingPositionTracker -class NodeCreatorFactory(object): +class NodeCreatorFactory: """Will return instances of the given node_creator. This is used as a means of creating new InstructionNodeCreator instances, @@ -133,17 +135,18 @@ def from_list(self, roll_rows): ) -def get_corrected_end_time(caption): - """If the last caption was never explicitly ended, set its end time to +def fix_last_captions_without_ending(caption_list): + """ + If the last captions were never explicitly ended, set their end time to start + 4 seconds - :param Caption caption: the last caption - :rtype: int + :param caption_list: the entire list of captions """ - if caption.end: - return caption.end - return caption.start + 4 * 1000 * 1000 + for caption in reversed(caption_list): + if caption.end: + return + caption.end = caption.start + 4 * 1000 * 1000 class SCCReader(BaseReader): @@ -171,16 +174,43 @@ def __init__(self, *args, **kw): self.buffer_dict.add_change_observer(self._flush_implicit_buffers) self.buffer_dict.set_active('pop') + self.pop_ons_queue = deque() + self.roll_rows = [] self.roll_rows_expected = 0 self.simulate_roll_up = False self.time = 0 + def _group_captions_by_start_time(self, caps): + # group captions that have the same start time + caps_start_time = OrderedDict() + for i, cap in enumerate(caps): + if cap.start not in caps_start_time: + caps_start_time[cap.start] = [cap] + else: + caps_start_time[cap.start].append(cap) + # order by start timestamp + caps_start_time = OrderedDict(sorted(caps_start_time.items(), key=lambda item: item[0])) + + # check if captions with the same start time also have the same end time + # fail if different end times are found - this is not (yet?) supported + caps_final = [] + for start_time, caps_list in caps_start_time.items(): + if len(caps_list) == 1: + caps_final.append(caps_list) + else: + end_times = list(set([c.end for c in caps_list])) + if len(end_times) != 1: + raise ValueError("Unsupported subtitles - overlapping subtitles with different end times found") + else: + caps_final.append(caps_list) + return caps_final + def detect(self, content): """Checks whether the given content is a proper SCC file - :type content: unicode + :type content: str :rtype: bool """ @@ -190,13 +220,13 @@ def detect(self, content): else: return False - def read(self, content, lang='en-US', simulate_roll_up=False, offset=0): + def read(self, content, lang="en-US", simulate_roll_up=False, offset=0, merge_captions=False): """Converts the unicode string into a CaptionSet - :type content: six.text_type + :type content: str :param content: The SCC content to be converted to a CaptionSet - :type lang: six.text_type + :type lang: str :param lang: The language of the caption :type simulate_roll_up: bool @@ -207,9 +237,14 @@ def read(self, content, lang='en-US', simulate_roll_up=False, offset=0): :type offset: int :param offset: + :type merge_captions: bool + :param merge_captions: If True, we will merge captions that have the same + start and end time. We do this by merging their nodes together, separating + them with a line break. + :rtype: CaptionSet """ - if type(content) != six.text_type: + if not isinstance(content, str): raise InvalidInputError('The content is not a unicode string.') self.simulate_roll_up = simulate_roll_up @@ -221,51 +256,44 @@ def read(self, content, lang='en-US', simulate_roll_up=False, offset=0): for line in lines[1:]: self._translate_line(line) - self._flush_implicit_buffers() + self._flush_implicit_buffers(self.buffer_dict.active_key) - captions = CaptionSet({lang: self.caption_stash.get_all()}) + captions_raw = self.caption_stash.get_all() + if merge_captions: + _captions_by_start = self._group_captions_by_start_time(captions_raw) - # check captions for incorrect lengths - for cap in captions.get_captions(lang): - # if there's an end time on a caption and the difference is - # less than .05s kill it (this is likely caused by a standalone - # EOC marker in the SCC file) - if 0 < cap.end - cap.start < 50000: - raise ValueError('unsupported length found in SCC input file: ' + str(cap)) - - if captions.is_empty(): - raise CaptionReadNoCaptions("empty caption file") - else: - last_caption = captions.get_captions(lang)[-1] - last_caption.end = get_corrected_end_time(last_caption) + all_captions_with_same_time = [l for l in _captions_by_start if len(l) > 1] + for current_captions_with_same_time in all_captions_with_same_time: + nodes_to_append = [CaptionNode(CaptionNode.BREAK)] + for dupe_caption in current_captions_with_same_time[1:]: + nodes_to_append.extend(dupe_caption.nodes) + nodes_to_append.append(CaptionNode(CaptionNode.BREAK)) + captions_raw.remove(dupe_caption) - return captions + current_captions_with_same_time[0].nodes.extend(nodes_to_append) - def _fix_last_timing(self, timing): - """HACK HACK: Certain Paint-On captions don't specify the 942f [EOC] - (End Of Caption) command on the same line. - If this is a 942f line, also simulate a 942c (Erase Displayed Memory) - to properly set the timing on the last caption. + caption_set = CaptionSet({lang: captions_raw}) - This method needs some serious attention, because it proves the timing - calculation is not done well for Pop-On captions - """ - # Calculate the end time from the current line - time_translator = _SccTimeTranslator() - time_translator.start_at(timing) - time_translator.offset = self.time_translator.offset + # check captions for incorrect lengths + # for cap in captions.get_captions(lang): + # # if there's an end time on a caption and the difference is + # # less than .05s kill it (this is likely caused by a standalone + # # EOC marker in the SCC file) + # if 0 < cap.end - cap.start < 50000: + # raise ValueError('unsupported length found in SCC ' + # f'input file: {cap}') - # But use the current time translator for the start time - self.caption_stash.create_and_store( - self.buffer, self.time_translator.get_time()) + if caption_set.is_empty(): + raise CaptionReadNoCaptions("empty caption file") + else: + fix_last_captions_without_ending(caption_set.get_captions(lang)) - self.caption_stash.correct_last_timing(time_translator.get_time()) - self.buffer = self.node_creator_factory.node_creator() + return caption_set def _flush_implicit_buffers(self, old_key=None, *args): """Convert to Captions those buffers whose behavior is implicit. - The Paint-On buffer is explicit. New captions are created from it + The Pop-On buffer is explicit. New captions are created from it with the command 'End Of Caption' [EOC], '942f' The other 2 buffers, Roll-Up and Paint-On we treat as "more" implicit, @@ -274,17 +302,17 @@ def _flush_implicit_buffers(self, old_key=None, *args): we make sure to convert the buffers to text, so we don't lose any info. """ if old_key == 'pop': - return + if self.pop_ons_queue: + self._pop_on() - elif old_key is None or old_key == 'roll': + elif old_key == 'roll': if not self.buffer.is_empty(): self._roll_up() - elif old_key is None or old_key == 'paint': - # xxx - perhaps the self.buffer property is sufficient - if not self.buffer_dict['paint'].is_empty(): - self.caption_stash.create_and_store( - self.buffer_dict['paint'], self.time) + elif old_key == 'paint': + if not self.buffer.is_empty(): + self.caption_stash.create_and_store(self.buffer, self.time) + self.buffer = self.node_creator_factory.new_creator() def _translate_line(self, line): # ignore blank lines @@ -295,22 +323,20 @@ def _translate_line(self, line): r = re.compile(r"([0-9:;]*)([\s\t]*)((.)*)") parts = r.findall(line.lower()) - # XXX!!!!!! THESE 2 LINES ARE A HACK - if parts[0][2].strip() == '942f': - self._fix_last_timing(timing=parts[0][0]) - self.time_translator.start_at(parts[0][0]) # loop through each word for word in parts[0][2].split(' '): - # ignore empty results - if word.strip() != '': + # ignore empty results or invalid commands + word = word.strip() + if len(word) == 4: self._translate_word(word) def _translate_word(self, word): - # count frames for timing - self.time_translator.increment_frames() - + if self._handle_double_command(word): + # count frames for timing + self.time_translator.increment_frames() + return # first check if word is a command # TODO - check that all the positioning commands are here, or use # some other strategy to determine if the word is a command. @@ -328,36 +354,43 @@ def _translate_word(self, word): else: self._translate_characters(word) + # count frames for timing only after processing a command + self.time_translator.increment_frames() + def _handle_double_command(self, word): - # ensure we don't accidentally use the same command twice - if word == self.last_command: - self.last_command = '' - return True - else: - self.last_command = word - return False + # If the caption is to be broadcast, each of the commands are doubled + # up for redundancy in case the signal is garbled in transmission. + # The decoder is programmed to ignore a second command when it is the + # same as the first. + if word in COMMANDS or _is_pac_command(word): + if word == self.last_command: + self.last_command = '' + return True + # Fix for the + # repetition + elif _is_pac_command(word) and word in self.last_command: + self.last_command = '' + return True + elif word in PAC_TAB_OFFSET_COMMANDS: + if _is_pac_command(self.last_command): + self.last_command += f" {word}" + return False + else: + return True + + self.last_command = word + return False def _translate_special_char(self, word): - # XXX - this looks highly buggy. Why should special chars be ignored - # when printed 2 times one after another? - if self._handle_double_command(word): - return - self.buffer.add_chars(SPECIAL_CHARS[word]) def _translate_extended_char(self, word): - # XXX - this looks highly buggy. Why would a special char be ignored - # if it's printed 2 times one after another? - if self._handle_double_command(word): - return + self.buffer.remove_ascii_duplicate(EXTENDED_CHARS[word]) # add to buffer self.buffer.add_chars(EXTENDED_CHARS[word]) def _translate_command(self, word): - if self._handle_double_command(word): - return - # if command is pop_up if word == '9420': self.buffer_dict.set_active('pop') @@ -404,7 +437,13 @@ def _translate_command(self, word): # display pop_on buffer [End Of Caption] elif word == '942f': self.time = self.time_translator.get_time() - self.caption_stash.create_and_store(self.buffer, self.time) + if self.pop_ons_queue: + # there's a pop-on cue not ended by the 942c command + self._pop_on(end=self.time) + if self.buffer.is_empty(): + return + cue = PopOnCue(buffer=deepcopy(self.buffer), start=self.time, end=0) + self.pop_ons_queue.appendleft(cue) self.buffer = self.node_creator_factory.new_creator() # roll up captions [Carriage Return] @@ -413,25 +452,12 @@ def _translate_command(self, word): if not self.buffer.is_empty(): self._roll_up() - # clear screen - elif word == '942c': - self.roll_rows = [] - - # XXX - The 942c command has nothing to do with paint-ons - # This however is legacy code, and will break lots of tests if - # the proper buffer (self.buffer) is used. - # Most likely using `self.buffer` instead of the paint buffer - # is the right thing to do, but this needs some further attention. - if not self.buffer_dict['paint'].is_empty(): - self.caption_stash.create_and_store( - self.buffer_dict['paint'], self.time) - self.buffer = self.node_creator_factory.new_creator() + # 942c - Erase Displayed Memory - Clear the current screen of any + # displayed captions or text. + elif word == '942c' and self.pop_ons_queue: + self._pop_on(end=self.time_translator.get_time()) - # attempt to add proper end time to last caption(s) - self.caption_stash.correct_last_timing( - self.time_translator.get_time()) - - # if command not one of the aforementioned, add to buffer + # If command is not one of the aforementioned, add it to buffer else: self.buffer.interpret_command(word) @@ -448,8 +474,7 @@ def _translate_characters(self, word): @property def buffer(self): - """Returns the currently active buffer - """ + """Returns the currently active buffer""" return self.buffer_dict.get_active() @buffer.setter @@ -465,7 +490,7 @@ def buffer(self, value): pass def _roll_up(self): - # We expect the active buffer to be the rol buffer + # We expect the active buffer to be the roll buffer if self.simulate_roll_up: if self.roll_rows_expected > 1: if len(self.roll_rows) >= self.roll_rows_expected: @@ -485,11 +510,15 @@ def _roll_up(self): # try to insert the proper ending time for the previous caption self.caption_stash.correct_last_timing(self.time, force=True) + def _pop_on(self, end=0): + pop_on_cue = self.pop_ons_queue.pop() + self.caption_stash.create_and_store( + pop_on_cue.buffer, pop_on_cue.start, end) + class SCCWriter(BaseWriter): - def __init__(self, *args, **kw): - super(SCCWriter, self).__init__(*args, **kw) + super().__init__(*args, **kw) def write(self, caption_set): output = HEADER + '\n\n' @@ -516,20 +545,20 @@ def write(self, caption_set): code_start = start - code_time_microseconds if index == 0: continue - previous_code, previous_start, previous_end = codes[index-1] + previous_code, previous_start, previous_end = codes[index - 1] if previous_end + 3 * MICROSECONDS_PER_CODEWORD >= code_start: - codes[index-1] = (previous_code, previous_start, None) + codes[index - 1] = (previous_code, previous_start, None) codes[index] = (code, code_start, end) # PASS 3: # Write captions. for (code, start, end) in codes: - output += ('%s\t' % self._format_timestamp(start)) + output += f'{self._format_timestamp(start)}\t' output += '94ae 94ae 9420 9420 ' output += code output += '942c 942c 942f 942f\n\n' if end is not None: - output += '%s\t942c 942c\n\n' % self._format_timestamp(end) + output += f'{self._format_timestamp(end)}\t942c 942c\n\n' return output @@ -538,9 +567,11 @@ def write(self, caption_set): def _layout_line(caption): def caption_node_to_text(caption_node): if caption_node.type_ == CaptionNode.TEXT: - return six.text_type(caption_node.content) + return caption_node.content elif caption_node.type_ == CaptionNode.BREAK: return '\n' + else: + return '' caption_text = ''.join( [caption_node_to_text(node) for node in caption.nodes]) inner_lines = caption_text.split('\n') @@ -584,8 +615,8 @@ def _text_to_code(self, s): row += 16 - len(lines) # Move cursor to column 0 of the destination row for _ in range(2): - code += ('%s%s ' % (PAC_HIGH_BYTE_BY_ROW[row], - PAC_LOW_BYTE_BY_ROW_RESTRICTED[row])) + code += (PAC_HIGH_BYTE_BY_ROW[row] + + f'{PAC_LOW_BYTE_BY_ROW_RESTRICTED[row]} ') # Print the line using the SCC encoding for char in line: code = self._print_character(code, char) @@ -605,12 +636,12 @@ def _format_timestamp(microseconds): seconds = math.floor(seconds_float) seconds_float -= seconds frames = math.floor(seconds_float * 30) - return '%02d:%02d:%02d:%02d' % (hours, minutes, seconds, frames) + return f'{hours:02}:{minutes:02}:{seconds:02}:{frames:02}' -class _SccTimeTranslator(object): - """Converts SCC time to microseconds, keeping track of frames passed - """ +class _SccTimeTranslator: + """Converts SCC time to microseconds, keeping track of frames passed""" + def __init__(self): self._time = '00:00:00;00' @@ -625,7 +656,7 @@ def get_time(self): :rtype: int """ return self._translate_time( - self._time[:-2] + six.text_type(int(self._time[-2:]) + self._frames), + self._time[:-2] + str(int(self._time[-2:]) + self._frames), self.offset ) @@ -648,10 +679,10 @@ def _translate_time(stamp, offset): time_split = stamp.replace(';', ':').split(':') - timestamp_seconds = (int(time_split[0]) * 3600 + - int(time_split[1]) * 60 + - int(time_split[2]) + - int(time_split[3]) / 30.0) + timestamp_seconds = (int(time_split[0]) * 3600 + + int(time_split[1]) * 60 + + int(time_split[2]) + + int(time_split[3]) / 30.0) seconds = timestamp_seconds * seconds_per_timestamp_second microseconds = seconds * 1000 * 1000 - offset @@ -664,28 +695,24 @@ def _translate_time(stamp, offset): def start_at(self, timespec): """Reset the counter to the given time - :type timespec: unicode + :type timespec: str """ self._time = timespec self._frames = 0 def increment_frames(self): - """After a command was processed, we'd increment the number of frames - """ + """After a command was processed, we'd increment the number of frames""" self._frames += 1 def _is_pac_command(word): """Checks whether the given word is a Preamble Address Code [PAC] command - :type word: unicode + :type word: str :param word: 4 letter unicode command :rtype: bool """ - if not word or len(word) != 4: - return False - byte1, byte2 = word[:2], word[2:] try: diff --git a/pycaption/scc/constants.py b/pycaption/scc/constants.py index ba7ca2a2..a8e16f23 100644 --- a/pycaption/scc/constants.py +++ b/pycaption/scc/constants.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- - from itertools import product -from future.utils import viewitems COMMANDS = { '9420': '', @@ -328,7 +325,7 @@ '94d9': '<$>{break}<$>', '94d6': '<$>{break}<$>', '94d5': '<$>{break}<$>', - '15462': '<$>{break}<$>', + '1562': '<$>{break}<$>', '94d3': '<$>{break}<$>', '94d0': '<$>{break}<$>', '13e0': '<$>{break}<$>', @@ -367,7 +364,7 @@ '13cd': '<$>{break}<$>', '97da': '<$>{break}<$>', '13cb': '<$>{break}<$>', - '13462': '<$>{break}<$>', + '1362': '<$>{break}<$>', '16ec': '<$>{break}<$>', '16ea': '<$>{break}<$>', '16ef': '<$>{break}<$>{italic}<$>', @@ -390,7 +387,7 @@ '1676': '<$>{break}<$>', '1670': '<$>{break}<$>', '1673': '<$>{break}<$>', - '16462': '<$>{break}<$>', + '1662': '<$>{break}<$>', '97cb': '<$>{break}<$>', '97ce': '<$>{break}<$>{italic}<$>', '97cd': '<$>{break}<$>', @@ -654,7 +651,7 @@ '9131': '°', '9132': '½', '91b3': '¿', - '91b4': '™', + '9134': '™', '91b5': '¢', '91b6': '£', '9137': '♪', @@ -779,11 +776,11 @@ # Any of the values in that list, coupled with the high order byte will # map to the (row, column) tuple. # This particular dictionary will get transformed to a more suitable form for -# usage like PAC_BYTES_TO_POSITIONING_MAP[u'91'][u'd6'] = (1, 12) +# usage like PAC_BYTES_TO_POSITIONING_MAP['91']['d6'] = (1, 12) PAC_BYTES_TO_POSITIONING_MAP = { '91': { - ('d0', '51', 'c2', '43', 'c4', '45', '46', 'c7', 'c8', '49', '4a', 'cb', '4c', 'cd'): (1, 0), # noqa - ('70', 'f1', '62', 'e3', '64', 'e5', 'e6', '67', '68', 'e9', 'ea', '6b', 'ec', '6d'): (2, 0), # noqa + ('40', 'c1', 'd0', '51', 'c2', '43', 'c4', '45', '46', 'c7', 'c8', '49', '4a', 'cb', '4c', 'cd', 'ce', '4f'): (1, 0), # noqa + ('e0', '70', 'f1', '62', '61', 'e3', '64', 'e5', 'e6', '67', '68', 'e9', 'ea', '6b', 'ec', '6d', '6e', 'ef'): (2, 0), # noqa ('52', 'd3'): (1, 4), ('54', 'd5'): (1, 8), ('d6', '57'): (1, 12), @@ -797,12 +794,12 @@ ('76', 'f7'): (2, 12), ('f8', '79'): (2, 16), ('7a', 'fb'): (2, 20), - ('7c', 'fd'): (2, 24), + ('7c', 'fc', 'fd'): (2, 24), ('fe', '7f'): (2, 28) }, '92': { - ('d0', '51', 'c2', '43', 'c4', '45', '46', 'c7', 'c8', '49', '4a', 'cb', '4c', 'cd'): (3, 0), # noqa - ('70', 'f1', '62', 'e3', '64', 'e5', 'e6', '67', '68', 'e9', 'ea', '6b', 'ec', '6d'): (4, 0), # noqa + ('40', 'c1', '4f', 'd0', '51', 'c2', '43', 'c4', '45', '46', 'c7', 'c8', '49', '4a', 'cb', '4c', 'cd', 'ce'): (3, 0), # noqa + ('e0', '61', 'ef', '70', 'f1', '62', 'e3', '64', 'e5', 'e6', '67', '68', 'e9', 'ea', '6b', 'ec', '6d', '6e'): (4, 0), # noqa ('52', 'd3'): (3, 4), ('54', 'd5'): (3, 8), ('d6', '57'): (3, 12), @@ -815,13 +812,13 @@ ('f4', '75'): (4, 8), ('76', 'f7'): (4, 12), ('f8', '79'): (4, 16), - ('7a', 'fb'): (4, 20), + ('7a', 'fc', 'fb'): (4, 20), ('7c', 'fd'): (4, 24), ('fe', '7f'): (4, 28) }, '15': { - ('d0', '51', 'c2', '43', 'c4', '45', '46', 'c7', 'c8', '49', '4a', 'cb', '4c', 'cd'): (5, 0), # noqa - ('70', 'f1', '62', 'e3', '64', 'e5', 'e6', '67', '68', 'e9', 'ea', '6b', 'ec', '6d'): (6, 0), # noqa + ('40', 'ce', '4f', 'd0', '51', 'c1', 'c2', '43', 'c4', '45', '46', 'c7', 'c8', '49', '4a', 'cb', '4c', 'cd'): (5, 0), # noqa + ('e0', 'ef', '70', 'f1', '61', '62', 'e3', '64', 'e5', 'e6', '67', '68', 'e9', 'ea', '6b', 'ec', '6d', '6e'): (6, 0), # noqa ('52', 'd3'): (5, 4), ('54', 'd5'): (5, 8), ('d6', '57'): (5, 12), @@ -835,12 +832,12 @@ ('76', 'f7'): (6, 12), ('f8', '79'): (6, 16), ('7a', 'fb'): (6, 20), - ('7c', 'fd'): (6, 24), + ('7c', 'fc', 'fd'): (6, 24), ('fe', '7f'): (6, 28) }, '16': { - ('d0', '51', 'c2', '43', 'c4', '45', '46', 'c7', 'c8', '49', '4a', 'cb', '4c', 'cd'): (7, 0), # noqa - ('70', 'f1', '62', 'e3', '64', 'e5', 'e6', '67', '68', 'e9', 'ea', '6b', 'ec', '6d'): (8, 0), # noqa + ('40', 'c1', 'ce', '4f', 'd0', '51', 'c2', '43', 'c4', '45', '46', 'c7', 'c8', '49', '4a', 'cb', '4c', 'cd'): (7, 0), # noqa + ('e0', '61', '62', '6e', 'ef', '70', 'f1', '62', 'e3', '64', 'e5', 'e6', '67', '68', 'e9', 'ea', '6b', 'ec', '6d'): (8, 0), # noqa ('52', 'd3'): (7, 4), ('54', 'd5'): (7, 8), ('d6', '57'): (7, 12), @@ -854,12 +851,12 @@ ('76', 'f7'): (8, 12), ('f8', '79'): (8, 16), ('7a', 'fb'): (8, 20), - ('7c', 'fd'): (8, 24), + ('fc', '7c', 'fd'): (8, 24), ('fe', '7f'): (8, 28) }, '97': { - ('d0', '51', 'c2', '43', 'c4', '45', '46', 'c7', 'c8', '49', '4a', 'cb', '4c', 'cd'): (9, 0), # noqa - ('70', 'f1', '62', 'e3', '64', 'e5', 'e6', '67', '68', 'e9', 'ea', '6b', 'ec', '6d'): (10, 0), # noqa + ('40', 'c1', 'ce', '4f', 'd0', '51', 'c2', '43', 'c4', '45', '46', 'c7', 'c8', '49', '4a', 'cb', '4c', 'cd'): (9, 0), # noqa + ('e0', '61', '6e', 'ef', '70', 'f1', '62', 'e3', '64', 'e5', 'e6', '67', '68', 'e9', 'ea', '6b', 'ec', '6d'): (10, 0), # noqa ('52', 'd3'): (9, 4), ('54', 'd5'): (9, 8), ('d6', '57'): (9, 12), @@ -873,11 +870,11 @@ ('76', 'f7'): (10, 12), ('f8', '79'): (10, 16), ('7a', 'fb'): (10, 20), - ('7c', 'fd'): (10, 24), + ('fc', '7c', 'fd'): (10, 24), ('fe', '7f'): (10, 28) }, '10': { - ('d0', '51', 'c2', '43', 'c4', '45', '46', 'c7', 'c8', '49', '4a', 'cb', '4c', 'cd'): (11, 0), # noqa + ('40', 'c1', 'ce', '4f', 'd0', '51', 'c2', '43', 'c4', '45', '46', 'c7', 'c8', '49', '4a', 'cb', '4c', 'cd'): (11, 0), # noqa ('52', 'd3'): (11, 4), ('54', 'd5'): (11, 8), ('d6', '57'): (11, 12), @@ -887,8 +884,8 @@ ('5e', 'df'): (11, 28), }, '13': { - ('d0', '51', 'c2', '43', 'c4', '45', '46', 'c7', 'c8', '49', '4a', 'cb', '4c', 'cd'): (12, 0), # noqa - ('70', 'f1', '62', 'e3', '64', 'e5', 'e6', '67', '68', 'e9', 'ea', '6b', 'ec', '6d'): (13, 0), # noqa + ('40', 'c1', 'ce', '4f', 'd0', '51', 'c2', '43', 'c4', '45', '46', 'c7', 'c8', '49', '4a', 'cb', '4c', 'cd'): (12, 0), # noqa + ('e0', '61', '62', '6e', 'ef', '70', 'f1', '62', 'e3', '64', 'e5', 'e6', '67', '68', 'e9', 'ea', '6b', 'ec', '6d'): (13, 0), # noqa ('52', 'd3'): (12, 4), ('54', 'd5'): (12, 8), ('d6', '57'): (12, 12), @@ -902,12 +899,12 @@ ('76', 'f7'): (13, 12), ('f8', '79'): (13, 16), ('7a', 'fb'): (13, 20), - ('7c', 'fd'): (13, 24), + ('7c', 'fc', 'fd'): (13, 24), ('fe', '7f'): (13, 28) }, '94': { - ('d0', '51', 'c2', '43', 'c4', '45', '46', 'c7', 'c8', '49', '4a', 'cb', '4c', 'cd'): (14, 0), # noqa - ('70', 'f1', '62', 'e3', '64', 'e5', 'e6', '67', '68', 'e9', 'ea', '6b', 'ec', '6d'): (15, 0), # noqa + ('40', 'c1', 'ce', '4f', 'd0', '51', 'c2', '43', 'c4', '45', '46', 'c7', 'c8', '49', '4a', 'cb', '4c', 'cd'): (14, 0), # noqa + ('e0', '61', '6e', 'ef', '70', 'f1', '62', 'e3', '64', 'e5', 'e6', '67', '68', 'e9', 'ea', '6b', 'ec', '6d'): (15, 0), # noqa ('52', 'd3'): (14, 4), ('54', 'd5'): (14, 8), ('d6', '57'): (14, 12), @@ -921,11 +918,15 @@ ('76', 'f7'): (15, 12), ('f8', '79'): (15, 16), ('7a', 'fb'): (15, 20), - ('7c', 'fd'): (15, 24), + ('7c', 'fc', 'fd'): (15, 24), ('fe', '7f'): (15, 28) } } +# Tab Offset command +# - this moves the positioning 1, 2, or 3 columns to the right +PAC_TAB_OFFSET_COMMANDS = {'97a1': 1, '97a2': 2, '9723': 3} + def _create_position_to_bytes_map(bytes_to_pos): result = {} @@ -939,12 +940,12 @@ def _create_position_to_bytes_map(bytes_to_pos): if row not in result: result[row] = {} - result[row][column] = ( - tuple(product([high_byte], low_byte_list))) + result[row][column] = tuple(product([high_byte], low_byte_list)) return result + # (Almost) the reverse of PAC_BYTES_TO_POSITIONING_MAP. Call with arguments -# like for example [15][4] to get the tuple ((u'94', u'f2'), (u'94', u'73')) +# like for example [15][4] to get the tuple (('94', 'f2'), ('94', '73')) POSITIONING_TO_PAC_MAP = _create_position_to_bytes_map( PAC_BYTES_TO_POSITIONING_MAP ) @@ -954,12 +955,14 @@ def _restructure_bytes_to_position_map(byte_to_pos_map): return { k_: { low_byte: byte_to_pos_map[k_][low_byte_list] - for low_byte_list in list(v_.keys()) for low_byte in low_byte_list + for low_byte_list in list(v_.keys()) + for low_byte in low_byte_list } for k_, v_ in list(byte_to_pos_map.items()) } -# Now use the dict with arguments like [u'91'][u'75'] directly. + +# Now use the dict with arguments like ['91']['75'] directly. PAC_BYTES_TO_POSITIONING_MAP = _restructure_bytes_to_position_map( PAC_BYTES_TO_POSITIONING_MAP) @@ -967,14 +970,14 @@ def _restructure_bytes_to_position_map(byte_to_pos_map): # Inverted character lookup CHARACTER_TO_CODE = { character: code - for code, character in viewitems(CHARACTERS) + for code, character in CHARACTERS.items() } SPECIAL_OR_EXTENDED_CHAR_TO_CODE = { - character: code for code, character in viewitems(EXTENDED_CHARS) + character: code for code, character in EXTENDED_CHARS.items() } SPECIAL_OR_EXTENDED_CHAR_TO_CODE.update( - {character: code for code, character in viewitems(SPECIAL_CHARS)} + {character: code for code, character in SPECIAL_CHARS.items()} ) # Time to transmit a single codeword = 1 second / 29.97 diff --git a/pycaption/scc/specialized_collections.py b/pycaption/scc/specialized_collections.py index 31dd6085..2396cec7 100644 --- a/pycaption/scc/specialized_collections.py +++ b/pycaption/scc/specialized_collections.py @@ -1,12 +1,20 @@ +import collections +import unicodedata + from ..base import CaptionList, Caption, CaptionNode -from ..geometry import (UnitEnum, Size, Layout, Point, Alignment, - VerticalAlignmentEnum, HorizontalAlignmentEnum) +from ..geometry import ( + UnitEnum, Size, Layout, Point, Alignment, + VerticalAlignmentEnum, HorizontalAlignmentEnum +) +from .constants import ( + PAC_BYTES_TO_POSITIONING_MAP, COMMANDS, PAC_TAB_OFFSET_COMMANDS, + MICROSECONDS_PER_CODEWORD, +) -from .constants import PAC_BYTES_TO_POSITIONING_MAP, COMMANDS -import collections +PopOnCue = collections.namedtuple("PopOnCue", "buffer, start, end") -class PreCaption(object): +class PreCaption: """ The Caption class has been refactored and now its instances must be used as immutable objects. Some of the code in this module, however, relied on the @@ -38,7 +46,7 @@ class TimingCorrectingCaptionList(list): Also, doesn't allow Nones or empty captions """ def __init__(self, *args, **kwargs): - super(TimingCorrectingCaptionList, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self._last_batch = () def append(self, p_object): @@ -54,7 +62,7 @@ def append(self, p_object): self._last_batch = (p_object,) - super(TimingCorrectingCaptionList, self).append(p_object) + super().append(p_object) def extend(self, iterable): """Adds the elements in the iterable to the list, regarding the first @@ -68,7 +76,7 @@ def extend(self, iterable): self._last_batch = tuple(appendable_items) - super(TimingCorrectingCaptionList, self).extend(appendable_items) + super().extend(appendable_items) @staticmethod def _update_last_batch(batch, *new_captions): @@ -90,7 +98,9 @@ def _update_last_batch(batch, *new_captions): new_caption = new_captions[0] - if batch and batch[-1].end == 0: + if batch and (batch[-1].end == 0 + or new_caption.start - batch[-1].end + < 5 * MICROSECONDS_PER_CODEWORD + 1): for caption in batch: caption.end = new_caption.start @@ -104,7 +114,7 @@ class NotifyingDict(dict): _guard = {} def __init__(self, *args, **kwargs): - super(NotifyingDict, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.active_key = self._guard self.observers = [] @@ -124,8 +134,7 @@ def set_active(self, key): self.active_key = key def get_active(self): - """Returns the value corresponding to the active key - """ + """Returns the value corresponding to the active key""" if self.active_key is self._guard: raise KeyError('No active key set') @@ -140,15 +149,19 @@ def add_change_observer(self, observer): :param observer: any callable that can be called with 2 positional arguments """ - if not isinstance(observer, collections.Callable): + try: + check_type = collections.Callable + except AttributeError: + # python 3.10+ + check_type = collections.abc.Callable + if not isinstance(observer, check_type): raise TypeError('The observer should be callable') self.observers.append(observer) -class CaptionCreator(object): - """Creates and maintains a collection of Captions - """ +class CaptionCreator: + """Creates and maintains a collection of Captions""" def __init__(self): self._collection = TimingCorrectingCaptionList() @@ -182,7 +195,7 @@ def correct_last_timing(self, end_time, force=False): for caption in captions_to_correct: caption.end = end_time - def create_and_store(self, node_buffer, start): + def create_and_store(self, node_buffer, start, end=0): """Interpreter method, will convert the buffer into one or more Caption objects, storing them internally. @@ -194,13 +207,15 @@ def create_and_store(self, node_buffer, start): :type start: float :param start: the start time in microseconds + :type end: float + :param end: the end time in microseconds """ if node_buffer.is_empty(): return caption = PreCaption() caption.start = start - caption.end = 0 # Not yet known; filled in later + caption.end = end self._still_editing = [caption] for instruction in node_buffer: @@ -211,7 +226,7 @@ def create_and_store(self, node_buffer, start): elif instruction.requires_repositioning(): caption = PreCaption() caption.start = start - caption.end = 0 + caption.end = end self._still_editing.append(caption) # handle line breaks @@ -261,7 +276,7 @@ def get_all(self): return caption_list -class InstructionNodeCreator(object): +class InstructionNodeCreator: """Creates _InstructionNode instances from characters and commands, storing them internally """ @@ -280,14 +295,13 @@ def __init__(self, collection=None, position_tracker=None): self._position_tracer = position_tracker def is_empty(self): - """Whether any text was added to the buffer - """ + """Whether any text was added to the buffer""" return not any(element.text for element in self._collection) def add_chars(self, *chars): """Adds characters to a text node (last text node, or a new one) - :param chars: tuple containing text (unicode) + :param chars: tuple containing text (Unicode string) """ if not chars: return @@ -326,12 +340,12 @@ def add_chars(self, *chars): node.add_chars(*chars) def interpret_command(self, command): - """Given a command determines whether tu turn italics on or off, + """Given a command determines whether to turn italics on or off, or to set the positioning This is mostly used to convert from the legacy-style commands - :type command: unicode + :type command: str """ self._update_positioning(command) @@ -354,19 +368,21 @@ def interpret_command(self, command): def _update_positioning(self, command): """Sets the positioning information to use for the next nodes - :type command: unicode + :type command: str """ - if len(command) != 4: - return - - first, second = command[:2], command[2:] - - try: - positioning = PAC_BYTES_TO_POSITIONING_MAP[first][second] - except KeyError: - pass + if command in PAC_TAB_OFFSET_COMMANDS: + tab_offset = PAC_TAB_OFFSET_COMMANDS[command] + prev_positioning = self._position_tracer.default + positioning = (prev_positioning[0], + prev_positioning[1] + tab_offset) else: - self._position_tracer.update_positioning(positioning) + first, second = command[:2], command[2:] + + try: + positioning = PAC_BYTES_TO_POSITIONING_MAP[first][second] + except KeyError: + return + self._position_tracer.update_positioning(positioning) def __iter__(self): return iter(_format_italics(self._collection)) @@ -401,6 +417,24 @@ def from_list(cls, stash_list, position_tracker): return instance + def remove_ascii_duplicate(self, accented_character): + """ + Characters from the Extended Characters list are usually preceded by + their ASCII substitute, in case the decoder is not able to display + the special character. + + This is used to remove the substitute character in order to avoid + displaying both. + + :type accented_character: str + """ + if self._collection and self._collection[-1].is_text_node() and \ + self._collection[-1].text: + ascii_char = unicodedata.normalize('NFD', accented_character)\ + .encode('ascii', 'ignore').decode("utf-8") + if ascii_char and self._collection[-1].text[-1] == ascii_char: + self._collection[-1].text = self._collection[-1].text[:-1] + def _get_layout_from_tuple(position_tuple): """Create a Layout object from the positioning information given @@ -417,15 +451,17 @@ def _get_layout_from_tuple(position_tuple): row, column = position_tuple - horizontal = Size(100 * column / 32.0, UnitEnum.PERCENT) - vertical = Size(100 * (row - 1) / 15.0, UnitEnum.PERCENT) + # Horizontal safe area between 10% and 90% + horizontal = Size(80 * column / 32.0 + 10, UnitEnum.PERCENT) + # Vertical safe area between 5% and 95% + vertical = Size(90 * (row - 1) / 15.0 + 5, UnitEnum.PERCENT) return Layout(origin=Point(horizontal, vertical), alignment=Alignment(HorizontalAlignmentEnum.LEFT, VerticalAlignmentEnum.TOP) ) -class _InstructionNode(object): +class _InstructionNode: """Value object, that can contain text information, or interpretable commands (such as explicit line breaks or turning italics on/off). @@ -440,7 +476,7 @@ class _InstructionNode(object): def __init__(self, text=None, position=None, type_=0): """ - :type text: unicode + :type text: str :param position: a tuple of ints (row, column) :param type_: self.TEXT | self.BREAK | self.ITALICS :type type_: int @@ -452,7 +488,7 @@ def __init__(self, text=None, position=None, type_=0): def add_chars(self, *args): """This being a text node, add characters to it. :param args: - :type args: tuple[unicode] + :type args: tuple[str] :return: """ if self.text is None: @@ -507,8 +543,7 @@ def requires_repositioning(self): return self._type == self.CHANGE_POSITION def get_text(self): - """A little legacy code. - """ + """A little legacy code.""" return ' '.join(self.text.split()) @classmethod @@ -529,7 +564,7 @@ def create_text(cls, position, *chars): :type position: tuple[int] :param position: a tuple (row, col) to mark the positioning - :type chars: tuple[unicode] + :type chars: tuple[str] :param chars: characters to add to the text :rtype: _InstructionNode @@ -566,7 +601,7 @@ def __repr__(self): # pragma: no cover if self._type == self.BREAK: extra = 'BR' elif self._type == self.TEXT: - extra = '"{}"'.format(self.text) + extra = f'"{self.text}"' elif self._type in (self.ITALICS_ON, self.ITALICS_OFF): extra = 'italics {}'.format( 'on' if self._type == self.ITALICS_ON else 'off' @@ -574,7 +609,7 @@ def __repr__(self): # pragma: no cover else: extra = 'change position' - return ''.format(extra=extra) + return f'' def _format_italics(collection): diff --git a/pycaption/scc/state_machines.py b/pycaption/scc/state_machines.py index 89bc24c3..04fc632b 100644 --- a/pycaption/scc/state_machines.py +++ b/pycaption/scc/state_machines.py @@ -1,7 +1,7 @@ from ..exceptions import CaptionReadSyntaxError -class _PositioningTracker(object): +class _PositioningTracker: """Helps determine the positioning of a node, having kept track of positioning-related commands. """ @@ -13,6 +13,10 @@ def __init__(self, positioning=None): self._positions = [positioning] self._break_required = False self._repositioning_required = False + # Since the actual column is not applied when encountering a line break + # this attribute is used to store it and determine by comparison if the + # next positioning is actually a Tab Offset + self._last_column = None def update_positioning(self, positioning): """Being notified of a position change, updates the internal state, @@ -26,21 +30,31 @@ def update_positioning(self, positioning): if not current: if positioning: - # set the positioning for the first time + # Set the positioning for the first time self._positions = [positioning] return row, col = current - new_row, _ = positioning + if self._break_required: + col = self._last_column + new_row, new_col = positioning + is_tab_offset = new_row == row and col + 1 <= new_col <= col + 3 - # is the new position simply one line below? + # One line below will be treated as line break, not repositioning if new_row == row + 1: self._positions.append((new_row, col)) self._break_required = True + self._last_column = new_col + # Tab offsets after line breaks will be ignored to avoid repositioning + elif self._break_required and is_tab_offset: + return else: - # reset the "current" position altogether. + # Reset the "current" position altogether. self._positions = [positioning] - self._repositioning_required = True + # Tab offsets are not interpreted as repositioning, but adjustments + # to the previous PAC command + if not is_tab_offset: + self._repositioning_required = True def get_current_position(self): """Returns the current usable position @@ -65,8 +79,7 @@ def is_repositioning_required(self): return self._repositioning_required def acknowledge_position_changed(self): - """Acknowledge the position tracer that the position was changed - """ + """Acknowledge the position tracer that the position was changed""" self._repositioning_required = False def is_linebreak_required(self): @@ -76,8 +89,7 @@ def is_linebreak_required(self): return self._break_required def acknowledge_linebreak_consumed(self): - """Call to acknowledge that the line required was consumed - """ + """Call to acknowledge that the line required was consumed""" self._break_required = False @@ -95,7 +107,7 @@ def __init__(self, positioning=None, default=None): :type default: tuple[int] :param default: a tuple of ints (row, column) to use as fallback """ - super(DefaultProvidingPositionTracker, self).__init__(positioning) + super().__init__(positioning) if default: self.default = default @@ -107,10 +119,7 @@ def get_current_position(self): :rtype: tuple[int] """ try: - return ( - super(DefaultProvidingPositionTracker, self). - get_current_position() - ) + return super().get_current_position() except CaptionReadSyntaxError: return self.default @@ -124,5 +133,4 @@ def update_positioning(self, positioning): if positioning: self.default = positioning - super(DefaultProvidingPositionTracker, self).update_positioning( - positioning) \ No newline at end of file + super().update_positioning(positioning) diff --git a/pycaption/scc/translator.py b/pycaption/scc/translator.py new file mode 100644 index 00000000..851ccb6c --- /dev/null +++ b/pycaption/scc/translator.py @@ -0,0 +1,577 @@ +from pycaption.scc.constants import CHARACTERS, SPECIAL_CHARS, EXTENDED_CHARS + +ALL_CHARACTERS = {**CHARACTERS, **SPECIAL_CHARS, **EXTENDED_CHARS} +COMMAND_LABELS = { + "9420": "Resume Caption Loading", + "9429": "Resume Direct Captioning", + "9425": "Roll-Up Captions--2 Rows", + "9426": "Roll-Up Captions--3 Rows", + "94a7": "Roll-Up Captions--4 Rows", + "942a": "Text Restart", + "94ab": "Resume Text Display", + "942c": "Erase Displayed Memory", + "94ae": "Erase Non-displayed Memory", + "942f": "End Of Caption", + "9140": "row 01, column 00, with plain white text.", + "91c1": "row 01, column 00, with white underlined text.", + "91c2": "row 01, column 00, with plain green text.", + "9143": "row 01, column 00, with green underlined text.", + "91c4": "row 01, column 00, with plain blue text.", + "9145": "row 01, column 00, with blue underlined text.", + "9146": "row 01, column 00, with plain cyan text.", + "91c7": "row 01, column 00, with cyan underlined text.", + "91c8": "row 01, column 00, with plain red text.", + "9149": "row 01, column 00, with red underlined text.", + "914a": "row 01, column 00, with plain yellow text.", + "91cb": "row 01, column 00, with yellow underlined text.", + "914c": "row 01, column 00, with plain magenta text.", + "91cd": "row 01, column 00, with magenta underlined text.", + "91ce": "row 01, column 00, with white italicized text.", + "914f": "row 01, column 00, with white underlined italicized text.", + "91d0": "row 01, column 00, with plain white text.", + "9151": "row 01, column 00, with white underlined text.", + "9152": "row 01, column 04, with plain white text.", + "91d3": "row 01, column 04, with white underlined text.", + "9154": "row 01, column 08, with plain white text.", + "91d5": "row 01, column 08, with white underlined text.", + "91d6": "row 01, column 12, with plain white text.", + "9157": "row 01, column 12, with white underlined text.", + "9158": "row 01, column 16, with plain white text.", + "91d9": "row 01, column 16, with white underlined text.", + "91da": "row 01, column 20, with plain white text.", + "915b": "row 01, column 20, with white underlined text.", + "91dc": "row 01, column 24, with plain white text.", + "915d": "row 01, column 24, with white underlined text.", + "915e": "row 01, column 28, with plain white text.", + "91df": "row 01, column 28, with white underlined text.", + "91e0": "row 02, column 00, with plain white text.", + "9161": "row 02, column 00, with white underlined text.", + "9162": "row 02, column 00, with plain green text.", + "91e3": "row 02, column 00, with green underlined text.", + "9164": "row 02, column 00, with plain blue text.", + "91e5": "row 02, column 00, with blue underlined text.", + "91e6": "row 02, column 00, with plain cyan text.", + "9167": "row 02, column 00, with cyan underlined text.", + "9168": "row 02, column 00, with plain red text.", + "91e9": "row 02, column 00, with red underlined text.", + "91ea": "row 02, column 00, with plain yellow text.", + "916b": "row 02, column 00, with yellow underlined text.", + "91ec": "row 02, column 00, with plain magenta text.", + "916d": "row 02, column 00, with magenta underlined text.", + "916e": "row 02, column 00, with white italicized text.", + "91ef": "row 02, column 00, with white underlined italicized text.", + "9170": "row 02, column 00, with plain white text.", + "91f1": "row 02, column 00, with white underlined text.", + "91f2": "row 02, column 04, with plain white text.", + "9173": "row 02, column 04, with white underlined text.", + "91f4": "row 02, column 08, with plain white text.", + "9175": "row 02, column 08, with white underlined text.", + "9176": "row 02, column 12, with plain white text.", + "91f7": "row 02, column 12, with white underlined text.", + "91f8": "row 02, column 16, with plain white text.", + "9179": "row 02, column 16, with white underlined text.", + "917a": "row 02, column 20, with plain white text.", + "91fb": "row 02, column 20, with white underlined text.", + "91fc": "row 02, column 24, with plain white text.", + "91fd": "row 02, column 24, with white underlined text.", + "91fe": "row 02, column 28, with plain white text.", + "917f": "row 02, column 28, with white underlined text.", + "9240": "row 03, column 00, with plain white text.", + "92c1": "row 03, column 00, with white underlined text.", + "92c2": "row 03, column 00, with plain green text.", + "9243": "row 03, column 00, with green underlined text.", + "92c4": "row 03, column 00, with plain blue text.", + "9245": "row 03, column 00, with blue underlined text.", + "9246": "row 03, column 00, with plain cyan text.", + "92c7": "row 03, column 00, with cyan underlined text.", + "92c8": "row 03, column 00, with plain red text.", + "9249": "row 03, column 00, with red underlined text.", + "924a": "row 03, column 00, with plain yellow text.", + "92cb": "row 03, column 00, with yellow underlined text.", + "924c": "row 03, column 00, with plain magenta text.", + "92cd": "row 03, column 00, with magenta underlined text.", + "92ce": "row 03, column 00, with white italicized text.", + "924f": "row 03, column 00, with white underlined italicized text.", + "92d0": "row 03, column 00, with plain white text.", + "9251": "row 03, column 00, with white underlined text.", + "9252": "row 03, column 04, with plain white text.", + "92d3": "row 03, column 04, with white underlined text.", + "9254": "row 03, column 08, with plain white text.", + "92d5": "row 03, column 08, with white underlined text.", + "92d6": "row 03, column 12, with plain white text.", + "9257": "row 03, column 12, with white underlined text.", + "9258": "row 03, column 16, with plain white text.", + "92d9": "row 03, column 16, with white underlined text.", + "92da": "row 03, column 20, with plain white text.", + "925b": "row 03, column 20, with white underlined text.", + "92dc": "row 03, column 24, with plain white text.", + "925d": "row 03, column 24, with white underlined text.", + "925e": "row 03, column 28, with plain white text.", + "92df": "row 03, column 28, with white underlined text.", + "92e0": "row 04, column 00, with plain white text.", + "9261": "row 04, column 00, with white underlined text.", + "9262": "row 04, column 00, with plain green text.", + "92e3": "row 04, column 00, with green underlined text.", + "9264": "row 04, column 00, with plain blue text.", + "92e5": "row 04, column 00, with blue underlined text.", + "92e6": "row 04, column 00, with plain cyan text.", + "9267": "row 04, column 00, with cyan underlined text.", + "9268": "row 04, column 00, with plain red text.", + "92e9": "row 04, column 00, with red underlined text.", + "92ea": "row 04, column 00, with plain yellow text.", + "926b": "row 04, column 00, with yellow underlined text.", + "92ec": "row 04, column 00, with plain magenta text.", + "926d": "row 04, column 00, with magenta underlined text.", + "926e": "row 04, column 00, with white italicized text.", + "92ef": "row 04, column 00, with white underlined italicized text.", + "9270": "row 04, column 00, with plain white text.", + "92f1": "row 04, column 00, with white underlined text.", + "92f2": "row 04, column 04, with plain white text.", + "9273": "row 04, column 04, with white underlined text.", + "92f4": "row 04, column 08, with plain white text.", + "9275": "row 04, column 08, with white underlined text.", + "9276": "row 04, column 12, with plain white text.", + "92f7": "row 04, column 12, with white underlined text.", + "92f8": "row 04, column 16, with plain white text.", + "9279": "row 04, column 16, with white underlined text.", + "927a": "row 04, column 20, with plain white text.", + "92fb": "row 04, column 20, with white underlined text.", + "92fc": "row 04, column 24, with plain white text.", + "92fd": "row 04, column 24, with white underlined text.", + "92fe": "row 04, column 28, with plain white text.", + "927f": "row 04, column 28, with white underlined text.", + "1540": "row 05, column 00, with plain white text.", + "15c1": "row 05, column 00, with white underlined text.", + "15c2": "row 05, column 00, with plain green text.", + "1543": "row 05, column 00, with green underlined text.", + "15c4": "row 05, column 00, with plain blue text.", + "1545": "row 05, column 00, with blue underlined text.", + "1546": "row 05, column 00, with plain cyan text.", + "15c7": "row 05, column 00, with cyan underlined text.", + "15c8": "row 05, column 00, with plain red text.", + "1549": "row 05, column 00, with red underlined text.", + "154a": "row 05, column 00, with plain yellow text.", + "15cb": "row 05, column 00, with yellow underlined text.", + "154c": "row 05, column 00, with plain magenta text.", + "15cd": "row 05, column 00, with magenta underlined text.", + "15ce": "row 05, column 00, with white italicized text.", + "154f": "row 05, column 00, with white underlined italicized text.", + "15d0": "row 05, column 00, with plain white text.", + "1551": "row 05, column 00, with white underlined text.", + "1552": "row 05, column 04, with plain white text.", + "15d3": "row 05, column 04, with white underlined text.", + "1554": "row 05, column 08, with plain white text.", + "15d5": "row 05, column 08, with white underlined text.", + "15d6": "row 05, column 12, with plain white text.", + "1557": "row 05, column 12, with white underlined text.", + "1558": "row 05, column 16, with plain white text.", + "15d9": "row 05, column 16, with white underlined text.", + "15da": "row 05, column 20, with plain white text.", + "155b": "row 05, column 20, with white underlined text.", + "15dc": "row 05, column 24, with plain white text.", + "155d": "row 05, column 24, with white underlined text.", + "155e": "row 05, column 28, with plain white text.", + "15df": "row 05, column 28, with white underlined text.", + "15e0": "row 06, column 00, with plain white text.", + "1561": "row 06, column 00, with white underlined text.", + "15462": "row 06, column 00, with plain green text.", + "15e3": "row 06, column 00, with green underlined text.", + "1564": "row 06, column 00, with plain blue text.", + "15e5": "row 06, column 00, with blue underlined text.", + "15e6": "row 06, column 00, with plain cyan text.", + "1567": "row 06, column 00, with cyan underlined text.", + "1568": "row 06, column 00, with plain red text.", + "15e9": "row 06, column 00, with red underlined text.", + "15ea": "row 06, column 00, with plain yellow text.", + "156b": "row 06, column 00, with yellow underlined text.", + "15ec": "row 06, column 00, with plain magenta text.", + "156d": "row 06, column 00, with magenta underlined text.", + "156e": "row 06, column 00, with white italicized text.", + "15ef": "row 06, column 00, with white underlined italicized text.", + "1570": "row 06, column 00, with plain white text.", + "15f1": "row 06, column 00, with white underlined text.", + "15f2": "row 06, column 04, with plain white text.", + "1573": "row 06, column 04, with white underlined text.", + "15f4": "row 06, column 08, with plain white text.", + "1575": "row 06, column 08, with white underlined text.", + "1576": "row 06, column 12, with plain white text.", + "15f7": "row 06, column 12, with white underlined text.", + "15f8": "row 06, column 16, with plain white text.", + "1579": "row 06, column 16, with white underlined text.", + "157a": "row 06, column 20, with plain white text.", + "15fb": "row 06, column 20, with white underlined text.", + "15fc": "row 06, column 24, with plain white text.", + "15fd": "row 06, column 24, with white underlined text.", + "15fe": "row 06, column 28, with plain white text.", + "157f": "row 06, column 28, with white underlined text.", + "1640": "row 07, column 00, with plain white text.", + "16c1": "row 07, column 00, with white underlined text.", + "16c2": "row 07, column 00, with plain green text.", + "1643": "row 07, column 00, with green underlined text.", + "16c4": "row 07, column 00, with plain blue text.", + "1645": "row 07, column 00, with blue underlined text.", + "1646": "row 07, column 00, with plain cyan text.", + "16c7": "row 07, column 00, with cyan underlined text.", + "16c8": "row 07, column 00, with plain red text.", + "1649": "row 07, column 00, with red underlined text.", + "164a": "row 07, column 00, with plain yellow text.", + "16cb": "row 07, column 00, with yellow underlined text.", + "164c": "row 07, column 00, with plain magenta text.", + "16cd": "row 07, column 00, with magenta underlined text.", + "16ce": "row 07, column 00, with white italicized text.", + "164f": "row 07, column 00, with white underlined italicized text.", + "16d0": "row 07, column 00, with plain white text.", + "1651": "row 07, column 00, with white underlined text.", + "1652": "row 07, column 04, with plain white text.", + "16d3": "row 07, column 04, with white underlined text.", + "1654": "row 07, column 08, with plain white text.", + "16d5": "row 07, column 08, with white underlined text.", + "16d6": "row 07, column 12, with plain white text.", + "1657": "row 07, column 12, with white underlined text.", + "1658": "row 07, column 16, with plain white text.", + "16d9": "row 07, column 16, with white underlined text.", + "16da": "row 07, column 20, with plain white text.", + "165b": "row 07, column 20, with white underlined text.", + "16dc": "row 07, column 24, with plain white text.", + "165d": "row 07, column 24, with white underlined text.", + "165e": "row 07, column 28, with plain white text.", + "16df": "row 07, column 28, with white underlined text.", + "16e0": "row 08, column 00, with plain white text.", + "1661": "row 08, column 00, with white underlined text.", + "16462": "row 08, column 00, with plain green text.", + "16e3": "row 08, column 00, with green underlined text.", + "1664": "row 08, column 00, with plain blue text.", + "16e5": "row 08, column 00, with blue underlined text.", + "16e6": "row 08, column 00, with plain cyan text.", + "1667": "row 08, column 00, with cyan underlined text.", + "1668": "row 08, column 00, with plain red text.", + "16e9": "row 08, column 00, with red underlined text.", + "16ea": "row 08, column 00, with plain yellow text.", + "166b": "row 08, column 00, with yellow underlined text.", + "16ec": "row 08, column 00, with plain magenta text.", + "166d": "row 08, column 00, with magenta underlined text.", + "166e": "row 08, column 00, with white italicized text.", + "16ef": "row 08, column 00, with white underlined italicized text.", + "1670": "row 08, column 00, with plain white text.", + "16f1": "row 08, column 00, with white underlined text.", + "16f2": "row 08, column 04, with plain white text.", + "1673": "row 08, column 04, with white underlined text.", + "16f4": "row 08, column 08, with plain white text.", + "1675": "row 08, column 08, with white underlined text.", + "1676": "row 08, column 12, with plain white text.", + "16f7": "row 08, column 12, with white underlined text.", + "16f8": "row 08, column 16, with plain white text.", + "1679": "row 08, column 16, with white underlined text.", + "167a": "row 08, column 20, with plain white text.", + "16fb": "row 08, column 20, with white underlined text.", + "16fc": "row 08, column 24, with plain white text.", + "16fd": "row 08, column 24, with white underlined text.", + "16fe": "row 08, column 28, with plain white text.", + "167f": "row 08, column 28, with white underlined text.", + "9740": "row 09, column 00, with plain white text.", + "97c1": "row 09, column 00, with white underlined text.", + "97c2": "row 09, column 00, with plain green text.", + "9743": "row 09, column 00, with green underlined text.", + "97c4": "row 09, column 00, with plain blue text.", + "9745": "row 09, column 00, with blue underlined text.", + "9746": "row 09, column 00, with plain cyan text.", + "97c7": "row 09, column 00, with cyan underlined text.", + "97c8": "row 09, column 00, with plain red text.", + "9749": "row 09, column 00, with red underlined text.", + "974a": "row 09, column 00, with plain yellow text.", + "97cb": "row 09, column 00, with yellow underlined text.", + "974c": "row 09, column 00, with plain magenta text.", + "97cd": "row 09, column 00, with magenta underlined text.", + "97ce": "row 09, column 00, with white italicized text.", + "974f": "row 09, column 00, with white underlined italicized text.", + "97d0": "row 09, column 00, with plain white text.", + "9751": "row 09, column 00, with white underlined text.", + "9752": "row 09, column 04, with plain white text.", + "97d3": "row 09, column 04, with white underlined text.", + "9754": "row 09, column 08, with plain white text.", + "97d5": "row 09, column 08, with white underlined text.", + "97d6": "row 09, column 12, with plain white text.", + "9757": "row 09, column 12, with white underlined text.", + "9758": "row 09, column 16, with plain white text.", + "97d9": "row 09, column 16, with white underlined text.", + "97da": "row 09, column 20, with plain white text.", + "975b": "row 09, column 20, with white underlined text.", + "97dc": "row 09, column 24, with plain white text.", + "975d": "row 09, column 24, with white underlined text.", + "975e": "row 09, column 28, with plain white text.", + "97df": "row 09, column 28, with white underlined text.", + "97e0": "row 10, column 00, with plain white text.", + "9761": "row 10, column 00, with white underlined text.", + "9762": "row 10, column 00, with plain green text.", + "97e3": "row 10, column 00, with green underlined text.", + "9764": "row 10, column 00, with plain blue text.", + "97e5": "row 10, column 00, with blue underlined text.", + "97e6": "row 10, column 00, with plain cyan text.", + "9767": "row 10, column 00, with cyan underlined text.", + "9768": "row 10, column 00, with plain red text.", + "97e9": "row 10, column 00, with red underlined text.", + "97ea": "row 10, column 00, with plain yellow text.", + "976b": "row 10, column 00, with yellow underlined text.", + "97ec": "row 10, column 00, with plain magenta text.", + "976d": "row 10, column 00, with magenta underlined text.", + "976e": "row 10, column 00, with white italicized text.", + "97ef": "row 10, column 00, with white underlined italicized text.", + "9770": "row 10, column 00, with plain white text.", + "97f1": "row 10, column 00, with white underlined text.", + "97f2": "row 10, column 04, with plain white text.", + "9773": "row 10, column 04, with white underlined text.", + "97f4": "row 10, column 08, with plain white text.", + "9775": "row 10, column 08, with white underlined text.", + "9776": "row 10, column 12, with plain white text.", + "97f7": "row 10, column 12, with white underlined text.", + "97f8": "row 10, column 16, with plain white text.", + "9779": "row 10, column 16, with white underlined text.", + "977a": "row 10, column 20, with plain white text.", + "97fb": "row 10, column 20, with white underlined text.", + "97fc": "row 10, column 24, with plain white text.", + "97fd": "row 10, column 24, with white underlined text.", + "97fe": "row 10, column 28, with plain white text.", + "977f": "row 10, column 28, with white underlined text.", + "1040": "row 11, column 00, with plain white text.", + "10c1": "row 11, column 00, with white underlined text.", + "10c2": "row 11, column 00, with plain green text.", + "1043": "row 11, column 00, with green underlined text.", + "10c4": "row 11, column 00, with plain blue text.", + "1045": "row 11, column 00, with blue underlined text.", + "1046": "row 11, column 00, with plain cyan text.", + "10c7": "row 11, column 00, with cyan underlined text.", + "10c8": "row 11, column 00, with plain red text.", + "1049": "row 11, column 00, with red underlined text.", + "104a": "row 11, column 00, with plain yellow text.", + "10cb": "row 11, column 00, with yellow underlined text.", + "104c": "row 11, column 00, with plain magenta text.", + "10cd": "row 11, column 00, with magenta underlined text.", + "10ce": "row 11, column 00, with white italicized text.", + "104f": "row 11, column 00, with white underlined italicized text.", + "10d0": "row 11, column 00, with plain white text.", + "1051": "row 11, column 00, with white underlined text.", + "1052": "row 11, column 04, with plain white text.", + "10d3": "row 11, column 04, with white underlined text.", + "1054": "row 11, column 08, with plain white text.", + "10d5": "row 11, column 08, with white underlined text.", + "10d6": "row 11, column 12, with plain white text.", + "1057": "row 11, column 12, with white underlined text.", + "1058": "row 11, column 16, with plain white text.", + "10d9": "row 11, column 16, with white underlined text.", + "10da": "row 11, column 20, with plain white text.", + "105b": "row 11, column 20, with white underlined text.", + "10dc": "row 11, column 24, with plain white text.", + "105d": "row 11, column 24, with white underlined text.", + "105e": "row 11, column 28, with plain white text.", + "10df": "row 11, column 28, with white underlined text.", + "1340": "row 12, column 00, with plain white text.", + "13c1": "row 12, column 00, with white underlined text.", + "13c2": "row 12, column 00, with plain green text.", + "1343": "row 12, column 00, with green underlined text.", + "13c4": "row 12, column 00, with plain blue text.", + "1345": "row 12, column 00, with blue underlined text.", + "1346": "row 12, column 00, with plain cyan text.", + "13c7": "row 12, column 00, with cyan underlined text.", + "13c8": "row 12, column 00, with plain red text.", + "1349": "row 12, column 00, with red underlined text.", + "134a": "row 12, column 00, with plain yellow text.", + "13cb": "row 12, column 00, with yellow underlined text.", + "134c": "row 12, column 00, with plain magenta text.", + "13cd": "row 12, column 00, with magenta underlined text.", + "13ce": "row 12, column 00, with white italicized text.", + "134f": "row 12, column 00, with white underlined italicized text.", + "13d0": "row 12, column 00, with plain white text.", + "1351": "row 12, column 00, with white underlined text.", + "1352": "row 12, column 04, with plain white text.", + "13d3": "row 12, column 04, with white underlined text.", + "1354": "row 12, column 08, with plain white text.", + "13d5": "row 12, column 08, with white underlined text.", + "13d6": "row 12, column 12, with plain white text.", + "1357": "row 12, column 12, with white underlined text.", + "1358": "row 12, column 16, with plain white text.", + "13d9": "row 12, column 16, with white underlined text.", + "13da": "row 12, column 20, with plain white text.", + "135b": "row 12, column 20, with white underlined text.", + "13dc": "row 12, column 24, with plain white text.", + "135d": "row 12, column 24, with white underlined text.", + "135e": "row 12, column 28, with plain white text.", + "13df": "row 12, column 28, with white underlined text.", + "13e0": "row 13, column 00, with plain white text.", + "1361": "row 13, column 00, with white underlined text.", + "13462": "row 13, column 00, with plain green text.", + "13e3": "row 13, column 00, with green underlined text.", + "1364": "row 13, column 00, with plain blue text.", + "13e5": "row 13, column 00, with blue underlined text.", + "13e6": "row 13, column 00, with plain cyan text.", + "1367": "row 13, column 00, with cyan underlined text.", + "1368": "row 13, column 00, with plain red text.", + "13e9": "row 13, column 00, with red underlined text.", + "13ea": "row 13, column 00, with plain yellow text.", + "136b": "row 13, column 00, with yellow underlined text.", + "13ec": "row 13, column 00, with plain magenta text.", + "136d": "row 13, column 00, with magenta underlined text.", + "136e": "row 13, column 00, with white italicized text.", + "13ef": "row 13, column 00, with white underlined italicized text.", + "1370": "row 13, column 00, with plain white text.", + "13f1": "row 13, column 00, with white underlined text.", + "13f2": "row 13, column 04, with plain white text.", + "1373": "row 13, column 04, with white underlined text.", + "13f4": "row 13, column 08, with plain white text.", + "1375": "row 13, column 08, with white underlined text.", + "1376": "row 13, column 12, with plain white text.", + "13f7": "row 13, column 12, with white underlined text.", + "13f8": "row 13, column 16, with plain white text.", + "1379": "row 13, column 16, with white underlined text.", + "137a": "row 13, column 20, with plain white text.", + "13fb": "row 13, column 20, with white underlined text.", + "13fc": "row 13, column 24, with plain white text.", + "13fd": "row 13, column 24, with white underlined text.", + "13fe": "row 13, column 28, with plain white text.", + "137f": "row 13, column 28, with white underlined text.", + "9440": "row 14, column 00, with plain white text.", + "94c1": "row 14, column 00, with white underlined text.", + "94c2": "row 14, column 00, with plain green text.", + "9443": "row 14, column 00, with green underlined text.", + "94c4": "row 14, column 00, with plain blue text.", + "9445": "row 14, column 00, with blue underlined text.", + "9446": "row 14, column 00, with plain cyan text.", + "94c7": "row 14, column 00, with cyan underlined text.", + "94c8": "row 14, column 00, with plain red text.", + "9449": "row 14, column 00, with red underlined text.", + "944a": "row 14, column 00, with plain yellow text.", + "94cb": "row 14, column 00, with yellow underlined text.", + "944c": "row 14, column 00, with plain magenta text.", + "94cd": "row 14, column 00, with magenta underlined text.", + "94ce": "row 14, column 00, with white italicized text.", + "944f": "row 14, column 00, with white underlined italicized text.", + "94d0": "row 14, column 00, with plain white text.", + "9451": "row 14, column 00, with white underlined text.", + "9452": "row 14, column 04, with plain white text.", + "94d3": "row 14, column 04, with white underlined text.", + "9454": "row 14, column 08, with plain white text.", + "94d5": "row 14, column 08, with white underlined text.", + "94d6": "row 14, column 12, with plain white text.", + "9457": "row 14, column 12, with white underlined text.", + "9458": "row 14, column 16, with plain white text.", + "94d9": "row 14, column 16, with white underlined text.", + "94da": "row 14, column 20, with plain white text.", + "945b": "row 14, column 20, with white underlined text.", + "94dc": "row 14, column 24, with plain white text.", + "945d": "row 14, column 24, with white underlined text.", + "945e": "row 14, column 28, with plain white text.", + "94df": "row 14, column 28, with white underlined text.", + "94e0": "row 15, column 00, with plain white text.", + "9461": "row 15, column 00, with white underlined text.", + "9462": "row 15, column 00, with plain green text.", + "94e3": "row 15, column 00, with green underlined text.", + "9464": "row 15, column 00, with plain blue text.", + "94e5": "row 15, column 00, with blue underlined text.", + "94e6": "row 15, column 00, with plain cyan text.", + "9467": "row 15, column 00, with cyan underlined text.", + "9468": "row 15, column 00, with plain red text.", + "94e9": "row 15, column 00, with red underlined text.", + "94ea": "row 15, column 00, with plain yellow text.", + "946b": "row 15, column 00, with yellow underlined text.", + "94ec": "row 15, column 00, with plain magenta text.", + "946d": "row 15, column 00, with magenta underlined text.", + "946e": "row 15, column 00, with white italicized text.", + "94ef": "row 15, column 00, with white underlined italicized text.", + "9470": "row 15, column 00, with plain white text.", + "94f1": "row 15, column 00, with white underlined text.", + "94f2": "row 15, column 04, with plain white text.", + "9473": "row 15, column 04, with white underlined text.", + "94f4": "row 15, column 08, with plain white text.", + "9475": "row 15, column 08, with white underlined text.", + "9476": "row 15, column 12, with plain white text.", + "94f7": "row 15, column 12, with white underlined text.", + "94f8": "row 15, column 16, with plain white text.", + "9479": "row 15, column 16, with white underlined text.", + "947a": "row 15, column 20, with plain white text.", + "94fb": "row 15, column 20, with white underlined text.", + "94fc": "row 15, column 24, with plain white text.", + "94fd": "row 15, column 24, with white underlined text.", + "94fe": "row 15, column 28, with plain white text.", + "947f": "row 15, column 28, with white underlined text.", + "97a1": "Tab Offset 1 column", + "97a2": "Tab Offset 2 columns", + "9723": "Tab Offset 3 columns", + "94a1": "BackSpace", + "94a4": "Delete to End of Row", + "94ad": "Carriage Return", + "1020": "Background White", + "10a1": "Background Semi-Transparent White", + "10a2": "Background Green", + "1023": "Background Semi-Transparent Green", + "10a4": "Background Blue", + "1025": "Background Semi-Transparent Blue", + "1026": "Background Cyan", + "10a7": "Background Semi-Transparent Cyan", + "10a8": "Background Red", + "1029": "Background Semi-Transparent Red", + "102a": "Background Yellow", + "10ab": "Background Semi-Transparent Yellow", + "102c": "Background Magenta", + "10ad": "Background Semi-Transparent Magenta", + "10ae": "Background Black", + "102f": "Background Semi-Transparent Black", + "97ad": "Background Transparent", + "97a4": "Standard Character Set", + "9725": "Double-Size Character Set", + "9726": "First Private Character Set", + "97a7": "Second Private Character Set", + "97a8": "People`s Republic of China Character Set", + "9729": "Korean Standard Character Set", + "972a": "First Registered Character Set", + "9120": "White", + "91a1": "White Underline", + "91a2": "Green", + "9123": "Green Underline", + "91a4": "Blue", + "9125": "Blue Underline", + "9126": "Cyan", + "91a7": "Cyan Underline", + "91a8": "Red", + "9129": "Red Underline", + "912a": "Yellow", + "91ab": "Yellow Underline", + "912c": "Magenta", + "91ad": "Magenta Underline", + "97ae": "Black", + "972f": "Black Underline", + "91ae": "Italics", + "912f": "Italics Underline", + "94a8": "Flash ON", + "9423": "Alarm Off", + "94a2": "Alarm On" +} + + +def translate_scc(scc_content, brackets='[]'): + """ + Replaces hexadecimal words with their meaning + + In order to make SCC files more human readable and easier to debug, + this function is used to replace command codes with their labels and + character bytes with their actual characters + + :param scc_content: SCC captions to be translated + :type scc_content: str + :param brackets: Brackets to group the translated content of a command + :type brackets: str + :return: Translated SCC captions + :rtype: str + """ + opening_bracket, closing_bracket = brackets if brackets else ('', '') + scc_elements = set(scc_content.split()) + for elem in scc_elements: + name = COMMAND_LABELS.get(elem) + # If a 2 byte command was not found, try retrieving 1 byte characters + if not name: + char1 = ALL_CHARACTERS.get(elem[:2]) + char2 = ALL_CHARACTERS.get(elem[2:]) + if char1 is not None and char2 is not None: + name = f"{char1}{char2}" + if name: + scc_content = scc_content.replace( + elem, f"{opening_bracket}{name}{closing_bracket}") + return scc_content diff --git a/pycaption/scenarist.py b/pycaption/scenarist.py new file mode 100644 index 00000000..458045fe --- /dev/null +++ b/pycaption/scenarist.py @@ -0,0 +1,387 @@ +import os +import tempfile +import zipfile +from collections import OrderedDict +from datetime import timedelta +from io import BytesIO + +from PIL import Image, ImageFont, ImageDraw +import arabic_reshaper +from bidi.algorithm import get_display +from fontTools.ttLib import TTFont +from langcodes import Language, tag_distance + +from pycaption.base import BaseWriter, CaptionSet, Caption, CaptionNode +from pycaption.geometry import UnitEnum, Size + + +def get_sst_pixel_display_params(video_width, video_height): + py0 = 2 + py1 = video_height - 1 + + dx0 = 0 + dy0 = 2 + + dx1 = video_width - 1 + dy1 = video_height - 1 + + return py0, py1, dy0, dy1, dx0, dx1 + + +HEADER = """st_format 2 +SubTitle\tFace_Painting +Tape_Type\t{tape_type} +Display_Start\tnon_forced +Pixel_Area\t({py0} {py1}) +Display_Area\t({dx0} {dy0} {dx1} {dy1}) +Color\t{color} +Contrast\t{contrast} +BG\t({bg_red} {bg_green} {bg_blue} = = =) +PA\t({pa_red} {pa_green} {pa_blue} = = =) +E1\t({e1_red} {e1_green} {e1_blue} = = =) +E2\t({e2_red} {e2_green} {e2_blue} = = =) +directory\tC:\\ +Base_Time\t00:00:00:00 +################################################ +SP_NUMBER START END FILE_NAME +""" + +a = """ +0001 01:00:30:12 01:00:35:08 eng0001.tif +0002 01:00:35:13 01:00:40:07 eng0002.tif +0003 01:00:41:17 01:00:44:08 eng0003.tif +0004 01:00:44:13 01:00:48:02 eng0004.tif + +""" + + +def zipit(path, arch, mode='w'): + archive = zipfile.ZipFile(arch, mode, zipfile.ZIP_DEFLATED) + if os.path.isdir(path): + if not path.endswith('tmp'): + _zippy(path, path, archive) + else: + _, name = os.path.split(path) + archive.write(path, name) + archive.close() + + +def _zippy(base_path, path, archive): + paths = os.listdir(path) + for p in paths: + p = os.path.join(path, p) + if os.path.isdir(p): + _zippy(base_path, p, archive) + else: + archive.write(p, os.path.relpath(p, base_path)) + + +class ScenaristDVDWriter(BaseWriter): + VALID_POSITION = ['top', 'bottom', 'source'] + + paColor = (255, 255, 255) # letter body + e1Color = (190, 190, 190) # antialiasing color + e2Color = (0, 0, 0) # border color + bgColor = (0, 255, 0) # background color + + palette = [paColor, e1Color, e2Color, bgColor] + + palette_image = Image.new("P", (1, 1)) + palette_image.putpalette([*paColor, *e1Color, *e2Color, *bgColor] + [0, 0, 0] * 252) + + font_langs = { + Language.get('en'): {'fontfile': f"{os.path.dirname(__file__)}/NotoSansDisplay-Regular-Note-Math.ttf", + 'align': 'left'}, + Language.get('ru'): {'fontfile': f"{os.path.dirname(__file__)}/NotoSansDisplay-Regular-Note-Math.ttf", + 'align': 'left'}, + Language.get('ar'): {'fontfile': f"{os.path.dirname(__file__)}/NotoSansDisplay-RegularAndArabic.ttf", 'align': 'right'}, + Language.get('ja-JP'): {'fontfile': f"{os.path.dirname(__file__)}/NotoSansJP+Math-Regular.ttf", 'align': 'left'}, + Language.get('zh-TW'): {'fontfile': f"{os.path.dirname(__file__)}/NotoSansTC+Math-Regular.ttf", 'align': 'left'}, + Language.get('zh-CN'): {'fontfile': f"{os.path.dirname(__file__)}/NotoSansSC+Math-Regular.ttf", 'align': 'left'}, + Language.get('ko-KR'): {'fontfile': f"{os.path.dirname(__file__)}/NotoSansKR+Math-Regular.ttf", 'align': 'left'}, + } + + def __init__(self, relativize=True, video_width=720, video_height=480, fit_to_screen=True, tape_type='NON_DROP', + frame_rate=25, compat=False): + super().__init__(relativize, video_width, video_height, fit_to_screen) + self.tape_type = tape_type + self.frame_rate = frame_rate + + if compat: + self.color = '(1 2 3 4)' + self.contrast = '(15 15 15 0)' + else: + self.color = '(0 1 2 3)' + self.contrast = '(7 7 7 7)' + + def get_characters(self, captions): + all_characters = [] + for caption_list in captions: + for caption in caption_list: + all_characters.extend([char for char in caption.get_text() if char and char.strip()]) + unique_characters = list(set(all_characters)) + return unique_characters + + def get_characters_with_captions(self, captions): # -> dict[str, list[int]]: + chars_with_captions = {} + for caption_list in captions: + for caption in caption_list: + current_caption_chars = [char for char in caption.get_text() if char and char.strip()] + for char in current_caption_chars: + if char not in chars_with_captions: + chars_with_captions[char] = [] + chars_with_captions[char].append(caption) + return chars_with_captions + + def get_missing_glyphs(self, font, characters): + ttf_font = TTFont(font) + glyphs = {c: self._has_glyph(ttf_font, c) for c in characters} + + missing_glyphs = {k: v for k, v in glyphs.items() if not v} + + return missing_glyphs + + @staticmethod + def _has_glyph(fnt, glyph): + NOT_ACTUAL_GLYPHS = [ + '\u202A', # Left-to-Right Embedding (LRE) + '\u202B', # Right-to-Left Embedding (RLE) + '\u202C', # Pop Directional Formatting (PDF) + '\u202D', # Left-to-Right Override (LRO) + '\u202E', # Right-to-Left Override (RLO) + '\u200E', # Left-to-Right Mark (LRM) + '\u200F' # Right-to-Left Mark (RLM) + ] + + if glyph in NOT_ACTUAL_GLYPHS: + return True + + for table in fnt['cmap'].tables: + if ord(glyph) in table.cmap.keys(): + return True + + return False + + def get_missing_glyphs_with_timestamps( + self, font, characters_with_timestamps # : dict[str, list[int]] + ): # -> dict[str, list[int]]: + ttf_font = TTFont(font) + + missing_glyphs_with_timestamps = {} + for glyph, timestamps in characters_with_timestamps.items(): + is_glyph_in_font = self._has_glyph(ttf_font, glyph) + if not is_glyph_in_font: + missing_glyphs_with_timestamps[glyph] = timestamps + + return missing_glyphs_with_timestamps + + @staticmethod + def group_captions_by_start_time(caps): + # group captions that have the same start time + caps_start_time = OrderedDict() + for i, cap in enumerate(caps): + if cap.start not in caps_start_time: + caps_start_time[cap.start] = [cap] + else: + caps_start_time[cap.start].append(cap) + + # order by start timestamp + caps_start_time = OrderedDict(sorted(caps_start_time.items(), key=lambda item: item[0])) + return caps_start_time + + def check_overlapping_subs(self, captions_by_start_time): + caps_final = [] + overlapping = [] + for start_time, caps_list in captions_by_start_time.items(): + if len(caps_list) == 1: + caps_final.append(caps_list) + else: + end_times = list(set([c.end for c in caps_list])) + if len(end_times) != 1: + overlapping.append(caps_list) + else: + caps_final.append(caps_list) + return caps_final, overlapping + + def get_distances(self, lang, font_langs): + requested_lang = Language.get(lang) + distances = [ + (tag_distance(requested_lang, l), fnt) + for l, fnt in font_langs.items() + if tag_distance(requested_lang, l) < 100 + ] + if not distances: + return distances + + distances.sort(key=lambda l: l[0]) + return distances + + def write( + self, + caption_set: CaptionSet, + position='bottom', + avoid_same_next_start_prev_end=False, + tiff_compression='tiff_deflate', + ): + if tiff_compression not in ['tiff_deflate', 'raw']: + raise ValueError('Unknown tiff_compression. Supported: {}'.format('tiff_deflate, raw')) + + position = position.lower().strip() + if position not in ScenaristDVDWriter.VALID_POSITION: + raise ValueError('Unknown position. Supported: {}'.format(','.join(ScenaristDVDWriter.VALID_POSITION))) + + lang = caption_set.get_languages().pop() + caps = caption_set.get_captions(lang) + + # group captions that have the same start time + caps_start_time = self.group_captions_by_start_time(caps) + + # check if captions with the same start time also have the same end time + # fail if different end times are found - this is not (yet?) supported + caps_final, overlapping = self.check_overlapping_subs(caps_start_time) + if overlapping: + raise ValueError('Unsupported subtitles - overlapping subtitles with different end times found') + + if avoid_same_next_start_prev_end: + min_diff = (1 / self.frame_rate) * 1000000 + for i, caps_list in enumerate(caps_final): + if i == 0: + continue + + prev_end_time = caps_final[i - 1][0].end + current_start_time = caps_list[0].start + + if (current_start_time == prev_end_time) or ((current_start_time - prev_end_time) < min_diff): + for c in caps_list: + c.start = min(c.start + min_diff, c.end) + + distances = self.get_distances(lang, self.font_langs) + if not distances: + raise ValueError('Cannot find appropriate font for selected language') + + fnt = distances[0][1]['fontfile'] + align = distances[0][1]['align'] + missing_glyphs = self.get_missing_glyphs(fnt, self.get_characters(caps_final)) + + if missing_glyphs: + raise ValueError(f'Selected font was missing glyphs: {" ".join(missing_glyphs.keys())}') + + font_size = 30 + if self.video_width < 500: + font_size = 16 + + print(font_size) + + fnt = ImageFont.truetype(fnt, font_size) + + buf = BytesIO() + with tempfile.TemporaryDirectory() as tmpDir: + with open(tmpDir + '/subtitles.sst', 'w+') as sst: + index = 1 + py0, py1, dy0, dy1, dx0, dx1 = get_sst_pixel_display_params(self.video_width, self.video_height) + sst.write(HEADER.format( + py0=py0, py1=py1, + dx0=dx0, dy0=dy0, dx1=dx1, dy1=dy1, + bg_red=self.bgColor[0], bg_green=self.bgColor[1], bg_blue=self.bgColor[2], + pa_red=self.paColor[0], pa_green=self.paColor[1], pa_blue=self.paColor[2], + e1_red=self.e1Color[0], e1_green=self.e1Color[1], e1_blue=self.e1Color[2], + e2_red=self.e2Color[0], e2_green=self.e2Color[1], e2_blue=self.e2Color[2], + tape_type=self.tape_type, color=self.color, contrast=self.contrast + )) + + for i, cap_list in enumerate(caps_final): + sst.write("%04d %s %s subtitle%04d.tif\n" % ( + index, + self.format_ts(cap_list[0].start), + self.format_ts(cap_list[0].end), + index + )) + + img = Image.new('RGB', (self.video_width, self.video_height), self.bgColor) + draw = ImageDraw.Draw(img) + self.printLine(draw, cap_list, fnt, position, align) + + # quantize the image to our palette + img_quant = img.quantize(palette=self.palette_image, dither=0) + img_quant.save(tmpDir + '/subtitle%04d.tif' % index, compression=tiff_compression) + + index = index + 1 + zipit(tmpDir, buf) + buf.seek(0) + return buf.read() + + def format_ts(self, value): + datetime_value = timedelta(seconds=(int(value / 1000000))) + str_value = str(datetime_value)[:11] + + # make sure all numbers are padded with 0 to two places + str_value = ':'.join([n.zfill(2) for n in str_value.split(':')]) + + str_value = str_value + ':%02d' % (int((int(value / 1000) % 1000) / int(1000 / self.frame_rate))) + return str_value + + def printLine(self, draw: ImageDraw, caption_list: Caption, fnt: ImageFont, position: str = 'bottom', align: str = 'left'): + for caption in caption_list: + text = caption.get_text() + l, t, r, b = draw.textbbox((0, 0), text, font=fnt, align=align) + + x = None + y = None + + # if position is specified as source, get the layout info + # fall back to "bottom" position if we can't get it + if position == 'source': + try: + x_ = caption.layout_info.origin.x + y_ = caption.layout_info.origin.y + + if isinstance(x_, Size) \ + and isinstance(y_, Size) \ + and x_.unit == UnitEnum.PERCENT \ + and y_.unit == UnitEnum.PERCENT: + x = self.video_width * (x_.value / 100) + y = self.video_height * (y_.value / 100) + + # make sure the text doesn't go out of the screen + box_rightmost_edge = x + r + if box_rightmost_edge > self.video_width: + x = float(self.video_width) - float(r) - float(10) + + # padding for readability + if y_.value > 70: + y = y - 10 + else: + position = 'bottom' + except: + position = 'bottom' + + if position != 'source': + x = self.video_width / 2 - r / 2 + if position == 'bottom': + y = self.video_height - b - 10 # padding for readability + elif position == 'top': + y = 10 + else: + raise ValueError('Unknown "position": {}'.format(position)) + + borderColor = self.e2Color + fontColor = self.paColor + for adj in range(2): + # move right + draw.text((x - adj, y), text, font=fnt, fill=borderColor, align=align) + # move left + draw.text((x + adj, y), text, font=fnt, fill=borderColor, align=align) + # move up + draw.text((x, y + adj), text, font=fnt, fill=borderColor, align=align) + # move down + draw.text((x, y - adj), text, font=fnt, fill=borderColor, align=align) + # diagnal left up + draw.text((x - adj, y + adj), text, font=fnt, fill=borderColor, align=align) + # diagnal right up + draw.text((x + adj, y + adj), text, font=fnt, fill=borderColor, align=align) + # diagnal left down + draw.text((x - adj, y - adj), text, font=fnt, fill=borderColor, align=align) + # diagnal right down + draw.text((x + adj, y - adj), text, font=fnt, fill=borderColor, align=align) + + draw.text((x, y), text, font=fnt, fill=fontColor, align=align) diff --git a/pycaption/srt.py b/pycaption/srt.py index 9d9692d4..f69d0d24 100644 --- a/pycaption/srt.py +++ b/pycaption/srt.py @@ -1,12 +1,18 @@ +import os from copy import deepcopy -import six from .base import ( BaseReader, BaseWriter, CaptionSet, CaptionList, Caption, CaptionNode) from .exceptions import CaptionReadNoCaptions, InvalidInputError +import re +from PIL import Image, ImageFont, ImageDraw + class SRTReader(BaseReader): + RE_HTML = re.compile(r'<[^>]+>') + RE_ASS = re.compile(r'{[^}]+}') + def detect(self, content): lines = content.splitlines() if lines[0].isdigit() and '-->' in lines[1]: @@ -14,8 +20,8 @@ def detect(self, content): else: return False - def read(self, content, lang='en-US'): - if type(content) != six.text_type: + def read(self, content, lang='en-US', strip_html=False, strip_ass_tags=False): + if type(content) != str: raise InvalidInputError('The content is not a unicode string.') lines = content.splitlines() @@ -37,7 +43,14 @@ def read(self, content, lang='en-US'): for line in lines[start_line + 2:end_line - 1]: # skip extra blank lines if not nodes or line != '': - nodes.append(CaptionNode.create_text(line)) + txt = line + if strip_html: + txt = SRTReader.RE_HTML.sub('', txt) + + if strip_ass_tags: + txt = SRTReader.RE_ASS.sub('', txt) + + nodes.append(CaptionNode.create_text(txt)) nodes.append(CaptionNode.create_break()) if len(nodes): @@ -57,9 +70,19 @@ def read(self, content, lang='en-US'): def _srttomicro(self, stamp): timesplit = stamp.split(':') + + # TODO(apeterka): add better support for "extended SRT" + # This is a workaround for "extended SRT" format, whose timestamp looks like this: + # 00:00:18,208 --> 00:00:20,792 X1:230 X2:490 Y1:393 Y2:431 + if len(timesplit) > 3: + timesplit = timesplit[0:3] + if ' ' in timesplit[2]: + timesplit[2] = timesplit[2].split(' ')[0] + if ',' not in timesplit[2]: timesplit[2] += ',000' secsplit = timesplit[2].split(',') + microseconds = (int(timesplit[0]) * 3600000000 + int(timesplit[1]) * 60000000 + int(secsplit[0]) * 1000000 + @@ -83,32 +106,42 @@ def _find_text_line(self, start_line, lines): class SRTWriter(BaseWriter): - def write(self, caption_set): + VALID_POSITION = ['top', 'bottom'] + + def write(self, caption_set, position='bottom'): + position = position.lower().strip() + if position not in SRTWriter.VALID_POSITION: + raise ValueError('Unknown position. Supported: {}'.format(','.join(SRTWriter.VALID_POSITION))) + + if position == 'top' and not all([self.video_width, self.video_height]): + raise ValueError('Top position requires video width and height.') + caption_set = deepcopy(caption_set) srt_captions = [] for lang in caption_set.get_languages(): srt_captions.append( - self._recreate_lang(caption_set.get_captions(lang)) + self._recreate_lang(caption_set.get_captions(lang), position) ) caption_content = 'MULTI-LANGUAGE SRT\n'.join(srt_captions) return caption_content - def _recreate_lang(self, captions): + def _recreate_lang(self, captions, position='bottom'): srt = '' count = 1 - for caption in captions: - srt += '%s\n' % count + fnt = ImageFont.truetype(os.path.dirname(__file__) + '/NotoSansDisplay-Regular-Note-Math.ttf', 30) - start = caption.format_start(msec_separator=',') - end = caption.format_end(msec_separator=',') - timestamp = '%s --> %s\n' % (start[:12], end[:12]) - - srt += timestamp.replace('.', ',') + img = None + draw = None + if position == 'top': + img = Image.new('RGB', (self.video_width, self.video_height), (0, 255, 0)) + draw = ImageDraw.Draw(img) + for caption in captions: + # Generate the text new_content = '' for node in caption.nodes: new_content = self._recreate_line(new_content, node) @@ -118,6 +151,27 @@ def _recreate_lang(self, captions): while '\n\n' in new_content: new_content = new_content.replace('\n\n', '\n') + srt += '%s\n' % count + + start = caption.format_start(msec_separator=',') + end = caption.format_end(msec_separator=',') + if position == 'bottom': + # "bottom" is standard (no position info). + # Use the old behavior, output just the timestamp, no coordinates. + timestamp = '%s --> %s\n' % (start[:12], end[:12]) + elif position == 'top': + padding_top = 10 + l, t, r, b = draw.textbbox((0, 0), new_content, font=fnt) + l, t, r, b = draw.textbbox((self.video_width / 2 - r / 2, padding_top), new_content, font=fnt) + x1 = str(round(l)).zfill(3) + x2 = str(round(r)).zfill(3) + y1 = str(round(t)).zfill(3) + y2 = str(round(b)).zfill(3) + timestamp = '%s --> %s X1:%s X2:%s Y1:%s Y2:%s\n' % (start[:12], end[:12], x1, x2, y1, y2) + else: + raise ValueError('Unsupported position: %s' % position) + + srt += timestamp.replace('.', ',') srt += "%s%s" % (new_content, '\n\n') count += 1 diff --git a/pycaption/stl/__init__.py b/pycaption/stl/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pycaption/stl/ebu_stl_reader.py b/pycaption/stl/ebu_stl_reader.py new file mode 100644 index 00000000..041fa972 --- /dev/null +++ b/pycaption/stl/ebu_stl_reader.py @@ -0,0 +1,450 @@ +import codecs +import struct + +import logging + +import unicodedata +from io import BytesIO + +from pycaption.base import BaseReader, Caption, CaptionNode, CaptionSet, CaptionList + +lang_code = { + b"00": "und", + b"16": "smi", + b"01": "sq", + b"17": "la", + b"02": "br", + b"18": "lv", + b"03": "ca", + b"19": "lb", + b"04": "hr", + b"1A": "lt", + b"05": "cy", + b"1B": "hu", + b"06": "cs", + b"1C": "mt", + b"07": "da", + b"1D": "nl", + b"08": "de", + b"1E": "no", + b"09": "en", + b"1F": "oc", + b"0A": "es", + b"20": "pl", + b"0B": "eo", + b"21": "po", + b"0C": "et", + b"22": "ro", + b"0D": "eu", + b"23": "rm", + b"0E": "fo", + b"24": "sr", + b"0F": "fr", + b"25": "sk", + b"10": "fy", + b"26": "sl", + b"11": "ga", + b"27": "fi", + b"12": "gd", + b"28": "sv", + b"13": "gl", + b"29": "sl", + b"14": "is", + b"2A": "nl", + b"15": "it", + b"2B": "wa", + b"7F": "am", + b"69": "ja", + b"53": "sn", + b"7E": "ar", + b"68": "kn", + b"52": "si", + b"7D": "hy", + b"67": "kk", + b"51": "so", + b"7C": "as", + b"66": "km", + b"50": "srn", + b"7B": "az", + b"65": "ko", + b"4F": "sw", + b"7A": "bm", + b"64": "lo", + b"4E": "tg", + b"79": "be", + b"63": "mk", + b"4D": "ta", + b"78": "bn", + b"62": "mg", + b"4C": "tt", + b"77": "bg", + b"61": "zsm", + b"4B": "te", + b"76": "my", + b"60": "mo", + b"4A": "th", + b"75": "cmn-Hant", + b"5F": "mr", + b"49": "uk", + b"74": "cv", + b"5E": "nd", + b"48": "ur", + b"73": "prs", + b"5D": "ne", + b"47": "uz", + b"72": "ff", + b"5C": "ory", + b"46": "vi", + b"71": "ka", + b"5B": "pap", + b"45": "zu", + b"70": "grk", + b"5A": "fa", + b"6F": "gu", + b"59": "pa", + b"6E": "gug", + b"58": "ps", + b"6D": "ha", + b"57": "qu", + b"6C": "he", + b"56": "ru", + b"6B": "hi", + b"55": "rue", + b"6A": "id", + b"54": "sh", +} + + +class iso6937(codecs.Codec): + identical = set(range(0x20, 0x7F)) + identical |= { + 0xA, + 0xA0, + 0xA1, + 0xA2, + 0xA3, + 0xA5, + 0xA7, + 0xAB, + 0xB0, + 0xB1, + 0xB2, + 0xB3, + 0xB5, + 0xB6, + 0xB7, + 0xBB, + 0xBC, + 0xBD, + 0xBE, + 0xBF, + } + direct_mapping = { + 0x8A: 0x000A, # line break + 0xA8: 0x00A4, # ¤ + 0xA9: 0x2018, # ‘ + 0xAA: 0x201C, # “ + 0xAB: 0x00AB, # « + 0xAC: 0x2190, # ← + 0xAD: 0x2191, # ↑ + 0xAE: 0x2192, # → + 0xAF: 0x2193, # ↓ + 0xB4: 0x00D7, # × + 0xB8: 0x00F7, # ÷ + 0xB9: 0x2019, # ’ + 0xBA: 0x201D, # ” + 0xBC: 0x00BC, # ¼ + 0xBD: 0x00BD, # ½ + 0xBE: 0x00BE, # ¾ + 0xBF: 0x00BF, # ¿ + 0xD0: 0x2015, # ― + 0xD1: 0x00B9, # ¹ + 0xD2: 0x00AE, # ® + 0xD3: 0x00A9, # © + 0xD4: 0x2122, # ™ + 0xD5: 0x266A, # ♪ + 0xD6: 0x00AC, # ¬ + 0xD7: 0x00A6, # ¦ + 0xDC: 0x215B, # ⅛ + 0xDD: 0x215C, # ⅜ + 0xDE: 0x215D, # ⅝ + 0xDF: 0x215E, # ⅞ + 0xE0: 0x2126, # Ohm Ω + 0xE1: 0x00C6, # Æ + 0xE2: 0x0110, # Đ + 0xE3: 0x00AA, # ª + 0xE4: 0x0126, # Ħ + 0xE6: 0x0132, # IJ + 0xE7: 0x013F, # Ŀ + 0xE8: 0x0141, # Ł + 0xE9: 0x00D8, # Ø + 0xEA: 0x0152, # Œ + 0xEB: 0x00BA, # º + 0xEC: 0x00DE, # Þ + 0xED: 0x0166, # Ŧ + 0xEE: 0x014A, # Ŋ + 0xEF: 0x0149, # ʼn + 0xF0: 0x0138, # ĸ + 0xF1: 0x00E6, # æ + 0xF2: 0x0111, # đ + 0xF3: 0x00F0, # ð + 0xF4: 0x0127, # ħ + 0xF5: 0x0131, # ı + 0xF6: 0x0133, # ij + 0xF7: 0x0140, # ŀ + 0xF8: 0x0142, # ł + 0xF9: 0x00F8, # ø + 0xFA: 0x0153, # œ + 0xFB: 0x00DF, # ß + 0xFC: 0x00FE, # þ + 0xFD: 0x0167, # ŧ + 0xFE: 0x014B, # ŋ + 0xFF: 0x00AD, # Soft hyphen + } + diacritic = { + 0xC1: 0x0300, # grave accent + 0xC2: 0x0301, # acute accent + 0xC3: 0x0302, # circumflex + 0xC4: 0x0303, # tilde + 0xC5: 0x0304, # macron + 0xC6: 0x0306, # breve + 0xC7: 0x0307, # dot + 0xC8: 0x0308, # umlaut + 0xCA: 0x030A, # ring + 0xCB: 0x0327, # cedilla + 0xCD: 0x030B, # double acute accent + 0xCE: 0x0328, # ogonek + 0xCF: 0x030C, # caron + } + + def decode(self, input, errors="strict"): + output = "" + state = None + count = 0 + for char in input: + # End of a subtitle text + count += 1 + if not state and char in self.identical: + output += chr(char) + elif not state and char in self.direct_mapping: + output += chr(self.direct_mapping[char]) + elif not state and char in self.diacritic: + state = self.diacritic[char] + elif state: + combined = unicodedata.normalize("NFC", chr(char) + chr(state)) + if combined and len(combined) == 1: + output += combined + state = None + return output, len(input) + + def search(name): + if name in ("iso6937", "iso_6937-2", "iso_6937_2"): + return codecs.CodecInfo( + name="iso_6937-2", + encode=iso6937().encode, + decode=iso6937().decode, + ) + + def encode(self, input, errors="strict"): + pass + + +codecs.register(iso6937.search) + + +class STLReader(BaseReader): + """A class that behaves like a file object and reads an STL file""" + + GSIfields = "CPN DFC DSC CCT LC OPT OET TPT TET TN TCD SLR CD RD RN TNB TNS TNG MNC MNR TCS TCP TCF TND DSN CO PUB EN ECD UDA".split( + " " + ) + TTIfields = "SGN SN EBN CS TCIh TCIm TCIs TCIf TCOh TCOm TCOs TCOf VP JC CF TF".split(" ") + + def detect(self, content): + return content[3:11] == b"STL24.01" or content[3:11] == b"STL25.01" or content[3:11] == b"STL30.01" + + def read(self, content, language=None): + self.file = BytesIO(content) + self._readGSI() + + captions = CaptionList() + try: + while True: + captions.append(self._readTTI()) + except StopIteration: + pass + if language: + return CaptionSet({language: captions}) + else: + return CaptionSet({lang_code.get(self.GSI["LC"], "und"): captions}) + + def __bcdTimestampDecode(self, timestamp): + # Special case for people that can't bother to read a spec + if timestamp == b"________": + return 0.0 + + # BCD coded time with limited significant bits as per EBU Tech. 3097-E + safe_bytes = map( + lambda x: x[0] & x[1], zip((0x2, 0xF, 0x7, 0xF, 0x7, 0xF, 0x3, 0xF), struct.unpack("8B", timestamp)) + ) + return sum( + map(lambda x: x[0] * x[1], zip((36000, 3600, 600, 60, 10, 1, 10.0 / self.fps, 1.0 / self.fps), safe_bytes)) + ) + + def _readGSI(self): + self.GSI = dict( + zip( + self.GSIfields, + struct.unpack( + "3s8sc2s2s32s32s32s32s32s32s16s6s6s2s5s5s3s2s2s1s8s8s1s1s3s32s32s32s75x576s", self.file.read(1024) + ), + ) + ) + GSI = self.GSI + logging.debug(GSI) + # self.gsiCodePage = 'cp%s' % GSI['CPN'] + if GSI["DFC"] == b"STL24.01": + self.fps = 24 + elif GSI["DFC"] == b"STL25.01": + self.fps = 25 + elif GSI["DFC"] == b"STL30.01": + self.fps = 30 + else: + raise Exception("Invalid DFC") + self.codePage = { + b"00": "iso_6937-2", + b"01": "iso-8859-5", + b"02": "iso-8859-6", + b"03": "iso-8859-7", + b"04": "iso-8859-8", + }[GSI["CCT"]] + self.numberOfTTI = int(GSI["TNB"]) + if GSI["TCS"] == b"1": + # BCD coded time with limited significant bits + + self.startTime = self.__bcdTimestampDecode(GSI["TCP"]) + else: + self.startTime = 0.0 + logging.debug(self.__dict__) + + def __timecodeDecode(self, h, m, s, f): + return 3600 * h + 60 * m + s + float(f) / self.fps + + def __parseFormatting(self, text, encoding): + colorCodes = [ + "#000000", # black + "#ff0000", # red + "#00ff00", # green + "#ffff00", # yellow + "#0000ff", # blue + "#ff00ff", # magenta + "#00ffff", # cyan + "#ffffff", # white + ] + currentColor = 7 # White is the default color + + first_line = True + nodes = [] + + buffer = b"" + + def drain_buffer(buffer): + nodes.append(CaptionNode.create_text(buffer.decode(encoding))) + return b"" + + for ochar in text: + if ochar == 0x80: + buffer = drain_buffer(buffer) + nodes.append(CaptionNode.create_style(True, {"italics": True})) + elif ochar == 0x81: + buffer = drain_buffer(buffer) + nodes.append(CaptionNode.create_style(False, {"italics": True})) + elif ochar == 0x82: + buffer = drain_buffer(buffer) + nodes.append(CaptionNode.create_style(True, {"underline": True})) + elif ochar == 0x83: + buffer = drain_buffer(buffer) + nodes.append(CaptionNode.create_style(False, {"underline": True})) + elif ochar == 0xE: + buffer = drain_buffer(buffer) + nodes.append(CaptionNode.create_style(True, {"bold": True})) + elif ochar == 0xC: + buffer = drain_buffer(buffer) + nodes.append(CaptionNode.create_style(False, {"bold": True})) + elif ochar in (0, 1, 2, 3, 4, 5, 6, 7, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17): + color = ochar % 0x10 + if color != currentColor: + currentColor = color + buffer = drain_buffer(buffer) + nodes.append(CaptionNode.create_style(True, {"color": colorCodes[currentColor]})) + elif ochar == 0x8A and first_line: + buffer = drain_buffer(buffer) + nodes.append(CaptionNode.create_break()) + first_line = False + elif ochar == 0x8F: + buffer = drain_buffer(buffer) + break + elif (ochar & 0x7F) >= 0x20: + buffer += bytes( + [ + ochar, + ] + ) + + return nodes + + def _readTTI(self): + while True: + tci = None + tco = None + nodes = [] + + while True: + data = self.file.read(128) + if not data: + raise StopIteration() + TTI = dict(zip(self.TTIfields, struct.unpack(" 0 and tci >= 0: + opennodes = [] + for node in reversed(nodes): + if node.type_ == CaptionNode.STYLE: + styletype = node.content[next(iter(node.content.keys()))] + if node.start: + opennodes.append(styletype) + else: + if opennodes[-1] == styletype: + opennodes.pop() + else: + raise Exception( + "STL style overlapping no supported as it would result in something like italicbolditalicbold" + ) + + for opennode in opennodes: + nodes.append(CaptionNode.create_style(False, {opennode: True})) + return Caption(tci * 1000000, tco * 1000000, nodes) + break + + def __iter__(self): + return self + + def __next__(self): + return self._readTTI() + + +if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) + content = open("chernobyl_004_spanish_latin_american_2997.stl", mode="rb").read() + subs = STLReader().read(content) + print(subs) diff --git a/pycaption/webvtt.py b/pycaption/webvtt.py index 11efa8a7..9d6228eb 100644 --- a/pycaption/webvtt.py +++ b/pycaption/webvtt.py @@ -1,5 +1,4 @@ import re -import six import sys import datetime from copy import deepcopy @@ -32,14 +31,12 @@ WEBVTT_VERSION_OF = { HorizontalAlignmentEnum.LEFT: 'left', - HorizontalAlignmentEnum.CENTER: 'middle', + HorizontalAlignmentEnum.CENTER: 'center', HorizontalAlignmentEnum.RIGHT: 'right', HorizontalAlignmentEnum.START: 'start', HorizontalAlignmentEnum.END: 'end' } -DEFAULT_ALIGNMENT = 'middle' - def microseconds(h, m, s, f): """ @@ -60,7 +57,7 @@ def detect(self, content): return 'WEBVTT' in content def read(self, content, lang='en-US'): - if type(content) != six.text_type: + if type(content) != str: raise InvalidInputError('The content is not a unicode string.') caption_set = CaptionSet({lang: self._parse(content.splitlines())}) @@ -88,8 +85,8 @@ def _parse(self, lines): start, end, layout_info = self._parse_timing_line( line, last_start_time) except CaptionReadError as e: - new_message = '%s (line %d)' % (e.args[0], timing_line) - six.reraise(type(e), type(e)(new_message), sys.exc_info()[2]) + e.new_message = '%s (line %d)' % (e.args[0], timing_line) + raise elif '' == line: if found_timing: @@ -206,9 +203,10 @@ class WebVTTWriter(BaseWriter): video_width = None video_height = None - def write(self, caption_set): + def write(self, caption_set, force_hours=False): """ :type caption_set: CaptionSet + @param force_hours: force writing timestamps in full (hh:mm:ss.xxx) even when "hour" is 0 """ output = self.HEADER @@ -230,15 +228,15 @@ def write(self, caption_set): captions = caption_set.get_captions(lang) return output + '\n'.join( - [self._write_caption(caption_set, caption) for caption in captions]) + [self._write_caption(caption_set, caption, force_hours) for caption in captions]) - def _timestamp(self, ts): + def _timestamp(self, ts, force_hours): td = datetime.timedelta(microseconds=ts) mm, ss = divmod(td.seconds, 60) hh, mm = divmod(mm, 60) s = "%02d:%02d.%03d" % (mm, ss, td.microseconds/1000) - if hh: - s = "%d:%s" % (hh, s) + if hh or force_hours: + s = "%02d:%s" % (hh, s) return s def _tags_for_style(self, style): @@ -269,14 +267,14 @@ def _calculate_resulting_style(self, style, caption_set): return resulting_style - def _write_caption(self, caption_set, caption): + def _write_caption(self, caption_set, caption, force_hours): """ :type caption: Caption """ layout_groups = self._layout_groups(caption.nodes, caption_set) - start = self._timestamp(caption.start) - end = self._timestamp(caption.end) + start = self._timestamp(caption.start, force_hours) + end = self._timestamp(caption.end, force_hours) timespan = "{} --> {}".format(start, end) output = '' @@ -372,20 +370,21 @@ def _cue_settings_from(self, layout): # out try: - alignment = WEBVTT_VERSION_OF[layout.alignment.horizontal] + alignment = layout.alignment.horizontal except (AttributeError, KeyError): pass cue_settings = '' - if alignment and alignment != 'middle': - cue_settings += " align:" + alignment + + if alignment and alignment != HorizontalAlignmentEnum.CENTER: + cue_settings += " align:" + WEBVTT_VERSION_OF[alignment] if left_offset: - cue_settings += " position:{},start".format(six.text_type(left_offset)) + cue_settings += " position:{},line-left".format(str(left_offset)) if top_offset: - cue_settings += " line:" + six.text_type(top_offset) + cue_settings += " line:" + str(top_offset) if cue_width: - cue_settings += " size:" + six.text_type(cue_width) + cue_settings += " size:" + str(cue_width) return cue_settings @@ -454,6 +453,11 @@ def _encode(self, s): """ s = s.replace('&', '&') s = s.replace('<', '<') + # revert the tags that are allowed + allowed_tags = ['c', 'i', 'u', 'b'] + for tag in allowed_tags: + s = s.replace(f'<{tag}>', f'<{tag}>') + s = s.replace(f'</{tag}>', f'') # The substring "-->" is also not allowed according to this: # - http://dev.w3.org/html5/webvtt/#dfn-webvtt-cue-block diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..b3b332b5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +beautifulsoup4>=4.8.1 +lxml>=3.2.3 +cssutils>=0.9.10 +Pillow>=7.0.0 +fonttools~=4.33.3 +langcodes~=3.3.0 +arabic-reshaper==3.0.0 +python-bidi==0.4.2 +git+https://github.com/castlabs/striprtf.git@037f05cdf5ba230ae29a6bf629d13cee52da7c3f#egg=striprtf \ No newline at end of file diff --git a/setup.py b/setup.py index e5c84269..b885f7a9 100644 --- a/setup.py +++ b/setup.py @@ -5,25 +5,22 @@ README_PATH = os.path.join( os.path.abspath(os.path.dirname(__file__)), 'README.rst') +with open('requirements.txt') as f: + requirements = f.read().splitlines() +install_requires = [req for req in requirements if not req.startswith('git+')] +install_requires += [req.split('#egg=')[1] + ' @ ' + req for req in requirements if req.startswith('git+')] -dependencies = [ - 'beautifulsoup4>=4.2.1,<4.5.0', - 'lxml>=3.2.3', - 'cssutils>=0.9.10', - 'future', - 'enum34', - 'six>=1.9.0' -] +print(install_requires) setup( name='pycaption', - version='1.1.0', + version='1.2.0', description='Closed caption converter', long_description=open(README_PATH).read(), - author='Joe Norton', - author_email='joey@nortoncrew.com', - url='https://github.com/pbs/pycaption', - install_requires=dependencies, + author='Sebastian Annies', + author_email='sebastian.annies@castlabs.com', + url='https://github.com/castlabs/pycaption', + install_requires=install_requires, packages=find_packages(), include_package_data=True, classifiers=[ diff --git a/test.py b/test.py new file mode 100644 index 00000000..f05ff39a --- /dev/null +++ b/test.py @@ -0,0 +1,9 @@ + +from pycaption.srt import SRTReader +from pycaption.scenarist import ScenaristDVDWriter + + +srtReader = SRTReader() +c = srtReader.read(content=open("a.srt", "rb").read().decode('UTF-8-SIG'), lang='zh-Hans') +w = ScenaristDVDWriter() +w.write(c) diff --git a/tests/ebu1991/test.stl b/tests/ebu1991/test.stl new file mode 100644 index 00000000..2c4041f6 Binary files /dev/null and b/tests/ebu1991/test.stl differ diff --git a/tests/ebu1991/test2.stl b/tests/ebu1991/test2.stl new file mode 100644 index 00000000..f83b2295 Binary files /dev/null and b/tests/ebu1991/test2.stl differ diff --git a/tests/samples/pl_stt.py b/tests/samples/pl_stt.py new file mode 100644 index 00000000..c2989b26 --- /dev/null +++ b/tests/samples/pl_stt.py @@ -0,0 +1,58 @@ +SAMPLE_PL_STT_HEADER = """[HEADER] +SUBTITLING_COMPANY=Pixelogic Media +TIME_FRAME_RATE=24 +TIME_FORMAT=NONDROP +TIME_CONTENT_IN=00:00:00:00 +LANGUAGE=Spanish (Latin) +TITLE=Menu, The +[/HEADER]""" + +SAMPLE_PL_STT_HEADER_NO_FRAMERATE = """[HEADER] +SUBTITLING_COMPANY=Pixelogic Media +TIME_FORMAT=NONDROP +TIME_CONTENT_IN=00:00:00:00 +LANGUAGE=Spanish (Latin) +TITLE=Menu, The +[/HEADER]""" + +SAMPLE_PL_STT_HEADER_WRONG_FORMAT = """[HEADER] +FOOBAR +[/HEADER]""" + +SAMPLE_PL_STT_BODY = """[BODY] +[1] +00:00:50:00 +00:00:54:00 +[CENTER]First caption +With a line break +[2] +00:00:55:00 +00:00:58:12 +[TOP]Second [I]caption[/I], no line break +[3] +00:01:05:00 +00:01:06:12 +[BOTTOM]Third caption +[4] +00:01:06:20 +00:01:09:08 +Three +Line +Caption +[5] +00:01:09:10 +00:01:11:13 +Last caption, +also has a line break +[/BODY]""" + +SAMPLE_PL_STT = f"""{SAMPLE_PL_STT_HEADER} +{SAMPLE_PL_STT_BODY} +""" + +SAMPLE_PL_STT_NO_HEADER = f"""{SAMPLE_PL_STT_BODY} +""" + +SAMPLE_PL_STT_BAD_HEADER_1 = f"""{SAMPLE_PL_STT_HEADER_WRONG_FORMAT} +{SAMPLE_PL_STT_BODY} +""" diff --git a/tests/samples/srt.py b/tests/samples/srt.py index f98b8e96..3f38ebb8 100644 --- a/tests/samples/srt.py +++ b/tests/samples/srt.py @@ -128,4 +128,24 @@ +""" +SRT_ARABIC = """\ +1 +00:00:40,000 --> 00:00:43,250 +‫‎انها‎ ‎مرحلة‎ ‎سوداء،‬ +‫لا‎ ‎شك‎ ‎في‎ ‎ذلك‬ + +2 +00:00:44,542 --> 00:00:49,500 +‫‎لم‎ ‎يواجه‎ ‎عالمنا‎ ABC?"`´ ‎تهديداً‎ ‎خطيراً‬ +‫كما‎ ‎اليوم‬ + +3 +00:00:51,125 --> 00:00:54,417 +‫‎لكن‎ ‎هذا‎ ‎ما‎ ‎أقوله‎ ‎إلى‎ ‎جماعة‬ +‫المواطنين‬ + +4 +00:00:55,292 --> 00:00:58,917 +‫‎سنستمر‎ ‎في‎ ‎خدمتكم‬ """ diff --git a/tests/samples/webvtt.py b/tests/samples/webvtt.py index 228897a5..10f22441 100644 --- a/tests/samples/webvtt.py +++ b/tests/samples/webvtt.py @@ -93,32 +93,32 @@ SAMPLE_WEBVTT_FROM_DFXP_WITH_POSITIONING = """WEBVTT -00:01.000 --> 00:03.000 position:25%,start line:25% size:50% +00:01.000 --> 00:03.000 position:25%,line-left line:25% size:50% You might not remember us. We are a typical transparent region with centered text that has an outline. -00:03.500 --> 00:05.000 align:right position:25%,start line:25% size:50% +00:03.500 --> 00:05.000 align:right position:25%,line-left line:25% size:50% had personality. -00:05.500 --> 00:07.000 align:left position:50%,start line:50% size:25% +00:05.500 --> 00:07.000 align:left position:50%,line-left line:50% size:25% Hello there, children! Have you seen any visitors? -00:07.500 --> 00:09.000 align:right position:25%,start line:75% size:25% +00:07.500 --> 00:09.000 align:right position:25%,line-left line:75% size:25% This is the last cue """ SAMPLE_WEBVTT_FROM_DFXP_WITH_POSITIONING_AND_STYLE = """WEBVTT -00:01.000 --> 00:03.000 position:25%,start line:25% size:50% +00:01.000 --> 00:03.000 position:25%,line-left line:25% size:50% You might not remember us. We are a typical transparent region with centered text that has an outline. -00:03.500 --> 00:05.000 align:right position:25%,start line:25% size:50% +00:03.500 --> 00:05.000 align:right position:25%,line-left line:25% size:50% had personality. -00:05.500 --> 00:07.000 align:left position:50%,start line:50% size:25% +00:05.500 --> 00:07.000 align:left position:50%,line-left line:50% size:25% Hello there, children! Have you seen any visitors? -00:07.500 --> 00:09.000 align:right position:25%,start line:75% size:25% +00:07.500 --> 00:09.000 align:right position:25%,line-left line:75% size:25% This is the last cue """ @@ -206,7 +206,7 @@ 00:01.000 --> 00:02.000 NARRATOR: -00:02.000 --> 00:03.000 position:25%,start line:25% size:75% +00:02.000 --> 00:03.000 position:25%,line-left line:25% size:75% They built the largest, most incredible, wildest, craziest, 00:03.000 --> 00:04.000 @@ -236,7 +236,7 @@ SAMPLE_WEBVTT_FROM_SCC_PROPERLY_WRITES_NEWLINES_OUTPUT = """\ WEBVTT -21:30.033 --> 21:34.033 align:left position:12.5%,start line:86.67% size:87.5% +21:30.033 --> 21:34.033 align:left position:12.5%,line-left line:86.67% size:87.5% aa bb """ diff --git a/tests/test_dfxp.py b/tests/test_dfxp.py index 131b73a0..62967801 100644 --- a/tests/test_dfxp.py +++ b/tests/test_dfxp.py @@ -117,27 +117,6 @@ def test_empty_paragraph(self): except CaptionReadError: self.fail("Failing on empty paragraph") - def test_empty_cue(self): - caption_set = DFXPReader().read( - SAMPLE_DFXP_EMPTY_CUE) - caps = caption_set.get_captions('en-US') - self.assertEquals(caps[1], []) - -SAMPLE_DFXP_EMPTY_CUE = """\ - - - - - - - - -

-

abc

-

-
- -""" SAMPLE_DFXP_INVALID_POSITIONING_VALUE_TEMPLATE = """\ diff --git a/tests/test_dfxp_conversion.py b/tests/test_dfxp_conversion.py index f5935998..4d9774fa 100644 --- a/tests/test_dfxp_conversion.py +++ b/tests/test_dfxp_conversion.py @@ -3,8 +3,6 @@ import unittest from bs4 import BeautifulSoup -from six import text_type -import six from pycaption import ( DFXPReader, DFXPWriter, SRTWriter, SAMIWriter, WebVTTWriter) @@ -51,7 +49,7 @@ class DFXPtoDFXPTestCase(unittest.TestCase, DFXPTestingMixIn): def test_dfxp_to_dfxp_conversion(self): caption_set = DFXPReader().read(SAMPLE_DFXP) results = DFXPWriter().write(caption_set) - self.assertTrue(isinstance(results, text_type)) + self.assertTrue(isinstance(results, str)) self.assertDFXPEquals(SAMPLE_DFXP_OUTPUT, results) def test_default_styling_tag(self): @@ -117,18 +115,10 @@ def test_incorrectly_specified_positioning_is_explicitly_accepted(self): fit_to_screen=False, write_inline_positioning=True).write(caption_set) - if six.PY2: - self.assertDFXPEquals( - result, - SAMPLE_DFXP_INVALID_BUT_SUPPORTED_POSITIONING_OUTPUT - ) - else: - # attributes are sorted differently I guess testing for same - # length is close enough - self.assertDFXPEquals( - result, - SAMPLE_DFXP_INVALID_BUT_SUPPORTED_POSITIONING_OUTPUT - ) + self.assertDFXPEquals( + result, + SAMPLE_DFXP_INVALID_BUT_SUPPORTED_POSITIONING_OUTPUT + ) def test_dont_create_style_tags_with_no_id(self): # The