Issue 0062 support reading ebuttd

Pipfile

@@ -0,0 +1,34 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[[source]]
+url = "https://pypi.python.org/simple/"
+verify_ssl = true
+name = "pypipython"
+
+[packages]
+ipython = "*"
+sphinx = "*"
+sphinx-autobuild = "*"
+sphinxcontrib-plantuml = "*"
+recommonmark = "*"
+sphinx-rtd-theme = "*"
+pytest-bdd = "<4.0.0"
+pytest-cov = "*"
+pytest-mock = "*"
+pytest-subtests = "*"
+pytest-twisted = "*"
+coverage = "*"
+pytest-runner = "*"
+pytest = "*"
+jinja2 = "*"
+mock = "*"
+importlib-metadata = "<4.3,>2.0.0"
+ebu-tt-live = {editable = true, file = "file:///Users/megitn02/Code/bbc/ebu-tt-live-toolkit"}
+
+[dev-packages]
+
+[requires]
+python_version = "3.7"

ebu_tt_live/adapters/document_data.py

@@ -73,7 +73,11 @@ class XMLtoEBUTTDAdapter(IDocumentDataAdapter):
     _provides = EBUTTDDocument
 
     def convert_data(self, data, **kwargs):
-        return EBUTTDDocument.create_from_xml(data), kwargs
+        doc = EBUTTDDocument.create_from_xml(data)
+        kwargs.update(dict(
+            raw_xml=data
+        ))
+        return doc, kwargs
 
 
 class EBUTTDtoXMLAdapter(IDocumentDataAdapter):

ebu_tt_live/adapters/test/test_data/testEbuttd.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<tt:tt ttp:timeBase="media" xml:lang="en-GB" xmlns:ebuttm="urn:ebu:tt:metadata" xmlns:ebuttp="urn:ebu:tt:parameters" xmlns:tt="http://www.w3.org/ns/ttml" xmlns:ttp="http://www.w3.org/ns/ttml#parameter" xmlns:tts="http://www.w3.org/ns/ttml#styling" xmlns:xml="http://www.w3.org/XML/1998/namespace">
+    <tt:head>
+        <tt:metadata>
+            <ebuttm:documentMetadata/>
+        </tt:metadata>
+        <tt:styling>
+            <tt:style xml:id="s0"/>
+        </tt:styling>
+        <tt:layout>
+            <tt:region xml:id="r0" tts:origin="10% 10%" tts:extent="80% 80%"></tt:region>
+        </tt:layout>
+    </tt:head>
+    <tt:body>
+        <tt:div>
+            <tt:p xml:id="ID001" begin="01:23:45.670" end="01:23:45.890">It only took me six days.</tt:p>
+        </tt:div>
+    </tt:body>
+</tt:tt>
+

ebu_tt_live/adapters/test/test_document_data_adapters.py

@@ -136,11 +136,49 @@ def test_sequence_id_mismatch(self):
 
 
 class TestXMLtoEBUTTDAdapter(TestCase):
-    _output_type = documents.EBUTTDDocument
     _adapter_class = document_data.XMLtoEBUTTDAdapter
-    _expected_keys = []
+    _test_xml_file = 'testEbuttd.xml'
+    _test_data_dir_path = os.path.join(os.path.dirname(__file__), 'test_data')
+    _test_xml_path = os.path.join(_test_data_dir_path, _test_xml_file)
+    _output_type = documents.EBUTTDDocument
+    _expected_keys = [
+        'raw_xml'
+    ]
+    instance = None
+
+    def setUp(self):
+        self.instance = self._adapter_class()
+        self.assertIsInstance(self.instance, IDocumentDataAdapter)
 
-    # TODO: Finish this once we have EBUTT-D parsing
+    def _assert_output_type(self, result):
+        self.assertIsInstance(result, self._output_type)
+
+    def _assert_kwargs_passtrough(self, result_kwargs, expected_keys):
+        self.assertEqual(set(result_kwargs.keys()), set(expected_keys))
+
+    def _get_xml(self):
+        with open(self._test_xml_path, 'r') as xml_file:
+            xml_data = xml_file.read()
+        return xml_data
+
+    def _get_input(self):
+        return self._get_xml()
+
+    def test_success(self):
+        expected_keys = []
+        expected_keys.extend(self._expected_keys)
+        result, res_kwargs = self.instance.convert_data(self._get_input())
+        self._assert_output_type(result)
+        self._assert_kwargs_passtrough(res_kwargs, expected_keys)
+
+    def test_kwargs_passthrough(self):
+        in_kwargs = {
+            'foo': 'bar'
+        }
+        expected_keys = ['foo']
+        expected_keys.extend(self._expected_keys)
+        result, res_kwargs = self.instance.convert_data(self._get_input(), **in_kwargs)
+        self._assert_kwargs_passtrough(res_kwargs, expected_keys)
 
 
 class TestEBUTT3toXMLAdapter(TestXMLtoEBUTT3Adapter):
@@ -164,20 +202,18 @@ def test_sequence_id_match(self):
         pass
 
 
-class TestEBUTTDtoXMLAdapter(TestEBUTT3toXMLAdapter):
+class TestEBUTTDtoXMLAdapter(TestXMLtoEBUTTDAdapter):
+    _output_type = six.text_type
     _adapter_class = document_data.EBUTTDtoXMLAdapter
     _expected_keys = []
 
+    def _get_input(self):
+        return documents.EBUTTDDocument.create_from_xml(self._get_xml())
+
     def _get_input(self):
         input_doc = documents.EBUTTDDocument(lang='en-GB')
         return input_doc
 
-    def test_sequence_id_mismatch(self):
-        pass
-
-    def test_sequence_id_match(self):
-        pass
-
 
 class TestEBUTT3toEBUTTDAdapter(TestXMLtoEBUTT3Adapter):
     _adapter_class = document_data.EBUTT3toEBUTTDAdapter

ebu_tt_live/bindings/__init__.py

@@ -1539,6 +1539,11 @@ def _validateBinding_vx(self):
 
         super(d_tt_type, self)._validateBinding_vx()
 
+    def get_timing_type(self, timedelta_in):
+        if self.timeBase == 'media':
+            return ebuttdt.FullClockTimingType(timedelta_in)
+        else:
+            log.error('d_tt_type.get_timing_type() where self.timeBase == {}'.format(self.timeBase))
 
 raw.d_tt_type._SetSupersedingClass(d_tt_type)
 
@@ -2041,6 +2046,9 @@ def _validateBinding_vx(self):
         raw.layout: layout,
         raw.body_type: body_type,
         },
+    'ebuttd': {
+        raw.d_tt_type: d_tt_type,
+        },
     }
 
 

ebu_tt_live/bindings/_ebuttdt.py

@@ -68,8 +68,11 @@ def _ConvertArguments_vx(cls, args, kw):
         context = get_xml_parsing_context()
         if context is not None:
             # This means we are in XML parsing context. There should be a timeBase and a timing_attribute_name in the
-            # context object.
-            time_base = context['timeBase']
+            # context object. But if there's no timeBase, in the context
+            # of EBU-TT-D, we will assume media. Some files in the wild
+            # trigger this behaviour, for reasons not yet identified, i.e.
+            # we somehow get here without having a timeBase context set.
+            time_base = context.get('timeBase', 'media')
             # It is possible for a timing type to exist as the value of an element not an attribute,
             # in which case no timing_attribute_name is in the context; in that case don't attempt
             # to validate the data against a timebase. At the moment this only affects the

ebu_tt_live/bindings/pyxb_utils.py

@@ -23,7 +23,7 @@ def get_xml_parsing_context():
     into account the timeBase attribute on the tt element. In that case when the timeBase element is encountered by the
     parser is is added to the parsing context object to help PyXB make the right type in the timingType union.
 
-    :return: dict that is te parsing context for the currently running parser
+    :return: dict that is the parsing context for the currently running parser
     :return: None if not in parsing mode
     """
     log.debug('Accessing xml_parsing_context: {}'.format(__xml_parsing_context))

ebu_tt_live/documents/ebuttd.py

@@ -18,6 +18,7 @@ class EBUTTDDocument(SubtitleDocument, TimelineUtilMixin):
     _encoding = 'UTF-8'
 
     def __init__(self, lang):
+        self.load_types_for_document()
         self._ebuttd_content = bindings.ttd(
             timeBase='media',
             head=bindings.d_head_type(
@@ -46,13 +47,18 @@ def validate(self):
             document=self
         )
 
+    @classmethod
+    def load_types_for_document(cls):
+        bindings.load_types_for_document('ebuttd')
+
     @classmethod
     def create_from_xml(cls, xml):
         # NOTE: This is a workaround to make the bindings accept separate root element identities
         # for the same name. tt comes in but we rename it to ttd to make the xsd validate.
+        cls.load_types_for_document()
         xml_dom = minidom.parseString(xml)
-        if xml_dom.documentElement.tagName == 'tt':
-            xml_dom.documentElement.tagName = 'ttd'
+        if xml_dom.documentElement.tagName == xml_dom.documentElement.prefix + ':tt':
+            xml_dom.documentElement.tagName = xml_dom.documentElement.prefix +  ':ttd'
         instance = cls.create_from_raw_binding(
             binding=bindings.CreateFromDOM(
                 xml_dom
@@ -62,6 +68,7 @@ def create_from_xml(cls, xml):
 
     @classmethod
     def create_from_raw_binding(cls, binding):
+        cls.load_types_for_document()
         instance = cls.__new__(cls)
         instance._ebuttd_content = binding
         return instance

p0bmslf8_gaps.json

@@ -0,0 +1,4 @@
+> /Users/megitn02/Code/bbc/ebu-tt-live-toolkit/ebu_tt_live/scripts/get_times_from_ebuttd.py(1)<module>()
+-> from ebu_tt_live.documents import EBUTTDDocument
+(Pdb) --KeyboardInterrupt--
+(Pdb) 

testing/bdd/conftest.py

@@ -16,6 +16,18 @@
 import os
 import unittest
 
+
+# if os.getenv('_PYTEST_RAISE', "0") != "0":
+#
+#     @pytest.hookimpl(tryfirst=True)
+#     def pytest_exception_interact(call):
+#         raise call.excinfo.value
+#
+#     @pytest.hookimpl(tryfirst=True)
+#     def pytest_internalerror(excinfo):
+#         raise excinfo.value
+
+
 denester_node = DenesterNode(
     node_id = "denester_node",
     sequence_identifier = "denestedSequenceT"
@@ -221,6 +233,11 @@ def when_converter_set_to_use_fixed_offset_smpte_converter(test_context):
     else:
         print('tried making a FixedOffsetSMPTEConvverter but document timebase was not SMPTE')
 
+@when('the document\'s timeBase is set to <timebase>')
+def when_document_timebase(template_dict, timebase):
+    # timeBase in ebutt1_template.xml is 'media' by default
+    template_dict['timeBase'] = timebase
+
 @when('the EBU-TT-1 document is converted to EBU-TT-Live')
 def when_ebutt1_converted_to_ebutt3(test_context, template_file, template_dict):
     use_doc_id_as_seq_id = test_context.get('use_doc_id_as_seq_id', False)

testing/bdd/features/ebutt1/ebutt1_conversion.feature

@@ -55,3 +55,35 @@ Feature: Converting EBU-TT Part 1 files
     And the EBU-TT-1 document is converted to EBU-TT-Live
     Then the EBU-TT-Live document is valid
     And the sequenceIdentifier is "TestConverter"
+
+  Scenario: Pass conversion check with non-smpte timebase
+    Given an xml file <xml_file>
+    When the document contains a "styling" element
+    And the document contains a "style" element
+    And the document contains a "layout" element
+    And the document contains a "region" element
+    And the document's timeBase is set to <timebase>
+    And the XML is parsed as a valid EBU-TT-1 document
+    And the EBU-TT-1 document is converted to EBU-TT-Live
+    Then the EBU-TT-Live document is valid
+
+    Examples:
+      | timebase |
+      | media    |
+      | clock    |
+
+  Scenario: Pass conversion check with smpte timebase
+    Given an xml file <xml_file>
+    When the document contains a "styling" element
+    And the document contains a "style" element
+    And the document contains a "layout" element
+    And the document contains a "region" element
+    And the document's timeBase is set to <timebase>
+    And the XML is parsed as a valid EBU-TT-1 document
+    And the EBU-TT-1 converter is set to use a FixedOffsetSMPTEConverter
+    And the EBU-TT-1 document is converted to EBU-TT-Live
+    Then the EBU-TT-Live document is valid
+
+    Examples:
+      | timebase |
+      | smpte    |

testing/bdd/templates/ebutt1_template.xml

@@ -4,6 +4,7 @@
           ttp:timeBase="{{timeBase}}"
           {% if timeBase == 'smpte' %}
             ttp:frameRate="25"
+            ttp:frameRateMultiplier="1 1"
             ttp:dropMode="nonDrop"
             ttp:markerMode="continuous"
           {% elif timeBase == 'clock' %}
@@ -55,7 +56,7 @@
       <tt:p begin="{% if timeBase == 'smpte' %}00:00:00:12{% else %}500ms{% endif %}" end="{% if timeBase == 'smpte' %}00:00:03:12{% else %}3420ms{% endif %}" xml:id="ID005">
         <tt:span>Some example text...</tt:span>
         <tt:br/>
-        <tt:span>And another line</tt:span>
+        <tt:span {% if timeBase == 'smpte' %} begin="00:00:01:04" end="00:00:03:20" {% endif %}>And another line</tt:span>
       </tt:p>
     </tt:div>
   </tt:body>

testing/bdd/test_ebutt1_parsing.py

@@ -16,12 +16,6 @@ def when_document_body_contains_attribute(template_dict, attribute):
     template_dict[attribute] = True
 
 
-@when('the document\'s timeBase is set to <timebase>')
-def when_document_timebase(template_dict, timebase):
-    # timeBase in ebutt1_template.xml is 'media' by default
-    template_dict['timeBase'] = timebase
-
-
 @when('the document contains an ebuttp attribute <attribute>')
 def when_document_contains_ebuttp_attribute(template_dict, attribute):
     template_dict['ebuttp'] = True