From 9abd60654148db810550c5b5b52f8a9c867c0a44 Mon Sep 17 00:00:00 2001 From: Nigel Megitt Date: Tue, 23 Nov 2021 13:46:01 +0000 Subject: [PATCH 1/4] wip --- Pipfile | 26 ++++++++++++++++++++++++++ requirements.txt | 1 + 2 files changed, 27 insertions(+) create mode 100644 Pipfile diff --git a/Pipfile b/Pipfile new file mode 100644 index 000000000..6532612c7 --- /dev/null +++ b/Pipfile @@ -0,0 +1,26 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +ipython = "*" +sphinx-autobuild = "*" +sphinxcontrib-plantuml = "*" +recommonmark = "*" +sphinx-rtd-theme = "*" +pytest-bdd = "<4.0.0" +pytest-cov = "*" +pytest-mock = "*" +pytest-catchlog = "*" +pytest-subtests = "*" +pytest-twisted = "*" +coverage = "*" +pytest-runner = "*" +pytest = "*" +mock = "*" +ebu-tt-live = {editable = true,file = "file:///Users/megitn02/Code/bbc/ebu-tt-live-toolkit"} +Sphinx = "*" +Jinja2 = "*" diff --git a/requirements.txt b/requirements.txt index b2029b4db..85eb9e7ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,7 @@ sphinx-rtd-theme pytest-bdd<4.0.0 pytest-cov pytest-mock +pytest-catchlog pytest-subtests pytest-twisted coverage From 89ad293f8912f9ae94e90ddd82e6009e213f9d81 Mon Sep 17 00:00:00 2001 From: Nigel Megitt Date: Fri, 26 Nov 2021 12:09:12 +0000 Subject: [PATCH 2/4] Remove Pipfile and don't need pytest-catchlog --- Pipfile | 26 -------------------------- requirements.txt | 1 - 2 files changed, 27 deletions(-) delete mode 100644 Pipfile diff --git a/Pipfile b/Pipfile deleted file mode 100644 index 6532612c7..000000000 --- a/Pipfile +++ /dev/null @@ -1,26 +0,0 @@ -[[source]] -name = "pypi" -url = "https://pypi.org/simple" -verify_ssl = true - -[dev-packages] - -[packages] -ipython = "*" -sphinx-autobuild = "*" -sphinxcontrib-plantuml = "*" -recommonmark = "*" -sphinx-rtd-theme = "*" -pytest-bdd = "<4.0.0" -pytest-cov = "*" -pytest-mock = "*" -pytest-catchlog = "*" -pytest-subtests = "*" -pytest-twisted = "*" -coverage = "*" -pytest-runner = "*" -pytest = "*" -mock = "*" -ebu-tt-live = {editable = true,file = "file:///Users/megitn02/Code/bbc/ebu-tt-live-toolkit"} -Sphinx = "*" -Jinja2 = "*" diff --git a/requirements.txt b/requirements.txt index 85eb9e7ff..b2029b4db 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,6 @@ sphinx-rtd-theme pytest-bdd<4.0.0 pytest-cov pytest-mock -pytest-catchlog pytest-subtests pytest-twisted coverage From 4232dea1aa9bdb6a55d699dfed8c49f21df31cbd Mon Sep 17 00:00:00 2001 From: Nigel Megitt Date: Tue, 17 Jan 2023 16:11:52 +0000 Subject: [PATCH 3/4] Load EBU-TT-D documents from XML --- ebu_tt_live/bindings/__init__.py | 8 ++++++++ ebu_tt_live/documents/ebuttd.py | 11 +++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/ebu_tt_live/bindings/__init__.py b/ebu_tt_live/bindings/__init__.py index 9be639faf..11743398d 100644 --- a/ebu_tt_live/bindings/__init__.py +++ b/ebu_tt_live/bindings/__init__.py @@ -1539,6 +1539,11 @@ def _validateBinding_vx(self): super(d_tt_type, self)._validateBinding_vx() + def get_timing_type(self, timedelta_in): + if self.timeBase == 'media': + return ebuttdt.FullClockTimingType(timedelta_in) + else: + log.error('d_tt_type.get_timing_type() where self.timeBase == {}'.format(self.timeBase)) raw.d_tt_type._SetSupersedingClass(d_tt_type) @@ -2041,6 +2046,9 @@ def _validateBinding_vx(self): raw.layout: layout, raw.body_type: body_type, }, + 'ebuttd': { + raw.d_tt_type: d_tt_type, + }, } diff --git a/ebu_tt_live/documents/ebuttd.py b/ebu_tt_live/documents/ebuttd.py index ae2ebb84b..ac424d9d1 100644 --- a/ebu_tt_live/documents/ebuttd.py +++ b/ebu_tt_live/documents/ebuttd.py @@ -18,6 +18,7 @@ class EBUTTDDocument(SubtitleDocument, TimelineUtilMixin): _encoding = 'UTF-8' def __init__(self, lang): + self.load_types_for_document() self._ebuttd_content = bindings.ttd( timeBase='media', head=bindings.d_head_type( @@ -46,13 +47,18 @@ def validate(self): document=self ) + @classmethod + def load_types_for_document(cls): + bindings.load_types_for_document('ebuttd') + @classmethod def create_from_xml(cls, xml): # NOTE: This is a workaround to make the bindings accept separate root element identities # for the same name. tt comes in but we rename it to ttd to make the xsd validate. + cls.load_types_for_document() xml_dom = minidom.parseString(xml) - if xml_dom.documentElement.tagName == 'tt': - xml_dom.documentElement.tagName = 'ttd' + if xml_dom.documentElement.tagName == xml_dom.documentElement.prefix + ':tt': + xml_dom.documentElement.tagName = xml_dom.documentElement.prefix + ':ttd' instance = cls.create_from_raw_binding( binding=bindings.CreateFromDOM( xml_dom @@ -62,6 +68,7 @@ def create_from_xml(cls, xml): @classmethod def create_from_raw_binding(cls, binding): + cls.load_types_for_document() instance = cls.__new__(cls) instance._ebuttd_content = binding return instance From 5727b299aa4810de04c8685867fea1364e922121 Mon Sep 17 00:00:00 2001 From: Nigel Megitt Date: Fri, 20 Oct 2023 14:34:10 +0100 Subject: [PATCH 4/4] Test EBU-TT-D <--> XML --- Pipfile | 34 +++++++++++ ebu_tt_live/adapters/document_data.py | 6 +- .../adapters/test/test_data/testEbuttd.xml | 20 +++++++ .../test/test_document_data_adapters.py | 56 +++++++++++++++---- ebu_tt_live/bindings/_ebuttdt.py | 7 ++- ebu_tt_live/bindings/pyxb_utils.py | 2 +- p0bmslf8_gaps.json | 4 ++ package-lock.json | 12 ++-- testing/bdd/conftest.py | 17 ++++++ .../features/ebutt1/ebutt1_conversion.feature | 32 +++++++++++ testing/bdd/templates/ebutt1_template.xml | 3 +- testing/bdd/test_ebutt1_parsing.py | 6 -- 12 files changed, 172 insertions(+), 27 deletions(-) create mode 100644 Pipfile create mode 100644 ebu_tt_live/adapters/test/test_data/testEbuttd.xml create mode 100644 p0bmslf8_gaps.json diff --git a/Pipfile b/Pipfile new file mode 100644 index 000000000..3f07297be --- /dev/null +++ b/Pipfile @@ -0,0 +1,34 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[[source]] +url = "https://pypi.python.org/simple/" +verify_ssl = true +name = "pypipython" + +[packages] +ipython = "*" +sphinx = "*" +sphinx-autobuild = "*" +sphinxcontrib-plantuml = "*" +recommonmark = "*" +sphinx-rtd-theme = "*" +pytest-bdd = "<4.0.0" +pytest-cov = "*" +pytest-mock = "*" +pytest-subtests = "*" +pytest-twisted = "*" +coverage = "*" +pytest-runner = "*" +pytest = "*" +jinja2 = "*" +mock = "*" +importlib-metadata = "<4.3,>2.0.0" +ebu-tt-live = {editable = true, file = "file:///Users/megitn02/Code/bbc/ebu-tt-live-toolkit"} + +[dev-packages] + +[requires] +python_version = "3.7" diff --git a/ebu_tt_live/adapters/document_data.py b/ebu_tt_live/adapters/document_data.py index 34e7caed0..05543b8f5 100644 --- a/ebu_tt_live/adapters/document_data.py +++ b/ebu_tt_live/adapters/document_data.py @@ -73,7 +73,11 @@ class XMLtoEBUTTDAdapter(IDocumentDataAdapter): _provides = EBUTTDDocument def convert_data(self, data, **kwargs): - return EBUTTDDocument.create_from_xml(data), kwargs + doc = EBUTTDDocument.create_from_xml(data) + kwargs.update(dict( + raw_xml=data + )) + return doc, kwargs class EBUTTDtoXMLAdapter(IDocumentDataAdapter): diff --git a/ebu_tt_live/adapters/test/test_data/testEbuttd.xml b/ebu_tt_live/adapters/test/test_data/testEbuttd.xml new file mode 100644 index 000000000..8061a303a --- /dev/null +++ b/ebu_tt_live/adapters/test/test_data/testEbuttd.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + It only took me six days. + + + + diff --git a/ebu_tt_live/adapters/test/test_document_data_adapters.py b/ebu_tt_live/adapters/test/test_document_data_adapters.py index d3b593bb8..c06a214f6 100644 --- a/ebu_tt_live/adapters/test/test_document_data_adapters.py +++ b/ebu_tt_live/adapters/test/test_document_data_adapters.py @@ -136,11 +136,49 @@ def test_sequence_id_mismatch(self): class TestXMLtoEBUTTDAdapter(TestCase): - _output_type = documents.EBUTTDDocument _adapter_class = document_data.XMLtoEBUTTDAdapter - _expected_keys = [] + _test_xml_file = 'testEbuttd.xml' + _test_data_dir_path = os.path.join(os.path.dirname(__file__), 'test_data') + _test_xml_path = os.path.join(_test_data_dir_path, _test_xml_file) + _output_type = documents.EBUTTDDocument + _expected_keys = [ + 'raw_xml' + ] + instance = None + + def setUp(self): + self.instance = self._adapter_class() + self.assertIsInstance(self.instance, IDocumentDataAdapter) - # TODO: Finish this once we have EBUTT-D parsing + def _assert_output_type(self, result): + self.assertIsInstance(result, self._output_type) + + def _assert_kwargs_passtrough(self, result_kwargs, expected_keys): + self.assertEqual(set(result_kwargs.keys()), set(expected_keys)) + + def _get_xml(self): + with open(self._test_xml_path, 'r') as xml_file: + xml_data = xml_file.read() + return xml_data + + def _get_input(self): + return self._get_xml() + + def test_success(self): + expected_keys = [] + expected_keys.extend(self._expected_keys) + result, res_kwargs = self.instance.convert_data(self._get_input()) + self._assert_output_type(result) + self._assert_kwargs_passtrough(res_kwargs, expected_keys) + + def test_kwargs_passthrough(self): + in_kwargs = { + 'foo': 'bar' + } + expected_keys = ['foo'] + expected_keys.extend(self._expected_keys) + result, res_kwargs = self.instance.convert_data(self._get_input(), **in_kwargs) + self._assert_kwargs_passtrough(res_kwargs, expected_keys) class TestEBUTT3toXMLAdapter(TestXMLtoEBUTT3Adapter): @@ -164,20 +202,18 @@ def test_sequence_id_match(self): pass -class TestEBUTTDtoXMLAdapter(TestEBUTT3toXMLAdapter): +class TestEBUTTDtoXMLAdapter(TestXMLtoEBUTTDAdapter): + _output_type = six.text_type _adapter_class = document_data.EBUTTDtoXMLAdapter _expected_keys = [] + def _get_input(self): + return documents.EBUTTDDocument.create_from_xml(self._get_xml()) + def _get_input(self): input_doc = documents.EBUTTDDocument(lang='en-GB') return input_doc - def test_sequence_id_mismatch(self): - pass - - def test_sequence_id_match(self): - pass - class TestEBUTT3toEBUTTDAdapter(TestXMLtoEBUTT3Adapter): _adapter_class = document_data.EBUTT3toEBUTTDAdapter diff --git a/ebu_tt_live/bindings/_ebuttdt.py b/ebu_tt_live/bindings/_ebuttdt.py index 000f61fd8..b4223e2e5 100644 --- a/ebu_tt_live/bindings/_ebuttdt.py +++ b/ebu_tt_live/bindings/_ebuttdt.py @@ -68,8 +68,11 @@ def _ConvertArguments_vx(cls, args, kw): context = get_xml_parsing_context() if context is not None: # This means we are in XML parsing context. There should be a timeBase and a timing_attribute_name in the - # context object. - time_base = context['timeBase'] + # context object. But if there's no timeBase, in the context + # of EBU-TT-D, we will assume media. Some files in the wild + # trigger this behaviour, for reasons not yet identified, i.e. + # we somehow get here without having a timeBase context set. + time_base = context.get('timeBase', 'media') # It is possible for a timing type to exist as the value of an element not an attribute, # in which case no timing_attribute_name is in the context; in that case don't attempt # to validate the data against a timebase. At the moment this only affects the diff --git a/ebu_tt_live/bindings/pyxb_utils.py b/ebu_tt_live/bindings/pyxb_utils.py index 2fcfba8cb..8bcfa627d 100644 --- a/ebu_tt_live/bindings/pyxb_utils.py +++ b/ebu_tt_live/bindings/pyxb_utils.py @@ -23,7 +23,7 @@ def get_xml_parsing_context(): into account the timeBase attribute on the tt element. In that case when the timeBase element is encountered by the parser is is added to the parsing context object to help PyXB make the right type in the timingType union. - :return: dict that is te parsing context for the currently running parser + :return: dict that is the parsing context for the currently running parser :return: None if not in parsing mode """ log.debug('Accessing xml_parsing_context: {}'.format(__xml_parsing_context)) diff --git a/p0bmslf8_gaps.json b/p0bmslf8_gaps.json new file mode 100644 index 000000000..cf50b65e6 --- /dev/null +++ b/p0bmslf8_gaps.json @@ -0,0 +1,4 @@ +> /Users/megitn02/Code/bbc/ebu-tt-live-toolkit/ebu_tt_live/scripts/get_times_from_ebuttd.py(1)() +-> from ebu_tt_live.documents import EBUTTDDocument +(Pdb) --KeyboardInterrupt-- +(Pdb) \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index e4c235073..a86532859 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,9 +27,9 @@ } }, "node_modules/nunjucks": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/nunjucks/-/nunjucks-3.2.3.tgz", - "integrity": "sha512-psb6xjLj47+fE76JdZwskvwG4MYsQKXUtMsPh6U0YMvmyjRtKRFcxnlXGWglNybtNTNVmGdp94K62/+NjF5FDQ==", + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/nunjucks/-/nunjucks-3.2.4.tgz", + "integrity": "sha512-26XRV6BhkgK0VOxfbU5cQI+ICFUtMLixv1noZn1tGU38kQH5A5nmmbk/O45xdyBhD1esk47nKrY0mvQpZIhRjQ==", "dependencies": { "a-sync-waterfall": "^1.0.0", "asap": "^2.0.3", @@ -68,9 +68,9 @@ "integrity": "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==" }, "nunjucks": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/nunjucks/-/nunjucks-3.2.3.tgz", - "integrity": "sha512-psb6xjLj47+fE76JdZwskvwG4MYsQKXUtMsPh6U0YMvmyjRtKRFcxnlXGWglNybtNTNVmGdp94K62/+NjF5FDQ==", + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/nunjucks/-/nunjucks-3.2.4.tgz", + "integrity": "sha512-26XRV6BhkgK0VOxfbU5cQI+ICFUtMLixv1noZn1tGU38kQH5A5nmmbk/O45xdyBhD1esk47nKrY0mvQpZIhRjQ==", "requires": { "a-sync-waterfall": "^1.0.0", "asap": "^2.0.3", diff --git a/testing/bdd/conftest.py b/testing/bdd/conftest.py index a87922008..8e0457323 100644 --- a/testing/bdd/conftest.py +++ b/testing/bdd/conftest.py @@ -16,6 +16,18 @@ import os import unittest + +# if os.getenv('_PYTEST_RAISE', "0") != "0": +# +# @pytest.hookimpl(tryfirst=True) +# def pytest_exception_interact(call): +# raise call.excinfo.value +# +# @pytest.hookimpl(tryfirst=True) +# def pytest_internalerror(excinfo): +# raise excinfo.value + + denester_node = DenesterNode( node_id = "denester_node", sequence_identifier = "denestedSequenceT" @@ -221,6 +233,11 @@ def when_converter_set_to_use_fixed_offset_smpte_converter(test_context): else: print('tried making a FixedOffsetSMPTEConvverter but document timebase was not SMPTE') +@when('the document\'s timeBase is set to ') +def when_document_timebase(template_dict, timebase): + # timeBase in ebutt1_template.xml is 'media' by default + template_dict['timeBase'] = timebase + @when('the EBU-TT-1 document is converted to EBU-TT-Live') def when_ebutt1_converted_to_ebutt3(test_context, template_file, template_dict): use_doc_id_as_seq_id = test_context.get('use_doc_id_as_seq_id', False) diff --git a/testing/bdd/features/ebutt1/ebutt1_conversion.feature b/testing/bdd/features/ebutt1/ebutt1_conversion.feature index cac3e25fb..8b41d6052 100644 --- a/testing/bdd/features/ebutt1/ebutt1_conversion.feature +++ b/testing/bdd/features/ebutt1/ebutt1_conversion.feature @@ -55,3 +55,35 @@ Feature: Converting EBU-TT Part 1 files And the EBU-TT-1 document is converted to EBU-TT-Live Then the EBU-TT-Live document is valid And the sequenceIdentifier is "TestConverter" + + Scenario: Pass conversion check with non-smpte timebase + Given an xml file + When the document contains a "styling" element + And the document contains a "style" element + And the document contains a "layout" element + And the document contains a "region" element + And the document's timeBase is set to + And the XML is parsed as a valid EBU-TT-1 document + And the EBU-TT-1 document is converted to EBU-TT-Live + Then the EBU-TT-Live document is valid + + Examples: + | timebase | + | media | + | clock | + + Scenario: Pass conversion check with smpte timebase + Given an xml file + When the document contains a "styling" element + And the document contains a "style" element + And the document contains a "layout" element + And the document contains a "region" element + And the document's timeBase is set to + And the XML is parsed as a valid EBU-TT-1 document + And the EBU-TT-1 converter is set to use a FixedOffsetSMPTEConverter + And the EBU-TT-1 document is converted to EBU-TT-Live + Then the EBU-TT-Live document is valid + + Examples: + | timebase | + | smpte | diff --git a/testing/bdd/templates/ebutt1_template.xml b/testing/bdd/templates/ebutt1_template.xml index 80c065597..6449184c7 100644 --- a/testing/bdd/templates/ebutt1_template.xml +++ b/testing/bdd/templates/ebutt1_template.xml @@ -4,6 +4,7 @@ ttp:timeBase="{{timeBase}}" {% if timeBase == 'smpte' %} ttp:frameRate="25" + ttp:frameRateMultiplier="1 1" ttp:dropMode="nonDrop" ttp:markerMode="continuous" {% elif timeBase == 'clock' %} @@ -55,7 +56,7 @@ Some example text... - And another line + And another line diff --git a/testing/bdd/test_ebutt1_parsing.py b/testing/bdd/test_ebutt1_parsing.py index 66d3bf79e..47f75afc8 100644 --- a/testing/bdd/test_ebutt1_parsing.py +++ b/testing/bdd/test_ebutt1_parsing.py @@ -16,12 +16,6 @@ def when_document_body_contains_attribute(template_dict, attribute): template_dict[attribute] = True -@when('the document\'s timeBase is set to ') -def when_document_timebase(template_dict, timebase): - # timeBase in ebutt1_template.xml is 'media' by default - template_dict['timeBase'] = timebase - - @when('the document contains an ebuttp attribute ') def when_document_contains_ebuttp_attribute(template_dict, attribute): template_dict['ebuttp'] = True