diff --git a/.gitignore b/.gitignore index 35b7a77b..37c20cc9 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,5 @@ src/*.egg-info *.swp src/tests/pipeline/mtr-build gg-iss-16 +gg-iss-5 +bm3d-build diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 19c66926..173fe924 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -133,7 +133,6 @@ pypi: - cd $GiftGrab_venv - source bin/activate - pip install pytest - - pip install pyyaml - ls -alh ./* - PyPI_INSTALLER="../$GiftGrab_PyPI_DIST_DIR/$(ls ../$GiftGrab_PyPI_DIST_DIR | grep tar.gz)" - if [ -z "$PyPI_INSTALLER" ]; then exit 1; fi @@ -173,6 +172,10 @@ pypi: - pip install numpy - $TEST_LAUNCHER numpy bgra; exit_on_fail - $TEST_LAUNCHER numpy i420; exit_on_fail + # Runt stereo frame tests + - $TEST_LAUNCHER stereo bgra; exit_on_fail + - $TEST_LAUNCHER stereo i420; exit_on_fail + - $TEST_LAUNCHER stereo uyvy; exit_on_fail # test support for video files - pip install -vvv --install-option="--hevc" --install-option="--enable-nonfree" --install-option="--nvenc" --install-option="--xvid" --install-option="--vp9" --install-option="--numpy" --install-option="--files" --upgrade "$PyPI_INSTALLER" - $TEST_LAUNCHER decode hevc bgra; exit_on_fail @@ -430,8 +433,9 @@ pypi-blackmagic-decklink-sdi-4k: - PyPI_INSTALLER="../$GiftGrab_PyPI_DIST_DIR/$(ls ../$GiftGrab_PyPI_DIST_DIR | grep tar.gz)" - if [ -z "$PyPI_INSTALLER" ]; then exit 1; fi # because pip seems to be getting confused about install options sometimes: - - pip install PyYAML - pip install pytest + - pip install numpy + - pip install scipy - pip install -vvv --install-option="--blackmagic-decklink-sdi-4k" --install-option="--enable-nonfree" --upgrade "$PyPI_INSTALLER" # run tests - $TEST_LAUNCHER blackmagic-decklinksdi4k uyvy; exit_on_fail @@ -444,6 +448,9 @@ pypi-blackmagic-decklink-sdi-4k: - 23-support-for-blackmagic-decklink-4k-extreme-12g - 20-support-for-capturing-bgra-frames-with-blackmagic-devices - 14-unable-to-free-blackmagic-video-source + - 5-support-for-3d-capture-on-blackmagic-cards + - 18-extend-api-to-support-stereo-streams + - 30-12-bit-rgb-to-bgra-converter-for-decklink blackmagic-decklink-sdi-4k: stage: test_cmake @@ -466,6 +473,9 @@ blackmagic-decklink-sdi-4k: - 23-support-for-blackmagic-decklink-4k-extreme-12g - 20-support-for-capturing-bgra-frames-with-blackmagic-devices - 14-unable-to-free-blackmagic-video-source + - 5-support-for-3d-capture-on-blackmagic-cards + - 18-extend-api-to-support-stereo-streams + - 30-12-bit-rgb-to-bgra-converter-for-decklink ################## Device: Blackmagic DeckLink 4K Extreme 12G ################## pypi-blackmagic-decklink-4k-extreme-12g: @@ -494,9 +504,10 @@ pypi-blackmagic-decklink-4k-extreme-12g: - PyPI_INSTALLER="../$GiftGrab_PyPI_DIST_DIR/$(ls ../$GiftGrab_PyPI_DIST_DIR | grep tar.gz)" - if [ -z "$PyPI_INSTALLER" ]; then exit 1; fi # because pip seems to be getting confused about install options sometimes: - - pip install PyYAML - pip install pytest - - pip install -vvv --install-option="--blackmagic-decklink-4k-extreme-12g" --install-option="--enable-nonfree" --upgrade "$PyPI_INSTALLER" + - pip install numpy + - pip install scipy + - pip install -vvv --install-option="--blackmagic-decklink-4k-extreme-12g" --install-option="--enable-nonfree" --install-option="--numpy" --upgrade "$PyPI_INSTALLER" # run tests - $TEST_LAUNCHER blackmagic-decklink4kextreme12g uyvy; exit_on_fail - $TEST_LAUNCHER blackmagic-decklink4kextreme12g bgra; exit_on_fail @@ -510,6 +521,9 @@ pypi-blackmagic-decklink-4k-extreme-12g: - 20-support-for-capturing-bgra-frames-with-blackmagic-devices - 32-videosourcefactory-destructor-does-not-free-all-devices - 14-unable-to-free-blackmagic-video-source + - 5-support-for-3d-capture-on-blackmagic-cards + - 18-extend-api-to-support-stereo-streams + - 30-12-bit-rgb-to-bgra-converter-for-decklink blackmagic-decklink-4k-extreme-12g: stage: test_cmake @@ -520,7 +534,7 @@ blackmagic-decklink-4k-extreme-12g: - rm -rf "$GiftGrab_BUILD_DIR" - mkdir -p "$GiftGrab_BUILD_DIR" - cd "$GiftGrab_BUILD_DIR" - - cmake -D BUILD_PYTHON=ON -D BUILD_TESTS=ON -D USE_BLACKMAGIC_DECKLINK_4K_EXTREME_12G=ON -D ENABLE_NONFREE=ON "$GiftGrab_SOURCE_DIR" + - cmake -D BUILD_PYTHON=ON -D USE_NUMPY=ON -D BUILD_TESTS=ON -D USE_BLACKMAGIC_DECKLINK_4K_EXTREME_12G=ON -D ENABLE_NONFREE=ON "$GiftGrab_SOURCE_DIR" - make -j; exit_on_fail - ctest; exit_on_fail tags: @@ -531,3 +545,6 @@ blackmagic-decklink-4k-extreme-12g: - 20-support-for-capturing-bgra-frames-with-blackmagic-devices - 32-videosourcefactory-destructor-does-not-free-all-devices - 14-unable-to-free-blackmagic-video-source + - 5-support-for-3d-capture-on-blackmagic-cards + - 18-extend-api-to-support-stereo-streams + - 30-12-bit-rgb-to-bgra-converter-for-decklink diff --git a/doc/requirements.md b/doc/requirements.md index af848ea7..5d559613 100644 --- a/doc/requirements.md +++ b/doc/requirements.md @@ -52,7 +52,7 @@ The parantheses in the version column mean that the listed version has been test | [libvpx](https://www.webmproject.org/code/) | 1.3.0 | | | [libVLC (VLC SDK)](https://wiki.videolan.org/LibVLC/) | 3.0.0 release candidate: [nighly build ID: 20160913-0237](http://nightlies.videolan.org/build/source/?C=M;O=D) | Please see the note in the [libVLC installation instructions](tips.md#libvlc) about using libVLC for capturing network streams. | | [Epiphan Video Grabber SDK](https://www.epiphan.com/support/) | 3.30.3.0007 | Epiphan Video Grabber SDK has a proprietary licence: enabling Epiphan Video Grabber SDK makes GIFT-Grab undistributable. | -| [Blackmagic Desktop Video SDK](https://www.blackmagicdesign.com/support) | 10.4 | Blackmagic Desktop Video SDK has a proprietary licence: enabling support for Blackmagic cards makes GIFT-Grab undistributable. | +| [Blackmagic Desktop Video SDK](https://www.blackmagicdesign.com/support) | 10.11 | Blackmagic Desktop Video SDK has a proprietary licence: enabling support for Blackmagic cards makes GIFT-Grab undistributable. | | [Python](https://www.python.org/) | 2.7 | | | [Boost.Python](http://www.boost.org/doc/libs/release/libs/python/) | 1.54.0, and 1.63.0 beta 1 for NumPy support | | | [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/) | | | diff --git a/doc/source/index.rst b/doc/source/index.rst index 8e9af514..fbaf6550 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -52,6 +52,7 @@ Examples scipy encoding complex + stereo Citing GIFT-Grab ^^^^^^^^^^^^^^^^ diff --git a/doc/source/stereo.rst b/doc/source/stereo.rst new file mode 100644 index 00000000..511c1e4b --- /dev/null +++ b/doc/source/stereo.rst @@ -0,0 +1,25 @@ +.. _Stereo: + +Capturing stereo video +====================== + +GIFT-Grab allows for capturing stereo video streams from frame grabbers supporting this +feature. +The example below demonstrates how stereo frames can be acquired and saved to individual +image files. +Running this example requires GIFT-Grab built/installed with support for +`Blackmagic DeckLink 4K Extreme 12G`_ and NumPy_. +The example uses OpenCV_ to save video frames to disk. + +.. _`Blackmagic DeckLink 4K Extreme 12G`: https://github.com/gift-surg/GIFT-Grab/blob/master/doc/pypi.md#blackmagic-decklink-4k-extreme-12g +.. _NumPy: https://github.com/gift-surg/GIFT-Grab/blob/master/doc/pypi.md#numpy +.. _OpenCV: http://www.opencv.org/ + +The full source code of the example is below. +Please follow the comments and the flow of code. +This example is also available on GitHub_: + +.. literalinclude:: ../../src/tests/blackmagic/stereo_capture.py + :language: python + +.. _GitHub: https://github.com/gift-surg/GIFT-Grab/blob/master/src/tests/blackmagic/stereo_capture.py diff --git a/doc/tips.md b/doc/tips.md index b4a0ac5f..6fd7a130 100644 --- a/doc/tips.md +++ b/doc/tips.md @@ -17,6 +17,9 @@ If you encounter problems installing any dependency, please have a look at the [ ## Epiphan Video Grabbing SDK +These instructions are provided for convenience only. +Always check the manufacturer's manuals before proceeding. + 1. Download Epiphan video grabbing SDK from [Epiphan support](https://www.epiphan.com/support/) and unpack it, e.g. `wget https://www.epiphan.com/downloads/products/epiphan_sdk-3.30.3.0007.zip; unzip epiphan_sdk-3.30.3.0007.zip`. 1. Change into the sub-folder `epiphan/samples/v2u` and run `make`. This should create a `build` folder here. 1. (Optional) Move the top folder (i.e. the one resulting from unpacking the archive) to a system folder, e.g. `/opt`, for easy access by all users. @@ -25,14 +28,25 @@ If you encounter problems installing any dependency, please have a look at the [ ## Blackmagic Drivers and Blackmagic Desktop Video SDK -1. Download and unpack [Blackmagic Desktop Video SDK](https://www.blackmagicdesign.com/support). -1. If the resulting folder name has spaces e.g. `Blackmagic DeckLink SDK 10.4`, replace spaces with an underscore, e.g. `Blackmagic_DeckLink_SDK_10.4`. -1. (Optional) Move the resulting folder to a system folder, e.g. `/opt`, for easy access by all users. -1. Specify the **absolute** path of the `SDK` sub-folder of this folder as the `BlackmagicSDK_DIR` environment variable, e.g. `export BlackmagicSDK_DIR="/opt/Blackmagic DeckLink SDK 10.4/SDK"`. -1. Install the driver package appropriate for your system (e.g. `/opt/Blackmagic DeckLink SDK 10.4/deb/amd64/desktopvideo_10.8.4a4_amd64.deb` for a 64-bit Ubuntu Linux). -1. (Optional) Install the MediaExpress application using the package appropriate for your system (e.g. `/opt/Blackmagic DeckLink SDK 10.4/deb/amd64/mediaexpress_3.5.3a1_amd64.deb` for a 64-bit Ubuntu Linux). -1. Check your Blackmagic card's firmware status, and update it if necessary. +These instructions are provided for convenience only. +Always check the manufacturer's manuals before proceeding. +1. Download and unpack [Blackmagic Desktop Video SDK][blackmagic-support]. +1. If the resulting folder name has spaces e.g. `Blackmagic DeckLink SDK 10.11.1`, replace spaces with an underscore, +e.g. `Blackmagic_DeckLink_SDK_10.11.1`. +1. (Optional) Move the resulting folder to a system folder, e.g. `/opt`, for easy access by all users. +1. Specify the **absolute** path of this folder as the `BlackmagicSDK_DIR` environment variable, e.g. +`export BlackmagicSDK_DIR="/opt/Blackmagic_DeckLink_SDK_10.11.1"`. +1. Download and unpack [Blackmagic Desktop Video][blackmagic-support]. +1. Install the driver package appropriate for your system, e.g. +`dpkg -i Blackmagic_Desktop_Video_Linux_10.11.1/deb/x86_64/desktopvideo_10.11.1a4_amd64.deb` +1. (Optional) Install the GUI components, e.g. +`dpkg -i Blackmagic_Desktop_Video_Linux_10.11.1/deb/x86_64/desktopvideo-gui_10.11.1a4_amd64.deb` and +`dpkg -i Blackmagic_Desktop_Video_Linux_10.11.1/deb/x86_64/mediaexpress_3.5.6a2_amd64.deb` +1. Check your Blackmagic card's firmware status: `BlackmagicFirmwareUpdater status`, and update if necessary, e.g. +`BlackmagicFirmwareUpdater update 0` + +[blackmagic-support]: https://www.blackmagicdesign.com/support ## OpenCV diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a91bcf6d..14106b63 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -279,6 +279,8 @@ if(USE_BLACKMAGICSDK) LIST(APPEND SOURCES blackmagicsdk/blackmagicsdk_video_source.cpp) LIST(APPEND HEADERS blackmagicsdk/deck_link_display_mode_detector.h) LIST(APPEND SOURCES blackmagicsdk/deck_link_display_mode_detector.cpp) + LIST(APPEND HEADERS blackmagicsdk/deck_link_bgra_video_frame.h) + LIST(APPEND SOURCES blackmagicsdk/deck_link_bgra_video_frame.cpp) endif(USE_BLACKMAGICSDK) # Performance benchmarking diff --git a/src/api/videoframe.cpp b/src/api/videoframe.cpp index 0a6cc49f..16962ebc 100644 --- a/src/api/videoframe.cpp +++ b/src/api/videoframe.cpp @@ -10,22 +10,26 @@ VideoFrame::VideoFrame() } VideoFrame::VideoFrame(enum ColourSpace colour, bool manage_data) - : _colour(colour), - _manage_data(manage_data), - _data(nullptr), - _data_length(0) + : _colour(colour) + , _manage_data(manage_data) + , _data(nullptr) + , _data_length(0) + , _stereo_count(1) { set_dimensions(0, 0); } -VideoFrame::VideoFrame(ColourSpace colour, size_t cols, size_t rows) +VideoFrame::VideoFrame(ColourSpace colour, size_t cols, size_t rows, + size_t stereo_count) : _colour(colour) , _manage_data(true) , _data(nullptr) , _data_length(0) + , _stereo_count(stereo_count) { set_dimensions(cols, rows); size_t data_length = required_data_length(_colour, _cols, _rows); + data_length *= _stereo_count; allocate_memory(data_length); set_pixels_black(); } @@ -35,6 +39,7 @@ VideoFrame::VideoFrame(const VideoFrame & rhs) , _manage_data(true) , _data(nullptr) , _data_length(0) + , _stereo_count(1) { clone(rhs); } @@ -145,8 +150,10 @@ void VideoFrame::operator=(const VideoFrame & rhs) } void VideoFrame::init_from_specs(unsigned char * data, size_t data_length, - size_t cols, size_t rows) + size_t cols, size_t rows, size_t stereo_count) { + _stereo_count = stereo_count; + if (_manage_data) { allocate_memory(data_length); @@ -161,6 +168,20 @@ void VideoFrame::init_from_specs(unsigned char * data, size_t data_length, set_dimensions(cols, rows); } +const size_t VideoFrame::data_length(size_t stereo_index) const +{ + validate_stereo_index(stereo_index); + return _data_length / _stereo_count; +} + +unsigned char * const VideoFrame::data(size_t stereo_index) const +{ + validate_stereo_index(stereo_index); + if (_data == nullptr) + return nullptr; + return &_data[stereo_index * (_data_length / _stereo_count)]; +} + void VideoFrame::clone(const VideoFrame & rhs) { if (not _manage_data) @@ -172,7 +193,7 @@ void VideoFrame::clone(const VideoFrame & rhs) _manage_data = true; _colour = rhs._colour; init_from_specs(rhs._data, rhs._data_length, - rhs._cols, rhs._rows); + rhs._cols, rhs._rows, rhs._stereo_count); } void VideoFrame::set_dimensions(size_t cols, size_t rows) @@ -213,6 +234,7 @@ void VideoFrame::free_memory() _data = nullptr; _data_length = 0; set_dimensions(0, 0); + _stereo_count = 0; } } diff --git a/src/api/videoframe.h b/src/api/videoframe.h index e5a97919..35567e9c 100644 --- a/src/api/videoframe.h +++ b/src/api/videoframe.h @@ -40,6 +40,10 @@ enum ColourSpace //! In that case, the caller is responsible for ensuring the data //! pointer is valid during the lifetime of the frame object. //! +//! Stereo frame data from a particular point in time are included +//! within a single frame object, so as to preserve temporal +//! relations and to facilitate potential stereo encoding. +//! class VideoFrame { protected: @@ -74,8 +78,11 @@ class VideoFrame //! \param colour //! \param cols //! \param rows + //! \param stereo_count if larger than 1, then this frame + //! will keep the specified number of stereo frames //! - VideoFrame(enum ColourSpace colour, size_t cols, size_t rows); + VideoFrame(enum ColourSpace colour, size_t cols, size_t rows, + size_t stereo_count = 1); //! //! \brief Create a video frame by copying the data and @@ -131,22 +138,37 @@ class VideoFrame //! available yet. //! //! \param data - //! \param data_length - //! \param cols - //! \param rows + //! \param data_length in case of stereo frames, this is the + //! \b total length of passed data buffer. Only stereo frames + //! with exactly the same dimensions are supports, so this is + //! assumed to be divisible by the number of stereo frames. + //! \param cols the width of a \b single video frame + //! \param rows the height of a \b single video frame + //! \param stereo_count the number of stereo frames included in + //! passed data //! \sa manages_own_data //! \sa VideoFrame(enum ColourSpace, size_t, size_t) //! void init_from_specs(unsigned char * data, size_t data_length, - size_t cols, size_t rows); + size_t cols, size_t rows, size_t stereo_count = 1); //! //! \brief Get length of data buffer + //! \param stereo_index index of stereo frame whose + //! length is requested + //! \throw std::out_of_range if provided index value is + //! invalid (i.e. out of range) + //! \return + //! + const size_t data_length(size_t stereo_index = 0) const; + + //! + //! \brief Get number of stereo frames stored in this object //! \return //! - const size_t data_length() const + size_t stereo_count() const { - return _data_length; + return _stereo_count; } //! @@ -207,15 +229,24 @@ class VideoFrame //! if frame.colour() == ColourSpace.BGRA: //! struc_arr = frame.data(True) # True => structured NumPy array //! # struc_arr.shape => (frame.rows(), frame.cols(), 4) + //! + //! # stereo index is the second (optional) parameter to this + //! # function in Python, e.g: + //! struc_arr_of_2nd_stereo_frame = frame.data(True, 1) + //! + //! # the following calls are equivalent + //! frame.data() == frame.data(False) + //! frame.data() == frame.data(False, 0) + //! frame.data(True) == frame.data(True, 0) //! \endcode //! + //! \param stereo_index index of requested stereo frame + //! \throw std::out_of_range if provided index value is + //! invalid (i.e. out of range) //! \return //! \sa VideoFrame(const VideoFrame & rhs) //! - unsigned char * const data() const - { - return _data; - } + unsigned char * const data(size_t stereo_index = 0) const; //! //! \brief Get what colour space \c this frame uses @@ -266,6 +297,12 @@ class VideoFrame //! size_t _data_length; + //! + //! \brief Number of stereo frames stored, i.e. + //! the total number of frames + //! + size_t _stereo_count; + //! //! \brief Always use \c set_dimensions() to set //! this @@ -334,6 +371,25 @@ class VideoFrame //! \brief Set all pixels of frame to black //! void set_pixels_black(); + + //! + //! \brief + //! \param stereo_index + //! \throw std::out_of_range if passed stereo + //! index is invalid + //! + inline void validate_stereo_index(size_t stereo_index) const + { + if (stereo_index >= _stereo_count) + { + std::string msg = "This frame has "; + msg.append(std::to_string(_stereo_count)) + .append(" stereo frames (requested ") + .append(std::to_string(stereo_index + 1)) + .append(". stereo frame)"); + throw std::out_of_range(msg); + } + } }; } diff --git a/src/api/videosourcefactory.cpp b/src/api/videosourcefactory.cpp index 5546a79a..fba3d160 100644 --- a/src/api/videosourcefactory.cpp +++ b/src/api/videosourcefactory.cpp @@ -1,3 +1,4 @@ +#include #include "videosourcefactory.h" #ifdef USE_OPENCV #include "opencv_video_source.h" @@ -222,11 +223,24 @@ IVideoSource * VideoSourceFactory::get_device(Device device, { // check querying frame dimensions int width = -1, height = -1; - if (not src->get_frame_dimensions(width, height)) + bool device_online = false; + for (int attempts = 0; attempts < 5; attempts++) { + if (src->get_frame_dimensions(width, height)) + { + device_online = true; + break; + } + // artificial sleep introduced to allow for making sure + // video capture has been launched (this was originally + // in BlackmagicSDKVideoSource, but moved here to + // enable a more adaptive waiting logic, to avoid + // dependencies on hardware specifics + std::this_thread::sleep_for(std::chrono::milliseconds(75)); + } + if (not device_online) throw DeviceOffline( "Device connected but does not return frame dimensions"); - } // check meaningful frame dimensions if (width <= 0 or height <= 0) diff --git a/src/blackmagicsdk/blackmagicsdk_video_source.cpp b/src/blackmagicsdk/blackmagicsdk_video_source.cpp index 238bad8a..72a040f9 100644 --- a/src/blackmagicsdk/blackmagicsdk_video_source.cpp +++ b/src/blackmagicsdk/blackmagicsdk_video_source.cpp @@ -1,6 +1,5 @@ #include "blackmagicsdk_video_source.h" #include "deck_link_display_mode_detector.h" -#include #include namespace gg @@ -31,6 +30,9 @@ VideoSourceBlackmagicSDK::VideoSourceBlackmagicSDK(size_t deck_link_index, , _buffer_video_frame(VideoFrame(colour, false)) // TODO manage data? , _deck_link(nullptr) , _deck_link_input(nullptr) + , _video_input_flags(bmdVideoInputFlagDefault | bmdVideoInputDualStream3D) + , _12_bit_rgb_to_bgra_converter(nullptr) + , _bgra_frame_buffers{nullptr, nullptr} , _running(false) { // Pixel format, i.e. colour space @@ -41,7 +43,9 @@ VideoSourceBlackmagicSDK::VideoSourceBlackmagicSDK(size_t deck_link_index, pixel_format = bmdFormat8BitYUV; break; case BGRA: - pixel_format = bmdFormat8BitBGRA; + // We currently only support BGRA with DeckLink 4K Extreme 12G, + // and that card supports only this YUV format: + pixel_format = bmdFormat10BitYUV; break; case I420: default: @@ -81,10 +85,25 @@ VideoSourceBlackmagicSDK::VideoSourceBlackmagicSDK(size_t deck_link_index, // Set the input format (i.e. display mode) BMDDisplayMode display_mode; + BMDFrameFlags frame_flags; std::string error_msg = ""; - BMDVideoInputFlags video_input_flags = bmdVideoInputFlagDefault | bmdVideoInputEnableFormatDetection; - if (not detect_input_format(pixel_format, video_input_flags, display_mode, _frame_rate, error_msg)) - bail(error_msg); + size_t cols = 0, rows = 0; + if (not detect_input_format(pixel_format, _video_input_flags, display_mode, + _frame_rate, cols, rows, frame_flags, error_msg)) + { + _video_input_flags ^= bmdVideoInputDualStream3D; + if (not detect_input_format(pixel_format, _video_input_flags, display_mode, + _frame_rate, cols, rows, frame_flags, error_msg)) + bail(error_msg); + } + + // Get a post-capture converter if necessary + if (_colour == BGRA) + { + _12_bit_rgb_to_bgra_converter = CreateVideoConversionInstance(); + if (_12_bit_rgb_to_bgra_converter == nullptr) + bail("Could not create colour converter for Blackmagic source"); + } // Set this object (IDeckLinkInputCallback instance) as callback res = _deck_link_input->SetCallback(this); @@ -95,7 +114,7 @@ VideoSourceBlackmagicSDK::VideoSourceBlackmagicSDK(size_t deck_link_index, // Enable video input res = _deck_link_input->EnableVideoInput(display_mode, pixel_format, - video_input_flags); + _video_input_flags); // No glory if (res != S_OK) bail("Could not enable video input of Blackmagic DeckLink device"); @@ -109,10 +128,6 @@ VideoSourceBlackmagicSDK::VideoSourceBlackmagicSDK(size_t deck_link_index, _running = false; bail("Could not start streaming from the Blackmagic DeckLink device"); } - - // artificial sleep introduced to allow for starting of streams - // the value is determined empirically - std::this_thread::sleep_for(std::chrono::milliseconds(75)); } @@ -230,37 +245,74 @@ HRESULT STDMETHODCALLTYPE VideoSourceBlackmagicSDK::VideoInputFrameArrived( // nop if no data return S_OK; - // Nr. of bytes of received data - size_t n_bytes = video_frame->GetRowBytes() * video_frame->GetHeight(); - { // Artificial scope for data lock // Make sure only this thread is accessing the buffer now std::lock_guard data_lock_guard(_data_lock); - // Extend buffer if more memory needed than already allocated - if (n_bytes > _video_buffer_length) - _video_buffer = reinterpret_cast( - realloc(_video_buffer, n_bytes * sizeof(uint8_t)) - ); + smart_allocate_buffers( + video_frame->GetWidth(), video_frame->GetHeight(), + video_frame->GetFlags() + ); + if (_video_buffer == nullptr) // something's terribly wrong! // nop if something's terribly wrong! return S_OK; - // Get the new data into the buffer - HRESULT res = video_frame->GetBytes( - reinterpret_cast(&_video_buffer) - ); - // If data could not be read into the buffer, return + HRESULT res; + + if (need_conversion()) + // convert to BGRA from capture format + res = _12_bit_rgb_to_bgra_converter->ConvertFrame( + video_frame, _bgra_frame_buffers[0] + ); + else + // Get the new data into the buffer + res = video_frame->GetBytes( + reinterpret_cast(&_video_buffer) + ); + + // If the last operation failed, return if (FAILED(res)) return res; - // Set video frame specs according to new data - _video_buffer_length = n_bytes; - _cols = video_frame->GetWidth(); - _rows = video_frame->GetHeight(); + + if (is_stereo()) + { + IDeckLinkVideoFrame *right_eye_frame = nullptr; + IDeckLinkVideoFrame3DExtensions *three_d_extensions = nullptr; + if ((video_frame->QueryInterface( + IID_IDeckLinkVideoFrame3DExtensions, + (void **) &three_d_extensions) != S_OK) || + (three_d_extensions->GetFrameForRightEye( + &right_eye_frame) != S_OK)) + { + right_eye_frame = nullptr; + } + + if (three_d_extensions != nullptr) + three_d_extensions->Release(); + + if (right_eye_frame != nullptr) + { + if (need_conversion()) + // convert to BGRA from capture format + res = _12_bit_rgb_to_bgra_converter->ConvertFrame( + right_eye_frame, _bgra_frame_buffers[1] + ); + else + res = right_eye_frame->GetBytes( + reinterpret_cast(&_video_buffer[_video_buffer_length / 2]) + ); + right_eye_frame->Release(); + // If data could not be read into the buffer, return + if (FAILED(res)) + return res; + } + } // Propagate new video frame to observers _buffer_video_frame.init_from_specs( - _video_buffer, _video_buffer_length, _cols, _rows + _video_buffer, _video_buffer_length, _cols, _rows, + is_stereo() ? 2 : 1 ); } @@ -281,13 +333,70 @@ void VideoSourceBlackmagicSDK::release_deck_link() noexcept if (_deck_link != nullptr) _deck_link->Release(); + + if (_12_bit_rgb_to_bgra_converter != nullptr) + { + _12_bit_rgb_to_bgra_converter->Release(); + _12_bit_rgb_to_bgra_converter = nullptr; + } + + for (size_t i = 0; i < 2; i++) + if (_bgra_frame_buffers[i] != nullptr) + { + _bgra_frame_buffers[i]->Release(); + delete _bgra_frame_buffers[i]; + _bgra_frame_buffers[i] = nullptr; + } +} + + +inline void VideoSourceBlackmagicSDK::smart_allocate_buffers( + size_t cols, size_t rows, BMDFrameFlags frame_flags +) noexcept +{ + if (cols <= 0 or rows <= 0) + return; + + if (cols == _cols and rows == _rows) + return; + + _cols = cols; + _rows = rows; + + // Allocate pixel buffer + _video_buffer_length = VideoFrame::required_data_length(_colour, _cols, _rows); + if (is_stereo()) + _video_buffer_length *= 2; + _video_buffer = reinterpret_cast( + realloc(_video_buffer, _video_buffer_length * sizeof(uint8_t)) + ); + + // Colour converter for post-capture colour conversion + if (need_conversion()) + { + for (size_t i = 0; i < (is_stereo() ? 2 : 1); i++) + { + if (_bgra_frame_buffers[i] != nullptr) + { + _bgra_frame_buffers[i]->Release(); + delete _bgra_frame_buffers[i]; + _bgra_frame_buffers[i] = nullptr; + } + _bgra_frame_buffers[i] = new DeckLinkBGRAVideoFrame( + _cols, _rows, + &_video_buffer[i * _video_buffer_length / 2], frame_flags + ); + } + } } bool VideoSourceBlackmagicSDK::detect_input_format(BMDPixelFormat pixel_format, - BMDVideoInputFlags video_input_flags, + BMDVideoInputFlags & video_input_flags, BMDDisplayMode & display_mode, double & frame_rate, + size_t & cols, size_t & rows, + BMDFrameFlags & frame_flags, std::string & error_msg) noexcept { std::vector display_modes = @@ -315,7 +424,10 @@ bool VideoSourceBlackmagicSDK::detect_input_format(BMDPixelFormat pixel_format, if (display_mode_ != bmdModeUnknown) { frame_rate = detector.get_frame_rate(); + detector.get_frame_dimensions(cols, rows); + frame_flags = detector.get_frame_flags(); display_mode = display_mode_; + video_input_flags = detector.get_video_input_flags(); return true; } else diff --git a/src/blackmagicsdk/blackmagicsdk_video_source.h b/src/blackmagicsdk/blackmagicsdk_video_source.h index 136bf633..578b462e 100644 --- a/src/blackmagicsdk/blackmagicsdk_video_source.h +++ b/src/blackmagicsdk/blackmagicsdk_video_source.h @@ -2,6 +2,7 @@ #include "ivideosource.h" #include "macros.h" +#include "deck_link_bgra_video_frame.h" #include #include @@ -67,6 +68,22 @@ class VideoSourceBlackmagicSDK //! IDeckLinkInput * _deck_link_input; + //! + //! \brief Detected video input flags + //! + BMDVideoInputFlags _video_input_flags; + + //! + //! \brief Converter needed in case of BGRA captures + //! + IDeckLinkVideoConversion *_12_bit_rgb_to_bgra_converter; + + //! + //! \brief Internal frame buffers for post-capture + //! conversion + //! + DeckLinkBGRAVideoFrame *_bgra_frame_buffers[2]; + //! //! \brief Flag indicating streaming status //! @@ -149,6 +166,17 @@ class VideoSourceBlackmagicSDK //! void release_deck_link() noexcept; + //! + //! \brief (Re-)allocate internal buffers ONLY IF the new + //! dimensions are different than the previous ones + //! \param cols new frame width + //! \param rows new frame height + //! \param frame_flags + //! + inline void smart_allocate_buffers( + size_t cols, size_t rows, BMDFrameFlags frame_flags + ) noexcept; + //! //! \brief Try to detect the input video format, i.e. //! the display mode as well as the frame rate @@ -156,15 +184,20 @@ class VideoSourceBlackmagicSDK //! \param video_input_flags Use these video flags //! \param display_mode //! \param frame_rate + //! \param cols detected frame width + //! \param rows detected frame height + //! \param frame_flags //! \param error_msg //! \return \c true on success, \c false otherwise, //! accompanied by a detailed error message, which //! could be used for instance for throwing an exception //! bool detect_input_format(BMDPixelFormat pixel_format, - BMDVideoInputFlags video_input_flags, + BMDVideoInputFlags & video_input_flags, BMDDisplayMode & display_mode, double & frame_rate, + size_t & cols, size_t & rows, + BMDFrameFlags & frame_flags, std::string & error_msg) noexcept; //! @@ -180,6 +213,26 @@ class VideoSourceBlackmagicSDK throw VideoSourceError(error_msg); } + //! + //! \brief + //! \return whether this source is in stereo mode + //! + inline bool is_stereo() + { + return _video_input_flags & bmdVideoInputDualStream3D; + } + + //! + //! \brief + //! \return whether a post-capture colour conversion + //! is needed + //! + inline bool need_conversion() + { + return _colour == BGRA and + _12_bit_rgb_to_bgra_converter != nullptr; + } + private: DISALLOW_COPY_AND_ASSIGNMENT(VideoSourceBlackmagicSDK); }; diff --git a/src/blackmagicsdk/deck_link_bgra_video_frame.cpp b/src/blackmagicsdk/deck_link_bgra_video_frame.cpp new file mode 100644 index 00000000..a64f4508 --- /dev/null +++ b/src/blackmagicsdk/deck_link_bgra_video_frame.cpp @@ -0,0 +1,101 @@ +#include "deck_link_bgra_video_frame.h" +#include + +namespace gg +{ + +DeckLinkBGRAVideoFrame::DeckLinkBGRAVideoFrame( + size_t width, size_t height, + LPVOID pixel_buffer, BMDFrameFlags frame_flags +) + : _width(width) + , _height(height) + , _pixel_buffer(pixel_buffer) + , _flags(frame_flags) +{ + // nop +} + +DeckLinkBGRAVideoFrame::~DeckLinkBGRAVideoFrame() +{ + _width = 0; + _height = 0; + _pixel_buffer = nullptr; +} + +long STDMETHODCALLTYPE +DeckLinkBGRAVideoFrame::GetWidth(void) +{ + return _width; +} + +long STDMETHODCALLTYPE +DeckLinkBGRAVideoFrame::GetHeight(void) +{ + return _height; +} + +long STDMETHODCALLTYPE +DeckLinkBGRAVideoFrame::GetRowBytes(void) +{ + return 4 * _width; +} + +HRESULT STDMETHODCALLTYPE +DeckLinkBGRAVideoFrame::GetBytes(void **buffer) +{ + if (_pixel_buffer == nullptr) + return S_FALSE; + *buffer = _pixel_buffer; + return S_OK; +} + +BMDFrameFlags STDMETHODCALLTYPE +DeckLinkBGRAVideoFrame::GetFlags(void) +{ + return _flags; +} + +BMDPixelFormat STDMETHODCALLTYPE +DeckLinkBGRAVideoFrame::GetPixelFormat(void) +{ + return bmdFormat8BitBGRA; +} + +HRESULT STDMETHODCALLTYPE +DeckLinkBGRAVideoFrame::QueryInterface( + REFIID iid, LPVOID *ppv +) +{ + return E_NOINTERFACE; +} + +ULONG STDMETHODCALLTYPE DeckLinkBGRAVideoFrame::AddRef() +{ + __sync_add_and_fetch(&_ref_count, 1); + return _ref_count; +} + +ULONG STDMETHODCALLTYPE DeckLinkBGRAVideoFrame::Release() +{ + __sync_sub_and_fetch(&_ref_count, 1); + return _ref_count; +} + +HRESULT STDMETHODCALLTYPE +DeckLinkBGRAVideoFrame::GetAncillaryData( + IDeckLinkVideoFrameAncillary **ancillary +) +{ + // nop +} + +HRESULT STDMETHODCALLTYPE +DeckLinkBGRAVideoFrame::GetTimecode( + BMDTimecodeFormat format, IDeckLinkTimecode** timecode +) +{ + // nop +} + +} diff --git a/src/blackmagicsdk/deck_link_bgra_video_frame.h b/src/blackmagicsdk/deck_link_bgra_video_frame.h new file mode 100644 index 00000000..366646bd --- /dev/null +++ b/src/blackmagicsdk/deck_link_bgra_video_frame.h @@ -0,0 +1,63 @@ +#pragma once + +#include +#include "DeckLinkAPI.h" + +namespace gg +{ + +//! +//! \brief A very un-intelligent implementation of +//! Blackmagic SDK's video frame interface. This class is +//! not for external use, as such it does absolutely NO +//! sanity checking, or management of data passed to it. +//! It expects ALL of that to be done by the caller. +//! +class DeckLinkBGRAVideoFrame : public IDeckLinkVideoFrame +{ +protected: + // members for implementing promised API + long _width; + long _height; + BMDFrameFlags _flags; + LPVOID _pixel_buffer = nullptr; + ULONG _ref_count; + +public: + //! + //! \brief Wrap passed parameters into a Blackmagic + //! SDK video frame, with absolutely NO sanity + //! checking + //! \param width + //! \param height + //! \param pixel_buffer + //! \param frame_flags + //! + DeckLinkBGRAVideoFrame(size_t width, size_t height, + LPVOID pixel_buffer, BMDFrameFlags frame_flags); + + //! + //! \brief Simply destroy this object, DO NOTHING + //! about data + //! + virtual ~DeckLinkBGRAVideoFrame(); + +public: + // inherited methods + virtual long STDMETHODCALLTYPE GetWidth(void); + virtual long STDMETHODCALLTYPE GetHeight(void); + virtual long STDMETHODCALLTYPE GetRowBytes(void); + virtual HRESULT STDMETHODCALLTYPE GetBytes(void** buffer); + virtual BMDFrameFlags STDMETHODCALLTYPE GetFlags(void); + virtual BMDPixelFormat STDMETHODCALLTYPE GetPixelFormat(void); + + virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID iid, LPVOID *ppv); + virtual ULONG STDMETHODCALLTYPE AddRef(); + virtual ULONG STDMETHODCALLTYPE Release(); + + // Dummy implementations of remaining methods + virtual HRESULT STDMETHODCALLTYPE GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary); + virtual HRESULT STDMETHODCALLTYPE GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode); +}; + +} diff --git a/src/blackmagicsdk/deck_link_display_mode_detector.cpp b/src/blackmagicsdk/deck_link_display_mode_detector.cpp index 3d82822e..9688b167 100644 --- a/src/blackmagicsdk/deck_link_display_mode_detector.cpp +++ b/src/blackmagicsdk/deck_link_display_mode_detector.cpp @@ -25,6 +25,8 @@ DeckLinkDisplayModeDetector::DeckLinkDisplayModeDetector(IDeckLinkInput * deck_l , _video_input_flags(video_input_flags) , _display_mode(bmdModeUnknown) , _frame_rate(0.0) + , _cols(0) + , _rows(0) // to be reset upon determining display mode , _error_msg("Could not determine display mode (unknown reason)") , _running(false) @@ -108,7 +110,12 @@ DeckLinkDisplayModeDetector::DeckLinkDisplayModeDetector(IDeckLinkInput * deck_l _error_msg = "Could not infer frame rate of Blackmagic DeckLink device"; } else + { _frame_rate = (double) frame_rate_scale / (double) frame_rate_duration; + if (_video_input_flags & bmdVideoInputDualStream3D) + if (not (deck_link_display_mode->GetFlags() & bmdDisplayModeSupports3D)) + _video_input_flags ^= bmdVideoInputDualStream3D; + } } // Release the DeckLink display mode object @@ -132,12 +139,33 @@ BMDDisplayMode DeckLinkDisplayModeDetector::get_display_mode() noexcept } +BMDVideoInputFlags DeckLinkDisplayModeDetector::get_video_input_flags() noexcept +{ + return _video_input_flags; +} + + double DeckLinkDisplayModeDetector::get_frame_rate() noexcept { return _frame_rate; } +void DeckLinkDisplayModeDetector::get_frame_dimensions( + size_t &cols, size_t &rows +) noexcept +{ + cols = _cols; + rows = _rows; +} + + +BMDFrameFlags DeckLinkDisplayModeDetector::get_frame_flags() noexcept +{ + return _frame_flags; +} + + std::string DeckLinkDisplayModeDetector::get_error_msg() noexcept { return _error_msg; @@ -186,6 +214,9 @@ HRESULT STDMETHODCALLTYPE DeckLinkDisplayModeDetector::VideoInputFrameArrived( { if (video_frame->GetFlags() & bmdFrameHasNoInputSource) _display_mode = bmdModeUnknown; + _cols = video_frame->GetWidth(); + _rows = video_frame->GetHeight(); + _frame_flags = video_frame->GetFlags(); _running = false; } } diff --git a/src/blackmagicsdk/deck_link_display_mode_detector.h b/src/blackmagicsdk/deck_link_display_mode_detector.h index 821d974b..819ba43f 100644 --- a/src/blackmagicsdk/deck_link_display_mode_detector.h +++ b/src/blackmagicsdk/deck_link_display_mode_detector.h @@ -51,6 +51,21 @@ class DeckLinkDisplayModeDetector : public IDeckLinkInputCallback //! double _frame_rate; + //! + //! \brief Frame width + //! + size_t _cols; + + //! + //! \brief Frame height + //! + size_t _rows; + + //! + //! \brief + //! + BMDFrameFlags _frame_flags; + //! //! \brief //! @@ -110,12 +125,33 @@ class DeckLinkDisplayModeDetector : public IDeckLinkInputCallback //! BMDDisplayMode get_display_mode() noexcept; + //! + //! \brief + //! + //! \return The discovered video input flags, provided + //! \c getDisplayMode does not return \c bmdModeUnknown + //! + BMDVideoInputFlags get_video_input_flags() noexcept; + //! //! \brief //! \return //! double get_frame_rate() noexcept; + //! + //! \brief + //! \param cols + //! \param rows + //! + void get_frame_dimensions(size_t &cols, size_t &rows) noexcept; + + //! + //! \brief + //! \return + //! + BMDFrameFlags get_frame_flags() noexcept; + //! //! \brief Get the last detailed error message set //! \return a detailed message describing the last diff --git a/src/python/wrapper.cpp b/src/python/wrapper.cpp index 11f7e72f..0d35e142 100644 --- a/src/python/wrapper.cpp +++ b/src/python/wrapper.cpp @@ -91,6 +91,25 @@ class VideoFrameNumPyWrapper : public gg::VideoFrame, public wrapperdata_length(); + } + + const size_t data_length_stereo_frame(size_t stereo_index) const + { + return _frame->data_length(stereo_index); + } + #ifdef USE_NUMPY //! - //! \brief see: + //! \brief default arguments for both structured flag (i.e. false) + //! and the stereo index (i.e. 0), see: + //! http://www.boost.org/doc/libs/1_63_0/ + //! libs/python/doc/html/tutorial/tutorial/ + //! functions.html#tutorial.functions.default_arguments + //! \return the first stereo frame's data as a flat NumPy array + //! \sa stereo_data_as_ndarray() + //! + numpy::ndarray first_stereo_data_as_flat_ndarray() const + { + return stereo_data_as_ndarray(false, 0); + } + + //! + //! \brief default argument for stereo index (i.e. 0), see: //! http://www.boost.org/doc/libs/1_63_0/ //! libs/python/doc/html/tutorial/tutorial/ //! functions.html#tutorial.functions.default_arguments - //! \return a flat NumPy array - //! \sa data_as_ndarray() + //! \param structured + //! \return the first stereo frame's data as desired NumPy array + //! \throw gg::BasicException if wrapped gg::VideoFrame has colour + //! other than BGRA (currently only BGRA data supported for + //! structured ndarray exposure) + //! \sa stereo_data_as_ndarray() //! - numpy::ndarray data_as_flat_ndarray() const + numpy::ndarray first_stereo_data_as_ndarray(bool structured) const { - return data_as_ndarray(false); + return stereo_data_as_ndarray(structured, 0); } //! //! \brief Create a NumPy array referencing gg::VideoFrame::data() //! \param structured + //! \param stereo_index //! \return a flat NumPy array if not \c structured; otherwise one //! that conforms to the shape SciPy routines expect: (height, //! width, channels), e.g. (9, 16, 4) for BGRA data of a 16 x 9 @@ -137,8 +185,10 @@ class VideoFrameNumPyWrapper : public gg::VideoFrame, public wrapper(); @@ -163,12 +213,12 @@ class VideoFrameNumPyWrapper : public gg::VideoFrame, public wrapperdata_length()); + shape = make_tuple(_frame->data_length(stereo_index)); strides = make_tuple(sizeof(uint8_t)); } return numpy::from_data( - _frame->data(), data_type, shape, strides, + _frame->data(stereo_index), data_type, shape, strides, // owner (dangerous to pass None) object() ); @@ -183,6 +233,7 @@ class VideoFrameNumPyWrapper : public gg::VideoFrame, public wrapperrows(); _data = _frame->data(); _data_length = _frame->data_length(); + _stereo_count = _frame->stereo_count(); } }; @@ -401,6 +452,13 @@ void translate_ObserverError(gg::ObserverError const & e) PyErr_SetString(PyExc_RuntimeError, msg.c_str()); } +void translate_out_of_range(std::out_of_range const &e) +{ + std::string msg; + msg.append("std::out_of_range: ").append(e.what()); + PyErr_SetString(PyExc_IndexError, msg.c_str()); +} + BOOST_PYTHON_MODULE(pygiftgrab) { PyEval_InitThreads(); @@ -416,6 +474,7 @@ BOOST_PYTHON_MODULE(pygiftgrab) register_exception_translator(&translate_NetworkSourceUnavailable); register_exception_translator(&translate_VideoTargetError); register_exception_translator(&translate_ObserverError); + register_exception_translator(&translate_out_of_range); enum_("ColourSpace") .value("BGRA", gg::ColourSpace::BGRA) @@ -438,17 +497,21 @@ BOOST_PYTHON_MODULE(pygiftgrab) class_("VideoFrame", init()) .def(init()) + .def(init()) .def("colour", &VideoFrameNumPyWrapper::colour) .def("rows", &VideoFrameNumPyWrapper::rows) .def("cols", &VideoFrameNumPyWrapper::cols) - .def("data_length", &VideoFrameNumPyWrapper::data_length) + .def("data_length", &VideoFrameNumPyWrapper::data_length_default_frame) + .def("data_length", &VideoFrameNumPyWrapper::data_length_stereo_frame) + .def("stereo_count", &VideoFrameNumPyWrapper::stereo_count) .def("required_data_length", &VideoFrameNumPyWrapper::required_data_length) .staticmethod("required_data_length") .def("required_pixel_length", &VideoFrameNumPyWrapper::required_pixel_length) .staticmethod("required_pixel_length") #ifdef USE_NUMPY - .def("data", &VideoFrameNumPyWrapper::data_as_flat_ndarray) - .def("data", &VideoFrameNumPyWrapper::data_as_ndarray) + .def("data", &VideoFrameNumPyWrapper::first_stereo_data_as_flat_ndarray) + .def("data", &VideoFrameNumPyWrapper::first_stereo_data_as_ndarray) + .def("data", &VideoFrameNumPyWrapper::stereo_data_as_ndarray) #endif ; diff --git a/src/tests/blackmagic/CMakeLists.txt b/src/tests/blackmagic/CMakeLists.txt index 15d90d5a..d200d041 100644 --- a/src/tests/blackmagic/CMakeLists.txt +++ b/src/tests/blackmagic/CMakeLists.txt @@ -1,6 +1,7 @@ FILE(COPY test_unit.py test_observer.py + test_stereo.py conftest.py DESTINATION ${CMAKE_CURRENT_BINARY_DIR} ) @@ -45,11 +46,24 @@ foreach(BLACKMAGIC_DEVICE ${BLACKMAGIC_DEVICES}) # Blackmagic device using the observer design pattern SET(NAME_TEST Test_Blackmagic_${BLACKMAGIC_DEVICE}_ObserverPattern_${COLOUR_SPACE}) - SET(FRAME_RATE 27) + if(BLACKMAGIC_DEVICE STREQUAL DeckLink4KExtreme12G) + SET(FRAME_RATE 22) # frame rate seems to be reduced, due to stereo? + else() + SET(FRAME_RATE 27) + endif() ADD_TEST(NAME ${NAME_TEST} COMMAND py.test --device=${BLACKMAGIC_DEVICE} --colour-space=${COLOUR_SPACE} --frame-rate=${FRAME_RATE} --observers=3 test_observer.py ) LIST(APPEND TESTS_LIST ${NAME_TEST}) + + # Blackmagic device supporting stereo (3D) video streams + if(BLACKMAGIC_DEVICE STREQUAL DeckLink4KExtreme12G AND USE_NUMPY) + SET(NAME_TEST Test_Blackmagic_${BLACKMAGIC_DEVICE}_StereoFrames_${COLOUR_SPACE}) + ADD_TEST(NAME ${NAME_TEST} + COMMAND py.test --device=${BLACKMAGIC_DEVICE} --colour-space=${COLOUR_SPACE} test_stereo.py + ) + LIST(APPEND TESTS_LIST ${NAME_TEST}) + endif() endforeach(COLOUR_SPACE) endforeach(BLACKMAGIC_DEVICE) diff --git a/src/tests/blackmagic/stereo_capture.py b/src/tests/blackmagic/stereo_capture.py new file mode 100755 index 00000000..995497bd --- /dev/null +++ b/src/tests/blackmagic/stereo_capture.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +""" +Example demonstrating how stereo video frames can be captured +using a frame grabber card that supports this feature. +""" + +import time +import cv2 +import numpy as np +from pygiftgrab import (IObserver, VideoSourceFactory, + ColourSpace, Device, VideoFrame) + + +class StereoFrameSaver(IObserver): + """ + Simple class that demonstrates how mono and stereo frames, + and their respective parameters can be queried and the actual + frame data can be saved using the GIFT-Grab stereo API. + """ + + def __init__(self): + super(StereoFrameSaver, self).__init__() + self.current = 0 + + def update(self, frame): + self.current += 1 + + # 4 is the number of variations of stereo/mono + # calls to the data method, using it here as well to + # avoid flooding the user's terminal + if self.current <= 4: + # display number of stereo frames, should be 2 + # for this device + print( + 'Got {} stereo frames'.format( + frame.stereo_count() + ) + ) + # display length of data of each stereo frame, + # each stereo frame should consist of same number + # of bytes for this device + print( + 'Stereo data length (bytes):\n' + '\tdata_length(): {}\n' + '\tdata_length(0): {}\n' + '\tdata_length(1): {}\n'.format( + frame.data_length(), frame.data_length(0), + frame.data_length(1) + ) + ) + + frame_shape = (frame.rows(), frame.cols(), 4) + + # the slicing below, i.e. [:, :, :3], is due to OpenCV's + # imwrite expecting BGR data, so we strip out the alpha + # channel of each frame when saving it + + if self.current == 1: + # all three calls below save the same frame, + # that is the first of the two stereo frames + cv2.imwrite( + 'mono-frame.data.png', + np.reshape(frame.data(), frame_shape)[:, :, :3] + ) + cv2.imwrite( + 'mono-frame.data-False.png', + np.reshape(frame.data(False), frame_shape)[:, :, :3] + ) + cv2.imwrite( + 'mono-frame.data-False-0.png', + np.reshape(frame.data(False, 0), frame_shape)[:, :, :3] + ) + + elif self.current == 2: + # the two calls below save the two stereo frames, + # however the data needs to be reshaped, as the + # call to the data method yields a flat NumPy array + cv2.imwrite( + 'stereo-frame.data-False-0.png', + np.reshape(frame.data(False, 0), frame_shape)[:, :, :3] + ) + cv2.imwrite( + 'stereo-frame.data-False-1.png', + np.reshape(frame.data(False, 1), frame_shape)[:, :, :3] + ) + + elif self.current == 3: + # the two calls below save the two stereo frames, + # without the need for reshaping the data, as the + # call to the data method already yields a + # structured NumPy array + cv2.imwrite( + 'mono-frame.data-True.png', + frame.data(True)[:, :, :3] + ) + cv2.imwrite( + 'mono-frame.data-True-0.png', + frame.data(True, 0)[:, :, :3] + ) + + elif self.current == 4: + # the two calls below save the two stereo frames, + # without the need for reshaping the data, as the + # call to the data method already yields a + # structured NumPy array + cv2.imwrite( + 'stereo-frame.data-True-0.png', + frame.data(True, 0)[:, :, :3] + ) + cv2.imwrite( + 'stereo-frame.data-True-1.png', + frame.data(True, 1)[:, :, :3] + ) + + +if __name__ == '__main__': + sfac = VideoSourceFactory.get_instance() + source = sfac.get_device( + Device.DeckLink4KExtreme12G, ColourSpace.BGRA + ) + + saver = StereoFrameSaver() + + source.attach(saver) + + time.sleep(2) # operate pipeline for 2 sec + + source.detach(saver) diff --git a/src/tests/blackmagic/test_stereo.py b/src/tests/blackmagic/test_stereo.py new file mode 100644 index 00000000..e9103f95 --- /dev/null +++ b/src/tests/blackmagic/test_stereo.py @@ -0,0 +1,36 @@ +import time +from pytest import mark +try: + # in case of PyPI installation, this will work: + from giftgrab.tests.utils import (StereoFrameConsistencyChecker, + StereoFrameNumpyCompatibilityChecker, + StereoFrameBackwardsCompatibilityChecker) +except ImportError: + # in case of installation from source, this will work: + from utils import (StereoFrameConsistencyChecker, + StereoFrameNumpyCompatibilityChecker, + StereoFrameBackwardsCompatibilityChecker) +import pygiftgrab as pgg + + +@mark.stereo_frames +def test_stereo_frames(device, colour_space): + factory = pgg.VideoSourceFactory.get_instance() + source = factory.get_device(device, colour_space) + consistency_checker = StereoFrameConsistencyChecker() + numpy_checker = StereoFrameNumpyCompatibilityChecker(colour_space) + backwards_compatibility_checker = StereoFrameBackwardsCompatibilityChecker() + + source.attach(consistency_checker) + source.attach(numpy_checker) + source.attach(backwards_compatibility_checker) + + time.sleep(15) + + source.detach(consistency_checker) + source.detach(numpy_checker) + source.detach(backwards_compatibility_checker) + + assert consistency_checker + assert numpy_checker + assert backwards_compatibility_checker diff --git a/src/tests/run-tests.sh b/src/tests/run-tests.sh index 8a094944..a0aca4e7 100755 --- a/src/tests/run-tests.sh +++ b/src/tests/run-tests.sh @@ -98,6 +98,14 @@ elif [ "$1" = "numpy" ]; then test_cmd="$test_cmd $test_dir/videoframe -m numpy_compatibility" fi fi +elif [ "$1" = "stereo" ]; then + if [ $# -ne "2" ]; then + args_ok=false + else + parse_colour $2 + test_cmd="$test_cmd --colour-space=$test_colour_space" + test_cmd="$test_cmd $test_dir/videoframe -m stereo_frames" + fi elif [ "$1" = "epiphan-dvi2pcieduo" ]; then test_device=$1 test_device=${test_device:8} @@ -123,14 +131,25 @@ elif [ "$1" = "blackmagic-decklinksdi4k" ] || [ "$1" = "blackmagic-decklink4kext if [ $# -ne "2" ]; then args_ok=false else + if [ "$1" = "blackmagic-decklinksdi4k" ]; then + frame_rate=27 + elif [ "$1" = "blackmagic-decklink4kextreme12g" ]; then + frame_rate=22 + fi parse_colour $2 test_cmd="$test_cmd --device=$test_device" test_cmd="$test_cmd --colour-space=$test_colour_space" test_cmd_working_dir="$test_dir/blackmagic" test_cmd_unit="$test_cmd $test_cmd_working_dir -m unit" - test_cmd_observer="$test_cmd --frame-rate=27 --observers=3" + test_cmd_observer="$test_cmd --frame-rate=$frame_rate --observers=3" test_cmd_observer="$test_cmd_observer $test_cmd_working_dir -m observer_pattern" + if [ "$1" = "blackmagic-decklink4kextreme12g" ]; then + test_cmd_stereo="$test_cmd $test_cmd_working_dir -m stereo_frames" + fi test_cmd="$test_cmd_unit && $test_cmd_observer" + if [ "$1" = "blackmagic-decklink4kextreme12g" ]; then + test_cmd="$test_cmd && $test_cmd_stereo" + fi fi else args_ok=false diff --git a/src/tests/utils.py b/src/tests/utils.py index b6902816..f84d38e3 100644 --- a/src/tests/utils.py +++ b/src/tests/utils.py @@ -9,6 +9,137 @@ use_numpy = False +class StereoFrameBackwardsCompatibilityChecker(pgg.IObserver): + """Descendant of GIFT-Grab's `Observer`, which will + listen to `Observable`s for some time and when asked, + will report whether the video source has been sending + stereo frames that are backwards compatible with the + GIFT-Grab NumPy data interface. + """ + + def __init__(self): + super(StereoFrameBackwardsCompatibilityChecker, self).__init__() + self.obtained_backwards_compatible_frames = [] + + def update(self, frame): + frame_backwards_compatible = True + frame_backwards_compatible &= np.array_equal(frame.data(), frame.data(False)) + frame_backwards_compatible &= np.array_equal(frame.data(), frame.data(False, 0)) + frame_backwards_compatible &= frame.data_length() == frame.data_length(0) + self.obtained_backwards_compatible_frames.append(frame_backwards_compatible) + + def __bool__(self): + if not self.obtained_backwards_compatible_frames: + return False + for backwards_compatibility in self.obtained_backwards_compatible_frames: + if not backwards_compatibility: + return False + return True + + +class StereoFrameNumpyCompatibilityChecker(pgg.IObserver): + """Descendant of GIFT-Grab's `Observer`, which will + listen to `Observable`s for some time and when asked, + will report whether the video source has been sending + stereo frames that are compatible with the GIFT-Grab + NumPy data interface. + """ + + def __init__(self, colour): + super(StereoFrameNumpyCompatibilityChecker, self).__init__() + self.obtained_numpy_compatible_stereo_frames = [] + # currently structured NumPy arrays are supported + # only for BGRA frames + self.structured_flags = [colour == pgg.ColourSpace.BGRA] + if self.structured_flags[-1]: + self.structured_flags.append(False) + + def update(self, frame): + self.obtained_numpy_compatible_stereo_frames.append(True) + if frame.stereo_count() <= 1: + self.obtained_numpy_compatible_stereo_frames[-1] = False + return + + frames_numpy_compatible = True + + for structured_flag in self.structured_flags: + frames_numpy_compatible &= np.array_equal(frame.data(structured_flag), frame.data(structured_flag, 0)) + if not frames_numpy_compatible: + self.obtained_numpy_compatible_stereo_frames[-1] = False + return + + for index in range(frame.stereo_count()): + data_np = frame.data(structured_flag, index) + frames_numpy_compatible &= data_np.dtype == np.uint8 + data_len = frame.data_length(index) + frames_numpy_compatible &= data_len == data_np.size + if structured_flag: + frames_numpy_compatible &= data_np.shape[:2] == (frame.rows(), frame.cols()) + else: + try: + data_np[data_len] + except IndexError: + pass + else: + frames_numpy_compatible = False + if not frames_numpy_compatible: + self.obtained_numpy_compatible_stereo_frames[-1] = False + return + + self.obtained_numpy_compatible_stereo_frames[-1] = frames_numpy_compatible + + def __bool__(self): + if not self.obtained_numpy_compatible_stereo_frames: + return False + for numpy_compatibility in self.obtained_numpy_compatible_stereo_frames: + if not numpy_compatibility: + return False + return True + + +class StereoFrameConsistencyChecker(pgg.IObserver): + """Descendant of GIFT-Grab's `Observer`, which + will listen to `Observable`s for some time and + when asked, will report whether the video + source has been sending consistent stereo frames + consistently. + """ + + def __init__(self): + super(StereoFrameConsistencyChecker, self).__init__() + self.obtained_consistent_stereo_frames = [] + + def update(self, frame): + self.obtained_consistent_stereo_frames.append(True) + if frame.stereo_count() <= 1: + self.obtained_consistent_stereo_frames[-1] = False + return + + frames_consistent = True + for index in range(frame.stereo_count() - 1): + this_data = frame.data(False, index) + next_data = frame.data(False, index + 1) + if this_data.size == 0: + frames_consistent = False + break + if this_data.shape != next_data.shape: + frames_consistent = False + break + if np.array_equal(this_data, next_data): + frames_consistent = False + break + if not frames_consistent: + self.obtained_consistent_stereo_frames[-1] = False + + def __bool__(self): + if not self.obtained_consistent_stereo_frames: + return False + for consistency in self.obtained_consistent_stereo_frames: + if not consistency: + return False + return True + + class FrameRateTimer(pgg.IObserver): """Descendant of GIFT-Grab's `Observer`, which will listen to `Observable`s for some time and diff --git a/src/tests/videoframe/CMakeLists.txt b/src/tests/videoframe/CMakeLists.txt index dccad7c3..fae25006 100644 --- a/src/tests/videoframe/CMakeLists.txt +++ b/src/tests/videoframe/CMakeLists.txt @@ -2,6 +2,7 @@ if(USE_NUMPY) FILE(COPY ${CMAKE_SOURCE_DIR}/tests/videoframe/test_numpy_compatibility.py + ${CMAKE_SOURCE_DIR}/tests/videoframe/test_stereo.py ${CMAKE_SOURCE_DIR}/tests/videoframe/conftest.py DESTINATION ${CMAKE_CURRENT_BINARY_DIR} ) @@ -14,6 +15,14 @@ if(USE_NUMPY) LIST(APPEND TESTS_LIST ${NAME_TEST}) endforeach(COLOUR_SPACE) + foreach(COLOUR_SPACE ${COLOUR_SPACES}) + SET(NAME_TEST Test_VideoFrame_Stereo_${COLOUR_SPACE}) + ADD_TEST(NAME ${NAME_TEST} + COMMAND py.test --colour-space=${COLOUR_SPACE} test_stereo.py + ) + LIST(APPEND TESTS_LIST ${NAME_TEST}) + endforeach() + # to avoid copying stuff around SET_TESTS_PROPERTIES(${TESTS_LIST} PROPERTIES ENVIRONMENT "PYTHONPATH=${PYTHONPATH}" diff --git a/src/tests/videoframe/test_numpy_compatibility.py b/src/tests/videoframe/test_numpy_compatibility.py index 3b5df7b6..4d5e9238 100644 --- a/src/tests/videoframe/test_numpy_compatibility.py +++ b/src/tests/videoframe/test_numpy_compatibility.py @@ -4,6 +4,7 @@ frame = None +stereo_frame, stereo_count = None, 3 cols = 1920 rows = 1080 @@ -14,6 +15,10 @@ def peri_test(colour_space): frame = VideoFrame(colour_space, cols, rows) assert frame is not None + global stereo_frame, stereo_count + stereo_frame = VideoFrame(colour_space, cols, rows, stereo_count) + assert stereo_frame is not None + yield @@ -25,6 +30,14 @@ def test_data(): assert frame.data_length() == data_np.size +@mark.numpy_compatibility +def test_stereo_data(): + for stereo_index in range(stereo_count): + data_np = stereo_frame.data(False, stereo_index) + assert data_np.dtype == np.uint8 + assert stereo_frame.data_length(stereo_index) == data_np.size + + @mark.numpy_compatibility def test_data_length(): global frame, cols, rows @@ -40,6 +53,29 @@ def test_data_length(): fail(e.message) +@mark.numpy_compatibility +def test_stereo_data_length(): + global stereo_frame + + for stereo_index in range(stereo_count): + data_np = stereo_frame.data(False, stereo_index) + data_len = stereo_frame.data_length(stereo_index) + + with raises(IndexError): + data_np[data_len] + try: + data_np[data_len - 1] + except IndexError as e: + fail(e.message) + + +@mark.numpy_compatibility +def test_invalid_stereo_data_index_raises(): + for stereo_index in range(stereo_count, 2 * stereo_count): + with raises(IndexError): + stereo_frame.data(False, stereo_index) + + @mark.numpy_compatibility def test_read_access(colour_space): global frame diff --git a/src/tests/videoframe/test_stereo.py b/src/tests/videoframe/test_stereo.py new file mode 100644 index 00000000..71e82a5f --- /dev/null +++ b/src/tests/videoframe/test_stereo.py @@ -0,0 +1,44 @@ +from pytest import (mark, yield_fixture, raises) +from pygiftgrab import (VideoFrame, ColourSpace) + + +stereo_frame, stereo_count = None, None +data_length = 0 + + +@yield_fixture(autouse=True) +def peri_test(colour_space): + global stereo_frame, stereo_count, data_length + stereo_count = 2 + cols, rows = 1920, 1080 + stereo_frame = VideoFrame(colour_space, cols, rows, stereo_count) + data_length = VideoFrame.required_data_length(colour_space, cols, rows) + + +@mark.stereo_frames +def test_default_index_is_0(): + assert data_length == stereo_frame.data_length() + assert stereo_frame.data_length() == stereo_frame.data_length(0) + + +@mark.stereo_frames +def test_valid_index_returns_data_length(): + for stereo_index in range(stereo_count): + assert stereo_frame.data_length(stereo_index) == data_length + + +@mark.stereo_frames +def test_invalid_index_raises(): + for stereo_index in range(stereo_count, 2 * stereo_count): + with raises(IndexError): + stereo_frame.data_length(stereo_index) + + +@mark.stereo_frames +def test_stereo_frame_constructor(colour_space): + cols, rows = 1920, 1080 + frame = VideoFrame(colour_space, cols, rows) + assert frame.stereo_count() == 1 + for _stereo_count in range(2, 5): + frame = VideoFrame(colour_space, cols, rows, _stereo_count) + assert frame.stereo_count() == _stereo_count