From e7aa8efb91fb1ac82cb5bfcdc92222d9709f88a0 Mon Sep 17 00:00:00 2001 From: Awawa <69086569+awawa-dev@users.noreply.github.com> Date: Thu, 28 Nov 2024 14:17:14 +0100 Subject: [PATCH] Implementation of automatic tone mapping control --- include/base/AutomaticToneMapping.h | 11 +- include/base/Grabber.h | 1 + sources/base/AutomaticToneMapping.cpp | 32 +++-- sources/base/Grabber.cpp | 10 ++ sources/base/GrabberWrapper.cpp | 1 + .../schema/schema-automaticToneMapping.json | 2 +- .../flatbuffers/server/FlatBuffersServer.cpp | 9 +- sources/utils/FrameDecoder.cpp | 111 +++++++++++++++++- 8 files changed, 160 insertions(+), 17 deletions(-) diff --git a/include/base/AutomaticToneMapping.h b/include/base/AutomaticToneMapping.h index 09d4fbbd8..5b17fa83a 100644 --- a/include/base/AutomaticToneMapping.h +++ b/include/base/AutomaticToneMapping.h @@ -42,20 +42,24 @@ class AutomaticToneMapping AutomaticToneMapping* prepare(); void finilize(); void setConfig(bool enabled, const ToneMappingThresholds& newConfig, int timeInSec); + void setToneMapping(bool enabled); - constexpr void checkY(uint8_t& y) + constexpr uint8_t checkY(uint8_t y) { if (y > _running.y) _running.y = y; + return y; } - constexpr void checkU(uint8_t& u) + constexpr uint8_t checkU(uint8_t u) { if (u > _running.u) _running.u = u; + return u; } - constexpr void checkV(uint8_t& v) + constexpr uint8_t checkV(uint8_t v) { if (v > _running.v) _running.v = v; + return v; } struct ToneMappingThresholds { @@ -72,6 +76,7 @@ class AutomaticToneMapping bool _modeSDR; long _startedTime; + int _gracefulTimeout; Logger* _log; }; diff --git a/include/base/Grabber.h b/include/base/Grabber.h index 3f2f3c1f1..6fe994856 100644 --- a/include/base/Grabber.h +++ b/include/base/Grabber.h @@ -127,6 +127,7 @@ class Grabber : public DetectionAutomatic, public DetectionManual, protected Lut QString getConfigurationPath(); void setAutomaticToneMappingConfig(bool enabled, const AutomaticToneMapping::ToneMappingThresholds& newConfig, int timeInSec); + void setAutoToneMappingCurrentStateEnabled(bool enabled); struct DevicePropertiesItem { diff --git a/sources/base/AutomaticToneMapping.cpp b/sources/base/AutomaticToneMapping.cpp index 4a2005571..c7b51e776 100644 --- a/sources/base/AutomaticToneMapping.cpp +++ b/sources/base/AutomaticToneMapping.cpp @@ -32,11 +32,16 @@ #include #include -AutomaticToneMapping::AutomaticToneMapping(): - _enabled(false), +#define DEFAULT_GRACEFUL_TIMEOUT 10 + +AutomaticToneMapping::AutomaticToneMapping() : + _enabled(false), _timeInSec(30), + _config{}, + _running{}, _modeSDR(true), _startedTime(0), + _gracefulTimeout(DEFAULT_GRACEFUL_TIMEOUT), _log(Logger::getInstance(QString("AUTOTONEMAPPING"))) { } @@ -44,8 +49,7 @@ AutomaticToneMapping::AutomaticToneMapping(): AutomaticToneMapping* AutomaticToneMapping::prepare() { if (_enabled) - { - _running = _config; + { return this; } else @@ -60,7 +64,7 @@ void AutomaticToneMapping::finilize() { bool triggered = (_config.y != _running.y || _config.u != _running.u || _config.v != _running.v); - if (triggered && !_modeSDR) + if (triggered && !_modeSDR && _gracefulTimeout-- <= 0) { QString message = "Tone mapping OFF triggered by: "; if (_config.y != _running.y) @@ -85,22 +89,32 @@ void AutomaticToneMapping::finilize() else if (_startedTime + _timeInSec < now) { _modeSDR = false; - QString message = "Tone mapping ON triggered by configured time"; + Info(_log, "Tone mapping ON triggered by configured time"); GlobalSignals::getInstance()->SignalRequestComponent(hyperhdr::Components::COMP_HDR, -1, true); } } + + if (!triggered) + _gracefulTimeout = DEFAULT_GRACEFUL_TIMEOUT; + + _running = _config; } } void AutomaticToneMapping::setConfig(bool enabled, const ToneMappingThresholds& newConfig, int timeInSec) { - _modeSDR = true; - _startedTime = 0; - _enabled = enabled; _config = newConfig; _timeInSec = timeInSec; + _running = _config; Info(_log, "Enabled: %s, Time: %i, Thresholds: %i, %i, %i", (enabled) ? "yes" : "no", timeInSec, _config.y, _config.u, _config.v); } +void AutomaticToneMapping::setToneMapping(bool enabled) +{ + _modeSDR = !enabled; + _startedTime = 0; + _gracefulTimeout = DEFAULT_GRACEFUL_TIMEOUT; +} + diff --git a/sources/base/Grabber.cpp b/sources/base/Grabber.cpp index 5f637e2e6..9974b26bd 100644 --- a/sources/base/Grabber.cpp +++ b/sources/base/Grabber.cpp @@ -970,4 +970,14 @@ void Grabber::signalSetLutHandler(MemoryBuffer* lut) void Grabber::setAutomaticToneMappingConfig(bool enabled, const AutomaticToneMapping::ToneMappingThresholds& newConfig, int timeInSec) { _automaticToneMapping.setConfig(enabled, newConfig, timeInSec); + if (_automaticToneMapping.prepare() && !_qframe) + Error(_log, "Automatic tone mapping requires 'Quarter of frame' mode enabled"); + if (_automaticToneMapping.prepare() && (_enc != PixelFormat::YUYV && _enc != PixelFormat::NV12 && _enc != PixelFormat::P010 )) + Warning(_log, "Automatic tone mapping requires YUYV/NV12/P010 video format"); + +} + +void Grabber::setAutoToneMappingCurrentStateEnabled(bool enabled) +{ + _automaticToneMapping.setToneMapping(enabled); } diff --git a/sources/base/GrabberWrapper.cpp b/sources/base/GrabberWrapper.cpp index bee904bcb..2f9db2c5b 100644 --- a/sources/base/GrabberWrapper.cpp +++ b/sources/base/GrabberWrapper.cpp @@ -344,6 +344,7 @@ void GrabberWrapper::setHdrToneMappingEnabled(int mode) if (_grabber != NULL) { _grabber->setHdrToneMappingEnabled(mode); + _grabber->setAutoToneMappingCurrentStateEnabled(mode); } } diff --git a/sources/base/schema/schema-automaticToneMapping.json b/sources/base/schema/schema-automaticToneMapping.json index 187d80ef8..94cf81b07 100644 --- a/sources/base/schema/schema-automaticToneMapping.json +++ b/sources/base/schema/schema-automaticToneMapping.json @@ -20,7 +20,7 @@ "title" : "edt_automatic_tone_mapping_y_threshold_title", "minimum" : 150, "maximum" : 255, - "default" : 155, + "default" : 180, "required" : true, "options": { "dependencies": { diff --git a/sources/flatbuffers/server/FlatBuffersServer.cpp b/sources/flatbuffers/server/FlatBuffersServer.cpp index c58523d7d..a0c88da0a 100644 --- a/sources/flatbuffers/server/FlatBuffersServer.cpp +++ b/sources/flatbuffers/server/FlatBuffersServer.cpp @@ -348,6 +348,12 @@ void FlatBuffersServer::handlerImageReceived(int priority, FlatBuffersParser::Fl Image image(flatImage->width, flatImage->height); memmove(image.rawMem(), flatImage->firstPlane.data, flatImage->size); + if (_hdrToneMappingEnabled && !_lutBufferInit) + { + emit GlobalSignals::getInstance()->SignalLutRequest(); + Error(_log, "The LUT file is not loaded. A new LUT was requested. It usually takes less than a minute"); + } + if (getHdrToneMappingEnabled()) FrameDecoder::applyLUT((uint8_t*)image.rawMem(), image.width(), image.height(), _lut.data(), getHdrToneMappingEnabled()); @@ -369,7 +375,8 @@ void FlatBuffersServer::handlerImageReceived(int priority, FlatBuffersParser::Fl if (!_lutBufferInit) { - Error(_log, "The LUT file is not loaded"); + emit GlobalSignals::getInstance()->SignalLutRequest(); + Error(_log, "The LUT file is not loaded. A new LUT was requested. It usually takes less than a minute"); } else if (flatImage->size != ((flatImage->width * flatImage->height * 3) / 2) || flatImage->size == 0) { diff --git a/sources/utils/FrameDecoder.cpp b/sources/utils/FrameDecoder.cpp index 6ffea2fa5..5b053e461 100644 --- a/sources/utils/FrameDecoder.cpp +++ b/sources/utils/FrameDecoder.cpp @@ -524,7 +524,7 @@ void FrameDecoder::processQImage( int destLineSize = outputImage.width() * 3; - if (pixelFormat == PixelFormat::YUYV) + if (pixelFormat == PixelFormat::YUYV && automaticToneMapping == nullptr) { for (int yDest = 0, ySource = 0; yDest < outputHeight; ySource += 2, ++yDest) { @@ -545,6 +545,28 @@ void FrameDecoder::processQImage( } return; } + if (pixelFormat == PixelFormat::YUYV && automaticToneMapping != nullptr) + { + for (int yDest = 0, ySource = 0; yDest < outputHeight; ySource += 2, ++yDest) + { + uint8_t* currentDest = destMemory + ((uint64_t)destLineSize) * yDest; + uint8_t* endDest = currentDest + destLineSize; + uint8_t* currentSource = (uint8_t*)data + (((uint64_t)lineLength * ySource)); + + while (currentDest < endDest) + { + *((uint32_t*)&buffer) = *((uint32_t*)currentSource); + + ind_lutd = LUT_INDEX((automaticToneMapping->checkY(buffer[0])), (automaticToneMapping->checkU(buffer[1])), (automaticToneMapping->checkV(buffer[3]))); + + *((uint32_t*)currentDest) = *((uint32_t*)(&lutBuffer[ind_lutd])); + currentDest += 3; + currentSource += 4; + } + } + automaticToneMapping->finilize(); + return; + } if (pixelFormat == PixelFormat::UYVY) { @@ -662,7 +684,7 @@ void FrameDecoder::processQImage( return; } - if (pixelFormat == PixelFormat::P010) + if (pixelFormat == PixelFormat::P010 && automaticToneMapping == nullptr) { uint16_t p010[2] = {}; @@ -713,8 +735,65 @@ void FrameDecoder::processQImage( } return; } + if (pixelFormat == PixelFormat::P010 && automaticToneMapping != nullptr) + { + uint16_t p010[2] = {}; - if (pixelFormat == PixelFormat::NV12) + if (!FrameDecoderUtils::initialized) + { + initP010(); + } + + uint8_t* deltaUV = (dataUV != nullptr) ? (uint8_t*)dataUV : (uint8_t*)data + lineLength * height; + for (int yDest = 0, ySource = 0; yDest < outputHeight; ySource += 2, ++yDest) + { + uint8_t* currentDest = destMemory + ((uint64_t)destLineSize) * yDest; + uint8_t* endDest = currentDest + destLineSize; + uint8_t* currentSource = (uint8_t*)data + (((uint64_t)lineLength * ySource)); + uint8_t* currentSourceU = deltaUV + (((uint64_t)ySource / 2) * lineLength); + + while (currentDest < endDest) + { + memcpy(((uint16_t*)&p010), ((uint16_t*)currentSource), 2); + if (toneMapping) + { + automaticToneMapping->checkY(p010[0] >> 8); + + buffer[0] = lutP010_y[p010[0] >> 6]; + } + else + { + buffer[0] = automaticToneMapping->checkY(p010[0] >> 8); + } + currentSource += 4; + memcpy(((uint32_t*)&p010), ((uint32_t*)currentSourceU), 4); + if (toneMapping) + { + automaticToneMapping->checkU(p010[0] >> 8); + automaticToneMapping->checkV(p010[1] >> 8); + + buffer[2] = lutP010_uv[p010[0] >> 6]; + buffer[3] = lutP010_uv[p010[1] >> 6]; + } + else + { + buffer[2] = automaticToneMapping->checkU(p010[0] >> 8); + buffer[3] = automaticToneMapping->checkV(p010[1] >> 8); + } + + currentSourceU += 4; + + ind_lutd = LUT_INDEX(buffer[0], buffer[2], buffer[3]); + + *((uint32_t*)currentDest) = *((uint32_t*)(&lutBuffer[ind_lutd])); + currentDest += 3; + } + } + automaticToneMapping->finilize(); + return; + } + + if (pixelFormat == PixelFormat::NV12 && automaticToneMapping == nullptr) { uint8_t* deltaUV = (dataUV != nullptr) ? (uint8_t*)dataUV : (uint8_t*)data + lineLength * height; for (int yDest = 0, ySource = 0; yDest < outputHeight; ySource += 2, ++yDest) @@ -739,6 +818,32 @@ void FrameDecoder::processQImage( } return; } + if (pixelFormat == PixelFormat::NV12 && automaticToneMapping != nullptr) + { + uint8_t* deltaUV = (dataUV != nullptr) ? (uint8_t*)dataUV : (uint8_t*)data + lineLength * height; + for (int yDest = 0, ySource = 0; yDest < outputHeight; ySource += 2, ++yDest) + { + uint8_t* currentDest = destMemory + ((uint64_t)destLineSize) * yDest; + uint8_t* endDest = currentDest + destLineSize; + uint8_t* currentSource = (uint8_t*)data + (((uint64_t)lineLength * ySource)); + uint8_t* currentSourceU = deltaUV + (((uint64_t)ySource / 2) * lineLength); + + while (currentDest < endDest) + { + *((uint8_t*)&buffer) = *((uint8_t*)currentSource); + currentSource += 2; + *((uint16_t*)&(buffer[2])) = *((uint16_t*)currentSourceU); + currentSourceU += 2; + + ind_lutd = LUT_INDEX((automaticToneMapping->checkY(buffer[0])), (automaticToneMapping->checkU(buffer[2])), (automaticToneMapping->checkV(buffer[3]))); + + *((uint32_t*)currentDest) = *((uint32_t*)(&lutBuffer[ind_lutd])); + currentDest += 3; + } + } + automaticToneMapping->finilize(); + return; + } }