From c88510df141defd568cc1791454aa81b740d0604 Mon Sep 17 00:00:00 2001 From: Jonathan Lennox Date: Mon, 6 Jan 2025 15:19:24 -0500 Subject: [PATCH] Handle the case where VLA-provided frame rate is less than the onstage preferred frame rate. Correctly calculate temporal layers' frame rates from VLA. --- .../nlj/rtp/codec/av1/Av1DDRtpLayerDesc.kt | 3 +-- .../jitsi/nlj/rtp/codec/vpx/VpxRtpLayerDesc.kt | 2 +- .../transform/node/incoming/VlaReaderNode.kt | 10 +++++++--- .../cc/allocation/BandwidthAllocator.kt | 2 +- .../cc/allocation/BitrateController.kt | 6 ++++++ .../cc/allocation/SingleSourceAllocation.kt | 17 +++++++++++++++-- 6 files changed, 31 insertions(+), 9 deletions(-) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDRtpLayerDesc.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDRtpLayerDesc.kt index 2d1eccbf0b..aeb07634ca 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDRtpLayerDesc.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDRtpLayerDesc.kt @@ -79,8 +79,7 @@ class Av1DDRtpLayerDesc( * {@inheritDoc} */ override fun toString(): String { - return "subjective_quality=" + index + - ",DT=" + dt + return "subjective_quality=$index,DT=$dt,height=$height,frameRate=$frameRate" } companion object { diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vpx/VpxRtpLayerDesc.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vpx/VpxRtpLayerDesc.kt index 0244e0bd1a..ace8521fb2 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vpx/VpxRtpLayerDesc.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vpx/VpxRtpLayerDesc.kt @@ -118,7 +118,7 @@ constructor( * {@inheritDoc} */ override fun toString(): String { - return "subjective_quality=$index,temporal_id=$tid,spatial_id=$sid,height=$height" + return "subjective_quality=$index,temporal_id=$tid,spatial_id=$sid,height=$height,frameRate=$frameRate" } /** diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VlaReaderNode.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VlaReaderNode.kt index b4375482de..b2eab047cd 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VlaReaderNode.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VlaReaderNode.kt @@ -76,22 +76,26 @@ class VlaReaderNode( vla.forEachIndexed { streamIdx, stream -> val rtpEncoding = sourceDesc?.rtpEncodings?.get(streamIdx) stream.spatialLayers.forEach { spatialLayer -> + val maxTl = spatialLayer.targetBitratesKbps.size - 1 + spatialLayer.targetBitratesKbps.forEachIndexed { tlIdx, targetBitrateKbps -> rtpEncoding?.layers?.find { // With VP8 simulcast all layers have sid -1 (it.sid == spatialLayer.id || it.sid == -1) && it.tid == tlIdx }?.let { layer -> - logger.debug( + logger.debug { "Setting target bitrate for rtpEncoding=$rtpEncoding layer=$layer to " + "${targetBitrateKbps.kbps} (res=${spatialLayer.res})" - ) + } layer.targetBitrate = targetBitrateKbps.kbps spatialLayer.res?.let { res -> if (layer.height > 0 && layer.height != res.height) { logger.warn("Updating layer height from ${layer.height} to ${res.height}") } layer.height = res.height - layer.frameRate = res.maxFramerate.toDouble() + /* Presume 2:1 frame rate ratios for temporal layers */ + val framerateFraction = 1.0 / (1 shl (maxTl - tlIdx)) + layer.frameRate = res.maxFramerate.toDouble() * framerateFraction } } } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt index d3732502f6..77a4ec7800 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt @@ -174,7 +174,7 @@ internal class BandwidthAllocator( logger.trace { "Allocating: sortedSources=${sortedSources.map { it.sourceName }}, " + - " effectiveConstraints=${newEffectiveConstraints.map { "${it.key.sourceName}=${it.value}" }}" + "effectiveConstraints=${newEffectiveConstraints.map { "${it.key.sourceName}=${it.value}" }}" } // Compute the bandwidth allocation. diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt index 8aaa5d9f64..9ccb8f519b 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt @@ -26,6 +26,7 @@ import org.jitsi.utils.event.SyncEventEmitter import org.jitsi.utils.logging.DiagnosticContext import org.jitsi.utils.logging.TimeSeriesLogger import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger import org.jitsi.utils.secs import org.jitsi.videobridge.cc.config.BitrateControllerConfig.Companion.config import org.jitsi.videobridge.message.ReceiverVideoConstraintsMessage @@ -53,6 +54,8 @@ class BitrateController @JvmOverloads constructor( ) { val eventEmitter = SyncEventEmitter() + private val logger = createChildLogger(parentLogger) + private val bitrateAllocatorEventHandler = BitrateAllocatorEventHandler() /** @@ -133,6 +136,9 @@ class BitrateController @JvmOverloads constructor( fun accept(packetInfo: PacketInfo): Boolean { if (packetInfo.layeringChanged) { // This needs to be done synchronously, so it's complete before the accept, below. + logger.debug { + "Layering information changed for packet from ${packetInfo.endpointId}, updating bandwidth allocation" + } bandwidthAllocator.update() } return packetHandler.accept(packetInfo) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt index 12f730edaf..6a15b60746 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt @@ -26,6 +26,7 @@ import org.jitsi.utils.logging2.LoggerImpl import org.jitsi.videobridge.cc.config.BitrateControllerConfig.Companion.config import java.lang.Integer.max import java.time.Clock +import kotlin.math.min /** * A bitrate allocation that pertains to a specific source. This is the internal representation used in the allocation @@ -68,6 +69,7 @@ internal class SingleSourceAllocation( } timeSeriesLogger.trace(ratesTimeSeriesPoint) } + logger.addContext(mapOf("remote_endpoint_id" to endpointId)) } fun isOnStage() = onStage @@ -294,9 +296,15 @@ internal class SingleSourceAllocation( * oversending. */ private fun selectLayersForCamera(layers: List, constraints: VideoConstraints): Layers { - val minHeight = layers.map { it.layer.height }.minOrNull() ?: return Layers.noLayers + val minHeight = layers.minOfOrNull { it.layer.height } ?: return Layers.noLayers + val maxFps = layers.maxOfOrNull { it.layer.frameRate } ?: return Layers.noLayers val noActiveLayers = layers.none { (_, bitrate) -> bitrate > 0 } val (preferredHeight, preferredFps) = getPreferred(constraints) + val effectivePreferredFps = if (maxFps > 0) { + min(maxFps, preferredFps) + } else { + preferredFps + } val ratesList: MutableList = ArrayList() // Initialize the list of layers to be considered. These are the layers that satisfy the constraints, with @@ -306,7 +314,7 @@ internal class SingleSourceAllocation( val lessThanPreferredHeight = layer.height < preferredHeight val lessThanOrEqualMaxHeight = layer.height <= constraints.maxHeight || !constraints.heightIsLimited() // If frame rate is unknown, consider it to be sufficient. - val atLeastPreferredFps = layer.frameRate < 0 || layer.frameRate >= preferredFps + val atLeastPreferredFps = layer.frameRate < 0 || layer.frameRate >= effectivePreferredFps if (lessThanPreferredHeight || (lessThanOrEqualMaxHeight && atLeastPreferredFps) || layer.height == minHeight @@ -321,6 +329,11 @@ internal class SingleSourceAllocation( val effectivePreferredHeight = max(preferredHeight, minHeight) val preferredIndex = ratesList.lastIndexWhich { it.layer.height <= effectivePreferredHeight } + logger.trace { + "Selected rates list $ratesList, preferred index $preferredIndex " + + "from layers $layers with constraints $constraints" + } + return Layers(ratesList, preferredIndex, -1) }