From f9975f948048e6976782288ae536733d9006f671 Mon Sep 17 00:00:00 2001 From: Andrei Karasev Date: Wed, 27 Apr 2022 12:07:59 -0400 Subject: [PATCH 01/13] AV1 add Dependency Descriptor extension to enum --- .../src/main/kotlin/org/jitsi/nlj/rtp/RtpExtensions.kt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RtpExtensions.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RtpExtensions.kt index dc730843ee..2922ebd9d0 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RtpExtensions.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RtpExtensions.kt @@ -96,7 +96,14 @@ enum class RtpExtensionType(val uri: String) { /** * The URN which identifies the RTP Header Extension for Video Orientation. */ - VIDEO_ORIENTATION("urn:3gpp:video-orientation"); + VIDEO_ORIENTATION("urn:3gpp:video-orientation"), + + /** + * [VOWEL] Dependency Descriptor RTP Header Extension + */ + AV1_DEPENDENCY_DESCRIPTOR( + "https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension" + ); companion object { private val uriMap = RtpExtensionType.values().associateBy(RtpExtensionType::uri) From 8cac7204f82c0123c2bbc0283c5fce8df8572506 Mon Sep 17 00:00:00 2001 From: Andrei Karasev Date: Wed, 27 Apr 2022 21:34:25 -0400 Subject: [PATCH 02/13] add av1 enum and payload subtype --- .../src/main/kotlin/org/jitsi/nlj/format/PayloadType.kt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/format/PayloadType.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/format/PayloadType.kt index bb29e90f91..e6bd3ce1eb 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/format/PayloadType.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/format/PayloadType.kt @@ -74,6 +74,7 @@ enum class PayloadTypeEncoding { VP8, VP9, H264, + AV1, RED, RTX, OPUS; @@ -124,6 +125,12 @@ class H264PayloadType( rtcpFeedbackSet: RtcpFeedbackSet = emptySet() ) : VideoPayloadType(pt, PayloadTypeEncoding.H264, parameters = parameters, rtcpFeedbackSet = rtcpFeedbackSet) +class Av1PayloadType( + pt: Byte, + parameters: PayloadTypeParams = ConcurrentHashMap(), + rtcpFeedbackSet: RtcpFeedbackSet = emptySet() +) : VideoPayloadType(pt, PayloadTypeEncoding.AV1, parameters = parameters, rtcpFeedbackSet = rtcpFeedbackSet) + class RtxPayloadType( pt: Byte, parameters: PayloadTypeParams = ConcurrentHashMap() From 741b0005c64a011484be2e084d7bd1a873ff71bb Mon Sep 17 00:00:00 2001 From: Jack Z <8870013+j000z@users.noreply.github.com> Date: Thu, 28 Apr 2022 04:31:30 -0500 Subject: [PATCH 03/13] AV1 Dependency Descriptors --- .../transform/node/incoming/VideoParser.kt | 17 ++ .../nlj/rtp/codec/av1/Av1PacketConverter.kt | 59 +++++ .../org/jitsi/nlj/rtp/codec/av1/Av1Parser.kt | 39 ++++ .../org/jitsi/nlj/rtp/codec/av1/Av1packet.kt | 72 ++++++ .../jitsi/nlj/rtp/codec/av1/dd/BytesView.kt | 39 ++++ .../codec/av1/dd/DecodeTargetIndication.kt | 20 ++ .../rtp/codec/av1/dd/DependencyDescriptor.kt | 19 ++ .../av1/dd/DependencyDescriptorReader.kt | 212 ++++++++++++++++++ .../codec/av1/dd/FrameDependencyStructure.kt | 18 ++ .../codec/av1/dd/FrameDependencyTemplate.kt | 19 ++ .../nlj/rtp/codec/av1/dd/NextLayerIdc.kt | 16 ++ .../jitsi/nlj/rtp/codec/av1/dd/Resolution.kt | 3 + 12 files changed, 533 insertions(+) create mode 100644 src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1PacketConverter.kt create mode 100644 src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1Parser.kt create mode 100644 src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1packet.kt create mode 100644 src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/BytesView.kt create mode 100644 src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DecodeTargetIndication.kt create mode 100644 src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptor.kt create mode 100644 src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptorReader.kt create mode 100644 src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/FrameDependencyStructure.kt create mode 100644 src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/FrameDependencyTemplate.kt create mode 100644 src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/NextLayerIdc.kt create mode 100644 src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/Resolution.kt diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoParser.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoParser.kt index a31326f595..b48fbeb607 100644 --- a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoParser.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoParser.kt @@ -19,9 +19,12 @@ import org.jitsi.nlj.Event import org.jitsi.nlj.MediaSourceDesc import org.jitsi.nlj.PacketInfo import org.jitsi.nlj.SetMediaSourcesEvent +import org.jitsi.nlj.format.Av1PayloadType import org.jitsi.nlj.format.Vp8PayloadType import org.jitsi.nlj.format.Vp9PayloadType import org.jitsi.nlj.rtp.codec.VideoCodecParser +import org.jitsi.nlj.rtp.codec.av1.Av1PacketConverter +import org.jitsi.nlj.rtp.codec.av1.Av1Parser import org.jitsi.nlj.rtp.codec.vp8.Vp8Packet import org.jitsi.nlj.rtp.codec.vp8.Vp8Parser import org.jitsi.nlj.rtp.codec.vp9.Vp9Packet @@ -50,6 +53,7 @@ class VideoParser( private var signaledSources: Array = sources private var videoCodecParser: VideoCodecParser? = null + private val av1PacketConverter = Av1PacketConverter(streamInformationStore) override fun transform(packetInfo: PacketInfo): PacketInfo? { val packet = packetInfo.packetAs() @@ -90,6 +94,19 @@ class VideoParser( } vp9Packet } + is Av1PayloadType -> { + val av1packet = av1PacketConverter.parse(packet) + packetInfo.packet = av1packet + packetInfo.resetPayloadVerification() + + if (videoCodecParser !is Av1Parser) { + resetSources() + packetInfo.layeringChanged = true + videoCodecParser = Av1Parser(sources, logger) + } + + av1packet + } else -> { if (videoCodecParser != null) { logger.cdebug { diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1PacketConverter.kt b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1PacketConverter.kt new file mode 100644 index 0000000000..880eef99fa --- /dev/null +++ b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1PacketConverter.kt @@ -0,0 +1,59 @@ +/* + * Copyright @ 2023 - present 8x8, Inc. + * Copyright @ 2023 - Vowel, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtp.codec.av1 + +import org.jitsi.nlj.rtp.RtpExtensionType +import org.jitsi.nlj.rtp.codec.av1.dd.DependencyDescriptorReader +import org.jitsi.nlj.rtp.codec.av1.dd.FrameDependencyStructure +import org.jitsi.nlj.util.ReadOnlyStreamInformationStore +import org.jitsi.rtp.rtp.RtpPacket +import java.util.concurrent.ConcurrentHashMap + +class Av1PacketConverter(val streamInformationStore: ReadOnlyStreamInformationStore) { + private var ddExtId: Int? = null + private var structures = ConcurrentHashMap() + + init { + streamInformationStore.onRtpExtensionMapping(RtpExtensionType.AV1_DEPENDENCY_DESCRIPTOR) { + ddExtId = it + } + } + + fun parse(rtpPacket: RtpPacket): Av1packet { + val ssrc = rtpPacket.ssrc + val lastStructure = structures[ssrc] + + val extId = checkNotNull(ddExtId) { "missing dd ext id" } + val ddExt = checkNotNull(rtpPacket.getHeaderExtension(extId)) { "missing dd ext from $rtpPacket" } + val (descriptor, structure) = DependencyDescriptorReader(ddExt, lastStructure).parse() + + structures[ssrc] = structure + + return Av1packet( + rtpPacket.buffer, + rtpPacket.offset, + rtpPacket.length, + isKeyframe = true, + isStartOfFrame = descriptor.firstPacketInFrame, + isEndOfFrame = descriptor.lastPacketInFrame, + frameNumber = descriptor.frameNumber, + temporalLayerIndex = descriptor.frameDependencies!!.temporalId, + spatialLayerIndex = descriptor.frameDependencies!!.spatialId + ) + } +} diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1Parser.kt b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1Parser.kt new file mode 100644 index 0000000000..d941eca454 --- /dev/null +++ b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1Parser.kt @@ -0,0 +1,39 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtp.codec.av1 + +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.rtp.codec.VideoCodecParser +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger + +class Av1Parser( + sources: Array, + parentLogger: Logger +) : VideoCodecParser(sources) { + private val logger = createChildLogger(parentLogger) + + /** Encodings we've actually seen. Used to clear out inferred-from-signaling encoding information. */ + private val ssrcsSeen = HashSet() + + override fun parse(packetInfo: PacketInfo) { + val av1packet = packetInfo.packetAs() + + ssrcsSeen.add(av1packet.ssrc) + } +} diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1packet.kt b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1packet.kt new file mode 100644 index 0000000000..f13bd78459 --- /dev/null +++ b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1packet.kt @@ -0,0 +1,72 @@ +/* + * Copyright @ 2023 - present 8x8, Inc. + * Copyright @ 2023 - Vowel, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtp.codec.av1 + +import org.jitsi.nlj.RtpLayerDesc +import org.jitsi.nlj.rtp.ParsedVideoPacket +import org.jitsi.rtp.extensions.bytearray.hashCodeOfSegment +import org.jitsi.utils.logging2.createLogger +import org.jitsi_modified.impl.neomedia.codec.video.vp9.DePacketizer + +class Av1packet( + buffer: ByteArray, + offset: Int, + length: Int, + override val isKeyframe: Boolean, + override val isStartOfFrame: Boolean, + override val isEndOfFrame: Boolean, + val frameNumber: Int, + val temporalLayerIndex: Int, + val spatialLayerIndex: Int +) : ParsedVideoPacket(buffer, offset, length, null) { + + override val layerId: Int + get() = RtpLayerDesc.getIndex(0, spatialLayerIndex, temporalLayerIndex) + + override val payloadVerification: String + get() { + val rtpPayloadLength = payloadLength + val rtpPayloadOffset = payloadOffset + val vp9pdSize = DePacketizer.VP9PayloadDescriptor.getSize(buffer, rtpPayloadOffset, rtpPayloadLength) + val vp9PayloadLength = rtpPayloadLength - vp9pdSize + val hashCode = buffer.hashCodeOfSegment(rtpPayloadOffset + vp9pdSize, rtpPayloadOffset + rtpPayloadLength) + return "type=AV1Packet len=$vp9PayloadLength hashCode=$hashCode" + } + + override fun toString(): String { + return super.toString() + ", SID=$spatialLayerIndex, TID=$temporalLayerIndex, FrameNumber=$frameNumber" + } + + override fun clone(): Av1packet { + return Av1packet( + cloneBuffer(BYTES_TO_LEAVE_AT_START_OF_PACKET), + BYTES_TO_LEAVE_AT_START_OF_PACKET, + length, + isKeyframe = isKeyframe, + isStartOfFrame = isStartOfFrame, + isEndOfFrame = isEndOfFrame, + frameNumber = frameNumber, + temporalLayerIndex = temporalLayerIndex, + spatialLayerIndex = spatialLayerIndex + ) + } + + companion object { + private val logger = createLogger() + } +} diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/BytesView.kt b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/BytesView.kt new file mode 100644 index 0000000000..e3d4656aee --- /dev/null +++ b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/BytesView.kt @@ -0,0 +1,39 @@ +package org.jitsi.nlj.rtp.codec.av1.dd + +import org.apache.commons.compress.utils.BitInputStream +import org.jitsi.rtp.rtp.RtpPacket +import java.io.ByteArrayInputStream +import java.nio.ByteOrder + +class BytesView(val bytes: ByteArray, val offset: Int, val length: Int) { + private val bitStream = BitInputStream(ByteArrayInputStream(bytes, offset, length), ByteOrder.BIG_ENDIAN) + constructor(ext: RtpPacket.HeaderExtension) : this( + ext.currExtBuffer, + // TODO what about 2 bytes header + ext.currExtOffset + 1, + ext.currExtLength - 1 + ) + + fun readBoolean(): Boolean { + return readInt(1) == 1 + } + + fun readInt(bitCount: Int): Int { + return bitStream.readBits(bitCount).toInt() + } + + fun readNonSymmetric(n: Int): Int { + var w = 0 + var x = n + while (x != 0) { + x = x shr 1 + w++ + } + val m = (1 shl w) - n + val v = readInt(w - 1) + if (v < m) + return v + val extraBit = readInt(1) + return (v shl 1) - m + extraBit + } +} diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DecodeTargetIndication.kt b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DecodeTargetIndication.kt new file mode 100644 index 0000000000..c750c71613 --- /dev/null +++ b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DecodeTargetIndication.kt @@ -0,0 +1,20 @@ +package org.jitsi.nlj.rtp.codec.av1.dd + +enum class DecodeTargetIndication(val id: Int) { + // DecodeTargetInfo symbol '-' + NOT_PRESENT(0), + // DecodeTargetInfo symbol 'D' + DISCARDABLE(1), + // DecodeTargetInfo symbol 'S' + SWITCH(2), + // DecodeTargetInfo symbol 'R' + REQUIRED(3); + + companion object { + private val map = values().associateBy { it.id } + + fun parse(id: Int): DecodeTargetIndication { + return map.getValue(id) + } + } +} diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptor.kt b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptor.kt new file mode 100644 index 0000000000..68b97e58bf --- /dev/null +++ b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptor.kt @@ -0,0 +1,19 @@ +package org.jitsi.nlj.rtp.codec.av1.dd + +data class DependencyDescriptor( + var firstPacketInFrame: Boolean = true, + var lastPacketInFrame: Boolean = true, + var frameNumber: Int = 0, + var frameDependencies: FrameDependencyTemplate? = null, + var resolution: Resolution? = null, + // uint32_t + var activeDecodeTargetsBitmask: Int? = null, + var attachedStructure: FrameDependencyStructure? = null +) { + companion object { + const val kMaxSpatialIds = 4 + const val kMaxTemporalIds = 8 + const val kMaxDecodeTargets = 32 + const val kMaxTemplates = 64 + } +} diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptorReader.kt b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptorReader.kt new file mode 100644 index 0000000000..c9dd6ef81d --- /dev/null +++ b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptorReader.kt @@ -0,0 +1,212 @@ +package org.jitsi.nlj.rtp.codec.av1.dd + +import org.jitsi.rtp.rtp.RtpPacket + +class DependencyDescriptorReader( + ext: RtpPacket.HeaderExtension, + private var structure: FrameDependencyStructure? +) { + private val rawData = BytesView(ext) + private var frameDependencyTemplateId = 0 + private var activeDecodeTargetsPresentFlag = false + private var customDTIsFlag = false + private var customFDiffsFlag = false + private var customChainsFlag = false + + fun parse(): Pair { + val descriptor = DependencyDescriptor() + + readMandatoryFields(descriptor) + + if (rawData.length > 3) { + readExtendedFields(descriptor) + } else { + customDTIsFlag = false + customFDiffsFlag = false + customChainsFlag = false + } + + val nonNullStructure = checkNotNull(descriptor.attachedStructure ?: structure) + + if (activeDecodeTargetsPresentFlag) { + descriptor.activeDecodeTargetsBitmask = rawData.readInt(nonNullStructure.numDecodeTargets) + } + + readFrameDependencyDefinition(descriptor, nonNullStructure) + + return descriptor to nonNullStructure + } + + private fun readExtendedFields(descriptor: DependencyDescriptor) { + val templateDependencyStructurePresentFlag = rawData.readBoolean() + activeDecodeTargetsPresentFlag = rawData.readBoolean() + customDTIsFlag = rawData.readBoolean() + customFDiffsFlag = rawData.readBoolean() + customChainsFlag = rawData.readBoolean() + + if (templateDependencyStructurePresentFlag) { + readTemplateDependencyStructure(descriptor) + val attachedStructure = checkNotNull(descriptor.attachedStructure) + descriptor.activeDecodeTargetsBitmask = (1 shl attachedStructure.numDecodeTargets) - 1 + } + } + + private fun readTemplateDependencyStructure(descriptor: DependencyDescriptor) { + descriptor.attachedStructure = FrameDependencyStructure().apply { + structureId = rawData.readInt(6) + numDecodeTargets = rawData.readInt(5) + 1 + readTemplateLayers(this) + readTemplateDTIs(this) + readTemplateFDiffs(this) + readTemplateChains(this) + + val hasResolutions = rawData.readBoolean() + if (hasResolutions) + readResolutions(this) + } + } + + private fun readTemplateLayers(structure: FrameDependencyStructure) { + val templates = mutableListOf() + var temporalId = 0 + var spatialId = 0 + var nextLayerIdc: NextLayerIdc + + do { + check(templates.size < DependencyDescriptor.kMaxTemplates) { "template overflow" } + + templates.add( + FrameDependencyTemplate().apply { + this.temporalId = temporalId + this.spatialId = spatialId + } + ) + + nextLayerIdc = NextLayerIdc.parse(rawData.readInt(2)) + if (nextLayerIdc == NextLayerIdc.NEXT_TEMPORAL_LAYER) { + temporalId++ + check(temporalId < DependencyDescriptor.kMaxTemporalIds) { "temporal id $temporalId overflow" } + } else if (nextLayerIdc == NextLayerIdc.NEXT_SPATIAL_LAYER) { + temporalId = 0 + spatialId++ + check(spatialId < DependencyDescriptor.kMaxSpatialIds) { "spatial id $spatialId overflow" } + } + } while (nextLayerIdc != NextLayerIdc.NO_MORE_TEMPLATES) + + structure.templates = templates.toList() + } + + private fun readTemplateDTIs(structure: FrameDependencyStructure) { + structure.templates.forEach { currentTemplate -> + currentTemplate.decodeTargetIndications = (0 until structure.numDecodeTargets).map { + DecodeTargetIndication.parse(rawData.readInt(2)) + } + } + } + + private fun readTemplateFDiffs(structure: FrameDependencyStructure) { + for (currentTemplate in structure.templates) { + val frameDiffs = mutableListOf() + var fDiffFollows = rawData.readBoolean() + while (fDiffFollows) { + val fDiffMinusOne = rawData.readInt(4) + frameDiffs.add(fDiffMinusOne + 1) + fDiffFollows = rawData.readBoolean() + } + currentTemplate.frameDiffs = frameDiffs.toList() + } + } + + private fun readTemplateChains(structure: FrameDependencyStructure) { + structure.numChains = rawData.readNonSymmetric(structure.numDecodeTargets + 1) + if (structure.numChains == 0) { + return + } + + structure.decodeTargetProtectedByChain = (0 until structure.numDecodeTargets).map { + rawData.readNonSymmetric(structure.numChains) + } + + structure.templates.forEach { frameTemplate -> + frameTemplate.chainDiffs = (0 until structure.numChains).map { + rawData.readInt(4) + } + } + } + + private fun readResolutions(structure: FrameDependencyStructure) { + // The way templates are bitpacked, they are always ordered by spatial_id. + val spatialLayers = structure.templates.last().spatialId + 1 + structure.resolutions = (0 until spatialLayers).map { + val widthMinusOne = rawData.readInt(16) + val heightMinusOne = rawData.readInt(16) + Resolution(widthMinusOne + 1, heightMinusOne + 1) + } + } + + private fun readFrameDependencyDefinition( + descriptor: DependencyDescriptor, + structure: FrameDependencyStructure + ) { + val templateIndex = + (frameDependencyTemplateId + DependencyDescriptor.kMaxTemplates - structure.structureId) % + DependencyDescriptor.kMaxTemplates + + check(templateIndex < structure.templates.size) { "template index $templateIndex overflow" } + + // Copy all the fields from the matching template + val descriptorFrameDependencies = structure.templates[templateIndex].copy() + descriptor.frameDependencies = descriptorFrameDependencies + + if (customDTIsFlag) + readFrameDTIs(descriptorFrameDependencies, structure) + if (customFDiffsFlag) + readFrameFDiffs(descriptorFrameDependencies) + if (customChainsFlag) + readFrameChains(descriptorFrameDependencies, structure) + + if (structure.resolutions.isEmpty()) { + descriptor.resolution = null + } else { + // Format guarantees that if there were resolutions in the last structure, then each spatial layer got one. + check(descriptorFrameDependencies.spatialId <= structure.resolutions.size) + descriptor.resolution = structure.resolutions[descriptorFrameDependencies.spatialId] + } + } + + private fun readFrameDTIs( + frameDependencies: FrameDependencyTemplate, + structure: FrameDependencyStructure + ) { + check(frameDependencies.decodeTargetIndications.size == structure.numDecodeTargets) + + frameDependencies.decodeTargetIndications = frameDependencies.decodeTargetIndications.map { + DecodeTargetIndication.parse(rawData.readInt(2)) + } + } + + private fun readFrameFDiffs(frameDependencies: FrameDependencyTemplate) { + val frameDiffs = mutableListOf() + var nextFDiffSize = rawData.readInt(2) + while (nextFDiffSize > 0) { + val fDiffMinusOne = rawData.readInt(4 * nextFDiffSize) + frameDiffs.add(fDiffMinusOne + 1) + nextFDiffSize = rawData.readInt(2) + } + frameDependencies.frameDiffs = frameDiffs + } + + private fun readFrameChains(frameDependencies: FrameDependencyTemplate, structure: FrameDependencyStructure) { + check(frameDependencies.chainDiffs.size == structure.numChains) + + frameDependencies.chainDiffs = frameDependencies.chainDiffs.map { rawData.readInt(8) } + } + + // 3 bytes + private fun readMandatoryFields(descriptor: DependencyDescriptor) { + descriptor.firstPacketInFrame = rawData.readBoolean() + descriptor.lastPacketInFrame = rawData.readBoolean() + frameDependencyTemplateId = rawData.readInt(6) + descriptor.frameNumber = rawData.readInt(16) + } +} diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/FrameDependencyStructure.kt b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/FrameDependencyStructure.kt new file mode 100644 index 0000000000..24b4a63a67 --- /dev/null +++ b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/FrameDependencyStructure.kt @@ -0,0 +1,18 @@ +package org.jitsi.nlj.rtp.codec.av1.dd + +data class FrameDependencyStructure( + /** + * template_id_offset + */ + var structureId: Int = 0, + /** + * DtCnt + */ + var numDecodeTargets: Int = 0, + var numChains: Int = 0, + // If chains are used (num_chains > 0), maps decode target index into index of + // the chain protecting that target. + var decodeTargetProtectedByChain: List = emptyList(), + var resolutions: List = emptyList(), + var templates: List = emptyList() +) diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/FrameDependencyTemplate.kt b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/FrameDependencyTemplate.kt new file mode 100644 index 0000000000..e473309689 --- /dev/null +++ b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/FrameDependencyTemplate.kt @@ -0,0 +1,19 @@ +package org.jitsi.nlj.rtp.codec.av1.dd + +data class FrameDependencyTemplate( + var spatialId: Int = 0, + var temporalId: Int = 0, + var decodeTargetIndications: List = emptyList(), + var frameDiffs: List = emptyList(), + var chainDiffs: List = emptyList() +) { + fun copy(): FrameDependencyTemplate { + return FrameDependencyTemplate( + spatialId = spatialId, + temporalId = temporalId, + decodeTargetIndications = ArrayList(decodeTargetIndications), + frameDiffs = ArrayList(frameDiffs), + chainDiffs = ArrayList(chainDiffs) + ) + } +} diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/NextLayerIdc.kt b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/NextLayerIdc.kt new file mode 100644 index 0000000000..7e50101743 --- /dev/null +++ b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/NextLayerIdc.kt @@ -0,0 +1,16 @@ +package org.jitsi.nlj.rtp.codec.av1.dd + +enum class NextLayerIdc(val id: Int) { + SAME_LAYER(0), + NEXT_TEMPORAL_LAYER(1), + NEXT_SPATIAL_LAYER(2), + NO_MORE_TEMPLATES(3); + + companion object { + private val map = values().associateBy { it.id } + + fun parse(id: Int): NextLayerIdc { + return map.getValue(id) + } + } +} diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/Resolution.kt b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/Resolution.kt new file mode 100644 index 0000000000..3648068778 --- /dev/null +++ b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/Resolution.kt @@ -0,0 +1,3 @@ +package org.jitsi.nlj.rtp.codec.av1.dd + +data class Resolution(val width: Int, val height: Int) From 74276080c89bd37318f1577deec0601acd031dfd Mon Sep 17 00:00:00 2001 From: Andrei Karasev Date: Thu, 28 Apr 2022 13:31:53 -0400 Subject: [PATCH 04/13] AV1. Set key frame flag --- .../org/jitsi/nlj/rtp/codec/av1/Av1PacketConverter.kt | 2 +- .../org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptor.kt | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1PacketConverter.kt b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1PacketConverter.kt index 880eef99fa..fce0bf8772 100644 --- a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1PacketConverter.kt +++ b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1PacketConverter.kt @@ -48,7 +48,7 @@ class Av1PacketConverter(val streamInformationStore: ReadOnlyStreamInformationSt rtpPacket.buffer, rtpPacket.offset, rtpPacket.length, - isKeyframe = true, + isKeyframe = descriptor.isKeyFrame(), isStartOfFrame = descriptor.firstPacketInFrame, isEndOfFrame = descriptor.lastPacketInFrame, frameNumber = descriptor.frameNumber, diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptor.kt b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptor.kt index 68b97e58bf..4a3e3eb960 100644 --- a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptor.kt +++ b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptor.kt @@ -10,6 +10,12 @@ data class DependencyDescriptor( var activeDecodeTargetsBitmask: Int? = null, var attachedStructure: FrameDependencyStructure? = null ) { + fun isKeyFrame(): Boolean { + return frameDependencies?.let { + it.chainDiffs.size == 1 && it.chainDiffs.first() == 0 && it.frameDiffs.isEmpty() + } ?: false + } + companion object { const val kMaxSpatialIds = 4 const val kMaxTemporalIds = 8 From eb9511eff6c00eee41376e6a62a62f839f30c3a0 Mon Sep 17 00:00:00 2001 From: Pawel Domas <2965063+paweldomas@users.noreply.github.com> Date: Thu, 28 Apr 2022 13:51:30 -0500 Subject: [PATCH 05/13] log DD ext id --- .../kotlin/org/jitsi/nlj/rtp/codec/av1/Av1PacketConverter.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1PacketConverter.kt b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1PacketConverter.kt index fce0bf8772..ab3cff2cb6 100644 --- a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1PacketConverter.kt +++ b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1PacketConverter.kt @@ -39,7 +39,7 @@ class Av1PacketConverter(val streamInformationStore: ReadOnlyStreamInformationSt val lastStructure = structures[ssrc] val extId = checkNotNull(ddExtId) { "missing dd ext id" } - val ddExt = checkNotNull(rtpPacket.getHeaderExtension(extId)) { "missing dd ext from $rtpPacket" } + val ddExt = checkNotNull(rtpPacket.getHeaderExtension(extId)) { "missing dd ext($ddExtId) from $rtpPacket" } val (descriptor, structure) = DependencyDescriptorReader(ddExt, lastStructure).parse() structures[ssrc] = structure From 3e81c6d3084407f191261e482363bdc7f6308d6f Mon Sep 17 00:00:00 2001 From: Jack Z <8870013+j000z@users.noreply.github.com> Date: Thu, 28 Apr 2022 08:13:40 -0500 Subject: [PATCH 06/13] pcap disable pcap test & lint --- jitsi-media-transform/pom.xml | 39 +++++++ .../src/test/kotlin/org/jitsi/Pcap.kt | 103 ++++++++++++++++++ 2 files changed, 142 insertions(+) create mode 100644 jitsi-media-transform/src/test/kotlin/org/jitsi/Pcap.kt diff --git a/jitsi-media-transform/pom.xml b/jitsi-media-transform/pom.xml index 8524f27c9e..3e8c52885b 100644 --- a/jitsi-media-transform/pom.xml +++ b/jitsi-media-transform/pom.xml @@ -84,6 +84,45 @@ spotbugs-annotations ${spotbugs.version} + + + org.apache.commons + commons-compress + 1.21 + + + io.github.microutils + kotlin-logging-jvm + 2.1.21 + + + commons-codec + commons-codec + 1.15 + + + com.github.kotlin-graphics + kotlin-unsigned + v3.1.3 + + + org.jcodec + jcodec + 0.2.2 + + + org.pcap4j + pcap4j-core + 1.8.2 + + + + org.pcap4j + pcap4j-packetfactory-static + 1.8.2 + + + diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/Pcap.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/Pcap.kt new file mode 100644 index 0000000000..5ae8c606fa --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/Pcap.kt @@ -0,0 +1,103 @@ +package org.jitsi + +import io.kotest.core.spec.style.ShouldSpec +import org.jcodec.common.io.NIOUtils +import org.jitsi.nlj.rtp.codec.av1.dd.DependencyDescriptor +import org.jitsi.nlj.rtp.codec.av1.dd.DependencyDescriptorReader +import org.jitsi.nlj.rtp.codec.av1.dd.FrameDependencyStructure +import org.jitsi.rtp.UnparsedPacket +import org.jitsi.rtp.extensions.looksLikeRtcp +import org.jitsi.rtp.extensions.looksLikeRtp +import org.jitsi.rtp.rtcp.CompoundRtcpPacket +import org.jitsi.rtp.rtp.RtpPacket +import org.pcap4j.core.PcapHandle +import org.pcap4j.core.Pcaps +import org.pcap4j.packet.IpV4Packet +import org.pcap4j.packet.Packet +import org.pcap4j.packet.UdpPacket +import java.io.EOFException +import java.nio.ByteBuffer + +class Pcap : ShouldSpec() { + init { +// context("parse") { +// parse() +// } + } + + fun parse() { + val rawPackets = Pcaps.openOffline("/Users/jackz/Downloads/av1_1.pcapng").getPackets().toList() + // BsdLoopbackPacket + val udpPackets = rawPackets + .mapNotNull { it.payload as? IpV4Packet } + .mapNotNull { it.payload as? UdpPacket } + .toList() + + val rtpPackets = udpPackets + .asSequence() + .mapNotNull { ByteBuffer.wrap(it.payload.rawData.clone()).toPacket() } + .filterIsInstance() + .toList() + + val av1PayloadType = 41 + val rtxPayloadType = 42 + val ddExtId = 8 + val vlaExtId = 9 + + val av1PacketsPerSSRC = rtpPackets + .filter { it.payloadType == av1PayloadType } + .groupBy { it.ssrc } + + val descriptors = mutableListOf() + var lastStructure: FrameDependencyStructure? = null + + av1PacketsPerSSRC.entries.first().value.forEach { rtpPacket -> + // TODO check if there are 2 bytes header + rtpPacket.getHeaderExtension(ddExtId)?.let { + val (descriptor, structure) = DependencyDescriptorReader(it, lastStructure).parse() + lastStructure = structure + descriptors.add(descriptor) + } + + rtpPacket.getHeaderExtension(vlaExtId) + ?.let { VideoLayersAllocation.parse(it) } + ?.let { vlas.add(it) } + } + + descriptors.forEach { + println(it) + } + } + + companion object { + fun PcapHandle.getPackets(): Sequence { + return generateSequence { + try { + this.nextPacketEx + } catch (e: EOFException) { + null + } + } + } + + fun ByteBuffer.toPacket(): org.jitsi.rtp.Packet? { + val unparsed = if (this.hasArray()) { + // use duplicate to reset cursor + this.duplicate().let { UnparsedPacket(it.array(), it.position(), it.remaining()) } + } else { + UnparsedPacket(this.toArray()) + } + + if (unparsed.looksLikeRtp()) { + return unparsed.toOtherType(::RtpPacket) + } else if (unparsed.looksLikeRtcp()) { + return unparsed.toOtherType(::CompoundRtcpPacket) + } + return null + } + + fun ByteBuffer.toArray(): ByteArray { + return NIOUtils.toArray(this) + } + } +} From b26ba95838d7fc9f018bfe72c963a39b2a0e6fb3 Mon Sep 17 00:00:00 2001 From: Pawel Domas <2965063+paweldomas@users.noreply.github.com> Date: Thu, 28 Apr 2022 13:47:51 -0500 Subject: [PATCH 07/13] check with-without descriptors --- jitsi-media-transform/pom.xml | 5 +++ .../src/test/kotlin/org/jitsi/Pcap.kt | 42 ++++++++++++++----- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/jitsi-media-transform/pom.xml b/jitsi-media-transform/pom.xml index 3e8c52885b..e2e01bb8a7 100644 --- a/jitsi-media-transform/pom.xml +++ b/jitsi-media-transform/pom.xml @@ -90,6 +90,11 @@ commons-compress 1.21 + + org.jcodec + jcodec + 0.2.5 + io.github.microutils kotlin-logging-jvm diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/Pcap.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/Pcap.kt index 5ae8c606fa..20db2dc579 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/Pcap.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/Pcap.kt @@ -19,14 +19,14 @@ import java.io.EOFException import java.nio.ByteBuffer class Pcap : ShouldSpec() { - init { -// context("parse") { -// parse() -// } - } + /*init { + context("parse") { + parse() + } + }*/ fun parse() { - val rawPackets = Pcaps.openOffline("/Users/jackz/Downloads/av1_1.pcapng").getPackets().toList() + val rawPackets = Pcaps.openOffline("/Users/paweldomas/Desktop/av1-svc.pcapng").getPackets().toList() // BsdLoopbackPacket val udpPackets = rawPackets .mapNotNull { it.payload as? IpV4Packet } @@ -41,8 +41,8 @@ class Pcap : ShouldSpec() { val av1PayloadType = 41 val rtxPayloadType = 42 - val ddExtId = 8 - val vlaExtId = 9 + val ddExtId = 11 + val vlaExtId = 12 val av1PacketsPerSSRC = rtpPackets .filter { it.payloadType == av1PayloadType } @@ -50,23 +50,43 @@ class Pcap : ShouldSpec() { val descriptors = mutableListOf() var lastStructure: FrameDependencyStructure? = null + val av1WithDescriptors = mutableListOf() + val av1WithoutDescriptors = mutableListOf() av1PacketsPerSSRC.entries.first().value.forEach { rtpPacket -> - // TODO check if there are 2 bytes header - rtpPacket.getHeaderExtension(ddExtId)?.let { + val ddExt = rtpPacket.getHeaderExtension(ddExtId) + + ddExt?.let { val (descriptor, structure) = DependencyDescriptorReader(it, lastStructure).parse() lastStructure = structure descriptors.add(descriptor) } - rtpPacket.getHeaderExtension(vlaExtId) + val vlaExt = rtpPacket.getHeaderExtension(vlaExtId) + + vlaExt ?.let { VideoLayersAllocation.parse(it) } ?.let { vlas.add(it) } + + if (ddExt != null) { + av1WithDescriptors.add(rtpPacket) + } else { + av1WithoutDescriptors.add(rtpPacket) + } } descriptors.forEach { println(it) } + + println("AV1 with descriptors: ") + av1WithDescriptors.forEach { + println(it.sequenceNumber) + } + println("AV1 without descriptors: ") + av1WithoutDescriptors.forEach { + println(it.sequenceNumber) + } } companion object { From 6bff290c7c209209dad8934313d9d54fb83d1b9b Mon Sep 17 00:00:00 2001 From: Jack Z <8870013+J000Z@users.noreply.github.com> Date: Fri, 29 Apr 2022 17:14:12 +0800 Subject: [PATCH 08/13] TwoBytesExtNormalizer.kt --- .../src/test/kotlin/org/jitsi/Pcap.kt | 38 ++++++---- .../nlj/rtp/codec/av1/Av1PacketConverter.kt | 16 +++- .../jitsi/nlj/rtp/codec/av1/dd/BytesView.kt | 7 +- .../av1/dd/DependencyDescriptorReader.kt | 5 +- .../rtp/codec/av1/dd/TwoBytesExtNormalizer.kt | 75 +++++++++++++++++++ 5 files changed, 118 insertions(+), 23 deletions(-) create mode 100644 src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/TwoBytesExtNormalizer.kt diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/Pcap.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/Pcap.kt index 20db2dc579..27732a4d87 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/Pcap.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/Pcap.kt @@ -2,9 +2,8 @@ package org.jitsi import io.kotest.core.spec.style.ShouldSpec import org.jcodec.common.io.NIOUtils -import org.jitsi.nlj.rtp.codec.av1.dd.DependencyDescriptor -import org.jitsi.nlj.rtp.codec.av1.dd.DependencyDescriptorReader -import org.jitsi.nlj.rtp.codec.av1.dd.FrameDependencyStructure +import org.jitsi.nlj.rtp.RtpExtension +import org.jitsi.nlj.rtp.codec.av1.dd.* import org.jitsi.rtp.UnparsedPacket import org.jitsi.rtp.extensions.looksLikeRtcp import org.jitsi.rtp.extensions.looksLikeRtp @@ -12,6 +11,7 @@ import org.jitsi.rtp.rtcp.CompoundRtcpPacket import org.jitsi.rtp.rtp.RtpPacket import org.pcap4j.core.PcapHandle import org.pcap4j.core.Pcaps +import org.pcap4j.packet.EthernetPacket import org.pcap4j.packet.IpV4Packet import org.pcap4j.packet.Packet import org.pcap4j.packet.UdpPacket @@ -19,17 +19,18 @@ import java.io.EOFException import java.nio.ByteBuffer class Pcap : ShouldSpec() { - /*init { + init { context("parse") { parse() } - }*/ + } fun parse() { - val rawPackets = Pcaps.openOffline("/Users/paweldomas/Desktop/av1-svc.pcapng").getPackets().toList() + val rawPackets = Pcaps.openOffline("/Users/jackz/Downloads/av1_2.pcapng").getPackets().toList() // BsdLoopbackPacket val udpPackets = rawPackets .mapNotNull { it.payload as? IpV4Packet } + .filter { it.header.dstAddr.hostAddress == "217.61.26.65" } .mapNotNull { it.payload as? UdpPacket } .toList() @@ -41,8 +42,9 @@ class Pcap : ShouldSpec() { val av1PayloadType = 41 val rtxPayloadType = 42 - val ddExtId = 11 - val vlaExtId = 12 +// val ddExtId = 11 + val ddExtId = 12 +// val vlaExtId = 12 val av1PacketsPerSSRC = rtpPackets .filter { it.payloadType == av1PayloadType } @@ -52,23 +54,27 @@ class Pcap : ShouldSpec() { var lastStructure: FrameDependencyStructure? = null val av1WithDescriptors = mutableListOf() val av1WithoutDescriptors = mutableListOf() + val twoBytesExtNormalizer = TwoBytesExtNormalizer() av1PacketsPerSSRC.entries.first().value.forEach { rtpPacket -> - val ddExt = rtpPacket.getHeaderExtension(ddExtId) + val twoBytesExtensions = twoBytesExtNormalizer.handle(rtpPacket) + + val ddView = twoBytesExtensions.find { it.id == ddExtId }?.let { BytesView(it) } + ?: rtpPacket.getHeaderExtension(ddExtId)?.let { BytesView(it) } - ddExt?.let { + ddView?.let { val (descriptor, structure) = DependencyDescriptorReader(it, lastStructure).parse() lastStructure = structure descriptors.add(descriptor) } - val vlaExt = rtpPacket.getHeaderExtension(vlaExtId) - - vlaExt - ?.let { VideoLayersAllocation.parse(it) } - ?.let { vlas.add(it) } +// val vlaExt = rtpPacket.getHeaderExtension(vlaExtId) +// +// vlaExt +// ?.let { VideoLayersAllocation.parse(it) } +// ?.let { vlas.add(it) } - if (ddExt != null) { + if (ddView != null) { av1WithDescriptors.add(rtpPacket) } else { av1WithoutDescriptors.add(rtpPacket) diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1PacketConverter.kt b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1PacketConverter.kt index ab3cff2cb6..4c5ed899cf 100644 --- a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1PacketConverter.kt +++ b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1PacketConverter.kt @@ -18,8 +18,10 @@ package org.jitsi.nlj.rtp.codec.av1 import org.jitsi.nlj.rtp.RtpExtensionType +import org.jitsi.nlj.rtp.codec.av1.dd.BytesView import org.jitsi.nlj.rtp.codec.av1.dd.DependencyDescriptorReader import org.jitsi.nlj.rtp.codec.av1.dd.FrameDependencyStructure +import org.jitsi.nlj.rtp.codec.av1.dd.TwoBytesExtNormalizer import org.jitsi.nlj.util.ReadOnlyStreamInformationStore import org.jitsi.rtp.rtp.RtpPacket import java.util.concurrent.ConcurrentHashMap @@ -27,6 +29,7 @@ import java.util.concurrent.ConcurrentHashMap class Av1PacketConverter(val streamInformationStore: ReadOnlyStreamInformationStore) { private var ddExtId: Int? = null private var structures = ConcurrentHashMap() + private val twoBytesExtNormalizer = TwoBytesExtNormalizer() init { streamInformationStore.onRtpExtensionMapping(RtpExtensionType.AV1_DEPENDENCY_DESCRIPTOR) { @@ -39,8 +42,17 @@ class Av1PacketConverter(val streamInformationStore: ReadOnlyStreamInformationSt val lastStructure = structures[ssrc] val extId = checkNotNull(ddExtId) { "missing dd ext id" } - val ddExt = checkNotNull(rtpPacket.getHeaderExtension(extId)) { "missing dd ext($ddExtId) from $rtpPacket" } - val (descriptor, structure) = DependencyDescriptorReader(ddExt, lastStructure).parse() + + val twoBytesExtensions = twoBytesExtNormalizer.handle(rtpPacket) + + val ddView = checkNotNull( + twoBytesExtensions.find { it.id == extId }?.let { BytesView(it) } + ?: rtpPacket.getHeaderExtension(extId)?.let { BytesView(it) } + ) { + "missing dd ext($ddExtId) from $rtpPacket" + } + + val (descriptor, structure) = DependencyDescriptorReader(ddView, lastStructure).parse() structures[ssrc] = structure diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/BytesView.kt b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/BytesView.kt index e3d4656aee..1d931f8ada 100644 --- a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/BytesView.kt +++ b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/BytesView.kt @@ -9,11 +9,16 @@ class BytesView(val bytes: ByteArray, val offset: Int, val length: Int) { private val bitStream = BitInputStream(ByteArrayInputStream(bytes, offset, length), ByteOrder.BIG_ENDIAN) constructor(ext: RtpPacket.HeaderExtension) : this( ext.currExtBuffer, - // TODO what about 2 bytes header ext.currExtOffset + 1, ext.currExtLength - 1 ) + constructor(ext: TwoBytesExtNormalizer.TwoBytesExtension) : this( + ext.payload, + 0, + ext.payloadLength + ) + fun readBoolean(): Boolean { return readInt(1) == 1 } diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptorReader.kt b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptorReader.kt index c9dd6ef81d..b05082d434 100644 --- a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptorReader.kt +++ b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptorReader.kt @@ -1,12 +1,9 @@ package org.jitsi.nlj.rtp.codec.av1.dd -import org.jitsi.rtp.rtp.RtpPacket - class DependencyDescriptorReader( - ext: RtpPacket.HeaderExtension, + private val rawData: BytesView, private var structure: FrameDependencyStructure? ) { - private val rawData = BytesView(ext) private var frameDependencyTemplateId = 0 private var activeDecodeTargetsPresentFlag = false private var customDTIsFlag = false diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/TwoBytesExtNormalizer.kt b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/TwoBytesExtNormalizer.kt new file mode 100644 index 0000000000..13e78d88c4 --- /dev/null +++ b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/TwoBytesExtNormalizer.kt @@ -0,0 +1,75 @@ +package org.jitsi.nlj.rtp.codec.av1.dd + +import org.jitsi.rtp.extensions.bytearray.toHex +import org.jitsi.rtp.rtp.RtpHeader +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.rtp.header_extensions.HeaderExtensionHelpers +import org.jitsi.rtp.util.getByteAsInt +import org.jitsi.rtp.util.getShortAsInt + +class TwoBytesExtNormalizer { + /** + * strip off 2 bytes extensions + */ + fun handle(rtpPacket: RtpPacket): List { + if (!rtpPacket.hasExtensions) { + return emptyList() + } + + val extensionBlockOffset = rtpPacket.offset + RtpHeader.FIXED_HEADER_SIZE_BYTES + rtpPacket.csrcCount * 4 + val profile = rtpPacket.buffer.getShortAsInt(extensionBlockOffset) + + // 0x1000 + if (profile != 4096) { + return emptyList() + } + + val extensionLength = HeaderExtensionHelpers.getExtensionsTotalLength(rtpPacket.buffer, extensionBlockOffset) + val extensionDataLength = extensionLength - HeaderExtensionHelpers.TOP_LEVEL_EXT_HEADER_SIZE_BYTES + + if (extensionDataLength == 0) { + return emptyList() + } + + var curOffset = extensionBlockOffset + HeaderExtensionHelpers.TOP_LEVEL_EXT_HEADER_SIZE_BYTES + val extensions = mutableListOf() + + while (curOffset < rtpPacket.payloadOffset - 1) { + val id = rtpPacket.buffer.getByteAsInt(curOffset) + val payloadLength = rtpPacket.buffer.getByteAsInt(curOffset + 1) + + if (payloadLength > 0) { + extensions.add( + TwoBytesExtension( + id = id, + payloadLength = payloadLength, + payload = rtpPacket.buffer.copyOfRange(curOffset + 2, curOffset + 2 + payloadLength) + ) + ) + } + + curOffset += (2 + payloadLength) + } + + rtpPacket.removeHeaderExtensionsExcept(emptySet()) + extensions.forEach { extension -> + if (extension.payloadLength <= 16) { + val newExt = rtpPacket.addHeaderExtension(extension.id, extension.payloadLength) + extension.payload.copyInto(newExt.currExtBuffer, newExt.currExtOffset + 1) + } + } + rtpPacket.encodeHeaderExtensions() + + return extensions + } + + class TwoBytesExtension( + val id: Int, + val payloadLength: Int, + val payload: ByteArray + ) { + override fun toString(): String { + return "TwoBytesExtension(id=$id, length=$payloadLength, data=${payload.toHex()})" + } + } +} From 44eac6b743a8dcb19422962fefc6138b49bea3fc Mon Sep 17 00:00:00 2001 From: Jack Z <8870013+J000Z@users.noreply.github.com> Date: Fri, 29 Apr 2022 17:51:02 +0800 Subject: [PATCH 09/13] RtpEncodingDesc --- .../src/test/kotlin/org/jitsi/Pcap.kt | 8 ++- .../nlj/rtp/codec/av1/Av1PacketConverter.kt | 5 +- .../org/jitsi/nlj/rtp/codec/av1/Av1Parser.kt | 61 +++++++++++++++++++ .../org/jitsi/nlj/rtp/codec/av1/Av1packet.kt | 7 ++- .../rtp/codec/av1/dd/DependencyDescriptor.kt | 4 +- 5 files changed, 75 insertions(+), 10 deletions(-) diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/Pcap.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/Pcap.kt index 27732a4d87..05b0110362 100644 --- a/jitsi-media-transform/src/test/kotlin/org/jitsi/Pcap.kt +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/Pcap.kt @@ -2,8 +2,11 @@ package org.jitsi import io.kotest.core.spec.style.ShouldSpec import org.jcodec.common.io.NIOUtils -import org.jitsi.nlj.rtp.RtpExtension -import org.jitsi.nlj.rtp.codec.av1.dd.* +import org.jitsi.nlj.rtp.codec.av1.dd.DependencyDescriptorReader +import org.jitsi.nlj.rtp.codec.av1.dd.DependencyDescriptor +import org.jitsi.nlj.rtp.codec.av1.dd.FrameDependencyStructure +import org.jitsi.nlj.rtp.codec.av1.dd.TwoBytesExtNormalizer +import org.jitsi.nlj.rtp.codec.av1.dd.BytesView import org.jitsi.rtp.UnparsedPacket import org.jitsi.rtp.extensions.looksLikeRtcp import org.jitsi.rtp.extensions.looksLikeRtp @@ -11,7 +14,6 @@ import org.jitsi.rtp.rtcp.CompoundRtcpPacket import org.jitsi.rtp.rtp.RtpPacket import org.pcap4j.core.PcapHandle import org.pcap4j.core.Pcaps -import org.pcap4j.packet.EthernetPacket import org.pcap4j.packet.IpV4Packet import org.pcap4j.packet.Packet import org.pcap4j.packet.UdpPacket diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1PacketConverter.kt b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1PacketConverter.kt index 4c5ed899cf..c2c05b54a6 100644 --- a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1PacketConverter.kt +++ b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1PacketConverter.kt @@ -47,7 +47,7 @@ class Av1PacketConverter(val streamInformationStore: ReadOnlyStreamInformationSt val ddView = checkNotNull( twoBytesExtensions.find { it.id == extId }?.let { BytesView(it) } - ?: rtpPacket.getHeaderExtension(extId)?.let { BytesView(it) } + ?: rtpPacket.getHeaderExtension(extId)?.let { BytesView(it) } ) { "missing dd ext($ddExtId) from $rtpPacket" } @@ -65,7 +65,8 @@ class Av1PacketConverter(val streamInformationStore: ReadOnlyStreamInformationSt isEndOfFrame = descriptor.lastPacketInFrame, frameNumber = descriptor.frameNumber, temporalLayerIndex = descriptor.frameDependencies!!.temporalId, - spatialLayerIndex = descriptor.frameDependencies!!.spatialId + spatialLayerIndex = descriptor.frameDependencies!!.spatialId, + structure = descriptor.attachedStructure ) } } diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1Parser.kt b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1Parser.kt index d941eca454..673ee4b11b 100644 --- a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1Parser.kt +++ b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1Parser.kt @@ -18,7 +18,10 @@ package org.jitsi.nlj.rtp.codec.av1 import org.jitsi.nlj.MediaSourceDesc import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.RtpEncodingDesc +import org.jitsi.nlj.RtpLayerDesc import org.jitsi.nlj.rtp.codec.VideoCodecParser +import org.jitsi.nlj.rtp.codec.av1.dd.FrameDependencyStructure import org.jitsi.utils.logging2.Logger import org.jitsi.utils.logging2.createChildLogger @@ -30,10 +33,68 @@ class Av1Parser( /** Encodings we've actually seen. Used to clear out inferred-from-signaling encoding information. */ private val ssrcsSeen = HashSet() + private var numSpatialLayers = -1 override fun parse(packetInfo: PacketInfo) { val av1packet = packetInfo.packetAs() ssrcsSeen.add(av1packet.ssrc) + + val structure = av1packet.structure ?: return + + val packetSpatialLayers = structure.resolutions.size + if (packetSpatialLayers > 0) { + if (numSpatialLayers != -1 && numSpatialLayers != packetSpatialLayers) { + packetInfo.layeringChanged = true + } + numSpatialLayers = packetSpatialLayers + } + + findSourceDescAndRtpEncodingDesc(av1packet)?.let { (src, enc) -> + val desc = getScalabilityStructure( + eid = enc.eid, + structure = structure, + ssrc = av1packet.ssrc + ) + src.setEncodingLayers(desc.layers, av1packet.ssrc) + + for (otherEnc in src.rtpEncodings) { + if (!ssrcsSeen.contains(otherEnc.primarySSRC)) { + src.setEncodingLayers(emptyArray(), otherEnc.primarySSRC) + } + } + } + } + + fun getScalabilityStructure(eid: Int, structure: FrameDependencyStructure, ssrc: Long): RtpEncodingDesc { + val spatialIds = structure.templates.map { it.spatialId }.distinct().sorted() + val temporalIds = structure.templates.map { it.temporalId }.distinct().sorted() + val layers = ArrayList() + + for (s in spatialIds) { + for (t in temporalIds) { + val dependencies = ArrayList() + val softDependencies = ArrayList() + if (s > 0) { + /* Because of K-SVC, spatial layer dependencies are soft */ + layers.find { it.sid == s - 1 && it.tid == t }?.let { softDependencies.add(it) } + } + if (t > 0) { + layers.find { it.sid == s && it.tid == t - 1 }?.let { dependencies.add(it) } + } + val layerDesc = RtpLayerDesc( + eid = eid, + tid = t, + sid = s, + height = structure.resolutions[s].height, + frameRate = RtpLayerDesc.NO_FRAME_RATE, + dependencyLayers = dependencies.toArray(arrayOf()), + softDependencyLayers = softDependencies.toArray(arrayOf()) + ) + layers.add(layerDesc) + } + } + + return RtpEncodingDesc(ssrc, layers.toArray(arrayOf()), eid) } } diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1packet.kt b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1packet.kt index f13bd78459..33ebef747f 100644 --- a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1packet.kt +++ b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1packet.kt @@ -19,6 +19,7 @@ package org.jitsi.nlj.rtp.codec.av1 import org.jitsi.nlj.RtpLayerDesc import org.jitsi.nlj.rtp.ParsedVideoPacket +import org.jitsi.nlj.rtp.codec.av1.dd.FrameDependencyStructure import org.jitsi.rtp.extensions.bytearray.hashCodeOfSegment import org.jitsi.utils.logging2.createLogger import org.jitsi_modified.impl.neomedia.codec.video.vp9.DePacketizer @@ -32,7 +33,8 @@ class Av1packet( override val isEndOfFrame: Boolean, val frameNumber: Int, val temporalLayerIndex: Int, - val spatialLayerIndex: Int + val spatialLayerIndex: Int, + val structure: FrameDependencyStructure? ) : ParsedVideoPacket(buffer, offset, length, null) { override val layerId: Int @@ -62,7 +64,8 @@ class Av1packet( isEndOfFrame = isEndOfFrame, frameNumber = frameNumber, temporalLayerIndex = temporalLayerIndex, - spatialLayerIndex = spatialLayerIndex + spatialLayerIndex = spatialLayerIndex, + structure = structure ) } diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptor.kt b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptor.kt index 4a3e3eb960..4e8d40324b 100644 --- a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptor.kt +++ b/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptor.kt @@ -11,9 +11,7 @@ data class DependencyDescriptor( var attachedStructure: FrameDependencyStructure? = null ) { fun isKeyFrame(): Boolean { - return frameDependencies?.let { - it.chainDiffs.size == 1 && it.chainDiffs.first() == 0 && it.frameDiffs.isEmpty() - } ?: false + return frameDependencies?.decodeTargetIndications?.all { it == DecodeTargetIndication.SWITCH } ?: false } companion object { From dc022116cf9b61a45591b15f5ce8f89aeee5c241 Mon Sep 17 00:00:00 2001 From: Pawel Domas <2965063+paweldomas@users.noreply.github.com> Date: Wed, 30 Nov 2022 20:24:22 -0600 Subject: [PATCH 10/13] move the files around for mono repo --- .../kotlin/org/jitsi/nlj/rtp/codec/av1/Av1PacketConverter.kt | 0 .../src}/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1Parser.kt | 3 ++- .../src}/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1packet.kt | 0 .../main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/BytesView.kt | 0 .../org/jitsi/nlj/rtp/codec/av1/dd/DecodeTargetIndication.kt | 0 .../org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptor.kt | 0 .../jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptorReader.kt | 2 +- .../org/jitsi/nlj/rtp/codec/av1/dd/FrameDependencyStructure.kt | 0 .../org/jitsi/nlj/rtp/codec/av1/dd/FrameDependencyTemplate.kt | 0 .../main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/NextLayerIdc.kt | 0 .../main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/Resolution.kt | 0 .../org/jitsi/nlj/rtp/codec/av1/dd/TwoBytesExtNormalizer.kt | 0 12 files changed, 3 insertions(+), 2 deletions(-) rename {src => jitsi-media-transform/src}/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1PacketConverter.kt (100%) rename {src => jitsi-media-transform/src}/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1Parser.kt (98%) rename {src => jitsi-media-transform/src}/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1packet.kt (100%) rename {src => jitsi-media-transform/src}/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/BytesView.kt (100%) rename {src => jitsi-media-transform/src}/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DecodeTargetIndication.kt (100%) rename {src => jitsi-media-transform/src}/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptor.kt (100%) rename {src => jitsi-media-transform/src}/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptorReader.kt (99%) rename {src => jitsi-media-transform/src}/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/FrameDependencyStructure.kt (100%) rename {src => jitsi-media-transform/src}/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/FrameDependencyTemplate.kt (100%) rename {src => jitsi-media-transform/src}/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/NextLayerIdc.kt (100%) rename {src => jitsi-media-transform/src}/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/Resolution.kt (100%) rename {src => jitsi-media-transform/src}/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/TwoBytesExtNormalizer.kt (100%) diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1PacketConverter.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1PacketConverter.kt similarity index 100% rename from src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1PacketConverter.kt rename to jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1PacketConverter.kt diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1Parser.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1Parser.kt similarity index 98% rename from src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1Parser.kt rename to jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1Parser.kt index 673ee4b11b..3489317c66 100644 --- a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1Parser.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1Parser.kt @@ -1,5 +1,6 @@ /* - * Copyright @ 2018 - present 8x8, Inc. + * Copyright @ 2023 - present 8x8, Inc. + * Copyright @ 2023 - Vowel, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1packet.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1packet.kt similarity index 100% rename from src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1packet.kt rename to jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1packet.kt diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/BytesView.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/BytesView.kt similarity index 100% rename from src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/BytesView.kt rename to jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/BytesView.kt diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DecodeTargetIndication.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DecodeTargetIndication.kt similarity index 100% rename from src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DecodeTargetIndication.kt rename to jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DecodeTargetIndication.kt diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptor.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptor.kt similarity index 100% rename from src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptor.kt rename to jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptor.kt diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptorReader.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptorReader.kt similarity index 99% rename from src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptorReader.kt rename to jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptorReader.kt index b05082d434..1b850fa7b1 100644 --- a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptorReader.kt +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/DependencyDescriptorReader.kt @@ -147,7 +147,7 @@ class DependencyDescriptorReader( ) { val templateIndex = (frameDependencyTemplateId + DependencyDescriptor.kMaxTemplates - structure.structureId) % - DependencyDescriptor.kMaxTemplates + DependencyDescriptor.kMaxTemplates check(templateIndex < structure.templates.size) { "template index $templateIndex overflow" } diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/FrameDependencyStructure.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/FrameDependencyStructure.kt similarity index 100% rename from src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/FrameDependencyStructure.kt rename to jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/FrameDependencyStructure.kt diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/FrameDependencyTemplate.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/FrameDependencyTemplate.kt similarity index 100% rename from src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/FrameDependencyTemplate.kt rename to jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/FrameDependencyTemplate.kt diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/NextLayerIdc.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/NextLayerIdc.kt similarity index 100% rename from src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/NextLayerIdc.kt rename to jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/NextLayerIdc.kt diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/Resolution.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/Resolution.kt similarity index 100% rename from src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/Resolution.kt rename to jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/Resolution.kt diff --git a/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/TwoBytesExtNormalizer.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/TwoBytesExtNormalizer.kt similarity index 100% rename from src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/TwoBytesExtNormalizer.kt rename to jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/dd/TwoBytesExtNormalizer.kt From 23dbab3a77ac1b8caeef7b233da561f5ad56c5ce Mon Sep 17 00:00:00 2001 From: Andrei Karasev Date: Wed, 27 Apr 2022 22:31:44 -0400 Subject: [PATCH 11/13] Add av1 payload type --- .../main/kotlin/org/jitsi/videobridge/util/PayloadTypeUtil.kt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/util/PayloadTypeUtil.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/util/PayloadTypeUtil.kt index 24503bec86..59fa4b2acb 100644 --- a/jvb/src/main/kotlin/org/jitsi/videobridge/util/PayloadTypeUtil.kt +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/util/PayloadTypeUtil.kt @@ -16,11 +16,13 @@ package org.jitsi.videobridge.util import org.jitsi.nlj.format.AudioRedPayloadType +import org.jitsi.nlj.format.Av1PayloadType import org.jitsi.nlj.format.H264PayloadType import org.jitsi.nlj.format.OpusPayloadType import org.jitsi.nlj.format.OtherAudioPayloadType import org.jitsi.nlj.format.OtherVideoPayloadType import org.jitsi.nlj.format.PayloadType +import org.jitsi.nlj.format.PayloadTypeEncoding.AV1 import org.jitsi.nlj.format.PayloadTypeEncoding.Companion.createFrom import org.jitsi.nlj.format.PayloadTypeEncoding.H264 import org.jitsi.nlj.format.PayloadTypeEncoding.OPUS @@ -95,6 +97,7 @@ class PayloadTypeUtil { VP8 -> Vp8PayloadType(id, parameters, rtcpFeedbackSet) VP9 -> Vp9PayloadType(id, parameters, rtcpFeedbackSet) H264 -> H264PayloadType(id, parameters, rtcpFeedbackSet) + AV1 -> Av1PayloadType(id, parameters, rtcpFeedbackSet) RTX -> RtxPayloadType(id, parameters) OPUS -> OpusPayloadType(id, parameters) RED -> when (mediaType) { From be484da9da52dbd9a994a225a26054e618178ef9 Mon Sep 17 00:00:00 2001 From: Pawel Domas <2965063+paweldomas@users.noreply.github.com> Date: Wed, 30 Nov 2022 20:56:31 -0600 Subject: [PATCH 12/13] cleanup pom.xml --- jitsi-media-transform/pom.xml | 37 ----------------------------------- 1 file changed, 37 deletions(-) diff --git a/jitsi-media-transform/pom.xml b/jitsi-media-transform/pom.xml index e2e01bb8a7..0f785a7808 100644 --- a/jitsi-media-transform/pom.xml +++ b/jitsi-media-transform/pom.xml @@ -90,43 +90,6 @@ commons-compress 1.21 - - org.jcodec - jcodec - 0.2.5 - - - io.github.microutils - kotlin-logging-jvm - 2.1.21 - - - commons-codec - commons-codec - 1.15 - - - com.github.kotlin-graphics - kotlin-unsigned - v3.1.3 - - - org.jcodec - jcodec - 0.2.2 - - - org.pcap4j - pcap4j-core - 1.8.2 - - - - org.pcap4j - pcap4j-packetfactory-static - 1.8.2 - - From c4a7fedbe4f2cd699c35d3ca65010532826a6666 Mon Sep 17 00:00:00 2001 From: Pawel Domas <2965063+paweldomas@users.noreply.github.com> Date: Sat, 24 Dec 2022 08:19:02 -0600 Subject: [PATCH 13/13] example config and fix test dependency --- jitsi-media-transform/pom.xml | 7 +++ local-docker-config/.gitignore | 3 ++ local-docker-config/jvb1.conf.example | 72 +++++++++++++++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 local-docker-config/.gitignore create mode 100644 local-docker-config/jvb1.conf.example diff --git a/jitsi-media-transform/pom.xml b/jitsi-media-transform/pom.xml index 0f785a7808..4066ba0b81 100644 --- a/jitsi-media-transform/pom.xml +++ b/jitsi-media-transform/pom.xml @@ -133,6 +133,13 @@ 2.3.1 test + + + org.jcodec + jcodec + 0.2.5 + test + diff --git a/local-docker-config/.gitignore b/local-docker-config/.gitignore new file mode 100644 index 0000000000..b2acc17720 --- /dev/null +++ b/local-docker-config/.gitignore @@ -0,0 +1,3 @@ +jvb1.conf +jvb2.conf +jvb3.conf diff --git a/local-docker-config/jvb1.conf.example b/local-docker-config/jvb1.conf.example new file mode 100644 index 0000000000..759a4784d7 --- /dev/null +++ b/local-docker-config/jvb1.conf.example @@ -0,0 +1,72 @@ +videobridge { + ice { + udp { + port = 10000 + } + advertise-private-candidates = true + } + apis { + xmpp-client { + configs { + + shard0 { + HOSTNAME = "localhost" + PORT = "5222" + DOMAIN = "auth.meet.jitsi" + USERNAME = "jvb" + PASSWORD = "1d94cb1ad2a8eb6a2ffde0f89dee15b2" + MUC_JIDS = "jvbbrewery@internal-muc.meet.jitsi" + MUC_NICKNAME = "bcae248fa1a4" + DISABLE_CERTIFICATE_VERIFICATION = true + } +} + } + rest { + enabled = false + } + } + rest { + shutdown { + enabled = false + } + } + stats { + enabled = true + } + websockets { + enabled = true + domain = "localhost:9090" + tls = false + server-id = "127.0.0.1" + } + multi-stream { + enabled = false + } + http-servers { + private { + host = 0.0.0.0 + } + public { + host = 0.0.0.0 + port = 9090 + } + } + + } + +ice4j { + harvest { + mapping { + stun { +addresses = [ "meet-jit-si-turnrelay.jitsi.net:443" ] +} + static-mappings = [ +{ + local-address = "127.0.0.1" + public-address = "" + name = "ip-0" + }, +] + } + } +}