Merge pull request #123 from shiguredo/feature/e2e-test-simulcast

Feature/e2e test simulcast

.github/workflows/build.yml

@@ -5,6 +5,8 @@ on:
     paths-ignore:
       - "examples/**"
       - "**.md"
+    branches-ignore:
+      - "feature/e2e-test*"
   # schedule:
   #   # UTC の 01:00 は JST だと 10:00 。
   #   # 1-5 で 月曜日から金曜日

tests/test_macos.py

@@ -60,13 +60,44 @@ def test_macos_video_hwa_sendonly(setup, video_codec_type):
 
 @pytest.mark.skipif(sys.platform != "darwin", reason="macOS でのみ実行する")
 @pytest.mark.parametrize(
-    ("video_codec_type", "expected_implementation"),
+    (
+        "video_codec_type",
+        "expected_implementation",
+        "video_bit_rate",
+        "video_width",
+        "video_height",
+        "simulcast_count",
+    ),
     [
-        ("H264", "VideoToolbox"),
-        ("H265", "VideoToolbox"),
+        # 1080p
+        ("H264", "VideoToolbox", 5000, 1920, 1080, 3),
+        ("H265", "VideoToolbox", 5000, 1920, 1080, 3),
+        # 720p
+        ("H264", "VideoToolbox", 2500, 1280, 720, 3),
+        ("H265", "VideoToolbox", 2500, 1280, 720, 3),
+        # 540p
+        ("H264", "VideoToolbox", 1200, 960, 540, 3),
+        ("H265", "VideoToolbox", 1200, 960, 540, 3),
+        # 360p
+        ("H264", "VideoToolbox", 700, 640, 360, 2),
+        ("H265", "VideoToolbox", 700, 640, 360, 2),
+        # 270p
+        ("H264", "VideoToolbox", 450, 480, 270, 2),
+        ("H265", "VideoToolbox", 450, 480, 270, 2),
+        # 180p
+        ("H264", "VideoToolbox", 200, 320, 180, 1),
+        ("H265", "VideoToolbox", 200, 320, 180, 1),
     ],
 )
-def test_macos_simulcast(setup, video_codec_type, expected_implementation):
+def test_macos_simulcast(
+    setup,
+    video_codec_type,
+    expected_implementation,
+    video_bit_rate,
+    video_width,
+    video_height,
+    simulcast_count,
+):
     signaling_urls = setup.get("signaling_urls")
     channel_id_prefix = setup.get("channel_id_prefix")
     metadata = setup.get("metadata")
@@ -81,10 +112,10 @@ def test_macos_simulcast(setup, video_codec_type, expected_implementation):
         audio=False,
         video=True,
         video_codec_type=video_codec_type,
-        video_bit_rate=3000,
+        video_bit_rate=video_bit_rate,
         metadata=metadata,
-        video_width=1280,
-        video_height=720,
+        video_width=video_width,
+        video_height=video_height,
         use_hwa=True,
     )
     sendonly.connect(fake_video=True)
@@ -113,12 +144,45 @@ def test_macos_simulcast(setup, video_codec_type, expected_implementation):
     # rid でソート
     sorted_stats = sorted(outbound_rtp_stats, key=lambda x: x.get("rid", ""))
 
-    for i, rtp_stat in enumerate(sorted_stats):
-        assert rtp_stat["rid"] == f"r{i}"
-        assert "SimulcastEncoderAdapter" in rtp_stat["encoderImplementation"]
-        assert expected_implementation in rtp_stat["encoderImplementation"]
-        assert rtp_stat["bytesSent"] > 0
-        assert rtp_stat["packetsSent"] > 0
+    for i, s in enumerate(sorted_stats):
+        assert s["rid"] == f"r{i}"
+        # simulcast_count が 2 の場合、rid r2 の bytesSent/packetsSent は 0 or 1 になる
+        # simulcast_count が 1 の場合、rid r2 と r1 の bytesSent/packetsSent は 0 or 1 になる
+        if i < simulcast_count:
+            # 1 本になると simulcastEncodingAdapter がなくなる
+            if simulcast_count > 1:
+                assert "SimulcastEncoderAdapter" in s["encoderImplementation"]
+            assert expected_implementation in s["encoderImplementation"]
+
+            assert s["bytesSent"] > 1000
+            assert s["packetsSent"] > 5
+            # targetBitrate が指定したビットレートの 90% 以上、100% 以下に収まることを確認
+            expected_bitrate = video_bit_rate * 1000
+            print(
+                s["rid"],
+                video_codec_type,
+                expected_implementation,
+                expected_bitrate,
+                s["targetBitrate"],
+                s["frameWidth"],
+                s["frameHeight"],
+                s["bytesSent"],
+                s["packetsSent"],
+            )
+            # 期待値の 20% 以上、100% 以下に収まることを確認
+            assert expected_bitrate * 0.2 <= s["targetBitrate"] <= expected_bitrate
+        else:
+            # 本来は 0 なのだが、simulcast_count が 1 の場合、
+            # packetSent が 0 ではなく 1 や 2 になる場合がある
+            # byteSent は 0
+            assert s["bytesSent"] == 0
+            assert s["packetsSent"] <= 2
+            print(
+                s["rid"],
+                video_codec_type,
+                s["bytesSent"],
+                s["packetsSent"],
+            )
 
 
 @pytest.mark.skip(reason="ローカルでは成功する")

tests/test_openh264.py

@@ -70,12 +70,38 @@ def test_openh264_sendonly_recvonly(setup):
 
 @pytest.mark.skipif(sys.platform not in ["darwin", "linux"], reason="macOSとLinuxでのみ実行する")
 @pytest.mark.parametrize(
-    "video_codec_type,expected_implementation",
+    (
+        "video_codec_type",
+        "expected_implementation",
+        "video_bit_rate",
+        "video_width",
+        "video_height",
+        "simulcast_count",
+    ),
     [
-        ("H264", "OpenH264"),
+        # 1080p
+        ("H264", "OpenH264", 5000, 1920, 1080, 3),
+        # 720p
+        ("H264", "OpenH264", 2500, 1280, 720, 3),
+        # 540p
+        ("H264", "OpenH264", 1200, 960, 540, 3),
+        # 360p
+        ("H264", "OpenH264", 700, 640, 360, 2),
+        # 270p
+        ("H264", "OpenH264", 450, 480, 270, 2),
+        # 180p
+        ("H264", "OpenH264", 200, 320, 180, 1),
     ],
 )
-def test_openh264_simulcast(setup, video_codec_type, expected_implementation):
+def test_openh264_simulcast(
+    setup,
+    video_codec_type,
+    expected_implementation,
+    video_bit_rate,
+    video_width,
+    video_height,
+    simulcast_count,
+):
     signaling_urls = setup.get("signaling_urls")
     channel_id_prefix = setup.get("channel_id_prefix")
     metadata = setup.get("metadata")
@@ -92,11 +118,12 @@ def test_openh264_simulcast(setup, video_codec_type, expected_implementation):
         audio=False,
         video=True,
         video_codec_type=video_codec_type,
-        video_bit_rate=3000,
+        video_bit_rate=video_bit_rate,
         metadata=metadata,
-        video_width=1280,
-        video_height=720,
+        video_width=video_width,
+        video_height=video_height,
         openh264_path=openh264_path,
+        # HWA を無効化
         use_hwa=False,
     )
     sendonly.connect(fake_video=True)
@@ -120,9 +147,42 @@ def test_openh264_simulcast(setup, video_codec_type, expected_implementation):
     # rid でソート
     sorted_stats = sorted(outbound_rtp_stats, key=lambda x: x.get("rid", ""))
 
-    for i, rtp_stat in enumerate(sorted_stats):
-        assert rtp_stat["rid"] == f"r{i}"
-        assert "SimulcastEncoderAdapter" in rtp_stat["encoderImplementation"]
-        assert expected_implementation in rtp_stat["encoderImplementation"]
-        assert rtp_stat["bytesSent"] > 0
-        assert rtp_stat["packetsSent"] > 0
+    for i, s in enumerate(sorted_stats):
+        assert s["rid"] == f"r{i}"
+        # simulcast_count が 2 の場合、rid r2 の bytesSent/packetsSent は 0 or 1 になる
+        # simulcast_count が 1 の場合、rid r2 と r1 の bytesSent/packetsSent は 0 or 1 になる
+        if i < simulcast_count:
+            # 1 本になると simulcastEncodingAdapter がなくなる
+            if simulcast_count > 1:
+                assert "SimulcastEncoderAdapter" in s["encoderImplementation"]
+            assert expected_implementation in s["encoderImplementation"]
+
+            assert s["bytesSent"] > 1000
+            assert s["packetsSent"] > 20
+            # targetBitrate が指定したビットレートの 90% 以上、100% 以下に収まることを確認
+            expected_bitrate = video_bit_rate * 1000
+            print(
+                s["rid"],
+                video_codec_type,
+                expected_implementation,
+                expected_bitrate,
+                s["targetBitrate"],
+                s["frameWidth"],
+                s["frameHeight"],
+                s["bytesSent"],
+                s["packetsSent"],
+            )
+            # 期待値の 20% 以上、100% 以下に収まることを確認
+            assert expected_bitrate * 0.2 <= s["targetBitrate"] <= expected_bitrate
+        else:
+            # 本来は 0 なのだが、simulcast_count が 1 の場合、
+            # packetSent が 0 ではなく 1 や 2 になる場合がある
+            # byteSent は 0
+            assert s["bytesSent"] == 0
+            assert s["packetsSent"] <= 2
+            print(
+                s["rid"],
+                video_codec_type,
+                s["bytesSent"],
+                s["packetsSent"],
+            )

tests/test_simulcast.py

@@ -7,14 +7,54 @@
 
 
 @pytest.mark.parametrize(
-    ("video_codec_type", "expected_implementation"),
+    (
+        "video_codec_type",
+        "expected_implementation",
+        "video_bit_rate",
+        "video_width",
+        "video_height",
+        "simulcast_count",
+    ),
     [
-        ("VP8", "libvpx"),
-        ("VP9", "libvpx"),
-        ("AV1", "libaom"),
+        # AV1 は VP8 と同じビットレートとして扱う
+        # https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/video/config/simulcast.cc;l=219-222
+        # 1080p
+        ("VP8", "libvpx", 5000, 1920, 1080, 3),
+        ("VP9", "libvpx", 3367, 1920, 1080, 3),
+        ("AV1", "libaom", 5000, 1920, 1080, 3),
+        # 720p
+        ("VP8", "libvpx", 2500, 1280, 720, 3),
+        ("VP9", "libvpx", 1524, 1280, 720, 3),
+        ("AV1", "libaom", 2500, 1280, 720, 3),
+        # 540p
+        ("VP8", "libvpx", 1200, 960, 540, 3),
+        ("VP9", "libvpx", 879, 960, 540, 3),
+        ("AV1", "libaom", 1200, 960, 540, 3),
+        # 360p
+        ("VP8", "libvpx", 700, 640, 360, 2),
+        ("VP9", "libvpx", 420, 640, 360, 2),
+        ("AV1", "libaom", 700, 640, 360, 2),
+        # 270p
+        ("VP8", "libvpx", 450, 480, 270, 2),
+        ("VP9", "libvpx", 257, 480, 270, 2),
+        ("AV1", "libaom", 450, 480, 270, 2),
+        # 180p
+        ("VP8", "libvpx", 200, 320, 180, 1),
+        ("VP9", "libvpx", 142, 320, 180, 1),
+        ("AV1", "libaom", 200, 320, 180, 1),
+        # 135p
+        ("VP9", "libvpx", 101, 240, 135, 1),
     ],
 )
-def test_simulcast(setup, video_codec_type, expected_implementation):
+def test_simulcast(
+    setup,
+    video_codec_type,
+    expected_implementation,
+    video_bit_rate,
+    video_width,
+    video_height,
+    simulcast_count,
+):
     signaling_urls = setup.get("signaling_urls")
     channel_id_prefix = setup.get("channel_id_prefix")
     metadata = setup.get("metadata")
@@ -29,10 +69,10 @@ def test_simulcast(setup, video_codec_type, expected_implementation):
         audio=False,
         video=True,
         video_codec_type=video_codec_type,
-        video_bit_rate=3000,
+        video_bit_rate=video_bit_rate,
         metadata=metadata,
-        video_width=1280,
-        video_height=720,
+        video_width=video_width,
+        video_height=video_height,
     )
     sendonly.connect(fake_video=True)
 
@@ -55,14 +95,49 @@ def test_simulcast(setup, video_codec_type, expected_implementation):
     outbound_rtp_stats = [
         s for s in sendonly_stats if s.get("type") == "outbound-rtp" and s.get("kind") == "video"
     ]
+    # simulcast_count に関係なく統計情報はかならず 3 本出力される
+    # これは SDP で rid で ~r0 とかやる減るはず
     assert len(outbound_rtp_stats) == 3
 
     # rid でソート
     sorted_stats = sorted(outbound_rtp_stats, key=lambda x: x.get("rid", ""))
 
-    for i, rtp_stat in enumerate(sorted_stats):
-        assert rtp_stat["rid"] == f"r{i}"
-        assert "SimulcastEncoderAdapter" in rtp_stat["encoderImplementation"]
-        assert expected_implementation in rtp_stat["encoderImplementation"]
-        assert rtp_stat["bytesSent"] > 0
-        assert rtp_stat["packetsSent"] > 0
+    for i, s in enumerate(sorted_stats):
+        assert s["rid"] == f"r{i}"
+        # simulcast_count が 2 の場合、rid r2 の bytesSent/packetsSent は 0 or 1 になる
+        # simulcast_count が 1 の場合、rid r2 と r1 の bytesSent/packetsSent は 0 or 1 になる
+        if i < simulcast_count:
+            # 1 本になると simulcastEncodingAdapter がなくなる
+            if simulcast_count > 1:
+                assert "SimulcastEncoderAdapter" in s["encoderImplementation"]
+            assert expected_implementation in s["encoderImplementation"]
+
+            assert s["bytesSent"] > 1000
+            assert s["packetsSent"] > 20
+            # targetBitrate が指定したビットレートの 90% 以上、100% 以下に収まることを確認
+            expected_bitrate = video_bit_rate * 1000
+            print(
+                s["rid"],
+                video_codec_type,
+                expected_implementation,
+                expected_bitrate,
+                s["targetBitrate"],
+                s["frameWidth"],
+                s["frameHeight"],
+                s["bytesSent"],
+                s["packetsSent"],
+            )
+            # 期待値の 20% 以上、100% 以下に収まることを確認
+            assert expected_bitrate * 0.2 <= s["targetBitrate"] <= expected_bitrate
+        else:
+            # 本来は 0 なのだが、simulcast_count が 1 の場合、
+            # packetSent が 0 ではなく 1 や 2 になる場合がある
+            # byteSent は 0
+            assert s["bytesSent"] == 0
+            assert s["packetsSent"] <= 2
+            print(
+                s["rid"],
+                video_codec_type,
+                s["bytesSent"],
+                s["packetsSent"],
+            )