From 1235d0808ed7697874085fc8528f22aec988c85e Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 15 Nov 2024 13:18:50 -0800 Subject: [PATCH] Use livekit's Rust SDK instead of their swift SDK (#13343) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/livekit/rust-sdks/pull/355 Todo: * [x] make `call` / `live_kit_client` crates use the livekit rust sdk * [x] create a fake version of livekit rust API for integration tests * [x] capture local audio * [x] play remote audio * [x] capture local video tracks * [x] play remote video tracks * [x] tests passing * bugs * [x] deafening does not work (https://github.com/livekit/rust-sdks/issues/359) * [x] mute and speaking status are not replicated properly: (https://github.com/livekit/rust-sdks/issues/358) * [x] **linux** - crash due to symbol conflict between WebRTC's BoringSSL and libcurl's openssl (https://github.com/livekit/rust-sdks/issues/89) * [x] **linux** - libwebrtc-sys adds undesired dependencies on `libGL` and `libXext` * [x] **windows** - linker error, maybe related to the C++ stdlib (https://github.com/livekit/rust-sdks/issues/364) ``` libwebrtc_sys-54978c6ad5066a35.rlib(video_frame.obj) : error LNK2038: mismatch detected for 'RuntimeLibrary': value 'MT_StaticRelease' doesn't match value 'MD_DynamicRelease' in libtree_sitter_yaml-df6b0adf8f009e8f.rlib(2e40c9e35e9506f4-scanner.o) ``` * [x] audio problems Release Notes: - Switch from Swift to Rust LiveKit SDK 🦀 --------- Co-authored-by: Mikayla Maki Co-authored-by: Conrad Irwin Co-authored-by: Kirill Bulatov Co-authored-by: Michael Sloan --- .cargo/config.toml | 6 + Cargo.lock | 434 +++++++- Cargo.toml | 6 + crates/call/Cargo.toml | 1 + crates/call/src/call.rs | 9 + crates/call/src/participant.rs | 28 +- crates/call/src/room.rs | 824 ++++++++------- crates/collab/src/tests.rs | 3 + .../collab/src/tests/channel_guest_tests.rs | 24 +- crates/collab/src/tests/following_tests.rs | 13 +- crates/collab/src/tests/integration_tests.rs | 27 +- crates/collab/src/tests/test_server.rs | 6 +- crates/collab_ui/src/collab_panel.rs | 5 +- crates/gpui/build.rs | 1 + crates/gpui/src/app.rs | 11 +- crates/gpui/src/app/test_context.rs | 10 +- crates/gpui/src/geometry.rs | 5 + crates/gpui/src/platform.rs | 26 + crates/gpui/src/platform/linux.rs | 2 + crates/gpui/src/platform/linux/platform.rs | 12 +- crates/gpui/src/platform/mac.rs | 5 + crates/gpui/src/platform/mac/platform.rs | 14 +- .../gpui/src/platform/mac/screen_capture.rs | 239 +++++ crates/gpui/src/platform/test.rs | 2 + crates/gpui/src/platform/test/platform.rs | 58 +- crates/gpui/src/platform/windows.rs | 2 + crates/gpui/src/platform/windows/platform.rs | 8 + crates/http_client/Cargo.toml | 2 +- crates/live_kit_client/Cargo.toml | 29 +- .../LiveKitBridge/Package.resolved | 52 - .../LiveKitBridge/Package.swift | 27 - .../live_kit_client/LiveKitBridge/README.md | 3 - .../Sources/LiveKitBridge/LiveKitBridge.swift | 383 ------- crates/live_kit_client/build.rs | 185 ---- crates/live_kit_client/examples/test_app.rs | 494 +++++++-- crates/live_kit_client/src/live_kit_client.rs | 410 +++++++- crates/live_kit_client/src/prod.rs | 981 ------------------ .../src/remote_video_track_view.rs | 61 ++ crates/live_kit_client/src/test.rs | 843 +++++++-------- .../live_kit_client/src/test/participant.rs | 111 ++ .../live_kit_client/src/test/publication.rs | 116 +++ crates/live_kit_client/src/test/track.rs | 201 ++++ crates/live_kit_client/src/test/webrtc.rs | 136 +++ crates/media/Cargo.toml | 1 + crates/media/src/media.rs | 11 +- crates/title_bar/src/collab.rs | 63 +- crates/workspace/src/shared_screen.rs | 52 +- crates/workspace/src/workspace.rs | 13 +- 48 files changed, 3181 insertions(+), 2774 deletions(-) create mode 100644 crates/gpui/src/platform/mac/screen_capture.rs delete mode 100644 crates/live_kit_client/LiveKitBridge/Package.resolved delete mode 100644 crates/live_kit_client/LiveKitBridge/Package.swift delete mode 100644 crates/live_kit_client/LiveKitBridge/README.md delete mode 100644 crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift delete mode 100644 crates/live_kit_client/build.rs delete mode 100644 crates/live_kit_client/src/prod.rs create mode 100644 crates/live_kit_client/src/remote_video_track_view.rs create mode 100644 crates/live_kit_client/src/test/participant.rs create mode 100644 crates/live_kit_client/src/test/publication.rs create mode 100644 crates/live_kit_client/src/test/track.rs create mode 100644 crates/live_kit_client/src/test/webrtc.rs diff --git a/.cargo/config.toml b/.cargo/config.toml index a657ae61b9cf8..043adf6b30e2c 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -13,6 +13,12 @@ rustflags = ["-C", "link-arg=-fuse-ld=mold"] linker = "clang" rustflags = ["-C", "link-arg=-fuse-ld=mold"] +[target.aarch64-apple-darwin] +rustflags = ["-C", "link-args=-Objc -all_load"] + +[target.x86_64-apple-darwin] +rustflags = ["-C", "link-args=-Objc -all_load"] + # This cfg will reduce the size of `windows::core::Error` from 16 bytes to 4 bytes [target.'cfg(target_os = "windows")'] rustflags = ["--cfg", "windows_slim_errors"] diff --git a/Cargo.lock b/Cargo.lock index 9feb65c45814c..0c4ea47526610 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -915,6 +915,22 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "async-tungstenite" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cca750b12e02c389c1694d35c16539f88b8bbaa5945934fdc1b41a776688589" +dependencies = [ + "async-native-tls", + "async-std", + "async-tls", + "futures-io", + "futures-util", + "log", + "pin-project-lite", + "tungstenite 0.21.0", +] + [[package]] name = "async-tungstenite" version = "0.28.0" @@ -1789,7 +1805,7 @@ dependencies = [ "arrayvec", "cc", "cfg-if", - "constant_time_eq", + "constant_time_eq 0.3.1", ] [[package]] @@ -1975,6 +1991,27 @@ dependencies = [ "either", ] +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "call" version = "0.1.0" @@ -1983,6 +2020,7 @@ dependencies = [ "audio", "client", "collections", + "feature_flags", "fs", "futures 0.3.30", "gpui", @@ -2446,7 +2484,7 @@ dependencies = [ "anyhow", "async-native-tls", "async-recursion 0.3.2", - "async-tungstenite", + "async-tungstenite 0.28.0", "chrono", "clock", "cocoa 0.26.0", @@ -2579,7 +2617,7 @@ dependencies = [ "assistant", "async-stripe", "async-trait", - "async-tungstenite", + "async-tungstenite 0.28.0", "audio", "aws-config", "aws-sdk-kinesis", @@ -2630,7 +2668,7 @@ dependencies = [ "pretty_assertions", "project", "prometheus", - "prost", + "prost 0.9.0", "rand 0.8.5", "recent_projects", "release_channel", @@ -2831,6 +2869,12 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + [[package]] name = "constant_time_eq" version = "0.3.1" @@ -3054,8 +3098,7 @@ dependencies = [ [[package]] name = "cpal" version = "0.15.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "873dab07c8f743075e57f524c583985fbaf745602acbe916a01539364369a779" +source = "git+https://github.com/zed-industries/cpal?rev=fd8bc2fd39f1f5fdee5a0690656caff9a26d9d50#fd8bc2fd39f1f5fdee5a0690656caff9a26d9d50" dependencies = [ "alsa", "core-foundation-sys", @@ -3391,6 +3434,50 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96a6ac251f4a2aca6b3f91340350eab87ae57c3f127ffeb585e92bd336717991" +[[package]] +name = "cxx" +version = "1.0.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23c042a0ba58aaff55299632834d1ea53ceff73d62373f62c9ae60890ad1b942" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45dc1c88d0fdac57518a9b1f6c4f4fb2aca8f3c30c0d03d7d8518b47ca0bcea6" +dependencies = [ + "cc", + "codespan-reporting", + "proc-macro2", + "quote", + "scratch", + "syn 2.0.87", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa7ed7d30b289e2592cc55bc2ccd89803a63c913e008e6eb59f06cddf45bb52f" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b8c465d22de46b851c04630a5fc749a26005b263632ed2e0d9cc81518ead78d" +dependencies = [ + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.87", +] + [[package]] name = "dashmap" version = "5.5.3" @@ -4654,6 +4741,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "fsevent" version = "0.1.0" @@ -6139,6 +6236,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.12.1" @@ -6617,6 +6723,29 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libwebrtc" +version = "0.3.7" +source = "git+https://github.com/zed-industries/rust-sdks?rev=4262308983646ab5b0e0802c3d8bc52154f99aab#4262308983646ab5b0e0802c3d8bc52154f99aab" +dependencies = [ + "cxx", + "jni", + "js-sys", + "lazy_static", + "livekit-protocol", + "livekit-runtime", + "log", + "parking_lot", + "serde", + "serde_json", + "thiserror", + "tokio", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webrtc-sys", +] + [[package]] name = "libz-sys" version = "1.1.20" @@ -6629,6 +6758,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "link-cplusplus" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d240c6f7e1ba3a28b0249f774e6a9dd0175054b52dfbb61b16eb8505c3785c9" +dependencies = [ + "cc", +] + [[package]] name = "linkify" version = "0.10.0" @@ -6675,13 +6813,16 @@ name = "live_kit_client" version = "0.1.0" dependencies = [ "anyhow", - "async-broadcast", "async-trait", "collections", "core-foundation 0.9.4", + "cpal", "futures 0.3.30", "gpui", + "http 0.2.12", + "http_client", "live_kit_server", + "livekit", "log", "media", "nanoid", @@ -6691,6 +6832,7 @@ dependencies = [ "serde_json", "sha2", "simplelog", + "util", ] [[package]] @@ -6701,13 +6843,88 @@ dependencies = [ "async-trait", "jsonwebtoken", "log", - "prost", - "prost-build", - "prost-types", + "prost 0.9.0", + "prost-build 0.9.0", + "prost-types 0.9.0", "reqwest 0.12.8", "serde", ] +[[package]] +name = "livekit" +version = "0.7.0" +source = "git+https://github.com/zed-industries/rust-sdks?rev=4262308983646ab5b0e0802c3d8bc52154f99aab#4262308983646ab5b0e0802c3d8bc52154f99aab" +dependencies = [ + "chrono", + "futures-util", + "lazy_static", + "libwebrtc", + "livekit-api", + "livekit-protocol", + "livekit-runtime", + "log", + "parking_lot", + "prost 0.12.6", + "semver", + "serde", + "serde_json", + "thiserror", + "tokio", +] + +[[package]] +name = "livekit-api" +version = "0.4.1" +source = "git+https://github.com/zed-industries/rust-sdks?rev=4262308983646ab5b0e0802c3d8bc52154f99aab#4262308983646ab5b0e0802c3d8bc52154f99aab" +dependencies = [ + "async-tungstenite 0.25.1", + "futures-util", + "http 0.2.12", + "jsonwebtoken", + "livekit-protocol", + "livekit-runtime", + "log", + "parking_lot", + "prost 0.12.6", + "reqwest 0.11.27", + "scopeguard", + "serde", + "serde_json", + "sha2", + "thiserror", + "tokio", + "tokio-tungstenite 0.20.1", + "url", +] + +[[package]] +name = "livekit-protocol" +version = "0.3.6" +source = "git+https://github.com/zed-industries/rust-sdks?rev=4262308983646ab5b0e0802c3d8bc52154f99aab#4262308983646ab5b0e0802c3d8bc52154f99aab" +dependencies = [ + "futures-util", + "livekit-runtime", + "parking_lot", + "pbjson", + "pbjson-types", + "prost 0.12.6", + "prost-types 0.12.6", + "serde", + "thiserror", + "tokio", +] + +[[package]] +name = "livekit-runtime" +version = "0.3.1" +source = "git+https://github.com/zed-industries/rust-sdks?rev=4262308983646ab5b0e0802c3d8bc52154f99aab#4262308983646ab5b0e0802c3d8bc52154f99aab" +dependencies = [ + "async-io 2.3.4", + "async-std", + "async-task", + "futures 0.3.30", +] + [[package]] name = "lmdb-master-sys" version = "0.2.4" @@ -6993,6 +7210,7 @@ dependencies = [ "anyhow", "bindgen 0.70.1", "core-foundation 0.9.4", + "ctor", "foreign-types 0.5.0", "metal", "objc", @@ -7695,7 +7913,7 @@ dependencies = [ "md-5", "num", "num-bigint-dig", - "pbkdf2", + "pbkdf2 0.12.2", "rand 0.8.5", "serde", "sha2", @@ -8015,6 +8233,17 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "password-hash" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" +dependencies = [ + "base64ct", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "password-hash" version = "0.5.0" @@ -8065,6 +8294,55 @@ dependencies = [ "util", ] +[[package]] +name = "pbjson" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1030c719b0ec2a2d25a5df729d6cff1acf3cc230bf766f4f97833591f7577b90" +dependencies = [ + "base64 0.21.7", + "serde", +] + +[[package]] +name = "pbjson-build" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2580e33f2292d34be285c5bc3dba5259542b083cfad6037b6d70345f24dcb735" +dependencies = [ + "heck 0.4.1", + "itertools 0.11.0", + "prost 0.12.6", + "prost-types 0.12.6", +] + +[[package]] +name = "pbjson-types" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18f596653ba4ac51bdecbb4ef6773bc7f56042dc13927910de1684ad3d32aa12" +dependencies = [ + "bytes 1.7.2", + "chrono", + "pbjson", + "pbjson-build", + "prost 0.12.6", + "prost-build 0.12.6", + "serde", +] + +[[package]] +name = "pbkdf2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +dependencies = [ + "digest", + "hmac", + "password-hash 0.4.2", + "sha2", +] + [[package]] name = "pbkdf2" version = "0.12.2" @@ -9072,7 +9350,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001" dependencies = [ "bytes 1.7.2", - "prost-derive", + "prost-derive 0.9.0", +] + +[[package]] +name = "prost" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" +dependencies = [ + "bytes 1.7.2", + "prost-derive 0.12.6", ] [[package]] @@ -9088,13 +9376,34 @@ dependencies = [ "log", "multimap", "petgraph", - "prost", - "prost-types", + "prost 0.9.0", + "prost-types 0.9.0", "regex", "tempfile", "which 4.4.2", ] +[[package]] +name = "prost-build" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" +dependencies = [ + "bytes 1.7.2", + "heck 0.4.1", + "itertools 0.10.5", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost 0.12.6", + "prost-types 0.12.6", + "regex", + "syn 2.0.87", + "tempfile", +] + [[package]] name = "prost-derive" version = "0.9.0" @@ -9108,6 +9417,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "prost-derive" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" +dependencies = [ + "anyhow", + "itertools 0.10.5", + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "prost-types" version = "0.9.0" @@ -9115,7 +9437,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a" dependencies = [ "bytes 1.7.2", - "prost", + "prost 0.9.0", +] + +[[package]] +name = "prost-types" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" +dependencies = [ + "prost 0.12.6", ] [[package]] @@ -9124,8 +9455,8 @@ version = "0.1.0" dependencies = [ "anyhow", "collections", - "prost", - "prost-build", + "prost 0.9.0", + "prost-build 0.9.0", "serde", ] @@ -9645,7 +9976,7 @@ dependencies = [ "log", "parking_lot", "paths", - "prost", + "prost 0.9.0", "release_channel", "rpc", "serde", @@ -9774,6 +10105,7 @@ dependencies = [ "http 0.2.12", "http-body 0.4.6", "hyper 0.14.31", + "hyper-rustls 0.24.2", "hyper-tls", "ipnet", "js-sys", @@ -9783,6 +10115,8 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", + "rustls 0.21.12", + "rustls-native-certs 0.6.3", "rustls-pemfile 1.0.4", "serde", "serde_json", @@ -9791,6 +10125,7 @@ dependencies = [ "system-configuration 0.5.1", "tokio", "tokio-native-tls", + "tokio-rustls 0.24.1", "tower-service", "url", "wasm-bindgen", @@ -10015,7 +10350,7 @@ name = "rpc" version = "0.1.0" dependencies = [ "anyhow", - "async-tungstenite", + "async-tungstenite 0.28.0", "base64 0.22.1", "chrono", "collections", @@ -10390,14 +10725,20 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scratch" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3cf7c11c38cb994f3d40e8a8cde3bbd1f72a435e4c49e85d6553d8312306152" + [[package]] name = "scrypt" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" dependencies = [ - "password-hash", - "pbkdf2", + "password-hash 0.5.0", + "pbkdf2 0.12.2", "salsa20", "sha2", ] @@ -12519,7 +12860,10 @@ checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" dependencies = [ "futures-util", "log", + "rustls 0.21.12", + "rustls-native-certs 0.6.3", "tokio", + "tokio-rustls 0.24.1", "tungstenite 0.20.1", ] @@ -13027,6 +13371,7 @@ dependencies = [ "httparse", "log", "rand 0.8.5", + "rustls 0.21.12", "sha1", "thiserror", "url", @@ -13045,6 +13390,7 @@ dependencies = [ "http 1.1.0", "httparse", "log", + "native-tls", "rand 0.8.5", "sha1", "thiserror", @@ -14121,6 +14467,32 @@ version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" +[[package]] +name = "webrtc-sys" +version = "0.3.5" +source = "git+https://github.com/zed-industries/rust-sdks?rev=4262308983646ab5b0e0802c3d8bc52154f99aab#4262308983646ab5b0e0802c3d8bc52154f99aab" +dependencies = [ + "cc", + "cxx", + "cxx-build", + "glob", + "log", + "webrtc-sys-build", +] + +[[package]] +name = "webrtc-sys-build" +version = "0.3.5" +source = "git+https://github.com/zed-industries/rust-sdks?rev=4262308983646ab5b0e0802c3d8bc52154f99aab#4262308983646ab5b0e0802c3d8bc52154f99aab" +dependencies = [ + "fs2", + "regex", + "reqwest 0.11.27", + "scratch", + "semver", + "zip", +] + [[package]] name = "weezl" version = "0.1.8" @@ -15504,6 +15876,26 @@ dependencies = [ "uuid", ] +[[package]] +name = "zip" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +dependencies = [ + "aes", + "byteorder", + "bzip2", + "constant_time_eq 0.1.5", + "crc32fast", + "crossbeam-utils", + "flate2", + "hmac", + "pbkdf2 0.11.0", + "sha1", + "time", + "zstd", +] + [[package]] name = "zstd" version = "0.11.2+zstd.1.5.2" diff --git a/Cargo.toml b/Cargo.toml index c6d182fd5808a..9d86bc1b20711 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -363,6 +363,7 @@ heed = { version = "0.20.1", features = ["read-txn-no-tls"] } hex = "0.4.3" html5ever = "0.27.0" hyper = "0.14" +http = "1.1" ignore = "0.4.22" image = "0.25.1" indexmap = { version = "1.6.2", features = ["serde"] } @@ -371,6 +372,7 @@ itertools = "0.13.0" jsonwebtoken = "9.3" libc = "0.2" linkify = "0.10.0" +livekit = { git = "https://github.com/zed-industries/rust-sdks", rev="4262308983646ab5b0e0802c3d8bc52154f99aab", features = ["dispatcher", "services-dispatcher", "rustls-tls-native-roots"], default-features = false } log = { version = "0.4.16", features = ["kv_unstable_serde", "serde"] } markup5ever_rcdom = "0.3.0" nanoid = "0.4" @@ -549,6 +551,10 @@ features = [ "Win32_UI_WindowsAndMessaging", ] +# TODO livekit https://github.com/RustAudio/cpal/pull/891 +[patch.crates-io] +cpal = { git = "https://github.com/zed-industries/cpal", rev = "fd8bc2fd39f1f5fdee5a0690656caff9a26d9d50" } + [profile.dev] split-debuginfo = "unpacked" debug = "limited" diff --git a/crates/call/Cargo.toml b/crates/call/Cargo.toml index 974c860c08bdb..3ac55078e73bc 100644 --- a/crates/call/Cargo.toml +++ b/crates/call/Cargo.toml @@ -27,6 +27,7 @@ anyhow.workspace = true audio.workspace = true client.workspace = true collections.workspace = true +feature_flags.workspace = true fs.workspace = true futures.workspace = true gpui.workspace = true diff --git a/crates/call/src/call.rs b/crates/call/src/call.rs index c7993f365880d..eb91ff08851cb 100644 --- a/crates/call/src/call.rs +++ b/crates/call/src/call.rs @@ -18,6 +18,11 @@ use room::Event; use settings::Settings; use std::sync::Arc; +#[cfg(not(target_os = "windows"))] +pub use live_kit_client::play_remote_video_track; +pub use live_kit_client::{ + track::RemoteVideoTrack, RemoteVideoTrackView, RemoteVideoTrackViewEvent, +}; pub use participant::ParticipantLocation; pub use room::Room; @@ -26,6 +31,10 @@ struct GlobalActiveCall(Model); impl Global for GlobalActiveCall {} pub fn init(client: Arc, user_store: Model, cx: &mut AppContext) { + live_kit_client::init( + cx.background_executor().dispatcher.clone(), + cx.http_client(), + ); CallSettings::register(cx); let active_call = cx.new_model(|cx| ActiveCall::new(client, user_store, cx)); diff --git a/crates/call/src/participant.rs b/crates/call/src/participant.rs index 9faefc63c3697..67bb95a3b7ed8 100644 --- a/crates/call/src/participant.rs +++ b/crates/call/src/participant.rs @@ -1,13 +1,17 @@ +#![cfg_attr(target_os = "windows", allow(unused))] + use anyhow::{anyhow, Result}; -use client::ParticipantIndex; -use client::{proto, User}; +use client::{proto, ParticipantIndex, User}; use collections::HashMap; use gpui::WeakModel; -pub use live_kit_client::Frame; -pub use live_kit_client::{RemoteAudioTrack, RemoteVideoTrack}; +use live_kit_client::AudioStream; use project::Project; use std::sync::Arc; +#[cfg(not(target_os = "windows"))] +pub use live_kit_client::id::TrackSid; +pub use live_kit_client::track::{RemoteAudioTrack, RemoteVideoTrack}; + #[derive(Copy, Clone, Debug, Eq, PartialEq)] pub enum ParticipantLocation { SharedProject { project_id: u64 }, @@ -39,7 +43,6 @@ pub struct LocalParticipant { pub role: proto::ChannelRole, } -#[derive(Clone, Debug)] pub struct RemoteParticipant { pub user: Arc, pub peer_id: proto::PeerId, @@ -49,6 +52,17 @@ pub struct RemoteParticipant { pub participant_index: ParticipantIndex, pub muted: bool, pub speaking: bool, - pub video_tracks: HashMap>, - pub audio_tracks: HashMap>, + #[cfg(not(target_os = "windows"))] + pub video_tracks: HashMap, + #[cfg(not(target_os = "windows"))] + pub audio_tracks: HashMap, +} + +impl RemoteParticipant { + pub fn has_video_tracks(&self) -> bool { + #[cfg(not(target_os = "windows"))] + return !self.video_tracks.is_empty(); + #[cfg(target_os = "windows")] + return false; + } } diff --git a/crates/call/src/room.rs b/crates/call/src/room.rs index 3eb98f3109ff4..a0e9de201a180 100644 --- a/crates/call/src/room.rs +++ b/crates/call/src/room.rs @@ -1,3 +1,5 @@ +#![cfg_attr(target_os = "windows", allow(unused))] + use crate::{ call_settings::CallSettings, participant::{LocalParticipant, ParticipantLocation, RemoteParticipant}, @@ -15,11 +17,23 @@ use gpui::{ AppContext, AsyncAppContext, Context, EventEmitter, Model, ModelContext, Task, WeakModel, }; use language::LanguageRegistry; -use live_kit_client::{LocalAudioTrack, LocalTrackPublication, LocalVideoTrack, RoomUpdate}; +use live_kit_client as livekit; +#[cfg(not(target_os = "windows"))] +use livekit::{ + capture_local_audio_track, capture_local_video_track, + id::ParticipantIdentity, + options::{TrackPublishOptions, VideoCodec}, + play_remote_audio_track, + publication::LocalTrackPublication, + track::{TrackKind, TrackSource}, + RoomEvent, RoomOptions, +}; +#[cfg(target_os = "windows")] +use livekit::{publication::LocalTrackPublication, RoomEvent}; use postage::{sink::Sink, stream::Stream, watch}; use project::Project; use settings::Settings as _; -use std::{future::Future, mem, sync::Arc, time::Duration}; +use std::{any::Any, future::Future, mem, sync::Arc, time::Duration}; use util::{post_inc, ResultExt, TryFutureExt}; pub const RECONNECT_TIMEOUT: Duration = Duration::from_secs(30); @@ -92,13 +106,10 @@ impl Room { !self.shared_projects.is_empty() } - #[cfg(any(test, feature = "test-support"))] + #[cfg(all(any(test, feature = "test-support"), not(target_os = "windows")))] pub fn is_connected(&self) -> bool { if let Some(live_kit) = self.live_kit.as_ref() { - matches!( - *live_kit.room.status().borrow(), - live_kit_client::ConnectionState::Connected { .. } - ) + live_kit.room.connection_state() == livekit::ConnectionState::Connected } else { false } @@ -112,77 +123,7 @@ impl Room { user_store: Model, cx: &mut ModelContext, ) -> Self { - let live_kit_room = if let Some(connection_info) = live_kit_connection_info { - let room = live_kit_client::Room::new(); - let mut status = room.status(); - // Consume the initial status of the room. - let _ = status.try_recv(); - let _maintain_room = cx.spawn(|this, mut cx| async move { - while let Some(status) = status.next().await { - let this = if let Some(this) = this.upgrade() { - this - } else { - break; - }; - - if status == live_kit_client::ConnectionState::Disconnected { - this.update(&mut cx, |this, cx| this.leave(cx).log_err()) - .ok(); - break; - } - } - }); - - let _handle_updates = cx.spawn({ - let room = room.clone(); - move |this, mut cx| async move { - let mut updates = room.updates(); - while let Some(update) = updates.next().await { - let this = if let Some(this) = this.upgrade() { - this - } else { - break; - }; - - this.update(&mut cx, |this, cx| { - this.live_kit_room_updated(update, cx).log_err() - }) - .ok(); - } - } - }); - - let connect = room.connect(&connection_info.server_url, &connection_info.token); - cx.spawn(|this, mut cx| async move { - connect.await?; - this.update(&mut cx, |this, cx| { - if this.can_use_microphone() { - if let Some(live_kit) = &this.live_kit { - if !live_kit.muted_by_user && !live_kit.deafened { - return this.share_microphone(cx); - } - } - } - Task::ready(Ok(())) - })? - .await - }) - .detach_and_log_err(cx); - - Some(LiveKitRoom { - room, - screen_track: LocalTrack::None, - microphone_track: LocalTrack::None, - next_publish_id: 0, - muted_by_user: Self::mute_on_join(cx), - deafened: false, - speaking: false, - _maintain_room, - _handle_updates, - }) - } else { - None - }; + spawn_room_connection(live_kit_connection_info, cx); let maintain_connection = cx.spawn({ let client = client.clone(); @@ -196,7 +137,7 @@ impl Room { Self { id, channel_id, - live_kit: live_kit_room, + live_kit: None, status: RoomStatus::Online, shared_projects: Default::default(), joined_projects: Default::default(), @@ -706,11 +647,45 @@ impl Room { this.update(&mut cx, |this, cx| this.apply_room_update(room, cx))? } - fn apply_room_update( - &mut self, + fn apply_room_update(&mut self, room: proto::Room, cx: &mut ModelContext) -> Result<()> { + log::trace!( + "client {:?}. room update: {:?}", + self.client.user_id(), + &room + ); + + self.pending_room_update = Some(self.start_room_connection(room, cx)); + + cx.notify(); + Ok(()) + } + + pub fn room_update_completed(&mut self) -> impl Future { + let mut done_rx = self.room_update_completed_rx.clone(); + async move { + while let Some(result) = done_rx.next().await { + if result.is_some() { + break; + } + } + } + } + + #[cfg(target_os = "windows")] + fn start_room_connection( + &self, mut room: proto::Room, cx: &mut ModelContext, - ) -> Result<()> { + ) -> Task<()> { + Task::ready(()) + } + + #[cfg(not(target_os = "windows"))] + fn start_room_connection( + &self, + mut room: proto::Room, + cx: &mut ModelContext, + ) -> Task<()> { // Filter ourselves out from the room's participants. let local_participant_ix = room .participants @@ -737,8 +712,7 @@ impl Room { user_store.get_users(pending_participant_user_ids, cx), ) }); - - self.pending_room_update = Some(cx.spawn(|this, mut cx| async move { + cx.spawn(|this, mut cx| async move { let (remote_participants, pending_participants) = futures::join!(remote_participants, pending_participants); @@ -776,6 +750,11 @@ impl Room { this.local_participant.projects.clear(); } + let livekit_participants = this + .live_kit + .as_ref() + .map(|live_kit| live_kit.room.remote_participants()); + if let Some(participants) = remote_participants.log_err() { for (participant, user) in room.participants.into_iter().zip(participants) { let Some(peer_id) = participant.peer_id else { @@ -858,40 +837,31 @@ impl Room { muted: true, speaking: false, video_tracks: Default::default(), + #[cfg(not(target_os = "windows"))] audio_tracks: Default::default(), }, ); Audio::play_sound(Sound::Joined, cx); - - if let Some(live_kit) = this.live_kit.as_ref() { - let video_tracks = - live_kit.room.remote_video_tracks(&user.id.to_string()); - let audio_tracks = - live_kit.room.remote_audio_tracks(&user.id.to_string()); - let publications = live_kit - .room - .remote_audio_track_publications(&user.id.to_string()); - - for track in video_tracks { - this.live_kit_room_updated( - RoomUpdate::SubscribedToRemoteVideoTrack(track), - cx, - ) - .log_err(); - } - - for (track, publication) in - audio_tracks.iter().zip(publications.iter()) + if let Some(livekit_participants) = &livekit_participants { + if let Some(livekit_participant) = livekit_participants + .get(&ParticipantIdentity(user.id.to_string())) { - this.live_kit_room_updated( - RoomUpdate::SubscribedToRemoteAudioTrack( - track.clone(), - publication.clone(), - ), - cx, - ) - .log_err(); + for publication in + livekit_participant.track_publications().into_values() + { + if let Some(track) = publication.track() { + this.live_kit_room_updated( + RoomEvent::TrackSubscribed { + track, + publication, + participant: livekit_participant.clone(), + }, + cx, + ) + .warn_on_err(); + } + } } } } @@ -959,61 +929,89 @@ impl Room { cx.notify(); }) .ok(); - })); - - cx.notify(); - Ok(()) - } - - pub fn room_update_completed(&mut self) -> impl Future { - let mut done_rx = self.room_update_completed_rx.clone(); - async move { - while let Some(result) = done_rx.next().await { - if result.is_some() { - break; - } - } - } + }) } fn live_kit_room_updated( &mut self, - update: RoomUpdate, + event: RoomEvent, cx: &mut ModelContext, ) -> Result<()> { - match update { - RoomUpdate::SubscribedToRemoteVideoTrack(track) => { - let user_id = track.publisher_id().parse()?; - let track_id = track.sid().to_string(); - let participant = self - .remote_participants - .get_mut(&user_id) - .ok_or_else(|| anyhow!("subscribed to track by unknown participant"))?; - participant.video_tracks.insert(track_id.clone(), track); - cx.emit(Event::RemoteVideoTracksChanged { - participant_id: participant.peer_id, - }); + log::trace!( + "client {:?}. livekit event: {:?}", + self.client.user_id(), + &event + ); + + match event { + #[cfg(not(target_os = "windows"))] + RoomEvent::TrackSubscribed { + track, + participant, + publication, + } => { + let user_id = participant.identity().0.parse()?; + let track_id = track.sid(); + let participant = self.remote_participants.get_mut(&user_id).ok_or_else(|| { + anyhow!( + "{:?} subscribed to track by unknown participant {user_id}", + self.client.user_id() + ) + })?; + if self.live_kit.as_ref().map_or(true, |kit| kit.deafened) { + track.rtc_track().set_enabled(false); + } + match track { + livekit::track::RemoteTrack::Audio(track) => { + cx.emit(Event::RemoteAudioTracksChanged { + participant_id: participant.peer_id, + }); + let stream = play_remote_audio_track(&track, cx); + participant.audio_tracks.insert(track_id, (track, stream)); + participant.muted = publication.is_muted(); + } + livekit::track::RemoteTrack::Video(track) => { + cx.emit(Event::RemoteVideoTracksChanged { + participant_id: participant.peer_id, + }); + participant.video_tracks.insert(track_id, track); + } + } } - RoomUpdate::UnsubscribedFromRemoteVideoTrack { - publisher_id, - track_id, + #[cfg(not(target_os = "windows"))] + RoomEvent::TrackUnsubscribed { + track, participant, .. } => { - let user_id = publisher_id.parse()?; - let participant = self - .remote_participants - .get_mut(&user_id) - .ok_or_else(|| anyhow!("unsubscribed from track by unknown participant"))?; - participant.video_tracks.remove(&track_id); - cx.emit(Event::RemoteVideoTracksChanged { - participant_id: participant.peer_id, - }); + let user_id = participant.identity().0.parse()?; + let participant = self.remote_participants.get_mut(&user_id).ok_or_else(|| { + anyhow!( + "{:?}, unsubscribed from track by unknown participant {user_id}", + self.client.user_id() + ) + })?; + match track { + livekit::track::RemoteTrack::Audio(track) => { + participant.audio_tracks.remove(&track.sid()); + participant.muted = true; + cx.emit(Event::RemoteAudioTracksChanged { + participant_id: participant.peer_id, + }); + } + livekit::track::RemoteTrack::Video(track) => { + participant.video_tracks.remove(&track.sid()); + cx.emit(Event::RemoteVideoTracksChanged { + participant_id: participant.peer_id, + }); + } + } } - RoomUpdate::ActiveSpeakersChanged { speakers } => { + #[cfg(not(target_os = "windows"))] + RoomEvent::ActiveSpeakersChanged { speakers } => { let mut speaker_ids = speakers .into_iter() - .filter_map(|speaker_sid| speaker_sid.parse().ok()) + .filter_map(|speaker| speaker.identity().0.parse().ok()) .collect::>(); speaker_ids.sort_unstable(); for (sid, participant) in &mut self.remote_participants { @@ -1026,82 +1024,65 @@ impl Room { } } - RoomUpdate::RemoteAudioTrackMuteChanged { track_id, muted } => { + #[cfg(not(target_os = "windows"))] + RoomEvent::TrackMuted { + participant, + publication, + } + | RoomEvent::TrackUnmuted { + participant, + publication, + } => { let mut found = false; - for participant in &mut self.remote_participants.values_mut() { - for track in participant.audio_tracks.values() { + let user_id = participant.identity().0.parse()?; + let track_id = publication.sid(); + if let Some(participant) = self.remote_participants.get_mut(&user_id) { + for (track, _) in participant.audio_tracks.values() { if track.sid() == track_id { found = true; break; } } if found { - participant.muted = muted; - break; - } - } - } - - RoomUpdate::SubscribedToRemoteAudioTrack(track, publication) => { - if let Some(live_kit) = &self.live_kit { - if live_kit.deafened { - track.stop(); - cx.foreground_executor() - .spawn(publication.set_enabled(false)) - .detach(); + participant.muted = publication.is_muted(); } } - - let user_id = track.publisher_id().parse()?; - let track_id = track.sid().to_string(); - let participant = self - .remote_participants - .get_mut(&user_id) - .ok_or_else(|| anyhow!("subscribed to track by unknown participant"))?; - participant.audio_tracks.insert(track_id.clone(), track); - participant.muted = publication.is_muted(); - - cx.emit(Event::RemoteAudioTracksChanged { - participant_id: participant.peer_id, - }); - } - - RoomUpdate::UnsubscribedFromRemoteAudioTrack { - publisher_id, - track_id, - } => { - let user_id = publisher_id.parse()?; - let participant = self - .remote_participants - .get_mut(&user_id) - .ok_or_else(|| anyhow!("unsubscribed from track by unknown participant"))?; - participant.audio_tracks.remove(&track_id); - cx.emit(Event::RemoteAudioTracksChanged { - participant_id: participant.peer_id, - }); - } - - RoomUpdate::LocalAudioTrackUnpublished { publication } => { - log::info!("unpublished audio track {}", publication.sid()); - if let Some(room) = &mut self.live_kit { - room.microphone_track = LocalTrack::None; - } } - RoomUpdate::LocalVideoTrackUnpublished { publication } => { - log::info!("unpublished video track {}", publication.sid()); + #[cfg(not(target_os = "windows"))] + RoomEvent::LocalTrackUnpublished { publication, .. } => { + log::info!("unpublished track {}", publication.sid()); if let Some(room) = &mut self.live_kit { - room.screen_track = LocalTrack::None; + if let LocalTrack::Published { + track_publication, .. + } = &room.microphone_track + { + if track_publication.sid() == publication.sid() { + room.microphone_track = LocalTrack::None; + } + } + if let LocalTrack::Published { + track_publication, .. + } = &room.screen_track + { + if track_publication.sid() == publication.sid() { + room.screen_track = LocalTrack::None; + } + } } } - RoomUpdate::LocalAudioTrackPublished { publication } => { - log::info!("published audio track {}", publication.sid()); + #[cfg(not(target_os = "windows"))] + RoomEvent::LocalTrackPublished { publication, .. } => { + log::info!("published track {:?}", publication.sid()); } - RoomUpdate::LocalVideoTrackPublished { publication } => { - log::info!("published video track {}", publication.sid()); + #[cfg(not(target_os = "windows"))] + RoomEvent::Disconnected { reason } => { + log::info!("disconnected from room: {reason:?}"); + self.leave(cx).detach_and_log_err(cx); } + _ => {} } cx.notify(); @@ -1317,8 +1298,17 @@ impl Room { self.live_kit.as_ref().map(|live_kit| live_kit.deafened) } - pub fn can_use_microphone(&self) -> bool { + pub fn can_use_microphone(&self, _cx: &AppContext) -> bool { use proto::ChannelRole::*; + + #[cfg(not(any(test, feature = "test-support")))] + { + use feature_flags::FeatureFlagAppExt as _; + if cfg!(target_os = "windows") || (cfg!(target_os = "linux") && !_cx.is_staff()) { + return false; + } + } + match self.local_participant.role { Admin | Member | Talker => true, Guest | Banned => false, @@ -1333,161 +1323,177 @@ impl Room { } } + #[cfg(target_os = "windows")] + pub fn share_microphone(&mut self, cx: &mut ModelContext) -> Task> { + Task::ready(Err(anyhow!("Windows is not supported yet"))) + } + + #[cfg(not(target_os = "windows"))] #[track_caller] pub fn share_microphone(&mut self, cx: &mut ModelContext) -> Task> { if self.status.is_offline() { return Task::ready(Err(anyhow!("room is offline"))); } - let publish_id = if let Some(live_kit) = self.live_kit.as_mut() { + let (participant, publish_id) = if let Some(live_kit) = self.live_kit.as_mut() { let publish_id = post_inc(&mut live_kit.next_publish_id); live_kit.microphone_track = LocalTrack::Pending { publish_id }; cx.notify(); - publish_id + (live_kit.room.local_participant(), publish_id) } else { return Task::ready(Err(anyhow!("live-kit was not initialized"))); }; cx.spawn(move |this, mut cx| async move { - let publish_track = async { - let track = LocalAudioTrack::create(); - this.upgrade() - .ok_or_else(|| anyhow!("room was dropped"))? - .update(&mut cx, |this, _| { - this.live_kit - .as_ref() - .map(|live_kit| live_kit.room.publish_audio_track(track)) - })? - .ok_or_else(|| anyhow!("live-kit was not initialized"))? - .await - }; - let publication = publish_track.await; - this.upgrade() - .ok_or_else(|| anyhow!("room was dropped"))? - .update(&mut cx, |this, cx| { - let live_kit = this - .live_kit - .as_mut() - .ok_or_else(|| anyhow!("live-kit was not initialized"))?; - - let canceled = if let LocalTrack::Pending { - publish_id: cur_publish_id, - } = &live_kit.microphone_track - { - *cur_publish_id != publish_id - } else { - true - }; - - match publication { - Ok(publication) => { - if canceled { - live_kit.room.unpublish_track(publication); - } else { - if live_kit.muted_by_user || live_kit.deafened { - cx.background_executor() - .spawn(publication.set_mute(true)) - .detach(); - } - live_kit.microphone_track = LocalTrack::Published { - track_publication: publication, - }; - cx.notify(); + let (track, stream) = cx.update(capture_local_audio_track)??; + + let publication = participant + .publish_track( + livekit::track::LocalTrack::Audio(track), + TrackPublishOptions { + source: TrackSource::Microphone, + ..Default::default() + }, + ) + .await + .map_err(|error| anyhow!("failed to publish track: {error}")); + this.update(&mut cx, |this, cx| { + let live_kit = this + .live_kit + .as_mut() + .ok_or_else(|| anyhow!("live-kit was not initialized"))?; + + let canceled = if let LocalTrack::Pending { + publish_id: cur_publish_id, + } = &live_kit.microphone_track + { + *cur_publish_id != publish_id + } else { + true + }; + + match publication { + Ok(publication) => { + if canceled { + cx.background_executor() + .spawn(async move { + participant.unpublish_track(&publication.sid()).await + }) + .detach_and_log_err(cx) + } else { + if live_kit.muted_by_user || live_kit.deafened { + publication.mute(); } - Ok(()) + live_kit.microphone_track = LocalTrack::Published { + track_publication: publication, + _stream: Box::new(stream), + }; + cx.notify(); } - Err(error) => { - if canceled { - Ok(()) - } else { - live_kit.microphone_track = LocalTrack::None; - cx.notify(); - Err(error) - } + Ok(()) + } + Err(error) => { + if canceled { + Ok(()) + } else { + live_kit.microphone_track = LocalTrack::None; + cx.notify(); + Err(error) } } - })? + } + })? }) } + #[cfg(target_os = "windows")] + pub fn share_screen(&mut self, cx: &mut ModelContext) -> Task> { + Task::ready(Err(anyhow!("Windows is not supported yet"))) + } + + #[cfg(not(target_os = "windows"))] pub fn share_screen(&mut self, cx: &mut ModelContext) -> Task> { if self.status.is_offline() { return Task::ready(Err(anyhow!("room is offline"))); - } else if self.is_screen_sharing() { + } + if self.is_screen_sharing() { return Task::ready(Err(anyhow!("screen was already shared"))); } - let (displays, publish_id) = if let Some(live_kit) = self.live_kit.as_mut() { + let (participant, publish_id) = if let Some(live_kit) = self.live_kit.as_mut() { let publish_id = post_inc(&mut live_kit.next_publish_id); live_kit.screen_track = LocalTrack::Pending { publish_id }; cx.notify(); - (live_kit.room.display_sources(), publish_id) + (live_kit.room.local_participant(), publish_id) } else { return Task::ready(Err(anyhow!("live-kit was not initialized"))); }; - cx.spawn(move |this, mut cx| async move { - let publish_track = async { - let displays = displays.await?; - let display = displays - .first() - .ok_or_else(|| anyhow!("no display found"))?; - let track = LocalVideoTrack::screen_share_for_display(display); - this.upgrade() - .ok_or_else(|| anyhow!("room was dropped"))? - .update(&mut cx, |this, _| { - this.live_kit - .as_ref() - .map(|live_kit| live_kit.room.publish_video_track(track)) - })? - .ok_or_else(|| anyhow!("live-kit was not initialized"))? - .await - }; - - let publication = publish_track.await; - this.upgrade() - .ok_or_else(|| anyhow!("room was dropped"))? - .update(&mut cx, |this, cx| { - let live_kit = this - .live_kit - .as_mut() - .ok_or_else(|| anyhow!("live-kit was not initialized"))?; - - let canceled = if let LocalTrack::Pending { - publish_id: cur_publish_id, - } = &live_kit.screen_track - { - *cur_publish_id != publish_id - } else { - true - }; + let sources = cx.screen_capture_sources(); - match publication { - Ok(publication) => { - if canceled { - live_kit.room.unpublish_track(publication); - } else { - live_kit.screen_track = LocalTrack::Published { - track_publication: publication, - }; - cx.notify(); - } + cx.spawn(move |this, mut cx| async move { + let sources = sources.await??; + let source = sources.first().ok_or_else(|| anyhow!("no display found"))?; + + let (track, stream) = capture_local_video_track(&**source).await?; + + let publication = participant + .publish_track( + livekit::track::LocalTrack::Video(track), + TrackPublishOptions { + source: TrackSource::Screenshare, + video_codec: VideoCodec::H264, + ..Default::default() + }, + ) + .await + .map_err(|error| anyhow!("error publishing screen track {error:?}")); - Audio::play_sound(Sound::StartScreenshare, cx); + this.update(&mut cx, |this, cx| { + let live_kit = this + .live_kit + .as_mut() + .ok_or_else(|| anyhow!("live-kit was not initialized"))?; + + let canceled = if let LocalTrack::Pending { + publish_id: cur_publish_id, + } = &live_kit.screen_track + { + *cur_publish_id != publish_id + } else { + true + }; + + match publication { + Ok(publication) => { + if canceled { + cx.background_executor() + .spawn(async move { + participant.unpublish_track(&publication.sid()).await + }) + .detach() + } else { + live_kit.screen_track = LocalTrack::Published { + track_publication: publication, + _stream: Box::new(stream), + }; + cx.notify(); + } + Audio::play_sound(Sound::StartScreenshare, cx); + Ok(()) + } + Err(error) => { + if canceled { Ok(()) - } - Err(error) => { - if canceled { - Ok(()) - } else { - live_kit.screen_track = LocalTrack::None; - cx.notify(); - Err(error) - } + } else { + live_kit.screen_track = LocalTrack::None; + cx.notify(); + Err(error) } } - })? + } + })? }) } @@ -1512,9 +1518,7 @@ impl Room { } if should_undeafen { - if let Some(task) = self.set_deafened(false, cx) { - task.detach_and_log_err(cx); - } + self.set_deafened(false, cx); } } } @@ -1527,9 +1531,7 @@ impl Room { live_kit.deafened = deafened; let should_change_mute = !live_kit.muted_by_user; - if let Some(task) = self.set_deafened(deafened, cx) { - task.detach_and_log_err(cx); - } + self.set_deafened(deafened, cx); if should_change_mute { if let Some(task) = self.set_mute(deafened, cx) { @@ -1557,47 +1559,36 @@ impl Room { LocalTrack::Published { track_publication, .. } => { - live_kit.room.unpublish_track(track_publication); - cx.notify(); - + #[cfg(not(target_os = "windows"))] + { + let local_participant = live_kit.room.local_participant(); + let sid = track_publication.sid(); + cx.background_executor() + .spawn(async move { local_participant.unpublish_track(&sid).await }) + .detach_and_log_err(cx); + cx.notify(); + } Audio::play_sound(Sound::StopScreenshare, cx); Ok(()) } } } - fn set_deafened( - &mut self, - deafened: bool, - cx: &mut ModelContext, - ) -> Option>> { - let live_kit = self.live_kit.as_mut()?; - cx.notify(); - - let mut track_updates = Vec::new(); - for participant in self.remote_participants.values() { - for publication in live_kit - .room - .remote_audio_track_publications(&participant.user.id.to_string()) - { - track_updates.push(publication.set_enabled(!deafened)); - } - - for track in participant.audio_tracks.values() { - if deafened { - track.stop(); - } else { - track.start(); + fn set_deafened(&mut self, deafened: bool, cx: &mut ModelContext) -> Option<()> { + #[cfg(not(target_os = "windows"))] + { + let live_kit = self.live_kit.as_mut()?; + cx.notify(); + for (_, participant) in live_kit.room.remote_participants() { + for (_, publication) in participant.track_publications() { + if publication.kind() == TrackKind::Audio { + publication.set_enabled(!deafened); + } } } } - Some(cx.foreground_executor().spawn(async move { - for result in futures::future::join_all(track_updates).await { - result?; - } - Ok(()) - })) + None } fn set_mute( @@ -1623,25 +1614,84 @@ impl Room { } } LocalTrack::Pending { .. } => None, - LocalTrack::Published { track_publication } => Some( - cx.foreground_executor() - .spawn(track_publication.set_mute(should_mute)), - ), + LocalTrack::Published { + track_publication, .. + } => { + #[cfg(not(target_os = "windows"))] + { + if should_mute { + track_publication.mute() + } else { + track_publication.unmute() + } + } + None + } } } +} - #[cfg(any(test, feature = "test-support"))] - pub fn set_display_sources(&self, sources: Vec) { - self.live_kit - .as_ref() - .unwrap() - .room - .set_display_sources(sources); +#[cfg(target_os = "windows")] +fn spawn_room_connection( + live_kit_connection_info: Option, + cx: &mut ModelContext<'_, Room>, +) { +} + +#[cfg(not(target_os = "windows"))] +fn spawn_room_connection( + live_kit_connection_info: Option, + cx: &mut ModelContext<'_, Room>, +) { + if let Some(connection_info) = live_kit_connection_info { + cx.spawn(|this, mut cx| async move { + let (room, mut events) = livekit::Room::connect( + &connection_info.server_url, + &connection_info.token, + RoomOptions::default(), + ) + .await?; + + this.update(&mut cx, |this, cx| { + let _handle_updates = cx.spawn(|this, mut cx| async move { + while let Some(event) = events.recv().await { + if this + .update(&mut cx, |this, cx| { + this.live_kit_room_updated(event, cx).warn_on_err(); + }) + .is_err() + { + break; + } + } + }); + + let muted_by_user = Room::mute_on_join(cx); + this.live_kit = Some(LiveKitRoom { + room: Arc::new(room), + screen_track: LocalTrack::None, + microphone_track: LocalTrack::None, + next_publish_id: 0, + muted_by_user, + deafened: false, + speaking: false, + _handle_updates, + }); + + if !muted_by_user && this.can_use_microphone(cx) { + this.share_microphone(cx) + } else { + Task::ready(Ok(())) + } + })? + .await + }) + .detach_and_log_err(cx); } } struct LiveKitRoom { - room: Arc, + room: Arc, screen_track: LocalTrack, microphone_track: LocalTrack, /// Tracks whether we're currently in a muted state due to auto-mute from deafening or manual mute performed by user. @@ -1649,17 +1699,21 @@ struct LiveKitRoom { deafened: bool, speaking: bool, next_publish_id: usize, - _maintain_room: Task<()>, _handle_updates: Task<()>, } impl LiveKitRoom { + #[cfg(target_os = "windows")] + fn stop_publishing(&mut self, _cx: &mut ModelContext) {} + + #[cfg(not(target_os = "windows"))] fn stop_publishing(&mut self, cx: &mut ModelContext) { + let mut tracks_to_unpublish = Vec::new(); if let LocalTrack::Published { track_publication, .. } = mem::replace(&mut self.microphone_track, LocalTrack::None) { - self.room.unpublish_track(track_publication); + tracks_to_unpublish.push(track_publication.sid()); cx.notify(); } @@ -1667,9 +1721,18 @@ impl LiveKitRoom { track_publication, .. } = mem::replace(&mut self.screen_track, LocalTrack::None) { - self.room.unpublish_track(track_publication); + tracks_to_unpublish.push(track_publication.sid()); cx.notify(); } + + let participant = self.room.local_participant(); + cx.background_executor() + .spawn(async move { + for sid in tracks_to_unpublish { + participant.unpublish_track(&sid).await.log_err(); + } + }) + .detach(); } } @@ -1680,6 +1743,7 @@ enum LocalTrack { }, Published { track_publication: LocalTrackPublication, + _stream: Box, }, } diff --git a/crates/collab/src/tests.rs b/crates/collab/src/tests.rs index 29373bc6ea170..2ce69efc9b406 100644 --- a/crates/collab/src/tests.rs +++ b/crates/collab/src/tests.rs @@ -1,3 +1,6 @@ +// todo(windows): Actually run the tests +#![cfg(not(target_os = "windows"))] + use std::sync::Arc; use call::Room; diff --git a/crates/collab/src/tests/channel_guest_tests.rs b/crates/collab/src/tests/channel_guest_tests.rs index 5a091fe3083b1..006a3e5d1cf22 100644 --- a/crates/collab/src/tests/channel_guest_tests.rs +++ b/crates/collab/src/tests/channel_guest_tests.rs @@ -107,7 +107,9 @@ async fn test_channel_guest_promotion(cx_a: &mut TestAppContext, cx_b: &mut Test }); assert!(project_b.read_with(cx_b, |project, cx| project.is_read_only(cx))); assert!(editor_b.update(cx_b, |e, cx| e.read_only(cx))); - assert!(room_b.read_with(cx_b, |room, _| !room.can_use_microphone())); + cx_b.update(|cx_b| { + assert!(room_b.read_with(cx_b, |room, cx| !room.can_use_microphone(cx))); + }); assert!(room_b .update(cx_b, |room, cx| room.share_microphone(cx)) .await @@ -133,7 +135,9 @@ async fn test_channel_guest_promotion(cx_a: &mut TestAppContext, cx_b: &mut Test assert!(editor_b.update(cx_b, |editor, cx| !editor.read_only(cx))); // B sees themselves as muted, and can unmute. - assert!(room_b.read_with(cx_b, |room, _| room.can_use_microphone())); + cx_b.update(|cx_b| { + assert!(room_b.read_with(cx_b, |room, cx| room.can_use_microphone(cx))); + }); room_b.read_with(cx_b, |room, _| assert!(room.is_muted())); room_b.update(cx_b, |room, cx| room.toggle_mute(cx)); cx_a.run_until_parked(); @@ -226,7 +230,9 @@ async fn test_channel_requires_zed_cla(cx_a: &mut TestAppContext, cx_b: &mut Tes let room_b = cx_b .read(ActiveCall::global) .update(cx_b, |call, _| call.room().unwrap().clone()); - assert!(room_b.read_with(cx_b, |room, _| !room.can_use_microphone())); + cx_b.update(|cx_b| { + assert!(room_b.read_with(cx_b, |room, cx| !room.can_use_microphone(cx))); + }); // A tries to grant write access to B, but cannot because B has not // yet signed the zed CLA. @@ -244,7 +250,9 @@ async fn test_channel_requires_zed_cla(cx_a: &mut TestAppContext, cx_b: &mut Tes .unwrap_err(); cx_a.run_until_parked(); assert!(room_b.read_with(cx_b, |room, _| !room.can_share_projects())); - assert!(room_b.read_with(cx_b, |room, _| !room.can_use_microphone())); + cx_b.update(|cx_b| { + assert!(room_b.read_with(cx_b, |room, cx| !room.can_use_microphone(cx))); + }); // A tries to grant write access to B, but cannot because B has not // yet signed the zed CLA. @@ -262,7 +270,9 @@ async fn test_channel_requires_zed_cla(cx_a: &mut TestAppContext, cx_b: &mut Tes .unwrap(); cx_a.run_until_parked(); assert!(room_b.read_with(cx_b, |room, _| !room.can_share_projects())); - assert!(room_b.read_with(cx_b, |room, _| room.can_use_microphone())); + cx_b.update(|cx_b| { + assert!(room_b.read_with(cx_b, |room, cx| room.can_use_microphone(cx))); + }); // User B signs the zed CLA. server @@ -287,5 +297,7 @@ async fn test_channel_requires_zed_cla(cx_a: &mut TestAppContext, cx_b: &mut Tes .unwrap(); cx_a.run_until_parked(); assert!(room_b.read_with(cx_b, |room, _| room.can_share_projects())); - assert!(room_b.read_with(cx_b, |room, _| room.can_use_microphone())); + cx_b.update(|cx_b| { + assert!(room_b.read_with(cx_b, |room, cx| room.can_use_microphone(cx))); + }); } diff --git a/crates/collab/src/tests/following_tests.rs b/crates/collab/src/tests/following_tests.rs index d708194f58396..778d67b81dcad 100644 --- a/crates/collab/src/tests/following_tests.rs +++ b/crates/collab/src/tests/following_tests.rs @@ -9,10 +9,9 @@ use collab_ui::{ use editor::{Editor, ExcerptRange, MultiBuffer}; use gpui::{ point, BackgroundExecutor, BorrowAppContext, Context, Entity, SharedString, TestAppContext, - View, VisualContext, VisualTestContext, + TestScreenCaptureSource, View, VisualContext, VisualTestContext, }; use language::Capability; -use live_kit_client::MacOSDisplay; use project::WorktreeSettings; use rpc::proto::PeerId; use serde_json::json; @@ -429,17 +428,17 @@ async fn test_basic_following( ); // Client B activates an external window, which causes a new screen-sharing item to be added to the pane. - let display = MacOSDisplay::new(); + let display = TestScreenCaptureSource::new(); active_call_b .update(cx_b, |call, cx| call.set_location(None, cx)) .await .unwrap(); + cx_b.set_screen_capture_sources(vec![display]); active_call_b .update(cx_b, |call, cx| { - call.room().unwrap().update(cx, |room, cx| { - room.set_display_sources(vec![display.clone()]); - room.share_screen(cx) - }) + call.room() + .unwrap() + .update(cx, |room, cx| room.share_screen(cx)) }) .await .unwrap(); diff --git a/crates/collab/src/tests/integration_tests.rs b/crates/collab/src/tests/integration_tests.rs index 5ec9a574a1e20..94b7ad81c5a30 100644 --- a/crates/collab/src/tests/integration_tests.rs +++ b/crates/collab/src/tests/integration_tests.rs @@ -15,7 +15,7 @@ use futures::{channel::mpsc, StreamExt as _}; use git::repository::GitFileStatus; use gpui::{ px, size, AppContext, BackgroundExecutor, Model, Modifiers, MouseButton, MouseDownEvent, - TestAppContext, UpdateGlobal, + TestAppContext, TestScreenCaptureSource, UpdateGlobal, }; use language::{ language_settings::{ @@ -24,7 +24,6 @@ use language::{ tree_sitter_rust, tree_sitter_typescript, Diagnostic, DiagnosticEntry, FakeLspAdapter, Language, LanguageConfig, LanguageMatcher, LineEnding, OffsetRangeExt, Point, Rope, }; -use live_kit_client::MacOSDisplay; use lsp::LanguageServerId; use parking_lot::Mutex; use project::lsp_store::FormatTarget; @@ -241,15 +240,15 @@ async fn test_basic_calls( ); // User A shares their screen - let display = MacOSDisplay::new(); + let display = TestScreenCaptureSource::new(); let events_b = active_call_events(cx_b); let events_c = active_call_events(cx_c); + cx_a.set_screen_capture_sources(vec![display]); active_call_a .update(cx_a, |call, cx| { - call.room().unwrap().update(cx, |room, cx| { - room.set_display_sources(vec![display.clone()]); - room.share_screen(cx) - }) + call.room() + .unwrap() + .update(cx, |room, cx| room.share_screen(cx)) }) .await .unwrap(); @@ -1942,7 +1941,7 @@ async fn test_mute_deafen( room_a.read_with(cx_a, |room, _| assert!(!room.is_muted())); room_b.read_with(cx_b, |room, _| assert!(!room.is_muted())); - // Users A and B are both muted. + // Users A and B are both unmuted. assert_eq!( participant_audio_state(&room_a, cx_a), &[ParticipantAudioState { @@ -2074,7 +2073,7 @@ async fn test_mute_deafen( audio_tracks_playing: participant .audio_tracks .values() - .map(|track| track.is_playing()) + .map(|(track, _)| track.rtc_track().enabled()) .collect(), }) .collect::>() @@ -6057,13 +6056,13 @@ async fn test_join_call_after_screen_was_shared( assert_eq!(call_b.calling_user.github_login, "user_a"); // User A shares their screen - let display = MacOSDisplay::new(); + let display = TestScreenCaptureSource::new(); + cx_a.set_screen_capture_sources(vec![display]); active_call_a .update(cx_a, |call, cx| { - call.room().unwrap().update(cx, |room, cx| { - room.set_display_sources(vec![display.clone()]); - room.share_screen(cx) - }) + call.room() + .unwrap() + .update(cx, |room, cx| room.share_screen(cx)) }) .await .unwrap(); diff --git a/crates/collab/src/tests/test_server.rs b/crates/collab/src/tests/test_server.rs index 17cd1b51c42cd..ae9e101031f74 100644 --- a/crates/collab/src/tests/test_server.rs +++ b/crates/collab/src/tests/test_server.rs @@ -47,7 +47,7 @@ use workspace::{Workspace, WorkspaceStore}; pub struct TestServer { pub app_state: Arc, - pub test_live_kit_server: Arc, + pub test_live_kit_server: Arc, server: Arc, next_github_user_id: i32, connection_killers: Arc>>>, @@ -89,7 +89,7 @@ impl TestServer { TestDb::sqlite(deterministic.clone()) }; let live_kit_server_id = NEXT_LIVE_KIT_SERVER_ID.fetch_add(1, SeqCst); - let live_kit_server = live_kit_client::TestServer::create( + let live_kit_server = live_kit_client::test::TestServer::create( format!("http://livekit.{}.test", live_kit_server_id), format!("devkey-{}", live_kit_server_id), format!("secret-{}", live_kit_server_id), @@ -499,7 +499,7 @@ impl TestServer { pub async fn build_app_state( test_db: &TestDb, - live_kit_test_server: &live_kit_client::TestServer, + live_kit_test_server: &live_kit_client::test::TestServer, executor: Executor, ) -> Arc { Arc::new(AppState { diff --git a/crates/collab_ui/src/collab_panel.rs b/crates/collab_ui/src/collab_panel.rs index c93a48096a4b4..fa3ab0219b77d 100644 --- a/crates/collab_ui/src/collab_panel.rs +++ b/crates/collab_ui/src/collab_panel.rs @@ -474,11 +474,10 @@ impl CollabPanel { project_id: project.id, worktree_root_names: project.worktree_root_names.clone(), host_user_id: participant.user.id, - is_last: projects.peek().is_none() - && participant.video_tracks.is_empty(), + is_last: projects.peek().is_none() && !participant.has_video_tracks(), }); } - if !participant.video_tracks.is_empty() { + if participant.has_video_tracks() { self.entries.push(ListEntry::ParticipantScreen { peer_id: Some(participant.peer_id), is_last: true, diff --git a/crates/gpui/build.rs b/crates/gpui/build.rs index 5a015106c722a..e5917a0f052f2 100644 --- a/crates/gpui/build.rs +++ b/crates/gpui/build.rs @@ -48,6 +48,7 @@ mod macos { fn generate_dispatch_bindings() { println!("cargo:rustc-link-lib=framework=System"); + println!("cargo:rustc-link-lib=framework=ScreenCaptureKit"); println!("cargo:rerun-if-changed=src/platform/mac/dispatch.h"); let bindings = bindgen::Builder::default() diff --git a/crates/gpui/src/app.rs b/crates/gpui/src/app.rs index 2a8da1c506e59..da14927fa21f0 100644 --- a/crates/gpui/src/app.rs +++ b/crates/gpui/src/app.rs @@ -33,8 +33,8 @@ use crate::{ Entity, EventEmitter, ForegroundExecutor, Global, KeyBinding, Keymap, Keystroke, LayoutId, Menu, MenuItem, OwnedMenu, PathPromptOptions, Pixels, Platform, PlatformDisplay, Point, PromptBuilder, PromptHandle, PromptLevel, Render, RenderablePromptHandle, Reservation, - SharedString, SubscriberSet, Subscription, SvgRenderer, Task, TextSystem, View, ViewContext, - Window, WindowAppearance, WindowContext, WindowHandle, WindowId, + ScreenCaptureSource, SharedString, SubscriberSet, Subscription, SvgRenderer, Task, TextSystem, + View, ViewContext, Window, WindowAppearance, WindowContext, WindowHandle, WindowId, }; mod async_context; @@ -599,6 +599,13 @@ impl AppContext { self.platform.primary_display() } + /// Returns a list of available screen capture sources. + pub fn screen_capture_sources( + &self, + ) -> oneshot::Receiver>>> { + self.platform.screen_capture_sources() + } + /// Returns the display with the given ID, if one exists. pub fn find_display(&self, id: DisplayId) -> Option> { self.displays() diff --git a/crates/gpui/src/app/test_context.rs b/crates/gpui/src/app/test_context.rs index 34449c91ec732..2f5053a382f6a 100644 --- a/crates/gpui/src/app/test_context.rs +++ b/crates/gpui/src/app/test_context.rs @@ -4,8 +4,8 @@ use crate::{ Element, Empty, Entity, EventEmitter, ForegroundExecutor, Global, InputEvent, Keystroke, Model, ModelContext, Modifiers, ModifiersChangedEvent, MouseButton, MouseDownEvent, MouseMoveEvent, MouseUpEvent, Pixels, Platform, Point, Render, Result, Size, Task, TestDispatcher, - TestPlatform, TestWindow, TextSystem, View, ViewContext, VisualContext, WindowBounds, - WindowContext, WindowHandle, WindowOptions, + TestPlatform, TestScreenCaptureSource, TestWindow, TextSystem, View, ViewContext, + VisualContext, WindowBounds, WindowContext, WindowHandle, WindowOptions, }; use anyhow::{anyhow, bail}; use futures::{channel::oneshot, Stream, StreamExt}; @@ -287,6 +287,12 @@ impl TestAppContext { self.test_window(window_handle).simulate_resize(size); } + /// Causes the given sources to be returned if the application queries for screen + /// capture sources. + pub fn set_screen_capture_sources(&self, sources: Vec) { + self.test_platform.set_screen_capture_sources(sources); + } + /// Returns all windows open in the test. pub fn windows(&self) -> Vec { self.app.borrow().windows().clone() diff --git a/crates/gpui/src/geometry.rs b/crates/gpui/src/geometry.rs index 9e0b9b9014039..b636c95a614e6 100644 --- a/crates/gpui/src/geometry.rs +++ b/crates/gpui/src/geometry.rs @@ -704,6 +704,11 @@ pub struct Bounds { pub size: Size, } +/// Create a bounds with the given origin and size +pub fn bounds(origin: Point, size: Size) -> Bounds { + Bounds { origin, size } +} + impl Bounds { /// Generate a centered bounds for the given display or primary display if none is provided pub fn centered(display_id: Option, size: Size, cx: &AppContext) -> Self { diff --git a/crates/gpui/src/platform.rs b/crates/gpui/src/platform.rs index a8424d197a762..727ca952daba6 100644 --- a/crates/gpui/src/platform.rs +++ b/crates/gpui/src/platform.rs @@ -70,6 +70,9 @@ pub(crate) use test::*; #[cfg(target_os = "windows")] pub(crate) use windows::*; +#[cfg(any(test, feature = "test-support"))] +pub use test::TestScreenCaptureSource; + #[cfg(target_os = "macos")] pub(crate) fn current_platform(headless: bool) -> Rc { Rc::new(MacPlatform::new(headless)) @@ -149,6 +152,10 @@ pub(crate) trait Platform: 'static { None } + fn screen_capture_sources( + &self, + ) -> oneshot::Receiver>>>; + fn open_window( &self, handle: AnyWindowHandle, @@ -228,6 +235,25 @@ pub trait PlatformDisplay: Send + Sync + Debug { } } +/// A source of on-screen video content that can be captured. +pub trait ScreenCaptureSource { + /// Returns the video resolution of this source. + fn resolution(&self) -> Result>; + + /// Start capture video from this source, invoking the given callback + /// with each frame. + fn stream( + &self, + frame_callback: Box, + ) -> oneshot::Receiver>>; +} + +/// A video stream captured from a screen. +pub trait ScreenCaptureStream {} + +/// A frame of video captured from a screen. +pub struct ScreenCaptureFrame(pub PlatformScreenCaptureFrame); + /// An opaque identifier for a hardware display #[derive(PartialEq, Eq, Hash, Copy, Clone)] pub struct DisplayId(pub(crate) u32); diff --git a/crates/gpui/src/platform/linux.rs b/crates/gpui/src/platform/linux.rs index 04998693616fd..089b52cf1e730 100644 --- a/crates/gpui/src/platform/linux.rs +++ b/crates/gpui/src/platform/linux.rs @@ -20,3 +20,5 @@ pub(crate) use text_system::*; pub(crate) use wayland::*; #[cfg(feature = "x11")] pub(crate) use x11::*; + +pub(crate) type PlatformScreenCaptureFrame = (); diff --git a/crates/gpui/src/platform/linux/platform.rs b/crates/gpui/src/platform/linux/platform.rs index a2e9af691b554..5865e500926b5 100644 --- a/crates/gpui/src/platform/linux/platform.rs +++ b/crates/gpui/src/platform/linux/platform.rs @@ -35,8 +35,8 @@ use crate::{ px, Action, AnyWindowHandle, BackgroundExecutor, ClipboardItem, CursorStyle, DisplayId, ForegroundExecutor, Keymap, Keystroke, LinuxDispatcher, Menu, MenuItem, Modifiers, OwnedMenu, PathPromptOptions, Pixels, Platform, PlatformDisplay, PlatformInputHandler, PlatformTextSystem, - PlatformWindow, Point, PromptLevel, Result, SemanticVersion, SharedString, Size, Task, - WindowAppearance, WindowOptions, WindowParams, + PlatformWindow, Point, PromptLevel, Result, ScreenCaptureSource, SemanticVersion, SharedString, + Size, Task, WindowAppearance, WindowOptions, WindowParams, }; pub(crate) const SCROLL_LINES: f32 = 3.0; @@ -242,6 +242,14 @@ impl Platform for P { self.displays() } + fn screen_capture_sources( + &self, + ) -> oneshot::Receiver>>> { + let (mut tx, rx) = oneshot::channel(); + tx.send(Err(anyhow!("screen capture not implemented"))).ok(); + rx + } + fn active_window(&self) -> Option { self.active_window() } diff --git a/crates/gpui/src/platform/mac.rs b/crates/gpui/src/platform/mac.rs index 396fd49d04896..bd3d8f35ac9d1 100644 --- a/crates/gpui/src/platform/mac.rs +++ b/crates/gpui/src/platform/mac.rs @@ -4,12 +4,14 @@ mod dispatcher; mod display; mod display_link; mod events; +mod screen_capture; #[cfg(not(feature = "macos-blade"))] mod metal_atlas; #[cfg(not(feature = "macos-blade"))] pub mod metal_renderer; +use media::core_video::CVImageBuffer; #[cfg(not(feature = "macos-blade"))] use metal_renderer as renderer; @@ -49,6 +51,9 @@ pub(crate) use window::*; #[cfg(feature = "font-kit")] pub(crate) use text_system::*; +/// A frame of video captured from a screen. +pub(crate) type PlatformScreenCaptureFrame = CVImageBuffer; + trait BoolExt { fn to_objc(self) -> BOOL; } diff --git a/crates/gpui/src/platform/mac/platform.rs b/crates/gpui/src/platform/mac/platform.rs index b744c658cee1b..d0fd8a85f43cc 100644 --- a/crates/gpui/src/platform/mac/platform.rs +++ b/crates/gpui/src/platform/mac/platform.rs @@ -1,14 +1,14 @@ use super::{ attributed_string::{NSAttributedString, NSMutableAttributedString}, events::key_to_native, - BoolExt, + renderer, screen_capture, BoolExt, }; use crate::{ hash, Action, AnyWindowHandle, BackgroundExecutor, ClipboardEntry, ClipboardItem, ClipboardString, CursorStyle, ForegroundExecutor, Image, ImageFormat, Keymap, MacDispatcher, MacDisplay, MacWindow, Menu, MenuItem, PathPromptOptions, Platform, PlatformDisplay, - PlatformTextSystem, PlatformWindow, Result, SemanticVersion, Task, WindowAppearance, - WindowParams, + PlatformTextSystem, PlatformWindow, Result, ScreenCaptureSource, SemanticVersion, Task, + WindowAppearance, WindowParams, }; use anyhow::anyhow; use block::ConcreteBlock; @@ -58,8 +58,6 @@ use std::{ }; use strum::IntoEnumIterator; -use super::renderer; - #[allow(non_upper_case_globals)] const NSUTF8StringEncoding: NSUInteger = 4; @@ -550,6 +548,12 @@ impl Platform for MacPlatform { .collect() } + fn screen_capture_sources( + &self, + ) -> oneshot::Receiver>>> { + screen_capture::get_sources() + } + fn active_window(&self) -> Option { MacWindow::active_window() } diff --git a/crates/gpui/src/platform/mac/screen_capture.rs b/crates/gpui/src/platform/mac/screen_capture.rs new file mode 100644 index 0000000000000..a2b535996fa1a --- /dev/null +++ b/crates/gpui/src/platform/mac/screen_capture.rs @@ -0,0 +1,239 @@ +use crate::{ + platform::{ScreenCaptureFrame, ScreenCaptureSource, ScreenCaptureStream}, + px, size, Pixels, Size, +}; +use anyhow::{anyhow, Result}; +use block::ConcreteBlock; +use cocoa::{ + base::{id, nil, YES}, + foundation::NSArray, +}; +use core_foundation::base::TCFType; +use ctor::ctor; +use futures::channel::oneshot; +use media::core_media::{CMSampleBuffer, CMSampleBufferRef}; +use metal::NSInteger; +use objc::{ + class, + declare::ClassDecl, + msg_send, + runtime::{Class, Object, Sel}, + sel, sel_impl, +}; +use std::{cell::RefCell, ffi::c_void, mem, ptr, rc::Rc}; + +#[derive(Clone)] +pub struct MacScreenCaptureSource { + sc_display: id, +} + +pub struct MacScreenCaptureStream { + sc_stream: id, + sc_stream_output: id, +} + +#[link(name = "ScreenCaptureKit", kind = "framework")] +extern "C" {} + +static mut DELEGATE_CLASS: *const Class = ptr::null(); +static mut OUTPUT_CLASS: *const Class = ptr::null(); +const FRAME_CALLBACK_IVAR: &str = "frame_callback"; + +#[allow(non_upper_case_globals)] +const SCStreamOutputTypeScreen: NSInteger = 0; + +impl ScreenCaptureSource for MacScreenCaptureSource { + fn resolution(&self) -> Result> { + unsafe { + let width: i64 = msg_send![self.sc_display, width]; + let height: i64 = msg_send![self.sc_display, height]; + Ok(size(px(width as f32), px(height as f32))) + } + } + + fn stream( + &self, + frame_callback: Box, + ) -> oneshot::Receiver>> { + unsafe { + let stream: id = msg_send![class!(SCStream), alloc]; + let filter: id = msg_send![class!(SCContentFilter), alloc]; + let configuration: id = msg_send![class!(SCStreamConfiguration), alloc]; + let delegate: id = msg_send![DELEGATE_CLASS, alloc]; + let output: id = msg_send![OUTPUT_CLASS, alloc]; + + let excluded_windows = NSArray::array(nil); + let filter: id = msg_send![filter, initWithDisplay:self.sc_display excludingWindows:excluded_windows]; + let configuration: id = msg_send![configuration, init]; + let delegate: id = msg_send![delegate, init]; + let output: id = msg_send![output, init]; + + output.as_mut().unwrap().set_ivar( + FRAME_CALLBACK_IVAR, + Box::into_raw(Box::new(frame_callback)) as *mut c_void, + ); + + let stream: id = msg_send![stream, initWithFilter:filter configuration:configuration delegate:delegate]; + + let (mut tx, rx) = oneshot::channel(); + + let mut error: id = nil; + let _: () = msg_send![stream, addStreamOutput:output type:SCStreamOutputTypeScreen sampleHandlerQueue:0 error:&mut error as *mut id]; + if error != nil { + let message: id = msg_send![error, localizedDescription]; + tx.send(Err(anyhow!("failed to add stream output {message:?}"))) + .ok(); + return rx; + } + + let tx = Rc::new(RefCell::new(Some(tx))); + let handler = ConcreteBlock::new({ + move |error: id| { + let result = if error == nil { + let stream = MacScreenCaptureStream { + sc_stream: stream, + sc_stream_output: output, + }; + Ok(Box::new(stream) as Box) + } else { + let message: id = msg_send![error, localizedDescription]; + Err(anyhow!("failed to stop screen capture stream {message:?}")) + }; + if let Some(tx) = tx.borrow_mut().take() { + tx.send(result).ok(); + } + } + }); + let handler = handler.copy(); + let _: () = msg_send![stream, startCaptureWithCompletionHandler:handler]; + rx + } + } +} + +impl Drop for MacScreenCaptureSource { + fn drop(&mut self) { + unsafe { + let _: () = msg_send![self.sc_display, release]; + } + } +} + +impl ScreenCaptureStream for MacScreenCaptureStream {} + +impl Drop for MacScreenCaptureStream { + fn drop(&mut self) { + unsafe { + let mut error: id = nil; + let _: () = msg_send![self.sc_stream, removeStreamOutput:self.sc_stream_output type:SCStreamOutputTypeScreen error:&mut error as *mut _]; + if error != nil { + let message: id = msg_send![error, localizedDescription]; + log::error!("failed to add stream output {message:?}"); + } + + let handler = ConcreteBlock::new(move |error: id| { + if error != nil { + let message: id = msg_send![error, localizedDescription]; + log::error!("failed to stop screen capture stream {message:?}"); + } + }); + let block = handler.copy(); + let _: () = msg_send![self.sc_stream, stopCaptureWithCompletionHandler:block]; + let _: () = msg_send![self.sc_stream, release]; + let _: () = msg_send![self.sc_stream_output, release]; + } + } +} + +pub(crate) fn get_sources() -> oneshot::Receiver>>> { + unsafe { + let (mut tx, rx) = oneshot::channel(); + let tx = Rc::new(RefCell::new(Some(tx))); + + let block = ConcreteBlock::new(move |shareable_content: id, error: id| { + let Some(mut tx) = tx.borrow_mut().take() else { + return; + }; + let result = if error == nil { + let displays: id = msg_send![shareable_content, displays]; + let mut result = Vec::new(); + for i in 0..displays.count() { + let display = displays.objectAtIndex(i); + let source = MacScreenCaptureSource { + sc_display: msg_send![display, retain], + }; + result.push(Box::new(source) as Box); + } + Ok(result) + } else { + let msg: id = msg_send![error, localizedDescription]; + Err(anyhow!("Failed to register: {:?}", msg)) + }; + tx.send(result).ok(); + }); + let block = block.copy(); + + let _: () = msg_send![ + class!(SCShareableContent), + getShareableContentExcludingDesktopWindows:YES + onScreenWindowsOnly:YES + completionHandler:block]; + rx + } +} + +#[ctor] +unsafe fn build_classes() { + let mut decl = ClassDecl::new("GPUIStreamDelegate", class!(NSObject)).unwrap(); + decl.add_method( + sel!(outputVideoEffectDidStartForStream:), + output_video_effect_did_start_for_stream as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(outputVideoEffectDidStopForStream:), + output_video_effect_did_stop_for_stream as extern "C" fn(&Object, Sel, id), + ); + decl.add_method( + sel!(stream:didStopWithError:), + stream_did_stop_with_error as extern "C" fn(&Object, Sel, id, id), + ); + DELEGATE_CLASS = decl.register(); + + let mut decl = ClassDecl::new("GPUIStreamOutput", class!(NSObject)).unwrap(); + decl.add_method( + sel!(stream:didOutputSampleBuffer:ofType:), + stream_did_output_sample_buffer_of_type as extern "C" fn(&Object, Sel, id, id, NSInteger), + ); + decl.add_ivar::<*mut c_void>(FRAME_CALLBACK_IVAR); + + OUTPUT_CLASS = decl.register(); +} + +extern "C" fn output_video_effect_did_start_for_stream(_this: &Object, _: Sel, _stream: id) {} + +extern "C" fn output_video_effect_did_stop_for_stream(_this: &Object, _: Sel, _stream: id) {} + +extern "C" fn stream_did_stop_with_error(_this: &Object, _: Sel, _stream: id, _error: id) {} + +extern "C" fn stream_did_output_sample_buffer_of_type( + this: &Object, + _: Sel, + _stream: id, + sample_buffer: id, + buffer_type: NSInteger, +) { + if buffer_type != SCStreamOutputTypeScreen { + return; + } + + unsafe { + let sample_buffer = sample_buffer as CMSampleBufferRef; + let sample_buffer = CMSampleBuffer::wrap_under_get_rule(sample_buffer); + if let Some(buffer) = sample_buffer.image_buffer() { + let callback: Box> = + Box::from_raw(*this.get_ivar::<*mut c_void>(FRAME_CALLBACK_IVAR) as *mut _); + callback(ScreenCaptureFrame(buffer)); + mem::forget(callback); + } + } +} diff --git a/crates/gpui/src/platform/test.rs b/crates/gpui/src/platform/test.rs index d17739239eede..70462cb5e2cae 100644 --- a/crates/gpui/src/platform/test.rs +++ b/crates/gpui/src/platform/test.rs @@ -7,3 +7,5 @@ pub(crate) use dispatcher::*; pub(crate) use display::*; pub(crate) use platform::*; pub(crate) use window::*; + +pub use platform::TestScreenCaptureSource; diff --git a/crates/gpui/src/platform/test/platform.rs b/crates/gpui/src/platform/test/platform.rs index aadbe9b5953d8..67227b60fec04 100644 --- a/crates/gpui/src/platform/test/platform.rs +++ b/crates/gpui/src/platform/test/platform.rs @@ -1,7 +1,7 @@ use crate::{ - AnyWindowHandle, BackgroundExecutor, ClipboardItem, CursorStyle, ForegroundExecutor, Keymap, - Platform, PlatformDisplay, PlatformTextSystem, Task, TestDisplay, TestWindow, WindowAppearance, - WindowParams, + px, size, AnyWindowHandle, BackgroundExecutor, ClipboardItem, CursorStyle, ForegroundExecutor, + Keymap, Platform, PlatformDisplay, PlatformTextSystem, ScreenCaptureFrame, ScreenCaptureSource, + ScreenCaptureStream, Task, TestDisplay, TestWindow, WindowAppearance, WindowParams, }; use anyhow::Result; use collections::VecDeque; @@ -31,6 +31,7 @@ pub(crate) struct TestPlatform { #[cfg(any(target_os = "linux", target_os = "freebsd"))] current_primary_item: Mutex>, pub(crate) prompts: RefCell, + screen_capture_sources: RefCell>, pub opened_url: RefCell>, pub text_system: Arc, #[cfg(target_os = "windows")] @@ -38,6 +39,31 @@ pub(crate) struct TestPlatform { weak: Weak, } +#[derive(Clone)] +/// A fake screen capture source, used for testing. +pub struct TestScreenCaptureSource {} + +pub struct TestScreenCaptureStream {} + +impl ScreenCaptureSource for TestScreenCaptureSource { + fn resolution(&self) -> Result> { + Ok(size(px(1.), px(1.))) + } + + fn stream( + &self, + _frame_callback: Box, + ) -> oneshot::Receiver>> { + let (mut tx, rx) = oneshot::channel(); + let stream = TestScreenCaptureStream {}; + tx.send(Ok(Box::new(stream) as Box)) + .ok(); + rx + } +} + +impl ScreenCaptureStream for TestScreenCaptureStream {} + #[derive(Default)] pub(crate) struct TestPrompts { multiple_choice: VecDeque>, @@ -72,6 +98,7 @@ impl TestPlatform { background_executor: executor, foreground_executor, prompts: Default::default(), + screen_capture_sources: Default::default(), active_cursor: Default::default(), active_display: Rc::new(TestDisplay::new()), active_window: Default::default(), @@ -114,6 +141,10 @@ impl TestPlatform { !self.prompts.borrow().multiple_choice.is_empty() } + pub(crate) fn set_screen_capture_sources(&self, sources: Vec) { + *self.screen_capture_sources.borrow_mut() = sources; + } + pub(crate) fn prompt(&self, msg: &str, detail: Option<&str>) -> oneshot::Receiver { let (tx, rx) = oneshot::channel(); self.background_executor() @@ -202,6 +233,20 @@ impl Platform for TestPlatform { Some(self.active_display.clone()) } + fn screen_capture_sources( + &self, + ) -> oneshot::Receiver>>> { + let (mut tx, rx) = oneshot::channel(); + tx.send(Ok(self + .screen_capture_sources + .borrow() + .iter() + .map(|source| Box::new(source.clone()) as Box) + .collect())) + .ok(); + rx + } + fn active_window(&self) -> Option { self.active_window .borrow() @@ -330,6 +375,13 @@ impl Platform for TestPlatform { } } +impl TestScreenCaptureSource { + /// Create a fake screen capture source, for testing. + pub fn new() -> Self { + Self {} + } +} + #[cfg(target_os = "windows")] impl Drop for TestPlatform { fn drop(&mut self) { diff --git a/crates/gpui/src/platform/windows.rs b/crates/gpui/src/platform/windows.rs index 84cf107c70516..51d09f0013f96 100644 --- a/crates/gpui/src/platform/windows.rs +++ b/crates/gpui/src/platform/windows.rs @@ -21,3 +21,5 @@ pub(crate) use window::*; pub(crate) use wrapper::*; pub(crate) use windows::Win32::Foundation::HWND; + +pub(crate) type PlatformScreenCaptureFrame = (); diff --git a/crates/gpui/src/platform/windows/platform.rs b/crates/gpui/src/platform/windows/platform.rs index 29443afabbbd6..c2bbc9890c346 100644 --- a/crates/gpui/src/platform/windows/platform.rs +++ b/crates/gpui/src/platform/windows/platform.rs @@ -325,6 +325,14 @@ impl Platform for WindowsPlatform { WindowsDisplay::primary_monitor().map(|display| Rc::new(display) as Rc) } + fn screen_capture_sources( + &self, + ) -> oneshot::Receiver>>> { + let (mut tx, rx) = oneshot::channel(); + tx.send(Err(anyhow!("screen capture not implemented"))).ok(); + rx + } + fn active_window(&self) -> Option { let active_window_hwnd = unsafe { GetActiveWindow() }; self.try_get_windows_inner_from_hwnd(active_window_hwnd) diff --git a/crates/http_client/Cargo.toml b/crates/http_client/Cargo.toml index ac8e254b84f60..a4f10cff18082 100644 --- a/crates/http_client/Cargo.toml +++ b/crates/http_client/Cargo.toml @@ -20,7 +20,7 @@ bytes.workspace = true anyhow.workspace = true derive_more.workspace = true futures.workspace = true -http = "1.1" +http.workspace = true log.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/crates/live_kit_client/Cargo.toml b/crates/live_kit_client/Cargo.toml index e23c63453e176..921c048f23c8f 100644 --- a/crates/live_kit_client/Cargo.toml +++ b/crates/live_kit_client/Cargo.toml @@ -2,7 +2,7 @@ name = "live_kit_client" version = "0.1.0" edition = "2021" -description = "Bindings to LiveKit Swift client SDK" +description = "Logic for using LiveKit with GPUI" publish = false license = "GPL-3.0-or-later" @@ -19,42 +19,37 @@ name = "test_app" [features] no-webrtc = [] test-support = [ - "async-trait", "collections/test-support", "gpui/test-support", - "live_kit_server", "nanoid", ] [dependencies] anyhow.workspace = true -async-broadcast = "0.7" -async-trait = { workspace = true, optional = true } -collections = { workspace = true, optional = true } +async-trait.workspace = true +collections.workspace = true +cpal = "0.15" futures.workspace = true -gpui = { workspace = true, optional = true } -live_kit_server = { workspace = true, optional = true } +gpui.workspace = true +http_2 = { package = "http", version = "0.2.1" } +live_kit_server.workspace = true log.workspace = true media.workspace = true nanoid = { workspace = true, optional = true} parking_lot.workspace = true postage.workspace = true +util.workspace = true +http_client.workspace = true + +[target.'cfg(not(target_os = "windows"))'.dependencies] +livekit.workspace = true [target.'cfg(target_os = "macos")'.dependencies] core-foundation.workspace = true -[target.'cfg(all(not(target_os = "macos")))'.dependencies] -async-trait = { workspace = true } -collections = { workspace = true } -gpui = { workspace = true } -live_kit_server.workspace = true -nanoid.workspace = true - [dev-dependencies] -async-trait.workspace = true collections = { workspace = true, features = ["test-support"] } gpui = { workspace = true, features = ["test-support"] } -live_kit_server.workspace = true nanoid.workspace = true sha2.workspace = true simplelog.workspace = true diff --git a/crates/live_kit_client/LiveKitBridge/Package.resolved b/crates/live_kit_client/LiveKitBridge/Package.resolved deleted file mode 100644 index b925bc8f0d5ef..0000000000000 --- a/crates/live_kit_client/LiveKitBridge/Package.resolved +++ /dev/null @@ -1,52 +0,0 @@ -{ - "object": { - "pins": [ - { - "package": "LiveKit", - "repositoryURL": "https://github.com/livekit/client-sdk-swift.git", - "state": { - "branch": null, - "revision": "7331b813a5ab8a95cfb81fb2b4ed10519428b9ff", - "version": "1.0.12" - } - }, - { - "package": "Promises", - "repositoryURL": "https://github.com/google/promises.git", - "state": { - "branch": null, - "revision": "ec957ccddbcc710ccc64c9dcbd4c7006fcf8b73a", - "version": "2.2.0" - } - }, - { - "package": "WebRTC", - "repositoryURL": "https://github.com/webrtc-sdk/Specs.git", - "state": { - "branch": null, - "revision": "2f6bab30c8df0fe59ab3e58bc99097f757f85f65", - "version": "104.5112.17" - } - }, - { - "package": "swift-log", - "repositoryURL": "https://github.com/apple/swift-log.git", - "state": { - "branch": null, - "revision": "32e8d724467f8fe623624570367e3d50c5638e46", - "version": "1.5.2" - } - }, - { - "package": "SwiftProtobuf", - "repositoryURL": "https://github.com/apple/swift-protobuf.git", - "state": { - "branch": null, - "revision": "ce20dc083ee485524b802669890291c0d8090170", - "version": "1.22.1" - } - } - ] - }, - "version": 1 -} diff --git a/crates/live_kit_client/LiveKitBridge/Package.swift b/crates/live_kit_client/LiveKitBridge/Package.swift deleted file mode 100644 index d7b5c271b9549..0000000000000 --- a/crates/live_kit_client/LiveKitBridge/Package.swift +++ /dev/null @@ -1,27 +0,0 @@ -// swift-tools-version: 5.5 - -import PackageDescription - -let package = Package( - name: "LiveKitBridge", - platforms: [ - .macOS(.v10_15) - ], - products: [ - // Products define the executables and libraries a package produces, and make them visible to other packages. - .library( - name: "LiveKitBridge", - type: .static, - targets: ["LiveKitBridge"]), - ], - dependencies: [ - .package(url: "https://github.com/livekit/client-sdk-swift.git", .exact("1.0.12")), - ], - targets: [ - // Targets are the basic building blocks of a package. A target can define a module or a test suite. - // Targets can depend on other targets in this package, and on products in packages this package depends on. - .target( - name: "LiveKitBridge", - dependencies: [.product(name: "LiveKit", package: "client-sdk-swift")]), - ] -) diff --git a/crates/live_kit_client/LiveKitBridge/README.md b/crates/live_kit_client/LiveKitBridge/README.md deleted file mode 100644 index b982c672866a3..0000000000000 --- a/crates/live_kit_client/LiveKitBridge/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# LiveKitBridge - -A description of this package. diff --git a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift deleted file mode 100644 index 7468c08791f8b..0000000000000 --- a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ /dev/null @@ -1,383 +0,0 @@ -import Foundation -import LiveKit -import WebRTC -import ScreenCaptureKit - -class LKRoomDelegate: RoomDelegate { - var data: UnsafeRawPointer - var onDidDisconnect: @convention(c) (UnsafeRawPointer) -> Void - var onDidSubscribeToRemoteAudioTrack: @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer, UnsafeRawPointer) -> Void - var onDidUnsubscribeFromRemoteAudioTrack: @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void - var onMuteChangedFromRemoteAudioTrack: @convention(c) (UnsafeRawPointer, CFString, Bool) -> Void - var onActiveSpeakersChanged: @convention(c) (UnsafeRawPointer, CFArray) -> Void - var onDidSubscribeToRemoteVideoTrack: @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer) -> Void - var onDidUnsubscribeFromRemoteVideoTrack: @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void - var onDidPublishOrUnpublishLocalAudioTrack: @convention(c) (UnsafeRawPointer, UnsafeRawPointer, Bool) -> Void - var onDidPublishOrUnpublishLocalVideoTrack: @convention(c) (UnsafeRawPointer, UnsafeRawPointer, Bool) -> Void - - init( - data: UnsafeRawPointer, - onDidDisconnect: @escaping @convention(c) (UnsafeRawPointer) -> Void, - onDidSubscribeToRemoteAudioTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer, UnsafeRawPointer) -> Void, - onDidUnsubscribeFromRemoteAudioTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void, - onMuteChangedFromRemoteAudioTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, Bool) -> Void, - onActiveSpeakersChanged: @convention(c) (UnsafeRawPointer, CFArray) -> Void, - onDidSubscribeToRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer) -> Void, - onDidUnsubscribeFromRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void, - onDidPublishOrUnpublishLocalAudioTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer, Bool) -> Void, - onDidPublishOrUnpublishLocalVideoTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer, Bool) -> Void - ) - { - self.data = data - self.onDidDisconnect = onDidDisconnect - self.onDidSubscribeToRemoteAudioTrack = onDidSubscribeToRemoteAudioTrack - self.onDidUnsubscribeFromRemoteAudioTrack = onDidUnsubscribeFromRemoteAudioTrack - self.onDidSubscribeToRemoteVideoTrack = onDidSubscribeToRemoteVideoTrack - self.onDidUnsubscribeFromRemoteVideoTrack = onDidUnsubscribeFromRemoteVideoTrack - self.onMuteChangedFromRemoteAudioTrack = onMuteChangedFromRemoteAudioTrack - self.onActiveSpeakersChanged = onActiveSpeakersChanged - self.onDidPublishOrUnpublishLocalAudioTrack = onDidPublishOrUnpublishLocalAudioTrack - self.onDidPublishOrUnpublishLocalVideoTrack = onDidPublishOrUnpublishLocalVideoTrack - } - - func room(_ room: Room, didUpdate connectionState: ConnectionState, oldValue: ConnectionState) { - if connectionState.isDisconnected { - self.onDidDisconnect(self.data) - } - } - - func room(_ room: Room, participant: RemoteParticipant, didSubscribe publication: RemoteTrackPublication, track: Track) { - if track.kind == .video { - self.onDidSubscribeToRemoteVideoTrack(self.data, participant.identity as CFString, track.sid! as CFString, Unmanaged.passUnretained(track).toOpaque()) - } else if track.kind == .audio { - self.onDidSubscribeToRemoteAudioTrack(self.data, participant.identity as CFString, track.sid! as CFString, Unmanaged.passUnretained(track).toOpaque(), Unmanaged.passUnretained(publication).toOpaque()) - } - } - - func room(_ room: Room, participant: Participant, didUpdate publication: TrackPublication, muted: Bool) { - if publication.kind == .audio { - self.onMuteChangedFromRemoteAudioTrack(self.data, publication.sid as CFString, muted) - } - } - - func room(_ room: Room, didUpdate speakers: [Participant]) { - guard let speaker_ids = speakers.compactMap({ $0.identity as CFString }) as CFArray? else { return } - self.onActiveSpeakersChanged(self.data, speaker_ids) - } - - func room(_ room: Room, participant: RemoteParticipant, didUnsubscribe publication: RemoteTrackPublication, track: Track) { - if track.kind == .video { - self.onDidUnsubscribeFromRemoteVideoTrack(self.data, participant.identity as CFString, track.sid! as CFString) - } else if track.kind == .audio { - self.onDidUnsubscribeFromRemoteAudioTrack(self.data, participant.identity as CFString, track.sid! as CFString) - } - } - - func room(_ room: Room, localParticipant: LocalParticipant, didPublish publication: LocalTrackPublication) { - if publication.kind == .video { - self.onDidPublishOrUnpublishLocalVideoTrack(self.data, Unmanaged.passUnretained(publication).toOpaque(), true) - } else if publication.kind == .audio { - self.onDidPublishOrUnpublishLocalAudioTrack(self.data, Unmanaged.passUnretained(publication).toOpaque(), true) - } - } - - func room(_ room: Room, localParticipant: LocalParticipant, didUnpublish publication: LocalTrackPublication) { - if publication.kind == .video { - self.onDidPublishOrUnpublishLocalVideoTrack(self.data, Unmanaged.passUnretained(publication).toOpaque(), false) - } else if publication.kind == .audio { - self.onDidPublishOrUnpublishLocalAudioTrack(self.data, Unmanaged.passUnretained(publication).toOpaque(), false) - } - } -} - -class LKVideoRenderer: NSObject, VideoRenderer { - var data: UnsafeRawPointer - var onFrame: @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Bool - var onDrop: @convention(c) (UnsafeRawPointer) -> Void - var adaptiveStreamIsEnabled: Bool = false - var adaptiveStreamSize: CGSize = .zero - weak var track: VideoTrack? - - init(data: UnsafeRawPointer, onFrame: @escaping @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Bool, onDrop: @escaping @convention(c) (UnsafeRawPointer) -> Void) { - self.data = data - self.onFrame = onFrame - self.onDrop = onDrop - } - - deinit { - self.onDrop(self.data) - } - - func setSize(_ size: CGSize) { - } - - func renderFrame(_ frame: RTCVideoFrame?) { - let buffer = frame?.buffer as? RTCCVPixelBuffer - if let pixelBuffer = buffer?.pixelBuffer { - if !self.onFrame(self.data, pixelBuffer) { - DispatchQueue.main.async { - self.track?.remove(videoRenderer: self) - } - } - } - } -} - -@_cdecl("LKRoomDelegateCreate") -public func LKRoomDelegateCreate( - data: UnsafeRawPointer, - onDidDisconnect: @escaping @convention(c) (UnsafeRawPointer) -> Void, - onDidSubscribeToRemoteAudioTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer, UnsafeRawPointer) -> Void, - onDidUnsubscribeFromRemoteAudioTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void, - onMuteChangedFromRemoteAudioTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, Bool) -> Void, - onActiveSpeakerChanged: @escaping @convention(c) (UnsafeRawPointer, CFArray) -> Void, - onDidSubscribeToRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer) -> Void, - onDidUnsubscribeFromRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void, - onDidPublishOrUnpublishLocalAudioTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer, Bool) -> Void, - onDidPublishOrUnpublishLocalVideoTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer, Bool) -> Void -) -> UnsafeMutableRawPointer { - let delegate = LKRoomDelegate( - data: data, - onDidDisconnect: onDidDisconnect, - onDidSubscribeToRemoteAudioTrack: onDidSubscribeToRemoteAudioTrack, - onDidUnsubscribeFromRemoteAudioTrack: onDidUnsubscribeFromRemoteAudioTrack, - onMuteChangedFromRemoteAudioTrack: onMuteChangedFromRemoteAudioTrack, - onActiveSpeakersChanged: onActiveSpeakerChanged, - onDidSubscribeToRemoteVideoTrack: onDidSubscribeToRemoteVideoTrack, - onDidUnsubscribeFromRemoteVideoTrack: onDidUnsubscribeFromRemoteVideoTrack, - onDidPublishOrUnpublishLocalAudioTrack: onDidPublishOrUnpublishLocalAudioTrack, - onDidPublishOrUnpublishLocalVideoTrack: onDidPublishOrUnpublishLocalVideoTrack - ) - return Unmanaged.passRetained(delegate).toOpaque() -} - -@_cdecl("LKRoomCreate") -public func LKRoomCreate(delegate: UnsafeRawPointer) -> UnsafeMutableRawPointer { - let delegate = Unmanaged.fromOpaque(delegate).takeUnretainedValue() - return Unmanaged.passRetained(Room(delegate: delegate)).toOpaque() -} - -@_cdecl("LKRoomConnect") -public func LKRoomConnect(room: UnsafeRawPointer, url: CFString, token: CFString, callback: @escaping @convention(c) (UnsafeRawPointer, CFString?) -> Void, callback_data: UnsafeRawPointer) { - let room = Unmanaged.fromOpaque(room).takeUnretainedValue() - - room.connect(url as String, token as String).then { _ in - callback(callback_data, UnsafeRawPointer(nil) as! CFString?) - }.catch { error in - callback(callback_data, error.localizedDescription as CFString) - } -} - -@_cdecl("LKRoomDisconnect") -public func LKRoomDisconnect(room: UnsafeRawPointer) { - let room = Unmanaged.fromOpaque(room).takeUnretainedValue() - room.disconnect() -} - -@_cdecl("LKRoomPublishVideoTrack") -public func LKRoomPublishVideoTrack(room: UnsafeRawPointer, track: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer, UnsafeMutableRawPointer?, CFString?) -> Void, callback_data: UnsafeRawPointer) { - let room = Unmanaged.fromOpaque(room).takeUnretainedValue() - let track = Unmanaged.fromOpaque(track).takeUnretainedValue() - room.localParticipant?.publishVideoTrack(track: track).then { publication in - callback(callback_data, Unmanaged.passRetained(publication).toOpaque(), nil) - }.catch { error in - callback(callback_data, nil, error.localizedDescription as CFString) - } -} - -@_cdecl("LKRoomPublishAudioTrack") -public func LKRoomPublishAudioTrack(room: UnsafeRawPointer, track: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer, UnsafeMutableRawPointer?, CFString?) -> Void, callback_data: UnsafeRawPointer) { - let room = Unmanaged.fromOpaque(room).takeUnretainedValue() - let track = Unmanaged.fromOpaque(track).takeUnretainedValue() - room.localParticipant?.publishAudioTrack(track: track).then { publication in - callback(callback_data, Unmanaged.passRetained(publication).toOpaque(), nil) - }.catch { error in - callback(callback_data, nil, error.localizedDescription as CFString) - } -} - - -@_cdecl("LKRoomUnpublishTrack") -public func LKRoomUnpublishTrack(room: UnsafeRawPointer, publication: UnsafeRawPointer) { - let room = Unmanaged.fromOpaque(room).takeUnretainedValue() - let publication = Unmanaged.fromOpaque(publication).takeUnretainedValue() - let _ = room.localParticipant?.unpublish(publication: publication) -} - -@_cdecl("LKRoomAudioTracksForRemoteParticipant") -public func LKRoomAudioTracksForRemoteParticipant(room: UnsafeRawPointer, participantId: CFString) -> CFArray? { - let room = Unmanaged.fromOpaque(room).takeUnretainedValue() - - for (_, participant) in room.remoteParticipants { - if participant.identity == participantId as String { - return participant.audioTracks.compactMap { $0.track as? RemoteAudioTrack } as CFArray? - } - } - - return nil; -} - -@_cdecl("LKRoomAudioTrackPublicationsForRemoteParticipant") -public func LKRoomAudioTrackPublicationsForRemoteParticipant(room: UnsafeRawPointer, participantId: CFString) -> CFArray? { - let room = Unmanaged.fromOpaque(room).takeUnretainedValue() - - for (_, participant) in room.remoteParticipants { - if participant.identity == participantId as String { - return participant.audioTracks.compactMap { $0 as? RemoteTrackPublication } as CFArray? - } - } - - return nil; -} - -@_cdecl("LKRoomVideoTracksForRemoteParticipant") -public func LKRoomVideoTracksForRemoteParticipant(room: UnsafeRawPointer, participantId: CFString) -> CFArray? { - let room = Unmanaged.fromOpaque(room).takeUnretainedValue() - - for (_, participant) in room.remoteParticipants { - if participant.identity == participantId as String { - return participant.videoTracks.compactMap { $0.track as? RemoteVideoTrack } as CFArray? - } - } - - return nil; -} - -@_cdecl("LKLocalAudioTrackCreateTrack") -public func LKLocalAudioTrackCreateTrack() -> UnsafeMutableRawPointer { - let track = LocalAudioTrack.createTrack(options: AudioCaptureOptions( - echoCancellation: true, - noiseSuppression: true - )) - - return Unmanaged.passRetained(track).toOpaque() -} - - -@_cdecl("LKCreateScreenShareTrackForDisplay") -public func LKCreateScreenShareTrackForDisplay(display: UnsafeMutableRawPointer) -> UnsafeMutableRawPointer { - let display = Unmanaged.fromOpaque(display).takeUnretainedValue() - let track = LocalVideoTrack.createMacOSScreenShareTrack(source: display, preferredMethod: .legacy) - return Unmanaged.passRetained(track).toOpaque() -} - -@_cdecl("LKVideoRendererCreate") -public func LKVideoRendererCreate(data: UnsafeRawPointer, onFrame: @escaping @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Bool, onDrop: @escaping @convention(c) (UnsafeRawPointer) -> Void) -> UnsafeMutableRawPointer { - Unmanaged.passRetained(LKVideoRenderer(data: data, onFrame: onFrame, onDrop: onDrop)).toOpaque() -} - -@_cdecl("LKVideoTrackAddRenderer") -public func LKVideoTrackAddRenderer(track: UnsafeRawPointer, renderer: UnsafeRawPointer) { - let track = Unmanaged.fromOpaque(track).takeUnretainedValue() as! VideoTrack - let renderer = Unmanaged.fromOpaque(renderer).takeRetainedValue() - renderer.track = track - track.add(videoRenderer: renderer) -} - -@_cdecl("LKRemoteVideoTrackGetSid") -public func LKRemoteVideoTrackGetSid(track: UnsafeRawPointer) -> CFString { - let track = Unmanaged.fromOpaque(track).takeUnretainedValue() - return track.sid! as CFString -} - -@_cdecl("LKRemoteAudioTrackGetSid") -public func LKRemoteAudioTrackGetSid(track: UnsafeRawPointer) -> CFString { - let track = Unmanaged.fromOpaque(track).takeUnretainedValue() - return track.sid! as CFString -} - -@_cdecl("LKRemoteAudioTrackStart") -public func LKRemoteAudioTrackStart(track: UnsafeRawPointer) { - let track = Unmanaged.fromOpaque(track).takeUnretainedValue() - track.start() -} - -@_cdecl("LKRemoteAudioTrackStop") -public func LKRemoteAudioTrackStop(track: UnsafeRawPointer) { - let track = Unmanaged.fromOpaque(track).takeUnretainedValue() - track.stop() -} - -@_cdecl("LKDisplaySources") -public func LKDisplaySources(data: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer, CFArray?, CFString?) -> Void) { - MacOSScreenCapturer.sources(for: .display, includeCurrentApplication: false, preferredMethod: .legacy).then { displaySources in - callback(data, displaySources as CFArray, nil) - }.catch { error in - callback(data, nil, error.localizedDescription as CFString) - } -} - -@_cdecl("LKLocalTrackPublicationSetMute") -public func LKLocalTrackPublicationSetMute( - publication: UnsafeRawPointer, - muted: Bool, - on_complete: @escaping @convention(c) (UnsafeRawPointer, CFString?) -> Void, - callback_data: UnsafeRawPointer -) { - let publication = Unmanaged.fromOpaque(publication).takeUnretainedValue() - - if muted { - publication.mute().then { - on_complete(callback_data, nil) - }.catch { error in - on_complete(callback_data, error.localizedDescription as CFString) - } - } else { - publication.unmute().then { - on_complete(callback_data, nil) - }.catch { error in - on_complete(callback_data, error.localizedDescription as CFString) - } - } -} - -@_cdecl("LKLocalTrackPublicationIsMuted") -public func LKLocalTrackPublicationIsMuted( - publication: UnsafeRawPointer -) -> Bool { - let publication = Unmanaged.fromOpaque(publication).takeUnretainedValue() - return publication.muted -} - -@_cdecl("LKRemoteTrackPublicationSetEnabled") -public func LKRemoteTrackPublicationSetEnabled( - publication: UnsafeRawPointer, - enabled: Bool, - on_complete: @escaping @convention(c) (UnsafeRawPointer, CFString?) -> Void, - callback_data: UnsafeRawPointer -) { - let publication = Unmanaged.fromOpaque(publication).takeUnretainedValue() - - publication.set(enabled: enabled).then { - on_complete(callback_data, nil) - }.catch { error in - on_complete(callback_data, error.localizedDescription as CFString) - } -} - -@_cdecl("LKRemoteTrackPublicationIsMuted") -public func LKRemoteTrackPublicationIsMuted( - publication: UnsafeRawPointer -) -> Bool { - let publication = Unmanaged.fromOpaque(publication).takeUnretainedValue() - - return publication.muted -} - -@_cdecl("LKRemoteTrackPublicationGetSid") -public func LKRemoteTrackPublicationGetSid( - publication: UnsafeRawPointer -) -> CFString { - let publication = Unmanaged.fromOpaque(publication).takeUnretainedValue() - - return publication.sid as CFString -} - -@_cdecl("LKLocalTrackPublicationGetSid") -public func LKLocalTrackPublicationGetSid( - publication: UnsafeRawPointer -) -> CFString { - let publication = Unmanaged.fromOpaque(publication).takeUnretainedValue() - - return publication.sid as CFString -} diff --git a/crates/live_kit_client/build.rs b/crates/live_kit_client/build.rs deleted file mode 100644 index 2fdfd982bf9c1..0000000000000 --- a/crates/live_kit_client/build.rs +++ /dev/null @@ -1,185 +0,0 @@ -use serde::Deserialize; -use std::{ - env, - path::{Path, PathBuf}, - process::Command, -}; - -const SWIFT_PACKAGE_NAME: &str = "LiveKitBridge"; - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SwiftTargetInfo { - pub triple: String, - pub unversioned_triple: String, - pub module_triple: String, - pub swift_runtime_compatibility_version: String, - #[serde(rename = "librariesRequireRPath")] - pub libraries_require_rpath: bool, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SwiftPaths { - pub runtime_library_paths: Vec, - pub runtime_library_import_paths: Vec, - pub runtime_resource_path: String, -} - -#[derive(Debug, Deserialize)] -pub struct SwiftTarget { - pub target: SwiftTargetInfo, - pub paths: SwiftPaths, -} - -const MACOS_TARGET_VERSION: &str = "10.15.7"; - -fn main() { - if cfg!(all( - target_os = "macos", - not(any(test, feature = "test-support", feature = "no-webrtc")), - )) { - let swift_target = get_swift_target(); - - build_bridge(&swift_target); - link_swift_stdlib(&swift_target); - link_webrtc_framework(&swift_target); - - // Register exported Objective-C selectors, protocols, etc when building example binaries. - println!("cargo:rustc-link-arg=-Wl,-ObjC"); - } -} - -fn build_bridge(swift_target: &SwiftTarget) { - println!("cargo:rerun-if-env-changed=MACOSX_DEPLOYMENT_TARGET"); - println!("cargo:rerun-if-changed={}/Sources", SWIFT_PACKAGE_NAME); - println!( - "cargo:rerun-if-changed={}/Package.swift", - SWIFT_PACKAGE_NAME - ); - println!( - "cargo:rerun-if-changed={}/Package.resolved", - SWIFT_PACKAGE_NAME - ); - - let swift_package_root = swift_package_root(); - let swift_target_folder = swift_target_folder(); - let swift_cache_folder = swift_cache_folder(); - if !Command::new("swift") - .arg("build") - .arg("--disable-automatic-resolution") - .args(["--configuration", &env::var("PROFILE").unwrap()]) - .args(["--triple", &swift_target.target.triple]) - .args(["--build-path".into(), swift_target_folder]) - .args(["--cache-path".into(), swift_cache_folder]) - .current_dir(&swift_package_root) - .status() - .unwrap() - .success() - { - panic!( - "Failed to compile swift package in {}", - swift_package_root.display() - ); - } - - println!( - "cargo:rustc-link-search=native={}", - swift_target.out_dir_path().display() - ); - println!("cargo:rustc-link-lib=static={}", SWIFT_PACKAGE_NAME); -} - -fn link_swift_stdlib(swift_target: &SwiftTarget) { - for path in &swift_target.paths.runtime_library_paths { - println!("cargo:rustc-link-search=native={}", path); - } -} - -fn link_webrtc_framework(swift_target: &SwiftTarget) { - let swift_out_dir_path = swift_target.out_dir_path(); - println!("cargo:rustc-link-lib=framework=WebRTC"); - println!( - "cargo:rustc-link-search=framework={}", - swift_out_dir_path.display() - ); - // Find WebRTC.framework as a sibling of the executable when running tests. - println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path"); - // Find WebRTC.framework in parent directory of the executable when running examples. - println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path/.."); - - let source_path = swift_out_dir_path.join("WebRTC.framework"); - let deps_dir_path = - PathBuf::from(env::var("OUT_DIR").unwrap()).join("../../../deps/WebRTC.framework"); - let target_dir_path = - PathBuf::from(env::var("OUT_DIR").unwrap()).join("../../../WebRTC.framework"); - copy_dir(&source_path, &deps_dir_path); - copy_dir(&source_path, &target_dir_path); -} - -fn get_swift_target() -> SwiftTarget { - let mut arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap(); - if arch == "aarch64" { - arch = "arm64".into(); - } - let target = format!("{}-apple-macosx{}", arch, MACOS_TARGET_VERSION); - - let swift_target_info_str = Command::new("swift") - .args(["-target", &target, "-print-target-info"]) - .output() - .unwrap() - .stdout; - - serde_json::from_slice(&swift_target_info_str).unwrap() -} - -fn swift_package_root() -> PathBuf { - env::current_dir().unwrap().join(SWIFT_PACKAGE_NAME) -} - -fn swift_target_folder() -> PathBuf { - let target = env::var("TARGET").unwrap(); - env::current_dir() - .unwrap() - .join(format!("../../target/{target}/{SWIFT_PACKAGE_NAME}_target")) -} - -fn swift_cache_folder() -> PathBuf { - let target = env::var("TARGET").unwrap(); - env::current_dir() - .unwrap() - .join(format!("../../target/{target}/{SWIFT_PACKAGE_NAME}_cache")) -} - -fn copy_dir(source: &Path, destination: &Path) { - assert!( - Command::new("rm") - .arg("-rf") - .arg(destination) - .status() - .unwrap() - .success(), - "could not remove {:?} before copying", - destination - ); - - assert!( - Command::new("cp") - .arg("-R") - .args([source, destination]) - .status() - .unwrap() - .success(), - "could not copy {:?} to {:?}", - source, - destination - ); -} - -impl SwiftTarget { - fn out_dir_path(&self) -> PathBuf { - swift_target_folder() - .join(&self.target.unversioned_triple) - .join(env::var("PROFILE").unwrap()) - } -} diff --git a/crates/live_kit_client/examples/test_app.rs b/crates/live_kit_client/examples/test_app.rs index de8be97e86c59..8edd171d8465b 100644 --- a/crates/live_kit_client/examples/test_app.rs +++ b/crates/live_kit_client/examples/test_app.rs @@ -1,18 +1,53 @@ -use std::time::Duration; +#![cfg_attr(windows, allow(unused))] + +use gpui::{ + actions, bounds, div, point, + prelude::{FluentBuilder as _, IntoElement}, + px, rgb, size, AsyncAppContext, Bounds, InteractiveElement, KeyBinding, Menu, MenuItem, + ParentElement, Pixels, Render, ScreenCaptureStream, SharedString, + StatefulInteractiveElement as _, Styled, Task, View, ViewContext, VisualContext, WindowBounds, + WindowHandle, WindowOptions, +}; +#[cfg(not(target_os = "windows"))] +use live_kit_client::{ + capture_local_audio_track, capture_local_video_track, + id::ParticipantIdentity, + options::{TrackPublishOptions, VideoCodec}, + participant::{Participant, RemoteParticipant}, + play_remote_audio_track, + publication::{LocalTrackPublication, RemoteTrackPublication}, + track::{LocalTrack, RemoteTrack, RemoteVideoTrack, TrackSource}, + AudioStream, RemoteVideoTrackView, Room, RoomEvent, RoomOptions, +}; + +#[cfg(target_os = "windows")] +use live_kit_client::{ + participant::{Participant, RemoteParticipant}, + publication::{LocalTrackPublication, RemoteTrackPublication}, + track::{LocalTrack, RemoteTrack, RemoteVideoTrack}, + AudioStream, RemoteVideoTrackView, Room, RoomEvent, +}; -use futures::StreamExt; -use gpui::{actions, KeyBinding, Menu, MenuItem}; -use live_kit_client::{LocalAudioTrack, LocalVideoTrack, Room, RoomUpdate}; use live_kit_server::token::{self, VideoGrant}; use log::LevelFilter; +use postage::stream::Stream as _; use simplelog::SimpleLogger; actions!(live_kit_client, [Quit]); +#[cfg(windows)] +fn main() {} + +#[cfg(not(windows))] fn main() { SimpleLogger::init(LevelFilter::Info, Default::default()).expect("could not initialize logger"); gpui::App::new().run(|cx| { + live_kit_client::init( + cx.background_executor().dispatcher.clone(), + cx.http_client(), + ); + #[cfg(any(test, feature = "test-support"))] println!("USING TEST LIVEKIT"); @@ -20,10 +55,8 @@ fn main() { println!("USING REAL LIVEKIT"); cx.activate(true); - cx.on_action(quit); cx.bind_keys([KeyBinding::new("cmd-q", Quit, None)]); - cx.set_menus(vec![Menu { name: "Zed".into(), items: vec![MenuItem::Action { @@ -36,137 +69,368 @@ fn main() { let live_kit_url = std::env::var("LIVE_KIT_URL").unwrap_or("http://localhost:7880".into()); let live_kit_key = std::env::var("LIVE_KIT_KEY").unwrap_or("devkey".into()); let live_kit_secret = std::env::var("LIVE_KIT_SECRET").unwrap_or("secret".into()); + let height = px(800.); + let width = px(800.); cx.spawn(|cx| async move { - let user_a_token = token::create( - &live_kit_key, - &live_kit_secret, - Some("test-participant-1"), - VideoGrant::to_join("test-room"), - ) + let mut windows = Vec::new(); + for i in 0..3 { + let token = token::create( + &live_kit_key, + &live_kit_secret, + Some(&format!("test-participant-{i}")), + VideoGrant::to_join("test-room"), + ) + .unwrap(); + + let bounds = bounds(point(width * i, px(0.0)), size(width, height)); + let window = + LivekitWindow::new(live_kit_url.as_str(), token.as_str(), bounds, cx.clone()) + .await; + windows.push(window); + } + }) + .detach(); + }); +} + +fn quit(_: &Quit, cx: &mut gpui::AppContext) { + cx.quit(); +} + +struct LivekitWindow { + room: Room, + microphone_track: Option, + screen_share_track: Option, + microphone_stream: Option, + screen_share_stream: Option>, + #[cfg(not(target_os = "windows"))] + remote_participants: Vec<(ParticipantIdentity, ParticipantState)>, + _events_task: Task<()>, +} + +#[derive(Default)] +struct ParticipantState { + audio_output_stream: Option<(RemoteTrackPublication, AudioStream)>, + muted: bool, + screen_share_output_view: Option<(RemoteVideoTrack, View)>, + speaking: bool, +} + +#[cfg(not(windows))] +impl LivekitWindow { + async fn new( + url: &str, + token: &str, + bounds: Bounds, + cx: AsyncAppContext, + ) -> WindowHandle { + let (room, mut events) = Room::connect(url, token, RoomOptions::default()) + .await .unwrap(); - let room_a = Room::new(); - room_a.connect(&live_kit_url, &user_a_token).await.unwrap(); - - let user2_token = token::create( - &live_kit_key, - &live_kit_secret, - Some("test-participant-2"), - VideoGrant::to_join("test-room"), + + cx.update(|cx| { + cx.open_window( + WindowOptions { + window_bounds: Some(WindowBounds::Windowed(bounds)), + ..Default::default() + }, + |cx| { + cx.new_view(|cx| { + let _events_task = cx.spawn(|this, mut cx| async move { + while let Some(event) = events.recv().await { + this.update(&mut cx, |this: &mut LivekitWindow, cx| { + this.handle_room_event(event, cx) + }) + .ok(); + } + }); + + Self { + room, + microphone_track: None, + microphone_stream: None, + screen_share_track: None, + screen_share_stream: None, + remote_participants: Vec::new(), + _events_task, + } + }) + }, ) - .unwrap(); - let room_b = Room::new(); - room_b.connect(&live_kit_url, &user2_token).await.unwrap(); - - let mut room_updates = room_b.updates(); - let audio_track = LocalAudioTrack::create(); - let audio_track_publication = room_a.publish_audio_track(audio_track).await.unwrap(); - - if let RoomUpdate::SubscribedToRemoteAudioTrack(track, _) = - room_updates.next().await.unwrap() - { - let remote_tracks = room_b.remote_audio_tracks("test-participant-1"); - assert_eq!(remote_tracks.len(), 1); - assert_eq!(remote_tracks[0].publisher_id(), "test-participant-1"); - assert_eq!(track.publisher_id(), "test-participant-1"); - } else { - panic!("unexpected message"); - } + .unwrap() + }) + .unwrap() + } - audio_track_publication.set_mute(true).await.unwrap(); + fn handle_room_event(&mut self, event: RoomEvent, cx: &mut ViewContext) { + eprintln!("event: {event:?}"); - println!("waiting for mute changed!"); - if let RoomUpdate::RemoteAudioTrackMuteChanged { track_id, muted } = - room_updates.next().await.unwrap() - { - let remote_tracks = room_b.remote_audio_tracks("test-participant-1"); - assert_eq!(remote_tracks[0].sid(), track_id); - assert!(muted); - } else { - panic!("unexpected message"); + match event { + RoomEvent::TrackUnpublished { + publication, + participant, + } => { + let output = self.remote_participant(participant); + let unpublish_sid = publication.sid(); + if output + .audio_output_stream + .as_ref() + .map_or(false, |(track, _)| track.sid() == unpublish_sid) + { + output.audio_output_stream.take(); + } + if output + .screen_share_output_view + .as_ref() + .map_or(false, |(track, _)| track.sid() == unpublish_sid) + { + output.screen_share_output_view.take(); + } + cx.notify(); } - audio_track_publication.set_mute(false).await.unwrap(); - - if let RoomUpdate::RemoteAudioTrackMuteChanged { track_id, muted } = - room_updates.next().await.unwrap() - { - let remote_tracks = room_b.remote_audio_tracks("test-participant-1"); - assert_eq!(remote_tracks[0].sid(), track_id); - assert!(!muted); - } else { - panic!("unexpected message"); + RoomEvent::TrackSubscribed { + publication, + participant, + track, + } => { + let output = self.remote_participant(participant); + match track { + RemoteTrack::Audio(track) => { + output.audio_output_stream = + Some((publication.clone(), play_remote_audio_track(&track, cx))); + } + RemoteTrack::Video(track) => { + output.screen_share_output_view = Some(( + track.clone(), + cx.new_view(|cx| RemoteVideoTrackView::new(track, cx)), + )); + } + } + cx.notify(); } - println!("Pausing for 5 seconds to test audio, make some noise!"); - let timer = cx.background_executor().timer(Duration::from_secs(5)); - timer.await; - let remote_audio_track = room_b - .remote_audio_tracks("test-participant-1") - .pop() - .unwrap(); - room_a.unpublish_track(audio_track_publication); + RoomEvent::TrackMuted { participant, .. } => { + if let Participant::Remote(participant) = participant { + self.remote_participant(participant).muted = true; + cx.notify(); + } + } - // Clear out any active speakers changed messages - let mut next = room_updates.next().await.unwrap(); - while let RoomUpdate::ActiveSpeakersChanged { speakers } = next { - println!("Speakers changed: {:?}", speakers); - next = room_updates.next().await.unwrap(); + RoomEvent::TrackUnmuted { participant, .. } => { + if let Participant::Remote(participant) = participant { + self.remote_participant(participant).muted = false; + cx.notify(); + } } - if let RoomUpdate::UnsubscribedFromRemoteAudioTrack { - publisher_id, - track_id, - } = next - { - assert_eq!(publisher_id, "test-participant-1"); - assert_eq!(remote_audio_track.sid(), track_id); - assert_eq!(room_b.remote_audio_tracks("test-participant-1").len(), 0); - } else { - panic!("unexpected message"); + RoomEvent::ActiveSpeakersChanged { speakers } => { + for (identity, output) in &mut self.remote_participants { + output.speaking = speakers.iter().any(|speaker| { + if let Participant::Remote(speaker) = speaker { + speaker.identity() == *identity + } else { + false + } + }); + } + cx.notify(); } - let displays = room_a.display_sources().await.unwrap(); - let display = displays.into_iter().next().unwrap(); + _ => {} + } - let local_video_track = LocalVideoTrack::screen_share_for_display(&display); - let local_video_track_publication = - room_a.publish_video_track(local_video_track).await.unwrap(); + cx.notify(); + } - if let RoomUpdate::SubscribedToRemoteVideoTrack(track) = - room_updates.next().await.unwrap() - { - let remote_video_tracks = room_b.remote_video_tracks("test-participant-1"); - assert_eq!(remote_video_tracks.len(), 1); - assert_eq!(remote_video_tracks[0].publisher_id(), "test-participant-1"); - assert_eq!(track.publisher_id(), "test-participant-1"); - } else { - panic!("unexpected message"); + fn remote_participant(&mut self, participant: RemoteParticipant) -> &mut ParticipantState { + match self + .remote_participants + .binary_search_by_key(&&participant.identity(), |row| &row.0) + { + Ok(ix) => &mut self.remote_participants[ix].1, + Err(ix) => { + self.remote_participants + .insert(ix, (participant.identity(), ParticipantState::default())); + &mut self.remote_participants[ix].1 } + } + } - let remote_video_track = room_b - .remote_video_tracks("test-participant-1") - .pop() - .unwrap(); - room_a.unpublish_track(local_video_track_publication); - if let RoomUpdate::UnsubscribedFromRemoteVideoTrack { - publisher_id, - track_id, - } = room_updates.next().await.unwrap() - { - assert_eq!(publisher_id, "test-participant-1"); - assert_eq!(remote_video_track.sid(), track_id); - assert_eq!(room_b.remote_video_tracks("test-participant-1").len(), 0); + fn toggle_mute(&mut self, cx: &mut ViewContext) { + if let Some(track) = &self.microphone_track { + if track.is_muted() { + track.unmute(); } else { - panic!("unexpected message"); + track.mute(); } + cx.notify(); + } else { + let participant = self.room.local_participant(); + cx.spawn(|this, mut cx| async move { + let (track, stream) = cx.update(|cx| capture_local_audio_track(cx))??; + let publication = participant + .publish_track( + LocalTrack::Audio(track), + TrackPublishOptions { + source: TrackSource::Microphone, + ..Default::default() + }, + ) + .await + .unwrap(); + this.update(&mut cx, |this, cx| { + this.microphone_track = Some(publication); + this.microphone_stream = Some(stream); + cx.notify(); + }) + }) + .detach(); + } + } - cx.update(|cx| cx.shutdown()).ok(); - }) - .detach(); - }); + fn toggle_screen_share(&mut self, cx: &mut ViewContext) { + if let Some(track) = self.screen_share_track.take() { + self.screen_share_stream.take(); + let participant = self.room.local_participant(); + cx.background_executor() + .spawn(async move { + participant.unpublish_track(&track.sid()).await.unwrap(); + }) + .detach(); + cx.notify(); + } else { + let participant = self.room.local_participant(); + let sources = cx.screen_capture_sources(); + cx.spawn(|this, mut cx| async move { + let sources = sources.await.unwrap()?; + let source = sources.into_iter().next().unwrap(); + let (track, stream) = capture_local_video_track(&*source).await?; + let publication = participant + .publish_track( + LocalTrack::Video(track), + TrackPublishOptions { + source: TrackSource::Screenshare, + video_codec: VideoCodec::H264, + ..Default::default() + }, + ) + .await + .unwrap(); + this.update(&mut cx, |this, cx| { + this.screen_share_track = Some(publication); + this.screen_share_stream = Some(stream); + cx.notify(); + }) + }) + .detach(); + } + } + + fn toggle_remote_audio_for_participant( + &mut self, + identity: &ParticipantIdentity, + cx: &mut ViewContext, + ) -> Option<()> { + let participant = self.remote_participants.iter().find_map(|(id, state)| { + if id == identity { + Some(state) + } else { + None + } + })?; + let publication = &participant.audio_output_stream.as_ref()?.0; + publication.set_enabled(!publication.is_enabled()); + cx.notify(); + Some(()) + } } -fn quit(_: &Quit, cx: &mut gpui::AppContext) { - cx.quit(); +#[cfg(not(windows))] +impl Render for LivekitWindow { + fn render(&mut self, cx: &mut ViewContext) -> impl IntoElement { + fn button() -> gpui::Div { + div() + .w(px(180.0)) + .h(px(30.0)) + .px_2() + .m_2() + .bg(rgb(0x8888ff)) + } + + div() + .bg(rgb(0xffffff)) + .size_full() + .flex() + .flex_col() + .child( + div().bg(rgb(0xffd4a8)).flex().flex_row().children([ + button() + .id("toggle-mute") + .child(if let Some(track) = &self.microphone_track { + if track.is_muted() { + "Unmute" + } else { + "Mute" + } + } else { + "Publish mic" + }) + .on_click(cx.listener(|this, _, cx| this.toggle_mute(cx))), + button() + .id("toggle-screen-share") + .child(if self.screen_share_track.is_none() { + "Share screen" + } else { + "Unshare screen" + }) + .on_click(cx.listener(|this, _, cx| this.toggle_screen_share(cx))), + ]), + ) + .child( + div() + .id("remote-participants") + .overflow_y_scroll() + .flex() + .flex_col() + .flex_grow() + .children(self.remote_participants.iter().map(|(identity, state)| { + div() + .h(px(300.0)) + .flex() + .flex_col() + .m_2() + .px_2() + .bg(rgb(0x8888ff)) + .child(SharedString::from(if state.speaking { + format!("{} (speaking)", &identity.0) + } else if state.muted { + format!("{} (muted)", &identity.0) + } else { + identity.0.clone() + })) + .when_some(state.audio_output_stream.as_ref(), |el, state| { + el.child( + button() + .id(SharedString::from(identity.0.clone())) + .child(if state.0.is_enabled() { + "Deafen" + } else { + "Undeafen" + }) + .on_click(cx.listener({ + let identity = identity.clone(); + move |this, _, cx| { + this.toggle_remote_audio_for_participant( + &identity, cx, + ); + } + })), + ) + }) + .children(state.screen_share_output_view.as_ref().map(|e| e.1.clone())) + })), + ) + } } diff --git a/crates/live_kit_client/src/live_kit_client.rs b/crates/live_kit_client/src/live_kit_client.rs index 4820a4eedb854..ad2e72d67f07d 100644 --- a/crates/live_kit_client/src/live_kit_client.rs +++ b/crates/live_kit_client/src/live_kit_client.rs @@ -1,37 +1,387 @@ -#![allow(clippy::arc_with_non_send_sync)] +#![cfg_attr(target_os = "windows", allow(unused))] -use std::sync::Arc; +mod remote_video_track_view; +#[cfg(any(test, feature = "test-support", target_os = "windows"))] +pub mod test; -#[cfg(all(target_os = "macos", not(any(test, feature = "test-support"))))] -pub mod prod; +use anyhow::{anyhow, Context as _, Result}; +use cpal::{ + traits::{DeviceTrait, HostTrait, StreamTrait as _}, + StreamConfig, +}; +use futures::{io, Stream, StreamExt as _}; +use gpui::{AppContext, ScreenCaptureFrame, ScreenCaptureSource, ScreenCaptureStream, Task}; +use parking_lot::Mutex; +use std::{borrow::Cow, future::Future, pin::Pin, sync::Arc}; +use util::{debug_panic, ResultExt as _, TryFutureExt}; +#[cfg(not(target_os = "windows"))] +use webrtc::{ + audio_frame::AudioFrame, + audio_source::{native::NativeAudioSource, AudioSourceOptions, RtcAudioSource}, + audio_stream::native::NativeAudioStream, + video_frame::{VideoBuffer, VideoFrame, VideoRotation}, + video_source::{native::NativeVideoSource, RtcVideoSource, VideoResolution}, + video_stream::native::NativeVideoStream, +}; -#[cfg(all(target_os = "macos", not(any(test, feature = "test-support"))))] -pub use prod::*; +#[cfg(all(not(any(test, feature = "test-support")), not(target_os = "windows")))] +pub use livekit::*; +#[cfg(any(test, feature = "test-support", target_os = "windows"))] +pub use test::*; -#[cfg(any(test, feature = "test-support", not(target_os = "macos")))] -pub mod test; +pub use remote_video_track_view::{RemoteVideoTrackView, RemoteVideoTrackViewEvent}; -#[cfg(any(test, feature = "test-support", not(target_os = "macos")))] -pub use test::*; +pub struct AudioStream { + _tasks: [Task>; 2], +} + +struct Dispatcher(Arc); + +#[cfg(not(target_os = "windows"))] +impl livekit::dispatcher::Dispatcher for Dispatcher { + fn dispatch(&self, runnable: livekit::dispatcher::Runnable) { + self.0.dispatch(runnable, None); + } + + fn dispatch_after( + &self, + duration: std::time::Duration, + runnable: livekit::dispatcher::Runnable, + ) { + self.0.dispatch_after(duration, runnable); + } +} + +struct HttpClientAdapter(Arc); + +fn http_2_status(status: http_client::http::StatusCode) -> http_2::StatusCode { + http_2::StatusCode::from_u16(status.as_u16()) + .expect("valid status code to status code conversion") +} + +#[cfg(not(target_os = "windows"))] +impl livekit::dispatcher::HttpClient for HttpClientAdapter { + fn get( + &self, + url: &str, + ) -> Pin> + Send>> { + let http_client = self.0.clone(); + let url = url.to_string(); + Box::pin(async move { + let response = http_client + .get(&url, http_client::AsyncBody::empty(), false) + .await + .map_err(io::Error::other)?; + Ok(livekit::dispatcher::Response { + status: http_2_status(response.status()), + body: Box::pin(response.into_body()), + }) + }) + } + + fn send_async( + &self, + request: http_2::Request>, + ) -> Pin> + Send>> { + let http_client = self.0.clone(); + let mut builder = http_client::http::Request::builder() + .method(request.method().as_str()) + .uri(request.uri().to_string()); + + for (key, value) in request.headers().iter() { + builder = builder.header(key.as_str(), value.as_bytes()); + } + + if !request.extensions().is_empty() { + debug_panic!( + "Livekit sent an HTTP request with a protocol extension that Zed doesn't support!" + ); + } + + let request = builder + .body(http_client::AsyncBody::from_bytes( + request.into_body().into(), + )) + .unwrap(); + + Box::pin(async move { + let response = http_client.send(request).await.map_err(io::Error::other)?; + Ok(livekit::dispatcher::Response { + status: http_2_status(response.status()), + body: Box::pin(response.into_body()), + }) + }) + } +} + +#[cfg(target_os = "windows")] +pub fn init( + dispatcher: Arc, + http_client: Arc, +) { +} + +#[cfg(not(target_os = "windows"))] +pub fn init( + dispatcher: Arc, + http_client: Arc, +) { + livekit::dispatcher::set_dispatcher(Dispatcher(dispatcher)); + livekit::dispatcher::set_http_client(HttpClientAdapter(http_client)); +} + +#[cfg(not(target_os = "windows"))] +pub async fn capture_local_video_track( + capture_source: &dyn ScreenCaptureSource, +) -> Result<(track::LocalVideoTrack, Box)> { + let resolution = capture_source.resolution()?; + let track_source = NativeVideoSource::new(VideoResolution { + width: resolution.width.0 as u32, + height: resolution.height.0 as u32, + }); + + let capture_stream = capture_source + .stream({ + let track_source = track_source.clone(); + Box::new(move |frame| { + if let Some(buffer) = video_frame_buffer_to_webrtc(frame) { + track_source.capture_frame(&VideoFrame { + rotation: VideoRotation::VideoRotation0, + timestamp_us: 0, + buffer, + }); + } + }) + }) + .await??; + + Ok(( + track::LocalVideoTrack::create_video_track( + "screen share", + RtcVideoSource::Native(track_source), + ), + capture_stream, + )) +} + +#[cfg(not(target_os = "windows"))] +pub fn capture_local_audio_track( + cx: &mut AppContext, +) -> Result<(track::LocalAudioTrack, AudioStream)> { + let (frame_tx, mut frame_rx) = futures::channel::mpsc::unbounded(); + + let sample_rate; + let channels; + let stream; + if cfg!(any(test, feature = "test-support")) { + sample_rate = 1; + channels = 1; + stream = None; + } else { + let device = cpal::default_host() + .default_input_device() + .ok_or_else(|| anyhow!("no audio input device available"))?; + let config = device + .default_input_config() + .context("failed to get default input config")?; + sample_rate = config.sample_rate().0; + channels = config.channels() as u32; + stream = Some( + device + .build_input_stream_raw( + &config.config(), + cpal::SampleFormat::I16, + move |data, _: &_| { + frame_tx + .unbounded_send(AudioFrame { + data: Cow::Owned(data.as_slice::().unwrap().to_vec()), + sample_rate, + num_channels: channels, + samples_per_channel: data.len() as u32 / channels, + }) + .ok(); + }, + |err| log::error!("error capturing audio track: {:?}", err), + None, + ) + .context("failed to build input stream")?, + ); + } + + let source = NativeAudioSource::new( + AudioSourceOptions { + echo_cancellation: true, + noise_suppression: true, + auto_gain_control: false, + }, + sample_rate, + channels, + // TODO livekit: Pull these out of a proto later + 100, + ); + + let stream_task = cx.foreground_executor().spawn(async move { + if let Some(stream) = &stream { + stream.play().log_err(); + } + futures::future::pending().await + }); + + let transmit_task = cx.background_executor().spawn({ + let source = source.clone(); + async move { + while let Some(frame) = frame_rx.next().await { + source.capture_frame(&frame).await.ok(); + } + Some(()) + } + }); + + let track = + track::LocalAudioTrack::create_audio_track("microphone", RtcAudioSource::Native(source)); + + Ok(( + track, + AudioStream { + _tasks: [stream_task, transmit_task], + }, + )) +} + +#[cfg(not(target_os = "windows"))] +pub fn play_remote_audio_track( + track: &track::RemoteAudioTrack, + cx: &mut AppContext, +) -> AudioStream { + let buffer = Arc::new(Mutex::new(Vec::::new())); + let (stream_config_tx, mut stream_config_rx) = futures::channel::mpsc::unbounded(); + // TODO livekit: Pull these out of a proto later + let mut stream = NativeAudioStream::new(track.rtc_track(), 48000, 1); + + let receive_task = cx.background_executor().spawn({ + let buffer = buffer.clone(); + async move { + let mut stream_config = None; + while let Some(frame) = stream.next().await { + let mut buffer = buffer.lock(); + let buffer_size = frame.samples_per_channel * frame.num_channels; + debug_assert!(frame.data.len() == buffer_size as usize); + + let frame_config = StreamConfig { + channels: frame.num_channels as u16, + sample_rate: cpal::SampleRate(frame.sample_rate), + buffer_size: cpal::BufferSize::Fixed(buffer_size), + }; + + if stream_config.as_ref().map_or(true, |c| *c != frame_config) { + buffer.resize(buffer_size as usize, 0); + stream_config = Some(frame_config.clone()); + stream_config_tx.unbounded_send(frame_config).ok(); + } + + if frame.data.len() == buffer.len() { + buffer.copy_from_slice(&frame.data); + } else { + buffer.iter_mut().for_each(|x| *x = 0); + } + } + Some(()) + } + }); + + let play_task = cx.foreground_executor().spawn( + { + let buffer = buffer.clone(); + async move { + if cfg!(any(test, feature = "test-support")) { + return Err(anyhow!("can't play audio in tests")); + } + + let device = cpal::default_host() + .default_output_device() + .ok_or_else(|| anyhow!("no audio output device available"))?; + + let mut _output_stream = None; + while let Some(config) = stream_config_rx.next().await { + _output_stream = Some(device.build_output_stream( + &config, + { + let buffer = buffer.clone(); + move |data, _info| { + let buffer = buffer.lock(); + if data.len() == buffer.len() { + data.copy_from_slice(&buffer); + } else { + data.iter_mut().for_each(|x| *x = 0); + } + } + }, + |error| log::error!("error playing audio track: {:?}", error), + None, + )?); + } + + Ok(()) + } + } + .log_err(), + ); + + AudioStream { + _tasks: [receive_task, play_task], + } +} + +#[cfg(target_os = "windows")] +pub fn play_remote_video_track( + track: &track::RemoteVideoTrack, +) -> impl Stream { + futures::stream::empty() +} + +#[cfg(not(target_os = "windows"))] +pub fn play_remote_video_track( + track: &track::RemoteVideoTrack, +) -> impl Stream { + NativeVideoStream::new(track.rtc_track()) + .filter_map(|frame| async move { video_frame_buffer_from_webrtc(frame.buffer) }) +} + +#[cfg(target_os = "macos")] +fn video_frame_buffer_from_webrtc(buffer: Box) -> Option { + use core_foundation::base::TCFType as _; + use media::core_video::CVImageBuffer; + + let buffer = buffer.as_native()?; + let pixel_buffer = buffer.get_cv_pixel_buffer(); + if pixel_buffer.is_null() { + return None; + } + + unsafe { + Some(ScreenCaptureFrame(CVImageBuffer::wrap_under_get_rule( + pixel_buffer as _, + ))) + } +} + +#[cfg(not(any(target_os = "macos", target_os = "windows")))] +fn video_frame_buffer_from_webrtc(_buffer: Box) -> Option { + None +} + +#[cfg(target_os = "macos")] +fn video_frame_buffer_to_webrtc(frame: ScreenCaptureFrame) -> Option> { + use core_foundation::base::TCFType as _; + + let pixel_buffer = frame.0.as_concrete_TypeRef(); + std::mem::forget(frame.0); + unsafe { + Some(webrtc::video_frame::native::NativeBuffer::from_cv_pixel_buffer(pixel_buffer as _)) + } +} -pub type Sid = String; - -#[derive(Clone, Eq, PartialEq)] -pub enum ConnectionState { - Disconnected, - Connected { url: String, token: String }, -} - -#[derive(Clone)] -pub enum RoomUpdate { - ActiveSpeakersChanged { speakers: Vec }, - RemoteAudioTrackMuteChanged { track_id: Sid, muted: bool }, - SubscribedToRemoteVideoTrack(Arc), - SubscribedToRemoteAudioTrack(Arc, Arc), - UnsubscribedFromRemoteVideoTrack { publisher_id: Sid, track_id: Sid }, - UnsubscribedFromRemoteAudioTrack { publisher_id: Sid, track_id: Sid }, - LocalAudioTrackPublished { publication: LocalTrackPublication }, - LocalAudioTrackUnpublished { publication: LocalTrackPublication }, - LocalVideoTrackPublished { publication: LocalTrackPublication }, - LocalVideoTrackUnpublished { publication: LocalTrackPublication }, +#[cfg(not(any(target_os = "macos", target_os = "windows")))] +fn video_frame_buffer_to_webrtc(_frame: ScreenCaptureFrame) -> Option> { + None as Option> } diff --git a/crates/live_kit_client/src/prod.rs b/crates/live_kit_client/src/prod.rs deleted file mode 100644 index f7452da71036d..0000000000000 --- a/crates/live_kit_client/src/prod.rs +++ /dev/null @@ -1,981 +0,0 @@ -use crate::{ConnectionState, RoomUpdate, Sid}; -use anyhow::{anyhow, Context, Result}; -use core_foundation::{ - array::{CFArray, CFArrayRef}, - base::{CFRelease, CFRetain, TCFType}, - string::{CFString, CFStringRef}, -}; -use futures::{ - channel::{mpsc, oneshot}, - Future, -}; -pub use media::core_video::CVImageBuffer; -use media::core_video::CVImageBufferRef; -use parking_lot::Mutex; -use postage::watch; -use std::{ - ffi::c_void, - sync::{Arc, Weak}, -}; - -macro_rules! pointer_type { - ($pointer_name:ident) => { - #[repr(transparent)] - #[derive(Copy, Clone, Debug)] - pub struct $pointer_name(pub *const std::ffi::c_void); - unsafe impl Send for $pointer_name {} - }; -} - -mod swift { - pointer_type!(Room); - pointer_type!(LocalAudioTrack); - pointer_type!(RemoteAudioTrack); - pointer_type!(LocalVideoTrack); - pointer_type!(RemoteVideoTrack); - pointer_type!(LocalTrackPublication); - pointer_type!(RemoteTrackPublication); - pointer_type!(MacOSDisplay); - pointer_type!(RoomDelegate); -} - -extern "C" { - fn LKRoomDelegateCreate( - callback_data: *mut c_void, - on_did_disconnect: extern "C" fn(callback_data: *mut c_void), - on_did_subscribe_to_remote_audio_track: extern "C" fn( - callback_data: *mut c_void, - publisher_id: CFStringRef, - track_id: CFStringRef, - remote_track: swift::RemoteAudioTrack, - remote_publication: swift::RemoteTrackPublication, - ), - on_did_unsubscribe_from_remote_audio_track: extern "C" fn( - callback_data: *mut c_void, - publisher_id: CFStringRef, - track_id: CFStringRef, - ), - on_mute_changed_from_remote_audio_track: extern "C" fn( - callback_data: *mut c_void, - track_id: CFStringRef, - muted: bool, - ), - on_active_speakers_changed: extern "C" fn( - callback_data: *mut c_void, - participants: CFArrayRef, - ), - on_did_subscribe_to_remote_video_track: extern "C" fn( - callback_data: *mut c_void, - publisher_id: CFStringRef, - track_id: CFStringRef, - remote_track: swift::RemoteVideoTrack, - ), - on_did_unsubscribe_from_remote_video_track: extern "C" fn( - callback_data: *mut c_void, - publisher_id: CFStringRef, - track_id: CFStringRef, - ), - on_did_publish_or_unpublish_local_audio_track: extern "C" fn( - callback_data: *mut c_void, - publication: swift::LocalTrackPublication, - is_published: bool, - ), - on_did_publish_or_unpublish_local_video_track: extern "C" fn( - callback_data: *mut c_void, - publication: swift::LocalTrackPublication, - is_published: bool, - ), - ) -> swift::RoomDelegate; - - fn LKRoomCreate(delegate: swift::RoomDelegate) -> swift::Room; - fn LKRoomConnect( - room: swift::Room, - url: CFStringRef, - token: CFStringRef, - callback: extern "C" fn(*mut c_void, CFStringRef), - callback_data: *mut c_void, - ); - fn LKRoomDisconnect(room: swift::Room); - fn LKRoomPublishVideoTrack( - room: swift::Room, - track: swift::LocalVideoTrack, - callback: extern "C" fn(*mut c_void, swift::LocalTrackPublication, CFStringRef), - callback_data: *mut c_void, - ); - fn LKRoomPublishAudioTrack( - room: swift::Room, - track: swift::LocalAudioTrack, - callback: extern "C" fn(*mut c_void, swift::LocalTrackPublication, CFStringRef), - callback_data: *mut c_void, - ); - fn LKRoomUnpublishTrack(room: swift::Room, publication: swift::LocalTrackPublication); - - fn LKRoomAudioTracksForRemoteParticipant( - room: swift::Room, - participant_id: CFStringRef, - ) -> CFArrayRef; - - fn LKRoomAudioTrackPublicationsForRemoteParticipant( - room: swift::Room, - participant_id: CFStringRef, - ) -> CFArrayRef; - - fn LKRoomVideoTracksForRemoteParticipant( - room: swift::Room, - participant_id: CFStringRef, - ) -> CFArrayRef; - - fn LKVideoRendererCreate( - callback_data: *mut c_void, - on_frame: extern "C" fn(callback_data: *mut c_void, frame: CVImageBufferRef) -> bool, - on_drop: extern "C" fn(callback_data: *mut c_void), - ) -> *const c_void; - - fn LKRemoteAudioTrackGetSid(track: swift::RemoteAudioTrack) -> CFStringRef; - fn LKRemoteVideoTrackGetSid(track: swift::RemoteVideoTrack) -> CFStringRef; - fn LKRemoteAudioTrackStart(track: swift::RemoteAudioTrack); - fn LKRemoteAudioTrackStop(track: swift::RemoteAudioTrack); - fn LKVideoTrackAddRenderer(track: swift::RemoteVideoTrack, renderer: *const c_void); - - fn LKDisplaySources( - callback_data: *mut c_void, - callback: extern "C" fn( - callback_data: *mut c_void, - sources: CFArrayRef, - error: CFStringRef, - ), - ); - fn LKCreateScreenShareTrackForDisplay(display: swift::MacOSDisplay) -> swift::LocalVideoTrack; - fn LKLocalAudioTrackCreateTrack() -> swift::LocalAudioTrack; - - fn LKLocalTrackPublicationSetMute( - publication: swift::LocalTrackPublication, - muted: bool, - on_complete: extern "C" fn(callback_data: *mut c_void, error: CFStringRef), - callback_data: *mut c_void, - ); - - fn LKRemoteTrackPublicationSetEnabled( - publication: swift::RemoteTrackPublication, - enabled: bool, - on_complete: extern "C" fn(callback_data: *mut c_void, error: CFStringRef), - callback_data: *mut c_void, - ); - - fn LKLocalTrackPublicationIsMuted(publication: swift::LocalTrackPublication) -> bool; - fn LKRemoteTrackPublicationIsMuted(publication: swift::RemoteTrackPublication) -> bool; - fn LKLocalTrackPublicationGetSid(publication: swift::LocalTrackPublication) -> CFStringRef; - fn LKRemoteTrackPublicationGetSid(publication: swift::RemoteTrackPublication) -> CFStringRef; -} - -pub struct Room { - native_room: swift::Room, - connection: Mutex<( - watch::Sender, - watch::Receiver, - )>, - update_subscribers: Mutex>>, - _delegate: RoomDelegate, -} - -impl Room { - pub fn new() -> Arc { - Arc::new_cyclic(|weak_room| { - let delegate = RoomDelegate::new(weak_room.clone()); - Self { - native_room: unsafe { LKRoomCreate(delegate.native_delegate) }, - connection: Mutex::new(watch::channel_with(ConnectionState::Disconnected)), - update_subscribers: Default::default(), - _delegate: delegate, - } - }) - } - - pub fn status(&self) -> watch::Receiver { - self.connection.lock().1.clone() - } - - pub fn connect(self: &Arc, url: &str, token: &str) -> impl Future> { - let url = CFString::new(url); - let token = CFString::new(token); - let (did_connect, tx, rx) = Self::build_done_callback(); - unsafe { - LKRoomConnect( - self.native_room, - url.as_concrete_TypeRef(), - token.as_concrete_TypeRef(), - did_connect, - tx, - ) - } - - let this = self.clone(); - let url = url.to_string(); - let token = token.to_string(); - async move { - rx.await.unwrap().context("error connecting to room")?; - *this.connection.lock().0.borrow_mut() = ConnectionState::Connected { url, token }; - Ok(()) - } - } - - fn did_disconnect(&self) { - *self.connection.lock().0.borrow_mut() = ConnectionState::Disconnected; - } - - pub fn display_sources(self: &Arc) -> impl Future>> { - extern "C" fn callback(tx: *mut c_void, sources: CFArrayRef, error: CFStringRef) { - unsafe { - let tx = Box::from_raw(tx as *mut oneshot::Sender>>); - - if sources.is_null() { - let _ = tx.send(Err(anyhow!("{}", CFString::wrap_under_get_rule(error)))); - } else { - let sources = CFArray::wrap_under_get_rule(sources) - .into_iter() - .map(|source| MacOSDisplay::new(swift::MacOSDisplay(*source))) - .collect(); - - let _ = tx.send(Ok(sources)); - } - } - } - - let (tx, rx) = oneshot::channel(); - - unsafe { - LKDisplaySources(Box::into_raw(Box::new(tx)) as *mut _, callback); - } - - async move { rx.await.unwrap() } - } - - pub fn publish_video_track( - self: &Arc, - track: LocalVideoTrack, - ) -> impl Future> { - let (tx, rx) = oneshot::channel::>(); - extern "C" fn callback( - tx: *mut c_void, - publication: swift::LocalTrackPublication, - error: CFStringRef, - ) { - let tx = - unsafe { Box::from_raw(tx as *mut oneshot::Sender>) }; - if error.is_null() { - let _ = tx.send(Ok(LocalTrackPublication::new(publication))); - } else { - let error = unsafe { CFString::wrap_under_get_rule(error).to_string() }; - let _ = tx.send(Err(anyhow!(error))); - } - } - unsafe { - LKRoomPublishVideoTrack( - self.native_room, - track.0, - callback, - Box::into_raw(Box::new(tx)) as *mut c_void, - ); - } - async { rx.await.unwrap().context("error publishing video track") } - } - - pub fn publish_audio_track( - self: &Arc, - track: LocalAudioTrack, - ) -> impl Future> { - let (tx, rx) = oneshot::channel::>(); - extern "C" fn callback( - tx: *mut c_void, - publication: swift::LocalTrackPublication, - error: CFStringRef, - ) { - let tx = - unsafe { Box::from_raw(tx as *mut oneshot::Sender>) }; - if error.is_null() { - let _ = tx.send(Ok(LocalTrackPublication::new(publication))); - } else { - let error = unsafe { CFString::wrap_under_get_rule(error).to_string() }; - let _ = tx.send(Err(anyhow!(error))); - } - } - unsafe { - LKRoomPublishAudioTrack( - self.native_room, - track.0, - callback, - Box::into_raw(Box::new(tx)) as *mut c_void, - ); - } - async { rx.await.unwrap().context("error publishing audio track") } - } - - pub fn unpublish_track(&self, publication: LocalTrackPublication) { - unsafe { - LKRoomUnpublishTrack(self.native_room, publication.0); - } - } - - pub fn remote_video_tracks(&self, participant_id: &str) -> Vec> { - unsafe { - let tracks = LKRoomVideoTracksForRemoteParticipant( - self.native_room, - CFString::new(participant_id).as_concrete_TypeRef(), - ); - - if tracks.is_null() { - Vec::new() - } else { - let tracks = CFArray::wrap_under_get_rule(tracks); - tracks - .into_iter() - .map(|native_track| { - let native_track = swift::RemoteVideoTrack(*native_track); - let id = - CFString::wrap_under_get_rule(LKRemoteVideoTrackGetSid(native_track)) - .to_string(); - Arc::new(RemoteVideoTrack::new( - native_track, - id, - participant_id.into(), - )) - }) - .collect() - } - } - } - - pub fn remote_audio_tracks(&self, participant_id: &str) -> Vec> { - unsafe { - let tracks = LKRoomAudioTracksForRemoteParticipant( - self.native_room, - CFString::new(participant_id).as_concrete_TypeRef(), - ); - - if tracks.is_null() { - Vec::new() - } else { - let tracks = CFArray::wrap_under_get_rule(tracks); - tracks - .into_iter() - .map(|native_track| { - let native_track = swift::RemoteAudioTrack(*native_track); - let id = - CFString::wrap_under_get_rule(LKRemoteAudioTrackGetSid(native_track)) - .to_string(); - Arc::new(RemoteAudioTrack::new( - native_track, - id, - participant_id.into(), - )) - }) - .collect() - } - } - } - - pub fn remote_audio_track_publications( - &self, - participant_id: &str, - ) -> Vec> { - unsafe { - let tracks = LKRoomAudioTrackPublicationsForRemoteParticipant( - self.native_room, - CFString::new(participant_id).as_concrete_TypeRef(), - ); - - if tracks.is_null() { - Vec::new() - } else { - let tracks = CFArray::wrap_under_get_rule(tracks); - tracks - .into_iter() - .map(|native_track_publication| { - let native_track_publication = - swift::RemoteTrackPublication(*native_track_publication); - Arc::new(RemoteTrackPublication::new(native_track_publication)) - }) - .collect() - } - } - } - - pub fn updates(&self) -> mpsc::UnboundedReceiver { - let (tx, rx) = mpsc::unbounded(); - self.update_subscribers.lock().push(tx); - rx - } - - fn did_subscribe_to_remote_audio_track( - &self, - track: RemoteAudioTrack, - publication: RemoteTrackPublication, - ) { - let track = Arc::new(track); - let publication = Arc::new(publication); - self.update_subscribers.lock().retain(|tx| { - tx.unbounded_send(RoomUpdate::SubscribedToRemoteAudioTrack( - track.clone(), - publication.clone(), - )) - .is_ok() - }); - } - - fn did_unsubscribe_from_remote_audio_track(&self, publisher_id: String, track_id: String) { - self.update_subscribers.lock().retain(|tx| { - tx.unbounded_send(RoomUpdate::UnsubscribedFromRemoteAudioTrack { - publisher_id: publisher_id.clone(), - track_id: track_id.clone(), - }) - .is_ok() - }); - } - - fn mute_changed_from_remote_audio_track(&self, track_id: String, muted: bool) { - self.update_subscribers.lock().retain(|tx| { - tx.unbounded_send(RoomUpdate::RemoteAudioTrackMuteChanged { - track_id: track_id.clone(), - muted, - }) - .is_ok() - }); - } - - fn active_speakers_changed(&self, speakers: Vec) { - self.update_subscribers.lock().retain(move |tx| { - tx.unbounded_send(RoomUpdate::ActiveSpeakersChanged { - speakers: speakers.clone(), - }) - .is_ok() - }); - } - - fn did_subscribe_to_remote_video_track(&self, track: RemoteVideoTrack) { - let track = Arc::new(track); - self.update_subscribers.lock().retain(|tx| { - tx.unbounded_send(RoomUpdate::SubscribedToRemoteVideoTrack(track.clone())) - .is_ok() - }); - } - - fn did_unsubscribe_from_remote_video_track(&self, publisher_id: String, track_id: String) { - self.update_subscribers.lock().retain(|tx| { - tx.unbounded_send(RoomUpdate::UnsubscribedFromRemoteVideoTrack { - publisher_id: publisher_id.clone(), - track_id: track_id.clone(), - }) - .is_ok() - }); - } - - fn build_done_callback() -> ( - extern "C" fn(*mut c_void, CFStringRef), - *mut c_void, - oneshot::Receiver>, - ) { - let (tx, rx) = oneshot::channel(); - extern "C" fn done_callback(tx: *mut c_void, error: CFStringRef) { - let tx = unsafe { Box::from_raw(tx as *mut oneshot::Sender>) }; - if error.is_null() { - let _ = tx.send(Ok(())); - } else { - let error = unsafe { CFString::wrap_under_get_rule(error).to_string() }; - let _ = tx.send(Err(anyhow!(error))); - } - } - ( - done_callback, - Box::into_raw(Box::new(tx)) as *mut c_void, - rx, - ) - } - - pub fn set_display_sources(&self, _: Vec) { - unreachable!("This is a test-only function") - } -} - -impl Drop for Room { - fn drop(&mut self) { - unsafe { - LKRoomDisconnect(self.native_room); - CFRelease(self.native_room.0); - } - } -} - -struct RoomDelegate { - native_delegate: swift::RoomDelegate, - weak_room: *mut c_void, -} - -impl RoomDelegate { - fn new(weak_room: Weak) -> Self { - let weak_room = weak_room.into_raw() as *mut c_void; - let native_delegate = unsafe { - LKRoomDelegateCreate( - weak_room, - Self::on_did_disconnect, - Self::on_did_subscribe_to_remote_audio_track, - Self::on_did_unsubscribe_from_remote_audio_track, - Self::on_mute_change_from_remote_audio_track, - Self::on_active_speakers_changed, - Self::on_did_subscribe_to_remote_video_track, - Self::on_did_unsubscribe_from_remote_video_track, - Self::on_did_publish_or_unpublish_local_audio_track, - Self::on_did_publish_or_unpublish_local_video_track, - ) - }; - Self { - native_delegate, - weak_room, - } - } - - extern "C" fn on_did_disconnect(room: *mut c_void) { - let room = unsafe { Weak::from_raw(room as *mut Room) }; - if let Some(room) = room.upgrade() { - room.did_disconnect(); - } - let _ = Weak::into_raw(room); - } - - extern "C" fn on_did_subscribe_to_remote_audio_track( - room: *mut c_void, - publisher_id: CFStringRef, - track_id: CFStringRef, - track: swift::RemoteAudioTrack, - publication: swift::RemoteTrackPublication, - ) { - let room = unsafe { Weak::from_raw(room as *mut Room) }; - let publisher_id = unsafe { CFString::wrap_under_get_rule(publisher_id).to_string() }; - let track_id = unsafe { CFString::wrap_under_get_rule(track_id).to_string() }; - let track = RemoteAudioTrack::new(track, track_id, publisher_id); - let publication = RemoteTrackPublication::new(publication); - if let Some(room) = room.upgrade() { - room.did_subscribe_to_remote_audio_track(track, publication); - } - let _ = Weak::into_raw(room); - } - - extern "C" fn on_did_unsubscribe_from_remote_audio_track( - room: *mut c_void, - publisher_id: CFStringRef, - track_id: CFStringRef, - ) { - let room = unsafe { Weak::from_raw(room as *mut Room) }; - let publisher_id = unsafe { CFString::wrap_under_get_rule(publisher_id).to_string() }; - let track_id = unsafe { CFString::wrap_under_get_rule(track_id).to_string() }; - if let Some(room) = room.upgrade() { - room.did_unsubscribe_from_remote_audio_track(publisher_id, track_id); - } - let _ = Weak::into_raw(room); - } - - extern "C" fn on_mute_change_from_remote_audio_track( - room: *mut c_void, - track_id: CFStringRef, - muted: bool, - ) { - let room = unsafe { Weak::from_raw(room as *mut Room) }; - let track_id = unsafe { CFString::wrap_under_get_rule(track_id).to_string() }; - if let Some(room) = room.upgrade() { - room.mute_changed_from_remote_audio_track(track_id, muted); - } - let _ = Weak::into_raw(room); - } - - extern "C" fn on_active_speakers_changed(room: *mut c_void, participants: CFArrayRef) { - if participants.is_null() { - return; - } - - let room = unsafe { Weak::from_raw(room as *mut Room) }; - let speakers = unsafe { - CFArray::wrap_under_get_rule(participants) - .into_iter() - .map( - |speaker: core_foundation::base::ItemRef<'_, *const c_void>| { - CFString::wrap_under_get_rule(*speaker as CFStringRef).to_string() - }, - ) - .collect() - }; - - if let Some(room) = room.upgrade() { - room.active_speakers_changed(speakers); - } - let _ = Weak::into_raw(room); - } - - extern "C" fn on_did_subscribe_to_remote_video_track( - room: *mut c_void, - publisher_id: CFStringRef, - track_id: CFStringRef, - track: swift::RemoteVideoTrack, - ) { - let room = unsafe { Weak::from_raw(room as *mut Room) }; - let publisher_id = unsafe { CFString::wrap_under_get_rule(publisher_id).to_string() }; - let track_id = unsafe { CFString::wrap_under_get_rule(track_id).to_string() }; - let track = RemoteVideoTrack::new(track, track_id, publisher_id); - if let Some(room) = room.upgrade() { - room.did_subscribe_to_remote_video_track(track); - } - let _ = Weak::into_raw(room); - } - - extern "C" fn on_did_unsubscribe_from_remote_video_track( - room: *mut c_void, - publisher_id: CFStringRef, - track_id: CFStringRef, - ) { - let room = unsafe { Weak::from_raw(room as *mut Room) }; - let publisher_id = unsafe { CFString::wrap_under_get_rule(publisher_id).to_string() }; - let track_id = unsafe { CFString::wrap_under_get_rule(track_id).to_string() }; - if let Some(room) = room.upgrade() { - room.did_unsubscribe_from_remote_video_track(publisher_id, track_id); - } - let _ = Weak::into_raw(room); - } - - extern "C" fn on_did_publish_or_unpublish_local_audio_track( - room: *mut c_void, - publication: swift::LocalTrackPublication, - is_published: bool, - ) { - let room = unsafe { Weak::from_raw(room as *mut Room) }; - if let Some(room) = room.upgrade() { - let publication = LocalTrackPublication::new(publication); - let update = if is_published { - RoomUpdate::LocalAudioTrackPublished { publication } - } else { - RoomUpdate::LocalAudioTrackUnpublished { publication } - }; - room.update_subscribers - .lock() - .retain(|tx| tx.unbounded_send(update.clone()).is_ok()); - } - let _ = Weak::into_raw(room); - } - - extern "C" fn on_did_publish_or_unpublish_local_video_track( - room: *mut c_void, - publication: swift::LocalTrackPublication, - is_published: bool, - ) { - let room = unsafe { Weak::from_raw(room as *mut Room) }; - if let Some(room) = room.upgrade() { - let publication = LocalTrackPublication::new(publication); - let update = if is_published { - RoomUpdate::LocalVideoTrackPublished { publication } - } else { - RoomUpdate::LocalVideoTrackUnpublished { publication } - }; - room.update_subscribers - .lock() - .retain(|tx| tx.unbounded_send(update.clone()).is_ok()); - } - let _ = Weak::into_raw(room); - } -} - -impl Drop for RoomDelegate { - fn drop(&mut self) { - unsafe { - CFRelease(self.native_delegate.0); - let _ = Weak::from_raw(self.weak_room as *mut Room); - } - } -} - -pub struct LocalAudioTrack(swift::LocalAudioTrack); - -impl LocalAudioTrack { - pub fn create() -> Self { - Self(unsafe { LKLocalAudioTrackCreateTrack() }) - } -} - -impl Drop for LocalAudioTrack { - fn drop(&mut self) { - unsafe { CFRelease(self.0 .0) } - } -} - -pub struct LocalVideoTrack(swift::LocalVideoTrack); - -impl LocalVideoTrack { - pub fn screen_share_for_display(display: &MacOSDisplay) -> Self { - Self(unsafe { LKCreateScreenShareTrackForDisplay(display.0) }) - } -} - -impl Drop for LocalVideoTrack { - fn drop(&mut self) { - unsafe { CFRelease(self.0 .0) } - } -} - -pub struct LocalTrackPublication(swift::LocalTrackPublication); - -impl LocalTrackPublication { - pub fn new(native_track_publication: swift::LocalTrackPublication) -> Self { - unsafe { - CFRetain(native_track_publication.0); - } - Self(native_track_publication) - } - - pub fn sid(&self) -> String { - unsafe { CFString::wrap_under_get_rule(LKLocalTrackPublicationGetSid(self.0)).to_string() } - } - - pub fn set_mute(&self, muted: bool) -> impl Future> { - let (tx, rx) = futures::channel::oneshot::channel(); - - extern "C" fn complete_callback(callback_data: *mut c_void, error: CFStringRef) { - let tx = unsafe { Box::from_raw(callback_data as *mut oneshot::Sender>) }; - if error.is_null() { - tx.send(Ok(())).ok(); - } else { - let error = unsafe { CFString::wrap_under_get_rule(error).to_string() }; - tx.send(Err(anyhow!(error))).ok(); - } - } - - unsafe { - LKLocalTrackPublicationSetMute( - self.0, - muted, - complete_callback, - Box::into_raw(Box::new(tx)) as *mut c_void, - ) - } - - async move { rx.await.unwrap() } - } - - pub fn is_muted(&self) -> bool { - unsafe { LKLocalTrackPublicationIsMuted(self.0) } - } -} - -impl Clone for LocalTrackPublication { - fn clone(&self) -> Self { - unsafe { - CFRetain(self.0 .0); - } - Self(self.0) - } -} - -impl Drop for LocalTrackPublication { - fn drop(&mut self) { - unsafe { CFRelease(self.0 .0) } - } -} - -pub struct RemoteTrackPublication(swift::RemoteTrackPublication); - -impl RemoteTrackPublication { - pub fn new(native_track_publication: swift::RemoteTrackPublication) -> Self { - unsafe { - CFRetain(native_track_publication.0); - } - Self(native_track_publication) - } - - pub fn sid(&self) -> String { - unsafe { CFString::wrap_under_get_rule(LKRemoteTrackPublicationGetSid(self.0)).to_string() } - } - - pub fn is_muted(&self) -> bool { - unsafe { LKRemoteTrackPublicationIsMuted(self.0) } - } - - pub fn set_enabled(&self, enabled: bool) -> impl Future> { - let (tx, rx) = futures::channel::oneshot::channel(); - - extern "C" fn complete_callback(callback_data: *mut c_void, error: CFStringRef) { - let tx = unsafe { Box::from_raw(callback_data as *mut oneshot::Sender>) }; - if error.is_null() { - tx.send(Ok(())).ok(); - } else { - let error = unsafe { CFString::wrap_under_get_rule(error).to_string() }; - tx.send(Err(anyhow!(error))).ok(); - } - } - - unsafe { - LKRemoteTrackPublicationSetEnabled( - self.0, - enabled, - complete_callback, - Box::into_raw(Box::new(tx)) as *mut c_void, - ) - } - - async move { rx.await.unwrap() } - } -} - -impl Drop for RemoteTrackPublication { - fn drop(&mut self) { - unsafe { CFRelease(self.0 .0) } - } -} - -#[derive(Debug)] -pub struct RemoteAudioTrack { - native_track: swift::RemoteAudioTrack, - sid: Sid, - publisher_id: String, -} - -impl RemoteAudioTrack { - fn new(native_track: swift::RemoteAudioTrack, sid: Sid, publisher_id: String) -> Self { - unsafe { - CFRetain(native_track.0); - } - Self { - native_track, - sid, - publisher_id, - } - } - - pub fn sid(&self) -> &str { - &self.sid - } - - pub fn publisher_id(&self) -> &str { - &self.publisher_id - } - - pub fn start(&self) { - unsafe { LKRemoteAudioTrackStart(self.native_track) } - } - - pub fn stop(&self) { - unsafe { LKRemoteAudioTrackStop(self.native_track) } - } -} - -impl Drop for RemoteAudioTrack { - fn drop(&mut self) { - // todo: uncomment this `CFRelease`, unless we find that it was causing - // the crash in the `livekit.multicast` thread. - // - // unsafe { CFRelease(self.native_track.0) } - let _ = self.native_track; - } -} - -#[derive(Debug)] -pub struct RemoteVideoTrack { - native_track: swift::RemoteVideoTrack, - sid: Sid, - publisher_id: String, -} - -impl RemoteVideoTrack { - fn new(native_track: swift::RemoteVideoTrack, sid: Sid, publisher_id: String) -> Self { - unsafe { - CFRetain(native_track.0); - } - Self { - native_track, - sid, - publisher_id, - } - } - - pub fn sid(&self) -> &str { - &self.sid - } - - pub fn publisher_id(&self) -> &str { - &self.publisher_id - } - - pub fn frames(&self) -> async_broadcast::Receiver { - extern "C" fn on_frame(callback_data: *mut c_void, frame: CVImageBufferRef) -> bool { - unsafe { - let tx = Box::from_raw(callback_data as *mut async_broadcast::Sender); - let buffer = CVImageBuffer::wrap_under_get_rule(frame); - let result = tx.try_broadcast(Frame(buffer)); - let _ = Box::into_raw(tx); - match result { - Ok(_) => true, - Err(async_broadcast::TrySendError::Closed(_)) - | Err(async_broadcast::TrySendError::Inactive(_)) => { - log::warn!("no active receiver for frame"); - false - } - Err(async_broadcast::TrySendError::Full(_)) => { - log::warn!("skipping frame as receiver is not keeping up"); - true - } - } - } - } - - extern "C" fn on_drop(callback_data: *mut c_void) { - unsafe { - let _ = Box::from_raw(callback_data as *mut async_broadcast::Sender); - } - } - - let (tx, rx) = async_broadcast::broadcast(64); - unsafe { - let renderer = LKVideoRendererCreate( - Box::into_raw(Box::new(tx)) as *mut c_void, - on_frame, - on_drop, - ); - LKVideoTrackAddRenderer(self.native_track, renderer); - rx - } - } -} - -impl Drop for RemoteVideoTrack { - fn drop(&mut self) { - unsafe { CFRelease(self.native_track.0) } - } -} - -pub struct MacOSDisplay(swift::MacOSDisplay); - -impl MacOSDisplay { - fn new(ptr: swift::MacOSDisplay) -> Self { - unsafe { - CFRetain(ptr.0); - } - Self(ptr) - } -} - -impl Drop for MacOSDisplay { - fn drop(&mut self) { - unsafe { CFRelease(self.0 .0) } - } -} - -#[derive(Clone)] -pub struct Frame(CVImageBuffer); - -impl Frame { - pub fn width(&self) -> usize { - self.0.width() - } - - pub fn height(&self) -> usize { - self.0.height() - } - - pub fn image(&self) -> CVImageBuffer { - self.0.clone() - } -} diff --git a/crates/live_kit_client/src/remote_video_track_view.rs b/crates/live_kit_client/src/remote_video_track_view.rs new file mode 100644 index 0000000000000..bbfaea1875cf1 --- /dev/null +++ b/crates/live_kit_client/src/remote_video_track_view.rs @@ -0,0 +1,61 @@ +use crate::track::RemoteVideoTrack; +use anyhow::Result; +use futures::StreamExt as _; +use gpui::{ + Empty, EventEmitter, IntoElement, Render, ScreenCaptureFrame, Task, View, ViewContext, + VisualContext as _, +}; + +pub struct RemoteVideoTrackView { + track: RemoteVideoTrack, + frame: Option, + _maintain_frame: Task>, +} + +#[derive(Debug)] +pub enum RemoteVideoTrackViewEvent { + Close, +} + +impl RemoteVideoTrackView { + pub fn new(track: RemoteVideoTrack, cx: &mut ViewContext) -> Self { + cx.focus_handle(); + let frames = super::play_remote_video_track(&track); + + Self { + track, + frame: None, + _maintain_frame: cx.spawn(|this, mut cx| async move { + futures::pin_mut!(frames); + while let Some(frame) = frames.next().await { + this.update(&mut cx, |this, cx| { + this.frame = Some(frame); + cx.notify(); + })?; + } + this.update(&mut cx, |_, cx| cx.emit(RemoteVideoTrackViewEvent::Close))?; + Ok(()) + }), + } + } + + pub fn clone(&self, cx: &mut ViewContext) -> View { + cx.new_view(|cx| Self::new(self.track.clone(), cx)) + } +} + +impl EventEmitter for RemoteVideoTrackView {} + +impl Render for RemoteVideoTrackView { + fn render(&mut self, _cx: &mut ViewContext) -> impl IntoElement { + #[cfg(target_os = "macos")] + if let Some(frame) = &self.frame { + use gpui::Styled as _; + return gpui::surface(frame.0.clone()) + .size_full() + .into_any_element(); + } + + Empty.into_any_element() + } +} diff --git a/crates/live_kit_client/src/test.rs b/crates/live_kit_client/src/test.rs index 2c26c88f72346..1bd2e60d17c3c 100644 --- a/crates/live_kit_client/src/test.rs +++ b/crates/live_kit_client/src/test.rs @@ -1,32 +1,42 @@ -use crate::{ConnectionState, RoomUpdate, Sid}; +pub mod participant; +pub mod publication; +pub mod track; + +#[cfg(not(windows))] +pub mod webrtc; + +#[cfg(not(windows))] +use self::id::*; +use self::{participant::*, publication::*, track::*}; use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; use collections::{btree_map::Entry as BTreeEntry, hash_map::Entry, BTreeMap, HashMap, HashSet}; -use futures::Stream; -use gpui::{BackgroundExecutor, SurfaceSource}; +use gpui::BackgroundExecutor; use live_kit_server::{proto, token}; - +#[cfg(not(windows))] +use livekit::options::TrackPublishOptions; use parking_lot::Mutex; -use postage::watch; -use std::{ - future::Future, - mem, - sync::{ - atomic::{AtomicBool, Ordering::SeqCst}, - Arc, Weak, - }, +use postage::{mpsc, sink::Sink}; +use std::sync::{ + atomic::{AtomicBool, Ordering::SeqCst}, + Arc, Weak, }; +#[cfg(not(windows))] +pub use livekit::{id, options, ConnectionState, DisconnectReason, RoomOptions}; + static SERVERS: Mutex>> = Mutex::new(BTreeMap::new()); pub struct TestServer { pub url: String, pub api_key: String, pub secret_key: String, + #[cfg(not(target_os = "windows"))] rooms: Mutex>, executor: BackgroundExecutor, } +#[cfg(not(target_os = "windows"))] impl TestServer { pub fn create( url: String, @@ -73,9 +83,8 @@ impl TestServer { } pub async fn create_room(&self, room: String) -> Result<()> { - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] self.executor.simulate_random_delay().await; + let mut server_rooms = self.rooms.lock(); if let Entry::Vacant(e) = server_rooms.entry(room.clone()) { e.insert(Default::default()); @@ -86,10 +95,8 @@ impl TestServer { } async fn delete_room(&self, room: String) -> Result<()> { - // TODO: clear state associated with all `Room`s. - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] self.executor.simulate_random_delay().await; + let mut server_rooms = self.rooms.lock(); server_rooms .remove(&room) @@ -97,46 +104,64 @@ impl TestServer { Ok(()) } - async fn join_room(&self, token: String, client_room: Arc) -> Result<()> { - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] + async fn join_room(&self, token: String, client_room: Room) -> Result { self.executor.simulate_random_delay().await; let claims = live_kit_server::token::validate(&token, &self.secret_key)?; - let identity = claims.sub.unwrap().to_string(); + let identity = ParticipantIdentity(claims.sub.unwrap().to_string()); let room_name = claims.video.room.unwrap(); let mut server_rooms = self.rooms.lock(); let room = (*server_rooms).entry(room_name.to_string()).or_default(); if let Entry::Vacant(e) = room.client_rooms.entry(identity.clone()) { - for track in &room.video_tracks { + for server_track in &room.video_tracks { + let track = RemoteTrack::Video(RemoteVideoTrack { + server_track: server_track.clone(), + _room: client_room.downgrade(), + }); client_room .0 .lock() .updates_tx - .try_broadcast(RoomUpdate::SubscribedToRemoteVideoTrack(Arc::new( - RemoteVideoTrack { - server_track: track.clone(), + .blocking_send(RoomEvent::TrackSubscribed { + track: track.clone(), + publication: RemoteTrackPublication { + sid: server_track.sid.clone(), + room: client_room.downgrade(), + track, + }, + participant: RemoteParticipant { + room: client_room.downgrade(), + identity: server_track.publisher_id.clone(), }, - ))) + }) .unwrap(); } - for track in &room.audio_tracks { + for server_track in &room.audio_tracks { + let track = RemoteTrack::Audio(RemoteAudioTrack { + server_track: server_track.clone(), + room: client_room.downgrade(), + }); client_room .0 .lock() .updates_tx - .try_broadcast(RoomUpdate::SubscribedToRemoteAudioTrack( - Arc::new(RemoteAudioTrack { - server_track: track.clone(), - room: Arc::downgrade(&client_room), - }), - Arc::new(RemoteTrackPublication), - )) + .blocking_send(RoomEvent::TrackSubscribed { + track: track.clone(), + publication: RemoteTrackPublication { + sid: server_track.sid.clone(), + room: client_room.downgrade(), + track, + }, + participant: RemoteParticipant { + room: client_room.downgrade(), + identity: server_track.publisher_id.clone(), + }, + }) .unwrap(); } e.insert(client_room); - Ok(()) + Ok(identity) } else { Err(anyhow!( "{:?} attempted to join room {:?} twice", @@ -147,11 +172,10 @@ impl TestServer { } async fn leave_room(&self, token: String) -> Result<()> { - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] self.executor.simulate_random_delay().await; + let claims = live_kit_server::token::validate(&token, &self.secret_key)?; - let identity = claims.sub.unwrap().to_string(); + let identity = ParticipantIdentity(claims.sub.unwrap().to_string()); let room_name = claims.video.room.unwrap(); let mut server_rooms = self.rooms.lock(); let room = server_rooms @@ -167,10 +191,44 @@ impl TestServer { Ok(()) } - async fn remove_participant(&self, room_name: String, identity: String) -> Result<()> { - // TODO: clear state associated with the `Room`. - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] + fn remote_participants( + &self, + token: String, + ) -> Result> { + let claims = live_kit_server::token::validate(&token, &self.secret_key)?; + let local_identity = ParticipantIdentity(claims.sub.unwrap().to_string()); + let room_name = claims.video.room.unwrap().to_string(); + + if let Some(server_room) = self.rooms.lock().get(&room_name) { + let room = server_room + .client_rooms + .get(&local_identity) + .unwrap() + .downgrade(); + Ok(server_room + .client_rooms + .iter() + .filter(|(identity, _)| *identity != &local_identity) + .map(|(identity, _)| { + ( + identity.clone(), + RemoteParticipant { + room: room.clone(), + identity: identity.clone(), + }, + ) + }) + .collect()) + } else { + Ok(Default::default()) + } + } + + async fn remove_participant( + &self, + room_name: String, + identity: ParticipantIdentity, + ) -> Result<()> { self.executor.simulate_random_delay().await; let mut server_rooms = self.rooms.lock(); @@ -193,25 +251,32 @@ impl TestServer { identity: String, permission: proto::ParticipantPermission, ) -> Result<()> { - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] self.executor.simulate_random_delay().await; + let mut server_rooms = self.rooms.lock(); let room = server_rooms .get_mut(&room_name) .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; - room.participant_permissions.insert(identity, permission); + room.participant_permissions + .insert(ParticipantIdentity(identity), permission); Ok(()) } pub async fn disconnect_client(&self, client_identity: String) { - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] + let client_identity = ParticipantIdentity(client_identity); + self.executor.simulate_random_delay().await; + let mut server_rooms = self.rooms.lock(); for room in server_rooms.values_mut() { if let Some(room) = room.client_rooms.remove(&client_identity) { - *room.0.lock().connection.0.borrow_mut() = ConnectionState::Disconnected; + let mut room = room.0.lock(); + room.connection_state = ConnectionState::Disconnected; + room.updates_tx + .blocking_send(RoomEvent::Disconnected { + reason: DisconnectReason::SignalClose, + }) + .ok(); } } } @@ -219,13 +284,12 @@ impl TestServer { async fn publish_video_track( &self, token: String, - local_track: LocalVideoTrack, - ) -> Result { - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] + _local_track: LocalVideoTrack, + ) -> Result { self.executor.simulate_random_delay().await; + let claims = live_kit_server::token::validate(&token, &self.secret_key)?; - let identity = claims.sub.unwrap().to_string(); + let identity = ParticipantIdentity(claims.sub.unwrap().to_string()); let room_name = claims.video.room.unwrap(); let mut server_rooms = self.rooms.lock(); @@ -244,26 +308,38 @@ impl TestServer { return Err(anyhow!("user is not allowed to publish")); } - let sid = nanoid::nanoid!(17); - let track = Arc::new(TestServerVideoTrack { + let sid: TrackSid = format!("TR_{}", nanoid::nanoid!(17)).try_into().unwrap(); + let server_track = Arc::new(TestServerVideoTrack { sid: sid.clone(), publisher_id: identity.clone(), - frames_rx: local_track.frames_rx.clone(), }); - room.video_tracks.push(track.clone()); - - for (id, client_room) in &room.client_rooms { - if *id != identity { - let _ = client_room + room.video_tracks.push(server_track.clone()); + + for (room_identity, client_room) in &room.client_rooms { + if *room_identity != identity { + let track = RemoteTrack::Video(RemoteVideoTrack { + server_track: server_track.clone(), + _room: client_room.downgrade(), + }); + let publication = RemoteTrackPublication { + sid: sid.clone(), + room: client_room.downgrade(), + track: track.clone(), + }; + let participant = RemoteParticipant { + identity: identity.clone(), + room: client_room.downgrade(), + }; + client_room .0 .lock() .updates_tx - .try_broadcast(RoomUpdate::SubscribedToRemoteVideoTrack(Arc::new( - RemoteVideoTrack { - server_track: track.clone(), - }, - ))) + .blocking_send(RoomEvent::TrackSubscribed { + track, + publication, + participant, + }) .unwrap(); } } @@ -275,13 +351,11 @@ impl TestServer { &self, token: String, _local_track: &LocalAudioTrack, - ) -> Result { - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] + ) -> Result { self.executor.simulate_random_delay().await; let claims = live_kit_server::token::validate(&token, &self.secret_key)?; - let identity = claims.sub.unwrap().to_string(); + let identity = ParticipantIdentity(claims.sub.unwrap().to_string()); let room_name = claims.video.room.unwrap(); let mut server_rooms = self.rooms.lock(); @@ -300,41 +374,54 @@ impl TestServer { return Err(anyhow!("user is not allowed to publish")); } - let sid = nanoid::nanoid!(17); - let track = Arc::new(TestServerAudioTrack { + let sid: TrackSid = format!("TR_{}", nanoid::nanoid!(17)).try_into().unwrap(); + let server_track = Arc::new(TestServerAudioTrack { sid: sid.clone(), publisher_id: identity.clone(), muted: AtomicBool::new(false), }); - let publication = Arc::new(RemoteTrackPublication); - - room.audio_tracks.push(track.clone()); - - for (id, client_room) in &room.client_rooms { - if *id != identity { - let _ = client_room + room.audio_tracks.push(server_track.clone()); + + for (room_identity, client_room) in &room.client_rooms { + if *room_identity != identity { + let track = RemoteTrack::Audio(RemoteAudioTrack { + server_track: server_track.clone(), + room: client_room.downgrade(), + }); + let publication = RemoteTrackPublication { + sid: sid.clone(), + room: client_room.downgrade(), + track: track.clone(), + }; + let participant = RemoteParticipant { + identity: identity.clone(), + room: client_room.downgrade(), + }; + client_room .0 .lock() .updates_tx - .try_broadcast(RoomUpdate::SubscribedToRemoteAudioTrack( - Arc::new(RemoteAudioTrack { - server_track: track.clone(), - room: Arc::downgrade(client_room), - }), - publication.clone(), - )) - .unwrap(); + .blocking_send(RoomEvent::TrackSubscribed { + track, + publication, + participant, + }) + .ok(); } } Ok(sid) } - fn set_track_muted(&self, token: &str, track_sid: &str, muted: bool) -> Result<()> { - let claims = live_kit_server::token::validate(token, &self.secret_key)?; + async fn unpublish_track(&self, _token: String, _track: &TrackSid) -> Result<()> { + Ok(()) + } + + fn set_track_muted(&self, token: &str, track_sid: &TrackSid, muted: bool) -> Result<()> { + let claims = live_kit_server::token::validate(&token, &self.secret_key)?; let room_name = claims.video.room.unwrap(); - let identity = claims.sub.unwrap(); + let identity = ParticipantIdentity(claims.sub.unwrap().to_string()); let mut server_rooms = self.rooms.lock(); let room = server_rooms .get_mut(&*room_name) @@ -342,19 +429,42 @@ impl TestServer { if let Some(track) = room .audio_tracks .iter_mut() - .find(|track| track.sid == track_sid) + .find(|track| track.sid == *track_sid) { track.muted.store(muted, SeqCst); for (id, client_room) in room.client_rooms.iter() { if *id != identity { + let participant = Participant::Remote(RemoteParticipant { + identity: identity.clone(), + room: client_room.downgrade(), + }); + let track = RemoteTrack::Audio(RemoteAudioTrack { + server_track: track.clone(), + room: client_room.downgrade(), + }); + let publication = TrackPublication::Remote(RemoteTrackPublication { + sid: track_sid.clone(), + room: client_room.downgrade(), + track, + }); + + let event = if muted { + RoomEvent::TrackMuted { + participant, + publication, + } + } else { + RoomEvent::TrackUnmuted { + participant, + publication, + } + }; + client_room .0 .lock() .updates_tx - .try_broadcast(RoomUpdate::RemoteAudioTrackMuteChanged { - track_id: track_sid.to_string(), - muted, - }) + .blocking_send(event) .unwrap(); } } @@ -362,14 +472,14 @@ impl TestServer { Ok(()) } - fn is_track_muted(&self, token: &str, track_sid: &str) -> Option { - let claims = live_kit_server::token::validate(token, &self.secret_key).ok()?; + fn is_track_muted(&self, token: &str, track_sid: &TrackSid) -> Option { + let claims = live_kit_server::token::validate(&token, &self.secret_key).ok()?; let room_name = claims.video.room.unwrap(); let mut server_rooms = self.rooms.lock(); let room = server_rooms.get_mut(&*room_name)?; room.audio_tracks.iter().find_map(|track| { - if track.sid == track_sid { + if track.sid == *track_sid { Some(track.muted.load(SeqCst)) } else { None @@ -377,33 +487,33 @@ impl TestServer { }) } - fn video_tracks(&self, token: String) -> Result>> { + fn video_tracks(&self, token: String) -> Result> { let claims = live_kit_server::token::validate(&token, &self.secret_key)?; let room_name = claims.video.room.unwrap(); - let identity = claims.sub.unwrap(); + let identity = ParticipantIdentity(claims.sub.unwrap().to_string()); let mut server_rooms = self.rooms.lock(); let room = server_rooms .get_mut(&*room_name) .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; - room.client_rooms - .get(identity.as_ref()) + let client_room = room + .client_rooms + .get(&identity) .ok_or_else(|| anyhow!("not a participant in room"))?; Ok(room .video_tracks .iter() - .map(|track| { - Arc::new(RemoteVideoTrack { - server_track: track.clone(), - }) + .map(|track| RemoteVideoTrack { + server_track: track.clone(), + _room: client_room.downgrade(), }) .collect()) } - fn audio_tracks(&self, token: String) -> Result>> { + fn audio_tracks(&self, token: String) -> Result> { let claims = live_kit_server::token::validate(&token, &self.secret_key)?; let room_name = claims.video.room.unwrap(); - let identity = claims.sub.unwrap(); + let identity = ParticipantIdentity(claims.sub.unwrap().to_string()); let mut server_rooms = self.rooms.lock(); let room = server_rooms @@ -411,49 +521,125 @@ impl TestServer { .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; let client_room = room .client_rooms - .get(identity.as_ref()) + .get(&identity) .ok_or_else(|| anyhow!("not a participant in room"))?; Ok(room .audio_tracks .iter() - .map(|track| { - Arc::new(RemoteAudioTrack { - server_track: track.clone(), - room: Arc::downgrade(client_room), - }) + .map(|track| RemoteAudioTrack { + server_track: track.clone(), + room: client_room.downgrade(), }) .collect()) } } -#[derive(Default)] +#[cfg(not(target_os = "windows"))] +#[derive(Default, Debug)] struct TestServerRoom { - client_rooms: HashMap>, + client_rooms: HashMap, video_tracks: Vec>, audio_tracks: Vec>, - participant_permissions: HashMap, + participant_permissions: HashMap, } +#[cfg(not(target_os = "windows"))] #[derive(Debug)] struct TestServerVideoTrack { - sid: Sid, - publisher_id: Sid, - frames_rx: async_broadcast::Receiver, + sid: TrackSid, + publisher_id: ParticipantIdentity, + // frames_rx: async_broadcast::Receiver, } +#[cfg(not(target_os = "windows"))] #[derive(Debug)] struct TestServerAudioTrack { - sid: Sid, - publisher_id: Sid, + sid: TrackSid, + publisher_id: ParticipantIdentity, muted: AtomicBool, } -impl TestServerRoom {} - pub struct TestApiClient { url: String, } +#[derive(Clone, Debug)] +#[non_exhaustive] +pub enum RoomEvent { + ParticipantConnected(RemoteParticipant), + ParticipantDisconnected(RemoteParticipant), + LocalTrackPublished { + publication: LocalTrackPublication, + track: LocalTrack, + participant: LocalParticipant, + }, + LocalTrackUnpublished { + publication: LocalTrackPublication, + participant: LocalParticipant, + }, + TrackSubscribed { + track: RemoteTrack, + publication: RemoteTrackPublication, + participant: RemoteParticipant, + }, + TrackUnsubscribed { + track: RemoteTrack, + publication: RemoteTrackPublication, + participant: RemoteParticipant, + }, + TrackSubscriptionFailed { + participant: RemoteParticipant, + error: String, + #[cfg(not(target_os = "windows"))] + track_sid: TrackSid, + }, + TrackPublished { + publication: RemoteTrackPublication, + participant: RemoteParticipant, + }, + TrackUnpublished { + publication: RemoteTrackPublication, + participant: RemoteParticipant, + }, + TrackMuted { + participant: Participant, + publication: TrackPublication, + }, + TrackUnmuted { + participant: Participant, + publication: TrackPublication, + }, + RoomMetadataChanged { + old_metadata: String, + metadata: String, + }, + ParticipantMetadataChanged { + participant: Participant, + old_metadata: String, + metadata: String, + }, + ParticipantNameChanged { + participant: Participant, + old_name: String, + name: String, + }, + ActiveSpeakersChanged { + speakers: Vec, + }, + #[cfg(not(target_os = "windows"))] + ConnectionStateChanged(ConnectionState), + Connected { + participants_with_tracks: Vec<(RemoteParticipant, Vec)>, + }, + #[cfg(not(target_os = "windows"))] + Disconnected { + reason: DisconnectReason, + }, + Reconnecting, + Reconnected, +} + +#[cfg(not(target_os = "windows"))] #[async_trait] impl live_kit_server::api::Client for TestApiClient { fn url(&self) -> &str { @@ -474,7 +660,9 @@ impl live_kit_server::api::Client for TestApiClient { async fn remove_participant(&self, room: String, identity: String) -> Result<()> { let server = TestServer::get(&self.url)?; - server.remove_participant(room, identity).await?; + server + .remove_participant(room, ParticipantIdentity(identity)) + .await?; Ok(()) } @@ -513,370 +701,125 @@ impl live_kit_server::api::Client for TestApiClient { } struct RoomState { - connection: ( - watch::Sender, - watch::Receiver, - ), - display_sources: Vec, - paused_audio_tracks: HashSet, - updates_tx: async_broadcast::Sender, - updates_rx: async_broadcast::Receiver, + url: String, + token: String, + #[cfg(not(target_os = "windows"))] + local_identity: ParticipantIdentity, + #[cfg(not(target_os = "windows"))] + connection_state: ConnectionState, + #[cfg(not(target_os = "windows"))] + paused_audio_tracks: HashSet, + updates_tx: mpsc::Sender, } -pub struct Room(Mutex); +#[derive(Clone, Debug)] +pub struct Room(Arc>); -impl Room { - pub fn new() -> Arc { - let (updates_tx, updates_rx) = async_broadcast::broadcast(128); - Arc::new(Self(Mutex::new(RoomState { - connection: watch::channel_with(ConnectionState::Disconnected), - display_sources: Default::default(), - paused_audio_tracks: Default::default(), - updates_tx, - updates_rx, - }))) - } +#[derive(Clone, Debug)] +pub(crate) struct WeakRoom(Weak>); - pub fn status(&self) -> watch::Receiver { - self.0.lock().connection.1.clone() +#[cfg(not(target_os = "windows"))] +impl std::fmt::Debug for RoomState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Room") + .field("url", &self.url) + .field("token", &self.token) + .field("local_identity", &self.local_identity) + .field("connection_state", &self.connection_state) + .field("paused_audio_tracks", &self.paused_audio_tracks) + .finish() } +} - pub fn connect(self: &Arc, url: &str, token: &str) -> impl Future> { - let this = self.clone(); - let url = url.to_string(); - let token = token.to_string(); - async move { - let server = TestServer::get(&url)?; - server - .join_room(token.clone(), this.clone()) - .await - .context("room join")?; - *this.0.lock().connection.0.borrow_mut() = ConnectionState::Connected { url, token }; - Ok(()) - } +#[cfg(target_os = "windows")] +impl std::fmt::Debug for RoomState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Room") + .field("url", &self.url) + .field("token", &self.token) + .finish() } +} - pub fn display_sources(self: &Arc) -> impl Future>> { - let this = self.clone(); - async move { - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] - { - let server = this.test_server(); - server.executor.simulate_random_delay().await; - } - - Ok(this.0.lock().display_sources.clone()) - } +#[cfg(not(target_os = "windows"))] +impl Room { + fn downgrade(&self) -> WeakRoom { + WeakRoom(Arc::downgrade(&self.0)) } - pub fn publish_video_track( - self: &Arc, - track: LocalVideoTrack, - ) -> impl Future> { - let this = self.clone(); - let track = track.clone(); - async move { - let sid = this - .test_server() - .publish_video_track(this.token(), track) - .await?; - Ok(LocalTrackPublication { - room: Arc::downgrade(&this), - sid, - }) - } + pub fn connection_state(&self) -> ConnectionState { + self.0.lock().connection_state } - pub fn publish_audio_track( - self: &Arc, - track: LocalAudioTrack, - ) -> impl Future> { - let this = self.clone(); - let track = track.clone(); - async move { - let sid = this - .test_server() - .publish_audio_track(this.token(), &track) - .await?; - Ok(LocalTrackPublication { - room: Arc::downgrade(&this), - sid, - }) + pub fn local_participant(&self) -> LocalParticipant { + let identity = self.0.lock().local_identity.clone(); + LocalParticipant { + identity, + room: self.clone(), } } - pub fn unpublish_track(&self, _publication: LocalTrackPublication) {} - - pub fn remote_audio_tracks(&self, publisher_id: &str) -> Vec> { - if !self.is_connected() { - return Vec::new(); - } - - self.test_server() - .audio_tracks(self.token()) - .unwrap() - .into_iter() - .filter(|track| track.publisher_id() == publisher_id) - .collect() - } + pub async fn connect( + url: &str, + token: &str, + _options: RoomOptions, + ) -> Result<(Self, mpsc::Receiver)> { + let server = TestServer::get(&url)?; + let (updates_tx, updates_rx) = mpsc::channel(1024); + let this = Self(Arc::new(Mutex::new(RoomState { + local_identity: ParticipantIdentity(String::new()), + url: url.to_string(), + token: token.to_string(), + connection_state: ConnectionState::Disconnected, + paused_audio_tracks: Default::default(), + updates_tx, + }))); - pub fn remote_audio_track_publications( - &self, - publisher_id: &str, - ) -> Vec> { - if !self.is_connected() { - return Vec::new(); + let identity = server + .join_room(token.to_string(), this.clone()) + .await + .context("room join")?; + { + let mut state = this.0.lock(); + state.local_identity = identity; + state.connection_state = ConnectionState::Connected; } - self.test_server() - .audio_tracks(self.token()) - .unwrap() - .into_iter() - .filter(|track| track.publisher_id() == publisher_id) - .map(|_track| Arc::new(RemoteTrackPublication {})) - .collect() + Ok((this, updates_rx)) } - pub fn remote_video_tracks(&self, publisher_id: &str) -> Vec> { - if !self.is_connected() { - return Vec::new(); - } - + pub fn remote_participants(&self) -> HashMap { self.test_server() - .video_tracks(self.token()) + .remote_participants(self.0.lock().token.clone()) .unwrap() - .into_iter() - .filter(|track| track.publisher_id() == publisher_id) - .collect() - } - - pub fn updates(&self) -> impl Stream { - self.0.lock().updates_rx.clone() - } - - pub fn set_display_sources(&self, sources: Vec) { - self.0.lock().display_sources = sources; } fn test_server(&self) -> Arc { - match self.0.lock().connection.1.borrow().clone() { - ConnectionState::Disconnected => panic!("must be connected to call this method"), - ConnectionState::Connected { url, .. } => TestServer::get(&url).unwrap(), - } + TestServer::get(&self.0.lock().url).unwrap() } fn token(&self) -> String { - match self.0.lock().connection.1.borrow().clone() { - ConnectionState::Disconnected => panic!("must be connected to call this method"), - ConnectionState::Connected { token, .. } => token, - } - } - - fn is_connected(&self) -> bool { - match *self.0.lock().connection.1.borrow() { - ConnectionState::Disconnected => false, - ConnectionState::Connected { .. } => true, - } + self.0.lock().token.clone() } } -impl Drop for Room { +#[cfg(not(target_os = "windows"))] +impl Drop for RoomState { fn drop(&mut self) { - if let ConnectionState::Connected { token, .. } = mem::replace( - &mut *self.0.lock().connection.0.borrow_mut(), - ConnectionState::Disconnected, - ) { - if let Ok(server) = TestServer::get(&token) { + if self.connection_state == ConnectionState::Connected { + if let Ok(server) = TestServer::get(&self.url) { let executor = server.executor.clone(); + let token = self.token.clone(); executor - .spawn(async move { server.leave_room(token).await.unwrap() }) + .spawn(async move { server.leave_room(token).await.ok() }) .detach(); } } } } -#[derive(Clone)] -pub struct LocalTrackPublication { - sid: String, - room: Weak, -} - -impl LocalTrackPublication { - pub fn set_mute(&self, mute: bool) -> impl Future> { - let sid = self.sid.clone(); - let room = self.room.clone(); - async move { - if let Some(room) = room.upgrade() { - room.test_server() - .set_track_muted(&room.token(), &sid, mute) - } else { - Err(anyhow!("no such room")) - } - } - } - - pub fn is_muted(&self) -> bool { - if let Some(room) = self.room.upgrade() { - room.test_server() - .is_track_muted(&room.token(), &self.sid) - .unwrap_or(false) - } else { - false - } - } - - pub fn sid(&self) -> String { - self.sid.clone() - } -} - -pub struct RemoteTrackPublication; - -impl RemoteTrackPublication { - pub fn set_enabled(&self, _enabled: bool) -> impl Future> { - async { Ok(()) } - } - - pub fn is_muted(&self) -> bool { - false - } - - pub fn sid(&self) -> String { - "".to_string() - } -} - -#[derive(Clone)] -pub struct LocalVideoTrack { - frames_rx: async_broadcast::Receiver, -} - -impl LocalVideoTrack { - pub fn screen_share_for_display(display: &MacOSDisplay) -> Self { - Self { - frames_rx: display.frames.1.clone(), - } - } -} - -#[derive(Clone)] -pub struct LocalAudioTrack; - -impl LocalAudioTrack { - pub fn create() -> Self { - Self - } -} - -#[derive(Debug)] -pub struct RemoteVideoTrack { - server_track: Arc, -} - -impl RemoteVideoTrack { - pub fn sid(&self) -> &str { - &self.server_track.sid - } - - pub fn publisher_id(&self) -> &str { - &self.server_track.publisher_id - } - - pub fn frames(&self) -> async_broadcast::Receiver { - self.server_track.frames_rx.clone() - } -} - -#[derive(Debug)] -pub struct RemoteAudioTrack { - server_track: Arc, - room: Weak, -} - -impl RemoteAudioTrack { - pub fn sid(&self) -> &str { - &self.server_track.sid - } - - pub fn publisher_id(&self) -> &str { - &self.server_track.publisher_id - } - - pub fn start(&self) { - if let Some(room) = self.room.upgrade() { - room.0 - .lock() - .paused_audio_tracks - .remove(&self.server_track.sid); - } - } - - pub fn stop(&self) { - if let Some(room) = self.room.upgrade() { - room.0 - .lock() - .paused_audio_tracks - .insert(self.server_track.sid.clone()); - } - } - - pub fn is_playing(&self) -> bool { - !self - .room - .upgrade() - .unwrap() - .0 - .lock() - .paused_audio_tracks - .contains(&self.server_track.sid) - } -} - -#[derive(Clone)] -pub struct MacOSDisplay { - frames: ( - async_broadcast::Sender, - async_broadcast::Receiver, - ), -} - -impl Default for MacOSDisplay { - fn default() -> Self { - Self::new() - } -} - -impl MacOSDisplay { - pub fn new() -> Self { - Self { - frames: async_broadcast::broadcast(128), - } - } - - pub fn send_frame(&self, frame: Frame) { - self.frames.0.try_broadcast(frame).unwrap(); - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Frame { - pub label: String, - pub width: usize, - pub height: usize, -} - -impl Frame { - pub fn width(&self) -> usize { - self.width - } - - pub fn height(&self) -> usize { - self.height - } - - pub fn image(&self) -> SurfaceSource { - unimplemented!("you can't call this in test mode") +impl WeakRoom { + fn upgrade(&self) -> Option { + self.0.upgrade().map(Room) } } diff --git a/crates/live_kit_client/src/test/participant.rs b/crates/live_kit_client/src/test/participant.rs new file mode 100644 index 0000000000000..8d476b15379dd --- /dev/null +++ b/crates/live_kit_client/src/test/participant.rs @@ -0,0 +1,111 @@ +use super::*; + +#[derive(Clone, Debug)] +pub enum Participant { + Local(LocalParticipant), + Remote(RemoteParticipant), +} + +#[derive(Clone, Debug)] +pub struct LocalParticipant { + #[cfg(not(target_os = "windows"))] + pub(super) identity: ParticipantIdentity, + pub(super) room: Room, +} + +#[derive(Clone, Debug)] +pub struct RemoteParticipant { + #[cfg(not(target_os = "windows"))] + pub(super) identity: ParticipantIdentity, + pub(super) room: WeakRoom, +} + +#[cfg(not(target_os = "windows"))] +impl Participant { + pub fn identity(&self) -> ParticipantIdentity { + match self { + Participant::Local(participant) => participant.identity.clone(), + Participant::Remote(participant) => participant.identity.clone(), + } + } +} + +#[cfg(not(target_os = "windows"))] +impl LocalParticipant { + pub async fn unpublish_track(&self, track: &TrackSid) -> Result<()> { + self.room + .test_server() + .unpublish_track(self.room.token(), track) + .await + } + + pub async fn publish_track( + &self, + track: LocalTrack, + _options: TrackPublishOptions, + ) -> Result { + let this = self.clone(); + let track = track.clone(); + let server = this.room.test_server(); + let sid = match track { + LocalTrack::Video(track) => { + server.publish_video_track(this.room.token(), track).await? + } + LocalTrack::Audio(track) => { + server + .publish_audio_track(this.room.token(), &track) + .await? + } + }; + Ok(LocalTrackPublication { + room: self.room.downgrade(), + sid, + }) + } +} + +#[cfg(not(target_os = "windows"))] +impl RemoteParticipant { + pub fn track_publications(&self) -> HashMap { + if let Some(room) = self.room.upgrade() { + let server = room.test_server(); + let audio = server + .audio_tracks(room.token()) + .unwrap() + .into_iter() + .filter(|track| track.publisher_id() == self.identity) + .map(|track| { + ( + track.sid(), + RemoteTrackPublication { + sid: track.sid(), + room: self.room.clone(), + track: RemoteTrack::Audio(track), + }, + ) + }); + let video = server + .video_tracks(room.token()) + .unwrap() + .into_iter() + .filter(|track| track.publisher_id() == self.identity) + .map(|track| { + ( + track.sid(), + RemoteTrackPublication { + sid: track.sid(), + room: self.room.clone(), + track: RemoteTrack::Video(track), + }, + ) + }); + audio.chain(video).collect() + } else { + HashMap::default() + } + } + + pub fn identity(&self) -> ParticipantIdentity { + self.identity.clone() + } +} diff --git a/crates/live_kit_client/src/test/publication.rs b/crates/live_kit_client/src/test/publication.rs new file mode 100644 index 0000000000000..6a3dfa0a51ce9 --- /dev/null +++ b/crates/live_kit_client/src/test/publication.rs @@ -0,0 +1,116 @@ +use super::*; + +#[derive(Clone, Debug)] +pub enum TrackPublication { + Local(LocalTrackPublication), + Remote(RemoteTrackPublication), +} + +#[derive(Clone, Debug)] +pub struct LocalTrackPublication { + #[cfg(not(target_os = "windows"))] + pub(crate) sid: TrackSid, + pub(crate) room: WeakRoom, +} + +#[derive(Clone, Debug)] +pub struct RemoteTrackPublication { + #[cfg(not(target_os = "windows"))] + pub(crate) sid: TrackSid, + pub(crate) room: WeakRoom, + pub(crate) track: RemoteTrack, +} + +#[cfg(not(target_os = "windows"))] +impl TrackPublication { + pub fn sid(&self) -> TrackSid { + match self { + TrackPublication::Local(track) => track.sid(), + TrackPublication::Remote(track) => track.sid(), + } + } + + pub fn is_muted(&self) -> bool { + match self { + TrackPublication::Local(track) => track.is_muted(), + TrackPublication::Remote(track) => track.is_muted(), + } + } +} + +#[cfg(not(target_os = "windows"))] +impl LocalTrackPublication { + pub fn sid(&self) -> TrackSid { + self.sid.clone() + } + + pub fn mute(&self) { + self.set_mute(true) + } + + pub fn unmute(&self) { + self.set_mute(false) + } + + fn set_mute(&self, mute: bool) { + if let Some(room) = self.room.upgrade() { + room.test_server() + .set_track_muted(&room.token(), &self.sid, mute) + .ok(); + } + } + + pub fn is_muted(&self) -> bool { + if let Some(room) = self.room.upgrade() { + room.test_server() + .is_track_muted(&room.token(), &self.sid) + .unwrap_or(false) + } else { + false + } + } +} + +#[cfg(not(target_os = "windows"))] +impl RemoteTrackPublication { + pub fn sid(&self) -> TrackSid { + self.sid.clone() + } + + pub fn track(&self) -> Option { + Some(self.track.clone()) + } + + pub fn kind(&self) -> TrackKind { + self.track.kind() + } + + pub fn is_muted(&self) -> bool { + if let Some(room) = self.room.upgrade() { + room.test_server() + .is_track_muted(&room.token(), &self.sid) + .unwrap_or(false) + } else { + false + } + } + + pub fn is_enabled(&self) -> bool { + if let Some(room) = self.room.upgrade() { + !room.0.lock().paused_audio_tracks.contains(&self.sid) + } else { + false + } + } + + pub fn set_enabled(&self, enabled: bool) { + if let Some(room) = self.room.upgrade() { + let paused_audio_tracks = &mut room.0.lock().paused_audio_tracks; + if enabled { + paused_audio_tracks.remove(&self.sid); + } else { + paused_audio_tracks.insert(self.sid.clone()); + } + } + } +} diff --git a/crates/live_kit_client/src/test/track.rs b/crates/live_kit_client/src/test/track.rs new file mode 100644 index 0000000000000..302177a10a6f6 --- /dev/null +++ b/crates/live_kit_client/src/test/track.rs @@ -0,0 +1,201 @@ +use super::*; +#[cfg(not(windows))] +use webrtc::{audio_source::RtcAudioSource, video_source::RtcVideoSource}; + +#[cfg(not(windows))] +pub use livekit::track::{TrackKind, TrackSource}; + +#[derive(Clone, Debug)] +pub enum LocalTrack { + Audio(LocalAudioTrack), + Video(LocalVideoTrack), +} + +#[derive(Clone, Debug)] +pub enum RemoteTrack { + Audio(RemoteAudioTrack), + Video(RemoteVideoTrack), +} + +#[derive(Clone, Debug)] +pub struct LocalVideoTrack {} + +#[derive(Clone, Debug)] +pub struct LocalAudioTrack {} + +#[derive(Clone, Debug)] +pub struct RemoteVideoTrack { + #[cfg(not(target_os = "windows"))] + pub(super) server_track: Arc, + pub(super) _room: WeakRoom, +} + +#[derive(Clone, Debug)] +pub struct RemoteAudioTrack { + #[cfg(not(target_os = "windows"))] + pub(super) server_track: Arc, + pub(super) room: WeakRoom, +} + +pub enum RtcTrack { + Audio(RtcAudioTrack), + Video(RtcVideoTrack), +} + +pub struct RtcAudioTrack { + #[cfg(not(target_os = "windows"))] + pub(super) server_track: Arc, + pub(super) room: WeakRoom, +} + +pub struct RtcVideoTrack { + #[cfg(not(target_os = "windows"))] + pub(super) _server_track: Arc, +} + +#[cfg(not(target_os = "windows"))] +impl RemoteTrack { + pub fn sid(&self) -> TrackSid { + match self { + RemoteTrack::Audio(track) => track.sid(), + RemoteTrack::Video(track) => track.sid(), + } + } + + pub fn kind(&self) -> TrackKind { + match self { + RemoteTrack::Audio(_) => TrackKind::Audio, + RemoteTrack::Video(_) => TrackKind::Video, + } + } + + pub fn publisher_id(&self) -> ParticipantIdentity { + match self { + RemoteTrack::Audio(track) => track.publisher_id(), + RemoteTrack::Video(track) => track.publisher_id(), + } + } + + pub fn rtc_track(&self) -> RtcTrack { + match self { + RemoteTrack::Audio(track) => RtcTrack::Audio(track.rtc_track()), + RemoteTrack::Video(track) => RtcTrack::Video(track.rtc_track()), + } + } +} + +#[cfg(not(windows))] +impl LocalVideoTrack { + pub fn create_video_track(_name: &str, _source: RtcVideoSource) -> Self { + Self {} + } +} + +#[cfg(not(windows))] +impl LocalAudioTrack { + pub fn create_audio_track(_name: &str, _source: RtcAudioSource) -> Self { + Self {} + } +} + +#[cfg(not(target_os = "windows"))] +impl RemoteAudioTrack { + pub fn sid(&self) -> TrackSid { + self.server_track.sid.clone() + } + + pub fn publisher_id(&self) -> ParticipantIdentity { + self.server_track.publisher_id.clone() + } + + pub fn start(&self) { + if let Some(room) = self.room.upgrade() { + room.0 + .lock() + .paused_audio_tracks + .remove(&self.server_track.sid); + } + } + + pub fn stop(&self) { + if let Some(room) = self.room.upgrade() { + room.0 + .lock() + .paused_audio_tracks + .insert(self.server_track.sid.clone()); + } + } + + pub fn rtc_track(&self) -> RtcAudioTrack { + RtcAudioTrack { + server_track: self.server_track.clone(), + room: self.room.clone(), + } + } +} + +#[cfg(not(target_os = "windows"))] +impl RemoteVideoTrack { + pub fn sid(&self) -> TrackSid { + self.server_track.sid.clone() + } + + pub fn publisher_id(&self) -> ParticipantIdentity { + self.server_track.publisher_id.clone() + } + + pub fn rtc_track(&self) -> RtcVideoTrack { + RtcVideoTrack { + _server_track: self.server_track.clone(), + } + } +} + +#[cfg(not(target_os = "windows"))] +impl RtcTrack { + pub fn enabled(&self) -> bool { + match self { + RtcTrack::Audio(track) => track.enabled(), + RtcTrack::Video(track) => track.enabled(), + } + } + + pub fn set_enabled(&self, enabled: bool) { + match self { + RtcTrack::Audio(track) => track.set_enabled(enabled), + RtcTrack::Video(_) => {} + } + } +} + +#[cfg(not(target_os = "windows"))] +impl RtcAudioTrack { + pub fn set_enabled(&self, enabled: bool) { + if let Some(room) = self.room.upgrade() { + let paused_audio_tracks = &mut room.0.lock().paused_audio_tracks; + if enabled { + paused_audio_tracks.remove(&self.server_track.sid); + } else { + paused_audio_tracks.insert(self.server_track.sid.clone()); + } + } + } + + pub fn enabled(&self) -> bool { + if let Some(room) = self.room.upgrade() { + !room + .0 + .lock() + .paused_audio_tracks + .contains(&self.server_track.sid) + } else { + false + } + } +} + +impl RtcVideoTrack { + pub fn enabled(&self) -> bool { + true + } +} diff --git a/crates/live_kit_client/src/test/webrtc.rs b/crates/live_kit_client/src/test/webrtc.rs new file mode 100644 index 0000000000000..6ac06e04846d7 --- /dev/null +++ b/crates/live_kit_client/src/test/webrtc.rs @@ -0,0 +1,136 @@ +use super::track::{RtcAudioTrack, RtcVideoTrack}; +use futures::Stream; +use livekit::webrtc as real; +use std::{ + pin::Pin, + task::{Context, Poll}, +}; + +pub mod video_stream { + use super::*; + + pub mod native { + use super::*; + use real::video_frame::BoxVideoFrame; + + pub struct NativeVideoStream { + pub track: RtcVideoTrack, + } + + impl NativeVideoStream { + pub fn new(track: RtcVideoTrack) -> Self { + Self { track } + } + } + + impl Stream for NativeVideoStream { + type Item = BoxVideoFrame; + + fn poll_next(self: Pin<&mut Self>, _cx: &mut Context) -> Poll> { + Poll::Pending + } + } + } +} + +pub mod audio_stream { + use super::*; + + pub mod native { + use super::*; + use real::audio_frame::AudioFrame; + + pub struct NativeAudioStream { + pub track: RtcAudioTrack, + } + + impl NativeAudioStream { + pub fn new(track: RtcAudioTrack, _sample_rate: i32, _num_channels: i32) -> Self { + Self { track } + } + } + + impl Stream for NativeAudioStream { + type Item = AudioFrame<'static>; + + fn poll_next(self: Pin<&mut Self>, _cx: &mut Context) -> Poll> { + Poll::Pending + } + } + } +} + +pub mod audio_source { + use super::*; + + pub use real::audio_source::AudioSourceOptions; + + pub mod native { + use std::sync::Arc; + + use super::*; + use real::{audio_frame::AudioFrame, RtcError}; + + #[derive(Clone)] + pub struct NativeAudioSource { + pub options: Arc, + pub sample_rate: u32, + pub num_channels: u32, + } + + impl NativeAudioSource { + pub fn new( + options: AudioSourceOptions, + sample_rate: u32, + num_channels: u32, + _queue_size_ms: u32, + ) -> Self { + Self { + options: Arc::new(options), + sample_rate, + num_channels, + } + } + + pub async fn capture_frame(&self, _frame: &AudioFrame<'_>) -> Result<(), RtcError> { + Ok(()) + } + } + } + + pub enum RtcAudioSource { + Native(native::NativeAudioSource), + } +} + +pub use livekit::webrtc::audio_frame; +pub use livekit::webrtc::video_frame; + +pub mod video_source { + use super::*; + pub use real::video_source::VideoResolution; + + pub struct RTCVideoSource; + + pub mod native { + use super::*; + use real::video_frame::{VideoBuffer, VideoFrame}; + + #[derive(Clone)] + pub struct NativeVideoSource { + pub resolution: VideoResolution, + } + + impl NativeVideoSource { + pub fn new(resolution: super::VideoResolution) -> Self { + Self { resolution } + } + + pub fn capture_frame>(&self, _frame: &VideoFrame) {} + } + } + + pub enum RtcVideoSource { + Native(native::NativeVideoSource), + } +} diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index 92940d1c526e8..70478eeb759f8 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -17,6 +17,7 @@ anyhow.workspace = true [target.'cfg(target_os = "macos")'.dependencies] core-foundation.workspace = true +ctor.workspace = true foreign-types = "0.5" metal = "0.29" objc = "0.2" diff --git a/crates/media/src/media.rs b/crates/media/src/media.rs index 8757249c31940..3f55475589814 100644 --- a/crates/media/src/media.rs +++ b/crates/media/src/media.rs @@ -253,11 +253,14 @@ pub mod core_media { } } - pub fn image_buffer(&self) -> CVImageBuffer { + pub fn image_buffer(&self) -> Option { unsafe { - CVImageBuffer::wrap_under_get_rule(CMSampleBufferGetImageBuffer( - self.as_concrete_TypeRef(), - )) + let ptr = CMSampleBufferGetImageBuffer(self.as_concrete_TypeRef()); + if ptr.is_null() { + None + } else { + Some(CVImageBuffer::wrap_under_get_rule(ptr)) + } } } diff --git a/crates/title_bar/src/collab.rs b/crates/title_bar/src/collab.rs index 805c0e72029b2..f043194a03e3e 100644 --- a/crates/title_bar/src/collab.rs +++ b/crates/title_bar/src/collab.rs @@ -296,9 +296,9 @@ impl TitleBar { let is_muted = room.is_muted(); let is_deafened = room.is_deafened().unwrap_or(false); let is_screen_sharing = room.is_screen_sharing(); - let can_use_microphone = room.can_use_microphone(); + let can_use_microphone = room.can_use_microphone(cx); let can_share_projects = room.can_share_projects(); - let platform_supported = match self.platform_style { + let screen_sharing_supported = match self.platform_style { PlatformStyle::Mac => true, PlatformStyle::Linux | PlatformStyle::Windows => false, }; @@ -365,9 +365,7 @@ impl TitleBar { ) .tooltip(move |cx| { Tooltip::text( - if !platform_supported { - "Cannot share microphone" - } else if is_muted { + if is_muted { "Unmute microphone" } else { "Mute microphone" @@ -377,56 +375,45 @@ impl TitleBar { }) .style(ButtonStyle::Subtle) .icon_size(IconSize::Small) - .selected(platform_supported && is_muted) - .disabled(!platform_supported) + .selected(is_muted) .selected_style(ButtonStyle::Tinted(TintColor::Negative)) .on_click(move |_, cx| { toggle_mute(&Default::default(), cx); }) .into_any_element(), ); - } - children.push( - IconButton::new( - "mute-sound", - if is_deafened { - ui::IconName::AudioOff - } else { - ui::IconName::AudioOn - }, - ) - .style(ButtonStyle::Subtle) - .selected_style(ButtonStyle::Tinted(TintColor::Negative)) - .icon_size(IconSize::Small) - .selected(is_deafened) - .disabled(!platform_supported) - .tooltip(move |cx| { - if !platform_supported { - Tooltip::text("Cannot share microphone", cx) - } else if can_use_microphone { + children.push( + IconButton::new( + "mute-sound", + if is_deafened { + ui::IconName::AudioOff + } else { + ui::IconName::AudioOn + }, + ) + .style(ButtonStyle::Subtle) + .selected_style(ButtonStyle::Tinted(TintColor::Negative)) + .icon_size(IconSize::Small) + .selected(is_deafened) + .tooltip(move |cx| { Tooltip::with_meta("Deafen Audio", None, "Mic will be muted", cx) - } else { - Tooltip::text("Deafen Audio", cx) - } - }) - .on_click(move |_, cx| toggle_deafen(&Default::default(), cx)) - .into_any_element(), - ); + }) + .on_click(move |_, cx| toggle_deafen(&Default::default(), cx)) + .into_any_element(), + ); + } - if can_share_projects { + if screen_sharing_supported { children.push( IconButton::new("screen-share", ui::IconName::Screen) .style(ButtonStyle::Subtle) .icon_size(IconSize::Small) .selected(is_screen_sharing) - .disabled(!platform_supported) .selected_style(ButtonStyle::Tinted(TintColor::Accent)) .tooltip(move |cx| { Tooltip::text( - if !platform_supported { - "Cannot share screen" - } else if is_screen_sharing { + if is_screen_sharing { "Stop Sharing Screen" } else { "Share Screen" diff --git a/crates/workspace/src/shared_screen.rs b/crates/workspace/src/shared_screen.rs index 59df859488093..285946cce0709 100644 --- a/crates/workspace/src/shared_screen.rs +++ b/crates/workspace/src/shared_screen.rs @@ -2,16 +2,13 @@ use crate::{ item::{Item, ItemEvent}, ItemNavHistory, WorkspaceId, }; -use anyhow::Result; -use call::participant::{Frame, RemoteVideoTrack}; +use call::{RemoteVideoTrack, RemoteVideoTrackView}; use client::{proto::PeerId, User}; -use futures::StreamExt; use gpui::{ - div, surface, AppContext, EventEmitter, FocusHandle, FocusableView, InteractiveElement, - ParentElement, Render, SharedString, Styled, Task, View, ViewContext, VisualContext, - WindowContext, + div, AppContext, EventEmitter, FocusHandle, FocusableView, InteractiveElement, ParentElement, + Render, SharedString, Styled, View, ViewContext, VisualContext, WindowContext, }; -use std::sync::{Arc, Weak}; +use std::sync::Arc; use ui::{prelude::*, Icon, IconName}; pub enum Event { @@ -19,40 +16,30 @@ pub enum Event { } pub struct SharedScreen { - track: Weak, - frame: Option, pub peer_id: PeerId, user: Arc, nav_history: Option, - _maintain_frame: Task>, + view: View, focus: FocusHandle, } impl SharedScreen { pub fn new( - track: &Arc, + track: RemoteVideoTrack, peer_id: PeerId, user: Arc, cx: &mut ViewContext, ) -> Self { - cx.focus_handle(); - let mut frames = track.frames(); + let view = cx.new_view(|cx| RemoteVideoTrackView::new(track.clone(), cx)); + cx.subscribe(&view, |_, _, ev, cx| match ev { + call::RemoteVideoTrackViewEvent::Close => cx.emit(Event::Close), + }) + .detach(); Self { - track: Arc::downgrade(track), - frame: None, + view, peer_id, user, nav_history: Default::default(), - _maintain_frame: cx.spawn(|this, mut cx| async move { - while let Some(frame) = frames.next().await { - this.update(&mut cx, |this, cx| { - this.frame = Some(frame); - cx.notify(); - })?; - } - this.update(&mut cx, |_, cx| cx.emit(Event::Close))?; - Ok(()) - }), focus: cx.focus_handle(), } } @@ -72,11 +59,7 @@ impl Render for SharedScreen { .track_focus(&self.focus) .key_context("SharedScreen") .size_full() - .children( - self.frame - .as_ref() - .map(|frame| surface(frame.image()).size_full()), - ) + .child(self.view.clone()) } } @@ -114,8 +97,13 @@ impl Item for SharedScreen { _workspace_id: Option, cx: &mut ViewContext, ) -> Option> { - let track = self.track.upgrade()?; - Some(cx.new_view(|cx| Self::new(&track, self.peer_id, self.user.clone(), cx))) + Some(cx.new_view(|cx| Self { + view: self.view.update(cx, |view, cx| view.clone(cx)), + peer_id: self.peer_id, + user: self.user.clone(), + nav_history: Default::default(), + focus: cx.focus_handle(), + })) } fn to_item_events(event: &Self::Event, mut f: impl FnMut(ItemEvent)) { diff --git a/crates/workspace/src/workspace.rs b/crates/workspace/src/workspace.rs index aa3d859f8cef7..047e47e814834 100644 --- a/crates/workspace/src/workspace.rs +++ b/crates/workspace/src/workspace.rs @@ -3939,6 +3939,17 @@ impl Workspace { None } + #[cfg(target_os = "windows")] + fn shared_screen_for_peer( + &self, + _peer_id: PeerId, + _pane: &View, + _cx: &mut WindowContext, + ) -> Option> { + None + } + + #[cfg(not(target_os = "windows"))] fn shared_screen_for_peer( &self, peer_id: PeerId, @@ -3957,7 +3968,7 @@ impl Workspace { } } - Some(cx.new_view(|cx| SharedScreen::new(&track, peer_id, user.clone(), cx))) + Some(cx.new_view(|cx| SharedScreen::new(track, peer_id, user.clone(), cx))) } pub fn on_window_activation_changed(&mut self, cx: &mut ViewContext) {