From 3561849fabd84dca9f08989ed07d873f87687af5 Mon Sep 17 00:00:00 2001 From: Calin-Teodor Date: Mon, 30 Sep 2024 13:10:42 +0300 Subject: [PATCH 1/8] feat(mobile/picture-in-picture): ios implementation --- react/features/app/types.ts | 2 + .../media/components/native/FallbackView.tsx | 10 +++ .../base/media/components/native/Video.tsx | 69 +++++++++++++++++-- .../components/ParticipantView.native.tsx | 3 +- .../mobile/picture-in-picture/actionTypes.ts | 12 ++++ .../mobile/picture-in-picture/actions.ts | 9 ++- .../picture-in-picture/middleware.native.ts | 24 +++++++ .../mobile/picture-in-picture/reducer.ts | 29 ++++++++ 8 files changed, 150 insertions(+), 8 deletions(-) create mode 100644 react/features/base/media/components/native/FallbackView.tsx create mode 100644 react/features/mobile/picture-in-picture/middleware.native.ts create mode 100644 react/features/mobile/picture-in-picture/reducer.ts diff --git a/react/features/app/types.ts b/react/features/app/types.ts index 272d5bd2ca9c..898b27338f8f 100644 --- a/react/features/app/types.ts +++ b/react/features/app/types.ts @@ -52,6 +52,7 @@ import { IBackgroundState } from '../mobile/background/reducer'; import { ICallIntegrationState } from '../mobile/call-integration/reducer'; import { IMobileExternalApiState } from '../mobile/external-api/reducer'; import { IFullScreenState } from '../mobile/full-screen/reducer'; +import { IMobilePictureInPictureState } from '../mobile/picture-in-picture/reducer'; import { IMobileWatchOSState } from '../mobile/watchos/reducer'; import { INoAudioSignalState } from '../no-audio-signal/reducer'; import { INoiseDetectionState } from '../noise-detection/reducer'; @@ -142,6 +143,7 @@ export interface IReduxState { 'features/lobby': ILobbyState; 'features/mobile/audio-mode': IMobileAudioModeState; 'features/mobile/external-api': IMobileExternalApiState; + 'features/mobile/picture-in-picture': IMobilePictureInPictureState; 'features/mobile/watchos': IMobileWatchOSState; 'features/no-audio-signal': INoAudioSignalState; 'features/noise-detection': INoiseDetectionState; diff --git a/react/features/base/media/components/native/FallbackView.tsx b/react/features/base/media/components/native/FallbackView.tsx new file mode 100644 index 000000000000..626d3032d908 --- /dev/null +++ b/react/features/base/media/components/native/FallbackView.tsx @@ -0,0 +1,10 @@ +import { View } from 'react-native'; + +const FallbackView = () => { + + return( + + ) +} + +export default FallbackView; diff --git a/react/features/base/media/components/native/Video.tsx b/react/features/base/media/components/native/Video.tsx index e6003c9588bc..f2965ce89ee8 100644 --- a/react/features/base/media/components/native/Video.tsx +++ b/react/features/base/media/components/native/Video.tsx @@ -1,16 +1,27 @@ -import React, { Component } from 'react'; +import React, {Component, RefObject} from 'react'; import { GestureResponderEvent } from 'react-native'; -import { MediaStream, RTCView } from 'react-native-webrtc'; +import { connect } from 'react-redux'; +import { MediaStream, RTCPIPView, startIOSPIP, stopIOSPIP } from 'react-native-webrtc'; + +import { IReduxState} from '../../../../app/types'; +import { translate } from '../../../i18n/functions'; import Pressable from '../../../react/components/native/Pressable'; +import logger from '../../logger'; + import VideoTransform from './VideoTransform'; import styles from './styles'; +import FallbackView from "./FallbackView"; + /** * The type of the React {@code Component} props of {@link Video}. */ interface IProps { + + _enableIosPIP?: boolean; + mirror: boolean; onPlaying: Function; @@ -58,7 +69,15 @@ interface IProps { * {@code HTMLVideoElement} and wraps around react-native-webrtc's * {@link RTCView}. */ -export default class Video extends Component { +class Video extends Component { + + _ref: RefObject; + + constructor(props) { + super(props); + this._ref = React.createRef(); + } + /** * React Component method that executes once component is mounted. * @@ -79,7 +98,28 @@ export default class Video extends Component { * @returns {ReactElement|null} */ render() { - const { onPress, stream, zoomEnabled } = this.props; + const { _enableIosPIP, onPress, stream, zoomEnabled } = this.props; + + const iosPIPOptions = { + startAutomatically: true, + fallbackView: (), + preferredSize: { + width: 400, + height: 800, + } + } + + if (_enableIosPIP) { + console.log('TESTING _enableIosPIP is', _enableIosPIP); + console.log(this._ref?.current, 'Picture in picture mode on'); + // logger.warn(this._ref?.current, `Picture in picture mode on`); + // startIOSPIP(this._ref?.current); + } else { + console.log('TESTING _enableIosPIP is', _enableIosPIP); + console.log('TESTING Picture in picture mode off'); + // logger.warn(`Picture in picture mode off`); + // stopIOSPIP(this._ref?.current); + } if (stream) { // RTCView @@ -90,9 +130,11 @@ export default class Video extends Component { : 'cover'; const rtcView = ( - @@ -132,3 +174,20 @@ export default class Video extends Component { return null; } } + +/** + * Maps part of the Redux state to the props of this component. + * + * @param {Object} state - The Redux state. + * @returns {Object} + */ +function _mapStateToProps(state: IReduxState) { + const iosPIP = state['features/mobile/picture-in-picture']?.enableIosPIP + + return { + _enableIosPIP: iosPIP + }; +} + +// @ts-ignore +export default translate(connect(_mapStateToProps)(Video)); diff --git a/react/features/base/participants/components/ParticipantView.native.tsx b/react/features/base/participants/components/ParticipantView.native.tsx index 390bbd5e930a..4b5a6b7415c6 100644 --- a/react/features/base/participants/components/ParticipantView.native.tsx +++ b/react/features/base/participants/components/ParticipantView.native.tsx @@ -21,6 +21,7 @@ import TestHint from '../../testing/components/TestHint'; import { getVideoTrackByParticipant } from '../../tracks/functions'; import { ITrack } from '../../tracks/types'; import { getParticipantById, getParticipantDisplayName, isSharedVideoParticipant } from '../functions'; +import { FakeParticipant } from '../types'; import styles from './styles'; @@ -71,7 +72,7 @@ interface IProps { /** * Whether video should be disabled for his view. */ - disableVideo?: boolean; + disableVideo?: boolean | FakeParticipant; /** * Callback to invoke when the {@code ParticipantView} is clicked/pressed. diff --git a/react/features/mobile/picture-in-picture/actionTypes.ts b/react/features/mobile/picture-in-picture/actionTypes.ts index 318b31264d57..5ffa11ef4f11 100644 --- a/react/features/mobile/picture-in-picture/actionTypes.ts +++ b/react/features/mobile/picture-in-picture/actionTypes.ts @@ -1,3 +1,15 @@ +/** + * The type of redux action to enable picture-in-picture for ios. + * + * { + * type: ENABLE_IOS_PIP, + * enableIosPIP: boolean + * } + * + * @public + */ +export const ENABLE_IOS_PIP = 'ENABLE_IOS_PIP'; + /** * The type of redux action to enter (or rather initiate entering) * picture-in-picture. diff --git a/react/features/mobile/picture-in-picture/actions.ts b/react/features/mobile/picture-in-picture/actions.ts index 4b27aff53109..dc98c15f28ae 100644 --- a/react/features/mobile/picture-in-picture/actions.ts +++ b/react/features/mobile/picture-in-picture/actions.ts @@ -3,7 +3,7 @@ import { NativeModules } from 'react-native'; import { IStore } from '../../app/types'; import Platform from '../../base/react/Platform.native'; -import { ENTER_PICTURE_IN_PICTURE } from './actionTypes'; +import { ENABLE_IOS_PIP, ENTER_PICTURE_IN_PICTURE } from './actionTypes'; import { isPipEnabled } from './functions'; import logger from './logger'; @@ -24,8 +24,13 @@ export function enterPictureInPicture() { // fine to enter PiP mode. if (isPipEnabled(getState())) { const { PictureInPicture } = NativeModules; + const { enableIosPIP } = getState()['features/mobile/picture-in-picture']; const p - = Platform.OS === 'android' + = Platform.OS === 'ios' + ? dispatch({ + type: ENABLE_IOS_PIP, + enableIosPIP: !enableIosPIP }) + : Platform.OS === 'android' ? PictureInPicture ? PictureInPicture.enterPictureInPicture() : Promise.reject( diff --git a/react/features/mobile/picture-in-picture/middleware.native.ts b/react/features/mobile/picture-in-picture/middleware.native.ts new file mode 100644 index 000000000000..a27a9be905df --- /dev/null +++ b/react/features/mobile/picture-in-picture/middleware.native.ts @@ -0,0 +1,24 @@ +import MiddlewareRegistry from '../../base/redux/MiddlewareRegistry'; +import { APP_STATE_CHANGED } from '../background/actionTypes'; +import { ENABLE_IOS_PIP } from './actionTypes'; + +MiddlewareRegistry.register(store => next => action => { + const result = next(action); + + switch (action.type) { + case APP_STATE_CHANGED: { + const state = store.getState(); + const { dispatch } = store; + const { appState } = state['features/background']; + + if (appState === 'inactive') { + dispatch({ + type: ENABLE_IOS_PIP, + enableIosPIP: false }) + } + break; + } + } + + return result; +}) diff --git a/react/features/mobile/picture-in-picture/reducer.ts b/react/features/mobile/picture-in-picture/reducer.ts new file mode 100644 index 000000000000..9466165cd956 --- /dev/null +++ b/react/features/mobile/picture-in-picture/reducer.ts @@ -0,0 +1,29 @@ +import ReducerRegistry from '../../base/redux/ReducerRegistry'; +import { ENABLE_IOS_PIP } from './actionTypes'; + +const DEFAULT_STATE = { + enableIosPIP: false +}; + +export interface IMobilePictureInPictureState { + enableIosPIP: boolean; +} + +const STORE_NAME = 'features/mobile/picture-in-picture'; + +ReducerRegistry.register(STORE_NAME, (state = DEFAULT_STATE, action): IMobilePictureInPictureState => { + switch (action.type) { + + case ENABLE_IOS_PIP: { + const { enableIosPIP } = action; + + return { + ...state, + enableIosPIP + } + } + + default: + return state; + } +}) From 2543e9e076f60527adf60acc000a441c0e47c0f7 Mon Sep 17 00:00:00 2001 From: Calin-Teodor Date: Mon, 30 Sep 2024 13:12:02 +0300 Subject: [PATCH 2/8] dep(rnwebrtc): use ios pip rnwebrtc commit --- package-lock.json | 13 +++++++------ package.json | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index 270119234c40..4d20248d3efb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -100,7 +100,7 @@ "react-native-url-polyfill": "2.0.0", "react-native-video": "6.0.0-alpha.11", "react-native-watch-connectivity": "1.1.0", - "react-native-webrtc": "124.0.4", + "react-native-webrtc": "git+https://git@github.com/react-native-webrtc/react-native-webrtc.git#4288a1630d6dcfa141ff7c1c8c30c64c67c0dd0e", "react-native-webview": "13.8.7", "react-native-youtube-iframe": "2.3.0", "react-redux": "7.2.9", @@ -16063,8 +16063,9 @@ }, "node_modules/react-native-webrtc": { "version": "124.0.4", - "resolved": "https://registry.npmjs.org/react-native-webrtc/-/react-native-webrtc-124.0.4.tgz", - "integrity": "sha512-ZbhSz1f+kc1v5VE0B84+v6ujIWTHa2fIuocrYzGUIFab7E5izmct7PNHb9dzzs0xhBGqh4c2rUa49jbL+P/e2w==", + "resolved": "git+https://git@github.com/react-native-webrtc/react-native-webrtc.git#4288a1630d6dcfa141ff7c1c8c30c64c67c0dd0e", + "integrity": "sha512-Z5G4+wSqLYNtdORjoDXkbFQKRW9AI7dvys9E8L642bAA+k1g9kZSL3FC9eWmsQKq/0gEgeYaFiJwUiENzqlnfw==", + "license": "MIT", "dependencies": { "base64-js": "1.5.1", "debug": "4.3.4", @@ -30642,9 +30643,9 @@ } }, "react-native-webrtc": { - "version": "124.0.4", - "resolved": "https://registry.npmjs.org/react-native-webrtc/-/react-native-webrtc-124.0.4.tgz", - "integrity": "sha512-ZbhSz1f+kc1v5VE0B84+v6ujIWTHa2fIuocrYzGUIFab7E5izmct7PNHb9dzzs0xhBGqh4c2rUa49jbL+P/e2w==", + "version": "git+https://git@github.com/react-native-webrtc/react-native-webrtc.git#4288a1630d6dcfa141ff7c1c8c30c64c67c0dd0e", + "integrity": "sha512-Z5G4+wSqLYNtdORjoDXkbFQKRW9AI7dvys9E8L642bAA+k1g9kZSL3FC9eWmsQKq/0gEgeYaFiJwUiENzqlnfw==", + "from": "react-native-webrtc@git+https://git@github.com/react-native-webrtc/react-native-webrtc.git#4288a1630d6dcfa141ff7c1c8c30c64c67c0dd0e", "requires": { "base64-js": "1.5.1", "debug": "4.3.4", diff --git a/package.json b/package.json index 4eb40d99a4b1..42a75dfc5ac8 100644 --- a/package.json +++ b/package.json @@ -106,7 +106,7 @@ "react-native-url-polyfill": "2.0.0", "react-native-video": "6.0.0-alpha.11", "react-native-watch-connectivity": "1.1.0", - "react-native-webrtc": "124.0.4", + "react-native-webrtc": "git+https://git@github.com/react-native-webrtc/react-native-webrtc.git#4288a1630d6dcfa141ff7c1c8c30c64c67c0dd0e", "react-native-webview": "13.8.7", "react-native-youtube-iframe": "2.3.0", "react-redux": "7.2.9", From 15db4a29ec6235b9d2ab4e3d17f5bb2e78b98416 Mon Sep 17 00:00:00 2001 From: Calin-Teodor Date: Fri, 4 Oct 2024 14:40:45 +0300 Subject: [PATCH 3/8] feat(base/media): refactoring/lint fixes --- .../media/components/native/FallbackView.tsx | 9 +- .../base/media/components/native/Video.tsx | 82 +++++++++---------- 2 files changed, 42 insertions(+), 49 deletions(-) diff --git a/react/features/base/media/components/native/FallbackView.tsx b/react/features/base/media/components/native/FallbackView.tsx index 626d3032d908..e64970a8e412 100644 --- a/react/features/base/media/components/native/FallbackView.tsx +++ b/react/features/base/media/components/native/FallbackView.tsx @@ -1,10 +1,7 @@ import { View } from 'react-native'; -const FallbackView = () => { - - return( - - ) -} +const FallbackView = () => ( + +) export default FallbackView; diff --git a/react/features/base/media/components/native/Video.tsx b/react/features/base/media/components/native/Video.tsx index f2965ce89ee8..6da58c9bcea7 100644 --- a/react/features/base/media/components/native/Video.tsx +++ b/react/features/base/media/components/native/Video.tsx @@ -8,7 +8,7 @@ import { IReduxState} from '../../../../app/types'; import { translate } from '../../../i18n/functions'; import Pressable from '../../../react/components/native/Pressable'; -import logger from '../../logger'; +// import logger from '../../logger'; import VideoTransform from './VideoTransform'; import styles from './styles'; @@ -101,7 +101,6 @@ class Video extends Component { const { _enableIosPIP, onPress, stream, zoomEnabled } = this.props; const iosPIPOptions = { - startAutomatically: true, fallbackView: (), preferredSize: { width: 400, @@ -121,52 +120,49 @@ class Video extends Component { // stopIOSPIP(this._ref?.current); } - if (stream) { - // RTCView - const style = styles.video; - const objectFit - = zoomEnabled - ? 'contain' - : 'cover'; - const rtcView - = ( - - ); - - // VideoTransform implements "pinch to zoom". As part of "pinch to - // zoom", it implements onPress, of course. - if (zoomEnabled) { - return ( - - { rtcView } - - ); - } - - // XXX Unfortunately, VideoTransform implements a custom press - // detection which has been observed to be very picky about the - // precision of the press unlike the builtin/default/standard press - // detection which is forgiving to imperceptible movements while - // pressing. It's not acceptable to be so picky, especially when - // "pinch to zoom" is not enabled. + // RTCView + const style = styles.video; + const objectFit + = zoomEnabled + ? 'contain' + : 'cover'; + const rtcView + = ( + + ); + + // VideoTransform implements "pinch to zoom". As part of "pinch to + // zoom", it implements onPress, of course. + if (zoomEnabled) { return ( - + { rtcView } - + ); } + // XXX Unfortunately, VideoTransform implements a custom press + // detection which has been observed to be very picky about the + // precision of the press unlike the builtin/default/standard press + // detection which is forgiving to imperceptible movements while + // pressing. It's not acceptable to be so picky, especially when + // "pinch to zoom" is not enabled. + return ( + + { rtcView } + + ); + // RTCView has peculiarities which may or may not be platform specific. // For example, it doesn't accept an empty streamURL. If the execution // reached here, it means that we explicitly chose to not initialize an From 8fa68b1462354f1924bdb0537b069b0c30ad2a9e Mon Sep 17 00:00:00 2001 From: Calin-Teodor Date: Thu, 24 Oct 2024 12:21:57 +0300 Subject: [PATCH 4/8] feat(mobile/picture-in-picture): updates --- package-lock.json | 12 +- package.json | 2 +- react/features/app/reducers.native.ts | 1 + .../media/components/AbstractVideoTrack.tsx | 8 +- .../media/components/native/FallbackView.tsx | 7 - .../base/media/components/native/Video.tsx | 173 +++++++----------- .../media/components/native/VideoTrack.tsx | 14 +- .../components/ParticipantView.native.tsx | 124 +++++-------- react/features/base/participants/functions.ts | 71 +++++-- .../mobile/picture-in-picture/actions.ts | 27 +-- .../components/PictureInPictureButton.ts | 2 +- .../prejoin/components/native/Prejoin.tsx | 2 +- 12 files changed, 209 insertions(+), 234 deletions(-) delete mode 100644 react/features/base/media/components/native/FallbackView.tsx diff --git a/package-lock.json b/package-lock.json index 4d20248d3efb..5d7be208477a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -100,7 +100,7 @@ "react-native-url-polyfill": "2.0.0", "react-native-video": "6.0.0-alpha.11", "react-native-watch-connectivity": "1.1.0", - "react-native-webrtc": "git+https://git@github.com/react-native-webrtc/react-native-webrtc.git#4288a1630d6dcfa141ff7c1c8c30c64c67c0dd0e", + "react-native-webrtc": "git+https://git@github.com/react-native-webrtc/react-native-webrtc.git#6953b21a33382478c9d56674e4d4bd35a30149f9", "react-native-webview": "13.8.7", "react-native-youtube-iframe": "2.3.0", "react-redux": "7.2.9", @@ -16063,8 +16063,8 @@ }, "node_modules/react-native-webrtc": { "version": "124.0.4", - "resolved": "git+https://git@github.com/react-native-webrtc/react-native-webrtc.git#4288a1630d6dcfa141ff7c1c8c30c64c67c0dd0e", - "integrity": "sha512-Z5G4+wSqLYNtdORjoDXkbFQKRW9AI7dvys9E8L642bAA+k1g9kZSL3FC9eWmsQKq/0gEgeYaFiJwUiENzqlnfw==", + "resolved": "git+https://git@github.com/react-native-webrtc/react-native-webrtc.git#6953b21a33382478c9d56674e4d4bd35a30149f9", + "integrity": "sha512-JMunPsk83dw+Vn1YaK8VuzBkJtE6WznjdIn4tj84DWqzj0h8L+JGBaYTR6QRQArNvL9Dh+0dwadmXPjuh3DtUw==", "license": "MIT", "dependencies": { "base64-js": "1.5.1", @@ -30643,9 +30643,9 @@ } }, "react-native-webrtc": { - "version": "git+https://git@github.com/react-native-webrtc/react-native-webrtc.git#4288a1630d6dcfa141ff7c1c8c30c64c67c0dd0e", - "integrity": "sha512-Z5G4+wSqLYNtdORjoDXkbFQKRW9AI7dvys9E8L642bAA+k1g9kZSL3FC9eWmsQKq/0gEgeYaFiJwUiENzqlnfw==", - "from": "react-native-webrtc@git+https://git@github.com/react-native-webrtc/react-native-webrtc.git#4288a1630d6dcfa141ff7c1c8c30c64c67c0dd0e", + "version": "git+https://git@github.com/react-native-webrtc/react-native-webrtc.git#6953b21a33382478c9d56674e4d4bd35a30149f9", + "integrity": "sha512-JMunPsk83dw+Vn1YaK8VuzBkJtE6WznjdIn4tj84DWqzj0h8L+JGBaYTR6QRQArNvL9Dh+0dwadmXPjuh3DtUw==", + "from": "react-native-webrtc@git+https://git@github.com/react-native-webrtc/react-native-webrtc.git#6953b21a33382478c9d56674e4d4bd35a30149f9", "requires": { "base64-js": "1.5.1", "debug": "4.3.4", diff --git a/package.json b/package.json index 42a75dfc5ac8..56b11448d8b3 100644 --- a/package.json +++ b/package.json @@ -106,7 +106,7 @@ "react-native-url-polyfill": "2.0.0", "react-native-video": "6.0.0-alpha.11", "react-native-watch-connectivity": "1.1.0", - "react-native-webrtc": "git+https://git@github.com/react-native-webrtc/react-native-webrtc.git#4288a1630d6dcfa141ff7c1c8c30c64c67c0dd0e", + "react-native-webrtc": "git+https://git@github.com/react-native-webrtc/react-native-webrtc.git#6953b21a33382478c9d56674e4d4bd35a30149f9", "react-native-webview": "13.8.7", "react-native-youtube-iframe": "2.3.0", "react-redux": "7.2.9", diff --git a/react/features/app/reducers.native.ts b/react/features/app/reducers.native.ts index 0ffc5800b794..a0a632cc4009 100644 --- a/react/features/app/reducers.native.ts +++ b/react/features/app/reducers.native.ts @@ -3,6 +3,7 @@ import '../mobile/background/reducer'; import '../mobile/call-integration/reducer'; import '../mobile/external-api/reducer'; import '../mobile/full-screen/reducer'; +import '../mobile/picture-in-picture/reducer'; import '../mobile/watchos/reducer'; import '../share-room/reducer'; diff --git a/react/features/base/media/components/AbstractVideoTrack.tsx b/react/features/base/media/components/AbstractVideoTrack.tsx index 03c405153901..9bc4a8b834a8 100644 --- a/react/features/base/media/components/AbstractVideoTrack.tsx +++ b/react/features/base/media/components/AbstractVideoTrack.tsx @@ -1,4 +1,4 @@ -import React, { Component } from 'react'; +import React, {Component, ReactElement} from 'react'; import { IStore } from '../../../app/types'; import { trackVideoStarted } from '../../tracks/actions'; @@ -16,6 +16,11 @@ export interface IProps { */ dispatch: IStore['dispatch']; + /** + * iOS component for PiP view. + */ + fallbackView?: ReactElement; + /** * Callback to invoke when the {@link Video} of {@code AbstractVideoTrack} * is clicked/pressed. @@ -111,6 +116,7 @@ export default class AbstractVideoTrack

extends Component

{ return (