Max jibri retries (#337)

* logic to limit how many times we'll retry a Jibri request

* clean up the retry code a bit to make it clearer

src/main/java/org/jitsi/jicofo/JitsiMeetGlobalConfig.java

@@ -76,6 +76,20 @@ public class JitsiMeetGlobalConfig
      */
     private static final int JIBRI_DEFAULT_PENDING_TIMEOUT = 90;
 
+    /**
+     * The name of the config property which specifies how many times
+     * we'll retry a given Jibri request before giving up.  Set to
+     * -1 to allow infinite retries.
+     */
+    public static final String NUM_JIBRI_RETRIES_PNAME
+            = "org.jitsi.jicofo.NUM_JIBRI_RETRIES";
+
+    /**
+     * The default value for {@link #NUM_JIBRI_RETRIES_PNAME}
+     */
+    private static final int DEFAULT_NUM_JIBRI_RETRIES = 5;
+
+
     /**
      * Flag indicates whether auto owner feature is active. First participant to
      * join the room will become conference owner. When the owner leaves the
@@ -90,6 +104,12 @@ public class JitsiMeetGlobalConfig
      */
     private int jibriPendingTimeout;
 
+    /**
+     * How many attempts we'll make to retry a given Jibri request if the Jibri
+     * fails.
+     */
+    private int numJibriRetries;
+
     /**
      * Maximal amount of sources per media that can be advertised by
      * conference participant.
@@ -189,6 +209,17 @@ private void init(ConfigurationService configService)
             logger.warn("Jibri PENDING timeouts are disabled");
         }
 
+        numJibriRetries = configService.getInt(NUM_JIBRI_RETRIES_PNAME, DEFAULT_NUM_JIBRI_RETRIES);
+        if (numJibriRetries >= 0)
+        {
+            logger.info("Will attempt a maximum of " + numJibriRetries +
+                    " Jibri retries after failure");
+        }
+        else
+        {
+            logger.info("Will retry Jibri requests infinitely (if a Jibri is available)");
+        }
+
         singleParticipantTimeout
             = configService.getLong(
                     SINGLE_PARTICIPANT_TIMEOUT_CONFIG_PNAME,
@@ -245,6 +276,16 @@ public int getJibriPendingTimeout()
         return jibriPendingTimeout;
     }
 
+    /**
+     * Tells how many retry attempts we'll make for a Jibri request when a Jibri fails
+     * @return the amount of retry attempts we'll make for a Jibri request when
+     * a Jibri fails
+     */
+    public int getNumJibriRetries()
+    {
+        return numJibriRetries;
+    }
+
     /**
      * Indicates whether auto owner feature is active. First participant to join
      * the room will become conference owner. When the owner leaves the room

src/main/java/org/jitsi/jicofo/recording/jibri/JibriRecorder.java

@@ -139,6 +139,7 @@ protected IQ handleStartRequest(JibriIq iq)
                     this,
                     conference.getRoomName(),
                     globalConfig.getJibriPendingTimeout(),
+                    globalConfig.getNumJibriRetries(),
                     connection,
                     scheduledExecutor,
                     jibriDetector,

src/main/java/org/jitsi/jicofo/recording/jibri/JibriSession.java

@@ -160,6 +160,16 @@ static private boolean isStartingStatus(JibriIq.Status status)
      */
     private final XmppConnection xmpp;
 
+    /**
+     * The maximum amount of retries we'll attempt
+     */
+    private final int maxNumRetries;
+
+    /**
+     * How many times we've retried this request to another Jibri
+     */
+    private int numRetries = 0;
+
     /**
      * Creates new {@link JibriSession} instance.
      * @param owner the session owner which will be notified about this session
@@ -189,6 +199,7 @@ static private boolean isStartingStatus(JibriIq.Status status)
             JibriSession.Owner owner,
             EntityBareJid roomName,
             long pendingTimeout,
+            int maxNumRetries,
             XmppConnection connection,
             ScheduledExecutorService scheduledExecutor,
             JibriDetector jibriDetector,
@@ -206,6 +217,7 @@ static private boolean isStartingStatus(JibriIq.Status status)
         this.scheduledExecutor
             = Objects.requireNonNull(scheduledExecutor, "scheduledExecutor");
         this.pendingTimeout = pendingTimeout;
+        this.maxNumRetries = maxNumRetries;
         this.isSIP = isSIP;
         this.jibriDetector = jibriDetector;
         this.sipAddress = sipAddress;
@@ -290,6 +302,7 @@ private void cleanupSession()
     {
         logger.info("Cleaning up current JibriSession");
         currentJibriJid = null;
+        numRetries = 0;
         try
         {
             jibriEventHandler.stop(FocusBundleActivator.bundleContext);
@@ -439,6 +452,26 @@ private void reschedulePendingTimeout()
         }
     }
 
+    /**
+     * Check whether or not we should retry the current request to another Jibri
+     * @return true if we should retry again, false otherwise
+     */
+    private boolean shouldRetryRequest()
+    {
+        return (maxNumRetries < 0 || numRetries < maxNumRetries);
+    }
+
+    /**
+     * Retry the current request with another Jibri (if one is available)
+     * @return true if we were able to find another Jibri to retry the request with,
+     * false otherwise
+     */
+    private boolean retryRequestWithAnotherJibri()
+    {
+        numRetries++;
+        return start();
+    }
+
     /**
      * Handle a Jibri status update (this could come from an IQ response, a new IQ from Jibri, an XMPP event, etc.).
      * This will handle:
@@ -474,19 +507,30 @@ private void handleJibriStatusUpdate(
         // Now, if there was a failure of any kind we'll try and find another Jibri to keep things going
         if (failureReason != null)
         {
-            // There was an error with the current Jibri, see if we can resume the session with another Jibri
-            logger.info("Jibri failed, trying to fall back to another Jibri");
-            if (!start())
+            // There was an error with the current Jibri, see if we should retry
+            if (shouldRetryRequest())
             {
-                logger.info("Failed to fall back to another Jibri, this session has now failed");
-                // Propagate up that the session has failed entirely.  We'll pass the original failure reason.
-                owner.onSessionStateChanged(this, newStatus, failureReason);
-                cleanupSession();
+                logger.info("Jibri failed, trying to fall back to another Jibri");
+                if (retryRequestWithAnotherJibri())
+                {
+                    // The fallback to another Jibri succeeded.
+                    logger.info("Successfully resumed session with another Jibri");
+                }
+                else
+                {
+                    logger.info("Failed to fall back to another Jibri, this session has now failed");
+                    // Propagate up that the session has failed entirely.  We'll pass the original failure reason.
+                    owner.onSessionStateChanged(this, newStatus, failureReason);
+                    cleanupSession();
+                }
             }
             else
             {
-                // The fallback to another Jibri succeeded.
-                logger.info("Successfully resumed session with another Jibri");
+                // The Jibri we tried failed and we've reached the maxmium amount of retries we've been
+                // configured to attempt, so we'll give up trying to handle this request.
+                logger.info("Jibri failed, but max amount of retries (" + maxNumRetries + ") reached, giving up");
+                owner.onSessionStateChanged(this, newStatus, failureReason);
+                cleanupSession();
             }
         }
         else if (Status.OFF.equals(newStatus))

src/main/java/org/jitsi/jicofo/recording/jibri/JibriSipGateway.java

@@ -132,6 +132,7 @@ protected IQ handleStartRequest(JibriIq iq)
                         this,
                         conference.getRoomName(),
                         globalConfig.getJibriPendingTimeout(),
+                        globalConfig.getNumJibriRetries(),
                         connection,
                         scheduledExecutor,
                         jibriDetector,