Add dedicated Mattermost job server (#388)

This change adds new configuration for deploying Mattermost with
a dedicated job server.

apis/mattermost/v1beta1/mattermost_types.go

@@ -128,6 +128,10 @@ type MattermostSpec struct {
 	// +optional
 	UpdateJob *UpdateJob `json:"updateJob,omitempty"`
 
+	// JobServer defines configuration for the Mattermost job server.
+	// +optional
+	JobServer *JobServer `json:"jobServer,omitempty"`
+
 	// PodExtensions specify custom extensions for Mattermost pods.
 	// This can be used for custom readiness checks etc.
 	// These settings generally don't need to be changed.
@@ -284,6 +288,16 @@ type UpdateJob struct {
 	ExtraLabels map[string]string `json:"extraLabels,omitempty"`
 }
 
+// JobServer defines configuration for the Mattermost job server.
+type JobServer struct {
+	// Determines whether to create a dedicated Mattermost server deployment
+	// which is configured to run scheduled jobs. This deployment will recieve
+	// no user traffic and the primary Mattermost deployment will no longer be
+	// configured to run jobs.
+	// +optional
+	DedicatedJobServer bool `json:"dedicatedJobServer,omitempty"`
+}
+
 // PodExtensions specify customized extensions for a pod.
 type PodExtensions struct {
 	// Additional InitContainers injected into pods.

apis/mattermost/v1beta1/mattermost_utils.go

@@ -45,6 +45,10 @@ const (
 	// MattermostAppContainerName is the name of the container which runs the
 	// Mattermost application
 	MattermostAppContainerName = "mattermost"
+
+	// MattermostJobServerContainerName is the name of the container which runs
+	// an optional dedicated Mattermost job server.
+	MattermostJobServerContainerName = MattermostAppContainerName + "-jobserver"
 )
 
 // SetDefaults sets the missing values in the manifest to the default ones
@@ -272,9 +276,39 @@ func (mm *Mattermost) MattermostPodLabels(name string) map[string]string {
 	return l
 }
 
+// MattermostJobServerPodLabels returns the labels for selecting
+// the pods belonging to the given mattermost dedicated job server.
+func (mm *Mattermost) MattermostJobServerPodLabels(name string) map[string]string {
+	l := map[string]string{}
+	// Set resourceLabels ("global") as the initial labels
+	if mm.Spec.ResourceLabels != nil {
+		l = mm.Spec.ResourceLabels
+	}
+	if mm.Spec.PodTemplate != nil {
+		// Overwrite with pod specific labels
+		for k, v := range mm.Spec.PodTemplate.ExtraLabels {
+			l[k] = v
+		}
+	}
+	// Overwrite with default labels
+	for k, v := range MattermostResourceLabels(name) {
+		l[k] = v
+	}
+	l[ClusterLabel] = name
+	l["app"] = MattermostJobServerContainerName
+
+	return l
+}
+
 // MattermostResourceLabels returns the labels for selecting a given
 // Mattermost as well as any external dependency resources that were
 // created for the installation.
 func MattermostResourceLabels(name string) map[string]string {
 	return map[string]string{ClusterResourceLabel: name}
 }
+
+// DedicatedJobServerName returns the standard name for the optional
+// dedicated Mattermost job server.
+func (mm *Mattermost) DedicatedJobServerName() string {
+	return mm.Name + "-jobserver"
+}

config/crd/bases/installation.mattermost.com_mattermosts.yaml

@@ -494,6 +494,18 @@ spec:
                   IngressName defines the host to be used when creating the ingress rules.
                   Deprecated: Use Spec.Ingress.Host instead.
                 type: string
+              jobServer:
+                description: JobServer defines configuration for the Mattermost job
+                  server.
+                properties:
+                  dedicatedJobServer:
+                    description: |-
+                      Determines whether to create a dedicated Mattermost server deployment
+                      which is configured to run scheduled jobs. This deployment will recieve
+                      no user traffic and the primary Mattermost deployment will no longer be
+                      configured to run jobs.
+                    type: boolean
+                type: object
               licenseSecret:
                 description: LicenseSecret is the name of the secret containing a
                   Mattermost license.

controllers/mattermost/mattermost/controller.go

@@ -177,6 +177,12 @@ func (r *MattermostReconciler) Reconcile(ctx context.Context, request ctrl.Reque
 		return ctrl.Result{RequeueAfter: resourcesReadyDelay}, nil
 	}
 
+	err = r.checkMattermostJobServer(mattermost, dbConfig, fileStoreConfig, &status, reqLogger)
+	if err != nil {
+		r.updateStatusReconcilingAndLogError(mattermost, status, reqLogger, err)
+		return reconcile.Result{}, err
+	}
+
 	status, err = r.checkMattermostHealth(mattermost, status, reqLogger)
 	if err != nil {
 		statusErr := r.updateStatus(mattermost, status, reqLogger)

controllers/mattermost/mattermost/health_check.go

@@ -87,8 +87,43 @@ func (r *MattermostReconciler) checkMattermostHealth(mattermost *mmv1beta.Matter
 		return status, fmt.Errorf("found %d pods, but wanted %d", podsStatus.Replicas, replicas)
 	}
 
+	if mattermost.Spec.JobServer != nil && mattermost.Spec.JobServer.DedicatedJobServer {
+		err = r.checkMattermostJobServerHealth(mattermost, logger)
+		if err != nil {
+			return status, errors.Wrap(err, "failed to check job server health")
+		}
+	}
+
 	// Everything checks out. The installation is stable.
 	status.State = mmv1beta.Stable
 
 	return status, nil
 }
+
+func (r *MattermostReconciler) checkMattermostJobServerHealth(mattermost *mmv1beta.Mattermost, logger logr.Logger) error {
+	labels := mattermost.MattermostJobServerPodLabels(mattermost.Name)
+	listOptions := []client.ListOption{
+		client.InNamespace(mattermost.Namespace),
+		client.MatchingLabels(labels),
+	}
+
+	healthChecker := healthcheck.NewHealthChecker(r.NonCachedAPIReader, listOptions, logger)
+
+	jobServerPodsStatus, err := healthChecker.CheckReplicaSetRollout(mattermost.DedicatedJobServerName(), mattermost.Namespace)
+	if err != nil {
+		return errors.Wrap(err, "job server pod rollout not yet started")
+	}
+
+	if jobServerPodsStatus.UpdatedReplicas == 0 {
+		return errors.New("mattermost job server pods not yet updated")
+	}
+
+	if jobServerPodsStatus.UpdatedReplicas != 1 {
+		return fmt.Errorf("found %d updated job server replicas, but wanted 1", jobServerPodsStatus.UpdatedReplicas)
+	}
+	if jobServerPodsStatus.Replicas != 1 {
+		return fmt.Errorf("found job server %d pods, but wanted 1", jobServerPodsStatus.Replicas)
+	}
+
+	return nil
+}

controllers/mattermost/mattermost/mattermost.go

@@ -241,6 +241,68 @@ func (r *MattermostReconciler) checkMattermostIngressClass(mattermost *mmv1beta.
 	return r.Resources.Update(current, desired, reqLogger)
 }
 
+func (r *MattermostReconciler) checkMattermostJobServer(
+	mattermost *mmv1beta.Mattermost,
+	dbInfo mattermostApp.DatabaseConfig,
+	fsConfig mattermostApp.FileStoreConfig,
+	status *mmv1beta.MattermostStatus,
+	reqLogger logr.Logger) error {
+	reqLogger = reqLogger.WithValues("Reconcile", "mattermost-jobserver")
+
+	return r.checkMattermostJobServerDeployment(mattermost, dbInfo, fsConfig, status, reqLogger)
+}
+
+func (r *MattermostReconciler) checkMattermostJobServerDeployment(
+	mattermost *mmv1beta.Mattermost,
+	dbConfig mattermostApp.DatabaseConfig,
+	fsConfig mattermostApp.FileStoreConfig,
+	status *mmv1beta.MattermostStatus,
+	reqLogger logr.Logger) error {
+	desired := mattermostApp.GenerateJobServerDeploymentV1Beta(
+		mattermost,
+		dbConfig,
+		fsConfig,
+		mattermost.Name,
+		mattermost.GetIngressHost(),
+		mattermost.Name,
+		mattermost.GetImageName(),
+	)
+
+	if mattermost.Spec.JobServer == nil || !mattermost.Spec.JobServer.DedicatedJobServer {
+		// Ensure any existing dedicated job server deployments are removed.
+		err := r.Resources.DeleteDeployment(types.NamespacedName{Namespace: desired.Namespace, Name: desired.Name}, reqLogger)
+		if err != nil {
+			return errors.Wrap(err, "failed to delete job server deployment")
+		}
+		return nil
+	}
+
+	patchedObj, applied, err := mattermost.Spec.ResourcePatch.ApplyToDeployment(desired)
+	if err != nil {
+		reqLogger.Error(err, "Failed to patch deployment", "patch", mattermost.Status.ResourcePatch)
+		status.SetDeploymentPatchStatus(false, errors.Wrap(err, "failed to apply patch to Deployment"))
+	} else if applied {
+		reqLogger.Info("Applied patch to deployment")
+		desired = patchedObj
+		status.SetDeploymentPatchStatus(true, nil)
+	} else {
+		status.ClearDeploymentPatchStatus()
+	}
+
+	err = r.Resources.CreateDeploymentIfNotExists(mattermost, desired, reqLogger)
+	if err != nil {
+		return errors.Wrap(err, "failed to create mattermost job server deployment")
+	}
+
+	current := &appsv1.Deployment{}
+	err = r.Client.Get(context.TODO(), types.NamespacedName{Name: desired.Name, Namespace: desired.Namespace}, current)
+	if err != nil {
+		return errors.Wrap(err, "failed to get mattermost job server deployment")
+	}
+
+	return r.Resources.Update(current, desired, reqLogger)
+}
+
 func (r *MattermostReconciler) checkMattermostDeployment(
 	mattermost *mmv1beta.Mattermost,
 	dbConfig mattermostApp.DatabaseConfig,

pkg/mattermost/mattermost_v1beta.go

@@ -1,6 +1,7 @@
 package mattermost
 
 import (
+	"fmt"
 	"strconv"
 
 	mmv1beta "github.com/mattermost/mattermost-operator/apis/mattermost/v1beta1"
@@ -316,6 +317,18 @@ func GenerateDeploymentV1Beta(mattermost *mmv1beta.Mattermost, db DatabaseConfig
 		Value: bodySize,
 	})
 
+	// Apply optional job server settings
+	if mattermost.Spec.JobServer != nil && mattermost.Spec.JobServer.DedicatedJobServer {
+		envVarGeneral = append(envVarGeneral, corev1.EnvVar{
+			Name:  "MM_JOBSETTINGS_RUNSCHEDULER",
+			Value: "false",
+		})
+		envVarGeneral = append(envVarGeneral, corev1.EnvVar{
+			Name:  "MM_JOBSETTINGS_RUNJOBS",
+			Value: "false",
+		})
+	}
+
 	// Prepare annotations
 	podAnnotations := map[string]string{}
 
@@ -429,6 +442,39 @@ func GenerateDeploymentV1Beta(mattermost *mmv1beta.Mattermost, db DatabaseConfig
 	}
 }
 
+// GenerateJobServerDeploymentV1Beta returns the deployment for Mattermost app dedicated job server.
+func GenerateJobServerDeploymentV1Beta(mattermost *mmv1beta.Mattermost, db DatabaseConfig, fileStore FileStoreConfig, deploymentName, ingressHost, serviceAccountName, containerImage string) *appsv1.Deployment {
+	deployment := GenerateDeploymentV1Beta(
+		mattermost,
+		db,
+		fileStore,
+		mattermost.Name,
+		mattermost.GetIngressHost(),
+		mattermost.Name,
+		mattermost.GetImageName(),
+	)
+
+	// Apply metadata overrides for dedicated job server configuration.
+	deployment.Name = fmt.Sprintf("%s-jobserver", deploymentName)
+	replicas := int32(1)
+	deployment.Spec.Replicas = &replicas
+	deployment.Spec.Template.ObjectMeta.Labels = mattermost.MattermostJobServerPodLabels(deploymentName)
+	deployment.Spec.Selector.MatchLabels = mattermost.MattermostJobServerPodLabels(deploymentName)
+
+	// Apply dedicated job server container configuration to existing
+	// Mattermost container configuration.
+	mattermostContainer := deployment.Spec.Template.Spec.Containers[0]
+	mattermostContainer.Name = mmv1beta.MattermostJobServerContainerName
+	mattermostContainer.Command = []string{"mattermost", "jobserver"}
+	mattermostContainer.Ports = nil
+	mattermostContainer.LivenessProbe = nil
+	mattermostContainer.ReadinessProbe = nil
+
+	deployment.Spec.Template.Spec.Containers = []corev1.Container{mattermostContainer}
+
+	return deployment
+}
+
 // GenerateSecretV1Beta returns the secret for Mattermost
 func GenerateSecretV1Beta(mattermost *mmv1beta.Mattermost, secretName string, labels map[string]string, values map[string][]byte) *corev1.Secret {
 	return &corev1.Secret{