From 6747c59c29d2ca43752706449a03a5de78a5b528 Mon Sep 17 00:00:00 2001 From: Craig O'Donnell Date: Tue, 5 Mar 2024 10:46:46 -0500 Subject: [PATCH 1/7] support deploying without minio --- templates/kotsadm-config.yaml | 1 + templates/kotsadm-deployment.yaml | 2 + templates/kotsadm-statefulset.yaml | 302 +++++++++++++++++++++++++++++ templates/minio-service.yaml | 2 +- templates/minio-statefulset.yaml | 2 +- values.yaml.tmpl | 1 + 6 files changed, 308 insertions(+), 2 deletions(-) create mode 100644 templates/kotsadm-statefulset.yaml diff --git a/templates/kotsadm-config.yaml b/templates/kotsadm-config.yaml index 9575acc..b3b6fd2 100644 --- a/templates/kotsadm-config.yaml +++ b/templates/kotsadm-config.yaml @@ -9,6 +9,7 @@ data: minio-enabled-snapshots: "false" registry-is-read-only: "false" skip-preflights: {{ .Values.automation.skipPreflights | quote }} + with-minio: {{ .Values.withMinio | quote }} {{- if .Values.automation.appVersionLabel }} app-version-label: {{ .Values.automation.appVersionLabel | quote }} {{- end }} diff --git a/templates/kotsadm-deployment.yaml b/templates/kotsadm-deployment.yaml index e13c1ac..813f20a 100644 --- a/templates/kotsadm-deployment.yaml +++ b/templates/kotsadm-deployment.yaml @@ -1,3 +1,4 @@ +{{ if .Values.withMinio }} apiVersion: apps/v1 kind: Deployment metadata: @@ -250,3 +251,4 @@ spec: - emptyDir: {} name: tmp status: {} +{{ end }} diff --git a/templates/kotsadm-statefulset.yaml b/templates/kotsadm-statefulset.yaml new file mode 100644 index 0000000..fbd397a --- /dev/null +++ b/templates/kotsadm-statefulset.yaml @@ -0,0 +1,302 @@ +{{ if not .Values.withMinio }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + {{- include "admin-console.labels" . | nindent 4 }} + name: kotsadm +spec: + selector: + matchLabels: + app: kotsadm + serviceName: kotsadm + template: + metadata: + annotations: + backup.velero.io/backup-volumes: backup + pre.hook.backup.velero.io/command: '["/backup.sh"]' + pre.hook.backup.velero.io/timeout: 10m + labels: + app: kotsadm + {{- include "admin-console.labels" . | nindent 8 }} + spec: + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - env: + - name: SHARED_PASSWORD_BCRYPT + valueFrom: + secretKeyRef: + key: passwordBcrypt + name: kotsadm-password + - name: AUTO_CREATE_CLUSTER_TOKEN + valueFrom: + secretKeyRef: + key: kotsadm-cluster-token + name: kotsadm-cluster-token + - name: SESSION_KEY + valueFrom: + secretKeyRef: + key: key + name: kotsadm-session +{{ if not .Values.isHelmManaged }} + - name: RQLITE_PASSWORD + valueFrom: + secretKeyRef: + key: password + name: kotsadm-rqlite + - name: RQLITE_URI + valueFrom: + secretKeyRef: + key: uri + name: kotsadm-rqlite +{{ end }} + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_OWNER_KIND + value: deployment + - name: API_ENCRYPTION_KEY + valueFrom: + secretKeyRef: + key: encryptionKey + name: kotsadm-encryption + - name: API_ENDPOINT + value: http://kotsadm.{{ .Release.Namespace }}.svc.cluster.local:3000 + - name: API_ADVERTISE_ENDPOINT + value: http://localhost:8800 +{{ if .Values.embeddedClusterID }} + - name: EMBEDDED_CLUSTER_ID + value: {{ .Values.embeddedClusterID | quote }} +{{ end }} +{{ if .Values.embeddedClusterVersion }} + - name: EMBEDDED_CLUSTER_VERSION + value: {{ .Values.embeddedClusterVersion | quote }} +{{ end }} + - name: HTTP_PROXY + - name: HTTPS_PROXY + - name: NO_PROXY + value: kotsadm-rqlite,kotsadm-api-node + - name: IS_HELM_MANAGED + value: {{ .Values.isHelmManaged | quote }} + image: {{ .Values.images.kotsadm }} + imagePullPolicy: IfNotPresent + name: kotsadm + ports: + - containerPort: 3000 + name: http + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 3000 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - mountPath: /kotsadmdata + name: kotsadmdata + - mountPath: /backup + name: backup + - mountPath: /tmp + name: tmp +{{ if not .Values.isHelmManaged }} + initContainers: + - args: + - plan + env: + - name: SCHEMAHERO_DRIVER + value: rqlite + - name: SCHEMAHERO_SPEC_FILE + value: /tables + - name: SCHEMAHERO_OUT + value: /migrations/plan.yaml + - name: SCHEMAHERO_URI + valueFrom: + secretKeyRef: + key: uri + name: kotsadm-rqlite + image: {{ .Values.images.migrations }} + imagePullPolicy: IfNotPresent + name: schemahero-plan + resources: + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 50m + memory: 50Mi + volumeMounts: + - mountPath: /migrations + name: migrations + - args: + - apply + env: + - name: SCHEMAHERO_DRIVER + value: rqlite + - name: SCHEMAHERO_DDL + value: /migrations/plan.yaml + - name: SCHEMAHERO_URI + valueFrom: + secretKeyRef: + key: uri + name: kotsadm-rqlite + image: {{ .Values.images.migrations }} + imagePullPolicy: IfNotPresent + name: schemahero-apply + resources: + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 50m + memory: 50Mi + volumeMounts: + - mountPath: /migrations + name: migrations + - command: + - /restore.sh + env: + - name: RQLITE_PASSWORD + valueFrom: + secretKeyRef: + key: password + name: kotsadm-rqlite + image: {{ .Values.images.kotsadm }} + imagePullPolicy: IfNotPresent + name: restore-data + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - mountPath: /kotsadmdata + name: kotsadmdata + - mountPath: /backup + name: backup + - mountPath: /tmp + name: tmp +{{ end }} + restartPolicy: Always + securityContext: + fsGroup: 1001 + runAsUser: 1001 + serviceAccountName: kotsadm + volumes: + - persistentVolumeClaim: + claimName: kotsadmdata + name: kotsadmdata + - emptyDir: + medium: Memory + name: migrations + - emptyDir: {} + name: backup + - emptyDir: {} + name: tmp +{{ $kotsminio := lookup "apps/v1" "StatefulSet" .Release.Namespace "kotsadm-minio" }} +{{ if not $kotsminio }} +{{ $existingpvc := lookup "v1" "PersistentVolumeClaim" .Release.Namespace "kotsadmdata" }} +{{ if not $existingpvc }} +# if the migration occurred, the pvc will already exist +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: kotsadmdata + labels: + {{- include "admin-console.immutableLabels" . | nindent 4 }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 4Gi +{{ end }} +{{ else }} +# if minio exists, we need to migrate the data to a pvc +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: kotsadm-migrate-s3 + annotations: + helm.sh/hook: pre-upgrade + helm.sh/hook-weight: "1" + labels: + {{- include "admin-console.labels" . | nindent 4 }} +spec: + template: + metadata: + labels: + {{- include "admin-console.labels" . | nindent 8 }} + spec: + containers: + - command: + - /migrate-s3.sh + env: + - name: S3_ENDPOINT + value: http://kotsadm-minio:9000 + - name: S3_BUCKET_NAME + value: kotsadm + - name: S3_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: accesskey + name: kotsadm-minio + - name: S3_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: secretkey + name: kotsadm-minio + - name: S3_BUCKET_ENDPOINT + value: "true" + image: {{ .Values.images.kotsadm }} + imagePullPolicy: IfNotPresent + name: migrate-s3 + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - mountPath: /kotsadmdata + name: kotsadmdata + restartPolicy: OnFailure + volumes: + - persistentVolumeClaim: + claimName: kotsadmdata + name: kotsadmdata +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: kotsadmdata + annotations: + helm.sh/hook: pre-upgrade + helm.sh/hook-weight: "0" + labels: + {{- include "admin-console.immutableLabels" . | nindent 4 }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 4Gi +{{ end }} +{{ end }} + diff --git a/templates/minio-service.yaml b/templates/minio-service.yaml index 9ddab3d..287e0b6 100644 --- a/templates/minio-service.yaml +++ b/templates/minio-service.yaml @@ -1,4 +1,4 @@ -{{ if not .Values.isHelmManaged }} +{{ if and (not .Values.isHelmManaged) (.Values.withMinio) }} apiVersion: v1 kind: Service metadata: diff --git a/templates/minio-statefulset.yaml b/templates/minio-statefulset.yaml index 767b2c0..82c9fb4 100644 --- a/templates/minio-statefulset.yaml +++ b/templates/minio-statefulset.yaml @@ -1,4 +1,4 @@ -{{ if not .Values.isHelmManaged }} +{{ if and (not .Values.isHelmManaged) (.Values.withMinio) }} apiVersion: apps/v1 kind: StatefulSet metadata: diff --git a/values.yaml.tmpl b/values.yaml.tmpl index ca90f5c..baee242 100644 --- a/values.yaml.tmpl +++ b/values.yaml.tmpl @@ -11,6 +11,7 @@ images: password: "" minimalRBAC: true isHelmManaged: true +withMinio: true embeddedClusterID: "" embeddedClusterVersion: "" automation: From 85f46013c8febfeeaf308bdadd6bec3554888fc8 Mon Sep 17 00:00:00 2001 From: Craig O'Donnell Date: Tue, 5 Mar 2024 13:01:28 -0500 Subject: [PATCH 2/7] scale down kotsadm before migration --- templates/kotsadm-statefulset.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/templates/kotsadm-statefulset.yaml b/templates/kotsadm-statefulset.yaml index fbd397a..34b8620 100644 --- a/templates/kotsadm-statefulset.yaml +++ b/templates/kotsadm-statefulset.yaml @@ -243,6 +243,20 @@ spec: labels: {{- include "admin-console.labels" . | nindent 8 }} spec: + initContainers: + - command: + - /bin/sh + - -c + - | + kubectl scale deploy/kotsadm -n {{ .Release.Namespace }} --replicas=0 + kubectl wait --for delete pod --selector=app=kotsadm -n {{ .Release.Namespace }} --timeout=300s + image: {{ .Values.images.kotsadm }} + imagePullPolicy: IfNotPresent + name: scale-down-kotsadm + resources: + requests: + cpu: 100m + memory: 100Mi containers: - command: - /migrate-s3.sh @@ -277,6 +291,7 @@ spec: - mountPath: /kotsadmdata name: kotsadmdata restartPolicy: OnFailure + serviceAccountName: kotsadm volumes: - persistentVolumeClaim: claimName: kotsadmdata From b14256cd915d444ef2c885c980aa990c402a193f Mon Sep 17 00:00:00 2001 From: Craig O'Donnell Date: Tue, 5 Mar 2024 13:27:15 -0500 Subject: [PATCH 3/7] don't create kotsadm-s3 secret if withMinio: false --- templates/secret-s3.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/secret-s3.yaml b/templates/secret-s3.yaml index cbc6233..f182431 100644 --- a/templates/secret-s3.yaml +++ b/templates/secret-s3.yaml @@ -1,4 +1,4 @@ -{{ if not .Values.isHelmManaged }} +{{ if and (not .Values.isHelmManaged) (.Values.withMinio) }} {{- $accesskey := uuidv4 | b64enc | quote }} {{- $secretkey := uuidv4 | b64enc | quote }} {{- $secret := (lookup "v1" "Secret" .Release.Namespace "kotsadm-minio") }} From 3c6fb719812b15e9a1646665f6a766363e83ea14 Mon Sep 17 00:00:00 2001 From: Craig O'Donnell Date: Tue, 5 Mar 2024 14:30:25 -0500 Subject: [PATCH 4/7] remove POD_OWNER_KIND --- templates/kotsadm-statefulset.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/templates/kotsadm-statefulset.yaml b/templates/kotsadm-statefulset.yaml index 34b8620..7b0d008 100644 --- a/templates/kotsadm-statefulset.yaml +++ b/templates/kotsadm-statefulset.yaml @@ -57,8 +57,6 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - - name: POD_OWNER_KIND - value: deployment - name: API_ENCRYPTION_KEY valueFrom: secretKeyRef: From 6040bdfd4ecd7f00e0741702f15ad0cbf60fe5a9 Mon Sep 17 00:00:00 2001 From: Craig O'Donnell Date: Wed, 6 Mar 2024 11:21:24 -0500 Subject: [PATCH 5/7] split statefulset and migrate-s3 hook --- templates/kotsadm-statefulset.yaml | 91 +----------------------------- templates/migrate-s3-hook.yaml | 89 +++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 90 deletions(-) create mode 100644 templates/migrate-s3-hook.yaml diff --git a/templates/kotsadm-statefulset.yaml b/templates/kotsadm-statefulset.yaml index 7b0d008..6fbee64 100644 --- a/templates/kotsadm-statefulset.yaml +++ b/templates/kotsadm-statefulset.yaml @@ -205,9 +205,8 @@ spec: - emptyDir: {} name: tmp {{ $kotsminio := lookup "apps/v1" "StatefulSet" .Release.Namespace "kotsadm-minio" }} -{{ if not $kotsminio }} {{ $existingpvc := lookup "v1" "PersistentVolumeClaim" .Release.Namespace "kotsadmdata" }} -{{ if not $existingpvc }} +{{ if and (not $kotsminio) (not $existingpvc) }} # if the migration occurred, the pvc will already exist --- apiVersion: v1 @@ -223,93 +222,5 @@ spec: requests: storage: 4Gi {{ end }} -{{ else }} -# if minio exists, we need to migrate the data to a pvc ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: kotsadm-migrate-s3 - annotations: - helm.sh/hook: pre-upgrade - helm.sh/hook-weight: "1" - labels: - {{- include "admin-console.labels" . | nindent 4 }} -spec: - template: - metadata: - labels: - {{- include "admin-console.labels" . | nindent 8 }} - spec: - initContainers: - - command: - - /bin/sh - - -c - - | - kubectl scale deploy/kotsadm -n {{ .Release.Namespace }} --replicas=0 - kubectl wait --for delete pod --selector=app=kotsadm -n {{ .Release.Namespace }} --timeout=300s - image: {{ .Values.images.kotsadm }} - imagePullPolicy: IfNotPresent - name: scale-down-kotsadm - resources: - requests: - cpu: 100m - memory: 100Mi - containers: - - command: - - /migrate-s3.sh - env: - - name: S3_ENDPOINT - value: http://kotsadm-minio:9000 - - name: S3_BUCKET_NAME - value: kotsadm - - name: S3_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - key: accesskey - name: kotsadm-minio - - name: S3_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - key: secretkey - name: kotsadm-minio - - name: S3_BUCKET_ENDPOINT - value: "true" - image: {{ .Values.images.kotsadm }} - imagePullPolicy: IfNotPresent - name: migrate-s3 - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: 100m - memory: 100Mi - volumeMounts: - - mountPath: /kotsadmdata - name: kotsadmdata - restartPolicy: OnFailure - serviceAccountName: kotsadm - volumes: - - persistentVolumeClaim: - claimName: kotsadmdata - name: kotsadmdata ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: kotsadmdata - annotations: - helm.sh/hook: pre-upgrade - helm.sh/hook-weight: "0" - labels: - {{- include "admin-console.immutableLabels" . | nindent 4 }} -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 4Gi -{{ end }} {{ end }} diff --git a/templates/migrate-s3-hook.yaml b/templates/migrate-s3-hook.yaml new file mode 100644 index 0000000..c63ace5 --- /dev/null +++ b/templates/migrate-s3-hook.yaml @@ -0,0 +1,89 @@ + +{{ $kotsminio := lookup "apps/v1" "StatefulSet" .Release.Namespace "kotsadm-minio" }} +{{ if and (not .Values.withMinio) ($kotsminio) }} +# if minio exists, we need to migrate the data to a PVC +apiVersion: batch/v1 +kind: Job +metadata: + name: kotsadm-migrate-s3 + annotations: + helm.sh/hook: pre-upgrade + helm.sh/hook-weight: "1" + labels: + {{- include "admin-console.labels" . | nindent 4 }} +spec: + template: + metadata: + labels: + {{- include "admin-console.labels" . | nindent 8 }} + spec: + initContainers: + - command: + - /bin/sh + - -c + - | + kubectl scale deploy/kotsadm -n {{ .Release.Namespace }} --replicas=0 + kubectl wait --for delete pod --selector=app=kotsadm -n {{ .Release.Namespace }} --timeout=300s + image: {{ .Values.images.kotsadm }} + imagePullPolicy: IfNotPresent + name: scale-down-kotsadm + resources: + requests: + cpu: 100m + memory: 100Mi + containers: + - command: + - /migrate-s3.sh + env: + - name: S3_ENDPOINT + value: http://kotsadm-minio:9000 + - name: S3_BUCKET_NAME + value: kotsadm + - name: S3_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: accesskey + name: kotsadm-minio + - name: S3_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: secretkey + name: kotsadm-minio + - name: S3_BUCKET_ENDPOINT + value: "true" + image: {{ .Values.images.kotsadm }} + imagePullPolicy: IfNotPresent + name: migrate-s3 + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - mountPath: /kotsadmdata + name: kotsadmdata + restartPolicy: OnFailure + serviceAccountName: kotsadm + volumes: + - persistentVolumeClaim: + claimName: kotsadmdata + name: kotsadmdata +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: kotsadmdata + annotations: + helm.sh/hook: pre-upgrade + helm.sh/hook-weight: "0" + labels: + {{- include "admin-console.immutableLabels" . | nindent 4 }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 4Gi +{{ end }} From 2e41fef9bee3e03fadf3472a44bc640be26439f1 Mon Sep 17 00:00:00 2001 From: Craig O'Donnell Date: Wed, 6 Mar 2024 13:12:53 -0500 Subject: [PATCH 6/7] remove minio --- templates/kotsadm-config.yaml | 2 +- templates/kotsadm-deployment.yaml | 254 ----------------------------- templates/kotsadm-statefulset.yaml | 2 - templates/migrate-s3-hook.yaml | 2 +- templates/minio-service.yaml | 18 -- templates/minio-statefulset.yaml | 116 ------------- templates/secret-s3.yaml | 20 --- values.yaml.tmpl | 2 - 8 files changed, 2 insertions(+), 414 deletions(-) delete mode 100644 templates/kotsadm-deployment.yaml delete mode 100644 templates/minio-service.yaml delete mode 100644 templates/minio-statefulset.yaml delete mode 100644 templates/secret-s3.yaml diff --git a/templates/kotsadm-config.yaml b/templates/kotsadm-config.yaml index b3b6fd2..94c75d5 100644 --- a/templates/kotsadm-config.yaml +++ b/templates/kotsadm-config.yaml @@ -9,7 +9,7 @@ data: minio-enabled-snapshots: "false" registry-is-read-only: "false" skip-preflights: {{ .Values.automation.skipPreflights | quote }} - with-minio: {{ .Values.withMinio | quote }} + with-minio: "false" {{- if .Values.automation.appVersionLabel }} app-version-label: {{ .Values.automation.appVersionLabel | quote }} {{- end }} diff --git a/templates/kotsadm-deployment.yaml b/templates/kotsadm-deployment.yaml deleted file mode 100644 index 813f20a..0000000 --- a/templates/kotsadm-deployment.yaml +++ /dev/null @@ -1,254 +0,0 @@ -{{ if .Values.withMinio }} -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - {{- include "admin-console.labels" . | nindent 4 }} - name: kotsadm -spec: - selector: - matchLabels: - app: kotsadm - strategy: {} - template: - metadata: - annotations: - backup.velero.io/backup-volumes: backup - pre.hook.backup.velero.io/command: '["/backup.sh"]' - pre.hook.backup.velero.io/timeout: 10m - creationTimestamp: null - labels: - app: kotsadm - {{- include "admin-console.labels" . | nindent 8 }} - spec: - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - env: - - name: SHARED_PASSWORD_BCRYPT - valueFrom: - secretKeyRef: - key: passwordBcrypt - name: kotsadm-password - - name: AUTO_CREATE_CLUSTER_TOKEN - valueFrom: - secretKeyRef: - key: kotsadm-cluster-token - name: kotsadm-cluster-token - - name: SESSION_KEY - valueFrom: - secretKeyRef: - key: key - name: kotsadm-session -{{ if not .Values.isHelmManaged }} - - name: RQLITE_PASSWORD - valueFrom: - secretKeyRef: - key: password - name: kotsadm-rqlite - - name: RQLITE_URI - valueFrom: - secretKeyRef: - key: uri - name: kotsadm-rqlite -{{ end }} - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_OWNER_KIND - value: deployment - - name: API_ENCRYPTION_KEY - valueFrom: - secretKeyRef: - key: encryptionKey - name: kotsadm-encryption - - name: API_ENDPOINT - value: http://kotsadm.{{ .Release.Namespace }}.svc.cluster.local:3000 - - name: API_ADVERTISE_ENDPOINT - value: http://localhost:8800 -{{ if not .Values.isHelmManaged }} - - name: S3_ENDPOINT - value: http://kotsadm-minio:9000 - - name: S3_BUCKET_NAME - value: kotsadm - - name: S3_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - key: accesskey - name: kotsadm-minio - - name: S3_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - key: secretkey - name: kotsadm-minio - - name: S3_BUCKET_ENDPOINT - value: "true" -{{ end }} -{{ if .Values.embeddedClusterID }} - - name: EMBEDDED_CLUSTER_ID - value: {{ .Values.embeddedClusterID | quote }} -{{ end }} -{{ if .Values.embeddedClusterVersion }} - - name: EMBEDDED_CLUSTER_VERSION - value: {{ .Values.embeddedClusterVersion | quote }} -{{ end }} - - name: HTTP_PROXY - - name: HTTPS_PROXY - - name: NO_PROXY - value: kotsadm-rqlite,kotsadm-minio,kotsadm-api-node - - name: IS_HELM_MANAGED - value: {{ .Values.isHelmManaged | quote }} - image: {{ .Values.images.kotsadm }} - imagePullPolicy: IfNotPresent - name: kotsadm - ports: - - containerPort: 3000 - name: http - readinessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: 3000 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: 100m - memory: 100Mi - volumeMounts: - - mountPath: /backup - name: backup - - mountPath: /tmp - name: tmp -{{ if not .Values.isHelmManaged }} - initContainers: - - args: - - plan - env: - - name: SCHEMAHERO_DRIVER - value: rqlite - - name: SCHEMAHERO_SPEC_FILE - value: /tables - - name: SCHEMAHERO_OUT - value: /migrations/plan.yaml - - name: SCHEMAHERO_URI - valueFrom: - secretKeyRef: - key: uri - name: kotsadm-rqlite - image: {{ .Values.images.migrations }} - imagePullPolicy: IfNotPresent - name: schemahero-plan - resources: - limits: - cpu: 100m - memory: 100Mi - requests: - cpu: 50m - memory: 50Mi - volumeMounts: - - mountPath: /migrations - name: migrations - - args: - - apply - env: - - name: SCHEMAHERO_DRIVER - value: rqlite - - name: SCHEMAHERO_DDL - value: /migrations/plan.yaml - - name: SCHEMAHERO_URI - valueFrom: - secretKeyRef: - key: uri - name: kotsadm-rqlite - image: {{ .Values.images.migrations }} - imagePullPolicy: IfNotPresent - name: schemahero-apply - resources: - limits: - cpu: 100m - memory: 100Mi - requests: - cpu: 50m - memory: 50Mi - volumeMounts: - - mountPath: /migrations - name: migrations - - command: - - /restore-db.sh - env: - - name: RQLITE_PASSWORD - valueFrom: - secretKeyRef: - key: password - name: kotsadm-rqlite - image: {{ .Values.images.kotsadm }} - imagePullPolicy: IfNotPresent - name: restore-db - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: 100m - memory: 100Mi - volumeMounts: - - mountPath: /backup - name: backup - - mountPath: /tmp - name: tmp - - command: - - /restore-s3.sh - env: - - name: S3_ENDPOINT - value: http://kotsadm-minio:9000 - - name: S3_BUCKET_NAME - value: kotsadm - - name: S3_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - key: accesskey - name: kotsadm-minio - - name: S3_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - key: secretkey - name: kotsadm-minio - - name: S3_BUCKET_ENDPOINT - value: "true" - image: {{ .Values.images.kotsadm }} - imagePullPolicy: IfNotPresent - name: restore-s3 - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: 100m - memory: 100Mi - volumeMounts: - - mountPath: /backup - name: backup -{{ end }} - restartPolicy: Always - securityContext: - fsGroup: 1001 - runAsUser: 1001 - serviceAccountName: kotsadm - volumes: - - emptyDir: - medium: Memory - name: migrations - - emptyDir: {} - name: backup - - emptyDir: {} - name: tmp -status: {} -{{ end }} diff --git a/templates/kotsadm-statefulset.yaml b/templates/kotsadm-statefulset.yaml index 6fbee64..8769d7f 100644 --- a/templates/kotsadm-statefulset.yaml +++ b/templates/kotsadm-statefulset.yaml @@ -1,4 +1,3 @@ -{{ if not .Values.withMinio }} apiVersion: apps/v1 kind: StatefulSet metadata: @@ -222,5 +221,4 @@ spec: requests: storage: 4Gi {{ end }} -{{ end }} diff --git a/templates/migrate-s3-hook.yaml b/templates/migrate-s3-hook.yaml index c63ace5..d6a485b 100644 --- a/templates/migrate-s3-hook.yaml +++ b/templates/migrate-s3-hook.yaml @@ -1,6 +1,6 @@ {{ $kotsminio := lookup "apps/v1" "StatefulSet" .Release.Namespace "kotsadm-minio" }} -{{ if and (not .Values.withMinio) ($kotsminio) }} +{{ if $kotsminio }} # if minio exists, we need to migrate the data to a PVC apiVersion: batch/v1 kind: Job diff --git a/templates/minio-service.yaml b/templates/minio-service.yaml deleted file mode 100644 index 287e0b6..0000000 --- a/templates/minio-service.yaml +++ /dev/null @@ -1,18 +0,0 @@ -{{ if and (not .Values.isHelmManaged) (.Values.withMinio) }} -apiVersion: v1 -kind: Service -metadata: - labels: - {{- include "admin-console.labels" . | nindent 4 }} - name: kotsadm-minio -spec: - ports: - - name: service - port: 9000 - targetPort: 9000 - selector: - app: kotsadm-minio - type: ClusterIP -status: - loadBalancer: {} -{{ end }} \ No newline at end of file diff --git a/templates/minio-statefulset.yaml b/templates/minio-statefulset.yaml deleted file mode 100644 index 82c9fb4..0000000 --- a/templates/minio-statefulset.yaml +++ /dev/null @@ -1,116 +0,0 @@ -{{ if and (not .Values.isHelmManaged) (.Values.withMinio) }} -apiVersion: apps/v1 -kind: StatefulSet -metadata: - labels: - {{- include "admin-console.labels" . | nindent 4 }} - name: kotsadm-minio -spec: - selector: - matchLabels: - app: kotsadm-minio - serviceName: "" - template: - metadata: - annotations: - backup.velero.io/backup-volumes: kotsadm-minio,minio-config-dir,minio-cert-dir - labels: - app: kotsadm-minio - {{- include "admin-console.labels" . | nindent 8 }} - spec: - containers: - - command: - - /bin/sh - - -ce - - minio -C /home/minio/.minio/ --quiet server /export - env: - - name: MINIO_ACCESS_KEY - valueFrom: - secretKeyRef: - key: accesskey - name: kotsadm-minio - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - key: secretkey - name: kotsadm-minio - - name: MINIO_BROWSER - value: "on" - - name: MINIO_UPDATE - value: "off" - image: {{ .Values.images.minio }} - imagePullPolicy: IfNotPresent - livenessProbe: - failureThreshold: 3 - httpGet: - path: /minio/health/live - port: service - scheme: HTTP - initialDelaySeconds: 5 - periodSeconds: 30 - successThreshold: 1 - timeoutSeconds: 1 - name: kotsadm-minio - ports: - - containerPort: 9000 - name: service - readinessProbe: - failureThreshold: 3 - httpGet: - path: /minio/health/ready - port: service - scheme: HTTP - initialDelaySeconds: 5 - periodSeconds: 15 - successThreshold: 1 - timeoutSeconds: 1 - resources: - limits: - cpu: 100m - memory: 200Mi - requests: - cpu: 50m - memory: 100Mi - securityContext: - allowPrivilegeEscalation: false - privileged: false - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /export - name: kotsadm-minio - - mountPath: /home/minio/.minio/ - name: minio-config-dir - - mountPath: /.minio/ - name: minio-cert-dir - securityContext: - fsGroup: 1001 - runAsGroup: 1001 - runAsNonRoot: true - runAsUser: 1001 - seccompProfile: - type: RuntimeDefault - supplementalGroups: - - 1001 - volumes: - - name: kotsadm-minio - persistentVolumeClaim: - claimName: kotsadm-minio - - emptyDir: {} - name: minio-config-dir - - emptyDir: {} - name: minio-cert-dir - updateStrategy: - type: RollingUpdate - volumeClaimTemplates: - - metadata: - labels: - {{- include "admin-console.immutableLabels" . | nindent 8 }} - name: kotsadm-minio - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 4Gi - status: {} -{{ end }} diff --git a/templates/secret-s3.yaml b/templates/secret-s3.yaml deleted file mode 100644 index f182431..0000000 --- a/templates/secret-s3.yaml +++ /dev/null @@ -1,20 +0,0 @@ -{{ if and (not .Values.isHelmManaged) (.Values.withMinio) }} -{{- $accesskey := uuidv4 | b64enc | quote }} -{{- $secretkey := uuidv4 | b64enc | quote }} -{{- $secret := (lookup "v1" "Secret" .Release.Namespace "kotsadm-minio") }} -{{- if $secret }} -{{- $accesskey = index $secret.data "accesskey" }} -{{- $secretkey = index $secret.data "secretkey" }} -{{- end -}} -apiVersion: v1 -kind: Secret -metadata: - labels: - {{- include "admin-console.labels" . | nindent 4 }} - name: kotsadm-minio - annotations: - "helm.sh/resource-policy": "keep" -data: - accesskey: {{ $accesskey }} - secretkey: {{ $secretkey }} -{{ end }} \ No newline at end of file diff --git a/values.yaml.tmpl b/values.yaml.tmpl index baee242..8aba86c 100644 --- a/values.yaml.tmpl +++ b/values.yaml.tmpl @@ -5,13 +5,11 @@ images: kotsadm: ${KOTSADM_REGISTRY}/kotsadm:${KOTS_TAG} migrations: ${KOTSADM_REGISTRY}/kotsadm-migrations:${KOTS_TAG} - minio: ${KOTSADM_REGISTRY}/minio:${MINIO_TAG} rqlite: ${KOTSADM_REGISTRY}/rqlite:${RQLITE_TAG} kurlProxy: ${KOTSADM_REGISTRY}/kurl-proxy:${KOTS_TAG} password: "" minimalRBAC: true isHelmManaged: true -withMinio: true embeddedClusterID: "" embeddedClusterVersion: "" automation: From f83dc671619834463026f4e3d89b793ab5f66971 Mon Sep 17 00:00:00 2001 From: Craig O'Donnell Date: Wed, 6 Mar 2024 14:07:33 -0500 Subject: [PATCH 7/7] use statefulset naming convention for migration pvc --- templates/kotsadm-statefulset.yaml | 27 +++++++++------------------ templates/migrate-s3-hook.yaml | 4 ++-- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/templates/kotsadm-statefulset.yaml b/templates/kotsadm-statefulset.yaml index 8769d7f..661d3fc 100644 --- a/templates/kotsadm-statefulset.yaml +++ b/templates/kotsadm-statefulset.yaml @@ -203,22 +203,13 @@ spec: name: backup - emptyDir: {} name: tmp -{{ $kotsminio := lookup "apps/v1" "StatefulSet" .Release.Namespace "kotsadm-minio" }} -{{ $existingpvc := lookup "v1" "PersistentVolumeClaim" .Release.Namespace "kotsadmdata" }} -{{ if and (not $kotsminio) (not $existingpvc) }} -# if the migration occurred, the pvc will already exist ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: kotsadmdata - labels: - {{- include "admin-console.immutableLabels" . | nindent 4 }} -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 4Gi -{{ end }} + volumeClaimTemplates: + - metadata: + name: kotsadmdata + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 4Gi diff --git a/templates/migrate-s3-hook.yaml b/templates/migrate-s3-hook.yaml index d6a485b..8fb3cc4 100644 --- a/templates/migrate-s3-hook.yaml +++ b/templates/migrate-s3-hook.yaml @@ -68,13 +68,13 @@ spec: serviceAccountName: kotsadm volumes: - persistentVolumeClaim: - claimName: kotsadmdata + claimName: kotsadmdata-kotsadm-0 name: kotsadmdata --- apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: kotsadmdata + name: kotsadmdata-kotsadm-0 annotations: helm.sh/hook: pre-upgrade helm.sh/hook-weight: "0"