Switch back to local Minio in development

The Minio testing service play.min.io is down for almost a day blocking the CI, so switching back to local deployment in development.
redhat-appstudio · Jan 9, 2025 · 910ba80 · 910ba80
1 parent 6f59f7b
commit 910ba80
Show file tree

Hide file tree

Showing 4 changed files with 250 additions and 15 deletions.
diff --git a/components/pipeline-service/development/dev-only-pipeline-service-storage-configuration.yaml b/components/pipeline-service/development/dev-only-pipeline-service-storage-configuration.yaml
@@ -1,3 +1,49 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+  name: openshift-minio-apply-tenants
+rules:
+- apiGroups:
+  - minio.min.io
+  resources:
+  - tenants
+  verbs:
+  - get
+  - create
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - apps
+  resources:
+  - deployments
+  verbs:
+  - '*'
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  - services
+  verbs:
+  - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+  name: openshift-minio-apply-tenants
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: openshift-minio-apply-tenants
+subjects:
+- kind: ServiceAccount
+  name: openshift-gitops-argocd-application-controller
+  namespace: openshift-gitops
+---
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
@@ -70,3 +116,66 @@ spec:
     syncOptions:
     - CreateNamespace=false
     - Validate=false
+---
+apiVersion: minio.min.io/v2
+kind: Tenant
+metadata:
+  annotations:
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+    argocd.argoproj.io/sync-wave: "0"
+    prometheus.io/path: /minio/v2/metrics/cluster
+    prometheus.io/port: "9000"
+    prometheus.io/scrape: "true"
+  labels:
+    app: minio
+  name: storage
+  namespace: tekton-results
+spec:
+  certConfig: {}
+  configuration:
+    name: minio-storage-configuration
+  env: []
+  exposeServices:
+    minio: false
+  externalCaCertSecret: []
+  externalCertSecret: []
+  externalClientCertSecrets: []
+  features:
+    bucketDNS: false
+    domains: {}
+  image: quay.io/minio/minio:RELEASE.2024-08-26T15-33-07Z
+  imagePullSecret: {}
+  mountPath: /export
+  podManagementPolicy: Parallel
+  pools:
+  - affinity:
+      nodeAffinity: {}
+      podAffinity: {}
+      podAntiAffinity: {}
+    containerSecurityContext: {}
+    name: pool-0
+    nodeSelector: {}
+    resources: {}
+    securityContext: {}
+    servers: 1
+    tolerations: []
+    volumeClaimTemplate:
+      apiVersion: v1
+      kind: persistentvolumeclaims
+      metadata: {}
+      spec:
+        accessModes:
+        - ReadWriteOnce
+        resources:
+          requests:
+            storage: 1Gi
+      status: {}
+    volumesPerServer: 2
+  priorityClassName: ""
+  requestAutoCert: true
+  serviceMetadata:
+    consoleServiceAnnotations: {}
+    consoleServiceLabels: {}
+    minioServiceAnnotations: {}
+    minioServiceLabels: {}
+  subPath: ""
diff --git a/components/pipeline-service/development/kustomization.yaml b/components/pipeline-service/development/kustomization.yaml
@@ -8,6 +8,55 @@ commonAnnotations:
   argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
 
 resources:
+  - github.com/minio/operator?ref=v5.0.15
   - main-pipeline-service-configuration.yaml
   - dev-only-pipeline-service-storage-configuration.yaml
   - ../base/rbac
+
+patches:
+  - target:
+      kind: Service
+      name: operator
+      namespace: minio-operator
+    patch: |
+      - op: add
+        path: /metadata/annotations/ignore-check.kube-linter.io~1dangling-service
+        value: This service is not dangling. The minio operator assigns the missing labels at runtime.
+  - target:
+      kind: Deployment
+      name: minio-operator
+      namespace: minio-operator
+    patch: |
+      - op: add
+        path: /metadata/annotations/ignore-check.kube-linter.io~1no-read-only-root-fs
+        value: The operator needs to be able to write to /tmp
+      - op: add
+        path: /spec/template/spec/containers/0/resources/limits
+        value:
+          cpu: 200m
+          memory: 256Mi
+      - op: remove
+        path: /spec/template/spec/containers/0/securityContext/runAsUser
+      - op: remove
+        path: /spec/template/spec/containers/0/securityContext/runAsGroup
+  - target:
+      kind: Deployment
+      name: console
+      namespace: minio-operator
+    patch: |
+      - op: add
+        path: /spec/template/spec/containers/0/resources
+        value:
+          requests:
+            cpu: 50m
+            memory: 64Mi
+          limits:
+            cpu: 100m
+            memory: 128Mi
+      - op: remove
+        path: /spec/template/spec/containers/0/securityContext/runAsUser
+      - op: remove
+        path: /spec/template/spec/containers/0/securityContext/runAsGroup
+      - op: add
+        path: /spec/template/spec/containers/0/securityContext/readOnlyRootFilesystem
+        value: true
diff --git a/components/pipeline-service/development/main-pipeline-service-configuration.yaml b/components/pipeline-service/development/main-pipeline-service-configuration.yaml
@@ -1101,6 +1101,9 @@ spec:
           initialDelaySeconds: 5
           periodSeconds: 10
         volumeMounts:
+        - mountPath: /etc/ssl/certs/s3-cert.crt
+          name: ca-s3
+          subPath: s3-cert.crt
         - mountPath: /etc/tls/db
           name: db-tls-ca
           readOnly: true
@@ -1137,8 +1140,70 @@ spec:
           runAsNonRoot: true
           seccompProfile:
             type: RuntimeDefault
+      initContainers:
+      - args:
+        - -c
+        - |
+          mc --config-dir /tmp config host add minio "$S3_ENDPOINT" "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" &&
+          if [ -z "$(mc --config-dir /tmp ls minio | grep "$S3_BUCKET_NAME")" ]; then
+            mc --config-dir /tmp mb --with-lock --region "$S3_REGION" minio/"$S3_BUCKET_NAME" &&
+            echo "Minio bucket $S3_BUCKET_NAME successfully created."
+          fi
+        command:
+        - /bin/bash
+        env:
+        - name: S3_ACCESS_KEY_ID
+          valueFrom:
+            secretKeyRef:
+              key: aws_access_key_id
+              name: tekton-results-s3
+        - name: S3_SECRET_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              key: aws_secret_access_key
+              name: tekton-results-s3
+        - name: S3_REGION
+          valueFrom:
+            secretKeyRef:
+              key: aws_region
+              name: tekton-results-s3
+        - name: S3_BUCKET_NAME
+          valueFrom:
+            secretKeyRef:
+              key: bucket
+              name: tekton-results-s3
+        - name: S3_ENDPOINT
+          valueFrom:
+            secretKeyRef:
+              key: endpoint
+              name: tekton-results-s3
+        image: quay.io/minio/mc:RELEASE.2023-01-28T20-29-38Z
+        imagePullPolicy: Always
+        name: mc
+        resources:
+          limits:
+            cpu: 100m
+            memory: 128Mi
+          requests:
+            cpu: 5m
+            memory: 32Mi
+        securityContext:
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+        volumeMounts:
+        - mountPath: /etc/ssl/certs/s3-cert.crt
+          name: ca-s3
+          subPath: s3-cert.crt
+        - mountPath: /tmp
+          name: tmp-mc-volume
       serviceAccountName: tekton-results-api
       volumes:
+      - name: ca-s3
+        secret:
+          items:
+          - key: public.crt
+            path: s3-cert.crt
+          secretName: storage-tls
       - emptyDir: {}
         name: tmp-mc-volume
       - configMap:

diff --git a/hack/secret-creator/create-plnsvc-secrets.sh b/hack/secret-creator/create-plnsvc-secrets.sh
@@ -8,16 +8,6 @@ main() {
     create_s3_secret tekton-results tekton-results-s3
     create_s3_secret tekton-logging tekton-results-s3
     create_db_cert_secret_and_configmap
-    if ! [ -x "$(command -v mc)" ]; then
-        curl https://dl.min.io/client/mc/release/linux-amd64/mc \
-          --create-dirs \
-        -o $HOME/minio-binaries/mc && chmod +x $HOME/minio-binaries/mc
-        export PATH=$PATH:$HOME/minio-binaries/
-
-    fi
-
-    mc alias set myPlayMinio https://play.min.io:9000  Q3AM3UQ867SPQQA43P2F zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
-    mc mb myPlayMinio/tekton-logs || true
 }
 
 create_namespace() {
@@ -47,12 +37,34 @@ create_s3_secret() {
         echo "S3 secret already exists, skipping creation"
         return
     fi
+    USER=minio
+    PASS="$(openssl rand -base64 20)"
     kubectl create secret generic -n $1 $2 \
-      --from-literal=aws_access_key_id="Q3AM3UQ867SPQQA43P2F" \
-      --from-literal=aws_secret_access_key="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" \
-      --from-literal=aws_region='us-east-1' \
-      --from-literal=bucket=tekton-logs \
-      --from-literal=endpoint='https://play.min.io:9000'
+      --from-literal=aws_access_key_id="$USER" \
+      --from-literal=aws_secret_access_key="$PASS" \
+      --from-literal=aws_region='not-applicable' \
+      --from-literal=bucket=tekton-results \
+      --from-literal=endpoint='https://minio.tekton-results.svc.cluster.local'
+
+    echo "Creating MinIO config" >&2
+    if kubectl get secret -n tekton-results minio-storage-configuration &>/dev/null; then
+        echo "MinIO config already exists, skipping creation"
+        return
+    fi
+    cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: Secret
+metadata:
+  name: minio-storage-configuration
+  namespace: tekton-results
+type: Opaque
+stringData:
+  config.env: |-
+    export MINIO_ROOT_USER="$USER"
+    export MINIO_ROOT_PASSWORD="$PASS"
+    export MINIO_STORAGE_CLASS_STANDARD="EC:1"
+    export MINIO_BROWSER="on"
+EOF
 }
 
 create_db_cert_secret_and_configmap() {