Refactor containerisation (#964)

* Refactor containerisation 1 (#952) * Introduce Services and Containers on BitriseDataModel * Validate containers * Test containerization config * Fix test names * Log output of failing docker test commands * Update docker tests * Refactor docker test bitrise.ymls * Fix docker-service-start-succeeds-after-retries test * Update models/models_methods.go Co-authored-by: Olivér Falvai <[email protected]> * Strip whitespace before validating container images * Update containerization validation error messages --------- Co-authored-by: Olivér Falvai <[email protected]> * rebase --------- Co-authored-by: Olivér Falvai <[email protected]>
bitrise-io · Jun 5, 2024 · 75cb964 · 75cb964
1 parent 246c4bc
commit 75cb964
Show file tree

Hide file tree

Showing 15 changed files with 989 additions and 343 deletions.
diff --git a/_tests/integration/docker_create_bitrise.yml b/_tests/integration/docker_create_bitrise.yml
@@ -1,47 +1,60 @@
 format_version: 1.3.0
 default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git
+containers:
+  invalid-port:
+    image: frolvlad/alpine-bash:latest
+    ports:
+    - 22:22
+  valid-port:
+    image: frolvlad/alpine-bash:latest
+    ports:
+    - 12341:12341
+  unhealthy-container:
+    image: frolvlad/alpine-bash:latest
+    options: --health-cmd "redis-cli ping" --health-interval 1s --health-timeout 3s --health-retries 2
+  invalid-option:
+    image: frolvlad/alpine-bash:latest
+    options: --invalid-option "fail now!"
 workflows:
   docker-create-fails-invalid-port:
     title: Expected to fail on docker create, invalid port provided
-    container:
-      image: frolvlad/alpine-bash:latest
-      ports:
-      - 22:22
     steps:
-    - script:
-        title: Should not run due to prev error
-        inputs:
-        - content: exit 0
+    - with:
+        container: invalid-port
+        steps:
+        - script:
+            title: Should not run due to prev error
+            inputs:
+            - content: exit 0
   docker-create-succeeds-valid-port:
     title: Expected to pass on docker create, valid port provided
-    container:
-      image: frolvlad/alpine-bash:latest
-      ports:
-      - 12341:12341
     steps:
-    - script:
-        title: Should succeed
-        inputs:
-        - content: exit 0
+    - with:
+        container: valid-port
+        steps:
+        - script:
+            title: Should succeed
+            inputs:
+            - content: exit 0
   docker-create-succeeds-with-false-unhealthy-container:
     title: Expected to log error on docker create
     description: Expected to log error on docker create, because healthchecks are wrong, however execution should continue
-    container:
-      image: frolvlad/alpine-bash:latest
-      options: --health-cmd "redis-cli ping" --health-interval 1s --health-timeout 3s --health-retries 2
     steps:
-    - script:
-        title: Should succceed
-        inputs:
-        - content: exit 0
+    - with:
+        container: unhealthy-container
+        steps:
+        - script:
+            title: Should succceed
+            inputs:
+            - content: exit 0
   docker-create-fails-invalid-option:
     title: Expected to log error on docker create
     description: Expected to log error on docker create, because healthcheck are wrong, however execution should continue
-    container:
-      image: frolvlad/alpine-bash:latest
-      options: --invalid-option "fail now!"
     steps:
-    - script:
-        title: Should fail
-        inputs:
-        - content: exit 0
+    - with:
+        container: invalid-option
+        steps:
+        - script:
+            title: Should fail
+            inputs:
+            - content: exit 0
diff --git a/_tests/integration/docker_multiple_containers_bitrise.yml b/_tests/integration/docker_multiple_containers_bitrise.yml
@@ -0,0 +1,108 @@
+format_version: 1.3.0
+default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git
+containers:
+  step_execution_container:
+    image: localhost:5001/healthy-image
+    credentials:
+      username: $DOCKER_USR_STEP_EXECUTION_CONTAINER
+      password: $DOCKER_PW_STEP_EXECUTION_CONTAINER
+services:
+  service_1_container:
+    image: localhost:5002/healthy-image
+    credentials:
+      username: $DOCKER_USR_SERVICE_1_CONTAINER
+      password: $DOCKER_PW_SERVICE_1_CONTAINER
+    options: --health-cmd "stat /ready || exit 1" --health-interval 1s --health-timeout 3s --health-retries 3
+  service_2_container:
+    image: localhost:5003/healthy-image
+    credentials:
+      username: $DOCKER_USR_SERVICE_2_CONTAINER
+      password: $DOCKER_PW_SERVICE_2_CONTAINER
+    options: --health-cmd "stat /ready || exit 1" --health-interval 1s --health-timeout 3s --health-retries 3
+workflows:
+  docker-login-multiple-containers:
+    before_run:
+    - _start_mock_registry_for_step_execution_container
+    - _start_mock_registry_for_service_1_container
+    - _start_mock_registry_for_service_2_container
+    after_run:
+    - _cleanup_mock_registry_for_step_execution_container
+    - _cleanup_mock_registry_for_service_1_container
+    - _cleanup_mock_registry_for_service_2_container
+    title: Expected to pass docker login
+    steps:
+    - with:
+        container: step_execution_container
+        services:
+        - service_1_container
+        - service_2_container
+        steps:
+        - script:
+            title: Should pass
+            inputs:
+            - content: exit 0
+  _start_mock_registry_for_step_execution_container:
+    envs:
+    - PORT: 5001
+    - USR: $DOCKER_USR_STEP_EXECUTION_CONTAINER
+    - PASS: $DOCKER_PW_STEP_EXECUTION_CONTAINER
+    after_run:
+    - _start_mock_registry
+  _start_mock_registry_for_service_1_container:
+    envs:
+    - PORT: 5002
+    - USR: $DOCKER_USR_SERVICE_1_CONTAINER
+    - PASS: $DOCKER_PW_SERVICE_1_CONTAINER
+    after_run:
+    - _start_mock_registry
+  _start_mock_registry_for_service_2_container:
+    envs:
+    - PORT: 5003
+    - USR: $DOCKER_USR_SERVICE_2_CONTAINER
+    - PASS: $DOCKER_PW_SERVICE_2_CONTAINER
+    after_run:
+    - _start_mock_registry
+  _start_mock_registry:
+    steps:
+    - script:
+        title: setup mock registry for step execution container
+        inputs:
+        - content: |-
+            mkdir auth_$PORT
+            docker run --entrypoint htpasswd httpd:2 -Bbn $USR $PASS > auth_$PORT/htpasswd
+            docker pull --platform linux/amd64 registry:latest
+            docker run -d -p $PORT:5000 --restart always --name registry_$PORT \
+              -v "$(pwd)"/auth_$PORT:/auth_$PORT \
+              -e "REGISTRY_AUTH=htpasswd" \
+              -e "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" \
+              -e REGISTRY_AUTH_HTPASSWD_PATH=/auth_$PORT/htpasswd \
+              registry
+            docker login localhost:$PORT -u $USR -p $PASS
+            docker build -t healthy-image -f ${SRC_DIR_IN_GOPATH}/_tests/integration/docker_test.Dockerfile.healthy-container .
+            docker tag healthy-image localhost:$PORT/healthy-image
+            docker push localhost:$PORT/healthy-image
+            docker logout localhost:$PORT
+  _cleanup_mock_registry_for_step_execution_container:
+    envs:
+    - PORT: 5001
+    after_run:
+    - _cleanup_mock_registry
+  _cleanup_mock_registry_for_service_1_container:
+    envs:
+    - PORT: 5002
+    after_run:
+    - _cleanup_mock_registry
+  _cleanup_mock_registry_for_service_2_container:
+    envs:
+    - PORT: 5003
+    after_run:
+    - _cleanup_mock_registry
+  _cleanup_mock_registry:
+    steps:
+    - script:
+        is_always_run: true
+        title: cleanup mock registry
+        inputs:
+        - content: |-
+            docker stop registry_$PORT
+            docker rm registry_$PORT
diff --git a/_tests/integration/docker_multiple_containers_secrets.yml b/_tests/integration/docker_multiple_containers_secrets.yml
@@ -0,0 +1,7 @@
+envs:
+- DOCKER_USR_STEP_EXECUTION_CONTAINER: test_usr_step_execution_container
+- DOCKER_PW_STEP_EXECUTION_CONTAINER: test_pwd_step_execution_container
+- DOCKER_USR_SERVICE_1_CONTAINER: test_usr_service_1_container
+- DOCKER_PW_SERVICE_1_CONTAINER: test_pwd_service_1_container
+- DOCKER_USR_SERVICE_2_CONTAINER: test_usr_service_2_container
+- DOCKER_PW_SERVICE_2_CONTAINER: test_pwd_service_2_container
diff --git a/_tests/integration/docker_pull_bitrise.yml b/_tests/integration/docker_pull_bitrise.yml
@@ -1,52 +1,65 @@
 format_version: 1.3.0
 default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git
+containers:
+  success:
+    image: frolvlad/alpine-bash:latest
+  fails-404:
+    image: localhost.hu/noimage:3cb48a46a66e
+  login-fail:
+    image: us-central1-docker.pkg.dev/ip-kubernetes-dev/sandbox/ruby:zstd
+    credentials:
+      username: _json_key_base64
+      password: bad pw
+  login-success:
+    image: localhost:5001/frolvlad/alpine-bash:latest
+    credentials:
+      username: test
+      password: $DOCKER_PW
 workflows:
   docker-pull-success:
     title: Expected to pass docker pull
-    container:
-      image: frolvlad/alpine-bash:latest
     steps:
-    - script:
-        title: Should pass
-        inputs:
-        - content: exit 0
+    - with:
+        container: success
+        steps:
+        - script:
+            title: Should pass
+            inputs:
+            - content: exit 0
   docker-pull-fails-404:
     title: Expected to fail docker pull
-    container:
-      image: localhost.hu/noimage:3cb48a46a66e
     steps:
-    - script:
-        title: Should fail
-        inputs:
-        - content: exit 0
+    - with:
+        container: fails-404
+        steps:
+        - script:
+            title: Should fail
+            inputs:
+            - content: exit 0
   docker-login-fail:
     title: Expected to fail on docker login
-    container:
-      image: us-central1-docker.pkg.dev/ip-kubernetes-dev/sandbox/ruby:zstd
-      credentials:
-        username: _json_key_base64
-        password: bad pw
     steps:
-    - script:
-        title: Should fail
-        inputs:
-        - content: exit 0
+    - with:
+        container: login-fail
+        steps:
+        - script:
+            title: Should fail
+            inputs:
+            - content: exit 0
   docker-login-success:
     before_run:
     - _start_mock_registry
     after_run:
     - _cleanup_mock_registry
     title: Expected to pass docker login
-    container:
-      image: localhost:5001/frolvlad/alpine-bash:latest
-      credentials:
-        username: test
-        password: $DOCKER_PW
     steps:
-    - script:
-        title: Should pass
-        inputs:
-        - content: exit 0
+    - with:
+        container: login-success
+        steps:
+        - script:
+            title: Should pass
+            inputs:
+            - content: exit 0
   _start_mock_registry:
     steps:
     - script:

diff --git a/_tests/integration/docker_service_bitrise.yml b/_tests/integration/docker_service_bitrise.yml
@@ -1,30 +1,36 @@
 format_version: 1.3.0
 default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git
+services:
+  failing-service:
+    image: test-failing-image
+  slow-booting-service:
+    image: test-slow-booting-image
+    options: --health-cmd "stat /ready || exit 1" --health-interval 1s --health-timeout 3s --health-retries 16
 workflows:
   docker-service-start-fails:
     before_run:
     - _build-failing-image
-    services:
-      failing-service:
-        image: test-failing-image
     steps:
-    - script:
-        title: Should succeed, but services related errors are logged
-        inputs:
-        - content: exit 0
+    - with:
+        services:
+        - failing-service
+        steps:
+        - script:
+            title: Should succeed, but services related errors are logged
+            inputs:
+            - content: exit 0
   docker-service-start-succeeds-after-retries:
     before_run:
     - _build-slow-starting-image
-    services:
-      slow-bootin-service:
-        image: test-slow-booting-image
-        options: --health-cmd "stat /ready || exit 1" --health-interval 1s --health-timeout 3s --health-retries 16
     steps:
-    - script:
-        title: Should succeed, but services related errors are logged
-        inputs:
-        - content: exit 0
-
+    - with:
+        services:
+        - slow-booting-service
+        steps:
+        - script:
+            title: Should succeed, but services related errors are logged
+            inputs:
+            - content: exit 0
   _build-failing-image:
     steps:
     - script:

diff --git a/_tests/integration/docker_start_fails_bitrise.yml b/_tests/integration/docker_start_fails_bitrise.yml
@@ -1,17 +1,21 @@
 format_version: 1.3.0
 default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git
+containers:
+  failing-image:
+    image: test-failing-image:latest
 workflows:
   docker-start-fails:
     before_run:
     - _build-failing-image
     title: Expected to fail on docker start, failing image is used
-    container:
-      image: test-failing-image:latest
     steps:
-    - script:
-        title: Should not run due to prev error
-        inputs:
-        - content: exit 0
+    - with:
+        container: failing-image
+        steps:
+        - script:
+            title: Should not run due to prev error
+            inputs:
+            - content: exit 0
   _build-failing-image:
     steps:
     - script:

diff --git a/_tests/integration/docker_test.Dockerfile.healthy-container b/_tests/integration/docker_test.Dockerfile.healthy-container
@@ -0,0 +1,2 @@
+FROM frolvlad/alpine-bash:latest
+CMD ["bash", "-c", "sleep 3; touch /ready; echo 'Im healthy now'; sleep infinity"]
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		FROM frolvlad/alpine-bash:latest
		CMD ["bash", "-c", "sleep 3; touch /ready; echo 'Im healthy now'; sleep infinity"]