diff --git a/_tests/integration/docker_create_bitrise.yml b/_tests/integration/docker_create_bitrise.yml index 2b57ad4f5..ae001fc9a 100644 --- a/_tests/integration/docker_create_bitrise.yml +++ b/_tests/integration/docker_create_bitrise.yml @@ -1,47 +1,60 @@ format_version: 1.3.0 default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git +containers: + invalid-port: + image: frolvlad/alpine-bash:latest + ports: + - 22:22 + valid-port: + image: frolvlad/alpine-bash:latest + ports: + - 12341:12341 + unhealthy-container: + image: frolvlad/alpine-bash:latest + options: --health-cmd "redis-cli ping" --health-interval 1s --health-timeout 3s --health-retries 2 + invalid-option: + image: frolvlad/alpine-bash:latest + options: --invalid-option "fail now!" workflows: docker-create-fails-invalid-port: title: Expected to fail on docker create, invalid port provided - container: - image: frolvlad/alpine-bash:latest - ports: - - 22:22 steps: - - script: - title: Should not run due to prev error - inputs: - - content: exit 0 + - with: + container: invalid-port + steps: + - script: + title: Should not run due to prev error + inputs: + - content: exit 0 docker-create-succeeds-valid-port: title: Expected to pass on docker create, valid port provided - container: - image: frolvlad/alpine-bash:latest - ports: - - 12341:12341 steps: - - script: - title: Should succeed - inputs: - - content: exit 0 + - with: + container: valid-port + steps: + - script: + title: Should succeed + inputs: + - content: exit 0 docker-create-succeeds-with-false-unhealthy-container: title: Expected to log error on docker create description: Expected to log error on docker create, because healthchecks are wrong, however execution should continue - container: - image: frolvlad/alpine-bash:latest - options: --health-cmd "redis-cli ping" --health-interval 1s --health-timeout 3s --health-retries 2 steps: - - script: - title: Should succceed - inputs: - - content: exit 0 + - with: + container: unhealthy-container + steps: + - script: + title: Should succceed + inputs: + - content: exit 0 docker-create-fails-invalid-option: title: Expected to log error on docker create description: Expected to log error on docker create, because healthcheck are wrong, however execution should continue - container: - image: frolvlad/alpine-bash:latest - options: --invalid-option "fail now!" steps: - - script: - title: Should fail - inputs: - - content: exit 0 + - with: + container: invalid-option + steps: + - script: + title: Should fail + inputs: + - content: exit 0 diff --git a/_tests/integration/docker_multiple_containers_bitrise.yml b/_tests/integration/docker_multiple_containers_bitrise.yml new file mode 100644 index 000000000..386541f52 --- /dev/null +++ b/_tests/integration/docker_multiple_containers_bitrise.yml @@ -0,0 +1,108 @@ +format_version: 1.3.0 +default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git +containers: + step_execution_container: + image: localhost:5001/healthy-image + credentials: + username: $DOCKER_USR_STEP_EXECUTION_CONTAINER + password: $DOCKER_PW_STEP_EXECUTION_CONTAINER +services: + service_1_container: + image: localhost:5002/healthy-image + credentials: + username: $DOCKER_USR_SERVICE_1_CONTAINER + password: $DOCKER_PW_SERVICE_1_CONTAINER + options: --health-cmd "stat /ready || exit 1" --health-interval 1s --health-timeout 3s --health-retries 3 + service_2_container: + image: localhost:5003/healthy-image + credentials: + username: $DOCKER_USR_SERVICE_2_CONTAINER + password: $DOCKER_PW_SERVICE_2_CONTAINER + options: --health-cmd "stat /ready || exit 1" --health-interval 1s --health-timeout 3s --health-retries 3 +workflows: + docker-login-multiple-containers: + before_run: + - _start_mock_registry_for_step_execution_container + - _start_mock_registry_for_service_1_container + - _start_mock_registry_for_service_2_container + after_run: + - _cleanup_mock_registry_for_step_execution_container + - _cleanup_mock_registry_for_service_1_container + - _cleanup_mock_registry_for_service_2_container + title: Expected to pass docker login + steps: + - with: + container: step_execution_container + services: + - service_1_container + - service_2_container + steps: + - script: + title: Should pass + inputs: + - content: exit 0 + _start_mock_registry_for_step_execution_container: + envs: + - PORT: 5001 + - USR: $DOCKER_USR_STEP_EXECUTION_CONTAINER + - PASS: $DOCKER_PW_STEP_EXECUTION_CONTAINER + after_run: + - _start_mock_registry + _start_mock_registry_for_service_1_container: + envs: + - PORT: 5002 + - USR: $DOCKER_USR_SERVICE_1_CONTAINER + - PASS: $DOCKER_PW_SERVICE_1_CONTAINER + after_run: + - _start_mock_registry + _start_mock_registry_for_service_2_container: + envs: + - PORT: 5003 + - USR: $DOCKER_USR_SERVICE_2_CONTAINER + - PASS: $DOCKER_PW_SERVICE_2_CONTAINER + after_run: + - _start_mock_registry + _start_mock_registry: + steps: + - script: + title: setup mock registry for step execution container + inputs: + - content: |- + mkdir auth_$PORT + docker run --entrypoint htpasswd httpd:2 -Bbn $USR $PASS > auth_$PORT/htpasswd + docker pull --platform linux/amd64 registry:latest + docker run -d -p $PORT:5000 --restart always --name registry_$PORT \ + -v "$(pwd)"/auth_$PORT:/auth_$PORT \ + -e "REGISTRY_AUTH=htpasswd" \ + -e "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" \ + -e REGISTRY_AUTH_HTPASSWD_PATH=/auth_$PORT/htpasswd \ + registry + docker login localhost:$PORT -u $USR -p $PASS + docker build -t healthy-image -f ${SRC_DIR_IN_GOPATH}/_tests/integration/docker_test.Dockerfile.healthy-container . + docker tag healthy-image localhost:$PORT/healthy-image + docker push localhost:$PORT/healthy-image + docker logout localhost:$PORT + _cleanup_mock_registry_for_step_execution_container: + envs: + - PORT: 5001 + after_run: + - _cleanup_mock_registry + _cleanup_mock_registry_for_service_1_container: + envs: + - PORT: 5002 + after_run: + - _cleanup_mock_registry + _cleanup_mock_registry_for_service_2_container: + envs: + - PORT: 5003 + after_run: + - _cleanup_mock_registry + _cleanup_mock_registry: + steps: + - script: + is_always_run: true + title: cleanup mock registry + inputs: + - content: |- + docker stop registry_$PORT + docker rm registry_$PORT diff --git a/_tests/integration/docker_multiple_containers_secrets.yml b/_tests/integration/docker_multiple_containers_secrets.yml new file mode 100644 index 000000000..da4125fb8 --- /dev/null +++ b/_tests/integration/docker_multiple_containers_secrets.yml @@ -0,0 +1,7 @@ +envs: +- DOCKER_USR_STEP_EXECUTION_CONTAINER: test_usr_step_execution_container +- DOCKER_PW_STEP_EXECUTION_CONTAINER: test_pwd_step_execution_container +- DOCKER_USR_SERVICE_1_CONTAINER: test_usr_service_1_container +- DOCKER_PW_SERVICE_1_CONTAINER: test_pwd_service_1_container +- DOCKER_USR_SERVICE_2_CONTAINER: test_usr_service_2_container +- DOCKER_PW_SERVICE_2_CONTAINER: test_pwd_service_2_container diff --git a/_tests/integration/docker_pull_bitrise.yml b/_tests/integration/docker_pull_bitrise.yml index 11a5e7d87..e853c989f 100644 --- a/_tests/integration/docker_pull_bitrise.yml +++ b/_tests/integration/docker_pull_bitrise.yml @@ -1,52 +1,65 @@ format_version: 1.3.0 default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git +containers: + success: + image: frolvlad/alpine-bash:latest + fails-404: + image: localhost.hu/noimage:3cb48a46a66e + login-fail: + image: us-central1-docker.pkg.dev/ip-kubernetes-dev/sandbox/ruby:zstd + credentials: + username: _json_key_base64 + password: bad pw + login-success: + image: localhost:5001/frolvlad/alpine-bash:latest + credentials: + username: test + password: $DOCKER_PW workflows: docker-pull-success: title: Expected to pass docker pull - container: - image: frolvlad/alpine-bash:latest steps: - - script: - title: Should pass - inputs: - - content: exit 0 + - with: + container: success + steps: + - script: + title: Should pass + inputs: + - content: exit 0 docker-pull-fails-404: title: Expected to fail docker pull - container: - image: localhost.hu/noimage:3cb48a46a66e steps: - - script: - title: Should fail - inputs: - - content: exit 0 + - with: + container: fails-404 + steps: + - script: + title: Should fail + inputs: + - content: exit 0 docker-login-fail: title: Expected to fail on docker login - container: - image: us-central1-docker.pkg.dev/ip-kubernetes-dev/sandbox/ruby:zstd - credentials: - username: _json_key_base64 - password: bad pw steps: - - script: - title: Should fail - inputs: - - content: exit 0 + - with: + container: login-fail + steps: + - script: + title: Should fail + inputs: + - content: exit 0 docker-login-success: before_run: - _start_mock_registry after_run: - _cleanup_mock_registry title: Expected to pass docker login - container: - image: localhost:5001/frolvlad/alpine-bash:latest - credentials: - username: test - password: $DOCKER_PW steps: - - script: - title: Should pass - inputs: - - content: exit 0 + - with: + container: login-success + steps: + - script: + title: Should pass + inputs: + - content: exit 0 _start_mock_registry: steps: - script: diff --git a/_tests/integration/docker_service_bitrise.yml b/_tests/integration/docker_service_bitrise.yml index 2fabcfb29..d9d1c1288 100644 --- a/_tests/integration/docker_service_bitrise.yml +++ b/_tests/integration/docker_service_bitrise.yml @@ -1,30 +1,36 @@ format_version: 1.3.0 default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git +services: + failing-service: + image: test-failing-image + slow-booting-service: + image: test-slow-booting-image + options: --health-cmd "stat /ready || exit 1" --health-interval 1s --health-timeout 3s --health-retries 16 workflows: docker-service-start-fails: before_run: - _build-failing-image - services: - failing-service: - image: test-failing-image steps: - - script: - title: Should succeed, but services related errors are logged - inputs: - - content: exit 0 + - with: + services: + - failing-service + steps: + - script: + title: Should succeed, but services related errors are logged + inputs: + - content: exit 0 docker-service-start-succeeds-after-retries: before_run: - _build-slow-starting-image - services: - slow-bootin-service: - image: test-slow-booting-image - options: --health-cmd "stat /ready || exit 1" --health-interval 1s --health-timeout 3s --health-retries 16 steps: - - script: - title: Should succeed, but services related errors are logged - inputs: - - content: exit 0 - + - with: + services: + - slow-booting-service + steps: + - script: + title: Should succeed, but services related errors are logged + inputs: + - content: exit 0 _build-failing-image: steps: - script: diff --git a/_tests/integration/docker_start_fails_bitrise.yml b/_tests/integration/docker_start_fails_bitrise.yml index 9236f0155..3c2c22d16 100644 --- a/_tests/integration/docker_start_fails_bitrise.yml +++ b/_tests/integration/docker_start_fails_bitrise.yml @@ -1,17 +1,21 @@ format_version: 1.3.0 default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git +containers: + failing-image: + image: test-failing-image:latest workflows: docker-start-fails: before_run: - _build-failing-image title: Expected to fail on docker start, failing image is used - container: - image: test-failing-image:latest steps: - - script: - title: Should not run due to prev error - inputs: - - content: exit 0 + - with: + container: failing-image + steps: + - script: + title: Should not run due to prev error + inputs: + - content: exit 0 _build-failing-image: steps: - script: diff --git a/_tests/integration/docker_test.Dockerfile.healthy-container b/_tests/integration/docker_test.Dockerfile.healthy-container new file mode 100644 index 000000000..5beb706c3 --- /dev/null +++ b/_tests/integration/docker_test.Dockerfile.healthy-container @@ -0,0 +1,2 @@ +FROM frolvlad/alpine-bash:latest +CMD ["bash", "-c", "sleep 3; touch /ready; echo 'Im healthy now'; sleep infinity"] diff --git a/_tests/integration/docker_test.go b/_tests/integration/docker_test.go index d071f78e4..5917d43c5 100644 --- a/_tests/integration/docker_test.go +++ b/_tests/integration/docker_test.go @@ -7,16 +7,18 @@ import ( "testing" "github.com/bitrise-io/go-utils/command" + "github.com/ryanuber/go-glob" "github.com/stretchr/testify/require" ) func Test_Docker(t *testing.T) { testCases := map[string]struct { - configPath string - inventoryPath string - workflowName string - requireErr bool - requireLogs []string + configPath string + inventoryPath string + workflowName string + requireErr bool + requireLogs []string + requiredLogPatterns []string }{ "docker pull succeeds with existing image": { configPath: "docker_pull_bitrise.yml", @@ -38,7 +40,7 @@ func Test_Docker(t *testing.T) { workflowName: "docker-login-fail", requireErr: true, requireLogs: []string{ - "workflow has docker credentials provided, but the authentication failed", + "docker credentials provided, but the authentication failed", }, }, "docker login succeeds when correct credentials are provided": { @@ -74,9 +76,11 @@ func Test_Docker(t *testing.T) { workflowName: "docker-create-succeeds-with-false-unhealthy-container", requireErr: false, requireLogs: []string{ - "Container (bitrise-workflow-docker-create-succeeds-with-false-unhealthy-container) is unhealthy...", "Step is running in container: frolvlad/alpine-bash:latest", }, + requiredLogPatterns: []string{ + "*Container (bitrise-workflow-*) is unhealthy...*", + }, }, "docker create fails when invalid option is provided": { configPath: "docker_create_bitrise.yml", @@ -110,7 +114,18 @@ func Test_Docker(t *testing.T) { workflowName: "docker-service-start-succeeds-after-retries", requireErr: false, requireLogs: []string{ - "Waiting for container (slow-bootin-service) to be healthy", + "Waiting for container (slow-booting-service) to be healthy", + }, + }, + "docker start container and services with credentials": { + configPath: "docker_multiple_containers_bitrise.yml", + workflowName: "docker-login-multiple-containers", + inventoryPath: "docker_multiple_containers_secrets.yml", + requireErr: false, + requireLogs: []string{ + "Container (service_1_container) is healthy...", + "Container (service_2_container) is healthy...", + "Step is running in container: localhost:5001/healthy-image", }, }, } @@ -126,11 +141,15 @@ func Test_Docker(t *testing.T) { if testCase.requireErr { require.Error(t, err) } else { - require.NoError(t, err) + require.NoError(t, err, out) } for _, log := range testCase.requireLogs { require.Contains(t, out, log) } + for _, logPattern := range testCase.requiredLogPatterns { + contains := glob.Glob(logPattern, out) + require.True(t, contains, out) + } }) } } diff --git a/cli/docker/container_manager.go b/cli/docker/container_manager.go index d61872bc4..ff51a021f 100644 --- a/cli/docker/container_manager.go +++ b/cli/docker/container_manager.go @@ -122,38 +122,20 @@ func NewContainerManager(logger log.Logger, secrets []string) *ContainerManager } } -func (cm *ContainerManager) Login(container models.Container, envs map[string]string) error { - if container.Credentials.Username != "" && container.Credentials.Password != "" { - cm.logger.Infof("ℹ️ Logging into docker registry: %s", container.Image) - - resolvedPassword := resolveEnvVariable(container.Credentials.Password, envs) - args := []string{"login", "--username", container.Credentials.Username, "--password", resolvedPassword} - - if container.Credentials.Server != "" { - args = append(args, container.Credentials.Server) - } else { - args = append(args, container.Image) - } - - cm.logger.Infof("ℹ️ Running command: docker %s", strings.Join(args, " ")) - - out, err := command.New("docker", args...).RunAndReturnTrimmedCombinedOutput() - if err != nil { - cm.logger.Errorf(out) - return fmt.Errorf("run docker login: %w", err) - } - } - return nil -} - -func (cm *ContainerManager) StartWorkflowContainer( +func (cm *ContainerManager) StartContainerForStepGroup( container models.Container, - workflowID string, + groupID string, envs map[string]string, ) (*RunningContainer, error) { cm.mu.Lock() defer cm.mu.Unlock() - containerName := fmt.Sprintf("bitrise-workflow-%s", workflowID) + + if err := cm.login(container, envs); err != nil { + log.Errorf("docker credentials provided, but the authentication failed.") + return nil, fmt.Errorf("authentication failed: %w", err) + } + + containerName := fmt.Sprintf("bitrise-workflow-%s", groupID) // TODO: handle default mounts if BITRISE_DOCKER_MOUNT_OVERRIDES is not provided dockerMountOverrides := strings.Split(os.Getenv("BITRISE_DOCKER_MOUNT_OVERRIDES"), ",") @@ -168,7 +150,7 @@ func (cm *ContainerManager) StartWorkflowContainer( // Even on failure we save the reference to make sure containers will be cleaned up if runningContainer != nil { - cm.workflowContainers[workflowID] = runningContainer + cm.workflowContainers[groupID] = runningContainer } if err != nil { @@ -182,18 +164,27 @@ func (cm *ContainerManager) StartWorkflowContainer( return runningContainer, nil } -func (cm *ContainerManager) StartServiceContainers( +func (cm *ContainerManager) StartServiceContainersForStepGroup( services map[string]models.Container, - workflowID string, + groupID string, envs map[string]string, ) ([]*RunningContainer, error) { - var containers []*RunningContainer cm.mu.Lock() defer cm.mu.Unlock() + + var containers []*RunningContainer failedServices := make(map[string]error) + for serviceName := range services { + serviceContainer := services[serviceName] + + if err := cm.login(serviceContainer, envs); err != nil { + failedServices[serviceName] = err + continue + } + // Naming the container other than the service name, can cause issues with network calls - runningContainer, err := cm.runContainer(services[serviceName], containerCreateOptions{ + runningContainer, err := cm.runContainer(serviceContainer, containerCreateOptions{ name: serviceName, }, envs) if runningContainer != nil { @@ -204,7 +195,7 @@ func (cm *ContainerManager) StartServiceContainers( } } // Even on failure we save the references to make sure containers will be cleaned up - cm.serviceContainers[workflowID] = append(cm.serviceContainers[workflowID], containers...) + cm.serviceContainers[groupID] = append(cm.serviceContainers[groupID], containers...) if len(failedServices) != 0 { errServices := fmt.Errorf("failed to start services") @@ -224,12 +215,12 @@ func (cm *ContainerManager) StartServiceContainers( return containers, nil } -func (cm *ContainerManager) GetWorkflowContainer(workflowID string) *RunningContainer { - return cm.workflowContainers[workflowID] +func (cm *ContainerManager) GetContainerForStepGroup(groupID string) *RunningContainer { + return cm.workflowContainers[groupID] } -func (cm *ContainerManager) GetServiceContainers(workflowID string) []*RunningContainer { - return cm.serviceContainers[workflowID] +func (cm *ContainerManager) GetServiceContainersForStepGroup(groupID string) []*RunningContainer { + return cm.serviceContainers[groupID] } func (cm *ContainerManager) DestroyAllContainers() error { @@ -258,6 +249,31 @@ func (cm *ContainerManager) DestroyAllContainers() error { return nil } +func (cm *ContainerManager) login(container models.Container, envs map[string]string) error { + if container.Credentials.Username != "" && container.Credentials.Password != "" { + cm.logger.Infof("ℹ️ Logging into docker registry: %s", container.Image) + + resolvedPassword := resolveEnvVariable(container.Credentials.Password, envs) + resolvedUsername := resolveEnvVariable(container.Credentials.Username, envs) + args := []string{"login", "--username", resolvedUsername, "--password", resolvedPassword} + + if container.Credentials.Server != "" { + args = append(args, container.Credentials.Server) + } else { + args = append(args, container.Image) + } + + cm.logger.Infof("ℹ️ Running command: docker %s", strings.Join(args, " ")) + + out, err := command.New("docker", args...).RunAndReturnTrimmedCombinedOutput() + if err != nil { + cm.logger.Errorf(out) + return fmt.Errorf("run docker login: %w", err) + } + } + return nil +} + // We are not using the docker sdk for pull, start and create commands because: // - We want to make sure the end user can easily debug using the same docker command we issue // (hard to convert between sdk and cli api) diff --git a/cli/run.go b/cli/run.go index b039a4c99..46d3735b0 100644 --- a/cli/run.go +++ b/cli/run.go @@ -30,11 +30,9 @@ import ( ) const ( - // DefaultBitriseConfigFileName ... DefaultBitriseConfigFileName = "bitrise.yml" - // DefaultSecretsFileName ... - DefaultSecretsFileName = ".bitrise.secrets.yml" - OutputFormatKey = "output-format" + DefaultSecretsFileName = ".bitrise.secrets.yml" + OutputFormatKey = "output-format" depManagerBrew = "brew" secretFilteringFlag = "secret-filtering" @@ -166,6 +164,14 @@ func setupAgentConfig() (*configs.AgentConfig, error) { return &config, nil } +type DockerManager interface { + StartContainerForStepGroup(models.Container, string, map[string]string) (*docker.RunningContainer, error) + StartServiceContainersForStepGroup(services map[string]models.Container, workflowID string, envs map[string]string) ([]*docker.RunningContainer, error) + GetContainerForStepGroup(string) *docker.RunningContainer + GetServiceContainersForStepGroup(string) []*docker.RunningContainer + DestroyAllContainers() error +} + type WorkflowRunner struct { config RunConfig @@ -231,7 +237,7 @@ func (r WorkflowRunner) runWorkflows(tracker analytics.Tracker) (models.BuildRun // Register run modes if err := registerRunModes(r.config.Modes); err != nil { - return models.BuildRunResultsModel{}, fmt.Errorf("failed to register workflow run modes: %s", err) + return models.BuildRunResultsModel{}, fmt.Errorf("failed to register workflow run modes: %w", err) } targetWorkflow := r.config.Config.Workflows[r.config.Workflow] @@ -241,25 +247,25 @@ func (r WorkflowRunner) runWorkflows(tracker analytics.Tracker) (models.BuildRun // Envman setup if err := os.Setenv(configs.EnvstorePathEnvKey, configs.OutputEnvstorePath); err != nil { - return models.BuildRunResultsModel{}, fmt.Errorf("failed to add env, err: %s", err) + return models.BuildRunResultsModel{}, fmt.Errorf("failed to set %s env: %w", configs.EnvstorePathEnvKey, err) } if err := os.Setenv(configs.FormattedOutputPathEnvKey, configs.FormattedOutputPath); err != nil { - return models.BuildRunResultsModel{}, fmt.Errorf("failed to add env, err: %s", err) + return models.BuildRunResultsModel{}, fmt.Errorf("failed to set %s env: %w", configs.FormattedOutputPathEnvKey, err) } if err := tools.EnvmanInit(configs.OutputEnvstorePath, false); err != nil { - return models.BuildRunResultsModel{}, fmt.Errorf("failed to run envman init: %s", err) + return models.BuildRunResultsModel{}, fmt.Errorf("failed to run envman init: %w", err) } // App level environment environments := append(r.config.Secrets, r.config.Config.App.Environments...) if err := os.Setenv("BITRISE_TRIGGERED_WORKFLOW_ID", r.config.Workflow); err != nil { - return models.BuildRunResultsModel{}, fmt.Errorf("failed to set BITRISE_TRIGGERED_WORKFLOW_ID env: %s", err) + return models.BuildRunResultsModel{}, fmt.Errorf("failed to set BITRISE_TRIGGERED_WORKFLOW_ID env: %w", err) } if err := os.Setenv("BITRISE_TRIGGERED_WORKFLOW_TITLE", targetWorkflow.Title); err != nil { - return models.BuildRunResultsModel{}, fmt.Errorf("failed to set BITRISE_TRIGGERED_WORKFLOW_TITLE env: %s", err) + return models.BuildRunResultsModel{}, fmt.Errorf("failed to set BITRISE_TRIGGERED_WORKFLOW_TITLE env: %w", err) } environments = append(environments, targetWorkflow.Environments...) @@ -272,8 +278,7 @@ func (r WorkflowRunner) runWorkflows(tracker analytics.Tracker) (models.BuildRun // the toolkit's `PrepareForStepRun` can bootstrap for itself later if required // or if the system installed version is not sufficient if err := aToolkit.Bootstrap(); err != nil { - return models.BuildRunResultsModel{}, fmt.Errorf("failed to bootstrap the required toolkit for the step (%s), error: %s", - toolkitName, err) + return models.BuildRunResultsModel{}, fmt.Errorf("failed to bootstrap %s toolkit: %w", toolkitName, err) } } } @@ -285,7 +290,7 @@ func (r WorkflowRunner) runWorkflows(tracker analytics.Tracker) (models.BuildRun ProjectType: r.config.Config.ProjectType, } if err := plugins.TriggerEvent(plugins.WillStartRun, buildRunStartModel); err != nil { - log.Warnf("Failed to trigger WillStartRun, error: %s", err) + log.Warnf("Failed to trigger WillStartRun: %s", err) } // Prepare workflow run parameters @@ -296,7 +301,10 @@ func (r WorkflowRunner) runWorkflows(tracker analytics.Tracker) (models.BuildRun ProjectType: r.config.Config.ProjectType, } - plan := createWorkflowRunPlan(r.config.Modes, r.config.Workflow, r.config.Config.Workflows, func() string { return uuid.Must(uuid.NewV4()).String() }) + plan, err := createWorkflowRunPlan(r.config.Modes, r.config.Workflow, r.config.Config.Workflows, func() string { return uuid.Must(uuid.NewV4()).String() }) + if err != nil { + return models.BuildRunResultsModel{}, fmt.Errorf("failed to create workflow execution plan: %w", err) + } if len(plan.ExecutionPlan) < 1 { return models.BuildRunResultsModel{}, fmt.Errorf("execution plan doesn't have any workflow to run") } @@ -317,10 +325,8 @@ func (r WorkflowRunner) runWorkflows(tracker analytics.Tracker) (models.BuildRun for i, workflowRunPlan := range plan.ExecutionPlan { isLastWorkflow := i == len(plan.ExecutionPlan)-1 workflowToRun := r.config.Config.Workflows[workflowRunPlan.WorkflowID] - if workflowToRun.Title == "" { - workflowToRun.Title = workflowRunPlan.WorkflowID - } - buildRunResults = r.runWorkflow(workflowRunPlan, workflowRunPlan.WorkflowID, workflowToRun, r.config.Config.DefaultStepLibSource, buildRunResults, &environments, r.config.Secrets, isLastWorkflow, tracker, buildIDProperties) + environments = append(environments, workflowToRun.Environments...) + buildRunResults = r.runWorkflow(workflowRunPlan, r.config.Config.DefaultStepLibSource, buildRunResults, &environments, r.config.Secrets, isLastWorkflow, tracker, buildIDProperties) } // Build finished @@ -329,12 +335,31 @@ func (r WorkflowRunner) runWorkflows(tracker analytics.Tracker) (models.BuildRun // Trigger WorkflowRunDidFinish buildRunResults.EventName = string(plugins.DidFinishRun) if err := plugins.TriggerEvent(plugins.DidFinishRun, buildRunResults); err != nil { - log.Warnf("Failed to trigger WorkflowRunDidFinish, error: %s", err) + log.Warnf("Failed to trigger WorkflowRunDidFinish: %s", err) } return buildRunResults, nil } +func (r WorkflowRunner) ContainerDefinition(id string) *models.Container { + container, ok := r.config.Config.Containers[id] + if ok { + return &container + } + return nil +} + +func (r WorkflowRunner) ServiceDefinitions(ids ...string) map[string]models.Container { + services := map[string]models.Container{} + for _, id := range ids { + service, ok := r.config.Config.Services[id] + if ok { + services[id] = service + } + } + return services +} + func processArgs(c *cli.Context) (*RunConfig, error) { workflowToRunID := c.String(WorkflowKey) if workflowToRunID == "" && len(c.Args()) > 0 { @@ -464,25 +489,59 @@ func registerRunModes(modes models.WorkflowRunModes) error { return nil } -func createWorkflowRunPlan(modes models.WorkflowRunModes, targetWorkflow string, workflows map[string]models.WorkflowModel, uuidProvider func() string) models.WorkflowRunPlan { +func createWorkflowRunPlan(modes models.WorkflowRunModes, targetWorkflow string, workflows map[string]models.WorkflowModel, uuidProvider func() string) (models.WorkflowRunPlan, error) { var executionPlan []models.WorkflowExecutionPlan + workflowList := walkWorkflows(targetWorkflow, workflows, nil) for _, workflowID := range workflowList { workflow := workflows[workflowID] var stepPlan []models.StepExecutionPlan - for _, stepItem := range workflow.Steps { - stepID, _ := stepItem.GetStepIDAndStep() - stepPlan = append(stepPlan, models.StepExecutionPlan{ - UUID: uuidProvider(), - StepID: stepID, - }) + + for _, stepListItem := range workflow.Steps { + key, step, with, err := stepListItem.GetStepListItemKeyAndValue() + if err != nil { + return models.WorkflowRunPlan{}, err + } + + if key == models.StepListItemWithKey { + groupID := uuidProvider() + + for _, stepListStepItem := range with.Steps { + stepID, step, err := stepListStepItem.GetStepIDAndStep() + if err != nil { + return models.WorkflowRunPlan{}, err + } + + stepPlan = append(stepPlan, models.StepExecutionPlan{ + UUID: uuidProvider(), + StepID: stepID, + Step: step, + GroupID: groupID, + ContainerID: with.ContainerID, + ServiceIDs: with.ServiceIDs, + }) + } + } else { + stepID := key + stepPlan = append(stepPlan, models.StepExecutionPlan{ + UUID: uuidProvider(), + StepID: stepID, + Step: step, + }) + } + } + + workflowTitle := workflow.Title + if workflowTitle == "" { + workflowTitle = workflowID } executionPlan = append(executionPlan, models.WorkflowExecutionPlan{ UUID: uuidProvider(), WorkflowID: workflowID, Steps: stepPlan, + WorkflowTitle: workflowTitle, IsSteplibOfflineMode: modes.IsSteplibOfflineMode, }) } @@ -503,7 +562,7 @@ func createWorkflowRunPlan(modes models.WorkflowRunModes, targetWorkflow string, SecretFilteringMode: modes.SecretFilteringMode, SecretEnvsFilteringMode: modes.SecretEnvsFilteringMode, ExecutionPlan: executionPlan, - } + }, nil } func walkWorkflows(workflowID string, workflows map[string]models.WorkflowModel, workflowStack []string) []string { diff --git a/cli/run_util.go b/cli/run_util.go index f187083e4..c783197f2 100644 --- a/cli/run_util.go +++ b/cli/run_util.go @@ -179,7 +179,6 @@ func isDirEmpty(path string) (bool, error) { return len(entries) == 0, nil } -// GetBitriseConfigFromBase64Data ... func GetBitriseConfigFromBase64Data(configBase64Str string) (models.BitriseDataModel, []string, error) { configBase64Bytes, err := base64.StdEncoding.DecodeString(configBase64Str) if err != nil { @@ -194,7 +193,6 @@ func GetBitriseConfigFromBase64Data(configBase64Str string) (models.BitriseDataM return config, warnings, nil } -// GetBitriseConfigFilePath ... func GetBitriseConfigFilePath(bitriseConfigPath string) (string, error) { if bitriseConfigPath == "" { bitriseConfigPath = filepath.Join(configs.CurrentDir, DefaultBitriseConfigFileName) @@ -209,7 +207,6 @@ func GetBitriseConfigFilePath(bitriseConfigPath string) (string, error) { return bitriseConfigPath, nil } -// CreateBitriseConfigFromCLIParams ... func CreateBitriseConfigFromCLIParams(bitriseConfigBase64Data, bitriseConfigPath string) (models.BitriseDataModel, []string, error) { bitriseConfig := models.BitriseDataModel{} warnings := []string{} @@ -249,7 +246,6 @@ func CreateBitriseConfigFromCLIParams(bitriseConfigBase64Data, bitriseConfigPath return bitriseConfig, warnings, nil } -// GetInventoryFromBase64Data ... func GetInventoryFromBase64Data(inventoryBase64Str string) ([]envmanModels.EnvironmentItemModel, error) { inventoryBase64Bytes, err := base64.StdEncoding.DecodeString(inventoryBase64Str) if err != nil { @@ -264,7 +260,6 @@ func GetInventoryFromBase64Data(inventoryBase64Str string) ([]envmanModels.Envir return inventory.Envs, nil } -// GetInventoryFilePath ... func GetInventoryFilePath(inventoryPath string) (string, error) { if inventoryPath == "" { log.Debug("[BITRISE_CLI] - Inventory path not defined, searching for " + DefaultSecretsFileName + " in current folder...") @@ -280,7 +275,6 @@ func GetInventoryFilePath(inventoryPath string) (string, error) { return inventoryPath, nil } -// CreateInventoryFromCLIParams ... func CreateInventoryFromCLIParams(inventoryBase64Data, inventoryPath string) ([]envmanModels.EnvironmentItemModel, error) { inventoryEnvironments := []envmanModels.EnvironmentItemModel{} @@ -400,8 +394,8 @@ func (r WorkflowRunner) executeStep( step stepmanModels.StepModel, sIDData stepid.CanonicalID, stepAbsDirPath, bitriseSourceDir string, secrets []string, - workflow models.WorkflowModel, - workflowID string, + containerID string, + groupID string, ) (int, error) { toolkitForStep := toolkits.ToolkitForStep(step) @@ -445,7 +439,8 @@ func (r WorkflowRunner) executeStep( var args []string var envs []string - if workflow.Container.Image != "" { + containerDef := r.ContainerDefinition(containerID) + if containerDef != nil { envs, err = envman.ReadAndEvaluateEnvs(configs.InputEnvstorePath, &docker.EnvironmentSource{ Logger: logger, }) @@ -454,17 +449,17 @@ func (r WorkflowRunner) executeStep( } name = "docker" - container := r.dockerManager.GetWorkflowContainer(workflowID) - if container == nil { + runningContainer := r.dockerManager.GetContainerForStepGroup(groupID) + if runningContainer == nil { return 1, fmt.Errorf("Docker container does not exist") } - args = container.ExecuteCommandArgs(envs) + args = runningContainer.ExecuteCommandArgs(envs) args = append(args, cmdArgs...) cmd := stepruncmd.New(name, args, bitriseSourceDir, envs, stepSecrets, timeout, noOutputTimeout, stdout, logV2.NewLogger()) - logger.Infof("Step is running in container: %s", workflow.Container.Image) + logger.Infof("Step is running in container: %s", containerDef.Image) return cmd.Run() } @@ -490,8 +485,8 @@ func (r WorkflowRunner) runStep( stepDir string, environments []envmanModels.EnvironmentItemModel, secrets []string, - workflow models.WorkflowModel, - workflowID string, + containerID string, + groupID string, ) (int, []envmanModels.EnvironmentItemModel, error) { log.Debugf("[BITRISE_CLI] - Try running step: %s (%s)", stepIDData.IDorURI, stepIDData.Version) @@ -529,7 +524,7 @@ func (r WorkflowRunner) runStep( bitriseSourceDir = configs.CurrentDir } - if exit, err := r.executeStep(stepUUID, step, stepIDData, stepDir, bitriseSourceDir, secrets, workflow, workflowID); err != nil { + if exit, err := r.executeStep(stepUUID, step, stepIDData, stepDir, bitriseSourceDir, secrets, containerID, groupID); err != nil { stepOutputs, envErr := bitrise.CollectEnvironmentsFromFile(configs.OutputEnvstorePath) if envErr != nil { return 1, []envmanModels.EnvironmentItemModel{}, envErr @@ -576,18 +571,63 @@ func (r WorkflowRunner) runStep( return 0, updatedStepOutputs, nil } -type DockerManager interface { - Login(models.Container, map[string]string) error - StartWorkflowContainer(models.Container, string, map[string]string) (*docker.RunningContainer, error) - StartServiceContainers(services map[string]models.Container, workflowID string, envs map[string]string) ([]*docker.RunningContainer, error) - GetWorkflowContainer(string) *docker.RunningContainer - GetServiceContainers(string) []*docker.RunningContainer - DestroyAllContainers() error +func (r WorkflowRunner) startContainersForStepGroup(containerID string, serviceIDs []string, environments *[]envmanModels.EnvironmentItemModel, groupID, workflowTitle string) { + if containerID == "" && len(serviceIDs) == 0 { + return + } + + if err := tools.EnvmanInit(configs.InputEnvstorePath, true); err != nil { + log.Debugf("Couldn't initialize envman.") + } + if err := tools.EnvmanAddEnvs(configs.InputEnvstorePath, *environments); err != nil { + log.Debugf("Couldn't add envs.") + } + + envList, err := tools.EnvmanReadEnvList(configs.InputEnvstorePath) + if err != nil { + log.Debugf("Couldn't read envs from envman.") + } + + if containerID != "" { + containerDef := r.ContainerDefinition(containerID) + if containerDef != nil { + log.Infof("ℹ️ Running workflow in docker container: %s", containerDef.Image) + + _, err := r.dockerManager.StartContainerForStepGroup(*containerDef, groupID, envList) + if err != nil { + log.Errorf("Could not start the specified docker image for workflow: %s", workflowTitle) + } + } + } + + if len(serviceIDs) > 0 { + servicesDefs := r.ServiceDefinitions(serviceIDs...) + _, err := r.dockerManager.StartServiceContainersForStepGroup(servicesDefs, groupID, envList) + if err != nil { + log.Errorf("❌ Some services failed to start properly!") + } + } +} + +func (r WorkflowRunner) stopContainersForStepGroup(groupID, workflowTitle string) { + if container := r.dockerManager.GetContainerForStepGroup(groupID); container != nil { + // TODO: Feature idea, make this configurable, so that we can keep the container for debugging purposes. + if err := container.Destroy(); err != nil { + log.Errorf("Attempted to stop the docker container for workflow: %s: %s", workflowTitle, err) + } + } + + if services := r.dockerManager.GetServiceContainersForStepGroup(groupID); services != nil { + for _, container := range services { + if err := container.Destroy(); err != nil { + log.Errorf("Attempted to stop the docker container for service: %s: %s", container.Name, err) + } + } + } } func (r WorkflowRunner) activateAndRunSteps( plan models.WorkflowExecutionPlan, - workflow models.WorkflowModel, defaultStepLibSource string, buildRunResults models.BuildRunResultsModel, environments *[]envmanModels.EnvironmentItemModel, @@ -595,82 +635,57 @@ func (r WorkflowRunner) activateAndRunSteps( isLastWorkflow bool, tracker analytics.Tracker, workflowIDProperties coreanalytics.Properties, - workflowID string, ) models.BuildRunResultsModel { log.Debug("[BITRISE_CLI] - Activating and running steps") - if len(workflow.Steps) == 0 { - log.Warnf("%s workflow has no steps to run, moving on to the next workflow...", workflow.Title) + if len(plan.Steps) == 0 { + log.Warnf("%s workflow has no steps to run, moving on to the next workflow...", plan.WorkflowTitle) return buildRunResults } - envList := envmanModels.EnvsJSONListModel{} - if workflow.Container.Image != "" || len(workflow.Services) > 0 { - if err := tools.EnvmanInit(configs.InputEnvstorePath, true); err != nil { - log.Debugf("Couldn't initialize envman.") - } - if err := tools.EnvmanAddEnvs(configs.InputEnvstorePath, *environments); err != nil { - log.Debugf("Couldn't add envs.") - } - - var err error - if envList, err = tools.EnvmanReadEnvList(configs.InputEnvstorePath); err != nil { - log.Debugf("Couldn't read envs from envman.") - } - } + // ------------------------------------------ + // In function global variables - serviceContainers, err := r.dockerManager.StartServiceContainers(workflow.Services, workflowID, envList) - if err != nil { - log.Errorf("❌ Some services failed to start properly!") - } + // These are global for easy use in local register step run result methods. + var stepStartTime time.Time + runResultCollector := newBuildRunResultCollector(tracker) + // Global variables for synchronising container's lifecycle + // and shutting down the last set of containers at the end of the workflow run + currentStepGroupID := "" + lastStepGroupID := "" defer func() { - for _, container := range serviceContainers { - if err := container.Destroy(); err != nil { - log.Errorf("Attempted to stop the docker container for service: %s: %w", container.Name, err.Error()) - } + if lastStepGroupID != "" { + r.stopContainersForStepGroup(lastStepGroupID, plan.WorkflowTitle) } }() - if workflow.Container.Image != "" { - log.Infof("ℹ️ Running workflow in docker container: %s", workflow.Container.Image) - - if err := r.dockerManager.Login(workflow.Container, envList); err != nil { - log.Errorf("%s workflow has docker credentials provided, but the authentication failed.", workflow.Title) - } - - runningContainer, err := r.dockerManager.StartWorkflowContainer(workflow.Container, workflowID, envList) - if err != nil { - log.Errorf("Could not start the specified docker image for workflow: %s", workflow.Title) - } - - defer func() { - if runningContainer == nil { - return + // ------------------------------------------ + // Main - Preparing & running the steps + for idx, stepPlan := range plan.Steps { + if stepPlan.GroupID != currentStepGroupID { + if currentStepGroupID != "" { + r.stopContainersForStepGroup(currentStepGroupID, plan.WorkflowTitle) } - // TODO: Feature idea, make this configurable, so that we can keep the container for debugging purposes. - if err := runningContainer.Destroy(); err != nil { - log.Errorf("Attempted to stop the docker container for workflow: %s: %w", workflow.Title, err.Error()) + if stepPlan.GroupID != "" { + if len(stepPlan.ContainerID) > 0 || len(stepPlan.ServiceIDs) > 0 { + r.startContainersForStepGroup(stepPlan.ContainerID, stepPlan.ServiceIDs, environments, stepPlan.GroupID, plan.WorkflowTitle) + } } - }() - } - // ------------------------------------------ - // In function global variables - These are global for easy use in local register step run result methods. - var stepStartTime time.Time - runResultCollector := newBuildRunResultCollector(tracker) + currentStepGroupID = stepPlan.GroupID + if stepPlan.GroupID != "" { + lastStepGroupID = stepPlan.GroupID + } + } - // ------------------------------------------ - // Main - Preparing & running the steps - for idx, stepListItm := range workflow.Steps { - stepPlan := plan.Steps[idx] stepExecutionID := stepPlan.UUID stepIDProperties := coreanalytics.Properties{analytics.StepExecutionID: stepExecutionID} stepStartedProperties := workflowIDProperties.Merge(stepIDProperties) // Per step variables stepStartTime = time.Now() - isLastStep := isLastWorkflow && (idx == len(workflow.Steps)-1) + isLastStep := isLastWorkflow && (idx == len(plan.Steps)-1) // TODO: stepInfoPtr.Step is not a real step, only stores presentation properties (printed in the step boxes) stepInfoPtr := stepmanModels.StepInfoModel{} stepIdxPtr := idx @@ -700,13 +715,9 @@ func (r WorkflowRunner) activateAndRunSteps( continue } - // Get step id & version data - compositeStepIDStr, workflowStep, err := models.GetStepIDStepDataPair(stepListItm) - if err != nil { - runResultCollector.registerStepRunResults(&buildRunResults, stepExecutionID, stepStartTime, stepmanModels.StepModel{}, stepInfoPtr, stepIdxPtr, - models.StepRunStatusCodePreparationFailed, 1, err, isLastStep, true, map[string]string{}, stepStartedProperties) - continue - } + compositeStepIDStr := stepPlan.StepID + workflowStep := stepPlan.Step + stepInfoPtr.ID = compositeStepIDStr if workflowStep.Title != nil && *workflowStep.Title != "" { stepInfoPtr.Step.Title = pointers.NewStringPtr(*workflowStep.Title) @@ -923,7 +934,7 @@ func (r WorkflowRunner) activateAndRunSteps( tracker.SendStepStartedEvent(stepStartedProperties, prepareAnalyticsStepInfo(mergedStep, stepInfoPtr), redactedInputsWithType, redactedOriginalInputs) - exit, outEnvironments, err := r.runStep(stepExecutionID, mergedStep, stepIDData, stepDir, stepDeclaredEnvironments, stepSecretValues, workflow, workflowID) + exit, outEnvironments, err := r.runStep(stepExecutionID, mergedStep, stepIDData, stepDir, stepDeclaredEnvironments, stepSecretValues, stepPlan.ContainerID, stepPlan.GroupID) if stepTestDir != "" { if err := addTestMetadata(stepTestDir, models.TestResultStepInfo{Number: idx, Title: *mergedStep.Title, ID: stepIDData.IDorURI, Version: stepIDData.Version}); err != nil { @@ -985,18 +996,15 @@ func prepareAnalyticsStepInfo(step stepmanModels.StepModel, stepInfoPtr stepmanM func (r WorkflowRunner) runWorkflow( plan models.WorkflowExecutionPlan, - workflowID string, - workflow models.WorkflowModel, steplibSource string, buildRunResults models.BuildRunResultsModel, environments *[]envmanModels.EnvironmentItemModel, secrets []envmanModels.EnvironmentItemModel, isLastWorkflow bool, tracker analytics.Tracker, buildIDProperties coreanalytics.Properties) models.BuildRunResultsModel { workflowIDProperties := coreanalytics.Properties{analytics.WorkflowExecutionID: plan.UUID} - bitrise.PrintRunningWorkflow(workflow.Title) - tracker.SendWorkflowStarted(buildIDProperties.Merge(workflowIDProperties), workflowID, workflow.Title) - *environments = append(*environments, workflow.Environments...) - results := r.activateAndRunSteps(plan, workflow, steplibSource, buildRunResults, environments, secrets, isLastWorkflow, tracker, workflowIDProperties, workflowID) + bitrise.PrintRunningWorkflow(plan.WorkflowTitle) + tracker.SendWorkflowStarted(buildIDProperties.Merge(workflowIDProperties), plan.WorkflowID, plan.WorkflowTitle) + results := r.activateAndRunSteps(plan, steplibSource, buildRunResults, environments, secrets, isLastWorkflow, tracker, workflowIDProperties) tracker.SendWorkflowFinished(workflowIDProperties, results.IsBuildFailed()) collectToolVersions(tracker) return results diff --git a/models/models.go b/models/models.go index 87b14d4d9..a0705dc1f 100644 --- a/models/models.go +++ b/models/models.go @@ -10,14 +10,22 @@ import ( ) const ( - // FormatVersion ... - FormatVersion = "14" + FormatVersion = "14" + StepListItemWithKey = "with" ) -// StepListItemModel ... -type StepListItemModel map[string]stepmanModels.StepModel +type WithModel struct { + ContainerID string `json:"container,omitempty" yaml:"container,omitempty"` + ServiceIDs []string `json:"services,omitempty" yaml:"services,omitempty"` + Steps []StepListStepItemModel `json:"steps,omitempty" yaml:"steps,omitempty"` +} + +type StepListWithItemModel map[string]WithModel + +type StepListStepItemModel map[string]stepmanModels.StepModel + +type StepListItemModel map[string]interface{} -// PipelineModel ... type PipelineModel struct { Title string `json:"title,omitempty" yaml:"title,omitempty"` Summary string `json:"summary,omitempty" yaml:"summary,omitempty"` @@ -25,10 +33,8 @@ type PipelineModel struct { Stages []StageListItemModel `json:"stages,omitempty" yaml:"stages,omitempty"` } -// StageListItemModel ... type StageListItemModel map[string]StageModel -// StageModel ... type StageModel struct { Title string `json:"title,omitempty" yaml:"title,omitempty"` Summary string `json:"summary,omitempty" yaml:"summary,omitempty"` @@ -39,21 +45,15 @@ type StageModel struct { Workflows []StageWorkflowListItemModel `json:"workflows,omitempty" yaml:"workflows,omitempty"` } -// StageWorkflowListItemModel ... type StageWorkflowListItemModel map[string]StageWorkflowModel -// StageWorkflowModel ... type StageWorkflowModel struct { RunIf string `json:"run_if,omitempty" yaml:"run_if,omitempty"` } -// WorkflowListItemModel ... type WorkflowListItemModel map[string]WorkflowModel -// WorkflowModel ... type WorkflowModel struct { - Container Container `json:"container,omitempty" yaml:"container,omitempty"` - Services map[string]Container `json:"services,omitempty" yaml:"services,omitempty"` Title string `json:"title,omitempty" yaml:"title,omitempty"` Summary string `json:"summary,omitempty" yaml:"summary,omitempty"` Description string `json:"description,omitempty" yaml:"description,omitempty"` @@ -78,7 +78,6 @@ type Container struct { Options string `json:"options,omitempty" yaml:"options,omitempty"` } -// AppModel ... type AppModel struct { Title string `json:"title,omitempty" yaml:"title,omitempty"` Summary string `json:"summary,omitempty" yaml:"summary,omitempty"` @@ -86,7 +85,6 @@ type AppModel struct { Environments []envmanModels.EnvironmentItemModel `json:"envs,omitempty" yaml:"envs,omitempty"` } -// BitriseDataModel ... type BitriseDataModel struct { FormatVersion string `json:"format_version" yaml:"format_version"` DefaultStepLibSource string `json:"default_step_lib_source,omitempty" yaml:"default_step_lib_source,omitempty"` @@ -96,6 +94,8 @@ type BitriseDataModel struct { Summary string `json:"summary,omitempty" yaml:"summary,omitempty"` Description string `json:"description,omitempty" yaml:"description,omitempty"` // + Services map[string]Container `json:"services" yaml:"services"` + Containers map[string]Container `json:"containers" yaml:"containers"` App AppModel `json:"app,omitempty" yaml:"app,omitempty"` Meta map[string]interface{} `json:"meta,omitempty" yaml:"meta,omitempty"` TriggerMap TriggerMapModel `json:"trigger_map,omitempty" yaml:"trigger_map,omitempty"` @@ -104,14 +104,12 @@ type BitriseDataModel struct { Workflows map[string]WorkflowModel `json:"workflows,omitempty" yaml:"workflows,omitempty"` } -// BuildRunStartModel ... type BuildRunStartModel struct { EventName string `json:"event_name" yaml:"event_name"` ProjectType string `json:"project_type" yaml:"project_type"` StartTime time.Time `json:"start_time" yaml:"start_time"` } -// BuildRunResultsModel ... type BuildRunResultsModel struct { WorkflowID string `json:"workflow_id" yaml:"workflow_id"` EventName string `json:"event_name" yaml:"event_name"` @@ -124,7 +122,6 @@ type BuildRunResultsModel struct { SkippedSteps []StepRunResultsModel `json:"skipped_steps" yaml:"skipped_steps"` } -// StepRunResultsModel ... type StepRunResultsModel struct { StepInfo stepmanModels.StepInfoModel `json:"step_info" yaml:"step_info"` StepInputs map[string]string `json:"step_inputs" yaml:"step_inputs"` @@ -139,7 +136,6 @@ type StepRunResultsModel struct { NoOutputTimeout time.Duration `json:"-"` } -// StepError ... type StepError struct { Code int `json:"code"` Message string `json:"message"` @@ -238,7 +234,6 @@ func formatStatusReasonTimeInterval(timeInterval time.Duration) string { return formattedTimeInterval } -// TestResultStepInfo ... type TestResultStepInfo struct { ID string `json:"id" yaml:"id"` Version string `json:"version" yaml:"version"` diff --git a/models/models_methods.go b/models/models_methods.go index 9b2ce59d1..8b858374e 100644 --- a/models/models_methods.go +++ b/models/models_methods.go @@ -105,7 +105,6 @@ func (config *BitriseDataModel) getPipelineIDs() []string { // ---------------------------- // --- Normalize -// Normalize ... func (workflow *WorkflowModel) Normalize() error { for _, env := range workflow.Environments { if err := env.Normalize(); err != nil { @@ -114,20 +113,21 @@ func (workflow *WorkflowModel) Normalize() error { } for _, stepListItem := range workflow.Steps { - stepID, step, err := GetStepIDStepDataPair(stepListItem) + key, step, _, err := stepListItem.GetStepListItemKeyAndValue() if err != nil { return err } - if err := step.Normalize(); err != nil { - return err + if key != StepListItemWithKey { + if err := step.Normalize(); err != nil { + return err + } + stepListItem[key] = step } - stepListItem[stepID] = step } return nil } -// Normalize ... func (app *AppModel) Normalize() error { for _, env := range app.Environments { if err := env.Normalize(); err != nil { @@ -137,7 +137,6 @@ func (app *AppModel) Normalize() error { return nil } -// Normalize ... func (config *BitriseDataModel) Normalize() error { if err := config.App.Normalize(); err != nil { return err @@ -166,50 +165,103 @@ func (config *BitriseDataModel) Normalize() error { // ---------------------------- // --- Validate -// Validate ... -func (workflow *WorkflowModel) Validate() ([]string, error) { - for _, env := range workflow.Environments { - if err := env.Validate(); err != nil { - return []string{}, err +func (with WithModel) Validate(workflowID string, containers, services map[string]Container) ([]string, error) { + var warnings []string + + if with.ContainerID != "" { + if _, ok := containers[with.ContainerID]; !ok { + return warnings, fmt.Errorf("container (%s) referenced in workflow (%s), but this container is not defined", with.ContainerID, workflowID) } } - warnings := []string{} - for _, stepListItem := range workflow.Steps { - stepID, step, err := GetStepIDStepDataPair(stepListItem) + serviceIDs := map[string]bool{} + for _, serviceID := range with.ServiceIDs { + if _, ok := services[serviceID]; !ok { + return warnings, fmt.Errorf("service (%s) referenced in workflow (%s), but this service is not defined", serviceID, workflowID) + } + + if _, ok := serviceIDs[serviceID]; ok { + return warnings, fmt.Errorf("service (%s) specified multiple times for workflow (%s)", serviceID, workflowID) + } + serviceIDs[serviceID] = true + } + + for _, stepListItem := range with.Steps { + stepID, step, err := stepListItem.GetStepIDAndStep() if err != nil { return warnings, err } - if err := stepid.Validate(stepID); err != nil { + warns, err := validateStep(stepID, step) + warnings = append(warnings, warns...) + if err != nil { return warnings, err } + } + + return warnings, nil + +} + +func (workflow *WorkflowModel) Validate() ([]string, error) { + var warnings []string - if err := step.ValidateInputAndOutputEnvs(false); err != nil { + for _, env := range workflow.Environments { + if err := env.Validate(); err != nil { return warnings, err } + } - stepInputMap := map[string]bool{} - for _, input := range step.Inputs { - key, _, err := input.GetKeyValuePair() + for _, stepListItem := range workflow.Steps { + key, step, _, err := stepListItem.GetStepListItemKeyAndValue() + if err != nil { + return warnings, err + } + + if key != StepListItemWithKey { + stepID := key + warns, err := validateStep(stepID, step) + warnings = append(warnings, warns...) if err != nil { return warnings, err } - _, found := stepInputMap[key] - if found { - warnings = append(warnings, fmt.Sprintf("invalid step: duplicated input found: (%s)", key)) - } - stepInputMap[key] = true + // TODO: Why is this assignment needed? + stepListItem[stepID] = step } + } + + return warnings, nil +} - stepListItem[stepID] = step +func validateStep(stepID string, step stepmanModels.StepModel) ([]string, error) { + var warnings []string + + if err := stepid.Validate(stepID); err != nil { + return warnings, err + } + + if err := step.ValidateInputAndOutputEnvs(false); err != nil { + return warnings, err + } + + stepInputMap := map[string]bool{} + for _, input := range step.Inputs { + key, _, err := input.GetKeyValuePair() + if err != nil { + return warnings, err + } + + _, found := stepInputMap[key] + if found { + warnings = append(warnings, fmt.Sprintf("invalid step: duplicated input found: (%s)", key)) + } + stepInputMap[key] = true } return warnings, nil } -// Validate ... func (app *AppModel) Validate() error { for _, env := range app.Environments { if err := env.Validate(); err != nil { @@ -219,9 +271,8 @@ func (app *AppModel) Validate() error { return nil } -// Validate ... func (config *BitriseDataModel) Validate() ([]string, error) { - warnings := []string{} + var warnings []string if config.FormatVersion == "" { return warnings, fmt.Errorf("missing format_version") @@ -243,6 +294,26 @@ func (config *BitriseDataModel) Validate() ([]string, error) { } // --- + // containers + for containerID, containerDef := range config.Containers { + if containerID == "" { + return nil, fmt.Errorf("service (image: %s) has empty ID defined", containerDef.Image) + } + if strings.TrimSpace(containerDef.Image) == "" { + return warnings, fmt.Errorf("service (%s) has no image defined", containerID) + } + } + + for serviceID, serviceDef := range config.Services { + if serviceID == "" { + return nil, fmt.Errorf("service (image: %s) has empty ID defined", serviceDef.Image) + } + if strings.TrimSpace(serviceDef.Image) == "" { + return warnings, fmt.Errorf("service (%s) has no image defined", serviceID) + } + } + // --- + // pipelines pipelineWarnings, err := validatePipelines(config) warnings = append(warnings, pipelineWarnings...) @@ -265,6 +336,22 @@ func (config *BitriseDataModel) Validate() ([]string, error) { if err != nil { return warnings, err } + + for workflowID, workflow := range config.Workflows { + for _, stepListItem := range workflow.Steps { + key, _, with, err := stepListItem.GetStepListItemKeyAndValue() + if err != nil { + return warnings, err + } + if key == StepListItemWithKey { + warns, err := with.Validate(workflowID, config.Containers, config.Services) + warnings = append(warnings, warns...) + if err != nil { + return warnings, err + } + } + } + } // --- return warnings, nil @@ -392,7 +479,6 @@ func validateID(id, modelType string) (string, error) { // ---------------------------- // --- FillMissingDefaults -// FillMissingDefaults ... func (workflow *WorkflowModel) FillMissingDefaults(title string) error { // Don't call step.FillMissingDefaults() // StepLib versions of steps (which are the default versions), @@ -413,7 +499,6 @@ func (workflow *WorkflowModel) FillMissingDefaults(title string) error { return nil } -// FillMissingDefaults ... func (app *AppModel) FillMissingDefaults() error { for _, env := range app.Environments { if err := env.FillMissingDefaults(); err != nil { @@ -423,7 +508,6 @@ func (app *AppModel) FillMissingDefaults() error { return nil } -// FillMissingDefaults ... func (config *BitriseDataModel) FillMissingDefaults() error { if err := config.App.FillMissingDefaults(); err != nil { return err @@ -559,7 +643,6 @@ func (app *AppModel) removeRedundantFields() error { return nil } -// RemoveRedundantFields ... func (config *BitriseDataModel) RemoveRedundantFields() error { if err := config.App.removeRedundantFields(); err != nil { return err @@ -575,7 +658,6 @@ func (config *BitriseDataModel) RemoveRedundantFields() error { // ---------------------------- // --- Merge -// MergeEnvironmentWith ... func MergeEnvironmentWith(env *envmanModels.EnvironmentItemModel, otherEnv envmanModels.EnvironmentItemModel) error { // merge key-value key, _, err := env.GetKeyValuePair() @@ -671,7 +753,6 @@ func getOutputByKey(step stepmanModels.StepModel, key string) (envmanModels.Envi return envmanModels.EnvironmentItemModel{}, false } -// MergeStepWith ... func MergeStepWith(step, otherStep stepmanModels.StepModel) (stepmanModels.StepModel, error) { if otherStep.Title != nil { step.Title = pointers.NewStringPtr(*otherStep.Title) @@ -809,28 +890,105 @@ func getStageID(stageListItem StageListItemModel) (string, error) { // ---------------------------- // --- StepIDData -// GetStepIDAndStep returns the Step ID and Step model described by the stepListItem. -// Use this on validated BitriseDataModels. -func (stepListItem StepListItemModel) GetStepIDAndStep() (string, stepmanModels.StepModel) { - for key, value := range stepListItem { - return key, value +func (stepListItem *StepListItemModel) UnmarshalYAML(unmarshal func(interface{}) error) error { + var raw map[string]interface{} + if err := unmarshal(&raw); err != nil { + return err + } + + var key string + for k := range raw { + key = k + break + } + + if key == StepListItemWithKey { + var withItem StepListWithItemModel + if err := unmarshal(&withItem); err != nil { + return err + } + + *stepListItem = map[string]interface{}{} + for k, v := range withItem { + (*stepListItem)[k] = v + } + } else { + var stepItem StepListStepItemModel + if err := unmarshal(&stepItem); err != nil { + return err + } + + *stepListItem = map[string]interface{}{} + for k, v := range stepItem { + (*stepListItem)[k] = v + } } - return "", stepmanModels.StepModel{} + + return nil } -// GetStepIDStepDataPair ... -func GetStepIDStepDataPair(stepListItem StepListItemModel) (string, stepmanModels.StepModel, error) { - if len(stepListItem) == 0 { - return "", stepmanModels.StepModel{}, errors.New("StepListItem does not contain a key-value pair") +func (stepListStepItem *StepListStepItemModel) GetStepIDAndStep() (string, stepmanModels.StepModel, error) { + if stepListStepItem == nil { + return "", stepmanModels.StepModel{}, nil + } + + if len(*stepListStepItem) == 0 { + return "", stepmanModels.StepModel{}, errors.New("stepListStepItem does not contain a key-value pair") } - if len(stepListItem) > 1 { - return "", stepmanModels.StepModel{}, errors.New("StepListItem contains more than 1 key-value pair") + if len(*stepListStepItem) > 1 { + return "", stepmanModels.StepModel{}, errors.New("stepListStepItem contains more than 1 key-value pair") } - stepID, step := stepListItem.GetStepIDAndStep() + + var stepID string + var step stepmanModels.StepModel + for k, v := range *stepListStepItem { + stepID = k + step = v + break + } + return stepID, step, nil } +// GetStepListItemKeyAndValue returns the Step List Item key and value. The key is either a Step ID or 'with'. +// If the key is 'with' the returned WithModel is relevant otherwise the StepModel. +func (stepListItem *StepListItemModel) GetStepListItemKeyAndValue() (string, stepmanModels.StepModel, WithModel, error) { + if stepListItem == nil { + return "", stepmanModels.StepModel{}, WithModel{}, nil + } + + if len(*stepListItem) == 0 { + return "", stepmanModels.StepModel{}, WithModel{}, errors.New("StepListItem does not contain a key-value pair") + } + + if len(*stepListItem) > 1 { + return "", stepmanModels.StepModel{}, WithModel{}, errors.New("StepListItem contains more than 1 key-value pair") + } + + for key, value := range *stepListItem { + if key == StepListItemWithKey { + with := value.(WithModel) + return key, stepmanModels.StepModel{}, with, nil + } else { + step, ok := value.(stepmanModels.StepModel) + if ok { + return key, step, WithModel{}, nil + } + + // StepListItemModel is a map[string]interface{}, when it comes from a JSON/YAML unmarshal + // the StepModel has a pointer type. + stepPtr, ok := value.(*stepmanModels.StepModel) + if ok { + return key, *stepPtr, WithModel{}, nil + } + + return key, stepmanModels.StepModel{}, WithModel{}, nil + } + } + return "", stepmanModels.StepModel{}, WithModel{}, nil +} + // ---------------------------- // --- BuildRunResults diff --git a/models/models_methods_test.go b/models/models_methods_test.go index cadd06c53..bcfe94687 100644 --- a/models/models_methods_test.go +++ b/models/models_methods_test.go @@ -76,6 +76,58 @@ workflows: test: check:`, }, + { + name: "Containers are normalized", + config: ` +format_version: '11' +default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git +services: + postgres: + image: postgres:13 + envs: + - POSTGRES_PASSWORD: password + ports: + - 5435:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 +containers: + ruby: + image: ruby:3.2 +workflows: + test: + steps: + - with: + container: ruby + services: + - postgres + steps: + - script: + title: Setup DB + inputs: + - content: bundle exec rails db:setup + - script: + title: Run tests + inputs: + - content: bundle exec rspec + test_features: + steps: + - with: + container: ruby + services: + - postgres + steps: + - script: + title: Setup DB + inputs: + - content: bundle exec rails db:setup + - script: + title: Run tests + inputs: + - content: bundle exec rspec spec/features/`, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -391,6 +443,187 @@ func TestValidateConfig(t *testing.T) { } } +func TestValidateConfig_Containers(t *testing.T) { + tests := []struct { + name string + config BitriseDataModel + wantErr string + }{ + { + name: "Valid bitrise.yml with a service and container", + config: createConfig(t, ` +format_version: '11' +default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git +services: + postgres: + image: postgres:13 + envs: + - POSTGRES_PASSWORD: password + ports: + - 5435:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 +containers: + ruby: + image: ruby:3.2 +workflows: + test: + steps: + - with: + container: ruby + services: + - postgres + steps: + - script: + title: Setup DB + inputs: + - content: bundle exec rails db:setup + - script: + title: Run tests + inputs: + - content: bundle exec rspec + test_features: + steps: + - with: + container: ruby + services: + - postgres + steps: + - script: + title: Setup DB + inputs: + - content: bundle exec rails db:setup + - script: + title: Run tests + inputs: + - content: bundle exec rspec spec/features/`), + }, + { + name: "Invalid bitrise.yml: missing service id", + config: createConfig(t, ` +format_version: '11' +default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git +services: + "": + image: postgres:13`), + wantErr: "service (image: postgres:13) has empty ID defined", + }, + { + name: "Invalid bitrise.yml: missing service image", + config: createConfig(t, ` +format_version: '11' +default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git +services: + "postgres": + image: ""`), + wantErr: "service (postgres) has no image defined", + }, + { + name: "Invalid bitrise.yml: missing service image (whitespace)", + config: createConfig(t, ` +format_version: '11' +default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git +services: + "postgres": + image: " "`), + wantErr: "service (postgres) has no image defined", + }, + { + name: "Invalid bitrise.yml: missing container id", + config: createConfig(t, ` +format_version: '11' +default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git +containers: + "": + image: ruby:3.2`), + wantErr: "service (image: ruby:3.2) has empty ID defined", + }, + { + name: "Invalid bitrise.yml: missing container image", + config: createConfig(t, ` +format_version: '11' +default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git +containers: + "ruby": + image: ""`), + wantErr: "service (ruby) has no image defined", + }, + { + name: "Invalid bitrise.yml: missing container image (whitespace)", + config: createConfig(t, ` +format_version: '11' +default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git +containers: + "ruby": + image: " "`), + wantErr: "service (ruby) has no image defined", + }, + { + name: "Invalid bitrise.yml: non-existing container referenced", + config: createConfig(t, ` +format_version: '11' +default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git +containers: + ruby: + image: ruby:3.2 +workflows: + primary: + steps: + - with: + container: ruby_3_2`), + wantErr: "container (ruby_3_2) referenced in workflow (primary), but this container is not defined", + }, + { + name: "Invalid bitrise.yml: non-existing service referenced", + config: createConfig(t, ` +format_version: '11' +default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git +services: + postgres: + image: postgres:13 +workflows: + primary: + steps: + - with: + services: + - postgres_13`), + wantErr: "service (postgres_13) referenced in workflow (primary), but this service is not defined", + }, + { + name: "Invalid bitrise.yml: service referenced multiple times", + config: createConfig(t, ` +format_version: '11' +default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git +services: + postgres: + image: postgres:13 +workflows: + primary: + steps: + - with: + services: + - postgres + - postgres`), + wantErr: "service (postgres) specified multiple times for workflow (primary)", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + warns, err := tt.config.Validate() + require.Empty(t, warns) + + if tt.wantErr != "" { + require.EqualError(t, err, tt.wantErr) + } else { + require.NoError(t, err) + } + }) + } +} + // Workflow func TestValidateWorkflow(t *testing.T) { t.Log("before-after test") @@ -586,8 +819,7 @@ workflows: _deps-update: ` - config, err := configModelFromYAMLBytes([]byte(configStr)) - require.NoError(t, err) + config := createConfig(t, configStr) warnings, err := config.Validate() require.NoError(t, err) @@ -618,10 +850,9 @@ workflows: ci: ` - config, err := configModelFromYAMLBytes([]byte(configStr)) - require.NoError(t, err) + config := createConfig(t, configStr) - _, err = config.Validate() + _, err := config.Validate() require.EqualError(t, err, "trigger item #1: non-existent pipeline defined as trigger target: release") } @@ -639,10 +870,9 @@ workflows: ci: ` - config, err := configModelFromYAMLBytes([]byte(configStr)) - require.NoError(t, err) + config := createConfig(t, configStr) - _, err = config.Validate() + _, err := config.Validate() require.EqualError(t, err, "trigger item #1: non-existent workflow defined as trigger target: release") } } @@ -903,7 +1133,7 @@ func TestGetStepIDStepDataPair(t *testing.T) { "step1": stepData, } - id, _, err := GetStepIDStepDataPair(stepListItem) + id, _, _, err := stepListItem.GetStepListItemKeyAndValue() require.NoError(t, err) require.Equal(t, "step1", id) } @@ -915,7 +1145,7 @@ func TestGetStepIDStepDataPair(t *testing.T) { "step2": stepData, } - id, _, err := GetStepIDStepDataPair(stepListItem) + id, _, _, err := stepListItem.GetStepListItemKeyAndValue() require.Error(t, err) require.Equal(t, "", id) } @@ -924,7 +1154,7 @@ func TestGetStepIDStepDataPair(t *testing.T) { { stepListItem := StepListItemModel{} - id, _, err := GetStepIDStepDataPair(stepListItem) + id, _, _, err := stepListItem.GetStepListItemKeyAndValue() require.Error(t, err) require.Equal(t, "", id) } @@ -1044,13 +1274,6 @@ func TestRemoveEnvironmentRedundantFields(t *testing.T) { } } -func configModelFromYAMLBytes(configBytes []byte) (bitriseData BitriseDataModel, err error) { - if err = yaml.Unmarshal(configBytes, &bitriseData); err != nil { - return - } - return -} - func TestRemoveWorkflowRedundantFields(t *testing.T) { configStr := `format_version: 2 default_step_lib_source: "https://github.com/bitrise-io/bitrise-steplib.git" @@ -1075,10 +1298,9 @@ workflows: description: test ` - config, err := configModelFromYAMLBytes([]byte(configStr)) - require.NoError(t, err) + config := createConfig(t, configStr) - err = config.RemoveRedundantFields() + err := config.RemoveRedundantFields() require.NoError(t, err) require.Equal(t, "2", config.FormatVersion) @@ -1121,7 +1343,7 @@ workflows: } for _, stepListItem := range workflow.Steps { - _, step, err := GetStepIDStepDataPair(stepListItem) + _, step, _, err := stepListItem.GetStepListItemKeyAndValue() require.NoError(t, err) require.Nil(t, step.Title) @@ -1178,3 +1400,9 @@ func TestBitriseDataModelValidateWorkflowsCircularDependency(t *testing.T) { require.Equal(t, "0", os.Getenv("BITRISE_BUILD_STATUS")) require.Equal(t, "0", os.Getenv("STEPLIB_BUILD_STATUS")) } + +func createConfig(t *testing.T, yamlContent string) BitriseDataModel { + config := BitriseDataModel{} + require.NoError(t, yaml.Unmarshal([]byte(yamlContent), &config)) + return config +} diff --git a/models/workflow_run_plan.go b/models/workflow_run_plan.go index fbf0ae195..35a920fd5 100644 --- a/models/workflow_run_plan.go +++ b/models/workflow_run_plan.go @@ -1,6 +1,10 @@ package models -import "time" +import ( + "time" + + stepmanModels "github.com/bitrise-io/stepman/models" +) type WorkflowRunModes struct { CIMode bool @@ -12,17 +16,23 @@ type WorkflowRunModes struct { IsSteplibOfflineMode bool } +// TODO: dispatch Plans from JSON event logging and actual workflow execution type StepExecutionPlan struct { UUID string `json:"uuid"` StepID string `json:"step_id"` + + Step stepmanModels.StepModel `json:"-"` + GroupID string `json:"-"` + ContainerID string `json:"-"` + ServiceIDs []string `json:"-"` } type WorkflowExecutionPlan struct { - UUID string `json:"uuid"` - WorkflowID string `json:"workflow_id"` - Steps []StepExecutionPlan `json:"steps"` - - IsSteplibOfflineMode bool `json:"-"` + UUID string `json:"uuid"` + WorkflowID string `json:"workflow_id"` + Steps []StepExecutionPlan `json:"steps"` + WorkflowTitle string `json:"-"` + IsSteplibOfflineMode bool `json:"-"` } type WorkflowRunPlan struct {