Multi Region Support

.github/workflows/ci.yml

@@ -21,6 +21,7 @@ jobs:
       go_version: '1.23'
       build_linux_packages: true
       VERSION_BUMPER_APPID: ${{ vars.GH_APP_STEADYBIT_APP_ID }}
+      force_push_docker_image: true
     secrets:
       SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
       PAT_TOKEN_EXTENSION_DEPLOYER: ${{ secrets.PAT_TOKEN_EXTENSION_DEPLOYER }}

Dockerfile

@@ -5,7 +5,8 @@
 ##
 FROM --platform=$BUILDPLATFORM goreleaser/goreleaser:v2.4.8 AS build
 
-ARG TARGETOS TARGETARCH
+ARG TARGETOS
+ARG TARGETARCH
 ARG BUILD_WITH_COVERAGE
 ARG BUILD_SNAPSHOT=true
 ARG SKIP_LICENSES_REPORT=false
@@ -14,13 +15,16 @@ WORKDIR /app
 
 COPY . .
 RUN GOOS=$TARGETOS GOARCH=$TARGETARCH goreleaser build --snapshot="${BUILD_SNAPSHOT}" --single-target -o extension
+
 ##
 ## Runtime
 ##
 FROM alpine:3.20
 
 LABEL "steadybit.com.discovery-disabled"="true"
 
+RUN apk --no-cache add aws-cli
+
 ARG USERNAME=steadybit
 ARG USER_UID=10000
 

README.md

@@ -14,6 +14,7 @@ our [Reliability Hub](https://hub.steadybit.com/extension/com.steadybit.extensio
 |-----------------------------------------------------------------|-------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-----------------------------------------------------------------------------------------------------------------------------------------------|
 | `STEADYBIT_EXTENSION_WORKER_THREADS`                            |                                                 | How many parallel workers should call aws apis (only used if `STEADYBIT_EXTENSION_ASSUME_ROLES` is used)                                                      | no       | 1                                                                                                                                             |
 | `STEADYBIT_EXTENSION_ASSUME_ROLES`                              | `aws.assumeRoles`                               | See detailed description below                                                                                                                                | no       |                                                                                                                                               |
+| `STEADYBIT_EXTENSION_REGIONS`                                   |                                                 | See detailed description below                                                                                                                                | no       |                                                                                                                                               |
 | `STEADYBIT_EXTENSION_DISCOVERY_DISABLED_EC2`                    | `aws.discovery.disabled.ec2`                    | Disable EC2-Discovery and all EC2 related definitions                                                                                                         | no       | false                                                                                                                                         |
 | `STEADYBIT_EXTENSION_DISCOVERY_INTERVAL_EC2`                    |                                                 | Discovery-Interval in seconds                                                                                                                                 | no       | 30                                                                                                                                            |
 | `STEADYBIT_EXTENSION_DISCOVERY_DISABLED_ECS`                    | `aws.discovery.disabled.ecs`                    | Disable ECS-Discovery and all ECS related definitions                                                                                                         | no       | false                                                                                                                                         |
@@ -100,7 +101,8 @@ by tweaking the `Resource` clause.
         "ec2:DescribeTags",
         "ec2:StopInstances",
         "ec2:RebootInstances",
-        "ec2:TerminateInstances"
+        "ec2:TerminateInstances",
+				"ec2:StartInstances",
       ],
       "Resource": "*"
     }
@@ -518,6 +520,17 @@ steps:
    }
    ```
 
+### Multi Region Support
+
+By default, the extension will discover targets only in the AWS Region that is provided by the current authentication (environment variable `AWS_REGION`).
+
+If you want to discover targets in multiple regions, you can set the `STEADYBIT_EXTENSION_REGIONS` environment variable to a comma-separated list of regions. Example:
+
+```sh
+STEADYBIT_EXTENSION_REGIONS='us-east-1,us-west-2'
+```
+
+
 ### Agent Lockout - Requirements
 
 In order to prevent the agent or the extension of beeing locked out by their own attacks, we implemented some security

config/config.go

@@ -6,6 +6,7 @@ package config
 import (
 	"github.com/kelseyhightower/envconfig"
 	"github.com/rs/zerolog/log"
+	"strings"
 )
 
 var (
@@ -14,7 +15,18 @@ var (
 
 func ParseConfiguration() {
 	err := envconfig.Process("steadybit_extension", &Config)
+	Config.AssumeRoles = trimSpaces(Config.AssumeRoles)
+	Config.Regions = trimSpaces(Config.Regions)
+	Config.EnrichEc2DataForTargetTypes = trimSpaces(Config.EnrichEc2DataForTargetTypes)
 	if err != nil {
 		log.Fatal().Err(err).Msgf("Failed to parse configuration from environment.")
 	}
 }
+
+func trimSpaces(orig []string) []string {
+	var trimmed []string
+	for _, s := range orig {
+		trimmed = append(trimmed, strings.TrimSpace(s))
+	}
+	return trimmed
+}

config/specification.go

@@ -5,6 +5,7 @@ package config
 
 type Specification struct {
 	AssumeRoles                                  []string `json:"assumeRoles" split_words:"true" required:"false"`
+	Regions                                      []string `json:"regions" split_words:"true" required:"false"`
 	WorkerThreads                                int      `json:"workerThreads" split_words:"true" required:"false" default:"1"`
 	AwsEndpointOverride                          string   `json:"awsEndpointOverride" split_words:"true" required:"false"`
 	DiscoveryDisabledEc2                         bool     `json:"discoveryDisabledEc2" split_words:"true" required:"false" default:"false"`

e2e/integration_test.go

@@ -23,7 +23,7 @@ func TestWithMinikube(t *testing.T) {
 				"--set", "logging.level=INFO",
 				"--set", "extraEnv[0].name=STEADYBIT_EXTENSION_AWS_ENDPOINT_OVERRIDE",
 				"--set", "extraEnv[0].value=http://localstack.default.svc.cluster.local:4566",
-				"--set", "extraEnv[1].name=AWS_DEFAULT_REGION",
+				"--set", "extraEnv[1].name=AWS_REGION",
 				"--set", "extraEnv[1].value=us-east-1",
 				"--set", "extraEnv[2].name=AWS_ACCESS_KEY_ID",
 				"--set", "extraEnv[2].value=test",

extaz/availability_zone_attack_blackhole.go

@@ -23,7 +23,7 @@ import (
 )
 
 type azBlackholeAction struct {
-	clientProvider             func(account string) (azBlackholeEC2Api, azBlackholeImdsApi, error)
+	clientProvider             func(account string, region string) (azBlackholeEC2Api, azBlackholeImdsApi, error)
 	extensionRootAccountNumber string
 }
 
@@ -35,6 +35,7 @@ type BlackholeState struct {
 	AgentAWSAccount     string
 	ExtensionAwsAccount string
 	TargetZone          string
+	TargetRegion        string
 	NetworkAclIds       []string
 	OldNetworkAclIds    map[string]string   // map[NewAssociationId] = oldNetworkAclId
 	TargetSubnets       map[string][]string // map[vpcId] = [subnetIds]
@@ -57,7 +58,7 @@ type azBlackholeImdsApi interface {
 func NewAzBlackholeAction() action_kit_sdk.Action[BlackholeState] {
 	return &azBlackholeAction{
 		clientProvider:             defaultClientProvider,
-		extensionRootAccountNumber: utils.Accounts.GetRootAccount().AccountNumber,
+		extensionRootAccountNumber: utils.GetRootAccountNumber(),
 	}
 }
 
@@ -108,9 +109,10 @@ func (e *azBlackholeAction) Describe() action_kit_api.ActionDescription {
 func (e *azBlackholeAction) Prepare(ctx context.Context, state *BlackholeState, request action_kit_api.PrepareActionRequestBody) (*action_kit_api.PrepareResult, error) {
 	targetAccount := extutil.MustHaveValue(request.Target.Attributes, "aws.account")[0]
 	targetZone := extutil.MustHaveValue(request.Target.Attributes, "aws.zone")[0]
+	targetRegion := extutil.MustHaveValue(request.Target.Attributes, "aws.region")[0]
 
 	// Get AWS Clients
-	clientEc2, clientImds, err := e.clientProvider(targetAccount)
+	clientEc2, clientImds, err := e.clientProvider(targetAccount, targetRegion)
 	if err != nil {
 		return nil, extension_kit.ToError(fmt.Sprintf("Failed to initialize AWS clients for AWS targetAccount %s", targetAccount), err)
 	}
@@ -145,17 +147,18 @@ func (e *azBlackholeAction) Prepare(ctx context.Context, state *BlackholeState,
 	state.AgentAWSAccount = agentAwsAccountId
 	state.ExtensionAwsAccount = targetAccount
 	state.TargetZone = targetZone
+	state.TargetRegion = targetRegion
 	state.TargetSubnets = targetSubnets
 	state.AttackExecutionId = request.ExecutionId
 	return nil, nil
 }
 
 func (e *azBlackholeAction) Start(ctx context.Context, state *BlackholeState) (*action_kit_api.StartResult, error) {
-	clientEc2, _, err := e.clientProvider(state.ExtensionAwsAccount)
+	clientEc2, _, err := e.clientProvider(state.ExtensionAwsAccount, state.TargetRegion)
 	if err != nil {
 		return nil, extension_kit.ToError(fmt.Sprintf("Failed to initialize EC2 client for AWS account %s", state.ExtensionAwsAccount), err)
 	}
-	log.Info().Msgf("Starting AZ Blackhole attack against AWS account %s", state.ExtensionAwsAccount)
+	log.Info().Msgf("Starting AZ Blackhole attack against AWS account %s and region %s", state.ExtensionAwsAccount, state.TargetRegion)
 	log.Debug().Msgf("Attack state: %+v", state)
 
 	state.OldNetworkAclIds = make(map[string]string)
@@ -371,9 +374,9 @@ func getNetworkAclAssociations(ctx context.Context, clientEc2 azBlackholeEC2Api,
 }
 
 func (e *azBlackholeAction) Stop(ctx context.Context, state *BlackholeState) (*action_kit_api.StopResult, error) {
-	clientEc2, _, err := e.clientProvider(state.ExtensionAwsAccount)
+	clientEc2, _, err := e.clientProvider(state.ExtensionAwsAccount, state.TargetRegion)
 	if err != nil {
-		return nil, extension_kit.ToError(fmt.Sprintf("Failed to initialize EC2 client for AWS account %s", state.ExtensionAwsAccount), err)
+		return nil, extension_kit.ToError(fmt.Sprintf("Failed to initialize EC2 client for AWS account %s and region %s", state.ExtensionAwsAccount, state.TargetRegion), err)
 	}
 
 	return nil, rollbackBlackholeViaTags(ctx, state, clientEc2)
@@ -465,13 +468,13 @@ func getAllNACLsCreatedBySteadybit(clientEc2 azBlackholeEC2Api, ctx context.Cont
 	return &result, nil
 }
 
-func defaultClientProvider(account string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
-	awsAccount, err := utils.Accounts.GetAccount(account)
+func defaultClientProvider(account string, region string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
+	awsAccess, err := utils.GetAwsAccess(account, region)
 	if err != nil {
 		return nil, nil, err
 	}
-	clientEc2 := ec2.NewFromConfig(awsAccount.AwsConfig)
-	clientImds := imds.NewFromConfig(awsAccount.AwsConfig)
+	clientEc2 := ec2.NewFromConfig(awsAccess.AwsConfig)
+	clientImds := imds.NewFromConfig(awsAccess.AwsConfig)
 	if err != nil {
 		return nil, nil, err
 	}

extaz/availability_zone_attack_blackhole_localstack_test.go

@@ -55,6 +55,7 @@ func testPrepareAndStartAndStopBlackholeLocalStack(t *testing.T, clientEc2 *ec2.
 	assert.Equal(t, "41", state.AgentAWSAccount)
 	assert.Equal(t, "42", state.ExtensionAwsAccount)
 	assert.Equal(t, "eu-west-1a", state.TargetZone)
+	assert.Equal(t, "eu-west-1", state.TargetRegion)
 	assert.Len(t, state.TargetSubnets, 2)               //default vpc and the one we created
 	assert.Len(t, state.TargetSubnets[defaultVpcId], 1) //default vpc has 1 subnet
 	assert.Len(t, state.TargetSubnets[createdVpcId], 2) //our vpc with 2 subnets
@@ -75,6 +76,7 @@ func testPrepareAndStartAndStopBlackholeLocalStack(t *testing.T, clientEc2 *ec2.
 	assert.Equal(t, "41", state.AgentAWSAccount)
 	assert.Equal(t, "42", state.ExtensionAwsAccount)
 	assert.Equal(t, "eu-west-1a", state.TargetZone)
+	assert.Equal(t, "eu-west-1", state.TargetRegion)
 	assert.Len(t, state.NetworkAclIds, 2) //one per vpc
 	newAssociationIds := reflect.ValueOf(state.OldNetworkAclIds).MapKeys()
 	assert.NotEqual(t, "", state.OldNetworkAclIds[newAssociationIds[0].String()])
@@ -261,7 +263,7 @@ func testApiThrottlingDuringStopWhileDeletingNACLs(t *testing.T, clientEc2 *ec2.
 func prepareActionCall(clientEc2 *ec2.Client, clientImds *imds.Client) (azBlackholeAction, BlackholeState, action_kit_api.PrepareActionRequestBody) {
 	action := azBlackholeAction{
 		extensionRootAccountNumber: "41",
-		clientProvider: func(account string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
+		clientProvider: func(account string, region string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
 			return clientEc2, clientImds, nil
 		}}
 	state := action.NewEmptyState()
@@ -273,6 +275,7 @@ func prepareActionCall(clientEc2 *ec2.Client, clientImds *imds.Client) (azBlackh
 		Target: extutil.Ptr(action_kit_api.Target{
 			Attributes: map[string][]string{
 				"aws.zone":    {"eu-west-1a"},
+				"aws.region":  {"eu-west-1"},
 				"aws.account": {"42"},
 			},
 		}),

extaz/availability_zone_attack_blackhole_test.go

@@ -110,7 +110,7 @@ func TestPrepareBlackhole(t *testing.T) {
 	ctx := context.Background()
 	action := azBlackholeAction{
 		extensionRootAccountNumber: "",
-		clientProvider: func(account string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
+		clientProvider: func(account string, region string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
 			return clientEc2, clientImds, nil
 		}}
 	state := action.NewEmptyState()
@@ -122,6 +122,7 @@ func TestPrepareBlackhole(t *testing.T) {
 		Target: extutil.Ptr(action_kit_api.Target{
 			Attributes: map[string][]string{
 				"aws.zone":    {"eu-west-1a"},
+				"aws.region":  {"eu-west-1"},
 				"aws.account": {"42"},
 			},
 		}),
@@ -138,6 +139,7 @@ func TestPrepareBlackhole(t *testing.T) {
 	assert.Equal(t, "41", state.AgentAWSAccount)
 	assert.Equal(t, "42", state.ExtensionAwsAccount)
 	assert.Equal(t, "eu-west-1a", state.TargetZone)
+	assert.Equal(t, "eu-west-1", state.TargetRegion)
 	assert.Equal(t, []string{"subnet-1", "subnet-2"}, state.TargetSubnets["vpcId-1"])
 	assert.NotNil(t, state.AttackExecutionId)
 	clientEc2.AssertExpectations(t)
@@ -156,7 +158,7 @@ func TestShouldNotAttackWhenExtensionIsInTargetAccountId(t *testing.T) {
 	ctx := context.Background()
 	action := azBlackholeAction{
 		extensionRootAccountNumber: "42",
-		clientProvider: func(account string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
+		clientProvider: func(account string, region string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
 			return nil, clientImds, nil
 		}}
 	state := action.NewEmptyState()
@@ -168,6 +170,7 @@ func TestShouldNotAttackWhenExtensionIsInTargetAccountId(t *testing.T) {
 		Target: extutil.Ptr(action_kit_api.Target{
 			Attributes: map[string][]string{
 				"aws.zone":    {"eu-west-1a"},
+				"aws.region":  {"eu-west-1"},
 				"aws.account": {"42"},
 			},
 		}),
@@ -191,7 +194,7 @@ func TestShouldNotAttackWhenExtensionIsInTargetAccountIdViaStsClient(t *testing.
 	ctx := context.Background()
 	action := azBlackholeAction{
 		extensionRootAccountNumber: "42",
-		clientProvider: func(account string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
+		clientProvider: func(account string, region string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
 			return nil, clientImds, nil
 		}}
 	state := action.NewEmptyState()
@@ -203,6 +206,7 @@ func TestShouldNotAttackWhenExtensionIsInTargetAccountIdViaStsClient(t *testing.
 		Target: extutil.Ptr(action_kit_api.Target{
 			Attributes: map[string][]string{
 				"aws.zone":    {"eu-west-1a"},
+				"aws.region":  {"eu-west-1"},
 				"aws.account": {"42"},
 			},
 		}),
@@ -226,7 +230,7 @@ func TestShouldNotAttackWhenExtensionAccountIsUnknown(t *testing.T) {
 	ctx := context.Background()
 	action := azBlackholeAction{
 		extensionRootAccountNumber: "",
-		clientProvider: func(account string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
+		clientProvider: func(account string, region string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
 			return nil, clientImds, nil
 		}}
 	state := action.NewEmptyState()
@@ -238,6 +242,7 @@ func TestShouldNotAttackWhenExtensionAccountIsUnknown(t *testing.T) {
 		Target: extutil.Ptr(action_kit_api.Target{
 			Attributes: map[string][]string{
 				"aws.zone":    {"eu-west-1a"},
+				"aws.region":  {"eu-west-1"},
 				"aws.account": {"42"},
 			},
 		}),
@@ -261,7 +266,7 @@ func TestShouldNotAttackWhenAgentAccountIsUnknown(t *testing.T) {
 	ctx := context.Background()
 	action := azBlackholeAction{
 		extensionRootAccountNumber: "",
-		clientProvider: func(account string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
+		clientProvider: func(account string, region string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
 			return nil, clientImds, nil
 		}}
 	state := action.NewEmptyState()
@@ -273,6 +278,7 @@ func TestShouldNotAttackWhenAgentAccountIsUnknown(t *testing.T) {
 		Target: extutil.Ptr(action_kit_api.Target{
 			Attributes: map[string][]string{
 				"aws.zone":    {"eu-west-1a"},
+				"aws.region":  {"eu-west-1"},
 				"aws.account": {"42"},
 			},
 		}),
@@ -300,7 +306,7 @@ func TestShouldNotAttackWhenAgentIsInTargetAccountId(t *testing.T) {
 	ctx := context.Background()
 	action := azBlackholeAction{
 		extensionRootAccountNumber: "",
-		clientProvider: func(account string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
+		clientProvider: func(account string, region string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
 			return nil, clientImds, nil
 		}}
 	state := action.NewEmptyState()
@@ -312,6 +318,7 @@ func TestShouldNotAttackWhenAgentIsInTargetAccountId(t *testing.T) {
 		Target: extutil.Ptr(action_kit_api.Target{
 			Attributes: map[string][]string{
 				"aws.zone":    {"eu-west-1a"},
+				"aws.region":  {"eu-west-1"},
 				"aws.account": {"42"},
 			},
 		}),
@@ -395,14 +402,15 @@ func TestStartBlackhole(t *testing.T) {
 	}), nil)
 
 	ctx := context.Background()
-	action := azBlackholeAction{clientProvider: func(account string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
+	action := azBlackholeAction{clientProvider: func(account string, region string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
 		return clientEc2, nil, nil
 	}}
 
 	state := BlackholeState{
 		AgentAWSAccount:     "41",
 		ExtensionAwsAccount: "43",
 		TargetZone:          "eu-west-1a",
+		TargetRegion:        "eu-west-1",
 		TargetSubnets: map[string][]string{
 			"vpcId-1": {"subnet-1", "subnet-2"},
 		},
@@ -482,14 +490,15 @@ func TestStopBlackhole(t *testing.T) {
 	}), mock.Anything).Return(extutil.Ptr(ec2.DeleteNetworkAclOutput{}), nil)
 
 	ctx := context.Background()
-	action := azBlackholeAction{clientProvider: func(account string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
+	action := azBlackholeAction{clientProvider: func(account string, region string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
 		return clientEc2, nil, nil
 	}}
 
 	state := BlackholeState{
 		AgentAWSAccount:     "41",
 		ExtensionAwsAccount: "43",
 		TargetZone:          "eu-west-1a",
+		TargetRegion:        "eu-west-1",
 		TargetSubnets: map[string][]string{
 			"vpcId-1": {"subnet-1", "subnet-2"},
 		},