Skip to content

Commit

Permalink
feat: multi region support
Browse files Browse the repository at this point in the history
  • Loading branch information
@ReuDa
ReuDa authored Dec 2, 2024
1 parent 8ac68af commit 6002ef0
Show file tree
Hide file tree
Showing 71 changed files with 901 additions and 819 deletions.
1 change: 1 addition & 0 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ jobs:
go_version: '1.23'
build_linux_packages: true
VERSION_BUMPER_APPID: ${{ vars.GH_APP_STEADYBIT_APP_ID }}
force_push_docker_image: true
secrets:
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
PAT_TOKEN_EXTENSION_DEPLOYER: ${{ secrets.PAT_TOKEN_EXTENSION_DEPLOYER }}
Expand Down
6 changes: 5 additions & 1 deletion Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
##
FROM --platform=$BUILDPLATFORM goreleaser/goreleaser:v2.4.8 AS build

ARG TARGETOS TARGETARCH
ARG TARGETOS
ARG TARGETARCH
ARG BUILD_WITH_COVERAGE
ARG BUILD_SNAPSHOT=true
ARG SKIP_LICENSES_REPORT=false
Expand All @@ -14,13 +15,16 @@ WORKDIR /app

COPY . .
RUN GOOS=$TARGETOS GOARCH=$TARGETARCH goreleaser build --snapshot="${BUILD_SNAPSHOT}" --single-target -o extension

##
## Runtime
##
FROM alpine:3.20

LABEL "steadybit.com.discovery-disabled"="true"

RUN apk --no-cache add aws-cli

ARG USERNAME=steadybit
ARG USER_UID=10000

Expand Down
15 changes: 14 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ our [Reliability Hub](https://hub.steadybit.com/extension/com.steadybit.extensio
|-----------------------------------------------------------------|-------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-----------------------------------------------------------------------------------------------------------------------------------------------|
| `STEADYBIT_EXTENSION_WORKER_THREADS` | | How many parallel workers should call aws apis (only used if `STEADYBIT_EXTENSION_ASSUME_ROLES` is used) | no | 1 |
| `STEADYBIT_EXTENSION_ASSUME_ROLES` | `aws.assumeRoles` | See detailed description below | no | |
| `STEADYBIT_EXTENSION_REGIONS` | | See detailed description below | no | |
| `STEADYBIT_EXTENSION_DISCOVERY_DISABLED_EC2` | `aws.discovery.disabled.ec2` | Disable EC2-Discovery and all EC2 related definitions | no | false |
| `STEADYBIT_EXTENSION_DISCOVERY_INTERVAL_EC2` | | Discovery-Interval in seconds | no | 30 |
| `STEADYBIT_EXTENSION_DISCOVERY_DISABLED_ECS` | `aws.discovery.disabled.ecs` | Disable ECS-Discovery and all ECS related definitions | no | false |
Expand Down Expand Up @@ -100,7 +101,8 @@ by tweaking the `Resource` clause.
"ec2:DescribeTags",
"ec2:StopInstances",
"ec2:RebootInstances",
"ec2:TerminateInstances"
"ec2:TerminateInstances",
"ec2:StartInstances",
],
"Resource": "*"
}
Expand Down Expand Up @@ -518,6 +520,17 @@ steps:
}
```

### Multi Region Support

By default, the extension will discover targets only in the AWS Region that is provided by the current authentication (environment variable `AWS_REGION`).

If you want to discover targets in multiple regions, you can set the `STEADYBIT_EXTENSION_REGIONS` environment variable to a comma-separated list of regions. Example:

```sh
STEADYBIT_EXTENSION_REGIONS='us-east-1,us-west-2'
```


### Agent Lockout - Requirements

In order to prevent the agent or the extension of beeing locked out by their own attacks, we implemented some security
Expand Down
12 changes: 12 additions & 0 deletions config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package config
import (
"github.com/kelseyhightower/envconfig"
"github.com/rs/zerolog/log"
"strings"
)

var (
Expand All @@ -14,7 +15,18 @@ var (

func ParseConfiguration() {
err := envconfig.Process("steadybit_extension", &Config)
Config.AssumeRoles = trimSpaces(Config.AssumeRoles)
Config.Regions = trimSpaces(Config.Regions)
Config.EnrichEc2DataForTargetTypes = trimSpaces(Config.EnrichEc2DataForTargetTypes)
if err != nil {
log.Fatal().Err(err).Msgf("Failed to parse configuration from environment.")
}
}

func trimSpaces(orig []string) []string {
var trimmed []string
for _, s := range orig {
trimmed = append(trimmed, strings.TrimSpace(s))
}
return trimmed
}
1 change: 1 addition & 0 deletions config/specification.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package config

type Specification struct {
AssumeRoles []string `json:"assumeRoles" split_words:"true" required:"false"`
Regions []string `json:"regions" split_words:"true" required:"false"`
WorkerThreads int `json:"workerThreads" split_words:"true" required:"false" default:"1"`
AwsEndpointOverride string `json:"awsEndpointOverride" split_words:"true" required:"false"`
DiscoveryDisabledEc2 bool `json:"discoveryDisabledEc2" split_words:"true" required:"false" default:"false"`
Expand Down
2 changes: 1 addition & 1 deletion e2e/integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ func TestWithMinikube(t *testing.T) {
"--set", "logging.level=INFO",
"--set", "extraEnv[0].name=STEADYBIT_EXTENSION_AWS_ENDPOINT_OVERRIDE",
"--set", "extraEnv[0].value=http://localstack.default.svc.cluster.local:4566",
"--set", "extraEnv[1].name=AWS_DEFAULT_REGION",
"--set", "extraEnv[1].name=AWS_REGION",
"--set", "extraEnv[1].value=us-east-1",
"--set", "extraEnv[2].name=AWS_ACCESS_KEY_ID",
"--set", "extraEnv[2].value=test",
Expand Down
25 changes: 14 additions & 11 deletions extaz/availability_zone_attack_blackhole.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import (
)

type azBlackholeAction struct {
clientProvider func(account string) (azBlackholeEC2Api, azBlackholeImdsApi, error)
clientProvider func(account string, region string) (azBlackholeEC2Api, azBlackholeImdsApi, error)
extensionRootAccountNumber string
}

Expand All @@ -35,6 +35,7 @@ type BlackholeState struct {
AgentAWSAccount string
ExtensionAwsAccount string
TargetZone string
TargetRegion string
NetworkAclIds []string
OldNetworkAclIds map[string]string // map[NewAssociationId] = oldNetworkAclId
TargetSubnets map[string][]string // map[vpcId] = [subnetIds]
Expand All @@ -57,7 +58,7 @@ type azBlackholeImdsApi interface {
func NewAzBlackholeAction() action_kit_sdk.Action[BlackholeState] {
return &azBlackholeAction{
clientProvider: defaultClientProvider,
extensionRootAccountNumber: utils.Accounts.GetRootAccount().AccountNumber,
extensionRootAccountNumber: utils.GetRootAccountNumber(),
}
}

Expand Down Expand Up @@ -108,9 +109,10 @@ func (e *azBlackholeAction) Describe() action_kit_api.ActionDescription {
func (e *azBlackholeAction) Prepare(ctx context.Context, state *BlackholeState, request action_kit_api.PrepareActionRequestBody) (*action_kit_api.PrepareResult, error) {
targetAccount := extutil.MustHaveValue(request.Target.Attributes, "aws.account")[0]
targetZone := extutil.MustHaveValue(request.Target.Attributes, "aws.zone")[0]
targetRegion := extutil.MustHaveValue(request.Target.Attributes, "aws.region")[0]

// Get AWS Clients
clientEc2, clientImds, err := e.clientProvider(targetAccount)
clientEc2, clientImds, err := e.clientProvider(targetAccount, targetRegion)
if err != nil {
return nil, extension_kit.ToError(fmt.Sprintf("Failed to initialize AWS clients for AWS targetAccount %s", targetAccount), err)
}
Expand Down Expand Up @@ -145,17 +147,18 @@ func (e *azBlackholeAction) Prepare(ctx context.Context, state *BlackholeState,
state.AgentAWSAccount = agentAwsAccountId
state.ExtensionAwsAccount = targetAccount
state.TargetZone = targetZone
state.TargetRegion = targetRegion
state.TargetSubnets = targetSubnets
state.AttackExecutionId = request.ExecutionId
return nil, nil
}

func (e *azBlackholeAction) Start(ctx context.Context, state *BlackholeState) (*action_kit_api.StartResult, error) {
clientEc2, _, err := e.clientProvider(state.ExtensionAwsAccount)
clientEc2, _, err := e.clientProvider(state.ExtensionAwsAccount, state.TargetRegion)
if err != nil {
return nil, extension_kit.ToError(fmt.Sprintf("Failed to initialize EC2 client for AWS account %s", state.ExtensionAwsAccount), err)
}
log.Info().Msgf("Starting AZ Blackhole attack against AWS account %s", state.ExtensionAwsAccount)
log.Info().Msgf("Starting AZ Blackhole attack against AWS account %s and region %s", state.ExtensionAwsAccount, state.TargetRegion)
log.Debug().Msgf("Attack state: %+v", state)

state.OldNetworkAclIds = make(map[string]string)
Expand Down Expand Up @@ -371,9 +374,9 @@ func getNetworkAclAssociations(ctx context.Context, clientEc2 azBlackholeEC2Api,
}

func (e *azBlackholeAction) Stop(ctx context.Context, state *BlackholeState) (*action_kit_api.StopResult, error) {
clientEc2, _, err := e.clientProvider(state.ExtensionAwsAccount)
clientEc2, _, err := e.clientProvider(state.ExtensionAwsAccount, state.TargetRegion)
if err != nil {
return nil, extension_kit.ToError(fmt.Sprintf("Failed to initialize EC2 client for AWS account %s", state.ExtensionAwsAccount), err)
return nil, extension_kit.ToError(fmt.Sprintf("Failed to initialize EC2 client for AWS account %s and region %s", state.ExtensionAwsAccount, state.TargetRegion), err)
}

return nil, rollbackBlackholeViaTags(ctx, state, clientEc2)
Expand Down Expand Up @@ -465,13 +468,13 @@ func getAllNACLsCreatedBySteadybit(clientEc2 azBlackholeEC2Api, ctx context.Cont
return &result, nil
}

func defaultClientProvider(account string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
awsAccount, err := utils.Accounts.GetAccount(account)
func defaultClientProvider(account string, region string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
awsAccess, err := utils.GetAwsAccess(account, region)
if err != nil {
return nil, nil, err
}
clientEc2 := ec2.NewFromConfig(awsAccount.AwsConfig)
clientImds := imds.NewFromConfig(awsAccount.AwsConfig)
clientEc2 := ec2.NewFromConfig(awsAccess.AwsConfig)
clientImds := imds.NewFromConfig(awsAccess.AwsConfig)
if err != nil {
return nil, nil, err
}
Expand Down
5 changes: 4 additions & 1 deletion extaz/availability_zone_attack_blackhole_localstack_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ func testPrepareAndStartAndStopBlackholeLocalStack(t *testing.T, clientEc2 *ec2.
assert.Equal(t, "41", state.AgentAWSAccount)
assert.Equal(t, "42", state.ExtensionAwsAccount)
assert.Equal(t, "eu-west-1a", state.TargetZone)
assert.Equal(t, "eu-west-1", state.TargetRegion)
assert.Len(t, state.TargetSubnets, 2) //default vpc and the one we created
assert.Len(t, state.TargetSubnets[defaultVpcId], 1) //default vpc has 1 subnet
assert.Len(t, state.TargetSubnets[createdVpcId], 2) //our vpc with 2 subnets
Expand All @@ -75,6 +76,7 @@ func testPrepareAndStartAndStopBlackholeLocalStack(t *testing.T, clientEc2 *ec2.
assert.Equal(t, "41", state.AgentAWSAccount)
assert.Equal(t, "42", state.ExtensionAwsAccount)
assert.Equal(t, "eu-west-1a", state.TargetZone)
assert.Equal(t, "eu-west-1", state.TargetRegion)
assert.Len(t, state.NetworkAclIds, 2) //one per vpc
newAssociationIds := reflect.ValueOf(state.OldNetworkAclIds).MapKeys()
assert.NotEqual(t, "", state.OldNetworkAclIds[newAssociationIds[0].String()])
Expand Down Expand Up @@ -261,7 +263,7 @@ func testApiThrottlingDuringStopWhileDeletingNACLs(t *testing.T, clientEc2 *ec2.
func prepareActionCall(clientEc2 *ec2.Client, clientImds *imds.Client) (azBlackholeAction, BlackholeState, action_kit_api.PrepareActionRequestBody) {
action := azBlackholeAction{
extensionRootAccountNumber: "41",
clientProvider: func(account string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
clientProvider: func(account string, region string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
return clientEc2, clientImds, nil
}}
state := action.NewEmptyState()
Expand All @@ -273,6 +275,7 @@ func prepareActionCall(clientEc2 *ec2.Client, clientImds *imds.Client) (azBlackh
Target: extutil.Ptr(action_kit_api.Target{
Attributes: map[string][]string{
"aws.zone": {"eu-west-1a"},
"aws.region": {"eu-west-1"},
"aws.account": {"42"},
},
}),
Expand Down
25 changes: 17 additions & 8 deletions extaz/availability_zone_attack_blackhole_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ func TestPrepareBlackhole(t *testing.T) {
ctx := context.Background()
action := azBlackholeAction{
extensionRootAccountNumber: "",
clientProvider: func(account string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
clientProvider: func(account string, region string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
return clientEc2, clientImds, nil
}}
state := action.NewEmptyState()
Expand All @@ -122,6 +122,7 @@ func TestPrepareBlackhole(t *testing.T) {
Target: extutil.Ptr(action_kit_api.Target{
Attributes: map[string][]string{
"aws.zone": {"eu-west-1a"},
"aws.region": {"eu-west-1"},
"aws.account": {"42"},
},
}),
Expand All @@ -138,6 +139,7 @@ func TestPrepareBlackhole(t *testing.T) {
assert.Equal(t, "41", state.AgentAWSAccount)
assert.Equal(t, "42", state.ExtensionAwsAccount)
assert.Equal(t, "eu-west-1a", state.TargetZone)
assert.Equal(t, "eu-west-1", state.TargetRegion)
assert.Equal(t, []string{"subnet-1", "subnet-2"}, state.TargetSubnets["vpcId-1"])
assert.NotNil(t, state.AttackExecutionId)
clientEc2.AssertExpectations(t)
Expand All @@ -156,7 +158,7 @@ func TestShouldNotAttackWhenExtensionIsInTargetAccountId(t *testing.T) {
ctx := context.Background()
action := azBlackholeAction{
extensionRootAccountNumber: "42",
clientProvider: func(account string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
clientProvider: func(account string, region string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
return nil, clientImds, nil
}}
state := action.NewEmptyState()
Expand All @@ -168,6 +170,7 @@ func TestShouldNotAttackWhenExtensionIsInTargetAccountId(t *testing.T) {
Target: extutil.Ptr(action_kit_api.Target{
Attributes: map[string][]string{
"aws.zone": {"eu-west-1a"},
"aws.region": {"eu-west-1"},
"aws.account": {"42"},
},
}),
Expand All @@ -191,7 +194,7 @@ func TestShouldNotAttackWhenExtensionIsInTargetAccountIdViaStsClient(t *testing.
ctx := context.Background()
action := azBlackholeAction{
extensionRootAccountNumber: "42",
clientProvider: func(account string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
clientProvider: func(account string, region string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
return nil, clientImds, nil
}}
state := action.NewEmptyState()
Expand All @@ -203,6 +206,7 @@ func TestShouldNotAttackWhenExtensionIsInTargetAccountIdViaStsClient(t *testing.
Target: extutil.Ptr(action_kit_api.Target{
Attributes: map[string][]string{
"aws.zone": {"eu-west-1a"},
"aws.region": {"eu-west-1"},
"aws.account": {"42"},
},
}),
Expand All @@ -226,7 +230,7 @@ func TestShouldNotAttackWhenExtensionAccountIsUnknown(t *testing.T) {
ctx := context.Background()
action := azBlackholeAction{
extensionRootAccountNumber: "",
clientProvider: func(account string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
clientProvider: func(account string, region string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
return nil, clientImds, nil
}}
state := action.NewEmptyState()
Expand All @@ -238,6 +242,7 @@ func TestShouldNotAttackWhenExtensionAccountIsUnknown(t *testing.T) {
Target: extutil.Ptr(action_kit_api.Target{
Attributes: map[string][]string{
"aws.zone": {"eu-west-1a"},
"aws.region": {"eu-west-1"},
"aws.account": {"42"},
},
}),
Expand All @@ -261,7 +266,7 @@ func TestShouldNotAttackWhenAgentAccountIsUnknown(t *testing.T) {
ctx := context.Background()
action := azBlackholeAction{
extensionRootAccountNumber: "",
clientProvider: func(account string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
clientProvider: func(account string, region string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
return nil, clientImds, nil
}}
state := action.NewEmptyState()
Expand All @@ -273,6 +278,7 @@ func TestShouldNotAttackWhenAgentAccountIsUnknown(t *testing.T) {
Target: extutil.Ptr(action_kit_api.Target{
Attributes: map[string][]string{
"aws.zone": {"eu-west-1a"},
"aws.region": {"eu-west-1"},
"aws.account": {"42"},
},
}),
Expand Down Expand Up @@ -300,7 +306,7 @@ func TestShouldNotAttackWhenAgentIsInTargetAccountId(t *testing.T) {
ctx := context.Background()
action := azBlackholeAction{
extensionRootAccountNumber: "",
clientProvider: func(account string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
clientProvider: func(account string, region string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
return nil, clientImds, nil
}}
state := action.NewEmptyState()
Expand All @@ -312,6 +318,7 @@ func TestShouldNotAttackWhenAgentIsInTargetAccountId(t *testing.T) {
Target: extutil.Ptr(action_kit_api.Target{
Attributes: map[string][]string{
"aws.zone": {"eu-west-1a"},
"aws.region": {"eu-west-1"},
"aws.account": {"42"},
},
}),
Expand Down Expand Up @@ -395,14 +402,15 @@ func TestStartBlackhole(t *testing.T) {
}), nil)

ctx := context.Background()
action := azBlackholeAction{clientProvider: func(account string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
action := azBlackholeAction{clientProvider: func(account string, region string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
return clientEc2, nil, nil
}}

state := BlackholeState{
AgentAWSAccount: "41",
ExtensionAwsAccount: "43",
TargetZone: "eu-west-1a",
TargetRegion: "eu-west-1",
TargetSubnets: map[string][]string{
"vpcId-1": {"subnet-1", "subnet-2"},
},
Expand Down Expand Up @@ -482,14 +490,15 @@ func TestStopBlackhole(t *testing.T) {
}), mock.Anything).Return(extutil.Ptr(ec2.DeleteNetworkAclOutput{}), nil)

ctx := context.Background()
action := azBlackholeAction{clientProvider: func(account string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
action := azBlackholeAction{clientProvider: func(account string, region string) (azBlackholeEC2Api, azBlackholeImdsApi, error) {
return clientEc2, nil, nil
}}

state := BlackholeState{
AgentAWSAccount: "41",
ExtensionAwsAccount: "43",
TargetZone: "eu-west-1a",
TargetRegion: "eu-west-1",
TargetSubnets: map[string][]string{
"vpcId-1": {"subnet-1", "subnet-2"},
},
Expand Down
Loading

0 comments on commit 6002ef0

Please sign in to comment.