From 17e6265ccbae113814f0f8877175788492089ff8 Mon Sep 17 00:00:00 2001 From: Daniel Reuter Date: Mon, 13 Jan 2025 15:14:06 +0100 Subject: [PATCH] chore: add vpc id/name to all resources in a vpc --- CHANGELOG.md | 1 + README.md | 20 +-- charts/steadybit-extension-aws/Chart.yaml | 2 +- charts/steadybit-extension-aws/values.yaml | 2 + config/specification.go | 1 + extec2/availablity_zone_discovery.go | 9 +- extec2/availablity_zone_discovery_test.go | 9 +- extec2/common_test.go | 14 +- extec2/ec2_util.go | 132 ++++++++++++++++++ .../ec2_util_test.go | 33 ++--- extec2/instance_discovery.go | 17 ++- extec2/instance_discovery_test.go | 27 ++-- extecs/task_discovery.go | 16 ++- extecs/task_discovery_test.go | 10 +- extelb/alb_discovery.go | 21 ++- extelb/alb_discovery_test.go | 15 +- extlambda/discovery.go | 13 +- extlambda/discovery_test.go | 27 +++- extrds/common_instance_test.go | 5 + extrds/instance_discovery.go | 20 ++- extrds/instance_discovery_test.go | 9 +- main.go | 2 +- utils/aws_zones.go | 92 ------------ 23 files changed, 314 insertions(+), 183 deletions(-) create mode 100644 extec2/ec2_util.go rename utils/aws_zones_test.go => extec2/ec2_util_test.go (50%) delete mode 100644 utils/aws_zones.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 503f1af..6fb23b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - ignore ecs services and tasks with a tag `steadybit.com/discovery-disabled` set to `true` - don't cache zones forever (for example removed permissions should lead to removed targets in the platform) - include tags in the discovery of Lambda functions (requires new permission `tag:GetResources`) +- add vpc name to targets (requires new permission `ec2:DescribeVpcs`, can be disabled by `STEADYBIT_EXTENSION_DISCOVERY_DISABLED_VPC`) ## v2.3.6 diff --git a/README.md b/README.md index ac50c23..9f9b449 100644 --- a/README.md +++ b/README.md @@ -15,24 +15,25 @@ our [Reliability Hub](https://hub.steadybit.com/extension/com.steadybit.extensio | `STEADYBIT_EXTENSION_WORKER_THREADS` | | How many parallel workers should call aws apis (only used if `STEADYBIT_EXTENSION_ASSUME_ROLES` is used) | no | 1 | | `STEADYBIT_EXTENSION_ASSUME_ROLES` | `aws.assumeRoles` | See detailed description below | no | | | `STEADYBIT_EXTENSION_REGIONS` | `aws.regions` | See detailed description below | no | | -| `STEADYBIT_EXTENSION_DISCOVERY_DISABLED_EC2` | `aws.discovery.disabled.ec2` | Disable EC2-Discovery and all EC2 related definitions | no | false | +| `STEADYBIT_EXTENSION_DISCOVERY_DISABLED_EC2` | `aws.discovery.disabled.ec2` | Disable EC2-Discovery and all related definitions | no | false | | `STEADYBIT_EXTENSION_DISCOVERY_INTERVAL_EC2` | | Discovery-Interval in seconds | no | 30 | -| `STEADYBIT_EXTENSION_DISCOVERY_DISABLED_ECS` | `aws.discovery.disabled.ecs` | Disable ECS-Discovery and all ECS related definitions | no | false | +| `STEADYBIT_EXTENSION_DISCOVERY_DISABLED_ECS` | `aws.discovery.disabled.ecs` | Disable ECS-Discovery and all related definitions | no | false | | `STEADYBIT_EXTENSION_DISCOVERY_INTERVAL_ECS_TASK` | | Discovery-Interval in seconds | no | 30 | | `STEADYBIT_EXTENSION_DISCOVERY_INTERVAL_ECS_SERVICE` | | Discovery-Interval in seconds | no | 30 | -| `STEADYBIT_EXTENSION_DISCOVERY_DISABLED_ELASTICACHE` | `aws.discovery.disabled.elasticache` | Disable Elasticache-Discovery and all Elasticache related definitions | no | true | +| `STEADYBIT_EXTENSION_DISCOVERY_DISABLED_ELASTICACHE` | `aws.discovery.disabled.elasticache` | Disable Elasticache-Discovery and all related definitions | no | true | | `STEADYBIT_EXTENSION_DISCOVERY_INTERVAL_ELASTICACHE` | | Discovery-Interval in seconds | no | 30 | -| `STEADYBIT_EXTENSION_DISCOVERY_DISABLED_ELB` | `aws.discovery.disabled.elb` | Disable ELB-Discovery and all ECS related definitions | no | false | +| `STEADYBIT_EXTENSION_DISCOVERY_DISABLED_ELB` | `aws.discovery.disabled.elb` | Disable ELB-Discovery and all related definitions | no | false | | `STEADYBIT_EXTENSION_DISCOVERY_INTERVAL_ELB_ALB` | | Discovery-Interval in seconds | no | 30 | -| `STEADYBIT_EXTENSION_DISCOVERY_DISABLED_FIS` | `aws.discovery.disabled.fis` | Disable FIS-Discovery and all FIS related definitions | no | false | +| `STEADYBIT_EXTENSION_DISCOVERY_DISABLED_FIS` | `aws.discovery.disabled.fis` | Disable FIS-Discovery and all related definitions | no | false | | `STEADYBIT_EXTENSION_DISCOVERY_INTERVAL_FIS` | | Discovery-Interval in seconds | no | 300 | -| `STEADYBIT_EXTENSION_DISCOVERY_DISABLED_MSK` | `aws.discovery.disabled.msk` | Disable MSK-Discovery and all MSK related definitions | no | false | +| `STEADYBIT_EXTENSION_DISCOVERY_DISABLED_MSK` | `aws.discovery.disabled.msk` | Disable MSK-Discovery and all related definitions | no | false | | `STEADYBIT_EXTENSION_DISCOVERY_INTERVAL_MSK` | | Discovery-Interval in seconds | no | 30 | -| `STEADYBIT_EXTENSION_DISCOVERY_DISABLED_LAMBDA` | `aws.discovery.disabled.lambda` | Disable Lambda-Discovery and all Lambda related definitions | no | false | +| `STEADYBIT_EXTENSION_DISCOVERY_DISABLED_LAMBDA` | `aws.discovery.disabled.lambda` | Disable Lambda-Discovery and all related definitions | no | false | | `STEADYBIT_EXTENSION_DISCOVERY_INTERVAL_LAMBDA` | | Discovery-Interval in seconds | no | 60 | -| `STEADYBIT_EXTENSION_DISCOVERY_DISABLED_RDS` | `aws.discovery.disabled.rds` | Disable RDS-Discovery and all RDS related definitions | no | false | +| `STEADYBIT_EXTENSION_DISCOVERY_DISABLED_RDS` | `aws.discovery.disabled.rds` | Disable RDS-Discovery and all related definitions | no | false | | `STEADYBIT_EXTENSION_DISCOVERY_INTERVAL_RDS` | | Discovery-Interval in seconds | no | 30 | -| `STEADYBIT_EXTENSION_DISCOVERY_DISABLED_ZONE` | `aws.discovery.disabled.zone` | Disable Zone-Discovery and all Zone related definitions | no | false | +| `STEADYBIT_EXTENSION_DISCOVERY_DISABLED_VPC` | `aws.discovery.disabled.vpc` | Disable VPC-Discovery and all related definitions | no | false | +| `STEADYBIT_EXTENSION_DISCOVERY_DISABLED_ZONE` | `aws.discovery.disabled.zone` | Disable Zone-Discovery and all related definitions | no | false | | `STEADYBIT_EXTENSION_DISCOVERY_INTERVAL_ZONE` | | Discovery-Interval in seconds | no | 300 | | `STEADYBIT_EXTENSION_ENRICH_EC2_DATA_FOR_TARGET_TYPES` | | These target types will be enriched with EC2 data. They must have the attribute specified by 'STEADYBIT_EXTENSION_ENRICH_EC2_DATA_MATCHER_ATTRIBUTE' for this | no | com.steadybit.extension_jvm.jvm-instance,com.steadybit.extension_container.container,com.steadybit.extension_kubernetes.kubernetes-deployment | | `STEADYBIT_EXTENSION_ENRICH_EC2_DATA_MATCHER_ATTRIBUTE` | | Targets for EC2 Data enrichment will be matched by this attribute. | no | host.hostname | @@ -73,6 +74,7 @@ by tweaking the `Resource` clause. "ec2:DescribeAvailabilityZones", "ec2:DescribeSubnets", "ec2:DescribeNetworkAcls", + "ec2:DescribeVpcs", "ec2:CreateNetworkAcl", "ec2:CreateNetworkAclEntry", "ec2:ReplaceNetworkAclAssociation", diff --git a/charts/steadybit-extension-aws/Chart.yaml b/charts/steadybit-extension-aws/Chart.yaml index 29c3006..006040d 100644 --- a/charts/steadybit-extension-aws/Chart.yaml +++ b/charts/steadybit-extension-aws/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: steadybit-extension-aws description: Steadybit AWS extension Helm chart for Kubernetes. -version: 2.1.23 +version: 2.1.24 appVersion: v2.3.6 home: https://www.steadybit.com/ icon: https://steadybit-website-assets.s3.amazonaws.com/logo-symbol-transparent.png diff --git a/charts/steadybit-extension-aws/values.yaml b/charts/steadybit-extension-aws/values.yaml index 7aef7a1..4bc3d3f 100644 --- a/charts/steadybit-extension-aws/values.yaml +++ b/charts/steadybit-extension-aws/values.yaml @@ -25,6 +25,8 @@ aws: lambda: false # aws.discovery.disabled.rds -- Disables RDS discovery and the related actions. rds: false + # aws.discovery.disabled.vpc -- Disables VPC discovery and the related actions. + vpc: false # aws.discovery.disabled.zone -- Disables AZ discovery and the related actions. zone: false attributes: diff --git a/config/specification.go b/config/specification.go index 1d34647..2d30581 100644 --- a/config/specification.go +++ b/config/specification.go @@ -18,6 +18,7 @@ type Specification struct { DiscoveryDisabledRds bool `json:"discoveryDisabledRds" split_words:"true" required:"false" default:"false"` DiscoveryDisabledZone bool `json:"discoveryDisabledZone" split_words:"true" required:"false" default:"false"` DiscoveryIntervalEc2 int `json:"discoveryIntervalEc2" split_words:"true" required:"false" default:"30"` + DiscoveryDisabledVpc bool `json:"discoveryDisabledVpc" split_words:"true" required:"false" default:"false"` DiscoveryIntervalEcsService int `json:"discoveryIntervalEcsService" split_words:"true" required:"false" default:"30"` DiscoveryIntervalEcsTask int `json:"discoveryIntervalEcsTask" split_words:"true" required:"false" default:"30"` DiscoveryIntervalElasticacheReplicationGroup int `json:"discoveryIntervalElasticacheReplicationGroup" split_words:"true" required:"false" default:"30"` diff --git a/extec2/availablity_zone_discovery.go b/extec2/availablity_zone_discovery.go index c2845cc..2d54a9e 100644 --- a/extec2/availablity_zone_discovery.go +++ b/extec2/availablity_zone_discovery.go @@ -45,7 +45,7 @@ func (a *azDiscovery) Describe() discovery_kit_api.DiscoveryDescription { func (a *azDiscovery) DescribeTarget() discovery_kit_api.TargetDescription { return discovery_kit_api.TargetDescription{ Id: azTargetType, - Label: discovery_kit_api.PluralLabel{One: "Availability Zone", Other: "Availability Zones"}, + Label: discovery_kit_api.PluralLabel{One: "Availability Zone", Other: "Availability Util"}, Category: extutil.Ptr("cloud"), Version: extbuild.GetSemverVersionStringOrUnknown(), Icon: extutil.Ptr(azIcon), @@ -70,12 +70,13 @@ func (a *azDiscovery) DiscoverTargets(ctx context.Context) ([]discovery_kit_api. } func getAllAvailabilityZonesForAccount(account *utils.AwsAccess, ctx context.Context) ([]discovery_kit_api.Target, error) { - return getAllAvailabilityZones(utils.Zones, account, ctx), nil + _, _ = InitEc2UtilForAccount(account, ctx) + return getAllAvailabilityZonesFromCache(Util, account), nil } -func getAllAvailabilityZones(zones utils.GetZonesUtil, account *utils.AwsAccess, ctx context.Context) []discovery_kit_api.Target { +func getAllAvailabilityZonesFromCache(getZonesUtil GetZonesUtil, account *utils.AwsAccess) []discovery_kit_api.Target { result := make([]discovery_kit_api.Target, 0, 20) - for _, availabilityZone := range zones.GetZones(account, ctx, true) { + for _, availabilityZone := range getZonesUtil.GetZones(account) { result = append(result, toAvailabilityZoneTarget(availabilityZone, account.AccountNumber)) } return discovery_kit_commons.ApplyAttributeExcludes(result, config.Config.DiscoveryAttributesExcludesZone) diff --git a/extec2/availablity_zone_discovery_test.go b/extec2/availablity_zone_discovery_test.go index 70cf1d0..109365c 100644 --- a/extec2/availablity_zone_discovery_test.go +++ b/extec2/availablity_zone_discovery_test.go @@ -4,7 +4,6 @@ package extec2 import ( - "context" "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/steadybit/discovery-kit/go/discovery_kit_api" "github.com/steadybit/extension-aws/utils" @@ -16,7 +15,7 @@ import ( func TestGetAllAvailabilityZones(t *testing.T) { // Given - mockedApi := new(zoneMock) + mockedApi := new(ec2UtilMock) mockedReturnValue := []types.AvailabilityZone{ { ZoneName: discovery_kit_api.Ptr("eu-central-1b"), @@ -24,12 +23,12 @@ func TestGetAllAvailabilityZones(t *testing.T) { ZoneId: discovery_kit_api.Ptr("euc1-az3"), }, } - mockedApi.On("GetZones", mock.Anything, mock.Anything, mock.Anything).Return(mockedReturnValue) + mockedApi.On("GetZones", mock.Anything).Return(mockedReturnValue) // When - targets := getAllAvailabilityZones(mockedApi, &utils.AwsAccess{ + targets := getAllAvailabilityZonesFromCache(mockedApi, &utils.AwsAccess{ AccountNumber: "42", - }, context.Background()) + }) // Then assert.Equal(t, 1, len(targets)) diff --git a/extec2/common_test.go b/extec2/common_test.go index 1348074..8df5d56 100644 --- a/extec2/common_test.go +++ b/extec2/common_test.go @@ -4,22 +4,24 @@ package extec2 import ( - "context" "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/steadybit/extension-aws/utils" "github.com/stretchr/testify/mock" ) -type zoneMock struct { +type ec2UtilMock struct { mock.Mock } -func (m *zoneMock) GetZones(account *utils.AwsAccess, ctx context.Context, updateCache bool) []types.AvailabilityZone { - args := m.Called(account, ctx, updateCache) +func (m *ec2UtilMock) GetZones(account *utils.AwsAccess) []types.AvailabilityZone { + args := m.Called(account) return args.Get(0).([]types.AvailabilityZone) } - -func (m *zoneMock) GetZone(awsAccountNumber string, awsZone string, region string) *types.AvailabilityZone { +func (m *ec2UtilMock) GetZone(awsAccountNumber string, awsZone string, region string) *types.AvailabilityZone { args := m.Called(awsAccountNumber, awsZone, region) return args.Get(0).(*types.AvailabilityZone) } +func (m *ec2UtilMock) GetVpcName(awsAccountNumber string, region string, vpcId string) string { + args := m.Called(awsAccountNumber, region, vpcId) + return args.Get(0).(string) +} diff --git a/extec2/ec2_util.go b/extec2/ec2_util.go new file mode 100644 index 0000000..1cad7bb --- /dev/null +++ b/extec2/ec2_util.go @@ -0,0 +1,132 @@ +package extec2 + +import ( + "context" + "errors" + "github.com/aws/aws-sdk-go-v2/aws" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/rs/zerolog/log" + "github.com/steadybit/discovery-kit/go/discovery_kit_api" + "github.com/steadybit/extension-aws/config" + "github.com/steadybit/extension-aws/utils" + "sync" +) + +var ( + Util *util +) + +type util struct { + zones sync.Map + vpcs sync.Map +} + +func InitializeEc2Util() { + Util = &util{ + zones: sync.Map{}, + vpcs: sync.Map{}, + } + _, _ = utils.ForEveryConfiguredAwsAccess(InitEc2UtilForAccount, context.Background(), "availability zone") +} + +func InitEc2UtilForAccount(account *utils.AwsAccess, ctx context.Context) ([]discovery_kit_api.Target, error) { + initZonesCache(ec2.NewFromConfig(account.AwsConfig), account.AccountNumber, account.Region, ctx) + initVpcCache(ec2.NewFromConfig(account.AwsConfig), account.AccountNumber, account.Region, ctx) + return nil, nil +} + +type AZDescribeAvailabilityZonesApi interface { + DescribeAvailabilityZones(ctx context.Context, params *ec2.DescribeAvailabilityZonesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeAvailabilityZonesOutput, error) +} + +func initZonesCache(client AZDescribeAvailabilityZonesApi, awsAccountNumber string, region string, ctx context.Context) { + output, err := client.DescribeAvailabilityZones(ctx, &ec2.DescribeAvailabilityZonesInput{ + AllAvailabilityZones: aws.Bool(false), + }) + if err != nil { + var re *awshttp.ResponseError + if errors.As(err, &re) && re.HTTPStatusCode() == 403 { + log.Error().Msgf("Not Authorized to discover availability zones for account %s and region %s. If this is intended, you can disable the discovery by setting STEADYBIT_EXTENSION_DISCOVERY_DISABLED_ZONE=true. Details: %s", awsAccountNumber, region, re.Error()) + Util.zones.Store(awsAccountNumber+"-"+region, []types.AvailabilityZone{}) + return + } + log.Error().Err(err).Msgf("Failed to load availability zones for account %s and region %s.", awsAccountNumber, region) + Util.zones.Store(awsAccountNumber+"-"+region, []types.AvailabilityZone{}) + return + } + Util.zones.Store(awsAccountNumber+"-"+region, output.AvailabilityZones) +} + +func initVpcCache(client ec2.DescribeVpcsAPIClient, awsAccountNumber string, region string, ctx context.Context) { + if !config.Config.DiscoveryDisabledVpc { + output, err := client.DescribeVpcs(ctx, &ec2.DescribeVpcsInput{}) + if err != nil { + var re *awshttp.ResponseError + if errors.As(err, &re) && re.HTTPStatusCode() == 403 { + log.Error().Msgf("Not Authorized to discover vpcs for account %s and region %s. If this is intended, you can disable the discovery by setting STEADYBIT_EXTENSION_DISCOVERY_DISABLED_VPC=true. Details: %s", awsAccountNumber, region, re.Error()) + Util.vpcs.Store(awsAccountNumber+"-"+region, []types.Vpc{}) + return + } + log.Error().Err(err).Msgf("Failed to load vpcs for account %s and region %s.", awsAccountNumber, region) + Util.vpcs.Store(awsAccountNumber+"-"+region, []types.Vpc{}) + return + } + Util.vpcs.Store(awsAccountNumber+"-"+region, output.Vpcs) + } else { + Util.vpcs.Store(awsAccountNumber+"-"+region, []types.Vpc{}) + } +} + +type GetZoneUtil interface { + GetZone(awsAccountNumber string, region string, awsZone string) *types.AvailabilityZone +} + +func (zones *util) GetZones(account *utils.AwsAccess) []types.AvailabilityZone { + awsAccountNumber := account.AccountNumber + region := account.Region + value, ok := zones.zones.Load(awsAccountNumber + "-" + region) + if !ok { + return []types.AvailabilityZone{} + } + return value.([]types.AvailabilityZone) +} + +type GetVpcNameUtil interface { + GetVpcName(awsAccountNumber string, region string, vpcId string) string +} + +func (zones *util) GetVpcName(awsAccountNumber string, region string, vpcId string) string { + value, ok := zones.vpcs.Load(awsAccountNumber + "-" + region) + if !ok { + return vpcId + } + for _, vpc := range value.([]types.Vpc) { + if aws.ToString(vpc.VpcId) == vpcId { + for _, tag := range vpc.Tags { + if aws.ToString(tag.Key) == "Name" { + return aws.ToString(tag.Value) + " (" + vpcId + ")" + } + } + } + } + return vpcId +} + +type GetZonesUtil interface { + GetZones(account *utils.AwsAccess) []types.AvailabilityZone +} + +func (zones *util) GetZone(awsAccountNumber string, region string, awsZone string) *types.AvailabilityZone { + value, ok := zones.zones.Load(awsAccountNumber + "-" + region) + if !ok { + return nil + } + for _, zone := range value.([]types.AvailabilityZone) { + if aws.ToString(zone.ZoneName) == awsZone { + return &zone + } + } + return nil +} diff --git a/utils/aws_zones_test.go b/extec2/ec2_util_test.go similarity index 50% rename from utils/aws_zones_test.go rename to extec2/ec2_util_test.go index e8800cd..0204e7f 100644 --- a/utils/aws_zones_test.go +++ b/extec2/ec2_util_test.go @@ -1,21 +1,22 @@ -package utils +package extec2 import ( "context" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/steadybit/discovery-kit/go/discovery_kit_api" + "github.com/steadybit/extension-aws/utils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "sync" "testing" ) -type ec2ClientMock struct { +type ec2UtilsClientMock struct { mock.Mock } -func (m *ec2ClientMock) DescribeAvailabilityZones(ctx context.Context, params *ec2.DescribeAvailabilityZonesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeAvailabilityZonesOutput, error) { +func (m *ec2UtilsClientMock) DescribeAvailabilityZones(ctx context.Context, params *ec2.DescribeAvailabilityZonesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeAvailabilityZonesOutput, error) { args := m.Called(ctx, params, optFns) if args.Get(0) == nil { return nil, args.Error(1) @@ -25,10 +26,10 @@ func (m *ec2ClientMock) DescribeAvailabilityZones(ctx context.Context, params *e func TestAwsZones(t *testing.T) { // Given - Zones = &AwsZones{ + Util = &util{ zones: sync.Map{}, } - mockedApi42 := new(ec2ClientMock) + mockedApi42 := new(ec2UtilsClientMock) mockedReturnValue42 := ec2.DescribeAvailabilityZonesOutput{ AvailabilityZones: []types.AvailabilityZone{ { @@ -45,26 +46,22 @@ func TestAwsZones(t *testing.T) { } mockedApi42.On("DescribeAvailabilityZones", mock.Anything, mock.Anything, mock.Anything).Return(&mockedReturnValue42, nil) - mockedApi4711 := new(ec2ClientMock) + mockedApi4711 := new(ec2UtilsClientMock) mockedReturnValue4711 := ec2.DescribeAvailabilityZonesOutput{ AvailabilityZones: []types.AvailabilityZone{}, } mockedApi4711.On("DescribeAvailabilityZones", mock.Anything, mock.Anything, mock.Anything).Return(&mockedReturnValue4711, nil) // When - result, err := initAwsZonesForAccountWithClient(mockedApi42, "42", "eu-central-1", context.Background()) - assert.Nil(t, result) - assert.Nil(t, err) - result, err = initAwsZonesForAccountWithClient(mockedApi4711, "4711", "eu-central-1", context.Background()) - assert.Nil(t, result) - assert.Nil(t, err) + initZonesCache(mockedApi42, "42", "eu-central-1", context.Background()) + initZonesCache(mockedApi4711, "4711", "eu-central-1", context.Background()) // Then - assert.Equal(t, &mockedReturnValue42.AvailabilityZones[0], Zones.GetZone("42", "eu-central-1", "eu-central-1a")) - assert.Nil(t, Zones.GetZone("42", "eu-central-1", "eu-central-1c")) - assert.Nil(t, Zones.GetZone("4711", "eu-central-1", "eu-central-1a")) + assert.Equal(t, &mockedReturnValue42.AvailabilityZones[0], Util.GetZone("42", "eu-central-1", "eu-central-1a")) + assert.Nil(t, Util.GetZone("42", "eu-central-1", "eu-central-1c")) + assert.Nil(t, Util.GetZone("4711", "eu-central-1", "eu-central-1a")) - assert.Equal(t, mockedReturnValue42.AvailabilityZones, Zones.GetZones(&AwsAccess{AccountNumber: "42", Region: "eu-central-1"}, context.Background(), false)) - assert.Equal(t, []types.AvailabilityZone{}, Zones.GetZones(&AwsAccess{AccountNumber: "4711", Region: "eu-central-1"}, context.Background(), false)) - assert.Equal(t, []types.AvailabilityZone{}, Zones.GetZones(&AwsAccess{AccountNumber: "0815", Region: "eu-central-1"}, context.Background(), false)) + assert.Equal(t, mockedReturnValue42.AvailabilityZones, Util.GetZones(&utils.AwsAccess{AccountNumber: "42", Region: "eu-central-1"})) + assert.Equal(t, []types.AvailabilityZone{}, Util.GetZones(&utils.AwsAccess{AccountNumber: "4711", Region: "eu-central-1"})) + assert.Equal(t, []types.AvailabilityZone{}, Util.GetZones(&utils.AwsAccess{AccountNumber: "0815", Region: "eu-central-1"})) } diff --git a/extec2/instance_discovery.go b/extec2/instance_discovery.go index 90b377c..73a1900 100644 --- a/extec2/instance_discovery.go +++ b/extec2/instance_discovery.go @@ -254,7 +254,7 @@ func (e *ec2Discovery) DiscoverTargets(ctx context.Context) ([]discovery_kit_api func getEc2InstancesForAccount(account *utils.AwsAccess, ctx context.Context) ([]discovery_kit_api.Target, error) { client := ec2.NewFromConfig(account.AwsConfig) - result, err := GetAllEc2Instances(ctx, client, utils.Zones, account.AccountNumber, account.AwsConfig.Region) + result, err := GetAllEc2Instances(ctx, client, Util, account.AccountNumber, account.AwsConfig.Region) if err != nil { var re *awshttp.ResponseError if errors.As(err, &re) && re.HTTPStatusCode() == 403 { @@ -266,7 +266,12 @@ func getEc2InstancesForAccount(account *utils.AwsAccess, ctx context.Context) ([ return result, nil } -func GetAllEc2Instances(ctx context.Context, ec2Api ec2.DescribeInstancesAPIClient, zoneUtil utils.GetZoneUtil, awsAccountNumber string, awsRegion string) ([]discovery_kit_api.Target, error) { +type instanceDiscoveryEc2Util interface { + GetZoneUtil + GetVpcNameUtil +} + +func GetAllEc2Instances(ctx context.Context, ec2Api ec2.DescribeInstancesAPIClient, ec2Util instanceDiscoveryEc2Util, awsAccountNumber string, awsRegion string) ([]discovery_kit_api.Target, error) { result := make([]discovery_kit_api.Target, 0, 20) paginator := ec2.NewDescribeInstancesPaginator(ec2Api, &ec2.DescribeInstancesInput{}) @@ -277,7 +282,7 @@ func GetAllEc2Instances(ctx context.Context, ec2Api ec2.DescribeInstancesAPIClie } for _, reservation := range output.Reservations { for _, ec2Instance := range reservation.Instances { - result = append(result, toEc2InstanceTarget(ec2Instance, zoneUtil, awsAccountNumber, awsRegion)) + result = append(result, toEc2InstanceTarget(ec2Instance, ec2Util, awsAccountNumber, awsRegion)) } } } @@ -285,7 +290,7 @@ func GetAllEc2Instances(ctx context.Context, ec2Api ec2.DescribeInstancesAPIClie return discovery_kit_commons.ApplyAttributeExcludes(result, config.Config.DiscoveryAttributesExcludesEc2), nil } -func toEc2InstanceTarget(ec2Instance types.Instance, zoneUtil utils.GetZoneUtil, awsAccountNumber string, awsRegion string) discovery_kit_api.Target { +func toEc2InstanceTarget(ec2Instance types.Instance, ec2Util instanceDiscoveryEc2Util, awsAccountNumber string, awsRegion string) discovery_kit_api.Target { var name *string for _, tag := range ec2Instance.Tags { if *tag.Key == "Name" { @@ -299,7 +304,7 @@ func toEc2InstanceTarget(ec2Instance types.Instance, zoneUtil utils.GetZoneUtil, label = label + " / " + *name } availabilityZoneName := aws.ToString(ec2Instance.Placement.AvailabilityZone) - availabilityZoneApi := zoneUtil.GetZone(awsAccountNumber, availabilityZoneName, awsRegion) + availabilityZoneApi := ec2Util.GetZone(awsAccountNumber, availabilityZoneName, awsRegion) attributes := make(map[string][]string) attributes["aws.account"] = []string{awsAccountNumber} @@ -323,6 +328,8 @@ func toEc2InstanceTarget(ec2Instance types.Instance, zoneUtil utils.GetZoneUtil, } attributes["aws-ec2.arn"] = []string{arn} attributes["aws-ec2.vpc"] = []string{aws.ToString(ec2Instance.VpcId)} + attributes["aws.vpc.id"] = []string{aws.ToString(ec2Instance.VpcId)} + attributes["aws.vpc.name"] = []string{ec2Util.GetVpcName(awsAccountNumber, awsRegion, aws.ToString(ec2Instance.VpcId))} if ec2Instance.State != nil { attributes["aws-ec2.state"] = []string{string(ec2Instance.State.Name)} } diff --git a/extec2/instance_discovery_test.go b/extec2/instance_discovery_test.go index 45ba431..85c33fc 100644 --- a/extec2/instance_discovery_test.go +++ b/extec2/instance_discovery_test.go @@ -16,11 +16,11 @@ import ( "testing" ) -type ec2ClientMock struct { +type instanceDiscoveryEc2UtilMock struct { mock.Mock } -func (m *ec2ClientMock) DescribeInstances(ctx context.Context, params *ec2.DescribeInstancesInput, _ ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) { +func (m *instanceDiscoveryEc2UtilMock) DescribeInstances(ctx context.Context, params *ec2.DescribeInstancesInput, _ ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) { args := m.Called(ctx, params) if args.Get(0) == nil { return nil, args.Error(1) @@ -49,7 +49,7 @@ var instance = types.Instance{ func TestGetAllEc2Instances(t *testing.T) { // Given - mockedApi := new(ec2ClientMock) + mockedApi := new(instanceDiscoveryEc2UtilMock) mockedReturnValue := ec2.DescribeInstancesOutput{ Reservations: []types.Reservation{ { @@ -61,14 +61,14 @@ func TestGetAllEc2Instances(t *testing.T) { } mockedApi.On("DescribeInstances", mock.Anything, mock.Anything).Return(&mockedReturnValue, nil) - mockedZoneUtil := new(zoneMock) + mockedZoneUtil := new(ec2UtilMock) mockedZone := types.AvailabilityZone{ ZoneName: discovery_kit_api.Ptr("us-east-1b"), RegionName: discovery_kit_api.Ptr("us-east-1"), ZoneId: discovery_kit_api.Ptr("us-east-1b-id"), } mockedZoneUtil.On("GetZone", mock.Anything, mock.Anything, mock.Anything).Return(&mockedZone) - + mockedZoneUtil.On("GetVpcName", mock.Anything, mock.Anything, mock.Anything).Return("vpc-123-name") // When targets, err := GetAllEc2Instances(context.Background(), mockedApi, mockedZoneUtil, "42", "us-east-1") @@ -99,7 +99,7 @@ func TestGetAllEc2InstancesWithFilteredAttributes(t *testing.T) { // Given // set env var to filter out all attributes starting with "aws-ec2" config.Config.DiscoveryAttributesExcludesEc2 = []string{"aws-ec2.label.*", "aws-ec2.image"} - mockedApi := new(ec2ClientMock) + mockedApi := new(instanceDiscoveryEc2UtilMock) mockedReturnValue := ec2.DescribeInstancesOutput{ Reservations: []types.Reservation{ { @@ -111,13 +111,14 @@ func TestGetAllEc2InstancesWithFilteredAttributes(t *testing.T) { } mockedApi.On("DescribeInstances", mock.Anything, mock.Anything).Return(&mockedReturnValue, nil) - mockedZoneUtil := new(zoneMock) + mockedZoneUtil := new(ec2UtilMock) mockedZone := types.AvailabilityZone{ ZoneName: discovery_kit_api.Ptr("us-east-1b"), RegionName: discovery_kit_api.Ptr("us-east-1"), ZoneId: discovery_kit_api.Ptr("us-east-1b-id"), } mockedZoneUtil.On("GetZone", mock.Anything, mock.Anything, mock.Anything).Return(&mockedZone) + mockedZoneUtil.On("GetVpcName", mock.Anything, mock.Anything, mock.Anything).Return("vpc-123-name") // When targets, err := GetAllEc2Instances(context.Background(), mockedApi, mockedZoneUtil, "42", "us-east-1") @@ -138,6 +139,8 @@ func TestGetAllEc2InstancesWithFilteredAttributes(t *testing.T) { assert.Equal(t, []string{"ip-10-3-92-28.eu-central-1.compute.internal"}, target.Attributes["aws-ec2.hostname.internal"]) assert.Equal(t, []string{"arn:aws:ec2:us-east-1:42:instance/i-0ef9adc9fbd3b19c5"}, target.Attributes["aws-ec2.arn"]) assert.Equal(t, []string{"vpc-003cf5dda88c814c6"}, target.Attributes["aws-ec2.vpc"]) + assert.Equal(t, []string{"vpc-003cf5dda88c814c6"}, target.Attributes["aws.vpc.id"]) + assert.Equal(t, []string{"vpc-123-name"}, target.Attributes["aws.vpc.name"]) assert.Equal(t, []string{"running"}, target.Attributes["aws-ec2.state"]) assert.NotContains(t, target.Attributes, "aws-ec2.label.specialtag") assert.NotContains(t, target.Attributes, "aws-ec2.image") @@ -147,7 +150,7 @@ func TestGetAllEc2InstancesWithFilteredAttributes(t *testing.T) { func TestNameNotSet(t *testing.T) { // Given - mockedApi := new(ec2ClientMock) + mockedApi := new(instanceDiscoveryEc2UtilMock) mockedReturnValue := ec2.DescribeInstancesOutput{ Reservations: []types.Reservation{ { @@ -164,13 +167,14 @@ func TestNameNotSet(t *testing.T) { } mockedApi.On("DescribeInstances", mock.Anything, mock.Anything).Return(&mockedReturnValue, nil) - mockedZoneUtil := new(zoneMock) + mockedZoneUtil := new(ec2UtilMock) mockedZone := types.AvailabilityZone{ ZoneName: discovery_kit_api.Ptr("us-east-1b"), RegionName: discovery_kit_api.Ptr("us-east-1"), ZoneId: discovery_kit_api.Ptr("us-east-1b-id"), } mockedZoneUtil.On("GetZone", mock.Anything, mock.Anything, mock.Anything).Return(&mockedZone) + mockedZoneUtil.On("GetVpcName", mock.Anything, mock.Anything, mock.Anything).Return("vpc-123-name") // When targets, err := GetAllEc2Instances(context.Background(), mockedApi, mockedZoneUtil, "42", "us-east-1") @@ -185,17 +189,18 @@ func TestNameNotSet(t *testing.T) { func TestGetAllEc2InstancesError(t *testing.T) { // Given - mockedApi := new(ec2ClientMock) + mockedApi := new(instanceDiscoveryEc2UtilMock) mockedApi.On("DescribeInstances", mock.Anything, mock.Anything).Return(nil, errors.New("expected")) - mockedZoneUtil := new(zoneMock) + mockedZoneUtil := new(ec2UtilMock) mockedZone := types.AvailabilityZone{ ZoneName: discovery_kit_api.Ptr("us-east-1b"), RegionName: discovery_kit_api.Ptr("us-east-1"), ZoneId: discovery_kit_api.Ptr("us-east-1b-id"), } mockedZoneUtil.On("GetZone", mock.Anything, mock.Anything, mock.Anything).Return(&mockedZone) + mockedZoneUtil.On("GetVpcName", mock.Anything, mock.Anything, mock.Anything).Return("vpc-123-name") // When _, err := GetAllEc2Instances(context.Background(), mockedApi, mockedZoneUtil, "42", "us-east-1") diff --git a/extecs/task_discovery.go b/extecs/task_discovery.go index 8853929..744eeef 100644 --- a/extecs/task_discovery.go +++ b/extecs/task_discovery.go @@ -20,6 +20,7 @@ import ( "github.com/steadybit/discovery-kit/go/discovery_kit_commons" "github.com/steadybit/discovery-kit/go/discovery_kit_sdk" "github.com/steadybit/extension-aws/config" + "github.com/steadybit/extension-aws/extec2" "github.com/steadybit/extension-aws/utils" "github.com/steadybit/extension-kit/extbuild" "github.com/steadybit/extension-kit/extutil" @@ -112,7 +113,7 @@ func (e *ecsTaskDiscovery) DiscoverTargets(ctx context.Context) ([]discovery_kit func getTargetsForAccount(account *utils.AwsAccess, ctx context.Context) ([]discovery_kit_api.Target, error) { client := ecs.NewFromConfig(account.AwsConfig) - result, err := GetAllEcsTasks(ctx, client, utils.Zones, account.AccountNumber, account.AwsConfig.Region) + result, err := GetAllEcsTasks(ctx, client, extec2.Util, account.AccountNumber, account.AwsConfig.Region) if err != nil { var re *awshttp.ResponseError if errors.As(err, &re) && re.HTTPStatusCode() == 403 { @@ -130,7 +131,12 @@ type EcsTasksApi interface { ecs.ListClustersAPIClient } -func GetAllEcsTasks(ctx context.Context, ecsApi EcsTasksApi, zoneUtil utils.GetZoneUtil, awsAccountNumber string, awsRegion string) ([]discovery_kit_api.Target, error) { +type taskDiscoveryEc2Util interface { + extec2.GetZoneUtil + extec2.GetVpcNameUtil +} + +func GetAllEcsTasks(ctx context.Context, ecsApi EcsTasksApi, ec2Util taskDiscoveryEc2Util, awsAccountNumber string, awsRegion string) ([]discovery_kit_api.Target, error) { result := make([]discovery_kit_api.Target, 0, 20) listClusterOutput, err := ecsApi.ListClusters(ctx, &ecs.ListClustersInput{}) @@ -160,7 +166,7 @@ func GetAllEcsTasks(ctx context.Context, ecsApi EcsTasksApi, zoneUtil utils.GetZ for _, task := range describeTasksOutput.Tasks { if task.LastStatus != nil && *task.LastStatus == "RUNNING" && !ignoreTask(task) { - result = append(result, toTarget(task, zoneUtil, awsAccountNumber, awsRegion)) + result = append(result, toTarget(task, ec2Util, awsAccountNumber, awsRegion)) } } } @@ -182,7 +188,7 @@ func ignoreTask(service types.Task) bool { return false } -func toTarget(task types.Task, zoneUtil utils.GetZoneUtil, awsAccountNumber string, awsRegion string) discovery_kit_api.Target { +func toTarget(task types.Task, ec2Util taskDiscoveryEc2Util, awsAccountNumber string, awsRegion string) discovery_kit_api.Target { var service, clusterName *string for _, tag := range task.Tags { if *tag.Key == "aws:ecs:serviceName" { @@ -195,7 +201,7 @@ func toTarget(task types.Task, zoneUtil utils.GetZoneUtil, awsAccountNumber stri arn := aws.ToString(task.TaskArn) availabilityZoneName := aws.ToString(task.AvailabilityZone) - availabilityZoneApi := zoneUtil.GetZone(awsAccountNumber, availabilityZoneName, awsRegion) + availabilityZoneApi := ec2Util.GetZone(awsAccountNumber, availabilityZoneName, awsRegion) attributes := make(map[string][]string) attributes["aws.account"] = []string{awsAccountNumber} diff --git a/extecs/task_discovery_test.go b/extecs/task_discovery_test.go index ac52897..e7694c8 100644 --- a/extecs/task_discovery_test.go +++ b/extecs/task_discovery_test.go @@ -43,14 +43,18 @@ func (m *ecsClientMock) ListClusters(ctx context.Context, params *ecs.ListCluste return args.Get(0).(*ecs.ListClustersOutput), args.Error(1) } -type zoneMock struct { +type taskDiscoveryEc2UtilMock struct { mock.Mock } -func (m *zoneMock) GetZone(awsAccountNumber string, awsZone string, region string) *ec2types.AvailabilityZone { +func (m *taskDiscoveryEc2UtilMock) GetZone(awsAccountNumber string, awsZone string, region string) *ec2types.AvailabilityZone { args := m.Called(awsAccountNumber, awsZone, region) return args.Get(0).(*ec2types.AvailabilityZone) } +func (m *taskDiscoveryEc2UtilMock) GetVpcName(awsAccountNumber string, region string, vpcId string) string { + args := m.Called(awsAccountNumber, region, vpcId) + return args.Get(0).(string) +} var taskArn = "arn:aws:ecs:eu-central-1:42:task/sandbox-demo-ecs-fargate/15ac9bc28dce4a6fb757580ac87eb854" var clusterArn = "arn:aws:ecs:eu-central-1:42:cluster/sandbox-demo-ecs-fargate" @@ -98,7 +102,7 @@ func TestGetAllEcsTasks(t *testing.T) { Tasks: []types.Task{task, taskStopped}, }, nil) - mockedZoneUtil := new(zoneMock) + mockedZoneUtil := new(taskDiscoveryEc2UtilMock) mockedZone := ec2types.AvailabilityZone{ ZoneName: discovery_kit_api.Ptr("us-east-1b"), RegionName: discovery_kit_api.Ptr("us-east-1"), diff --git a/extelb/alb_discovery.go b/extelb/alb_discovery.go index 97bf193..9604cc8 100644 --- a/extelb/alb_discovery.go +++ b/extelb/alb_discovery.go @@ -20,6 +20,7 @@ import ( "github.com/steadybit/discovery-kit/go/discovery_kit_commons" "github.com/steadybit/discovery-kit/go/discovery_kit_sdk" "github.com/steadybit/extension-aws/config" + "github.com/steadybit/extension-aws/extec2" "github.com/steadybit/extension-aws/utils" extension_kit "github.com/steadybit/extension-kit" "github.com/steadybit/extension-kit/extbuild" @@ -104,7 +105,7 @@ func (e *albDiscovery) DiscoverTargets(ctx context.Context) ([]discovery_kit_api func getTargetsForAccount(account *utils.AwsAccess, ctx context.Context) ([]discovery_kit_api.Target, error) { client := elasticloadbalancingv2.NewFromConfig(account.AwsConfig) - result, err := GetAlbs(ctx, client, utils.Zones, account.AccountNumber, account.AwsConfig.Region) + result, err := GetAlbs(ctx, client, extec2.Util, account.AccountNumber, account.AwsConfig.Region) if err != nil { var re *awshttp.ResponseError if errors.As(err, &re) && re.HTTPStatusCode() == 403 { @@ -122,7 +123,12 @@ type AlbDiscoveryApi interface { DescribeTags(ctx context.Context, params *elasticloadbalancingv2.DescribeTagsInput, optFns ...func(*elasticloadbalancingv2.Options)) (*elasticloadbalancingv2.DescribeTagsOutput, error) } -func GetAlbs(ctx context.Context, albDiscoveryApi AlbDiscoveryApi, zoneUtil utils.GetZoneUtil, awsAccountNumber string, awsRegion string) ([]discovery_kit_api.Target, error) { +type albDiscoveryEc2Util interface { + extec2.GetZoneUtil + extec2.GetVpcNameUtil +} + +func GetAlbs(ctx context.Context, albDiscoveryApi AlbDiscoveryApi, ec2Util albDiscoveryEc2Util, awsAccountNumber string, awsRegion string) ([]discovery_kit_api.Target, error) { result := make([]discovery_kit_api.Target, 0, 20) paginator := elasticloadbalancingv2.NewDescribeLoadBalancersPaginator(albDiscoveryApi, &elasticloadbalancingv2.DescribeLoadBalancersInput{}) @@ -164,22 +170,21 @@ func GetAlbs(ctx context.Context, albDiscoveryApi AlbDiscoveryApi, zoneUtil util return nil, extension_kit.ToError("Failed to fetch load balancer listeners.", err) } - result = append(result, toTarget(&loadBalancer, tags, describeListenersResult.Listeners, zoneUtil, awsAccountNumber, awsRegion)) + result = append(result, toTarget(&loadBalancer, tags, describeListenersResult.Listeners, ec2Util, awsAccountNumber, awsRegion)) } } } return discovery_kit_commons.ApplyAttributeExcludes(result, config.Config.DiscoveryAttributesExcludesEcs), nil } -func toTarget(lb *types.LoadBalancer, tags []types.Tag, listeners []types.Listener, zoneUtil utils.GetZoneUtil, awsAccountNumber string, awsRegion string) discovery_kit_api.Target { +func toTarget(lb *types.LoadBalancer, tags []types.Tag, listeners []types.Listener, ec2Util albDiscoveryEc2Util, awsAccountNumber string, awsRegion string) discovery_kit_api.Target { arn := aws.ToString(lb.LoadBalancerArn) name := aws.ToString(lb.LoadBalancerName) - zones := make([]string, 0, len(lb.AvailabilityZones)) zoneIds := make([]string, 0, len(lb.AvailabilityZones)) for _, zone := range lb.AvailabilityZones { zones = append(zones, aws.ToString(zone.ZoneName)) - zoneApi := zoneUtil.GetZone(awsAccountNumber, aws.ToString(zone.ZoneName), awsRegion) + zoneApi := ec2Util.GetZone(awsAccountNumber, aws.ToString(zone.ZoneName), awsRegion) if zoneApi != nil { zoneIds = append(zoneIds, *zoneApi.ZoneId) } @@ -198,6 +203,10 @@ func toTarget(lb *types.LoadBalancer, tags []types.Tag, listeners []types.Listen attributes["aws-elb.alb.listener.port"] = listenerPorts attributes["aws.account"] = []string{awsAccountNumber} attributes["aws.region"] = []string{awsRegion} + if lb.VpcId != nil { + attributes["aws.vpc.id"] = []string{aws.ToString(lb.VpcId)} + attributes["aws.vpc.name"] = []string{ec2Util.GetVpcName(awsAccountNumber, awsRegion, aws.ToString(lb.VpcId))} + } attributes["aws.zone"] = zones attributes["aws.zone.id"] = zoneIds diff --git a/extelb/alb_discovery_test.go b/extelb/alb_discovery_test.go index 0f48cc6..53e4d70 100644 --- a/extelb/alb_discovery_test.go +++ b/extelb/alb_discovery_test.go @@ -44,21 +44,27 @@ func (m *albDiscoveryApiMock) DescribeListeners(ctx context.Context, params *ela return args.Get(0).(*elasticloadbalancingv2.DescribeListenersOutput), args.Error(1) } -type zoneMock struct { +type albDiscoveryEc2UtilMock struct { mock.Mock } -func (m *zoneMock) GetZone(awsAccountNumber string, awsZone string, region string) *ec2types.AvailabilityZone { +func (m *albDiscoveryEc2UtilMock) GetZone(awsAccountNumber string, awsZone string, region string) *ec2types.AvailabilityZone { args := m.Called(awsAccountNumber, awsZone, region) return args.Get(0).(*ec2types.AvailabilityZone) } +func (m *albDiscoveryEc2UtilMock) GetVpcName(awsAccountNumber string, region string, vpcId string) string { + args := m.Called(awsAccountNumber, region, vpcId) + return args.Get(0).(string) +} + var albArn = "arn:aws:elasticloadbalancing:us-east-1:123456789012:loadbalancer/app/my-app-balancer/123" var alb = types.LoadBalancer{ LoadBalancerArn: extutil.Ptr(albArn), DNSName: extutil.Ptr("my-app-balancer-1234567890.us-east-1.elb.amazonaws.com"), LoadBalancerName: extutil.Ptr("my-app-balancer"), Type: types.LoadBalancerTypeEnumApplication, + VpcId: extutil.Ptr("vpc-123"), AvailabilityZones: []types.AvailabilityZone{ { ZoneName: extutil.Ptr("us-east-1a"), @@ -133,7 +139,7 @@ func TestGetAllAlbTargets(t *testing.T) { }, }, nil) - mockedZoneUtil := new(zoneMock) + mockedZoneUtil := new(albDiscoveryEc2UtilMock) mockedZone1a := ec2types.AvailabilityZone{ ZoneName: discovery_kit_api.Ptr("us-east-1a"), RegionName: discovery_kit_api.Ptr("us-east-1"), @@ -150,6 +156,7 @@ func TestGetAllAlbTargets(t *testing.T) { mockedZoneUtil.On("GetZone", mock.Anything, mock.MatchedBy(func(params string) bool { return params == "us-east-1b" }), mock.Anything).Return(&mockedZone1b) + mockedZoneUtil.On("GetVpcName", mock.Anything, mock.Anything, mock.Anything).Return("vpc-123-name") // When targets, err := GetAlbs(context.Background(), mockedApi, mockedZoneUtil, "42", "us-east-1") @@ -166,6 +173,8 @@ func TestGetAllAlbTargets(t *testing.T) { assert.Equal(t, []string{"us-east-1"}, target.Attributes["aws.region"]) assert.Equal(t, []string{"us-east-1a", "us-east-1b"}, target.Attributes["aws.zone"]) assert.Equal(t, []string{"us-east-1a-id", "us-east-1b-id"}, target.Attributes["aws.zone.id"]) + assert.Equal(t, []string{"vpc-123"}, target.Attributes["aws.vpc.id"]) + assert.Equal(t, []string{"vpc-123-name"}, target.Attributes["aws.vpc.name"]) assert.Equal(t, []string{"my-app-balancer"}, target.Attributes["aws-elb.alb.name"]) assert.Equal(t, []string{"my-app-balancer-1234567890.us-east-1.elb.amazonaws.com"}, target.Attributes["aws-elb.alb.dns"]) assert.Equal(t, []string{albArn}, target.Attributes["aws-elb.alb.arn"]) diff --git a/extlambda/discovery.go b/extlambda/discovery.go index 381ba15..c3fe5ac 100644 --- a/extlambda/discovery.go +++ b/extlambda/discovery.go @@ -19,6 +19,7 @@ import ( "github.com/steadybit/discovery-kit/go/discovery_kit_commons" "github.com/steadybit/discovery-kit/go/discovery_kit_sdk" "github.com/steadybit/extension-aws/config" + "github.com/steadybit/extension-aws/extec2" "github.com/steadybit/extension-aws/utils" "github.com/steadybit/extension-kit/extbuild" "github.com/steadybit/extension-kit/extutil" @@ -184,7 +185,7 @@ func getTargetsForAccount(account *utils.AwsAccess, ctx context.Context) ([]disc lambdaClient := lambda.NewFromConfig(account.AwsConfig) tagsClient := resourcegroupstaggingapi.NewFromConfig(account.AwsConfig) - result, err := getAllAwsLambdaFunctions(ctx, lambdaClient, tagsClient, account.AccountNumber, account.AwsConfig.Region) + result, err := getAllAwsLambdaFunctions(ctx, lambdaClient, tagsClient, extec2.Util, account.AccountNumber, account.AwsConfig.Region) if err != nil { var re *awshttp.ResponseError if errors.As(err, &re) && re.HTTPStatusCode() == 403 { @@ -196,7 +197,7 @@ func getTargetsForAccount(account *utils.AwsAccess, ctx context.Context) ([]disc return result, nil } -func getAllAwsLambdaFunctions(ctx context.Context, lambdaClient lambda.ListFunctionsAPIClient, tagsClient resourcegroupstaggingapi.GetResourcesAPIClient, awsAccountNumber string, awsAccountRegion string) ([]discovery_kit_api.Target, error) { +func getAllAwsLambdaFunctions(ctx context.Context, lambdaClient lambda.ListFunctionsAPIClient, tagsClient resourcegroupstaggingapi.GetResourcesAPIClient, ec2Util extec2.GetVpcNameUtil, awsAccountNumber string, awsAccountRegion string) ([]discovery_kit_api.Target, error) { result := make([]discovery_kit_api.Target, 0, 100) var marker *string = nil for { @@ -215,7 +216,7 @@ func getAllAwsLambdaFunctions(ctx context.Context, lambdaClient lambda.ListFunct tags = tagsResponse.Tags } } - result = append(result, toTarget(function, awsAccountNumber, awsAccountRegion, tags)) + result = append(result, toTarget(function, ec2Util, awsAccountNumber, awsAccountRegion, tags)) } if output.NextMarker == nil { @@ -246,7 +247,7 @@ func getTags(ctx context.Context, output *lambda.ListFunctionsOutput, tagsClient return tags } -func toTarget(function types.FunctionConfiguration, awsAccountNumber string, awsAccountRegion string, tags []tagTypes.Tag) discovery_kit_api.Target { +func toTarget(function types.FunctionConfiguration, ec2Util extec2.GetVpcNameUtil, awsAccountNumber string, awsAccountRegion string, tags []tagTypes.Tag) discovery_kit_api.Target { arn := aws.ToString(function.FunctionArn) name := aws.ToString(function.FunctionName) @@ -254,6 +255,10 @@ func toTarget(function types.FunctionConfiguration, awsAccountNumber string, aws attributes["aws.account"] = []string{awsAccountNumber} attributes["aws.region"] = []string{awsAccountRegion} attributes["aws.arn"] = []string{arn} + if function.VpcConfig != nil && function.VpcConfig.VpcId != nil { + attributes["aws.vpc.id"] = []string{aws.ToString(function.VpcConfig.VpcId)} + attributes["aws.vpc.name"] = []string{ec2Util.GetVpcName(awsAccountNumber, awsAccountRegion, aws.ToString(function.VpcConfig.VpcId))} + } attributes["aws.lambda.function-name"] = []string{name} attributes["aws.lambda.runtime"] = []string{string(function.Runtime)} attributes["aws.role"] = []string{aws.ToString(function.Role)} diff --git a/extlambda/discovery_test.go b/extlambda/discovery_test.go index 96af7a4..ca3817e 100644 --- a/extlambda/discovery_test.go +++ b/extlambda/discovery_test.go @@ -42,9 +42,19 @@ func (m *tagClientMock) GetResources(ctx context.Context, params *resourcegroups return args.Get(0).(*resourcegroupstaggingapi.GetResourcesOutput), args.Error(1) } +type lambdaDiscoveryEc2UtilMock struct { + mock.Mock +} + +func (m *lambdaDiscoveryEc2UtilMock) GetVpcName(awsAccountNumber string, region string, vpcId string) string { + args := m.Called(awsAccountNumber, region, vpcId) + return args.Get(0).(string) +} + func Test_getAllAwsLambdaFunctions(t *testing.T) { lambdaApi := new(lambdaClientMock) tagApi := new(tagClientMock) + ec2util := new(lambdaDiscoveryEc2UtilMock) listedFunction := lambda.ListFunctionsOutput{ Functions: []types.FunctionConfiguration{ { @@ -56,6 +66,9 @@ func Test_getAllAwsLambdaFunctions(t *testing.T) { "FAILURE_INJECTION_PARAM": "env-fip", }, }), + VpcConfig: extutil.Ptr(types.VpcConfigResponse{ + VpcId: extutil.Ptr("vpc-123"), + }), FunctionArn: extutil.Ptr("arn"), FunctionName: extutil.Ptr("name"), LastModified: extutil.Ptr("last-modified"), @@ -87,8 +100,10 @@ func Test_getAllAwsLambdaFunctions(t *testing.T) { } tagApi.On("GetResources", mock.Anything, mock.Anything, mock.Anything).Return(&tags, nil) + ec2util.On("GetVpcName", mock.Anything, mock.Anything, mock.Anything).Return("vpc-123-name") + // When - targets, err := getAllAwsLambdaFunctions(context.Background(), lambdaApi, tagApi, "42", "us-east-1") + targets, err := getAllAwsLambdaFunctions(context.Background(), lambdaApi, tagApi, ec2util, "42", "us-east-1") // Then assert.Equal(t, nil, err) @@ -98,18 +113,21 @@ func Test_getAllAwsLambdaFunctions(t *testing.T) { assert.Equal(t, lambdaTargetID, target.TargetType) assert.Equal(t, "name", target.Label) assert.Equal(t, "arn", target.Id) - assert.Equal(t, 19, len(target.Attributes)) + assert.Equal(t, 21, len(target.Attributes)) assert.Equal(t, []string{"42"}, target.Attributes["aws.account"]) assert.Equal(t, []string{"us-east-1"}, target.Attributes["aws.region"]) assert.Equal(t, []string{"name"}, target.Attributes["aws.lambda.function-name"]) assert.Equal(t, []string{"env-fip"}, target.Attributes["aws.lambda.failure-injection-param"]) assert.Equal(t, []string{"Tag123"}, target.Attributes["aws.lambda.label.example"]) + assert.Equal(t, []string{"vpc-123"}, target.Attributes["aws.vpc.id"]) + assert.Equal(t, []string{"vpc-123-name"}, target.Attributes["aws.vpc.name"]) } func Test_getAllAwsLambdaFunctions_withPagination(t *testing.T) { // Given mockedApi := new(lambdaClientMock) tagApi := new(tagClientMock) + ec2util := new(lambdaDiscoveryEc2UtilMock) withMarker := mock.MatchedBy(func(arg *lambda.ListFunctionsInput) bool { return arg.Marker != nil @@ -148,7 +166,7 @@ func Test_getAllAwsLambdaFunctions_withPagination(t *testing.T) { tagApi.On("GetResources", mock.Anything, mock.Anything, mock.Anything).Return(&tags, nil) // When - targets, err := getAllAwsLambdaFunctions(context.Background(), mockedApi, tagApi, "42", "us-east-1") + targets, err := getAllAwsLambdaFunctions(context.Background(), mockedApi, tagApi, ec2util, "42", "us-east-1") // Then assert.Equal(t, nil, err) @@ -161,9 +179,10 @@ func Test_getAllAwsLambdaFunctions_withError(t *testing.T) { // Given clientApi := new(lambdaClientMock) tagApi := new(tagClientMock) + ec2util := new(lambdaDiscoveryEc2UtilMock) clientApi.On("ListFunctions", mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.New("error")) // When - _, err := getAllAwsLambdaFunctions(context.Background(), clientApi, tagApi, "42", "us-east-1") + _, err := getAllAwsLambdaFunctions(context.Background(), clientApi, tagApi, ec2util, "42", "us-east-1") assert.Equal(t, "error", err.Error()) } diff --git a/extrds/common_instance_test.go b/extrds/common_instance_test.go index 2f8d51e..98773a0 100644 --- a/extrds/common_instance_test.go +++ b/extrds/common_instance_test.go @@ -44,3 +44,8 @@ func (m *zoneMock) GetZone(awsAccountNumber string, awsZone string, region strin args := m.Called(awsAccountNumber, awsZone, region) return args.Get(0).(*types.AvailabilityZone) } + +func (m *zoneMock) GetVpcName(awsAccountNumber string, region string, vpcId string) string { + args := m.Called(awsAccountNumber, region, vpcId) + return args.Get(0).(string) +} diff --git a/extrds/instance_discovery.go b/extrds/instance_discovery.go index de66555..c741682 100644 --- a/extrds/instance_discovery.go +++ b/extrds/instance_discovery.go @@ -7,6 +7,7 @@ import ( "context" "errors" "fmt" + "github.com/steadybit/extension-aws/extec2" "strings" "time" @@ -99,7 +100,7 @@ func (r *rdsInstanceDiscovery) DiscoverTargets(ctx context.Context) ([]discovery func getInstanceTargetsForAccount(account *utils.AwsAccess, ctx context.Context) ([]discovery_kit_api.Target, error) { client := rds.NewFromConfig(account.AwsConfig) - result, err := getAllRdsInstances(ctx, client, utils.Zones, account.AccountNumber, account.AwsConfig.Region) + result, err := getAllRdsInstances(ctx, client, extec2.Util, account.AccountNumber, account.AwsConfig.Region) if err != nil { var re *awshttp.ResponseError if errors.As(err, &re) && re.HTTPStatusCode() == 403 { @@ -111,7 +112,12 @@ func getInstanceTargetsForAccount(account *utils.AwsAccess, ctx context.Context) return result, nil } -func getAllRdsInstances(ctx context.Context, rdsApi rdsDBInstanceApi, zoneUtil utils.GetZoneUtil, awsAccountNumber string, awsRegion string) ([]discovery_kit_api.Target, error) { +type rdsInstanceDiscoveryEc2Util interface { + extec2.GetZoneUtil + extec2.GetVpcNameUtil +} + +func getAllRdsInstances(ctx context.Context, rdsApi rdsDBInstanceApi, ec2Util rdsInstanceDiscoveryEc2Util, awsAccountNumber string, awsRegion string) ([]discovery_kit_api.Target, error) { result := make([]discovery_kit_api.Target, 0, 20) paginator := rds.NewDescribeDBInstancesPaginator(rdsApi, &rds.DescribeDBInstancesInput{}) @@ -122,18 +128,18 @@ func getAllRdsInstances(ctx context.Context, rdsApi rdsDBInstanceApi, zoneUtil u } for _, dbInstance := range output.DBInstances { - result = append(result, toInstanceTarget(dbInstance, zoneUtil, awsAccountNumber, awsRegion)) + result = append(result, toInstanceTarget(dbInstance, ec2Util, awsAccountNumber, awsRegion)) } } return discovery_kit_commons.ApplyAttributeExcludes(result, config.Config.DiscoveryAttributesExcludesRds), nil } -func toInstanceTarget(dbInstance types.DBInstance, zoneUtil utils.GetZoneUtil, awsAccountNumber string, awsRegion string) discovery_kit_api.Target { +func toInstanceTarget(dbInstance types.DBInstance, ec2util rdsInstanceDiscoveryEc2Util, awsAccountNumber string, awsRegion string) discovery_kit_api.Target { arn := aws.ToString(dbInstance.DBInstanceArn) label := aws.ToString(dbInstance.DBInstanceIdentifier) availabilityZoneName := aws.ToString(dbInstance.AvailabilityZone) - availabilityZoneApi := zoneUtil.GetZone(awsAccountNumber, availabilityZoneName, awsRegion) + availabilityZoneApi := ec2util.GetZone(awsAccountNumber, availabilityZoneName, awsRegion) attributes := make(map[string][]string) attributes["aws.account"] = []string{awsAccountNumber} @@ -143,6 +149,10 @@ func toInstanceTarget(dbInstance types.DBInstance, zoneUtil utils.GetZoneUtil, a attributes["aws.zone.id"] = []string{*availabilityZoneApi.ZoneId} } attributes["aws.region"] = []string{awsRegion} + if dbInstance.DBSubnetGroup != nil && dbInstance.DBSubnetGroup.VpcId != nil { + attributes["aws.vpc.id"] = []string{aws.ToString(dbInstance.DBSubnetGroup.VpcId)} + attributes["aws.vpc.name"] = []string{ec2util.GetVpcName(awsAccountNumber, awsRegion, aws.ToString(dbInstance.DBSubnetGroup.VpcId))} + } attributes["aws.rds.engine"] = []string{aws.ToString(dbInstance.Engine)} attributes["aws.rds.instance.id"] = []string{label} attributes["aws.rds.instance.status"] = []string{aws.ToString(dbInstance.DBInstanceStatus)} diff --git a/extrds/instance_discovery_test.go b/extrds/instance_discovery_test.go index 7f4bc9d..1fc508f 100644 --- a/extrds/instance_discovery_test.go +++ b/extrds/instance_discovery_test.go @@ -6,6 +6,7 @@ package extrds import ( "context" "errors" + "github.com/steadybit/extension-kit/extutil" ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/aws/aws-sdk-go-v2/service/rds" @@ -23,6 +24,9 @@ func TestGetAllRdsInstances(t *testing.T) { mockedReturnValue := rds.DescribeDBInstancesOutput{ DBInstances: []types.DBInstance{ { + DBSubnetGroup: extutil.Ptr(types.DBSubnetGroup{ + VpcId: discovery_kit_api.Ptr("vpc-123-id"), + }), DBInstanceArn: discovery_kit_api.Ptr("arn"), DBInstanceIdentifier: discovery_kit_api.Ptr("identifier"), AvailabilityZone: discovery_kit_api.Ptr("us-east-1a"), @@ -44,6 +48,7 @@ func TestGetAllRdsInstances(t *testing.T) { ZoneId: discovery_kit_api.Ptr("us-east-1a-id"), } mockedZoneUtil.On("GetZone", mock.Anything, mock.Anything, mock.Anything).Return(&mockedZone) + mockedZoneUtil.On("GetVpcName", mock.Anything, mock.Anything, mock.Anything).Return("vpc-123-name") // When targets, err := getAllRdsInstances(context.Background(), mockedApi, mockedZoneUtil, "42", "us-east-1") @@ -56,13 +61,15 @@ func TestGetAllRdsInstances(t *testing.T) { assert.Equal(t, rdsInstanceTargetId, target.TargetType) assert.Equal(t, "identifier", target.Label) assert.Equal(t, "arn", target.Id) - assert.Equal(t, 10, len(target.Attributes)) + assert.Equal(t, 12, len(target.Attributes)) assert.Equal(t, []string{"cluster"}, target.Attributes["aws.rds.cluster"]) assert.Equal(t, []string{"status"}, target.Attributes["aws.rds.instance.status"]) assert.Equal(t, []string{"42"}, target.Attributes["aws.account"]) assert.Equal(t, []string{"us-east-1"}, target.Attributes["aws.region"]) assert.Equal(t, []string{"us-east-1a"}, target.Attributes["aws.zone"]) assert.Equal(t, []string{"us-east-1a-id"}, target.Attributes["aws.zone.id"]) + assert.Equal(t, []string{"vpc-123-id"}, target.Attributes["aws.vpc.id"]) + assert.Equal(t, []string{"vpc-123-name"}, target.Attributes["aws.vpc.name"]) assert.Equal(t, []string{"Great Thing"}, target.Attributes["aws.rds.label.specialtag"]) } diff --git a/main.go b/main.go index 9a5e67e..4216495 100644 --- a/main.go +++ b/main.go @@ -45,7 +45,7 @@ func main() { config.ValidateConfiguration() utils.InitializeAwsAccess(config.Config) - utils.InitializeAwsZones() + extec2.InitializeEc2Util() ctx, cancel := SignalCanceledContext() diff --git a/utils/aws_zones.go b/utils/aws_zones.go deleted file mode 100644 index eef79bc..0000000 --- a/utils/aws_zones.go +++ /dev/null @@ -1,92 +0,0 @@ -package utils - -import ( - "context" - "errors" - "github.com/aws/aws-sdk-go-v2/aws" - awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" - "github.com/aws/aws-sdk-go-v2/service/ec2" - "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/rs/zerolog/log" - "github.com/steadybit/discovery-kit/go/discovery_kit_api" - "sync" -) - -var ( - Zones *AwsZones -) - -type AwsZones struct { - zones sync.Map -} - -func InitializeAwsZones() { - Zones = &AwsZones{ - zones: sync.Map{}, - } - _, _ = ForEveryConfiguredAwsAccess(initAwsZonesForAccount, context.Background(), "availability zone") -} - -func initAwsZonesForAccount(account *AwsAccess, ctx context.Context) ([]discovery_kit_api.Target, error) { - return initAwsZonesForAccountWithClient(ec2.NewFromConfig(account.AwsConfig), account.AccountNumber, account.Region, ctx) -} - -func initAwsZonesForAccountWithClient(client AZDescribeAvailabilityZonesApi, awsAccountNumber string, region string, ctx context.Context) ([]discovery_kit_api.Target, error) { - result := getAllAvailabilityZones(ctx, client, awsAccountNumber, region) - Zones.zones.Store(awsAccountNumber+"-"+region, result) - return nil, nil -} - -type AZDescribeAvailabilityZonesApi interface { - DescribeAvailabilityZones(ctx context.Context, params *ec2.DescribeAvailabilityZonesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeAvailabilityZonesOutput, error) -} - -func getAllAvailabilityZones(ctx context.Context, ec2Api AZDescribeAvailabilityZonesApi, awsAccountNumber string, region string) []types.AvailabilityZone { - output, err := ec2Api.DescribeAvailabilityZones(ctx, &ec2.DescribeAvailabilityZonesInput{ - AllAvailabilityZones: aws.Bool(false), - }) - if err != nil { - var re *awshttp.ResponseError - if errors.As(err, &re) && re.HTTPStatusCode() == 403 { - log.Error().Msgf("Not Authorized to discover availability zones for account %s and region %s. If this is intended, you can disable the discovery by setting STEADYBIT_EXTENSION_DISCOVERY_DISABLED_ZONE=true. Details: %s", awsAccountNumber, region, re.Error()) - return []types.AvailabilityZone{} - } - log.Error().Err(err).Msgf("Failed to load availability zones for account %s and region %s.", awsAccountNumber, region) - return []types.AvailabilityZone{} - } - return output.AvailabilityZones -} - -type GetZoneUtil interface { - GetZone(awsAccountNumber string, region string, awsZone string) *types.AvailabilityZone -} -type GetZonesUtil interface { - GetZones(account *AwsAccess, ctx context.Context, updateCache bool) []types.AvailabilityZone -} - -func (zones *AwsZones) GetZones(account *AwsAccess, ctx context.Context, updateCache bool) []types.AvailabilityZone { - awsAccountNumber := account.AccountNumber - region := account.Region - if updateCache { - _, _ = initAwsZonesForAccountWithClient(ec2.NewFromConfig(account.AwsConfig), awsAccountNumber, region, ctx) - } - - value, ok := zones.zones.Load(awsAccountNumber + "-" + region) - if !ok { - return []types.AvailabilityZone{} - } - return value.([]types.AvailabilityZone) -} - -func (zones *AwsZones) GetZone(awsAccountNumber string, region string, awsZone string) *types.AvailabilityZone { - value, ok := zones.zones.Load(awsAccountNumber + "-" + region) - if !ok { - return nil - } - for _, zone := range value.([]types.AvailabilityZone) { - if aws.ToString(zone.ZoneName) == awsZone { - return &zone - } - } - return nil -}