Skip to content

Commit

Permalink
✨ Assume prerequisite role on hub if initailize with aws-irsa (#807)
Browse files Browse the repository at this point in the history
* Adding managedcluster identity creator role arn

Signed-off-by: Gaurav Jaswal <[email protected]>

* Addressing review comments

Signed-off-by: Gaurav Jaswal <[email protected]>

---------

Signed-off-by: Gaurav Jaswal <[email protected]>
Co-authored-by: Amrutha <[email protected]>
  • Loading branch information
jaswalkiranavtar and amrcoder authored Jan 17, 2025
1 parent d323b60 commit f62242d
Show file tree
Hide file tree
Showing 22 changed files with 310 additions and 31 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,31 @@ spec:
- feature
type: object
type: array
registrationDrivers:
description: |-
RegistrationDrivers represent the list of hub registration drivers that contain information used by hub to initialize the hub cluster
A RegistrationDriverHub contains details of authentication type and the hub cluster ARN
items:
properties:
authType:
default: csr
description: Type of the authentication used by hub to initialize
the Hub cluster. Possible values are csr and awsirsa.
enum:
- csr
- awsirsa
type: string
hubClusterArn:
description: |-
This represents the hub cluster ARN
Example - arn:eks:us-west-2:12345678910:cluster/hub-cluster1
pattern: ^arn:aws:eks:([a-zA-Z0-9-]+):(\d{12}):cluster/([a-zA-Z0-9-]+)$
type: string
type: object
type: array
x-kubernetes-list-map-keys:
- authType
x-kubernetes-list-type: map
type: object
registrationImagePullSpec:
default: quay.io/open-cluster-management/registration
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,31 @@ spec:
- feature
type: object
type: array
registrationDrivers:
description: |-
RegistrationDrivers represent the list of hub registration drivers that contain information used by hub to initialize the hub cluster
A RegistrationDriverHub contains details of authentication type and the hub cluster ARN
items:
properties:
authType:
default: csr
description: Type of the authentication used by hub to initialize
the Hub cluster. Possible values are csr and awsirsa.
enum:
- csr
- awsirsa
type: string
hubClusterArn:
description: |-
This represents the hub cluster ARN
Example - arn:eks:us-west-2:12345678910:cluster/hub-cluster1
pattern: ^arn:aws:eks:([a-zA-Z0-9-]+):(\d{12}):cluster/([a-zA-Z0-9-]+)$
type: string
type: object
type: array
x-kubernetes-list-map-keys:
- authType
x-kubernetes-list-type: map
type: object
registrationImagePullSpec:
default: quay.io/open-cluster-management/registration
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ metadata:
categories: Integration & Delivery,OpenShift Optional
certified: "false"
containerImage: quay.io/open-cluster-management/registration-operator:latest
createdAt: "2025-01-06T02:51:43Z"
createdAt: "2025-01-16T23:45:52Z"
description: Manages the installation and upgrade of the ClusterManager.
operators.operatorframework.io/builder: operator-sdk-v1.32.0
operators.operatorframework.io/project_layout: go.kubebuilder.io/v3
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,31 @@ spec:
- feature
type: object
type: array
registrationDrivers:
description: |-
RegistrationDrivers represent the list of hub registration drivers that contain information used by hub to initialize the hub cluster
A RegistrationDriverHub contains details of authentication type and the hub cluster ARN
items:
properties:
authType:
default: csr
description: Type of the authentication used by hub to initialize
the Hub cluster. Possible values are csr and awsirsa.
enum:
- csr
- awsirsa
type: string
hubClusterArn:
description: |-
This represents the hub cluster ARN
Example - arn:eks:us-west-2:12345678910:cluster/hub-cluster1
pattern: ^arn:aws:eks:([a-zA-Z0-9-]+):(\d{12}):cluster/([a-zA-Z0-9-]+)$
type: string
type: object
type: array
x-kubernetes-list-map-keys:
- authType
x-kubernetes-list-type: map
type: object
registrationImagePullSpec:
default: quay.io/open-cluster-management/registration
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ metadata:
categories: Integration & Delivery,OpenShift Optional
certified: "false"
containerImage: quay.io/open-cluster-management/registration-operator:latest
createdAt: "2024-12-18T07:51:42Z"
createdAt: "2025-01-16T23:45:52Z"
description: Manages the installation and upgrade of the Klusterlet.
operators.operatorframework.io/builder: operator-sdk-v1.32.0
operators.operatorframework.io/project_layout: go.kubebuilder.io/v3
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ require (
k8s.io/kube-aggregator v0.31.4
k8s.io/utils v0.0.0-20240921022957-49e7df575cb6
open-cluster-management.io/addon-framework v0.11.1-0.20241129080247-57b1d2859f50
open-cluster-management.io/api v0.15.1-0.20250109024121-1a5e25a78a43
open-cluster-management.io/api v0.15.1-0.20250116010516-3a595d6a4e40
open-cluster-management.io/sdk-go v0.15.1-0.20241125015855-1536c3970f8f
sigs.k8s.io/cluster-inventory-api v0.0.0-20240730014211-ef0154379848
sigs.k8s.io/controller-runtime v0.19.3
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -453,8 +453,8 @@ k8s.io/utils v0.0.0-20240921022957-49e7df575cb6 h1:MDF6h2H/h4tbzmtIKTuctcwZmY0tY
k8s.io/utils v0.0.0-20240921022957-49e7df575cb6/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
open-cluster-management.io/addon-framework v0.11.1-0.20241129080247-57b1d2859f50 h1:TXRd6OdGjArh6cwlCYOqlIcyx21k81oUIYj4rmHlYx0=
open-cluster-management.io/addon-framework v0.11.1-0.20241129080247-57b1d2859f50/go.mod h1:tsBSNs9mGfVQQjXBnjgpiX6r0UM+G3iNfmzQgKhEfw4=
open-cluster-management.io/api v0.15.1-0.20250109024121-1a5e25a78a43 h1:9kgKRQQHMGNM1t+J+OrmF7hgZmND9kRwyRVnHIULzqw=
open-cluster-management.io/api v0.15.1-0.20250109024121-1a5e25a78a43/go.mod h1:9erZEWEn4bEqh0nIX2wA7f/s3KCuFycQdBrPrRzi0QM=
open-cluster-management.io/api v0.15.1-0.20250116010516-3a595d6a4e40 h1:LckTHZ68rcy3hDFu6wa7BVOJ9wbWItJLZXmi0bpMyh8=
open-cluster-management.io/api v0.15.1-0.20250116010516-3a595d6a4e40/go.mod h1:9erZEWEn4bEqh0nIX2wA7f/s3KCuFycQdBrPrRzi0QM=
open-cluster-management.io/sdk-go v0.15.1-0.20241125015855-1536c3970f8f h1:zeC7QrFNarfK2zY6jGtd+mX+yDrQQmnH/J8A7n5Nh38=
open-cluster-management.io/sdk-go v0.15.1-0.20241125015855-1536c3970f8f/go.mod h1:fi5WBsbC5K3txKb8eRLuP0Sim/Oqz/PHX18skAEyjiA=
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 h1:2770sDpzrjjsAtVhSeUFseziht227YAWYHLGNM8QPwY=
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,10 @@ kind: ServiceAccount
metadata:
name: registration-controller-sa
namespace: {{ .ClusterManagerNamespace }}
{{ if .ManagedClusterIdentityCreatorRole }}
annotations:
eks.amazonaws.com/role-arn: {{ .ManagedClusterIdentityCreatorRole }}
{{ else }}
annotations:
eks.amazonaws.com/role-arn-: ""
{{end}}
3 changes: 2 additions & 1 deletion manifests/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ type HubConfig struct {
ResourceRequirementResourceType operatorapiv1.ResourceQosClass
// ResourceRequirements is the resource requirements for the cluster manager managed containers.
// The type has to be []byte to use "indent" template function.
ResourceRequirements []byte
ResourceRequirements []byte
ManagedClusterIdentityCreatorRole string
}

type Webhook struct {
Expand Down
21 changes: 21 additions & 0 deletions pkg/common/helpers/aws.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package helpers

import "strings"

// GetAwsAccountIdAndClusterName Parses aws accountId and cluster-name from clusterArn
// e.g. if clusterArn is arn:aws:eks:us-west-2:123456789012:cluster/hub-cluster1
// accountId is 123456789012 and clusterName is hub-cluster1
func GetAwsAccountIdAndClusterName(clusterArn string) (string, string) {
clusterStringParts := strings.Split(clusterArn, ":")
clusterName := strings.Split(clusterStringParts[5], "/")[1]
awsAccountId := clusterStringParts[4]
return awsAccountId, clusterName
}

// GetAwsRegion Parses aws accountId and cluster-name from clusterArn
// e.g. if clusterArn is arn:aws:eks:us-west-2:123456789012:cluster/hub-cluster1
// awsRegion is us-west-2
func GetAwsRegion(clusterArn string) string {
clusterStringParts := strings.Split(clusterArn, ":")
return clusterStringParts[3]
}
14 changes: 14 additions & 0 deletions pkg/common/helpers/aws_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package helpers

import (
"testing"
)

func TestGetAwsAccountIdAndClusterName(t *testing.T) {

awsAccountId, clusterName := GetAwsAccountIdAndClusterName("arn:aws:eks:us-west-2:123456789012:cluster/hub-cluster")
if awsAccountId != "123456789012" && clusterName != "hub-cluster" {
t.Errorf("awsAccountId and cluster id are not valid")
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,9 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f
// Check if addon management is enabled by the feature gate
config.AddOnManagerEnabled = helpers.FeatureGateEnabled(addonFeatureGates, ocmfeature.DefaultHubAddonManagerFeatureGates, ocmfeature.AddonManagement)

// Compute and populate the value of managed cluster identity creator role to be used in cluster manager registration service account
config.ManagedClusterIdentityCreatorRole = getManagedClusterIdentityCreatorRolename(*clusterManager)

// If we are deploying in the hosted mode, it requires us to create webhook in a different way with the default mode.
// In the hosted mode, the webhook servers is running in the management cluster but the users are accessing the hub cluster.
// So we need to add configuration to make the apiserver of the hub cluster could access the webhook servers on the management cluster.
Expand Down Expand Up @@ -420,3 +423,16 @@ func (n *clusterManagerController) getImagePullSecret(ctx context.Context) (stri

return helpers.ImagePullSecret, nil
}

func getManagedClusterIdentityCreatorRolename(cm operatorapiv1.ClusterManager) string {
if cm.Spec.RegistrationConfiguration != nil {
for _, registrationDriver := range cm.Spec.RegistrationConfiguration.RegistrationDrivers {
if registrationDriver.AuthType == "awsirsa" {
hubClusterArn := registrationDriver.HubClusterArn
hubClusterAccountId, hubClusterName := commonhelper.GetAwsAccountIdAndClusterName(hubClusterArn)
return "arn:aws:iam::" + hubClusterAccountId + ":role/" + hubClusterName + "_managed-cluster-identity-creator"
}
}
}
return ""
}
Original file line number Diff line number Diff line change
Expand Up @@ -130,16 +130,16 @@ type ManagedClusterIamRole struct {
}

func (managedClusterIamRole *ManagedClusterIamRole) arn() string {
managedClusterAccountId, _ := GetAwsAccountIdAndClusterName(managedClusterIamRole.AwsIrsa.ManagedClusterArn)
managedClusterAccountId, _ := commonhelpers.GetAwsAccountIdAndClusterName(managedClusterIamRole.AwsIrsa.ManagedClusterArn)
md5HashUniqueIdentifier := managedClusterIamRole.md5HashSuffix()

//arn:aws:iam::<managed-cluster-account-id>:role/ocm-managed-cluster-<md5-hash-unique-identifier>
return "arn:aws:iam::" + managedClusterAccountId + ":role/ocm-managed-cluster-" + md5HashUniqueIdentifier
}

func (managedClusterIamRole *ManagedClusterIamRole) md5HashSuffix() string {
hubClusterAccountId, hubClusterName := GetAwsAccountIdAndClusterName(managedClusterIamRole.AwsIrsa.HubClusterArn)
managedClusterAccountId, managedClusterName := GetAwsAccountIdAndClusterName(managedClusterIamRole.AwsIrsa.ManagedClusterArn)
hubClusterAccountId, hubClusterName := commonhelpers.GetAwsAccountIdAndClusterName(managedClusterIamRole.AwsIrsa.HubClusterArn)
managedClusterAccountId, managedClusterName := commonhelpers.GetAwsAccountIdAndClusterName(managedClusterIamRole.AwsIrsa.ManagedClusterArn)

hash := md5.Sum([]byte(strings.Join([]string{hubClusterAccountId, hubClusterName, managedClusterAccountId, managedClusterName}, "#"))) // #nosec G401
return hex.EncodeToString(hash[:])
Expand Down Expand Up @@ -573,15 +573,3 @@ func serviceAccountName(suffix string, klusterlet *operatorapiv1.Klusterlet) str
}
return fmt.Sprintf("%s-%s", klusterlet.Name, suffix)
}

func GetAwsAccountIdAndClusterName(clusterArn string) (string, string) {
clusterStringParts := strings.Split(clusterArn, ":")
clusterName := strings.Split(clusterStringParts[5], "/")[1]
awsAccountId := clusterStringParts[4]
return awsAccountId, clusterName
}

func GetAwsRegion(clusterArn string) string {
clusterStringParts := strings.Split(clusterArn, ":")
return clusterStringParts[3]
}
6 changes: 3 additions & 3 deletions pkg/registration/register/aws_irsa/aws_irsa.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import (
clusterv1 "open-cluster-management.io/api/cluster/v1"
operatorv1 "open-cluster-management.io/api/operator/v1"

"open-cluster-management.io/ocm/pkg/operator/operators/klusterlet/controllers/klusterletcontroller"
"open-cluster-management.io/ocm/pkg/common/helpers"
"open-cluster-management.io/ocm/pkg/registration/register"
)

Expand Down Expand Up @@ -59,8 +59,8 @@ func (c *AWSIRSADriver) Process(
}

func (c *AWSIRSADriver) BuildKubeConfigFromTemplate(kubeConfig *clientcmdapi.Config) *clientcmdapi.Config {
hubClusterAccountId, hubClusterName := klusterletcontroller.GetAwsAccountIdAndClusterName(c.hubClusterArn)
awsRegion := klusterletcontroller.GetAwsRegion(c.hubClusterArn)
hubClusterAccountId, hubClusterName := helpers.GetAwsAccountIdAndClusterName(c.hubClusterArn)
awsRegion := helpers.GetAwsRegion(c.hubClusterArn)
kubeConfig.AuthInfos = map[string]*clientcmdapi.AuthInfo{register.DefaultKubeConfigAuth: {
Exec: &clientcmdapi.ExecConfig{
APIVersion: "client.authentication.k8s.io/v1beta1",
Expand Down
73 changes: 73 additions & 0 deletions test/integration/operator/clustermanager_aws_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
package operator

import (
"context"

"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

operatorapiv1 "open-cluster-management.io/api/operator/v1"
)

var _ = ginkgo.Describe("ClusterManager Default Mode with aws registration", func() {
var cancel context.CancelFunc
var hubRegistrationSA = "registration-controller-sa"

ginkgo.BeforeEach(func() {
var ctx context.Context
ctx, cancel = context.WithCancel(context.Background())
go startHubOperator(ctx, operatorapiv1.InstallModeDefault)
})

ginkgo.AfterEach(func() {
// delete deployment for clustermanager here so tests are not impacted with each other
err := kubeClient.AppsV1().Deployments(hubNamespace).DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred())
if cancel != nil {
cancel()
}
})

ginkgo.Context("Deploy hub with aws auth", func() {

ginkgo.It("should have IAM role annotation when initialized with awsirsa", func() {

clusterManager, err := operatorClient.OperatorV1().ClusterManagers().Get(context.Background(), clusterManagerName, metav1.GetOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())

if clusterManager.Spec.RegistrationConfiguration == nil {
clusterManager.Spec.RegistrationConfiguration = &operatorapiv1.RegistrationHubConfiguration{}
clusterManager.Spec.RegistrationConfiguration.RegistrationDrivers = []operatorapiv1.RegistrationDriverHub{
{
AuthType: "awsirsa",
HubClusterArn: "arn:aws:eks:us-west-2:123456789012:cluster/hub-cluster",
},
}
}
_, err = operatorClient.OperatorV1().ClusterManagers().Update(context.Background(), clusterManager, metav1.UpdateOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())

gomega.Eventually(func() bool {
registrationControllerSA, err := kubeClient.CoreV1().ServiceAccounts(hubNamespace).Get(
context.Background(), hubRegistrationSA, metav1.GetOptions{})
if err != nil {
return false
}
annotation := registrationControllerSA.Annotations["eks.amazonaws.com/role-arn"]

// The same cluster-manager CR is used for other tests.
// Hence updating it here, so that annotation is removed for other test testing with csr or empty registration
clusterManager, err := operatorClient.OperatorV1().ClusterManagers().Get(context.Background(), clusterManagerName, metav1.GetOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
clusterManager.Spec.RegistrationConfiguration = nil
_, err = operatorClient.OperatorV1().ClusterManagers().Update(context.Background(), clusterManager, metav1.UpdateOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())

return annotation == "arn:aws:iam::123456789012:role/hub-cluster_managed-cluster-identity-creator"
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())

})
})

})
8 changes: 7 additions & 1 deletion test/integration/operator/clustermanager_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -153,9 +153,14 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() {

// Check service account
gomega.Eventually(func() error {
if _, err := kubeClient.CoreV1().ServiceAccounts(hubNamespace).Get(context.Background(), hubRegistrationSA, metav1.GetOptions{}); err != nil {
registrationControllerSA, err := kubeClient.CoreV1().ServiceAccounts(hubNamespace).Get(context.Background(), hubRegistrationSA, metav1.GetOptions{})
if err != nil {
return err
}

if _, ok := registrationControllerSA.Annotations["eks.amazonaws.com/role-arn"]; ok {
return fmt.Errorf("Annotation applicable to awsirsa registration only")
}
return nil
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil())
gomega.Eventually(func() error {
Expand Down Expand Up @@ -1294,4 +1299,5 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() {
gomega.ContainElement("manager"))
})
})

})
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ import (
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/tools/clientcmd"

"open-cluster-management.io/ocm/pkg/common/helpers"
commonoptions "open-cluster-management.io/ocm/pkg/common/options"
"open-cluster-management.io/ocm/pkg/operator/operators/klusterlet/controllers/klusterletcontroller"
"open-cluster-management.io/ocm/pkg/registration/register"
"open-cluster-management.io/ocm/pkg/registration/spoke"
"open-cluster-management.io/ocm/test/integration/util"
Expand Down Expand Up @@ -113,8 +113,8 @@ var _ = ginkgo.Describe("Joining Process for aws flow", func() {
return fmt.Errorf("user exec plugun command is invalid")
}

hubClusterAccountId, hubClusterName := klusterletcontroller.GetAwsAccountIdAndClusterName(hubClusterArn)
awsRegion := klusterletcontroller.GetAwsRegion(hubClusterArn)
hubClusterAccountId, hubClusterName := helpers.GetAwsAccountIdAndClusterName(hubClusterArn)
awsRegion := helpers.GetAwsRegion(hubClusterArn)

if !contains(hubUser.Exec.Args, fmt.Sprintf("arn:aws:iam::%s:role/ocm-hub-%s", hubClusterAccountId, managedClusterRoleSuffix)) ||
!contains(hubUser.Exec.Args, hubClusterName) ||
Expand Down
Loading

0 comments on commit f62242d

Please sign in to comment.