Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

✨ Assume prerequisite role on hub if initailize with aws-irsa #807

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,31 @@ spec:
- feature
type: object
type: array
registrationDrivers:
description: |-
RegistrationDrivers represent the list of hub registration drivers that contain information used by hub to initialize the hub cluster
A RegistrationDriverHub contains details of authentication type and the hub cluster ARN
items:
properties:
authType:
default: csr
description: Type of the authentication used by hub to initialize
the Hub cluster. Possible values are csr and awsirsa.
enum:
- csr
- awsirsa
type: string
hubClusterArn:
description: |-
This represents the hub cluster ARN
Example - arn:eks:us-west-2:12345678910:cluster/hub-cluster1
pattern: ^arn:aws:eks:([a-zA-Z0-9-]+):(\d{12}):cluster/([a-zA-Z0-9-]+)$
type: string
type: object
type: array
x-kubernetes-list-map-keys:
- authType
x-kubernetes-list-type: map
type: object
registrationImagePullSpec:
default: quay.io/open-cluster-management/registration
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,31 @@ spec:
- feature
type: object
type: array
registrationDrivers:
description: |-
RegistrationDrivers represent the list of hub registration drivers that contain information used by hub to initialize the hub cluster
A RegistrationDriverHub contains details of authentication type and the hub cluster ARN
items:
properties:
authType:
default: csr
description: Type of the authentication used by hub to initialize
the Hub cluster. Possible values are csr and awsirsa.
enum:
- csr
- awsirsa
type: string
hubClusterArn:
description: |-
This represents the hub cluster ARN
Example - arn:eks:us-west-2:12345678910:cluster/hub-cluster1
pattern: ^arn:aws:eks:([a-zA-Z0-9-]+):(\d{12}):cluster/([a-zA-Z0-9-]+)$
type: string
type: object
type: array
x-kubernetes-list-map-keys:
- authType
x-kubernetes-list-type: map
type: object
registrationImagePullSpec:
default: quay.io/open-cluster-management/registration
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ metadata:
categories: Integration & Delivery,OpenShift Optional
certified: "false"
containerImage: quay.io/open-cluster-management/registration-operator:latest
createdAt: "2025-01-06T02:51:43Z"
createdAt: "2025-01-16T23:45:52Z"
description: Manages the installation and upgrade of the ClusterManager.
operators.operatorframework.io/builder: operator-sdk-v1.32.0
operators.operatorframework.io/project_layout: go.kubebuilder.io/v3
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,31 @@ spec:
- feature
type: object
type: array
registrationDrivers:
description: |-
RegistrationDrivers represent the list of hub registration drivers that contain information used by hub to initialize the hub cluster
A RegistrationDriverHub contains details of authentication type and the hub cluster ARN
items:
properties:
authType:
default: csr
description: Type of the authentication used by hub to initialize
the Hub cluster. Possible values are csr and awsirsa.
enum:
- csr
- awsirsa
type: string
hubClusterArn:
description: |-
This represents the hub cluster ARN
Example - arn:eks:us-west-2:12345678910:cluster/hub-cluster1
pattern: ^arn:aws:eks:([a-zA-Z0-9-]+):(\d{12}):cluster/([a-zA-Z0-9-]+)$
type: string
type: object
type: array
x-kubernetes-list-map-keys:
- authType
x-kubernetes-list-type: map
type: object
registrationImagePullSpec:
default: quay.io/open-cluster-management/registration
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ metadata:
categories: Integration & Delivery,OpenShift Optional
certified: "false"
containerImage: quay.io/open-cluster-management/registration-operator:latest
createdAt: "2024-12-18T07:51:42Z"
createdAt: "2025-01-16T23:45:52Z"
description: Manages the installation and upgrade of the Klusterlet.
operators.operatorframework.io/builder: operator-sdk-v1.32.0
operators.operatorframework.io/project_layout: go.kubebuilder.io/v3
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ require (
k8s.io/kube-aggregator v0.31.4
k8s.io/utils v0.0.0-20240921022957-49e7df575cb6
open-cluster-management.io/addon-framework v0.11.1-0.20241129080247-57b1d2859f50
open-cluster-management.io/api v0.15.1-0.20250109024121-1a5e25a78a43
open-cluster-management.io/api v0.15.1-0.20250116010516-3a595d6a4e40
open-cluster-management.io/sdk-go v0.15.1-0.20241125015855-1536c3970f8f
sigs.k8s.io/cluster-inventory-api v0.0.0-20240730014211-ef0154379848
sigs.k8s.io/controller-runtime v0.19.3
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -453,8 +453,8 @@ k8s.io/utils v0.0.0-20240921022957-49e7df575cb6 h1:MDF6h2H/h4tbzmtIKTuctcwZmY0tY
k8s.io/utils v0.0.0-20240921022957-49e7df575cb6/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
open-cluster-management.io/addon-framework v0.11.1-0.20241129080247-57b1d2859f50 h1:TXRd6OdGjArh6cwlCYOqlIcyx21k81oUIYj4rmHlYx0=
open-cluster-management.io/addon-framework v0.11.1-0.20241129080247-57b1d2859f50/go.mod h1:tsBSNs9mGfVQQjXBnjgpiX6r0UM+G3iNfmzQgKhEfw4=
open-cluster-management.io/api v0.15.1-0.20250109024121-1a5e25a78a43 h1:9kgKRQQHMGNM1t+J+OrmF7hgZmND9kRwyRVnHIULzqw=
open-cluster-management.io/api v0.15.1-0.20250109024121-1a5e25a78a43/go.mod h1:9erZEWEn4bEqh0nIX2wA7f/s3KCuFycQdBrPrRzi0QM=
open-cluster-management.io/api v0.15.1-0.20250116010516-3a595d6a4e40 h1:LckTHZ68rcy3hDFu6wa7BVOJ9wbWItJLZXmi0bpMyh8=
open-cluster-management.io/api v0.15.1-0.20250116010516-3a595d6a4e40/go.mod h1:9erZEWEn4bEqh0nIX2wA7f/s3KCuFycQdBrPrRzi0QM=
open-cluster-management.io/sdk-go v0.15.1-0.20241125015855-1536c3970f8f h1:zeC7QrFNarfK2zY6jGtd+mX+yDrQQmnH/J8A7n5Nh38=
open-cluster-management.io/sdk-go v0.15.1-0.20241125015855-1536c3970f8f/go.mod h1:fi5WBsbC5K3txKb8eRLuP0Sim/Oqz/PHX18skAEyjiA=
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 h1:2770sDpzrjjsAtVhSeUFseziht227YAWYHLGNM8QPwY=
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,10 @@ kind: ServiceAccount
metadata:
name: registration-controller-sa
namespace: {{ .ClusterManagerNamespace }}
{{ if .ManagedClusterIdentityCreatorRole }}
annotations:
eks.amazonaws.com/role-arn: {{ .ManagedClusterIdentityCreatorRole }}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is not quite extensible, e.g. the key is specifically for eks. How do you think the keys/values could be configured from the template?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think same needs to be applied on klusterlet SA as well. Will address it in a seperate PR.

{{ else }}
annotations:
eks.amazonaws.com/role-arn-: ""
{{end}}
3 changes: 2 additions & 1 deletion manifests/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ type HubConfig struct {
ResourceRequirementResourceType operatorapiv1.ResourceQosClass
// ResourceRequirements is the resource requirements for the cluster manager managed containers.
// The type has to be []byte to use "indent" template function.
ResourceRequirements []byte
ResourceRequirements []byte
ManagedClusterIdentityCreatorRole string
}

type Webhook struct {
Expand Down
21 changes: 21 additions & 0 deletions pkg/common/helpers/aws.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package helpers

import "strings"

// GetAwsAccountIdAndClusterName Parses aws accountId and cluster-name from clusterArn
// e.g. if clusterArn is arn:aws:eks:us-west-2:123456789012:cluster/hub-cluster1
// accountId is 123456789012 and clusterName is hub-cluster1
func GetAwsAccountIdAndClusterName(clusterArn string) (string, string) {
clusterStringParts := strings.Split(clusterArn, ":")
clusterName := strings.Split(clusterStringParts[5], "/")[1]
awsAccountId := clusterStringParts[4]
return awsAccountId, clusterName
}

// GetAwsRegion Parses aws accountId and cluster-name from clusterArn
// e.g. if clusterArn is arn:aws:eks:us-west-2:123456789012:cluster/hub-cluster1
// awsRegion is us-west-2
func GetAwsRegion(clusterArn string) string {
clusterStringParts := strings.Split(clusterArn, ":")
return clusterStringParts[3]
}
14 changes: 14 additions & 0 deletions pkg/common/helpers/aws_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package helpers

import (
"testing"
)

func TestGetAwsAccountIdAndClusterName(t *testing.T) {

awsAccountId, clusterName := GetAwsAccountIdAndClusterName("arn:aws:eks:us-west-2:123456789012:cluster/hub-cluster")
if awsAccountId != "123456789012" && clusterName != "hub-cluster" {
t.Errorf("awsAccountId and cluster id are not valid")
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,9 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f
// Check if addon management is enabled by the feature gate
config.AddOnManagerEnabled = helpers.FeatureGateEnabled(addonFeatureGates, ocmfeature.DefaultHubAddonManagerFeatureGates, ocmfeature.AddonManagement)

// Compute and populate the value of managed cluster identity creator role to be used in cluster manager registration service account
config.ManagedClusterIdentityCreatorRole = getManagedClusterIdentityCreatorRolename(*clusterManager)

// If we are deploying in the hosted mode, it requires us to create webhook in a different way with the default mode.
// In the hosted mode, the webhook servers is running in the management cluster but the users are accessing the hub cluster.
// So we need to add configuration to make the apiserver of the hub cluster could access the webhook servers on the management cluster.
Expand Down Expand Up @@ -419,3 +422,16 @@ func (n *clusterManagerController) getImagePullSecret(ctx context.Context) (stri

return helpers.ImagePullSecret, nil
}

func getManagedClusterIdentityCreatorRolename(cm operatorapiv1.ClusterManager) string {
if cm.Spec.RegistrationConfiguration != nil {
for _, registrationDriver := range cm.Spec.RegistrationConfiguration.RegistrationDrivers {
if registrationDriver.AuthType == "awsirsa" {
hubClusterArn := registrationDriver.HubClusterArn
hubClusterAccountId, hubClusterName := commonhelper.GetAwsAccountIdAndClusterName(hubClusterArn)
return "arn:aws:iam::" + hubClusterAccountId + ":role/" + hubClusterName + "_managed-cluster-identity-creator"
}
}
}
return ""
}
Original file line number Diff line number Diff line change
Expand Up @@ -130,16 +130,16 @@ type ManagedClusterIamRole struct {
}

func (managedClusterIamRole *ManagedClusterIamRole) arn() string {
managedClusterAccountId, _ := GetAwsAccountIdAndClusterName(managedClusterIamRole.AwsIrsa.ManagedClusterArn)
managedClusterAccountId, _ := commonhelpers.GetAwsAccountIdAndClusterName(managedClusterIamRole.AwsIrsa.ManagedClusterArn)
md5HashUniqueIdentifier := managedClusterIamRole.md5HashSuffix()

//arn:aws:iam::<managed-cluster-account-id>:role/ocm-managed-cluster-<md5-hash-unique-identifier>
return "arn:aws:iam::" + managedClusterAccountId + ":role/ocm-managed-cluster-" + md5HashUniqueIdentifier
}

func (managedClusterIamRole *ManagedClusterIamRole) md5HashSuffix() string {
hubClusterAccountId, hubClusterName := GetAwsAccountIdAndClusterName(managedClusterIamRole.AwsIrsa.HubClusterArn)
managedClusterAccountId, managedClusterName := GetAwsAccountIdAndClusterName(managedClusterIamRole.AwsIrsa.ManagedClusterArn)
hubClusterAccountId, hubClusterName := commonhelpers.GetAwsAccountIdAndClusterName(managedClusterIamRole.AwsIrsa.HubClusterArn)
managedClusterAccountId, managedClusterName := commonhelpers.GetAwsAccountIdAndClusterName(managedClusterIamRole.AwsIrsa.ManagedClusterArn)

hash := md5.Sum([]byte(strings.Join([]string{hubClusterAccountId, hubClusterName, managedClusterAccountId, managedClusterName}, "#"))) // #nosec G401
return hex.EncodeToString(hash[:])
Expand Down Expand Up @@ -573,15 +573,3 @@ func serviceAccountName(suffix string, klusterlet *operatorapiv1.Klusterlet) str
}
return fmt.Sprintf("%s-%s", klusterlet.Name, suffix)
}

func GetAwsAccountIdAndClusterName(clusterArn string) (string, string) {
clusterStringParts := strings.Split(clusterArn, ":")
clusterName := strings.Split(clusterStringParts[5], "/")[1]
awsAccountId := clusterStringParts[4]
return awsAccountId, clusterName
}

func GetAwsRegion(clusterArn string) string {
clusterStringParts := strings.Split(clusterArn, ":")
return clusterStringParts[3]
}
6 changes: 3 additions & 3 deletions pkg/registration/register/aws_irsa/aws_irsa.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import (
clusterv1 "open-cluster-management.io/api/cluster/v1"
operatorv1 "open-cluster-management.io/api/operator/v1"

"open-cluster-management.io/ocm/pkg/operator/operators/klusterlet/controllers/klusterletcontroller"
"open-cluster-management.io/ocm/pkg/common/helpers"
"open-cluster-management.io/ocm/pkg/registration/register"
)

Expand Down Expand Up @@ -59,8 +59,8 @@ func (c *AWSIRSADriver) Process(
}

func (c *AWSIRSADriver) BuildKubeConfigFromTemplate(kubeConfig *clientcmdapi.Config) *clientcmdapi.Config {
hubClusterAccountId, hubClusterName := klusterletcontroller.GetAwsAccountIdAndClusterName(c.hubClusterArn)
awsRegion := klusterletcontroller.GetAwsRegion(c.hubClusterArn)
hubClusterAccountId, hubClusterName := helpers.GetAwsAccountIdAndClusterName(c.hubClusterArn)
awsRegion := helpers.GetAwsRegion(c.hubClusterArn)
kubeConfig.AuthInfos = map[string]*clientcmdapi.AuthInfo{register.DefaultKubeConfigAuth: {
Exec: &clientcmdapi.ExecConfig{
APIVersion: "client.authentication.k8s.io/v1beta1",
Expand Down
73 changes: 73 additions & 0 deletions test/integration/operator/clustermanager_aws_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
package operator

import (
"context"

"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

operatorapiv1 "open-cluster-management.io/api/operator/v1"
)

var _ = ginkgo.Describe("ClusterManager Default Mode with aws registration", func() {
var cancel context.CancelFunc
var hubRegistrationSA = "registration-controller-sa"

ginkgo.BeforeEach(func() {
var ctx context.Context
ctx, cancel = context.WithCancel(context.Background())
go startHubOperator(ctx, operatorapiv1.InstallModeDefault)
})

ginkgo.AfterEach(func() {
// delete deployment for clustermanager here so tests are not impacted with each other
err := kubeClient.AppsV1().Deployments(hubNamespace).DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred())
if cancel != nil {
cancel()
}
})

ginkgo.Context("Deploy hub with aws auth", func() {

ginkgo.It("should have IAM role annotation when initialized with awsirsa", func() {

clusterManager, err := operatorClient.OperatorV1().ClusterManagers().Get(context.Background(), clusterManagerName, metav1.GetOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())

if clusterManager.Spec.RegistrationConfiguration == nil {
clusterManager.Spec.RegistrationConfiguration = &operatorapiv1.RegistrationHubConfiguration{}
clusterManager.Spec.RegistrationConfiguration.RegistrationDrivers = []operatorapiv1.RegistrationDriverHub{
{
AuthType: "awsirsa",
HubClusterArn: "arn:aws:eks:us-west-2:123456789012:cluster/hub-cluster",
},
}
}
_, err = operatorClient.OperatorV1().ClusterManagers().Update(context.Background(), clusterManager, metav1.UpdateOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())

gomega.Eventually(func() bool {
registrationControllerSA, err := kubeClient.CoreV1().ServiceAccounts(hubNamespace).Get(
context.Background(), hubRegistrationSA, metav1.GetOptions{})
if err != nil {
return false
}
annotation := registrationControllerSA.Annotations["eks.amazonaws.com/role-arn"]

// The same cluster-manager CR is used for other tests.
// Hence updating it here, so that annotation is removed for other test testing with csr or empty registration
clusterManager, err := operatorClient.OperatorV1().ClusterManagers().Get(context.Background(), clusterManagerName, metav1.GetOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
clusterManager.Spec.RegistrationConfiguration = nil
_, err = operatorClient.OperatorV1().ClusterManagers().Update(context.Background(), clusterManager, metav1.UpdateOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred())

return annotation == "arn:aws:iam::123456789012:role/hub-cluster_managed-cluster-identity-creator"
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())

})
})

})
8 changes: 7 additions & 1 deletion test/integration/operator/clustermanager_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -153,9 +153,14 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() {

// Check service account
gomega.Eventually(func() error {
if _, err := kubeClient.CoreV1().ServiceAccounts(hubNamespace).Get(context.Background(), hubRegistrationSA, metav1.GetOptions{}); err != nil {
registrationControllerSA, err := kubeClient.CoreV1().ServiceAccounts(hubNamespace).Get(context.Background(), hubRegistrationSA, metav1.GetOptions{})
if err != nil {
return err
}

if _, ok := registrationControllerSA.Annotations["eks.amazonaws.com/role-arn"]; ok {
return fmt.Errorf("Annotation applicable to awsirsa registration only")
}
return nil
}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil())
gomega.Eventually(func() error {
Expand Down Expand Up @@ -1294,4 +1299,5 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", func() {
gomega.ContainElement("manager"))
})
})

})
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ import (
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/tools/clientcmd"

"open-cluster-management.io/ocm/pkg/common/helpers"
commonoptions "open-cluster-management.io/ocm/pkg/common/options"
"open-cluster-management.io/ocm/pkg/operator/operators/klusterlet/controllers/klusterletcontroller"
"open-cluster-management.io/ocm/pkg/registration/register"
"open-cluster-management.io/ocm/pkg/registration/spoke"
"open-cluster-management.io/ocm/test/integration/util"
Expand Down Expand Up @@ -113,8 +113,8 @@ var _ = ginkgo.Describe("Joining Process for aws flow", func() {
return fmt.Errorf("user exec plugun command is invalid")
}

hubClusterAccountId, hubClusterName := klusterletcontroller.GetAwsAccountIdAndClusterName(hubClusterArn)
awsRegion := klusterletcontroller.GetAwsRegion(hubClusterArn)
hubClusterAccountId, hubClusterName := helpers.GetAwsAccountIdAndClusterName(hubClusterArn)
awsRegion := helpers.GetAwsRegion(hubClusterArn)

if !contains(hubUser.Exec.Args, fmt.Sprintf("arn:aws:iam::%s:role/ocm-hub-%s", hubClusterAccountId, managedClusterRoleSuffix)) ||
!contains(hubUser.Exec.Args, hubClusterName) ||
Expand Down
Loading
Loading