Update cluster client secret fleet-local/local-kubeconfig

Update these fields:
apiServerURL: value of Rancher setting "internal-server-url".
apiServerCA: value of Rancher setting "internal-cacerts".

Fleet needs these values to be set after Rancher v2.7.5 to provision a local cluster

Signed-off-by: Kiefer Chang <[email protected]>

cmd/rancherd/main.go

@@ -1,16 +1,18 @@
 package main
 
 import (
+	cli "github.com/rancher/wrangler-cli"
+	"github.com/spf13/cobra"
+
 	"github.com/rancher/rancherd/cmd/rancherd/bootstrap"
 	"github.com/rancher/rancherd/cmd/rancherd/gettoken"
 	"github.com/rancher/rancherd/cmd/rancherd/gettpmhash"
 	"github.com/rancher/rancherd/cmd/rancherd/info"
 	"github.com/rancher/rancherd/cmd/rancherd/probe"
 	"github.com/rancher/rancherd/cmd/rancherd/resetadmin"
 	"github.com/rancher/rancherd/cmd/rancherd/retry"
+	"github.com/rancher/rancherd/cmd/rancherd/updateclientsecret"
 	"github.com/rancher/rancherd/cmd/rancherd/upgrade"
-	cli "github.com/rancher/wrangler-cli"
-	"github.com/spf13/cobra"
 )
 
 type Rancherd struct {
@@ -33,6 +35,7 @@ func main() {
 		upgrade.NewUpgrade(),
 		info.NewInfo(),
 		gettpmhash.NewGetTPMHash(),
+		updateclientsecret.NewUpdateClientSecret(),
 	)
 	cli.Main(root)
 }

cmd/rancherd/updateclientsecret/update.go

@@ -0,0 +1,22 @@
+package updateclientsecret
+
+import (
+	cli "github.com/rancher/wrangler-cli"
+	"github.com/spf13/cobra"
+
+	"github.com/rancher/rancherd/pkg/rancher"
+)
+
+func NewUpdateClientSecret() *cobra.Command {
+	return cli.Command(&UpdateClientSecret{}, cobra.Command{
+		Short: "Update cluster client secret to have API Server URL and CA Certs configured",
+	})
+}
+
+type UpdateClientSecret struct {
+	Kubeconfig string `usage:"Kubeconfig file" env:"KUBECONFIG"`
+}
+
+func (s *UpdateClientSecret) Run(cmd *cobra.Command, args []string) error {
+	return rancher.UpdateClientSecret(cmd.Context(), &rancher.Options{Kubeconfig: s.Kubeconfig})
+}

pkg/plan/bootstrap.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/rancher/system-agent/pkg/applyinator"
+
 	"github.com/rancher/rancherd/pkg/config"
 	"github.com/rancher/rancherd/pkg/discovery"
 	"github.com/rancher/rancherd/pkg/join"
@@ -14,7 +16,6 @@ import (
 	"github.com/rancher/rancherd/pkg/resources"
 	"github.com/rancher/rancherd/pkg/runtime"
 	"github.com/rancher/rancherd/pkg/versions"
-	"github.com/rancher/system-agent/pkg/applyinator"
 )
 
 type plan applyinator.Plan
@@ -106,6 +107,14 @@ func (p *plan) addInstructions(cfg *config.Config, dataDir string) error {
 		return err
 	}
 
+	if err := p.addInstruction(rancher.ToWaitClusterClientSecretInstruction(cfg.RancherInstallerImage, cfg.SystemDefaultRegistry, k8sVersion)); err != nil {
+		return err
+	}
+
+	if err := p.addInstruction(rancher.ToUpdateClientSecretInstruction(cfg.RancherInstallerImage, cfg.SystemDefaultRegistry, k8sVersion)); err != nil {
+		return err
+	}
+
 	if err := p.addInstruction(resources.ToInstruction(cfg.RancherInstallerImage, cfg.SystemDefaultRegistry, k8sVersion, dataDir)); err != nil {
 		return err
 	}

pkg/rancher/cluster.go

@@ -0,0 +1,92 @@
+package rancher
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/sirupsen/logrus"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+
+	"github.com/rancher/rancherd/pkg/kubectl"
+)
+
+const (
+	rancherSettingInternalServerURL = "internal-server-url"
+	rancherSettingInternalCACerts   = "internal-cacerts"
+	clusterClientSecret             = "local-kubeconfig"
+	clusterNamespace                = "fleet-local"
+)
+
+type Options struct {
+	Kubeconfig string
+}
+
+// Update cluster client secret (fleet-local/local-kubeconfig):
+// apiServerURL: value of Rancher setting "internal-server-url"
+// apiServerCA: value of Rancher setting "internal-cacerts"
+// Fleet needs these values to be set after Rancher v2.7.5 to provision a local cluster
+func UpdateClientSecret(ctx context.Context, opts *Options) error {
+	if opts == nil {
+		opts = &Options{}
+	}
+
+	kubeconfig, err := kubectl.GetKubeconfig(opts.Kubeconfig)
+	if err != nil {
+		return err
+	}
+
+	conf, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
+	if err != nil {
+		return err
+	}
+
+	client := dynamic.NewForConfigOrDie(conf)
+	settingClient := client.Resource(schema.GroupVersionResource{
+		Group:    "management.cattle.io",
+		Version:  "v3",
+		Resource: "settings",
+	})
+
+	internalServerURLSetting, err := settingClient.Get(ctx, rancherSettingInternalServerURL, v1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	internalServerURL := internalServerURLSetting.Object["value"].(string)
+	logrus.Infof("Rancher setting %s is %q", rancherSettingInternalServerURL, internalServerURL)
+
+	internalCACertSetting, err := settingClient.Get(ctx, rancherSettingInternalCACerts, v1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	internalCACerts := internalCACertSetting.Object["value"].(string)
+	logrus.Infof("Rancher setting %s is %q", rancherSettingInternalCACerts, internalCACerts)
+
+	if internalServerURL == "" || internalCACerts == "" {
+		return fmt.Errorf("both %s and %s settings must be configured", rancherSettingInternalCACerts, rancherSettingInternalCACerts)
+	}
+
+	k8s, err := kubernetes.NewForConfig(conf)
+	if err != nil {
+		return err
+	}
+
+	secret, err := k8s.CoreV1().Secrets(clusterNamespace).Get(ctx, clusterClientSecret, v1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	toUpdate := secret.DeepCopy()
+	toUpdate.Data["apiServerURL"] = []byte(internalServerURL)
+	toUpdate.Data["apiServerCA"] = []byte(internalCACerts)
+	_, err = k8s.CoreV1().Secrets(clusterNamespace).Update(ctx, toUpdate, v1.UpdateOptions{})
+
+	if err == nil {
+		fmt.Println("Cluster client secret is updated.")
+	}
+
+	return err
+}

pkg/rancher/wait.go

@@ -4,9 +4,10 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/rancher/system-agent/pkg/applyinator"
+
 	"github.com/rancher/rancherd/pkg/kubectl"
 	"github.com/rancher/rancherd/pkg/self"
-	"github.com/rancher/system-agent/pkg/applyinator"
 )
 
 func ToWaitRancherInstruction(imageOverride, systemDefaultRegistry, k8sVersion string) (*applyinator.Instruction, error) {
@@ -65,3 +66,32 @@ func ToWaitSUCPlanInstruction(imageOverride, systemDefaultRegistry, k8sVersion s
 		Command: cmd,
 	}, nil
 }
+
+func ToWaitClusterClientSecretInstruction(imageOverride, systemDefaultRegistry, k8sVersion string) (*applyinator.Instruction, error) {
+	cmd, err := self.Self()
+	if err != nil {
+		return nil, fmt.Errorf("resolving location of %s: %w", os.Args[0], err)
+	}
+	return &applyinator.Instruction{
+		Name:       "wait-cluster-client-secret-resolved",
+		SaveOutput: true,
+		Args: []string{"retry", kubectl.Command(k8sVersion), "-n", clusterNamespace, "get",
+			"secret", clusterClientSecret},
+		Env:     kubectl.Env(k8sVersion),
+		Command: cmd,
+	}, nil
+}
+
+func ToUpdateClientSecretInstruction(imageOverride, systemDefaultRegistry, k8sVersion string) (*applyinator.Instruction, error) {
+	cmd, err := self.Self()
+	if err != nil {
+		return nil, fmt.Errorf("resolving location of %s: %w", os.Args[0], err)
+	}
+	return &applyinator.Instruction{
+		Name:       "update-client-secret",
+		SaveOutput: true,
+		Args:       []string{"update-client-secret"},
+		Env:        kubectl.Env(k8sVersion),
+		Command:    cmd,
+	}, nil
+}