Option to remove a "not ready" taint at startup

.golangci.yaml

@@ -78,3 +78,7 @@ linters-settings:
         alias: digest
       - pkg: github.com/opencontainers/image-spec/specs-go/v1
         alias: ocispec
+      - pkg: k8s.io/api/core/v1
+        alias: corev1
+      - pkg: k8s.io/apimachinery/pkg/apis/meta/v1
+        alias: metav1

CHANGELOG.md

@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- [#xxx](https://github.com/spegel-org/spegel/pull/xxx) Remove a taint when spegel is ready.
+
 ### Changed
 
 ### Deprecated

charts/spegel/README.md

@@ -55,6 +55,8 @@ Read the [getting started](https://spegel.dev/docs/getting-started/) guide to de
 | spegel.logLevel | string | `"INFO"` | Minimum log level to output. Value should be DEBUG, INFO, WARN, or ERROR. |
 | spegel.mirrorResolveRetries | int | `3` | Max ammount of mirrors to attempt. |
 | spegel.mirrorResolveTimeout | string | `"20ms"` | Max duration spent finding a mirror. |
+| spegel.notReadyTaint.key | string | `"spegel/not-ready"` | Key of the taint to remove when spegel is ready. |
+| spegel.notReadyTaint.removeWhenReady | bool | `false` | If true remove the taint of the node when the registry is ready. |
 | spegel.registries | list | `["https://cgr.dev","https://docker.io","https://ghcr.io","https://quay.io","https://mcr.microsoft.com","https://public.ecr.aws","https://gcr.io","https://registry.k8s.io","https://k8s.gcr.io","https://lscr.io"]` | Registries for which mirror configuration will be created. |
 | spegel.resolveLatestTag | bool | `true` | When true latest tags will be resolved to digests. |
 | spegel.resolveTags | bool | `true` | When true Spegel will resolve tags to digests. |

charts/spegel/templates/daemonset.yaml

@@ -98,7 +98,13 @@ spec:
           {{- with .Values.spegel.containerdContentPath }}
           - --containerd-content-path={{ . }}
           {{- end }}
+          - --not-ready-taint-remove={{ .Values.spegel.notReadyTaint.removeWhenReady }}
+          - --not-ready-taint-key={{ .Values.spegel.notReadyTaint.key }}
         env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
         - name: NODE_IP
         {{- include "networking.nodeIp" . | nindent 10 }}
         ports:

charts/spegel/templates/rbac.yaml

@@ -9,3 +9,27 @@ metadata:
   annotations:
     {{- toYaml . | nindent 4 }}
   {{- end }}
+{{- if .Values.spegel.notReadyTaint.removeWhenReady }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "spegel.serviceAccountName" . }}
+rules:
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs: ["get", "update"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "spegel.serviceAccountName" . }}
+subjects:
+- kind: ServiceAccount
+  name: {{ include "spegel.serviceAccountName" . }}
+  namespace: {{ include "spegel.namespace" . }}
+roleRef:
+  kind: ClusterRole
+  name: {{ include "spegel.serviceAccountName" . }}
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}

charts/spegel/values.yaml

@@ -171,6 +171,12 @@ spegel:
   # -- When true existing mirror configuration will be appended to instead of replaced.
   appendMirrors: false
 
+  notReadyTaint:
+    # -- If true remove the taint of the node when the registry is ready.
+    removeWhenReady: false
+    # -- Key of the taint to remove when spegel is ready.
+    key: spegel/not-ready
+
 verticalPodAutoscaler:
   # -- If true creates a Vertical Pod Autoscaler.
   enabled: false

go.mod

@@ -25,6 +25,8 @@ require (
 	github.com/stretchr/testify v1.10.0
 	go.etcd.io/bbolt v1.3.11
 	golang.org/x/sync v0.10.0
+	k8s.io/api v0.32.1
+	k8s.io/apimachinery v0.32.1
 	k8s.io/client-go v0.32.1
 	k8s.io/cri-api v0.32.1
 	k8s.io/klog/v2 v2.130.1
@@ -214,8 +216,7 @@ require (
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	helm.sh/helm/v3 v3.15.2 // indirect
-	k8s.io/api v0.32.1 // indirect
-	k8s.io/apimachinery v0.32.1 // indirect
+
 	k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect
 	k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect
 	lukechampine.com/blake3 v1.3.0 // indirect

internal/kubernetes/kubernetes.go

@@ -1,9 +1,15 @@
 package kubernetes
 
 import (
+	"context"
+	"slices"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/util/retry"
 )
 
 func GetClientset(kubeconfigPath string) (*kubernetes.Clientset, error) {
@@ -28,3 +34,15 @@ func GetClientset(kubeconfigPath string) (*kubernetes.Clientset, error) {
 	}
 	return clientset, nil
 }
+
+func RemoveNodeTaint(ctx context.Context, cs *kubernetes.Clientset, nodeName, startupTaintKey string) error {
+	return retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		node, err := cs.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+		node.Spec.Taints = slices.DeleteFunc(node.Spec.Taints, func(t corev1.Taint) bool { return t.Key == startupTaintKey })
+		_, err = cs.CoreV1().Nodes().Update(ctx, node, metav1.UpdateOptions{FieldManager: "spegel"})
+		return err
+	})
+}

main.go

@@ -47,8 +47,15 @@ type BootstrapConfig struct {
 	HTTPBootstrapPeer       string `arg:"--http-bootstrap-peer,env:HTTP_BOOTSTRAP_PEER" help:"Peer to HTTP bootstrap with."`
 }
 
+type NotReadyTaintConfig struct {
+	NodeName            string `arg:"--node-name,env:NODE_NAME" help:"Name of the node."`
+	NotReadyTaintKey    string `arg:"--not-ready-taint-key,env:NOT_READY_TAINT_KEY" default:"spegel/not-ready" help:"Key of the taint to remove when spegel is ready."`
+	NotReadyTaintRemove bool   `arg:"--not-ready-taint-remove,env:NOT_READY_TAINT_REMOVE" help:"Remove the not ready taint when spegel is ready."`
+}
+
 type RegistryCmd struct {
 	BootstrapConfig
+	NotReadyTaintConfig
 	ContainerdRegistryConfigPath string        `arg:"--containerd-registry-config-path,env:CONTAINERD_REGISTRY_CONFIG_PATH" default:"/etc/containerd/certs.d" help:"Directory where mirror configuration is written."`
 	MetricsAddr                  string        `arg:"--metrics-addr,required,env:METRICS_ADDR" help:"address to serve metrics."`
 	LocalAddr                    string        `arg:"--local-addr,required,env:LOCAL_ADDR" help:"Address that the local Spegel instance will be reached at."`
@@ -116,6 +123,15 @@ func registryCommand(ctx context.Context, args *RegistryCmd) (err error) {
 	log := logr.FromContextOrDiscard(ctx)
 	g, ctx := errgroup.WithContext(ctx)
 
+	if args.NotReadyTaintRemove {
+		if args.NotReadyTaintKey == "" {
+			return errors.New("not ready taint key must be set when remove not ready taint is enabled")
+		}
+		if args.NodeName == "" {
+			return errors.New("node name must be set when remove not ready taint is enabled")
+		}
+	}
+
 	// OCI Client
 	ociClient, err := oci.NewContainerd(args.ContainerdSock, args.ContainerdNamespace, args.ContainerdRegistryConfigPath, args.Registries, oci.WithContentPath(args.ContainerdContentPath))
 	if err != nil {
@@ -209,6 +225,17 @@ func registryCommand(ctx context.Context, args *RegistryCmd) (err error) {
 		return regSrv.Shutdown(shutdownCtx)
 	})
 
+	if args.NotReadyTaintRemove {
+		log.Info("removing not ready taint", "key", args.NotReadyTaintKey)
+		cs, err := kubernetes.GetClientset(args.KubeconfigPath)
+		if err != nil {
+			return err
+		}
+		if err := kubernetes.RemoveNodeTaint(ctx, cs, args.NodeName, args.NotReadyTaintKey); err != nil {
+			return err
+		}
+	}
+
 	log.Info("running Spegel", "registry", args.RegistryAddr, "router", args.RouterAddr)
 	err = g.Wait()
 	if err != nil {

test/e2e/e2e_test.go

@@ -71,6 +71,10 @@ func TestE2E(t *testing.T) {
   capabilities = [push]`
 	command(ctx, t, fmt.Sprintf("docker exec %s-worker2 bash -c \"mkdir -p /etc/containerd/certs.d/docker.io; echo -e '%s' > /etc/containerd/certs.d/docker.io/hosts.toml\"", kindName, hostsToml))
 
+	// Taint nodes
+	t.Log("Tainting nodes")
+	command(ctx, t, fmt.Sprintf("kubectl --kubeconfig %s taint nodes --all spegel/not-ready=true:NoSchedule", kcPath))
+
 	// Deploy Spegel.
 	t.Log("Deploying Spegel")
 	command(ctx, t, fmt.Sprintf("kind load docker-image --name %s %s", kindName, imageRef))
@@ -86,14 +90,19 @@ func TestE2E(t *testing.T) {
 	for _, node := range nodes {
 		command(ctx, t, fmt.Sprintf("docker exec %s-%s ctr -n k8s.io image tag %s ghcr.io/spegel-org/spegel@%s", kindName, node, imageRef, imageDigest))
 	}
-	command(ctx, t, fmt.Sprintf("helm --kubeconfig %s upgrade --timeout 60s --create-namespace --wait --install --namespace=\"spegel\" spegel ../../charts/spegel --set \"image.pullPolicy=Never\" --set \"image.digest=%s\" --set \"nodeSelector.spegel=schedule\"", kcPath, imageDigest))
+	command(ctx, t, fmt.Sprintf("helm --kubeconfig %s upgrade --timeout 60s --create-namespace --wait --install --namespace=\"spegel\" spegel ../../charts/spegel --set \"image.pullPolicy=Never\" --set \"image.digest=%s\" --set \"nodeSelector.spegel=schedule\" --set \"spegel.notReadyTaint.removeWhenReady=true\"", kcPath, imageDigest))
 	podOutput := command(ctx, t, fmt.Sprintf("kubectl --kubeconfig %s --namespace spegel get pods --no-headers", kcPath))
 	require.Len(t, strings.Split(podOutput, "\n"), 5)
 
 	// Verify that configuration has been backed up.
 	backupHostToml := command(ctx, t, fmt.Sprintf("docker exec %s-worker2 cat /etc/containerd/certs.d/_backup/docker.io/hosts.toml", kindName))
 	require.Equal(t, hostsToml, backupHostToml)
 
+	// Verify that spegel has remove the not ready taint from nodes.
+	taintsTpl := `{{range .items}}{{$hasTaint := false}}{{range .spec.taints}}{{if eq .key "spegel/not-ready"}}{{$hasTaint = true}}{{end}}{{end}}{{if $hasTaint}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}`
+	nodeOutput := command(ctx, t, fmt.Sprintf("kubectl --kubeconfig %s get nodes -o go-template='%s'", kcPath, taintsTpl))
+	require.Empty(t, nodeOutput)
+
 	// Run conformance tests.
 	t.Log("Running conformance tests")
 	command(ctx, t, fmt.Sprintf("kubectl --kubeconfig %s create namespace conformance --dry-run=client -o yaml | kubectl --kubeconfig %s apply -f -", kcPath, kcPath))