From df10c7575efb994a2f185784b56c76d28e833f74 Mon Sep 17 00:00:00 2001 From: Patryk Strusiewicz-Surmacki Date: Mon, 22 Jul 2024 19:14:22 +0200 Subject: [PATCH] Standalone Egress NAT Signed-off-by: Patryk Strusiewicz-Surmacki Update v2/cmd/coil-egress-controller/sub/root.go Update v2/runners/coild_server.go Update v2/pkg/nodenet/pod.go Update v2/pkg/cnirpc/cni.proto Co-authored-by: Tomoya Terashima <49265363+terassyi@users.noreply.github.com> --- docs/cmd-coil-egress-controller.md | 27 +++ ...troller.md => cmd-coil-ipam-controller.md} | 22 +- docs/cni-grpc.md | 19 ++ docs/design.md | 13 +- docs/setup.md | 53 ++++- v2/Makefile | 37 +++- v2/cmd/coil-controller/main.go | 7 - v2/cmd/coil-egress-controller/main.go | 7 + v2/cmd/coil-egress-controller/sub/root.go | 56 +++++ v2/cmd/coil-egress-controller/sub/run.go | 120 +++++++++++ v2/cmd/coil-installer/sub/install.go | 9 +- v2/cmd/coil-ipam-controller/main.go | 7 + .../sub/root.go | 2 - .../sub/run.go | 28 +-- v2/cmd/coil/main.go | 29 ++- v2/cmd/coil/rpc.go | 40 +++- v2/cmd/coild/sub/root.go | 4 + v2/cmd/coild/sub/run.go | 59 +++--- v2/cmd/gencert/main.go | 83 +++++++- v2/config/cke/webhook-secret.yaml | 12 +- v2/config/crd/kustomization.yaml | 6 + v2/config/crd/patches/remove_status.yaml | 2 + .../crd/patches/webhook_in_addressblocks.yaml | 2 +- .../crd/patches/webhook_in_addresspools.yaml | 2 +- .../crd/patches/webhook_in_blockrequests.yaml | 2 +- .../crd/patches/webhook_in_egresses.yaml | 2 +- v2/config/default/.gitignore | 4 + v2/config/default/kustomization.yaml | 16 +- .../default/webhook_manifests_patch.yaml.tmpl | 23 ++- v2/config/pod/coil-egress-controller.yaml | 90 ++++++++ ...troller.yaml => coil-ipam-controller.yaml} | 18 +- v2/config/pod/coild.yaml | 2 + v2/config/pod/kustomization.yaml | 5 +- ....yaml => coil-egress-controller_role.yaml} | 47 +---- v2/config/rbac/coil-ipam-controller_role.yaml | 59 ++++++ v2/config/rbac/kustomization.yaml | 11 +- .../rbac/leader_election_role_binding.yaml | 5 +- v2/config/rbac/role_binding.yaml | 19 +- v2/config/rbac/serviceaccount.yaml | 8 +- v2/config/webhook/manifests.v1.yaml | 40 ++-- v2/config/webhook/service.yaml | 17 +- v2/controllers/egress_watcher.go | 2 +- v2/controllers/egress_watcher_test.go | 8 +- v2/e2e/Makefile | 3 +- v2/e2e/coil-controller_patch.yaml | 4 +- v2/e2e/controller_test.go | 26 ++- v2/kustomization.yaml | 2 +- v2/netconf.json | 6 +- v2/pkg/cnirpc/cni.pb.go | 138 ++++++++----- v2/pkg/cnirpc/cni.proto | 2 + v2/pkg/constants/constants.go | 7 + v2/pkg/nodenet/pod.go | 105 +++++++--- v2/pkg/nodenet/pod_test.go | 18 +- v2/runners/coild_server.go | 194 ++++++++++++++---- v2/runners/coild_server_test.go | 13 +- 55 files changed, 1177 insertions(+), 365 deletions(-) create mode 100644 docs/cmd-coil-egress-controller.md rename docs/{cmd-coil-controller.md => cmd-coil-ipam-controller.md} (60%) delete mode 100644 v2/cmd/coil-controller/main.go create mode 100644 v2/cmd/coil-egress-controller/main.go create mode 100644 v2/cmd/coil-egress-controller/sub/root.go create mode 100644 v2/cmd/coil-egress-controller/sub/run.go create mode 100644 v2/cmd/coil-ipam-controller/main.go rename v2/cmd/{coil-controller => coil-ipam-controller}/sub/root.go (93%) rename v2/cmd/{coil-controller => coil-ipam-controller}/sub/run.go (80%) create mode 100644 v2/config/pod/coil-egress-controller.yaml rename v2/config/pod/{coil-controller.yaml => coil-ipam-controller.yaml} (82%) rename v2/config/rbac/{coil-controller_role.yaml => coil-egress-controller_role.yaml} (61%) create mode 100644 v2/config/rbac/coil-ipam-controller_role.yaml diff --git a/docs/cmd-coil-egress-controller.md b/docs/cmd-coil-egress-controller.md new file mode 100644 index 00000000..c76c5ae8 --- /dev/null +++ b/docs/cmd-coil-egress-controller.md @@ -0,0 +1,27 @@ +coil-egress-controller +=============== + +`coil-egress-controller` is a Kubernetes controller for Coil custom resources related to on-demand NAT egress. +It is intended to be run as a Pod in `kube-system` namespace. + + +## Egress + +`coil-egress-controller` creates **Deployment** and **Service** for each Egress. + +It also creates `coil-egress` **ServiceAccount** in the namespace of Egress, +and binds it to the **ClusterRoles** for `coil-egress`. + +## Command-line flags + +``` +Flags: + --cert-dir string directory to locate TLS certs for webhook (default "/certs") + --egress-port int32 UDP port number used by coil-egress (default 5555) + --gc-interval duration garbage collection interval (default 1h0m0s) + --health-addr string bind address of health/readiness probes (default ":9387") + -h, --help help for coil-egress-controller + --metrics-addr string bind address of metrics endpoint (default ":9386") + -v, --version version for coil-egress-controller + --webhook-addr string bind address of admission webhook (default ":9443") +``` diff --git a/docs/cmd-coil-controller.md b/docs/cmd-coil-ipam-controller.md similarity index 60% rename from docs/cmd-coil-controller.md rename to docs/cmd-coil-ipam-controller.md index 49827174..5a939707 100644 --- a/docs/cmd-coil-controller.md +++ b/docs/cmd-coil-ipam-controller.md @@ -1,41 +1,33 @@ -coil-controller +coil-ipam-controller =============== -`coil-controller` is a Kubernetes controller for Coil custom resources. +`coil-ipam-controller` is a Kubernetes controller for Coil IPAM related custom resources. It is intended to be run as a Pod in `kube-system` namespace. ## AddressPool and AddressBlock -`coil-controller` has an in-memory database of address pools and +`coil-ipam-controller` has an in-memory database of address pools and address blocks to allocate address blocks quickly. ## BlockRequest -`coil-controller` watches newly created block requests and carve out +`coil-ipam-controller` watches newly created block requests and carve out address blocks from the requested pool. -## Egress - -`coil-controller` creates **Deployment** and **Service** for each Egress. - -It also creates `coil-egress` **ServiceAccount** in the namespace of Egress, -and binds it to the **ClusterRoles** for `coil-egress`. - ## Garbage collection -`coil-controller` periodically checks orphaned address blocks and deletes them. +`coil-ipam-controller` periodically checks orphaned address blocks and deletes them. ## Command-line flags ``` Flags: --cert-dir string directory to locate TLS certs for webhook (default "/certs") - --egress-port int32 UDP port number used by coil-egress (default 5555) --gc-interval duration garbage collection interval (default 1h0m0s) --health-addr string bind address of health/readiness probes (default ":9387") - -h, --help help for coil-controller + -h, --help help for coil-ipam-controller --metrics-addr string bind address of metrics endpoint (default ":9386") - -v, --version version for coil-controller + -v, --version version for coil-ipam-controller --webhook-addr string bind address of admission webhook (default ":9443") ``` diff --git a/docs/cni-grpc.md b/docs/cni-grpc.md index 954c3bac..9c556b00 100644 --- a/docs/cni-grpc.md +++ b/docs/cni-grpc.md @@ -7,6 +7,7 @@ - [AddResponse](#pkg-cnirpc-AddResponse) - [CNIArgs](#pkg-cnirpc-CNIArgs) - [CNIArgs.ArgsEntry](#pkg-cnirpc-CNIArgs-ArgsEntry) + - [CNIArgs.InterfacesEntry](#pkg-cnirpc-CNIArgs-InterfacesEntry) - [CNIError](#pkg-cnirpc-CNIError) - [ErrorCode](#pkg-cnirpc-ErrorCode) @@ -57,6 +58,8 @@ https://pkg.go.dev/github.com/containernetworking/cni@v0.8.0/pkg/skel?tab=doc#Cm | args | [CNIArgs.ArgsEntry](#pkg-cnirpc-CNIArgs-ArgsEntry) | repeated | Key-Value pairs parsed from CNI_ARGS | | path | [string](#string) | | | | stdin_data | [bytes](#bytes) | | | +| ips | [string](#string) | repeated | | +| interfaces | [CNIArgs.InterfacesEntry](#pkg-cnirpc-CNIArgs-InterfacesEntry) | repeated | | @@ -79,6 +82,22 @@ https://pkg.go.dev/github.com/containernetworking/cni@v0.8.0/pkg/skel?tab=doc#Cm + + +### CNIArgs.InterfacesEntry + + + +| Field | Type | Label | Description | +| ----- | ---- | ----- | ----------- | +| key | [string](#string) | | | +| value | [bool](#bool) | | | + + + + + + ### CNIError diff --git a/docs/design.md b/docs/design.md index b86c603c..ed73167d 100644 --- a/docs/design.md +++ b/docs/design.md @@ -73,7 +73,8 @@ Now that we have learned how to do these things, and want to add rich features s Coil v2 will consist of the following programs: -- `coil-controller`: Kubernetes controller managing custom resources. +- `coil-ipam-controller`: Kubernetes controller managing IPAM related custom resources. +- `coil-egress-controller`: Kubernetes controller managing on-demand NAT egress related custom resources. - `coild`: Daemon program running on nodes. - `coil`: CNI interface that delegates requests from `kubelet` to `coild`. - `coil-egress`: Administration program running in Egress pods. @@ -102,11 +103,11 @@ To make things simple, the default pool is the pool whose name is `default`. To reduce the number of advertised routes, addresses in an address pool are divided into fixed-size blocks. These blocks are called _address blocks_, and assigned to nodes. Since all IP addresses in an address block are routed to the same node, only one route per address block need to be advertised. -For example, if an address pool defines that the size of an address block is 25, `coil-controller` will carve an address block for IPv4 with `/27` subnet mask out of the pool, and assigns it to a node. +For example, if an address pool defines that the size of an address block is 25, `coil-ipam-controller` will carve an address block for IPv4 with `/27` subnet mask out of the pool, and assigns it to a node. In general, avoiding immediate reuse of IP addresses is better not to confuse other software or components. -To avoid such immediate reuse, `coil-controller` remembers the last used address, and it assigns the address from the next address. +To avoid such immediate reuse, `coil-ipam-controller` remembers the last used address, and it assigns the address from the next address. The same problem may occur when we use address blocks of the size `/32`. In this case, there is a high chance of reusing the same address immediately. @@ -333,7 +334,7 @@ Therefore, when the deletion of the owning `AddressPool` is directed, all `Addre That said, an `AddressBlock` should not be deleted until there are no more Pods with an address in the block. For this purpose, Coil adds a finalizer to each `AddressBlock`. **`coild` checks the usage of addresses in the block**, and once there are no more Pods using the addresses, it removes the finalizer to delete the `AddressBlock`. -`AddressBlock` should also be deleted when `Node` that acquired the block is deleted. Since `coild` running as a DaemonSet pod cannot do this, **`coil-controller` watches Node deletions and removes `AddressBlocks`**. `coil-controller` periodically checks dangling `AddressBlocks` and removes them. +`AddressBlock` should also be deleted when `Node` that acquired the block is deleted. Since `coild` running as a DaemonSet pod cannot do this, **`coil-ipam-controller` watches Node deletions and removes `AddressBlocks`**. `coil-ipam-controller` periodically checks dangling `AddressBlocks` and removes them. `coild` also deletes `AddressBlock` when it frees the last IP address used in the block. At startup, `coild` also checks each `AddressBlock` for the Node, and if no Pod is using the addresses in the block, it deletes the `AddressBlock`. @@ -342,7 +343,7 @@ Note that Coil does not include `Node` in the list of owner references of an `Ad ### AddressPool Similar to an `AddressBlock` and its addresses, an `AddressPool` should not be deleted until there are no more `AddressBlock`s derived from the pool. -For this purpose, Coil adds a finalizer to each `AddressPool`. `coil-controller` checks the usage of blocks in the pool. +For this purpose, Coil adds a finalizer to each `AddressPool`. `coil-ipam-controller` checks the usage of blocks in the pool. Note that `blockOwnerDeletion: true` in `AddressBlock`'s `ownerReferences` does not always block the deletion of the owning `AddressPool`. This directive has effect only when foreground cascading deletion is adopted. @@ -397,7 +398,7 @@ skinparam rectangle { together { actor User package Deployment { - [coil-controller] as controller + [coil-ipam-controller] as controller } database kube-apiserver as apiserver #lightblue { [AddressPool] as pool1 diff --git a/docs/setup.md b/docs/setup.md index b681d412..670105d6 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -17,6 +17,8 @@ You can tweak optional parameters by editing [`kustomization.yaml`](../v2/kustom - [IPv4/v6 dual stack pool](#ipv4v6-dual-stack-pool) - [(Option) Configure BIRD](#option-configure-bird) - [Note on CRI runtime compatibility](#note-on-cri-runtime-compatibility) +- [Standalone egress](#standalone-egress) + - [Configuration](#configuration) ## Install `kustomize` @@ -37,7 +39,8 @@ This should generate the following PEM files: ```console $ ls config/default/*.pem -config/default/cert.pem config/default/key.pem +config/default/cert.pem config/default/egress-key.pem config/default/ipam-key.pem +config/default/egress-cert.pem config/default/ipam-cert.pem config/default/key.pem ``` ## Edit `kustomization.yaml` @@ -57,7 +60,7 @@ $ vi kustomization.yaml (actually, the example is a network configuration list). You may edit the file to, say, add Cilium for network policies or to tune MTU. -Note that `coil` must be the first in the plugin list. +Note that `coil` must be the first in the plugin list if IPAM is enabled. ```console vi netconf.json @@ -76,7 +79,9 @@ The following example adds `tuning` and `bandwidth` plugins. "plugins": [ { "type": "coil", - "socket": "/run/coild.sock" + "socket": "/run/coild.sock", + "ipam": true, + "egress": true }, { "type": "tuning", @@ -231,3 +236,45 @@ host directory. [netconf]: https://github.com/containernetworking/cni/blob/spec-v0.4.0/SPEC.md#network-configuration [BIRD]: https://bird.network.cz/ + +## Standalone egress + +Coil can be run as standalone egress NAT controller, using CNI chaining with another CNI providing base connectivity. This chapter will guide you on how to achieve this. + +### Configuration + +To deploy Coil with only egress feature enabled the following changes are required in the configuration files: + +1. Comment all IPAM related pieces in the following `kustomization.yaml` files: + - `v2/config/crd/kustomization.yaml` + - `v2/config/default/kustomization.yaml` + - `v2/config/pod/kustomization.yaml` + - `v2/config/rbac/kustomization.yaml` + +2. Comment unnecessary resources in `config/crd/patches/remove_status.yaml`. +3. Add following arguments to the `coild` contianer executable in `config/pod/coild.yaml` + ```yaml + containers: + - name: coild + image: coil:dev + command: ["coild"] + args: + - --zap-stacktrace-level=panic + - --enable-ipam=false + - --enable-egress=true + - --pod-table-id=0 # 255 if IPv6 is being used + - --protocol-id=2 + ``` +4. Set coil capabilites in `v2/netconf.json` to: + ```json + { + "type": "coil", + "socket": "/run/coild.sock", + "capabilities": { + "ipam": false, + "egress": true + } + }, + ``` +5. Add configuration of your chosen CNI to `v2/netconf.json` before `coil` related configuration. +6. Deploy `coil` to existing cluster as described in [Compile and apply the manifest](#compile-and-apply-the-manifest). diff --git a/v2/Makefile b/v2/Makefile index 0a968ddd..23ef6eb2 100644 --- a/v2/Makefile +++ b/v2/Makefile @@ -15,7 +15,8 @@ CONTROLLER_GEN := $(shell pwd)/bin/controller-gen SETUP_ENVTEST := $(shell pwd)/bin/setup-envtest YQ := $(shell pwd)/bin/yq CRD_OPTIONS = "crd:crdVersions=v1" -ROLES = config/rbac/coil-controller_role.yaml \ +ROLES = config/rbac/coil-ipam-controller_role.yaml \ + config/rbac/coil-egress-controller_role.yaml \ config/rbac/coild_role.yaml \ config/rbac/coil-router_role.yaml \ config/rbac/coil-egress_role.yaml @@ -89,25 +90,32 @@ $(YQ): rm -f yq.tar.gz chmod +x $@ -COIL_CONTROLLER_ROLE_DEPENDS = controllers/addresspool_controller.go \ +COIL_IPAM_CONTROLLER_ROLE_DEPENDS = controllers/addresspool_controller.go \ controllers/blockrequest_controller.go \ - controllers/egress_controller.go \ - controllers/clusterrolebinding_controller.go \ pkg/ipam/pool.go \ runners/garbage_collector.go -config/rbac/coil-controller_role.yaml: $(COIL_CONTROLLER_ROLE_DEPENDS) +config/rbac/coil-ipam-controller_role.yaml: $(COIL_IPAM_CONTROLLER_ROLE_DEPENDS) -rm -rf work mkdir work sed '0,/^package/s/.*/package work/' controllers/addresspool_controller.go > work/addresspool_controller.go sed '0,/^package/s/.*/package work/' controllers/blockrequest_controller.go > work/blockrequest_controller.go - sed '0,/^package/s/.*/package work/' controllers/egress_controller.go > work/egress_controller.go - sed '0,/^package/s/.*/package work/' controllers/clusterrolebinding_controller.go > work/clusterrolebinding_controller.go sed '0,/^package/s/.*/package work/' pkg/ipam/pool.go > work/pool.go sed '0,/^package/s/.*/package work/' runners/garbage_collector.go > work/garbage_collector.go - $(CONTROLLER_GEN) rbac:roleName=coil-controller paths=./work output:stdout > $@ + $(CONTROLLER_GEN) rbac:roleName=coil-ipam-controller paths=./work output:stdout > $@ rm -rf work +COIL_EGRESS_CONTROLLER_ROLE_DEPENDS = controllers/egress_controller.go \ + controllers/clusterrolebinding_controller.go + +config/rbac/coil-egress-controller_role.yaml: $(COIL_EGRESS_CONTROLLER_ROLE_DEPENDS) + -rm -rf work + mkdir work + sed '0,/^package/s/.*/package work/' controllers/egress_controller.go > work/egress_controller.go + sed '0,/^package/s/.*/package work/' controllers/clusterrolebinding_controller.go > work/clusterrolebinding_controller.go + $(CONTROLLER_GEN) rbac:roleName=coil-egress-controller paths=./work output:stdout > $@ + # rm -rf work + COILD_DEPENDS = controllers/blockrequest_watcher.go \ pkg/ipam/node.go \ runners/coild_server.go @@ -142,10 +150,16 @@ config/rbac/coil-egress_role.yaml: controllers/pod_watcher.go # TLS certificates for webhook .PHONY: certs -certs: config/default/cert.pem config/default/key.pem config/default/webhook_manifests_patch.yaml +certs: config/default/cert.pem config/default/key.pem config/default/ipam-cert.pem config/default/ipam-key.pem config/default/egress-cert.pem config/default/egress-key.pem config/default/webhook_manifests_patch.yaml config/default/cert.pem config/default/key.pem: - go run ./cmd/gencert -outdir=$(PWD)/config/default + go run ./cmd/gencert -outdir=$(PWD)/config/default -cn=coilv2-webhook-service -host=coilv2-webhook-service.kube-system.svc -certname=cert.pem -keyname=key.pem + +config/default/ipam-cert.pem config/default/ipam-key.pem: + go run ./cmd/gencert -outdir=$(PWD)/config/default -cn=coilv2-ipam-webhook-service -host=coilv2-ipam-webhook-service.kube-system.svc -certname=ipam-cert.pem -keyname=ipam-key.pem -ca=cert.pem -cakey=key.pem + +config/default/egress-cert.pem config/default/egress-key.pem: + go run ./cmd/gencert -outdir=$(PWD)/config/default -cn=coilv2-egress-webhook-service -host=coilv2-egress-webhook-service.kube-system.svc -certname=egress-cert.pem -keyname=egress-key.pem -ca=cert.pem -cakey=key.pem config/default/webhook_manifests_patch.yaml: config/default/cert.pem config/default/webhook_manifests_patch.yaml.tmpl sed "s/%CACERT%/$$(base64 -w0 < $<)/g" $@.tmpl > $@ @@ -171,7 +185,8 @@ pkg/cnirpc/cni_grpc.pb.go: pkg/cnirpc/cni.proto .PHONY: build build: GOARCH=$(GOARCH) CGO_ENABLED=0 go build -o work/coil -ldflags="-s -w" cmd/coil/*.go - GOARCH=$(GOARCH) CGO_ENABLED=0 go build -o work/coil-controller -ldflags="-s -w" cmd/coil-controller/*.go + GOARCH=$(GOARCH) CGO_ENABLED=0 go build -o work/coil-ipam-controller -ldflags="-s -w" cmd/coil-ipam-controller/*.go + GOARCH=$(GOARCH) CGO_ENABLED=0 go build -o work/coil-egress-controller -ldflags="-s -w" cmd/coil-egress-controller/*.go GOARCH=$(GOARCH) CGO_ENABLED=0 go build -o work/coil-egress -ldflags="-s -w" cmd/coil-egress/*.go GOARCH=$(GOARCH) CGO_ENABLED=0 go build -o work/coil-installer -ldflags="-s -w" cmd/coil-installer/*.go GOARCH=$(GOARCH) CGO_ENABLED=0 go build -o work/coil-router -ldflags="-s -w" cmd/coil-router/*.go diff --git a/v2/cmd/coil-controller/main.go b/v2/cmd/coil-controller/main.go deleted file mode 100644 index 8e94d24e..00000000 --- a/v2/cmd/coil-controller/main.go +++ /dev/null @@ -1,7 +0,0 @@ -package main - -import "github.com/cybozu-go/coil/v2/cmd/coil-controller/sub" - -func main() { - sub.Execute() -} diff --git a/v2/cmd/coil-egress-controller/main.go b/v2/cmd/coil-egress-controller/main.go new file mode 100644 index 00000000..8c3d52c0 --- /dev/null +++ b/v2/cmd/coil-egress-controller/main.go @@ -0,0 +1,7 @@ +package main + +import "github.com/cybozu-go/coil/v2/cmd/coil-egress-controller/sub" + +func main() { + sub.Execute() +} diff --git a/v2/cmd/coil-egress-controller/sub/root.go b/v2/cmd/coil-egress-controller/sub/root.go new file mode 100644 index 00000000..92176c71 --- /dev/null +++ b/v2/cmd/coil-egress-controller/sub/root.go @@ -0,0 +1,56 @@ +package sub + +import ( + "flag" + "fmt" + "os" + + v2 "github.com/cybozu-go/coil/v2" + "github.com/spf13/cobra" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +var config struct { + metricsAddr string + healthAddr string + webhookAddr string + certDir string + egressPort int32 + zapOpts zap.Options +} + +var rootCmd = &cobra.Command{ + Use: "coil-egress-controller", + Short: "controller for coil egress related custom resources", + Long: `coil-egress-controller is a Kubernetes controller for coil egress related custom resources.`, + Version: v2.Version(), + RunE: func(cmd *cobra.Command, _ []string) error { + cmd.SilenceUsage = true + return subMain() + }, +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the rootCmd. +func Execute() { + if err := rootCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func init() { + pf := rootCmd.PersistentFlags() + pf.StringVar(&config.metricsAddr, "metrics-addr", ":9396", "bind address of metrics endpoint") + pf.StringVar(&config.healthAddr, "health-addr", ":9397", "bind address of health/readiness probes") + pf.StringVar(&config.webhookAddr, "webhook-addr", ":9444", "bind address of admission webhook") + pf.StringVar(&config.certDir, "cert-dir", "/certs", "directory to locate TLS certs for webhook") + pf.Int32Var(&config.egressPort, "egress-port", 5555, "UDP port number used by coil-egress") + + goflags := flag.NewFlagSet("klog", flag.ExitOnError) + klog.InitFlags(goflags) + config.zapOpts.BindFlags(goflags) + + pf.AddGoFlagSet(goflags) +} diff --git a/v2/cmd/coil-egress-controller/sub/run.go b/v2/cmd/coil-egress-controller/sub/run.go new file mode 100644 index 00000000..ca58fef1 --- /dev/null +++ b/v2/cmd/coil-egress-controller/sub/run.go @@ -0,0 +1,120 @@ +package sub + +import ( + "fmt" + "net" + "os" + "strconv" + "time" + + v2 "github.com/cybozu-go/coil/v2" + coilv2 "github.com/cybozu-go/coil/v2/api/v2" + "github.com/cybozu-go/coil/v2/controllers" + "github.com/cybozu-go/coil/v2/pkg/constants" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +const ( + gracefulTimeout = 20 * time.Second +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(coilv2.AddToScheme(scheme)) + + // +kubebuilder:scaffold:scheme +} + +func subMain() error { + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&config.zapOpts))) + + host, portStr, err := net.SplitHostPort(config.webhookAddr) + if err != nil { + return fmt.Errorf("invalid webhook address: %w", err) + } + port, err := strconv.Atoi(portStr) + if err != nil { + return fmt.Errorf("invalid webhook address: %w", err) + } + + timeout := gracefulTimeout + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + LeaderElection: true, + LeaderElectionID: "coil-egress-leader", + LeaderElectionNamespace: "kube-system", // coil should run in kube-system + Metrics: metricsserver.Options{ + BindAddress: config.metricsAddr, + }, + GracefulShutdownTimeout: &timeout, + HealthProbeBindAddress: config.healthAddr, + WebhookServer: webhook.NewServer(webhook.Options{ + Host: host, + Port: port, + CertDir: config.certDir, + }), + }) + if err != nil { + return err + } + + if err := mgr.AddHealthzCheck("ping", healthz.Ping); err != nil { + return err + } + if err := mgr.AddReadyzCheck("ping", healthz.Ping); err != nil { + return err + } + + // register controllers + + ctx := ctrl.SetupSignalHandler() + + podNS := os.Getenv(constants.EnvPodNamespace) + podName := os.Getenv(constants.EnvPodName) + img, err := controllers.GetImage(mgr.GetAPIReader(), client.ObjectKey{Namespace: podNS, Name: podName}) + if err != nil { + return err + } + egressctrl := controllers.EgressReconciler{ + Client: mgr.GetClient(), + Scheme: scheme, + Image: img, + Port: config.egressPort, + } + if err := egressctrl.SetupWithManager(mgr); err != nil { + return err + } + + if err := controllers.SetupCRBReconciler(mgr); err != nil { + return err + } + + // register webhooks + + if err := (&coilv2.Egress{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + + // start manager + + setupLog.Info(fmt.Sprintf("starting manager (version: %s)", v2.Version())) + if err := mgr.Start(ctx); err != nil { + setupLog.Error(err, "problem running manager") + return err + } + + return nil +} diff --git a/v2/cmd/coil-installer/sub/install.go b/v2/cmd/coil-installer/sub/install.go index 7c240790..2a50d46a 100644 --- a/v2/cmd/coil-installer/sub/install.go +++ b/v2/cmd/coil-installer/sub/install.go @@ -4,6 +4,7 @@ import ( "io" "os" "path/filepath" + "strings" ) func installCniConf(cniConfName, cniEtcDir, cniNetConf, cniNetConfFile string) error { @@ -29,9 +30,11 @@ func installCniConf(cniConfName, cniEtcDir, cniNetConf, cniNetConfFile string) e if fi.IsDir() { continue } - err := os.Remove(filepath.Join(cniEtcDir, fi.Name())) - if err != nil { - return err + if strings.Contains(fi.Name(), "conflist") { + err := os.Remove(filepath.Join(cniEtcDir, fi.Name())) + if err != nil { + return err + } } } diff --git a/v2/cmd/coil-ipam-controller/main.go b/v2/cmd/coil-ipam-controller/main.go new file mode 100644 index 00000000..4a7bd5d3 --- /dev/null +++ b/v2/cmd/coil-ipam-controller/main.go @@ -0,0 +1,7 @@ +package main + +import "github.com/cybozu-go/coil/v2/cmd/coil-ipam-controller/sub" + +func main() { + sub.Execute() +} diff --git a/v2/cmd/coil-controller/sub/root.go b/v2/cmd/coil-ipam-controller/sub/root.go similarity index 93% rename from v2/cmd/coil-controller/sub/root.go rename to v2/cmd/coil-ipam-controller/sub/root.go index 8e009cc1..e10e22d7 100644 --- a/v2/cmd/coil-controller/sub/root.go +++ b/v2/cmd/coil-ipam-controller/sub/root.go @@ -18,7 +18,6 @@ var config struct { webhookAddr string certDir string gcInterval time.Duration - egressPort int32 zapOpts zap.Options } @@ -49,7 +48,6 @@ func init() { pf.StringVar(&config.webhookAddr, "webhook-addr", ":9443", "bind address of admission webhook") pf.StringVar(&config.certDir, "cert-dir", "/certs", "directory to locate TLS certs for webhook") pf.DurationVar(&config.gcInterval, "gc-interval", 1*time.Hour, "garbage collection interval") - pf.Int32Var(&config.egressPort, "egress-port", 5555, "UDP port number used by coil-egress") goflags := flag.NewFlagSet("klog", flag.ExitOnError) klog.InitFlags(goflags) diff --git a/v2/cmd/coil-controller/sub/run.go b/v2/cmd/coil-ipam-controller/sub/run.go similarity index 80% rename from v2/cmd/coil-controller/sub/run.go rename to v2/cmd/coil-ipam-controller/sub/run.go index a2d04cc3..6c614ad2 100644 --- a/v2/cmd/coil-controller/sub/run.go +++ b/v2/cmd/coil-ipam-controller/sub/run.go @@ -3,14 +3,12 @@ package sub import ( "fmt" "net" - "os" "strconv" "time" v2 "github.com/cybozu-go/coil/v2" coilv2 "github.com/cybozu-go/coil/v2/api/v2" "github.com/cybozu-go/coil/v2/controllers" - "github.com/cybozu-go/coil/v2/pkg/constants" "github.com/cybozu-go/coil/v2/pkg/indexing" "github.com/cybozu-go/coil/v2/pkg/ipam" "github.com/cybozu-go/coil/v2/runners" @@ -18,7 +16,6 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" @@ -57,7 +54,7 @@ func subMain() error { mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, LeaderElection: true, - LeaderElectionID: "coil-leader", + LeaderElectionID: "coil-ipam-leader", LeaderElectionNamespace: "kube-system", // coil should run in kube-system Metrics: metricsserver.Options{ BindAddress: config.metricsAddr, @@ -107,34 +104,11 @@ func subMain() error { return err } - podNS := os.Getenv(constants.EnvPodNamespace) - podName := os.Getenv(constants.EnvPodName) - img, err := controllers.GetImage(mgr.GetAPIReader(), client.ObjectKey{Namespace: podNS, Name: podName}) - if err != nil { - return err - } - egressctrl := controllers.EgressReconciler{ - Client: mgr.GetClient(), - Scheme: scheme, - Image: img, - Port: config.egressPort, - } - if err := egressctrl.SetupWithManager(mgr); err != nil { - return err - } - - if err := controllers.SetupCRBReconciler(mgr); err != nil { - return err - } - // register webhooks if err := (&coilv2.AddressPool{}).SetupWebhookWithManager(mgr); err != nil { return err } - if err := (&coilv2.Egress{}).SetupWebhookWithManager(mgr); err != nil { - return err - } // other runners diff --git a/v2/cmd/coil/main.go b/v2/cmd/coil/main.go index 634fcc37..f47b8765 100644 --- a/v2/cmd/coil/main.go +++ b/v2/cmd/coil/main.go @@ -13,7 +13,11 @@ import ( "github.com/cybozu-go/coil/v2/pkg/cnirpc" ) -const rpcTimeout = 1 * time.Minute +const ( + rpcTimeout = 1 * time.Minute + ipamEnableKey = "ipam" + egressEnableKey = "egress" +) func cmdAdd(args *skel.CmdArgs) error { conf, err := parseConfig(args.StdinData) @@ -21,11 +25,16 @@ func cmdAdd(args *skel.CmdArgs) error { return err } - if conf.PrevResult != nil { - return types.NewError(types.ErrInvalidNetworkConfig, "coil must be called as the first plugin", "") + ipamEnablad, exists := conf.Capabilities[ipamEnableKey] + if !exists { + ipamEnablad = true + } + + if ipamEnablad && conf.PrevResult != nil { + return types.NewError(types.ErrInvalidNetworkConfig, "coil must be called as the first plugin when IPAM related features are enabled", "") } - cniArgs, err := makeCNIArgs(args) + cniArgs, err := makeCNIArgs(args, conf) if err != nil { return err } @@ -45,7 +54,13 @@ func cmdAdd(args *skel.CmdArgs) error { return convertError(err) } - result, err := current.NewResult(resp.Result) + var result types.Result + if conf.PrevResult != nil { + result, err = current.NewResultFromResult(conf.PrevResult) + } else { + result, err = current.NewResult(resp.Result) + } + if err != nil { return types.NewError(types.ErrDecodingFailure, "failed to unmarshal result", err.Error()) } @@ -59,7 +74,7 @@ func cmdDel(args *skel.CmdArgs) error { return err } - cniArgs, err := makeCNIArgs(args) + cniArgs, err := makeCNIArgs(args, conf) if err != nil { return err } @@ -87,7 +102,7 @@ func cmdCheck(args *skel.CmdArgs) error { return err } - cniArgs, err := makeCNIArgs(args) + cniArgs, err := makeCNIArgs(args, conf) if err != nil { return err } diff --git a/v2/cmd/coil/rpc.go b/v2/cmd/coil/rpc.go index b42d9330..72f2e580 100644 --- a/v2/cmd/coil/rpc.go +++ b/v2/cmd/coil/rpc.go @@ -2,11 +2,15 @@ package main import ( "context" + "fmt" "net" + "strconv" "github.com/containernetworking/cni/pkg/skel" "github.com/containernetworking/cni/pkg/types" + current "github.com/containernetworking/cni/pkg/types/100" "github.com/cybozu-go/coil/v2/pkg/cnirpc" + "github.com/cybozu-go/coil/v2/pkg/constants" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/resolver" @@ -14,20 +18,52 @@ import ( ) // makeCNIArgs creates *CNIArgs. -func makeCNIArgs(args *skel.CmdArgs) (*cnirpc.CNIArgs, error) { +func makeCNIArgs(args *skel.CmdArgs, conf *PluginConf) (*cnirpc.CNIArgs, error) { env := &PluginEnvArgs{} if err := types.LoadArgs(args.Args, env); err != nil { return nil, types.NewError(types.ErrInvalidEnvironmentVariables, "failed to load CNI_ARGS", err.Error()) } + argsData := env.Map() + ipamEnablad, exists := conf.Capabilities[ipamEnableKey] + if !exists { + ipamEnablad = true + } + + egressEnabled, exists := conf.Capabilities[egressEnableKey] + if !exists { + egressEnabled = true + } + + argsData[constants.EnableIPAM] = strconv.FormatBool(ipamEnablad) + argsData[constants.EnableEgress] = strconv.FormatBool(egressEnabled) + + ips := []string{} + interfaces := map[string]bool{} + if conf.PrevResult != nil { + prevResult, err := current.GetResult(conf.PrevResult) + if err != nil { + return nil, fmt.Errorf("error getting previous CNI result: %w", err) + } + for _, ip := range prevResult.IPs { + ips = append(ips, ip.Address.IP.String()) + } + for _, intf := range prevResult.Interfaces { + interfaces[intf.Name] = intf.Sandbox != "" + } + } + cniArgs := &cnirpc.CNIArgs{ ContainerId: args.ContainerID, Netns: args.Netns, Ifname: args.IfName, - Args: env.Map(), + Args: argsData, Path: args.Path, StdinData: args.StdinData, + Ips: ips, + Interfaces: interfaces, } + return cniArgs, nil } diff --git a/v2/cmd/coild/sub/root.go b/v2/cmd/coild/sub/root.go index 042c8c9e..e25af4f5 100644 --- a/v2/cmd/coild/sub/root.go +++ b/v2/cmd/coild/sub/root.go @@ -24,6 +24,8 @@ var config struct { egressPort int registerFromMain bool zapOpts zap.Options + enableIPAM bool + enableEgress bool } var rootCmd = &cobra.Command{ @@ -61,6 +63,8 @@ func init() { pf.BoolVar(&config.compatCalico, "compat-calico", false, "make veth name compatible with Calico") pf.IntVar(&config.egressPort, "egress-port", 5555, "UDP port number for egress NAT") pf.BoolVar(&config.registerFromMain, "register-from-main", false, "help migration from Coil 2.0.1") + pf.BoolVar(&config.enableIPAM, "enable-ipam", true, "enable IPAM related features") + pf.BoolVar(&config.enableEgress, "enable-egress", true, "enable IPAM related features") goflags := flag.NewFlagSet("klog", flag.ExitOnError) klog.InitFlags(goflags) diff --git a/v2/cmd/coild/sub/run.go b/v2/cmd/coild/sub/run.go index 109b517c..61c3cc9e 100644 --- a/v2/cmd/coild/sub/run.go +++ b/v2/cmd/coild/sub/run.go @@ -78,13 +78,15 @@ func subMain() error { exporter := nodenet.NewRouteExporter(config.exportTableId, config.protocolId, ctrl.Log.WithName("route-exporter")) nodeIPAM := ipam.NewNodeIPAM(nodeName, ctrl.Log.WithName("node-ipam"), mgr, exporter) - watcher := &controllers.BlockRequestWatcher{ - Client: mgr.GetClient(), - NodeIPAM: nodeIPAM, - NodeName: nodeName, - } - if err := watcher.SetupWithManager(mgr); err != nil { - return err + if config.enableIPAM { + watcher := &controllers.BlockRequestWatcher{ + Client: mgr.GetClient(), + NodeIPAM: nodeIPAM, + NodeName: nodeName, + } + if err := watcher.SetupWithManager(mgr); err != nil { + return err + } } ctx := context.Background() @@ -101,22 +103,26 @@ func subMain() error { ipv6, config.compatCalico, config.registerFromMain, - ctrl.Log.WithName("pod-network")) + ctrl.Log.WithName("pod-network"), + config.enableIPAM) if err := podNet.Init(); err != nil { return err } - podConfigs, err := podNet.List() - if err != nil { - return err - } - for _, c := range podConfigs { - if err := nodeIPAM.Register(ctx, c.PoolName, c.ContainerId, c.IFace, c.IPv4, c.IPv6); err != nil { + if config.enableIPAM { + podConfigs, err := podNet.List() + if err != nil { + return err + } + + for _, c := range podConfigs { + if err := nodeIPAM.Register(ctx, c.PoolName, c.ContainerId, c.IFace, c.IPv4, c.IPv6); err != nil { + return err + } + } + if err := nodeIPAM.GC(ctx); err != nil { return err } - } - if err := nodeIPAM.GC(ctx); err != nil { - return err } os.Remove(config.socketPath) @@ -129,15 +135,18 @@ func subMain() error { return err } - egressWatcher := &controllers.EgressWatcher{ - Client: mgr.GetClient(), - NodeName: nodeName, - PodNet: podNet, - EgressPort: config.egressPort, - } - if err := egressWatcher.SetupWithManager(mgr); err != nil { - return err + if config.enableEgress { + egressWatcher := &controllers.EgressWatcher{ + Client: mgr.GetClient(), + NodeName: nodeName, + PodNet: podNet, + EgressPort: config.egressPort, + } + if err := egressWatcher.SetupWithManager(mgr); err != nil { + return err + } } + ctx2 := ctrl.SetupSignalHandler() if err := indexing.SetupIndexForPodByNodeName(ctx2, mgr); err != nil { return err diff --git a/v2/cmd/gencert/main.go b/v2/cmd/gencert/main.go index 4e958024..ecbb342a 100644 --- a/v2/cmd/gencert/main.go +++ b/v2/cmd/gencert/main.go @@ -7,6 +7,7 @@ import ( "crypto/x509/pkix" "encoding/pem" "flag" + "fmt" "log" "math/big" "os" @@ -16,37 +17,65 @@ import ( ) var ( - host = flag.String("host", "coilv2-webhook-service.kube-system.svc", "TLS hostname") - validFor = flag.Duration("duration", 36500*24*time.Hour, "Duration that certificate is valid for") - outDir = flag.String("outdir", ".", "Directory where the certificate files are created") + host = flag.String("host", "coilv2-webhook-service.kube-system.svc", "TLS hostname") + validFor = flag.Duration("duration", 36500*24*time.Hour, "Duration that certificate is valid for") + outDir = flag.String("outdir", ".", "Directory where the certificate files are created") + commonName = flag.String("cn", "coilv2-webhook-service", "Certificate common name") + outCert = flag.String("certname", "cert.pem", "Certificate filename") + outKey = flag.String("keyname", "key.pem", "Key filename") + authority = flag.String("ca", "", "Certificate authority") + authorityKey = flag.String("cakey", "", "Certificate authority") ) func main() { flag.Parse() - priv, err := rsa.GenerateKey(rand.Reader, 4096) - if err != nil { - log.Fatal(err) + var ca *x509.Certificate + var priv *rsa.PrivateKey + var err error + if *authority != "" { + if ca, priv, err = readCA(filepath.Join(*outDir, *authority), filepath.Join(*outDir, *authorityKey)); err != nil { + log.Fatal(err) + } + } else { + priv, err = rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + log.Fatal(err) + } } + isCA := (ca == nil) + keyUsage := x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment + extKeyUsage := []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth} + if isCA { + keyUsage = x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign + extKeyUsage = append(extKeyUsage, x509.ExtKeyUsageClientAuth) + } + notBefore := time.Now() notAfter := notBefore.Add(*validFor) template := x509.Certificate{ SerialNumber: big.NewInt(1), Subject: pkix.Name{ - CommonName: "coilv2-webhook-service", + CommonName: *commonName, }, NotBefore: notBefore, NotAfter: notAfter, KeyUsage: keyUsage, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + ExtKeyUsage: extKeyUsage, BasicConstraintsValid: true, + IsCA: isCA, DNSNames: dnsAliases(*host), } - certBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, priv.Public(), priv) + parent := ca + if isCA { + parent = &template + } + + certBytes, err := x509.CreateCertificate(rand.Reader, &template, parent, priv.Public(), priv) if err != nil { log.Fatalf("failed to create certificate: %v", err) } @@ -68,8 +97,8 @@ func main() { log.Fatalf("failed to marshal private key: %v", err) } - outputPEM(filepath.Join(*outDir, "cert.pem"), "CERTIFICATE", certBytes) - outputPEM(filepath.Join(*outDir, "key.pem"), "PRIVATE KEY", privBytes) + outputPEM(filepath.Join(*outDir, *outCert), "CERTIFICATE", certBytes) + outputPEM(filepath.Join(*outDir, *outKey), "PRIVATE KEY", privBytes) } func dnsAliases(host string) []string { @@ -98,3 +127,35 @@ func outputPEM(fname string, pemType string, data []byte) { log.Fatalf("failed to fsync: %v", err) } } + +func readCA(certPath, keyPath string) (*x509.Certificate, *rsa.PrivateKey, error) { + certFile, err := os.ReadFile(certPath) + if err != nil { + return nil, nil, fmt.Errorf("error reading file '%s': %w", certPath, err) + } + + caData, _ := pem.Decode(certFile) + + ca, err := x509.ParseCertificate(caData.Bytes) + if err != nil { + log.Fatal(err) + } + + keyFile, err := os.ReadFile(keyPath) + if err != nil { + log.Fatal(err) + } + + cakData, _ := pem.Decode(keyFile) + key, err := x509.ParsePKCS8PrivateKey(cakData.Bytes) + if err != nil { + log.Fatal(err) + } + + privateKey, ok := key.(*rsa.PrivateKey) + if !ok { + return nil, nil, fmt.Errorf("type assertion error - object is not of type *rsa.PrivateKey") + } + + return ca, privateKey, nil +} diff --git a/v2/config/cke/webhook-secret.yaml b/v2/config/cke/webhook-secret.yaml index 6275f0aa..658ffee9 100644 --- a/v2/config/cke/webhook-secret.yaml +++ b/v2/config/cke/webhook-secret.yaml @@ -1,7 +1,15 @@ apiVersion: v1 kind: Secret metadata: - name: coilv2-webhook-server-cert + name: coilv2-ipam-webhook-server-cert namespace: system annotations: - cke.cybozu.com/issue-cert: coilv2-webhook-service + cke.cybozu.com/issue-cert: coilv2-ipam-webhook-service +--- +apiVersion: v1 +kind: Secret +metadata: + name: coilv2-egress-webhook-server-cert + namespace: system + annotations: + cke.cybozu.com/issue-cert: coilv2-egress-webhook-service diff --git a/v2/config/crd/kustomization.yaml b/v2/config/crd/kustomization.yaml index 7816c22b..d22f3656 100644 --- a/v2/config/crd/kustomization.yaml +++ b/v2/config/crd/kustomization.yaml @@ -2,9 +2,11 @@ # since it depends on service name and namespace that are out of this kustomize package. # It should be run by config/default resources: +# [IPAM] Following files should be uncommented to enable IPAM features. - bases/coil.cybozu.com_addresspools.yaml - bases/coil.cybozu.com_addressblocks.yaml - bases/coil.cybozu.com_blockrequests.yaml +# [EGRESS] Following files should be uncommented to enable Egress NAT features. - bases/coil.cybozu.com_egresses.yaml # +kubebuilder:scaffold:crdkustomizeresource @@ -12,17 +14,21 @@ patchesStrategicMerge: - patches/remove_status.yaml # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. # patches here are for enabling the conversion webhook for each CRD +# [IPAM] Following files should be uncommented to enable IPAM features. #- patches/webhook_in_addresspools.yaml #- patches/webhook_in_addressblocks.yaml #- patches/webhook_in_blockrequests.yaml +# [EGRESS] Following files should be uncommented to enable Egress NAT features. #- patches/webhook_in_egresses.yaml # +kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. # patches here are for enabling the CA injection for each CRD +# [IPAM] Following files should be uncommented to enable IPAM features. #- patches/cainjection_in_addresspools.yaml #- patches/cainjection_in_addressblocks.yaml #- patches/cainjection_in_blockrequests.yaml +# [EGRESS] Following files should be uncommented to enable Egress NAT features. #- patches/cainjection_in_egresses.yaml # +kubebuilder:scaffold:crdkustomizecainjectionpatch diff --git a/v2/config/crd/patches/remove_status.yaml b/v2/config/crd/patches/remove_status.yaml index 0683c1ed..43552f63 100644 --- a/v2/config/crd/patches/remove_status.yaml +++ b/v2/config/crd/patches/remove_status.yaml @@ -1,3 +1,4 @@ +# [IPAM] Following resources should be uncommented to enable IPAM features. apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -16,6 +17,7 @@ metadata: name: blockrequests.coil.cybozu.com status: null --- +# [EGRESS] Following resources be uncommented to enable Egress NAT features. apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: diff --git a/v2/config/crd/patches/webhook_in_addressblocks.yaml b/v2/config/crd/patches/webhook_in_addressblocks.yaml index b9cc9943..53cd94a0 100644 --- a/v2/config/crd/patches/webhook_in_addressblocks.yaml +++ b/v2/config/crd/patches/webhook_in_addressblocks.yaml @@ -13,5 +13,5 @@ spec: caBundle: Cg== service: namespace: system - name: webhook-service + name: ipam-webhook-service path: /convert diff --git a/v2/config/crd/patches/webhook_in_addresspools.yaml b/v2/config/crd/patches/webhook_in_addresspools.yaml index 6e1594a6..1c88c43a 100644 --- a/v2/config/crd/patches/webhook_in_addresspools.yaml +++ b/v2/config/crd/patches/webhook_in_addresspools.yaml @@ -13,5 +13,5 @@ spec: caBundle: Cg== service: namespace: system - name: webhook-service + name: ipam-webhook-service path: /convert diff --git a/v2/config/crd/patches/webhook_in_blockrequests.yaml b/v2/config/crd/patches/webhook_in_blockrequests.yaml index 9757563c..ad5a5f6c 100644 --- a/v2/config/crd/patches/webhook_in_blockrequests.yaml +++ b/v2/config/crd/patches/webhook_in_blockrequests.yaml @@ -13,5 +13,5 @@ spec: caBundle: Cg== service: namespace: system - name: webhook-service + name: ipam-webhook-service path: /convert diff --git a/v2/config/crd/patches/webhook_in_egresses.yaml b/v2/config/crd/patches/webhook_in_egresses.yaml index cadaf5a8..eb0bd60a 100644 --- a/v2/config/crd/patches/webhook_in_egresses.yaml +++ b/v2/config/crd/patches/webhook_in_egresses.yaml @@ -13,5 +13,5 @@ spec: caBundle: Cg== service: namespace: system - name: webhook-service + name: egress-webhook-service path: /convert diff --git a/v2/config/default/.gitignore b/v2/config/default/.gitignore index f933710b..3bdc987e 100644 --- a/v2/config/default/.gitignore +++ b/v2/config/default/.gitignore @@ -1,2 +1,6 @@ cert.pem key.pem +ipam-cert.pem +ipam-key.pem +egress-cert.pem +egress-key.pem \ No newline at end of file diff --git a/v2/config/default/kustomization.yaml b/v2/config/default/kustomization.yaml index dd042d0a..af24152b 100644 --- a/v2/config/default/kustomization.yaml +++ b/v2/config/default/kustomization.yaml @@ -11,9 +11,17 @@ generatorOptions: disableNameSuffixHash: true secretGenerator: -- name: coilv2-webhook-server-cert +# [IPAM] Following lines should be uncommented to enable IPAM features. +- name: coilv2-ipam-webhook-server-cert files: - - ca.crt=./cert.pem - - tls.crt=./cert.pem - - tls.key=./key.pem + - ca.crt=./ipam-cert.pem + - tls.crt=./ipam-cert.pem + - tls.key=./ipam-key.pem type: "kubernetes.io/tls" +# [EGRESS] Following lines be uncommented to enable Egress NAT features. +- name: coilv2-egress-webhook-server-cert + files: + - ca.crt=./egress-cert.pem + - tls.crt=./egress-cert.pem + - tls.key=./egress-key.pem + type: "kubernetes.io/tls" \ No newline at end of file diff --git a/v2/config/default/webhook_manifests_patch.yaml.tmpl b/v2/config/default/webhook_manifests_patch.yaml.tmpl index 29096510..3c7c7ca3 100644 --- a/v2/config/default/webhook_manifests_patch.yaml.tmpl +++ b/v2/config/default/webhook_manifests_patch.yaml.tmpl @@ -1,24 +1,35 @@ apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: - name: coilv2-mutating-webhook-configuration + name: coilv2-mutating-ipam-webhook-configuration webhooks: - name: maddresspool.kb.io clientConfig: caBundle: "%CACERT%" -- name: megress.kb.io - clientConfig: - caBundle: "%CACERT%" --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: - name: coilv2-validating-webhook-configuration + name: coilv2-validating-ipam-webhook-configuration webhooks: - name: vaddresspool.kb.io clientConfig: caBundle: "%CACERT%" -- name: vegress.kb.io +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: coilv2-mutating-egress-webhook-configuration +webhooks: +- name: megress.kb.io clientConfig: caBundle: "%CACERT%" --- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: coilv2-validating-egress-webhook-configuration +webhooks: +- name: vegress.kb.io + clientConfig: + caBundle: "%CACERT%" diff --git a/v2/config/pod/coil-egress-controller.yaml b/v2/config/pod/coil-egress-controller.yaml new file mode 100644 index 00000000..00f67a53 --- /dev/null +++ b/v2/config/pod/coil-egress-controller.yaml @@ -0,0 +1,90 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coil-egress-controller + namespace: system + labels: + app.kubernetes.io/component: coil-egress-controller +spec: + selector: + matchLabels: + app.kubernetes.io/component: coil-egress-controller + replicas: 2 + template: + metadata: + labels: + app.kubernetes.io/component: coil-egress-controller + spec: + hostNetwork: true + priorityClassName: system-cluster-critical + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + - key: node.kubernetes.io/not-ready + effect: NoSchedule + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/component + operator: In + values: ["coil-egress-controller"] + topologyKey: kubernetes.io/hostname + securityContext: + runAsUser: 10000 + runAsGroup: 10000 + serviceAccountName: coil-egress-controller + terminationGracePeriodSeconds: 10 + containers: + - name: coil-egress-controller + image: coil:dev + command: ["coil-egress-controller"] + args: + - --zap-stacktrace-level=panic + env: + - name: "COIL_POD_NAMESPACE" + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: "COIL_POD_NAME" + valueFrom: + fieldRef: + fieldPath: metadata.name + ports: + - name: metrics + containerPort: 9396 + protocol: TCP + - name: health + containerPort: 9397 + protocol: TCP + - name: webhook-server + containerPort: 9444 + protocol: TCP + resources: + requests: + cpu: 100m + memory: 200Mi + readinessProbe: + httpGet: + path: /readyz + port: health + host: localhost + livenessProbe: + httpGet: + path: /healthz + port: health + host: localhost + volumeMounts: + - mountPath: /certs + name: cert + readOnly: true + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: coilv2-egress-webhook-server-cert diff --git a/v2/config/pod/coil-controller.yaml b/v2/config/pod/coil-ipam-controller.yaml similarity index 82% rename from v2/config/pod/coil-controller.yaml rename to v2/config/pod/coil-ipam-controller.yaml index fdc8ab8c..5445d760 100644 --- a/v2/config/pod/coil-controller.yaml +++ b/v2/config/pod/coil-ipam-controller.yaml @@ -1,19 +1,19 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: coil-controller + name: coil-ipam-controller namespace: system labels: - app.kubernetes.io/component: coil-controller + app.kubernetes.io/component: coil-ipam-controller spec: selector: matchLabels: - app.kubernetes.io/component: coil-controller + app.kubernetes.io/component: coil-ipam-controller replicas: 2 template: metadata: labels: - app.kubernetes.io/component: coil-controller + app.kubernetes.io/component: coil-ipam-controller spec: hostNetwork: true priorityClassName: system-cluster-critical @@ -33,17 +33,17 @@ spec: matchExpressions: - key: app.kubernetes.io/component operator: In - values: ["coil-controller"] + values: ["coil-ipam-controller"] topologyKey: kubernetes.io/hostname securityContext: runAsUser: 10000 runAsGroup: 10000 - serviceAccountName: coil-controller + serviceAccountName: coil-ipam-controller terminationGracePeriodSeconds: 10 containers: - - name: coil-controller + - name: coil-ipam-controller image: coil:dev - command: ["coil-controller"] + command: ["coil-ipam-controller"] args: - --zap-stacktrace-level=panic env: @@ -87,4 +87,4 @@ spec: - name: cert secret: defaultMode: 420 - secretName: coilv2-webhook-server-cert + secretName: coilv2-ipam-webhook-server-cert diff --git a/v2/config/pod/coild.yaml b/v2/config/pod/coild.yaml index 7bdd7547..f7898950 100644 --- a/v2/config/pod/coild.yaml +++ b/v2/config/pod/coild.yaml @@ -30,6 +30,8 @@ spec: command: ["coild"] args: - --zap-stacktrace-level=panic + - --enable-ipam=true + - --enable-egress=true env: - name: COIL_NODE_NAME valueFrom: diff --git a/v2/config/pod/kustomization.yaml b/v2/config/pod/kustomization.yaml index 70b23f1a..b4bcc3ee 100644 --- a/v2/config/pod/kustomization.yaml +++ b/v2/config/pod/kustomization.yaml @@ -1,3 +1,6 @@ resources: -- coil-controller.yaml +# [IPAM] Next file should be uncommented to enable IPAM features. +- coil-ipam-controller.yaml +# [EGRESS] Following line should be uncommented to enable Egress NAT features. +- coil-egress-controller.yaml - coild.yaml diff --git a/v2/config/rbac/coil-controller_role.yaml b/v2/config/rbac/coil-egress-controller_role.yaml similarity index 61% rename from v2/config/rbac/coil-controller_role.yaml rename to v2/config/rbac/coil-egress-controller_role.yaml index 10e366f7..864cf169 100644 --- a/v2/config/rbac/coil-controller_role.yaml +++ b/v2/config/rbac/coil-egress-controller_role.yaml @@ -2,15 +2,8 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: coil-controller + name: coil-egress-controller rules: -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - apiGroups: - "" resources: @@ -42,44 +35,6 @@ rules: - patch - update - watch -- apiGroups: - - coil.cybozu.com - resources: - - addressblocks - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - coil.cybozu.com - resources: - - addresspools - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - coil.cybozu.com - resources: - - blockrequests - verbs: - - get - - list - - watch -- apiGroups: - - coil.cybozu.com - resources: - - blockrequests/status - verbs: - - get - - patch - - update - apiGroups: - coil.cybozu.com resources: diff --git a/v2/config/rbac/coil-ipam-controller_role.yaml b/v2/config/rbac/coil-ipam-controller_role.yaml new file mode 100644 index 00000000..814b83c5 --- /dev/null +++ b/v2/config/rbac/coil-ipam-controller_role.yaml @@ -0,0 +1,59 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: coil-ipam-controller +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - coil.cybozu.com + resources: + - addressblocks + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - coil.cybozu.com + resources: + - addresspools + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - coil.cybozu.com + resources: + - blockrequests + verbs: + - get + - list + - watch +- apiGroups: + - coil.cybozu.com + resources: + - blockrequests/status + verbs: + - get + - patch + - update diff --git a/v2/config/rbac/kustomization.yaml b/v2/config/rbac/kustomization.yaml index 20adc469..fdcbfd7d 100644 --- a/v2/config/rbac/kustomization.yaml +++ b/v2/config/rbac/kustomization.yaml @@ -1,13 +1,16 @@ resources: - serviceaccount.yaml -- coil-controller_role.yaml -- coil-router_role.yaml -- coild_role.yaml -- coil-egress_role.yaml - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml +- coild_role.yaml +# [IPAM] Following files should be uncommented to enable IPAM features. +- coil-ipam-controller_role.yaml +- coil-router_role.yaml - addressblock_viewer_role.yaml - addresspool_viewer_role.yaml - blockrequest_viewer_role.yaml +# [EGRESS] Following files should be uncommented to enable Egress NAT features. +- coil-egress-controller_role.yaml +- coil-egress_role.yaml - egress_viewer_role.yaml diff --git a/v2/config/rbac/leader_election_role_binding.yaml b/v2/config/rbac/leader_election_role_binding.yaml index 40e370df..1b237987 100644 --- a/v2/config/rbac/leader_election_role_binding.yaml +++ b/v2/config/rbac/leader_election_role_binding.yaml @@ -8,5 +8,8 @@ roleRef: name: coil-leader-election subjects: - kind: ServiceAccount - name: coil-controller + name: coil-ipam-controller namespace: system +- kind: ServiceAccount + name: coil-egress-controller + namespace: system \ No newline at end of file diff --git a/v2/config/rbac/role_binding.yaml b/v2/config/rbac/role_binding.yaml index faa422ef..bcf4fb74 100644 --- a/v2/config/rbac/role_binding.yaml +++ b/v2/config/rbac/role_binding.yaml @@ -1,14 +1,27 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: coil-controller + name: coil-ipam-controller roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: coil-controller + name: coil-ipam-controller subjects: - kind: ServiceAccount - name: coil-controller + name: coil-ipam-controller + namespace: system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: coil-egress-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: coil-egress-controller +subjects: +- kind: ServiceAccount + name: coil-egress-controller namespace: system --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/v2/config/rbac/serviceaccount.yaml b/v2/config/rbac/serviceaccount.yaml index 292656b2..9e519749 100644 --- a/v2/config/rbac/serviceaccount.yaml +++ b/v2/config/rbac/serviceaccount.yaml @@ -1,7 +1,13 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: coil-controller + name: coil-ipam-controller + namespace: system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: coil-egress-controller namespace: system --- apiVersion: v1 diff --git a/v2/config/webhook/manifests.v1.yaml b/v2/config/webhook/manifests.v1.yaml index b4facf51..1ce142fe 100644 --- a/v2/config/webhook/manifests.v1.yaml +++ b/v2/config/webhook/manifests.v1.yaml @@ -2,14 +2,14 @@ apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: - name: mutating-webhook-configuration + name: mutating-ipam-webhook-configuration webhooks: - admissionReviewVersions: - v1 - v1beta1 clientConfig: service: - name: webhook-service + name: ipam-webhook-service namespace: system path: /mutate-coil-cybozu-com-v2-addresspool failurePolicy: Fail @@ -24,16 +24,22 @@ webhooks: resources: - addresspools sideEffects: None +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validating-ipam-webhook-configuration +webhooks: - admissionReviewVersions: - v1 - v1beta1 clientConfig: service: - name: webhook-service + name: ipam-webhook-service namespace: system - path: /mutate-coil-cybozu-com-v2-egress + path: /validate-coil-cybozu-com-v2-addresspool failurePolicy: Fail - name: megress.kb.io + name: vaddresspool.kb.io rules: - apiGroups: - coil.cybozu.com @@ -41,25 +47,26 @@ webhooks: - v2 operations: - CREATE + - UPDATE resources: - - egresses + - addresspools sideEffects: None --- apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration +kind: MutatingWebhookConfiguration metadata: - name: validating-webhook-configuration + name: mutating-egress-webhook-configuration webhooks: - admissionReviewVersions: - v1 - v1beta1 clientConfig: service: - name: webhook-service + name: egress-webhook-service namespace: system - path: /validate-coil-cybozu-com-v2-addresspool + path: /mutate-coil-cybozu-com-v2-egress failurePolicy: Fail - name: vaddresspool.kb.io + name: megress.kb.io rules: - apiGroups: - coil.cybozu.com @@ -67,16 +74,21 @@ webhooks: - v2 operations: - CREATE - - UPDATE resources: - - addresspools + - egresses sideEffects: None +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validating-egress-webhook-configuration +webhooks: - admissionReviewVersions: - v1 - v1beta1 clientConfig: service: - name: webhook-service + name: egress-webhook-service namespace: system path: /validate-coil-cybozu-com-v2-egress failurePolicy: Fail diff --git a/v2/config/webhook/service.yaml b/v2/config/webhook/service.yaml index 1688b4e5..2bec62fb 100644 --- a/v2/config/webhook/service.yaml +++ b/v2/config/webhook/service.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: - name: webhook-service + name: ipam-webhook-service namespace: system spec: ports: @@ -10,4 +10,17 @@ spec: targetPort: 9443 protocol: TCP selector: - app.kubernetes.io/component: coil-controller + app.kubernetes.io/component: coil-ipam-controller +--- +apiVersion: v1 +kind: Service +metadata: + name: egress-webhook-service + namespace: system +spec: + ports: + - port: 443 + targetPort: 9444 + protocol: TCP + selector: + app.kubernetes.io/component: coil-egress-controller diff --git a/v2/controllers/egress_watcher.go b/v2/controllers/egress_watcher.go index 879eddbd..cdbf2c6a 100644 --- a/v2/controllers/egress_watcher.go +++ b/v2/controllers/egress_watcher.go @@ -125,7 +125,7 @@ func (r *EgressWatcher) reconcileEgressClient(ctx context.Context, eg *coilv2.Eg ipv6 = ip.To16() } } - if err := r.PodNet.Update(ipv4, ipv6, hook); err != nil { + if err := r.PodNet.Update(ipv4, ipv6, hook, pod); err != nil { return fmt.Errorf("failed to update NAT configuration: %w", err) } diff --git a/v2/controllers/egress_watcher_test.go b/v2/controllers/egress_watcher_test.go index b469adaa..c610d58a 100644 --- a/v2/controllers/egress_watcher_test.go +++ b/v2/controllers/egress_watcher_test.go @@ -180,11 +180,15 @@ func (p *mockPodNetwork) List() ([]*nodenet.PodNetConf, error) { panic("not implemented") } -func (p *mockPodNetwork) Setup(nsPath, podName, podNS string, conf *nodenet.PodNetConf, hook nodenet.SetupHook) (*current.Result, error) { +func (p *mockPodNetwork) SetupIPAM(nsPath, podName, podNS string, conf *nodenet.PodNetConf) (*current.Result, error) { panic("not implemented") } -func (p *mockPodNetwork) Update(podIPv4, podIPv6 net.IP, hook nodenet.SetupHook) error { +func (p *mockPodNetwork) SetupEgress(nsPath string, conf *nodenet.PodNetConf, hook nodenet.SetupHook) error { + panic("not implemented") +} + +func (p *mockPodNetwork) Update(podIPv4, podIPv6 net.IP, hook nodenet.SetupHook, pod *corev1.Pod) error { p.mu.Lock() defer p.mu.Unlock() diff --git a/v2/e2e/Makefile b/v2/e2e/Makefile index 2a2de9a4..3b88f725 100644 --- a/v2/e2e/Makefile +++ b/v2/e2e/Makefile @@ -29,7 +29,8 @@ install-coil: $(KUBECTL) label nodes coil-worker2 test=coil $(KIND) load docker-image --name coil coil:dev $(KUSTOMIZE) build --load-restrictor=LoadRestrictionsNone . | $(KUBECTL) apply -f - - $(KUBECTL) -n kube-system wait --timeout=3m --for=condition=available deployment/coil-controller + $(KUBECTL) -n kube-system wait --timeout=3m --for=condition=available deployment/coil-ipam-controller + $(KUBECTL) -n kube-system wait --timeout=3m --for=condition=available deployment/coil-egress-controller .PHONY: test test: diff --git a/v2/e2e/coil-controller_patch.yaml b/v2/e2e/coil-controller_patch.yaml index 13db1c02..f35a6eee 100644 --- a/v2/e2e/coil-controller_patch.yaml +++ b/v2/e2e/coil-controller_patch.yaml @@ -1,11 +1,11 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: coil-controller + name: coil-ipam-controller namespace: system spec: template: spec: containers: - - name: coil-controller + - name: coil-ipam-controller args: ["--gc-interval=10s"] diff --git a/v2/e2e/controller_test.go b/v2/e2e/controller_test.go index d6d3ff3e..33a1cf46 100644 --- a/v2/e2e/controller_test.go +++ b/v2/e2e/controller_test.go @@ -10,9 +10,9 @@ import ( corev1 "k8s.io/api/core/v1" ) -var _ = Describe("coil-controller", func() { +var _ = Describe("coil-ipam-controller", func() { It("should elect a leader instance of coil-controller", func() { - kubectlSafe(nil, "-n", "kube-system", "get", "leases", "coil-leader") + kubectlSafe(nil, "-n", "kube-system", "get", "leases", "coil-ipam-leader") }) It("should run the admission webhook", func() { @@ -23,7 +23,7 @@ var _ = Describe("coil-controller", func() { It("should export metrics", func() { pods := &corev1.PodList{} - getResourceSafe("kube-system", "pods", "", "app.kubernetes.io/component=coil-controller", pods) + getResourceSafe("kube-system", "pods", "", "app.kubernetes.io/component=coil-ipam-controller", pods) Expect(pods.Items).Should(HaveLen(2)) node := pods.Items[0].Spec.NodeName @@ -50,3 +50,23 @@ var _ = Describe("coil-controller", func() { }, 20).Should(BeEmpty()) }) }) + +var _ = Describe("coil-egress-controller", func() { + It("should elect a leader instance of coil-controller", func() { + kubectlSafe(nil, "-n", "kube-system", "get", "leases", "coil-egress-leader") + }) + + It("should export metrics", func() { + pods := &corev1.PodList{} + getResourceSafe("kube-system", "pods", "", "app.kubernetes.io/component=coil-egress-controller", pods) + Expect(pods.Items).Should(HaveLen(2)) + + node := pods.Items[0].Spec.NodeName + out, err := runOnNode(node, "curl", "-sf", "http://localhost:9396/metrics") + Expect(err).ShouldNot(HaveOccurred()) + + mfs, err := (&expfmt.TextParser{}).TextToMetricFamilies(bytes.NewReader(out)) + Expect(err).NotTo(HaveOccurred()) + Expect(mfs).NotTo(BeEmpty()) + }) +}) diff --git a/v2/kustomization.yaml b/v2/kustomization.yaml index b941ffe8..b9462034 100644 --- a/v2/kustomization.yaml +++ b/v2/kustomization.yaml @@ -5,7 +5,7 @@ images: resources: - config/default -# If you are using CKE (github.com/cybozu-go/cke) and wwant to use +# If you are using CKE (github.com/cybozu-go/cke) and want to use # its webhook installation feature, comment the above line and # uncomment the below line. #- config/cke diff --git a/v2/netconf.json b/v2/netconf.json index d3efb490..d0faa985 100644 --- a/v2/netconf.json +++ b/v2/netconf.json @@ -4,7 +4,11 @@ "plugins": [ { "type": "coil", - "socket": "/run/coild.sock" + "socket": "/run/coild.sock", + "capabilities": { + "ipam": true, + "egress": true + } }, { "type": "portmap", diff --git a/v2/pkg/cnirpc/cni.pb.go b/v2/pkg/cnirpc/cni.pb.go index e46d12c1..f40dbe82 100644 --- a/v2/pkg/cnirpc/cni.pb.go +++ b/v2/pkg/cnirpc/cni.pb.go @@ -105,6 +105,8 @@ type CNIArgs struct { Args map[string]string `protobuf:"bytes,4,rep,name=args,proto3" json:"args,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` // Key-Value pairs parsed from CNI_ARGS Path string `protobuf:"bytes,5,opt,name=path,proto3" json:"path,omitempty"` StdinData []byte `protobuf:"bytes,6,opt,name=stdin_data,json=stdinData,proto3" json:"stdin_data,omitempty"` + Ips []string `protobuf:"bytes,7,rep,name=ips,proto3" json:"ips,omitempty"` + Interfaces map[string]bool `protobuf:"bytes,8,rep,name=interfaces,proto3" json:"interfaces,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` } func (x *CNIArgs) Reset() { @@ -181,6 +183,20 @@ func (x *CNIArgs) GetStdinData() []byte { return nil } +func (x *CNIArgs) GetIps() []string { + if x != nil { + return x.Ips + } + return nil +} + +func (x *CNIArgs) GetInterfaces() map[string]bool { + if x != nil { + return x.Interfaces + } + return nil +} + // CNIError is a mirror of cin.pkg.types.Error struct. // https://pkg.go.dev/github.com/containernetworking/cni@v0.8.0/pkg/types?tab=doc#Error // @@ -307,7 +323,7 @@ var file_pkg_cnirpc_cni_proto_rawDesc = []byte{ 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x70, 0x6b, 0x67, 0x2e, 0x63, 0x6e, 0x69, 0x72, 0x70, 0x63, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, - 0xf9, 0x01, 0x0a, 0x07, 0x43, 0x4e, 0x49, 0x41, 0x72, 0x67, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x63, + 0x8f, 0x03, 0x0a, 0x07, 0x43, 0x4e, 0x49, 0x41, 0x72, 0x67, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x6e, 0x65, 0x74, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6e, @@ -319,47 +335,57 @@ var file_pkg_cnirpc_cni_proto_rawDesc = []byte{ 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x74, 0x64, 0x69, 0x6e, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x74, 0x64, 0x69, 0x6e, 0x44, 0x61, - 0x74, 0x61, 0x1a, 0x37, 0x0a, 0x09, 0x41, 0x72, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, - 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x61, 0x0a, 0x08, 0x43, - 0x4e, 0x49, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x29, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x15, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x63, 0x6e, 0x69, 0x72, - 0x70, 0x63, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x63, 0x6f, - 0x64, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x6d, 0x73, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x22, 0x25, - 0x0a, 0x0b, 0x41, 0x64, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, - 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x72, - 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2a, 0xed, 0x01, 0x0a, 0x09, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, - 0x6f, 0x64, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, - 0x12, 0x1c, 0x0a, 0x18, 0x49, 0x4e, 0x43, 0x4f, 0x4d, 0x50, 0x41, 0x54, 0x49, 0x42, 0x4c, 0x45, - 0x5f, 0x43, 0x4e, 0x49, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x10, 0x01, 0x12, 0x15, - 0x0a, 0x11, 0x55, 0x4e, 0x53, 0x55, 0x50, 0x50, 0x4f, 0x52, 0x54, 0x45, 0x44, 0x5f, 0x46, 0x49, - 0x45, 0x4c, 0x44, 0x10, 0x02, 0x12, 0x15, 0x0a, 0x11, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, - 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x41, 0x49, 0x4e, 0x45, 0x52, 0x10, 0x03, 0x12, 0x21, 0x0a, 0x1d, - 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x5f, 0x45, 0x4e, 0x56, 0x49, 0x52, 0x4f, 0x4e, 0x4d, - 0x45, 0x4e, 0x54, 0x5f, 0x56, 0x41, 0x52, 0x49, 0x41, 0x42, 0x4c, 0x45, 0x53, 0x10, 0x04, 0x12, - 0x0e, 0x0a, 0x0a, 0x49, 0x4f, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, 0x10, 0x05, 0x12, - 0x14, 0x0a, 0x10, 0x44, 0x45, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x46, 0x41, 0x49, 0x4c, - 0x55, 0x52, 0x45, 0x10, 0x06, 0x12, 0x1a, 0x0a, 0x16, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, - 0x5f, 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x5f, 0x43, 0x4f, 0x4e, 0x46, 0x49, 0x47, 0x10, - 0x07, 0x12, 0x13, 0x0a, 0x0f, 0x54, 0x52, 0x59, 0x5f, 0x41, 0x47, 0x41, 0x49, 0x4e, 0x5f, 0x4c, - 0x41, 0x54, 0x45, 0x52, 0x10, 0x0b, 0x12, 0x0d, 0x0a, 0x08, 0x49, 0x4e, 0x54, 0x45, 0x52, 0x4e, - 0x41, 0x4c, 0x10, 0xe7, 0x07, 0x32, 0xa4, 0x01, 0x0a, 0x03, 0x43, 0x4e, 0x49, 0x12, 0x33, 0x0a, - 0x03, 0x41, 0x64, 0x64, 0x12, 0x13, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x63, 0x6e, 0x69, 0x72, 0x70, - 0x63, 0x2e, 0x43, 0x4e, 0x49, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x17, 0x2e, 0x70, 0x6b, 0x67, 0x2e, - 0x63, 0x6e, 0x69, 0x72, 0x70, 0x63, 0x2e, 0x41, 0x64, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x32, 0x0a, 0x03, 0x44, 0x65, 0x6c, 0x12, 0x13, 0x2e, 0x70, 0x6b, 0x67, 0x2e, - 0x63, 0x6e, 0x69, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x4e, 0x49, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x16, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x34, 0x0a, 0x05, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, + 0x74, 0x61, 0x12, 0x10, 0x0a, 0x03, 0x69, 0x70, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x03, 0x69, 0x70, 0x73, 0x12, 0x43, 0x0a, 0x0a, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, + 0x65, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x63, + 0x6e, 0x69, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x4e, 0x49, 0x41, 0x72, 0x67, 0x73, 0x2e, 0x49, 0x6e, + 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x69, + 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x73, 0x1a, 0x37, 0x0a, 0x09, 0x41, 0x72, 0x67, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x1a, 0x3d, 0x0a, 0x0f, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, + 0x01, 0x22, 0x61, 0x0a, 0x08, 0x43, 0x4e, 0x49, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x29, 0x0a, + 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x15, 0x2e, 0x70, 0x6b, + 0x67, 0x2e, 0x63, 0x6e, 0x69, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, + 0x64, 0x65, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x73, 0x67, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6d, 0x73, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x65, + 0x74, 0x61, 0x69, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x64, 0x65, 0x74, + 0x61, 0x69, 0x6c, 0x73, 0x22, 0x25, 0x0a, 0x0b, 0x41, 0x64, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2a, 0xed, 0x01, 0x0a, 0x09, + 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, + 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x1c, 0x0a, 0x18, 0x49, 0x4e, 0x43, 0x4f, 0x4d, 0x50, + 0x41, 0x54, 0x49, 0x42, 0x4c, 0x45, 0x5f, 0x43, 0x4e, 0x49, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, + 0x4f, 0x4e, 0x10, 0x01, 0x12, 0x15, 0x0a, 0x11, 0x55, 0x4e, 0x53, 0x55, 0x50, 0x50, 0x4f, 0x52, + 0x54, 0x45, 0x44, 0x5f, 0x46, 0x49, 0x45, 0x4c, 0x44, 0x10, 0x02, 0x12, 0x15, 0x0a, 0x11, 0x55, + 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x41, 0x49, 0x4e, 0x45, 0x52, + 0x10, 0x03, 0x12, 0x21, 0x0a, 0x1d, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x5f, 0x45, 0x4e, + 0x56, 0x49, 0x52, 0x4f, 0x4e, 0x4d, 0x45, 0x4e, 0x54, 0x5f, 0x56, 0x41, 0x52, 0x49, 0x41, 0x42, + 0x4c, 0x45, 0x53, 0x10, 0x04, 0x12, 0x0e, 0x0a, 0x0a, 0x49, 0x4f, 0x5f, 0x46, 0x41, 0x49, 0x4c, + 0x55, 0x52, 0x45, 0x10, 0x05, 0x12, 0x14, 0x0a, 0x10, 0x44, 0x45, 0x43, 0x4f, 0x44, 0x49, 0x4e, + 0x47, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, 0x10, 0x06, 0x12, 0x1a, 0x0a, 0x16, 0x49, + 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x5f, 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x5f, 0x43, + 0x4f, 0x4e, 0x46, 0x49, 0x47, 0x10, 0x07, 0x12, 0x13, 0x0a, 0x0f, 0x54, 0x52, 0x59, 0x5f, 0x41, + 0x47, 0x41, 0x49, 0x4e, 0x5f, 0x4c, 0x41, 0x54, 0x45, 0x52, 0x10, 0x0b, 0x12, 0x0d, 0x0a, 0x08, + 0x49, 0x4e, 0x54, 0x45, 0x52, 0x4e, 0x41, 0x4c, 0x10, 0xe7, 0x07, 0x32, 0xa4, 0x01, 0x0a, 0x03, + 0x43, 0x4e, 0x49, 0x12, 0x33, 0x0a, 0x03, 0x41, 0x64, 0x64, 0x12, 0x13, 0x2e, 0x70, 0x6b, 0x67, + 0x2e, 0x63, 0x6e, 0x69, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x4e, 0x49, 0x41, 0x72, 0x67, 0x73, 0x1a, + 0x17, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x63, 0x6e, 0x69, 0x72, 0x70, 0x63, 0x2e, 0x41, 0x64, 0x64, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x32, 0x0a, 0x03, 0x44, 0x65, 0x6c, 0x12, 0x13, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x63, 0x6e, 0x69, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x4e, 0x49, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x42, 0x29, 0x5a, 0x27, - 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x79, 0x62, 0x6f, 0x7a, - 0x75, 0x2d, 0x67, 0x6f, 0x2f, 0x63, 0x6f, 0x69, 0x6c, 0x2f, 0x76, 0x32, 0x2f, 0x70, 0x6b, 0x67, - 0x2f, 0x63, 0x6e, 0x69, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x34, 0x0a, 0x05, + 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x13, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x63, 0x6e, 0x69, 0x72, + 0x70, 0x63, 0x2e, 0x43, 0x4e, 0x49, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, + 0x74, 0x79, 0x42, 0x29, 0x5a, 0x27, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x63, 0x79, 0x62, 0x6f, 0x7a, 0x75, 0x2d, 0x67, 0x6f, 0x2f, 0x63, 0x6f, 0x69, 0x6c, 0x2f, + 0x76, 0x32, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x63, 0x6e, 0x69, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -375,29 +401,31 @@ func file_pkg_cnirpc_cni_proto_rawDescGZIP() []byte { } var file_pkg_cnirpc_cni_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_pkg_cnirpc_cni_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_pkg_cnirpc_cni_proto_msgTypes = make([]protoimpl.MessageInfo, 5) var file_pkg_cnirpc_cni_proto_goTypes = []any{ (ErrorCode)(0), // 0: pkg.cnirpc.ErrorCode (*CNIArgs)(nil), // 1: pkg.cnirpc.CNIArgs (*CNIError)(nil), // 2: pkg.cnirpc.CNIError (*AddResponse)(nil), // 3: pkg.cnirpc.AddResponse nil, // 4: pkg.cnirpc.CNIArgs.ArgsEntry - (*emptypb.Empty)(nil), // 5: google.protobuf.Empty + nil, // 5: pkg.cnirpc.CNIArgs.InterfacesEntry + (*emptypb.Empty)(nil), // 6: google.protobuf.Empty } var file_pkg_cnirpc_cni_proto_depIdxs = []int32{ 4, // 0: pkg.cnirpc.CNIArgs.args:type_name -> pkg.cnirpc.CNIArgs.ArgsEntry - 0, // 1: pkg.cnirpc.CNIError.code:type_name -> pkg.cnirpc.ErrorCode - 1, // 2: pkg.cnirpc.CNI.Add:input_type -> pkg.cnirpc.CNIArgs - 1, // 3: pkg.cnirpc.CNI.Del:input_type -> pkg.cnirpc.CNIArgs - 1, // 4: pkg.cnirpc.CNI.Check:input_type -> pkg.cnirpc.CNIArgs - 3, // 5: pkg.cnirpc.CNI.Add:output_type -> pkg.cnirpc.AddResponse - 5, // 6: pkg.cnirpc.CNI.Del:output_type -> google.protobuf.Empty - 5, // 7: pkg.cnirpc.CNI.Check:output_type -> google.protobuf.Empty - 5, // [5:8] is the sub-list for method output_type - 2, // [2:5] is the sub-list for method input_type - 2, // [2:2] is the sub-list for extension type_name - 2, // [2:2] is the sub-list for extension extendee - 0, // [0:2] is the sub-list for field type_name + 5, // 1: pkg.cnirpc.CNIArgs.interfaces:type_name -> pkg.cnirpc.CNIArgs.InterfacesEntry + 0, // 2: pkg.cnirpc.CNIError.code:type_name -> pkg.cnirpc.ErrorCode + 1, // 3: pkg.cnirpc.CNI.Add:input_type -> pkg.cnirpc.CNIArgs + 1, // 4: pkg.cnirpc.CNI.Del:input_type -> pkg.cnirpc.CNIArgs + 1, // 5: pkg.cnirpc.CNI.Check:input_type -> pkg.cnirpc.CNIArgs + 3, // 6: pkg.cnirpc.CNI.Add:output_type -> pkg.cnirpc.AddResponse + 6, // 7: pkg.cnirpc.CNI.Del:output_type -> google.protobuf.Empty + 6, // 8: pkg.cnirpc.CNI.Check:output_type -> google.protobuf.Empty + 6, // [6:9] is the sub-list for method output_type + 3, // [3:6] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name } func init() { file_pkg_cnirpc_cni_proto_init() } @@ -449,7 +477,7 @@ func file_pkg_cnirpc_cni_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_pkg_cnirpc_cni_proto_rawDesc, NumEnums: 1, - NumMessages: 4, + NumMessages: 5, NumExtensions: 0, NumServices: 1, }, diff --git a/v2/pkg/cnirpc/cni.proto b/v2/pkg/cnirpc/cni.proto index df327f62..87bf4ed7 100644 --- a/v2/pkg/cnirpc/cni.proto +++ b/v2/pkg/cnirpc/cni.proto @@ -14,6 +14,8 @@ message CNIArgs { map args = 4; // Key-Value pairs parsed from CNI_ARGS string path = 5; bytes stdin_data = 6; + repeated string ips = 7; + map interfaces = 8; } // ErrorCode enumerates errors for CNIError diff --git a/v2/pkg/constants/constants.go b/v2/pkg/constants/constants.go index 7b3ea614..acc28e87 100644 --- a/v2/pkg/constants/constants.go +++ b/v2/pkg/constants/constants.go @@ -57,6 +57,13 @@ const ( EnvEgressName = "COIL_EGRESS_NAME" ) +// Config flags + +const ( + EnableIPAM = "ENABLE_IPAM" + EnableEgress = "ENABLE_EGRESS" +) + // MetricsNS is the namespace for Prometheus metrics const MetricsNS = "coil" diff --git a/v2/pkg/nodenet/pod.go b/v2/pkg/nodenet/pod.go index 98382611..077e8704 100644 --- a/v2/pkg/nodenet/pod.go +++ b/v2/pkg/nodenet/pod.go @@ -21,6 +21,7 @@ import ( "github.com/go-logr/logr" "github.com/vishvananda/netlink" "golang.org/x/sys/unix" + corev1 "k8s.io/api/core/v1" ) var ( @@ -48,14 +49,19 @@ type PodNetwork interface { // Init initializes the host network. Init() error - // Setup connects the host network and the container network with a veth pair. + // SetupIPAM connects the host network and the container network with a veth pair. // `nsPath` is the container network namespace's (possibly bind-mounted) file. // If `hook` is non-nil, it is called in the Pod network. - Setup(nsPath, podName, podNS string, conf *PodNetConf, hook SetupHook) (*current.Result, error) + SetupIPAM(nsPath, podName, podNS string, conf *PodNetConf) (*current.Result, error) + + // SetupEgress configures egress for container. + // `nsPath` is the container network namespace's (possibly bind-mounted) file. + // If `hook` is non-nil, it is called in the Pod network. + SetupEgress(nsPath string, conf *PodNetConf, hook SetupHook) error // Update updates the container network configuration // Currently, it only updates configuration using a SetupHook, e.g. NAT setting - Update(podIPv4, podIPv6 net.IP, hook SetupHook) error + Update(podIPv4, podIPv6 net.IP, hook SetupHook, pod *corev1.Pod) error // Check checks the pod network's status. Check(containerId, iface string) error @@ -69,7 +75,7 @@ type PodNetwork interface { } // NewPodNetwork creates a PodNetwork -func NewPodNetwork(podTableID, podRulePrio, protocolId int, hostIPv4, hostIPv6 net.IP, compatCalico, registerFromMain bool, log logr.Logger) PodNetwork { +func NewPodNetwork(podTableID, podRulePrio, protocolId int, hostIPv4, hostIPv6 net.IP, compatCalico, registerFromMain bool, log logr.Logger, enableIPAM bool) PodNetwork { return &podNetwork{ podTableId: podTableID, podRulePrio: podRulePrio, @@ -79,6 +85,7 @@ func NewPodNetwork(podTableID, podRulePrio, protocolId int, hostIPv4, hostIPv6 n compatCalico: compatCalico, registerFromMain: registerFromMain, log: log, + enableIPAM: enableIPAM, } } @@ -92,12 +99,13 @@ type podNetwork struct { compatCalico bool registerFromMain bool log logr.Logger + enableIPAM bool mu sync.Mutex } -func genAlias(conf *PodNetConf) string { - return fmt.Sprintf("COIL:%s:%s:%s", conf.PoolName, conf.ContainerId, conf.IFace) +func GenAlias(conf *PodNetConf, id string) string { + return fmt.Sprintf("COIL:%s:%s:%s", conf.PoolName, id, conf.IFace) } func parseLink(l netlink.Link) *PodNetConf { @@ -187,10 +195,23 @@ func (pn *podNetwork) initRule(family int) error { return nil } -func (pn *podNetwork) Setup(nsPath, podName, podNS string, conf *PodNetConf, hook SetupHook) (*current.Result, error) { +func (pn *podNetwork) SetupIPAM(nsPath, podName, podNS string, conf *PodNetConf) (*current.Result, error) { pn.mu.Lock() defer pn.mu.Unlock() + containerNS, err := ns.GetNS(nsPath) + if err != nil { + return nil, fmt.Errorf("failed to open netns path %s: %w", nsPath, err) + } + defer containerNS.Close() + + var hostIPv6 net.IP + var hLink netlink.Link + + result := ¤t.Result{ + CNIVersion: current.ImplementedSpecVersion, + } + // cleanup garbage veth switch l, err := lookup(conf.ContainerId, conf.IFace); err { case errNotFound: @@ -203,16 +224,7 @@ func (pn *podNetwork) Setup(nsPath, podName, podNS string, conf *PodNetConf, hoo return nil, err } - containerNS, err := ns.GetNS(nsPath) - if err != nil { - return nil, fmt.Errorf("failed to open netns path %s: %w", nsPath, err) - } - defer containerNS.Close() - // setup veth and configure IP addresses - result := ¤t.Result{ - CNIVersion: current.ImplementedSpecVersion, - } err = containerNS.Do(func(hostNS ns.NetNS) error { vethName := "" if pn.compatCalico { @@ -287,7 +299,7 @@ func (pn *podNetwork) Setup(nsPath, podName, podNS string, conf *PodNetConf, hoo // install cleanup handler upon errors hName := result.Interfaces[1].Name - hLink, err := netlink.LinkByName(hName) + hLink, err = netlink.LinkByName(hName) if err != nil { return nil, fmt.Errorf("netlink: failed to look up the host-side veth: %w", err) } @@ -297,14 +309,13 @@ func (pn *podNetwork) Setup(nsPath, podName, podNS string, conf *PodNetConf, hoo } }() - // give identifer as an alias of host veth - err = netlink.LinkSetAlias(hLink, genAlias(conf)) + // give identifier as an alias of host veth + err = netlink.LinkSetAlias(hLink, GenAlias(conf, conf.ContainerId)) if err != nil { return nil, fmt.Errorf("netlink: failed to set alias: %w", err) } // setup routing on the host side - var hostIPv6 net.IP if conf.IPv6 != nil { ip.SettleAddresses(hName, 10) @@ -398,9 +409,6 @@ func (pn *podNetwork) Setup(nsPath, podName, podNS string, conf *PodNetConf, hoo } } - if hook != nil { - return hook(conf.IPv4, conf.IPv6) - } return nil }) if err != nil { @@ -411,7 +419,31 @@ func (pn *podNetwork) Setup(nsPath, podName, podNS string, conf *PodNetConf, hoo return result, nil } -func (pn *podNetwork) Update(podIPv4, podIPv6 net.IP, hook SetupHook) error { +func (pn *podNetwork) SetupEgress(nsPath string, conf *PodNetConf, hook SetupHook) error { + pn.mu.Lock() + defer pn.mu.Unlock() + + containerNS, err := ns.GetNS(nsPath) + if err != nil { + return fmt.Errorf("failed to open netns path %s: %w", nsPath, err) + } + defer containerNS.Close() + + // setup routing on the container side + err = containerNS.Do(func(ns.NetNS) error { + if hook != nil { + return hook(conf.IPv4, conf.IPv6) + } + return nil + }) + if err != nil { + return fmt.Errorf("error executing hook: %w", err) + } + + return nil +} + +func (pn *podNetwork) Update(podIPv4, podIPv6 net.IP, hook SetupHook, pod *corev1.Pod) error { pn.mu.Lock() defer pn.mu.Unlock() @@ -422,12 +454,25 @@ func (pn *podNetwork) Update(podIPv4, podIPv6 net.IP, hook SetupHook) error { var netNsPath string for _, c := range podConfigs { - // When both c.IPvX and podIPvX are nil, net.IP.Equal() returns always true. - // To avoid comparing nil to nil, confirm c.IPvX is not nil. - if (c.IPv4 != nil && c.IPv4.Equal(podIPv4)) || (c.IPv6 != nil && c.IPv6.Equal(podIPv6)) { - netNsPath, err = getNetNsPath(c.HostVethName) - if err != nil { - return err + if pn.enableIPAM { + // When both c.IPvX and podIPvX are nil, net.IP.Equal() returns always true. + // To avoid comparing nil to nil, confirm c.IPvX is not nil. + if (c.IPv4 != nil && c.IPv4.Equal(podIPv4)) || (c.IPv6 != nil && c.IPv6.Equal(podIPv6)) { + netNsPath, err = getNetNsPath(c.HostVethName) + if err != nil { + return err + } + } + } else { + // We need to bind PodNetConf with Pod. When Coil is a primary CNI IP address can be used for that. + // However different CNIs manage IP addresses differently, therefore, in egress-only mode instead of + // storing container's ID in ContainerId, we store pod's UID there to be able to identify pod that was + // used to create particular PodNetConf. + if c.ContainerId == string(pod.UID) { + netNsPath, err = getNetNsPath(c.HostVethName) + if err != nil { + return err + } } } } diff --git a/v2/pkg/nodenet/pod_test.go b/v2/pkg/nodenet/pod_test.go index 952a9dfb..21c0cfc1 100644 --- a/v2/pkg/nodenet/pod_test.go +++ b/v2/pkg/nodenet/pod_test.go @@ -63,7 +63,7 @@ func TestPodNetwork(t *testing.T) { } pn := NewPodNetwork(116, 2000, 30, net.ParseIP("10.20.30.41"), net.ParseIP("fd10::41"), - false, false, ctrl.Log.WithName("pod-network")) + false, false, ctrl.Log.WithName("pod-network"), true) if err := pn.Init(); err != nil { t.Fatal(err) } @@ -72,7 +72,11 @@ func TestPodNetwork(t *testing.T) { for name, conf := range podConfMap { - result, err := pn.Setup(nsPath(name), name, "ns1", &conf, func(ipv4, ipv6 net.IP) error { + result, err := pn.SetupIPAM(nsPath(name), name, "ns1", &conf) + if err != nil { + t.Fatal(err) + } + err = pn.SetupEgress(nsPath(name), &conf, func(ipv4, ipv6 net.IP) error { givenIPv4 = ipv4 givenIPv6 = ipv6 return nil @@ -210,7 +214,7 @@ func TestPodNetwork(t *testing.T) { givenIPv4 = ipv4 givenIPv6 = ipv6 return nil - }) + }, nil) if err != nil { t.Fatal(err) } @@ -261,7 +265,7 @@ func TestPodNetwork(t *testing.T) { givenIPv4 = ipv4 givenIPv6 = ipv6 return nil - }) + }, nil) if err != nil { t.Fatal(err) } @@ -295,7 +299,7 @@ func TestPodNetwork(t *testing.T) { givenIPv4 = ipv4 givenIPv6 = ipv6 return nil - }) + }, nil) if err != nil { t.Fatal(err) } @@ -350,7 +354,7 @@ func TestPodNetwork(t *testing.T) { return err } return nil - }) + }, nil) if err != nil { t.Error(err) } @@ -364,7 +368,7 @@ func TestPodNetwork(t *testing.T) { return err } return nil - }) + }, nil) if err != nil { t.Error(err) } diff --git a/v2/runners/coild_server.go b/v2/runners/coild_server.go index 4792c5de..1923ea8f 100644 --- a/v2/runners/coild_server.go +++ b/v2/runners/coild_server.go @@ -6,8 +6,10 @@ import ( "errors" "fmt" "net" + "strconv" "strings" + current "github.com/containernetworking/cni/pkg/types/100" coilv2 "github.com/cybozu-go/coil/v2/api/v2" "github.com/cybozu-go/coil/v2/pkg/cnirpc" "github.com/cybozu-go/coil/v2/pkg/constants" @@ -17,6 +19,7 @@ import ( "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors" "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging" grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + "github.com/vishvananda/netlink" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -189,54 +192,91 @@ func (s *coildServer) Add(ctx context.Context, args *cnirpc.CNIArgs) (*cnirpc.Ad return nil, newInternalError(err, "failed to get pod") } - // fetch namespace to decide the pool name - ns := &corev1.Namespace{} - if err := s.client.Get(ctx, client.ObjectKey{Name: podNS}, ns); err != nil { - logger.Sugar().Errorw("failed to get namespace", "name", podNS, "error", err) - return nil, newInternalError(err, "failed to get namespace") - } - poolName := constants.DefaultPool - if v, ok := ns.Annotations[constants.AnnPool]; ok { - poolName = v + ipamEnabled, egressEnabled, err := getSettings(args) + if err != nil { + return nil, newInternalError(err, "error parsing settings") } - ipv4, ipv6, err := s.nodeIPAM.Allocate(ctx, poolName, args.ContainerId, args.Ifname) - if err != nil { - logger.Sugar().Errorw("failed to allocate address", "error", err) - return nil, newInternalError(err, "failed to allocate address") + if !ipamEnabled && !egressEnabled { + return nil, newInternalError(fmt.Errorf("configuration error"), "both ipam and egress are disabled") } - hook, err := s.getHook(ctx, pod) - if err != nil { - logger.Sugar().Errorw("failed to setup NAT hook", "error", err) - return nil, newInternalError(err, "failed to setup NAT hook") + var ipv4, ipv6 net.IP + var poolName string + + if ipamEnabled { + ns := &corev1.Namespace{} + if err := s.client.Get(ctx, client.ObjectKey{Name: podNS}, ns); err != nil { + logger.Sugar().Errorw("failed to get namespace", "name", podNS, "error", err) + return nil, newInternalError(err, "failed to get namespace") + } + poolName = constants.DefaultPool + if v, ok := ns.Annotations[constants.AnnPool]; ok { + poolName = v + } + + ipv4, ipv6, err = s.nodeIPAM.Allocate(ctx, poolName, args.ContainerId, args.Ifname) + if err != nil { + logger.Sugar().Errorw("failed to allocate address", "error", err) + return nil, newInternalError(err, "failed to allocate address") + } + } else { + ipv4, ipv6 = getPodIPs(args.Ips) } - if hook != nil { - logger.Sugar().Info("enabling NAT") + + result := ¤t.Result{ + CNIVersion: current.ImplementedSpecVersion, } - result, err := s.podNet.Setup(args.Netns, podName, podNS, &nodenet.PodNetConf{ + config := &nodenet.PodNetConf{ ContainerId: args.ContainerId, IFace: args.Ifname, IPv4: ipv4, IPv6: ipv6, PoolName: poolName, - }, hook) - if err != nil { - if err := s.nodeIPAM.Free(ctx, args.ContainerId, args.Ifname); err != nil { - logger.Sugar().Warnw("failed to deallocate address", "error", err) + } + + if ipamEnabled { + result, err = s.podNet.SetupIPAM(args.Netns, podName, podNS, config) + if err != nil { + if err := s.nodeIPAM.Free(ctx, args.ContainerId, args.Ifname); err != nil { + logger.Sugar().Warnw("failed to deallocate address", "error", err) + } + logger.Sugar().Errorw("failed to setup pod network", "error", err) + return nil, newInternalError(err, "failed to setup pod network IPAM") + } + } + + if egressEnabled { + if !ipamEnabled { + if err := setCoilInterfaceAlias(args.Interfaces, config, logger, pod); err != nil { + return nil, newInternalError(err, "failed to set interface alias") + } + } + + hook, err := s.getHook(ctx, pod) + if err != nil { + logger.Sugar().Errorw("failed to setup NAT hook", "error", err) + return nil, newInternalError(err, "failed to setup NAT hook") + } + + if hook != nil { + logger.Sugar().Info("enabling NAT") + if err := s.podNet.SetupEgress(args.Netns, config, hook); err != nil { + return nil, newInternalError(err, "failed to setup pod network egress") + } } - logger.Sugar().Errorw("failed to setup pod network", "error", err) - return nil, newInternalError(err, "failed to setup pod network") } data, err := json.Marshal(result) if err != nil { - if err := s.podNet.Destroy(args.ContainerId, args.Ifname); err != nil { - logger.Sugar().Warnw("failed to destroy pod network", "error", err) - } - if err := s.nodeIPAM.Free(ctx, args.ContainerId, args.Ifname); err != nil { - logger.Sugar().Warnw("failed to deallocate address", "error", err) + if ipamEnabled { + if err := s.podNet.Destroy(args.ContainerId, args.Ifname); err != nil { + logger.Sugar().Warnw("failed to destroy pod network", "error", err) + } + if err := s.nodeIPAM.Free(ctx, args.ContainerId, args.Ifname); err != nil { + logger.Sugar().Warnw("failed to deallocate address", "error", err) + } } logger.Sugar().Errorw("failed to marshal the result", "error", err) return nil, newInternalError(err, "failed to marshal the result") @@ -244,17 +284,64 @@ func (s *coildServer) Add(ctx context.Context, args *cnirpc.CNIArgs) (*cnirpc.Ad return &cnirpc.AddResponse{Result: data}, nil } +func setCoilInterfaceAlias(interfaces map[string]bool, conf *nodenet.PodNetConf, logger *zap.Logger, pod *corev1.Pod) error { + ifName := "" + for name, isSandbox := range interfaces { + if !isSandbox { + ifName = name + break + } + } + logger.Sugar().Infof("interface selected: %s", ifName) + hLink, err := netlink.LinkByName(ifName) + if err != nil { + return fmt.Errorf("netlink: failed to look up the host-side veth: %w", err) + } + logger.Sugar().Infof("link found: %v", hLink) + + // give identifier as an alias of host veth + if err := netlink.LinkSetAlias(hLink, nodenet.GenAlias(conf, string(pod.UID))); err != nil { + return fmt.Errorf("netlink: failed to set alias: %w", err) + } + return nil +} + +func getPodIPs(ips []string) (net.IP, net.IP) { + var ipv4, ipv6 net.IP + for _, ip := range ips { + addr := net.ParseIP(ip) + if addr != nil { + if ipv4 == nil && addr.To4() != nil { + ipv4 = addr + } else if ipv6 == nil { + ipv6 = addr + } + } + if ipv4 != nil && ipv6 != nil { + break + } + } + return ipv4, ipv6 +} + func (s *coildServer) Del(ctx context.Context, args *cnirpc.CNIArgs) (*emptypb.Empty, error) { logger := withCtxFields(ctx, s.logger) - if err := s.podNet.Destroy(args.ContainerId, args.Ifname); err != nil { - logger.Sugar().Errorw("failed to destroy pod network", "error", err) - return nil, newInternalError(err, "failed to destroy pod network") + ipamEnabled, _, err := getSettings(args) + if err != nil { + return nil, newInternalError(err, "error parsing settings") } - if err := s.nodeIPAM.Free(ctx, args.ContainerId, args.Ifname); err != nil { - logger.Sugar().Errorw("failed to free addresses", "error", err) - return nil, newInternalError(err, "failed to free addresses") + if ipamEnabled { + if err := s.podNet.Destroy(args.ContainerId, args.Ifname); err != nil { + logger.Sugar().Errorw("failed to destroy pod network", "error", err) + return nil, newInternalError(err, "failed to destroy pod network") + } + + if err := s.nodeIPAM.Free(ctx, args.ContainerId, args.Ifname); err != nil { + logger.Sugar().Errorw("failed to free addresses", "error", err) + return nil, newInternalError(err, "failed to free addresses") + } } return &emptypb.Empty{}, nil } @@ -262,10 +349,18 @@ func (s *coildServer) Del(ctx context.Context, args *cnirpc.CNIArgs) (*emptypb.E func (s *coildServer) Check(ctx context.Context, args *cnirpc.CNIArgs) (*emptypb.Empty, error) { logger := withCtxFields(ctx, s.logger) - if err := s.podNet.Check(args.ContainerId, args.Ifname); err != nil { - logger.Sugar().Errorw("check failed", "error", err) + ipamEnabled, _, err := getSettings(args) + if err != nil { return nil, newInternalError(err, "check failed") } + + if ipamEnabled { + if err := s.podNet.Check(args.ContainerId, args.Ifname); err != nil { + logger.Sugar().Errorw("check failed", "error", err) + return nil, newInternalError(err, "check failed") + } + } + return &emptypb.Empty{}, nil } @@ -346,6 +441,7 @@ func (s *coildServer) getHook(ctx context.Context, pod *corev1.Pod) (nodenet.Set if len(gwlist) > 0 { logger = logger.With(zap.String("pod_name", pod.Name), zap.String("pod_namespace", pod.Namespace)) + logger.Sugar().Infof("gwlist: %v", gwlist) return s.natSetup.Hook(gwlist, logger), nil } return nil, nil @@ -418,3 +514,23 @@ func loggingFields(_ context.Context, c interceptors.CallMeta) logging.Fields { func withCtxFields(ctx context.Context, l *zap.Logger) *zap.Logger { return l.With(toZapFields(logging.ExtractFields(ctx))...) } + +func getSettings(args *cnirpc.CNIArgs) (bool, bool, error) { + ipamEnabled := true + egressEnabled := true + + var err error + if args.Args[constants.EnableIPAM] != "" { + if ipamEnabled, err = strconv.ParseBool(args.Args[constants.EnableIPAM]); err != nil { + return false, false, newInternalError(err, "error parsing bool value for IPAM enable flag") + } + } + + if args.Args[constants.EnableEgress] != "" { + if egressEnabled, err = strconv.ParseBool(args.Args[constants.EnableEgress]); err != nil { + return false, false, newInternalError(err, "error parsing bool value for Egress enable flag") + } + } + + return ipamEnabled, egressEnabled, nil +} diff --git a/v2/runners/coild_server_test.go b/v2/runners/coild_server_test.go index 7209a08a..41e264cf 100644 --- a/v2/runners/coild_server_test.go +++ b/v2/runners/coild_server_test.go @@ -91,7 +91,7 @@ func (p *mockPodNetwork) List() ([]*nodenet.PodNetConf, error) { panic("not implemented") } -func (p *mockPodNetwork) Setup(nsPath, podName, podNS string, conf *nodenet.PodNetConf, hook nodenet.SetupHook) (*current.Result, error) { +func (p *mockPodNetwork) SetupIPAM(nsPath, podName, podNS string, conf *nodenet.PodNetConf) (*current.Result, error) { p.nSetup++ if p.errSetup { return nil, errors.New("setup failure") @@ -107,15 +107,14 @@ func (p *mockPodNetwork) Setup(nsPath, podName, podNS string, conf *nodenet.PodN Address: *netlink.NewIPNet(conf.IPv6), }) } - if hook != nil { - if err := hook(conf.IPv4, conf.IPv6); err != nil { - return nil, err - } - } return ¤t.Result{IPs: ips}, nil } -func (p *mockPodNetwork) Update(podIPv4, podIPv6 net.IP, hook nodenet.SetupHook) error { +func (p *mockPodNetwork) SetupEgress(nsPath string, conf *nodenet.PodNetConf, hook nodenet.SetupHook) error { + return nil +} + +func (p *mockPodNetwork) Update(podIPv4, podIPv6 net.IP, hook nodenet.SetupHook, pod *corev1.Pod) error { panic("not implemented") }