diff --git a/salt/kubernetes/README.md b/salt/kubernetes/README.md index 3b347183..9624d0a4 100644 --- a/salt/kubernetes/README.md +++ b/salt/kubernetes/README.md @@ -7,7 +7,7 @@ Deploys and configures the Kubernetes Nodes. - [`kubernetes.helm`](#kuberneteshelm) ## Usage -Some prerequisites must be met first: +Some prerequisites must be met first (pre 3005 version): - _Salt Minion_ config must contain: ``` use_superseded: @@ -46,7 +46,7 @@ x509_signing_policies: - authorityKeyIdentifier: keyid,issuer:always - days_valid: 365 ``` -2. `salt-run state.orchestrate kubernetes._orchestrate.cluster saltenv=server pillar='{"kubernetes": {"nodes": {"masters": [k8s1], "workers": [k8s2, k8s3]}}}'` +2. `salt-run state.orchestrate kubernetes._orchestrate.cluster saltenv=base pillar='{"kubernetes": {"nodes": {"masters": [k8s1], "workers": [k8s2, k8s3]}}}'` ### `kubernetes.master` Setup Kubernetes master node diff --git a/salt/kubernetes/distro/k3s/_install.macros.jinja b/salt/kubernetes/distro/k3s/_install.macros.jinja new file mode 100644 index 00000000..b80cbccf --- /dev/null +++ b/salt/kubernetes/distro/k3s/_install.macros.jinja @@ -0,0 +1,16 @@ +{% macro k3s_install(installer_url, envs_list) %} +k3s: + cmd.script: + - name: {{ installer_url }} + - env: {{ envs_list | tojson }} + - require: + - file: k3s_config + +k3s-running: + service.running: + - name: {{ k3s.config.unit_name }} + - enable: True + - require: + - cmd: k3s + +{% endmacro %} diff --git a/salt/kubernetes/distro/k3s/config.sls b/salt/kubernetes/distro/k3s/config.sls new file mode 100644 index 00000000..edc69561 --- /dev/null +++ b/salt/kubernetes/distro/k3s/config.sls @@ -0,0 +1,13 @@ +{%- from "kubernetes/distro/k3s/map.jinja" import k3s with context %} + +# rename? move to jinja? +k3s_config: + file.managed: + - name: {{ k3s.distro_config.installer_file }} + - contents: {{ k3s.distro_config.installer_config|yaml_encode }} + - makedirs: True + - replace: False + - user: {{ k3s.user }} + - group: {{ k3s.group|default(k3s.user) }} + - require: + - service: docker diff --git a/salt/kubernetes/distro/k3s/map.jinja b/salt/kubernetes/distro/k3s/map.jinja new file mode 100644 index 00000000..6f12190f --- /dev/null +++ b/salt/kubernetes/distro/k3s/map.jinja @@ -0,0 +1,18 @@ +{%- from "kubernetes/map.jinja" import kubernetes as kubernetes_defaults with context %} + +{% set installer_config_contents = '' %} +{% set k3s = salt['grains.filter_by']({ + 'default': { + 'distro_config': { + 'installer_file': "/etc/rancher/k3s/config.yaml", + 'installer_config': installer_config_contents, + 'installer_url': "https://get.k3s.io", + 'env': [], + 'token_file': "/var/lib/rancher/k3s/server/node-token" + }, + 'config': { + 'locations': ["/etc/rancher/k3s/k3s.yaml"], + 'unit_name': "k3s-agent", + } + }, +}, merge=kubernetes_defaults) %} diff --git a/salt/kubernetes/distro/k3s/master.sls b/salt/kubernetes/distro/k3s/master.sls new file mode 100644 index 00000000..37dc84ef --- /dev/null +++ b/salt/kubernetes/distro/k3s/master.sls @@ -0,0 +1,17 @@ +{%- from "kubernetes/distro/k3s/map.jinja" import k3s with context %} +{%- from "kubernetes/distro/k3s/_install.macros.jinja" import k3s_install with context %} + +include: + - kubernetes.distro.requisites + - kubernetes.distro.k3s.config + +{{ k3s_install(k3s.distro_config.installer_url, k3s.distro_config.env) }} + +propagate_token: + module.run: + - mine.send: + - kubernetes_token + - mine_function: file.read + - {{ k3s.distro_config.token_file }} + - require: + - cmd: k3s diff --git a/salt/kubernetes/distro/k3s/worker.sls b/salt/kubernetes/distro/k3s/worker.sls new file mode 100644 index 00000000..ee20f105 --- /dev/null +++ b/salt/kubernetes/distro/k3s/worker.sls @@ -0,0 +1,15 @@ +{%- from "kubernetes/distro/k3s/map.jinja" import k3s with context %} +{%- from "kubernetes/distro/k3s/_install.macros.jinja" import k3s_install with context %} + +{%- set masters = k3s.nodes.masters %} +{%- set tokens = salt['mine.get'](masters|first, "kubernetes_token") %} +{%- set envs = k3s.distro_config.env %} +{%- do envs.append({'K3S_TOKEN': tokens[masters|first] | regex_replace('\n','') })%} +# this is file content thus contains new line, which breaks agent join + +include: + - kubernetes.distro.requisites + - kubernetes.distro.k3s.config + +{{ k3s_install(k3s.distro_config.installer_url, envs) }} + diff --git a/salt/kubernetes/distro/kubeadm/init.sls b/salt/kubernetes/distro/kubeadm/init.sls new file mode 100644 index 00000000..3ff2b250 --- /dev/null +++ b/salt/kubernetes/distro/kubeadm/init.sls @@ -0,0 +1,3 @@ +include: + - kubernetes.distro.requisites + - kubernetes.distro.kubeadm.install diff --git a/salt/kubernetes/kubeadm/install.sls b/salt/kubernetes/distro/kubeadm/install.sls similarity index 90% rename from salt/kubernetes/kubeadm/install.sls rename to salt/kubernetes/distro/kubeadm/install.sls index a3607994..cff79421 100644 --- a/salt/kubernetes/kubeadm/install.sls +++ b/salt/kubernetes/distro/kubeadm/install.sls @@ -10,3 +10,4 @@ kubeadm: - require: - pkgrepo_ext: kube_repository - service: docker + - sls: kubernetes.distro.requisites diff --git a/salt/kubernetes/distro/kubeadm/master.sls b/salt/kubernetes/distro/kubeadm/master.sls new file mode 100644 index 00000000..6de22a6f --- /dev/null +++ b/salt/kubernetes/distro/kubeadm/master.sls @@ -0,0 +1,25 @@ +ensure_token: + module.run: + - kubeadm.token_create: [] + - unless: + - fun: kubeadm.token_list + - require: + - cmd: kubeadm_init + +propagate_token: + module.run: + - mine.send: + - kubernetes_token + - mine_function: kubeadm.token_list + - require: + - module: ensure_token + +propagate_hash: + module.run: + - mine.send: + - kubernetes_hash + - mine_function: cmd.run + - "openssl x509 -pubkey -in {{ kubernetes.master.pki.dir }}/{{ kubernetes.master.ca.pub }} | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'" + - python_shell: True + - require: + - cmd: kubeadm_init diff --git a/salt/kubernetes/master/setup/ca.sls b/salt/kubernetes/distro/kubeadm/setup/ca.sls similarity index 100% rename from salt/kubernetes/master/setup/ca.sls rename to salt/kubernetes/distro/kubeadm/setup/ca.sls diff --git a/salt/kubernetes/master/setup/first.sls b/salt/kubernetes/distro/kubeadm/setup/first.sls similarity index 100% rename from salt/kubernetes/master/setup/first.sls rename to salt/kubernetes/distro/kubeadm/setup/first.sls diff --git a/salt/kubernetes/master/setup/init.sls b/salt/kubernetes/distro/kubeadm/setup/init.sls similarity index 64% rename from salt/kubernetes/master/setup/init.sls rename to salt/kubernetes/distro/kubeadm/setup/init.sls index 2c7b921c..983fc9c2 100644 --- a/salt/kubernetes/master/setup/init.sls +++ b/salt/kubernetes/distro/kubeadm/setup/init.sls @@ -9,7 +9,7 @@ kubeadm_master_reset: - require: - pkg: kubeadm - require_in: - - cmd: kubeadm_init + - sls: kubernetes.distro.kubeadm {%- endif -%} {%- if grains['id'] == masters|first %} @@ -28,7 +28,7 @@ allow_schedule_on_master: - env: - KUBECONFIG: {{ kubernetes.config.locations|join(':') }} - require: - - cmd: kubeadm_init + - sls: kubernetes.distro.{{ kubernetes.distro }} # todo else -> taint the node {%- endif %} @@ -40,7 +40,7 @@ propagate_cert_key: - mine_function: grains.get - "kubernetes:master:certificate_key" - require: - - cmd: kubeadm_init + - sls: kubernetes.distro.{{ kubernetes.distro }} propagate_ip: module.run: - mine.send: @@ -48,7 +48,7 @@ propagate_ip: - mine_function: network.ip_addrs - cidr: {{ kubernetes_network.nodes.master_vip }} - require: - - cmd: kubeadm_init + - sls: kubernetes.distro.{{ kubernetes.distro }} {%- else %} propagate_ip: module.run: @@ -57,33 +57,7 @@ propagate_ip: - mine_function: network.ip_addrs - cidr: {{ kubernetes_network.nodes.cidr }} - require: - - cmd: kubeadm_init + - sls: kubernetes.distro.{{ kubernetes.distro }} {%- endif %} -ensure_token: - module.run: - - kubeadm.token_create: [] - - unless: - - fun: kubeadm.token_list - - require: - - cmd: kubeadm_init -propagate_token: - module.run: - - mine.send: - - kubernetes_token - - mine_function: kubeadm.token_list - - require: - - module: ensure_token - -propagate_hash: - module.run: - - mine.send: - - kubernetes_hash - - mine_function: cmd.run - - "openssl x509 -pubkey -in {{ kubernetes.master.pki.dir }}/{{ kubernetes.master.ca.pub }} | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'" - - python_shell: True - - require: - - cmd: kubeadm_init - - #todo the cmd.run should be wrapped with script and return stateful data diff --git a/salt/kubernetes/master/setup/other.sls b/salt/kubernetes/distro/kubeadm/setup/other.sls similarity index 100% rename from salt/kubernetes/master/setup/other.sls rename to salt/kubernetes/distro/kubeadm/setup/other.sls diff --git a/salt/kubernetes/worker/setup.sls b/salt/kubernetes/distro/kubeadm/worker.sls similarity index 86% rename from salt/kubernetes/worker/setup.sls rename to salt/kubernetes/distro/kubeadm/worker.sls index 422d129e..d5b04f53 100644 --- a/salt/kubernetes/worker/setup.sls +++ b/salt/kubernetes/distro/kubeadm/worker.sls @@ -1,6 +1,3 @@ -{%- from "kubernetes/worker/map.jinja" import kubernetes with context %} -{%- from "kubernetes/network/map.jinja" import kubernetes as kubernetes_network with context %} - #load modules ip_vs, ip_vs_rr, ip_vs_wrr, ip_vs_sh, nf_conntrack_ipv4 {%- set masters = kubernetes.nodes.masters %} {%- set main_master_id = kubernetes.nodes.masters|first %} @@ -8,6 +5,10 @@ {%- set ips = salt['mine.get'](masters|join(","), "kubernetes_master_ip", tgt_type="list") %} {%- set hashes = salt['mine.get'](masters|join(","), "kubernetes_hash", tgt_type="list") -%} + +include: + - kubernetes.distro.kubeadm + {%- if ips and tokens and hashes %} {%- if kubernetes.worker.reset %} kubeadm_worker_reset: @@ -18,7 +19,7 @@ kubeadm_worker_reset: - require_in: - cmd: join_master {%- endif %} -join_master: +join_master: # fixme from 3001 there is a module for this cmd.run: - name: "kubeadm join {{ ips[main_master_id][0] }}:{{ kubernetes_network.nodes.apiserver_port }} --token {{ tokens[main_master_id]|selectattr('usages', 'match', '.*authentication.*')|map(attribute="token")|first }} --discovery-token-ca-cert-hash sha256:{{ hashes[main_master_id] }}" - require: diff --git a/salt/kubernetes/minikube/init.sls b/salt/kubernetes/distro/minikube/init.sls similarity index 100% rename from salt/kubernetes/minikube/init.sls rename to salt/kubernetes/distro/minikube/init.sls diff --git a/salt/kubernetes/minikube/kvm2.sls b/salt/kubernetes/distro/minikube/kvm2.sls similarity index 100% rename from salt/kubernetes/minikube/kvm2.sls rename to salt/kubernetes/distro/minikube/kvm2.sls diff --git a/salt/kubernetes/minikube/map.jinja b/salt/kubernetes/distro/minikube/map.jinja similarity index 100% rename from salt/kubernetes/minikube/map.jinja rename to salt/kubernetes/distro/minikube/map.jinja diff --git a/salt/kubernetes/minikube/minikube_bin.sls b/salt/kubernetes/distro/minikube/minikube_bin.sls similarity index 100% rename from salt/kubernetes/minikube/minikube_bin.sls rename to salt/kubernetes/distro/minikube/minikube_bin.sls diff --git a/salt/kubernetes/minikube/minikube_setup.sls b/salt/kubernetes/distro/minikube/minikube_setup.sls similarity index 100% rename from salt/kubernetes/minikube/minikube_setup.sls rename to salt/kubernetes/distro/minikube/minikube_setup.sls diff --git a/salt/kubernetes/minikube/none.sls b/salt/kubernetes/distro/minikube/none.sls similarity index 100% rename from salt/kubernetes/minikube/none.sls rename to salt/kubernetes/distro/minikube/none.sls diff --git a/salt/kubernetes/minikube/pillar.example.sls b/salt/kubernetes/distro/minikube/pillar.example.sls similarity index 100% rename from salt/kubernetes/minikube/pillar.example.sls rename to salt/kubernetes/distro/minikube/pillar.example.sls diff --git a/salt/kubernetes/kubeadm/requisites.sls b/salt/kubernetes/distro/requisites.sls similarity index 69% rename from salt/kubernetes/kubeadm/requisites.sls rename to salt/kubernetes/distro/requisites.sls index 67f16d4d..a97aae61 100644 --- a/salt/kubernetes/kubeadm/requisites.sls +++ b/salt/kubernetes/distro/requisites.sls @@ -1,19 +1,15 @@ #!py - def run(): states = {} swaps = __salt__['mount.swaps']() # immediately disable currently mounted swap for dev, details in swaps.items(): - states["kubeadm_disable_swap_{}".format(dev)] = { + states["kubernetes_disable_swap_{}".format(dev)] = { 'module.run': [ {'mount.swapoff': [ {'name': dev}, - ]}, - {'require_in': [ - {'pkg': "kubeadm"} ]} ] } @@ -22,14 +18,11 @@ def run(): entries = __salt__['mount.fstab']() for name, details in entries.items(): if details['fstype'] == 'swap': - states["kubeadm_remove_swap_{}".format(name)] = { + states["kubernetes_remove_swap_{}".format(name)] = { 'module.run': [ {'mount.rm_fstab': [ {'name': name}, {'device': details['device']}, - ]}, - {'require_in': [ - {'pkg': "kubeadm"} ]} ] } diff --git a/salt/kubernetes/kubeadm/init.sls b/salt/kubernetes/kubeadm/init.sls deleted file mode 100644 index 46b00844..00000000 --- a/salt/kubernetes/kubeadm/init.sls +++ /dev/null @@ -1,3 +0,0 @@ -include: - - kubernetes.kubeadm.requisites - - kubernetes.kubeadm.install diff --git a/salt/kubernetes/map.jinja b/salt/kubernetes/map.jinja index ef843c42..b285d45b 100644 --- a/salt/kubernetes/map.jinja +++ b/salt/kubernetes/map.jinja @@ -1,35 +1,24 @@ +{% set version = "1.29" %} {% set kubernetes = salt['grains.filter_by']({ - 'RedHat': { - 'baseurl': "https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64", - 'repo_id': "kubernetes", - 'gpgkey': "https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg", - 'pkgs': ["kubelet", "kubeadm", "kubectl"], - 'config': { - 'locations': ["/etc/kubernetes/admin.conf"], - }, - 'user': "root", - 'nodes': { - 'masters': [], - 'workers': [] - } - }, - 'Debian': { + 'default': { 'names': [ - "deb http://apt.kubernetes.io/ kubernetes-xenial main" + "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v" ~ version ~ "/deb/ /" ], 'file': '/etc/apt/sources.list.d/kubernetes.list', - 'key_url': "https://packages.cloud.google.com/apt/doc/apt-key.gpg", + 'key_url': "https://pkgs.k8s.io/core:/stable:/v" ~ version ~ "/deb/Release.key", 'pkgs': ["kubelet", "kubeadm", "kubectl"], 'config': { 'locations': ["/etc/kubernetes/admin.conf"], }, 'user': "root", + "distro": "kubeadm", + 'version': version, 'nodes': { 'masters': [], 'workers': [] + }, + 'container': { + 'runtime': "docker" } - }, - 'Windows': { - - }, + } }, merge=salt['pillar.get']('kubernetes')) %} diff --git a/salt/kubernetes/master/init.sls b/salt/kubernetes/master/init.sls index bc8036b0..eadbf53b 100644 --- a/salt/kubernetes/master/init.sls +++ b/salt/kubernetes/master/init.sls @@ -5,7 +5,11 @@ {%- endif %} include: - - docker - - kubernetes.kubeadm - - kubernetes.master.setup + - {{ kubernetes.container.runtime }} +{% if kubernetes.distro == "kubeadm" %} + - kubernetes.distro.kubeadm.master + - kubernetes.master.kubeadm.setup # kubeadm specific, refactor to be used like worker +{% elif kubernetes.distro == "k3s" %} + - kubernetes.distro.k3s.master +{% endif %} - kubernetes.network diff --git a/salt/kubernetes/network/cilium.sls b/salt/kubernetes/network/cilium.sls new file mode 100644 index 00000000..bf7a87e3 --- /dev/null +++ b/salt/kubernetes/network/cilium.sls @@ -0,0 +1,19 @@ +{%- from "kubernetes/network/map.jinja" import kubernetes with context %} +{%- from "_common/util.jinja" import retry with context %} + +kubernetes_network: + archive.extracted: + - name: {{ kubernetes.network.config.extract }} + - source: {{ kubernetes.network.config.source }} + - skip_verify: True + - enforce_toplevel: False + - clean_parent: True + cmd.run: + - name: "{{ kubernetes.network.config.extract }}/cilium install" + - env: + - KUBECONFIG: {{ kubernetes.config.locations|join(':') }} + - require: + - archive: kubernetes_network + + +# add this bpffs mount \ No newline at end of file diff --git a/salt/kubernetes/network/map.jinja b/salt/kubernetes/network/map.jinja index a214310c..af4f649d 100644 --- a/salt/kubernetes/network/map.jinja +++ b/salt/kubernetes/network/map.jinja @@ -1,7 +1,12 @@ {%- from "kubernetes/map.jinja" import kubernetes as kubernetes_defaults with context %} # fixme delete flannel, add kube-router, calico and weave-net -{%- set default_network_provider = "flannel" %} +# https://github.com/cilium/cilium-cli/releases/tag/ +{%- set default_network_provider = "cilium" %} {%- set network_provider = { + 'cilium': { + 'source': "https://github.com/cilium/cilium-cli/releases/download/v0.15.20/cilium-linux-amd64.tar.gz", + 'extract': "/usr/local/bin/" + }, 'flannel' : { 'cidr': "10.244.0.0/16", 'source': "https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml" diff --git a/salt/kubernetes/worker/init.sls b/salt/kubernetes/worker/init.sls index 4f580975..ad34088b 100644 --- a/salt/kubernetes/worker/init.sls +++ b/salt/kubernetes/worker/init.sls @@ -1,5 +1,6 @@ +{%- from "kubernetes/worker/map.jinja" import kubernetes with context %} +{%- from "kubernetes/network/map.jinja" import kubernetes as kubernetes_network with context %} + include: - - docker - - kubernetes.kubeadm - - kubernetes.worker.setup - \ No newline at end of file + - {{ kubernetes.container.runtime }} + - kubernetes.distro.{{kubernetes.distro}}.worker