k3s setup

possibly broke the kubeadm but focus on k3s now

salt/kubernetes/README.md

@@ -7,7 +7,7 @@ Deploys and configures the Kubernetes Nodes.
  - [`kubernetes.helm`](#kuberneteshelm)
 
 ## Usage
-Some prerequisites must be met first:
+Some prerequisites must be met first (pre 3005 version):
  - _Salt Minion_ config must contain:
 ```
 use_superseded:
@@ -46,7 +46,7 @@ x509_signing_policies:
     - authorityKeyIdentifier: keyid,issuer:always
     - days_valid: 365
 ```
-2. `salt-run state.orchestrate kubernetes._orchestrate.cluster saltenv=server pillar='{"kubernetes": {"nodes": {"masters": [k8s1], "workers": [k8s2, k8s3]}}}'`
+2. `salt-run state.orchestrate kubernetes._orchestrate.cluster saltenv=base pillar='{"kubernetes": {"nodes": {"masters": [k8s1], "workers": [k8s2, k8s3]}}}'`
 
 ### `kubernetes.master`
 Setup Kubernetes master node

salt/kubernetes/distro/k3s/_install.macros.jinja

@@ -0,0 +1,16 @@
+{% macro k3s_install(installer_url, envs_list) %}
+k3s:
+  cmd.script:
+    - name: {{ installer_url }}
+    - env: {{ envs_list | tojson }}
+    - require:
+      - file: k3s_config
+
+k3s-running:
+  service.running:
+    - name: {{ k3s.config.unit_name }}
+    - enable: True
+    - require:
+      - cmd: k3s
+
+{% endmacro %}

salt/kubernetes/distro/k3s/config.sls

@@ -0,0 +1,13 @@
+{%- from "kubernetes/distro/k3s/map.jinja" import k3s with context %}
+
+# rename? move to jinja?
+k3s_config:
+  file.managed:
+    - name: {{ k3s.distro_config.installer_file }}
+    - contents: {{ k3s.distro_config.installer_config|yaml_encode }}
+    - makedirs: True
+    - replace: False
+    - user: {{ k3s.user }}
+    - group: {{ k3s.group|default(k3s.user) }}
+    - require:
+      - service: docker

salt/kubernetes/distro/k3s/map.jinja

@@ -0,0 +1,18 @@
+{%- from "kubernetes/map.jinja" import kubernetes as kubernetes_defaults with context %}
+
+{% set installer_config_contents = '' %}
+{% set k3s = salt['grains.filter_by']({
+  'default': {
+    'distro_config': {
+        'installer_file': "/etc/rancher/k3s/config.yaml",
+        'installer_config': installer_config_contents,
+        'installer_url': "https://get.k3s.io",
+        'env': [],
+        'token_file': "/var/lib/rancher/k3s/server/node-token"
+    },
+    'config': {
+        'locations': ["/etc/rancher/k3s/k3s.yaml"],
+        'unit_name': "k3s-agent",
+    }
+  },
+}, merge=kubernetes_defaults) %}

salt/kubernetes/distro/k3s/master.sls

@@ -0,0 +1,17 @@
+{%- from "kubernetes/distro/k3s/map.jinja" import k3s with context %}
+{%- from "kubernetes/distro/k3s/_install.macros.jinja" import k3s_install with context %}
+
+include:
+  - kubernetes.distro.requisites
+  - kubernetes.distro.k3s.config
+
+{{ k3s_install(k3s.distro_config.installer_url, k3s.distro_config.env) }}
+
+propagate_token:
+  module.run:
+    - mine.send:
+        - kubernetes_token
+        - mine_function: file.read
+        - {{ k3s.distro_config.token_file }}
+    - require:
+      - cmd: k3s

salt/kubernetes/distro/k3s/worker.sls

@@ -0,0 +1,15 @@
+{%- from "kubernetes/distro/k3s/map.jinja" import k3s with context %}
+{%- from "kubernetes/distro/k3s/_install.macros.jinja" import k3s_install with context %}
+
+{%- set masters = k3s.nodes.masters %}
+{%- set tokens = salt['mine.get'](masters|first, "kubernetes_token") %}
+{%- set envs = k3s.distro_config.env %}
+{%- do envs.append({'K3S_TOKEN': tokens[masters|first] | regex_replace('\n','') })%}
+# this is file content thus contains new line, which breaks agent join
+
+include:
+  - kubernetes.distro.requisites
+  - kubernetes.distro.k3s.config
+
+{{ k3s_install(k3s.distro_config.installer_url, envs) }}
+

salt/kubernetes/distro/kubeadm/init.sls

@@ -0,0 +1,3 @@
+include:
+  - kubernetes.distro.requisites
+  - kubernetes.distro.kubeadm.install

salt/kubernetes/kubeadm/install.sls → salt/kubernetes/distro/kubeadm/install.sls

@@ -10,3 +10,4 @@ kubeadm:
     - require:
       - pkgrepo_ext: kube_repository
       - service: docker
+      - sls: kubernetes.distro.requisites

salt/kubernetes/distro/kubeadm/master.sls

@@ -0,0 +1,25 @@
+ensure_token:
+  module.run:
+    - kubeadm.token_create: []
+    - unless:
+      - fun: kubeadm.token_list
+    - require:
+      - cmd: kubeadm_init
+
+propagate_token:
+  module.run:
+    - mine.send:
+        - kubernetes_token
+        - mine_function: kubeadm.token_list
+    - require:
+      - module: ensure_token
+
+propagate_hash:
+  module.run:
+    - mine.send:
+        - kubernetes_hash
+        - mine_function: cmd.run
+        - "openssl x509 -pubkey -in {{ kubernetes.master.pki.dir }}/{{ kubernetes.master.ca.pub }} | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'"
+        - python_shell: True
+    - require:
+      - cmd: kubeadm_init

salt/kubernetes/master/setup/init.sls → salt/kubernetes/distro/kubeadm/setup/init.sls

@@ -9,7 +9,7 @@ kubeadm_master_reset:
     - require:
         - pkg: kubeadm
     - require_in:
-        - cmd: kubeadm_init
+        - sls: kubernetes.distro.kubeadm
 {%- endif -%}
 
 {%- if grains['id'] == masters|first %}
@@ -28,7 +28,7 @@ allow_schedule_on_master:
     - env:
         - KUBECONFIG: {{ kubernetes.config.locations|join(':') }}
     - require:
-        - cmd: kubeadm_init
+        - sls: kubernetes.distro.{{ kubernetes.distro }}
 # todo else -> taint the node
 {%- endif %}
 
@@ -40,15 +40,15 @@ propagate_cert_key:
         - mine_function: grains.get
         - "kubernetes:master:certificate_key"
     - require:
-      - cmd: kubeadm_init
+      - sls: kubernetes.distro.{{ kubernetes.distro }}
 propagate_ip:
   module.run:
     - mine.send:
         - kubernetes_master_ip
         - mine_function: network.ip_addrs
         - cidr: {{ kubernetes_network.nodes.master_vip }}
     - require:
-      - cmd: kubeadm_init
+      - sls: kubernetes.distro.{{ kubernetes.distro }}
 {%- else %}
 propagate_ip:
   module.run:
@@ -57,33 +57,7 @@ propagate_ip:
         - mine_function: network.ip_addrs
         - cidr: {{ kubernetes_network.nodes.cidr }}
     - require:
-      - cmd: kubeadm_init
+      - sls: kubernetes.distro.{{ kubernetes.distro }}
 {%- endif %}
 
-ensure_token:
-  module.run:
-    - kubeadm.token_create: []
-    - unless:
-      - fun: kubeadm.token_list    
-    - require:
-      - cmd: kubeadm_init
-propagate_token:
-  module.run:
-    - mine.send:
-        - kubernetes_token
-        - mine_function: kubeadm.token_list
-    - require:
-      - module: ensure_token
-
-propagate_hash:
-  module.run:
-    - mine.send:
-        - kubernetes_hash
-        - mine_function: cmd.run
-        - "openssl x509 -pubkey -in {{ kubernetes.master.pki.dir }}/{{ kubernetes.master.ca.pub }} | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'"
-        - python_shell: True
-    - require:
-      - cmd: kubeadm_init
-
-
 #todo the cmd.run should be wrapped with script and return stateful data

salt/kubernetes/worker/setup.sls → salt/kubernetes/distro/kubeadm/worker.sls

@@ -1,13 +1,14 @@
-{%- from "kubernetes/worker/map.jinja" import kubernetes with context %}
-{%- from "kubernetes/network/map.jinja" import kubernetes as kubernetes_network with context %}
-
 #load modules ip_vs, ip_vs_rr, ip_vs_wrr, ip_vs_sh, nf_conntrack_ipv4
 {%- set masters = kubernetes.nodes.masters %}
 {%- set main_master_id = kubernetes.nodes.masters|first %}
 {%- set tokens = salt['mine.get'](masters|join(","), "kubernetes_token", tgt_type="list") %}
 {%- set ips = salt['mine.get'](masters|join(","), "kubernetes_master_ip", tgt_type="list") %}
 {%- set hashes = salt['mine.get'](masters|join(","), "kubernetes_hash", tgt_type="list") -%}
 
+
+include:
+    - kubernetes.distro.kubeadm
+
 {%- if ips and tokens and hashes %}
 {%- if kubernetes.worker.reset %}
 kubeadm_worker_reset:
@@ -18,7 +19,7 @@ kubeadm_worker_reset:
     - require_in:
         - cmd: join_master
 {%- endif %}
-join_master:
+join_master: # fixme from 3001 there is a module for this
     cmd.run:
         - name: "kubeadm join {{ ips[main_master_id][0] }}:{{ kubernetes_network.nodes.apiserver_port }} --token {{ tokens[main_master_id]|selectattr('usages', 'match', '.*authentication.*')|map(attribute="token")|first }} --discovery-token-ca-cert-hash sha256:{{ hashes[main_master_id] }}"
         - require:

salt/kubernetes/kubeadm/requisites.sls → salt/kubernetes/distro/requisites.sls

@@ -1,19 +1,15 @@
 #!py
 
-
 def run():
     states = {}
     swaps = __salt__['mount.swaps']()
 
     # immediately disable currently mounted swap
     for dev, details in swaps.items():
-        states["kubeadm_disable_swap_{}".format(dev)] = {
+        states["kubernetes_disable_swap_{}".format(dev)] = {
             'module.run': [
                 {'mount.swapoff': [
                     {'name': dev},
-                ]},
-                {'require_in': [
-                    {'pkg': "kubeadm"}
                 ]}
             ]
         }
@@ -22,14 +18,11 @@ def run():
     entries = __salt__['mount.fstab']()
     for name, details in entries.items():
         if details['fstype'] == 'swap':
-            states["kubeadm_remove_swap_{}".format(name)] = {
+            states["kubernetes_remove_swap_{}".format(name)] = {
                 'module.run': [
                     {'mount.rm_fstab': [
                         {'name': name},
                         {'device': details['device']},
-                    ]},
-                    {'require_in': [
-                        {'pkg': "kubeadm"}
                     ]}
                 ]
             }

salt/kubernetes/map.jinja

@@ -1,35 +1,24 @@
+{% set version = "1.29" %}
 {% set kubernetes = salt['grains.filter_by']({
-      'RedHat': {
-          'baseurl': "https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64",
-          'repo_id': "kubernetes",
-          'gpgkey': "https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg",
-          'pkgs': ["kubelet", "kubeadm", "kubectl"],
-          'config': {
-            'locations': ["/etc/kubernetes/admin.conf"],
-          },
-          'user': "root",
-          'nodes': {
-            'masters': [],
-            'workers': []
-          }
-      },
-      'Debian': {
+      'default': {
           'names': [
-              "deb http://apt.kubernetes.io/ kubernetes-xenial main"
+              "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v" ~ version ~ "/deb/ /"
           ],
           'file': '/etc/apt/sources.list.d/kubernetes.list',
-          'key_url': "https://packages.cloud.google.com/apt/doc/apt-key.gpg",
+          'key_url': "https://pkgs.k8s.io/core:/stable:/v" ~ version ~ "/deb/Release.key",
           'pkgs': ["kubelet", "kubeadm", "kubectl"],
           'config': {
             'locations': ["/etc/kubernetes/admin.conf"],
           },
           'user': "root",
+          "distro": "kubeadm",
+          'version': version,
           'nodes': {
             'masters': [],
             'workers': []
+          },
+          'container': {
+            'runtime': "docker"
           }
-      },
-      'Windows': {
-
-      },
+      }
 }, merge=salt['pillar.get']('kubernetes')) %}

salt/kubernetes/master/init.sls

@@ -5,7 +5,11 @@
 {%- endif %}
 
 include:
-  - docker
-  - kubernetes.kubeadm
-  - kubernetes.master.setup
+  - {{ kubernetes.container.runtime }}
+{% if kubernetes.distro == "kubeadm" %}
+  - kubernetes.distro.kubeadm.master
+  - kubernetes.master.kubeadm.setup # kubeadm specific, refactor to be used like worker
+{% elif kubernetes.distro == "k3s" %}
+  - kubernetes.distro.k3s.master
+{% endif %}
   - kubernetes.network

salt/kubernetes/network/cilium.sls

@@ -0,0 +1,19 @@
+{%- from "kubernetes/network/map.jinja" import kubernetes with context %}
+{%- from "_common/util.jinja" import retry with context %}
+
+kubernetes_network:
+  archive.extracted:
+    - name: {{ kubernetes.network.config.extract }}
+    - source: {{ kubernetes.network.config.source }}
+    - skip_verify: True
+    - enforce_toplevel: False
+    - clean_parent: True
+  cmd.run:
+    - name: "{{ kubernetes.network.config.extract }}/cilium install"
+    - env:
+        - KUBECONFIG: {{ kubernetes.config.locations|join(':') }}
+    - require:
+      - archive: kubernetes_network
+
+
+# add this bpffs mount

salt/kubernetes/network/map.jinja

@@ -1,7 +1,12 @@
 {%- from "kubernetes/map.jinja" import kubernetes as kubernetes_defaults with context %}
 # fixme delete flannel, add kube-router, calico and weave-net
-{%- set default_network_provider = "flannel" %}
+# https://github.com/cilium/cilium-cli/releases/tag/
+{%- set default_network_provider = "cilium" %}
 {%- set network_provider = {
+  'cilium': {
+      'source': "https://github.com/cilium/cilium-cli/releases/download/v0.15.20/cilium-linux-amd64.tar.gz",
+      'extract': "/usr/local/bin/"
+  },
   'flannel' : {
       'cidr': "10.244.0.0/16",
       'source': "https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml"

salt/kubernetes/worker/init.sls

@@ -1,5 +1,6 @@
+{%- from "kubernetes/worker/map.jinja" import kubernetes with context %}
+{%- from "kubernetes/network/map.jinja" import kubernetes as kubernetes_network with context %}
+
 include:
-  - docker
-  - kubernetes.kubeadm
-  - kubernetes.worker.setup
-
+  - {{ kubernetes.container.runtime }}
+  - kubernetes.distro.{{kubernetes.distro}}.worker