Implement coredns service discovery

group_vars/all.yml

@@ -1485,7 +1485,7 @@ varnish_manager_ssh_key_filename: varnish/sesame
 
 # Is varnish hosted on a separate, dedicated node?
 varnish_standalone: yes
-
+varnish_lambda_backends_update: "{{ varnish_standalone and not mageops_coredns_enabled }}"
 # Shall standalone varnish instance be used as a loadbalancer (instead of AWS ELB)?
 varnish_as_loadbalancer: "{{ varnish_standalone and mageops_https_termination_enable }}"
 
@@ -1825,3 +1825,13 @@ mageops_magento_vary_sign_enabled: no
 mageops_magento_vary_sign_secret: ""
 magento_vary_sign: "{{ mageops_magento_vary_sign_enabled }}"
 magento_vary_secret: "{{ mageops_magento_vary_sign_secret }}"
+
+# ---------------------------
+# --------  CoreDNS  --------
+# ---------------------------
+mageops_coredns_enabled: no
+mageops_coredns_dynamic_http_port: 2673
+# Required to be set
+# mageops_coredns_dynamic_secret:
+mageops_dynamic_node_enabled: "{{ mageops_coredns_enabled }}"
+varnish_dns_service_discovery: "{{ mageops_dynamic_node_enabled }}"

roles/cs.aws-security-group/tasks/main.yml

@@ -40,6 +40,9 @@
       - proto: tcp
         ports: ["{{ goaccess_vhost_port }}"]
         cidr_ip: "{{ mageops_trusted_cidr_blocks }}"
+      - proto: tcp
+        ports: [ "{{ mageops_coredns_dynamic_http_port }}" ]
+        group_name: "{{ aws_security_group_app_name }}"
     vpc_id: "{{ aws_vpc_id }}"
     tags: "{{ aws_tags_default | combine(ec2_sg_tags) }}"
   vars:
@@ -214,7 +217,3 @@
   vars:
     ec2_sg_tags:
       Name: "{{ aws_security_group_varnish_name }}"
-
-
-
-

roles/cs.coredns/defaults/main.yml

@@ -0,0 +1,5 @@
+coredns_dynamic_enabled: no
+coredns_dynamic_domain: dynamic.internal
+# coredns_dynamic_http_port:
+# coredns_dynamic_secret:
+coredns_dynamic_timeout: 2m

roles/cs.coredns/files/coredns.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Coredns serivce discovery and dns server
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/coredns -conf /etc/coredns/Corefile
+WorkingDirectory=/etc/coredns
+
+[Install]
+WantedBy=multi-user.target

roles/cs.coredns/meta/main.yml

@@ -0,0 +1,2 @@
+dependencies:
+ - name: cs.repo-mageops

roles/cs.coredns/tasks/disable.yml

@@ -0,0 +1,32 @@
+- name: Update dhclient config
+  template:
+    dest: /etc/dhcp/dhclient.conf
+    src: dhclient.conf.j2
+  register: _update_dhclient
+
+- name: Restart network
+  service:
+    name: network
+    state: restarted
+  when: _update_dhclient is changed
+
+- name: Disable coredns service
+  service:
+    name: coredns
+    state: stopped
+    enabled: no
+
+- name: Remove coredns config
+  file:
+    path: /etc/coredns/Corefile
+    state: absent
+
+- name: Remove coredns service file
+  file:
+    path: /etc/systemd/system/coredns.service
+    state: absent
+
+- name: Uninstall coredns
+  yum:
+    name: coredns
+    state: absent

roles/cs.coredns/tasks/enable.yml

@@ -0,0 +1,46 @@
+- name: Install coredns
+  yum:
+    name: coredns
+    state: latest
+
+- name: Install service file
+  copy:
+    dest: /etc/systemd/system/coredns.service
+    src: coredns.service
+
+- name: Create coredns config directory
+  file:
+    name: /etc/coredns
+    state: directory
+
+- name: Configure coredns
+  template:
+    dest: /etc/coredns/Corefile
+    src: Corefile.j2
+  register: _corefile
+
+- name: Restart coredns service
+  service:
+    name: coredns
+    state: restarted
+    enabled: yes
+  when: _corefile is changed
+
+- name: Enable coredns service
+  service:
+    name: coredns
+    state: started
+    enabled: yes
+  when: _corefile is not changed
+
+- name: Update dhclient config
+  template:
+    dest: /etc/dhcp/dhclient.conf
+    src: dhclient.conf.j2
+  register: _update_dhclient
+
+- name: Restart network
+  service:
+    name: network
+    state: restarted
+  when: _update_dhclient is changed

roles/cs.coredns/tasks/main.yml

@@ -0,0 +1,6 @@
+- name: Enable coredns
+  include_tasks: enable.yml
+  when: coredns_dynamic_enabled
+- name: Disable coredns
+  include_tasks: disable.yml
+  when: not coredns_dynamic_enabled

roles/cs.coredns/templates/Corefile.j2

@@ -0,0 +1,10 @@
+{% if coredns_dynamic_enabled %}
+{{ coredns_dynamic_domain }}:53 {
+    bind 127.0.0.53
+    dynamic {
+        addr            :{{ coredns_dynamic_http_port }}
+        secret          {{ coredns_dynamic_secret }}
+        host_timeout    {{ coredns_dynamic_timeout }}
+    }
+}
+{% endif %}

roles/cs.coredns/templates/dhclient.conf.j2

@@ -0,0 +1,5 @@
+{% if coredns_dynamic_enabled %}
+prepend domain-name-servers 127.0.0.53;
+{% endif %}
+timeout 300;
+retry 60;

roles/cs.dynamic-node/defaults/main.yml

@@ -0,0 +1,2 @@
+# Set node name to register
+# dynamic_node_backend_name:

roles/cs.dynamic-node/files/dynamic_node@.service

@@ -0,0 +1,6 @@
+[Unit]
+Description=Register node as dynamic backend named %i
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/mageopscli register_dynamic_bakcend %i

roles/cs.dynamic-node/files/dynamic_node@.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Timer to keep dynamic backend named %i registered
+
+[Timer]
+OnUnitActiveSec=30sec
+AccuracySec=30sec
+OnActiveSec=10sec
+
+[Install]
+WantedBy=timers.target

roles/cs.dynamic-node/meta/main.yml

@@ -0,0 +1,2 @@
+dependencies:
+  - cs.mageops-cli

roles/cs.dynamic-node/tasks/main.yml

@@ -0,0 +1,15 @@
+- name: Install dynamic node service
+  copy:
+    dest: "/etc/systemd/system/[email protected]"
+    src: "[email protected]"
+
+- name: Install dynamic node timer
+  copy:
+    dest: "/etc/systemd/system/[email protected]"
+    src: "[email protected]"
+
+- name: Enable dynamic node timer
+  service:
+    name: "dynamic_node@{{ dynamic_node_backend_name }}.timer"
+    state: started
+    enabled: yes

roles/cs.mageops-cli/files/libdynamicnode.bash

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -e
+
+dynamicnode::register_node() {
+    local node_name=$1
+    curl -Lsf -H "secret: ${config__dynamicnode_secret}" -H "backend: ${node_name}" "${config__dynamicnode_endpoint_addr}/register"
+}

roles/cs.mageops-cli/files/mageopscli

@@ -2,56 +2,73 @@
 set -euo pipefail
 
 source "$(dirname "${BASH_SOURCE[0]}")/../lib/mageops/config.bash"
-source "$(dirname "${BASH_SOURCE[0]}")/../lib/mageops/libaws.bash"
-source "$(dirname "${BASH_SOURCE[0]}")/../lib/mageops/libmageops.bash"
-source "$(dirname "${BASH_SOURCE[0]}")/../lib/mageops/libfeatures.bash"
+if [ "${config__aws_enabled}" = "yes" ];then
+    source "$(dirname "${BASH_SOURCE[0]}")/../lib/mageops/libaws.bash"
+    source "$(dirname "${BASH_SOURCE[0]}")/../lib/mageops/libmageops.bash"
+    source "$(dirname "${BASH_SOURCE[0]}")/../lib/mageops/libfeatures.bash"
+fi
+source "$(dirname "${BASH_SOURCE[0]}")/../lib/mageops/libdynamicnode.bash"
 
 main::main() {
     local cmd=${1:-}
 
     case $cmd in
     is_cron_node)
+        main::aws_only
         if [ $# != 1 ];then main::help; fi
         main::is_cron_node || exit 1
     ;;
     is_tag_present)
+        main::aws_only
         if [ $# != 2 ];then main::help; fi
         local tag=$2
         main::is_tag_present "$tag" || exit 1
     ;;
     get_tag_value)
+        main::aws_only
         if [ $# != 2 ];then main::help; fi
         local tag=$2
         main::get_tag_value "$tag" || exit 1
     ;;
     current_instance_id)
+        main::aws_only
         if [ $# != 1 ];then main::help; fi
         main::current_instance_id || exit 1
     ;;
     current_region)
+        main::aws_only
         if [ $# != 1 ];then main::help; fi
         main::current_region || exit 1
     ;;
     set_feature_flag)
+        main::aws_only
         if [ $# != 3 ];then main::help; fi
         local feature=$2
         local value=$3
         main::set_feature_flag "$feature" "$value"
     ;;
     read_feature_flag)
+        main::aws_only
         if [ $# != 2 ];then main::help; fi
         local feature=$2
         main::read_feature_flag "$feature"
     ;;
     is_feature_flag_set)
+        main::aws_only
         if [ $# != 2 ];then main::help; fi
         local feature=$2
         main::is_feature_flag_set "$feature" || exit 1
     ;;
     apply_features)
-    if [ $# != 1 ];then main::help; fi
+        main::aws_only
+        if [ $# != 1 ];then main::help; fi
         main::apply_features
     ;;
+    register_dynamic_bakcend)
+        if [ $# != 2 ];then main::help; fi
+        local backend_name=$2
+        main::register_dynamic_backend "$backend_name" || exit 1
+    ;;
     *) main::help ;;
     esac
 }
@@ -125,27 +142,58 @@ main::apply_features() {
     features::update_host_state
 }
 
+main::register_dynamic_backend() {
+    local backend_name=$1
+
+    dynamicnode::register_node "$backend_name" || main::panic "Failed to register this node as dynamic backend"
+}
+
+main::aws_only() {
+    if [ "${config__aws_enabled}" != "yes" ];then
+        main::panic "This command is only available on aws environment"
+    fi
+}
+
+main::dynamic_node_only() {
+    if [ "${config__dynamicnode_enabled}" != "yes" ];then
+        main::panic "This command is only available on nodes with dynamic node feature configured"
+    fi
+}
+
 main::help() {
+    local contains_commands=0
     main::eprintln "  $0 <cmd>"
     main::eprintln ""
     main::eprintln "Where cmd is one of:"
-    main::eprintln "  is_cron_node                         Checks if there is tag 'Cron' with value 'yes'"
-    main::eprintln "                                       status code 0 means tag present, 1 means missing tag or"
-    main::eprintln "                                       other error"
-    main::eprintln "  is_tag_present <tag_name>            Checks if specified tag name is set"
-    main::eprintln "                                       status code 0 means tag present, 1 means missing tag or"
-    main::eprintln "                                       other error"
-    main::eprintln "  get_tag_value <tag_name>             Reads value assigned to tag"
-    main::eprintln "  current_instance_id                  Reads current ec2 instance id"
-    main::eprintln "  current_region                       Reads current ec2 region name"
-    main::eprintln "  set_feature_flag <feature> <value>   Update feature flag value"
-    main::eprintln "  read_feature_flag <feature>          Reads current feature flag value"
-    main::eprintln "  is_feature_flag_set <feature>        Checks if there is any value set for feature flag"
-    main::eprintln "                                       status code 0 means flag is set, 1 otherwise"
-    main::eprintln "  apply_features                       Apply feature updates to this host"
+    if [ "${config__aws_enabled}" = "yes" ];then
+        contains_commands=1
+        main::eprintln "  is_cron_node                         Checks if there is tag 'Cron' with value 'yes'"
+        main::eprintln "                                       status code 0 means tag present, 1 means missing tag or"
+        main::eprintln "                                       other error"
+        main::eprintln "  is_tag_present <tag_name>            Checks if specified tag name is set"
+        main::eprintln "                                       status code 0 means tag present, 1 means missing tag or"
+        main::eprintln "                                       other error"
+        main::eprintln "  get_tag_value <tag_name>             Reads value assigned to tag"
+        main::eprintln "  current_instance_id                  Reads current ec2 instance id"
+        main::eprintln "  current_region                       Reads current ec2 region name"
+        main::eprintln "  set_feature_flag <feature> <value>   Update feature flag value"
+        main::eprintln "  read_feature_flag <feature>          Reads current feature flag value"
+        main::eprintln "  is_feature_flag_set <feature>        Checks if there is any value set for feature flag"
+        main::eprintln "                                       status code 0 means flag is set, 1 otherwise"
+        main::eprintln "  apply_features                       Apply feature updates to this host"
+    fi
+    if [ "${config__dynamicnode_enabled}" = "yes" ];then
+        contains_commands=1
+        main::eprintln "  register_dynamic_bakcend <name>      Register this node as dynamic backend with <name>."
+        main::eprintln "                                       This command need to be repeated constandly to keep"
+        main::eprintln "                                       this node in list."
+    fi
+    if [ "${contains_commands}" = "0" ];then
+        main::eprintln "  Sorry but this node does not contain any usable commands"
+    fi
     main::eprintln ""
     main::eprintln " Mageops cli tools"
-    main::eprintln "  (c) Creativestyle 2020"
+    main::eprintln "  (c) Creativestyle 2020-2021"
     exit 1
 }
 

roles/cs.mageops-cli/tasks/main.yml

@@ -16,6 +16,7 @@
     - { src: "libaws.bash", dest: "/usr/local/lib/mageops/libaws.bash", mode: "0644" }
     - { src: "libmageops.bash", dest: "/usr/local/lib/mageops/libmageops.bash", mode: "0644" }
     - { src: "libfeatures.bash", dest: "/usr/local/lib/mageops/libfeatures.bash", mode: "0644" }
+    - { src: "libdynamicnode.bash", dest: "/usr/local/lib/mageops/libdynamicnode.bash", mode: "0644" }
     - { src: "mageopscli", dest: "/usr/local/bin/mageopscli", mode: "0755" }
     - { src: "features/example_feature.bash", dest: "/usr/local/lib/mageops/features/example_feature.bash", mode: "0644" }
     - { src: "mageops-update-features.service", dest: "/etc/systemd/system/mageops-update-features.service", mode: "0644" }

roles/cs.mageops-cli/templates/config.bash

@@ -3,3 +3,7 @@
 config__features_s3_bucket="{{ aws_s3_secret_bucket }}"
 config__features_s3_config_path="s3://${config__features_s3_bucket}/features.json"
 config__features_modules_path="{{ mageops_cli_features_dir }}"
+config__dynamicnode_endpoint_addr="http://{{ mageops_varnish_host }}:{{ mageops_coredns_dynamic_http_port }}/"
+config__dynamicnode_secret="{{ mageops_coredns_dynamic_secret }}"
+config__aws_enabled="{{ aws_use | ternary('yes', 'no') }}"
+config__dynamicnode_enabled="{{ mageops_dynamic_node_enabled | ternary('yes', 'no') }}"

roles/cs.varnish/defaults/main.yml

@@ -172,3 +172,9 @@ varnish_301_cache_time: 3h
 # If you are on varnish_trusted_ips list you will
 # still receive those headers
 varnish_do_not_expose_caching: no
+
+varnish_dns_service_discovery: no
+varnish_dns_backend_domain: magento.dynamic.internal
+varnish_dns_extra_backend_domain: magento_extra.dynamic.internal
+varnish_dns_acl:
+  - 10/8 # Default aws network

roles/cs.varnish/meta/main.yml

@@ -1,5 +1,6 @@
 allow_duplicates: no
 dependencies:
+  - cs.repo-mageops
   - cs.ansible-plugins
   - cs.repo-varnish