From 3b7cdba348baeac20d6f8ac2c91966c7aaf0614c Mon Sep 17 00:00:00 2001 From: Piotr Rogowski Date: Thu, 10 Jun 2021 13:49:51 +0200 Subject: [PATCH] Implement efs logs storage --- group_vars/all.yml | 7 +++- roles/cs.aws-efs-logs/defaults/main.yml | 4 ++ roles/cs.aws-efs-logs/meta/main.yml | 14 +++++++ roles/cs.aws-efs-logs/tasks/disable.yml | 27 +++++++++++++ roles/cs.aws-efs-logs/tasks/enable.yml | 17 +++++++++ roles/cs.aws-efs-logs/tasks/main.yml | 6 +++ .../templates/aws-efs-logs.service | 13 +++++++ .../templates/move-logs-to-efs.sh | 38 +++++++++++++++++++ roles/cs.aws-security-group/tasks/main.yml | 28 ++++++++++++-- site.step-15-varnish.yml | 7 ++++ site.step-20-persistent.yml | 2 + site.step-40-app-node.yml | 9 +++++ 12 files changed, 167 insertions(+), 5 deletions(-) create mode 100644 roles/cs.aws-efs-logs/defaults/main.yml create mode 100644 roles/cs.aws-efs-logs/meta/main.yml create mode 100644 roles/cs.aws-efs-logs/tasks/disable.yml create mode 100644 roles/cs.aws-efs-logs/tasks/enable.yml create mode 100644 roles/cs.aws-efs-logs/tasks/main.yml create mode 100644 roles/cs.aws-efs-logs/templates/aws-efs-logs.service create mode 100644 roles/cs.aws-efs-logs/templates/move-logs-to-efs.sh diff --git a/group_vars/all.yml b/group_vars/all.yml index 2c376434..f16680ca 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -147,6 +147,10 @@ aws_tags_role_mysql_database: aws_tags_role_storage: Role: "storage" +aws_tags_role_logs_storage: + RoleStorage: "logs" + RoleStoragePublic: "no" + aws_tags_role_shared_storage: RoleStorage: "shared" RoleStoragePublic: "no" @@ -335,6 +339,7 @@ aws_security_group_rds_name: "{{ mageops_app_name }}-rds-sg" aws_security_group_redis_name: "{{ mageops_app_name }}-redis-sg" aws_security_group_elasticsearch_name: "{{ mageops_app_name }}-elastic-sg" aws_security_group_efs_name: "{{ mageops_app_name }}-efs-sg" +aws_security_group_efs_logs_name: "{{ mageops_app_name }}-efs-sg" # Allows to add extra ports to persistant server # e.g. to allow ssh from webnode: @@ -376,7 +381,6 @@ aws_app_node_webnodedown_hook_name: WebNodeGoingDown aws_varnish_node_launch_script_extra: '' - # --------------------------------------------- # -------- AWS Elastic Load Balancer -------- # --------------------------------------------- @@ -1138,6 +1142,7 @@ magento_efs_locks_data_app_path: "{{ mageops_app_web_dir }}/shared/var/lock" # EFS resource name magento_efs_app_node_name: "{{ mageops_app_name }}-app-shared" +mageops_efs_logs_name: "{{ mageops_app_name }}-logs" # Use this to set up app additional instance mounts per-project mageops_efs_app_node_mounts: [] diff --git a/roles/cs.aws-efs-logs/defaults/main.yml b/roles/cs.aws-efs-logs/defaults/main.yml new file mode 100644 index 00000000..5826d55f --- /dev/null +++ b/roles/cs.aws-efs-logs/defaults/main.yml @@ -0,0 +1,4 @@ +aws_efs_logs_enabled: no + +aws_efs_logs_efs_dir: /mnt/efs_logs +aws_efs_mount_script_path: /usr/local/libexec/move-logs-to-efs diff --git a/roles/cs.aws-efs-logs/meta/main.yml b/roles/cs.aws-efs-logs/meta/main.yml new file mode 100644 index 00000000..7532b7a5 --- /dev/null +++ b/roles/cs.aws-efs-logs/meta/main.yml @@ -0,0 +1,14 @@ +dependencies: + - role: cs.aws-vpc-facts + delegate_to: localhost + delegate_facts: no + become: no + - role: cs.aws-security-group-facts + delegate_to: localhost + delegate_facts: no + become: no + - role: cs.aws-efs + efs_name: "{{ mageops_efs_logs_name }}" + efs_tags: "{{ aws_tags_default | combine(aws_tags_role_storage, aws_tags_role_logs_storage) }}" + efs_root_mountpoint: "{{ aws_efs_logs_efs_dir }}" + when: aws_efs_logs_enabled diff --git a/roles/cs.aws-efs-logs/tasks/disable.yml b/roles/cs.aws-efs-logs/tasks/disable.yml new file mode 100644 index 00000000..9f4496f0 --- /dev/null +++ b/roles/cs.aws-efs-logs/tasks/disable.yml @@ -0,0 +1,27 @@ +- name: Check if /var/log is mounted + shell: + cmd: "mountpoint -q /var/log" + register: aws_efs_mount_stat + failed_when: False + changed_when: False + +- name: Restore /var/log + block: + - name: Disable service + service: + name: aws-efs-logs.service + state: stopped + enabled: no + + - name: Remove service files + file: + name: "{{ item }}" + state: absent + with_items: + - "/etc/systemd/system/aws-efs-logs.service" + - "{{ aws_efs_mount_script_path }}" + + - name: Schedule system reboot at 3:00 + shell: "shutdown -r 3:00" + + when: aws_efs_mount_stat.rc == 0 diff --git a/roles/cs.aws-efs-logs/tasks/enable.yml b/roles/cs.aws-efs-logs/tasks/enable.yml new file mode 100644 index 00000000..8d4c43e8 --- /dev/null +++ b/roles/cs.aws-efs-logs/tasks/enable.yml @@ -0,0 +1,17 @@ +- name: Install migration script + template: + src: move-logs-to-efs.sh + dest: "{{ aws_efs_mount_script_path }}" + mode: 0700 + +- name: Install service + template: + src: aws-efs-logs.service + dest: "/etc/systemd/system/aws-efs-logs.service" + mode: 0700 + +- name: Enable service + service: + name: aws-efs-logs.service + state: started + enabled: yes diff --git a/roles/cs.aws-efs-logs/tasks/main.yml b/roles/cs.aws-efs-logs/tasks/main.yml new file mode 100644 index 00000000..8d0664b1 --- /dev/null +++ b/roles/cs.aws-efs-logs/tasks/main.yml @@ -0,0 +1,6 @@ +- name: Enable AWS efs logs storage + include_tasks: enable.yml + when: aws_efs_logs_enabled +- name: Disable AWS efs logs storage + include_tasks: disable.yml + when: not aws_efs_logs_enabled diff --git a/roles/cs.aws-efs-logs/templates/aws-efs-logs.service b/roles/cs.aws-efs-logs/templates/aws-efs-logs.service new file mode 100644 index 00000000..27bf0e6c --- /dev/null +++ b/roles/cs.aws-efs-logs/templates/aws-efs-logs.service @@ -0,0 +1,13 @@ +[Unit] +Description=Move /var/log to efs mount +DefaultDependencies=no +RequiresMountsFor=/var/log {{ aws_efs_logs_efs_dir }} +IgnoreOnIsolate=yes + +[Service] +Type=oneshot +ExecStart={{ aws_efs_mount_script_path }} +RemainAfterExit=yes + +[Install] +WantedBy=network-online.target diff --git a/roles/cs.aws-efs-logs/templates/move-logs-to-efs.sh b/roles/cs.aws-efs-logs/templates/move-logs-to-efs.sh new file mode 100644 index 00000000..45bce5a7 --- /dev/null +++ b/roles/cs.aws-efs-logs/templates/move-logs-to-efs.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -euo pipefail + +EFS_MOUNTPOINT="{{ aws_efs_logs_efs_dir }}" +LOG_TARGET_PATH="$EFS_MOUNTPOINT/$( hostname )" + +if ! [ -d "$EFS_MOUNTPOINT" ];then + echo "Mount point does not exist!" + exit 2 +fi + +if [ -L "/var/log" ];then + echo "/var/log is already symlinked" + exit 0 +fi + +if [ -e "$LOG_TARGET_PATH" ];then + LOG_RENAME_TO="$LOG_TARGET_PATH-moved-$(date -Iseconds)" + echo "Found existing log target dir, renaming to $LOG_RENAME_TO" + mv "$LOG_TARGET_PATH" "$LOG_RENAME_TO" +fi + +echo "Moving logs to EFS" +mv /var/log "$LOG_TARGET_PATH" +mkdir /var/log + +# After reboot we will start logging on local filesystem again, we need to restore files structure to make sure nothing will crash + +echo "Recreating directories in local log" +find "$LOG_TARGET_PATH" -type d -printf '%P\n' \ + | xargs -I '{}' sh -c "mkdir '/var/log/{}' && chmod -v --reference='$LOG_TARGET_PATH/{}' '/var/log/{}' && chown -v --reference='$LOG_TARGET_PATH/{}' '/var/log/{}'" + +echo "Recreate files in local log" +find "$LOG_TARGET_PATH" -type f -printf '%P\n' \ + | grep -v -E '(\-|\.)[0-9]+(\.log|\.json)?(\.gz|\.zstd|.zst)?$' \ + | xargs -I '{}' sh -c "touch '/var/log/{}' && chmod -v --reference='$LOG_TARGET_PATH/{}' '/var/log/{}' && chown -v --reference='$LOG_TARGET_PATH/{}' '/var/log/{}'" + +mount -o bind,nonempty --make-private "$LOG_TARGET_PATH" /var/log diff --git a/roles/cs.aws-security-group/tasks/main.yml b/roles/cs.aws-security-group/tasks/main.yml index afe78b46..3a6ae197 100644 --- a/roles/cs.aws-security-group/tasks/main.yml +++ b/roles/cs.aws-security-group/tasks/main.yml @@ -192,6 +192,30 @@ Name: "{{ aws_security_group_efs_name }}" register: aws_security_group_efs + +- name: Create security group for EFS Logs + ec2_group: + name: "{{ aws_security_group_efs_logs_name }}" + description: "{{ mageops_app_name }} EFS security group" + region: "{{ aws_region }}" + purge_rules: no + rules: + - proto: tcp + ports: [2049] + group_name: "{{ aws_security_group_app_name }}" + - proto: tcp + ports: [2049] + group_name: "{{ aws_security_group_persistant_name }}" + - proto: tcp + ports: [2049] + group_name: "{{ aws_security_group_lb_name }}" + vpc_id: "{{ aws_vpc_id }}" + tags: "{{ aws_tags_default | combine(ec2_sg_tags) }}" + vars: + ec2_sg_tags: + Name: "{{ aws_security_group_efs_logs_name }}" + when: aws_efs_logs_enabled + - name: Allow app to access varnish ec2_group: name: "{{ aws_security_group_varnish_name }}" @@ -214,7 +238,3 @@ vars: ec2_sg_tags: Name: "{{ aws_security_group_varnish_name }}" - - - - diff --git a/site.step-15-varnish.yml b/site.step-15-varnish.yml index c60f3338..66606734 100644 --- a/site.step-15-varnish.yml +++ b/site.step-15-varnish.yml @@ -6,9 +6,16 @@ delegate_to: localhost become: no when: aws_use + - role: cs.aws-vpc-facts + delegate_to: localhost + delegate_facts: no + become: no + when: aws_use - role: cs.switch-to-dnf - role: pinkeen.selinux-disable - role: cs.swap + - role: cs.aws-efs-logs + when: aws_use - role: cs.earlyoom when: mageops_earlyoom_enable - role: cs.packages diff --git a/site.step-20-persistent.yml b/site.step-20-persistent.yml index 03e36cd6..12aa92c2 100644 --- a/site.step-20-persistent.yml +++ b/site.step-20-persistent.yml @@ -19,6 +19,8 @@ - role: cs.switch-to-dnf - role: pinkeen.selinux-disable - role: cs.swap + - role: cs.aws-efs-logs + when: aws_use - role: cs.earlyoom when: mageops_earlyoom_enable - role: cs.packages diff --git a/site.step-40-app-node.yml b/site.step-40-app-node.yml index c0283332..cfcffe9c 100644 --- a/site.step-40-app-node.yml +++ b/site.step-40-app-node.yml @@ -22,6 +22,12 @@ become: no when: aws_use + - role: cs.aws-vpc-facts + delegate_to: localhost + delegate_facts: no + become: no + when: aws_use + - role: cs.switch-to-dnf - role: pinkeen.selinux-disable @@ -37,6 +43,9 @@ swap_swappiness: "{{ mageops_app_node_swappiness }}" when: mageops_app_node_swap_enable + - role: cs.aws-efs-logs + when: aws_use + - role: cs.earlyoom when: mageops_earlyoom_enable