Implement efs logs storage

group_vars/all.yml

@@ -147,6 +147,10 @@ aws_tags_role_mysql_database:
 aws_tags_role_storage:
   Role: "storage"
 
+aws_tags_role_logs_storage:
+  RoleStorage: "logs"
+  RoleStoragePublic: "no"
+
 aws_tags_role_shared_storage:
   RoleStorage: "shared"
   RoleStoragePublic: "no"
@@ -335,6 +339,7 @@ aws_security_group_rds_name: "{{ mageops_app_name }}-rds-sg"
 aws_security_group_redis_name: "{{ mageops_app_name }}-redis-sg"
 aws_security_group_elasticsearch_name: "{{ mageops_app_name }}-elastic-sg"
 aws_security_group_efs_name: "{{ mageops_app_name }}-efs-sg"
+aws_security_group_efs_logs_name: "{{ mageops_app_name }}-efs-sg"
 
 # Allows to add extra ports to persistant server
 # e.g. to allow ssh from webnode:
@@ -376,7 +381,6 @@ aws_app_node_webnodedown_hook_name: WebNodeGoingDown
 
 aws_varnish_node_launch_script_extra: ''
 
-
 # ---------------------------------------------
 # --------  AWS Elastic Load Balancer  --------
 # ---------------------------------------------
@@ -1138,6 +1142,7 @@ magento_efs_locks_data_app_path: "{{ mageops_app_web_dir }}/shared/var/lock"
 
 # EFS resource name
 magento_efs_app_node_name: "{{ mageops_app_name }}-app-shared"
+mageops_efs_logs_name: "{{ mageops_app_name }}-logs"
 
 # Use this to set up app additional instance mounts per-project
 mageops_efs_app_node_mounts: []

roles/cs.aws-efs-logs/defaults/main.yml

@@ -0,0 +1,4 @@
+aws_efs_logs_enabled: no
+
+aws_efs_logs_efs_dir: /mnt/efs_logs
+aws_efs_mount_script_path: /usr/local/libexec/move-logs-to-efs

roles/cs.aws-efs-logs/meta/main.yml

@@ -0,0 +1,14 @@
+dependencies:
+  - role: cs.aws-vpc-facts
+    delegate_to: localhost
+    delegate_facts: no
+    become: no
+  - role: cs.aws-security-group-facts
+    delegate_to: localhost
+    delegate_facts: no
+    become: no
+  - role: cs.aws-efs
+    efs_name: "{{ mageops_efs_logs_name }}"
+    efs_tags: "{{ aws_tags_default | combine(aws_tags_role_storage, aws_tags_role_logs_storage) }}"
+    efs_root_mountpoint: "{{ aws_efs_logs_efs_dir }}"
+    when: aws_efs_logs_enabled

roles/cs.aws-efs-logs/tasks/disable.yml

@@ -0,0 +1,27 @@
+- name: Check if /var/log is mounted
+  shell:
+    cmd: "mountpoint -q /var/log"
+  register: aws_efs_mount_stat
+  failed_when: False
+  changed_when: False
+
+- name: Restore /var/log
+  block:
+  - name: Disable service
+    service:
+      name: aws-efs-logs.service
+      state: stopped
+      enabled: no
+
+  - name: Remove service files
+    file:
+      name: "{{ item }}"
+      state: absent
+    with_items:
+      - "/etc/systemd/system/aws-efs-logs.service"
+      - "{{ aws_efs_mount_script_path }}"
+
+  - name: Schedule system reboot at 3:00
+    shell: "shutdown -r 3:00"
+
+  when: aws_efs_mount_stat.rc == 0

roles/cs.aws-efs-logs/tasks/enable.yml

@@ -0,0 +1,17 @@
+- name: Install migration script
+  template:
+    src: move-logs-to-efs.sh
+    dest: "{{ aws_efs_mount_script_path }}"
+    mode: 0700
+
+- name: Install service
+  template:
+    src: aws-efs-logs.service
+    dest: "/etc/systemd/system/aws-efs-logs.service"
+    mode: 0700
+
+- name: Enable service
+  service:
+    name: aws-efs-logs.service
+    state: started
+    enabled: yes

roles/cs.aws-efs-logs/tasks/main.yml

@@ -0,0 +1,6 @@
+- name: Enable AWS efs logs storage
+  include_tasks: enable.yml
+  when: aws_efs_logs_enabled
+- name: Disable AWS efs logs storage
+  include_tasks: disable.yml
+  when: not aws_efs_logs_enabled

roles/cs.aws-efs-logs/templates/aws-efs-logs.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Move /var/log to efs mount
+DefaultDependencies=no
+RequiresMountsFor=/var/log {{ aws_efs_logs_efs_dir }}
+IgnoreOnIsolate=yes
+
+[Service]
+Type=oneshot
+ExecStart={{ aws_efs_mount_script_path }}
+RemainAfterExit=yes
+
+[Install]
+WantedBy=network-online.target

roles/cs.aws-efs-logs/templates/move-logs-to-efs.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+EFS_MOUNTPOINT="{{ aws_efs_logs_efs_dir }}"
+LOG_TARGET_PATH="$EFS_MOUNTPOINT/$( hostname )"
+
+if ! [ -d "$EFS_MOUNTPOINT" ];then
+    echo "Mount point does not exist!"
+    exit 2
+fi
+
+if [ -L "/var/log" ];then
+    echo "/var/log is already symlinked"
+    exit 0
+fi
+
+if [ -e "$LOG_TARGET_PATH" ];then
+    LOG_RENAME_TO="$LOG_TARGET_PATH-moved-$(date -Iseconds)"
+    echo "Found existing log target dir, renaming to $LOG_RENAME_TO"
+    mv "$LOG_TARGET_PATH" "$LOG_RENAME_TO"
+fi
+
+echo "Moving logs to EFS"
+mv /var/log "$LOG_TARGET_PATH"
+mkdir /var/log
+
+# After reboot we will start logging on local filesystem again, we need to restore files structure to make sure nothing will crash
+
+echo "Recreating directories in local log"
+find "$LOG_TARGET_PATH" -type d -printf '%P\n' \
+    | xargs -I '{}' sh -c "mkdir '/var/log/{}' && chmod -v --reference='$LOG_TARGET_PATH/{}' '/var/log/{}' && chown -v --reference='$LOG_TARGET_PATH/{}' '/var/log/{}'"
+
+echo "Recreate files in local log"
+find "$LOG_TARGET_PATH" -type f -printf '%P\n' \
+    | grep -v -E '(\-|\.)[0-9]+(\.log|\.json)?(\.gz|\.zstd|.zst)?$' \
+    | xargs -I '{}' sh -c "touch '/var/log/{}' && chmod -v --reference='$LOG_TARGET_PATH/{}' '/var/log/{}' && chown -v --reference='$LOG_TARGET_PATH/{}' '/var/log/{}'"
+
+mount -o bind,nonempty --make-private "$LOG_TARGET_PATH" /var/log

roles/cs.aws-security-group/tasks/main.yml

@@ -192,6 +192,30 @@
       Name: "{{ aws_security_group_efs_name }}"
   register: aws_security_group_efs
 
+
+- name: Create security group for EFS Logs
+  ec2_group:
+    name: "{{ aws_security_group_efs_logs_name }}"
+    description: "{{ mageops_app_name }} EFS security group"
+    region: "{{ aws_region }}"
+    purge_rules: no
+    rules:
+      - proto: tcp
+        ports: [2049]
+        group_name: "{{ aws_security_group_app_name }}"
+      - proto: tcp
+        ports: [2049]
+        group_name: "{{ aws_security_group_persistant_name }}"
+      - proto: tcp
+        ports: [2049]
+        group_name: "{{ aws_security_group_lb_name }}"
+    vpc_id: "{{ aws_vpc_id }}"
+    tags: "{{ aws_tags_default | combine(ec2_sg_tags) }}"
+  vars:
+    ec2_sg_tags:
+      Name: "{{ aws_security_group_efs_logs_name }}"
+  when: aws_efs_logs_enabled
+
 - name: Allow app to access varnish
   ec2_group:
     name: "{{ aws_security_group_varnish_name }}"
@@ -214,7 +238,3 @@
   vars:
     ec2_sg_tags:
       Name: "{{ aws_security_group_varnish_name }}"
-
-
-
-

site.step-15-varnish.yml

@@ -6,9 +6,16 @@
       delegate_to: localhost
       become: no
       when: aws_use
+    - role: cs.aws-vpc-facts
+      delegate_to: localhost
+      delegate_facts: no
+      become: no
+      when: aws_use
     - role: cs.switch-to-dnf
     - role: pinkeen.selinux-disable
     - role: cs.swap
+    - role: cs.aws-efs-logs
+      when: aws_use
     - role: cs.earlyoom
       when: mageops_earlyoom_enable
     - role: cs.packages

site.step-20-persistent.yml

@@ -19,6 +19,8 @@
     - role: cs.switch-to-dnf
     - role: pinkeen.selinux-disable
     - role: cs.swap
+    - role: cs.aws-efs-logs
+      when: aws_use
     - role: cs.earlyoom
       when: mageops_earlyoom_enable
     - role: cs.packages

site.step-40-app-node.yml

@@ -22,6 +22,12 @@
       become: no
       when: aws_use
 
+    - role: cs.aws-vpc-facts
+      delegate_to: localhost
+      delegate_facts: no
+      become: no
+      when: aws_use
+
     - role: cs.switch-to-dnf
 
     - role: pinkeen.selinux-disable
@@ -37,6 +43,9 @@
       swap_swappiness: "{{ mageops_app_node_swappiness }}"
       when: mageops_app_node_swap_enable
 
+    - role: cs.aws-efs-logs
+      when: aws_use
+
     - role: cs.earlyoom
       when: mageops_earlyoom_enable