Walg config yml

README.md

@@ -22,9 +22,15 @@ walg_version: 0.2.15
 # wal-g releases repository url
 walg_url: "https://github.com/wal-g/wal-g/releases/download/v{{ walg_version }}/wal-g.linux-amd64.tar.gz"
 
+# Name of binary file in tarball or downloaded from walg_url archive
+walg_archive_binary: "wal-g-pg-{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}-amd64"
+
 # wal-g binary place
 walg_binary: /usr/local/bin/wal-g
 
+# Ability to install wal-g from a local .tar.gz file. Should be path to file.
+walg_tarball: false
+
 # user for cronjob
 walg_user: postgres
 
@@ -61,6 +67,10 @@ walg_backups_retain: 2
 
 # Create cron jobs
 walg_cron_enabled: true
+
+# If host is set, sends extra time metrics of commands to prometheus pushgateway
+walg_pushgateway_host: ""
+walg_pushgateway_port: 9091
 ```
 
 ### Override

defaults/main.yml

@@ -3,10 +3,14 @@
 
 walg_version: 0.2.15
 walg_url: "https://github.com/wal-g/wal-g/releases/download/v{{ walg_version }}/wal-g.linux-amd64.tar.gz"
-walg_binary: /usr/local/bin/wal-g
+walg_archive_binary: "wal-g-pg-{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}-amd64"
+walg_binary: "/usr/local/bin/wal-g-pg-{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}-amd64-v{{ walg_version }}"
+walg_tarball: false
 
 walg_user: postgres
 
+walg_conf_type: "env"  # env or yml
+walg_conf_yml_file: /etc/wal-g/config.yml
 walg_env: /etc/default/wal-g
 walg_script_path: /usr/local/scripts
 walg_data_path: /var/lib/postgresql/12/main
@@ -21,6 +25,21 @@ walg_s3_prefix: ''
 walg_s3_storage_class: STANDARD_IA
 walg_s3_use_path_style: false
 
+# A map of storage config parameters
+# Like:
+# walg_failover_storages
+#   aws:
+#      walg_s3_id: "id"
+#      walg_s3_secret: "secret"
+#      walg_s3_region: "eu-west-1"
+#      walg_s3_endpoint: walg_s3_endpoint: "https://s3.dualstack.eu-west-1.amazonaws.com"
+#   other:
+#        ...
+walg_failover_storages: []
+walg_failover_check_timeout: 30s
+walg_failover_storages_cache_lifetime: 15m
+walg_failover_storages_check_size: 1mb
+
 walg_pg_socket: /var/run/postgresql
 
 walg_backup_upload_disk: 2
@@ -40,10 +59,20 @@ walg_gcs_override: []
 walg_swift_override: []
 walg_filesystem_override: []
 
+## Failover storages
+walg_failover_storages_override: []
+
 ## Databases
 walg_mongodb_override: []
 walg_mysql_override: []
 walg_postgresql_override: []
 
+# Statsd
+walg_statsd_override: []
+
 ## Commons
 walg_common_override: []
+
+## Pushgateway Prometheus metrics
+walg_pushgateway_host: ""
+walg_pushgateway_port: 9091

tasks/main.yml

@@ -22,43 +22,96 @@
   tags:
     - always
 
+- name: wal-g | ensure directory exists
+  file:
+    path: "{{ walg_binary | dirname }}"
+    state: directory
+
+- name: wal-g | ensure tmp directory exists
+  file:
+    path: "/tmp/wal-g-build"
+    state: directory
+  when:
+    - not walg_current_binary.stat.exists or walg_current_version.stdout is version(walg_version, '!=')
+
 - name: wal-g | download with unarchive when upgrade or downgrade
   unarchive:
     src: "{{ walg_url }}"
-    dest: "{{ walg_binary | dirname }}"
+    dest: "/tmp/wal-g-build"
     exclude:
       - 'README.md'
       - 'LICENSE'
     remote_src: true
-    force: true
     owner: root
     group: root
     mode: 0755
   when:
     - not walg_current_binary.stat.exists or walg_current_version.stdout is version(walg_version, '!=')
+    - not walg_tarball
   tags:
     - always
 
+- name: copy tarball to server
+  copy:
+    src: "{{ walg_tarball }}"
+    dest: /tmp/wal-g-build
+  when:
+    - not walg_current_binary.stat.exists or walg_current_version.stdout is version(walg_version, '!=')
+    - walg_tarball | default(false)
+
+- name: wal-g | unarchive tarball when upgrade or downgrade
+  unarchive:
+    src: "/tmp/wal-g-build/{{ walg_tarball | basename }}"
+    dest: "/tmp/wal-g-build"
+    exclude:
+      - 'README.md'
+      - 'LICENSE'
+    remote_src: true
+    owner: root
+    group: root
+    mode: 0755
+  when:
+    - not walg_current_binary.stat.exists or walg_current_version.stdout is version(walg_version, '!=')
+    - walg_tarball
+  tags:
+    - always
+
+- name: wal-g | move wal-g unarchived binary file to walg_binary location
+  command: "mv /tmp/wal-g-build/{{ walg_archive_binary }} {{ walg_binary }}"
+  when:
+    - not walg_current_binary.stat.exists or walg_current_version.stdout is version(walg_version, '!=')
+
+- name: wal-g | remove tmp directory
+  file:
+    path: "/tmp/wal-g-build"
+    state: absent
+  when:
+    - not walg_current_binary.stat.exists or walg_current_version.stdout is version(walg_version, '!=')
+
 - name: wal-g | combine env config
   block:
     - set_fact:
-        walg_aws_combine: "{{ walg_aws_config | combine(walg_aws_override) | dict2items(key_name='option', value_name='value') }}"
+        walg_aws_combine: "{{ walg_aws_config | combine(walg_aws_override) }}"
     - set_fact:
-        walg_azure_combine: "{{ walg_azure_config | combine(walg_azure_override) | dict2items(key_name='option', value_name='value') }}"
+        walg_azure_combine: "{{ walg_azure_config | combine(walg_azure_override) }}"
     - set_fact:
-        walg_gcs_combine: "{{ walg_gcs_config | combine(walg_gcs_override) | dict2items(key_name='option', value_name='value') }}"
+        walg_gcs_combine: "{{ walg_gcs_config | combine(walg_gcs_override) }}"
     - set_fact:
-        walg_swift_combine: "{{ walg_swift_config | combine(walg_swift_override) | dict2items(key_name='option', value_name='value') }}"
+        walg_swift_combine: "{{ walg_swift_config | combine(walg_swift_override) }}"
     - set_fact:
-        walg_filesystem_combine: "{{ walg_filesystem_config | combine(walg_filesystem_override) | dict2items(key_name='option', value_name='value') }}"
+        walg_filesystem_combine: "{{ walg_filesystem_config | combine(walg_filesystem_override) }}"
     - set_fact:
-        walg_mongodb_combine: "{{ walg_mongodb_config | combine(walg_mongodb_override) | dict2items(key_name='option', value_name='value') }}"
+        walg_mongodb_combine: "{{ walg_mongodb_config | combine(walg_mongodb_override) }}"
     - set_fact:
-        walg_mysql_combine: "{{ walg_mysql_config | combine(walg_mysql_override) | dict2items(key_name='option', value_name='value') }}"
+        walg_mysql_combine: "{{ walg_mysql_config | combine(walg_mysql_override) }}"
     - set_fact:
-        walg_postgresql_combine: "{{ walg_postgresql_config | combine(walg_postgresql_override) | dict2items(key_name='option', value_name='value') }}"
+        walg_postgresql_combine: "{{ walg_postgresql_config | combine(walg_postgresql_override) }}"
     - set_fact:
-        walg_common_combine: "{{ walg_common_config | combine(walg_common_override) | dict2items(key_name='option', value_name='value') }}"
+        walg_failover_storages_combine: "{{ walg_failover_storages_config | combine(walg_failover_storages_override) }}"
+    - set_fact:
+        walg_statsd_combine: "{{ walg_statsd_config | combine(walg_statsd_override) }}"
+    - set_fact:
+        walg_common_combine: "{{ walg_common_config | combine(walg_common_override) }}"
   tags:
     - always
 
@@ -67,6 +120,7 @@
     path: "{{ walg_env | dirname }}"
     state: directory
     mode: 0755
+  when: walg_conf_type == "env"
   tags:
     - always
 
@@ -77,6 +131,43 @@
     owner: root
     group: adm
     mode: 0644
+  when: walg_conf_type == "env"
+  tags:
+    - always
+
+- name: wal-g | remove unused env file
+  file:
+    path: "{{ walg_env }}"
+    state: absent
+  when: walg_conf_type != "env"
+  tags:
+    - always
+
+- name: wal-g | check yml directory
+  file:
+    path: "{{ walg_conf_yml_file | dirname }}"
+    state: directory
+    mode: 0755
+  when: walg_conf_type == "yml"
+  tags:
+    - always
+
+- name: wal-g | create yml file
+  template:
+    src: 'config.yml.j2'
+    dest: "{{ walg_conf_yml_file }}"
+    owner: root
+    group: adm
+    mode: 0644
+  when: walg_conf_type == "yml"
+  tags:
+    - always
+
+- name: wal-g | remove unused yaml conf file
+  file:
+    path: "{{ walg_conf_yml_file }}"
+    state: absent
+  when: walg_conf_type != "yml"
   tags:
     - always
 

templates/config.yml.j2

@@ -0,0 +1,86 @@
+---
+{{ ansible_managed | comment }}
+
+{% if walg_aws_combine | length > 0 %}
+# AWS S3 setup
+{%   for key, value in walg_aws_combine.items() %}
+{{      key | upper }}: {{ (value | int) if value in (0, "0") or (value | int) != 0 else ('"' + (value | string) +  '"') }}
+{%   endfor %}
+{% endif %}
+{% if walg_azure_combine | length > 0 %}
+# Azure Storage setup
+{%   for key, value in walg_azure_combine.items() %}
+{{      key | upper }}: {{ (value | int) if value in (0, "0") or (value | int) != 0 else ('"' + (value | string) +  '"') }}
+{%   endfor %}
+{% endif %}
+{% if walg_gcs_combine | length > 0 %}
+# Google Cloud Storage setup
+{%   for key, value in walg_gcs_combine.items() %}
+{{      key | upper }}: {{ (value | int) if value in (0, "0") or (value | int) != 0 else ('"' + (value | string) +  '"') }}
+{%   endfor %}
+{% endif %}
+{% if walg_swift_combine | length > 0 %}
+# Swift object storage setup
+{%   for key, value in walg_swift_combine.items() %}
+{{     key | upper }}: {{ (value | int) if value in (0, "0") or (value | int) != 0 else ('"' + (value | string) +  '"') }}
+{%   endfor %}
+{% endif %}
+{% if walg_filesystem_combine | length > 0 %}
+# Backups on filesystem setup
+{%   for key, value in walg_filesystem_combine.items() %}
+{{     key | upper }}: {{ (value | int) if value in (0, "0") or (value | int) != 0 else ('"' + (value | string) +  '"') }}
+{%   endfor %}
+{% endif %}
+
+{% if walg_mongodb_combine | length > 0 %}
+# MongoDB setup
+{%   for key, value in walg_mongodb_combine.items() %}
+{{     key | upper }}: {{ (value | int) if value in (0, "0") or (value | int) != 0 else ('"' + (value | string) +  '"') }}
+{%   endfor %}
+{% endif %}
+{% if walg_mysql_combine | length > 0 %}
+# MySQL setup
+{%   for key, value in walg_mysql_combine.items() %}
+{{     key | upper }}: {{ (value | int) if value in (0, "0") or (value | int) != 0 else ('"' + (value | string) +  '"') }}
+{%   endfor %}
+{% endif %}
+{% if walg_postgresql_combine | length > 0 %}
+# PostgreSQL setup
+{%   for key, value in walg_postgresql_combine.items() %}
+{{     key | upper }}: {{ (value | int) if value in (0, "0") or (value | int) != 0 else ('"' + (value | string) +  '"') }}
+{%   endfor %}
+{% endif %}
+
+{% if walg_failover_storages %}
+# Failover setup
+{%   for key, value in walg_failover_storages_combine.items() %}
+{%     if key != "walg_failover_storages" %}
+{{       key | upper }}: {{ (value | int) if value in (0, "0") or (value | int) != 0 else ('"' + (value | string) +  '"') }}
+{%     endif %}
+{%   endfor %}
+WALG_FAILOVER_STORAGES:
+{%   for storage_name, storage_config in walg_failover_storages_combine.walg_failover_storages.items() %}
+  {{ storage_name | upper }}:
+{%  for key, value in storage_config.items() %}
+    {{   key | upper }}: {{ (value | int) if value in (0, "0") or (value | int) != 0 else ('"' + (value | string) +  '"') }}
+{%  endfor %}
+{%   endfor %}
+{% endif %}
+
+{% if walg_statsd_combine %}
+# Statsd setup
+WALG_STATSD_ADDRESS: {{ walg_statsd_combine.walg_statsd_address }}
+{%   if walg_statsd_combine.walg_statsd_extra_tags %}
+WALG_STATSD_EXTRA_TAGS:
+{%     for key, value in walg_statsd_extra_tags.items() %}
+  {{ key }}: {{ (value | int) if value in (0, "0") or (value | int) != 0 else ('"' + (value | string) +  '"') }}
+{%     endfor %}
+{%   endif %}
+{% endif %}
+
+{% if walg_common_combine | length > 0 %}
+# Wal-G setup
+{%   for key, value in walg_common_combine.items() %}
+{{     key | upper }}: {{ (value | int) if value in (0, "0") or (value | int) != 0 else ('"' + (value | string) +  '"') }}
+{%   endfor %}
+{% endif %}

templates/walg-backup.sh.j2

@@ -8,8 +8,10 @@ set -o errexit    # Used to exit upon error, avoiding cascading errors
 set -o nounset    # Exposes unset variables
 set -o pipefail   # Unveils hidden failures
 
+{% if walg_conf_type == "env" %}
 set -o allexport
 source {{ walg_env }}
 set +o allexport
+{% endif %}
 
-{{ walg_binary }} backup-push {{ walg_data_path }}
+{{ walg_binary }} backup-push {{ walg_data_path }} {% if walg_conf_type == "yml" %}--config {{ walg_conf_yml_file }}{% endif %}

templates/walg-fetch.sh.j2

@@ -1,15 +1,27 @@
 #!/usr/bin/env bash
 {{ ansible_managed | comment }}
 
-# Created by [email protected]
-
 set -o noclobber  # Avoid overlay files (echo "hi" > foo)
 set -o errexit    # Used to exit upon error, avoiding cascading errors
 set -o nounset    # Exposes unset variables
 set -o pipefail   # Unveils hidden failures
 
+{% if walg_conf_type == "env" %}
 set -o allexport
 source {{ walg_env }}
 set +o allexport
+{% endif %}
+
+start=`date +%s%3N`
+{{ walg_binary }} wal-fetch $1 $2 {% if walg_conf_type == "yml" %}--config {{ walg_conf_yml_file }}{% endif %}
+
+end=`date +%s%3N`
+current_dt="`date +'%Y/%m/%d %H:%M:%S.%6N'`"
+echo "INFO: ${current_dt} WAL FETCH TIME: $(($end - $start)) ms"
 
-{{ walg_binary }} wal-fetch $1 $2
+{% if walg_pushgateway_host %}
+pushgateway_url="http://{{ walg_pushgateway_host }}:{{ walg_pushgateway_port }}/metrics/job/wal-g/hostname/$(hostname)"
+cat <<EOF | curl -s --data-binary @- "${pushgateway_url}" || true
+walg_wal_fetch_time $(($end - $start))
+EOF
+{% endif %}

templates/walg-list.sh.j2

@@ -8,8 +8,10 @@ set -o errexit    # Used to exit upon error, avoiding cascading errors
 set -o nounset    # Exposes unset variables
 set -o pipefail   # Unveils hidden failures
 
+{% if walg_conf_type == "env" %}
 set -o allexport
 source {{ walg_env }}
 set +o allexport
+{% endif %}
 
-{{ walg_binary }} backup-list --pretty --detail --json
+{{ walg_binary }} backup-list --pretty --detail --json {% if walg_conf_type == "yml" %}--config {{ walg_conf_yml_file }}{% endif %}