From 90865190837cd86f011cd6e2283e597566da6af7 Mon Sep 17 00:00:00 2001 From: Nicky Gerritsen Date: Sun, 21 Aug 2022 20:16:54 +0200 Subject: [PATCH] Allow to set up load balancing when installing DOMjudge. --- provision-contest/ansible/domserver.yml | 12 ++--- .../ansible/group_vars/all/all.yml | 4 ++ .../ansible/group_vars/all/secret.yml.example | 4 ++ .../ansible/roles/domserver/tasks/main.yml | 16 +++++-- .../domserver/templates/dbpasswords.secret.j2 | 4 ++ .../templates/nginx-domjudge-inner.j2 | 5 +++ .../templates/nginx-domjudge.conf.j2 | 45 +++++++++++++++++++ .../ansible/roles/mysql_server/tasks/main.yml | 20 +++++++++ 8 files changed, 101 insertions(+), 9 deletions(-) diff --git a/provision-contest/ansible/domserver.yml b/provision-contest/ansible/domserver.yml index e1c5b75d..4f830b0e 100644 --- a/provision-contest/ansible/domserver.yml +++ b/provision-contest/ansible/domserver.yml @@ -26,18 +26,18 @@ tags: ssh - role: mysql_server tags: mysql_server - - role: domjudge_checkout - tags: domjudge_checkout - - role: domjudge_build - tags: domjudge_build - - role: domserver - tags: domserver - role: mysql_replication tags: mysql_replication when: REPLICATION_PASSWORD is defined - role: keepalived tags: keepalived when: KEEPALIVED_PRIORITY is defined + - role: domjudge_checkout + tags: domjudge_checkout + - role: domjudge_build + tags: domjudge_build + - role: domserver + tags: domserver - role: prometheus_target_web tags: prometheus_target_web vars: diff --git a/provision-contest/ansible/group_vars/all/all.yml b/provision-contest/ansible/group_vars/all/all.yml index 47de5730..8642d909 100644 --- a/provision-contest/ansible/group_vars/all/all.yml +++ b/provision-contest/ansible/group_vars/all/all.yml @@ -26,6 +26,10 @@ ICPC_IMAGE: false # server. WF_RESTRICTED_NETWORK: false +# Set this to true to enable load balancing instead of (automatic) failover +# DBA_PASSWORD needs to be set for this in secret.yml +DOMSERVER_LOADBALANCING: false + TIMEZONE: "Europe/Moscow" PHP_FPM_MAX_CHILDREN: 400 diff --git a/provision-contest/ansible/group_vars/all/secret.yml.example b/provision-contest/ansible/group_vars/all/secret.yml.example index b166b42b..aacf72d8 100644 --- a/provision-contest/ansible/group_vars/all/secret.yml.example +++ b/provision-contest/ansible/group_vars/all/secret.yml.example @@ -2,6 +2,10 @@ # Set this to enable master-master replication between two domservers. #REPLICATION_PASSWORD: some-replication-password +# Database administrator password. Needed when DOMSERVER_LOADBALANCING is true +# Note: this user will have access from the whole SERVER_IP_PREFIX range +#DBA_PASSWORD: some-dba-password + # Database user password. DB_PASSWORD: some-database-password diff --git a/provision-contest/ansible/roles/domserver/tasks/main.yml b/provision-contest/ansible/roles/domserver/tasks/main.yml index db44a25e..3b8ba36d 100644 --- a/provision-contest/ansible/roles/domserver/tasks/main.yml +++ b/provision-contest/ansible/roles/domserver/tasks/main.yml @@ -19,16 +19,26 @@ owner: domjudge notify: fix permissions on domjudge inplace-install +- name: set the DBA credentials + set_fact: + dba_credentials: | + {% if DBA_PASSWORD is defined %} + -u domjudge_dba -p {{ DBA_PASSWORD }} + {% else %} + -u root + {% endif %} + # When using replication, the DB will be dropped and recreated on the slave later. - name: check if the database is configured - command: "{{ DJ_DIR }}/bin/dj_setup_database -u root status" + command: "{{ DJ_DIR }}/bin/dj_setup_database {{ dba_credentials }} status" register: db_status ignore_errors: true changed_when: false + when: not DOMSERVER_LOADBALANCING or groups['domserver'][0] == inventory_hostname - name: make sure the database is configured - command: "{{ DJ_DIR }}/bin/dj_setup_database -u root bare-install" - when: "'failed' in db_status.stdout" + command: "{{ DJ_DIR }}/bin/dj_setup_database {{ dba_credentials }} bare-install" + when: "(not DOMSERVER_LOADBALANCING or groups['domserver'][0] == inventory_hostname) and 'failed' in db_status.stdout" - name: install required packages apt: diff --git a/provision-contest/ansible/roles/domserver/templates/dbpasswords.secret.j2 b/provision-contest/ansible/roles/domserver/templates/dbpasswords.secret.j2 index 15706d84..711b48a1 100644 --- a/provision-contest/ansible/roles/domserver/templates/dbpasswords.secret.j2 +++ b/provision-contest/ansible/roles/domserver/templates/dbpasswords.secret.j2 @@ -1,3 +1,7 @@ # {{ansible_managed}} # Format: 'unused:::::' +{% if DOMSERVER_LOADBALANCING %} +unused:{{DOMSERVER_IP}}:domjudge:domjudge:{{DB_PASSWORD}}:3306 +{% else %} unused:localhost:domjudge:domjudge:{{DB_PASSWORD}}:3306 +{% endif %} diff --git a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge-inner.j2 b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge-inner.j2 index 2d19afe6..21ca9b2e 100644 --- a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge-inner.j2 +++ b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge-inner.j2 @@ -11,6 +11,11 @@ set $domjudgeRoot {{ DJ_DIR }}/webapp/public; set $prefix ''; location / { +{% if DOMSERVER_LOADBALANCING %} + if ($access_allowed = false) { + return 403; + } +{% endif %} root $domjudgeRoot; try_files $uri @domjudgeFront; } diff --git a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2 b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2 index b80a9bd0..13b9b95d 100644 --- a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2 +++ b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2 @@ -7,6 +7,36 @@ upstream domjudge { server unix:/var/run/php-fpm-domjudge.sock; # if using with etc/domjudge-fpm.conf } +{% if DOMSERVER_LOADBALANCING %} +upstream domjudge-loadbalanced { + least_conn; +{% for host in groups['domserver'] %} + server {{ hostvars[host].ansible_host }}:81; +{% endfor %} +} + +server { + listen 81; + listen [::]:81; + server_name _default_; + + add_header Strict-Transport-Security max-age=31556952; + include /etc/nginx/snippets/domjudge-inner; + + set_real_ip_from {{ SERVER_IP_PREFIX }}.0/24; + real_ip_header X-Forwarded-For; + real_ip_recursive on; +} + +map $realip_remote_addr $access_allowed { + default false; +{% for host in groups['domserver'] %} + {{ hostvars[host].ansible_host }} true; +{% endfor %} +} + +{% endif %} + server { listen 80; listen [::]:80; @@ -25,5 +55,20 @@ server { add_header Strict-Transport-Security max-age=31556952; +{% if DOMSERVER_LOADBALANCING %} + location / { + proxy_pass http://domjudge-loadbalanced; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_request_buffering off; + proxy_buffering off; + } +{% else %} include /etc/nginx/snippets/domjudge-inner; +{% endif %} } diff --git a/provision-contest/ansible/roles/mysql_server/tasks/main.yml b/provision-contest/ansible/roles/mysql_server/tasks/main.yml index 26479b04..21d8249c 100644 --- a/provision-contest/ansible/roles/mysql_server/tasks/main.yml +++ b/provision-contest/ansible/roles/mysql_server/tasks/main.yml @@ -67,3 +67,23 @@ loop: - load-db - dump-db + +- name: create mysql user for for DOMjudge database administration + mysql_user: + name: domjudge_dba + host: '{{ SERVER_IP_PREFIX }}.%' + password: "{{ DBA_PASSWORD }}" + append_privs: true + priv: 'domjudge.*:ALL,GRANT/*.*:CREATE USER,RELOAD' + state: present + when: DBA_PASSWORD is defined + +- name: create mysql user for for DOMjudge when we are doing loadbalancing + mysql_user: + name: domjudge + host: '{{ SERVER_IP_PREFIX }}.%' + password: "{{ DB_PASSWORD }}" + append_privs: true + priv: 'domjudge.*:SELECT,INSERT,UPDATE,DELETE' + state: present + when: DOMSERVER_LOADBALANCING