From 5198a1e4d4c99e67aa985e65fc7c42ece40f4316 Mon Sep 17 00:00:00 2001 From: Zulfikar Akbar Muzakki Date: Fri, 19 Jul 2024 11:02:25 +0700 Subject: [PATCH] Add Split DB --- deployment/.env.example | 3 + deployment/docker-compose-consumer.yml | 7 +- deployment/docker-compose-dev.yml | 24 +++- deployment/docker-compose.yml | 3 +- .../docker/g3w-suite/settings_docker.py | 11 ++ deployment/scripts/docker-entrypoint.sh | 1 + deployment/scripts/setup_suite.sh | 113 ++++++++++++++++++ 7 files changed, 156 insertions(+), 6 deletions(-) create mode 100755 deployment/scripts/setup_suite.sh diff --git a/deployment/.env.example b/deployment/.env.example index da0add2..299bd9a 100644 --- a/deployment/.env.example +++ b/deployment/.env.example @@ -5,6 +5,9 @@ WEBGIS_PUBLIC_HOSTNAME=dev.g3wsuite.it # I suggest not to use the /tmp/ folder, /tmp/ folder is cleaned at each reboot WEBGIS_DOCKER_SHARED_VOLUME=./shared-volume +# Dam Monitor path +DAM_MONITOR_CODE_PATH=../../dam_monitor + # Docker internal DB G3WSUITE_POSTGRES_USER_LOCAL=g3wsuite G3WSUITE_POSTGRES_PASS='89#kL8y3D' diff --git a/deployment/docker-compose-consumer.yml b/deployment/docker-compose-consumer.yml index 98a8c20..ff9c5ec 100644 --- a/deployment/docker-compose-consumer.yml +++ b/deployment/docker-compose-consumer.yml @@ -5,7 +5,7 @@ services: ports: - "5439:5432" environment: - - POSTGRES_DBNAME=${G3WSUITE_POSTGRES_DBNAME},data_testing,data_production + - POSTGRES_DBNAME=${G3WSUITE_POSTGRES_DBNAME},${TAILINGS_POSTGRES_DBNAME},data_testing,data_production - POSTGRES_USER=${G3WSUITE_POSTGRES_USER_LOCAL} - POSTGRES_PASS=${G3WSUITE_POSTGRES_PASS} - ALLOW_IP_RANGE=0.0.0.0/0 @@ -38,6 +38,11 @@ services: - G3WSUITE_POSTGRES_PASS - G3WSUITE_POSTGRES_HOST - G3WSUITE_POSTGRES_PORT + - TAILINGS_POSTGRES_USER + - TAILINGS_POSTGRES_USER_LOCAL + - TAILINGS_POSTGRES_PASS + - TAILINGS_POSTGRES_HOST + - TAILINGS_POSTGRES_PORT - G3WSUITE_ORS_API_ENDPOINT - G3WSUITE_ORS_API_KEY - TILESTACHE_CACHE_BUFFER_SIZE diff --git a/deployment/docker-compose-dev.yml b/deployment/docker-compose-dev.yml index 1428009..1cde012 100644 --- a/deployment/docker-compose-dev.yml +++ b/deployment/docker-compose-dev.yml @@ -5,7 +5,7 @@ services: ports: - "5439:5432" environment: - - POSTGRES_DBNAME=${G3WSUITE_POSTGRES_DBNAME},data_testing,data_production + - POSTGRES_DBNAME=${G3WSUITE_POSTGRES_DBNAME},${TAILINGS_POSTGRES_DBNAME},data_testing,data_production - POSTGRES_USER=${G3WSUITE_POSTGRES_USER_LOCAL} - POSTGRES_PASS=${G3WSUITE_POSTGRES_PASS} - ALLOW_IP_RANGE=0.0.0.0/0 @@ -27,9 +27,10 @@ services: networks: internal: - g3w-suite: - image: g3wsuite/g3w-suite:dev + build: + context: ../ + dockerfile: deployment/docker/Dockerfile environment: - DISPLAY=:99 - G3WSUITE_TILECACHE_PATH @@ -40,6 +41,12 @@ services: - G3WSUITE_POSTGRES_PASS - G3WSUITE_POSTGRES_HOST - G3WSUITE_POSTGRES_PORT + - TAILINGS_POSTGRES_DBNAME + - TAILINGS_POSTGRES_USER + - TAILINGS_POSTGRES_USER_LOCAL + - TAILINGS_POSTGRES_PASS + - TAILINGS_POSTGRES_HOST + - TAILINGS_POSTGRES_PORT - G3WSUITE_ORS_API_ENDPOINT - G3WSUITE_ORS_API_KEY - G3WSUITE_DEBUG @@ -70,8 +77,9 @@ services: - ./docker/g3w-suite/overrides/static:/code/static:ro - ./docker/g3w-suite/overrides/templates:/code/templates:ro - ./docker/g3w-suite/settings_docker.py:/code/g3w-admin/base/settings/local_settings.py - - ./scripts:/scripts + - ./scripts:/code/ci_scripts - ${G3WSUITE_LOCAL_CODE_PATH}:/code + - ${DAM_MONITOR_CODE_PATH}:/code/g3w-admin/dam_monitor - ./qgis/QGIS3.ini:/shared-volume/QGIS/QGIS3.ini entrypoint: /scripts/docker-entrypoint-dev.sh @@ -79,6 +87,14 @@ services: networks: internal: + redis: + image: redis:latest + expose: + - 6379 + networks: + internal: + restart: always + volumes: shared-volume: g3wsuite-pg-data: diff --git a/deployment/docker-compose.yml b/deployment/docker-compose.yml index 1b3974f..42dcbb0 100644 --- a/deployment/docker-compose.yml +++ b/deployment/docker-compose.yml @@ -5,7 +5,7 @@ services: ports: - "5439:5432" environment: - - POSTGRES_DBNAME=${G3WSUITE_POSTGRES_DBNAME},data_testing,data_production + - POSTGRES_DBNAME=${G3WSUITE_POSTGRES_DBNAME},${TAILINGS_POSTGRES_DBNAME},data_testing,data_production - POSTGRES_USER=${G3WSUITE_POSTGRES_USER_LOCAL} - POSTGRES_PASS=${G3WSUITE_POSTGRES_PASS} - ALLOW_IP_RANGE=0.0.0.0/0 @@ -68,6 +68,7 @@ services: - ${WEBGIS_DOCKER_SHARED_VOLUME}/node_modules:/code/node_modules - ./docker/g3w-suite/overrides/static:/code/static:ro - ./docker/g3w-suite/overrides/templates:/code/templates:ro + - ${G3WSUITE_LOCAL_CODE_PATH}:/code - ./docker/g3w-suite/settings_docker.py:/code/g3w-admin/base/settings/local_settings.py - ./secrets/pg_service.conf:${PGSERVICEFILE} - ./qgis/QGIS3.ini:/shared-volume/QGIS/QGIS3.ini diff --git a/deployment/docker/g3w-suite/settings_docker.py b/deployment/docker/g3w-suite/settings_docker.py index bf10c5d..50ae345 100644 --- a/deployment/docker/g3w-suite/settings_docker.py +++ b/deployment/docker/g3w-suite/settings_docker.py @@ -38,9 +38,20 @@ 'PASSWORD': os.getenv('G3WSUITE_POSTGRES_PASS'), 'HOST': os.getenv('G3WSUITE_POSTGRES_HOST'), 'PORT': os.getenv('G3WSUITE_POSTGRES_PORT'), + }, + 'tailings': { + 'ENGINE': 'django.contrib.gis.db.backends.postgis', + 'NAME': os.getenv('TAILINGS_POSTGRES_DBNAME'), + 'USER': os.getenv('TAILINGS_POSTGRES_USER_LOCAL') if os.getenv('TAILINGS_POSTGRES_USER_LOCAL') else "%s@%s" % ( + os.getenv('TAILINGS_POSTGRES_USER'), os.getenv('TAILINGS_POSTGRES_HOST')), + 'PASSWORD': os.getenv('TAILINGS_POSTGRES_PASS'), + 'HOST': os.getenv('TAILINGS_POSTGRES_HOST'), + 'PORT': os.getenv('TAILINGS_POSTGRES_PORT'), } } +DATABASE_ROUTERS = ['dam_monitor.router.TailingsRouter'] + MEDIA_ROOT = '/shared-volume/media/' MEDIA_URL = '/media/' STATIC_ROOT = '/shared-volume/static/' diff --git a/deployment/scripts/docker-entrypoint.sh b/deployment/scripts/docker-entrypoint.sh index 7ebd118..26d600f 100755 --- a/deployment/scripts/docker-entrypoint.sh +++ b/deployment/scripts/docker-entrypoint.sh @@ -32,6 +32,7 @@ wait-for-it -h ${G3WSUITE_REDIS_HOST:-redis} -p ${G3WSUITE_REDIS_PORT:-6379} -t # Build the suite /code/ci_scripts/build_suite.sh # Setup once +cp /scripts/setup-suite.sh /code/ci_scripts/setup_suite.sh /code/ci_scripts/setup_suite.sh gunicorn base.wsgi:application \ diff --git a/deployment/scripts/setup_suite.sh b/deployment/scripts/setup_suite.sh new file mode 100755 index 0000000..b20e872 --- /dev/null +++ b/deployment/scripts/setup_suite.sh @@ -0,0 +1,113 @@ +#!/bin/bash +# +# Setup script for g3w-suite, to be run inside the docker container +# +# Depending on the existence of "setup_done" file +# setup steps are performed, they consist in: +# - media and static re-creation from empty directories +# - Django setup +# - migrations +# - collect static +# - admin user and fake passwords +# - fixtures installation + +set -e + +CODE_DIRECTORY='/code' +DJANGO_DIRECTORY="${CODE_DIRECTORY}/g3w-admin" +DATASOURCE_PATH='/shared-volume/project_data' +MEDIA_ROOT='/shared-volume/media' +STATIC_ROOT='/shared-volume/static' +PROJECTS_DIR="${MEDIA_ROOT}/projects" +SECRET_KEY_FILE='/shared-volume/.secret_key' +SETUP_DONE_FILE='/shared-volume/setup_done' + +cd "${CODE_DIRECTORY}" + + +if [ ! -e ${SETUP_DONE_FILE} ]; then + echo "Setup started for G3W-Suite installation ..." + + if [ ! -e ./g3w-admin/base/settings/local_settings.py ]; then + echo "Copying docker_settings.py to base/settings/local_settings.py" + cp ./settings_docker.py ./g3w-admin/base/settings/local_settings.py + fi + + echo "Cleaning up some dirs before collecting statics ..." + ls ${MEDIA_ROOT} || mkdir ${MEDIA_ROOT} + pushd . + cd ${MEDIA_ROOT}/.. + ls static || ln -s media static + popd + + echo "Creating projects(_data) directores in ${PROJECTS_DIR} ..." + ls ${PROJECTS_DIR} || mkdir ${PROJECTS_DIR} + ls ${DATASOURCE_PATH} || mkdir ${DATASOURCE_PATH} + + wait-for-it -h ${G3WSUITE_POSTGRES_HOST:-postgis} -p ${G3WSUITE_POSTGRES_PORT:-5432} -t 60 + + pushd . + cd ${DJANGO_DIRECTORY}/core/static + rm -rf bower_components + ln -s "/code/node_modules/@bower_components" bower_components + popd + + echo "Creating a unique SECRET_KEY file ..." + python3 "${DJANGO_DIRECTORY}/manage.py" generate_secret_key_file -o ${SECRET_KEY_FILE} + + cd ${DJANGO_DIRECTORY} + if [[ -z ${G3WSUITE_DEBUG} || ${G3WSUITE_DEBUG} != "True" ]]; then + rm -rf ${STATIC_ROOT} + python3 manage.py collectstatic --noinput -v 0 + fi + python3 manage.py migrate --noinput + + echo "Installing fixtures ..." + for FIXTURE in 'BaseLayer.json' 'G3WGeneralDataSuite.json' 'G3WMapControls.json' 'G3WSpatialRefSys.json'; do + python3 manage.py loaddata core/fixtures/${FIXTURE} + done + # sync menu tree items + python3 manage.py sitetree_resync_apps + python3 manage.py createsuperuser --noinput --username admin --email admin@email.com || true + # Set fake password for all users + python3 manage.py set_passwords --password admin + + # For django-file-form: create /temp_uploads if not exists + ls ${MEDIA_ROOT}/temp_uploads || mkdir ${MEDIA_ROOT}/temp_uploads + + + python3 manage.py migrate --database=${TAILINGS_POSTGRES_DBNAME} --noinput + + + touch ${SETUP_DONE_FILE} + echo "Setup completed ..." +else + echo "Setup was already done, skipping ..." + # Wait for postgis + wait-for-it -h ${G3WSUITE_POSTGRES_HOST:-postgis} -p ${G3WSUITE_POSTGRES_PORT:-5432} -t 60 + + # Restore on restart ln -s for bower_component + cd ${DJANGO_DIRECTORY}/core/static + rm -rf bower_components + ln -s "/code/node_modules/@bower_components" bower_components + + # Create SECRET_KEY value file if not exists + if [ ! -e ${SECRET_KEY_FILE} ]; then + echo "Creating a unique SECRET_KEY file ..." + python3 "${DJANGO_DIRECTORY}/manage.py" generate_secret_key_file -o ${SECRET_KEY_FILE} + fi + + rm -rf ${STATIC_ROOT} + cd ${DJANGO_DIRECTORY} + if [[ -z ${G3WSUITE_DEBUG} || ${G3WSUITE_DEBUG} != "True" ]]; then + python3 manage.py collectstatic --noinput -v 0 + fi + python3 manage.py migrate --noinput + python3 manage.py sitetree_resync_apps + python3 manage.py migrate --database=${TAILINGS_POSTGRES_DBNAME} --noinput +fi + +# Make sure data are readable: +chmod -R 777 ${DATASOURCE_PATH} + +echo "end setup_suite" \ No newline at end of file