Skip to content

Commit

Permalink
Add Split DB
Browse files Browse the repository at this point in the history
  • Loading branch information
zamuzakki committed Jul 19, 2024
1 parent f0c7f4e commit 5198a1e
Show file tree
Hide file tree
Showing 7 changed files with 156 additions and 6 deletions.
3 changes: 3 additions & 0 deletions deployment/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ WEBGIS_PUBLIC_HOSTNAME=dev.g3wsuite.it
# I suggest not to use the /tmp/ folder, /tmp/ folder is cleaned at each reboot
WEBGIS_DOCKER_SHARED_VOLUME=./shared-volume

# Dam Monitor path
DAM_MONITOR_CODE_PATH=../../dam_monitor

# Docker internal DB
G3WSUITE_POSTGRES_USER_LOCAL=g3wsuite
G3WSUITE_POSTGRES_PASS='89#kL8y3D'
Expand Down
7 changes: 6 additions & 1 deletion deployment/docker-compose-consumer.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ services:
ports:
- "5439:5432"
environment:
- POSTGRES_DBNAME=${G3WSUITE_POSTGRES_DBNAME},data_testing,data_production
- POSTGRES_DBNAME=${G3WSUITE_POSTGRES_DBNAME},${TAILINGS_POSTGRES_DBNAME},data_testing,data_production
- POSTGRES_USER=${G3WSUITE_POSTGRES_USER_LOCAL}
- POSTGRES_PASS=${G3WSUITE_POSTGRES_PASS}
- ALLOW_IP_RANGE=0.0.0.0/0
Expand Down Expand Up @@ -38,6 +38,11 @@ services:
- G3WSUITE_POSTGRES_PASS
- G3WSUITE_POSTGRES_HOST
- G3WSUITE_POSTGRES_PORT
- TAILINGS_POSTGRES_USER
- TAILINGS_POSTGRES_USER_LOCAL
- TAILINGS_POSTGRES_PASS
- TAILINGS_POSTGRES_HOST
- TAILINGS_POSTGRES_PORT
- G3WSUITE_ORS_API_ENDPOINT
- G3WSUITE_ORS_API_KEY
- TILESTACHE_CACHE_BUFFER_SIZE
Expand Down
24 changes: 20 additions & 4 deletions deployment/docker-compose-dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ services:
ports:
- "5439:5432"
environment:
- POSTGRES_DBNAME=${G3WSUITE_POSTGRES_DBNAME},data_testing,data_production
- POSTGRES_DBNAME=${G3WSUITE_POSTGRES_DBNAME},${TAILINGS_POSTGRES_DBNAME},data_testing,data_production
- POSTGRES_USER=${G3WSUITE_POSTGRES_USER_LOCAL}
- POSTGRES_PASS=${G3WSUITE_POSTGRES_PASS}
- ALLOW_IP_RANGE=0.0.0.0/0
Expand All @@ -27,9 +27,10 @@ services:
networks:
internal:


g3w-suite:
image: g3wsuite/g3w-suite:dev
build:
context: ../
dockerfile: deployment/docker/Dockerfile
environment:
- DISPLAY=:99
- G3WSUITE_TILECACHE_PATH
Expand All @@ -40,6 +41,12 @@ services:
- G3WSUITE_POSTGRES_PASS
- G3WSUITE_POSTGRES_HOST
- G3WSUITE_POSTGRES_PORT
- TAILINGS_POSTGRES_DBNAME
- TAILINGS_POSTGRES_USER
- TAILINGS_POSTGRES_USER_LOCAL
- TAILINGS_POSTGRES_PASS
- TAILINGS_POSTGRES_HOST
- TAILINGS_POSTGRES_PORT
- G3WSUITE_ORS_API_ENDPOINT
- G3WSUITE_ORS_API_KEY
- G3WSUITE_DEBUG
Expand Down Expand Up @@ -70,15 +77,24 @@ services:
- ./docker/g3w-suite/overrides/static:/code/static:ro
- ./docker/g3w-suite/overrides/templates:/code/templates:ro
- ./docker/g3w-suite/settings_docker.py:/code/g3w-admin/base/settings/local_settings.py
- ./scripts:/scripts
- ./scripts:/code/ci_scripts
- ${G3WSUITE_LOCAL_CODE_PATH}:/code
- ${DAM_MONITOR_CODE_PATH}:/code/g3w-admin/dam_monitor
- ./qgis/QGIS3.ini:/shared-volume/QGIS/QGIS3.ini

entrypoint: /scripts/docker-entrypoint-dev.sh

networks:
internal:

redis:
image: redis:latest
expose:
- 6379
networks:
internal:
restart: always

volumes:
shared-volume:
g3wsuite-pg-data:
Expand Down
3 changes: 2 additions & 1 deletion deployment/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ services:
ports:
- "5439:5432"
environment:
- POSTGRES_DBNAME=${G3WSUITE_POSTGRES_DBNAME},data_testing,data_production
- POSTGRES_DBNAME=${G3WSUITE_POSTGRES_DBNAME},${TAILINGS_POSTGRES_DBNAME},data_testing,data_production
- POSTGRES_USER=${G3WSUITE_POSTGRES_USER_LOCAL}
- POSTGRES_PASS=${G3WSUITE_POSTGRES_PASS}
- ALLOW_IP_RANGE=0.0.0.0/0
Expand Down Expand Up @@ -68,6 +68,7 @@ services:
- ${WEBGIS_DOCKER_SHARED_VOLUME}/node_modules:/code/node_modules
- ./docker/g3w-suite/overrides/static:/code/static:ro
- ./docker/g3w-suite/overrides/templates:/code/templates:ro
- ${G3WSUITE_LOCAL_CODE_PATH}:/code
- ./docker/g3w-suite/settings_docker.py:/code/g3w-admin/base/settings/local_settings.py
- ./secrets/pg_service.conf:${PGSERVICEFILE}
- ./qgis/QGIS3.ini:/shared-volume/QGIS/QGIS3.ini
Expand Down
11 changes: 11 additions & 0 deletions deployment/docker/g3w-suite/settings_docker.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,20 @@
'PASSWORD': os.getenv('G3WSUITE_POSTGRES_PASS'),
'HOST': os.getenv('G3WSUITE_POSTGRES_HOST'),
'PORT': os.getenv('G3WSUITE_POSTGRES_PORT'),
},
'tailings': {
'ENGINE': 'django.contrib.gis.db.backends.postgis',
'NAME': os.getenv('TAILINGS_POSTGRES_DBNAME'),
'USER': os.getenv('TAILINGS_POSTGRES_USER_LOCAL') if os.getenv('TAILINGS_POSTGRES_USER_LOCAL') else "%s@%s" % (
os.getenv('TAILINGS_POSTGRES_USER'), os.getenv('TAILINGS_POSTGRES_HOST')),
'PASSWORD': os.getenv('TAILINGS_POSTGRES_PASS'),
'HOST': os.getenv('TAILINGS_POSTGRES_HOST'),
'PORT': os.getenv('TAILINGS_POSTGRES_PORT'),
}
}

DATABASE_ROUTERS = ['dam_monitor.router.TailingsRouter']

MEDIA_ROOT = '/shared-volume/media/'
MEDIA_URL = '/media/'
STATIC_ROOT = '/shared-volume/static/'
Expand Down
1 change: 1 addition & 0 deletions deployment/scripts/docker-entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ wait-for-it -h ${G3WSUITE_REDIS_HOST:-redis} -p ${G3WSUITE_REDIS_PORT:-6379} -t
# Build the suite
/code/ci_scripts/build_suite.sh
# Setup once
cp /scripts/setup-suite.sh /code/ci_scripts/setup_suite.sh
/code/ci_scripts/setup_suite.sh

gunicorn base.wsgi:application \
Expand Down
113 changes: 113 additions & 0 deletions deployment/scripts/setup_suite.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
#!/bin/bash
#
# Setup script for g3w-suite, to be run inside the docker container
#
# Depending on the existence of "setup_done" file
# setup steps are performed, they consist in:
# - media and static re-creation from empty directories
# - Django setup
# - migrations
# - collect static
# - admin user and fake passwords
# - fixtures installation

set -e

CODE_DIRECTORY='/code'
DJANGO_DIRECTORY="${CODE_DIRECTORY}/g3w-admin"
DATASOURCE_PATH='/shared-volume/project_data'
MEDIA_ROOT='/shared-volume/media'
STATIC_ROOT='/shared-volume/static'
PROJECTS_DIR="${MEDIA_ROOT}/projects"
SECRET_KEY_FILE='/shared-volume/.secret_key'
SETUP_DONE_FILE='/shared-volume/setup_done'

cd "${CODE_DIRECTORY}"


if [ ! -e ${SETUP_DONE_FILE} ]; then
echo "Setup started for G3W-Suite installation ..."

if [ ! -e ./g3w-admin/base/settings/local_settings.py ]; then
echo "Copying docker_settings.py to base/settings/local_settings.py"
cp ./settings_docker.py ./g3w-admin/base/settings/local_settings.py
fi

echo "Cleaning up some dirs before collecting statics ..."
ls ${MEDIA_ROOT} || mkdir ${MEDIA_ROOT}
pushd .
cd ${MEDIA_ROOT}/..
ls static || ln -s media static
popd

echo "Creating projects(_data) directores in ${PROJECTS_DIR} ..."
ls ${PROJECTS_DIR} || mkdir ${PROJECTS_DIR}
ls ${DATASOURCE_PATH} || mkdir ${DATASOURCE_PATH}

wait-for-it -h ${G3WSUITE_POSTGRES_HOST:-postgis} -p ${G3WSUITE_POSTGRES_PORT:-5432} -t 60

pushd .
cd ${DJANGO_DIRECTORY}/core/static
rm -rf bower_components
ln -s "/code/node_modules/@bower_components" bower_components
popd

echo "Creating a unique SECRET_KEY file ..."
python3 "${DJANGO_DIRECTORY}/manage.py" generate_secret_key_file -o ${SECRET_KEY_FILE}

cd ${DJANGO_DIRECTORY}
if [[ -z ${G3WSUITE_DEBUG} || ${G3WSUITE_DEBUG} != "True" ]]; then
rm -rf ${STATIC_ROOT}
python3 manage.py collectstatic --noinput -v 0
fi
python3 manage.py migrate --noinput

echo "Installing fixtures ..."
for FIXTURE in 'BaseLayer.json' 'G3WGeneralDataSuite.json' 'G3WMapControls.json' 'G3WSpatialRefSys.json'; do
python3 manage.py loaddata core/fixtures/${FIXTURE}
done
# sync menu tree items
python3 manage.py sitetree_resync_apps
python3 manage.py createsuperuser --noinput --username admin --email [email protected] || true
# Set fake password for all users
python3 manage.py set_passwords --password admin

# For django-file-form: create <media_directory>/temp_uploads if not exists
ls ${MEDIA_ROOT}/temp_uploads || mkdir ${MEDIA_ROOT}/temp_uploads


python3 manage.py migrate --database=${TAILINGS_POSTGRES_DBNAME} --noinput


touch ${SETUP_DONE_FILE}
echo "Setup completed ..."
else
echo "Setup was already done, skipping ..."
# Wait for postgis
wait-for-it -h ${G3WSUITE_POSTGRES_HOST:-postgis} -p ${G3WSUITE_POSTGRES_PORT:-5432} -t 60

# Restore on restart ln -s for bower_component
cd ${DJANGO_DIRECTORY}/core/static
rm -rf bower_components
ln -s "/code/node_modules/@bower_components" bower_components

# Create SECRET_KEY value file if not exists
if [ ! -e ${SECRET_KEY_FILE} ]; then
echo "Creating a unique SECRET_KEY file ..."
python3 "${DJANGO_DIRECTORY}/manage.py" generate_secret_key_file -o ${SECRET_KEY_FILE}
fi

rm -rf ${STATIC_ROOT}
cd ${DJANGO_DIRECTORY}
if [[ -z ${G3WSUITE_DEBUG} || ${G3WSUITE_DEBUG} != "True" ]]; then
python3 manage.py collectstatic --noinput -v 0
fi
python3 manage.py migrate --noinput
python3 manage.py sitetree_resync_apps
python3 manage.py migrate --database=${TAILINGS_POSTGRES_DBNAME} --noinput
fi

# Make sure data are readable:
chmod -R 777 ${DATASOURCE_PATH}

echo "end setup_suite"

0 comments on commit 5198a1e

Please sign in to comment.