From ca791827a4cd6e720d08c907924efefee3c1802b Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Mon, 10 May 2021 16:26:37 +0200 Subject: [PATCH 01/52] implementation of datapusher/datastore plugins --- ckan-docker/Dockerfile | 22 ++++++++++------------ ckan-docker/ckan-entrypoint.sh | 21 +++++++++++++-------- ckan-docker/docker-compose.yml | 6 +++--- 3 files changed, 26 insertions(+), 23 deletions(-) diff --git a/ckan-docker/Dockerfile b/ckan-docker/Dockerfile index e77e759..bf1fbf1 100644 --- a/ckan-docker/Dockerfile +++ b/ckan-docker/Dockerfile @@ -1,6 +1,6 @@ # See CKAN docs on installation from Docker Compose on usage FROM debian:buster -MAINTAINER Open Knowledge +LABEL Open Knowledge EXPOSE 5000 @@ -41,10 +41,10 @@ ENV TENANT_ID 00000000-0000-0000-0000-000000000000 ENV CLIENT_ID 00000000-0000-0000-0000-000000000000 ENV CLIENT_SECRET pKf_0000000_00000-0000-0000 +# Build-time variables specified by docker-compose.yml / .env ARG TENANT_ID ARG CLIENT_ID ARG CLIENT_SECRET -# Build-time variables specified by docker-compose.yml / .env ARG CKAN_SITE_URL # Create ckan user @@ -54,16 +54,7 @@ RUN useradd -r -u 900 -m -c "ckan account" -d $CKAN_HOME -s /bin/bash ckan RUN mkdir -p $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH && \ python3 -m venv $CKAN_VENV -# ln -s $CKAN_VENV/bin/pip /usr/local/bin/ckan-pip &&\ -# ln -s $CKAN_VENV/bin/ckan /usr/local/bin/ckan - -# Setup CKAN -#ADD . $CKAN_VENV/src/ckan/ COPY ./ckan_copy $CKAN_VENV/src/ckan/ -COPY ./ckan-entrypoint.sh /ckan-entrypoint.sh -COPY ./ckan-run.sh /ckan-run.sh -COPY ./gdb.commands /gdb.commands -COPY ./capture_gdb.sh /capture_gdb.sh RUN $CKAN_VENV/bin/pip install -U pip && \ $CKAN_VENV/bin/pip install --upgrade --no-cache-dir -r $CKAN_VENV/src/ckan/requirement-setuptools.txt && \ @@ -71,7 +62,6 @@ RUN $CKAN_VENV/bin/pip install -U pip && \ $CKAN_VENV/bin/pip install --upgrade --no-cache-dir -r $CKAN_VENV/src/ckan/dev-requirements.txt && \ $CKAN_VENV/bin/pip install -e $CKAN_VENV/src/ckan/ && \ ln -s $CKAN_VENV/src/ckan/ckan/config/who.ini $CKAN_CONFIG/who.ini && \ - chmod +x /ckan-entrypoint.sh /capture_gdb.sh /ckan-run.sh && \ chown -R ckan:ckan $CKAN_HOME $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH RUN $CKAN_VENV/bin/pip install -e git+https://github.com/geosolutions-it/ckanext-c195.git@main#egg=ckanext-c195 @@ -80,6 +70,14 @@ RUN $CKAN_VENV/bin/pip install -e git+https://github.com/ckan/ckanext-dcat.git@m RUN $CKAN_VENV/bin/pip install -e git+https://github.com/geosolutions-it/ckanext-azure-auth.git@main#egg=ckanext-azure-auth && \ $CKAN_VENV/bin/pip install -r $CKAN_VENV/src/ckanext-azure-auth/requirements.txt +# Setup CKAN custom scripts + +COPY ./ckan-entrypoint.sh /ckan-entrypoint.sh +COPY ./ckan-run.sh /ckan-run.sh +COPY ./gdb.commands /gdb.commands +COPY ./capture_gdb.sh /capture_gdb.sh +RUN chmod +x /ckan-entrypoint.sh /capture_gdb.sh /ckan-run.sh + #USER ckan SHELL ["/bin/bash", "-c"] diff --git a/ckan-docker/ckan-entrypoint.sh b/ckan-docker/ckan-entrypoint.sh index bb1ee7b..8a3783a 100644 --- a/ckan-docker/ckan-entrypoint.sh +++ b/ckan-docker/ckan-entrypoint.sh @@ -64,13 +64,8 @@ crudini --set --verbose --list --list-sep=\ ${CONFIG_INI} app:main ckan.plugins crudini --set --verbose --list --list-sep=\ ${CONFIG_INI} app:main ckan.plugins dcat_json_interface crudini --set --verbose --list --list-sep=\ ${CONFIG_INI} app:main ckan.plugins structured_data crudini --set --verbose --list --list-sep=\ ${CONFIG_INI} app:main ckan.plugins azure_auth - - -crudini --set --verbose ${CONFIG_INI} app:main sqlalchemy.pool_size 10 -crudini --set --verbose ${CONFIG_INI} app:main sqlalchemy.echo_pool True -crudini --set --verbose ${CONFIG_INI} app:main sqlalchemy.pool_pre_ping True -crudini --set --verbose ${CONFIG_INI} app:main sqlalchemy.pool_reset_on_return rollback -crudini --set --verbose ${CONFIG_INI} app:main sqlalchemy.pool_timeout 30 +crudini --set --verbose --list --list-sep=\ ${CONFIG_INI} app:main ckan.plugins datastore +crudini --set --verbose --list --list-sep=\ ${CONFIG_INI} app:main ckan.plugins datapusher crudini --set --verbose ${CONFIG_INI} DEFAULT debug True @@ -80,6 +75,12 @@ crudini --set --verbose ${CONFIG_INI} logger_ckan level DEBUG crudini --set --verbose ${CONFIG_INI} logger_ckanext level DEBUG crudini --set --verbose ${CONFIG_INI} handler_console level DEBUG +crudini --set --verbose ${CONFIG_INI} app:main sqlalchemy.pool_size 10 +crudini --set --verbose ${CONFIG_INI} app:main sqlalchemy.echo_pool True +crudini --set --verbose ${CONFIG_INI} app:main sqlalchemy.pool_pre_ping True +crudini --set --verbose ${CONFIG_INI} app:main sqlalchemy.pool_reset_on_return rollback +crudini --set --verbose ${CONFIG_INI} app:main sqlalchemy.pool_timeout 30 + #Azure auth plugin https://github.com/geosolutions-it/ckanext-azure-auth.git crudini --set --verbose ${CONFIG_INI} app:main ckanext.azure_auth.tenant_id ${TENANT_ID} @@ -89,7 +90,11 @@ crudini --set --verbose ${CONFIG_INI} app:main ckanext.azure_auth.client_secret crudini --set --verbose ${CONFIG_INI} app:main ckanext.azure_auth.auth_callback_path /azure/callback crudini --set --verbose ${CONFIG_INI} app:main ckanext.azure_auth.allow_create_users True -# END changes to the ini file +#Configure datastore SQL functions + +$CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions | grep -v ^"$(date +%Y-%m-%d)" | psql ${CKAN_DATASTORE_WRITE_URL} --set ON_ERROR_STOP=1 + +# END changes to the ini file # Get or create CKAN_SQLALCHEMY_URL if [ -z "$CKAN_SQLALCHEMY_URL" ]; then diff --git a/ckan-docker/docker-compose.yml b/ckan-docker/docker-compose.yml index 1b669e5..7434cfe 100644 --- a/ckan-docker/docker-compose.yml +++ b/ckan-docker/docker-compose.yml @@ -3,8 +3,8 @@ version: "3" volumes: - ckan_config: - ckan_home: + # ckan_config: + # ckan_home: ckan_storage: pg_data: solr_data: @@ -32,7 +32,7 @@ services: # Defaults work with linked containers, change to use own Postgres, SolR, Redis or Datapusher - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@ckan_db/ckan - CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@ckan_db/datastore - - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore + - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@ckan_db/datastore - CKAN_SOLR_URL=http://ckan_solr:8983/solr/ckan - CKAN_REDIS_URL=redis://ckan_redis:6379/1 - CKAN_DATAPUSHER_URL=http://datapusher:8800 From 64a30fec24767509cbeec7b6ef0f42e96482c7d2 Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Mon, 10 May 2021 16:29:10 +0200 Subject: [PATCH 02/52] implementation of datapusher container in azure compose, fixed readme --- azure/resourcegroup_deployment/README.md | 24 +++++++++++-------- .../ckan-compose/docker-compose.yml | 13 ++++++++-- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/azure/resourcegroup_deployment/README.md b/azure/resourcegroup_deployment/README.md index a190e25..1896ada 100644 --- a/azure/resourcegroup_deployment/README.md +++ b/azure/resourcegroup_deployment/README.md @@ -39,49 +39,53 @@ Here is a partial list: Partial customization can be also be done in file `setenv.sh`, for vars not extracted from the `parameters.json` file, but it's not needed. It is recommended **not** to modify the params in the following list, because they are considered as fixed values in some scripts: -- `param_postgres_username = ckan` +- `param_postgres_username = ckan` -## Deploy +## Deploy -- Deploy main resources in Azure. +- Deploy main resources in Azure. - This command will take up to 20-25 minutes to complete. - Run locally: + ```bash ./azure_main_deploy.sh ``` - Install docker stuff on VM and create images. - Run locally: + ```bash ./azure_ckan_vm_config.sh ``` - Create configuration from local files. - This script will also retrieve some info from Azure, so it's not immediate, but should be quite fast anyway. - - Run locally + - Run locally + ```bash env -i ./az_config_env.sh ``` + - Copy configuration to VM - Previous script should have printed a full `scp` command line. Run it locally to copy local generated configuration file to VM. - - Create CKAN DB, restart services - Create CKAN DBs and assign privs, restart containers in VM (solr, ckan, nginx) - - This command above is idempotent and can be run several times, due to a current bug in Azure CLI (https://github.com/Azure/azure-cli/issues/16705) this script may be needed to be run more than one for solr to be configured correctly. - Run locally (calls `az` commands) + ```bash ./azure_solr_config.sh ``` - Create API key 1. login into CKAN as admin - 1. go into admin / manage + 1. go into admin / manage 1. regenerate API key - Load initial datasets - Run the script + ```bash ./000_provision_initial_data.sh ``` @@ -103,9 +107,9 @@ To ensure CKAN is always responding, there's a script named `check_ckan_alive.sh date=$(date '+%Y-%m-%d %H:%M:%S') response="$(curl -I -s http://localhost:5000/ --max-time 10 --connect-timeout 10 | head -1 | tr -d '\r')" if [ "$response" != 'HTTP/1.0 200 OK' ]; then - docker exec -i ckan /capture_gdb.sh - docker restart ckan - echo "$date - restarted ckan because it was stuck" >> $HOME/ckan_restart_log + docker exec -i ckan /capture_gdb.sh + docker restart ckan + echo "$date - restarted ckan because it was stuck" >> $HOME/ckan_restart_log fi ``` diff --git a/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml b/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml index d838c25..0eeb0c5 100644 --- a/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml +++ b/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml @@ -4,6 +4,7 @@ version: "3" services: + ckan: container_name: ckan image: ${REGISTRY_NAME}.azurecr.io/crea_ckan @@ -17,7 +18,7 @@ services: environment: # Defaults work with linked containers, change to use own Postgres, SolR, Redis or Datapusher - CKAN_SQLALCHEMY_URL=postgresql://${CKAN_PG_USER}:${POSTGRES_PASSWORD}@${PG_HOST_FULL}/ckan - - CKAN_DATASTORE_WRITE_URL="postgresql://${CKAN_PG_USER}:${POSTGRES_PASSWORD}@${PG_HOST_FULL}/datastore" + - CKAN_DATASTORE_WRITE_URL=postgresql://${CKAN_PG_USER}:${POSTGRES_PASSWORD}@${PG_HOST_FULL}/datastore - CKAN_DATASTORE_READ_URL=postgresql://${DATASTORE_RO_PG_USER}:${DATASTORE_READONLY_PASSWORD}@${PG_HOST_FULL}/datastore - CKAN_SOLR_URL=http://${SOLR_HOST_FULL}:8983/solr/ckan - CKAN_REDIS_URL=redis://default:${REDIS_AUTHKEY}@${REDIS_HOST_FULL}:6379/1 @@ -28,6 +29,13 @@ services: - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} volumes: - ${CKAN_SHARE_MOUNT}:/var/lib/ckan + + datapusher: + container_name: ckan_datapusher + image: clementmouchet/datapusher + ports: + - "8800:8800" + ckan_solr: image: ${REGISTRY_NAME}.azurecr.io/crea_ckan_solr container_name: ckan_solr @@ -35,7 +43,8 @@ services: expose: - "8983" volumes: - - ${SOLR_SHARE_MOUNT}:/opt/solr/server/solr/ckan/data + - ${SOLR_SHARE_MOUNT}:/opt/solr/server/solr/ckan/data + proxy: image: ghcr.io/linuxserver/swag container_name: proxy From d2ba2b452961971ec96716ae5f998b8cd5107e7a Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Mon, 10 May 2021 18:41:49 +0200 Subject: [PATCH 03/52] added new datapusher docker image, configured datastore, datapusher ckan plugins, initted datastore db --- ckan-docker/ckan-entrypoint.sh | 1 + ckan-docker/datapusher_copy | 1 + ckan-docker/docker-compose.yml | 6 +++++- 3 files changed, 7 insertions(+), 1 deletion(-) create mode 160000 ckan-docker/datapusher_copy diff --git a/ckan-docker/ckan-entrypoint.sh b/ckan-docker/ckan-entrypoint.sh index 8a3783a..49beaf7 100644 --- a/ckan-docker/ckan-entrypoint.sh +++ b/ckan-docker/ckan-entrypoint.sh @@ -90,6 +90,7 @@ crudini --set --verbose ${CONFIG_INI} app:main ckanext.azure_auth.client_secret crudini --set --verbose ${CONFIG_INI} app:main ckanext.azure_auth.auth_callback_path /azure/callback crudini --set --verbose ${CONFIG_INI} app:main ckanext.azure_auth.allow_create_users True +crudini --set --verbose ${CONFIG_INI} app:main ckan.max_resource_size 5000 #Configure datastore SQL functions $CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions | grep -v ^"$(date +%Y-%m-%d)" | psql ${CKAN_DATASTORE_WRITE_URL} --set ON_ERROR_STOP=1 diff --git a/ckan-docker/datapusher_copy b/ckan-docker/datapusher_copy new file mode 160000 index 0000000..de2c460 --- /dev/null +++ b/ckan-docker/datapusher_copy @@ -0,0 +1 @@ +Subproject commit de2c460452ea087f53ac8ae31551fe0e2c5f93ec diff --git a/ckan-docker/docker-compose.yml b/ckan-docker/docker-compose.yml index 7434cfe..739ab36 100644 --- a/ckan-docker/docker-compose.yml +++ b/ckan-docker/docker-compose.yml @@ -48,9 +48,13 @@ services: datapusher: container_name: ckan_datapusher - image: clementmouchet/datapusher + image: crearegistry.azurecr.io/datapusher + build: + context: ./datapusher_copy ports: - "8800:8800" + environment: + - MAX_CONTENT_LENGTH="5000000000" ckan_db: image: crearegistry.azurecr.io/crea_ckan_db From d6d585d699f2998ab8e19ba9149153ad313dd076 Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Mon, 10 May 2021 18:46:16 +0200 Subject: [PATCH 04/52] aligned datapusher submodule --- ckan-docker/datapusher_copy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan-docker/datapusher_copy b/ckan-docker/datapusher_copy index de2c460..2794a62 160000 --- a/ckan-docker/datapusher_copy +++ b/ckan-docker/datapusher_copy @@ -1 +1 @@ -Subproject commit de2c460452ea087f53ac8ae31551fe0e2c5f93ec +Subproject commit 2794a62d164e4db0835e5c40f447410a3112925d From 759dd1ee847d2006c2cbe9b1eeecba866f8a35ce Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Tue, 11 May 2021 09:15:14 +0200 Subject: [PATCH 05/52] aligned datapusher submodule --- .gitmodules | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 32dce2f..9e08302 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,8 @@ [submodule "ckan-docker/ckan_copy"] path = ckan-docker/ckan_copy url = https://github.com/ckan/ckan.git - branch = master \ No newline at end of file + branch = master +[submodule "ckan-docker/datapusher_copy"] + path = ckan-docker/datapusher_copy + url = https://github.com/geosolutions-it/datapusher.git + branch = 228_docker From 8f10b67edb1ba97a7a1a9b8f2415a0f59e99b81e Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Tue, 11 May 2021 09:45:36 +0200 Subject: [PATCH 06/52] aligned datapusher in azure compose --- azure/resourcegroup_deployment/ckan-compose/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml b/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml index 0eeb0c5..14313c2 100644 --- a/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml +++ b/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml @@ -35,6 +35,8 @@ services: image: clementmouchet/datapusher ports: - "8800:8800" + environment: + - MAX_CONTENT_LENGTH="5000000000" ckan_solr: image: ${REGISTRY_NAME}.azurecr.io/crea_ckan_solr From 5909edc0e50d91c86dfd8f18ef9cc0cb9d7d41f3 Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Tue, 11 May 2021 13:08:23 +0200 Subject: [PATCH 07/52] removed unused module datapusher --- .gitmodules | 4 ---- .../001_deployment.json | 14 ++++++++++++- .../az_scripts/az_install_docker.sh | 2 +- ckan-docker/README.md | 9 +++++---- ckan-docker/ckan-entrypoint.sh | 6 +++++- ckan-docker/datapusher_copy | 1 - ckan-docker/docker-compose.yml | 20 +++++++++++++------ 7 files changed, 38 insertions(+), 18 deletions(-) delete mode 160000 ckan-docker/datapusher_copy diff --git a/.gitmodules b/.gitmodules index 9e08302..fd56dcf 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,7 +2,3 @@ path = ckan-docker/ckan_copy url = https://github.com/ckan/ckan.git branch = master -[submodule "ckan-docker/datapusher_copy"] - path = ckan-docker/datapusher_copy - url = https://github.com/geosolutions-it/datapusher.git - branch = 228_docker diff --git a/azure/resourcegroup_deployment/001_deployment.json b/azure/resourcegroup_deployment/001_deployment.json index 1dc0e3e..b971507 100644 --- a/azure/resourcegroup_deployment/001_deployment.json +++ b/azure/resourcegroup_deployment/001_deployment.json @@ -85,7 +85,19 @@ "param_vnetlink_solr_name": { "defaultValue": "crea-solr", "type": "String" - } + }, + "param_azure_auth_tenantid": { + "defaultValue": "00000000-0000-0000-0000-000000000000", + "type": "String" + }, + "param_azure_auth_clientid": { + "defaultValue": "00000000-0000-0000-0000-000000000000", + "type": "String" + }, + "param_azure_auth_client_secret": { + "defaultValue": "pKf_0000000_00000-0000-0000", + "type": "String" + } }, "variables": {}, "resources": [ diff --git a/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh b/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh index 8ede7a8..5ee4f26 100755 --- a/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh +++ b/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh @@ -17,7 +17,7 @@ sudo -u ${vmusername} git clone https://github.com/geosolutions-it/C195-azure-wo cd /home/${vmusername}/C195-azure-workspace ### remove this before merging to master -#sudo -u ${vmusername} git checkout move-solr-into-vm +sudo -u ${vmusername} git checkout datapusher-datastore-ckan ### sudo -u ${vmusername} git submodule init && sudo -u ${vmusername} git submodule update diff --git a/ckan-docker/README.md b/ckan-docker/README.md index 11d5707..6e42080 100644 --- a/ckan-docker/README.md +++ b/ckan-docker/README.md @@ -2,18 +2,19 @@ This directory contains the configuration scripts for creating docker images for Initial files have been copied from the ckan Docker configuration (https://github.com/ckan/ckan/tree/master/contrib/docker) at the time of version 2.9.1. -Main changes and fixes are related to +Main changes and fixes are related to - changing images registry, in order to be able to push images to Azure -- CKAN image: using python 3.7 instead of 2.7 +- CKAN image: using python 3.7 instead of 2.7 - removed the use of some mounted volumes, since the setup procedure would populate local directories, that would not be present when deploying the images remotely. - Please note that in this directory there is a git submodule including the CKAN repo. This is needed in order to have CKAN deployed in the docker image. Initial tests where made using master at around version 2.9.1 (Jan 2021) +For local testing please add ckan host as 127.0.0.1 in Unix or GNU/Linux /etc/hosts or similar file in Windows + Make sure you `docker` and `docker-compose` commands are up-to-date enough to deal with Azure context. -You can build images with the usual +You can build images with the usual docker-compose build diff --git a/ckan-docker/ckan-entrypoint.sh b/ckan-docker/ckan-entrypoint.sh index 49beaf7..6b0f9e2 100644 --- a/ckan-docker/ckan-entrypoint.sh +++ b/ckan-docker/ckan-entrypoint.sh @@ -67,7 +67,7 @@ crudini --set --verbose --list --list-sep=\ ${CONFIG_INI} app:main ckan.plugins crudini --set --verbose --list --list-sep=\ ${CONFIG_INI} app:main ckan.plugins datastore crudini --set --verbose --list --list-sep=\ ${CONFIG_INI} app:main ckan.plugins datapusher -crudini --set --verbose ${CONFIG_INI} DEFAULT debug True +crudini --set --verbose ${CONFIG_INI} DEFAULT debug False crudini --set --verbose ${CONFIG_INI} logger_root level DEBUG crudini --set --verbose ${CONFIG_INI} logger_werkzeug level DEBUG @@ -91,6 +91,10 @@ crudini --set --verbose ${CONFIG_INI} app:main ckanext.azure_auth.auth_callback_ crudini --set --verbose ${CONFIG_INI} app:main ckanext.azure_auth.allow_create_users True crudini --set --verbose ${CONFIG_INI} app:main ckan.max_resource_size 5000 +crudini --set --verbose ${CONFIG_INI} app:main ckan.datapusher.callback_url_base ${CKAN_SITE_URL} +crudini --set --verbose ${CONFIG_INI} app:main ckan.datapusher.url ${CKAN_DATAPUSHER_URL} +crudini --set --verbose ${CONFIG_INI} app:main ckan.datapusher.assume_task_stale_after 300 + #Configure datastore SQL functions $CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions | grep -v ^"$(date +%Y-%m-%d)" | psql ${CKAN_DATASTORE_WRITE_URL} --set ON_ERROR_STOP=1 diff --git a/ckan-docker/datapusher_copy b/ckan-docker/datapusher_copy deleted file mode 160000 index 2794a62..0000000 --- a/ckan-docker/datapusher_copy +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2794a62d164e4db0835e5c40f447410a3112925d diff --git a/ckan-docker/docker-compose.yml b/ckan-docker/docker-compose.yml index 739ab36..3c023f4 100644 --- a/ckan-docker/docker-compose.yml +++ b/ckan-docker/docker-compose.yml @@ -35,7 +35,7 @@ services: - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@ckan_db/datastore - CKAN_SOLR_URL=http://ckan_solr:8983/solr/ckan - CKAN_REDIS_URL=redis://ckan_redis:6379/1 - - CKAN_DATAPUSHER_URL=http://datapusher:8800 + - CKAN_DATAPUSHER_URL=http://datapusher:8000 - CKAN_SITE_URL=${CKAN_SITE_URL} - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} @@ -48,13 +48,21 @@ services: datapusher: container_name: ckan_datapusher - image: crearegistry.azurecr.io/datapusher - build: - context: ./datapusher_copy + image: keitaro/ckan-datapusher:latest + # build: + # dockerfile: ./Dockerfile + # context: ./datapusher_copy/ ports: - - "8800:8800" + - "8000:8000" environment: - - MAX_CONTENT_LENGTH="5000000000" + - DATAPUSHER_MAX_CONTENT_LENGTH=5000000000 + #- DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE} + #- DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS} + - DATAPUSHER_DOWNLOAD_TIMEOUT=30 + - DATAPUSHER_SSL_VERIFY=False + - DATAPUSHER_REWRITE_RESOURCES=True + - CKAN_SITE_URL=${CKAN_SITE_URL} + # - DATAPUSHER_REWRITE_URL=http://ckan:5000 ckan_db: image: crearegistry.azurecr.io/crea_ckan_db From d4cf77617fabcb1a9fd8ce22e62f6e22d15df179 Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Tue, 11 May 2021 13:16:44 +0200 Subject: [PATCH 08/52] fixed building compose in ckan-docker/docker-compose.yml --- azure/resourcegroup_deployment/parameters.json | 18 +++++++++--------- ckan-docker/docker-compose.yml | 9 +-------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/azure/resourcegroup_deployment/parameters.json b/azure/resourcegroup_deployment/parameters.json index d4ecc6e..af868b4 100644 --- a/azure/resourcegroup_deployment/parameters.json +++ b/azure/resourcegroup_deployment/parameters.json @@ -9,13 +9,13 @@ "Value": "YOUR_SUBSCRIPTION_ID" }, "param_redis_name": { - "Value": "YOUR_REDIS_NAME" + "Value": "testredis01aaa" }, "param_endpoint_redis_name": { - "Value": "YOUR_REDIS_NAME" + "Value": "testredis01aaa" }, "param_vm_ckan_hostname": { - "Value": "YOUR_VM_HOSTNAME" + "Value": "testvm01aaaxyz" }, "param_vm_ckan_username": { "Value": "geosolutions" @@ -30,28 +30,28 @@ "Value": "S3cre3tP4ssw0rd" }, "param_postgres_hostname": { - "Value": "YOUR_POSTGRES_HOSTNAME" + "Value": "testpostgres01aaaa" }, "param_endpoint_pg_name": { - "Value": "YOUR_POSTGRES_HOSTNAME" + "Value": "testpostgres01aaaa" }, "param_vnet_name": { "Value": "privnet01" }, "param_networkiface_vm_name": { - "Value": "ckan-vm195" + "Value": "testvm01aaa" }, "param_publicipaddress_vm_name": { - "Value": "ckan-vm-ip" + "Value": "testvm01aaa" }, "param_storageaccount_name": { - "Value": "YOUR_STORAGEACCOUNT_NAME" + "Value": "teststorage01aaa" }, "param_vnetlink_solr_name": { "Value": "crea-solr" }, "param_registry_name": { - "Value": "YOUR_REGISTRY_NAME" + "Value": "testregistry01aaa" }, "param_networksecgroup_name": { "Value": "ckan-vm-nsg" diff --git a/ckan-docker/docker-compose.yml b/ckan-docker/docker-compose.yml index 3c023f4..edee4a2 100644 --- a/ckan-docker/docker-compose.yml +++ b/ckan-docker/docker-compose.yml @@ -49,20 +49,13 @@ services: datapusher: container_name: ckan_datapusher image: keitaro/ckan-datapusher:latest - # build: - # dockerfile: ./Dockerfile - # context: ./datapusher_copy/ ports: - "8000:8000" environment: - DATAPUSHER_MAX_CONTENT_LENGTH=5000000000 - #- DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE} - #- DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS} - - DATAPUSHER_DOWNLOAD_TIMEOUT=30 + - DATAPUSHER_DOWNLOAD_TIMEOUT=60 - DATAPUSHER_SSL_VERIFY=False - DATAPUSHER_REWRITE_RESOURCES=True - - CKAN_SITE_URL=${CKAN_SITE_URL} - # - DATAPUSHER_REWRITE_URL=http://ckan:5000 ckan_db: image: crearegistry.azurecr.io/crea_ckan_db From f5e10df9c9c9613981f69f9a18f3899d5ba3d366 Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Tue, 11 May 2021 13:16:44 +0200 Subject: [PATCH 09/52] fixed building compose in ckan-docker/docker-compose.yml --- ckan-docker/docker-compose.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/ckan-docker/docker-compose.yml b/ckan-docker/docker-compose.yml index 3c023f4..edee4a2 100644 --- a/ckan-docker/docker-compose.yml +++ b/ckan-docker/docker-compose.yml @@ -49,20 +49,13 @@ services: datapusher: container_name: ckan_datapusher image: keitaro/ckan-datapusher:latest - # build: - # dockerfile: ./Dockerfile - # context: ./datapusher_copy/ ports: - "8000:8000" environment: - DATAPUSHER_MAX_CONTENT_LENGTH=5000000000 - #- DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE} - #- DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS} - - DATAPUSHER_DOWNLOAD_TIMEOUT=30 + - DATAPUSHER_DOWNLOAD_TIMEOUT=60 - DATAPUSHER_SSL_VERIFY=False - DATAPUSHER_REWRITE_RESOURCES=True - - CKAN_SITE_URL=${CKAN_SITE_URL} - # - DATAPUSHER_REWRITE_URL=http://ckan:5000 ckan_db: image: crearegistry.azurecr.io/crea_ckan_db From a33344229c1a5275cb97ff2d30a461e8328195e9 Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Tue, 11 May 2021 13:22:48 +0200 Subject: [PATCH 10/52] reverted parameters --- azure/resourcegroup_deployment/parameters.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/azure/resourcegroup_deployment/parameters.json b/azure/resourcegroup_deployment/parameters.json index af868b4..d4ecc6e 100644 --- a/azure/resourcegroup_deployment/parameters.json +++ b/azure/resourcegroup_deployment/parameters.json @@ -9,13 +9,13 @@ "Value": "YOUR_SUBSCRIPTION_ID" }, "param_redis_name": { - "Value": "testredis01aaa" + "Value": "YOUR_REDIS_NAME" }, "param_endpoint_redis_name": { - "Value": "testredis01aaa" + "Value": "YOUR_REDIS_NAME" }, "param_vm_ckan_hostname": { - "Value": "testvm01aaaxyz" + "Value": "YOUR_VM_HOSTNAME" }, "param_vm_ckan_username": { "Value": "geosolutions" @@ -30,28 +30,28 @@ "Value": "S3cre3tP4ssw0rd" }, "param_postgres_hostname": { - "Value": "testpostgres01aaaa" + "Value": "YOUR_POSTGRES_HOSTNAME" }, "param_endpoint_pg_name": { - "Value": "testpostgres01aaaa" + "Value": "YOUR_POSTGRES_HOSTNAME" }, "param_vnet_name": { "Value": "privnet01" }, "param_networkiface_vm_name": { - "Value": "testvm01aaa" + "Value": "ckan-vm195" }, "param_publicipaddress_vm_name": { - "Value": "testvm01aaa" + "Value": "ckan-vm-ip" }, "param_storageaccount_name": { - "Value": "teststorage01aaa" + "Value": "YOUR_STORAGEACCOUNT_NAME" }, "param_vnetlink_solr_name": { "Value": "crea-solr" }, "param_registry_name": { - "Value": "testregistry01aaa" + "Value": "YOUR_REGISTRY_NAME" }, "param_networksecgroup_name": { "Value": "ckan-vm-nsg" From cf2dd812caae515c6b92dbd4fde44c3078cb1445 Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Tue, 11 May 2021 15:52:07 +0200 Subject: [PATCH 11/52] datastore db provision --- .../az_scripts/az_postgres_datastore_provision.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh b/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh index e47447c..968d812 100755 --- a/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh +++ b/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh @@ -6,5 +6,7 @@ VMUSER=$arg6 psql -U $arg1@$arg3 -h $arg4 postgres -c 'create database ckan with owner ckan;' psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD '${arg5}';" -psql -U $arg1@$arg3 -h $arg4 ckan -f /home/$VMUSER/C195-azure-workspace/azure/azure_pg_test/00_create_datastore.sql -psql -U $arg1@$arg3 -h $arg4 datastore -f /home/$VMUSER/C195-azure-workspace/azure/azure_pg_test/20_postgis_permissions.sql +psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE DATABASE datastore OWNER ckan ENCODING 'utf-8';" +psql -U $arg1@$arg3 -h $arg4 postgres -c "GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan;" +psql -U $arg1@$arg3 -h $arg4 postgres -c "GRANT SELECT ON ALL TABLES IN DATABASE datastore TO datastore_ro;" +psql -U $arg1@$arg3 -h $arg4 datastore -c "CREATE EXTENSION IF NOT EXISTS postgis; ALTER VIEW geometry_columns OWNER TO ckan; ALTER TABLE spatial_ref_sys OWNER TO ckan;" \ No newline at end of file From 6d47ffa082559c7b27a992e4d00c739564c5cf05 Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Tue, 11 May 2021 16:01:40 +0200 Subject: [PATCH 12/52] changed psql command --- ckan-docker/ckan-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan-docker/ckan-entrypoint.sh b/ckan-docker/ckan-entrypoint.sh index 6b0f9e2..089d830 100644 --- a/ckan-docker/ckan-entrypoint.sh +++ b/ckan-docker/ckan-entrypoint.sh @@ -97,7 +97,7 @@ crudini --set --verbose ${CONFIG_INI} app:main ckan.datapusher.assume_task_stale #Configure datastore SQL functions -$CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions | grep -v ^"$(date +%Y-%m-%d)" | psql ${CKAN_DATASTORE_WRITE_URL} --set ON_ERROR_STOP=1 +$CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions | grep -v ^"$(date +%Y-%m-%d)" | psql -U datastore_ro -h ${PG_HOST_FULL} --set ON_ERROR_STOP=1 # END changes to the ini file From f7d303ed194d8970cd71588eb13ffcdb55bf0d91 Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Tue, 11 May 2021 16:18:32 +0200 Subject: [PATCH 13/52] changed psql command --- ckan-docker/ckan-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan-docker/ckan-entrypoint.sh b/ckan-docker/ckan-entrypoint.sh index 089d830..1c7f59a 100644 --- a/ckan-docker/ckan-entrypoint.sh +++ b/ckan-docker/ckan-entrypoint.sh @@ -97,7 +97,7 @@ crudini --set --verbose ${CONFIG_INI} app:main ckan.datapusher.assume_task_stale #Configure datastore SQL functions -$CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions | grep -v ^"$(date +%Y-%m-%d)" | psql -U datastore_ro -h ${PG_HOST_FULL} --set ON_ERROR_STOP=1 +$CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions | grep -v ^"$(date +%Y-%m-%d)" | psql -U datastore_ro%40${PG_HOST} -h ${PG_HOST_FULL} --set ON_ERROR_STOP=1 # END changes to the ini file From b13a1ef3403cf64ee9e323d9807fc3eee81e2da2 Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Tue, 11 May 2021 16:32:38 +0200 Subject: [PATCH 14/52] changed psql command --- ckan-docker/ckan-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan-docker/ckan-entrypoint.sh b/ckan-docker/ckan-entrypoint.sh index 1c7f59a..9949dde 100644 --- a/ckan-docker/ckan-entrypoint.sh +++ b/ckan-docker/ckan-entrypoint.sh @@ -97,7 +97,7 @@ crudini --set --verbose ${CONFIG_INI} app:main ckan.datapusher.assume_task_stale #Configure datastore SQL functions -$CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions | grep -v ^"$(date +%Y-%m-%d)" | psql -U datastore_ro%40${PG_HOST} -h ${PG_HOST_FULL} --set ON_ERROR_STOP=1 +$CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions | grep -v ^"$(date +%Y-%m-%d)" | psql --set ON_ERROR_STOP=1 -U datastore_ro%40${PG_HOST} -h ${PG_HOST_FULL} # END changes to the ini file From 9f23fab65aac1c3c76b123d46d3d4c2218d9b1ae Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Tue, 11 May 2021 16:40:12 +0200 Subject: [PATCH 15/52] changed psql command --- azure/resourcegroup_deployment/ckan-compose/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml b/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml index 14313c2..4df6290 100644 --- a/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml +++ b/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml @@ -27,6 +27,7 @@ services: - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} + - PG_HOST_FULL=${PG_HOST_FULL} volumes: - ${CKAN_SHARE_MOUNT}:/var/lib/ckan From 00b177eadcf6c0e4e16a078d7d70042872b3ccf5 Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Tue, 11 May 2021 16:41:36 +0200 Subject: [PATCH 16/52] changed psql command --- ckan-docker/ckan-entrypoint.sh | 2 +- ckan-docker/docker-compose.yml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ckan-docker/ckan-entrypoint.sh b/ckan-docker/ckan-entrypoint.sh index 9949dde..883e8bb 100644 --- a/ckan-docker/ckan-entrypoint.sh +++ b/ckan-docker/ckan-entrypoint.sh @@ -97,7 +97,7 @@ crudini --set --verbose ${CONFIG_INI} app:main ckan.datapusher.assume_task_stale #Configure datastore SQL functions -$CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions | grep -v ^"$(date +%Y-%m-%d)" | psql --set ON_ERROR_STOP=1 -U datastore_ro%40${PG_HOST} -h ${PG_HOST_FULL} +$CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions | grep -v ^"$(date +%Y-%m-%d)" | psql --set ON_ERROR_STOP=1 -U datastore_ro@${PG_HOST} -h ${PG_HOST_FULL} # END changes to the ini file diff --git a/ckan-docker/docker-compose.yml b/ckan-docker/docker-compose.yml index edee4a2..5898872 100644 --- a/ckan-docker/docker-compose.yml +++ b/ckan-docker/docker-compose.yml @@ -40,10 +40,9 @@ services: - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} + - PG_HOST_FULL=${PG_HOST_FULL} volumes: - # - ckan_config:/etc/ckan - # - ckan_home:/usr/lib/ckan - ckan_storage:/var/lib/ckan datapusher: From a1ebfa696bd2db19d141d436b688f0063eb7e1db Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Tue, 11 May 2021 16:48:35 +0200 Subject: [PATCH 17/52] changed psql command --- azure/resourcegroup_deployment/ckan-compose/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml b/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml index 4df6290..7846e9f 100644 --- a/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml +++ b/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml @@ -28,6 +28,7 @@ services: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} - PG_HOST_FULL=${PG_HOST_FULL} + - PG_HOST=${PG_HOST} volumes: - ${CKAN_SHARE_MOUNT}:/var/lib/ckan From b54b111051d5b2b720f3c322dfa6394dfb3568d1 Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Tue, 11 May 2021 17:02:35 +0200 Subject: [PATCH 18/52] changed psql command --- ckan-docker/.env.template | 2 +- ckan-docker/ckan-entrypoint.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan-docker/.env.template b/ckan-docker/.env.template index 6f153cc..b58b5c9 100644 --- a/ckan-docker/.env.template +++ b/ckan-docker/.env.template @@ -43,4 +43,4 @@ POSTGRES_PORT=5432 # Readonly user/pass will be datastore_ro:DATASTORE_READONLY_PASSWORD DATASTORE_READONLY_PASSWORD=datastore - +PG_HOST_FULL=ckan_db diff --git a/ckan-docker/ckan-entrypoint.sh b/ckan-docker/ckan-entrypoint.sh index 883e8bb..01f264a 100644 --- a/ckan-docker/ckan-entrypoint.sh +++ b/ckan-docker/ckan-entrypoint.sh @@ -97,7 +97,7 @@ crudini --set --verbose ${CONFIG_INI} app:main ckan.datapusher.assume_task_stale #Configure datastore SQL functions -$CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions | grep -v ^"$(date +%Y-%m-%d)" | psql --set ON_ERROR_STOP=1 -U datastore_ro@${PG_HOST} -h ${PG_HOST_FULL} +$CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions | grep -v ^"$(date +%Y-%m-%d)" | psql --set ON_ERROR_STOP=1 -U ckan@${PG_HOST} -h ${PG_HOST_FULL} datastore # END changes to the ini file From 8fd1f0694ffb7405dfe64b88b41c7c6775786a13 Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Tue, 11 May 2021 17:12:24 +0200 Subject: [PATCH 19/52] changed psql command --- ckan-docker/ckan-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan-docker/ckan-entrypoint.sh b/ckan-docker/ckan-entrypoint.sh index 01f264a..164ddc5 100644 --- a/ckan-docker/ckan-entrypoint.sh +++ b/ckan-docker/ckan-entrypoint.sh @@ -97,7 +97,7 @@ crudini --set --verbose ${CONFIG_INI} app:main ckan.datapusher.assume_task_stale #Configure datastore SQL functions -$CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions | grep -v ^"$(date +%Y-%m-%d)" | psql --set ON_ERROR_STOP=1 -U ckan@${PG_HOST} -h ${PG_HOST_FULL} datastore +$CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions | grep -v ^"$(date +%Y-%m-%d)" | PDPASSWORD=${POSTGRES_PASSWORD} psql --set ON_ERROR_STOP=1 -U ckan@${PG_HOST} -h ${PG_HOST_FULL} datastore # END changes to the ini file From 463325b6b8901dd2bc4ee6c010699f94e866f299 Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Tue, 11 May 2021 17:15:33 +0200 Subject: [PATCH 20/52] changed psql command --- ckan-docker/ckan-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan-docker/ckan-entrypoint.sh b/ckan-docker/ckan-entrypoint.sh index 164ddc5..b12cd1c 100644 --- a/ckan-docker/ckan-entrypoint.sh +++ b/ckan-docker/ckan-entrypoint.sh @@ -97,7 +97,7 @@ crudini --set --verbose ${CONFIG_INI} app:main ckan.datapusher.assume_task_stale #Configure datastore SQL functions -$CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions | grep -v ^"$(date +%Y-%m-%d)" | PDPASSWORD=${POSTGRES_PASSWORD} psql --set ON_ERROR_STOP=1 -U ckan@${PG_HOST} -h ${PG_HOST_FULL} datastore +$CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions | grep -v ^"$(date +%Y-%m-%d)" | PGPASSWORD=${POSTGRES_PASSWORD} psql --set ON_ERROR_STOP=1 -U ckan@${PG_HOST} -h ${PG_HOST_FULL} datastore # END changes to the ini file From 07851013c5addf4b5d6d93d2ffbc0b68858a5ac4 Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Tue, 11 May 2021 17:32:08 +0200 Subject: [PATCH 21/52] changed psql command --- .../az_scripts/az_postgres_datastore_provision.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh b/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh index 968d812..0cfd5d9 100755 --- a/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh +++ b/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh @@ -5,7 +5,7 @@ export PGPASSWORD=$arg2 VMUSER=$arg6 psql -U $arg1@$arg3 -h $arg4 postgres -c 'create database ckan with owner ckan;' -psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD '${arg5}';" +psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE datastore_ro@${arg3} NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD '${arg5}';" psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE DATABASE datastore OWNER ckan ENCODING 'utf-8';" psql -U $arg1@$arg3 -h $arg4 postgres -c "GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan;" psql -U $arg1@$arg3 -h $arg4 postgres -c "GRANT SELECT ON ALL TABLES IN DATABASE datastore TO datastore_ro;" From 9214235897423fa38949392756bd3224ae6027aa Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Tue, 11 May 2021 17:36:32 +0200 Subject: [PATCH 22/52] changed psql command --- .../az_scripts/az_postgres_datastore_provision.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh b/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh index 0cfd5d9..968d812 100755 --- a/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh +++ b/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh @@ -5,7 +5,7 @@ export PGPASSWORD=$arg2 VMUSER=$arg6 psql -U $arg1@$arg3 -h $arg4 postgres -c 'create database ckan with owner ckan;' -psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE datastore_ro@${arg3} NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD '${arg5}';" +psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD '${arg5}';" psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE DATABASE datastore OWNER ckan ENCODING 'utf-8';" psql -U $arg1@$arg3 -h $arg4 postgres -c "GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan;" psql -U $arg1@$arg3 -h $arg4 postgres -c "GRANT SELECT ON ALL TABLES IN DATABASE datastore TO datastore_ro;" From e4913a21c6a1deb68b293016b7e825ea25c9e1de Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Tue, 11 May 2021 18:05:39 +0200 Subject: [PATCH 23/52] pg for datastore --- .../az_scripts/az_postgres_datastore_provision.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh b/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh index 968d812..1094eb2 100755 --- a/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh +++ b/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh @@ -5,8 +5,8 @@ export PGPASSWORD=$arg2 VMUSER=$arg6 psql -U $arg1@$arg3 -h $arg4 postgres -c 'create database ckan with owner ckan;' -psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD '${arg5}';" -psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE DATABASE datastore OWNER ckan ENCODING 'utf-8';" +psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE datastore_ro NOCREATEDB NOCREATEROLE LOGIN PASSWORD '${arg5}';" +psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE DATABASE datastore with OWNER ckan ENCODING 'utf-8';" psql -U $arg1@$arg3 -h $arg4 postgres -c "GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan;" psql -U $arg1@$arg3 -h $arg4 postgres -c "GRANT SELECT ON ALL TABLES IN DATABASE datastore TO datastore_ro;" psql -U $arg1@$arg3 -h $arg4 datastore -c "CREATE EXTENSION IF NOT EXISTS postgis; ALTER VIEW geometry_columns OWNER TO ckan; ALTER TABLE spatial_ref_sys OWNER TO ckan;" \ No newline at end of file From 473656af43b0b4277f54d645121ca6687ee37935 Mon Sep 17 00:00:00 2001 From: etj Date: Wed, 26 May 2021 10:10:05 +0200 Subject: [PATCH 24/52] #39 make ckan config persistent --- .../resourcegroup_deployment/001_deployment.json | 14 ++++++++++++++ .../az_scripts/az_install_docker.sh | 16 +++++++++++++--- .../ckan-compose/docker-compose.yml | 1 + 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/azure/resourcegroup_deployment/001_deployment.json b/azure/resourcegroup_deployment/001_deployment.json index b971507..53e3f49 100644 --- a/azure/resourcegroup_deployment/001_deployment.json +++ b/azure/resourcegroup_deployment/001_deployment.json @@ -867,6 +867,20 @@ "enabledProtocols": "SMB" } }, + { + "type": "Microsoft.Storage/storageAccounts/fileServices/shares", + "apiVersion": "2020-08-01-preview", + "name": "[concat(parameters('param_storageaccount_name'), '/default/ckanconfig')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/fileServices', parameters('param_storageaccount_name'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', parameters('param_storageaccount_name'))]" + ], + "properties": { + "accessTier": "TransactionOptimized", + "shareQuota": 5120, + "enabledProtocols": "SMB" + } + }, { "type": "Microsoft.Storage/storageAccounts/fileServices/shares", "apiVersion": "2020-08-01-preview", diff --git a/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh b/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh index 5ee4f26..776b314 100755 --- a/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh +++ b/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh @@ -30,16 +30,19 @@ resourceGroupName="$arg1" storageAccountName="$arg2" fileShareName="$arg3" fileShareName2="$arg9" +fileShareName3="ckanconfig" storageAccountKey="$arg8" registryName="$arg4" registryUsername="$arg5" registryPassword="$arg6" mntPath1="/mnt/$fileShareName" mntPath2="/mnt/$fileShareName2" +mntPath3="/mnt/$fileShareName3" smbCredentialFile="/etc/smbcredentials/$storageAccountName.cred" httpEndpoint="$arg7" smbPath1=$(echo $httpEndpoint | cut -c7-$(expr length $httpEndpoint))$fileShareName smbPath2=$(echo $httpEndpoint | cut -c7-$(expr length $httpEndpoint))$fileShareName2 +smbPath3=$(echo $httpEndpoint | cut -c7-$(expr length $httpEndpoint))$fileShareName3 #build and push ckan and solr images @@ -56,7 +59,7 @@ sudo -u ${vmusername} docker pull ${registryName}.azurecr.io/crea_ckan || echo " sudo -u ${vmusername} docker pull ${registryName}.azurecr.io/crea_ckan_solr || echo "problem pulling from registry" # mount ckan share -sudo mkdir -p $mntPath1 $mntPath2 +sudo mkdir -p $mntPath1 $mntPath2 $mntPath3 if [ ! -d "/etc/smbcredentials" ]; then sudo mkdir "/etc/smbcredentials" @@ -74,12 +77,19 @@ sudo chmod 600 $smbCredentialFile if [ -z "$(grep $smbPath1\ $mntPath1 /etc/fstab)" ]; then echo "$smbPath1 $mntPath1 cifs nofail,vers=3.0,credentials=$smbCredentialFile,serverino,file_mode=0777,dir_mode=0777" | sudo tee -a /etc/fstab > /dev/null else - echo "/etc/fstab was not modified to avoid conflicting entries as this Azure file share was already present. You may want to double check /etc/fstab to ensure the configuration is as desired." + echo "/etc/fstab was not modified to avoid conflicting entries as this Azure file share $mntPath1 was already present. You may want to double check /etc/fstab to ensure the configuration is as desired." fi if [ -z "$(grep $smbPath2\ $mntPath2 /etc/fstab)" ]; then echo "$smbPath2 $mntPath2 cifs nofail,vers=3.0,credentials=$smbCredentialFile,serverino,file_mode=0777,dir_mode=0777" | sudo tee -a /etc/fstab > /dev/null else - echo "/etc/fstab was not modified to avoid conflicting entries as this Azure file share was already present. You may want to double check /etc/fstab to ensure the configuration is as desired." + echo "/etc/fstab was not modified to avoid conflicting entries as this Azure file share $mntPath2 was already present. You may want to double check /etc/fstab to ensure the configuration is as desired." fi + +if [ -z "$(grep $smbPath3\ $mntPath3 /etc/fstab)" ]; then + echo "$smbPath3 $mntPath3 cifs nofail,vers=3.0,credentials=$smbCredentialFile,serverino,file_mode=0777,dir_mode=0777" | sudo tee -a /etc/fstab > /dev/null +else + echo "/etc/fstab was not modified to avoid conflicting entries as this Azure file share $mntPath3 was already present. You may want to double check /etc/fstab to ensure the configuration is as desired." +fi + sudo mount -a diff --git a/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml b/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml index 7846e9f..4397108 100644 --- a/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml +++ b/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml @@ -31,6 +31,7 @@ services: - PG_HOST=${PG_HOST} volumes: - ${CKAN_SHARE_MOUNT}:/var/lib/ckan + - ckanconfig:/etc/ckan datapusher: container_name: ckan_datapusher From b4ce84886fed10eb4755b95dd08a34643470b3d7 Mon Sep 17 00:00:00 2001 From: etj Date: Wed, 26 May 2021 11:16:23 +0200 Subject: [PATCH 25/52] #39 make ckan config persistent --- azure/resourcegroup_deployment/ckan-compose/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml b/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml index 4397108..ba17fe2 100644 --- a/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml +++ b/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml @@ -31,7 +31,7 @@ services: - PG_HOST=${PG_HOST} volumes: - ${CKAN_SHARE_MOUNT}:/var/lib/ckan - - ckanconfig:/etc/ckan + - /mnt/ckanconfig:/etc/ckan datapusher: container_name: ckan_datapusher From 0584fff3c198c9a3aff44fb560f0934cd1edd6c1 Mon Sep 17 00:00:00 2001 From: etj Date: Wed, 26 May 2021 16:06:50 +0200 Subject: [PATCH 26/52] #39 make ckan config persistent --- ckan-docker/Dockerfile | 2 + ckan-docker/ckan-entrypoint.sh | 78 ++++++++++++++++++---------------- 2 files changed, 44 insertions(+), 36 deletions(-) diff --git a/ckan-docker/Dockerfile b/ckan-docker/Dockerfile index bf1fbf1..95fbf57 100644 --- a/ckan-docker/Dockerfile +++ b/ckan-docker/Dockerfile @@ -31,6 +31,8 @@ RUN apt-get -q -y update \ swig \ && apt-get -q clean \ && rm -rf /var/lib/apt/lists/* +RUN wget https://raw.githubusercontent.com/pixelb/crudini/0.9.3/crudini -O /usr/bin/crudini \ + && chmod a+x /usr/bin/crudini # Define environment variables ENV CKAN_HOME /usr/lib/ckan diff --git a/ckan-docker/ckan-entrypoint.sh b/ckan-docker/ckan-entrypoint.sh index b12cd1c..5b50ee5 100644 --- a/ckan-docker/ckan-entrypoint.sh +++ b/ckan-docker/ckan-entrypoint.sh @@ -58,49 +58,55 @@ else ls -l ${CONFIG_INI} fi -# changes to the ini file -- SHOULD BE IDEMPOTENT -crudini --set --verbose --list --list-sep=\ ${CONFIG_INI} app:main ckan.plugins c195 -crudini --set --verbose --list --list-sep=\ ${CONFIG_INI} app:main ckan.plugins dcat -crudini --set --verbose --list --list-sep=\ ${CONFIG_INI} app:main ckan.plugins dcat_json_interface -crudini --set --verbose --list --list-sep=\ ${CONFIG_INI} app:main ckan.plugins structured_data -crudini --set --verbose --list --list-sep=\ ${CONFIG_INI} app:main ckan.plugins azure_auth -crudini --set --verbose --list --list-sep=\ ${CONFIG_INI} app:main ckan.plugins datastore -crudini --set --verbose --list --list-sep=\ ${CONFIG_INI} app:main ckan.plugins datapusher - -crudini --set --verbose ${CONFIG_INI} DEFAULT debug False - -crudini --set --verbose ${CONFIG_INI} logger_root level DEBUG -crudini --set --verbose ${CONFIG_INI} logger_werkzeug level DEBUG -crudini --set --verbose ${CONFIG_INI} logger_ckan level DEBUG -crudini --set --verbose ${CONFIG_INI} logger_ckanext level DEBUG -crudini --set --verbose ${CONFIG_INI} handler_console level DEBUG - -crudini --set --verbose ${CONFIG_INI} app:main sqlalchemy.pool_size 10 -crudini --set --verbose ${CONFIG_INI} app:main sqlalchemy.echo_pool True -crudini --set --verbose ${CONFIG_INI} app:main sqlalchemy.pool_pre_ping True -crudini --set --verbose ${CONFIG_INI} app:main sqlalchemy.pool_reset_on_return rollback -crudini --set --verbose ${CONFIG_INI} app:main sqlalchemy.pool_timeout 30 +echo "Customizing CKAN configuration file ${CONFIG_INI}..." +CONFIG_TMP=/tmp/ckan.ini -#Azure auth plugin https://github.com/geosolutions-it/ckanext-azure-auth.git +# we need to use crudini in a local copy or we get a [Errno 13] Permission denied +cp ${CONFIG_INI} ${CONFIG_TMP} +#cp ${CONFIG_INI} "/etc/ckan/$(date -Ins)_ckan.ini" -crudini --set --verbose ${CONFIG_INI} app:main ckanext.azure_auth.tenant_id ${TENANT_ID} -crudini --set --verbose ${CONFIG_INI} app:main ckanext.azure_auth.client_id ${CLIENT_ID} -crudini --set --verbose ${CONFIG_INI} app:main ckanext.azure_auth.audience ${CLIENT_ID} -crudini --set --verbose ${CONFIG_INI} app:main ckanext.azure_auth.client_secret ${CLIENT_SECRET} -crudini --set --verbose ${CONFIG_INI} app:main ckanext.azure_auth.auth_callback_path /azure/callback -crudini --set --verbose ${CONFIG_INI} app:main ckanext.azure_auth.allow_create_users True +# changes to the ini file -- SHOULD BE IDEMPOTENT +crudini --set --verbose --list --list-sep=\ ${CONFIG_TMP} app:main ckan.plugins c195 +crudini --set --verbose --list --list-sep=\ ${CONFIG_TMP} app:main ckan.plugins dcat +crudini --set --verbose --list --list-sep=\ ${CONFIG_TMP} app:main ckan.plugins dcat_json_interface +crudini --set --verbose --list --list-sep=\ ${CONFIG_TMP} app:main ckan.plugins structured_data +crudini --set --verbose --list --list-sep=\ ${CONFIG_TMP} app:main ckan.plugins azure_auth +crudini --set --verbose --list --list-sep=\ ${CONFIG_TMP} app:main ckan.plugins datastore +crudini --set --verbose --list --list-sep=\ ${CONFIG_TMP} app:main ckan.plugins datapusher + +crudini --set --verbose ${CONFIG_TMP} DEFAULT debug False + +crudini --set --verbose ${CONFIG_TMP} logger_root level DEBUG +crudini --set --verbose ${CONFIG_TMP} logger_werkzeug level DEBUG +crudini --set --verbose ${CONFIG_TMP} logger_ckan level DEBUG +crudini --set --verbose ${CONFIG_TMP} logger_ckanext level DEBUG +crudini --set --verbose ${CONFIG_TMP} handler_console level DEBUG + +crudini --set --verbose ${CONFIG_TMP} app:main sqlalchemy.pool_size 10 +crudini --set --verbose ${CONFIG_TMP} app:main sqlalchemy.echo_pool True +crudini --set --verbose ${CONFIG_TMP} app:main sqlalchemy.pool_pre_ping True +crudini --set --verbose ${CONFIG_TMP} app:main sqlalchemy.pool_reset_on_return rollback +crudini --set --verbose ${CONFIG_TMP} app:main sqlalchemy.pool_timeout 30 -crudini --set --verbose ${CONFIG_INI} app:main ckan.max_resource_size 5000 -crudini --set --verbose ${CONFIG_INI} app:main ckan.datapusher.callback_url_base ${CKAN_SITE_URL} -crudini --set --verbose ${CONFIG_INI} app:main ckan.datapusher.url ${CKAN_DATAPUSHER_URL} -crudini --set --verbose ${CONFIG_INI} app:main ckan.datapusher.assume_task_stale_after 300 +#Azure auth plugin https://github.com/geosolutions-it/ckanext-azure-auth.git +crudini --set --verbose ${CONFIG_TMP} app:main ckanext.azure_auth.tenant_id ${TENANT_ID} +crudini --set --verbose ${CONFIG_TMP} app:main ckanext.azure_auth.client_id ${CLIENT_ID} +crudini --set --verbose ${CONFIG_TMP} app:main ckanext.azure_auth.audience ${CLIENT_ID} +crudini --set --verbose ${CONFIG_TMP} app:main ckanext.azure_auth.client_secret ${CLIENT_SECRET} +crudini --set --verbose ${CONFIG_TMP} app:main ckanext.azure_auth.auth_callback_path /azure/callback +crudini --set --verbose ${CONFIG_TMP} app:main ckanext.azure_auth.allow_create_users True + +crudini --set --verbose ${CONFIG_TMP} app:main ckan.max_resource_size 5000 +crudini --set --verbose ${CONFIG_TMP} app:main ckan.datapusher.callback_url_base ${CKAN_SITE_URL} +crudini --set --verbose ${CONFIG_TMP} app:main ckan.datapusher.url ${CKAN_DATAPUSHER_URL} +crudini --set --verbose ${CONFIG_TMP} app:main ckan.datapusher.assume_task_stale_after 300 +# END changes to the ini file +cp ${CONFIG_TMP} ${CONFIG_INI} #Configure datastore SQL functions - +echo "Configuring datastore..." $CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions | grep -v ^"$(date +%Y-%m-%d)" | PGPASSWORD=${POSTGRES_PASSWORD} psql --set ON_ERROR_STOP=1 -U ckan@${PG_HOST} -h ${PG_HOST_FULL} datastore -# END changes to the ini file - # Get or create CKAN_SQLALCHEMY_URL if [ -z "$CKAN_SQLALCHEMY_URL" ]; then abort "ERROR: no CKAN_SQLALCHEMY_URL specified in docker-compose.yml" From f7f58eb5c04f3c4d1bb7be2466ea6c146d6eb649 Mon Sep 17 00:00:00 2001 From: etj Date: Wed, 26 May 2021 16:36:40 +0200 Subject: [PATCH 27/52] #39 make ckan config persistent --- ckan-docker/ckan-entrypoint.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ckan-docker/ckan-entrypoint.sh b/ckan-docker/ckan-entrypoint.sh index 5b50ee5..cc0c74d 100644 --- a/ckan-docker/ckan-entrypoint.sh +++ b/ckan-docker/ckan-entrypoint.sh @@ -50,6 +50,13 @@ while ! pg_isready -h $CKAN_PG_HOST -U ckan; do sleep 1; done +# If we don't already have a who config file, bootstrap +if [ ! -e "$CKAN_CONFIG/who.ini" ]; then + cp $CKAN_VENV/src/ckan/ckan/config/who.ini $CKAN_CONFIG/who.ini +else + echo "who.ini already exists" +fi + # If we don't already have a config file, bootstrap if [ ! -e "$CONFIG_INI" ]; then write_config From 00789ebbf93f00e8f60514c751b532269311f799 Mon Sep 17 00:00:00 2001 From: etj Date: Wed, 26 May 2021 16:37:30 +0200 Subject: [PATCH 28/52] #27 fix datastore role creation --- .../az_scripts/az_postgres_datastore_provision.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh b/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh index 1094eb2..4717db6 100755 --- a/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh +++ b/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh @@ -5,8 +5,8 @@ export PGPASSWORD=$arg2 VMUSER=$arg6 psql -U $arg1@$arg3 -h $arg4 postgres -c 'create database ckan with owner ckan;' -psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE datastore_ro NOCREATEDB NOCREATEROLE LOGIN PASSWORD '${arg5}';" +psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE "datastore_ro@$arg3" NOCREATEDB NOCREATEROLE LOGIN PASSWORD '${arg5}';" psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE DATABASE datastore with OWNER ckan ENCODING 'utf-8';" psql -U $arg1@$arg3 -h $arg4 postgres -c "GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan;" -psql -U $arg1@$arg3 -h $arg4 postgres -c "GRANT SELECT ON ALL TABLES IN DATABASE datastore TO datastore_ro;" +psql -U $arg1@$arg3 -h $arg4 postgres -c "GRANT SELECT ON ALL TABLES IN DATABASE datastore TO \"datastore_ro@$arg3\";" psql -U $arg1@$arg3 -h $arg4 datastore -c "CREATE EXTENSION IF NOT EXISTS postgis; ALTER VIEW geometry_columns OWNER TO ckan; ALTER TABLE spatial_ref_sys OWNER TO ckan;" \ No newline at end of file From e5285e260396eb147a14939e07b91c1ccf851c0c Mon Sep 17 00:00:00 2001 From: etj Date: Wed, 26 May 2021 17:02:19 +0200 Subject: [PATCH 29/52] #27 fix datastore set-permission --- ckan-docker/ckan-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan-docker/ckan-entrypoint.sh b/ckan-docker/ckan-entrypoint.sh index cc0c74d..03de424 100644 --- a/ckan-docker/ckan-entrypoint.sh +++ b/ckan-docker/ckan-entrypoint.sh @@ -112,7 +112,7 @@ cp ${CONFIG_TMP} ${CONFIG_INI} #Configure datastore SQL functions echo "Configuring datastore..." -$CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions | grep -v ^"$(date +%Y-%m-%d)" | PGPASSWORD=${POSTGRES_PASSWORD} psql --set ON_ERROR_STOP=1 -U ckan@${PG_HOST} -h ${PG_HOST_FULL} datastore +$CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions | grep -v ^"$(date +%Y-%m-%d)" | sed -e "s/ckan@etj-pg3/ckan/g" | PGPASSWORD=${POSTGRES_PASSWORD} psql --set ON_ERROR_STOP=1 -U ckan@${PG_HOST} -h ${PG_HOST_FULL} datastore # Get or create CKAN_SQLALCHEMY_URL if [ -z "$CKAN_SQLALCHEMY_URL" ]; then From 1983ce0664e73838be9a4e8455690e5e7f14f8e6 Mon Sep 17 00:00:00 2001 From: etj Date: Wed, 26 May 2021 18:46:20 +0200 Subject: [PATCH 30/52] #27 fix datastore role creation --- .../az_scripts/az_postgres_datastore_provision.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh b/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh index 4717db6..57cda1d 100755 --- a/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh +++ b/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh @@ -5,7 +5,7 @@ export PGPASSWORD=$arg2 VMUSER=$arg6 psql -U $arg1@$arg3 -h $arg4 postgres -c 'create database ckan with owner ckan;' -psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE "datastore_ro@$arg3" NOCREATEDB NOCREATEROLE LOGIN PASSWORD '${arg5}';" +psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE \"datastore_ro@$arg3\" NOCREATEDB NOCREATEROLE LOGIN PASSWORD '${arg5}';" psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE DATABASE datastore with OWNER ckan ENCODING 'utf-8';" psql -U $arg1@$arg3 -h $arg4 postgres -c "GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan;" psql -U $arg1@$arg3 -h $arg4 postgres -c "GRANT SELECT ON ALL TABLES IN DATABASE datastore TO \"datastore_ro@$arg3\";" From 053291d8fc5ca0d66699b5bd89194ac7dbaa204b Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Thu, 27 May 2021 17:44:08 +0200 Subject: [PATCH 31/52] parametrized sed --- ckan-docker/ckan-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan-docker/ckan-entrypoint.sh b/ckan-docker/ckan-entrypoint.sh index 03de424..8b31b26 100644 --- a/ckan-docker/ckan-entrypoint.sh +++ b/ckan-docker/ckan-entrypoint.sh @@ -112,7 +112,7 @@ cp ${CONFIG_TMP} ${CONFIG_INI} #Configure datastore SQL functions echo "Configuring datastore..." -$CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions | grep -v ^"$(date +%Y-%m-%d)" | sed -e "s/ckan@etj-pg3/ckan/g" | PGPASSWORD=${POSTGRES_PASSWORD} psql --set ON_ERROR_STOP=1 -U ckan@${PG_HOST} -h ${PG_HOST_FULL} datastore +$CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions | grep -v ^"$(date +%Y-%m-%d)" | sed -e "s/ckan@${PG_HOST}/ckan/g" | PGPASSWORD=${POSTGRES_PASSWORD} psql --set ON_ERROR_STOP=1 -U ckan@${PG_HOST} -h ${PG_HOST_FULL} datastore # Get or create CKAN_SQLALCHEMY_URL if [ -z "$CKAN_SQLALCHEMY_URL" ]; then From 72f779bbe804c292aeecca7fbff5f82d3731a9c0 Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Fri, 28 May 2021 09:33:13 +0200 Subject: [PATCH 32/52] updated wrong image for datapusher on azure --- .../ckan-compose/docker-compose.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml b/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml index ba17fe2..82e0a32 100644 --- a/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml +++ b/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml @@ -35,11 +35,14 @@ services: datapusher: container_name: ckan_datapusher - image: clementmouchet/datapusher + image: keitaro/ckan-datapusher:latest ports: - - "8800:8800" + - "8000:8000" environment: - - MAX_CONTENT_LENGTH="5000000000" + - DATAPUSHER_MAX_CONTENT_LENGTH=5000000000 + - DATAPUSHER_DOWNLOAD_TIMEOUT=60 + - DATAPUSHER_SSL_VERIFY=False + - DATAPUSHER_REWRITE_RESOURCES=True ckan_solr: image: ${REGISTRY_NAME}.azurecr.io/crea_ckan_solr From b63f6ebd7811f499673a28c27ec186b3cf3922af Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Fri, 28 May 2021 10:15:23 +0200 Subject: [PATCH 33/52] datastore setup --- .../az_scripts/az_postgres_datastore_provision.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh b/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh index 57cda1d..a7ae8a6 100755 --- a/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh +++ b/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh @@ -6,7 +6,10 @@ VMUSER=$arg6 psql -U $arg1@$arg3 -h $arg4 postgres -c 'create database ckan with owner ckan;' psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE \"datastore_ro@$arg3\" NOCREATEDB NOCREATEROLE LOGIN PASSWORD '${arg5}';" +#psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE datastore_ro NOCREATEDB NOCREATEROLE LOGIN PASSWORD '${arg5}';" psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE DATABASE datastore with OWNER ckan ENCODING 'utf-8';" psql -U $arg1@$arg3 -h $arg4 postgres -c "GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan;" -psql -U $arg1@$arg3 -h $arg4 postgres -c "GRANT SELECT ON ALL TABLES IN DATABASE datastore TO \"datastore_ro@$arg3\";" +#psql -U $arg1@$arg3 -h $arg4 postgres -c "GRANT SELECT ON ALL TABLES IN DATABASE datastore TO \"datastore_ro@$arg3\";" +psql -U $arg1@$arg3 -h $arg4 datastore -c "GRANT SELECT ON ALL TABLES IN SCHEMA public TO \"datastore_ro@$arg3\";" +psql -U $arg1@$arg3 -h $arg4 datastore -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO \"datastore_ro@$arg3\";" psql -U $arg1@$arg3 -h $arg4 datastore -c "CREATE EXTENSION IF NOT EXISTS postgis; ALTER VIEW geometry_columns OWNER TO ckan; ALTER TABLE spatial_ref_sys OWNER TO ckan;" \ No newline at end of file From bbf0f959413ef199be4c54830dd68fd808cec430 Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Fri, 28 May 2021 10:39:27 +0200 Subject: [PATCH 34/52] updated wrong image for datapusher on azure --- .../az_scripts/az_postgres_datastore_provision.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh b/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh index a7ae8a6..357725d 100755 --- a/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh +++ b/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh @@ -4,10 +4,10 @@ export PGPASSWORD=$arg2 VMUSER=$arg6 -psql -U $arg1@$arg3 -h $arg4 postgres -c 'create database ckan with owner ckan;' -psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE \"datastore_ro@$arg3\" NOCREATEDB NOCREATEROLE LOGIN PASSWORD '${arg5}';" +psql -U $arg1@$arg3 -h $arg4 postgres -c 'create database IF NOT EXISTS ckan with owner ckan;' +psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE IF NOT EXISTS \"datastore_ro@$arg3\" NOCREATEDB NOCREATEROLE LOGIN PASSWORD \'${arg5}\';" #psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE datastore_ro NOCREATEDB NOCREATEROLE LOGIN PASSWORD '${arg5}';" -psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE DATABASE datastore with OWNER ckan ENCODING 'utf-8';" +psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE DATABASE IF NOT EXISTS datastore with OWNER ckan ENCODING 'utf-8';" psql -U $arg1@$arg3 -h $arg4 postgres -c "GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan;" #psql -U $arg1@$arg3 -h $arg4 postgres -c "GRANT SELECT ON ALL TABLES IN DATABASE datastore TO \"datastore_ro@$arg3\";" psql -U $arg1@$arg3 -h $arg4 datastore -c "GRANT SELECT ON ALL TABLES IN SCHEMA public TO \"datastore_ro@$arg3\";" From fe873f8a311ec3c97ed5326c9a8a1b50539b1777 Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Fri, 28 May 2021 10:41:50 +0200 Subject: [PATCH 35/52] updated wrong image for datapusher on azure --- .../az_scripts/az_postgres_datastore_provision.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh b/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh index 357725d..3e0518d 100755 --- a/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh +++ b/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh @@ -4,10 +4,10 @@ export PGPASSWORD=$arg2 VMUSER=$arg6 -psql -U $arg1@$arg3 -h $arg4 postgres -c 'create database IF NOT EXISTS ckan with owner ckan;' +psql -U $arg1@$arg3 -h $arg4 postgres -c 'create database ckan with owner ckan;' psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE IF NOT EXISTS \"datastore_ro@$arg3\" NOCREATEDB NOCREATEROLE LOGIN PASSWORD \'${arg5}\';" #psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE datastore_ro NOCREATEDB NOCREATEROLE LOGIN PASSWORD '${arg5}';" -psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE DATABASE IF NOT EXISTS datastore with OWNER ckan ENCODING 'utf-8';" +psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE DATABASE datastore with OWNER ckan ENCODING 'utf-8';" psql -U $arg1@$arg3 -h $arg4 postgres -c "GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan;" #psql -U $arg1@$arg3 -h $arg4 postgres -c "GRANT SELECT ON ALL TABLES IN DATABASE datastore TO \"datastore_ro@$arg3\";" psql -U $arg1@$arg3 -h $arg4 datastore -c "GRANT SELECT ON ALL TABLES IN SCHEMA public TO \"datastore_ro@$arg3\";" From 9e9c7b585eab65ba00325d200ef900309eb70d08 Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Fri, 28 May 2021 11:24:31 +0200 Subject: [PATCH 36/52] updated wrong image for datapusher on azure --- .../az_scripts/az_postgres_datastore_provision.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh b/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh index 3e0518d..7724d5a 100755 --- a/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh +++ b/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh @@ -5,11 +5,10 @@ export PGPASSWORD=$arg2 VMUSER=$arg6 psql -U $arg1@$arg3 -h $arg4 postgres -c 'create database ckan with owner ckan;' -psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE IF NOT EXISTS \"datastore_ro@$arg3\" NOCREATEDB NOCREATEROLE LOGIN PASSWORD \'${arg5}\';" -#psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE datastore_ro NOCREATEDB NOCREATEROLE LOGIN PASSWORD '${arg5}';" +#psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE IF NOT EXISTS \"datastore_ro@$arg3\" LOGIN NOSUPERUSER INHERIT NOCREATEDB NOCREATEROLE NOREPLICATION PASSWORD \'${arg5}\';" +psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE datastore_ro LOGIN NOSUPERUSER INHERIT NOCREATEDB NOCREATEROLE NOREPLICATION PASSWORD '${arg5}';" psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE DATABASE datastore with OWNER ckan ENCODING 'utf-8';" psql -U $arg1@$arg3 -h $arg4 postgres -c "GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan;" -#psql -U $arg1@$arg3 -h $arg4 postgres -c "GRANT SELECT ON ALL TABLES IN DATABASE datastore TO \"datastore_ro@$arg3\";" -psql -U $arg1@$arg3 -h $arg4 datastore -c "GRANT SELECT ON ALL TABLES IN SCHEMA public TO \"datastore_ro@$arg3\";" -psql -U $arg1@$arg3 -h $arg4 datastore -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO \"datastore_ro@$arg3\";" +psql -U $arg1@$arg3 -h $arg4 datastore -c "GRANT SELECT ON ALL TABLES IN SCHEMA public TO datastore_ro" +psql -U $arg1@$arg3 -h $arg4 datastore -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO datastore_ro;" psql -U $arg1@$arg3 -h $arg4 datastore -c "CREATE EXTENSION IF NOT EXISTS postgis; ALTER VIEW geometry_columns OWNER TO ckan; ALTER TABLE spatial_ref_sys OWNER TO ckan;" \ No newline at end of file From 29efb6edcbc77cfb431a44bed1cad92c43e4dd4d Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Fri, 28 May 2021 11:55:34 +0200 Subject: [PATCH 37/52] fixing datastore_ro privileges --- ckan-docker/ckan-entrypoint.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckan-docker/ckan-entrypoint.sh b/ckan-docker/ckan-entrypoint.sh index 8b31b26..217ecb2 100644 --- a/ckan-docker/ckan-entrypoint.sh +++ b/ckan-docker/ckan-entrypoint.sh @@ -112,7 +112,8 @@ cp ${CONFIG_TMP} ${CONFIG_INI} #Configure datastore SQL functions echo "Configuring datastore..." -$CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions | grep -v ^"$(date +%Y-%m-%d)" | sed -e "s/ckan@${PG_HOST}/ckan/g" | PGPASSWORD=${POSTGRES_PASSWORD} psql --set ON_ERROR_STOP=1 -U ckan@${PG_HOST} -h ${PG_HOST_FULL} datastore +$CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions > /tmp/check_datastore.sql +$CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions | grep -v ^"$(date +%Y-%m-%d)" | sed -e "s/ckan@${PG_HOST}/ckan/g" | sed -e "s/datastore_ro@${PG_HOST}/datastore_ro/g" | PGPASSWORD=${POSTGRES_PASSWORD} psql --set ON_ERROR_STOP=1 -U ckan@${PG_HOST} -h ${PG_HOST_FULL} datastore # Get or create CKAN_SQLALCHEMY_URL if [ -z "$CKAN_SQLALCHEMY_URL" ]; then From b0bad5181b93c512cff2c42517d0932650d382f7 Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Fri, 28 May 2021 12:18:49 +0200 Subject: [PATCH 38/52] fixing datastore_ro privileges --- .../az_scripts/az_postgres_datastore_provision.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh b/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh index 7724d5a..64201d1 100755 --- a/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh +++ b/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh @@ -5,8 +5,8 @@ export PGPASSWORD=$arg2 VMUSER=$arg6 psql -U $arg1@$arg3 -h $arg4 postgres -c 'create database ckan with owner ckan;' -#psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE IF NOT EXISTS \"datastore_ro@$arg3\" LOGIN NOSUPERUSER INHERIT NOCREATEDB NOCREATEROLE NOREPLICATION PASSWORD \'${arg5}\';" -psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE datastore_ro LOGIN NOSUPERUSER INHERIT NOCREATEDB NOCREATEROLE NOREPLICATION PASSWORD '${arg5}';" +psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE IF NOT EXISTS \"datastore_ro@$arg3\" LOGIN NOSUPERUSER INHERIT NOCREATEDB NOCREATEROLE NOREPLICATION PASSWORD \'${arg5}\';" +psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE IF NOT EXISTS datastore_ro LOGIN NOSUPERUSER INHERIT NOCREATEDB NOCREATEROLE NOREPLICATION PASSWORD '${arg5}';" psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE DATABASE datastore with OWNER ckan ENCODING 'utf-8';" psql -U $arg1@$arg3 -h $arg4 postgres -c "GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan;" psql -U $arg1@$arg3 -h $arg4 datastore -c "GRANT SELECT ON ALL TABLES IN SCHEMA public TO datastore_ro" From dd87e2f8a4b33de7cf769786fcdfd16a0e0591d5 Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Fri, 28 May 2021 12:25:52 +0200 Subject: [PATCH 39/52] fixing datastore_ro privileges --- ckan-docker/ckan-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan-docker/ckan-entrypoint.sh b/ckan-docker/ckan-entrypoint.sh index 217ecb2..8c9ccb3 100644 --- a/ckan-docker/ckan-entrypoint.sh +++ b/ckan-docker/ckan-entrypoint.sh @@ -113,7 +113,7 @@ cp ${CONFIG_TMP} ${CONFIG_INI} #Configure datastore SQL functions echo "Configuring datastore..." $CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions > /tmp/check_datastore.sql -$CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions | grep -v ^"$(date +%Y-%m-%d)" | sed -e "s/ckan@${PG_HOST}/ckan/g" | sed -e "s/datastore_ro@${PG_HOST}/datastore_ro/g" | PGPASSWORD=${POSTGRES_PASSWORD} psql --set ON_ERROR_STOP=1 -U ckan@${PG_HOST} -h ${PG_HOST_FULL} datastore +$CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions 2>/dev/null | grep -v ^"$(date +%Y-%m-%d)" | sed -e "s/ckan@${PG_HOST}/ckan/g" | PGPASSWORD=${POSTGRES_PASSWORD} psql --set ON_ERROR_STOP=1 -U ckan@${PG_HOST} -h ${PG_HOST_FULL} datastore # Get or create CKAN_SQLALCHEMY_URL if [ -z "$CKAN_SQLALCHEMY_URL" ]; then From a7e9b33ffce8518aa4d0ef36e79ff01e9eb628e0 Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Fri, 28 May 2021 12:30:39 +0200 Subject: [PATCH 40/52] fixing datastore_ro privileges --- .../az_scripts/az_postgres_datastore_provision.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh b/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh index 64201d1..5157100 100755 --- a/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh +++ b/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh @@ -5,8 +5,8 @@ export PGPASSWORD=$arg2 VMUSER=$arg6 psql -U $arg1@$arg3 -h $arg4 postgres -c 'create database ckan with owner ckan;' -psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE IF NOT EXISTS \"datastore_ro@$arg3\" LOGIN NOSUPERUSER INHERIT NOCREATEDB NOCREATEROLE NOREPLICATION PASSWORD \'${arg5}\';" -psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE IF NOT EXISTS datastore_ro LOGIN NOSUPERUSER INHERIT NOCREATEDB NOCREATEROLE NOREPLICATION PASSWORD '${arg5}';" +psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE \"datastore_ro@$arg3\" LOGIN NOSUPERUSER INHERIT NOCREATEDB NOCREATEROLE NOREPLICATION PASSWORD \'${arg5}\';" +psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE datastore_ro LOGIN NOSUPERUSER INHERIT NOCREATEDB NOCREATEROLE NOREPLICATION PASSWORD '${arg5}';" psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE DATABASE datastore with OWNER ckan ENCODING 'utf-8';" psql -U $arg1@$arg3 -h $arg4 postgres -c "GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan;" psql -U $arg1@$arg3 -h $arg4 datastore -c "GRANT SELECT ON ALL TABLES IN SCHEMA public TO datastore_ro" From b5f9dc46080cecbac53c7ece188fe5c1b905a18b Mon Sep 17 00:00:00 2001 From: Emanuele Tajariol Date: Fri, 28 May 2021 12:42:56 +0200 Subject: [PATCH 41/52] Provide APIKEY in command line --- .../000_provision_initial_data.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/azure/resourcegroup_deployment/000_provision_initial_data.sh b/azure/resourcegroup_deployment/000_provision_initial_data.sh index a6b633c..f7fcd69 100755 --- a/azure/resourcegroup_deployment/000_provision_initial_data.sh +++ b/azure/resourcegroup_deployment/000_provision_initial_data.sh @@ -4,9 +4,14 @@ set -x . ckan-compose/.env CKAN_HOST_FULL=${CKAN_VM_NAME}.${VM_DOMAIN} -export PGPASSWORD=$POSTGRES_PASSWORD -CKAN_APIKEY=$(psql -t -A -X -U ${CKAN_PG_USER_PARTIAL}@${PG_INSTANCE} -h ${PG_HOST_FULL} ckan -c "select apikey from \"user\" where name='admin';") +if [ $# -eq 0 ] ; then + export PGPASSWORD=$POSTGRES_PASSWORD + CKAN_APIKEY=$(psql -t -A -X -U ${CKAN_PG_USER_PARTIAL}@${PG_INSTANCE} -h ${PG_HOST_FULL} ckan -c "select apikey from \"user\" where name='admin';") +else + CKAN_APIKEY=$1 +fi + NEW_JSON=/tmp/load_org.json for json in data/orgs/* ; do From 38337e37d3fed1d5cbc2bf57474d6fa3c8d9fe6d Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Fri, 28 May 2021 13:16:52 +0200 Subject: [PATCH 42/52] fixing datastore_ro privileges --- .../resourcegroup_deployment/ckan-compose/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml b/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml index 82e0a32..95734b5 100644 --- a/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml +++ b/azure/resourcegroup_deployment/ckan-compose/docker-compose.yml @@ -22,7 +22,7 @@ services: - CKAN_DATASTORE_READ_URL=postgresql://${DATASTORE_RO_PG_USER}:${DATASTORE_READONLY_PASSWORD}@${PG_HOST_FULL}/datastore - CKAN_SOLR_URL=http://${SOLR_HOST_FULL}:8983/solr/ckan - CKAN_REDIS_URL=redis://default:${REDIS_AUTHKEY}@${REDIS_HOST_FULL}:6379/1 - - CKAN_DATAPUSHER_URL=http://datapusher:8800 + - CKAN_DATAPUSHER_URL=http://datapusher:8000 - CKAN_SITE_URL=${CKAN_SITE_URL} - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} @@ -42,7 +42,7 @@ services: - DATAPUSHER_MAX_CONTENT_LENGTH=5000000000 - DATAPUSHER_DOWNLOAD_TIMEOUT=60 - DATAPUSHER_SSL_VERIFY=False - - DATAPUSHER_REWRITE_RESOURCES=True + #- DATAPUSHER_REWRITE_RESOURCES=True ckan_solr: image: ${REGISTRY_NAME}.azurecr.io/crea_ckan_solr From bc1af9963a90d75c648cc891e89378ead966f50c Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Fri, 28 May 2021 13:29:51 +0200 Subject: [PATCH 43/52] fixing datastore_ro privileges --- ckan-docker/ckan-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan-docker/ckan-entrypoint.sh b/ckan-docker/ckan-entrypoint.sh index 8c9ccb3..3a045a3 100644 --- a/ckan-docker/ckan-entrypoint.sh +++ b/ckan-docker/ckan-entrypoint.sh @@ -113,7 +113,7 @@ cp ${CONFIG_TMP} ${CONFIG_INI} #Configure datastore SQL functions echo "Configuring datastore..." $CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions > /tmp/check_datastore.sql -$CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions 2>/dev/null | grep -v ^"$(date +%Y-%m-%d)" | sed -e "s/ckan@${PG_HOST}/ckan/g" | PGPASSWORD=${POSTGRES_PASSWORD} psql --set ON_ERROR_STOP=1 -U ckan@${PG_HOST} -h ${PG_HOST_FULL} datastore +$CKAN_VENV/bin/ckan -c ${CONFIG_INI} datastore set-permissions 2>/dev/null | grep -v ^"$(date +%Y-%m-%d)" | sed -e "s/ckan@${PG_HOST}/ckan/g" | sed -e "s/datastore_ro@${PG_HOST}/datastore_ro/g" | PGPASSWORD=${POSTGRES_PASSWORD} psql --set ON_ERROR_STOP=1 -U ckan@${PG_HOST} -h ${PG_HOST_FULL} datastore # Get or create CKAN_SQLALCHEMY_URL if [ -z "$CKAN_SQLALCHEMY_URL" ]; then From 57e3d7236b432c04526e47cd388ccc1d5ec5c4ac Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Fri, 28 May 2021 16:07:18 +0200 Subject: [PATCH 44/52] fixed after merge --- azure/resourcegroup_deployment/az_scripts/az_install_docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh b/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh index 776b314..3c09d18 100755 --- a/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh +++ b/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh @@ -17,7 +17,7 @@ sudo -u ${vmusername} git clone https://github.com/geosolutions-it/C195-azure-wo cd /home/${vmusername}/C195-azure-workspace ### remove this before merging to master -sudo -u ${vmusername} git checkout datapusher-datastore-ckan + sudo -u ${vmusername} git checkout datapusher-datastore-ckan ### sudo -u ${vmusername} git submodule init && sudo -u ${vmusername} git submodule update From 93522e1fb6843b7292a88e9d5a00ea56ea4ad6da Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Fri, 28 May 2021 16:14:08 +0200 Subject: [PATCH 45/52] last fixes --- azure/resourcegroup_deployment/az_scripts/az_install_docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh b/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh index 3c09d18..a7e51d9 100755 --- a/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh +++ b/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh @@ -24,7 +24,7 @@ sudo -u ${vmusername} git submodule init && sudo -u ${vmusername} git submodule cd /home/${vmusername}/C195-azure-workspace/ckan-docker/ckan_copy sudo -u ${vmusername} git fetch --all -sudo -u ${vmusername} git checkout master +sudo -u ${vmusername} git checkout datapusher-datastore-ckan resourceGroupName="$arg1" storageAccountName="$arg2" From c657d66dfc3be63c34b01ec147007f9057b8cd83 Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Fri, 28 May 2021 16:33:04 +0200 Subject: [PATCH 46/52] fixing datastore_ro privileges --- .../az_scripts/az_postgres_datastore_provision.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh b/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh index 5157100..ffdcbe0 100755 --- a/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh +++ b/azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh @@ -5,7 +5,7 @@ export PGPASSWORD=$arg2 VMUSER=$arg6 psql -U $arg1@$arg3 -h $arg4 postgres -c 'create database ckan with owner ckan;' -psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE \"datastore_ro@$arg3\" LOGIN NOSUPERUSER INHERIT NOCREATEDB NOCREATEROLE NOREPLICATION PASSWORD \'${arg5}\';" +psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE \"datastore_ro@$arg3\" LOGIN NOSUPERUSER INHERIT NOCREATEDB NOCREATEROLE NOREPLICATION PASSWORD '${arg5}';" psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE datastore_ro LOGIN NOSUPERUSER INHERIT NOCREATEDB NOCREATEROLE NOREPLICATION PASSWORD '${arg5}';" psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE DATABASE datastore with OWNER ckan ENCODING 'utf-8';" psql -U $arg1@$arg3 -h $arg4 postgres -c "GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan;" From 6322dfb832b18988bc5970620355d467ff82be35 Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Fri, 28 May 2021 16:43:04 +0200 Subject: [PATCH 47/52] wrong branch --- azure/resourcegroup_deployment/az_scripts/az_install_docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh b/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh index a7e51d9..3c09d18 100755 --- a/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh +++ b/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh @@ -24,7 +24,7 @@ sudo -u ${vmusername} git submodule init && sudo -u ${vmusername} git submodule cd /home/${vmusername}/C195-azure-workspace/ckan-docker/ckan_copy sudo -u ${vmusername} git fetch --all -sudo -u ${vmusername} git checkout datapusher-datastore-ckan +sudo -u ${vmusername} git checkout master resourceGroupName="$arg1" storageAccountName="$arg2" From bbf7f836d185f82c665f8d5dd14369086305bf48 Mon Sep 17 00:00:00 2001 From: Luca Pasquali Date: Fri, 28 May 2021 17:02:31 +0200 Subject: [PATCH 48/52] revert to master branch --- azure/resourcegroup_deployment/az_scripts/az_install_docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh b/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh index 3c09d18..6e8f0a3 100755 --- a/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh +++ b/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh @@ -17,7 +17,7 @@ sudo -u ${vmusername} git clone https://github.com/geosolutions-it/C195-azure-wo cd /home/${vmusername}/C195-azure-workspace ### remove this before merging to master - sudo -u ${vmusername} git checkout datapusher-datastore-ckan +# sudo -u ${vmusername} git checkout datapusher-datastore-ckan ### sudo -u ${vmusername} git submodule init && sudo -u ${vmusername} git submodule update From cef0b18da9649491df494575b21a2bb6f5aa6ff3 Mon Sep 17 00:00:00 2001 From: Emanuele Tajariol Date: Tue, 1 Jun 2021 11:52:40 +0200 Subject: [PATCH 49/52] Use datapusher-datastore-ckan for testing --- azure/resourcegroup_deployment/az_scripts/az_install_docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh b/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh index 6e8f0a3..776b314 100755 --- a/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh +++ b/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh @@ -17,7 +17,7 @@ sudo -u ${vmusername} git clone https://github.com/geosolutions-it/C195-azure-wo cd /home/${vmusername}/C195-azure-workspace ### remove this before merging to master -# sudo -u ${vmusername} git checkout datapusher-datastore-ckan +sudo -u ${vmusername} git checkout datapusher-datastore-ckan ### sudo -u ${vmusername} git submodule init && sudo -u ${vmusername} git submodule update From dad57dc879e1383e56223d9ab1129a88afcd4003 Mon Sep 17 00:00:00 2001 From: etj Date: Fri, 4 Jun 2021 11:47:52 +0200 Subject: [PATCH 50/52] Reinstate grace-period plugin --- ckan-docker/ckan-entrypoint.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ckan-docker/ckan-entrypoint.sh b/ckan-docker/ckan-entrypoint.sh index 85e70c1..30b01ea 100644 --- a/ckan-docker/ckan-entrypoint.sh +++ b/ckan-docker/ckan-entrypoint.sh @@ -74,6 +74,7 @@ cp ${CONFIG_INI} ${CONFIG_TMP} # changes to the ini file -- SHOULD BE IDEMPOTENT crudini --set --verbose --list --list-sep=\ ${CONFIG_TMP} app:main ckan.plugins c195 +crudini --set --verbose --list --list-sep=\ ${CONFIG_TMP} app:main ckan.plugins grace_period crudini --set --verbose --list --list-sep=\ ${CONFIG_TMP} app:main ckan.plugins dcat crudini --set --verbose --list --list-sep=\ ${CONFIG_TMP} app:main ckan.plugins dcat_json_interface crudini --set --verbose --list --list-sep=\ ${CONFIG_TMP} app:main ckan.plugins structured_data From b7308aef701230743e65318eb3706338185ea89c Mon Sep 17 00:00:00 2001 From: etj Date: Fri, 4 Jun 2021 11:49:40 +0200 Subject: [PATCH 51/52] Fix custom plugin order --- ckan-docker/ckan-entrypoint.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ckan-docker/ckan-entrypoint.sh b/ckan-docker/ckan-entrypoint.sh index 30b01ea..fc339bf 100644 --- a/ckan-docker/ckan-entrypoint.sh +++ b/ckan-docker/ckan-entrypoint.sh @@ -73,12 +73,14 @@ cp ${CONFIG_INI} ${CONFIG_TMP} #cp ${CONFIG_INI} "/etc/ckan/$(date -Ins)_ckan.ini" # changes to the ini file -- SHOULD BE IDEMPOTENT + +# Make sure azure_auth is before c195 +crudini --set --verbose --list --list-sep=\ ${CONFIG_TMP} app:main ckan.plugins azure_auth crudini --set --verbose --list --list-sep=\ ${CONFIG_TMP} app:main ckan.plugins c195 crudini --set --verbose --list --list-sep=\ ${CONFIG_TMP} app:main ckan.plugins grace_period crudini --set --verbose --list --list-sep=\ ${CONFIG_TMP} app:main ckan.plugins dcat crudini --set --verbose --list --list-sep=\ ${CONFIG_TMP} app:main ckan.plugins dcat_json_interface crudini --set --verbose --list --list-sep=\ ${CONFIG_TMP} app:main ckan.plugins structured_data -crudini --set --verbose --list --list-sep=\ ${CONFIG_TMP} app:main ckan.plugins azure_auth crudini --set --verbose --list --list-sep=\ ${CONFIG_TMP} app:main ckan.plugins datastore crudini --set --verbose --list --list-sep=\ ${CONFIG_TMP} app:main ckan.plugins datapusher From 0ffefa103694e95e33bcda7a46b2758b23778a56 Mon Sep 17 00:00:00 2001 From: etj Date: Fri, 4 Jun 2021 13:42:09 +0200 Subject: [PATCH 52/52] Switch back to master branch --- azure/resourcegroup_deployment/az_scripts/az_install_docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh b/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh index 776b314..6e8f0a3 100755 --- a/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh +++ b/azure/resourcegroup_deployment/az_scripts/az_install_docker.sh @@ -17,7 +17,7 @@ sudo -u ${vmusername} git clone https://github.com/geosolutions-it/C195-azure-wo cd /home/${vmusername}/C195-azure-workspace ### remove this before merging to master -sudo -u ${vmusername} git checkout datapusher-datastore-ckan +# sudo -u ${vmusername} git checkout datapusher-datastore-ckan ### sudo -u ${vmusername} git submodule init && sudo -u ${vmusername} git submodule update