Datapusher datastore ckan

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "ckan-docker/ckan_copy"]
 	path = ckan-docker/ckan_copy
 	url = https://github.com/ckan/ckan.git
-	branch = master
+	branch = master

azure/resourcegroup_deployment/000_provision_initial_data.sh

@@ -4,9 +4,14 @@
 set -x
 . ckan-compose/.env
 CKAN_HOST_FULL=${CKAN_VM_NAME}.${VM_DOMAIN}
-export PGPASSWORD=$POSTGRES_PASSWORD
 
-CKAN_APIKEY=$(psql -t -A -X -U ${CKAN_PG_USER_PARTIAL}@${PG_INSTANCE} -h ${PG_HOST_FULL} ckan -c "select apikey from \"user\" where name='admin';")
+if [ $# -eq 0 ] ; then
+   export PGPASSWORD=$POSTGRES_PASSWORD
+   CKAN_APIKEY=$(psql -t -A -X -U ${CKAN_PG_USER_PARTIAL}@${PG_INSTANCE} -h ${PG_HOST_FULL} ckan -c "select apikey from \"user\" where name='admin';")
+else
+   CKAN_APIKEY=$1
+fi
+
 NEW_JSON=/tmp/load_org.json
 
 for json in data/orgs/* ; do

azure/resourcegroup_deployment/001_deployment.json

@@ -97,7 +97,7 @@
         "param_azure_auth_client_secret": {
             "defaultValue": "pKf_0000000_00000-0000-0000",
             "type": "String"
-        }  
+        }
     },
     "variables": {},
     "resources": [
@@ -867,6 +867,20 @@
                 "enabledProtocols": "SMB"
             }
         },
+        {
+            "type": "Microsoft.Storage/storageAccounts/fileServices/shares",
+            "apiVersion": "2020-08-01-preview",
+            "name": "[concat(parameters('param_storageaccount_name'), '/default/ckanconfig')]",
+            "dependsOn": [
+                "[resourceId('Microsoft.Storage/storageAccounts/fileServices', parameters('param_storageaccount_name'), 'default')]",
+                "[resourceId('Microsoft.Storage/storageAccounts', parameters('param_storageaccount_name'))]"
+            ],
+            "properties": {
+                "accessTier": "TransactionOptimized",
+                "shareQuota": 5120,
+                "enabledProtocols": "SMB"
+            }
+        },
         {
             "type": "Microsoft.Storage/storageAccounts/fileServices/shares",
             "apiVersion": "2020-08-01-preview",

azure/resourcegroup_deployment/README.md

@@ -39,6 +39,7 @@ Here is a partial list:
 Partial customization can be also be done in file `setenv.sh`, for vars not extracted from the `parameters.json` file, but it's not needed.
 
 It is recommended **not** to modify the params in the following list, because they are considered as fixed values in some scripts:
+
 - `param_postgres_username = ckan`
 
 ### ADFS parameters
@@ -48,38 +49,40 @@ In order to setup ADFS integration, also edit:
 - `param_azure_auth_clientid`
 - `param_azure_auth_client_secret`  
 
-When configuring the AD client, also register a callback path in Azure as `PROT://YOUR_CKAN_HOST/azure/callback`
-
+When configuring the AD client, also register a callback path in Azure as `PORT://YOUR_CKAN_HOST/azure/callback`
 
-## Deploy 
+## Deploy
 
-- Deploy main resources in Azure. 
+- Deploy main resources in Azure.
   - This command will take up to 20-25 minutes to complete.
   - Run locally:
+
     ```bash
     ./azure_main_deploy.sh
     ```
 
 - Install docker stuff on VM and create images.
   - Run locally:
+
     ```bash
     ./azure_ckan_vm_config.sh
     ```
 
 - Create configuration from local files.
   - This script will also retrieve some info from Azure, so it's not immediate, but should be quite fast anyway.
-  - Run locally 
+  - Run locally
+
     ```bash
     env -i ./az_config_env.sh
     ```
+
 - Copy configuration to VM
   - Previous script should have printed a full `scp` command line. Run it locally to copy local generated configuration file to VM.
 
-
 - Create CKAN DB, restart services
   - Create CKAN DBs and assign privs, restart containers in VM (solr, ckan, nginx)
-  - This command above is idempotent and can be run several times, due to a current bug in Azure CLI (https://github.com/Azure/azure-cli/issues/16705) this script may be needed to be run more than one for solr to be configured correctly.
   - Run locally (calls `az` commands)
+
     ```bash
     ./azure_solr_config.sh
     ```
@@ -113,9 +116,9 @@ To ensure CKAN is always responding, there's a script named `check_ckan_alive.sh
 date=$(date '+%Y-%m-%d %H:%M:%S')
 response="$(curl -I -s http://localhost:5000/ --max-time 10 --connect-timeout 10 | head -1 | tr -d '\r')"
 if [ "$response" != 'HTTP/1.0 200 OK' ]; then
-    docker exec -i ckan /capture_gdb.sh      
-    docker restart ckan
-   	echo "$date - restarted ckan because it was stuck" >> $HOME/ckan_restart_log
+  docker exec -i ckan /capture_gdb.sh      
+  docker restart ckan
+  echo "$date - restarted ckan because it was stuck" >> $HOME/ckan_restart_log
 fi
 ```
 

azure/resourcegroup_deployment/az_scripts/az_install_docker.sh

@@ -17,7 +17,7 @@ sudo -u ${vmusername} git clone https://github.com/geosolutions-it/C195-azure-wo
 cd /home/${vmusername}/C195-azure-workspace 
 
 ### remove this before merging to master
-#sudo -u ${vmusername} git checkout move-solr-into-vm
+# sudo -u ${vmusername} git checkout datapusher-datastore-ckan
 ###
 
 sudo -u ${vmusername} git submodule init && sudo -u ${vmusername} git submodule update
@@ -30,16 +30,19 @@ resourceGroupName="$arg1"
 storageAccountName="$arg2"
 fileShareName="$arg3"
 fileShareName2="$arg9"
+fileShareName3="ckanconfig"
 storageAccountKey="$arg8"
 registryName="$arg4"
 registryUsername="$arg5"
 registryPassword="$arg6"
 mntPath1="/mnt/$fileShareName"
 mntPath2="/mnt/$fileShareName2"
+mntPath3="/mnt/$fileShareName3"
 smbCredentialFile="/etc/smbcredentials/$storageAccountName.cred"
 httpEndpoint="$arg7"
 smbPath1=$(echo $httpEndpoint | cut -c7-$(expr length $httpEndpoint))$fileShareName
 smbPath2=$(echo $httpEndpoint | cut -c7-$(expr length $httpEndpoint))$fileShareName2
+smbPath3=$(echo $httpEndpoint | cut -c7-$(expr length $httpEndpoint))$fileShareName3
 
 #build and push ckan and solr images
 
@@ -56,7 +59,7 @@ sudo -u ${vmusername} docker pull ${registryName}.azurecr.io/crea_ckan || echo "
 sudo -u ${vmusername} docker pull ${registryName}.azurecr.io/crea_ckan_solr || echo "problem pulling from registry"
 # mount ckan share
 
-sudo mkdir -p $mntPath1 $mntPath2
+sudo mkdir -p $mntPath1 $mntPath2 $mntPath3
 
 if [ ! -d "/etc/smbcredentials" ]; then
     sudo mkdir "/etc/smbcredentials"
@@ -74,12 +77,19 @@ sudo chmod 600 $smbCredentialFile
 if [ -z "$(grep $smbPath1\ $mntPath1 /etc/fstab)" ]; then
     echo "$smbPath1 $mntPath1 cifs nofail,vers=3.0,credentials=$smbCredentialFile,serverino,file_mode=0777,dir_mode=0777" | sudo tee -a /etc/fstab > /dev/null
 else
-    echo "/etc/fstab was not modified to avoid conflicting entries as this Azure file share was already present. You may want to double check /etc/fstab to ensure the configuration is as desired."
+    echo "/etc/fstab was not modified to avoid conflicting entries as this Azure file share $mntPath1 was already present. You may want to double check /etc/fstab to ensure the configuration is as desired."
 fi
 
 if [ -z "$(grep $smbPath2\ $mntPath2 /etc/fstab)" ]; then
     echo "$smbPath2 $mntPath2 cifs nofail,vers=3.0,credentials=$smbCredentialFile,serverino,file_mode=0777,dir_mode=0777" | sudo tee -a /etc/fstab > /dev/null
 else
-    echo "/etc/fstab was not modified to avoid conflicting entries as this Azure file share was already present. You may want to double check /etc/fstab to ensure the configuration is as desired."
+    echo "/etc/fstab was not modified to avoid conflicting entries as this Azure file share $mntPath2 was already present. You may want to double check /etc/fstab to ensure the configuration is as desired."
 fi
+
+if [ -z "$(grep $smbPath3\ $mntPath3 /etc/fstab)" ]; then
+    echo "$smbPath3 $mntPath3 cifs nofail,vers=3.0,credentials=$smbCredentialFile,serverino,file_mode=0777,dir_mode=0777" | sudo tee -a /etc/fstab > /dev/null
+else
+    echo "/etc/fstab was not modified to avoid conflicting entries as this Azure file share $mntPath3 was already present. You may want to double check /etc/fstab to ensure the configuration is as desired."
+fi
+
 sudo mount -a

azure/resourcegroup_deployment/az_scripts/az_postgres_datastore_provision.sh

@@ -5,6 +5,10 @@ export PGPASSWORD=$arg2
 VMUSER=$arg6
 
 psql -U $arg1@$arg3 -h $arg4 postgres -c 'create database ckan with owner ckan;'
-psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD '${arg5}';"
-psql -U $arg1@$arg3 -h $arg4 ckan -f /home/$VMUSER/C195-azure-workspace/azure/azure_pg_test/00_create_datastore.sql
-psql -U $arg1@$arg3 -h $arg4 datastore -f /home/$VMUSER/C195-azure-workspace/azure/azure_pg_test/20_postgis_permissions.sql
+psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE \"datastore_ro@$arg3\" LOGIN NOSUPERUSER INHERIT NOCREATEDB NOCREATEROLE NOREPLICATION PASSWORD '${arg5}';"
+psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE ROLE datastore_ro LOGIN NOSUPERUSER INHERIT NOCREATEDB NOCREATEROLE NOREPLICATION PASSWORD '${arg5}';"
+psql -U $arg1@$arg3 -h $arg4 postgres -c "CREATE DATABASE datastore with OWNER ckan ENCODING 'utf-8';"
+psql -U $arg1@$arg3 -h $arg4 postgres -c "GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan;"
+psql -U $arg1@$arg3 -h $arg4 datastore -c "GRANT SELECT ON ALL TABLES IN SCHEMA public TO datastore_ro"
+psql -U $arg1@$arg3 -h $arg4 datastore -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO datastore_ro;"
+psql -U $arg1@$arg3 -h $arg4 datastore -c "CREATE EXTENSION IF NOT EXISTS postgis; ALTER VIEW geometry_columns OWNER TO ckan; ALTER TABLE spatial_ref_sys OWNER TO ckan;"

azure/resourcegroup_deployment/ckan-compose/docker-compose.yml

@@ -4,6 +4,7 @@
 version: "3"
 
 services:
+
   ckan:
     container_name: ckan
     image: ${REGISTRY_NAME}.azurecr.io/crea_ckan
@@ -17,28 +18,45 @@ services:
     environment:
       # Defaults work with linked containers, change to use own Postgres, SolR, Redis or Datapusher
       - CKAN_SQLALCHEMY_URL=postgresql://${CKAN_PG_USER}:${POSTGRES_PASSWORD}@${PG_HOST_FULL}/ckan
-      - CKAN_DATASTORE_WRITE_URL="postgresql://${CKAN_PG_USER}:${POSTGRES_PASSWORD}@${PG_HOST_FULL}/datastore"
+      - CKAN_DATASTORE_WRITE_URL=postgresql://${CKAN_PG_USER}:${POSTGRES_PASSWORD}@${PG_HOST_FULL}/datastore
       - CKAN_DATASTORE_READ_URL=postgresql://${DATASTORE_RO_PG_USER}:${DATASTORE_READONLY_PASSWORD}@${PG_HOST_FULL}/datastore
       - CKAN_SOLR_URL=http://${SOLR_HOST_FULL}:8983/solr/ckan
       - CKAN_REDIS_URL=redis://default:${REDIS_AUTHKEY}@${REDIS_HOST_FULL}:6379/1
-      - CKAN_DATAPUSHER_URL=http://datapusher:8800
+      - CKAN_DATAPUSHER_URL=http://datapusher:8000
       - CKAN_SITE_URL=${CKAN_SITE_URL}
       - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
       - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD}
+      - PG_HOST_FULL=${PG_HOST_FULL}
+      - PG_HOST=${PG_HOST}
       - CLIENT_ID=${ADFS_CLIENT_ID}
       - CLIENT_SECRET=${ADFS_CLIENT_SECRET}
       - TENANT_ID=${ADFS_TENANT_ID}
+
     volumes:
       - ${CKAN_SHARE_MOUNT}:/var/lib/ckan
+      - /mnt/ckanconfig:/etc/ckan
+
+  datapusher:
+    container_name: ckan_datapusher
+    image: keitaro/ckan-datapusher:latest
+    ports:
+      - "8000:8000"
+    environment:
+      - DATAPUSHER_MAX_CONTENT_LENGTH=5000000000
+      - DATAPUSHER_DOWNLOAD_TIMEOUT=60
+      - DATAPUSHER_SSL_VERIFY=False
+      #- DATAPUSHER_REWRITE_RESOURCES=True    
+
   ckan_solr:
     image: ${REGISTRY_NAME}.azurecr.io/crea_ckan_solr
     container_name: ckan_solr      
     restart: always
     expose:
       - "8983"
     volumes:
-      - ${SOLR_SHARE_MOUNT}:/opt/solr/server/solr/ckan/data  
+      - ${SOLR_SHARE_MOUNT}:/opt/solr/server/solr/ckan/data
+
   proxy:
     image: ghcr.io/linuxserver/swag
     container_name: proxy

ckan-docker/.env.template

@@ -43,7 +43,8 @@ POSTGRES_PORT=5432
 # Readonly user/pass will be datastore_ro:DATASTORE_READONLY_PASSWORD
 DATASTORE_READONLY_PASSWORD=datastore
 
+PG_HOST_FULL=ckan_db
+
 ADFS_TENANT_ID=00000000-0000-0000-0000-000000000000
 ADFS_CLIENT_ID=00000000-0000-0000-0000-000000000000
 ADFS_CLIENT_SECRET=pKf_0000000_00000-0000-0000
-

ckan-docker/Dockerfile

@@ -1,6 +1,6 @@
 # See CKAN docs on installation from Docker Compose on usage
 FROM debian:buster
-MAINTAINER Open Knowledge
+LABEL Open Knowledge
 
 EXPOSE 5000
 
@@ -31,6 +31,8 @@ RUN apt-get -q -y update \
         swig \
     && apt-get -q clean \
     && rm -rf /var/lib/apt/lists/*
+RUN wget https://raw.githubusercontent.com/pixelb/crudini/0.9.3/crudini -O /usr/bin/crudini \
+    && chmod a+x /usr/bin/crudini
 
 # Define environment variables
 ENV CKAN_HOME /usr/lib/ckan
@@ -41,10 +43,10 @@ ENV TENANT_ID 00000000-0000-0000-0000-000000000000
 ENV CLIENT_ID 00000000-0000-0000-0000-000000000000
 ENV CLIENT_SECRET pKf_0000000_00000-0000-0000
 
+# Build-time variables specified by docker-compose.yml / .env
 ARG TENANT_ID
 ARG CLIENT_ID
 ARG CLIENT_SECRET
-# Build-time variables specified by docker-compose.yml / .env
 ARG CKAN_SITE_URL
 
 # Create ckan user
@@ -54,24 +56,14 @@ RUN useradd -r -u 900 -m -c "ckan account" -d $CKAN_HOME -s /bin/bash ckan
 RUN mkdir -p $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH && \
     python3 -m venv $CKAN_VENV
 
-#    ln -s $CKAN_VENV/bin/pip /usr/local/bin/ckan-pip &&\
-#    ln -s $CKAN_VENV/bin/ckan /usr/local/bin/ckan
-
-# Setup CKAN
-#ADD . $CKAN_VENV/src/ckan/
 COPY ./ckan_copy $CKAN_VENV/src/ckan/
-COPY ./ckan-entrypoint.sh /ckan-entrypoint.sh
-COPY ./ckan-run.sh /ckan-run.sh
-COPY ./gdb.commands /gdb.commands
-COPY ./capture_gdb.sh /capture_gdb.sh
 
 RUN $CKAN_VENV/bin/pip install -U pip && \
     $CKAN_VENV/bin/pip install --upgrade --no-cache-dir -r $CKAN_VENV/src/ckan/requirement-setuptools.txt && \
     $CKAN_VENV/bin/pip install --upgrade --no-cache-dir -r $CKAN_VENV/src/ckan/requirements.txt && \
     $CKAN_VENV/bin/pip install --upgrade --no-cache-dir -r $CKAN_VENV/src/ckan/dev-requirements.txt && \
     $CKAN_VENV/bin/pip install -e $CKAN_VENV/src/ckan/ && \
     ln -s $CKAN_VENV/src/ckan/ckan/config/who.ini $CKAN_CONFIG/who.ini && \
-    chmod +x /ckan-entrypoint.sh /capture_gdb.sh /ckan-run.sh && \
     chown -R ckan:ckan $CKAN_HOME $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH
 
 RUN $CKAN_VENV/bin/pip install -e git+https://github.com/geosolutions-it/ckanext-c195.git@main#egg=ckanext-c195
@@ -81,6 +73,14 @@ RUN $CKAN_VENV/bin/pip install -e git+https://github.com/geosolutions-it/ckanext
     $CKAN_VENV/bin/pip install -r $CKAN_VENV/src/ckanext-azure-auth/requirements.txt
 RUN $CKAN_VENV/bin/pip install -e git+https://github.com/geosolutions-it/ckanext-grace-period.git@main#egg=ckanext-grace-period
 
+# Setup CKAN custom scripts
+
+COPY ./ckan-entrypoint.sh /ckan-entrypoint.sh
+COPY ./ckan-run.sh /ckan-run.sh
+COPY ./gdb.commands /gdb.commands
+COPY ./capture_gdb.sh /capture_gdb.sh
+RUN chmod +x /ckan-entrypoint.sh /capture_gdb.sh /ckan-run.sh
+
 #USER ckan
 
 SHELL ["/bin/bash", "-c"]

ckan-docker/README.md

@@ -2,18 +2,19 @@ This directory contains the configuration scripts for creating docker images for
 
 Initial files have been copied from the ckan Docker configuration (https://github.com/ckan/ckan/tree/master/contrib/docker) at the time of version 2.9.1.
 
-Main changes and fixes are related to 
+Main changes and fixes are related to
 
 - changing images registry, in order to be able to push images to Azure
-- CKAN image: using python 3.7 instead of 2.7 
+- CKAN image: using python 3.7 instead of 2.7
 - removed the use of some mounted volumes, since the setup procedure would populate local directories, that would not be present when deploying the images remotely.
 
-
 Please note that in this directory there is a git submodule including the CKAN repo. This is needed in order to have CKAN deployed in the docker image.
 Initial tests where made using master at around version 2.9.1 (Jan 2021)
 
+For local testing please add ckan host as 127.0.0.1 in Unix or GNU/Linux /etc/hosts or similar file in Windows
+
 Make sure you `docker` and `docker-compose` commands are up-to-date enough to deal with Azure context.
-You can build images with the usual 
+You can build images with the usual
 
     docker-compose build