Add upgrade helper

.travis.yml

@@ -13,6 +13,7 @@ env:
   - SCENARIO=replications
   - SCENARIO=collations
   - SCENARIO=extensions
+  - SCENARIO=upgrade
 
 before_script:
   - ./build-test.sh

scenario_tests/upgrade/docker-compose.yml

@@ -0,0 +1,75 @@
+
+version: '2.1'
+
+volumes:
+  pg-previous-data-dir:
+  pg-previous-cluster-conf-dir:
+  new-pg-data-dir:
+  new-pg-cluster-conf-dir:
+
+services:
+  pg-previous-version:
+    image: 'kartoza/postgis:${PREV_VERSION:-11.0-2.5}'
+    restart: 'always'
+    # You can optionally mount to volume, to play with the persistence and
+    # observe how the node will behave after restarts.
+    volumes:
+      - pg-previous-data-dir:/var/lib/postgresql
+      - pg-previous-cluster-conf-dir:/etc/postgresql/11
+      - ./tests:/tests
+      - ../utils:/lib/utils
+    environment:
+      ALLOW_IP_RANGE: '0.0.0.0/0'
+      POSTGRES_HOST: 'localhost'
+      POSTGRES_DB: 'gis'
+      POSTGRES_DBNAME: 'gis'
+      POSTGRES_USER: 'docker'
+      POSTGRES_PASS: 'docker'
+      POSTGRES_PASSWORD: 'docker'
+      TEST_CLASS: test_upgrade.TestUpgradeInit
+    ports:
+      - "7777:5432"
+    healthcheck:
+      interval: 60s
+      timeout: 30s
+      retries: 3
+      test: "pg_isready"
+
+  pg-new:
+    image: 'kartoza/postgis:${TAG:-manual-build}'
+    restart: 'always'
+    # You can optionally mount to volume, to play with the persistence and
+    # observe how the node will behave after restarts.
+    entrypoint: ""
+    command: "tail -f /dev/null"
+    volumes:
+      - pg-previous-data-dir:/opt/data/postgis/previous
+      - pg-previous-cluster-conf-dir:/etc/postgresql/11
+      - new-pg-data-dir:/opt/data/postgis/new
+      - ./upgrade.d:/upgrade.d
+      - ./tests:/tests
+      - ../utils:/lib/utils
+    environment:
+      PGVERSIONOLD: "11"
+      PGVERSIONNEW: "12"
+      POSTGISVERSIONOLD: "2.5"
+      POSTGISVERSIONNEW: "3"
+      PGDATAOLD: "/opt/data/postgis/previous/11/main"
+      PGDATANEW: "/opt/data/postgis/new/12/main"
+      POSTGISDBNAME: "gis"
+      ALLOW_IP_RANGE: '0.0.0.0/0'
+      # For testing
+      POSTGRES_HOST: 'localhost'
+      POSTGRES_DB: 'gis'
+      POSTGRES_DBNAME: 'gis'
+      POSTGRES_USER: 'docker'
+      POSTGRES_PASS: 'docker'
+      POSTGRES_PASSWORD: 'docker'
+      TEST_CLASS: test_upgrade.TestUpgradeResult
+    ports:
+      - "7776:5432"
+    healthcheck:
+      interval: 60s
+      timeout: 30s
+      retries: 3
+      test: "pg_isready"

scenario_tests/upgrade/test.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+# exit immediately if test fails
+set -e
+
+source ../test-env.sh
+
+# Run service
+docker-compose up -d
+
+sleep 30
+
+# Initializing old clusters
+until docker-compose exec pg-previous-version pg_isready; do
+  sleep 30
+done;
+docker-compose exec pg-previous-version /bin/bash /tests/test.sh
+docker-compose stop pg-previous-version
+
+docker-compose exec pg-new /bin/bash /scripts/cluster-upgrade.sh
+docker-compose exec pg-new /bin/bash /tests/test.sh
+
+docker-compose down -v

scenario_tests/upgrade/tests/test.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -e
+
+# backward compatibility with older script locations
+source /scripts/env-data.sh || source /env-data.sh
+
+# execute tests
+pushd /tests
+
+cat << EOF
+Settings used:
+
+DEFAULT_COLLATION: ${DEFAULT_COLLATION}
+DEFAULT_CTYPE: ${DEFAULT_CTYPE}
+EOF
+
+PGHOST=localhost \
+PGDATABASE=gis \
+PYTHONPATH=/lib \
+  python3 -m unittest -v ${TEST_CLASS}

scenario_tests/upgrade/tests/test_upgrade.py

@@ -0,0 +1,48 @@
+import unittest
+import os
+from utils.utils import DBConnection
+
+
+class TestUpgradeBase(unittest.TestCase):
+
+    def setUp(self):
+        self.db = DBConnection()
+
+
+class TestUpgradeInit(TestUpgradeBase):
+
+    def test_upgrade_init(self):
+        # create new table
+        self.db.conn.autocommit = True
+        with self.db.cursor() as c:
+            c.execute(
+                """
+                create table if not exists layer (
+                    id integer not null primary key,
+                    value integer,
+                    geometry geometry(Point, 4326)
+                );
+                insert into layer values (1, 10, st_setsrid(st_makepoint(100, 6), 4326)) on conflict (id) do nothing;
+                """
+            )
+
+
+class TestUpgradeResult(TestUpgradeBase):
+
+    def test_upgrade_result(self):
+        self.db.conn.autocommit = True
+        with self.db.cursor() as c:
+            c.execute(
+                """
+                select id, value, st_astext(geometry) geometry from layer;
+                """
+            )
+            rows = c.fetchall()
+            self.assertTrue(rows)
+            row = rows[0]
+            self.assertEqual(row[1], 10)
+            self.assertEqual(row[2], 'POINT(100 6)')
+
+            # Check upgrade hook executed
+            self.assertTrue(os.path.exists('/tmp/pre.lock'))
+            self.assertTrue(os.path.exists('/tmp/post.lock'))

scenario_tests/upgrade/upgrade.d/post.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+echo "Post-upgrade hook script."
+
+touch /tmp/post.lock

scenario_tests/upgrade/upgrade.d/pre.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+echo "Pre-upgrade hook script."
+apt -y install postgresql-11-cron postgresql-12-cron
+touch /tmp/pre.lock

scripts/cluster-upgrade.sh

@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Variable sanity check
+if [[ -z ${PGVERSIONOLD} ]]; then
+  echo "Environment variable PGVERSIONOLD is empty."
+  echo "It must be set to postgresql version of the old cluster."
+fi
+if [[ -z ${PGVERSIONNEW} ]]; then
+  echo "Environment variable PGVERSIONNEW is empty."
+  echo "It must be set to postgresql version of the new cluster."
+fi
+if [[ -z ${PGDATAOLD} ]]; then
+  echo "Environment variable PGDATAOLD is empty."
+  echo "It must be set to the location of the old cluster."
+fi
+
+if [[ -z ${PGUNIXUSEROLD} ]]; then
+  PGUNIXUSEROLD=postgres
+fi
+
+if [[ -z ${PGDATANEW} ]]; then
+  PGDATANEW=/var/lib/postgresql/12/main
+fi
+
+# Inline replace
+# Change cluster data_directory
+sed -i 's|^data_directory|#data_directory|' /etc/postgresql/11/main/postgresql.conf
+echo "data_directory = '${PGDATAOLD}' # Added by cluster-upgrade.sh" >> /etc/postgresql/11/main/postgresql.conf
+# Disable ssl, because we won't be able to find the certificate
+sed -i 's|^ssl|#ssl|' /etc/postgresql/11/main/postgresql.conf
+
+apt -y update;
+# Install all default binary dependencies, which is:
+#  - Old postgres + Old postgis
+#  - New postgres + Old postgis
+#  - New postgres + New Postgis
+apt -y install \
+  postgresql-${PGVERSIONOLD} postgresql-${PGVERSIONOLD}-postgis-${POSTGISVERSIONOLD} \
+  postgresql-${PGVERSIONNEW} postgresql-${PGVERSIONNEW}-postgis-${POSTGISVERSIONNEW} \
+  postgresql-${PGVERSIONNEW}-postgis-${POSTGISVERSIONOLD}
+
+# show detected clusters
+pg_lsclusters
+
+if [[ -f "/upgrade.d/pre.sh" ]]; then
+  echo "Pre upgrade script exists. Executing pre upgrade..."
+  source /upgrade.d/pre.sh
+fi
+
+# We must change ownership of the data and config so it can be processed by this image
+echo "Attempting to change datadir and config permissions to user ${PGUNIXUSEROLD}."
+echo "This is an irreversible process."
+
+usermod -aG postgres ${PGUNIXUSEROLD}
+chown -R ${PGUNIXUSEROLD}:${PGUNIXUSEROLD} /etc/postgresql/${PGVERSIONOLD} ${PGDATAOLD} /var/log/postgresql /var/run/postgresql
+
+echo "Cluster list after permission change."
+
+pg_lsclusters
+
+# Shutdown default clusters
+echo "Shutting down default clusters"
+pg_ctlcluster ${PGVERSIONOLD} main stop || true
+pg_ctlcluster ${PGVERSIONNEW} main stop || true
+# We drop default cluster of the image because we don't need it.
+pg_dropcluster ${PGVERSIONNEW} main || true
+
+echo "Perform cluster upgrade"
+pg_upgradecluster -v ${PGVERSIONNEW} ${PGVERSIONOLD} main ${PGDATANEW}
+
+# TODO:
+# For some reason, pg cron database is not recognized and not upgraded.
+# We can only handle it from users perspective
+
+pg_ctlcluster ${PGVERSIONNEW} main start
+until pg_isready;
+do
+  sleep 5;
+done;
+
+if [[ "${POSTGISDBNAME}" ]]; then
+  echo "Upgrade postgis extensions in database"
+  for db in ${POSTGISDBNAME};
+  do
+    echo "Upgrade postgis in $db"
+    case ${POSTGISVERSIONNEW} in
+      3)
+        cat << EOF | su postgres -c "psql -d $db"
+        ALTER EXTENSION postgis UPDATE;
+        -- this next step repackages raster in its own extension
+        -- and upgrades all your other related postgis extensions
+        SELECT postgis_extensions_upgrade();
+EOF
+        ;;
+
+      2.5)
+        cat << EOF | su postgres -c "psql -d $db"
+        ALTER EXTENSION postgis UPDATE;
+        -- this next step repackages raster in its own extension
+        -- and upgrades all your other related postgis extensions
+        ALTER EXTENSION postgis_sfcgal UPDATE;
+        ALTER EXTENSION postgis_topology UPDATE;
+        ALTER EXTENSION postgis_tiger_geocoder UPDATE;
+EOF
+        ;;
+    esac
+  done
+fi
+
+if [[ -f "/upgrade.d/post.sh" ]]; then
+  echo "Post upgrade script exists. Executing post upgrade..."
+  source /upgrade.d/post.sh
+fi
+
+echo "Upgrade finished."
+echo "New cluster is in the location: ${PGDATANEW}"
+
+pg_lsclusters