From 578386b281bcc057eee8e0d1de3cb098ea1d6cf9 Mon Sep 17 00:00:00 2001 From: NyakudyaA Date: Tue, 31 Dec 2024 19:45:19 +0200 Subject: [PATCH 1/6] Fix running backup and restore plugin --- scripts/entrypoint.sh | 2 +- scripts/setup.sh | 6 +++--- scripts/start.sh | 8 ++++++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index 615b1bcc..52715aca 100644 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -106,7 +106,7 @@ export JAVA_OPTS="${JAVA_OPTS} ${GEOSERVER_OPTS}" # Chown again - seems to fix issue with resolving all created directories if [[ ${RUN_AS_ROOT} =~ [Ff][Aa][Ll][Ss][Ee] ]];then dir_ownership=("${CATALINA_HOME}" /home/"${USER_NAME}"/ "${COMMUNITY_PLUGINS_DIR}" - "${STABLE_PLUGINS_DIR}" "${REQUIRED_PLUGINS_DIR}" "${GEOSERVER_HOME}" /usr/share/fonts/ /tomcat_apps.zip + "${STABLE_PLUGINS_DIR}" "${REQUIRED_PLUGINS_DIR}" "${GEOSERVER_HOME}" /usr/share/fonts/ /tmp/ "${FOOTPRINTS_DATA_DIR}" "${CERT_DIR}" "${FONTS_DIR}" /scripts/ "${EXTRA_CONFIG_DIR}" "/docker-entrypoint-geoserver.d" "${MONITOR_AUDIT_PATH}") for directory in "${dir_ownership[@]}"; do diff --git a/scripts/setup.sh b/scripts/setup.sh index a575aea6..48442c66 100644 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -89,7 +89,7 @@ rm -f /tmp/resources/overlays/README.txt && # Package tomcat webapps - useful to activate later if [[ -d "${CATALINA_HOME}"/webapps.dist ]]; then mv "${CATALINA_HOME}"/webapps.dist /tomcat_apps - zip -r /tomcat_apps.zip /tomcat_apps + zip -r "${REQUIRED_PLUGINS_DIR}"/tomcat_apps.zip /tomcat_apps rm -r /tomcat_apps else cp -r "${CATALINA_HOME}"/webapps/ROOT /tomcat_apps @@ -97,8 +97,8 @@ else cp -r "${CATALINA_HOME}"/webapps/examples /tomcat_apps cp -r "${CATALINA_HOME}"/webapps/host-manager /tomcat_apps cp -r "${CATALINA_HOME}"/webapps/manager /tomcat_apps - zip -r /tomcat_apps.zip /tomcat_apps - rm -r /tomcat_apps + zip -r "${REQUIRED_PLUGINS_DIR}"/tomcat_apps.zip /tomcat_apps + rm -rf /tomcat_apps fi pushd ${CATALINA_HOME}/lib || exit diff --git a/scripts/start.sh b/scripts/start.sh index d5351d2d..23461c8e 100644 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -326,8 +326,12 @@ fi if [[ "${TOMCAT_EXTRAS}" =~ [Tt][Rr][Uu][Ee] ]]; then - unzip -qq /tomcat_apps.zip -d /tmp/ && - cp -r /tmp/tomcat_apps/webapps.dist/* "${CATALINA_HOME}"/webapps/ && + unzip -qq "${REQUIRED_PLUGINS_DIR}"/tomcat_apps.zip -d /tmp/ + if [[ -d /tmp/tomcat_apps/webapps.dist ]];then + cp -r /tmp/tomcat_apps/webapps.dist/* "${CATALINA_HOME}"/webapps/ + else + cp -r /tmp/tomcat_apps/* "${CATALINA_HOME}"/webapps/ + fi rm -r /tmp/tomcat_apps if [[ ${POSTGRES_JNDI} =~ [Ff][Aa][Ll][Ss][Ee] ]]; then if [[ -f ${EXTRA_CONFIG_DIR}/context.xml ]]; then From 6a3a9550e711b6ac2eb6019b033b7de6d3347e44 Mon Sep 17 00:00:00 2001 From: NyakudyaA Date: Fri, 3 Jan 2025 10:11:56 +0200 Subject: [PATCH 2/6] Add test for backup and restore --- .env | 4 +- .github/workflows/build-latest.yaml | 1 + .../backup_restore/docker-compose.yml | 53 ++++++++++++++ scenario_tests/backup_restore/test.sh | 44 +++++++++++ .../backup_restore/tests/__init__.py | 0 scenario_tests/backup_restore/tests/test.sh | 17 +++++ .../tests/test_geoserver_backup.py | 73 +++++++++++++++++++ .../tests/test_geoserver_restore.py | 67 +++++++++++++++++ 8 files changed, 257 insertions(+), 2 deletions(-) create mode 100644 scenario_tests/backup_restore/docker-compose.yml create mode 100755 scenario_tests/backup_restore/test.sh create mode 100644 scenario_tests/backup_restore/tests/__init__.py create mode 100644 scenario_tests/backup_restore/tests/test.sh create mode 100644 scenario_tests/backup_restore/tests/test_geoserver_backup.py create mode 100644 scenario_tests/backup_restore/tests/test_geoserver_restore.py diff --git a/.env b/.env index e841847f..fce24310 100644 --- a/.env +++ b/.env @@ -126,13 +126,13 @@ STABLE_EXTENSIONS= # Allow overriding default installed extension combined with stable extensions ACTIVE_EXTENSIONS= # Install the community edition plugins specified in https://github.com/kartoza/docker-geoserver/blob/master/build_data/community_plugins.txt -COMMUNITY_EXTENSIONS= +COMMUNITY_EXTENSIONS=backup-restore-plugin # Geoserver monitoring and audit logging https://docs.geoserver.org/main/en/user/extensions/monitoring/audit.html MONITORING_AUDIT_ENABLED=false MONITORING_AUDIT_ROLL_LIMIT=20 RESET_MONITORING_LOGS=false # Geoserver logging -GEOSERVER_LOG_PROFILE=DEFAULT_LOGGING +GEOSERVER_LOG_PROFILE=VERBOSE_LOGGING GEOSERVER_LOG_DIR=/opt/geoserver/data_dir/logs # Run as root RUN_AS_ROOT=true diff --git a/.github/workflows/build-latest.yaml b/.github/workflows/build-latest.yaml index 991b871f..fe52b3a5 100644 --- a/.github/workflows/build-latest.yaml +++ b/.github/workflows/build-latest.yaml @@ -190,6 +190,7 @@ jobs: # - clustering - jdbconfig - libjpeg + - backup_restore steps: - uses: actions/checkout@v4 - name: Download artifact diff --git a/scenario_tests/backup_restore/docker-compose.yml b/scenario_tests/backup_restore/docker-compose.yml new file mode 100644 index 00000000..afdcb5a0 --- /dev/null +++ b/scenario_tests/backup_restore/docker-compose.yml @@ -0,0 +1,53 @@ + +volumes: + geoserver-data-dir: + geoserver-data: + geoserver-backup-dir: + + +services: + + geoserver: + image: 'kartoza/geoserver:${TAG:-manual-build}' + restart: 'always' + volumes: + - geoserver-data-dir:/opt/geoserver/data_dir + - geoserver-backup-dir:/settings + - ./tests:/tests + environment: + GEOSERVER_ADMIN_PASSWORD: myawesomegeoserver + GEOSERVER_ADMIN_USER: admin + SAMPLE_DATA: true + CONSOLE_HANDLER_LEVEL: WARNING + COMMUNITY_EXTENSIONS: backup-restore-plugin + TEST_CLASS: test_geoserver_backup.TestGeoServerBackup + ports: + - "8080:8080" + healthcheck: + test: ["CMD-SHELL", "curl --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null -u $${GEOSERVER_ADMIN_USER}:$${GEOSERVER_ADMIN_PASSWORD} http://localhost:8080/geoserver/rest/about/version.xml"] + interval: 1m30s + timeout: 10s + retries: 3 + + restore: + image: 'kartoza/geoserver:${TAG:-manual-build}' + restart: 'always' + volumes: + - geoserver-data:/opt/geoserver/data_dir + - geoserver-backup-dir:/settings + - ./tests:/tests + environment: + GEOSERVER_ADMIN_PASSWORD: myawesomegeoserver + GEOSERVER_ADMIN_USER: admin + CONSOLE_HANDLER_LEVEL: WARNING + COMMUNITY_EXTENSIONS: backup-restore-plugin + TEST_CLASS: test_geoserver_backup.TestGeoServerRestore + ports: + - "8080:8080" + healthcheck: + test: [ "CMD-SHELL", "curl --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null -u $${GEOSERVER_ADMIN_USER}:$${GEOSERVER_ADMIN_PASSWORD} http://localhost:8080/geoserver/rest/about/version.xml" ] + interval: 1m30s + timeout: 10s + retries: 3 + + diff --git a/scenario_tests/backup_restore/test.sh b/scenario_tests/backup_restore/test.sh new file mode 100755 index 00000000..75972bf3 --- /dev/null +++ b/scenario_tests/backup_restore/test.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +# exit immediately if test fails +set -e + +source ../test-env.sh + +# Run service +if [[ $(dpkg -l | grep "docker-compose") > /dev/null ]];then + VERSION='docker-compose' + else + VERSION='docker compose' +fi + +${VERSION} -f docker-compose.yml up -d + +if [[ -n "${PRINT_TEST_LOGS}" ]]; then + ${VERSION} -f docker-compose.yml logs -f & +fi + + +services=("geoserver") + +for service in "${services[@]}"; do + + # Execute tests + test_url_availability http://localhost:8080/geoserver/rest/about/version.xml + echo "Execute test for $service" + ${VERSION} -f docker-compose.yml exec $service /bin/bash /tests/test.sh + +done + +services=("restore") + +for service in "${services[@]}"; do + + # Execute tests + test_url_availability http://localhost:8080/geoserver/rest/about/version.xml + echo "Execute test for $service" + ${VERSION} -f docker-compose.yml exec $service /bin/bash /tests/test.sh + +done + +${VERSION} -f docker-compose.yml down -v diff --git a/scenario_tests/backup_restore/tests/__init__.py b/scenario_tests/backup_restore/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/scenario_tests/backup_restore/tests/test.sh b/scenario_tests/backup_restore/tests/test.sh new file mode 100644 index 00000000..7337b414 --- /dev/null +++ b/scenario_tests/backup_restore/tests/test.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -e + +source /scripts/env-data.sh + +# execute tests +pushd /tests + +cat << EOF +Settings used: + +GEOSERVER_ADMIN_PASSWORD: ${GEOSERVER_ADMIN_PASSWORD} +GEOSERVER_ADMIN_USER: ${GEOSERVER_ADMIN_USER} +EOF + +python3 -m unittest -v ${TEST_CLASS} diff --git a/scenario_tests/backup_restore/tests/test_geoserver_backup.py b/scenario_tests/backup_restore/tests/test_geoserver_backup.py new file mode 100644 index 00000000..9c559dc0 --- /dev/null +++ b/scenario_tests/backup_restore/tests/test_geoserver_backup.py @@ -0,0 +1,73 @@ +import unittest +import requests +from requests.auth import HTTPBasicAuth +from os import environ, mkdir, chmod +from os.path import join, exists +from shutil import chown +import json +import time + + +class TestGeoServerBackup(unittest.TestCase): + + def setUp(self): + """Set up the base URL, authentication, and create the zip file.""" + self.gs_url = 'http://localhost:8080/geoserver' + self.geo_username = environ.get('GEOSERVER_ADMIN_USER', 'admin') + self.geo_password = environ.get('GEOSERVER_ADMIN_PASSWORD', 'myawesomegeoserver') + self.username = 'geoserveruser' + self.group_name = 'geoserverusers' + self.backup_path = "/settings/backup/" + + def test_create_backup(self): + """Test creating a GeoServer backup using the Backup and Restore plugin.""" + auth = HTTPBasicAuth('%s' % self.geo_username, '%s' % self.geo_password) + base_url = f"{self.gs_url}/rest/br/backup/" + if not exists(self.backup_path): + mkdir(self.backup_path) + backup_file = join(self.backup_path, 'geoserver.zip') + # Create the empty zip file + with open(backup_file, "wb") as f: + pass + + # Change ownership of the zip file + chmod(self.backup_path, 0o777) + chown(backup_file, user=self.username, group=self.group_name) + headers = { + "Content-Type": "application/json" + } + + payload = { + "backup": { + "archiveFile": backup_file, + "overwrite": True, + "options": {} + } + } + + # Send the POST request to trigger the backup + response = requests.post(base_url, json=payload, auth=auth, headers=headers) + response_data = json.loads(response.text) + execution_id = response_data["backup"]["execution"]["id"] + execution_url = f"{self.gs_url}/rest/br/backup/{execution_id}.json" + # wait for backup to complete + time.sleep(30) + response_execution_request = requests.get(execution_url, auth=auth) + if response_execution_request.status_code == 200: + try: + response_execution_json = response_execution_request.json() + response_status = response_execution_json["backup"]["execution"]["status"] + self.assertEqual(response_status, 'COMPLETED', "backup initiated successfully") + except ValueError as e: + print("Error parsing JSON:", e) + print("Raw response content:", response_execution_request.text) + else: + print(f"Request failed with status code {response_execution_request.status_code}") + print("Response content:", response_execution_request.text) + + # Verify the response status code + self.assertEqual(response.status_code, 201, "backup initiated successfully") + + +if __name__ == "__main__": + unittest.main() diff --git a/scenario_tests/backup_restore/tests/test_geoserver_restore.py b/scenario_tests/backup_restore/tests/test_geoserver_restore.py new file mode 100644 index 00000000..67d73413 --- /dev/null +++ b/scenario_tests/backup_restore/tests/test_geoserver_restore.py @@ -0,0 +1,67 @@ +import sys +import unittest +import requests +from requests.auth import HTTPBasicAuth +from os import environ, chmod +from os.path import join, exists +import json +import time + + +class TestGeoServerRestore(unittest.TestCase): + + def setUp(self): + """Set up the base URL, authentication, and create the zip file.""" + self.gs_url = 'http://localhost:8080/geoserver' + self.geo_username = environ.get('GEOSERVER_ADMIN_USER', 'admin') + self.geo_password = environ.get('GEOSERVER_ADMIN_PASSWORD', 'myawesomegeoserver') + self.backup_path = "/settings/backup/" + + def test_restore_backup(self): + """Test restoring an existing backup of a GeoServer instance using the Backup and Restore plugin.""" + auth = HTTPBasicAuth('%s' % self.geo_username, '%s' % self.geo_password) + base_url = f"{self.gs_url}/rest/br/restore/" + backup_file = join(self.backup_path, 'geoserver.zip') + chmod(self.backup_path, 0o777) + if not exists(backup_file): + sys.exit() + + headers = { + "Content-Type": "application/json" + } + + payload = { + "restore": { + "archiveFile": backup_file, + "options": { + "option": ["BK_BEST_EFFORT=true"] + } + } + } + + # Send the POST request to trigger the backup + response = requests.post(base_url, json=payload, auth=auth, headers=headers) + response_data = json.loads(response.text) + execution_id = response_data["restore"]["execution"]["id"] + execution_url = f"{self.gs_url}/rest/br/restore/{execution_id}.json" + # wait for backup to complete + time.sleep(30) + response_execution_request = requests.get(execution_url, auth=auth) + if response_execution_request.status_code == 200: + try: + response_execution_json = response_execution_request.json() + response_status = response_execution_json["restore"]["execution"]["status"] + self.assertEqual(response_status, 'COMPLETED', "backup initiated successfully") + except ValueError as e: + print("Error parsing JSON:", e) + print("Raw response content:", response_execution_request.text) + else: + print(f"Request failed with status code {response_execution_request.status_code}") + print("Response content:", response_execution_request.text) + + # Verify the response status code + self.assertEqual(response.status_code, 201, "backup initiated successfully") + + +if __name__ == "__main__": + unittest.main() From 89a523aec49686604573289f8362f8a759f7db01 Mon Sep 17 00:00:00 2001 From: NyakudyaA Date: Fri, 3 Jan 2025 12:03:00 +0200 Subject: [PATCH 3/6] fix port in restore --- scenario_tests/backup_restore/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scenario_tests/backup_restore/docker-compose.yml b/scenario_tests/backup_restore/docker-compose.yml index afdcb5a0..85aa872e 100644 --- a/scenario_tests/backup_restore/docker-compose.yml +++ b/scenario_tests/backup_restore/docker-compose.yml @@ -43,7 +43,7 @@ services: COMMUNITY_EXTENSIONS: backup-restore-plugin TEST_CLASS: test_geoserver_backup.TestGeoServerRestore ports: - - "8080:8080" + - "8080" healthcheck: test: [ "CMD-SHELL", "curl --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null -u $${GEOSERVER_ADMIN_USER}:$${GEOSERVER_ADMIN_PASSWORD} http://localhost:8080/geoserver/rest/about/version.xml" ] interval: 1m30s From e4528ea542acb789787a842563a5f475205feb7e Mon Sep 17 00:00:00 2001 From: NyakudyaA Date: Fri, 3 Jan 2025 18:02:35 +0200 Subject: [PATCH 4/6] Fix test --- scenario_tests/backup_restore/docker-compose.yml | 2 +- scenario_tests/backup_restore/tests/test_geoserver_backup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scenario_tests/backup_restore/docker-compose.yml b/scenario_tests/backup_restore/docker-compose.yml index 85aa872e..95598792 100644 --- a/scenario_tests/backup_restore/docker-compose.yml +++ b/scenario_tests/backup_restore/docker-compose.yml @@ -41,7 +41,7 @@ services: GEOSERVER_ADMIN_USER: admin CONSOLE_HANDLER_LEVEL: WARNING COMMUNITY_EXTENSIONS: backup-restore-plugin - TEST_CLASS: test_geoserver_backup.TestGeoServerRestore + TEST_CLASS: test_geoserver_restore.TestGeoServerRestore ports: - "8080" healthcheck: diff --git a/scenario_tests/backup_restore/tests/test_geoserver_backup.py b/scenario_tests/backup_restore/tests/test_geoserver_backup.py index 9c559dc0..5c3fcb4b 100644 --- a/scenario_tests/backup_restore/tests/test_geoserver_backup.py +++ b/scenario_tests/backup_restore/tests/test_geoserver_backup.py @@ -51,7 +51,7 @@ def test_create_backup(self): execution_id = response_data["backup"]["execution"]["id"] execution_url = f"{self.gs_url}/rest/br/backup/{execution_id}.json" # wait for backup to complete - time.sleep(30) + time.sleep(40) response_execution_request = requests.get(execution_url, auth=auth) if response_execution_request.status_code == 200: try: From 1fd78e684f36783a988a267f75c15fd5c0f7ff62 Mon Sep 17 00:00:00 2001 From: NyakudyaA Date: Fri, 3 Jan 2025 18:03:20 +0200 Subject: [PATCH 5/6] Fix test for restore --- scenario_tests/backup_restore/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/scenario_tests/backup_restore/docker-compose.yml b/scenario_tests/backup_restore/docker-compose.yml index 95598792..736bd69d 100644 --- a/scenario_tests/backup_restore/docker-compose.yml +++ b/scenario_tests/backup_restore/docker-compose.yml @@ -40,6 +40,7 @@ services: GEOSERVER_ADMIN_PASSWORD: myawesomegeoserver GEOSERVER_ADMIN_USER: admin CONSOLE_HANDLER_LEVEL: WARNING + RECREATE_DATADIR: TRUE COMMUNITY_EXTENSIONS: backup-restore-plugin TEST_CLASS: test_geoserver_restore.TestGeoServerRestore ports: From afe8e1d8ae8551faedf1cb7f8fe3d25b16225dc8 Mon Sep 17 00:00:00 2001 From: NyakudyaA Date: Fri, 3 Jan 2025 19:08:05 +0200 Subject: [PATCH 6/6] revert env variables --- .env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index fce24310..e841847f 100644 --- a/.env +++ b/.env @@ -126,13 +126,13 @@ STABLE_EXTENSIONS= # Allow overriding default installed extension combined with stable extensions ACTIVE_EXTENSIONS= # Install the community edition plugins specified in https://github.com/kartoza/docker-geoserver/blob/master/build_data/community_plugins.txt -COMMUNITY_EXTENSIONS=backup-restore-plugin +COMMUNITY_EXTENSIONS= # Geoserver monitoring and audit logging https://docs.geoserver.org/main/en/user/extensions/monitoring/audit.html MONITORING_AUDIT_ENABLED=false MONITORING_AUDIT_ROLL_LIMIT=20 RESET_MONITORING_LOGS=false # Geoserver logging -GEOSERVER_LOG_PROFILE=VERBOSE_LOGGING +GEOSERVER_LOG_PROFILE=DEFAULT_LOGGING GEOSERVER_LOG_DIR=/opt/geoserver/data_dir/logs # Run as root RUN_AS_ROOT=true