From a989b428fef328e0947913624684ba37552c2eb3 Mon Sep 17 00:00:00 2001 From: Admire Nyakudya Date: Mon, 23 Dec 2024 07:37:55 +0200 Subject: [PATCH 1/2] improve bucket creation --- docker-compose-s3.yml | 2 +- scenario_tests/s3/docker-compose.yml | 2 +- scripts/create_default_buckets.py | 106 +++++++++++++++++++++------ 3 files changed, 84 insertions(+), 26 deletions(-) diff --git a/docker-compose-s3.yml b/docker-compose-s3.yml index 835c256..e8dfbf5 100644 --- a/docker-compose-s3.yml +++ b/docker-compose-s3.yml @@ -21,7 +21,7 @@ services: entrypoint: /bin/bash command: -c 'minio server /data --console-address ":9001"' volumes: - - minio_data:/mapproxy + - minio_data:/data ports: - "9000:9000" - "9001:9001" diff --git a/scenario_tests/s3/docker-compose.yml b/scenario_tests/s3/docker-compose.yml index b7bd01a..1b1a37c 100644 --- a/scenario_tests/s3/docker-compose.yml +++ b/scenario_tests/s3/docker-compose.yml @@ -19,7 +19,7 @@ services: entrypoint: /bin/bash command: -c 'minio server /data --console-address ":9001"' volumes: - - minio_data:/mapproxy + - minio_data:/data ports: - "9000:9000" - "9001:9001" diff --git a/scripts/create_default_buckets.py b/scripts/create_default_buckets.py index 83c00c6..4791c9f 100644 --- a/scripts/create_default_buckets.py +++ b/scripts/create_default_buckets.py @@ -1,42 +1,100 @@ -from os import environ as env +import os import re -import boto3, botocore +import logging +from typing import List +import boto3 +from botocore.exceptions import ClientError -def check_bucket(s3, bucket_name): +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def get_environment_variable(name: str, required: bool = True) -> str: + """ + Retrieves an environment variable or raises an error if it's missing. + """ + value = os.getenv(name) + if required and not value: + raise ValueError(f"Missing required environment variable: {name}") + return value + + +def parse_bucket_list(bucket_list: str) -> List[str]: + """ + Parses a string of bucket names separated by commas, spaces, or semicolons into a list. + """ + return re.split(r'[ ,;]+', bucket_list.strip()) + + +def check_bucket(s3_client, bucket_name: str) -> bool: + """ + Checks if a bucket exists and is accessible. + + Returns True if the bucket exists or is private, False otherwise. + """ try: - s3.head_bucket(Bucket=bucket_name) - print(f"{bucket_name} available") + s3_client.head_bucket(Bucket=bucket_name) + logger.info(f"Bucket '{bucket_name}' is available.") return True - except botocore.exceptions.ClientError as e: + except ClientError as e: error_code = int(e.response['Error']['Code']) if error_code == 403: - print(f"{bucket_name} is Private. Access denied.") + logger.warning(f"Bucket '{bucket_name}' is private. Access denied.") return True elif error_code == 404: - print(f"{bucket_name} does not exist") + logger.info(f"Bucket '{bucket_name}' does not exist.") return False + else: + logger.error(f"Error checking bucket '{bucket_name}': {e}") + raise + + +def create_bucket(s3_client, bucket_name: str): + """ + Creates a bucket if it does not exist. + """ + try: + logger.info(f"Creating bucket '{bucket_name}'.") + s3_client.create_bucket(Bucket=bucket_name) + logger.info(f"Bucket '{bucket_name}' created successfully.") + except ClientError as e: + logger.error(f"Failed to create bucket '{bucket_name}': {e}") + raise + def main(): - buckets = env['S3_BUCKET_LIST'] - buckets = re.split(r',| |;', buckets) - end_point = env['S3_BUCKET_ENDPOINT'] + """ + Main function to check and create S3 buckets as needed. + """ + bucket_list_str = get_environment_variable('S3_BUCKET_LIST') + buckets = parse_bucket_list(bucket_list_str) + endpoint = get_environment_variable('S3_BUCKET_ENDPOINT') - session = boto3.session.Session() + aws_access_key_id = get_environment_variable('AWS_ACCESS_KEY_ID') + aws_secret_access_key = get_environment_variable('AWS_SECRET_ACCESS_KEY') + session = boto3.session.Session() s3 = session.client( service_name='s3', - aws_access_key_id=env['AWS_ACCESS_KEY_ID'], - aws_secret_access_key=env['AWS_SECRET_ACCESS_KEY'], - endpoint_url=end_point, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + endpoint_url=endpoint, ) - for i in buckets: - if not check_bucket(s3, i): - print(f"Creating {i}") - s3.create_bucket(Bucket=i) + for bucket_name in buckets: + if not check_bucket(s3, bucket_name): + create_bucket(s3, bucket_name) + -if __name__=="__main__": - create_buckets = env['CREATE_DEFAULT_S3_BUCKETS'] - if create_buckets.lower() == 'true': - print("Creating default buckets") - main() +if __name__ == "__main__": + try: + create_buckets = get_environment_variable('CREATE_DEFAULT_S3_BUCKETS', required=False) + if create_buckets and create_buckets.lower() == 'true': + logger.info("Starting bucket creation process.") + main() + else: + logger.info("CREATE_DEFAULT_S3_BUCKETS is not set to 'true'. Skipping bucket creation.") + except Exception as e: + logger.error(f"An error occurred: {e}") + exit(1) From 58f0dbe665cddeb945cdcfc3d3fb5043f23d0d90 Mon Sep 17 00:00:00 2001 From: Admire Nyakudya Date: Mon, 23 Dec 2024 20:56:54 +0200 Subject: [PATCH 2/2] update configs --- .github/dependabot.yml | 2 +- mapproxy_configuration/mapproxy-s3.yaml | 12 ++++++------ mapproxy_configuration/mapproxy.yaml | 12 ++++++------ .../s3/mapproxy_configuration/mapproxy.yaml | 13 ++++++------- scripts/start.sh | 2 ++ 5 files changed, 21 insertions(+), 20 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index adee0ed..120c689 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,4 +3,4 @@ updates: - package-ecosystem: "github-actions" directory: "/" schedule: - interval: "monthly" \ No newline at end of file + interval: "weekly" \ No newline at end of file diff --git a/mapproxy_configuration/mapproxy-s3.yaml b/mapproxy_configuration/mapproxy-s3.yaml index f337632..e6d6999 100644 --- a/mapproxy_configuration/mapproxy-s3.yaml +++ b/mapproxy_configuration/mapproxy-s3.yaml @@ -39,13 +39,13 @@ services: contact: person: Kartoza position: GIS Manager - organization: - address: - city: - postcode: + organization: Kartoza + address: Kartoza + city: Cape Town + postcode: 7700 country: south Africa - phone: - email: + phone: '276426345' + email: info@kartoza.com access_constraints: Insert license and copyright information for this service. fees: "None" tms: diff --git a/mapproxy_configuration/mapproxy.yaml b/mapproxy_configuration/mapproxy.yaml index 0fdc021..ee31d5a 100644 --- a/mapproxy_configuration/mapproxy.yaml +++ b/mapproxy_configuration/mapproxy.yaml @@ -39,13 +39,13 @@ services: contact: person: Kartoza position: GIS Manager - organization: - address: - city: - postcode: + organization: Kartoza + address: Kartoza + city: Cape Town + postcode: 7700 country: south Africa - phone: - email: + phone: '276426345' + email: info@kartoza.com access_constraints: Insert license and copyright information for this service. fees: 'None' diff --git a/scenario_tests/s3/mapproxy_configuration/mapproxy.yaml b/scenario_tests/s3/mapproxy_configuration/mapproxy.yaml index 7c08e53..fd06735 100644 --- a/scenario_tests/s3/mapproxy_configuration/mapproxy.yaml +++ b/scenario_tests/s3/mapproxy_configuration/mapproxy.yaml @@ -39,13 +39,13 @@ services: contact: person: Kartoza position: GIS Manager - organization: - address: - city: - postcode: + organization: Kartoza + address: Kartoza + city: Cape Town + postcode: 7700 country: south Africa - phone: - email: + phone: '276426345' + email: info@kartoza.com access_constraints: Insert license and copyright information for this service. fees: "None" tms: @@ -70,7 +70,6 @@ caches: grids: [osm_grid] meta_size: [5, 5] meta_buffer: 20 - concurrent_tile_creators: 2 cache: type: s3 directory: /hillshade/ diff --git a/scripts/start.sh b/scripts/start.sh index 56b8490..4a5c9b1 100755 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -33,6 +33,7 @@ function entry_point_script { for f in /docker-entrypoint-mapproxy.d/*; do case "$f" in *.sh) echo "$0: running $f"; . "$f" || true;; + *.py) echo "$0: running Python script $f"; python3 "$f" || true;; *) echo "$0: ignoring $f" ;; esac echo @@ -50,6 +51,7 @@ function cleanup_files(){ if [[ $proxy_count != 0 ]];then for X in ${PARAM}_*.${EXT}; do if [ "$X" != "${PARAM}_${HOSTNAME}.${EXT}" ]; then + echo "Removing $X" rm -rf "$X" fi done