From 529d14c3cbab4d9f9f22348aed02199bdf1a2a4f Mon Sep 17 00:00:00 2001 From: james-r-hmlr Date: Mon, 23 Sep 2024 12:24:00 +0000 Subject: [PATCH] Added elasticsearch7 commodity --- README.md | 38 +++++++++---- scripts/commodities.rb | 3 + scripts/docker/elasticsearch7/Dockerfile | 9 +++ .../elasticsearch7/compose-fragment.yml | 18 ++++++ .../docker-compose-fragment.3.7.yml | 18 ++++++ .../docker-compose-fragment.yml | 19 +++++++ scripts/provision_elasticsearch7.rb | 56 +++++++++++++++++++ 7 files changed, 149 insertions(+), 12 deletions(-) create mode 100644 scripts/docker/elasticsearch7/Dockerfile create mode 100644 scripts/docker/elasticsearch7/compose-fragment.yml create mode 100644 scripts/docker/elasticsearch7/docker-compose-fragment.3.7.yml create mode 100644 scripts/docker/elasticsearch7/docker-compose-fragment.yml create mode 100644 scripts/provision_elasticsearch7.rb diff --git a/README.md b/README.md index c5573ca..a4896e4 100644 --- a/README.md +++ b/README.md @@ -127,18 +127,19 @@ The list of allowable commodity values is: 1. postgres-13 2. db2_community 4. elasticsearch5 -5. nginx -6. rabbitmq -7. redis -8. swagger -9. wiremock -10. squid -11. auth -12. cadence -13. cadence-web -14. activemq -15. ibmmq -16. localstack +5. elasticsearch7 +6. nginx +7. rabbitmq +8. redis +9. swagger +10. wiremock +11. squid +12. auth +13. cadence +14. cadence-web +15. activemq +16. ibmmq +17. localstack The file may optionally also indicate that one or more services are resource intensive ("expensive") when starting up. The dev env will start those containers seperately - 3 at a time - and wait until each are declared healthy (or crash and get restarted 10 times) before starting any more. @@ -192,6 +193,19 @@ The ports 9300 and 9302 are exposed on the host. [Example](snippets/elasticsearch5-fragment.sh) +##### ElasticSearch 7 + +The ports 9207 and 9307 are exposed on the host. + +If the ElasticSearch 7 container is returning the follow error log message: +``` +max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144] +``` +Run the following command in a terminal to set the system's max map count. +``` +sysctl -w vm.max_map_count=262144 +``` + ##### Nginx **`/fragments/nginx-fragment.conf`** diff --git a/scripts/commodities.rb b/scripts/commodities.rb index e986d91..7256e49 100644 --- a/scripts/commodities.rb +++ b/scripts/commodities.rb @@ -6,6 +6,7 @@ require_relative 'provision_db2_community' require_relative 'provision_nginx' require_relative 'provision_elasticsearch5' +require_relative 'provision_elasticsearch7' require_relative 'provision_wiremock' require_relative 'provision_localstack' @@ -145,6 +146,8 @@ def provision_commodities(root_loc, new_containers) provision_nginx(root_loc, new_containers) # Elasticsearch5 provision_elasticsearch5(root_loc) + # Elasticsearch7 + provision_elasticsearch7(root_loc) # Auth provision_auth(root_loc, new_containers) # Wiremock mappings diff --git a/scripts/docker/elasticsearch7/Dockerfile b/scripts/docker/elasticsearch7/Dockerfile new file mode 100644 index 0000000..88a7a94 --- /dev/null +++ b/scripts/docker/elasticsearch7/Dockerfile @@ -0,0 +1,9 @@ +FROM docker.elastic.co/elasticsearch/elasticsearch:7.17.24 + +ENV ES_JAVA_OPTS -Xms1024m -Xmx1024m +ENV discovery.type single-node + +# Remove default heap size and add low-memory optimisations +RUN echo "bootstrap.memory_lock: true" >> /usr/share/elasticsearch/config/elasticsearch.yml && \ + echo "indices.fielddata.cache.size: 50%" >> /usr/share/elasticsearch/config/elasticsearch.yml && \ + echo "indices.memory.index_buffer_size: 50%" >> /usr/share/elasticsearch/config/elasticsearch.yml diff --git a/scripts/docker/elasticsearch7/compose-fragment.yml b/scripts/docker/elasticsearch7/compose-fragment.yml new file mode 100644 index 0000000..b496b22 --- /dev/null +++ b/scripts/docker/elasticsearch7/compose-fragment.yml @@ -0,0 +1,18 @@ +services: + elasticsearch7: + container_name: elasticsearch7 + build: ../scripts/docker/elasticsearch7 + ports: + - "9207:9200" + - "9307:9300" + # restart: on-failure + platform: "linux/amd64" + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + cap_add: + - IPC_LOCK diff --git a/scripts/docker/elasticsearch7/docker-compose-fragment.3.7.yml b/scripts/docker/elasticsearch7/docker-compose-fragment.3.7.yml new file mode 100644 index 0000000..8cd45cd --- /dev/null +++ b/scripts/docker/elasticsearch7/docker-compose-fragment.3.7.yml @@ -0,0 +1,18 @@ +version: '3.7' +services: + elasticsearch7: + container_name: elasticsearch7 + build: ../scripts/docker/elasticsearch7 + ports: + - "9207:9200" + - "9307:9300" + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + cap_add: + - IPC_LOCK + restart: on-failure diff --git a/scripts/docker/elasticsearch7/docker-compose-fragment.yml b/scripts/docker/elasticsearch7/docker-compose-fragment.yml new file mode 100644 index 0000000..28cd476 --- /dev/null +++ b/scripts/docker/elasticsearch7/docker-compose-fragment.yml @@ -0,0 +1,19 @@ +version: '2' +services: + elasticsearch7: + container_name: elasticsearch7 + build: ../scripts/docker/elasticsearch7 + ports: + - "9207:9200" + - "9307:9300" + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + mem_limit: 2048m + cap_add: + - IPC_LOCK + restart: on-failure diff --git a/scripts/provision_elasticsearch7.rb b/scripts/provision_elasticsearch7.rb new file mode 100644 index 0000000..3083ccb --- /dev/null +++ b/scripts/provision_elasticsearch7.rb @@ -0,0 +1,56 @@ +require_relative 'utilities' +require_relative 'commodities' +require 'yaml' + +def provision_elasticsearch7(root_loc) + puts colorize_lightblue('Searching for elasticsearch7 initialisation scripts in the apps') + + # Load configuration.yml into a Hash + config = YAML.load_file("#{root_loc}/dev-env-config/configuration.yml") + started = false + return unless config['applications'] + + config['applications'].each_key do |appname| + # To help enforce the accuracy of the app's dependency file, only search for init scripts + # if the app specifically specifies elasticsearch in it's commodity list + next unless File.exist?("#{root_loc}/apps/#{appname}/configuration.yml") + next unless commodity_required?(root_loc, appname, 'elasticsearch7') + + # Run any script contained in the app + if File.exist?("#{root_loc}/apps/#{appname}/fragments/elasticsearch7-fragment.sh") + started = start_elasticsearch7(root_loc, appname, started) + else + puts colorize_yellow("#{appname} says it uses Elasticsearch 7 but doesn't contain an init script. Oh well, " \ + 'onwards we go!') + end + end +end + +def start_elasticsearch7(root_loc, appname, started) + puts colorize_pink("Found some in #{appname}") + if commodity_provisioned?(root_loc, appname, 'elasticsearch7') + puts colorize_yellow("Elasticsearch7 has previously been provisioned for #{appname}, skipping") + else + unless started + run_command("#{ENV['DC_CMD']} up -d elasticsearch7") + # Better not run anything until elasticsearch is ready to accept connections... + puts colorize_lightblue('Waiting for Elasticsearch 7 to finish initialising') + + loop do + cmd_output = [] + run_command('curl --write-out "%{http_code}" --silent --output /dev/null http://localhost:9202', cmd_output) + break if cmd_output.include? '200' + + puts colorize_yellow('Elasticsearch 7 is unavailable - sleeping') + sleep(3) + end + + puts colorize_green('Elasticsearch 7 is ready') + started = true + end + run_command("sh #{root_loc}/apps/#{appname}/fragments/elasticsearch7-fragment.sh http://localhost:9202") + # Update the .commodities.yml to indicate that elasticsearch7 has now been provisioned + set_commodity_provision_status(root_loc, appname, 'elasticsearch7', true) + end + started +end