Merge pull request #159 from james-r-hmlr/elasticsearch7

Added elasticsearch7 commodity

README.md

@@ -128,18 +128,19 @@ The list of allowable commodity values is:
 2. postgres-17
 3. db2_community
 4. elasticsearch5
-5. nginx
-6. rabbitmq
-7. redis
-8. swagger
-9. wiremock
-10. squid
-11. auth
-12. cadence
-13. cadence-web
-14. activemq
-15. ibmmq
-16. localstack
+5. elasticsearch7
+6. nginx
+7. rabbitmq
+8. redis
+9. swagger
+10. wiremock
+11. squid
+12. auth
+13. cadence
+14. cadence-web
+15. activemq
+16. ibmmq
+17. localstack
 
 The file may optionally also indicate that one or more services are resource intensive ("expensive") when starting up. The dev env will start those containers seperately - 3 at a time - and wait until each are declared healthy (or crash and get restarted 10 times) before starting any more.
 
@@ -193,7 +194,20 @@ The ports 9300 and 9302 are exposed on the host.
 
 [Example](snippets/elasticsearch5-fragment.sh)
 
-##### nginx
+##### ElasticSearch 7
+
+The ports 9207 and 9307 are exposed on the host.
+
+If the ElasticSearch 7 container is returning the follow error log message:
+```
+max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]
+```
+Run the following command in a terminal to set the system's max map count.
+```
+sysctl -w vm.max_map_count=262144
+```
+
+##### Nginx
 
 **`/fragments/nginx-fragment.conf`**
 

scripts/commodities.rb

@@ -6,6 +6,7 @@
 require_relative 'provision_db2_community'
 require_relative 'provision_nginx'
 require_relative 'provision_elasticsearch5'
+require_relative 'provision_elasticsearch7'
 require_relative 'provision_wiremock'
 require_relative 'provision_localstack'
 
@@ -145,6 +146,8 @@ def provision_commodities(root_loc, new_containers)
   provision_nginx(root_loc, new_containers)
   # Elasticsearch5
   provision_elasticsearch5(root_loc)
+  # Elasticsearch7
+  provision_elasticsearch7(root_loc)
   # Auth
   provision_auth(root_loc, new_containers)
   # Wiremock mappings

scripts/docker/elasticsearch7/Dockerfile

@@ -0,0 +1,9 @@
+FROM docker.elastic.co/elasticsearch/elasticsearch:7.17.24
+
+ENV ES_JAVA_OPTS -Xms1024m -Xmx1024m
+ENV discovery.type single-node
+
+# Remove default heap size and add low-memory optimisations
+RUN echo "bootstrap.memory_lock: true" >> /usr/share/elasticsearch/config/elasticsearch.yml && \
+ echo "indices.fielddata.cache.size: 50%" >> /usr/share/elasticsearch/config/elasticsearch.yml && \
+ echo "indices.memory.index_buffer_size: 50%" >> /usr/share/elasticsearch/config/elasticsearch.yml

scripts/docker/elasticsearch7/compose-fragment.yml

@@ -0,0 +1,18 @@
+services:
+  elasticsearch7:
+    container_name: elasticsearch7
+    build: ../scripts/docker/elasticsearch7
+    ports:
+      - "9207:9200"
+      - "9307:9300"
+    # restart: on-failure
+    platform: "linux/amd64"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    cap_add:
+      - IPC_LOCK

scripts/docker/elasticsearch7/docker-compose-fragment.3.7.yml

@@ -0,0 +1,18 @@
+version: '3.7'
+services:
+  elasticsearch7:
+    container_name: elasticsearch7
+    build: ../scripts/docker/elasticsearch7
+    ports:
+      - "9207:9200"
+      - "9307:9300"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    cap_add:
+      - IPC_LOCK
+    restart: on-failure

scripts/docker/elasticsearch7/docker-compose-fragment.yml

@@ -0,0 +1,19 @@
+version: '2'
+services:
+  elasticsearch7:
+    container_name: elasticsearch7
+    build: ../scripts/docker/elasticsearch7
+    ports:
+      - "9207:9200"
+      - "9307:9300"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    mem_limit: 2048m
+    cap_add:
+      - IPC_LOCK
+    restart: on-failure

scripts/provision_elasticsearch7.rb

@@ -0,0 +1,56 @@
+require_relative 'utilities'
+require_relative 'commodities'
+require 'yaml'
+
+def provision_elasticsearch7(root_loc)
+  puts colorize_lightblue('Searching for elasticsearch7 initialisation scripts in the apps')
+
+  # Load configuration.yml into a Hash
+  config = YAML.load_file("#{root_loc}/dev-env-config/configuration.yml")
+  started = false
+  return unless config['applications']
+
+  config['applications'].each_key do |appname|
+    # To help enforce the accuracy of the app's dependency file, only search for init scripts
+    # if the app specifically specifies elasticsearch in it's commodity list
+    next unless File.exist?("#{root_loc}/apps/#{appname}/configuration.yml")
+    next unless commodity_required?(root_loc, appname, 'elasticsearch7')
+
+    # Run any script contained in the app
+    if File.exist?("#{root_loc}/apps/#{appname}/fragments/elasticsearch7-fragment.sh")
+      started = start_elasticsearch7(root_loc, appname, started)
+    else
+      puts colorize_yellow("#{appname} says it uses Elasticsearch 7 but doesn't contain an init script. Oh well, " \
+                            'onwards we go!')
+    end
+  end
+end
+
+def start_elasticsearch7(root_loc, appname, started)
+  puts colorize_pink("Found some in #{appname}")
+  if commodity_provisioned?(root_loc, appname, 'elasticsearch7')
+    puts colorize_yellow("Elasticsearch7 has previously been provisioned for #{appname}, skipping")
+  else
+    unless started
+      run_command("#{ENV['DC_CMD']} up -d elasticsearch7")
+      # Better not run anything until elasticsearch is ready to accept connections...
+      puts colorize_lightblue('Waiting for Elasticsearch 7 to finish initialising')
+
+      loop do
+        cmd_output = []
+        run_command('curl --write-out "%{http_code}" --silent --output /dev/null http://localhost:9202', cmd_output)
+        break if cmd_output.include? '200'
+
+        puts colorize_yellow('Elasticsearch 7 is unavailable - sleeping')
+        sleep(3)
+      end
+
+      puts colorize_green('Elasticsearch 7 is ready')
+      started = true
+    end
+    run_command("sh #{root_loc}/apps/#{appname}/fragments/elasticsearch7-fragment.sh http://localhost:9202")
+    # Update the .commodities.yml to indicate that elasticsearch7 has now been provisioned
+    set_commodity_provision_status(root_loc, appname, 'elasticsearch7', true)
+  end
+  started
+end