docker-compose.yml

version: '3.1'

services:
  dc:
    build: streamsets
    container_name: dc
    ports:
      - "18630:18630"
    restart: on-failure
    volumes:
      - sdc-data:/data
      - ./certs:/home/certs
      - ./test-datasets/test-directory-collector:/home/test-directory-collector
#      - ./test-datasets:/kerberos
#      - ./test-datasets/krb5.conf:/etc/krb5.conf
    environment:
      SDC_CONF_PRODUCTION_MAXBATCHSIZE: "50"
#      SDC_JAVA_OPTS: "-Djava.security.auth.login.config=/kerberos/kafka-consumer_jaas.conf -Dsun.security.krb5.debug=true -Djava.security.krb5.conf=/etc/krb5.conf"
#      SDC_CONF_PARSER_LIMIT: 2048576
      SDC_CONF_RUNNER_THREAD_POOL_SIZE: "2000"
#    extra_hosts:
#      - "daria-test2:10.0.0.28"

#  creating second streamsets to run tests on multiple instances
  dc2:
    build: streamsets
    container_name: dc2
    ports:
      - "18629:18630"
    restart: on-failure
    volumes:
      - sdc-data2:/data
      - ./certs:/home/certs
      - ./test-datasets/test-directory-collector:/home/test-directory-collector
    environment:
      SDC_CONF_PRODUCTION_MAXBATCHSIZE: "50"
      SDC_CONF_RUNNER_THREAD_POOL_SIZE: "2000"

  agent:
    build: agent
    restart: always
    container_name: anodot-agent
    hostname: agent
    environment:
      STREAMSETS_USERNAME: 'admin'
      STREAMSETS_PASSWORD: 'admin'
      STREAMSETS_URL: 'http://dc:18630'
      LOG_FILE_PATH: '/var/log/agent.log'
      ENV_PROD: 'false'
      AGENT_DB_HOST: 'db'
      AGENT_DB_USER: 'agent'
      AGENT_DB_PASSWORD: 'agent'
      PGPASSWORD: 'agent'
      AGENT_URL: 'http://anodot-agent'
#      VALIDATION_ENABLED: 'false'
    stdin_open: true
    tty: true
    depends_on:
      - dc
      - db
    volumes:
      - ./agent/backup-data:/usr/src/app/backup-data
#      - ./scripts:/usr/src/app/scripts
      - dummy-destination-data:/output
      - ./test-datasets:/home

  db:
    container_name: db
    image: postgres:9.6.19
    restart: always
    ports:
      - "5432:5432"
    environment:
      POSTGRES_USER: agent
      POSTGRES_PASSWORD: agent
      POSTGRES_DB: agent
    volumes:
      - /var/lib/postgresql/data

  mongo:
    image: mongo:4.4.1
    restart: always
#    ports:
#      - "27017:27017"
    environment:
      MONGO_INITDB_ROOT_USERNAME: root
      MONGO_INITDB_ROOT_PASSWORD: root
    volumes:
      - /data/db
      - ./test-datasets:/home
      - ./volumes/mongo-initdb:/docker-entrypoint-initdb.d

  zookeeper:
    image: wurstmeister/zookeeper
#    ports:
#      - "2181:2181"
  kafka:
#    image: wurstmeister/kafka:0.10.2.1
    image: wurstmeister/kafka:2.12-2.4.1
    container_name: agent-kafka
    depends_on:
      - zookeeper
#    ports:
#      - "9092:9092"
#    expose:
#      - "9093"
#      - "9094"
    environment:
      KAFKA_LISTENERS: PLAINTEXT://:29092,PLAINTEXT_HOST://:9092
#      KAFKA_LISTENERS: PLAINTEXT://:29092,PLAINTEXT_HOST://:9092,SSL://:9093,SSL_DOCKER://:9094
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
#      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092,SSL://localhost:9093,SSL_DOCKER://kafka:9094
#      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,SSL:SSL,SSL_DOCKER:SSL
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
#      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_CREATE_TOPICS: "test:1:1,test-json:1:1"
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
#      KAFKA_SSL_KEYSTORE_LOCATION: '/certs/docker.kafka.server.keystore.jks'
#      KAFKA_SSL_KEYSTORE_PASSWORD: 'kafkadocker'
#      KAFKA_SSL_KEY_PASSWORD: 'kafkadocker'
#      KAFKA_SSL_TRUSTSTORE_LOCATION: '/certs/docker.kafka.server.truststore.jks'
#      KAFKA_SSL_TRUSTSTORE_PASSWORD: 'kafkadocker'
#      KAFKA_SSL_CLIENT_AUTH: 'required'
#      KAFKA_SECURITY_INTER_BROKER_PROTOCOL: 'SSL'
#      KAFKA_SECURITY_PROTOCOL: 'SSL'
#      KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: ''
#      KAFKA_LOG4J_ROOT_LOGLEVEL: 'DEBUG, stdout'
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - ./test-datasets:/home
#      - ./certs:/certs

  influx:
    image: influxdb:1.8.3
#    ports:
#      - "8086:8086"
    volumes:
      - /var/lib/influxdb
      - ./volumes/influx-initdb:/docker-entrypoint-initdb.d
    environment:
      INFLUXDB_DB: test
#      INFLUXDB_HTTP_AUTH_ENABLED: 'true'
#      INFLUXDB_ADMIN_USER: admin
#      INFLUXDB_ADMIN_PASSWORD: admin
#      INFLUXDB_READ_USER: ro
#      INFLUXDB_READ_USER_PASSWORD: roro

  squid:
    image: datadog/squid
    extra_hosts:
      - "daria-test2:10.0.0.28"

  mysql:
    image: mysql:8.0.21
    container_name: agent-mysql
    command: --default-authentication-plugin=mysql_native_password
    environment:
      MYSQL_ROOT_HOST: "%"
      MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
      MYSQL_DATABASE: "zabbix"
      MYSQL_USER: "user"
      MYSQL_PASSWORD: "password"
    volumes:
      - /var/lib/mysql
      - ./volumes/mysql-initdb:/docker-entrypoint-initdb.d

  postgres:
    image: postgres:9.6.19
    environment:
      POSTGRES_DB: test
      POSTGRES_PASSWORD: password
    volumes:
      - /var/lib/postgresql/data
      - ./volumes/postgres-initdb:/docker-entrypoint-initdb.d

  es:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.5.2
    container_name: agent-es
    environment:
      - node.name=es01
      - cluster.name=es-docker-cluster
      - cluster.initial_master_nodes=es01
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    volumes:
      - /usr/share/elasticsearch/data
    ports:
      - 9200:9200

  dummy_destination:
    build: dummy_destination
    container_name: dummy_destination
    volumes:
      - dummy-destination-data:/app/log

  sage:
    build: sage_mock

  victoriametrics:
    container_name: victoriametrics
    image: victoriametrics/victoria-metrics:v1.43.0
    ports:
      - 8428:8428
    volumes:
      - /storage
    command:
      - '--storageDataPath=/storage'
      - '--httpListenAddr=:8428'
      - '--retentionPeriod=600'
    restart: always

  zabbix-server:
    image: zabbix/zabbix-server-mysql:alpine-4.4.10
    depends_on:
      - mysql
    environment:
      DB_SERVER_HOST: mysql
      MYSQL_USER: "user"
      MYSQL_PASSWORD: "password"

  zabbix-web:
    image: zabbix/zabbix-web-nginx-mysql:alpine-4.4.10
    ports:
      - 8888:8080
    depends_on:
      - zabbix-server
    environment:
      DB_SERVER_HOST: mysql
      MYSQL_USER: "user"
      MYSQL_PASSWORD: "password"

  zabbix-agent:
    image: zabbix/zabbix-agent:alpine-4.4.10
    hostname: zabbixagent
    depends_on:
      - zabbix-server

  clickhouse:
    image: yandex/clickhouse-server
    volumes:
      - /var/lib/clickhouse
      - ./volumes/clickhouse-initdb:/docker-entrypoint-initdb.d

volumes:
  sdc-data:
  sdc-data2:
  agent-data:
  dummy-destination-data: