docker-compose.yml

version: '3.7'

x-restart-policy: &restart_policy
  restart: unless-stopped

x-sentry-defaults: &sentry_defaults
  << : *restart_policy
  build:
    context: ./sentry
    args:
      - SENTRY_IMAGE
  image: sentry-onpremise-local
  depends_on:
    - redis
    - postgres
    - memcached
    - smtp
    - snuba-api
    - snuba-consumer
    - snuba-outcomes-consumer
    - snuba-replacer
    - symbolicator
    - kafka
  environment:
    SENTRY_CONF: '/etc/sentry'
    SNUBA: 'http://snuba-api:1218'
  volumes:
    - 'sentry-data:/data'
    - './sentry:/etc/sentry'
  networks:
    - host_network

x-snuba-defaults: &snuba_defaults
  << : *restart_policy
  depends_on:
    - redis
    - clickhouse
    - kafka
  image: 'getsentry/snuba:latest'
  environment:
    SNUBA_SETTINGS: docker
    CLICKHOUSE_HOST: clickhouse
    DEFAULT_BROKERS: 'kafka:9092'
    REDIS_HOST: redis
    UWSGI_MAX_REQUESTS: '10000'
    UWSGI_DISABLE_LOGGING: 'true'
  networks:
    - host_network

x-host-config: &host_config
  environment:
    - ASPNETCORE_ENVIRONMENT=Development
    - ASPNETCORE_URLS=http://+:80
  ports:
    - "5050:80"

services:
  host:
    << : *restart_policy
    << : *host_config
    depends_on:
      - postgres
      - worker
      - mongodb
      - proxy
      - selenium
      - scrapy
    container_name: maxim_emm_bots_host
    image: maxim_emm_bots/host:latest
    build:
      context: .
      dockerfile: Host/Dockerfile
    networks:
      - host_network
      - reviewbot
  
  scrapy:
    build:
      context: .
      dockerfile: scrapy/Dockerfile
    image: maxim_emm_bots/scrapy:latest

  postgres:
    << : *restart_policy
    image: postgres:latest
    environment:
      - POSTGRES_HOST_AUTH_METHOD=trust
      - POSTGRES_USER=postgres
      - POSTGRES_DB=auth
    command: ["-c", "shared_buffers=512MB", "-c", "max_connections=500"]
    volumes:
      - host_volume:/var/lib/postgresql/data
    networks:
      - host_network
        
  mongodb:
    << : *restart_policy
    image: mongo:4.0
    container_name: "mongodb"
    environment:
      - MONGO_DATA_DIR=/data/db
      - MONGO_LOG_DIR=/dev/null
    volumes:
      - mongodb_data:/data/db
    command: mongod --smallfiles --logpath=/dev/null
    networks:
      - reviewbot
        
  proxy:
    << : *restart_policy
    image: dperson/torproxy
    container_name: "proxy"
    networks:
      - reviewbot
        
  selenium:
    << : *restart_policy
    image: selenium/standalone-firefox:3
    container_name: "selenium"
    shm_size: '2g'
    environment:
      - START_XVFB=false
    networks:
      - reviewbot

  smtp:
    <<: *restart_policy
    image: tianon/exim4
    volumes:
      - 'sentry-smtp:/var/spool/exim4'
      - 'sentry-smtp-log:/var/log/exim4'
    networks:
      - host_network

  memcached:
    <<: *restart_policy
    image: 'memcached:1.5-alpine'
    networks:
      - host_network

  redis:
    <<: *restart_policy
    image: 'redis:5.0-alpine'
    volumes:
      - 'sentry-redis:/data'
    networks:
      - host_network

  zookeeper:
    <<: *restart_policy
    image: 'confluentinc/cp-zookeeper:5.5.0'
    environment:
      ZOOKEEPER_CLIENT_PORT: '2181'
      CONFLUENT_SUPPORT_METRICS_ENABLE: 'false'
      ZOOKEEPER_LOG4J_ROOT_LOGLEVEL: 'WARN'
      ZOOKEEPER_TOOLS_LOG4J_LOGLEVEL: 'WARN'
    volumes:
      - 'sentry-zookeeper:/var/lib/zookeeper/data'
      - 'sentry-zookeeper-log:/var/lib/zookeeper/log'
      - 'sentry-secrets:/etc/zookeeper/secrets'
    networks:
      - host_network

  kafka:
    <<: *restart_policy
    depends_on:
      - zookeeper
    image: 'confluentinc/cp-kafka:5.5.0'
    environment:
      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:9092'
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: '1'
      CONFLUENT_SUPPORT_METRICS_ENABLE: 'false'
      KAFKA_LOG4J_LOGGERS: 'kafka.cluster=WARN,kafka.controller=WARN,kafka.coordinator=WARN,kafka.log=WARN,kafka.server=WARN,kafka.zookeeper=WARN,state.change.logger=WARN'
      KAFKA_LOG4J_ROOT_LOGLEVEL: 'WARN'
      KAFKA_TOOLS_LOG4J_LOGLEVEL: 'WARN'
    volumes:
      - 'sentry-kafka:/var/lib/kafka/data'
      - 'sentry-kafka-log:/var/lib/kafka/log'
      - 'sentry-secrets:/etc/kafka/secrets'
    networks:
      - host_network

  clickhouse:
    <<: *restart_policy
    image: 'yandex/clickhouse-server:19.17'
    ulimits:
      nofile:
        soft: 262144
        hard: 262144
    volumes:
      - 'sentry-clickhouse:/var/lib/clickhouse'
      - 'sentry-clickhouse-log:/var/log/clickhouse-server'
    networks:
      - host_network

  snuba-api:
    <<: *snuba_defaults

  # Kafka consumer responsible for feeding events into Clickhouse
  snuba-consumer:
    <<: *snuba_defaults
    command: consumer --storage events --auto-offset-reset=latest --max-batch-time-ms 750

  # Kafka consumer responsible for feeding outcomes into Clickhouse
  # Use --auto-offset-reset=earliest to recover up to 7 days of TSDB data
  # since we did not do a proper migration
  snuba-outcomes-consumer:
    <<: *snuba_defaults
    command: consumer --storage outcomes_raw --auto-offset-reset=earliest --max-batch-time-ms 750

  snuba-replacer:
    <<: *snuba_defaults
    command: replacer --storage events --auto-offset-reset=latest --max-batch-size 3

  snuba-cleanup:
    <<: *snuba_defaults
    image: snuba-cleanup-onpremise-local
    build:
      context: ./cron
      args:
        BASE_IMAGE: 'getsentry/snuba:latest'
    command: '"*/5 * * * * gosu snuba snuba cleanup --dry-run False"'

  symbolicator:
    <<: *restart_policy
    image: 'getsentry/symbolicator:latest'
    volumes:
      - 'sentry-symbolicator:/data'
    networks:
      - host_network
    command: run

  symbolicator-cleanup:
    <<: *restart_policy
    image: symbolicator-cleanup-onpremise-local
    build:
      context: ./cron
      args:
        BASE_IMAGE: 'getsentry/symbolicator:latest'
    command: '"55 23 * * * gosu symbolicator symbolicator cleanup"'
    volumes:
      - 'sentry-symbolicator:/data'
    networks:
      - host_network

  web:
    <<: *sentry_defaults

  cron:
    <<: *sentry_defaults
    command: run cron

  worker:
    <<: *sentry_defaults
    command: run worker

  ingest-consumer:
    <<: *sentry_defaults
    command: run ingest-consumer --all-consumer-types

  post-process-forwarder:
    <<: *sentry_defaults
    # Increase `--commit-batch-size 1` below to deal with high-load environments.
    command: run post-process-forwarder --commit-batch-size 1

  sentry-cleanup:
    <<: *sentry_defaults
    image: sentry-cleanup-onpremise-local
    build:
      context: ./cron
      args:
        BASE_IMAGE: 'sentry-onpremise-local'
    command: '"0 0 * * * gosu sentry sentry cleanup --days $SENTRY_EVENT_RETENTION_DAYS"'

  relay:
    <<: *restart_policy
    image: "getsentry/relay:latest"
    volumes:
      - type: bind
        read_only: true
        source: ./relay
        target: /work/.relay
    depends_on:
      - kafka
      - redis
    networks:
      - host_network

volumes:
  host_volume:
  sentry-data:
  sentry-redis:
  sentry-zookeeper:
  sentry-kafka:
  sentry-clickhouse:
  sentry-symbolicator:
  sentry-secrets:
  sentry-smtp:
  sentry-zookeeper-log:
  sentry-kafka-log:
  sentry-smtp-log:
  sentry-clickhouse-log:
  mongodb_data:

networks:
  host_network:
  reviewbot: