Redirect Waldur logs to stdout instead of file [WAL-4028]

.gitignore

@@ -3,3 +3,4 @@
 pgsql
 waldur_libs
 waldur_logs
+.env

README.md

@@ -28,7 +28,7 @@ docker exec -t waldur-mastermind-worker status
 # Create user
 docker exec -t waldur-mastermind-worker waldur createstaffuser -u admin -p password -e [email protected]
 
-# Create demo categories for OpenStack: Virtual Private Cloud, VMs and Storage 
+# Create demo categories for OpenStack: Virtual Private Cloud, VMs and Storage
 docker exec -t waldur-mastermind-worker waldur load_categories vpc vm storage
 ```
 
@@ -43,6 +43,11 @@ Tearing down and cleaning up:
 docker-compose down
 ```
 
+## Logs
+
+Logs emitted by the containers are collected and saved in the `waldur_logs` folder. You can change the location by
+editing environment variable (`.env`) and updating `LOG_FOLDER` value.
+
 ## Known issues
 
 When Waldur is launched for the first time, it applies initial database migrations.
@@ -60,9 +65,8 @@ docker-compose restart
 ## Using TLS
 
 1. Add private key and certificate to ``./certs`` folder.
-2. Edit docker-compose.yml and replace port section with '80'. This is needed to force HTTP->HTTPS redirect from a TLS proxy:
-``sed -i 's/${WALDUR_INTERNAL_PORT}:80/80/' docker-compose.yml``.
-3. Change `WALDUR_PUBLIC_PORT` in .env to 443 and `WALDUR_PROTOCOL` to `https`.
+2. Edit docker-compose.yml and replace port section with '80'. This is needed to force HTTP->HTTPS redirect from a TLS proxy: `sed -i 's/${WALDUR_INTERNAL_PORT}:80/80/' docker-compose.yml`.
+3. Change `WALDUR_PUBLIC_PORT` in .env to 443 and `WALDUR_PROTOCOL` to https.
 4. Start docker-compose with an extra TLS proxy:
 
 ```bash

config/waldur-mastermind/beat

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+
+echo "INFO: Running Celery Beat..."
+source /etc/waldur/celery.conf
+mkdir -p /run/waldur/celerybeat
+chown -R waldur:waldur /run/waldur/celerybeat
+cd /run/waldur/celerybeat
+gosu waldur sh -c "celery -A $CELERY_APP beat --loglevel=$CELERYBEAT_LOG_LEVEL --pidfile=$CELERYBEAT_PID_FILE $CELERYBEAT_OPTS"
+
+exit 0

config/waldur-mastermind/logging.conf.py

@@ -0,0 +1,27 @@
+import sys
+
+
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+
+    'formatters': {
+        'simple': {
+            'format': '%(asctime)s %(levelname)s %(message)s',
+        },
+    },
+
+    'handlers': {
+        'console': {
+            'class': 'logging.StreamHandler',
+            'formatter': 'simple',
+            'level': 'DEBUG',
+            'stream': sys.stdout,
+        },
+    },
+
+    'root': {
+        'level': 'DEBUG',
+        'handlers': ['console'],
+    },
+}

config/waldur-mastermind/uwsgi.ini

@@ -0,0 +1,38 @@
+# Waldur uWSGI configuration file
+#
+# See also: http://uwsgi-docs.readthedocs.io/en/latest/Configuration.html#ini-files
+
+[uwsgi]
+buffer-size = 8192
+chmod-socket = 666
+enable-metrics = true
+gid = waldur
+# delegate logging to master process
+log-master = true
+# disable uWSGI standard logging
+disable-logging = true
+# ensure that critical errors are always logged
+log-4xx = true
+log-5xx = true
+module = waldur_core.server.wsgi:application
+plugins = python38
+processes = 8
+socket = :8000  # avoid localhost to fix binding issue
+static-map = /static=/usr/share/waldur/static
+static-map = /media=/var/lib/waldur/media
+uid = waldur
+enable-threads = true  # required by sentry-sdk
+
+http-socket = :8080
+http-enable-proxy-protocol = 1
+http-auto-chunked = true
+http-keepalive = 75
+http-timeout = 75
+offload-threads = $(UWSGI_OFFLOAD_THREADS)
+
+add-header = Access-Control-Allow-Credentials: true
+add-header = Access-Control-Allow-Headers: Accept, Accept-Encoding, Authorization, Content-Type, Origin, User-Agent, X-CSRFToken, X-Requested-With
+add-header = Access-Control-Allow-Methods: DELETE, GET, OPTIONS, PATCH, POST, PUT
+add-header = Access-Control-Allow-Origin: *
+add-header = Access-Control-Expose-Headers: Link, X-Result-Count
+add-header = Connection: Keep-Alive

config/waldur-mastermind/worker

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -e
+
+echo "INFO: Running Celery Worker..."
+source /etc/waldur/celery.conf
+mkdir -p /run/waldur/celery
+chown -R waldur:waldur /run/waldur/celery
+gosu waldur sh -c "celery -A $CELERY_APP worker --concurrency=$CELERYD_CONCURRENCY --loglevel=$CELERYD_LOG_LEVEL --pidfile=$CELERYD_PID_FILE $CELERYD_OPTS"
+
+exit 0

docker-compose.yml

@@ -7,23 +7,24 @@ x-mastermind-environment:
   - POSTGRESQL_PASSWORD=${POSTGRESQL_PASSWORD}
   - REDIS_HOST=waldur-queue
   - REDIS_PASSWORD=${REDIS_PASSWORD}
-  - LOGGING_LOG_FILE=/var/log/waldur/core.log
-  - EVENTS_LOG_FILE=/var/log/waldur/events.log
   - GLOBAL_DEFAULT_FROM_EMAIL=${GLOBAL_DEFAULT_FROM_EMAIL}
   - SENTRY_DSN=${SENTRY_DSN}
   - GLOBAL_DEBUG=${GLOBAL_DEBUG}
 
 x-mastermind-volumes:
   &default-mastermind-volumes
   - ${CONFIG_FOLDER}/waldur-mastermind/override.conf.py:/etc/waldur/override.conf.py
+  - ${CONFIG_FOLDER}/waldur-mastermind/logging.conf.py:/etc/waldur/logging.conf.py
+  - ${CONFIG_FOLDER}/waldur-mastermind/beat:/usr/local/bin/beat
+  - ${CONFIG_FOLDER}/waldur-mastermind/worker:/usr/local/bin/worker
+  - ${CONFIG_FOLDER}/waldur-mastermind/uwsgi.ini:/etc/waldur/uwsgi.ini
   - ${CONFIG_FOLDER}/waldur-mastermind/saml2.conf.py:/etc/waldur/saml2.conf.py
   - ${CONFIG_FOLDER}/waldur-mastermind/id_rsa:/etc/waldur/id_rsa
   - ${CONFIG_FOLDER}/waldur-mastermind/id_rsa.pub:/etc/waldur/id_rsa.pub
   - ${CONFIG_FOLDER}/waldur-mastermind/saml2/credentials:/etc/waldur/saml2/credentials
   - ${CONFIG_FOLDER}/waldur-mastermind/saml2/metadata:/etc/waldur/saml2/metadata
   - ${CONFIG_FOLDER}/waldur-mastermind/valimo:/etc/waldur/valimo
   - ${CONFIG_FOLDER}/waldur-mastermind/templates:/etc/waldur/templates
-  - ${LOG_FOLDER}:/var/log/waldur
   - ${LIB_FOLDER}:/var/lib/waldur
 
 services:
@@ -154,3 +155,26 @@ services:
       waldur-homeport:
         condition: service_healthy
     restart: always
+
+  logger:
+    image: umputun/docker-logger
+    container_name: "logger"
+    hostname: "logger"
+    restart: always
+    logging:
+      driver: json-file
+      options:
+          max-size: "10m"
+          max-file: "5"
+    environment:
+        - LOG_FILES=true
+        - LOG_SYSLOG=false
+        - MAX_FILES=10
+        - MAX_SIZE=50
+        - MAX_AGE=365
+        - MIX_ERR=true
+        - DEBUG=false
+        - INCLUDE=waldur-db,waldur-queue,waldur-mastermind-db-migration,waldur-mastermind-worker,waldur-mastermind-beat,waldur-mastermind-api,waldur-homeport,tls-proxy,nginx-proxy
+    volumes:
+        - ${LOG_FOLDER}:/srv/logs
+        - /var/run/docker.sock:/var/run/docker.sock