version 1.0.0

.env.example

@@ -13,20 +13,20 @@ UI_BADGE_CONTENT="Local -> Local"
 UI_BADGE_COLOR="brown"
 
 MINIO_ENABLED="true"
-MINIO_DATA_DIR="/tmp/eurydice/minio-data"
-MINIO_CONF_DIR="/tmp/eurydice/minio-conf"
+MINIO_DATA_DIR="./data/minio-data"
+MINIO_CONF_DIR="./data/minio-conf"
 MINIO_ACCESS_KEY="minio"
 MINIO_SECRET_KEY="gjhfdfdsh"
-TRANSFERABLE_STORAGE_DIR="/tmp/eurydice/storage-data"
+TRANSFERABLE_STORAGE_DIR="./data/storage-data"
 
 DB_PASSWORD="dbpass"
-DB_DATA_DIR="/tmp/eurydice/db-data"
-DB_LOGS_DIR="/tmp/eurydice/db-logs"
+DB_DATA_DIR="./data/db-data"
+DB_LOGS_DIR="./data/db-logs"
 
-FILEBEAT_LOGS_DIR="/tmp/eurydice/filebeat-logs"
-FILEBEAT_DATA_DIR="/tmp/eurydice/filebeat-data"
+FILEBEAT_LOGS_DIR="./data/filebeat-logs"
+FILEBEAT_DATA_DIR="./data/filebeat-data"
 FILEBEAT_CONFIG_PATH="./filebeat/filebeat.null.yml"
 
 LOG_TO_FILE=true
-PYTHON_LOGS_DIR="/tmp/eurydice/python-logs/"
+PYTHON_LOGS_DIR="./data/python-logs"
 ELASTICSEARCH_CERT_PATH="/etc/ssl/certs/ca-certificates.crt"

.gitignore

@@ -111,6 +111,7 @@ celerybeat.pid
 
 # Environments
 .env
+.env.prod
 .venv
 env/
 venv/
@@ -144,7 +145,6 @@ cython_debug/
 
 # Crash log files
 crash.log
-
 staticfiles/
 
 # Logs
@@ -269,4 +269,8 @@ dist
 # vim
 .*.swp
 
+# local volumes
 data/
+
+# sysctl backup file
+sysctl_backup.conf

Makefile

@@ -1,9 +1,127 @@
-# ------------------------------------------------------------------------------
-# Static Analysis
-# ------------------------------------------------------------------------------
+SHELL := bash
+.ONESHELL:
+.SHELLFLAGS := -eu -o pipefail -c
+.DELETE_ON_ERROR:
+MAKEFLAGS += --warn-undefined-variables
+MAKEFLAGS += --no-builtin-rules
+COMPOSE := PUID=$(shell id -u) PGID=$(shell id -g) docker compose
+COMPOSE_DEV := $(COMPOSE) -f compose.yml
+COMPOSE_PROD := $(COMPOSE) -f compose.prod.yml
+
+SYSCTL_OPTIONS = net.core.rmem_max net.core.rmem_default net.core.netdev_max_backlog net.ipv4.udp_mem
+SYSCTL_BACKUP_FILE = sysctl_backup.conf
+### config
+
+.PHONY: backup-sysctl
+backup-sysctl: ## export sysctl config to backup file
+	@for option in $(SYSCTL_OPTIONS); do \
+		value=$$(sysctl -n $$option); \
+		echo "$$option = $$value" >> $(SYSCTL_BACKUP_FILE); \
+	done
+
+.PHONY: restore-sysctl
+restore-sysctl: ## restore sysctl config from backup file
+	@while IFS= read -r line; do \
+		sysctl -w "$$line"; \
+	done < $(SYSCTL_BACKUP_FILE)
+
+.PHONY: config-sysctl
+config-sysctl: ## sysctl config of LIDI
+	# https://github.com/ANSSI-FR/lidi/blob/master/doc/tweaking.rst
+	$(MAKE) backup-sysctl
+	sudo sysctl -w net.core.rmem_max=67108864 \
+		-w net.core.rmem_default=67108864 \
+		-w net.core.netdev_max_backlog=10000 \
+		-w net.ipv4.udp_mem="12148128 16197504 24296256"
+
+.PHONY: config-ufw
+config-ufw: ## open sender and receiver firewall ports
+	sudo ufw allow 5000/tcp
+	sudo ufw allow 6000/udp
+
+.PHONY: reset-ufw
+reset-ufw: ## reset sender and receiver firewall ports
+	@for PORT_PROTO in "5000/tcp" "6000/udp"; do \
+		while sudo ufw status numbered | grep -q "$${PORT_PROTO}"; do \
+			RULE_NUMBER=$$(sudo ufw status numbered | grep "$${PORT_PROTO}" | awk -F'[][]' '{print $$2}' | head -n 1); \
+			if [ -n "$${RULE_NUMBER}" ]; then \
+				sudo ufw --force delete $${RULE_NUMBER}; \
+			fi; \
+		done; \
+	done
+
+### dev
+
+.PHONY: dev-config
+dev-config:
+	$(MAKE) config-sysctl config-ufw
+
+.PHONY: dev-up
+dev-up:
+	$(COMPOSE_DEV) up -d
+
+.PHONY: dev-up-elk
+dev-up-elk:
+	$(COMPOSE_DEV) -f compose.kibana.yml up -d
+
+.PHONY: dev-down
+dev-down:
+	$(COMPOSE_DEV) down --volumes --remove-orphans
+
+.PHONY: dev-reset
+dev-reset:
+	$(MAKE) down-dev
+	$(MAKE) up-dev
+
+### prod
+
+.PHONY: prod-config
+prod-config:
+	mkdir -p data/minio-data
+	mkdir -p data/minio-conf
+	mkdir -p data/db-data
+	mkdir -p data/db-logs
+	mkdir -p data/filebeat-logs
+	mkdir -p data/filebeat-data
+	mkdir -p data/python-logs/backend-origin
+	mkdir -p data/python-logs/backend-destination
+	mkdir -p data/python-logs/receiver
+	mkdir -p data/python-logs/dbtrimmer-origin
+	mkdir -p data/python-logs/dbtrimmer-destination
+	mkdir -p data/python-logs/db-migrations-origin
+	mkdir -p data/python-logs/db-migrations-destination
+
+.PHONY: prod-up-origin
+prod-up-origin:
+	$(COMPOSE_PROD) --profile origin --env-file .env.prod run --rm db-migrations-origin
+	$(COMPOSE_PROD) --profile origin --env-file .env.prod up -d
+
+.PHONY: prod-stop-origin
+prod-stop-origin:
+	$(COMPOSE_PROD) --profile origin --env-file .env.prod stop
+
+.PHONY: prod-up-destination
+prod-up-destination:
+	$(COMPOSE_PROD) --profile destination --env-file .env.prod run --rm db-migrations-destination
+	$(COMPOSE_PROD) --profile destination --env-file .env.prod up -d
+
+.PHONY: prod-stop-destination
+prod-stop-destination:
+	$(COMPOSE_PROD) --profile destination --env-file .env.prod stop
+
+### Static Analysis
 
 .PHONY: hadolint
 hadolint:	## Lint the Dockerfiles.
 	docker run --rm -i hadolint/hadolint:2.8.0-alpine < backend/docker/Dockerfile
 	docker run --rm -i hadolint/hadolint:2.8.0-alpine < frontend/docker/Dockerfile
 	docker run --rm -i hadolint/hadolint:2.8.0-alpine < pgadmin/Dockerfile
+
+### help
+
+.PHONY: help
+help: ## Show this help
+    @echo "Usage:"
+    @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+.DEFAULT_GOAL := help

README.md

@@ -54,9 +54,9 @@ See [docs/administrators.md](docs/administrators.md).
 
 See documentation directories:
 
-- [Common API documentation](eurydice/common/api/docs/static/).
-- [Origin API documentation and code snippets](eurydice/origin/api/docs/static/).
-- [Destination API documentation and code snippets](eurydice/destination/api/docs/static/).
+- [Common API documentation](backend/eurydice/common/api/docs/static/).
+- [Origin API documentation and code snippets](backend/eurydice/origin/api/docs/static/).
+- [Destination API documentation and code snippets](backend/eurydice/destination/api/docs/static/).
 
 ## 🙏 Credits