From 946377ef6356ac91b5bc74f35e45ccec2350e1e3 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Mon, 25 Nov 2024 16:02:39 +0100 Subject: [PATCH 01/11] Add production environment configuration --- .github/workflows/build_and_push.yml | 14 ++- Dockerfile | 2 +- config/environments/production.rb | 28 +++-- docker-compose.production.yml | 158 +++++++++++++++++++++++++++ prod-docker-entrypoint.sh | 51 +++++++++ 5 files changed, 241 insertions(+), 12 deletions(-) create mode 100644 docker-compose.production.yml create mode 100644 prod-docker-entrypoint.sh diff --git a/.github/workflows/build_and_push.yml b/.github/workflows/build_and_push.yml index 1be0cca5..812f071f 100644 --- a/.github/workflows/build_and_push.yml +++ b/.github/workflows/build_and_push.yml @@ -1,13 +1,23 @@ name: Docker image build and push + on: workflow_dispatch: + inputs: + branch: + description: "The branch to build the Docker image from" + required: false + default: "main" release: types: [created] + jobs: build-and-push-docker: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Checkout code + uses: actions/checkout@v2 + with: + ref: ${{ github.event.inputs.branch || github.ref_name }} - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx @@ -32,7 +42,7 @@ jobs: context: . file: ./Dockerfile push: true - tags: freikin/dawarich:latest,freikin/dawarich:${{ github.event.release.tag_name }} + tags: freikin/dawarich:latest,freikin/dawarich:${{ github.event.inputs.branch || github.ref_name }} platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6 cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache diff --git a/Dockerfile b/Dockerfile index d809b2d7..ef2b74c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM ruby:3.3.4-alpine ENV APP_PATH=/var/app -ENV BUNDLE_VERSION=2.5.9 +ENV BUNDLE_VERSION=2.5.21 ENV BUNDLE_PATH=/usr/local/bundle/gems ENV TMP_PATH=/tmp/ ENV RAILS_LOG_TO_STDOUT=true diff --git a/config/environments/production.rb b/config/environments/production.rb index f541929a..d0d64c4d 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -1,4 +1,6 @@ -require "active_support/core_ext/integer/time" +# frozen_string_literal: true + +require 'active_support/core_ext/integer/time' Rails.application.configure do # Settings specified here will take precedence over those in config/application.rb. @@ -27,7 +29,11 @@ # config.assets.css_compressor = :sass # Do not fallback to assets pipeline if a precompiled asset is missed. - config.assets.compile = false + config.assets.compile = true + + config.assets.content_type = { + geojson: 'application/geo+json' + } # Enable serving of images, stylesheets, and JavaScripts from an asset server. # config.asset_host = "http://assets.example.com" @@ -49,20 +55,20 @@ # config.assume_ssl = true # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. - config.force_ssl = true + config.force_ssl = ENV.fetch('APPLICATION_PROTOCOL', 'http').downcase == 'https' - # Log to STDOUT by default - config.logger = ActiveSupport::Logger.new(STDOUT) - .tap { |logger| logger.formatter = ::Logger::Formatter.new } - .then { |logger| ActiveSupport::TaggedLogging.new(logger) } + # Direct logs to STDOUT + config.logger = Logger.new($stdout) + config.lograge.enabled = true + config.lograge.formatter = Lograge::Formatters::Json.new # Prepend all log lines with the following tags. - config.log_tags = [ :request_id ] + config.log_tags = [:request_id] # Info include generic and useful information about system operation, but avoids logging too much # information to avoid inadvertent exposure of personally identifiable information (PII). If you # want to log everything, set the level to "debug". - config.log_level = ENV.fetch("RAILS_LOG_LEVEL", "info") + config.log_level = ENV.fetch('RAILS_LOG_LEVEL', 'info') # Use a different cache store in production. # config.cache_store = :mem_cache_store @@ -95,4 +101,8 @@ # ] # Skip DNS rebinding protection for the default health check endpoint. # config.host_authorization = { exclude: ->(request) { request.path == "/up" } } + config.hosts << ENV.fetch('APPLICATION_HOST', 'localhost') + + hosts = ENV.fetch('APPLICATION_HOSTS', 'localhost') + config.hosts.concat(hosts.split(',')) if hosts.present? end diff --git a/docker-compose.production.yml b/docker-compose.production.yml new file mode 100644 index 00000000..e6a39b90 --- /dev/null +++ b/docker-compose.production.yml @@ -0,0 +1,158 @@ +networks: + dawarich: +services: + dawarich_redis: + image: redis:7.0-alpine + container_name: dawarich_redis + command: redis-server + networks: + - dawarich + volumes: + - shared_data:/var/shared/redis + restart: always + healthcheck: + test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ] + interval: 10s + retries: 5 + start_period: 30s + timeout: 10s + dawarich_db: + image: postgres:14.2-alpine + container_name: dawarich_db + volumes: + - db_data:/var/lib/postgresql/data + - shared_data:/var/shared + networks: + - dawarich + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + restart: always + healthcheck: + test: [ "CMD-SHELL", "pg_isready -U postgres -d dawarich_development" ] + interval: 10s + retries: 5 + start_period: 30s + timeout: 10s + dawarich_app: + image: freikin/dawarich:latest + container_name: dawarich_app + volumes: + - gem_cache:/usr/local/bundle/gems_app + - public:/var/app/public + - watched:/var/app/tmp/imports/watched + networks: + - dawarich + ports: + - 3000:3000 + # - 9394:9394 # Prometheus exporter, uncomment if needed + stdin_open: true + tty: true + entrypoint: dev-entrypoint.sh + command: ['bin/dev'] + restart: on-failure + environment: + RAILS_ENV: production + REDIS_URL: redis://dawarich_redis:6379/0 + DATABASE_HOST: dawarich_db + DATABASE_USERNAME: postgres + DATABASE_PASSWORD: password + DATABASE_NAME: dawarich_development + MIN_MINUTES_SPENT_IN_CITY: 60 + APPLICATION_HOST: localhost + APPLICATION_HOSTS: localhost + TIME_ZONE: Europe/London + APPLICATION_PROTOCOL: http + DISTANCE_UNIT: km + PHOTON_API_HOST: photon.komoot.io + PHOTON_API_USE_HTTPS: true + PROMETHEUS_EXPORTER_ENABLED: false + PROMETHEUS_EXPORTER_HOST: 0.0.0.0 + PROMETHEUS_EXPORTER_PORT: 9394 + logging: + driver: "json-file" + options: + max-size: "100m" + max-file: "5" + healthcheck: + test: [ "CMD-SHELL", "wget -qO - http://127.0.0.1:3000/api/v1/health | grep -q '\"status\"\\s*:\\s*\"ok\"'" ] + interval: 10s + retries: 30 + start_period: 30s + timeout: 10s + depends_on: + dawarich_db: + condition: service_healthy + restart: true + dawarich_redis: + condition: service_healthy + restart: true + deploy: + resources: + limits: + cpus: '0.50' # Limit CPU usage to 50% of one core + memory: '2G' # Limit memory usage to 2GB + dawarich_sidekiq: + image: freikin/dawarich:latest + container_name: dawarich_sidekiq + volumes: + - gem_cache:/usr/local/bundle/gems_sidekiq + - public:/var/app/public + - watched:/var/app/tmp/imports/watched + networks: + - dawarich + stdin_open: true + tty: true + entrypoint: dev-entrypoint.sh + command: ['sidekiq'] + restart: on-failure + environment: + RAILS_ENV: production + REDIS_URL: redis://dawarich_redis:6379/0 + DATABASE_HOST: dawarich_db + DATABASE_USERNAME: postgres + DATABASE_PASSWORD: password + DATABASE_NAME: dawarich_development + APPLICATION_HOST: localhost + APPLICATION_HOSTS: localhost + BACKGROUND_PROCESSING_CONCURRENCY: 10 + APPLICATION_PROTOCOL: http + DISTANCE_UNIT: km + PHOTON_API_HOST: photon.komoot.io + PHOTON_API_USE_HTTPS: true + PROMETHEUS_EXPORTER_ENABLED: false + PROMETHEUS_EXPORTER_HOST: dawarich_app + PROMETHEUS_EXPORTER_PORT: 9394 + logging: + driver: "json-file" + options: + max-size: "100m" + max-file: "5" + healthcheck: + test: [ "CMD-SHELL", "bundle exec sidekiqmon processes | grep $${HOSTNAME}" ] + interval: 10s + retries: 30 + start_period: 30s + timeout: 10s + depends_on: + dawarich_db: + condition: service_healthy + restart: true + dawarich_redis: + condition: service_healthy + restart: true + dawarich_app: + condition: service_healthy + restart: true + deploy: + resources: + limits: + cpus: '0.50' # Limit CPU usage to 50% of one core + memory: '2G' # Limit memory usage to 2GB + +volumes: + db_data: + gem_cache: + shared_data: + public: + watched: diff --git a/prod-docker-entrypoint.sh b/prod-docker-entrypoint.sh new file mode 100644 index 00000000..52bac19a --- /dev/null +++ b/prod-docker-entrypoint.sh @@ -0,0 +1,51 @@ +#!/bin/sh + +unset BUNDLE_PATH +unset BUNDLE_BIN + +set -e + +echo "Environment: $RAILS_ENV" + +# set env var defaults +DATABASE_HOST=${DATABASE_HOST:-"dawarich_db"} +DATABASE_PORT=${DATABASE_PORT:-5432} +DATABASE_USERNAME=${DATABASE_USERNAME:-"postgres"} +DATABASE_PASSWORD=${DATABASE_PASSWORD:-"password"} +DATABASE_NAME=${DATABASE_NAME:-"dawarich_production"} + +# Remove pre-existing puma/passenger server.pid +rm -f $APP_PATH/tmp/pids/server.pid + +# Wait for the database to be ready +until nc -zv $DATABASE_HOST ${DATABASE_PORT:-5432}; do + echo "Waiting for PostgreSQL to be ready..." + sleep 1 +done + +# Install gems +gem update --system 3.5.23 +gem install bundler --version '2.5.21' + +# Create the database +if [ "$(psql "postgres://$DATABASE_USERNAME:$DATABASE_PASSWORD@$DATABASE_HOST:$DATABASE_PORT" -XtAc "SELECT 1 FROM pg_database WHERE datname='$DATABASE_NAME'")" = '1' ]; then + echo "Database $DATABASE_NAME already exists, skipping creation..." +else + echo "Creating database $DATABASE_NAME..." + bundle exec rails db:create +fi + +# Run database migrations +echo "PostgreSQL is ready. Running database migrations..." +bundle exec rails db:prepare + +# Run data migrations +echo "Running DATA migrations..." +bundle exec rake data:migrate + +# Run seeds +echo "Running seeds..." +bundle exec rake db:seed + +# run passed commands +bundle exec ${@} From 1476816418635c2e568ee299523bfd2172160b7c Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Tue, 7 Jan 2025 16:04:03 +0100 Subject: [PATCH 02/11] Update production environment --- config/environments/production.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/environments/production.rb b/config/environments/production.rb index 8484021a..4e8f5661 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -103,8 +103,8 @@ # ] # Skip DNS rebinding protection for the default health check endpoint. # config.host_authorization = { exclude: ->(request) { request.path == "/up" } } - config.hosts << ENV.fetch('APPLICATION_HOST', 'localhost') + hosts = ENV.fetch('APPLICATION_HOSTS', 'localhost').split(',') - hosts = ENV.fetch('APPLICATION_HOSTS', 'localhost') - config.hosts.concat(hosts.split(',')) if hosts.present? + config.action_mailer.default_url_options = { host: hosts.first, port: 3000 } + config.hosts.concat(hosts) if hosts.present? end From ba40b7d284fcdcfe515ca18e7c9bcc997b482a80 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Wed, 8 Jan 2025 13:06:50 +0100 Subject: [PATCH 03/11] Implement production environment --- .github/workflows/build_and_push.yml | 2 +- dev-docker-sidekiq-entrypoint.sh | 21 ----- .dockerignore => docker/.dockerignore | 0 Dockerfile => docker/Dockerfile | 8 +- Dockerfile.dev => docker/Dockerfile.dev | 0 .../dev-docker-entrypoint.sh | 6 +- .../docker-compose.production.yml | 23 +++-- .../docker-compose.yml | 0 .../docker-compose_mounted_volumes.yml | 0 docker/prod-docker-entrypoint.sh | 93 +++++++++++++++++++ prod-docker-entrypoint.sh | 51 ---------- 11 files changed, 115 insertions(+), 89 deletions(-) delete mode 100644 dev-docker-sidekiq-entrypoint.sh rename .dockerignore => docker/.dockerignore (100%) rename Dockerfile => docker/Dockerfile (83%) rename Dockerfile.dev => docker/Dockerfile.dev (100%) rename dev-docker-entrypoint.sh => docker/dev-docker-entrypoint.sh (92%) rename docker-compose.production.yml => docker/docker-compose.production.yml (89%) rename docker-compose.yml => docker/docker-compose.yml (100%) rename docker-compose_mounted_volumes.yml => docker/docker-compose_mounted_volumes.yml (100%) create mode 100644 docker/prod-docker-entrypoint.sh delete mode 100644 prod-docker-entrypoint.sh diff --git a/.github/workflows/build_and_push.yml b/.github/workflows/build_and_push.yml index 2c1ebe4c..05173d53 100644 --- a/.github/workflows/build_and_push.yml +++ b/.github/workflows/build_and_push.yml @@ -40,7 +40,7 @@ jobs: uses: docker/build-push-action@v2 with: context: . - file: ./Dockerfile + file: ./docker/Dockerfile push: true tags: freikin/dawarich:latest,freikin/dawarich:${{ github.event.inputs.branch || github.ref_name }} platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6 diff --git a/dev-docker-sidekiq-entrypoint.sh b/dev-docker-sidekiq-entrypoint.sh deleted file mode 100644 index db7c87e9..00000000 --- a/dev-docker-sidekiq-entrypoint.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/sh - -set -e - -echo "Environment: $RAILS_ENV" - -# set env var defaults -DATABASE_HOST=${DATABASE_HOST:-"dawarich_db"} -DATABASE_PORT=${DATABASE_PORT:-5432} -DATABASE_USER=${DATABASE_USER:-"postgres"} -DATABASE_PASSWORD=${DATABASE_PASSWORD:-"password"} -DATABASE_NAME=${DATABASE_NAME:-"dawarich_development"} - -# Wait for the database to be ready -until nc -zv $DATABASE_HOST ${DATABASE_PORT:-5432}; do - echo "Waiting for PostgreSQL to be ready..." - sleep 1 -done - -# run passed commands -bundle exec ${@} diff --git a/.dockerignore b/docker/.dockerignore similarity index 100% rename from .dockerignore rename to docker/.dockerignore diff --git a/Dockerfile b/docker/Dockerfile similarity index 83% rename from Dockerfile rename to docker/Dockerfile index ef2b74c5..07334a1f 100644 --- a/Dockerfile +++ b/docker/Dockerfile @@ -32,17 +32,17 @@ RUN gem install bundler --version "$BUNDLE_VERSION" \ # Navigate to app directory WORKDIR $APP_PATH -COPY Gemfile Gemfile.lock vendor .ruby-version ./ +COPY ../Gemfile ../Gemfile.lock ../vendor ../.ruby-version ./ # Install missing gems RUN bundle config set --local path 'vendor/bundle' \ && bundle install --jobs 20 --retry 5 -COPY . ./ +COPY ../. ./ # Copy entrypoint scripts and grant execution permissions -COPY ./dev-docker-entrypoint.sh /usr/local/bin/dev-entrypoint.sh -RUN chmod +x /usr/local/bin/dev-entrypoint.sh +COPY ./docker/prod-docker-entrypoint.sh /usr/local/bin/entrypoint.sh +RUN chmod +x /usr/local/bin/entrypoint.sh EXPOSE $RAILS_PORT diff --git a/Dockerfile.dev b/docker/Dockerfile.dev similarity index 100% rename from Dockerfile.dev rename to docker/Dockerfile.dev diff --git a/dev-docker-entrypoint.sh b/docker/dev-docker-entrypoint.sh similarity index 92% rename from dev-docker-entrypoint.sh rename to docker/dev-docker-entrypoint.sh index 385b50da..90a8ddfd 100644 --- a/dev-docker-entrypoint.sh +++ b/docker/dev-docker-entrypoint.sh @@ -24,8 +24,8 @@ until nc -zv $DATABASE_HOST ${DATABASE_PORT:-5432}; do done # Install gems -gem update --system 3.5.7 -gem install bundler --version '2.5.9' +gem update --system 3.6.2 +gem install bundler --version '2.5.21' # Create the database if [ "$(psql "postgres://$DATABASE_USERNAME:$DATABASE_PASSWORD@$DATABASE_HOST:$DATABASE_PORT" -XtAc "SELECT 1 FROM pg_database WHERE datname='$DATABASE_NAME'")" = '1' ]; then @@ -37,7 +37,7 @@ fi # Run database migrations echo "PostgreSQL is ready. Running database migrations..." -bundle exec rails db:prepare +bundle exec rails db:migrate # Run data migrations echo "Running DATA migrations..." diff --git a/docker-compose.production.yml b/docker/docker-compose.production.yml similarity index 89% rename from docker-compose.production.yml rename to docker/docker-compose.production.yml index e6a39b90..2399e731 100644 --- a/docker-compose.production.yml +++ b/docker/docker-compose.production.yml @@ -2,7 +2,7 @@ networks: dawarich: services: dawarich_redis: - image: redis:7.0-alpine + image: redis:7.4-alpine container_name: dawarich_redis command: redis-server networks: @@ -17,7 +17,7 @@ services: start_period: 30s timeout: 10s dawarich_db: - image: postgres:14.2-alpine + image: postgres:17-alpine container_name: dawarich_db volumes: - db_data:/var/lib/postgresql/data @@ -27,15 +27,16 @@ services: environment: POSTGRES_USER: postgres POSTGRES_PASSWORD: password + POSTGRES_DB: dawarich_production restart: always healthcheck: - test: [ "CMD-SHELL", "pg_isready -U postgres -d dawarich_development" ] + test: [ "CMD", "pg_isready", "-U", "postgres" ] interval: 10s retries: 5 start_period: 30s timeout: 10s dawarich_app: - image: freikin/dawarich:latest + image: dawarich:prod container_name: dawarich_app volumes: - gem_cache:/usr/local/bundle/gems_app @@ -48,16 +49,17 @@ services: # - 9394:9394 # Prometheus exporter, uncomment if needed stdin_open: true tty: true - entrypoint: dev-entrypoint.sh + entrypoint: entrypoint.sh command: ['bin/dev'] restart: on-failure environment: RAILS_ENV: production REDIS_URL: redis://dawarich_redis:6379/0 DATABASE_HOST: dawarich_db + DATABASE_PORT: 5432 DATABASE_USERNAME: postgres DATABASE_PASSWORD: password - DATABASE_NAME: dawarich_development + DATABASE_NAME: dawarich_production MIN_MINUTES_SPENT_IN_CITY: 60 APPLICATION_HOST: localhost APPLICATION_HOSTS: localhost @@ -69,6 +71,8 @@ services: PROMETHEUS_EXPORTER_ENABLED: false PROMETHEUS_EXPORTER_HOST: 0.0.0.0 PROMETHEUS_EXPORTER_PORT: 9394 + SECRET_KEY_BASE: 1234567890 + RAILS_LOG_TO_STDOUT: "true" logging: driver: "json-file" options: @@ -93,7 +97,7 @@ services: cpus: '0.50' # Limit CPU usage to 50% of one core memory: '2G' # Limit memory usage to 2GB dawarich_sidekiq: - image: freikin/dawarich:latest + image: dawarich:prod container_name: dawarich_sidekiq volumes: - gem_cache:/usr/local/bundle/gems_sidekiq @@ -103,16 +107,17 @@ services: - dawarich stdin_open: true tty: true - entrypoint: dev-entrypoint.sh + entrypoint: entrypoint.sh command: ['sidekiq'] restart: on-failure environment: RAILS_ENV: production REDIS_URL: redis://dawarich_redis:6379/0 DATABASE_HOST: dawarich_db + DATABASE_PORT: 5432 DATABASE_USERNAME: postgres DATABASE_PASSWORD: password - DATABASE_NAME: dawarich_development + DATABASE_NAME: dawarich_production APPLICATION_HOST: localhost APPLICATION_HOSTS: localhost BACKGROUND_PROCESSING_CONCURRENCY: 10 diff --git a/docker-compose.yml b/docker/docker-compose.yml similarity index 100% rename from docker-compose.yml rename to docker/docker-compose.yml diff --git a/docker-compose_mounted_volumes.yml b/docker/docker-compose_mounted_volumes.yml similarity index 100% rename from docker-compose_mounted_volumes.yml rename to docker/docker-compose_mounted_volumes.yml diff --git a/docker/prod-docker-entrypoint.sh b/docker/prod-docker-entrypoint.sh new file mode 100644 index 00000000..683ae2f0 --- /dev/null +++ b/docker/prod-docker-entrypoint.sh @@ -0,0 +1,93 @@ +#!/bin/sh + +unset BUNDLE_PATH +unset BUNDLE_BIN + +set -e + +echo "Environment: $RAILS_ENV" + +# Parse DATABASE_URL if present, otherwise use individual variables +if [ -n "$DATABASE_URL" ]; then + # Extract components from DATABASE_URL + DATABASE_HOST=$(echo $DATABASE_URL | awk -F[@/] '{print $4}') + DATABASE_PORT=$(echo $DATABASE_URL | awk -F[@/:] '{print $5}') + DATABASE_USERNAME=$(echo $DATABASE_URL | awk -F[:/@] '{print $4}') + DATABASE_PASSWORD=$(echo $DATABASE_URL | awk -F[:/@] '{print $5}') + DATABASE_NAME=$(echo $DATABASE_URL | awk -F[@/] '{print $5}') +else + # Use existing environment variables + DATABASE_HOST=${DATABASE_HOST:-dawarich_db} + DATABASE_PORT=${DATABASE_PORT:-5432} + DATABASE_USERNAME=${DATABASE_USERNAME:-postgres} + DATABASE_PASSWORD=${DATABASE_PASSWORD:-password} + DATABASE_NAME=${DATABASE_NAME:-dawarich_production} +fi + +# Function to test database connection +test_db_connection() { + echo "Testing connection to PostgreSQL..." + echo "Host: $DATABASE_HOST" + echo "Port: $DATABASE_PORT" + echo "Username: $DATABASE_USERNAME" + echo "Database: postgres (default database)" + + if PGPASSWORD=$DATABASE_PASSWORD psql -h "$DATABASE_HOST" -p "$DATABASE_PORT" -U "$DATABASE_USERNAME" -d postgres -c "SELECT 1" > /dev/null 2>&1; then + echo "✅ Successfully connected to PostgreSQL!" + return 0 + else + echo "❌ Failed to connect to PostgreSQL" + return 1 + fi +} + +# Try to connect to PostgreSQL, with a timeout +max_attempts=30 +attempt=1 + +while ! test_db_connection; do + if [ $attempt -ge $max_attempts ]; then + echo "Failed to connect to PostgreSQL after $max_attempts attempts. Exiting." + exit 1 + fi + + echo "Attempt $attempt of $max_attempts. Waiting 2 seconds before retry..." + attempt=$((attempt + 1)) + sleep 2 +done + +# Remove pre-existing puma/passenger server.pid +rm -f $APP_PATH/tmp/pids/server.pid + +# Install gems +gem update --system 3.6.2 +gem install bundler --version '2.5.21' + +# Create the database if it doesn't exist +if PGPASSWORD=$DATABASE_PASSWORD psql -h "$DATABASE_HOST" -p "$DATABASE_PORT" -U "$DATABASE_USERNAME" -c "SELECT 1 FROM pg_database WHERE datname='$DATABASE_NAME'" | grep -q 1; then + echo "Database $DATABASE_NAME already exists, skipping creation..." +else + echo "Creating database $DATABASE_NAME..." + bundle exec rails db:create +fi + +# Run database migrations +echo "PostgreSQL is ready. Running database migrations..." +bundle exec rails db:migrate + +# Run data migrations +echo "Running DATA migrations..." +bundle exec rake data:migrate + +# Run seeds +echo "Running seeds..." +bundle exec rake db:seed + +# Precompile assets +if [ "$RAILS_ENV" = "production" ]; then + echo "Precompiling assets..." + bundle exec rake assets:precompile +fi + +# run passed commands +bundle exec ${@} diff --git a/prod-docker-entrypoint.sh b/prod-docker-entrypoint.sh deleted file mode 100644 index 52bac19a..00000000 --- a/prod-docker-entrypoint.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/sh - -unset BUNDLE_PATH -unset BUNDLE_BIN - -set -e - -echo "Environment: $RAILS_ENV" - -# set env var defaults -DATABASE_HOST=${DATABASE_HOST:-"dawarich_db"} -DATABASE_PORT=${DATABASE_PORT:-5432} -DATABASE_USERNAME=${DATABASE_USERNAME:-"postgres"} -DATABASE_PASSWORD=${DATABASE_PASSWORD:-"password"} -DATABASE_NAME=${DATABASE_NAME:-"dawarich_production"} - -# Remove pre-existing puma/passenger server.pid -rm -f $APP_PATH/tmp/pids/server.pid - -# Wait for the database to be ready -until nc -zv $DATABASE_HOST ${DATABASE_PORT:-5432}; do - echo "Waiting for PostgreSQL to be ready..." - sleep 1 -done - -# Install gems -gem update --system 3.5.23 -gem install bundler --version '2.5.21' - -# Create the database -if [ "$(psql "postgres://$DATABASE_USERNAME:$DATABASE_PASSWORD@$DATABASE_HOST:$DATABASE_PORT" -XtAc "SELECT 1 FROM pg_database WHERE datname='$DATABASE_NAME'")" = '1' ]; then - echo "Database $DATABASE_NAME already exists, skipping creation..." -else - echo "Creating database $DATABASE_NAME..." - bundle exec rails db:create -fi - -# Run database migrations -echo "PostgreSQL is ready. Running database migrations..." -bundle exec rails db:prepare - -# Run data migrations -echo "Running DATA migrations..." -bundle exec rake data:migrate - -# Run seeds -echo "Running seeds..." -bundle exec rake db:seed - -# run passed commands -bundle exec ${@} From abff239d977eb6ce378ba8c25842f6455f375f80 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Wed, 8 Jan 2025 15:11:31 +0100 Subject: [PATCH 04/11] Run rails server instead of foreman --- config/database.yml | 2 +- docker/docker-compose.production.yml | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/config/database.yml b/config/database.yml index 1977b027..0dc98e97 100644 --- a/config/database.yml +++ b/config/database.yml @@ -20,4 +20,4 @@ test: production: <<: *default database: <%= ENV['DATABASE_NAME'] || 'dawarich_production' %> - url: <%= ENV['DATABASE_URL'] %> + # url: <%= ENV['DATABASE_URL'] %> diff --git a/docker/docker-compose.production.yml b/docker/docker-compose.production.yml index 2399e731..fea4fa75 100644 --- a/docker/docker-compose.production.yml +++ b/docker/docker-compose.production.yml @@ -50,7 +50,7 @@ services: stdin_open: true tty: true entrypoint: entrypoint.sh - command: ['bin/dev'] + command: ['bin/rails', 'server', '-p', '3000', '-b', '::'] restart: on-failure environment: RAILS_ENV: production @@ -61,8 +61,7 @@ services: DATABASE_PASSWORD: password DATABASE_NAME: dawarich_production MIN_MINUTES_SPENT_IN_CITY: 60 - APPLICATION_HOST: localhost - APPLICATION_HOSTS: localhost + APPLICATION_HOSTS: localhost,::1,127.0.0.1 TIME_ZONE: Europe/London APPLICATION_PROTOCOL: http DISTANCE_UNIT: km @@ -108,7 +107,7 @@ services: stdin_open: true tty: true entrypoint: entrypoint.sh - command: ['sidekiq'] + command: ['bundle', 'exec', 'sidekiq'] restart: on-failure environment: RAILS_ENV: production @@ -118,8 +117,7 @@ services: DATABASE_USERNAME: postgres DATABASE_PASSWORD: password DATABASE_NAME: dawarich_production - APPLICATION_HOST: localhost - APPLICATION_HOSTS: localhost + APPLICATION_HOSTS: localhost,::1,127.0.0.1 BACKGROUND_PROCESSING_CONCURRENCY: 10 APPLICATION_PROTOCOL: http DISTANCE_UNIT: km @@ -128,6 +126,8 @@ services: PROMETHEUS_EXPORTER_ENABLED: false PROMETHEUS_EXPORTER_HOST: dawarich_app PROMETHEUS_EXPORTER_PORT: 9394 + SECRET_KEY_BASE: 1234567890 + RAILS_LOG_TO_STDOUT: "true" logging: driver: "json-file" options: From 69af9710f558242ff164205f5a595d4525fcc963 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Thu, 9 Jan 2025 13:04:22 +0100 Subject: [PATCH 05/11] Clean up dockerfiles --- app/controllers/api/v1/health_controller.rb | 1 - docker/Dockerfile | 1 - docker/Dockerfile.dev | 34 ++++++++--------- docker/prod-docker-entrypoint.sh | 42 +++------------------ 4 files changed, 22 insertions(+), 56 deletions(-) diff --git a/app/controllers/api/v1/health_controller.rb b/app/controllers/api/v1/health_controller.rb index 53563cb0..62e594e9 100644 --- a/app/controllers/api/v1/health_controller.rb +++ b/app/controllers/api/v1/health_controller.rb @@ -8,4 +8,3 @@ def index render json: { status: 'ok' } end end - diff --git a/docker/Dockerfile b/docker/Dockerfile index 07334a1f..73644a6a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -21,7 +21,6 @@ RUN apk -U add --no-cache \ tzdata \ less \ yaml-dev \ - # gcompat for nokogiri on mac m1 gcompat \ && rm -rf /var/cache/apk/* \ && mkdir -p $APP_PATH diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 8d668cf3..ad2336aa 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -1,7 +1,7 @@ FROM ruby:3.3.4-alpine ENV APP_PATH=/var/app -ENV BUNDLE_VERSION=2.5.9 +ENV BUNDLE_VERSION=2.5.21 ENV BUNDLE_PATH=/usr/local/bundle/gems ENV TMP_PATH=/tmp/ ENV RAILS_LOG_TO_STDOUT=true @@ -9,24 +9,24 @@ ENV RAILS_PORT=3000 # install dependencies for application RUN apk -U add --no-cache \ - build-base \ - git \ - postgresql-dev \ - postgresql-client \ - libxml2-dev \ - libxslt-dev \ - nodejs \ - yarn \ - imagemagick \ - tzdata \ - less \ - # gcompat for nokogiri on mac m1 - gcompat \ - && rm -rf /var/cache/apk/* \ - && mkdir -p $APP_PATH + build-base \ + git \ + postgresql-dev \ + postgresql-client \ + libxml2-dev \ + libxslt-dev \ + nodejs \ + yarn \ + imagemagick \ + tzdata \ + less \ + # gcompat for nokogiri on mac m1 + gcompat \ + && rm -rf /var/cache/apk/* \ + && mkdir -p $APP_PATH RUN gem install bundler --version "$BUNDLE_VERSION" \ - && rm -rf $GEM_HOME/cache/* + && rm -rf $GEM_HOME/cache/* # copy entrypoint scripts and grant execution permissions COPY ./dev-docker-entrypoint.sh /usr/local/bin/dev-entrypoint.sh diff --git a/docker/prod-docker-entrypoint.sh b/docker/prod-docker-entrypoint.sh index 683ae2f0..50ff835c 100644 --- a/docker/prod-docker-entrypoint.sh +++ b/docker/prod-docker-entrypoint.sh @@ -17,45 +17,13 @@ if [ -n "$DATABASE_URL" ]; then DATABASE_NAME=$(echo $DATABASE_URL | awk -F[@/] '{print $5}') else # Use existing environment variables - DATABASE_HOST=${DATABASE_HOST:-dawarich_db} - DATABASE_PORT=${DATABASE_PORT:-5432} - DATABASE_USERNAME=${DATABASE_USERNAME:-postgres} - DATABASE_PASSWORD=${DATABASE_PASSWORD:-password} - DATABASE_NAME=${DATABASE_NAME:-dawarich_production} + DATABASE_HOST=${DATABASE_HOST} + DATABASE_PORT=${DATABASE_PORT} + DATABASE_USERNAME=${DATABASE_USERNAME} + DATABASE_PASSWORD=${DATABASE_PASSWORD} + DATABASE_NAME=${DATABASE_NAME} fi -# Function to test database connection -test_db_connection() { - echo "Testing connection to PostgreSQL..." - echo "Host: $DATABASE_HOST" - echo "Port: $DATABASE_PORT" - echo "Username: $DATABASE_USERNAME" - echo "Database: postgres (default database)" - - if PGPASSWORD=$DATABASE_PASSWORD psql -h "$DATABASE_HOST" -p "$DATABASE_PORT" -U "$DATABASE_USERNAME" -d postgres -c "SELECT 1" > /dev/null 2>&1; then - echo "✅ Successfully connected to PostgreSQL!" - return 0 - else - echo "❌ Failed to connect to PostgreSQL" - return 1 - fi -} - -# Try to connect to PostgreSQL, with a timeout -max_attempts=30 -attempt=1 - -while ! test_db_connection; do - if [ $attempt -ge $max_attempts ]; then - echo "Failed to connect to PostgreSQL after $max_attempts attempts. Exiting." - exit 1 - fi - - echo "Attempt $attempt of $max_attempts. Waiting 2 seconds before retry..." - attempt=$((attempt + 1)) - sleep 2 -done - # Remove pre-existing puma/passenger server.pid rm -f $APP_PATH/tmp/pids/server.pid From 4d25dbca2107c8455b4bbad13fb547f433e516f2 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Thu, 9 Jan 2025 13:38:13 +0100 Subject: [PATCH 06/11] Move some files around --- .devcontainer/Dockerfile | 4 +- .devcontainer/docker-compose.yml | 4 +- app/jobs/visit_suggesting_job.rb | 6 ++- ...70930_remove_points_without_coordinates.rb | 2 +- docker/Dockerfile | 2 +- docker/Dockerfile.dev | 41 --------------- docker/dev-docker-entrypoint.sh | 51 ------------------- docker/docker-compose.production.yml | 29 ++++++----- docker/docker-compose.yml | 10 ++-- ...ker-entrypoint.sh => docker-entrypoint.sh} | 2 +- 10 files changed, 32 insertions(+), 119 deletions(-) delete mode 100644 docker/Dockerfile.dev delete mode 100644 docker/dev-docker-entrypoint.sh rename docker/{prod-docker-entrypoint.sh => docker-entrypoint.sh} (97%) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 39a87c65..a2077946 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -2,7 +2,7 @@ FROM ruby:3.3.4-alpine ENV APP_PATH=/var/app -ENV BUNDLE_VERSION=2.5.9 +ENV BUNDLE_VERSION=2.5.21 ENV BUNDLE_PATH=/usr/local/bundle/gems ENV TMP_PATH=/tmp/ ENV RAILS_LOG_TO_STDOUT=true @@ -27,7 +27,7 @@ RUN apk -U add --no-cache \ && rm -rf /var/cache/apk/* \ && mkdir -p $APP_PATH -RUN gem update --system 3.5.7 && gem install bundler --version "$BUNDLE_VERSION" \ +RUN gem update --system 3.6.2 && gem install bundler --version "$BUNDLE_VERSION" \ && rm -rf $GEM_HOME/cache/* # FIXME It would be a good idea to use a other user than root, but this lead to permission error on export and maybe more yet. diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index e0bc7867..56fb8d5f 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -35,7 +35,7 @@ services: PROMETHEUS_EXPORTER_PORT: 9394 ENABLE_TELEMETRY: false # More on telemetry: https://dawarich.app/docs/tutorials/telemetry dawarich_redis: - image: redis:7.0-alpine + image: redis:7.4-alpine container_name: dawarich_redis command: redis-server networks: @@ -50,7 +50,7 @@ services: start_period: 30s timeout: 10s dawarich_db: - image: postgres:14.2-alpine + image: postgres:17-alpine container_name: dawarich_db volumes: - dawarich_db_data:/var/lib/postgresql/data diff --git a/app/jobs/visit_suggesting_job.rb b/app/jobs/visit_suggesting_job.rb index 06883b64..b1a3e13d 100644 --- a/app/jobs/visit_suggesting_job.rb +++ b/app/jobs/visit_suggesting_job.rb @@ -7,6 +7,10 @@ class VisitSuggestingJob < ApplicationJob def perform(user_ids: [], start_at: 1.day.ago, end_at: Time.current) users = user_ids.any? ? User.where(id: user_ids) : User.all - users.find_each { Visits::Suggest.new(_1, start_at:, end_at:).call } + users.find_each do |user| + next if user.tracked_points.empty? + + Visits::Suggest.new(user, start_at:, end_at:).call + end end end diff --git a/db/data/20240610170930_remove_points_without_coordinates.rb b/db/data/20240610170930_remove_points_without_coordinates.rb index b8647672..85ef7684 100644 --- a/db/data/20240610170930_remove_points_without_coordinates.rb +++ b/db/data/20240610170930_remove_points_without_coordinates.rb @@ -12,7 +12,7 @@ def up Rails.logger.info 'Points without coordinates removed.' - BulkStatsCalculatingJob.perform_later(User.pluck(:id)) + BulkStatsCalculatingJob.perform_later end def down diff --git a/docker/Dockerfile b/docker/Dockerfile index 73644a6a..330334bf 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -40,7 +40,7 @@ RUN bundle config set --local path 'vendor/bundle' \ COPY ../. ./ # Copy entrypoint scripts and grant execution permissions -COPY ./docker/prod-docker-entrypoint.sh /usr/local/bin/entrypoint.sh +COPY ./docker/docker-entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod +x /usr/local/bin/entrypoint.sh EXPOSE $RAILS_PORT diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev deleted file mode 100644 index ad2336aa..00000000 --- a/docker/Dockerfile.dev +++ /dev/null @@ -1,41 +0,0 @@ -FROM ruby:3.3.4-alpine - -ENV APP_PATH=/var/app -ENV BUNDLE_VERSION=2.5.21 -ENV BUNDLE_PATH=/usr/local/bundle/gems -ENV TMP_PATH=/tmp/ -ENV RAILS_LOG_TO_STDOUT=true -ENV RAILS_PORT=3000 - -# install dependencies for application -RUN apk -U add --no-cache \ - build-base \ - git \ - postgresql-dev \ - postgresql-client \ - libxml2-dev \ - libxslt-dev \ - nodejs \ - yarn \ - imagemagick \ - tzdata \ - less \ - # gcompat for nokogiri on mac m1 - gcompat \ - && rm -rf /var/cache/apk/* \ - && mkdir -p $APP_PATH - -RUN gem install bundler --version "$BUNDLE_VERSION" \ - && rm -rf $GEM_HOME/cache/* - -# copy entrypoint scripts and grant execution permissions -COPY ./dev-docker-entrypoint.sh /usr/local/bin/dev-entrypoint.sh -COPY ./test-docker-entrypoint.sh /usr/local/bin/test-entrypoint.sh -RUN chmod +x /usr/local/bin/dev-entrypoint.sh && chmod +x /usr/local/bin/test-entrypoint.sh - -# navigate to app directory -WORKDIR $APP_PATH - -EXPOSE $RAILS_PORT - -ENTRYPOINT [ "bundle", "exec" ] diff --git a/docker/dev-docker-entrypoint.sh b/docker/dev-docker-entrypoint.sh deleted file mode 100644 index 90a8ddfd..00000000 --- a/docker/dev-docker-entrypoint.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/sh - -unset BUNDLE_PATH -unset BUNDLE_BIN - -set -e - -echo "Environment: $RAILS_ENV" - -# set env var defaults -DATABASE_HOST=${DATABASE_HOST:-"dawarich_db"} -DATABASE_PORT=${DATABASE_PORT:-5432} -DATABASE_USERNAME=${DATABASE_USERNAME:-"postgres"} -DATABASE_PASSWORD=${DATABASE_PASSWORD:-"password"} -DATABASE_NAME=${DATABASE_NAME:-"dawarich_development"} - -# Remove pre-existing puma/passenger server.pid -rm -f $APP_PATH/tmp/pids/server.pid - -# Wait for the database to be ready -until nc -zv $DATABASE_HOST ${DATABASE_PORT:-5432}; do - echo "Waiting for PostgreSQL to be ready..." - sleep 1 -done - -# Install gems -gem update --system 3.6.2 -gem install bundler --version '2.5.21' - -# Create the database -if [ "$(psql "postgres://$DATABASE_USERNAME:$DATABASE_PASSWORD@$DATABASE_HOST:$DATABASE_PORT" -XtAc "SELECT 1 FROM pg_database WHERE datname='$DATABASE_NAME'")" = '1' ]; then - echo "Database $DATABASE_NAME already exists, skipping creation..." -else - echo "Creating database $DATABASE_NAME..." - bundle exec rails db:create -fi - -# Run database migrations -echo "PostgreSQL is ready. Running database migrations..." -bundle exec rails db:migrate - -# Run data migrations -echo "Running DATA migrations..." -bundle exec rake data:migrate - -# Run seeds -echo "Running seeds..." -bundle exec rake db:seed - -# run passed commands -bundle exec ${@} diff --git a/docker/docker-compose.production.yml b/docker/docker-compose.production.yml index fea4fa75..ce5ba6db 100644 --- a/docker/docker-compose.production.yml +++ b/docker/docker-compose.production.yml @@ -8,7 +8,7 @@ services: networks: - dawarich volumes: - - shared_data:/var/shared/redis + - dawarich_redis_data:/var/shared/redis restart: always healthcheck: test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ] @@ -18,10 +18,10 @@ services: timeout: 10s dawarich_db: image: postgres:17-alpine + shm_size: 1G container_name: dawarich_db volumes: - - db_data:/var/lib/postgresql/data - - shared_data:/var/shared + - dawarich_db_data:/var/lib/postgresql/data networks: - dawarich environment: @@ -39,9 +39,9 @@ services: image: dawarich:prod container_name: dawarich_app volumes: - - gem_cache:/usr/local/bundle/gems_app - - public:/var/app/public - - watched:/var/app/tmp/imports/watched + - dawarich_gem_cache_app:/usr/local/bundle/gems + - dawarich_public:/var/app/public + - dawarich_watched:/var/app/tmp/imports/watched networks: - dawarich ports: @@ -99,9 +99,9 @@ services: image: dawarich:prod container_name: dawarich_sidekiq volumes: - - gem_cache:/usr/local/bundle/gems_sidekiq - - public:/var/app/public - - watched:/var/app/tmp/imports/watched + - dawarich_gem_cache_sidekiq:/usr/local/bundle/gems + - dawarich_public:/var/app/public + - dawarich_watched:/var/app/tmp/imports/watched networks: - dawarich stdin_open: true @@ -156,8 +156,9 @@ services: memory: '2G' # Limit memory usage to 2GB volumes: - db_data: - gem_cache: - shared_data: - public: - watched: + dawarich_db_data: + dawarich_redis_data: + dawarich_gem_cache_app: + dawarich_gem_cache_sidekiq: + dawarich_public: + dawarich_watched: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 53586a39..ea7f5789 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -38,7 +38,7 @@ services: timeout: 10s # command: postgres -c config_file=/etc/postgresql/postgresql.conf # Use custom config, uncomment if you want to use a custom config dawarich_app: - image: freikin/dawarich:latest + image: dawarich:prod container_name: dawarich_app volumes: - dawarich_gem_cache_app:/usr/local/bundle/gems @@ -51,8 +51,8 @@ services: # - 9394:9394 # Prometheus exporter, uncomment if needed stdin_open: true tty: true - entrypoint: dev-entrypoint.sh - command: ['bin/dev'] + entrypoint: entrypoint.sh + command: ['bin/rails', 'server', '-p', '3000', '-b', '::'] restart: on-failure environment: RAILS_ENV: development @@ -94,7 +94,7 @@ services: cpus: '0.50' # Limit CPU usage to 50% of one core memory: '2G' # Limit memory usage to 2GB dawarich_sidekiq: - image: freikin/dawarich:latest + image: dawarich:prod container_name: dawarich_sidekiq volumes: - dawarich_gem_cache_sidekiq:/usr/local/bundle/gems @@ -104,7 +104,7 @@ services: - dawarich stdin_open: true tty: true - entrypoint: dev-entrypoint.sh + entrypoint: entrypoint.sh command: ['sidekiq'] restart: on-failure environment: diff --git a/docker/prod-docker-entrypoint.sh b/docker/docker-entrypoint.sh similarity index 97% rename from docker/prod-docker-entrypoint.sh rename to docker/docker-entrypoint.sh index 50ff835c..1fe36928 100644 --- a/docker/prod-docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -5,7 +5,7 @@ unset BUNDLE_BIN set -e -echo "Environment: $RAILS_ENV" +echo "⚠️ Environment: $RAILS_ENV ⚠️" # Parse DATABASE_URL if present, otherwise use individual variables if [ -n "$DATABASE_URL" ]; then From 3312ea794f2f65774eafd4527704654336f249f7 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Thu, 9 Jan 2025 14:44:16 +0100 Subject: [PATCH 07/11] Add separate entrypoints for web and sidekiq --- docker/Dockerfile | 25 ++++++++++---- docker/docker-compose.production.yml | 4 +-- docker/docker-compose.yml | 4 +-- docker/sidekiq-entrypoint.sh | 34 +++++++++++++++++++ ...docker-entrypoint.sh => web-entrypoint.sh} | 29 +++++++--------- 5 files changed, 70 insertions(+), 26 deletions(-) create mode 100644 docker/sidekiq-entrypoint.sh rename docker/{docker-entrypoint.sh => web-entrypoint.sh} (63%) diff --git a/docker/Dockerfile b/docker/Dockerfile index 330334bf..ce4d8455 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -3,7 +3,6 @@ FROM ruby:3.3.4-alpine ENV APP_PATH=/var/app ENV BUNDLE_VERSION=2.5.21 ENV BUNDLE_PATH=/usr/local/bundle/gems -ENV TMP_PATH=/tmp/ ENV RAILS_LOG_TO_STDOUT=true ENV RAILS_PORT=3000 @@ -22,10 +21,11 @@ RUN apk -U add --no-cache \ less \ yaml-dev \ gcompat \ - && rm -rf /var/cache/apk/* \ && mkdir -p $APP_PATH -RUN gem install bundler --version "$BUNDLE_VERSION" \ +# Update gem system and install bundler +RUN gem update --system 3.6.2 \ + && gem install bundler --version "$BUNDLE_VERSION" \ && rm -rf $GEM_HOME/cache/* # Navigate to app directory @@ -35,13 +35,26 @@ COPY ../Gemfile ../Gemfile.lock ../vendor ../.ruby-version ./ # Install missing gems RUN bundle config set --local path 'vendor/bundle' \ - && bundle install --jobs 20 --retry 5 + && if [ "$RAILS_ENV" = "production" ]; then \ + bundle install --jobs 4 --retry 3 --without development test; \ + else \ + bundle install --jobs 4 --retry 3; \ + fi COPY ../. ./ +# Precompile assets for production +RUN if [ "$RAILS_ENV" = "production" ]; then \ + bundle exec rake assets:precompile \ + && rm -rf node_modules tmp/cache; \ + fi + # Copy entrypoint scripts and grant execution permissions -COPY ./docker/docker-entrypoint.sh /usr/local/bin/entrypoint.sh -RUN chmod +x /usr/local/bin/entrypoint.sh +COPY ./docker/web-entrypoint.sh /usr/local/bin/web-entrypoint.sh +RUN chmod +x /usr/local/bin/web-entrypoint.sh + +COPY ./docker/sidekiq-entrypoint.sh /usr/local/bin/sidekiq-entrypoint.sh +RUN chmod +x /usr/local/bin/sidekiq-entrypoint.sh EXPOSE $RAILS_PORT diff --git a/docker/docker-compose.production.yml b/docker/docker-compose.production.yml index ce5ba6db..3603eefb 100644 --- a/docker/docker-compose.production.yml +++ b/docker/docker-compose.production.yml @@ -49,7 +49,7 @@ services: # - 9394:9394 # Prometheus exporter, uncomment if needed stdin_open: true tty: true - entrypoint: entrypoint.sh + entrypoint: web-entrypoint.sh command: ['bin/rails', 'server', '-p', '3000', '-b', '::'] restart: on-failure environment: @@ -106,7 +106,7 @@ services: - dawarich stdin_open: true tty: true - entrypoint: entrypoint.sh + entrypoint: sidekiq-entrypoint.sh command: ['bundle', 'exec', 'sidekiq'] restart: on-failure environment: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index ea7f5789..bee3df25 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -51,7 +51,7 @@ services: # - 9394:9394 # Prometheus exporter, uncomment if needed stdin_open: true tty: true - entrypoint: entrypoint.sh + entrypoint: web-entrypoint.sh command: ['bin/rails', 'server', '-p', '3000', '-b', '::'] restart: on-failure environment: @@ -104,7 +104,7 @@ services: - dawarich stdin_open: true tty: true - entrypoint: entrypoint.sh + entrypoint: sidekiq-entrypoint.sh command: ['sidekiq'] restart: on-failure environment: diff --git a/docker/sidekiq-entrypoint.sh b/docker/sidekiq-entrypoint.sh new file mode 100644 index 00000000..1083891b --- /dev/null +++ b/docker/sidekiq-entrypoint.sh @@ -0,0 +1,34 @@ +#!/bin/sh + +unset BUNDLE_PATH +unset BUNDLE_BIN + +set -e + +echo "⚠️ Starting Sidekiq in $RAILS_ENV environment ⚠️" + +# Parse DATABASE_URL if present, otherwise use individual variables +if [ -n "$DATABASE_URL" ]; then + # Extract components from DATABASE_URL + DATABASE_HOST=$(echo $DATABASE_URL | awk -F[@/] '{print $4}') + DATABASE_PORT=$(echo $DATABASE_URL | awk -F[@/:] '{print $5}') + DATABASE_USERNAME=$(echo $DATABASE_URL | awk -F[:/@] '{print $4}') + DATABASE_PASSWORD=$(echo $DATABASE_URL | awk -F[:/@] '{print $5}') +else + # Use existing environment variables + DATABASE_HOST=${DATABASE_HOST} + DATABASE_PORT=${DATABASE_PORT} + DATABASE_USERNAME=${DATABASE_USERNAME} + DATABASE_PASSWORD=${DATABASE_PASSWORD} +fi + +# Wait for the database to become available +echo "⏳ Waiting for database to be ready..." +until PGPASSWORD=$DATABASE_PASSWORD psql -h "$DATABASE_HOST" -p "$DATABASE_PORT" -U "$DATABASE_USERNAME" -c '\q'; do + >&2 echo "Postgres is unavailable - retrying..." + sleep 2 +done +echo "✅ PostgreSQL is ready!" + +# run sidekiq +bundle exec sidekiq diff --git a/docker/docker-entrypoint.sh b/docker/web-entrypoint.sh similarity index 63% rename from docker/docker-entrypoint.sh rename to docker/web-entrypoint.sh index 1fe36928..cfd52116 100644 --- a/docker/docker-entrypoint.sh +++ b/docker/web-entrypoint.sh @@ -5,7 +5,7 @@ unset BUNDLE_BIN set -e -echo "⚠️ Environment: $RAILS_ENV ⚠️" +echo "⚠️ Starting Rails environment: $RAILS_ENV ⚠️" # Parse DATABASE_URL if present, otherwise use individual variables if [ -n "$DATABASE_URL" ]; then @@ -27,14 +27,16 @@ fi # Remove pre-existing puma/passenger server.pid rm -f $APP_PATH/tmp/pids/server.pid -# Install gems -gem update --system 3.6.2 -gem install bundler --version '2.5.21' +# Wait for the database to become available +echo "⏳ Waiting for database to be ready..." +until PGPASSWORD=$DATABASE_PASSWORD psql -h "$DATABASE_HOST" -p "$DATABASE_PORT" -U "$DATABASE_USERNAME" -c '\q'; do + >&2 echo "Postgres is unavailable - retrying..." + sleep 2 +done +echo "✅ PostgreSQL is ready!" -# Create the database if it doesn't exist -if PGPASSWORD=$DATABASE_PASSWORD psql -h "$DATABASE_HOST" -p "$DATABASE_PORT" -U "$DATABASE_USERNAME" -c "SELECT 1 FROM pg_database WHERE datname='$DATABASE_NAME'" | grep -q 1; then - echo "Database $DATABASE_NAME already exists, skipping creation..." -else +# Create database if it doesn't exist +if ! PGPASSWORD=$DATABASE_PASSWORD psql -h "$DATABASE_HOST" -p "$DATABASE_PORT" -U "$DATABASE_USERNAME" -c "SELECT 1 FROM pg_database WHERE datname='$DATABASE_NAME'" | grep -q 1; then echo "Creating database $DATABASE_NAME..." bundle exec rails db:create fi @@ -47,14 +49,9 @@ bundle exec rails db:migrate echo "Running DATA migrations..." bundle exec rake data:migrate -# Run seeds -echo "Running seeds..." -bundle exec rake db:seed - -# Precompile assets -if [ "$RAILS_ENV" = "production" ]; then - echo "Precompiling assets..." - bundle exec rake assets:precompile +if [ "$RAILS_ENV" != "production" ]; then + echo "Running seeds..." + bundle exec rails db:seed fi # run passed commands From 1b6273ba1c11896bde27e5f92a83d0b8db2fac5c Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Thu, 9 Jan 2025 14:47:21 +0100 Subject: [PATCH 08/11] Fix visit suggesting job spec --- spec/jobs/visit_suggesting_job_spec.rb | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/spec/jobs/visit_suggesting_job_spec.rb b/spec/jobs/visit_suggesting_job_spec.rb index 70af4e74..f2ce47d9 100644 --- a/spec/jobs/visit_suggesting_job_spec.rb +++ b/spec/jobs/visit_suggesting_job_spec.rb @@ -3,9 +3,9 @@ require 'rails_helper' RSpec.describe VisitSuggestingJob, type: :job do - describe '#perform' do - let!(:users) { [create(:user)] } + let!(:users) { [create(:user)] } + describe '#perform' do subject { described_class.perform_now } before do @@ -13,10 +13,22 @@ allow_any_instance_of(Visits::Suggest).to receive(:call) end - it 'suggests visits' do - subject + context 'when user has no tracked points' do + it 'does not suggest visits' do + subject + + expect(Visits::Suggest).not_to have_received(:new) + end + end + + context 'when user has tracked points' do + let!(:tracked_point) { create(:point, user: users.first) } + + it 'suggests visits' do + subject - expect(Visits::Suggest).to have_received(:new) + expect(Visits::Suggest).to have_received(:new) + end end end end From c13ebe8d3c2e61ae2ded3fbb1d7382efc22a8c01 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Thu, 9 Jan 2025 15:04:05 +0100 Subject: [PATCH 09/11] Split docker files --- .app_version | 2 +- .github/workflows/build_and_push.yml | 2 +- CHANGELOG.md | 22 ++++++++++++ config/database.yml | 1 - docker/Dockerfile.dev | 50 ++++++++++++++++++++++++++ docker/{Dockerfile => Dockerfile.prod} | 16 +++------ docker/docker-compose.production.yml | 4 --- docker/docker-compose.yml | 4 +-- 8 files changed, 81 insertions(+), 20 deletions(-) create mode 100644 docker/Dockerfile.dev rename docker/{Dockerfile => Dockerfile.prod} (75%) diff --git a/.app_version b/.app_version index 78cfa5eb..21574090 100644 --- a/.app_version +++ b/.app_version @@ -1 +1 @@ -0.21.6 +0.22.0 diff --git a/.github/workflows/build_and_push.yml b/.github/workflows/build_and_push.yml index 05173d53..00a78a71 100644 --- a/.github/workflows/build_and_push.yml +++ b/.github/workflows/build_and_push.yml @@ -40,7 +40,7 @@ jobs: uses: docker/build-push-action@v2 with: context: . - file: ./docker/Dockerfile + file: ./docker/Dockerfile.dev push: true tags: freikin/dawarich:latest,freikin/dawarich:${{ github.event.inputs.branch || github.ref_name }} platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6 diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b2e1169..d02b51c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,28 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +# 0.22.0 - 2025-01-09 + +⚠️ This release introduces a breaking change. ⚠️ + +Please read this release notes carefully before upgrading. + +### Changed + +- All docker-related files were moved to the `docker` directory. +- Default memory limit for `dawarich_app` and `dawarich_sidekiq` services was increased to 4GB. +- `dawarich_app` and `dawarich_sidekiq` services now use separate entrypoint scripts. +- Gems (dependency libraries) are now being shipped as part of the Dawarich Docker image. + +### Fixed + +- Visit suggesting job does nothing if user has no tracked points. +- `BulkStatsCalculationJob` now being called without arguments in the data migration. + +### Added + +- A proper production Dockerfile, docker-compose and env files. + # 0.21.6 - 2025-01-07 ### Changed diff --git a/config/database.yml b/config/database.yml index 0dc98e97..fca7a51c 100644 --- a/config/database.yml +++ b/config/database.yml @@ -20,4 +20,3 @@ test: production: <<: *default database: <%= ENV['DATABASE_NAME'] || 'dawarich_production' %> - # url: <%= ENV['DATABASE_URL'] %> diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev new file mode 100644 index 00000000..381ece15 --- /dev/null +++ b/docker/Dockerfile.dev @@ -0,0 +1,50 @@ +FROM ruby:3.3.4-alpine + +ENV APP_PATH=/var/app +ENV BUNDLE_VERSION=2.5.21 +ENV BUNDLE_PATH=/usr/local/bundle/gems +ENV RAILS_LOG_TO_STDOUT=true +ENV RAILS_PORT=3000 + +# Install dependencies for application +RUN apk -U add --no-cache \ + build-base \ + git \ + postgresql-dev \ + postgresql-client \ + libxml2-dev \ + libxslt-dev \ + nodejs \ + yarn \ + imagemagick \ + tzdata \ + less \ + yaml-dev \ + gcompat \ + && mkdir -p $APP_PATH + +# Update gem system and install bundler +RUN gem update --system 3.6.2 \ + && gem install bundler --version "$BUNDLE_VERSION" \ + && rm -rf $GEM_HOME/cache/* + +WORKDIR $APP_PATH + +COPY ../Gemfile ../Gemfile.lock ../vendor ../.ruby-version ./ + +# Install all gems including development and test +RUN bundle config set --local path 'vendor/bundle' \ + && bundle install --jobs 4 --retry 3 + +COPY ../. ./ + +# Copy entrypoint scripts and grant execution permissions +COPY ./docker/web-entrypoint.sh /usr/local/bin/web-entrypoint.sh +RUN chmod +x /usr/local/bin/web-entrypoint.sh + +COPY ./docker/sidekiq-entrypoint.sh /usr/local/bin/sidekiq-entrypoint.sh +RUN chmod +x /usr/local/bin/sidekiq-entrypoint.sh + +EXPOSE $RAILS_PORT + +ENTRYPOINT [ "bundle", "exec" ] diff --git a/docker/Dockerfile b/docker/Dockerfile.prod similarity index 75% rename from docker/Dockerfile rename to docker/Dockerfile.prod index ce4d8455..a8f6a449 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile.prod @@ -5,6 +5,7 @@ ENV BUNDLE_VERSION=2.5.21 ENV BUNDLE_PATH=/usr/local/bundle/gems ENV RAILS_LOG_TO_STDOUT=true ENV RAILS_PORT=3000 +ENV RAILS_ENV=production # Install dependencies for application RUN apk -U add --no-cache \ @@ -28,26 +29,19 @@ RUN gem update --system 3.6.2 \ && gem install bundler --version "$BUNDLE_VERSION" \ && rm -rf $GEM_HOME/cache/* -# Navigate to app directory WORKDIR $APP_PATH COPY ../Gemfile ../Gemfile.lock ../vendor ../.ruby-version ./ -# Install missing gems +# Install production gems only RUN bundle config set --local path 'vendor/bundle' \ - && if [ "$RAILS_ENV" = "production" ]; then \ - bundle install --jobs 4 --retry 3 --without development test; \ - else \ - bundle install --jobs 4 --retry 3; \ - fi + && bundle install --jobs 4 --retry 3 --without development test COPY ../. ./ # Precompile assets for production -RUN if [ "$RAILS_ENV" = "production" ]; then \ - bundle exec rake assets:precompile \ - && rm -rf node_modules tmp/cache; \ - fi +RUN bundle exec rake assets:precompile \ + && rm -rf node_modules tmp/cache # Copy entrypoint scripts and grant execution permissions COPY ./docker/web-entrypoint.sh /usr/local/bin/web-entrypoint.sh diff --git a/docker/docker-compose.production.yml b/docker/docker-compose.production.yml index 3603eefb..df815317 100644 --- a/docker/docker-compose.production.yml +++ b/docker/docker-compose.production.yml @@ -65,8 +65,6 @@ services: TIME_ZONE: Europe/London APPLICATION_PROTOCOL: http DISTANCE_UNIT: km - PHOTON_API_HOST: photon.komoot.io - PHOTON_API_USE_HTTPS: true PROMETHEUS_EXPORTER_ENABLED: false PROMETHEUS_EXPORTER_HOST: 0.0.0.0 PROMETHEUS_EXPORTER_PORT: 9394 @@ -121,8 +119,6 @@ services: BACKGROUND_PROCESSING_CONCURRENCY: 10 APPLICATION_PROTOCOL: http DISTANCE_UNIT: km - PHOTON_API_HOST: photon.komoot.io - PHOTON_API_USE_HTTPS: true PROMETHEUS_EXPORTER_ENABLED: false PROMETHEUS_EXPORTER_HOST: dawarich_app PROMETHEUS_EXPORTER_PORT: 9394 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index bee3df25..e44ab15c 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -92,7 +92,7 @@ services: resources: limits: cpus: '0.50' # Limit CPU usage to 50% of one core - memory: '2G' # Limit memory usage to 2GB + memory: '4G' # Limit memory usage to 4GB dawarich_sidekiq: image: dawarich:prod container_name: dawarich_sidekiq @@ -147,7 +147,7 @@ services: resources: limits: cpus: '0.50' # Limit CPU usage to 50% of one core - memory: '2G' # Limit memory usage to 2GB + memory: '4G' # Limit memory usage to 4GB volumes: dawarich_db_data: From 1e83330d291fcb9574e1c07b985b11a63889d877 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Thu, 9 Jan 2025 15:15:56 +0100 Subject: [PATCH 10/11] Update changelog --- CHANGELOG.md | 4 ++++ docker/docker-compose.production.yml | 4 ++-- docker/docker-compose.yml | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d02b51c3..bd521f86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/). Please read this release notes carefully before upgrading. +Docker-related files were moved to the `docker` directory and some of them were renamed. Before upgrading, study carefully changes in the `docker/docker-compose.yml` file and update your docker-compose file accordingly, so it uses the new files and commands. Copying `docker/docker-compose.yml` blindly may lead to errors. + +No volumes were removed or renamed, so with a proper docker-compose file, you should be able to upgrade without any issues. To make it easier comparing your existing docker-compose file with the new one, you may use https://www.diffchecker.com/. + ### Changed - All docker-related files were moved to the `docker` directory. diff --git a/docker/docker-compose.production.yml b/docker/docker-compose.production.yml index df815317..1eb90902 100644 --- a/docker/docker-compose.production.yml +++ b/docker/docker-compose.production.yml @@ -36,7 +36,7 @@ services: start_period: 30s timeout: 10s dawarich_app: - image: dawarich:prod + image: freikin/dawarich:latest container_name: dawarich_app volumes: - dawarich_gem_cache_app:/usr/local/bundle/gems @@ -94,7 +94,7 @@ services: cpus: '0.50' # Limit CPU usage to 50% of one core memory: '2G' # Limit memory usage to 2GB dawarich_sidekiq: - image: dawarich:prod + image: freikin/dawarich:latest container_name: dawarich_sidekiq volumes: - dawarich_gem_cache_sidekiq:/usr/local/bundle/gems diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index e44ab15c..d2154c2c 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -38,7 +38,7 @@ services: timeout: 10s # command: postgres -c config_file=/etc/postgresql/postgresql.conf # Use custom config, uncomment if you want to use a custom config dawarich_app: - image: dawarich:prod + image: freikin/dawarich:latest container_name: dawarich_app volumes: - dawarich_gem_cache_app:/usr/local/bundle/gems @@ -94,7 +94,7 @@ services: cpus: '0.50' # Limit CPU usage to 50% of one core memory: '4G' # Limit memory usage to 4GB dawarich_sidekiq: - image: dawarich:prod + image: freikin/dawarich:latest container_name: dawarich_sidekiq volumes: - dawarich_gem_cache_sidekiq:/usr/local/bundle/gems From 7766fcbd6a2463b47f9dc8b0a5545f927cfc8dc8 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Thu, 9 Jan 2025 15:19:31 +0100 Subject: [PATCH 11/11] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd521f86..09c51769 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ Docker-related files were moved to the `docker` directory and some of them were No volumes were removed or renamed, so with a proper docker-compose file, you should be able to upgrade without any issues. To make it easier comparing your existing docker-compose file with the new one, you may use https://www.diffchecker.com/. +Although `docker-compose.production.yml` was added, it's not being used by default. It's just an example of how to configure Dawarich for production. The default `docker-compose.yml` file is still recommended for running the app. + ### Changed - All docker-related files were moved to the `docker` directory.