From cd31fb4cf0a70fdbe25f2ace12a1f94ac05e91e8 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Thu, 26 Dec 2024 21:34:10 +0100 Subject: [PATCH 1/7] Add database constraints for users table --- Gemfile | 2 ++ Gemfile.lock | 6 +++++ app/models/user.rb | 5 ++++ config/initializers/strong_migrations.rb | 26 +++++++++++++++++++ ...26202204_add_database_users_constraints.rb | 8 ++++++ ...validate_add_database_users_constraints.rb | 14 ++++++++++ db/schema.rb | 4 ++- 7 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 config/initializers/strong_migrations.rb create mode 100644 db/migrate/20241226202204_add_database_users_constraints.rb create mode 100644 db/migrate/20241226202831_validate_add_database_users_constraints.rb diff --git a/Gemfile b/Gemfile index 103b7688..1066cbfa 100644 --- a/Gemfile +++ b/Gemfile @@ -30,6 +30,7 @@ gem 'sidekiq-cron' gem 'sidekiq-limit_fetch' gem 'sprockets-rails' gem 'stimulus-rails' +gem 'strong_migrations' gem 'tailwindcss-rails' gem 'turbo-rails' gem 'tzinfo-data', platforms: %i[mingw mswin x64_mingw jruby] @@ -54,6 +55,7 @@ group :test do end group :development do + gem 'database_consistency', require: false gem 'foreman' gem 'rubocop-rails', require: false end diff --git a/Gemfile.lock b/Gemfile.lock index 47ea71bd..73829ee8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -109,6 +109,8 @@ GEM data_migrate (11.2.0) activerecord (>= 6.1) railties (>= 6.1) + database_consistency (2.0.0) + activerecord (>= 3.2) date (3.4.1) debug (1.10.0) irb (~> 1.10) @@ -389,6 +391,8 @@ GEM stimulus-rails (1.3.4) railties (>= 6.0.0) stringio (3.1.2) + strong_migrations (2.1.0) + activerecord (>= 6.1) super_diff (0.14.0) attr_extras (>= 6.2.4) diff-lcs @@ -437,6 +441,7 @@ DEPENDENCIES bootsnap chartkick data_migrate + database_consistency debug devise dotenv-rails @@ -473,6 +478,7 @@ DEPENDENCIES simplecov sprockets-rails stimulus-rails + strong_migrations super_diff tailwindcss-rails turbo-rails diff --git a/app/models/user.rb b/app/models/user.rb index 64e45425..0dd32104 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -18,6 +18,11 @@ class User < ApplicationRecord after_create :create_api_key before_save :strip_trailing_slashes + validates :email, presence: true + validates :reset_password_token, uniqueness: true, allow_nil: true + + attribute :admin, :boolean, default: false + def countries_visited stats.pluck(:toponyms).flatten.map { _1['country'] }.uniq.compact end diff --git a/config/initializers/strong_migrations.rb b/config/initializers/strong_migrations.rb new file mode 100644 index 00000000..04e43a9e --- /dev/null +++ b/config/initializers/strong_migrations.rb @@ -0,0 +1,26 @@ +# Mark existing migrations as safe +StrongMigrations.start_after = 20241225175637 + +# Set timeouts for migrations +# If you use PgBouncer in transaction mode, delete these lines and set timeouts on the database user +StrongMigrations.lock_timeout = 10.seconds +StrongMigrations.statement_timeout = 1.hour + +# Analyze tables after indexes are added +# Outdated statistics can sometimes hurt performance +StrongMigrations.auto_analyze = true + +# Set the version of the production database +# so the right checks are run in development +# StrongMigrations.target_version = 10 + +# Add custom checks +# StrongMigrations.add_check do |method, args| +# if method == :add_index && args[0].to_s == "users" +# stop! "No more indexes on the users table" +# end +# end + +# Make some operations safe by default +# See https://github.com/ankane/strong_migrations#safe-by-default +# StrongMigrations.safe_by_default = true diff --git a/db/migrate/20241226202204_add_database_users_constraints.rb b/db/migrate/20241226202204_add_database_users_constraints.rb new file mode 100644 index 00000000..04247aeb --- /dev/null +++ b/db/migrate/20241226202204_add_database_users_constraints.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +class AddDatabaseUsersConstraints < ActiveRecord::Migration[8.0] + def change + add_check_constraint :users, 'email IS NOT NULL', name: 'users_email_null', validate: false + add_check_constraint :users, 'admin IS NOT NULL', name: 'users_admin_null', validate: false + end +end diff --git a/db/migrate/20241226202831_validate_add_database_users_constraints.rb b/db/migrate/20241226202831_validate_add_database_users_constraints.rb new file mode 100644 index 00000000..d05c606b --- /dev/null +++ b/db/migrate/20241226202831_validate_add_database_users_constraints.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class ValidateAddDatabaseUsersConstraints < ActiveRecord::Migration[8.0] + def up + validate_check_constraint :users, name: 'users_email_null' + change_column_null :users, :email, false + remove_check_constraint :users, name: 'users_email_null' + end + + def down + add_check_constraint :users, 'email IS NOT NULL', name: 'users_email_null', validate: false + change_column_null :users, :email, true + end +end diff --git a/db/schema.rb b/db/schema.rb index a79c53a9..0b55b25e 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2024_12_11_113119) do +ActiveRecord::Schema[8.0].define(version: 2024_12_26_202831) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -219,6 +219,8 @@ t.index ["reset_password_token"], name: "index_users_on_reset_password_token", unique: true end + add_check_constraint "users", "admin IS NOT NULL", name: "users_admin_null", validate: false + create_table "visits", force: :cascade do |t| t.bigint "area_id" t.bigint "user_id", null: false From 356d790fe36feb836311ce7d4f311e0799640180 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Tue, 21 Jan 2025 23:46:33 +0100 Subject: [PATCH 2/7] Update build_and_push workflow to build rc Docker image --- .app_version | 2 +- .github/workflows/build_and_push.yml | 34 ++++++++++++++++++++++++---- CHANGELOG.md | 6 +++++ 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/.app_version b/.app_version index 9e40e75c..40a6dfed 100644 --- a/.app_version +++ b/.app_version @@ -1 +1 @@ -0.23.3 +0.23.4 diff --git a/.github/workflows/build_and_push.yml b/.github/workflows/build_and_push.yml index 00a78a71..90c78ae1 100644 --- a/.github/workflows/build_and_push.yml +++ b/.github/workflows/build_and_push.yml @@ -15,13 +15,16 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: ref: ${{ github.event.inputs.branch || github.ref_name }} + - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 + - name: Cache Docker layers uses: actions/cache@v4 with: @@ -29,20 +32,41 @@ jobs: key: ${{ runner.os }}-buildx-${{ github.sha }} restore-keys: | ${{ runner.os }}-buildx- + - name: Install dependencies run: npm install + - name: Login to Docker Hub uses: docker/login-action@v3.1.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set Docker tags + id: docker_meta + run: | + VERSION=${GITHUB_REF#refs/tags/} + TAGS="freikin/dawarich:${VERSION}" + + # Add :rc tag for pre-releases + if [ "${{ github.event.release.prerelease }}" = "true" ]; then + TAGS="${TAGS},freikin/dawarich:rc" + fi + + # Add :latest tag only if release is not a pre-release + if [ "${{ github.event.release.prerelease }}" != "true" ]; then + TAGS="${TAGS},freikin/dawarich:latest" + fi + + echo "tags=${TAGS}" >> $GITHUB_OUTPUT + - name: Build and push - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: context: . file: ./docker/Dockerfile.dev push: true - tags: freikin/dawarich:latest,freikin/dawarich:${{ github.event.inputs.branch || github.ref_name }} + tags: ${{ steps.docker_meta.outputs.tags }} platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6 cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache diff --git a/CHANGELOG.md b/CHANGELOG.md index 1608203b..81a8b8e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +# 0.23.4 - 2025-01-21 + +### Added + +- A test for building rc Docker image. + # 0.23.3 - 2025-01-21 ### Changed From c3243bdba0f5647ec65095a853e2ded56daffb03 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Wed, 22 Jan 2025 10:53:33 +0100 Subject: [PATCH 3/7] Fix authentication to `GET /api/v1/countries/visited_cities` --- CHANGELOG.md | 7 ++++++- .../api/v1/countries/visited_cities_controller.rb | 2 +- app/services/tasks/imports/google_records.rb | 5 ++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81a8b8e6..93bbcd22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,12 +5,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -# 0.23.4 - 2025-01-21 +# 0.23.4 - 2025-01-22 ### Added - A test for building rc Docker image. +### Fixed + +- Fix authentication to `GET /api/v1/countries/visited_cities` with header `Authorization: Bearer YOUR_API_KEY` instead of `api_key` query param. #679 + + # 0.23.3 - 2025-01-21 ### Changed diff --git a/app/controllers/api/v1/countries/visited_cities_controller.rb b/app/controllers/api/v1/countries/visited_cities_controller.rb index 125baf8e..85e53f7d 100644 --- a/app/controllers/api/v1/countries/visited_cities_controller.rb +++ b/app/controllers/api/v1/countries/visited_cities_controller.rb @@ -17,6 +17,6 @@ def index private def required_params - %i[start_at end_at api_key] + %i[start_at end_at] end end diff --git a/app/services/tasks/imports/google_records.rb b/app/services/tasks/imports/google_records.rb index 83174128..70b5d389 100644 --- a/app/services/tasks/imports/google_records.rb +++ b/app/services/tasks/imports/google_records.rb @@ -30,12 +30,11 @@ def create_import def process_file_in_batches(import_id) batch = [] + index = 0 Oj.load_file(@file_path, mode: :compat) do |record| next unless record.is_a?(Hash) && record['locations'] - index = 0 - record['locations'].each do |location| batch << location @@ -47,7 +46,7 @@ def process_file_in_batches(import_id) end end - Import::GoogleTakeoutJob.perform_later(import_id, Oj.dump(batch)) if batch.any? + Import::GoogleTakeoutJob.perform_later(import_id, Oj.dump(batch), index) if batch.any? end def log_start From 157d916f6bfd9a1c815cfeed92c460ec824b2d85 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Wed, 22 Jan 2025 11:15:55 +0100 Subject: [PATCH 4/7] Fix a bug where a gpx file with empty tracks was not being imported --- CHANGELOG.md | 5 +++ app/services/gpx/track_parser.rb | 4 ++- spec/fixtures/files/gpx/arc_example.gpx | 41 +++++++++++++++++++++++++ spec/services/gpx/track_parser_spec.rb | 10 ++++++ 4 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 spec/fixtures/files/gpx/arc_example.gpx diff --git a/CHANGELOG.md b/CHANGELOG.md index 81a8b8e6..d6609218 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - A test for building rc Docker image. +### Fixed + +- Fix authentication to `GET /api/v1/countries/visited_cities` with header `Authorization: Bearer YOUR_API_KEY` instead of `api_key` query param. #679 +- Fix a bug where a gpx file with empty tracks was not being imported. #646 + # 0.23.3 - 2025-01-21 ### Changed diff --git a/app/services/gpx/track_parser.rb b/app/services/gpx/track_parser.rb index 65cbc3be..10f13983 100644 --- a/app/services/gpx/track_parser.rb +++ b/app/services/gpx/track_parser.rb @@ -15,7 +15,7 @@ def call tracks = json['gpx']['trk'] tracks_arr = tracks.is_a?(Array) ? tracks : [tracks] - tracks_arr.map { parse_track(_1) }.flatten.each.with_index(1) do |point, index| + tracks_arr.map { parse_track(_1) }.flatten.compact.each.with_index(1) do |point, index| create_point(point, index) end end @@ -23,6 +23,8 @@ def call private def parse_track(track) + return if track['trkseg'].blank? + segments = track['trkseg'] segments_array = segments.is_a?(Array) ? segments : [segments] diff --git a/spec/fixtures/files/gpx/arc_example.gpx b/spec/fixtures/files/gpx/arc_example.gpx new file mode 100644 index 00000000..f944f776 --- /dev/null +++ b/spec/fixtures/files/gpx/arc_example.gpx @@ -0,0 +1,41 @@ + + + + + 89.9031832732575 + Topland Hotel & Convention Center + + + walking + + + + taxi + + + 49.96302288016834 + + + + 49.884678590538186 + + + + 49.71960135141746 + + + + 49.91594081568717 + + + + 50.344669848377556 + + + + 50.12800953488726 + + + + + diff --git a/spec/services/gpx/track_parser_spec.rb b/spec/services/gpx/track_parser_spec.rb index b1026143..c5980c91 100644 --- a/spec/services/gpx/track_parser_spec.rb +++ b/spec/services/gpx/track_parser_spec.rb @@ -74,5 +74,15 @@ expect(Point.first.velocity).to eq('2.8') end end + + context 'when file exported from Arc' do + context 'when file has empty tracks' do + let(:file_path) { Rails.root.join('spec/fixtures/files/gpx/arc_example.gpx') } + + it 'creates points' do + expect { parser }.to change { Point.count }.by(6) + end + end + end end end From f60c93ee4f9362a5d1bfa14b2a6c26f604929dee Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Wed, 22 Jan 2025 12:17:26 +0100 Subject: [PATCH 5/7] Fix a bug where rc version was being checked as a stable release --- CHANGELOG.md | 1 + app/services/check_app_version.rb | 5 ++++- spec/services/check_app_version_spec.rb | 9 +++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4e1952e..31b55115 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Fix authentication to `GET /api/v1/countries/visited_cities` with header `Authorization: Bearer YOUR_API_KEY` instead of `api_key` query param. #679 - Fix a bug where a gpx file with empty tracks was not being imported. #646 +- Fix a bug where rc version was being checked as a stable release. #711 # 0.23.3 - 2025-01-21 diff --git a/app/services/check_app_version.rb b/app/services/check_app_version.rb index 3ae5ed76..bb2fd449 100644 --- a/app/services/check_app_version.rb +++ b/app/services/check_app_version.rb @@ -17,7 +17,10 @@ def call def latest_version Rails.cache.fetch(VERSION_CACHE_KEY, expires_in: 6.hours) do - JSON.parse(Net::HTTP.get(URI.parse(@repo_url)))[0]['name'] + versions = JSON.parse(Net::HTTP.get(URI.parse(@repo_url))) + # Find first version that contains only numbers and dots + release_version = versions.find { |v| v['name'].match?(/^\d+\.\d+\.\d+$/) } + release_version ? release_version['name'] : APP_VERSION end end end diff --git a/spec/services/check_app_version_spec.rb b/spec/services/check_app_version_spec.rb index b58cc2e5..1e90b3af 100644 --- a/spec/services/check_app_version_spec.rb +++ b/spec/services/check_app_version_spec.rb @@ -29,6 +29,15 @@ it { is_expected.to be true } end + context 'when latest version is not a stable release' do + before do + stub_request(:any, 'https://api.github.com/repos/Freika/dawarich/tags') + .to_return(status: 200, body: '[{"name": "1.0.0-rc.1"}]', headers: {}) + end + + it { is_expected.to be false } + end + context 'when request fails' do before do allow(Net::HTTP).to receive(:get).and_raise(StandardError) From f4027551e62c0aa8477234869d84daee60086eea Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Wed, 22 Jan 2025 15:06:35 +0100 Subject: [PATCH 6/7] Update StrongMigrations start_after --- config/initializers/strong_migrations.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/initializers/strong_migrations.rb b/config/initializers/strong_migrations.rb index 04e43a9e..ec978211 100644 --- a/config/initializers/strong_migrations.rb +++ b/config/initializers/strong_migrations.rb @@ -1,5 +1,5 @@ # Mark existing migrations as safe -StrongMigrations.start_after = 20241225175637 +StrongMigrations.start_after = 20_250_122_150_500 # Set timeouts for migrations # If you use PgBouncer in transaction mode, delete these lines and set timeouts on the database user From b0e6d47648ebefb08976f8038fadcb24dce5d3b0 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Wed, 22 Jan 2025 15:11:49 +0100 Subject: [PATCH 7/7] Change version --- .app_version | 2 +- CHANGELOG.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.app_version b/.app_version index 40a6dfed..f6de0017 100644 --- a/.app_version +++ b/.app_version @@ -1 +1 @@ -0.23.4 +0.23.5 diff --git a/CHANGELOG.md b/CHANGELOG.md index 31b55115..6f117299 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -# 0.23.4 - 2025-01-22 +# 0.23.5 - 2025-01-22 ### Added