Merge pull request #699 from Freika/feature/api/points

.app_version

@@ -1 +1 @@
-0.22.5
+0.23.0

CHANGELOG.md

@@ -5,10 +5,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
-# 0.22.5 - 2025-01-20
+# 0.23.0 - 2025-01-20
+
+## ⚠️ IMPORTANT ⚠️
+
+This release includes a data migration to remove duplicated points from the database. It will not remove anything except for duplcates from the `points` table, but please make sure to create a [backup](https://dawarich.app/docs/tutorials/backup-and-restore) before updating to this version.
 
 ### Added
 
+- `POST /api/v1/points/create` endpoint added.
+- An index to guarantee uniqueness of points across `latitude`, `longitude`, `timestamp` and `user_id` values. This is introduced to make sure no duplicates will be created in the database in addition to previously existing validations.
 - `GET /api/v1/users/me` endpoint added to get current user.
 
 # 0.22.4 - 2025-01-20

app/controllers/api/v1/points_controller.rb

@@ -21,6 +21,12 @@ def index
     render json: serialized_points
   end
 
+  def create
+    Points::CreateJob.perform_later(batch_params, current_api_user.id)
+
+    render json: { message: 'Points are being processed' }
+  end
+
   def update
     point = current_api_user.tracked_points.find(params[:id])
 
@@ -42,6 +48,10 @@ def point_params
     params.require(:point).permit(:latitude, :longitude)
   end
 
+  def batch_params
+    params.permit(locations: [:type, { geometry: {}, properties: {} }], batch: {})
+  end
+
   def point_serializer
     params[:slim] == 'true' ? Api::SlimPointSerializer : Api::PointSerializer
   end

app/controllers/map_controller.rb

@@ -6,7 +6,6 @@ class MapController < ApplicationController
   def index
     @points = points.where('timestamp >= ? AND timestamp <= ?', start_at, end_at)
 
-    @countries_and_cities = CountriesAndCities.new(@points).call
     @coordinates =
       @points.pluck(:latitude, :longitude, :battery, :altitude, :timestamp, :velocity, :id, :country)
              .map { [_1.to_f, _2.to_f, _3.to_s, _4.to_s, _5.to_s, _6.to_s, _7.to_s, _8.to_s] }

app/jobs/points/create_job.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class Points::CreateJob < ApplicationJob
+  queue_as :default
+
+  def perform(params, user_id)
+    data = Points::Params.new(params, user_id).call
+
+    data.each_slice(1000) do |location_batch|
+      Point.upsert_all(
+        location_batch,
+        unique_by: %i[latitude longitude timestamp user_id],
+        returning: false
+      )
+    end
+  end
+end

app/models/point.rb

@@ -8,7 +8,11 @@ class Point < ApplicationRecord
   belongs_to :user
 
   validates :latitude, :longitude, :timestamp, presence: true
-
+  validates :timestamp, uniqueness: {
+    scope: %i[latitude longitude user_id],
+    message: 'already has a point at this location and time for this user',
+    index: true
+  }
   enum :battery_status, { unknown: 0, unplugged: 1, charging: 2, full: 3 }, suffix: true
   enum :trigger, {
     unknown: 0, background_event: 1, circular_region_event: 2, beacon_event: 3,

app/services/points/params.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+class Points::Params
+  attr_reader :data, :points, :user_id
+
+  def initialize(json, user_id)
+    @data = json.with_indifferent_access
+    @points = @data[:locations]
+    @user_id = user_id
+  end
+
+  def call
+    points.map do |point|
+      next unless params_valid?(point)
+
+      {
+        latitude:           point[:geometry][:coordinates][1],
+        longitude:          point[:geometry][:coordinates][0],
+        battery_status:     point[:properties][:battery_state],
+        battery:            battery_level(point[:properties][:battery_level]),
+        timestamp:          DateTime.parse(point[:properties][:timestamp]),
+        altitude:           point[:properties][:altitude],
+        tracker_id:         point[:properties][:device_id],
+        velocity:           point[:properties][:speed],
+        ssid:               point[:properties][:wifi],
+        accuracy:           point[:properties][:horizontal_accuracy],
+        vertical_accuracy:  point[:properties][:vertical_accuracy],
+        course_accuracy:    point[:properties][:course_accuracy],
+        course:             point[:properties][:course],
+        raw_data:           point,
+        user_id:            user_id
+      }
+    end.compact
+  end
+
+  private
+
+  def battery_level(level)
+    value = (level.to_f * 100).to_i
+
+    value.positive? ? value : nil
+  end
+
+  def params_valid?(point)
+    point[:geometry].present? &&
+      point[:geometry][:coordinates].present? &&
+      point.dig(:properties, :timestamp).present?
+  end
+end

config/routes.rb

@@ -68,7 +68,7 @@
       get   'users/me', to: 'users#me'
 
       resources :areas,     only: %i[index create update destroy]
-      resources :points,    only: %i[index destroy update]
+      resources :points,    only: %i[index create update destroy]
       resources :visits,    only: %i[update]
       resources :stats,     only: :index
 

db/data/20250120154554_remove_duplicate_points.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class RemoveDuplicatePoints < ActiveRecord::Migration[8.0]
+  def up
+    # Find duplicate groups using a subquery
+    duplicate_groups =
+      Point.select('latitude, longitude, timestamp, user_id, COUNT(*) as count')
+           .group('latitude, longitude, timestamp, user_id')
+           .having('COUNT(*) > 1')
+
+    puts "Duplicate groups found: #{duplicate_groups.length}"
+
+    duplicate_groups.each do |group|
+      points = Point.where(
+        latitude: group.latitude,
+        longitude: group.longitude,
+        timestamp: group.timestamp,
+        user_id: group.user_id
+      ).order(id: :asc)
+
+      # Keep the latest record and destroy all others
+      latest = points.last
+      points.where.not(id: latest.id).destroy_all
+    end
+  end
+
+  def down
+    # This migration cannot be reversed
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

db/data_schema.rb

@@ -1 +1 @@
-DataMigrate::Data.define(version: 20250104204852)
+DataMigrate::Data.define(version: 20250120154554)

db/migrate/20250120152014_add_course_and_course_accuracy_to_points.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class AddCourseAndCourseAccuracyToPoints < ActiveRecord::Migration[8.0]
+  def change
+    add_column :points, :course, :decimal, precision: 8, scale: 5
+    add_column :points, :course_accuracy, :decimal, precision: 8, scale: 5
+  end
+end

db/migrate/20250120152540_add_external_track_id_to_points.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class AddExternalTrackIdToPoints < ActiveRecord::Migration[8.0]
+  disable_ddl_transaction!
+
+  def change
+    add_column :points, :external_track_id, :string
+
+    add_index :points, :external_track_id, algorithm: :concurrently
+  end
+end

db/migrate/20250120154555_add_unique_index_to_points.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class AddUniqueIndexToPoints < ActiveRecord::Migration[8.0]
+  disable_ddl_transaction!
+
+  def up
+    add_index :points, %i[latitude longitude timestamp user_id],
+              unique: true,
+              name: 'unique_points_index',
+              algorithm: :concurrently
+  end
+
+  def down
+    remove_index :points, name: 'unique_points_index'
+  end
+end

spec/factories/points.rb

@@ -25,6 +25,10 @@
     import_id       { '' }
     city            { nil }
     country         { nil }
+    reverse_geocoded_at { nil }
+    course          { nil }
+    course_accuracy { nil }
+    external_track_id { nil }
     user
 
     trait :with_known_location do

spec/factories/trips.rb

@@ -10,11 +10,15 @@
 
     trait :with_points do
       after(:build) do |trip|
-        create_list(
-          :point, 25,
-          user: trip.user,
-          timestamp: trip.started_at + (1..1000).to_a.sample.minutes
-        )
+        (1..25).map do |i|
+          create(
+            :point,
+            :with_geodata,
+            :reverse_geocoded,
+            timestamp: trip.started_at + i.minutes,
+            user: trip.user
+          )
+        end
       end
     end
   end