From 6c0a954e8ec172338db89cba17118c8f03f117f5 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Mon, 20 Jan 2025 16:37:15 +0100 Subject: [PATCH 1/8] Implement dawarich points parsing --- app/jobs/points/create_job.rb | 26 ++++ app/services/points/params.rb | 41 ++++++ ...dd_course_and_course_accuracy_to_points.rb | 8 ++ ...0152540_add_external_track_id_to_points.rb | 11 ++ db/schema.rb | 6 +- .../files/points/geojson_example.json | 136 ++++++++++++++++++ spec/jobs/points/create_job_spec.rb | 5 + spec/services/points/params_spec.rb | 66 +++++++++ 8 files changed, 298 insertions(+), 1 deletion(-) create mode 100644 app/jobs/points/create_job.rb create mode 100644 app/services/points/params.rb create mode 100644 db/migrate/20250120152014_add_course_and_course_accuracy_to_points.rb create mode 100644 db/migrate/20250120152540_add_external_track_id_to_points.rb create mode 100644 spec/fixtures/files/points/geojson_example.json create mode 100644 spec/jobs/points/create_job_spec.rb create mode 100644 spec/services/points/params_spec.rb diff --git a/app/jobs/points/create_job.rb b/app/jobs/points/create_job.rb new file mode 100644 index 00000000..f079046d --- /dev/null +++ b/app/jobs/points/create_job.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class Points::CreateJob < ApplicationJob + queue_as :default + + def perform(params, user_id) + data = Overland::Params.new(params).call + + data.each do |location| + next if point_exists?(location, user_id) + + Point.create!(location.merge(user_id:)) + end + end + + private + + def point_exists?(params, user_id) + Point.exists?( + latitude: params[:latitude], + longitude: params[:longitude], + timestamp: params[:timestamp], + user_id: + ) + end +end diff --git a/app/services/points/params.rb b/app/services/points/params.rb new file mode 100644 index 00000000..1e2873ca --- /dev/null +++ b/app/services/points/params.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +class Points::Params + attr_reader :data, :points + + def initialize(json) + @data = json.with_indifferent_access + @points = @data[:locations] + end + + def call + points.map do |point| + next if point[:geometry].nil? || point.dig(:properties, :timestamp).nil? + + { + latitude: point[:geometry][:coordinates][1], + longitude: point[:geometry][:coordinates][0], + battery_status: point[:properties][:battery_state], + battery: battery_level(point[:properties][:battery_level]), + timestamp: DateTime.parse(point[:properties][:timestamp]), + altitude: point[:properties][:altitude], + tracker_id: point[:properties][:device_id], + velocity: point[:properties][:speed], + ssid: point[:properties][:wifi], + accuracy: point[:properties][:horizontal_accuracy], + vertical_accuracy: point[:properties][:vertical_accuracy], + course_accuracy: point[:properties][:course_accuracy], + course: point[:properties][:course], + raw_data: point + } + end.compact + end + + private + + def battery_level(level) + value = (level.to_f * 100).to_i + + value.positive? ? value : nil + end +end diff --git a/db/migrate/20250120152014_add_course_and_course_accuracy_to_points.rb b/db/migrate/20250120152014_add_course_and_course_accuracy_to_points.rb new file mode 100644 index 00000000..78e1feb0 --- /dev/null +++ b/db/migrate/20250120152014_add_course_and_course_accuracy_to_points.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +class AddCourseAndCourseAccuracyToPoints < ActiveRecord::Migration[8.0] + def change + add_column :points, :course, :decimal, precision: 8, scale: 5 + add_column :points, :course_accuracy, :decimal, precision: 8, scale: 5 + end +end diff --git a/db/migrate/20250120152540_add_external_track_id_to_points.rb b/db/migrate/20250120152540_add_external_track_id_to_points.rb new file mode 100644 index 00000000..4531b19d --- /dev/null +++ b/db/migrate/20250120152540_add_external_track_id_to_points.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class AddExternalTrackIdToPoints < ActiveRecord::Migration[8.0] + disable_ddl_transaction! + + def change + add_column :points, :external_track_id, :string + + add_index :points, :external_track_id, algorithm: :concurrently + end +end diff --git a/db/schema.rb b/db/schema.rb index 16db4226..31ce24e6 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2024_12_11_113119) do +ActiveRecord::Schema[8.0].define(version: 2025_01_20_152540) do # These are extensions that must be enabled in order to support this database enable_extension "pg_catalog.plpgsql" @@ -156,12 +156,16 @@ t.jsonb "geodata", default: {}, null: false t.bigint "visit_id" t.datetime "reverse_geocoded_at" + t.decimal "course", precision: 8, scale: 5 + t.decimal "course_accuracy", precision: 8, scale: 5 + t.string "external_track_id" t.index ["altitude"], name: "index_points_on_altitude" t.index ["battery"], name: "index_points_on_battery" t.index ["battery_status"], name: "index_points_on_battery_status" t.index ["city"], name: "index_points_on_city" t.index ["connection"], name: "index_points_on_connection" t.index ["country"], name: "index_points_on_country" + t.index ["external_track_id"], name: "index_points_on_external_track_id" t.index ["geodata"], name: "index_points_on_geodata", using: :gin t.index ["import_id"], name: "index_points_on_import_id" t.index ["latitude", "longitude"], name: "index_points_on_latitude_and_longitude" diff --git a/spec/fixtures/files/points/geojson_example.json b/spec/fixtures/files/points/geojson_example.json new file mode 100644 index 00000000..c1cac9e4 --- /dev/null +++ b/spec/fixtures/files/points/geojson_example.json @@ -0,0 +1,136 @@ +{ + "locations" : [ + { + "type" : "Feature", + "geometry" : { + "type" : "Point", + "coordinates" : [ + -122.40530871, + 37.744304130000003 + ] + }, + "properties" : { + "horizontal_accuracy" : 5, + "track_id" : "799F32F5-89BB-45FB-A639-098B1B95B09F", + "speed_accuracy" : 0, + "vertical_accuracy" : -1, + "course_accuracy" : 0, + "altitude" : 0, + "speed" : 92.087999999999994, + "course" : 27.07, + "timestamp" : "2025-01-17T21:03:01Z", + "device_id" : "8D5D4197-245B-4619-A88B-2049100ADE46" + } + }, + { + "type" : "Feature", + "properties" : { + "timestamp" : "2025-01-17T21:03:02Z", + "horizontal_accuracy" : 5, + "course" : 24.260000000000002, + "speed_accuracy" : 0, + "device_id" : "8D5D4197-245B-4619-A88B-2049100ADE46", + "vertical_accuracy" : -1, + "altitude" : 0, + "track_id" : "799F32F5-89BB-45FB-A639-098B1B95B09F", + "speed" : 92.448000000000008, + "course_accuracy" : 0 + }, + "geometry" : { + "type" : "Point", + "coordinates" : [ + -122.40518926999999, + 37.744513759999997 + ] + } + }, + { + "type" : "Feature", + "properties" : { + "altitude" : 0, + "horizontal_accuracy" : 5, + "speed" : 123.76800000000001, + "course_accuracy" : 0, + "speed_accuracy" : 0, + "course" : 309.73000000000002, + "track_id" : "F63A3CF9-2FF8-4076-8F59-5BB1EDC23888", + "device_id" : "8D5D4197-245B-4619-A88B-2049100ADE46", + "timestamp" : "2025-01-17T21:18:38Z", + "vertical_accuracy" : -1 + }, + "geometry" : { + "type" : "Point", + "coordinates" : [ + -122.28487643, + 37.454486080000002 + ] + } + }, + { + "type" : "Feature", + "properties" : { + "track_id" : "F63A3CF9-2FF8-4076-8F59-5BB1EDC23888", + "device_id" : "8D5D4197-245B-4619-A88B-2049100ADE46", + "speed_accuracy" : 0, + "course_accuracy" : 0, + "speed" : 123.3, + "horizontal_accuracy" : 5, + "course" : 309.38, + "altitude" : 0, + "timestamp" : "2025-01-17T21:18:39Z", + "vertical_accuracy" : -1 + }, + "geometry" : { + "coordinates" : [ + -122.28517332, + 37.454684899999997 + ], + "type" : "Point" + } + }, + { + "geometry" : { + "coordinates" : [ + -122.28547306, + 37.454883219999999 + ], + "type" : "Point" + }, + "properties" : { + "course_accuracy" : 0, + "device_id" : "8D5D4197-245B-4619-A88B-2049100ADE46", + "vertical_accuracy" : -1, + "course" : 309.73000000000002, + "speed_accuracy" : 0, + "timestamp" : "2025-01-17T21:18:40Z", + "horizontal_accuracy" : 5, + "speed" : 125.06400000000001, + "track_id" : "F63A3CF9-2FF8-4076-8F59-5BB1EDC23888", + "altitude" : 0 + }, + "type" : "Feature" + }, + { + "geometry" : { + "type" : "Point", + "coordinates" : [ + -122.28577665, + 37.455080109999997 + ] + }, + "properties" : { + "course_accuracy" : 0, + "speed_accuracy" : 0, + "speed" : 124.05600000000001, + "track_id" : "F63A3CF9-2FF8-4076-8F59-5BB1EDC23888", + "course" : 309.73000000000002, + "device_id" : "8D5D4197-245B-4619-A88B-2049100ADE46", + "altitude" : 0, + "horizontal_accuracy" : 5, + "vertical_accuracy" : -1, + "timestamp" : "2025-01-17T21:18:41Z" + }, + "type" : "Feature" + } + ] +} diff --git a/spec/jobs/points/create_job_spec.rb b/spec/jobs/points/create_job_spec.rb new file mode 100644 index 00000000..70baa6e5 --- /dev/null +++ b/spec/jobs/points/create_job_spec.rb @@ -0,0 +1,5 @@ +require 'rails_helper' + +RSpec.describe Points::CreateJob, type: :job do + pending "add some examples to (or delete) #{__FILE__}" +end diff --git a/spec/services/points/params_spec.rb b/spec/services/points/params_spec.rb new file mode 100644 index 00000000..6fb3f486 --- /dev/null +++ b/spec/services/points/params_spec.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe Points::Params do + describe '#call' do + let(:file_path) { 'spec/fixtures/files/points/geojson_example.json' } + let(:file) { File.open(file_path) } + let(:json) { JSON.parse(file.read) } + let(:expected_json) do + { + latitude: 37.74430413, + longitude: -122.40530871, + battery_status: nil, + battery: nil, + timestamp: DateTime.parse('2025-01-17T21:03:01Z'), + altitude: 0, + tracker_id: '8D5D4197-245B-4619-A88B-2049100ADE46', + velocity: 92.088, + ssid: nil, + accuracy: 5, + vertical_accuracy: -1, + course_accuracy: 0, + course: 27.07, + raw_data: { + type: 'Feature', + geometry: { + type: 'Point', + coordinates: [-122.40530871, 37.74430413] + }, + properties: { + horizontal_accuracy: 5, + track_id: '799F32F5-89BB-45FB-A639-098B1B95B09F', + speed_accuracy: 0, + vertical_accuracy: -1, + course_accuracy: 0, + altitude: 0, + speed: 92.088, + course: 27.07, + timestamp: '2025-01-17T21:03:01Z', + device_id: '8D5D4197-245B-4619-A88B-2049100ADE46' + } + }.with_indifferent_access + } + end + + subject(:params) { described_class.new(json).call } + + it 'returns an array of points' do + expect(params).to be_an(Array) + expect(params.first).to eq(expected_json) + end + + it 'returns the correct number of points' do + expect(params.size).to eq(6) + end + + it 'returns correct keys' do + expect(params.first.keys).to eq(expected_json.keys) + end + + it 'returns the correct values' do + expect(params.first).to eq(expected_json) + end + end +end From 6644fc9a132edc1f6b3c3173a746c25492275fbe Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Mon, 20 Jan 2025 17:59:13 +0100 Subject: [PATCH 2/8] Introduce uniqueness index and validation for points --- app/controllers/api/v1/points_controller.rb | 3 ++ app/controllers/map_controller.rb | 1 - app/jobs/points/create_job.rb | 11 ++++--- app/models/point.rb | 6 +++- .../20250120154554_remove_duplicate_points.rb | 31 +++++++++++++++++++ db/data_schema.rb | 2 +- ...250120154555_add_unique_index_to_points.rb | 16 ++++++++++ db/schema.rb | 3 +- spec/factories/points.rb | 4 +++ spec/factories/trips.rb | 14 ++++++--- .../files/geojson/export_same_points.json | 2 +- spec/jobs/bulk_stats_calculating_job_spec.rb | 13 ++++++-- spec/models/import_spec.rb | 6 +++- spec/models/stat_spec.rb | 10 ++++-- spec/models/user_spec.rb | 6 +++- spec/requests/api/v1/points_spec.rb | 18 ++++++----- spec/requests/api/v1/stats_spec.rb | 12 +++++-- spec/requests/exports_spec.rb | 6 +++- spec/requests/map_spec.rb | 6 +++- spec/serializers/export_serializer_spec.rb | 7 ++++- .../points/geojson_serializer_spec.rb | 7 ++++- .../serializers/points/gpx_serializer_spec.rb | 6 +++- spec/serializers/stats_serializer_spec.rb | 10 ++++-- spec/services/exports/create_spec.rb | 7 ++++- .../google_maps/records_parser_spec.rb | 6 ++-- spec/services/jobs/create_spec.rb | 20 ++++++++++-- spec/swagger/api/v1/points_controller_spec.rb | 6 +++- spec/swagger/api/v1/stats_controller_spec.rb | 14 +++++++-- 28 files changed, 204 insertions(+), 49 deletions(-) create mode 100644 db/data/20250120154554_remove_duplicate_points.rb create mode 100644 db/migrate/20250120154555_add_unique_index_to_points.rb diff --git a/app/controllers/api/v1/points_controller.rb b/app/controllers/api/v1/points_controller.rb index 7905ca68..016358ae 100644 --- a/app/controllers/api/v1/points_controller.rb +++ b/app/controllers/api/v1/points_controller.rb @@ -21,6 +21,9 @@ def index render json: serialized_points end + def create + end + def update point = current_api_user.tracked_points.find(params[:id]) diff --git a/app/controllers/map_controller.rb b/app/controllers/map_controller.rb index 7a7246c5..bad160d5 100644 --- a/app/controllers/map_controller.rb +++ b/app/controllers/map_controller.rb @@ -6,7 +6,6 @@ class MapController < ApplicationController def index @points = points.where('timestamp >= ? AND timestamp <= ?', start_at, end_at) - @countries_and_cities = CountriesAndCities.new(@points).call @coordinates = @points.pluck(:latitude, :longitude, :battery, :altitude, :timestamp, :velocity, :id, :country) .map { [_1.to_f, _2.to_f, _3.to_s, _4.to_s, _5.to_s, _6.to_s, _7.to_s, _8.to_s] } diff --git a/app/jobs/points/create_job.rb b/app/jobs/points/create_job.rb index f079046d..148349fe 100644 --- a/app/jobs/points/create_job.rb +++ b/app/jobs/points/create_job.rb @@ -4,12 +4,13 @@ class Points::CreateJob < ApplicationJob queue_as :default def perform(params, user_id) - data = Overland::Params.new(params).call + data = Points::Params.new(params, user_id).call - data.each do |location| - next if point_exists?(location, user_id) - - Point.create!(location.merge(user_id:)) + data.each_slice(1000) do |location_batch| + Point.upsert_all( + location_batch, + unique_by: %i[latitude longitude timestamp user_id] + ) end end diff --git a/app/models/point.rb b/app/models/point.rb index 040e6d41..f28b8043 100644 --- a/app/models/point.rb +++ b/app/models/point.rb @@ -8,7 +8,11 @@ class Point < ApplicationRecord belongs_to :user validates :latitude, :longitude, :timestamp, presence: true - + validates :timestamp, uniqueness: { + scope: %i[latitude longitude user_id], + message: 'already has a point at this location and time for this user', + index: true + } enum :battery_status, { unknown: 0, unplugged: 1, charging: 2, full: 3 }, suffix: true enum :trigger, { unknown: 0, background_event: 1, circular_region_event: 2, beacon_event: 3, diff --git a/db/data/20250120154554_remove_duplicate_points.rb b/db/data/20250120154554_remove_duplicate_points.rb new file mode 100644 index 00000000..2eaa2e4c --- /dev/null +++ b/db/data/20250120154554_remove_duplicate_points.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +class RemoveDuplicatePoints < ActiveRecord::Migration[8.0] + def up + # Find duplicate groups using a subquery + duplicate_groups = + Point.select('latitude, longitude, timestamp, user_id, COUNT(*) as count') + .group('latitude, longitude, timestamp, user_id') + .having('COUNT(*) > 1') + + puts "Duplicate groups found: #{duplicate_groups.length}" + + duplicate_groups.each do |group| + points = Point.where( + latitude: group.latitude, + longitude: group.longitude, + timestamp: group.timestamp, + user_id: group.user_id + ).order(id: :asc) + + # Keep the latest record and destroy all others + latest = points.last + points.where.not(id: latest.id).destroy_all + end + end + + def down + # This migration cannot be reversed + raise ActiveRecord::IrreversibleMigration + end +end diff --git a/db/data_schema.rb b/db/data_schema.rb index 222b8d11..56adf2dc 100644 --- a/db/data_schema.rb +++ b/db/data_schema.rb @@ -1 +1 @@ -DataMigrate::Data.define(version: 20250104204852) +DataMigrate::Data.define(version: 20250120154554) diff --git a/db/migrate/20250120154555_add_unique_index_to_points.rb b/db/migrate/20250120154555_add_unique_index_to_points.rb new file mode 100644 index 00000000..fc224ab0 --- /dev/null +++ b/db/migrate/20250120154555_add_unique_index_to_points.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class AddUniqueIndexToPoints < ActiveRecord::Migration[8.0] + disable_ddl_transaction! + + def up + add_index :points, %i[latitude longitude timestamp user_id], + unique: true, + name: 'unique_points_index', + algorithm: :concurrently + end + + def down + remove_index :points, name: 'unique_points_index' + end +end diff --git a/db/schema.rb b/db/schema.rb index 31ce24e6..8fc8554c 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2025_01_20_152540) do +ActiveRecord::Schema[8.0].define(version: 2025_01_20_154555) do # These are extensions that must be enabled in order to support this database enable_extension "pg_catalog.plpgsql" @@ -168,6 +168,7 @@ t.index ["external_track_id"], name: "index_points_on_external_track_id" t.index ["geodata"], name: "index_points_on_geodata", using: :gin t.index ["import_id"], name: "index_points_on_import_id" + t.index ["latitude", "longitude", "timestamp", "user_id"], name: "unique_points_index", unique: true t.index ["latitude", "longitude"], name: "index_points_on_latitude_and_longitude" t.index ["reverse_geocoded_at"], name: "index_points_on_reverse_geocoded_at" t.index ["timestamp"], name: "index_points_on_timestamp" diff --git a/spec/factories/points.rb b/spec/factories/points.rb index 6ae12ab2..2288a07d 100644 --- a/spec/factories/points.rb +++ b/spec/factories/points.rb @@ -25,6 +25,10 @@ import_id { '' } city { nil } country { nil } + reverse_geocoded_at { nil } + course { nil } + course_accuracy { nil } + external_track_id { nil } user trait :with_known_location do diff --git a/spec/factories/trips.rb b/spec/factories/trips.rb index 237a187b..4ef4041a 100644 --- a/spec/factories/trips.rb +++ b/spec/factories/trips.rb @@ -10,11 +10,15 @@ trait :with_points do after(:build) do |trip| - create_list( - :point, 25, - user: trip.user, - timestamp: trip.started_at + (1..1000).to_a.sample.minutes - ) + (1..25).map do |i| + create( + :point, + :with_geodata, + :reverse_geocoded, + timestamp: trip.started_at + i.minutes, + user: trip.user + ) + end end end end diff --git a/spec/fixtures/files/geojson/export_same_points.json b/spec/fixtures/files/geojson/export_same_points.json index 7a20a47f..f3961b32 100644 --- a/spec/fixtures/files/geojson/export_same_points.json +++ b/spec/fixtures/files/geojson/export_same_points.json @@ -1 +1 @@ -{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":"2021-01-01T00:00:00.000+01:00"}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":"2021-01-01T00:00:00.000+01:00"}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":"2021-01-01T00:00:00.000+01:00"}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":"2021-01-01T00:00:00.000+01:00"}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":"2021-01-01T00:00:00.000+01:00"}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":"2021-01-01T00:00:00.000+01:00"}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":"2021-01-01T00:00:00.000+01:00"}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":"2021-01-01T00:00:00.000+01:00"}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":"2021-01-01T00:00:00.000+01:00"}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":"2021-01-01T00:00:00.000+01:00"}}]} +{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459200,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":"2021-01-01T00:00:00.000+01:00","course":null,"course_accuracy":null,"external_track_id":null}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459201,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":"2021-01-01T00:00:00.000+01:00","course":null,"course_accuracy":null,"external_track_id":null}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459202,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":"2021-01-01T00:00:00.000+01:00","course":null,"course_accuracy":null,"external_track_id":null}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459203,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":"2021-01-01T00:00:00.000+01:00","course":null,"course_accuracy":null,"external_track_id":null}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459204,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":"2021-01-01T00:00:00.000+01:00","course":null,"course_accuracy":null,"external_track_id":null}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459205,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":"2021-01-01T00:00:00.000+01:00","course":null,"course_accuracy":null,"external_track_id":null}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459206,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":"2021-01-01T00:00:00.000+01:00","course":null,"course_accuracy":null,"external_track_id":null}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459207,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":"2021-01-01T00:00:00.000+01:00","course":null,"course_accuracy":null,"external_track_id":null}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459208,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":"2021-01-01T00:00:00.000+01:00","course":null,"course_accuracy":null,"external_track_id":null}},{"type":"Feature","geometry":{"type":"Point","coordinates":["37.6173","55.755826"]},"properties":{"battery_status":"unplugged","ping":"MyString","battery":1,"tracker_id":"MyString","topic":"MyString","altitude":1,"longitude":"37.6173","velocity":"0","trigger":"background_event","bssid":"MyString","ssid":"MyString","connection":"wifi","vertical_accuracy":1,"accuracy":1,"timestamp":1609459209,"latitude":"55.755826","mode":1,"inrids":[],"in_regions":[],"city":null,"country":null,"geodata":{},"reverse_geocoded_at":"2021-01-01T00:00:00.000+01:00","course":null,"course_accuracy":null,"external_track_id":null}}]} diff --git a/spec/jobs/bulk_stats_calculating_job_spec.rb b/spec/jobs/bulk_stats_calculating_job_spec.rb index 15bbc9fb..632fa47e 100644 --- a/spec/jobs/bulk_stats_calculating_job_spec.rb +++ b/spec/jobs/bulk_stats_calculating_job_spec.rb @@ -9,8 +9,17 @@ let(:timestamp) { DateTime.new(2024, 1, 1).to_i } - let!(:points1) { create_list(:point, 10, user_id: user1.id, timestamp:) } - let!(:points2) { create_list(:point, 10, user_id: user2.id, timestamp:) } + let!(:points1) do + (1..10).map do |i| + create(:point, user_id: user1.id, timestamp: timestamp + i.minutes) + end + end + + let!(:points2) do + (1..10).map do |i| + create(:point, user_id: user2.id, timestamp: timestamp + i.minutes) + end + end it 'enqueues Stats::CalculatingJob for each user' do expect(Stats::CalculatingJob).to receive(:perform_later).with(user1.id, 2024, 1) diff --git a/spec/models/import_spec.rb b/spec/models/import_spec.rb index d6c7efc8..8b682409 100644 --- a/spec/models/import_spec.rb +++ b/spec/models/import_spec.rb @@ -26,7 +26,11 @@ describe '#years_and_months_tracked' do let(:import) { create(:import) } let(:timestamp) { Time.zone.local(2024, 11, 1) } - let!(:points) { create_list(:point, 3, import:, timestamp:) } + let!(:points) do + (1..3).map do |i| + create(:point, import:, timestamp: timestamp + i.minutes) + end + end it 'returns years and months tracked' do expect(import.years_and_months_tracked).to eq([[2024, 11]]) diff --git a/spec/models/stat_spec.rb b/spec/models/stat_spec.rb index af8873b6..1208e006 100644 --- a/spec/models/stat_spec.rb +++ b/spec/models/stat_spec.rb @@ -89,8 +89,14 @@ subject { stat.points.to_a } let(:stat) { create(:stat, year:, month: 1, user:) } - let(:timestamp) { DateTime.new(year, 1, 1, 5, 0, 0) } - let!(:points) { create_list(:point, 3, user:, timestamp:) } + let(:base_timestamp) { DateTime.new(year, 1, 1, 5, 0, 0) } + let!(:points) do + [ + create(:point, user:, timestamp: base_timestamp), + create(:point, user:, timestamp: base_timestamp + 1.hour), + create(:point, user:, timestamp: base_timestamp + 2.hours) + ] + end it 'returns points' do expect(subject).to eq(points) diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index a1059d0a..398e436f 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -115,7 +115,11 @@ end describe '#years_tracked' do - let!(:points) { create_list(:point, 3, user:, timestamp: DateTime.new(2024, 1, 1, 5, 0, 0)) } + let!(:points) do + (1..3).map do |i| + create(:point, user:, timestamp: DateTime.new(2024, 1, 1, 5, 0, 0) + i.minutes) + end + end it 'returns years tracked' do expect(user.years_tracked).to eq([{ year: 2024, months: ['Jan'] }]) diff --git a/spec/requests/api/v1/points_spec.rb b/spec/requests/api/v1/points_spec.rb index 5120e5ce..3d5f49d8 100644 --- a/spec/requests/api/v1/points_spec.rb +++ b/spec/requests/api/v1/points_spec.rb @@ -4,7 +4,11 @@ RSpec.describe 'Api::V1::Points', type: :request do let!(:user) { create(:user) } - let!(:points) { create_list(:point, 150, user:) } + let!(:points) do + (1..15).map do |i| + create(:point, user:, timestamp: 1.day.ago + i.minutes) + end + end describe 'GET /index' do context 'when regular version of points is requested' do @@ -21,7 +25,7 @@ json_response = JSON.parse(response.body) - expect(json_response.size).to eq(100) + expect(json_response.size).to eq(15) end it 'returns a list of points with pagination' do @@ -31,7 +35,7 @@ json_response = JSON.parse(response.body) - expect(json_response.size).to eq(10) + expect(json_response.size).to eq(5) end it 'returns a list of points with pagination headers' do @@ -40,7 +44,7 @@ expect(response).to have_http_status(:ok) expect(response.headers['X-Current-Page']).to eq('2') - expect(response.headers['X-Total-Pages']).to eq('15') + expect(response.headers['X-Total-Pages']).to eq('2') end end @@ -58,7 +62,7 @@ json_response = JSON.parse(response.body) - expect(json_response.size).to eq(100) + expect(json_response.size).to eq(15) end it 'returns a list of points with pagination' do @@ -68,7 +72,7 @@ json_response = JSON.parse(response.body) - expect(json_response.size).to eq(10) + expect(json_response.size).to eq(5) end it 'returns a list of points with pagination headers' do @@ -77,7 +81,7 @@ expect(response).to have_http_status(:ok) expect(response.headers['X-Current-Page']).to eq('2') - expect(response.headers['X-Total-Pages']).to eq('15') + expect(response.headers['X-Total-Pages']).to eq('2') end it 'returns a list of points with slim attributes' do diff --git a/spec/requests/api/v1/stats_spec.rb b/spec/requests/api/v1/stats_spec.rb index d733ae3f..89cdc8e4 100644 --- a/spec/requests/api/v1/stats_spec.rb +++ b/spec/requests/api/v1/stats_spec.rb @@ -10,14 +10,20 @@ let!(:stats_in_2020) { create_list(:stat, 12, year: 2020, user:) } let!(:stats_in_2021) { create_list(:stat, 12, year: 2021, user:) } let!(:points_in_2020) do - create_list(:point, 85, :with_geodata, :reverse_geocoded, timestamp: Time.zone.local(2020), user:) + (1..85).map do |i| + create(:point, :with_geodata, :reverse_geocoded, timestamp: Time.zone.local(2020, 1, 1).to_i + i.hours, user:) + end + end + let!(:points_in_2021) do + (1..95).map do |i| + create(:point, :with_geodata, :reverse_geocoded, timestamp: Time.zone.local(2021, 1, 1).to_i + i.hours, user:) + end end - let!(:points_in_2021) { create_list(:point, 95, timestamp: Time.zone.local(2021), user:) } let(:expected_json) do { totalDistanceKm: stats_in_2020.map(&:distance).sum + stats_in_2021.map(&:distance).sum, totalPointsTracked: points_in_2020.count + points_in_2021.count, - totalReverseGeocodedPoints: points_in_2020.count, + totalReverseGeocodedPoints: points_in_2020.count + points_in_2021.count, totalCountriesVisited: 1, totalCitiesVisited: 1, yearlyStats: [ diff --git a/spec/requests/exports_spec.rb b/spec/requests/exports_spec.rb index c96ac744..0ec6fa61 100644 --- a/spec/requests/exports_spec.rb +++ b/spec/requests/exports_spec.rb @@ -37,7 +37,11 @@ before { sign_in user } context 'with valid parameters' do - let(:points) { create_list(:point, 10, user:, timestamp: 1.day.ago) } + let(:points) do + (1..10).map do |i| + create(:point, user:, timestamp: 1.day.ago + i.minutes) + end + end it 'creates a new Export' do expect { post exports_url, params: }.to change(Export, :count).by(1) diff --git a/spec/requests/map_spec.rb b/spec/requests/map_spec.rb index 3cda64a5..700a214a 100644 --- a/spec/requests/map_spec.rb +++ b/spec/requests/map_spec.rb @@ -11,7 +11,11 @@ describe 'GET /index' do context 'when user signed in' do let(:user) { create(:user) } - let(:points) { create_list(:point, 10, user:, timestamp: 1.day.ago) } + let(:points) do + (1..10).map do |i| + create(:point, user:, timestamp: 1.day.ago + i.minutes) + end + end before { sign_in user } diff --git a/spec/serializers/export_serializer_spec.rb b/spec/serializers/export_serializer_spec.rb index e77acff5..353d53fb 100644 --- a/spec/serializers/export_serializer_spec.rb +++ b/spec/serializers/export_serializer_spec.rb @@ -7,7 +7,12 @@ subject(:serializer) { described_class.new(points, user_email).call } let(:user_email) { 'ab@cd.com' } - let(:points) { create_list(:point, 2) } + let(:points) do + (1..2).map do |i| + create(:point, timestamp: 1.day.ago + i.minutes) + end + end + let(:expected_json) do { user_email => { diff --git a/spec/serializers/points/geojson_serializer_spec.rb b/spec/serializers/points/geojson_serializer_spec.rb index a532a192..e125c7b3 100644 --- a/spec/serializers/points/geojson_serializer_spec.rb +++ b/spec/serializers/points/geojson_serializer_spec.rb @@ -6,7 +6,12 @@ describe '#call' do subject(:serializer) { described_class.new(points).call } - let(:points) { create_list(:point, 3) } + let(:points) do + (1..3).map do |i| + create(:point, timestamp: 1.day.ago + i.minutes) + end + end + let(:expected_json) do { type: 'FeatureCollection', diff --git a/spec/serializers/points/gpx_serializer_spec.rb b/spec/serializers/points/gpx_serializer_spec.rb index e2b108b9..1434ca5d 100644 --- a/spec/serializers/points/gpx_serializer_spec.rb +++ b/spec/serializers/points/gpx_serializer_spec.rb @@ -6,7 +6,11 @@ describe '#call' do subject(:serializer) { described_class.new(points, 'some_name').call } - let(:points) { create_list(:point, 3) } + let(:points) do + (1..3).map do |i| + create(:point, timestamp: 1.day.ago + i.minutes) + end + end it 'returns GPX file' do expect(serializer).to be_a(GPX::GPXFile) diff --git a/spec/serializers/stats_serializer_spec.rb b/spec/serializers/stats_serializer_spec.rb index ad6f5bc8..2fba6656 100644 --- a/spec/serializers/stats_serializer_spec.rb +++ b/spec/serializers/stats_serializer_spec.rb @@ -29,16 +29,20 @@ let!(:stats_in_2020) { create_list(:stat, 12, year: 2020, user:) } let!(:stats_in_2021) { create_list(:stat, 12, year: 2021, user:) } let!(:points_in_2020) do - create_list(:point, 85, :with_geodata, :reverse_geocoded, timestamp: Time.zone.local(2020), user:) + (1..85).map do |i| + create(:point, :with_geodata, :reverse_geocoded, timestamp: Time.zone.local(2020, 1, 1).to_i + i.hours, user:) + end end let!(:points_in_2021) do - create_list(:point, 95, timestamp: Time.zone.local(2021), user:) + (1..95).map do |i| + create(:point, :with_geodata, :reverse_geocoded, timestamp: Time.zone.local(2021, 1, 1).to_i + i.hours, user:) + end end let(:expected_json) do { "totalDistanceKm": stats_in_2020.map(&:distance).sum + stats_in_2021.map(&:distance).sum, "totalPointsTracked": points_in_2020.count + points_in_2021.count, - "totalReverseGeocodedPoints": points_in_2020.count, + "totalReverseGeocodedPoints": points_in_2020.count + points_in_2021.count, "totalCountriesVisited": 1, "totalCitiesVisited": 1, "yearlyStats": [ diff --git a/spec/services/exports/create_spec.rb b/spec/services/exports/create_spec.rb index 2110b6b0..1bea40d2 100644 --- a/spec/services/exports/create_spec.rb +++ b/spec/services/exports/create_spec.rb @@ -15,7 +15,12 @@ let(:export_content) { Points::GeojsonSerializer.new(points).call } let(:reverse_geocoded_at) { Time.zone.local(2021, 1, 1) } let!(:points) do - create_list(:point, 10, :with_known_location, user:, timestamp: start_at.to_datetime.to_i, reverse_geocoded_at:) + 10.times.map do |i| + create(:point, :with_known_location, + user: user, + timestamp: start_at.to_datetime.to_i + i, + reverse_geocoded_at: reverse_geocoded_at) + end end before do diff --git a/spec/services/google_maps/records_parser_spec.rb b/spec/services/google_maps/records_parser_spec.rb index 44ec23b6..96495dad 100644 --- a/spec/services/google_maps/records_parser_spec.rb +++ b/spec/services/google_maps/records_parser_spec.rb @@ -7,7 +7,7 @@ subject(:parser) { described_class.new(import).call(json) } let(:import) { create(:import) } - let(:time) { Time.zone.now } + let(:time) { DateTime.new(2025, 1, 1, 12, 0, 0) } let(:json) do { 'latitudeE7' => 123_456_789, @@ -31,7 +31,7 @@ before do create( :point, user: import.user, import:, latitude: 12.3456789, longitude: 12.3456789, - timestamp: Time.zone.now.to_i + timestamp: time.to_i ) end @@ -78,4 +78,4 @@ end end end -end \ No newline at end of file +end diff --git a/spec/services/jobs/create_spec.rb b/spec/services/jobs/create_spec.rb index cc482b67..84988ff3 100644 --- a/spec/services/jobs/create_spec.rb +++ b/spec/services/jobs/create_spec.rb @@ -8,7 +8,12 @@ context 'when job_name is start_reverse_geocoding' do let(:user) { create(:user) } - let(:points) { create_list(:point, 4, user:) } + let(:points) do + (1..4).map do |i| + create(:point, user:, timestamp: 1.day.ago + i.minutes) + end + end + let(:job_name) { 'start_reverse_geocoding' } it 'enqueues reverse geocoding for all user points' do @@ -24,8 +29,17 @@ context 'when job_name is continue_reverse_geocoding' do let(:user) { create(:user) } - let(:points_without_address) { create_list(:point, 4, user:, country: nil, city: nil) } - let(:points_with_address) { create_list(:point, 5, user:, country: 'Country', city: 'City') } + let(:points_without_address) do + (1..4).map do |i| + create(:point, user:, country: nil, city: nil, timestamp: 1.day.ago + i.minutes) + end + end + + let(:points_with_address) do + (1..5).map do |i| + create(:point, user:, country: 'Country', city: 'City', timestamp: 1.day.ago + i.minutes) + end + end let(:job_name) { 'continue_reverse_geocoding' } diff --git a/spec/swagger/api/v1/points_controller_spec.rb b/spec/swagger/api/v1/points_controller_spec.rb index cbc31e6d..d4ff924c 100644 --- a/spec/swagger/api/v1/points_controller_spec.rb +++ b/spec/swagger/api/v1/points_controller_spec.rb @@ -58,7 +58,11 @@ let(:api_key) { user.api_key } let(:start_at) { Time.zone.now - 1.day } let(:end_at) { Time.zone.now } - let(:points) { create_list(:point, 10, user:, timestamp: 2.hours.ago) } + let(:points) do + (1..10).map do |i| + create(:point, user:, timestamp: 2.hours.ago + i.minutes) + end + end run_test! end diff --git a/spec/swagger/api/v1/stats_controller_spec.rb b/spec/swagger/api/v1/stats_controller_spec.rb index a6a49c0f..b1fda703 100644 --- a/spec/swagger/api/v1/stats_controller_spec.rb +++ b/spec/swagger/api/v1/stats_controller_spec.rb @@ -57,8 +57,18 @@ let!(:user) { create(:user) } let!(:stats_in_2020) { create_list(:stat, 12, year: 2020, user:) } let!(:stats_in_2021) { create_list(:stat, 12, year: 2021, user:) } - let!(:points_in_2020) { create_list(:point, 85, :with_geodata, timestamp: Time.zone.local(2020), user:) } - let!(:points_in_2021) { create_list(:point, 95, timestamp: Time.zone.local(2021), user:) } + let!(:points_in_2020) do + (1..85).map do |i| + create(:point, :with_geodata, :reverse_geocoded, timestamp: Time.zone.local(2020, 1, 1).to_i + i.hours, +user:) + end + end + let!(:points_in_2021) do + (1..95).map do |i| + create(:point, :with_geodata, :reverse_geocoded, timestamp: Time.zone.local(2021, 1, 1).to_i + i.hours, +user:) + end + end let(:api_key) { user.api_key } run_test! From 983768a5726fc134ae7cbe21feb1a68909b8d4a4 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Mon, 20 Jan 2025 20:07:46 +0100 Subject: [PATCH 3/8] Assign user_id to points on parsing --- app/controllers/api/v1/points_controller.rb | 7 +++++++ app/services/points/params.rb | 16 ++++++++++++---- spec/jobs/points/create_job_spec.rb | 15 ++++++++++++++- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/v1/points_controller.rb b/app/controllers/api/v1/points_controller.rb index 016358ae..f09340b8 100644 --- a/app/controllers/api/v1/points_controller.rb +++ b/app/controllers/api/v1/points_controller.rb @@ -22,6 +22,9 @@ def index end def create + Points::CreateJob.perform_later(batch_params, current_api_user.id) + + render json: { message: 'Points are being processed' } end def update @@ -45,6 +48,10 @@ def point_params params.require(:point).permit(:latitude, :longitude) end + def batch_params + params.permit(locations: [:type, { geometry: {}, properties: {} }], batch: {}) + end + def point_serializer params[:slim] == 'true' ? Api::SlimPointSerializer : Api::PointSerializer end diff --git a/app/services/points/params.rb b/app/services/points/params.rb index 1e2873ca..8c1b8a51 100644 --- a/app/services/points/params.rb +++ b/app/services/points/params.rb @@ -1,16 +1,17 @@ # frozen_string_literal: true class Points::Params - attr_reader :data, :points + attr_reader :data, :points, :user_id - def initialize(json) + def initialize(json, user_id) @data = json.with_indifferent_access @points = @data[:locations] + @user_id = user_id end def call points.map do |point| - next if point[:geometry].nil? || point.dig(:properties, :timestamp).nil? + next unless params_valid?(point) { latitude: point[:geometry][:coordinates][1], @@ -26,7 +27,8 @@ def call vertical_accuracy: point[:properties][:vertical_accuracy], course_accuracy: point[:properties][:course_accuracy], course: point[:properties][:course], - raw_data: point + raw_data: point, + user_id: user_id } end.compact end @@ -38,4 +40,10 @@ def battery_level(level) value.positive? ? value : nil end + + def params_valid?(point) + point[:geometry].present? && + point[:geometry][:coordinates].present? && + point.dig(:properties, :timestamp).present? + end end diff --git a/spec/jobs/points/create_job_spec.rb b/spec/jobs/points/create_job_spec.rb index 70baa6e5..7fa14b15 100644 --- a/spec/jobs/points/create_job_spec.rb +++ b/spec/jobs/points/create_job_spec.rb @@ -1,5 +1,18 @@ +# frozen_string_literal: true + require 'rails_helper' RSpec.describe Points::CreateJob, type: :job do - pending "add some examples to (or delete) #{__FILE__}" + describe '#perform' do + subject(:perform) { described_class.new.perform(json, user.id) } + + let(:file_path) { 'spec/fixtures/files/points/geojson_example.json' } + let(:file) { File.open(file_path) } + let(:json) { JSON.parse(file.read) } + let(:user) { create(:user) } + + it 'creates a point' do + expect { perform }.to change { Point.count }.by(6) + end + end end From 112f13587ce986b5e1a366277bb61972a8070b61 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Mon, 20 Jan 2025 20:17:27 +0100 Subject: [PATCH 4/8] Add swagger docs for POST /api/v1/points --- .app_version | 2 +- CHANGELOG.md | 6 ++ config/routes.rb | 2 +- spec/swagger/api/v1/points_controller_spec.rb | 81 +++++++++++++++++++ swagger/v1/swagger.yaml | 81 +++++++++++++++++++ 5 files changed, 170 insertions(+), 2 deletions(-) diff --git a/.app_version b/.app_version index 4240544f..03035cdd 100644 --- a/.app_version +++ b/.app_version @@ -1 +1 @@ -0.22.4 +0.22.5 diff --git a/CHANGELOG.md b/CHANGELOG.md index ecc13f47..f2622d79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +# 0.22.5 - 2025-01-20 + +### Added + +- `POST /api/v1/points/create` endpoint added to create points from a file. + # 0.22.4 - 2025-01-20 ### Added diff --git a/config/routes.rb b/config/routes.rb index 0befcca4..9e1384a9 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -67,7 +67,7 @@ get 'settings', to: 'settings#index' resources :areas, only: %i[index create update destroy] - resources :points, only: %i[index destroy update] + resources :points, only: %i[index create update destroy] resources :visits, only: %i[update] resources :stats, only: :index diff --git a/spec/swagger/api/v1/points_controller_spec.rb b/spec/swagger/api/v1/points_controller_spec.rb index d4ff924c..d3dc087c 100644 --- a/spec/swagger/api/v1/points_controller_spec.rb +++ b/spec/swagger/api/v1/points_controller_spec.rb @@ -67,6 +67,87 @@ run_test! end end + + post 'Creates a batch of points' do + request_body_example value: { + locations: [ + { + type: 'Feature', + geometry: { + type: 'Point', + coordinates: [-122.40530871, 37.74430413] + }, + properties: { + timestamp: '2025-01-17T21:03:01Z', + horizontal_accuracy: 5, + vertical_accuracy: -1, + altitude: 0, + speed: 92.088, + speed_accuracy: 0, + course: 27.07, + course_accuracy: 0, + track_id: '799F32F5-89BB-45FB-A639-098B1B95B09F', + device_id: '8D5D4197-245B-4619-A88B-2049100ADE46' + } + } + ] + } + tags 'Batches' + consumes 'application/json' + parameter name: :locations, in: :body, schema: { + type: :object, + properties: { + type: { type: :string }, + geometry: { + type: :object, + properties: { + type: { type: :string }, + coordinates: { type: :array, items: { type: :number } } + } + }, + properties: { + type: :object, + properties: { + timestamp: { type: :string }, + horizontal_accuracy: { type: :number }, + vertical_accuracy: { type: :number }, + altitude: { type: :number }, + speed: { type: :number }, + speed_accuracy: { type: :number }, + course: { type: :number }, + course_accuracy: { type: :number }, + track_id: { type: :string }, + device_id: { type: :string } + } + } + }, + required: %w[geometry properties] + } + + parameter name: :api_key, in: :query, type: :string, required: true, description: 'API Key' + + response '200', 'Batch of points being processed' do + let(:file_path) { 'spec/fixtures/files/points/geojson_example.json' } + let(:file) { File.open(file_path) } + let(:json) { JSON.parse(file.read) } + let(:params) { json } + let(:locations) { params['locations'] } + let(:api_key) { create(:user).api_key } + + run_test! + end + + response '401', 'Unauthorized' do + let(:file_path) { 'spec/fixtures/files/points/geojson_example.json' } + let(:file) { File.open(file_path) } + let(:json) { JSON.parse(file.read) } + let(:params) { json } + let(:locations) { params['locations'] } + let(:api_key) { 'invalid_api_key' } + + run_test! + end + end end path '/api/v1/points/{id}' do diff --git a/swagger/v1/swagger.yaml b/swagger/v1/swagger.yaml index beed0840..d40786d2 100644 --- a/swagger/v1/swagger.yaml +++ b/swagger/v1/swagger.yaml @@ -696,6 +696,87 @@ paths: type: string visit_id: type: string + post: + summary: Creates a batch of points + tags: + - Batches + parameters: + - name: api_key + in: query + required: true + description: API Key + schema: + type: string + responses: + '200': + description: Batch of points being processed + '401': + description: Unauthorized + requestBody: + content: + application/json: + schema: + type: object + properties: + type: + type: string + geometry: + type: object + properties: + type: + type: string + coordinates: + type: array + items: + type: number + properties: + type: object + properties: + timestamp: + type: string + horizontal_accuracy: + type: number + vertical_accuracy: + type: number + altitude: + type: number + speed: + type: number + speed_accuracy: + type: number + course: + type: number + course_accuracy: + type: number + track_id: + type: string + device_id: + type: string + required: + - geometry + - properties + examples: + '0': + summary: Creates a batch of points + value: + locations: + - type: Feature + geometry: + type: Point + coordinates: + - -122.40530871 + - 37.74430413 + properties: + timestamp: '2025-01-17T21:03:01Z' + horizontal_accuracy: 5 + vertical_accuracy: -1 + altitude: 0 + speed: 92.088 + speed_accuracy: 0 + course: 27.07 + course_accuracy: 0 + track_id: 799F32F5-89BB-45FB-A639-098B1B95B09F + device_id: 8D5D4197-245B-4619-A88B-2049100ADE46 "/api/v1/points/{id}": delete: summary: Deletes a point From 4e49e67fe521d4e6b1f747385b31e224ba0ed8cb Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Mon, 20 Jan 2025 20:22:47 +0100 Subject: [PATCH 5/8] Update changelog and app version --- .app_version | 2 +- CHANGELOG.md | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.app_version b/.app_version index 03035cdd..ca222b7c 100644 --- a/.app_version +++ b/.app_version @@ -1 +1 @@ -0.22.5 +0.23.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index f2622d79..332291f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -# 0.22.5 - 2025-01-20 +# 0.23.0 - 2025-01-20 + +⚠️ IMPORTANT ⚠️ + +This release includes a data migration to remove duplicated points from the database. It will not remove anything except for duplcates from the `points` table, but please make sure to create a backup before updating to this version. ### Added - `POST /api/v1/points/create` endpoint added to create points from a file. +- An index to guarantee uniqueness of points across `latitude`, `longitude`, `timestamp` and `user_id` values. This is introduced to make sure no duplicates will be created in the database in addition to previously existing validations. # 0.22.4 - 2025-01-20 From 85e38cae21784bbb5a04caa71d40510a1f71e1a4 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Mon, 20 Jan 2025 20:23:57 +0100 Subject: [PATCH 6/8] Provide link to a back up instructions --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 332291f1..f8566e4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/). # 0.23.0 - 2025-01-20 -⚠️ IMPORTANT ⚠️ +## ⚠️ IMPORTANT ⚠️ -This release includes a data migration to remove duplicated points from the database. It will not remove anything except for duplcates from the `points` table, but please make sure to create a backup before updating to this version. +This release includes a data migration to remove duplicated points from the database. It will not remove anything except for duplcates from the `points` table, but please make sure to create a [backup](https://dawarich.app/docs/tutorials/backup-and-restore) before updating to this version. ### Added From a311325c828635a76ae1b53d02106b1446864f56 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Mon, 20 Jan 2025 20:41:26 +0100 Subject: [PATCH 7/8] Fix failed tests --- app/jobs/points/create_job.rb | 14 ++------------ spec/services/points/params_spec.rb | 6 ++++-- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/app/jobs/points/create_job.rb b/app/jobs/points/create_job.rb index 148349fe..964c50f7 100644 --- a/app/jobs/points/create_job.rb +++ b/app/jobs/points/create_job.rb @@ -9,19 +9,9 @@ def perform(params, user_id) data.each_slice(1000) do |location_batch| Point.upsert_all( location_batch, - unique_by: %i[latitude longitude timestamp user_id] + unique_by: %i[latitude longitude timestamp user_id], + returning: false ) end end - - private - - def point_exists?(params, user_id) - Point.exists?( - latitude: params[:latitude], - longitude: params[:longitude], - timestamp: params[:timestamp], - user_id: - ) - end end diff --git a/spec/services/points/params_spec.rb b/spec/services/points/params_spec.rb index 6fb3f486..62f9b82b 100644 --- a/spec/services/points/params_spec.rb +++ b/spec/services/points/params_spec.rb @@ -4,6 +4,7 @@ RSpec.describe Points::Params do describe '#call' do + let(:user) { create(:user) } let(:file_path) { 'spec/fixtures/files/points/geojson_example.json' } let(:file) { File.open(file_path) } let(:json) { JSON.parse(file.read) } @@ -40,11 +41,12 @@ timestamp: '2025-01-17T21:03:01Z', device_id: '8D5D4197-245B-4619-A88B-2049100ADE46' } - }.with_indifferent_access + }.with_indifferent_access, + user_id: user.id } end - subject(:params) { described_class.new(json).call } + subject(:params) { described_class.new(json, user.id).call } it 'returns an array of points' do expect(params).to be_an(Array) From 1d820462f6c8667726803d9e1ba05dff4862ff87 Mon Sep 17 00:00:00 2001 From: Eugene Burmakin Date: Mon, 20 Jan 2025 20:42:15 +0100 Subject: [PATCH 8/8] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b394bef0..2f4efd0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ This release includes a data migration to remove duplicated points from the data ### Added -- `POST /api/v1/points/create` endpoint added to create points from a file. +- `POST /api/v1/points/create` endpoint added. - An index to guarantee uniqueness of points across `latitude`, `longitude`, `timestamp` and `user_id` values. This is introduced to make sure no duplicates will be created in the database in addition to previously existing validations. - `GET /api/v1/users/me` endpoint added to get current user.