Restore Changes to load_mongodump.sh and Sync with em-public-dashboard

docker/load_mongodump.sh

@@ -1,10 +1,74 @@
+#!/bin/bash
+
+# Directory of the script
+SCRIPT_DIR="$(dirname "$0")"
+
+# Path to the configuration file (one level up)
+CONFIG_FILE="$SCRIPT_DIR/../docker-compose-dev.yml"
+
+# Check if the correct number of arguments is provided
+if [ "$#" -ne 1 ]; then
+    echo "Usage: $0 <mongodump-file>"
+    echo "  <mongodump-file> : The path to the MongoDB dump file to be restored."
+    exit 1
+fi
+
 MONGODUMP_FILE=$1
 
-echo "Copying file to docker container"
-docker cp $MONGODUMP_FILE op-admin-dashboard-db-1:/tmp
+# Print debug information
+echo "Script Directory: $SCRIPT_DIR"
+echo "Configuration File Path: $CONFIG_FILE"
+echo "MongoDump File Path: $MONGODUMP_FILE"
+
+# Check if the provided file exists
+if [ ! -f "$MONGODUMP_FILE" ]; then
+    echo "Error: File '$MONGODUMP_FILE' does not exist."
+    exit 1
+fi
+
+# Check if the configuration file exists
+if [ ! -f "$CONFIG_FILE" ]; then
+    echo "Error: Configuration file '$CONFIG_FILE' does not exist."
+    exit 1
+fi
+
+# Print details about the configuration file
+echo "Configuration file details:"
+ls -l "$CONFIG_FILE"
+
+# Extract the database name from the mongodump file
+DB_NAME=$(tar -tf "$MONGODUMP_FILE" | grep '^dump/' | sed 's|^dump/||' | awk -F'/' '{if (NF > 0) {print $1; exit}}')
+
+# Output the database name
+echo "$DB_NAME"
+
+if [ -z "$DB_NAME" ]; then
+    echo "Error: Failed to extract database name from mongodump."
+    exit 1
+fi
+
+echo "Database Name: $DB_NAME"
+
+# Update the docker-compose configuration file with the actual DB_HOST
+DB_HOST="mongodb://db/$DB_NAME"
+sed -i.bak "s|DB_HOST:.*|DB_HOST: $DB_HOST|" "$CONFIG_FILE"
+
+# Update the docker-compose configuration file with the actual STUDY_CONFIG
+STUDY_CONFIG=$(echo "$DB_NAME" | sed -E 's/openpath_prod_(.*)$/\1/' | tr '_' '-')
+sed -i.bak "s|STUDY_CONFIG:.*|STUDY_CONFIG: \"$STUDY_CONFIG\"|" "$CONFIG_FILE"
+
+echo "Updated docker-compose file:"
+cat "$CONFIG_FILE"
+
+echo "Copying file to Docker container"
+docker cp "$MONGODUMP_FILE" op-admin-dashboard-db-1:/tmp
+
+FILE_NAME=$(basename "$MONGODUMP_FILE")
 
-FILE_NAME=`basename $MONGODUMP_FILE`
+echo "Clearing existing database"
+docker exec op-admin-dashboard-db-1 bash -c "mongo $DB_NAME --eval 'db.dropDatabase()'"
 
-echo "Restoring the dump from $FILE_NAME"
-docker exec -e MONGODUMP_FILE=$FILE_NAME op-admin-dashboard-db-1 bash -c 'cd /tmp && tar xvf $MONGODUMP_FILE && mongorestore'
+echo "Restoring the dump from $FILE_NAME to database $DB_NAME"
+docker exec -e MONGODUMP_FILE=$FILE_NAME op-admin-dashboard-db-1 bash -c "cd /tmp && tar xvf $FILE_NAME && mongorestore -d $DB_NAME dump/$DB_NAME"
 
+echo "Database restore complete."

utils/constants.py

@@ -35,7 +35,7 @@
     "inferred_section_summary",
 ]
 
-valid_uuids_columns = [
+VALID_UUIDS_COLS = [
     'user_token',
     'user_id',
     'update_ts',

utils/db_utils.py

@@ -437,57 +437,35 @@ def add_user_stats(user_data, batch_size=5):
         def process_user(user):
             with ect.Timer() as process_user_timer:
                 user_uuid = UUID(user['user_id'])
-
-                # Fetch aggregated data for all users once and cache it
-                ts_aggregate = esta.TimeSeries.get_aggregate_time_series()
-
-                # Fetch data for the user, cached for repeated queries
                 profile_data = edb.get_profile_db().find_one({'user_id': user_uuid})
+                # Fetch data for the user, cached for repeated queries
+                logging.info(f'keyspr: {profile_data}')
+                if not profile_data:
+                    profile_data = {}
+                # Assign existing profile attributes to the user dictionary
+                user['platform'] = profile_data.get('curr_platform')
+                user['manufacturer'] = profile_data.get('manufacturer')
+                user['app_version'] = profile_data.get('client_app_version')
+                user['os_version'] = profile_data.get('client_os_version')
+                user['phone_lang'] = profile_data.get('phone_lang')
 
-                total_trips = ts_aggregate.find_entries_count(
-                    key_list=["analysis/confirmed_trip"],
-                    extra_query_list=[{'user_id': user_uuid}]
-                )
-                labeled_trips = ts_aggregate.find_entries_count(
-                    key_list=["analysis/confirmed_trip"],
-                    extra_query_list=[{'user_id': user_uuid}, {'data.user_input': {'$ne': {}}}]
-                )
+                # Assign newly stored statistics to the user dictionary
+                user['total_trips'] = profile_data.get('total_trips')
+                user['labeled_trips'] = profile_data.get('labeled_trips')
 
-                user['total_trips'] = total_trips
-                user['labeled_trips'] = labeled_trips
-
-                if profile_data:
-                    user['platform'] = profile_data.get('curr_platform')
-                    user['manufacturer'] = profile_data.get('manufacturer')
-                    user['app_version'] = profile_data.get('client_app_version')
-                    user['os_version'] = profile_data.get('client_os_version')
-                    user['phone_lang'] = profile_data.get('phone_lang')
-
-                if total_trips > 0:
-                    ts = esta.TimeSeries.get_time_series(user_uuid)
-                    first_trip_ts = ts.get_first_value_for_field(
-                        key='analysis/confirmed_trip',
-                        field='data.end_ts',
-                        sort_order=pymongo.ASCENDING
-                    )
-                    if first_trip_ts != -1:
-                        user['first_trip'] = arrow.get(first_trip_ts).format(time_format)
-
-                    last_trip_ts = ts.get_first_value_for_field(
-                        key='analysis/confirmed_trip',
-                        field='data.end_ts',
-                        sort_order=pymongo.DESCENDING
-                    )
-                    if last_trip_ts != -1:
-                        user['last_trip'] = arrow.get(last_trip_ts).format(time_format)
-
-                    last_call_ts = ts.get_first_value_for_field(
-                        key='stats/server_api_time',
-                        field='data.ts',
-                        sort_order=pymongo.DESCENDING
-                    )
-                    if last_call_ts != -1:
-                        user['last_call'] = arrow.get(last_call_ts).format(time_format)
+                # Retrieve and assign pipeline range
+                pipeline_range = profile_data.get('pipeline_range', {})
+                start_ts = pipeline_range.get('start_ts')
+                end_ts = pipeline_range.get('end_ts')
+                if start_ts:
+                    user['first_trip'] = arrow.get(start_ts).format(time_format)
+                if end_ts:
+                    user['last_trip'] = arrow.get(end_ts).format(time_format)
+
+                # Retrieve and assign last API call timestamp
+                last_call_ts = profile_data.get('last_call_ts')
+                if last_call_ts:
+                    user['last_call'] = arrow.get(last_call_ts).format('YYYY-MM-DD')
 
             esdsq.store_dashboard_time(
                 "admin/db_utils/add_user_stats/process_user",

utils/permissions.py

@@ -93,7 +93,7 @@ def get_allowed_trip_columns():
 
 
 def get_uuids_columns():
-    columns = set(constants.valid_uuids_columns)
+    columns = set(constants.VALID_UUIDS_COLS)
     for column in permissions.get("data_uuids_columns_exclude", []):
         columns.discard(column)
     return columns