diff --git a/bin/debug/fix_usercache_processing.py b/bin/debug/fix_usercache_processing.py index 7187177e6..0158705b3 100644 --- a/bin/debug/fix_usercache_processing.py +++ b/bin/debug/fix_usercache_processing.py @@ -27,6 +27,7 @@ def fix_usercache_errors(): copy_to_usercache() + print(">" * 30) move_to_long_term() def copy_to_usercache(): @@ -37,8 +38,8 @@ def copy_to_usercache(): logging.info("Found %d errors in this round" % edb.get_timeseries_error_db().estimated_document_count()) for error in error_it: logging.debug("Copying entry %s" % error["metadata"]) - save_result = uc.save(error) - remove_result = te.remove(error["_id"]) + save_result = uc.replace_one({"_id": error['_id']}, error, upsert=True) + remove_result = te.delete_one({"_id": error["_id"]}) logging.debug("save_result = %s, remove_result = %s" % (save_result, remove_result)) logging.info("step copy_to_usercache DONE") diff --git a/emission/core/wrapper/bluetoothble.py b/emission/core/wrapper/bluetoothble.py new file mode 100644 index 000000000..a814e4303 --- /dev/null +++ b/emission/core/wrapper/bluetoothble.py @@ -0,0 +1,29 @@ +import logging +import enum as enum +import emission.core.wrapper.wrapperbase as ecwb + +class BLEEventTypes(enum.Enum): + REGION_ENTER = 0 + REGION_EXIT = 1 + RANGE_UPDATE = 2 + +class Bluetoothble(ecwb.WrapperBase): + props = {"eventType": ecwb.WrapperBase.Access.RO, # the type of event + "uuid": ecwb.WrapperBase.Access.RO, # UUID of the beacon. Will be a constant for beacons used by e-mission, consistent with https://github.com/e-mission/e-mission-docs/issues/1062#issuecomment-2026359038 + "major": ecwb.WrapperBase.Access.RO, # major value (matches deployment) + "minor": ecwb.WrapperBase.Access.RO, # minor value (matches the actual vehicle) + "ts": ecwb.WrapperBase.Access.RO, # timestamp (in seconds) + "proximity": ecwb.WrapperBase.Access.RO, # how close the beacon is (used as the second step in the process, https://github.com/e-mission/e-mission-docs/issues/1062#issuecomment-2026359038 + "local_dt": ecwb.WrapperBase.Access.RO, # searchable datetime in local time + "fmt_time": ecwb.WrapperBase.Access.RO, # formatted time + "accuracy": ecwb.WrapperBase.Access.RO, # only available for range updats + "rssi": ecwb.WrapperBase.Access.RO, # signal strength, only available for range updates + } + + enums = {"eventType": BLEEventTypes} + geojson = [] + nullable = ["major", "minor", "proximity", "accuracy", "rssi"] + local_dates = ['local_dt'] + + def _populateDependencies(self): + pass diff --git a/emission/core/wrapper/entry.py b/emission/core/wrapper/entry.py index b4d8520f7..c6f7b7dd3 100644 --- a/emission/core/wrapper/entry.py +++ b/emission/core/wrapper/entry.py @@ -42,6 +42,8 @@ def _getData2Wrapper(): "background/motion_activity": "motionactivity", # battery readings, to determine power drain empirically "background/battery": "battery", + # BLE events, including enter, exiting and ranging beacons + "background/bluetooth_ble": "bluetoothble", # transition events for the tracking finite state machine on the phone "statemachine/transition": "transition", # phone sensing configuration (e.g. sensing frequency, geofencing,...) diff --git a/emission/core/wrapper/transition.py b/emission/core/wrapper/transition.py index 9c6247272..634f7ac42 100644 --- a/emission/core/wrapper/transition.py +++ b/emission/core/wrapper/transition.py @@ -39,6 +39,8 @@ class TransitionType(enum.Enum): DATA_PUSHED = 16 # joint transition again START_TRACKING = 17 + BLE_BEACON_FOUND = 18 + BLE_BEACON_LOST = 19 class Transition(ecwb.WrapperBase): props = {"curr_state": ecwb.WrapperBase.Access.RO, diff --git a/emission/net/usercache/formatters/android/bluetooth_ble.py b/emission/net/usercache/formatters/android/bluetooth_ble.py new file mode 100644 index 000000000..bf43951eb --- /dev/null +++ b/emission/net/usercache/formatters/android/bluetooth_ble.py @@ -0,0 +1,26 @@ +import logging + +import emission.core.wrapper.bluetoothble as ecwb +import emission.net.usercache.formatters.common as fc +import attrdict as ad + +def format(entry): + formatted_entry = ad.AttrDict() + formatted_entry["_id"] = entry["_id"] + formatted_entry.user_id = entry.user_id + + metadata = entry.metadata + if "time_zone" not in metadata: + metadata.time_zone = "America/Los_Angeles" + fc.expand_metadata_times(metadata) + formatted_entry.metadata = metadata + + #logging.info('*** Motion Data write_ts: %d' % metadata.write_ts) + + data = entry.data + data.local_dt = formatted_entry.metadata.write_local_dt + data.fmt_time = formatted_entry.metadata.write_fmt_time + data.eventType = ecwb.BLEEventTypes[entry.data.eventType].value + formatted_entry.data = data + + return formatted_entry diff --git a/emission/net/usercache/formatters/android/transition.py b/emission/net/usercache/formatters/android/transition.py index 8352c6013..8dd689882 100644 --- a/emission/net/usercache/formatters/android/transition.py +++ b/emission/net/usercache/formatters/android/transition.py @@ -26,6 +26,8 @@ "local.transition.stopped_moving": et.TransitionType.STOPPED_MOVING, "local.transition.stop_tracking": et.TransitionType.STOP_TRACKING, "local.transition.start_tracking": et.TransitionType.START_TRACKING, + "local.transition.ble_beacon_found": et.TransitionType.BLE_BEACON_FOUND, + "local.transition.ble_beacon_lost": et.TransitionType.BLE_BEACON_LOST, "local.transition.tracking_error": et.TransitionType.TRACKING_ERROR } @@ -44,7 +46,11 @@ def format(entry): data = ad.AttrDict() data.curr_state = state_map[entry.data.currState].value logging.debug("Mapped %s -> %s" % (entry.data.currState, data.curr_state)) - data.transition = transition_map[entry.data.transition].value + if entry.data.transition is not None: + data.transition = transition_map[entry.data.transition].value + else: + data.transition = None + if "ts" not in data: data.ts = formatted_entry.metadata.write_ts logging.debug("No existing timestamp, copyied from metadata%s" % data.ts) diff --git a/emission/net/usercache/formatters/ios/bluetooth_ble.py b/emission/net/usercache/formatters/ios/bluetooth_ble.py new file mode 100644 index 000000000..a0883d424 --- /dev/null +++ b/emission/net/usercache/formatters/ios/bluetooth_ble.py @@ -0,0 +1,24 @@ +import logging + +import emission.core.wrapper.bluetoothble as ecwb +import emission.net.usercache.formatters.common as fc +import attrdict as ad +import pandas as pd +import numpy as np + +def format(entry): + formatted_entry = ad.AttrDict() + formatted_entry["_id"] = entry["_id"] + formatted_entry.user_id = entry.user_id + + metadata = entry.metadata + fc.expand_metadata_times(metadata) + formatted_entry.metadata = metadata + + data = entry.data + data.local_dt = formatted_entry.metadata.write_local_dt + data.fmt_time = formatted_entry.metadata.write_fmt_time + data.eventType = ecwb.BLEEventTypes[entry.data.eventType].value + formatted_entry.data = data + + return formatted_entry diff --git a/emission/net/usercache/formatters/ios/transition.py b/emission/net/usercache/formatters/ios/transition.py index 5c4f3cacd..04c19dcb1 100644 --- a/emission/net/usercache/formatters/ios/transition.py +++ b/emission/net/usercache/formatters/ios/transition.py @@ -34,6 +34,8 @@ "T_TRACKING_STOPPED": et.TransitionType.TRACKING_STOPPED, "T_VISIT_STARTED": et.TransitionType.VISIT_STARTED, "T_VISIT_ENDED": et.TransitionType.VISIT_ENDED, + "T_BLE_BEACON_FOUND": et.TransitionType.BLE_BEACON_FOUND, + "T_BLE_BEACON_LOST": et.TransitionType.BLE_BEACON_LOST, "T_NOP": et.TransitionType.NOP, "T_START_TRACKING": et.TransitionType.START_TRACKING } @@ -59,7 +61,6 @@ def format(entry): # deal with collapsing later # data.transition_raw = entry.data.transition - data.transition = transition_map[entry.data.transition].value if entry.data.transition is not None: data.transition = transition_map[entry.data.transition].value else: diff --git a/emission/storage/timeseries/builtin_timeseries.py b/emission/storage/timeseries/builtin_timeseries.py index 8204b4e29..15ba9f0ec 100644 --- a/emission/storage/timeseries/builtin_timeseries.py +++ b/emission/storage/timeseries/builtin_timeseries.py @@ -45,6 +45,7 @@ def __init__(self, user_id): "background/filtered_location": self.timeseries_db, "background/motion_activity": self.timeseries_db, "background/battery": self.timeseries_db, + "background/bluetooth_ble": self.timeseries_db, "statemachine/transition": self.timeseries_db, "config/sensor_config": self.timeseries_db, "config/sync_config": self.timeseries_db, diff --git a/emission/tests/netTests/TestBuiltinUserCacheHandlerInput.py b/emission/tests/netTests/TestBuiltinUserCacheHandlerInput.py index b7b2e8b97..3d024be06 100644 --- a/emission/tests/netTests/TestBuiltinUserCacheHandlerInput.py +++ b/emission/tests/netTests/TestBuiltinUserCacheHandlerInput.py @@ -138,7 +138,7 @@ def testMoveWhenEmpty(self): edb.get_usercache_db().insert_one({ 'user_id': self.testUserUUID1, '_id': boi.ObjectId('572d3621d282b8f30def7e85'), - 'data': {u'transition': None, + 'data': {u'transition': "MY_FAKE_TRANSITION", 'currState': u'STATE_ONGOING_TRIP'}, 'metadata': {'plugin': 'none', 'write_ts': self.curr_ts - 25,