From cd065698ec55f7d156afbf6072d3782ecc899e63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matu=CC=81s=CC=8C=20Tomlein?= Date: Sat, 6 Jan 2024 13:46:35 +0100 Subject: [PATCH] Add absolute_time_in_s to users and remove sessions_duration_s in favour of absolute_time_in_s in sessions and users table --- docs/markdown/snowplow_unified_common_cols.md | 12 ++++-------- integration_tests/README.md | 2 +- .../expected/snowplow_unified_users_expected.csv | 6 +++--- .../actual/snowplow_unified_sessions_actual.sql | 1 - .../models/actual/snowplow_unified_users_actual.sql | 2 +- ...unified_views_mobile_screen_engagement_actual.sql | 4 ++-- .../snowplow_unified_sessions_expected_stg.sql | 8 -------- .../expected/snowplow_unified_users_expected_stg.sql | 8 ++++---- .../scratch/snowplow_unified_sessions_this_run.sql | 3 --- models/sessions/sessions.yml | 2 -- models/users/scratch/snowplow_unified_users_aggs.sql | 3 ++- .../scratch/snowplow_unified_users_this_run.sql | 3 ++- models/users/scratch/users_scratch.yml | 2 ++ models/users/users.yml | 3 ++- 14 files changed, 23 insertions(+), 36 deletions(-) diff --git a/docs/markdown/snowplow_unified_common_cols.md b/docs/markdown/snowplow_unified_common_cols.md index e3a70a5a..2f3f8228 100644 --- a/docs/markdown/snowplow_unified_common_cols.md +++ b/docs/markdown/snowplow_unified_common_cols.md @@ -788,10 +788,6 @@ First application version. Last application version. {% enddocs %} -{% docs col_session_duration_s %} -Total duration of a session in seconds. -{% enddocs %} - {% docs col_device_user_id %} Unique device user id. {% enddocs %} @@ -816,10 +812,6 @@ Earliest timestamp for the user's activity, based on `derived_tstamp`. Latest timestamp for the user's activity, based on `derived_tstamp`. {% enddocs %} -{% docs col_sessions_duration_s %} -Total session duration for the specific user. -{% enddocs %} - {% docs col_active_days %} Total number of active days for the user. {% enddocs %} @@ -1308,6 +1300,10 @@ On mobile, it is the time that the app was in foreground + background during the The time in seconds between the `start_tstamp` and `end_tstamp` of the first and last event in the session. {% enddocs %} +{% docs col_user_absolute_time_in_s %} +The time in seconds between the `start_tstamp` and `end_tstamp` of the first and last event of sessions of the user. +{% enddocs %} + {% docs col_horizontal_pixels_scrolled %} Distance the user scrolled horizontally in pixels. diff --git a/integration_tests/README.md b/integration_tests/README.md index b0ed5028..80b24618 100644 --- a/integration_tests/README.md +++ b/integration_tests/README.md @@ -39,7 +39,7 @@ There are certain exceptions to how different warehouses process data and in pla - the non-deterministic nature of row_number() function for Redshift/Postgres/Databricks means that we had to hard-code actuals and expected models for cases where we are testing duplicate rows with exact same results / window - postgres / redshift needing the array format of : (within sessions_expected) -- bigquery handling of snowplow_utils.timestamp_diff() - absolute_time_in_s changes as well as sessions_duration_s +- bigquery handling of snowplow_utils.timestamp_diff() - absolute_time_in_s changes - rotating domain_userid per session is hard-coded in the integration test expectations, when run in one batch the user_identifier differs: 2e340eb6e94820ea8369c0174c612260d1cfe9d41f0fe46268994e28d9c0bbf17 0e9ab97b5d9d9a174112df13fe9c44788af3ac9088a8b41e0998d92a8b4b5a4fc - same with the number of quarantined sessions diff --git a/integration_tests/data/expected/snowplow_unified_users_expected.csv b/integration_tests/data/expected/snowplow_unified_users_expected.csv index 9a299dfd..e87e3b2c 100644 --- a/integration_tests/data/expected/snowplow_unified_users_expected.csv +++ b/integration_tests/data/expected/snowplow_unified_users_expected.csv @@ -1,4 +1,4 @@ -user_id,user_identifier,network_userid,start_tstamp,end_tstamp,model_tstamp,first_platform,last_platform,on_web,on_mobile,last_screen_resolution,last_os_type,last_os_version,last_mobile__device_manufacturer,last_mobile__device_model,last_mobile__carrier,first_mobile__device_manufacturer,first_mobile__device_model,mobile__os_type,mobile__os_version,mobile__android_idfa,mobile__apple_idfa,mobile__apple_idfv,first_mobile__carrier,mobile__open_idfa,mobile__network_technology,mobile__network_type,mobile__physical_memory,mobile__system_available_memory,mobile__app_available_memory,mobile__battery_level,mobile__battery_state,mobile__low_power_mode,mobile__available_storage,mobile__total_storage,mobile__is_portrait,mobile__resolution,mobile__scale,mobile__language,mobile__app_set_id,mobile__app_set_id_scope,first_geo_country,first_geo_country_name,first_geo_continent,first_geo_city,first_geo_region_name,last_geo_country,last_geo_country_name,last_geo_continent,last_geo_city,last_geo_region_name,geo_zipcode,geo_latitude,geo_longitude,geo_timezone,"views",sessions,active_days,engaged_time_in_s,screen_names_viewed,sessions_duration_s,page_referrer,refr_medium,refr_source,refr_term,first_page_title,first_page_url,first_page_urlscheme,first_page_urlhost,first_page_urlpath,first_page_urlquery,first_page_urlfragment,first_br_lang,first_br_lang_name,last_page_title,last_page_url,last_page_urlscheme,last_page_urlhost,last_page_urlpath,last_page_urlquery,last_page_urlfragment,last_br_lang,last_br_lang_name,refr_urlscheme,refr_urlhost,refr_urlpath,refr_urlquery,refr_urlfragment,first_screen_view__name,first_screen_view__type,last_screen_view__name,last_screen_view__transition_type,last_screen_view__type,mkt_medium,mkt_source,mkt_term,mkt_content,mkt_campaign,mkt_clickid,mkt_network,mkt_source_platform,default_channel_group,app_errors,fatal_app_errors,first_event_id,first_event_id2,last_event_id,last_event_id2 +user_id,user_identifier,network_userid,start_tstamp,end_tstamp,model_tstamp,first_platform,last_platform,on_web,on_mobile,last_screen_resolution,last_os_type,last_os_version,last_mobile__device_manufacturer,last_mobile__device_model,last_mobile__carrier,first_mobile__device_manufacturer,first_mobile__device_model,mobile__os_type,mobile__os_version,mobile__android_idfa,mobile__apple_idfa,mobile__apple_idfv,first_mobile__carrier,mobile__open_idfa,mobile__network_technology,mobile__network_type,mobile__physical_memory,mobile__system_available_memory,mobile__app_available_memory,mobile__battery_level,mobile__battery_state,mobile__low_power_mode,mobile__available_storage,mobile__total_storage,mobile__is_portrait,mobile__resolution,mobile__scale,mobile__language,mobile__app_set_id,mobile__app_set_id_scope,first_geo_country,first_geo_country_name,first_geo_continent,first_geo_city,first_geo_region_name,last_geo_country,last_geo_country_name,last_geo_continent,last_geo_city,last_geo_region_name,geo_zipcode,geo_latitude,geo_longitude,geo_timezone,"views",sessions,active_days,engaged_time_in_s,screen_names_viewed,absolute_time_in_s,page_referrer,refr_medium,refr_source,refr_term,first_page_title,first_page_url,first_page_urlscheme,first_page_urlhost,first_page_urlpath,first_page_urlquery,first_page_urlfragment,first_br_lang,first_br_lang_name,last_page_title,last_page_url,last_page_urlscheme,last_page_urlhost,last_page_urlpath,last_page_urlquery,last_page_urlfragment,last_br_lang,last_br_lang_name,refr_urlscheme,refr_urlhost,refr_urlpath,refr_urlquery,refr_urlfragment,first_screen_view__name,first_screen_view__type,last_screen_view__name,last_screen_view__transition_type,last_screen_view__type,mkt_medium,mkt_source,mkt_term,mkt_content,mkt_campaign,mkt_clickid,mkt_network,mkt_source_platform,default_channel_group,app_errors,fatal_app_errors,first_event_id,first_event_id2,last_event_id,last_event_id2 Unexpired session,12c98e1f5869e3f298fef62699ad9bbe5410d1685e8344a1a1f86e92029685f0,0af211ddbca948cb63c5c7eecf386e5920842b030126f30d121e24d1fa809313,2020-07-03 15:55:18.542000000,2020-07-03 15:55:18.542000000,2023-09-26 11:37:50.200000000,web,web,true,false,,MacOS,>=10.15.7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,GB,United Kingdom of Great Britain and Northern Ireland,Europe,Hackney,Hackney,GB,United Kingdom of Great Britain and Northern Ireland,Europe,Hackney,Hackney,E2,51.5507,-0.0537,Europe/London,"1",1,1,0,0,0,http://localhost:3000/177234df-d425-412e-ad8d-8b97515b2807/getting-started,,,,Snowplow,http://localhost:3000/177234df-d425-412e-ad8d-8b97515b2807/getting-started,http,localhost,/177234df-d425-412e-ad8d-8b97515b2807/getting-started,,,en-GB,English (United Kingdom),Snowplow,http://localhost:3000/177234df-d425-412e-ad8d-8b97515b2807/getting-started,http,localhost,/177234df-d425-412e-ad8d-8b97515b2807/getting-started,,,en-GB,English (United Kingdom),http,localhost,/177234df-d425-412e-ad8d-8b97515b2807/getting-started,,,,,,,,,,,,,,,,Unassigned,0,0,3b0fd63c907de8e2212972c8f409e51ff9628e48904185350d07ac8da380d335,3b0fd63c907de8e2212972c8f409e51ff9628e48904185350d07ac8da380d335-first,3b0fd63c907de8e2212972c8f409e51ff9628e48904185350d07ac8da380d335,3b0fd63c907de8e2212972c8f409e51ff9628e48904185350d07ac8da380d335-last long session,a7208f5a-5e77-4351-8840-f28a3d2a6507,73df60bfe16d26e0acb01d699a174243b6d4dbef08d60c4b5e8fd360f05264ea,2021-02-01 23:55:13.000000000,2021-02-27 14:17:24.000000000,2023-09-26 11:37:50.200000000,mob,mob,false,true,1440x2960,Google Android 8.1 (Oreo),8.1,Samsung,SM-N960N Galaxy Note9 TD-LTE KR 128GB,Unknown,Samsung,SM-N960N Galaxy Note9 TD-LTE KR 128GB,Google Android 8.1 (Oreo),8.1,00000000-0000-0000-0000-000000000000,na,na,Unknown,na,2G,wifi,1,1,1,42,charging,true,1,128000000000,false,1440x2960,1,na,na,na,IE,Ireland,Europe,Dublin,Leinster,IE,Ireland,Europe,Dublin,Leinster,D02,53.3338,-6.2488,Europe/Dublin,"32",2,2,0,2,87732,,,,,,,,,,,,English,,,,,,,,,English,,,,,,,na,na,na,na,na,,,,,,,,,Unassigned,0,0,7f536c8a-5ae9-44eb-aff5-78cbcddf3269,7f536c8a-5ae9-44eb-aff5-78cbcddf3269-first,1421c98b-7875-4b16-926c-4fb75800bd4f,1421c98b-7875-4b16-926c-4fb75800bd4f-last NULL screen_view_id,7a62ec9d-2aa0-4426-b014-eba2d0dcfebb,68db430d2fbbd2966db71f94c0c0c256fd5c0c3bdafe3672958d0884f91dcd30,2021-02-21 11:31:33.000000000,2021-02-21 11:34:28.000000000,2023-09-26 11:37:50.200000000,mob,mob,false,true,1440x2960,Google Android 8.1 (Oreo),8.1,Samsung,SM-N960N Galaxy Note9 TD-LTE KR 128GB,Unknown,Samsung,SM-N960N Galaxy Note9 TD-LTE KR 128GB,Google Android 8.1 (Oreo),8.1,00000000-0000-0000-0000-000000000000,na,na,Unknown,na,2G,wifi,1,1,1,42,charging,true,1,128000000000,false,1440x2960,1,na,na,na,DE,Germany,Europe,Frankfurt am Main,Hesse,DE,Germany,Europe,Frankfurt am Main,Hesse,60313,50.1188,8.6843,Europe/Berlin,"8",2,1,0,2,19,,,,,,,,,,,,en,English,,,,,,,,en,English,,,,,,na,na,na,na,na,,,,,,,,,Unassigned,0,0,994ba6d9-8b6e-42a3-b4cf-3e9ca969d28c,994ba6d9-8b6e-42a3-b4cf-3e9ca969d28c-first,1548be58-4ce7-4a32-a5e8-2696ece941f4,1548be58-4ce7-4a32-a5e8-2696ece941f4-last @@ -6,10 +6,10 @@ dupe: event_id different collector_tstamp,7813445e-40ed-4306-8d92-01cefe2235c7,0 long session,4f011c929234934761dfff0bca3b5509a1b6d139560439e894a8d26a363654ad,3dbf6369fde3ce359786fd49194f705f54af3a53a8986b5de5bfbd717e2960a7,2021-03-01 21:39:43.815000000,2021-03-03 21:39:50.989000000,2023-09-26 11:37:50.200000000,web,web,true,false,,MacOS,>=10.15.7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,GB,United Kingdom of Great Britain and Northern Ireland,Europe,Redhill,Surrey,GB,United Kingdom of Great Britain and Northern Ireland,Europe,Redhill,Surrey,RH1,51.24,-0.0973,Europe/London,"2",1,1,0,0,172807,https://www.google.co.uk,,,,White paper: Transform behavioral data into actional insights| Snowplow,https://snowplowanalytics.com/resources/data-utilization/?utm_source=google-ad&utm_medium=ppc&utm_campaign=data-utilization&utm_content=&creative=502863204369&keyword=data%20strategy&matchtype=e&network=g&device=m&campaignid=12477606708&adgroupid=119084231415&adid=502863204369&gclid=EAIaIQobChMI2K3yxpmp7wIVqhkGAB0wLAicEAAYAiAAEgKyYfD_BwE,https,snowplowanalytics.com,/resources/data-utilization/,utm_source=google-ad&utm_medium=ppc&utm_campaign=data-utilization&utm_content=&creative=502863204369&keyword=data%20strategy&matchtype=e&network=g&device=m&campaignid=12477606708&adgroupid=119084231415&adid=502863204369&gclid=EAIaIQobChMI2K3yxpmp7wIVqhkGAB0wLAicEAAYAiAAEgKyYfD_BwE,,en-gb,English (United Kingdom),"Collect, manage and operationalize behavioral data at scale | Snowplow",https://snowplowanalytics.com/,https,snowplowanalytics.com,/,,,en-gb,English (United Kingdom),https,www.google.co.uk,,,,,,,,,ppc,google-ad,,,data-utilization,EAIaIQobChMI2K3yxpmp7wIVqhkGAB0wLAicEAAYAiAAEgKyYfD_BwE,Google,,Paid Other,0,0,9f53ce75-cceb-4437-934e-68e1911bc599,9f53ce75-cceb-4437-934e-68e1911bc599-first,9f53ce75-cceb-4437-934e-68e1911bc599,9f53ce75-cceb-4437-934e-68e1911bc599-last dupe: event_id same collector_tstamp,bc0da66ea89bbb0991f1c37ecdc1830a97f1ed9e24296ef6dbf19635220a064e,508312389e03ec34ae2083cc3ba15ba6768b783a13d0816567fafce27ef83981,2021-03-01 20:39:52.944000000,2021-03-01 20:59:19.860000000,2023-09-26 11:37:50.200000000,web,web,true,false,,MacOS,>=10.15.7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,DE,Germany,Europe,,,DE,Germany,Europe,,,,51.2993,9.491,Europe/Berlin,"6",1,1,0,0,1167,,,,,eBook: Using data to develop killer products | Snowplow,https://snowplowanalytics.com/resources/product-analytics-ebook/?utm_source=google-ad&utm_medium=display&utm_campaign=product-analytics-new-ebook&creative=486771159105&keyword=&matchtype=&network=d&device=m&campaignid=11857971003&adgroupid=118244365394&adid=486771159105&gclid=Cj0KCQiAvvKBBhCXARIsACTePW8KY3WUt6QUTRyoI9fVb8Vftz_aZlW8mrqZ_ALTRCya8yuOOoKOvQUaAmChEALw_wcB,https,snowplowanalytics.com,/resources/product-analytics-ebook/,utm_source=google-ad&utm_medium=display&utm_campaign=product-analytics-new-ebook&creative=486771159105&keyword=&matchtype=&network=d&device=m&campaignid=11857971003&adgroupid=118244365394&adid=486771159105&gclid=Cj0KCQiAvvKBBhCXARIsACTePW8KY3WUt6QUTRyoI9fVb8Vftz_aZlW8mrqZ_ALTRCya8yuOOoKOvQUaAmChEALw_wcB,,en-US,English (United States),White paper: Transform behavioral data into actional insights| Snowplow,https://snowplowanalytics.com/resources/data-utilization/?success=true,https,snowplowanalytics.com,/resources/data-utilization/,success=true,,en-US,English (United States),,,,,,,,,,,display,google-ad,,,product-analytics-new-ebook,Cj0KCQiAvvKBBhCXARIsACTePW8KY3WUt6QUTRyoI9fVb8Vftz_aZlW8mrqZ_ALTRCya8yuOOoKOvQUaAmChEALw_wcB,Google,,Display,0,0,bcbeb513-27a7-4b9c-9755-af5936f42c0d,bcbeb513-27a7-4b9c-9755-af5936f42c0d-first,bcbeb513-27a7-4b9c-9755-af5936f42c0d,bcbeb513-27a7-4b9c-9755-af5936f42c0d-last user stitch - session: 1st id: 1,434dff58299fdc4f124ddf56a4f117d76f69bedb06f76d9858ffde85e16e14e1,769a8c9ba35df5018cf72c1650f846e3bd9146ad008a0c9dca0af5df5e8fc2e0,2021-03-01 23:57:04.705000000,2021-03-03 00:47:49.409000000,2023-09-26 11:37:50.200000000,web,web,true,false,,MacOS,>=10.15.7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,AU,Australia,Oceania,,New South Wales,AU,Australia,Oceania,,New South Wales,,-33.8592,151.2055,Australia/Sydney,"7",2,2,0,0,3048,,,,,Discourse – Test,https://discourse.snowplowanalytics.com/,https,discourse.snowplowanalytics.com,/,,,en-GB,English (United Kingdom),Discourse – Snowplow - Are you following us on Twitter yet? @Snowplowlabs,https://discourse.snowplowanalytics.com/,https,discourse.snowplowanalytics.com,/,,,en-GB,English (United Kingdom),,,,,,,,,,,,,,,,,,,Unassigned,0,0,0aada0f0f6e6772c8f5908619cfc235225c40217e288e39740654b32707a71bd,0aada0f0f6e6772c8f5908619cfc235225c40217e288e39740654b32707a71bd-first,4ef77ae4-1e5a-4d25-90ab-fda713772553,4ef77ae4-1e5a-4d25-90ab-fda713772553-last -stray page ping,b3c883845957060b8cbebdaec8366d1d4ec6ad283eba1c70e8a512ec43d10875,7ec2509c390620525e5850288e76a5e508272aca560f75d879100c04e5954233,2021-03-01 18:01:08.687000000,2021-03-03 18:01:38.658000000,2023-09-26 11:37:50.200000000,web,web,true,false,,MacOS,>=10.15.7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,GB,United Kingdom of Great Britain and Northern Ireland,Europe,Hackney,Hackney,GB,United Kingdom of Great Britain and Northern Ireland,Europe,Hackney,Hackney,E2,51.5507,-0.0537,Europe/London,"2",2,2,5,0,3,http://localhost:3000/75ea1aeb-4379-4958-8cbb-1a5cefbb4d29/users,,,,Snowplow,http://localhost:3000/75ea1aeb-4379-4958-8cbb-1a5cefbb4d29/users,http,localhost,/75ea1aeb-4379-4958-8cbb-1a5cefbb4d29/users,,,en-GB,English (United Kingdom),Snowplow Insights,http://localhost:3000/75ea1aeb-4379-4958-8cbb-1a5cefbb4d29/users,http,localhost,/75ea1aeb-4379-4958-8cbb-1a5cefbb4d29/users,,,en-GB,English (United Kingdom),http,localhost,/75ea1aeb-4379-4958-8cbb-1a5cefbb4d29/users,,,,,,,,,,,,,,,,Unassigned,0,0,2b87c7d1-739f-4471-b344-9a6c1eda4248,2b87c7d1-739f-4471-b344-9a6c1eda4248-first,2293979e-22b7-45c1-8ab6-35f865ad6094,2293979e-22b7-45c1-8ab6-35f865ad6094-last +stray page ping,b3c883845957060b8cbebdaec8366d1d4ec6ad283eba1c70e8a512ec43d10875,7ec2509c390620525e5850288e76a5e508272aca560f75d879100c04e5954233,2021-03-01 18:01:08.687000000,2021-03-03 18:01:38.658000000,2023-09-26 11:37:50.200000000,web,web,true,false,,MacOS,>=10.15.7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,GB,United Kingdom of Great Britain and Northern Ireland,Europe,Hackney,Hackney,GB,United Kingdom of Great Britain and Northern Ireland,Europe,Hackney,Hackney,E2,51.5507,-0.0537,Europe/London,"2",2,2,5,0,8,http://localhost:3000/75ea1aeb-4379-4958-8cbb-1a5cefbb4d29/users,,,,Snowplow,http://localhost:3000/75ea1aeb-4379-4958-8cbb-1a5cefbb4d29/users,http,localhost,/75ea1aeb-4379-4958-8cbb-1a5cefbb4d29/users,,,en-GB,English (United Kingdom),Snowplow Insights,http://localhost:3000/75ea1aeb-4379-4958-8cbb-1a5cefbb4d29/users,http,localhost,/75ea1aeb-4379-4958-8cbb-1a5cefbb4d29/users,,,en-GB,English (United Kingdom),http,localhost,/75ea1aeb-4379-4958-8cbb-1a5cefbb4d29/users,,,,,,,,,,,,,,,,Unassigned,0,0,2b87c7d1-739f-4471-b344-9a6c1eda4248,2b87c7d1-739f-4471-b344-9a6c1eda4248-first,2293979e-22b7-45c1-8ab6-35f865ad6094,2293979e-22b7-45c1-8ab6-35f865ad6094-last dupe: view_id different derived_tstamp,f897c486aa47cefa7478e3db62c797922b9fe006fc10fc05dd4b71abbbbdcae2,6948bc2069abe6386f0e961198b9afbf1b83e5b8c850f0eba62005b99dac3721,2021-03-02 19:25:22.488000000,2021-03-02 20:08:24.465000000,2023-09-26 11:37:50.200000000,web,web,true,false,,MacOS,>=10.15.7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,IE,Ireland,Europe,Dublin,Leinster,IE,Ireland,Europe,Dublin,Leinster,D02,53.3338,-6.2488,Europe/Dublin,"7",1,1,0,0,2582,https://www.google.com/,,,,"Collect, manage and operationalize behavioral data at scale | Snowplow",https://snowplowanalytics.com/,https,snowplowanalytics.com,/,,,en-US,English (United States),Is it possible to use confluent cloud kafka sink? - Collectors - Discourse – Snowplow,https://discourse.snowplowanalytics.com/t/is-it-possible-to-use-confluent-cloud-kafka-sink/2270,https,discourse.snowplowanalytics.com,/t/is-it-possible-to-use-confluent-cloud-kafka-sink/2270,,,en-US,English (United States),https,www.google.com,/,,,,,,,,,,,,,,,,Unassigned,0,0,0b088136-6ec8-4979-94f1-eb1daf732b29,0b088136-6ec8-4979-94f1-eb1daf732b29-first,0b088136-6ec8-4979-94f1-eb1daf732b29,0b088136-6ec8-4979-94f1-eb1daf732b29-last Rotating domain_userid per session,0e9ab97b5d9d9a174112df13fe9c44788af3ac9088a8b41e0998d92a8b4b5a4fc,cb14148aeccd3895346f44751cd42b054f92736c64bb2211da3cfde3d05ee285,2021-03-02 15:55:18.539000000,2021-03-04 15:55:27.084000000,2023-09-26 11:37:50.200000000,web,web,true,false,,MacOS,>=10.15.7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,GB,United Kingdom of Great Britain and Northern Ireland,Europe,Hackney,Hackney,GB,United Kingdom of Great Britain and Northern Ireland,Europe,Hackney,Hackney,E2,51.5507,-0.0537,Europe/London,"3",1,1,0,0,172809,http://localhost:3000/177234df-d425-412e-ad8d-8b97515b2807/getting-started,,,,Snowplow,http://localhost:3000/177234df-d425-412e-ad8d-8b97515b2807/getting-started,http,localhost,/177234df-d425-412e-ad8d-8b97515b2807/getting-started,,,en-GB,English (United Kingdom),Snowplow Insights,http://localhost:3000/177234df-d425-412e-ad8d-8b97515b2807/data-models,http,localhost,/177234df-d425-412e-ad8d-8b97515b2807/data-models,,,en-GB,English (United Kingdom),http,localhost,/177234df-d425-412e-ad8d-8b97515b2807/getting-started,,,,,,,,,,,,,,,,Unassigned,0,0,2a8fd470ecf59dbce20f2dbd419dc26ebc108ca026af2ae38ef0db7023da9353,2a8fd470ecf59dbce20f2dbd419dc26ebc108ca026af2ae38ef0db7023da9353-first,2a8fd470ecf59dbce20f2dbd419dc26ebc108ca026af2ae38ef0db7023da9353,2a8fd470ecf59dbce20f2dbd419dc26ebc108ca026af2ae38ef0db7023da9353-last Multiple last sessions per user,24837eb265f7e3f2e82196a3c3a05bf028135683819ab8ded7b8b79753bd52dd,7a5cf702c6cc3a461ec04d4ae58f3f5a71ec4c764e2df0267cc601c5cbe94d53,2021-03-02 02:01:19.675000000,2021-03-02 02:01:28.297000000,2023-09-26 11:37:50.200000000,web,web,true,false,,MacOS,>=10.15.7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,GB,United Kingdom of Great Britain and Northern Ireland,Europe,Hackney,Hackney,GB,United Kingdom of Great Britain and Northern Ireland,Europe,Hackney,Hackney,E2,51.5507,-0.0537,Europe/London,"4",2,1,0,0,9,http://localhost:3000/177234df-d425-412e-ad8d-8b97515b2807/getting-started,,,,Snowplow,http://localhost:3000/177234df-d425-412e-ad8d-8b97515b2807/getting-started,http,localhost,/177234df-d425-412e-ad8d-8b97515b2807/getting-started,,,en-GB,English (United Kingdom),Snowplow Insights,http://localhost:3000/177234df-d425-412e-ad8d-8b97515b2807/data-models,http,localhost,/177234df-d425-412e-ad8d-8b97515b2807/data-models,,,en-GB,English (United Kingdom),http,localhost,/177234df-d425-412e-ad8d-8b97515b2807/getting-started,,,,,,,,,,,,,,,,Unassigned,0,0,26e10987188f052eb033efeaaa1e938fdfa29ada31ed2eca82eecb6f4b0f5368,26e10987188f052eb033efeaaa1e938fdfa29ada31ed2eca82eecb6f4b0f5368-first,26e10987188f052eb033efeaaa1e938fdfa29ada31ed2eca82eecb6f4b0f5368,26e10987188f052eb033efeaaa1e938fdfa29ada31ed2eca82eecb6f4b0f5368-last -No edge cases,0691aaf040cd39a08408e62e5d015cc2cef433f1a95e5f6767d62d24bd4be237,cb14148aeccd3895346f44751cd42b054f92736c64bb2211da3cfde3d05ee285,2021-03-02 15:55:18.539000000,2021-03-04 22:59:29.752000000,2023-09-26 11:37:50.200000000,web,web,true,false,,MacOS,>=10.15.7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,GB,United Kingdom of Great Britain and Northern Ireland,Europe,Hackney,Hackney,GB,United Kingdom of Great Britain and Northern Ireland,Europe,Hackney,Hackney,E2,51.5507,-0.0537,Europe/London,"6",3,2,55,0,251,http://localhost:3000/177234df-d425-412e-ad8d-8b97515b2807/getting-started,,,,Snowplow,http://localhost:3000/177234df-d425-412e-ad8d-8b97515b2807/getting-started?utm_source_platform=my_fun_platform,http,localhost,/177234df-d425-412e-ad8d-8b97515b2807/getting-started,utm_source_platform=my_fun_platform,,en-GB,English (United Kingdom),Snowplow Insights,http://localhost:3000/177234df-d425-412e-ad8d-8b97515b2807/data-models/webmodel,http,localhost,/177234df-d425-412e-ad8d-8b97515b2807/data-models/webmodel,,,en-GB,English (United Kingdom),http,localhost,/177234df-d425-412e-ad8d-8b97515b2807/getting-started,,,,,,,,,,,,,,,my_fun_platform,Unassigned,0,0,b3278f28-800a-4b68-8846-395292399836,b3278f28-800a-4b68-8846-395292399836-first,ef9789a7-c267-4635-acd4-d60a3e69fg90,ef9789a7-c267-4635-acd4-d60a3e69fg90-last +No edge cases,0691aaf040cd39a08408e62e5d015cc2cef433f1a95e5f6767d62d24bd4be237,cb14148aeccd3895346f44751cd42b054f92736c64bb2211da3cfde3d05ee285,2021-03-02 15:55:18.539000000,2021-03-04 22:59:29.752000000,2023-09-26 11:37:50.200000000,web,web,true,false,,MacOS,>=10.15.7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,GB,United Kingdom of Great Britain and Northern Ireland,Europe,Hackney,Hackney,GB,United Kingdom of Great Britain and Northern Ireland,Europe,Hackney,Hackney,E2,51.5507,-0.0537,Europe/London,"6",3,2,55,0,261,http://localhost:3000/177234df-d425-412e-ad8d-8b97515b2807/getting-started,,,,Snowplow,http://localhost:3000/177234df-d425-412e-ad8d-8b97515b2807/getting-started?utm_source_platform=my_fun_platform,http,localhost,/177234df-d425-412e-ad8d-8b97515b2807/getting-started,utm_source_platform=my_fun_platform,,en-GB,English (United Kingdom),Snowplow Insights,http://localhost:3000/177234df-d425-412e-ad8d-8b97515b2807/data-models/webmodel,http,localhost,/177234df-d425-412e-ad8d-8b97515b2807/data-models/webmodel,,,en-GB,English (United Kingdom),http,localhost,/177234df-d425-412e-ad8d-8b97515b2807/getting-started,,,,,,,,,,,,,,,my_fun_platform,Unassigned,0,0,b3278f28-800a-4b68-8846-395292399836,b3278f28-800a-4b68-8846-395292399836-first,ef9789a7-c267-4635-acd4-d60a3e69fg90,ef9789a7-c267-4635-acd4-d60a3e69fg90-last Multiple first sessions per user,066d5a7eecd5792fc6700998b72e58da69e690d9a6fb23c3b70f2bdb41230a70,e1d85d92e5f6f0d068d4629581d887ccf688715eb6988fc37da26840b0bb2bf8,2021-03-02 07:23:19.675000000,2021-03-02 07:23:28.297000000,2023-09-26 11:37:50.200000000,web,web,true,false,,MacOS,>=10.15.7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,GB,United Kingdom of Great Britain and Northern Ireland,Europe,Hackney,Hackney,GB,United Kingdom of Great Britain and Northern Ireland,Europe,Hackney,Hackney,E2,51.5507,-0.0537,Europe/London,"4",2,1,0,0,14,http://localhost:3000/177234df-d425-412e-ad8d-8b97515b2807/getting-started,,,,Dummy title - will cause dupes,http://localhost:3000/177234df-d425-412e-ad8d-8b97515b2807/getting-started-dummy-row,http-dummy-row,localhost,/177234df-d425-412e-ad8d-8b97515b2807/getting-started-dummy-row,,,en-GB,English (United Kingdom),Snowplow Insights,http://localhost:3000/177234df-d425-412e-ad8d-8b97515b2807/data-models,http,localhost,/177234df-d425-412e-ad8d-8b97515b2807/data-models,,,en-GB,English (United Kingdom),http-dummy-row,localhost-dummy-row,/177234df-d425-412e-ad8d-8b97515b2807/getting-started-dummy-row,,,,,,,,,,,,,,,,Unassigned,0,0,53a0f3278633316a17c723da5d93f1ef646fa16b23fa2188119507f6776ffd00,53a0f3278633316a17c723da5d93f1ef646fa16b23fa2188119507f6776ffd00-first,68362751a6afb7785d07f7b80803289da2e7d48354b2e9eaa8781c0da850e224,68362751a6afb7785d07f7b80803289da2e7d48354b2e9eaa8781c0da850e224-last dupe: event_id different collector_tstamp,5d27ff97148de9e5f1c23e8fd0a4382c06852969f6495def03c66599e676f449,db1c72c5e85289ee6e249e8810079cd431a5910c2a2b07f898045e89392af5fa,2021-03-03 21:27:22.334000000,2021-03-03 21:27:35.176000000,2023-09-26 11:37:50.200000000,web,web,true,false,,MacOS,>=10.15.7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,AU,Australia,Oceania,Abbotsford,Victoria,AU,Australia,Oceania,Abbotsford,Victoria,3067,-37.8061,145.0015,Australia/Melbourne,"3",1,1,0,0,13,,,,,"True First Collect, manage and operationalize behavioral data at scale | Snowplow",https://snowplowanalytics.com/,https,snowplowanalytics.com,/,,,en-au,English (Australia),A data pipeline for advertisers and publishers | Snowplow,https://snowplowanalytics.com/advertising/,https,snowplowanalytics.com,/advertising/,,,en-au,English (Australia),,,,,,,,,,,,,,,,,,,Unassigned,0,0,1h2c6742-7a16-478d-b106-96f9a9a54547,1h2c6742-7a16-478d-b106-96f9a9a54547-first,1h2c6742-7a16-478d-b106-96f9a9a54547,1h2c6742-7a16-478d-b106-96f9a9a54547-last diff --git a/integration_tests/models/actual/snowplow_unified_sessions_actual.sql b/integration_tests/models/actual/snowplow_unified_sessions_actual.sql index e331355d..03f88e96 100644 --- a/integration_tests/models/actual/snowplow_unified_sessions_actual.sql +++ b/integration_tests/models/actual/snowplow_unified_sessions_actual.sql @@ -116,7 +116,6 @@ select ,is_engaged ,engaged_time_in_s ,absolute_time_in_s -,session_duration_s ,screen_names_viewed ,mkt_medium ,mkt_source diff --git a/integration_tests/models/actual/snowplow_unified_users_actual.sql b/integration_tests/models/actual/snowplow_unified_users_actual.sql index 2c671f99..5daf30ab 100644 --- a/integration_tests/models/actual/snowplow_unified_users_actual.sql +++ b/integration_tests/models/actual/snowplow_unified_users_actual.sql @@ -69,7 +69,7 @@ select ,active_days ,engaged_time_in_s ,screen_names_viewed - ,sessions_duration_s + ,absolute_time_in_s ,page_referrer ,refr_medium ,refr_source diff --git a/integration_tests/models/actual/snowplow_unified_views_mobile_screen_engagement_actual.sql b/integration_tests/models/actual/snowplow_unified_views_mobile_screen_engagement_actual.sql index a377b8d8..cb47e763 100644 --- a/integration_tests/models/actual/snowplow_unified_views_mobile_screen_engagement_actual.sql +++ b/integration_tests/models/actual/snowplow_unified_views_mobile_screen_engagement_actual.sql @@ -34,7 +34,7 @@ select session_identifier, user_identifier, engaged_time_in_s, - session_duration_s as absolute_time_in_s, + absolute_time_in_s, null as horizontal_pixels_scrolled, null as vertical_pixels_scrolled, null as horizontal_percentage_scrolled, @@ -53,7 +53,7 @@ select null as session_identifier, user_identifier, engaged_time_in_s, - sessions_duration_s as absolute_time_in_s, + absolute_time_in_s, null as horizontal_pixels_scrolled, null as vertical_pixels_scrolled, null as horizontal_percentage_scrolled, diff --git a/integration_tests/models/expected/snowplow_unified_sessions_expected_stg.sql b/integration_tests/models/expected/snowplow_unified_sessions_expected_stg.sql index 6116117b..24c21149 100644 --- a/integration_tests/models/expected/snowplow_unified_sessions_expected_stg.sql +++ b/integration_tests/models/expected/snowplow_unified_sessions_expected_stg.sql @@ -126,14 +126,6 @@ SELECT ,absolute_time_in_s {% endif %} - -- hard-coding due to different rounding in Bigquery -{% if target.type == 'bigquery' -%} - ,case when session_identifier in ('067bcc8ea082fcaf940893b64943edc6b718f0dd1bc1bd5d054a507c64048318', '79831faefff0edc56d0d93ccf192b5bb58c07bec8dfff185f5cc4b104c2f0a08', '37b340c11359852988ae9b4f77f0c4b283cf2d4e87e2b5e5bceda36a185b81ff', '4d34b56a2a474e7c5d7f125c9a0ed11b7e526bf99910d6d31afaf6a8cc25d7ae', '540b6c4f7c15de9093e0f4f6cf45a062fba7606ce64558a0b1c5a34d8bb33af5', '6b379f3eea11eb4ead878125af9cb708f8a91c455d9405b0bc617fa78c2e4291', '6f19f1f4f43224f646d6fed6358c39152faa088f7ab2d90f0a791ea1210a4656','cde5b90f34224b1a94351fa90fcb61d41b26a52a1a8381603cf538a11edd2bb2', '7605342329f0b916a4c1bcd8bddad038988920fb24f74ca9e7ada5d96df32c60') then session_duration_s -1 - else session_duration_s end as session_duration_s -{% else %} - ,session_duration_s -{% endif %} - ,screen_names_viewed ,mkt_medium ,mkt_source diff --git a/integration_tests/models/expected/snowplow_unified_users_expected_stg.sql b/integration_tests/models/expected/snowplow_unified_users_expected_stg.sql index da35a9a5..282f9865 100644 --- a/integration_tests/models/expected/snowplow_unified_users_expected_stg.sql +++ b/integration_tests/models/expected/snowplow_unified_users_expected_stg.sql @@ -71,11 +71,11 @@ select -- hard-coding due to different rounding in Bigquery {% if target.type == 'bigquery' -%} - ,case when user_identifier in ('066d5a7eecd5792fc6700998b72e58da69e690d9a6fb23c3b70f2bdb41230a70', '24837eb265f7e3f2e82196a3c3a05bf028135683819ab8ded7b8b79753bd52dd', '2e340eb6e94820ea8369c0174c612260d1cfe9d41f0fe46268994e28d9c0bbf17', '5d27ff97148de9e5f1c23e8fd0a4382c06852969f6495def03c66599e676f449','b3c883845957060b8cbebdaec8366d1d4ec6ad283eba1c70e8a512ec43d10875', 'bc0da66ea89bbb0991f1c37ecdc1830a97f1ed9e24296ef6dbf19635220a064e', 'f897c486aa47cefa7478e3db62c797922b9fe006fc10fc05dd4b71abbbbdcae2', '0e9ab97b5d9d9a174112df13fe9c44788af3ac9088a8b41e0998d92a8b4b5a4fc') then sessions_duration_s - 1 - when user_identifier = '434dff58299fdc4f124ddf56a4f117d76f69bedb06f76d9858ffde85e16e14e1' then sessions_duration_s - 2 - else sessions_duration_s end as sessions_duration_s + ,case when user_identifier in ('066d5a7eecd5792fc6700998b72e58da69e690d9a6fb23c3b70f2bdb41230a70', '24837eb265f7e3f2e82196a3c3a05bf028135683819ab8ded7b8b79753bd52dd', '2e340eb6e94820ea8369c0174c612260d1cfe9d41f0fe46268994e28d9c0bbf17', '5d27ff97148de9e5f1c23e8fd0a4382c06852969f6495def03c66599e676f449','b3c883845957060b8cbebdaec8366d1d4ec6ad283eba1c70e8a512ec43d10875', 'bc0da66ea89bbb0991f1c37ecdc1830a97f1ed9e24296ef6dbf19635220a064e', 'f897c486aa47cefa7478e3db62c797922b9fe006fc10fc05dd4b71abbbbdcae2', '0e9ab97b5d9d9a174112df13fe9c44788af3ac9088a8b41e0998d92a8b4b5a4fc') then absolute_time_in_s - 1 + when user_identifier = '434dff58299fdc4f124ddf56a4f117d76f69bedb06f76d9858ffde85e16e14e1' then absolute_time_in_s - 2 + else absolute_time_in_s end as absolute_time_in_s {% else %} - ,sessions_duration_s + ,absolute_time_in_s {% endif %} ,page_referrer diff --git a/models/sessions/scratch/snowplow_unified_sessions_this_run.sql b/models/sessions/scratch/snowplow_unified_sessions_this_run.sql index e1afb213..7f111c60 100644 --- a/models/sessions/scratch/snowplow_unified_sessions_this_run.sql +++ b/models/sessions/scratch/snowplow_unified_sessions_this_run.sql @@ -326,12 +326,9 @@ select , a.engaged_time_in_s {%- endif %} - {% if var('snowplow__enable_web') %} , a.absolute_time_in_s + case when f.event_name = 'page_ping' then {{ var("snowplow__min_visit_length", 5) }} else 0 end as absolute_time_in_s - {%- endif %} {% if var('snowplow__enable_mobile') %} - , a.absolute_time_in_s as session_duration_s , a.screen_names_viewed {%- endif %} diff --git a/models/sessions/sessions.yml b/models/sessions/sessions.yml index cf922168..864b38df 100644 --- a/models/sessions/sessions.yml +++ b/models/sessions/sessions.yml @@ -140,8 +140,6 @@ models: - name: geo_longitude - name: geo_timezone - name: user_ipaddress - - name: session_duration_s - description: '{{ doc("col_session_absolute_time_in_s") }}' - name: screen_names_viewed - name: mkt_medium - name: mkt_source diff --git a/models/users/scratch/snowplow_unified_users_aggs.sql b/models/users/scratch/snowplow_unified_users_aggs.sql index 94fe342d..9609541d 100644 --- a/models/users/scratch/snowplow_unified_users_aggs.sql +++ b/models/users/scratch/snowplow_unified_users_aggs.sql @@ -35,9 +35,10 @@ select , sum(engaged_time_in_s) as engaged_time_in_s {% endif %} + , sum(absolute_time_in_s) as absolute_time_in_s + {% if var('snowplow__enable_mobile') %} , sum(screen_names_viewed) as screen_names_viewed - , sum(session_duration_s) as sessions_duration_s {% endif %} {% if var('snowplow__enable_app_errors') %} diff --git a/models/users/scratch/snowplow_unified_users_this_run.sql b/models/users/scratch/snowplow_unified_users_this_run.sql index afb3adf6..19e31b52 100644 --- a/models/users/scratch/snowplow_unified_users_this_run.sql +++ b/models/users/scratch/snowplow_unified_users_this_run.sql @@ -90,9 +90,10 @@ select , b.engaged_time_in_s {% endif %} + , b.absolute_time_in_s + {% if var('snowplow__enable_mobile') %} , b.screen_names_viewed - , b.sessions_duration_s {% endif %} diff --git a/models/users/scratch/users_scratch.yml b/models/users/scratch/users_scratch.yml index f0c6ab9e..63011cc9 100644 --- a/models/users/scratch/users_scratch.yml +++ b/models/users/scratch/users_scratch.yml @@ -39,6 +39,8 @@ models: description: '{{ doc("col_user_engaged_time_in_s") }}' tests: - not_null + - name: absolute_time_in_s + description: '{{ doc("col_user_absolute_time_in_s") }}' - name: first_page_title description: The title of the first page visited by the user - name: first_page_url diff --git a/models/users/users.yml b/models/users/users.yml index 5ff6e9b0..89184a5f 100644 --- a/models/users/users.yml +++ b/models/users/users.yml @@ -170,7 +170,8 @@ models: - name: geo_timezone - name: active_days - name: screen_names_viewed - - name: sessions_duration_s + - name: absolute_time_in_s + description: '{{ doc("col_user_absolute_time_in_s") }}' - name: page_referrer - name: first_screen_view__name - name: first_screen_view__type