Add absolute_time_in_s to users and remove sessions_duration_s in fav…

…our of absolute_time_in_s in sessions and users table

docs/markdown/snowplow_unified_common_cols.md

@@ -788,10 +788,6 @@ First application version.
 Last application version.
 {% enddocs %}
 
-{% docs col_session_duration_s %}
-Total duration of a session in seconds.
-{% enddocs %}
-
 {% docs col_device_user_id %}
 Unique device user id.
 {% enddocs %}
@@ -816,10 +812,6 @@ Earliest timestamp for the user's activity, based on `derived_tstamp`.
 Latest timestamp for the user's activity, based on `derived_tstamp`.
 {% enddocs %}
 
-{% docs col_sessions_duration_s %}
-Total session duration for the specific user.
-{% enddocs %}
-
 {% docs col_active_days %}
 Total number of active days for the user.
 {% enddocs %}
@@ -1308,6 +1300,10 @@ On mobile, it is the time that the app was in foreground + background during the
 The time in seconds between the `start_tstamp` and `end_tstamp` of the first and last event in the session.
 {% enddocs %}
 
+{% docs col_user_absolute_time_in_s %}
+The time in seconds between the `start_tstamp` and `end_tstamp` of the first and last event of sessions of the user.
+{% enddocs %}
+
 {% docs col_horizontal_pixels_scrolled %}
 Distance the user scrolled horizontally in pixels.
 

integration_tests/README.md

@@ -39,7 +39,7 @@ There are certain exceptions to how different warehouses process data and in pla
 
 - the non-deterministic nature of row_number() function for Redshift/Postgres/Databricks means that we had to hard-code actuals and expected models for cases where we are testing duplicate rows with exact same results / window
 - postgres / redshift needing the array format of : (within sessions_expected)
-- bigquery handling of snowplow_utils.timestamp_diff() - absolute_time_in_s changes as well as sessions_duration_s
+- bigquery handling of snowplow_utils.timestamp_diff() - absolute_time_in_s changes
 - rotating domain_userid per session is hard-coded in the integration test expectations, when run in one batch the user_identifier differs: 2e340eb6e94820ea8369c0174c612260d1cfe9d41f0fe46268994e28d9c0bbf17
 0e9ab97b5d9d9a174112df13fe9c44788af3ac9088a8b41e0998d92a8b4b5a4fc
 - same with the number of quarantined sessions

integration_tests/models/actual/snowplow_unified_sessions_actual.sql

@@ -116,7 +116,6 @@ select
 ,is_engaged
 ,engaged_time_in_s
 ,absolute_time_in_s
-,session_duration_s
 ,screen_names_viewed
 ,mkt_medium
 ,mkt_source

integration_tests/models/actual/snowplow_unified_users_actual.sql

@@ -69,7 +69,7 @@ select
   ,active_days
   ,engaged_time_in_s
   ,screen_names_viewed
-  ,sessions_duration_s
+  ,absolute_time_in_s
   ,page_referrer
   ,refr_medium
   ,refr_source

integration_tests/models/actual/snowplow_unified_views_mobile_screen_engagement_actual.sql

@@ -34,7 +34,7 @@ select
   session_identifier,
   user_identifier,
   engaged_time_in_s,
-  session_duration_s as absolute_time_in_s,
+  absolute_time_in_s,
   null as horizontal_pixels_scrolled,
   null as vertical_pixels_scrolled,
   null as horizontal_percentage_scrolled,
@@ -53,7 +53,7 @@ select
   null as session_identifier,
   user_identifier,
   engaged_time_in_s,
-  sessions_duration_s as absolute_time_in_s,
+  absolute_time_in_s,
   null as horizontal_pixels_scrolled,
   null as vertical_pixels_scrolled,
   null as horizontal_percentage_scrolled,

integration_tests/models/expected/snowplow_unified_sessions_expected_stg.sql

@@ -126,14 +126,6 @@ SELECT
     ,absolute_time_in_s
 {% endif %}
 
-  -- hard-coding due to different rounding in Bigquery
-{% if target.type == 'bigquery' -%}
-  ,case when session_identifier in ('067bcc8ea082fcaf940893b64943edc6b718f0dd1bc1bd5d054a507c64048318', '79831faefff0edc56d0d93ccf192b5bb58c07bec8dfff185f5cc4b104c2f0a08', '37b340c11359852988ae9b4f77f0c4b283cf2d4e87e2b5e5bceda36a185b81ff', '4d34b56a2a474e7c5d7f125c9a0ed11b7e526bf99910d6d31afaf6a8cc25d7ae', '540b6c4f7c15de9093e0f4f6cf45a062fba7606ce64558a0b1c5a34d8bb33af5', '6b379f3eea11eb4ead878125af9cb708f8a91c455d9405b0bc617fa78c2e4291', '6f19f1f4f43224f646d6fed6358c39152faa088f7ab2d90f0a791ea1210a4656','cde5b90f34224b1a94351fa90fcb61d41b26a52a1a8381603cf538a11edd2bb2', '7605342329f0b916a4c1bcd8bddad038988920fb24f74ca9e7ada5d96df32c60') then session_duration_s -1
-   else session_duration_s end as session_duration_s
-{% else %}
-  ,session_duration_s
-{% endif %}
-
     ,screen_names_viewed
     ,mkt_medium
     ,mkt_source

integration_tests/models/expected/snowplow_unified_users_expected_stg.sql

@@ -71,11 +71,11 @@ select
 
   -- hard-coding due to different rounding in Bigquery
 {% if target.type == 'bigquery' -%}
-  ,case when user_identifier in ('066d5a7eecd5792fc6700998b72e58da69e690d9a6fb23c3b70f2bdb41230a70', '24837eb265f7e3f2e82196a3c3a05bf028135683819ab8ded7b8b79753bd52dd', '2e340eb6e94820ea8369c0174c612260d1cfe9d41f0fe46268994e28d9c0bbf17', '5d27ff97148de9e5f1c23e8fd0a4382c06852969f6495def03c66599e676f449','b3c883845957060b8cbebdaec8366d1d4ec6ad283eba1c70e8a512ec43d10875', 'bc0da66ea89bbb0991f1c37ecdc1830a97f1ed9e24296ef6dbf19635220a064e', 'f897c486aa47cefa7478e3db62c797922b9fe006fc10fc05dd4b71abbbbdcae2', '0e9ab97b5d9d9a174112df13fe9c44788af3ac9088a8b41e0998d92a8b4b5a4fc') then sessions_duration_s - 1
-   when user_identifier = '434dff58299fdc4f124ddf56a4f117d76f69bedb06f76d9858ffde85e16e14e1' then sessions_duration_s - 2
-   else sessions_duration_s end as sessions_duration_s
+  ,case when user_identifier in ('066d5a7eecd5792fc6700998b72e58da69e690d9a6fb23c3b70f2bdb41230a70', '24837eb265f7e3f2e82196a3c3a05bf028135683819ab8ded7b8b79753bd52dd', '2e340eb6e94820ea8369c0174c612260d1cfe9d41f0fe46268994e28d9c0bbf17', '5d27ff97148de9e5f1c23e8fd0a4382c06852969f6495def03c66599e676f449','b3c883845957060b8cbebdaec8366d1d4ec6ad283eba1c70e8a512ec43d10875', 'bc0da66ea89bbb0991f1c37ecdc1830a97f1ed9e24296ef6dbf19635220a064e', 'f897c486aa47cefa7478e3db62c797922b9fe006fc10fc05dd4b71abbbbdcae2', '0e9ab97b5d9d9a174112df13fe9c44788af3ac9088a8b41e0998d92a8b4b5a4fc') then absolute_time_in_s - 1
+   when user_identifier = '434dff58299fdc4f124ddf56a4f117d76f69bedb06f76d9858ffde85e16e14e1' then absolute_time_in_s - 2
+   else absolute_time_in_s end as absolute_time_in_s
 {% else %}
-  ,sessions_duration_s
+  ,absolute_time_in_s
 {% endif %}
 
   ,page_referrer

models/sessions/scratch/snowplow_unified_sessions_this_run.sql

@@ -326,12 +326,9 @@ select
     , a.engaged_time_in_s
   {%- endif %}
 
-  {% if var('snowplow__enable_web') %}
   , a.absolute_time_in_s + case when f.event_name = 'page_ping' then {{ var("snowplow__min_visit_length", 5) }} else 0 end as absolute_time_in_s
-  {%- endif %}
 
   {% if var('snowplow__enable_mobile') %}
-    , a.absolute_time_in_s as session_duration_s
     , a.screen_names_viewed
   {%- endif %}
 

models/sessions/sessions.yml

@@ -140,8 +140,6 @@ models:
       - name: geo_longitude
       - name: geo_timezone
       - name: user_ipaddress
-      - name: session_duration_s
-        description: '{{ doc("col_session_absolute_time_in_s") }}'
       - name: screen_names_viewed
       - name: mkt_medium
       - name: mkt_source

models/users/scratch/snowplow_unified_users_aggs.sql

@@ -35,9 +35,10 @@ select
     , sum(engaged_time_in_s) as engaged_time_in_s
   {% endif %}
 
+  , sum(absolute_time_in_s) as absolute_time_in_s
+
   {% if var('snowplow__enable_mobile') %}
     , sum(screen_names_viewed) as screen_names_viewed
-    , sum(session_duration_s) as sessions_duration_s
   {% endif %}
 
   {% if var('snowplow__enable_app_errors') %}

models/users/scratch/snowplow_unified_users_this_run.sql

@@ -90,9 +90,10 @@ select
     , b.engaged_time_in_s
   {% endif %}
 
+  , b.absolute_time_in_s
+
   {% if var('snowplow__enable_mobile') %}
     , b.screen_names_viewed
-    , b.sessions_duration_s
   {% endif %}
 
 

models/users/scratch/users_scratch.yml

@@ -39,6 +39,8 @@ models:
         description: '{{ doc("col_user_engaged_time_in_s") }}'
         tests:
           - not_null
+      - name: absolute_time_in_s
+        description: '{{ doc("col_user_absolute_time_in_s") }}'
       - name: first_page_title
         description: The title of the first page visited by the user
       - name: first_page_url

models/users/users.yml

@@ -170,7 +170,8 @@ models:
       - name: geo_timezone
       - name: active_days
       - name: screen_names_viewed
-      - name: sessions_duration_s
+      - name: absolute_time_in_s
+        description: '{{ doc("col_user_absolute_time_in_s") }}'
       - name: page_referrer
       - name: first_screen_view__name
       - name: first_screen_view__type