chore: update analytics for july 2023 (clevercanary#892, clevercanary…

…#893)

analytics/hca-analytics/analytics_hca.py

@@ -8,6 +8,8 @@
 import re
 from html import escape as escape_html
 
+users_over_time_file_name = "users_over_time_history.json"
+
 
 @cache
 def get_project_name(id):
@@ -43,16 +45,29 @@ def adjust_table_index_key(val):
 def format_project_id_key(val):
 	return ('<a href="' + escape_html("https://data.humancellatlas.org/explore/projects/" + val) + '">' + escape_html(get_project_name(val)) + '</a>', True)
 
-def plot_users_over_time(**other_params):
-	return ac.show_plot_over_time(
+def save_ga3_users_over_time_data(users_params, views_params, **other_params):
+	users_df = ac.get_data_df(["ga:30dayUsers"], ["ga:date"], df_processor=lambda df: df[::-1], **users_params, **other_params)
+	users_df.index = pd.to_datetime(users_df.index)
+	views_df = ac.get_data_df(["ga:pageviews"], ["ga:date"], df_processor=lambda df: df[::-1], **views_params, **other_params)
+	views_df.index = pd.to_datetime(views_df.index)
+
+	df = ac.make_month_filter(["ga:30dayUsers"])(users_df.join(views_df)).rename(columns={"ga:30dayUsers": "Users", "ga:pageviews": "Total Pageviews"})
+	df.to_json(users_over_time_file_name)
+
+def plot_users_over_time(load_json=True, use_api=True, **other_params):
+	old_data = pd.read_json(users_over_time_file_name) if load_json else None
+	df = ac.show_plot_over_time(
 		"Monthly Activity Overview",
-		["Users", "Total Unique Pageviews"],
-		["ga:30dayUsers", "ga:uniquePageviews"],
-		df_filter=ac.make_month_filter(["ga:30dayUsers"]),
-		df_processor=lambda df: df[::-1],
-		change_dir=-1,
+		["Users", "Total Pageviews"],
+		["activeUsers", "screenPageViews"] if use_api else None,
+		dimensions="yearMonth",
+		sort_results=["yearMonth"],
+		df_processor=(lambda df: df.set_index(df.index + "01")[-2::-1]) if use_api else None,
+		pre_plot_df_processor=None if old_data is None else (lambda df: df.add(old_data, fill_value=0).astype("int")[::-1]) if use_api else (lambda df: old_data),
+		format_table=False,
 		**other_params
 	)
+	return ac.format_change_over_time_table(df, change_dir=-1, **other_params)
 
 
 def plot_downloads():

analytics/hca-analytics/users_over_time_history.json

@@ -0,0 +1 @@
+{"Users":{"1685577600000":7778,"1682899200000":7384,"1680307200000":6467,"1677628800000":6574,"1675209600000":5855,"1672531200000":4470,"1669852800000":4684,"1667260800000":4965,"1664582400000":4891,"1661990400000":4766,"1659312000000":4304,"1656633600000":4112,"1654041600000":4182,"1651363200000":4960,"1648771200000":3899,"1646092800000":3412,"1643673600000":3104,"1640995200000":2836,"1638316800000":2394,"1635724800000":2938,"1633046400000":2584,"1630454400000":2405,"1627776000000":2496,"1625097600000":2954,"1622505600000":1891,"1619827200000":0,"1617235200000":0,"1614556800000":0,"1612137600000":0,"1609459200000":0},"Total Pageviews":{"1685577600000":44146,"1682899200000":48780,"1680307200000":39497,"1677628800000":38763,"1675209600000":30800,"1672531200000":25796,"1669852800000":26073,"1667260800000":28929,"1664582400000":29034,"1661990400000":28711,"1659312000000":27741,"1656633600000":29631,"1654041600000":30890,"1651363200000":33258,"1648771200000":29391,"1646092800000":27114,"1643673600000":22216,"1640995200000":22452,"1638316800000":19856,"1635724800000":24409,"1633046400000":22963,"1630454400000":20424,"1627776000000":21700,"1625097600000":25562,"1622505600000":16683,"1619827200000":0,"1617235200000":0,"1614556800000":0,"1612137600000":0,"1609459200000":0}}

analytics/lungmap-analytics/analytics_lungmap.py

@@ -1,8 +1,11 @@
 import analytics.charts as ac
+import pandas as pd
 import json
 import re
 from html import escape as escape_html
 
+users_over_time_file_name = "users_over_time_history.json"
+
 def format_export_url_info(type, secondary_type, filter):
 	result = escape_html(type.replace("-", " "))
 	if secondary_type:
@@ -24,15 +27,27 @@ def adjust_table_index_key(val):
 			return ('<a href="' + escape_html("https://data-browser.lungmap.net" + val) + '">' + escape_html(val) + '</a>', True)
 	return val
 
-def plot_users_over_time(**other_params):
+def save_ga3_users_over_time_data(users_params, views_params, **other_params):
+	users_df = ac.get_data_df(["ga:30dayUsers"], ["ga:date"], df_processor=lambda df: df[::-1], **users_params, **other_params)
+	users_df.index = pd.to_datetime(users_df.index)
+	views_df = ac.get_data_df(["ga:pageviews"], ["ga:date"], df_processor=lambda df: df[::-1], **views_params, **other_params)
+	views_df.index = pd.to_datetime(views_df.index)
+
+	df = ac.make_month_filter(["ga:30dayUsers"])(users_df.join(views_df)).rename(columns={"ga:30dayUsers": "Users", "ga:pageviews": "Total Pageviews"})
+	df.to_json(users_over_time_file_name)
+
+def plot_users_over_time(load_json=True, use_api=True, **other_params):
+	old_data = pd.read_json(users_over_time_file_name) if load_json else None
 	df = ac.show_plot_over_time(
 		"Monthly Activity Overview",
-		["Users Per Month", "Total Pageviews Per Month"],
-		["ga:30dayUsers", "ga:pageviews"],
-		df_filter=ac.make_month_filter(["ga:30dayUsers"]),
-		df_processor=lambda df: df[::-1],
+		["Users", "Total Pageviews"],
+		["activeUsers", "screenPageViews"] if use_api else None,
+		dimensions="yearMonth",
+		sort_results=["yearMonth"],
+		df_processor=(lambda df: df.set_index(df.index + "01")[-2::-1]) if use_api else None,
+		pre_plot_df_processor=None if old_data is None else (lambda df: df.add(old_data, fill_value=0).astype("int")[::-1]) if use_api else (lambda df: old_data),
 		format_table=False,
 		**other_params
-	).rename(columns={"Users Per Month": "Users", "Total Pageviews Per Month": "Total Pageviews"})
-	return ac.format_change_over_time_table(df, change_dir=-1,  **other_params)
+	)
+	return ac.format_change_over_time_table(df, change_dir=-1, **other_params)
 

analytics/lungmap-analytics/users_over_time_history.json

@@ -0,0 +1 @@
+{"Users":{"1685577600000":125,"1682899200000":186,"1680307200000":129,"1677628800000":165,"1675209600000":139,"1672531200000":100,"1669852800000":72,"1667260800000":98,"1664582400000":37,"1661990400000":18,"1659312000000":12,"1656633600000":10,"1654041600000":12,"1651363200000":10,"1648771200000":9,"1646092800000":8,"1643673600000":9,"1640995200000":10,"1638316800000":12,"1635724800000":14,"1633046400000":6,"1630454400000":1,"1627776000000":1,"1625097600000":3,"1622505600000":0,"1619827200000":0},"Total Pageviews":{"1685577600000":493,"1682899200000":1115,"1680307200000":679,"1677628800000":929,"1675209600000":725,"1672531200000":540,"1669852800000":444,"1667260800000":559,"1664582400000":230,"1661990400000":121,"1659312000000":38,"1656633600000":29,"1654041600000":39,"1651363200000":32,"1648771200000":120,"1646092800000":35,"1643673600000":24,"1640995200000":86,"1638316800000":36,"1635724800000":44,"1633046400000":26,"1630454400000":19,"1627776000000":3,"1625097600000":32,"1622505600000":0,"1619827200000":0}}

analytics/requirements.txt

@@ -1,5 +1,5 @@
 alabaster==0.7.13
--e git+https://github.com/DataBiosphere/data-browser.git@5042e623bfe6ad8404be5f60e1ddb3c0e8e7d6b3#egg=analytics&subdirectory=analytics/hdgar-book/analytics_package
+-e "git+https://github.com/DataBiosphere/data-browser.git@f674ffb9feae3fc9419e6c98d2ae34aacdc10704#egg=analytics&subdirectory=analytics/analytics_package"
 anyio==3.6.2
 appdirs==1.4.4
 appnope==0.1.3