feat(stats): arbitrum main page charts (#1210)

* add to interface

* save 3 seconds on each just test-with-db

* add new charts

* fix window charts incorrectly updated

* remove excess line chart from layout

* add the charts everywhere + update configs etc.

* fix index status req for new charts

* fix warn when arbitrum is disabled

* fix warn when calculating yesterday oper txns

* test disabled arbitrum

* fix warns

* clippy

stats/config/charts.json

@@ -65,6 +65,11 @@
             "title": "Yesterday txns",
             "description": "Number of transactions yesterday (0:00 - 23:59 UTC)"
         },
+        "yesterday_operational_txns": {
+            "enabled": false,
+            "title": "Yesterday operational txns",
+            "description": "Number of transactions yesterday (0:00 - 23:59 UTC) without block creation transactions"
+        },
         "last_new_contracts": {
             "title": "Number of contracts today",
             "description": "Number of deployed contracts today (UTC)"
@@ -150,6 +155,11 @@
             "title": "New operational transactions",
             "description": "Number of new transactions without block creation"
         },
+        "new_operational_txns_window": {
+            "enabled": false,
+            "title": "Daily operational transactions",
+            "description": "The chart displays daily transactions for the past 30 days (without block creation transactions)"
+        },
         "operational_txns_growth": {
             "enabled": false,
             "title": "Number of operational transactions",

stats/config/layout.json

@@ -42,8 +42,7 @@
                 "txns_growth",
                 "new_operational_txns",
                 "operational_txns_growth",
-                "txns_success_rate",
-                "new_txns_window"
+                "txns_success_rate"
             ]
         },
         {

stats/config/update_groups.json

@@ -8,7 +8,6 @@
         "total_blocks_group": "0 0 */2 * * * *",
         "total_tokens_group": "0 0 18 * * * *",
         "total_txns_group": "0 5 */2 * * * *",
-        "total_operational_txns_group": "0 5 1 * * * *",
         "yesterday_txns_group": "0 8 0 * * * *",
         "new_txns_window_group": "0 8 0 * * * *",
         "active_recurring_accounts_daily_recurrence_60_days_group": "0 0 2 * * * *",

stats/justfile

@@ -27,10 +27,18 @@ test *args:
 test-with-db *args:
     # remove db from previous run (if failed)
     -just                            docker-name="{{docker-name}}-test" stop-postgres 2> /dev/null
-    -just db-port="{{test-db-port}}" db-name="" docker-name="{{docker-name}}-test" start-postgres
+    just start-postgres-and-build-tests
     just db-port="{{test-db-port}}" db-name=""                                    test {{args}}
     just                            docker-name="{{docker-name}}-test" stop-postgres
 
+start-postgres-and-build-tests:
+    #!/bin/bash -eux
+    just db-port="{{test-db-port}}" db-name="" docker-name="{{docker-name}}-test" start-postgres &
+    cargo build --tests &
+    trap 'kill $(jobs -pr) || true' EXIT
+    wait
+    echo "finished postgres wait"
+
 check-envs:
     VALIDATE_ONLY=true cargo run --bin env-docs-generation
 

stats/stats-proto/proto/stats.proto

@@ -84,8 +84,11 @@ message MainPageStats {
   optional Counter total_blocks = 3;
   optional Counter total_transactions = 4;
   optional Counter yesterday_transactions = 5;
+  optional Counter total_operational_transactions = 6;
+  optional Counter yesterday_operational_transactions = 7;
 
-  optional LineChart daily_new_transactions = 6;
+  optional LineChart daily_new_transactions = 8;
+  optional LineChart daily_new_operational_transactions = 9;
 }
 
 message GetTransactionsPageStatsRequest {}

stats/stats-proto/swagger/stats.swagger.yaml

@@ -266,8 +266,14 @@ definitions:
         $ref: '#/definitions/v1Counter'
       yesterday_transactions:
         $ref: '#/definitions/v1Counter'
+      total_operational_transactions:
+        $ref: '#/definitions/v1Counter'
+      yesterday_operational_transactions:
+        $ref: '#/definitions/v1Counter'
       daily_new_transactions:
         $ref: '#/definitions/v1LineChart'
+      daily_new_operational_transactions:
+        $ref: '#/definitions/v1LineChart'
   v1Point:
     type: object
     properties:

stats/stats-server/src/read_service.rs

@@ -17,11 +17,11 @@ use sea_orm::{DatabaseConnection, DbErr};
 use stats::{
     counters::{
         AverageBlockTime, AverageTxnFee24h, NewContracts24h, NewTxns24h, NewVerifiedContracts24h,
-        PendingTxns30m, TotalAddresses, TotalBlocks, TotalContracts, TotalTxns,
-        TotalVerifiedContracts, TxnsFee24h, YesterdayTxns,
+        PendingTxns30m, TotalAddresses, TotalBlocks, TotalContracts, TotalOperationalTxns,
+        TotalTxns, TotalVerifiedContracts, TxnsFee24h, YesterdayOperationalTxns, YesterdayTxns,
     },
     data_source::{types::BlockscoutMigrations, UpdateContext, UpdateParameters},
-    lines::{NewTxnsWindow, NEW_TXNS_WINDOW_RANGE},
+    lines::{NewOperationalTxnsWindow, NewTxnsWindow, NEW_TXNS_WINDOW_RANGE},
     query_dispatch::{CounterHandle, LineHandle, QuerySerializedDyn},
     range::UniversalRange,
     types::{Timespan, TimespanDuration},
@@ -146,15 +146,21 @@ impl ReadService {
             total_blocks: None,
             total_transactions: None,
             yesterday_transactions: None,
+            total_operational_transactions: None,
+            yesterday_operational_transactions: None,
             daily_new_transactions: None,
+            daily_new_operational_transactions: None,
         };
         vec![
             AverageBlockTime::name(),
             TotalAddresses::name(),
             TotalBlocks::name(),
             TotalTxns::name(),
             YesterdayTxns::name(),
+            TotalOperationalTxns::name(),
+            YesterdayOperationalTxns::name(),
             NewTxnsWindow::name(),
+            NewOperationalTxnsWindow::name(),
         ]
     }
 
@@ -313,31 +319,37 @@ impl ReadService {
         Ok(chart_data)
     }
 
-    async fn query_new_txns_window(
+    async fn query_window_chart(
         &self,
+        name: String,
+        window_range: u64,
         query_time: DateTime<Utc>,
     ) -> Option<proto_v1::LineChart> {
-        // All `NEW_TXNS_WINDOW_RANGE` should be returned,
+        // `query_line_chart` will result in warn here even when querying a disabled chart.
+        if !self.charts.charts_info.contains_key(&name) {
+            return None;
+        }
+
+        // All `window_range` should be returned,
         // therefore we need to set exact query range to fill
         // zeroes (if any)
 
         let query_day = query_time.date_naive();
         // overshoot by two to account for
         // - last point being approximate
         // - chart last updated yesterday
-        let range_start =
-            query_day.saturating_sub(TimespanDuration::from_days(NEW_TXNS_WINDOW_RANGE + 1));
+        let range_start = query_day.saturating_sub(TimespanDuration::from_days(window_range + 1));
         let request_range = inclusive_date_range_to_query_range(Some(range_start), Some(query_day));
         let mut transactions = self
             .query_line_chart(
-                NewTxnsWindow::name(),
+                name.clone(),
                 ResolutionKind::Day,
                 request_range,
                 None,
                 query_time,
             )
             .await
-            .inspect_err(|e| tracing::warn!("Couldn't get transactions for main page: {}", e))
+            .inspect_err(|e| tracing::warn!("Couldn't get {} for the main page: {}", name, e))
             .ok()?;
         // return exactly `NEW_TXNS_WINDOW_RANGE` accurate points
         let data = transactions
@@ -429,14 +441,20 @@ impl StatsService for ReadService {
             total_blocks,
             total_transactions,
             yesterday_transactions,
+            total_operational_transactions,
+            yesterday_operational_transactions,
             daily_new_transactions,
+            daily_new_operational_transactions,
         ) = join!(
             self.query_counter(AverageBlockTime::name(), now),
             self.query_counter(TotalAddresses::name(), now),
             self.query_counter(TotalBlocks::name(), now),
             self.query_counter(TotalTxns::name(), now),
             self.query_counter(YesterdayTxns::name(), now),
-            self.query_new_txns_window(now)
+            self.query_counter(TotalOperationalTxns::name(), now),
+            self.query_counter(YesterdayOperationalTxns::name(), now),
+            self.query_window_chart(NewTxnsWindow::name(), NEW_TXNS_WINDOW_RANGE, now),
+            self.query_window_chart(NewOperationalTxnsWindow::name(), NEW_TXNS_WINDOW_RANGE, now),
         );
 
         Ok(Response::new(proto_v1::MainPageStats {
@@ -445,7 +463,10 @@ impl StatsService for ReadService {
             total_blocks,
             total_transactions,
             yesterday_transactions,
+            total_operational_transactions,
+            yesterday_operational_transactions,
             daily_new_transactions,
+            daily_new_operational_transactions,
         }))
     }
 

stats/stats-server/src/runtime_setup.rs

@@ -308,7 +308,6 @@ impl RuntimeSetup {
             Arc::new(TotalBlocksGroup),
             Arc::new(TotalTokensGroup),
             Arc::new(TotalTxnsGroup),
-            Arc::new(TotalOperationalTxnsGroup),
             Arc::new(YesterdayTxnsGroup),
             Arc::new(ActiveRecurringAccountsDailyRecurrence60DaysGroup),
             Arc::new(ActiveRecurringAccountsMonthlyRecurrence60DaysGroup),
@@ -412,6 +411,8 @@ impl RuntimeSetup {
             // compute, therefore this solution is ok (to not introduce
             // more update groups if not necessary)
             ("NewBlocksGroup", vec!["newTxns_DAY"]),
+            // Same logic as above
+            ("TotalBlocksGroup", vec!["totalTxns_DAY"]),
         ]
         .map(|(group_name, allowed_missing)| {
             (

stats/stats-server/src/settings.rs

@@ -8,8 +8,8 @@ use cron::Schedule;
 use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, DisplayFromStr};
 use stats::{
-    counters::TotalOperationalTxns,
-    lines::{NewOperationalTxns, OperationalTxnsGrowth},
+    counters::{TotalOperationalTxns, YesterdayOperationalTxns},
+    lines::{NewOperationalTxns, NewOperationalTxnsWindow, OperationalTxnsGrowth},
     ChartProperties, IndexingStatus,
 };
 use std::{collections::BTreeSet, net::SocketAddr, path::PathBuf, str::FromStr};
@@ -143,6 +143,8 @@ pub fn handle_enable_all_arbitrum(
             NewOperationalTxns::key().name(),
             OperationalTxnsGrowth::key().name(),
             TotalOperationalTxns::key().name(),
+            YesterdayOperationalTxns::key().name(),
+            NewOperationalTxnsWindow::key().name(),
         ] {
             let settings = match (
                 charts.lines.get_mut(enable_key),

stats/stats-server/tests/it/chart_endpoints/counters.rs

@@ -28,6 +28,7 @@ pub async fn test_counters_ok(base: Url) {
         "totalNativeCoinTransfers",
         "totalTokens",
         "totalTxns",
+        "totalOperationalTxns",
         "totalVerifiedContracts",
         // on a different page
         // "yesterdayTxns",

stats/stats-server/tests/it/chart_endpoints/lines.rs

@@ -52,8 +52,8 @@ pub async fn test_lines_ok(base: Url) {
         // "newTxnsWindow",
         "txnsFee",
         "txnsGrowth",
-        // "newOperationalTxns",
-        // "operationalTxnsGrowth",
+        "newOperationalTxns",
+        "operationalTxnsGrowth",
         "txnsSuccessRate",
         "newVerifiedContracts",
         "newContracts",

stats/stats-server/tests/it/chart_endpoints/main_page.rs

@@ -1,46 +1,57 @@
 use blockscout_service_launcher::test_server::send_get_request;
 
 use pretty_assertions::assert_eq;
-use stats::{
-    lines::{NewTxnsWindow, NEW_TXNS_WINDOW_RANGE},
-    Named,
-};
+use stats::lines::NEW_TXNS_WINDOW_RANGE;
 use stats_proto::blockscout::stats::v1::MainPageStats;
 use url::Url;
 
 use crate::array_of_variables_with_names;
 
-pub async fn test_main_page_ok(base: Url) {
+pub async fn test_main_page_ok(base: Url, expect_arbitrum: bool) {
     let main_page: MainPageStats = send_get_request(&base, "/api/v1/pages/main").await;
     let MainPageStats {
         average_block_time,
         total_addresses,
         total_blocks,
         total_transactions,
         yesterday_transactions,
+        total_operational_transactions,
+        yesterday_operational_transactions,
         daily_new_transactions,
+        daily_new_operational_transactions,
     } = main_page;
-    let counters = array_of_variables_with_names!([
+    let mut counters = array_of_variables_with_names!([
         average_block_time,
         total_addresses,
         total_blocks,
         total_transactions,
         yesterday_transactions,
-    ]);
+    ])
+    .to_vec();
+    if expect_arbitrum {
+        counters.extend(array_of_variables_with_names!([
+            total_operational_transactions,
+            yesterday_operational_transactions,
+        ]));
+    }
     for (name, counter) in counters {
-        #[allow(clippy::expect_fun_call)]
-        let counter = counter.expect(&format!("page counter {} must be available", name));
+        let counter = counter.unwrap_or_else(|| panic!("page counter {} must be available", name));
         assert!(!counter.description.is_empty());
         assert!(!counter.title.is_empty());
     }
 
-    let daily_new_transactions =
-        daily_new_transactions.expect("daily new transactions chart must be available");
-    let transactions_info = daily_new_transactions.info.unwrap();
-    assert_eq!(transactions_info.id, NewTxnsWindow::name());
-    assert_eq!(transactions_info.resolutions, vec!["DAY"]);
-    assert_eq!(
-        daily_new_transactions.chart.len(),
-        NEW_TXNS_WINDOW_RANGE as usize
-    );
+    let mut window_line_charts = array_of_variables_with_names!([daily_new_transactions]).to_vec();
+    if expect_arbitrum {
+        window_line_charts.extend(array_of_variables_with_names!([
+            daily_new_operational_transactions
+        ]));
+    }
+    for (name, window_chart) in window_line_charts {
+        let window_chart =
+            window_chart.unwrap_or_else(|| panic!("{} chart must be available", name));
+        let transactions_info = window_chart.info.unwrap();
+        assert!(!transactions_info.id.is_empty());
+        assert_eq!(transactions_info.resolutions, vec!["DAY"]);
+        assert_eq!(window_chart.chart.len(), NEW_TXNS_WINDOW_RANGE as usize);
+    }
 }

stats/stats-server/tests/it/chart_endpoints/mod.rs

@@ -46,12 +46,12 @@ async fn test_chart_endpoints_ok() {
     init_server(|| stats(settings), &base).await;
 
     // Sleep until server will start and calculate all values
-    tokio::time::sleep(std::time::Duration::from_secs(7)).await;
+    tokio::time::sleep(std::time::Duration::from_secs(8)).await;
 
     let tests: JoinSet<_> = [
         test_lines_ok(base.clone()).boxed(),
         test_counters_ok(base.clone()).boxed(),
-        test_main_page_ok(base.clone()).boxed(),
+        test_main_page_ok(base.clone(), true).boxed(),
         test_transactions_page_ok(base.clone()).boxed(),
         test_contracts_page_ok(base).boxed(),
     ]
@@ -81,14 +81,33 @@ async fn test_chart_endpoints_work_with_not_indexed_blockscout() {
     init_server(|| stats(settings), &base).await;
 
     // Sleep until server will start and calculate all values
-    tokio::time::sleep(std::time::Duration::from_secs(7)).await;
+    tokio::time::sleep(std::time::Duration::from_secs(8)).await;
 
     let tests: JoinSet<_> = [
-        test_main_page_ok(base.clone()).boxed(),
+        test_main_page_ok(base.clone(), true).boxed(),
         test_transactions_page_ok(base.clone()).boxed(),
         test_contracts_page_ok(base).boxed(),
     ]
     .into_iter()
     .collect();
     run_consolidated_tests(tests, test_name).await;
 }
+
+#[tokio::test]
+#[ignore = "needs database"]
+async fn test_chart_endpoints_work_with_disabled_arbitrum() {
+    let test_name = "test_chart_endpoints_work_with_disabled_arbitrum";
+    let (stats_db, blockscout_db) = init_db_all(test_name).await;
+    let blockscout_api = default_mock_blockscout_api().await;
+    fill_mock_blockscout_data(&blockscout_db, NaiveDate::from_str("2023-03-01").unwrap()).await;
+    std::env::set_var("STATS__CONFIG", "./tests/config/test.toml");
+    let (mut settings, base) = get_test_stats_settings(&stats_db, &blockscout_db, &blockscout_api);
+    settings.enable_all_arbitrum = false;
+
+    init_server(|| stats(settings), &base).await;
+
+    // Sleep until server will start and calculate all values
+    tokio::time::sleep(std::time::Duration::from_secs(8)).await;
+
+    test_main_page_ok(base, false).await;
+}

stats/stats-server/tests/it/common.rs

@@ -48,6 +48,7 @@ pub fn get_test_stats_settings(
     settings.db_url = stats_db.db_url();
     settings.blockscout_db_url = blockscout_db.db_url();
     settings.blockscout_api_url = Some(url::Url::from_str(&blockscout_api.uri()).unwrap());
+    settings.enable_all_arbitrum = true;
     (settings, base)
 }