From f42a961d86d27660e046db9cddd3cdce42a5e242 Mon Sep 17 00:00:00 2001 From: Kirill Ivanov <8144358+bragov4ik@users.noreply.github.com> Date: Mon, 27 Jan 2025 21:24:14 +0900 Subject: [PATCH] feat(stats): arbitrum main page charts (#1210) * add to interface * save 3 seconds on each just test-with-db * add new charts * fix window charts incorrectly updated * remove excess line chart from layout * add the charts everywhere + update configs etc. * fix index status req for new charts * fix warn when arbitrum is disabled * fix warn when calculating yesterday oper txns * test disabled arbitrum * fix warns * clippy --- stats/config/charts.json | 10 + stats/config/layout.json | 3 +- stats/config/update_groups.json | 1 - stats/justfile | 10 +- stats/stats-proto/proto/stats.proto | 5 +- stats/stats-proto/swagger/stats.swagger.yaml | 6 + stats/stats-server/src/read_service.rs | 41 +++- stats/stats-server/src/runtime_setup.rs | 3 +- stats/stats-server/src/settings.rs | 6 +- .../tests/it/chart_endpoints/counters.rs | 1 + .../tests/it/chart_endpoints/lines.rs | 4 +- .../tests/it/chart_endpoints/main_page.rs | 47 ++-- .../tests/it/chart_endpoints/mod.rs | 27 ++- stats/stats-server/tests/it/common.rs | 1 + stats/stats/src/charts/counters/mod.rs | 10 +- .../charts/counters/total_operational_txns.rs | 47 ++-- .../counters/yesterday_operational_txns.rs | 89 +++++++ .../src/charts/counters/yesterday_txns.rs | 45 ++-- stats/stats/src/charts/lines/mod.rs | 16 +- .../src/charts/lines/new_operational_txns.rs | 10 +- .../lines/new_operational_txns_window.rs | 226 ++++++++++++++++++ .../stats/src/charts/lines/new_txns_window.rs | 17 +- .../update/batching/parameters/mod.rs | 23 +- .../parameters/update/clear_and_query_all.rs | 70 ++++++ .../kinds/local_db/parameters/update/mod.rs | 35 +++ stats/stats/src/update_groups.rs | 33 ++- 26 files changed, 661 insertions(+), 125 deletions(-) create mode 100644 stats/stats/src/charts/counters/yesterday_operational_txns.rs create mode 100644 stats/stats/src/charts/lines/new_operational_txns_window.rs create mode 100644 stats/stats/src/data_source/kinds/local_db/parameters/update/clear_and_query_all.rs diff --git a/stats/config/charts.json b/stats/config/charts.json index e1d585d0e..66340dec4 100644 --- a/stats/config/charts.json +++ b/stats/config/charts.json @@ -65,6 +65,11 @@ "title": "Yesterday txns", "description": "Number of transactions yesterday (0:00 - 23:59 UTC)" }, + "yesterday_operational_txns": { + "enabled": false, + "title": "Yesterday operational txns", + "description": "Number of transactions yesterday (0:00 - 23:59 UTC) without block creation transactions" + }, "last_new_contracts": { "title": "Number of contracts today", "description": "Number of deployed contracts today (UTC)" @@ -150,6 +155,11 @@ "title": "New operational transactions", "description": "Number of new transactions without block creation" }, + "new_operational_txns_window": { + "enabled": false, + "title": "Daily operational transactions", + "description": "The chart displays daily transactions for the past 30 days (without block creation transactions)" + }, "operational_txns_growth": { "enabled": false, "title": "Number of operational transactions", diff --git a/stats/config/layout.json b/stats/config/layout.json index 576635341..b9e115b87 100644 --- a/stats/config/layout.json +++ b/stats/config/layout.json @@ -42,8 +42,7 @@ "txns_growth", "new_operational_txns", "operational_txns_growth", - "txns_success_rate", - "new_txns_window" + "txns_success_rate" ] }, { diff --git a/stats/config/update_groups.json b/stats/config/update_groups.json index cfa0370a0..6deba6864 100644 --- a/stats/config/update_groups.json +++ b/stats/config/update_groups.json @@ -8,7 +8,6 @@ "total_blocks_group": "0 0 */2 * * * *", "total_tokens_group": "0 0 18 * * * *", "total_txns_group": "0 5 */2 * * * *", - "total_operational_txns_group": "0 5 1 * * * *", "yesterday_txns_group": "0 8 0 * * * *", "new_txns_window_group": "0 8 0 * * * *", "active_recurring_accounts_daily_recurrence_60_days_group": "0 0 2 * * * *", diff --git a/stats/justfile b/stats/justfile index cb7f40c5a..3debdd3c7 100644 --- a/stats/justfile +++ b/stats/justfile @@ -27,10 +27,18 @@ test *args: test-with-db *args: # remove db from previous run (if failed) -just docker-name="{{docker-name}}-test" stop-postgres 2> /dev/null - -just db-port="{{test-db-port}}" db-name="" docker-name="{{docker-name}}-test" start-postgres + just start-postgres-and-build-tests just db-port="{{test-db-port}}" db-name="" test {{args}} just docker-name="{{docker-name}}-test" stop-postgres +start-postgres-and-build-tests: + #!/bin/bash -eux + just db-port="{{test-db-port}}" db-name="" docker-name="{{docker-name}}-test" start-postgres & + cargo build --tests & + trap 'kill $(jobs -pr) || true' EXIT + wait + echo "finished postgres wait" + check-envs: VALIDATE_ONLY=true cargo run --bin env-docs-generation diff --git a/stats/stats-proto/proto/stats.proto b/stats/stats-proto/proto/stats.proto index 9ac85da6e..89be8260c 100644 --- a/stats/stats-proto/proto/stats.proto +++ b/stats/stats-proto/proto/stats.proto @@ -84,8 +84,11 @@ message MainPageStats { optional Counter total_blocks = 3; optional Counter total_transactions = 4; optional Counter yesterday_transactions = 5; + optional Counter total_operational_transactions = 6; + optional Counter yesterday_operational_transactions = 7; - optional LineChart daily_new_transactions = 6; + optional LineChart daily_new_transactions = 8; + optional LineChart daily_new_operational_transactions = 9; } message GetTransactionsPageStatsRequest {} diff --git a/stats/stats-proto/swagger/stats.swagger.yaml b/stats/stats-proto/swagger/stats.swagger.yaml index f9970c6d3..63a197876 100644 --- a/stats/stats-proto/swagger/stats.swagger.yaml +++ b/stats/stats-proto/swagger/stats.swagger.yaml @@ -266,8 +266,14 @@ definitions: $ref: '#/definitions/v1Counter' yesterday_transactions: $ref: '#/definitions/v1Counter' + total_operational_transactions: + $ref: '#/definitions/v1Counter' + yesterday_operational_transactions: + $ref: '#/definitions/v1Counter' daily_new_transactions: $ref: '#/definitions/v1LineChart' + daily_new_operational_transactions: + $ref: '#/definitions/v1LineChart' v1Point: type: object properties: diff --git a/stats/stats-server/src/read_service.rs b/stats/stats-server/src/read_service.rs index 0787bdf2b..929cbadf7 100644 --- a/stats/stats-server/src/read_service.rs +++ b/stats/stats-server/src/read_service.rs @@ -17,11 +17,11 @@ use sea_orm::{DatabaseConnection, DbErr}; use stats::{ counters::{ AverageBlockTime, AverageTxnFee24h, NewContracts24h, NewTxns24h, NewVerifiedContracts24h, - PendingTxns30m, TotalAddresses, TotalBlocks, TotalContracts, TotalTxns, - TotalVerifiedContracts, TxnsFee24h, YesterdayTxns, + PendingTxns30m, TotalAddresses, TotalBlocks, TotalContracts, TotalOperationalTxns, + TotalTxns, TotalVerifiedContracts, TxnsFee24h, YesterdayOperationalTxns, YesterdayTxns, }, data_source::{types::BlockscoutMigrations, UpdateContext, UpdateParameters}, - lines::{NewTxnsWindow, NEW_TXNS_WINDOW_RANGE}, + lines::{NewOperationalTxnsWindow, NewTxnsWindow, NEW_TXNS_WINDOW_RANGE}, query_dispatch::{CounterHandle, LineHandle, QuerySerializedDyn}, range::UniversalRange, types::{Timespan, TimespanDuration}, @@ -146,7 +146,10 @@ impl ReadService { total_blocks: None, total_transactions: None, yesterday_transactions: None, + total_operational_transactions: None, + yesterday_operational_transactions: None, daily_new_transactions: None, + daily_new_operational_transactions: None, }; vec![ AverageBlockTime::name(), @@ -154,7 +157,10 @@ impl ReadService { TotalBlocks::name(), TotalTxns::name(), YesterdayTxns::name(), + TotalOperationalTxns::name(), + YesterdayOperationalTxns::name(), NewTxnsWindow::name(), + NewOperationalTxnsWindow::name(), ] } @@ -313,11 +319,18 @@ impl ReadService { Ok(chart_data) } - async fn query_new_txns_window( + async fn query_window_chart( &self, + name: String, + window_range: u64, query_time: DateTime, ) -> Option { - // All `NEW_TXNS_WINDOW_RANGE` should be returned, + // `query_line_chart` will result in warn here even when querying a disabled chart. + if !self.charts.charts_info.contains_key(&name) { + return None; + } + + // All `window_range` should be returned, // therefore we need to set exact query range to fill // zeroes (if any) @@ -325,19 +338,18 @@ impl ReadService { // overshoot by two to account for // - last point being approximate // - chart last updated yesterday - let range_start = - query_day.saturating_sub(TimespanDuration::from_days(NEW_TXNS_WINDOW_RANGE + 1)); + let range_start = query_day.saturating_sub(TimespanDuration::from_days(window_range + 1)); let request_range = inclusive_date_range_to_query_range(Some(range_start), Some(query_day)); let mut transactions = self .query_line_chart( - NewTxnsWindow::name(), + name.clone(), ResolutionKind::Day, request_range, None, query_time, ) .await - .inspect_err(|e| tracing::warn!("Couldn't get transactions for main page: {}", e)) + .inspect_err(|e| tracing::warn!("Couldn't get {} for the main page: {}", name, e)) .ok()?; // return exactly `NEW_TXNS_WINDOW_RANGE` accurate points let data = transactions @@ -429,14 +441,20 @@ impl StatsService for ReadService { total_blocks, total_transactions, yesterday_transactions, + total_operational_transactions, + yesterday_operational_transactions, daily_new_transactions, + daily_new_operational_transactions, ) = join!( self.query_counter(AverageBlockTime::name(), now), self.query_counter(TotalAddresses::name(), now), self.query_counter(TotalBlocks::name(), now), self.query_counter(TotalTxns::name(), now), self.query_counter(YesterdayTxns::name(), now), - self.query_new_txns_window(now) + self.query_counter(TotalOperationalTxns::name(), now), + self.query_counter(YesterdayOperationalTxns::name(), now), + self.query_window_chart(NewTxnsWindow::name(), NEW_TXNS_WINDOW_RANGE, now), + self.query_window_chart(NewOperationalTxnsWindow::name(), NEW_TXNS_WINDOW_RANGE, now), ); Ok(Response::new(proto_v1::MainPageStats { @@ -445,7 +463,10 @@ impl StatsService for ReadService { total_blocks, total_transactions, yesterday_transactions, + total_operational_transactions, + yesterday_operational_transactions, daily_new_transactions, + daily_new_operational_transactions, })) } diff --git a/stats/stats-server/src/runtime_setup.rs b/stats/stats-server/src/runtime_setup.rs index 2dce6ce3e..a148b85c3 100644 --- a/stats/stats-server/src/runtime_setup.rs +++ b/stats/stats-server/src/runtime_setup.rs @@ -308,7 +308,6 @@ impl RuntimeSetup { Arc::new(TotalBlocksGroup), Arc::new(TotalTokensGroup), Arc::new(TotalTxnsGroup), - Arc::new(TotalOperationalTxnsGroup), Arc::new(YesterdayTxnsGroup), Arc::new(ActiveRecurringAccountsDailyRecurrence60DaysGroup), Arc::new(ActiveRecurringAccountsMonthlyRecurrence60DaysGroup), @@ -412,6 +411,8 @@ impl RuntimeSetup { // compute, therefore this solution is ok (to not introduce // more update groups if not necessary) ("NewBlocksGroup", vec!["newTxns_DAY"]), + // Same logic as above + ("TotalBlocksGroup", vec!["totalTxns_DAY"]), ] .map(|(group_name, allowed_missing)| { ( diff --git a/stats/stats-server/src/settings.rs b/stats/stats-server/src/settings.rs index 10e90b984..88acb45c9 100644 --- a/stats/stats-server/src/settings.rs +++ b/stats/stats-server/src/settings.rs @@ -8,8 +8,8 @@ use cron::Schedule; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, DisplayFromStr}; use stats::{ - counters::TotalOperationalTxns, - lines::{NewOperationalTxns, OperationalTxnsGrowth}, + counters::{TotalOperationalTxns, YesterdayOperationalTxns}, + lines::{NewOperationalTxns, NewOperationalTxnsWindow, OperationalTxnsGrowth}, ChartProperties, IndexingStatus, }; use std::{collections::BTreeSet, net::SocketAddr, path::PathBuf, str::FromStr}; @@ -143,6 +143,8 @@ pub fn handle_enable_all_arbitrum( NewOperationalTxns::key().name(), OperationalTxnsGrowth::key().name(), TotalOperationalTxns::key().name(), + YesterdayOperationalTxns::key().name(), + NewOperationalTxnsWindow::key().name(), ] { let settings = match ( charts.lines.get_mut(enable_key), diff --git a/stats/stats-server/tests/it/chart_endpoints/counters.rs b/stats/stats-server/tests/it/chart_endpoints/counters.rs index 8acf5beba..70750fccd 100644 --- a/stats/stats-server/tests/it/chart_endpoints/counters.rs +++ b/stats/stats-server/tests/it/chart_endpoints/counters.rs @@ -28,6 +28,7 @@ pub async fn test_counters_ok(base: Url) { "totalNativeCoinTransfers", "totalTokens", "totalTxns", + "totalOperationalTxns", "totalVerifiedContracts", // on a different page // "yesterdayTxns", diff --git a/stats/stats-server/tests/it/chart_endpoints/lines.rs b/stats/stats-server/tests/it/chart_endpoints/lines.rs index 675718899..df2619511 100644 --- a/stats/stats-server/tests/it/chart_endpoints/lines.rs +++ b/stats/stats-server/tests/it/chart_endpoints/lines.rs @@ -52,8 +52,8 @@ pub async fn test_lines_ok(base: Url) { // "newTxnsWindow", "txnsFee", "txnsGrowth", - // "newOperationalTxns", - // "operationalTxnsGrowth", + "newOperationalTxns", + "operationalTxnsGrowth", "txnsSuccessRate", "newVerifiedContracts", "newContracts", diff --git a/stats/stats-server/tests/it/chart_endpoints/main_page.rs b/stats/stats-server/tests/it/chart_endpoints/main_page.rs index f184b18bf..c9b160227 100644 --- a/stats/stats-server/tests/it/chart_endpoints/main_page.rs +++ b/stats/stats-server/tests/it/chart_endpoints/main_page.rs @@ -1,16 +1,13 @@ use blockscout_service_launcher::test_server::send_get_request; use pretty_assertions::assert_eq; -use stats::{ - lines::{NewTxnsWindow, NEW_TXNS_WINDOW_RANGE}, - Named, -}; +use stats::lines::NEW_TXNS_WINDOW_RANGE; use stats_proto::blockscout::stats::v1::MainPageStats; use url::Url; use crate::array_of_variables_with_names; -pub async fn test_main_page_ok(base: Url) { +pub async fn test_main_page_ok(base: Url, expect_arbitrum: bool) { let main_page: MainPageStats = send_get_request(&base, "/api/v1/pages/main").await; let MainPageStats { average_block_time, @@ -18,29 +15,43 @@ pub async fn test_main_page_ok(base: Url) { total_blocks, total_transactions, yesterday_transactions, + total_operational_transactions, + yesterday_operational_transactions, daily_new_transactions, + daily_new_operational_transactions, } = main_page; - let counters = array_of_variables_with_names!([ + let mut counters = array_of_variables_with_names!([ average_block_time, total_addresses, total_blocks, total_transactions, yesterday_transactions, - ]); + ]) + .to_vec(); + if expect_arbitrum { + counters.extend(array_of_variables_with_names!([ + total_operational_transactions, + yesterday_operational_transactions, + ])); + } for (name, counter) in counters { - #[allow(clippy::expect_fun_call)] - let counter = counter.expect(&format!("page counter {} must be available", name)); + let counter = counter.unwrap_or_else(|| panic!("page counter {} must be available", name)); assert!(!counter.description.is_empty()); assert!(!counter.title.is_empty()); } - let daily_new_transactions = - daily_new_transactions.expect("daily new transactions chart must be available"); - let transactions_info = daily_new_transactions.info.unwrap(); - assert_eq!(transactions_info.id, NewTxnsWindow::name()); - assert_eq!(transactions_info.resolutions, vec!["DAY"]); - assert_eq!( - daily_new_transactions.chart.len(), - NEW_TXNS_WINDOW_RANGE as usize - ); + let mut window_line_charts = array_of_variables_with_names!([daily_new_transactions]).to_vec(); + if expect_arbitrum { + window_line_charts.extend(array_of_variables_with_names!([ + daily_new_operational_transactions + ])); + } + for (name, window_chart) in window_line_charts { + let window_chart = + window_chart.unwrap_or_else(|| panic!("{} chart must be available", name)); + let transactions_info = window_chart.info.unwrap(); + assert!(!transactions_info.id.is_empty()); + assert_eq!(transactions_info.resolutions, vec!["DAY"]); + assert_eq!(window_chart.chart.len(), NEW_TXNS_WINDOW_RANGE as usize); + } } diff --git a/stats/stats-server/tests/it/chart_endpoints/mod.rs b/stats/stats-server/tests/it/chart_endpoints/mod.rs index e48f0fce3..062d28740 100644 --- a/stats/stats-server/tests/it/chart_endpoints/mod.rs +++ b/stats/stats-server/tests/it/chart_endpoints/mod.rs @@ -46,12 +46,12 @@ async fn test_chart_endpoints_ok() { init_server(|| stats(settings), &base).await; // Sleep until server will start and calculate all values - tokio::time::sleep(std::time::Duration::from_secs(7)).await; + tokio::time::sleep(std::time::Duration::from_secs(8)).await; let tests: JoinSet<_> = [ test_lines_ok(base.clone()).boxed(), test_counters_ok(base.clone()).boxed(), - test_main_page_ok(base.clone()).boxed(), + test_main_page_ok(base.clone(), true).boxed(), test_transactions_page_ok(base.clone()).boxed(), test_contracts_page_ok(base).boxed(), ] @@ -81,10 +81,10 @@ async fn test_chart_endpoints_work_with_not_indexed_blockscout() { init_server(|| stats(settings), &base).await; // Sleep until server will start and calculate all values - tokio::time::sleep(std::time::Duration::from_secs(7)).await; + tokio::time::sleep(std::time::Duration::from_secs(8)).await; let tests: JoinSet<_> = [ - test_main_page_ok(base.clone()).boxed(), + test_main_page_ok(base.clone(), true).boxed(), test_transactions_page_ok(base.clone()).boxed(), test_contracts_page_ok(base).boxed(), ] @@ -92,3 +92,22 @@ async fn test_chart_endpoints_work_with_not_indexed_blockscout() { .collect(); run_consolidated_tests(tests, test_name).await; } + +#[tokio::test] +#[ignore = "needs database"] +async fn test_chart_endpoints_work_with_disabled_arbitrum() { + let test_name = "test_chart_endpoints_work_with_disabled_arbitrum"; + let (stats_db, blockscout_db) = init_db_all(test_name).await; + let blockscout_api = default_mock_blockscout_api().await; + fill_mock_blockscout_data(&blockscout_db, NaiveDate::from_str("2023-03-01").unwrap()).await; + std::env::set_var("STATS__CONFIG", "./tests/config/test.toml"); + let (mut settings, base) = get_test_stats_settings(&stats_db, &blockscout_db, &blockscout_api); + settings.enable_all_arbitrum = false; + + init_server(|| stats(settings), &base).await; + + // Sleep until server will start and calculate all values + tokio::time::sleep(std::time::Duration::from_secs(8)).await; + + test_main_page_ok(base, false).await; +} diff --git a/stats/stats-server/tests/it/common.rs b/stats/stats-server/tests/it/common.rs index cf0036574..bf1114f15 100644 --- a/stats/stats-server/tests/it/common.rs +++ b/stats/stats-server/tests/it/common.rs @@ -48,6 +48,7 @@ pub fn get_test_stats_settings( settings.db_url = stats_db.db_url(); settings.blockscout_db_url = blockscout_db.db_url(); settings.blockscout_api_url = Some(url::Url::from_str(&blockscout_api.uri()).unwrap()); + settings.enable_all_arbitrum = true; (settings, base) } diff --git a/stats/stats/src/charts/counters/mod.rs b/stats/stats/src/charts/counters/mod.rs index bb697d4df..ec1cb4883 100644 --- a/stats/stats/src/charts/counters/mod.rs +++ b/stats/stats/src/charts/counters/mod.rs @@ -20,6 +20,7 @@ mod yesterday_txns; #[cfg(test)] mod mock; +mod yesterday_operational_txns; pub use average_block_time::AverageBlockTime; pub use completed_txns::CompletedTxns; @@ -30,19 +31,24 @@ pub use new_verified_contracts_24h::NewVerifiedContracts24h; pub use pending_txns::PendingTxns30m; pub use total_accounts::TotalAccounts; pub use total_addresses::TotalAddresses; -pub use total_blocks::{TotalBlocks, TotalBlocksInt}; +pub use total_blocks::TotalBlocks; pub use total_contracts::TotalContracts; pub use total_native_coin_holders::TotalNativeCoinHolders; pub use total_native_coin_transfers::TotalNativeCoinTransfers; pub use total_operational_txns::TotalOperationalTxns; pub use total_tokens::TotalTokens; -pub use total_txns::{TotalTxns, TotalTxnsInt}; +pub use total_txns::TotalTxns; pub use total_verified_contracts::TotalVerifiedContracts; pub(crate) use txns_stats_24h::TxnsStatsValue; pub use txns_stats_24h::{ average_txn_fee_24h::AverageTxnFee24h, new_txns_24h::NewTxns24h, txns_fee_24h::TxnsFee24h, }; +pub use yesterday_operational_txns::YesterdayOperationalTxns; pub use yesterday_txns::YesterdayTxns; +pub(crate) use total_blocks::TotalBlocksInt; +pub(crate) use total_operational_txns::CalculateOperationalTxns; +pub(crate) use total_txns::TotalTxnsInt; + #[cfg(test)] pub use mock::MockCounter; diff --git a/stats/stats/src/charts/counters/total_operational_txns.rs b/stats/stats/src/charts/counters/total_operational_txns.rs index c13f52135..9503b85af 100644 --- a/stats/stats/src/charts/counters/total_operational_txns.rs +++ b/stats/stats/src/charts/counters/total_operational_txns.rs @@ -1,14 +1,14 @@ +use std::marker::PhantomData; + use crate::{ - data_source::{ - kinds::{ - data_manipulation::map::{Map, MapFunction}, - local_db::DirectPointLocalDbChartSource, - }, - DataSource, + data_source::kinds::{ + data_manipulation::map::{Map, MapFunction}, + local_db::DirectPointLocalDbChartSource, }, types::TimespanValue, - ChartProperties, MissingDatePolicy, Named, + ChartProperties, IndexingStatus, MissingDatePolicy, Named, }; +use std::fmt::Debug; use chrono::NaiveDate; use entity::sea_orm_active_enums::ChartType; @@ -34,22 +34,33 @@ impl ChartProperties for Properties { fn missing_date_policy() -> MissingDatePolicy { MissingDatePolicy::FillPrevious } + + fn indexing_status_requirement() -> IndexingStatus { + IndexingStatus::NoneIndexed + } } -pub struct Calculate; +pub struct CalculateOperationalTxns(PhantomData); -type Input = ( - ::Output, - ::Output, +type Input = ( + // blocks + TimespanValue, + // all transactions + TimespanValue, ); -impl MapFunction for Calculate { - type Output = TimespanValue; +impl MapFunction> for CalculateOperationalTxns +where + Resolution: Debug + PartialEq + Send, + ChartName: Named, +{ + type Output = TimespanValue; - fn function(inner_data: Input) -> Result { + fn function(inner_data: Input) -> Result { let (total_blocks_data, total_txns_data) = inner_data; if total_blocks_data.timespan != total_txns_data.timespan { - warn!("timespans for total blocks and total transactions do not match when calculating {}", Properties::name()); + warn!("timespans for total blocks and total transactions do not match when calculating {}: \ + {:?} != {:?}", ChartName::name(), total_blocks_data.timespan, total_txns_data.timespan); } let date = total_blocks_data.timespan; let value = total_txns_data @@ -62,8 +73,10 @@ impl MapFunction for Calculate { } } -pub type TotalOperationalTxns = - DirectPointLocalDbChartSource, Properties>; +pub type TotalOperationalTxns = DirectPointLocalDbChartSource< + Map<(TotalBlocksInt, TotalTxnsInt), CalculateOperationalTxns>, + Properties, +>; #[cfg(test)] mod tests { diff --git a/stats/stats/src/charts/counters/yesterday_operational_txns.rs b/stats/stats/src/charts/counters/yesterday_operational_txns.rs new file mode 100644 index 000000000..91ea1de76 --- /dev/null +++ b/stats/stats/src/charts/counters/yesterday_operational_txns.rs @@ -0,0 +1,89 @@ +use crate::{ + data_source::{ + kinds::{ + data_manipulation::map::{Map, MapParseTo}, + local_db::DirectPointLocalDbChartSource, + remote_db::{RemoteDatabaseSource, RemoteQueryBehaviour}, + }, + UpdateContext, + }, + lines::NewBlocksStatement, + range::UniversalRange, + types::TimespanValue, + ChartError, ChartProperties, IndexingStatus, MissingDatePolicy, Named, +}; + +use chrono::{DateTime, NaiveDate, Utc}; +use entity::sea_orm_active_enums::ChartType; + +use super::{ + yesterday_txns::{query_yesterday_data, YesterdayTxnsInt}, + CalculateOperationalTxns, +}; + +pub struct YesterdayBlocksQuery; + +impl RemoteQueryBehaviour for YesterdayBlocksQuery { + type Output = TimespanValue; + + async fn query_data( + cx: &UpdateContext<'_>, + _range: UniversalRange>, + ) -> Result { + let today = cx.time.date_naive(); + query_yesterday_data::(cx, today).await + } +} + +pub type YesterdayBlocksRemote = RemoteDatabaseSource; + +// should only be used in this chart for query efficiency. +// because is not directly stored in local DB. +pub type YesterdayBlocksRemoteInt = MapParseTo; + +pub struct Properties; + +impl Named for Properties { + fn name() -> String { + "yesterdayOperationalTxns".into() + } +} + +impl ChartProperties for Properties { + type Resolution = NaiveDate; + + fn chart_type() -> ChartType { + ChartType::Counter + } + + fn missing_date_policy() -> MissingDatePolicy { + MissingDatePolicy::FillPrevious + } + + fn indexing_status_requirement() -> IndexingStatus { + IndexingStatus::NoneIndexed + } +} + +pub type YesterdayOperationalTxns = DirectPointLocalDbChartSource< + Map<(YesterdayBlocksRemoteInt, YesterdayTxnsInt), CalculateOperationalTxns>, + Properties, +>; + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::{point_construction::dt, simple_test::simple_test_counter}; + + #[tokio::test] + #[ignore = "needs database to run"] + async fn update_yesterday_operational_txns() { + // 12 - 3 (txns - blocks) + simple_test_counter::( + "update_yesterday_operational_txns", + "9", + Some(dt("2022-11-11T00:00:00")), + ) + .await; + } +} diff --git a/stats/stats/src/charts/counters/yesterday_txns.rs b/stats/stats/src/charts/counters/yesterday_txns.rs index 8081e8fb7..e0f4d9687 100644 --- a/stats/stats/src/charts/counters/yesterday_txns.rs +++ b/stats/stats/src/charts/counters/yesterday_txns.rs @@ -1,6 +1,7 @@ use crate::{ data_source::{ kinds::{ + data_manipulation::map::MapParseTo, local_db::DirectPointLocalDbChartSource, remote_db::{RemoteDatabaseSource, RemoteQueryBehaviour, StatementFromRange}, }, @@ -16,6 +17,31 @@ use chrono::{DateTime, Days, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; use sea_orm::FromQueryResult; +// `DailyDataStatement` is assumed to have [`MissingDatePolicy::FillZero`] +pub(crate) async fn query_yesterday_data( + cx: &UpdateContext<'_>, + today: NaiveDate, +) -> Result, ChartError> { + let yesterday = today + .checked_sub_days(Days::new(1)) + .ok_or(ChartError::Internal( + "Update time is incorrect: ~ minimum possible date".into(), + ))?; + let yesterday_range = day_start(&yesterday)..day_start(&today); + let query = + DailyDataStatement::get_statement(Some(yesterday_range), &cx.blockscout_applied_migrations); + let mut data = TimespanValue::::find_by_statement(query) + .one(cx.blockscout) + .await + .map_err(ChartError::BlockscoutDB)? + // no data for yesterday + .unwrap_or(TimespanValue::with_zero_value(yesterday)); + // today's value is the number from the day before. + // still a value is considered to be "for today" (technically) + data.timespan = today; + Ok(data) +} + pub struct YesterdayTxnsQuery; impl RemoteQueryBehaviour for YesterdayTxnsQuery { @@ -26,23 +52,7 @@ impl RemoteQueryBehaviour for YesterdayTxnsQuery { _range: UniversalRange>, ) -> Result { let today = cx.time.date_naive(); - let yesterday = today - .checked_sub_days(Days::new(1)) - .ok_or(ChartError::Internal( - "Update time is incorrect: ~ minimum possible date".into(), - ))?; - let yesterday_range = day_start(&yesterday)..day_start(&today); - let query = NewTxnsStatement::get_statement( - Some(yesterday_range), - &cx.blockscout_applied_migrations, - ); - let data = Self::Output::find_by_statement(query) - .one(cx.blockscout) - .await - .map_err(ChartError::BlockscoutDB)? - // no transactions for yesterday - .unwrap_or(TimespanValue::with_zero_value(yesterday)); - Ok(data) + query_yesterday_data::(cx, today).await } } @@ -71,6 +81,7 @@ impl ChartProperties for Properties { } pub type YesterdayTxns = DirectPointLocalDbChartSource; +pub type YesterdayTxnsInt = MapParseTo; #[cfg(test)] mod tests { diff --git a/stats/stats/src/charts/lines/mod.rs b/stats/stats/src/charts/lines/mod.rs index 9b02295dc..de8c8bb77 100644 --- a/stats/stats/src/charts/lines/mod.rs +++ b/stats/stats/src/charts/lines/mod.rs @@ -17,6 +17,7 @@ mod new_contracts; mod new_native_coin_holders; mod new_native_coin_transfers; mod new_operational_txns; +mod new_operational_txns_window; mod new_txns; mod new_txns_window; mod new_verified_contracts; @@ -80,7 +81,6 @@ pub use native_coin_supply::{ NativeCoinSupply, NativeCoinSupplyMonthly, NativeCoinSupplyWeekly, NativeCoinSupplyYearly, }; pub use new_accounts::{NewAccounts, NewAccountsMonthly, NewAccountsWeekly, NewAccountsYearly}; -pub use new_block_rewards::{NewBlockRewardsInt, NewBlockRewardsMonthlyInt}; pub use new_blocks::{NewBlocks, NewBlocksMonthly, NewBlocksWeekly, NewBlocksYearly}; pub use new_contracts::{ NewContracts, NewContractsMonthly, NewContractsWeekly, NewContractsYearly, @@ -90,15 +90,15 @@ pub use new_native_coin_holders::{ NewNativeCoinHoldersYearly, }; pub use new_native_coin_transfers::{ - NewNativeCoinTransfers, NewNativeCoinTransfersInt, NewNativeCoinTransfersMonthly, - NewNativeCoinTransfersWeekly, NewNativeCoinTransfersYearly, + NewNativeCoinTransfers, NewNativeCoinTransfersMonthly, NewNativeCoinTransfersWeekly, + NewNativeCoinTransfersYearly, }; pub use new_operational_txns::{ NewOperationalTxns, NewOperationalTxnsMonthly, NewOperationalTxnsWeekly, NewOperationalTxnsYearly, }; -pub(crate) use new_txns::NewTxnsStatement; -pub use new_txns::{NewTxns, NewTxnsInt, NewTxnsMonthly, NewTxnsWeekly, NewTxnsYearly}; +pub use new_operational_txns_window::NewOperationalTxnsWindow; +pub use new_txns::{NewTxns, NewTxnsMonthly, NewTxnsWeekly, NewTxnsYearly}; pub use new_txns_window::NewTxnsWindow; pub use new_verified_contracts::{ NewVerifiedContracts, NewVerifiedContractsMonthly, NewVerifiedContractsWeekly, @@ -118,5 +118,11 @@ pub use verified_contracts_growth::{ VerifiedContractsGrowthYearly, }; +pub(crate) use new_block_rewards::{NewBlockRewardsInt, NewBlockRewardsMonthlyInt}; +pub(crate) use new_blocks::NewBlocksStatement; +pub(crate) use new_native_coin_transfers::NewNativeCoinTransfersInt; +pub(crate) use new_txns::{NewTxnsInt, NewTxnsStatement}; +pub(crate) use new_txns_window::NewTxnsWindowInt; + #[cfg(test)] pub use mock::{PredefinedMockSource, PseudoRandomMockLine, PseudoRandomMockRetrieve}; diff --git a/stats/stats/src/charts/lines/new_operational_txns.rs b/stats/stats/src/charts/lines/new_operational_txns.rs index 470f67937..b68879e66 100644 --- a/stats/stats/src/charts/lines/new_operational_txns.rs +++ b/stats/stats/src/charts/lines/new_operational_txns.rs @@ -1,3 +1,5 @@ +use std::fmt::Debug; + use crate::{ data_processing::zip_same_timespan, data_source::kinds::{ @@ -46,7 +48,7 @@ impl ChartProperties for Properties { } } -pub struct Calculate; +pub struct CalculateOperationalTxnsVec; type Input = ( // newBlocks @@ -55,9 +57,9 @@ type Input = ( Vec>, ); -impl MapFunction> for Calculate +impl MapFunction> for CalculateOperationalTxnsVec where - Resolution: Timespan + Send + Ord, + Resolution: Timespan + Send + Ord + Debug, { type Output = Vec>; @@ -89,7 +91,7 @@ define_and_impl_resolution_properties!( ); pub type NewOperationalTxns = DirectVecLocalDbChartSource< - Map<(NewBlocksInt, NewTxnsInt), Calculate>, + Map<(NewBlocksInt, NewTxnsInt), CalculateOperationalTxnsVec>, BatchMaxDays, Properties, >; diff --git a/stats/stats/src/charts/lines/new_operational_txns_window.rs b/stats/stats/src/charts/lines/new_operational_txns_window.rs new file mode 100644 index 000000000..1dad1764b --- /dev/null +++ b/stats/stats/src/charts/lines/new_operational_txns_window.rs @@ -0,0 +1,226 @@ +//! New operational transactions for the last N days +//! (usually 30). +//! +//! Basically an extension of [super::NewTxnsWindow] +//! but for operational txns + +use crate::{ + data_source::{ + kinds::{ + data_manipulation::map::{Map, MapParseTo}, + local_db::{ + parameters::{ + update::clear_and_query_all::ClearAllAndPassVec, DefaultCreate, DefaultQueryVec, + }, + LocalDbChartSource, + }, + remote_db::{RemoteDatabaseSource, RemoteQueryBehaviour, StatementFromRange}, + }, + types::BlockscoutMigrations, + UpdateContext, + }, + range::UniversalRange, + types::{Timespan, TimespanDuration, TimespanValue}, + utils::day_start, + ChartError, ChartProperties, IndexingStatus, Named, +}; + +use chrono::{DateTime, NaiveDate, Utc}; +use entity::sea_orm_active_enums::ChartType; +use sea_orm::{FromQueryResult, Statement}; + +use super::{ + new_operational_txns::CalculateOperationalTxnsVec, NewBlocksStatement, NewTxnsWindowInt, + NEW_TXNS_WINDOW_RANGE, +}; + +fn new_blocks_window_statement( + update_day: NaiveDate, + completed_migrations: &BlockscoutMigrations, +) -> Statement { + // `update_day` is not included because the data would + // be incomplete. + let window = day_start( + &update_day.saturating_sub(TimespanDuration::from_timespan_repeats( + NEW_TXNS_WINDOW_RANGE, + )), + )..day_start(&update_day); + NewBlocksStatement::get_statement(Some(window), completed_migrations) +} + +pub struct NewBlocksWindowQuery; + +impl RemoteQueryBehaviour for NewBlocksWindowQuery { + type Output = Vec>; + + async fn query_data( + cx: &UpdateContext<'_>, + _range: UniversalRange>, + ) -> Result>, ChartError> { + let update_day = cx.time.date_naive(); + let query = new_blocks_window_statement(update_day, &cx.blockscout_applied_migrations); + let mut data = TimespanValue::::find_by_statement(query) + .all(cx.blockscout) + .await + .map_err(ChartError::BlockscoutDB)?; + // linear time for sorted sequences + data.sort_unstable_by(|a, b| a.timespan.cmp(&b.timespan)); + Ok(data) + } +} + +// should only be used in this chart for query efficiency. +// because is not directly stored in local DB. +pub type NewBlocksWindowRemote = RemoteDatabaseSource; +pub type NewBlocksWindowRemoteInt = MapParseTo; + +pub struct Properties; + +impl Named for Properties { + fn name() -> String { + "newOperationalTxnsWindow".into() + } +} + +impl ChartProperties for Properties { + type Resolution = NaiveDate; + + fn chart_type() -> ChartType { + ChartType::Line + } + + fn indexing_status_requirement() -> IndexingStatus { + IndexingStatus::NoneIndexed + } +} + +pub type NewOperationalTxnsWindowCalculation = + Map<(NewBlocksWindowRemoteInt, NewTxnsWindowInt), CalculateOperationalTxnsVec>; +pub type NewOperationalTxnsWindow = LocalDbChartSource< + NewOperationalTxnsWindowCalculation, + (), + DefaultCreate, + ClearAllAndPassVec< + NewOperationalTxnsWindowCalculation, + DefaultQueryVec, + Properties, + >, + DefaultQueryVec, + Properties, +>; + +#[cfg(test)] +mod tests { + use pretty_assertions::assert_eq; + + use super::*; + use crate::{ + data_source::{DataSource, UpdateParameters}, + query_dispatch::QuerySerialized, + tests::{ + mock_blockscout::{fill_mock_blockscout_data, imitate_reindex}, + point_construction::dt, + simple_test::{chart_output_to_expected, map_str_tuple_to_owned, prepare_chart_test}, + }, + }; + + #[tokio::test] + #[ignore = "needs database to run"] + async fn update_operational_txns_window_clears_and_overwrites() { + let (init_time, db, blockscout) = prepare_chart_test::( + "update_operational_txns_window_clears_and_overwrites", + None, + ) + .await; + { + let current_date = init_time.date_naive(); + fill_mock_blockscout_data(&blockscout, current_date).await; + } + let current_time = dt("2022-12-01T00:00:00").and_utc(); + + let mut parameters = UpdateParameters { + db: &db, + blockscout: &blockscout, + blockscout_applied_migrations: BlockscoutMigrations::latest(), + update_time_override: Some(current_time), + force_full: false, + }; + let cx = UpdateContext::from_params_now_or_override(parameters.clone()); + NewOperationalTxnsWindow::update_recursively(&cx) + .await + .unwrap(); + assert_eq!( + &chart_output_to_expected( + NewOperationalTxnsWindow::query_data_static( + &cx, + UniversalRange::full(), + None, + false + ) + .await + .unwrap() + ), + &map_str_tuple_to_owned(vec![ + ("2022-11-09", "4"), + ("2022-11-10", "9"), + ("2022-11-11", "10"), + ("2022-11-12", "4"), + // update day is not included + ]), + ); + + let current_time = dt("2022-12-10T00:00:00").and_utc(); + parameters.update_time_override = Some(current_time); + let cx = UpdateContext::from_params_now_or_override(parameters.clone()); + NewOperationalTxnsWindow::update_recursively(&cx) + .await + .unwrap(); + assert_eq!( + &chart_output_to_expected( + NewOperationalTxnsWindow::query_data_static( + &cx, + UniversalRange::full(), + None, + false + ) + .await + .unwrap() + ), + &map_str_tuple_to_owned(vec![ + // values outside the window are removed + ("2022-11-10", "9"), + ("2022-11-11", "10"), + ("2022-11-12", "4"), + ("2022-12-01", "4"), + ]), + ); + + imitate_reindex(&blockscout, init_time.date_naive()).await; + + let current_time = dt("2022-12-11T00:00:00").and_utc(); + parameters.update_time_override = Some(current_time); + let cx = UpdateContext::from_params_now_or_override(parameters); + NewOperationalTxnsWindow::update_recursively(&cx) + .await + .unwrap(); + assert_eq!( + &chart_output_to_expected( + NewOperationalTxnsWindow::query_data_static( + &cx, + UniversalRange::full(), + None, + false + ) + .await + .unwrap() + ), + &map_str_tuple_to_owned(vec![ + // values outside the window are removed + // new values within the window are added + ("2022-11-11", "14"), + ("2022-11-12", "4"), + ("2022-12-01", "4"), + ]), + ); + } +} diff --git a/stats/stats/src/charts/lines/new_txns_window.rs b/stats/stats/src/charts/lines/new_txns_window.rs index c1c241e18..ecad545ac 100644 --- a/stats/stats/src/charts/lines/new_txns_window.rs +++ b/stats/stats/src/charts/lines/new_txns_window.rs @@ -8,13 +8,10 @@ use crate::{ data_source::{ kinds::{ + data_manipulation::map::{MapParseTo, StripExt}, local_db::{ parameters::{ - update::batching::{ - parameters::{BatchMaxDays, ClearAllAndPassStep}, - BatchUpdate, - }, - DefaultCreate, DefaultQueryVec, + update::clear_and_query_all::ClearAllAndPassVec, DefaultCreate, DefaultQueryVec, }, LocalDbChartSource, }, @@ -95,17 +92,11 @@ pub type NewTxnsWindow = LocalDbChartSource< NewTxnsWindowRemote, (), DefaultCreate, - BatchUpdate< - NewTxnsWindowRemote, - (), - ClearAllAndPassStep, - BatchMaxDays, - DefaultQueryVec, - Properties, - >, + ClearAllAndPassVec, Properties>, DefaultQueryVec, Properties, >; +pub type NewTxnsWindowInt = MapParseTo, i64>; #[cfg(test)] mod tests { diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/mod.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/mod.rs index 3d65a8545..3169498b8 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/mod.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/mod.rs @@ -1,8 +1,11 @@ +use std::fmt::Debug; + use chrono::{DateTime, NaiveDate, Utc}; use sea_orm::{ConnectionTrait, TransactionTrait}; use crate::{ - charts::db_interaction::write::{clear_all_chart_data, insert_data_many}, + charts::db_interaction::write::clear_all_chart_data, + data_source::kinds::local_db::parameters::update::pass_vec, gettable_const, types::{ timespans::{Month, Week, Year}, @@ -43,21 +46,7 @@ where main_data: Vec>, _resolution_data: (), ) -> Result { - let found = main_data.len(); - // note: right away cloning another chart will not result in exact copy, - // because if the other chart is `FillPrevious`, then omitted starting point - // within the range is set to the last known before the range - // i.e. some duplicate points might get added. - // - // however, it should not be a problem, since semantics remain the same + - // cloning already stored chart is counter-productive/not effective. - let values = main_data - .into_iter() - .map(|value| value.active_model(chart_id, Some(min_blockscout_block))); - insert_data_many(db, values) - .await - .map_err(ChartError::StatsDB)?; - Ok(found) + pass_vec(db, chart_id, min_blockscout_block, main_data).await } } @@ -71,7 +60,7 @@ pub struct ClearAllAndPassStep; impl BatchStepBehaviour>, ()> for ClearAllAndPassStep where - Resolution: Timespan + Clone + Send + Sync, + Resolution: Timespan + Clone + Send + Sync + Debug, { async fn batch_update_values_step_with( db: &C, diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/clear_and_query_all.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/clear_and_query_all.rs new file mode 100644 index 000000000..62829dad4 --- /dev/null +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/clear_and_query_all.rs @@ -0,0 +1,70 @@ +//! Batching for update process. +//! +//! Update for some period P can be done only with dependencies' +//! data for the same exact period P. + +use std::{fmt::Debug, marker::PhantomData, time::Instant}; + +use blockscout_metrics_tools::AggregateTimer; +use sea_orm::TransactionTrait; + +use crate::{ + charts::db_interaction::write::clear_all_chart_data, + data_source::{ + kinds::local_db::{parameter_traits::QueryBehaviour, UpdateBehaviour}, + source::DataSource, + UpdateContext, + }, + range::UniversalRange, + types::{ExtendedTimespanValue, Timespan, TimespanValue}, + ChartError, ChartProperties, +}; + +use super::pass_vec; + +pub struct ClearAllAndPassVec( + PhantomData<(MainDep, Query, ChartProps)>, +) +where + MainDep: DataSource, + Query: QueryBehaviour>>, + ChartProps: ChartProperties; + +impl UpdateBehaviour + for ClearAllAndPassVec +where + MainDep: DataSource>>, + Query: QueryBehaviour>>, + ChartProps: ChartProperties, + ChartProps::Resolution: Timespan + Ord + Clone + Debug + Send + Sync, +{ + async fn update_values( + cx: &UpdateContext<'_>, + chart_id: i32, + _last_accurate_point: Option>, + min_blockscout_block: i64, + dependency_data_fetch_timer: &mut AggregateTimer, + ) -> Result<(), ChartError> { + let now = Instant::now(); + let db = cx.db.begin().await.map_err(ChartError::StatsDB)?; + tracing::info!( + chart =% ChartProps::key(), + "clearing all data and querying from scratch" + ); + clear_all_chart_data(&db, chart_id) + .await + .map_err(ChartError::StatsDB)?; + // updating all at once => full range + let range = UniversalRange::full(); + let main_data = MainDep::query_data(cx, range, dependency_data_fetch_timer).await?; + let found = pass_vec(&db, chart_id, min_blockscout_block, main_data).await?; + tracing::info!( + found =? found, + elapsed =? now.elapsed(), + chart =% ChartProps::key(), + "updated" + ); + db.commit().await.map_err(ChartError::StatsDB)?; + Ok(()) + } +} diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/mod.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/mod.rs index 9035fe624..31f44c134 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/update/mod.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/mod.rs @@ -1,4 +1,39 @@ pub mod batching; +pub mod clear_and_query_all; pub mod point; pub use point::PassPoint; +use sea_orm::{ConnectionTrait, TransactionTrait}; + +use crate::{ + charts::db_interaction::write::insert_data_many, + types::{Timespan, TimespanValue}, + ChartError, +}; + +pub(crate) async fn pass_vec( + db: &C, + chart_id: i32, + min_blockscout_block: i64, + main_data: Vec>, +) -> Result +where + Resolution: Timespan + Clone + Send + Sync, + C: ConnectionTrait + TransactionTrait, +{ + let found = main_data.len(); + // note: right away cloning another chart will not result in exact copy, + // because if the other chart is `FillPrevious`, then omitted starting point + // within the range is set to the last known before the range + // i.e. some duplicate points might get added. + // + // however, it should not be a problem, since semantics remain the same + + // cloning already stored chart is counter-productive/not effective. + let values = main_data + .into_iter() + .map(|value| value.active_model(chart_id, Some(min_blockscout_block))); + insert_data_many(db, values) + .await + .map_err(ChartError::StatsDB)?; + Ok(found) +} diff --git a/stats/stats/src/update_groups.rs b/stats/stats/src/update_groups.rs index 5527754cc..59980c1a4 100644 --- a/stats/stats/src/update_groups.rs +++ b/stats/stats/src/update_groups.rs @@ -1,3 +1,8 @@ +//! It makes sense to focus on the slowest querying charts +//! when making update groups, as we want to synchronize +//! update of other lighter charts to perform simultaneously +//! and reuse the heavy query data. + use crate::{construct_update_group, counters::*, lines::*}; macro_rules! singleton_groups { @@ -22,14 +27,8 @@ singleton_groups!( CompletedTxns, PendingTxns30m, TotalAddresses, - TotalBlocks, TotalTxns, - // Even though it depends on `TotalTxns` and `TotalBlocks`, - // it's ok not to update it as frequently. - // Granular control over these 2 still seems useful. - TotalOperationalTxns, TotalTokens, - YesterdayTxns, // Each of the `ActiveRecurringAccounts*` charts includes quite heavy SQL query, // thus it's better to have granular control on update times. ActiveRecurringAccountsDailyRecurrence60Days, @@ -44,10 +43,24 @@ singleton_groups!( ActiveRecurringAccountsMonthlyRecurrence120Days, ActiveRecurringAccountsWeeklyRecurrence120Days, ActiveRecurringAccountsYearlyRecurrence120Days, - // Standalone chart - NewTxnsWindow, ); +// According to collected metrics, `TotalTxns` has +// very cheap query (a dependency of `TotalOperationalTxns`), +// but `TotalBlocks` doesn't. Also, all these charts +// are placed on the main page, thus they benefit +// from frequent updates. +// +// Therefore, we put the dependant (`TotalOperationalTxns`) in the same +// group as its heaviest dependency (`TotalBlocks`). +construct_update_group!(TotalBlocksGroup { + charts: [TotalBlocks, TotalOperationalTxns] +}); + +construct_update_group!(YesterdayTxnsGroup { + charts: [YesterdayTxns, YesterdayOperationalTxns] +}); + construct_update_group!(AverageBlockRewardsGroup { charts: [ AverageBlockRewards, @@ -187,6 +200,10 @@ construct_update_group!(NewTxnsGroup { ], }); +construct_update_group!(NewTxnsWindowGroup { + charts: [NewTxnsWindow, NewOperationalTxnsWindow], +}); + construct_update_group!(NewVerifiedContractsGroup { charts: [ NewVerifiedContracts,