Merge branch 'main' into shams-db-block-id-rename

CHANGELOG.md

@@ -2,8 +2,10 @@
 
 ## Next release
 
+- fix: rejected transaction block production panic
 - fix(sync): pending block retrying mechanism
 - feat(clean): dc_db: rename `DbBlockId::BlockN` to `DbBlockId::Number`
+- feat(cli): Environment variables can be used to specify Madara parameters
 - fix:(tests): Add testing feature to mc-db dev dependency (#294)
 - feat: new crate gateway client & server
 - test: Starknet-js basic tests added

README.md

@@ -342,12 +342,18 @@ Toggle details for each namespace to view additional settings:
 
 ### Environment Variables
 
-Set up your node's environment variables using the `STARKNET_` prefix. For example:
+Set up your node's environment variables using the `MADARA_` prefix. For example:
 
-- `STARKNET_BASE_PATH=/path/to/data`
-- `STARKNET_LOG=info`
+- `MADARA_BASE_PATH=/path/to/data`
+- `MADARA_RPC_PORT=1111`
 
-These variables allow you to adjust the node's configuration without using command-line arguments.
+These variables allow you to adjust the node's configuration without using command-line arguments. If the command-line
+argument is specified then it takes precedent over the environment variable.
+
+> [!CAUTION]
+> Environment variables can be visible beyond the current process and are not
+> encrypted. You should take special care when setting _secrets_ through
+> environment variables, such as `MADARA_L1_ENDPOINT` or `MADARA_GATEWAY_KEY`
 
 ### Configuration File
 

crates/client/mempool/src/block_production.rs

@@ -1,5 +1,8 @@
 // TODO: Move this into its own crate.
 
+use crate::close_block::close_block;
+use crate::header::make_pending_header;
+use crate::{clone_account_tx, L1DataProvider, MempoolProvider, MempoolTransaction};
 use blockifier::blockifier::transaction_executor::{TransactionExecutor, VisitedSegmentsMapping};
 use blockifier::bouncer::{Bouncer, BouncerWeights, BuiltinCount};
 use blockifier::state::cached_state::CommitmentStateDiff;
@@ -21,15 +24,12 @@ use mp_state_update::{
 use mp_transactions::TransactionWithHash;
 use mp_utils::graceful_shutdown;
 use starknet_types_core::felt::Felt;
+use std::borrow::Cow;
 use std::collections::VecDeque;
 use std::mem;
 use std::sync::Arc;
 use std::time::Instant;
 
-use crate::close_block::close_block;
-use crate::header::make_pending_header;
-use crate::{clone_account_tx, L1DataProvider, MempoolProvider, MempoolTransaction};
-
 #[derive(Default, Clone)]
 struct ContinueBlockStats {
     /// Number of batches executed before reaching the bouncer capacity.
@@ -52,6 +52,8 @@ pub enum Error {
     ExecutionContext(#[from] mc_exec::Error),
     #[error("Import error: {0:#}")]
     Import(#[from] mc_block_import::BlockImportError),
+    #[error("Unexpected error: {0:#}")]
+    Unexpected(Cow<'static, str>),
 }
 
 fn csd_to_state_diff(
@@ -235,8 +237,16 @@ impl<Mempool: MempoolProvider> BlockProductionTask<Mempool> {
         loop {
             // Take transactions from mempool.
             let to_take = batch_size.saturating_sub(txs_to_process.len());
+            let cur_len = txs_to_process.len();
             if to_take > 0 {
                 self.mempool.take_txs_chunk(/* extend */ &mut txs_to_process, batch_size);
+
+                txs_to_process_blockifier.extend(
+                    txs_to_process
+                        .iter()
+                        .skip(cur_len)
+                        .map(|tx| Transaction::AccountTransaction(clone_account_tx(&tx.tx))),
+                );
             }
 
             if txs_to_process.is_empty() {
@@ -246,16 +256,16 @@ impl<Mempool: MempoolProvider> BlockProductionTask<Mempool> {
 
             stats.n_batches += 1;
 
-            txs_to_process_blockifier
-                .extend(txs_to_process.iter().map(|tx| Transaction::AccountTransaction(clone_account_tx(&tx.tx))));
-
             // Execute the transactions.
             let all_results = self.executor.execute_txs(&txs_to_process_blockifier);
             // When the bouncer cap is reached, blockifier will return fewer results than what we asked for.
             let block_now_full = all_results.len() < txs_to_process_blockifier.len();
 
+            txs_to_process_blockifier.drain(..all_results.len()); // remove the used txs
+
             for exec_result in all_results {
-                let mut mempool_tx = txs_to_process.pop_front().expect("Vector length mismatch");
+                let mut mempool_tx =
+                    txs_to_process.pop_front().ok_or_else(|| Error::Unexpected("Vector length mismatch".into()))?;
                 match exec_result {
                     Ok(execution_info) => {
                         // Reverted transactions appear here as Ok too.
@@ -283,7 +293,10 @@ impl<Mempool: MempoolProvider> BlockProductionTask<Mempool> {
                         // errors during the execution of Declare and DeployAccount also appear here as they cannot be reverted.
                         // We reject them.
                         // Note that this is a big DoS vector.
-                        log::error!("Rejected transaction {} for unexpected error: {err:#}", mempool_tx.tx_hash());
+                        log::error!(
+                            "Rejected transaction {:#x} for unexpected error: {err:#}",
+                            mempool_tx.tx_hash().to_felt()
+                        );
                         stats.n_rejected += 1;
                     }
                 }

crates/node/Cargo.toml

@@ -44,7 +44,7 @@ alloy = { workspace = true }
 anyhow.workspace = true
 async-trait = { workspace = true }
 chrono = "0.4.38"
-clap = { workspace = true, features = ["derive"] }
+clap = { workspace = true, features = ["derive", "env"] }
 env_logger.workspace = true
 fdlimit.workspace = true
 forwarded-header-value = "0.1.1"

crates/node/src/cli/block_production.rs

@@ -3,15 +3,15 @@
 pub struct BlockProductionParams {
     /// Disable the block production service.
     /// The block production service is only enabled with the authority (sequencer) mode.
-    #[arg(long, alias = "no-block-production")]
+    #[arg(env = "MADARA_BLOCK_PRODUCTION_DISABLED", long, alias = "no-block-production")]
     pub block_production_disabled: bool,
 
-    /// Launch a devnet with a producton chain id (like SN_MAINNET, SN_SEPOLIA).
-    /// This in unsafe because your devnet transactiosn can be replayed on the actual network.
-    #[arg(long, default_value_t = false)]
+    /// Launch a devnet with a production chain id (like SN_MAINNET, SN_SEPOLIA).
+    /// This in unsafe because your devnet transactions can be replayed on the actual network.
+    #[arg(env = "MADARA_OVERRIDE_DEVNET_CHAIN_ID", long, default_value_t = false)]
     pub override_devnet_chain_id: bool,
 
     /// Create this number of contracts in the genesis block for the devnet configuration.
-    #[arg(long, default_value_t = 10)]
+    #[arg(env = "MADARA_DEVNET_CONTRACTS", long, default_value_t = 10)]
     pub devnet_contracts: u64,
 }

crates/node/src/cli/chain_config_overrides.rs

@@ -13,7 +13,7 @@ use mp_utils::parsers::parse_key_value;
 /// Format: "--chain-config-override chain_id=NEW_MADARA --chain-config-override chain_name=NEW_NAME"
 #[derive(clap::Parser, Clone, Debug)]
 pub struct ChainConfigOverrideParams {
-    #[clap(long = "chain-config-override", value_parser = parse_key_value, number_of_values = 1)]
+    #[clap(env = "MADARA_CHAIN_CONFIG_OVERRIDE", long = "chain-config-override", value_parser = parse_key_value, number_of_values = 1)]
     pub overrides: Vec<(String, String)>,
 }
 

crates/node/src/cli/db.rs

@@ -3,14 +3,14 @@ use std::path::PathBuf;
 #[derive(Clone, Debug, clap::Args)]
 pub struct DbParams {
     /// The path where madara will store the database. You should probably change it.
-    #[clap(long, default_value = "/tmp/madara", value_name = "PATH")]
+    #[clap(env = "MADARA_BASE_PATH", long, default_value = "/tmp/madara", value_name = "PATH")]
     pub base_path: PathBuf,
 
     /// Directory for backups. Use it with `--restore-from-latest-backup` or `--backup-every-n-blocks <NUMBER OF BLOCKS>`.
-    #[clap(long, value_name = "PATH")]
+    #[clap(env = "MADARA_BACKUP_DIR", long, value_name = "PATH")]
     pub backup_dir: Option<PathBuf>,
 
     /// Restore the database at startup from the latest backup version. Use it with `--backup-dir <PATH>`
-    #[clap(long)]
+    #[clap(env = "MADARA_RESTORE_FROM_LATEST_BACKUP", long)]
     pub restore_from_latest_backup: bool,
 }

crates/node/src/cli/gateway.rs

@@ -4,18 +4,18 @@ use clap::Args;
 #[derive(Debug, Clone, Args)]
 pub struct GatewayParams {
     /// Enable the feeder gateway server.
-    #[arg(long, alias = "feeder-gateway")]
+    #[arg(env = "MADARA_FEEDER_GATEWAY_ENABLE", long, alias = "feeder-gateway")]
     pub feeder_gateway_enable: bool,
 
     /// Enable the gateway server.
-    #[arg(long, alias = "gateway")]
+    #[arg(env = "MADARA_GATEWAY_ENABLE", long, alias = "gateway")]
     pub gateway_enable: bool,
 
     /// Listen on all network interfaces. This usually means the gateway server will be accessible externally.
-    #[arg(long)]
+    #[arg(env = "MADARA_GATEWAY_EXTERNAL", long)]
     pub gateway_external: bool,
 
     /// The gateway port to listen at.
-    #[arg(long, value_name = "GATEWAY PORT", default_value = "8080")]
+    #[arg(env = "MADARA_GATEWAY_PORT", long, value_name = "GATEWAY PORT", default_value = "8080")]
     pub gateway_port: u16,
 }

crates/node/src/cli/l1.rs

@@ -7,19 +7,20 @@ use mp_utils::parsers::{parse_duration, parse_url};
 #[derive(Clone, Debug, clap::Args)]
 pub struct L1SyncParams {
     /// Disable L1 sync.
-    #[clap(long, alias = "no-l1-sync", conflicts_with = "l1_endpoint")]
+    #[clap(env = "MADARA_SYNC_L1_DISABLED", long, alias = "no-l1-sync", conflicts_with = "l1_endpoint")]
     pub sync_l1_disabled: bool,
 
     /// The L1 rpc endpoint url for state verification.
-    #[clap(long, value_parser = parse_url, value_name = "ETHEREUM RPC URL")]
+    #[clap(env = "MADARA_L1_ENDPOINT", long, value_parser = parse_url, value_name = "ETHEREUM RPC URL")]
     pub l1_endpoint: Option<Url>,
 
     /// Disable the gas price sync service. The sync service is responsible to fetch the fee history from the ethereum.
-    #[clap(long, alias = "no-gas-price-sync")]
+    #[clap(env = "MADARA_GAS_PRICE_SYNC_DISABLED", long, alias = "no-gas-price-sync")]
     pub gas_price_sync_disabled: bool,
 
     /// Time in which the gas price worker will fetch the gas price.
     #[clap(
+		env = "MADARA_GAS_PRICE_POLL",
         long,
         default_value = "10s",
         value_parser = parse_duration,

crates/node/src/cli/mod.rs

@@ -50,7 +50,7 @@ use url::Url;
 pub struct RunCmd {
     /// The human-readable name for this node.
     /// It is used as the network node name.
-    #[arg(long, value_name = "NAME")]
+    #[arg(env = "MADARA_NAME", long, value_name = "NAME")]
     pub name: Option<String>,
 
     #[allow(missing_docs)]
@@ -86,27 +86,27 @@ pub struct RunCmd {
     pub block_production_params: BlockProductionParams,
 
     /// The node will run as a sequencer and produce its own state.
-    #[arg(long, group = "mode")]
+    #[arg(env = "MADARA_SEQUENCER", long, group = "mode")]
     pub sequencer: bool,
 
     /// The node will run as a full node and sync the state of a specific network from others.
-    #[arg(long, group = "mode")]
+    #[arg(env = "MADARA_FULL", long, group = "mode")]
     pub full: bool,
 
     /// The node will run as a testing sequencer with predeployed contracts.
-    #[arg(long, group = "mode")]
+    #[arg(env = "MADARA_DEVNET", long, group = "mode")]
     pub devnet: bool,
 
     /// The network chain configuration.
-    #[clap(long, short, group = "full_mode_config")]
+    #[clap(env = "MADARA_NETWORK", long, short, group = "full_mode_config")]
     pub network: Option<NetworkType>,
 
     /// Chain configuration file path.
-    #[clap(long, value_name = "CHAIN CONFIG FILE PATH", group = "chain_config")]
+    #[clap(env = "MADARA_CHAIN_CONFIG_PATH", long, value_name = "CHAIN CONFIG FILE PATH", group = "chain_config")]
     pub chain_config_path: Option<PathBuf>,
 
     /// Use preset as chain Config
-    #[clap(long, value_name = "PRESET NAME", group = "chain_config")]
+    #[clap(env = "MADARA_PRESET", long, value_name = "PRESET NAME", group = "chain_config")]
     pub preset: Option<ChainPreset>,
 
     /// Overrides parameters from the Chain Config.

crates/node/src/cli/prometheus.rs

@@ -4,12 +4,12 @@ use clap::Args;
 #[derive(Debug, Clone, Args)]
 pub struct PrometheusParams {
     /// The port used by the prometheus RPC service.
-    #[arg(long, value_name = "PORT", default_value = "9615")]
+    #[arg(env = "MADARA_PROMETHEUS_PORT", long, value_name = "PORT", default_value = "9615")]
     pub prometheus_port: u16,
     /// Listen on all network interfaces. This usually means the prometheus server will be accessible externally.
-    #[arg(long)]
+    #[arg(env = "MADARA_PROMETHEUS_EXTERNAL", long)]
     pub prometheus_external: bool,
     /// Disable the prometheus service.
-    #[arg(long, alias = "no-prometheus")]
+    #[arg(env = "MADARA_PROMETHEUS_DISABLED", long, alias = "no-prometheus")]
     pub prometheus_disabled: bool,
 }

crates/node/src/cli/rpc.rs

@@ -69,17 +69,18 @@ impl FromStr for Cors {
 #[derive(Clone, Debug, clap::Args)]
 pub struct RpcParams {
     /// Disable the RPC server.
-    #[arg(long, alias = "no-rpc")]
+    #[arg(env = "MADARA_RPC_DISABLED", long, alias = "no-rpc")]
     pub rpc_disabled: bool,
 
     /// Listen to all network interfaces. This usually means that the RPC server will be accessible externally.
     /// Please note that some endpoints should not be exposed to the outside world - by default, enabling remote access
     /// will disable these endpoints. To re-enable them, use `--rpc-methods unsafe`
-    #[arg(long)]
+    #[arg(env = "MADARA_RPC_EXTERNAL", long)]
     pub rpc_external: bool,
 
     /// RPC methods to expose.
     #[arg(
+		env = "MADARA_RPC_METHODS",
 		long,
 		value_name = "METHOD",
 		value_enum,
@@ -95,13 +96,13 @@ pub struct RpcParams {
     ///
     /// For example `--rpc-rate-limit 10` will maximum allow
     /// 10 calls per minute per connection.
-    #[arg(long)]
+    #[arg(env = "MADARA_RPC_RATE_LIMIT", long)]
     pub rpc_rate_limit: Option<NonZeroU32>,
 
     /// Disable RPC rate limiting for certain ip addresses or ranges.
     ///
     /// Each IP address must be in the following notation: `1.2.3.4/24`.
-    #[arg(long, num_args = 1..)]
+    #[arg(env = "MADARA_RPC_RATE_LIMIT_WHITELISTED_IPS", long, num_args = 1..)]
     pub rpc_rate_limit_whitelisted_ips: Vec<IpNetwork>,
 
     /// Trust proxy headers for disable rate limiting.
@@ -110,40 +111,40 @@ pub struct RpcParams {
     /// By default, the RPC server will not trust these headers.
     ///
     /// This is currently only useful for rate-limiting reasons.
-    #[arg(long)]
+    #[arg(env = "MADARA_RPC_RATE_LIMIT_TRUST_PROXY_HEADERS", long)]
     pub rpc_rate_limit_trust_proxy_headers: bool,
 
     /// Set the maximum RPC request payload size for both HTTP and WebSockets in megabytes.
-    #[arg(long, default_value_t = RPC_DEFAULT_MAX_REQUEST_SIZE_MB)]
+    #[arg(env = "MADARA_RPC_MAX_REQUEST_SIZE", long, default_value_t = RPC_DEFAULT_MAX_REQUEST_SIZE_MB)]
     pub rpc_max_request_size: u32,
 
     /// Set the maximum RPC response payload size for both HTTP and WebSockets in megabytes.
-    #[arg(long, default_value_t = RPC_DEFAULT_MAX_RESPONSE_SIZE_MB)]
+    #[arg(env = "MADARA_RPC_MAX_RESPONSE_SIZE", long, default_value_t = RPC_DEFAULT_MAX_RESPONSE_SIZE_MB)]
     pub rpc_max_response_size: u32,
 
     /// Set the maximum concurrent subscriptions per connection.
-    #[arg(long, default_value_t = RPC_DEFAULT_MAX_SUBS_PER_CONN)]
+    #[arg(env = "MADARA_RPC_MAX_SUBSCRIPTIONS_PER_CONNECTION", long, default_value_t = RPC_DEFAULT_MAX_SUBS_PER_CONN)]
     pub rpc_max_subscriptions_per_connection: u32,
 
     /// The RPC port to listen at.
-    #[arg(long, value_name = "PORT", default_value_t = RPC_DEFAULT_PORT)]
+    #[arg(env = "MADARA_RPC_PORT", long, value_name = "PORT", default_value_t = RPC_DEFAULT_PORT)]
     pub rpc_port: u16,
 
     /// Maximum number of RPC server connections at a given time.
-    #[arg(long, value_name = "COUNT", default_value_t = RPC_DEFAULT_MAX_CONNECTIONS)]
+    #[arg(env = "MADARA_RPC_MAX_CONNECTIONS", long, value_name = "COUNT", default_value_t = RPC_DEFAULT_MAX_CONNECTIONS)]
     pub rpc_max_connections: u32,
 
     /// The maximum number of messages that can be kept in memory at a given time, per connection.
     /// The server enforces backpressure, and this buffering is useful when the client cannot keep up with our server.
-    #[arg(long, default_value_t = RPC_DEFAULT_MESSAGE_CAPACITY_PER_CONN)]
+    #[arg(env = "MADARA_RPC_MESSAGE_BUFFER_CAPACITY_PER_CONNECTION", long, default_value_t = RPC_DEFAULT_MESSAGE_CAPACITY_PER_CONN)]
     pub rpc_message_buffer_capacity_per_connection: u32,
 
     /// Disable RPC batch requests.
-    #[arg(long, alias = "rpc_no_batch_requests", conflicts_with_all = &["rpc_max_batch_request_len"])]
+    #[arg(env = "MADARA_RPC_DISABLE_BATCH_REQUESTS", long, alias = "rpc_no_batch_requests", conflicts_with_all = &["rpc_max_batch_request_len"])]
     pub rpc_disable_batch_requests: bool,
 
     /// Limit the max length for an RPC batch request.
-    #[arg(long, conflicts_with_all = &["rpc_disable_batch_requests"], value_name = "LEN")]
+    #[arg(env = "MADARA_RPC_MAX_BATCH_REQUEST_LEN", long, conflicts_with_all = &["rpc_disable_batch_requests"], value_name = "LEN")]
     pub rpc_max_batch_request_len: Option<u32>,
 
     /// Specify browser *origins* allowed to access the HTTP & WebSocket RPC servers.
@@ -154,7 +155,7 @@ pub struct RpcParams {
     /// This argument is a comma separated list of origins, or the special `all` value.
     ///
     /// Learn more about CORS and web security at <https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS>.
-    #[arg(long, value_name = "ORIGINS")]
+    #[arg(env = "MADARA_RPC_CORS", long, value_name = "ORIGINS")]
     pub rpc_cors: Option<Cors>,
 }