feat(mempool): mempool transaction limits, saving to db and continue…

… block prod from pending block (#372)

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## Next release
 
+- feat(block_production): continue pending block on restart
+- feat(mempool): mempool transaction saving on db
+- feat(mempool): mempool transaction limits
 - feat(cli): madaraup quickfix
 - feat(cli): added madaraup for v0.7.0
 - refactor(rpc): replace starknet-rs by starknet-types-rpc

Cargo.toml

@@ -25,6 +25,7 @@ members = [
   "crates/proc-macros",
   "crates/tests",
   "crates/cairo-test-contracts",
+  "crates/client/block_production",
 ]
 resolver = "2"
 # Everything except test-related packages, so that they are not compiled when doing `cargo build`.
@@ -127,6 +128,7 @@ mc-gateway-server = { path = "crates/client/gateway/server" }
 mc-sync = { path = "crates/client/sync" }
 mc-eth = { path = "crates/client/eth" }
 mc-mempool = { path = "crates/client/mempool" }
+mc-block-production = { path = "crates/client/block_production" }
 mc-block-import = { path = "crates/client/block_import" }
 mc-devnet = { path = "crates/client/devnet" }
 
@@ -236,6 +238,7 @@ tracing-subscriber = { version = "0.3.18", features = [
   "registry",
   "std",
 ] }
+tracing-test = "0.2.5"
 tracing-opentelemetry = "0.26.0"
 
 [patch.crates-io]

configs/chain_config.example.yaml

@@ -69,3 +69,10 @@ bouncer_config:
 # /!\ Only used for block production.
 # Address of the sequencer (0x0 for a full node).
 sequencer_address: "0x0"
+
+# Transaction limit in the mempool.
+mempool_tx_limit: 10000
+# Transaction limit in the mempool, additional limit for declare transactions.
+mempool_declare_tx_limit: 20
+# Max age of a transaction in the mempool.
+mempool_tx_max_age: "5h"

configs/presets/devnet.yaml

@@ -29,3 +29,6 @@ bouncer_config:
 sequencer_address: "0x123"
 eth_core_contract_address: "0xe7f1725e7734ce288f8367e1bb143e90bb3f0512"
 eth_gps_statement_verifier: "0xf294781D719D2F4169cE54469C28908E6FA752C1"
+mempool_tx_limit: 10000
+mempool_declare_tx_limit: 20
+mempool_tx_max_age: "5h"

configs/presets/integration.yaml

@@ -29,3 +29,6 @@ bouncer_config:
 sequencer_address: "0x1176a1bd84444c89232ec27754698e5d2e7e1a7f1539f12027f28b23ec9f3d8"
 eth_core_contract_address: "0x4737c0c1B4D5b1A687B42610DdabEE781152359c"
 eth_gps_statement_verifier: "0x2046B966994Adcb88D83f467a41b75d64C2a619F"
+mempool_tx_limit: 10000
+mempool_declare_tx_limit: 20
+mempool_tx_max_age: "5h"

configs/presets/mainnet.yaml

@@ -29,3 +29,6 @@ bouncer_config:
 sequencer_address: "0x1176a1bd84444c89232ec27754698e5d2e7e1a7f1539f12027f28b23ec9f3d8"
 eth_core_contract_address: "0xc662c410C0ECf747543f5bA90660f6ABeBD9C8c4"
 eth_gps_statement_verifier: "0x47312450B3Ac8b5b8e247a6bB6d523e7605bDb60"
+mempool_tx_limit: 10000
+mempool_declare_tx_limit: 20
+mempool_tx_max_age: "5h"

configs/presets/sepolia.yaml

@@ -29,3 +29,6 @@ bouncer_config:
 sequencer_address: "0x1176a1bd84444c89232ec27754698e5d2e7e1a7f1539f12027f28b23ec9f3d8"
 eth_core_contract_address: "0xE2Bb56ee936fd6433DC0F6e7e3b8365C906AA057"
 eth_gps_statement_verifier: "0xf294781D719D2F4169cE54469C28908E6FA752C1"
+mempool_tx_limit: 10000
+mempool_declare_tx_limit: 20
+mempool_tx_max_age: "5h"

crates/client/block_import/src/pre_validate.rs

@@ -77,6 +77,7 @@ pub fn pre_validate_inner(
         unverified_global_state_root: block.commitments.global_state_root,
         unverified_block_hash: block.commitments.block_hash,
         unverified_block_number: block.unverified_block_number,
+        visited_segments: block.visited_segments,
     })
 }
 
@@ -97,6 +98,7 @@ pub fn pre_validate_pending_inner(
         state_diff: block.state_diff,
         receipts: block.receipts,
         converted_classes,
+        visited_segments: block.visited_segments,
     })
 }
 

crates/client/block_import/src/tests/block_import_utils.rs

@@ -104,6 +104,7 @@ pub fn create_dummy_block() -> PreValidatedBlock {
         receipts: vec![],
         state_diff: StateDiff::default(),
         converted_classes: Default::default(),
+        visited_segments: None,
     }
 }
 
@@ -128,6 +129,7 @@ pub fn create_dummy_unverified_full_block() -> UnverifiedFullBlock {
         declared_classes: vec![],
         commitments: UnverifiedCommitments::default(),
         trusted_converted_classes: vec![],
+        visited_segments: None,
     }
 }
 
@@ -151,5 +153,6 @@ pub fn create_dummy_pending_block() -> PreValidatedPendingBlock {
         receipts: vec![],
         state_diff: StateDiff::default(),
         converted_classes: Default::default(),
+        visited_segments: None,
     }
 }

crates/client/block_import/src/types.rs

@@ -3,7 +3,7 @@
 
 use mp_block::{
     header::{GasPrices, L1DataAvailabilityMode},
-    Header,
+    Header, VisitedSegments,
 };
 use mp_chain_config::StarknetVersion;
 use mp_class::{
@@ -142,13 +142,14 @@ pub struct UnverifiedCommitments {
 }
 
 /// An unverified pending full block as input for the block import pipeline.
-#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
+#[derive(Clone, Debug, Eq, PartialEq, Default, Serialize, Deserialize)]
 pub struct UnverifiedPendingFullBlock {
     pub header: UnverifiedHeader,
     pub state_diff: StateDiff,
     pub transactions: Vec<Transaction>,
     pub receipts: Vec<TransactionReceipt>,
     pub declared_classes: Vec<DeclaredClass>,
+    pub visited_segments: Option<VisitedSegments>,
 }
 
 /// An unverified full block as input for the block import pipeline.
@@ -165,6 +166,7 @@ pub struct UnverifiedFullBlock {
     #[serde(skip)]
     pub trusted_converted_classes: Vec<ConvertedClass>,
     pub commitments: UnverifiedCommitments,
+    pub visited_segments: Option<VisitedSegments>,
 }
 
 // Pre-validate outputs.
@@ -192,6 +194,7 @@ pub struct PreValidatedBlock {
     pub unverified_global_state_root: Option<Felt>,
     pub unverified_block_hash: Option<Felt>,
     pub unverified_block_number: Option<u64>,
+    pub visited_segments: Option<VisitedSegments>,
 }
 
 /// Output of the [`crate::pre_validate`] step.
@@ -202,6 +205,7 @@ pub struct PreValidatedPendingBlock {
     pub state_diff: StateDiff,
     pub receipts: Vec<TransactionReceipt>,
     pub converted_classes: Vec<ConvertedClass>,
+    pub visited_segments: Option<VisitedSegments>,
 }
 
 // Verify-apply output.

crates/client/block_import/src/verify_apply.rs

@@ -101,6 +101,8 @@ pub fn verify_apply_inner(
             },
             block.state_diff,
             block.converted_classes,
+            block.visited_segments,
+            None,
         )
         .map_err(make_db_error("storing block in db"))?;
 
@@ -144,6 +146,8 @@ pub fn verify_apply_pending_inner(
             },
             block.state_diff,
             block.converted_classes,
+            block.visited_segments,
+            None,
         )
         .map_err(make_db_error("storing block in db"))?;
 
@@ -407,7 +411,7 @@ mod verify_apply_tests {
         if populate_db {
             let header = create_dummy_header();
             let pending_block = finalized_block_zero(header);
-            backend.store_block(pending_block.clone(), finalized_state_diff_zero(), vec![]).unwrap();
+            backend.store_block(pending_block.clone(), finalized_state_diff_zero(), vec![], None, None).unwrap();
         }
 
         // Create a validation context with the specified ignore_block_order flag
@@ -661,7 +665,7 @@ mod verify_apply_tests {
             let mut header = create_dummy_header();
             header.block_number = 0;
             let pending_block = finalized_block_zero(header);
-            backend.store_block(pending_block.clone(), finalized_state_diff_zero(), vec![]).unwrap();
+            backend.store_block(pending_block.clone(), finalized_state_diff_zero(), vec![], None, None).unwrap();
 
             assert_eq!(backend.get_latest_block_n().unwrap(), Some(0));
 
@@ -687,7 +691,7 @@ mod verify_apply_tests {
             let mut header = create_dummy_header();
             header.block_number = 0;
             let pending_block = finalized_block_zero(header);
-            backend.store_block(pending_block.clone(), finalized_state_diff_zero(), vec![]).unwrap();
+            backend.store_block(pending_block.clone(), finalized_state_diff_zero(), vec![], None, None).unwrap();
 
             assert_eq!(backend.get_latest_block_n().unwrap(), Some(0));
 
@@ -723,7 +727,7 @@ mod verify_apply_tests {
             let mut genesis_header = create_dummy_header();
             genesis_header.block_number = 0;
             let genesis_block = finalized_block_zero(genesis_header.clone());
-            backend.store_block(genesis_block, finalized_state_diff_zero(), vec![]).unwrap();
+            backend.store_block(genesis_block, finalized_state_diff_zero(), vec![], None, None).unwrap();
 
             assert_eq!(backend.get_latest_block_n().unwrap(), Some(0));
 
@@ -769,7 +773,7 @@ mod verify_apply_tests {
             let mut genesis_header = create_dummy_header();
             genesis_header.block_number = 0;
             let genesis_block = finalized_block_zero(genesis_header.clone());
-            backend.store_block(genesis_block, finalized_state_diff_zero(), vec![]).unwrap();
+            backend.store_block(genesis_block, finalized_state_diff_zero(), vec![], None, None).unwrap();
 
             assert_eq!(backend.get_latest_block_n().unwrap(), Some(0));
 

crates/client/block_production/Cargo.toml

@@ -0,0 +1,78 @@
+[package]
+name = "mc-block-production"
+description = "Madara client block production service"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+version.workspace = true
+homepage.workspace = true
+
+[lints]
+workspace = true
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]
+
+[dev-dependencies]
+
+rstest = { workspace = true }
+mc-db = { workspace = true, features = ["testing"] }
+tokio = { workspace = true, features = ["rt-multi-thread"] }
+proptest.workspace = true
+proptest-derive.workspace = true
+bitvec.workspace = true
+blockifier = { workspace = true, features = ["testing"] }
+mockall.workspace = true
+assert_matches.workspace = true
+lazy_static.workspace = true
+serde_json.workspace = true
+
+[features]
+testing = ["blockifier/testing", "mc-db/testing", "mockall"]
+
+[dependencies]
+
+# Madara
+mc-analytics.workspace = true
+mc-block-import.workspace = true
+mc-db.workspace = true
+mc-exec.workspace = true
+mc-mempool.workspace = true
+mp-block.workspace = true
+mp-chain-config.workspace = true
+mp-class.workspace = true
+mp-convert.workspace = true
+mp-receipt.workspace = true
+mp-state-update.workspace = true
+mp-transactions.workspace = true
+mp-utils.workspace = true
+
+# Starknet
+blockifier.workspace = true
+starknet-core.workspace = true
+starknet-types-core.workspace = true
+starknet_api.workspace = true
+
+# Other
+anyhow.workspace = true
+mockall = { workspace = true, optional = true }
+thiserror.workspace = true
+tokio.workspace = true
+
+# Instrumentation
+once_cell = { workspace = true }
+opentelemetry = { workspace = true, features = ["metrics", "logs"] }
+opentelemetry-appender-tracing = { workspace = true, default-features = false }
+opentelemetry-otlp = { workspace = true, features = [
+  "tonic",
+  "metrics",
+  "logs",
+] }
+opentelemetry-semantic-conventions = { workspace = true }
+opentelemetry-stdout = { workspace = true }
+opentelemetry_sdk = { workspace = true, features = ["rt-tokio", "logs"] }
+tracing = { workspace = true }
+tracing-core = { workspace = true, default-features = false }
+tracing-opentelemetry = { workspace = true }
+tracing-subscriber = { workspace = true, features = ["env-filter"] }