feat(da): add exec cmd for da chunks execution and verification (#3054)

## Summary

1. enhance DA commands with new namespace and exec logic: Added full hash output for namespace command and refined the exec command with optional executor handling and improved transaction verification logic. Updated documentation with new usage instructions and examples.
2. use loaded genesis instance for consistency as DA default namespace
3. add exec cmd for da chunks execution and verification
4. feat(statedb): make export path optional in re-genesis command

crates/rooch-genesis/src/lib.rs

@@ -402,11 +402,6 @@ impl RoochGenesis {
             "Genesis output mismatch"
         );
 
-        let tx_hash = self.genesis_tx().tx_hash();
-        let (output, genesis_execution_info) = rooch_db
-            .moveos_store
-            .handle_tx_output(tx_hash, genesis_raw_output.clone())?;
-
         // Save the genesis txs to sequencer
         let genesis_tx_order: u64 = 0;
         let moveos_genesis_context = self
@@ -440,6 +435,11 @@ impl RoochGenesis {
             genesis_accumulator_unsaved_nodes,
         )?;
 
+        let tx_hash = self.genesis_tx().tx_hash();
+        let (output, genesis_execution_info) = rooch_db
+            .moveos_store
+            .handle_tx_output(tx_hash, genesis_raw_output.clone())?;
+
         // Save genesis tx state change set
         let state_change_set_ext = StateChangeSetExt::new(
             output.changeset.clone(),

crates/rooch-pipeline-processor/src/actor/processor.rs

@@ -135,7 +135,7 @@ impl PipelineProcessorActor {
                         }
                         None => {
                             return Err(anyhow::anyhow!(
-                                "The bitcoin client proxy should be initialized before processing the sequenced l1_block_tx(block: {:?} on startup", block
+                                "The bitcoin client proxy should be initialized before processing the sequenced l1_block_tx(block: {:?}) on startup", block
                             ));
                         }
                     }

crates/rooch-rpc-server/src/lib.rs

@@ -237,7 +237,7 @@ pub async fn run_start_server(opt: RoochOpt, server_opt: ServerOpt) -> Result<Se
         );
     }
 
-    let _genesis = RoochGenesis::load_or_init(network.clone(), &rooch_db)?;
+    let genesis = RoochGenesis::load_or_init(network.clone(), &rooch_db)?;
 
     let root = rooch_db
         .latest_root()?
@@ -296,7 +296,7 @@ pub async fn run_start_server(opt: RoochOpt, server_opt: ServerOpt) -> Result<Se
     let sequencer_proxy = SequencerProxy::new(sequencer.into());
 
     // Init DA
-    let genesis_bytes = RoochGenesis::build(network.clone())?.encode();
+    let genesis_bytes = genesis.encode();
     let genesis_namespace = derive_genesis_namespace(&genesis_bytes);
     let last_tx_order = sequencer_proxy.get_sequencer_order().await?;
     let (da_issues, da_fixed) = rooch_store.try_repair_da_meta(last_tx_order, false)?;

crates/rooch/Cargo.toml

@@ -17,7 +17,9 @@ rust-version = { workspace = true }
 [dependencies]
 anyhow = { workspace = true }
 bcs = { workspace = true }
+bitcoin = { workspace = true }
 clap = { features = ["derive"], workspace = true }
+coerce = { workspace = true }
 datatest-stable = { git = "https://github.com/rooch-network/diem-devtools", branch = "feature/pub-test-opts" }
 tokio = { features = ["full"], workspace = true }
 serde = { workspace = true }
@@ -47,7 +49,6 @@ xxhash-rust = { workspace = true, features = ["xxh3"] }
 vergen-pretty = { workspace = true }
 lazy_static = { workspace = true }
 schemars = { workspace = true }
-bitcoin = { workspace = true }
 ciborium = { workspace = true }
 wasmer = { workspace = true }
 tiny-keccak = { workspace = true }
@@ -81,6 +82,8 @@ framework-types = { workspace = true }
 raw-store = { workspace = true }
 smt = { workspace = true }
 
+bitcoin-client = { workspace = true }
+rooch-executor = { workspace = true }
 rooch-key = { workspace = true }
 rooch-types = { workspace = true }
 rooch-config = { workspace = true }

crates/rooch/src/commands/da/README.md

@@ -1,11 +1,93 @@
-## Rooch DA tool
+# Rooch DA tool
 
 Toolkits for RoochDA.
 
-### Usage
+## Usage
 
-1. Unpack tx list to human-readable format:
+### namespace
+
+Derive DA namespace from genesis file:
+
+```shell
+rooch da namespace --genesis-file-path {genesis-file}
+```
+
+### unpack
+
+#### download segments
+
+the easiest way to download segments is using `getda` tool from [here](https://github.com/popcnt1/roh) for downloading
+segments from cloud storage.
+
+```shell
+getda --output={segment-dir} --url={da-cloud-storage-path} --last_chunk={max-chunk-id-expected} --max_goroutines={max-goroutines}
+```
+
+#### unpack segments
+
+Unpack tx list from segments to human-readable format:
 
 ```shell
 rooch da unpack --segment-dir {segment-dir} --batch-dir {batch-dir}
 ```
+
+If you want to verify tx list order, you can use `--verify-order` flag:
+
+```shell
+rooch da unpack --segment-dir {segment-dir} --batch-dir {batch-dir} --verify-order
+```
+
+### exec
+
+Execute tx list with state root verification(compare with Rooch Network Mainnet/Testnet).
+It's a tool built for verification at development stage, not a full feature tool for sync states in production.
+
+Features includes:
+
+1. Execute tx list from segment dir and saving changes locally
+2. Compare state root with Rooch Network Mainnet/Testnet by tx_order:state_root list file
+3. We could collect performance data in execution process by tuning tools like `perf` if needed
+
+#### Prepare tx_order:state_root list file
+
+using [order_state.sh](https://github.com/popcnt1/roh/blob/main/scripts/order_state.sh) to generate tx_order:state_root
+list file:
+
+```shell
+rooch env switch -n {network}
+./order_state.sh {start_order} {end_order} {interval}
+```
+
+#### Prepare genesis
+
+if you just import data by `rooch statedb genesis`, and will execute transaction from tx_order 1 (genesis tx is tx_order
+0).
+
+For ensuring everything works as expected, you should:
+
+clean dirty genesis states:
+
+```shell
+rooch statedb re-genesis -d {data_dir} -n {network} --mode remove
+```
+
+genesis init(add builtin genesis back into db):
+
+```shell
+rooch genesis init -d {data_dir} -n {network}
+```
+
+we assume it's builtin genesis, because the target we want to verify is Rooch Network Mainnet/Testnet, all the two
+Network are using builtin genesis.
+
+#### Execute tx list
+
+```shell
+rooch da exec --segment-dir {segment-dir} --order-state-path {order-state-path} -d {data-dir} -n {network} --btc-rpc-url {btc-rpc-url} --btc-rpc-user-name {btc-rpc-user-name} --btc-rpc-password {btc-rpc-password}
+```
+
+If everything is ok, you will see this log in the end:
+
+```shell
+2024-12-16T05:48:26.924094Z  INFO rooch::commands::da::commands::exec: All transactions execution state root are strictly equal to RoochNetwork: [0, {end_order}]
+```