From 3d3eb7620bc7546b2cad35b5a8c81a31aeaff518 Mon Sep 17 00:00:00 2001 From: Chandra Penke Date: Wed, 18 Oct 2023 16:44:07 -0700 Subject: [PATCH 01/15] fix: add new_replica for agent --- crates/dscvr-canister-agent/src/lib.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/crates/dscvr-canister-agent/src/lib.rs b/crates/dscvr-canister-agent/src/lib.rs index 447ed1a..bb7ec0b 100644 --- a/crates/dscvr-canister-agent/src/lib.rs +++ b/crates/dscvr-canister-agent/src/lib.rs @@ -84,6 +84,18 @@ impl CanisterAgent { }) } + pub async fn new_replica( + caller: Arc, + replica: &str, + canister_id: &str, + ) -> Result { + let agent = Self { + agent: agent_impl::replica_impl::new(caller, replica).await?, + canister_id: Principal::from_text(canister_id)?, + }; + Ok(agent) + } + pub fn new_from_agent(agent: Agent, canister_id: Principal) -> Self where Agent: AgentImpl + 'static, From 108c8477f780250574fd3c0e60e368cdbe00ea8b Mon Sep 17 00:00:00 2001 From: Chandra Penke Date: Thu, 19 Oct 2023 02:17:37 -0700 Subject: [PATCH 02/15] fix: bump down ic-cdk --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 01a9986..659c16c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,7 +37,7 @@ enum-iterator = "1.2.0" flate2 = "1.0" futures = "0.3.25" ic-agent = { version = "0.25.0", features = ["pem"] } -ic-cdk = "0.11" +ic-cdk = "0.10" lazy_static = "1.4" num-traits = "0.2.15" ring = "0.16" From 0500eefa1682a0bf6b4ef26d8e33b2872b58ac52 Mon Sep 17 00:00:00 2001 From: Chandra Penke Date: Thu, 19 Oct 2023 10:31:28 -0700 Subject: [PATCH 03/15] fix: add library for creating an ingress verifier from a URL --- Cargo.toml | 3 ++- crates/ic-ingress-validator-util/Cargo.toml | 17 +++++++++++++++++ crates/ic-ingress-validator-util/src/lib.rs | 20 ++++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 crates/ic-ingress-validator-util/Cargo.toml create mode 100644 crates/ic-ingress-validator-util/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 659c16c..0ebe922 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ members = [ "crates/ic-canister-logger", "crates/ic-canister-stable-storage", "crates/ic-identity-util", + "crates/ic-ingress-validator-util", "crates/ic-rc-principal", "crates/instrumented-error", ] @@ -25,7 +26,7 @@ bincode = "1.3" candid = { git = "https://github.com/dscvr-one/candid.git", rev = "65d73d6", features = [ "parser", ] } -convert_case = "0.1" +convert_case = "0.6" deepsize = { git = "https://github.com/dscvr-one/deepsize.git", rev = "822ba27", features = [ "candid", "serde_bytes", diff --git a/crates/ic-ingress-validator-util/Cargo.toml b/crates/ic-ingress-validator-util/Cargo.toml new file mode 100644 index 0000000..6c0a171 --- /dev/null +++ b/crates/ic-ingress-validator-util/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "ic-ingress-validator-util" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +ic-agent.workspace = true +ic-crypto-utils-threshold-sig-der = { git = "https://github.com/dscvr-one/ic.git", rev = "cacda1", package = "ic-crypto-utils-threshold-sig-der" } +ic-error-types = { git = "https://github.com/dscvr-one/ic.git", rev = "cacda1", package = "ic-error-types" } +ic-http-endpoints-public = { git = "https://github.com/dscvr-one/ic.git", rev = "cacda1", package = "ic-http-endpoints-public" } +ic-types = { git = "https://github.com/dscvr-one/ic.git", rev = "cacda1", package = "ic-types" } +ic-validator-ingress-message = { git = "https://github.com/dscvr-one/ic.git", rev = "cacda1", package = "ic-validator-ingress-message" } + +ic-identity-util = { path = "../ic-identity-util" } +instrumented-error = { path = "../instrumented-error" } diff --git a/crates/ic-ingress-validator-util/src/lib.rs b/crates/ic-ingress-validator-util/src/lib.rs new file mode 100644 index 0000000..c00c403 --- /dev/null +++ b/crates/ic-ingress-validator-util/src/lib.rs @@ -0,0 +1,20 @@ +use ic_crypto_utils_threshold_sig_der::parse_threshold_sig_key_from_der; +use ic_validator_ingress_message::IngressMessageVerifier; +use instrumented_error::Result; +use std::sync::Arc; + +pub async fn init_ingress_verifier(url: &str) -> Result { + use ic_agent::agent::http_transport::ReqwestHttpReplicaV2Transport; + use ic_agent::identity::AnonymousIdentity; + use ic_agent::Agent; + + let agent: Agent = Agent::builder() + .with_transport(ReqwestHttpReplicaV2Transport::create(url)?) + .with_arc_identity(Arc::new(AnonymousIdentity)) + .build()?; + agent.fetch_root_key().await?; + let public_key = parse_threshold_sig_key_from_der(&agent.read_root_key())?; + Ok(IngressMessageVerifier::builder() + .with_root_of_trust(public_key) + .build()) +} From 3d91a12ad24fa2313a3caf2fc7ca9c49099457f5 Mon Sep 17 00:00:00 2001 From: Chandra Penke Date: Thu, 19 Oct 2023 11:27:46 -0700 Subject: [PATCH 04/15] fix: expose ic api --- crates/ic-ingress-validator-util/Cargo.toml | 1 + crates/ic-ingress-validator-util/src/lib.rs | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/ic-ingress-validator-util/Cargo.toml b/crates/ic-ingress-validator-util/Cargo.toml index 6c0a171..93d7f74 100644 --- a/crates/ic-ingress-validator-util/Cargo.toml +++ b/crates/ic-ingress-validator-util/Cargo.toml @@ -10,6 +10,7 @@ ic-agent.workspace = true ic-crypto-utils-threshold-sig-der = { git = "https://github.com/dscvr-one/ic.git", rev = "cacda1", package = "ic-crypto-utils-threshold-sig-der" } ic-error-types = { git = "https://github.com/dscvr-one/ic.git", rev = "cacda1", package = "ic-error-types" } ic-http-endpoints-public = { git = "https://github.com/dscvr-one/ic.git", rev = "cacda1", package = "ic-http-endpoints-public" } +ic-interfaces = { git = "https://github.com/dscvr-one/ic.git", rev = "cacda1", package = "ic-interfaces" } ic-types = { git = "https://github.com/dscvr-one/ic.git", rev = "cacda1", package = "ic-types" } ic-validator-ingress-message = { git = "https://github.com/dscvr-one/ic.git", rev = "cacda1", package = "ic-validator-ingress-message" } diff --git a/crates/ic-ingress-validator-util/src/lib.rs b/crates/ic-ingress-validator-util/src/lib.rs index c00c403..426b7f5 100644 --- a/crates/ic-ingress-validator-util/src/lib.rs +++ b/crates/ic-ingress-validator-util/src/lib.rs @@ -1,8 +1,10 @@ use ic_crypto_utils_threshold_sig_der::parse_threshold_sig_key_from_der; -use ic_validator_ingress_message::IngressMessageVerifier; use instrumented_error::Result; use std::sync::Arc; +pub use ic_interfaces::crypto::IngressSigVerifier; +pub use ic_validator_ingress_message::IngressMessageVerifier; + pub async fn init_ingress_verifier(url: &str) -> Result { use ic_agent::agent::http_transport::ReqwestHttpReplicaV2Transport; use ic_agent::identity::AnonymousIdentity; From 43c6eb35652f6f5099e7f0519db21e87f2bdf1ec Mon Sep 17 00:00:00 2001 From: Chandra Penke Date: Thu, 19 Oct 2023 11:29:53 -0700 Subject: [PATCH 05/15] fix: rename --- crates/ic-ingress-validator-util/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/ic-ingress-validator-util/src/lib.rs b/crates/ic-ingress-validator-util/src/lib.rs index 426b7f5..bb6b6b0 100644 --- a/crates/ic-ingress-validator-util/src/lib.rs +++ b/crates/ic-ingress-validator-util/src/lib.rs @@ -5,7 +5,7 @@ use std::sync::Arc; pub use ic_interfaces::crypto::IngressSigVerifier; pub use ic_validator_ingress_message::IngressMessageVerifier; -pub async fn init_ingress_verifier(url: &str) -> Result { +pub async fn new_ingress_verifier(url: &str) -> Result { use ic_agent::agent::http_transport::ReqwestHttpReplicaV2Transport; use ic_agent::identity::AnonymousIdentity; use ic_agent::Agent; From 35fdcf713f7404c427153fc7de9f1500ae32f436 Mon Sep 17 00:00:00 2001 From: Chandra Penke Date: Thu, 19 Oct 2023 11:32:18 -0700 Subject: [PATCH 06/15] fix: more renaming sigh --- crates/ic-ingress-validator-util/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/ic-ingress-validator-util/src/lib.rs b/crates/ic-ingress-validator-util/src/lib.rs index bb6b6b0..caf346f 100644 --- a/crates/ic-ingress-validator-util/src/lib.rs +++ b/crates/ic-ingress-validator-util/src/lib.rs @@ -5,7 +5,7 @@ use std::sync::Arc; pub use ic_interfaces::crypto::IngressSigVerifier; pub use ic_validator_ingress_message::IngressMessageVerifier; -pub async fn new_ingress_verifier(url: &str) -> Result { +pub async fn try_new_ingress_verifier(url: &str) -> Result { use ic_agent::agent::http_transport::ReqwestHttpReplicaV2Transport; use ic_agent::identity::AnonymousIdentity; use ic_agent::Agent; From 5133995d4bd1a0f868ffdd93387e283ab59f7280 Mon Sep 17 00:00:00 2001 From: Chandra Penke Date: Thu, 19 Oct 2023 12:23:55 -0700 Subject: [PATCH 07/15] fix: update ic validator --- crates/ic-ingress-validator-util/Cargo.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/ic-ingress-validator-util/Cargo.toml b/crates/ic-ingress-validator-util/Cargo.toml index 93d7f74..19a25d8 100644 --- a/crates/ic-ingress-validator-util/Cargo.toml +++ b/crates/ic-ingress-validator-util/Cargo.toml @@ -7,12 +7,12 @@ edition = "2021" [dependencies] ic-agent.workspace = true -ic-crypto-utils-threshold-sig-der = { git = "https://github.com/dscvr-one/ic.git", rev = "cacda1", package = "ic-crypto-utils-threshold-sig-der" } -ic-error-types = { git = "https://github.com/dscvr-one/ic.git", rev = "cacda1", package = "ic-error-types" } -ic-http-endpoints-public = { git = "https://github.com/dscvr-one/ic.git", rev = "cacda1", package = "ic-http-endpoints-public" } -ic-interfaces = { git = "https://github.com/dscvr-one/ic.git", rev = "cacda1", package = "ic-interfaces" } -ic-types = { git = "https://github.com/dscvr-one/ic.git", rev = "cacda1", package = "ic-types" } -ic-validator-ingress-message = { git = "https://github.com/dscvr-one/ic.git", rev = "cacda1", package = "ic-validator-ingress-message" } +ic-crypto-utils-threshold-sig-der = { git = "https://github.com/dscvr-one/ic.git", rev = "ed9a6915", package = "ic-crypto-utils-threshold-sig-der" } +ic-error-types = { git = "https://github.com/dscvr-one/ic.git", rev = "ed9a6915", package = "ic-error-types" } +ic-http-endpoints-public = { git = "https://github.com/dscvr-one/ic.git", rev = "ed9a6915", package = "ic-http-endpoints-public" } +ic-interfaces = { git = "https://github.com/dscvr-one/ic.git", rev = "ed9a6915", package = "ic-interfaces" } +ic-types = { git = "https://github.com/dscvr-one/ic.git", rev = "ed9a6915", package = "ic-types" } +ic-validator-ingress-message = { git = "https://github.com/dscvr-one/ic.git", rev = "ed9a6915", package = "ic-validator-ingress-message" } ic-identity-util = { path = "../ic-identity-util" } instrumented-error = { path = "../instrumented-error" } From c5040abfd21277cfcc3b294c8400bde8caec0598 Mon Sep 17 00:00:00 2001 From: Chandra Penke Date: Thu, 19 Oct 2023 14:22:15 -0700 Subject: [PATCH 08/15] fix: unwind some changes --- crates/ic-ingress-validator-util/Cargo.toml | 8 ++------ crates/ic-ingress-validator-util/src/lib.rs | 11 ++++------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/crates/ic-ingress-validator-util/Cargo.toml b/crates/ic-ingress-validator-util/Cargo.toml index 19a25d8..c9635cd 100644 --- a/crates/ic-ingress-validator-util/Cargo.toml +++ b/crates/ic-ingress-validator-util/Cargo.toml @@ -7,12 +7,8 @@ edition = "2021" [dependencies] ic-agent.workspace = true -ic-crypto-utils-threshold-sig-der = { git = "https://github.com/dscvr-one/ic.git", rev = "ed9a6915", package = "ic-crypto-utils-threshold-sig-der" } -ic-error-types = { git = "https://github.com/dscvr-one/ic.git", rev = "ed9a6915", package = "ic-error-types" } -ic-http-endpoints-public = { git = "https://github.com/dscvr-one/ic.git", rev = "ed9a6915", package = "ic-http-endpoints-public" } -ic-interfaces = { git = "https://github.com/dscvr-one/ic.git", rev = "ed9a6915", package = "ic-interfaces" } -ic-types = { git = "https://github.com/dscvr-one/ic.git", rev = "ed9a6915", package = "ic-types" } -ic-validator-ingress-message = { git = "https://github.com/dscvr-one/ic.git", rev = "ed9a6915", package = "ic-validator-ingress-message" } +ic-crypto-utils-threshold-sig-der = { git = "https://github.com/dscvr-one/ic.git", rev = "cacda1", package = "ic-crypto-utils-threshold-sig-der" } +ic-validator-ingress-message = { git = "https://github.com/dscvr-one/ic.git", rev = "cacda1", package = "ic-validator-ingress-message" } ic-identity-util = { path = "../ic-identity-util" } instrumented-error = { path = "../instrumented-error" } diff --git a/crates/ic-ingress-validator-util/src/lib.rs b/crates/ic-ingress-validator-util/src/lib.rs index caf346f..e658097 100644 --- a/crates/ic-ingress-validator-util/src/lib.rs +++ b/crates/ic-ingress-validator-util/src/lib.rs @@ -1,15 +1,12 @@ +use ic_agent::agent::http_transport::ReqwestHttpReplicaV2Transport; +use ic_agent::identity::AnonymousIdentity; +use ic_agent::Agent; use ic_crypto_utils_threshold_sig_der::parse_threshold_sig_key_from_der; +use ic_validator_ingress_message::IngressMessageVerifier; use instrumented_error::Result; use std::sync::Arc; -pub use ic_interfaces::crypto::IngressSigVerifier; -pub use ic_validator_ingress_message::IngressMessageVerifier; - pub async fn try_new_ingress_verifier(url: &str) -> Result { - use ic_agent::agent::http_transport::ReqwestHttpReplicaV2Transport; - use ic_agent::identity::AnonymousIdentity; - use ic_agent::Agent; - let agent: Agent = Agent::builder() .with_transport(ReqwestHttpReplicaV2Transport::create(url)?) .with_arc_identity(Arc::new(AnonymousIdentity)) From ac435e4e9b53c4fca14e5555885fc7ac0024282a Mon Sep 17 00:00:00 2001 From: Chandra Penke Date: Wed, 6 Mar 2024 15:03:16 -0800 Subject: [PATCH 09/15] feat: update dscvr-canister-agent + ic-ingress-validator-util crates --- Cargo.toml | 10 +- crates/dscvr-candid-generator/src/lib.rs | 18 - crates/dscvr-canister-agent/Cargo.toml | 7 +- crates/dscvr-canister-agent/src/agent_impl.rs | 6 +- .../src/agent_impl/embedded_canister_impl.rs | 39 +- .../src/agent_impl/replica_impl.rs | 27 +- .../src/agent_impl/state_machine_impl.rs | 26 +- crates/dscvr-canister-agent/src/lib.rs | 12 - .../src/rust_canister_agent.rs | 461 ++++++++++++++++++ .../src/stable_storage_restore_backup.rs | 47 +- crates/dscvr-canister-agent/src/stats.rs | 5 +- crates/dscvr-canister-agent/src/util.rs | 22 + 12 files changed, 532 insertions(+), 148 deletions(-) create mode 100644 crates/dscvr-canister-agent/src/rust_canister_agent.rs create mode 100644 crates/dscvr-canister-agent/src/util.rs diff --git a/Cargo.toml b/Cargo.toml index 0ebe922..9389c2f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,12 +23,12 @@ async-std = "1.12.0" async-trait = "0.1" # Note: Need to leave ring at 0.16 for compatibility with ic-agent bincode = "1.3" -candid = { git = "https://github.com/dscvr-one/candid.git", rev = "65d73d6", features = [ +candid = { git = "https://github.com/dscvr-one/candid.git", rev = "0.9.3-3", features = [ "parser", + "deepsize", ] } convert_case = "0.6" -deepsize = { git = "https://github.com/dscvr-one/deepsize.git", rev = "822ba27", features = [ - "candid", +deepsize = { git = "https://github.com/dscvr-one/deepsize.git", rev = "0.2.0-candid-0.9.3-2", features = [ "serde_bytes", "derive", "std", @@ -57,4 +57,6 @@ tracing-stackdriver = "0.8" tracing-subscriber = { version = "0.3", features = ["env-filter"] } [patch.crates-io] -candid = { git = "https://github.com/dscvr-one/candid.git", rev = "65d73d6" } +candid = { git = "https://github.com/dscvr-one/candid.git", rev = "0.9.3-3", features = [ + "deepsize", +] } diff --git a/crates/dscvr-candid-generator/src/lib.rs b/crates/dscvr-candid-generator/src/lib.rs index 58172c5..9caa462 100644 --- a/crates/dscvr-candid-generator/src/lib.rs +++ b/crates/dscvr-candid-generator/src/lib.rs @@ -3,21 +3,3 @@ pub mod rust_canister_agent; pub mod util; - -#[cfg(test)] -mod test { - use super::*; - use std::path::Path; - const DID: &str = "../../canisters/society_rs/society-common.did"; - #[test] - #[ignore] - fn test_generate() { - let output_dir: std::path::PathBuf = Path::new("src").join("gen"); - std::fs::create_dir_all("src/gen").unwrap(); - let _ = rust_canister_agent::generate( - Path::new(DID), - &output_dir.join("dscvr_tx_log_agent.rs"), - ) - .expect("Something good to happen"); - } -} diff --git a/crates/dscvr-canister-agent/Cargo.toml b/crates/dscvr-canister-agent/Cargo.toml index 8adc41e..9037c65 100644 --- a/crates/dscvr-canister-agent/Cargo.toml +++ b/crates/dscvr-canister-agent/Cargo.toml @@ -16,7 +16,6 @@ futures.workspace = true garcon = "0.2.3" hex = "0.4" ic-agent.workspace = true -ic-test-state-machine-client = "3.0" serde_bytes.workspace = true serde.workspace = true thiserror.workspace = true @@ -32,4 +31,10 @@ dscvr-canister-exports = { path = "../dscvr-canister-exports" } dscvr-interface = { path = "../dscvr-interface" } ic-canister-stable-storage = { path = "../ic-canister-stable-storage" } ic-identity-util = { path = "../ic-identity-util" } +ic-test-state-machine-client = "=3.0.0" instrumented-error = { path = "../instrumented-error" } + +[build-dependencies] + +dscvr-candid-generator = { path = "../dscvr-candid-generator" } +dscvr-tracing-util = { path = "../dscvr-tracing-util" } diff --git a/crates/dscvr-canister-agent/src/agent_impl.rs b/crates/dscvr-canister-agent/src/agent_impl.rs index 9425ad2..485145d 100644 --- a/crates/dscvr-canister-agent/src/agent_impl.rs +++ b/crates/dscvr-canister-agent/src/agent_impl.rs @@ -15,11 +15,7 @@ pub trait AgentImpl: Sync + Send { async fn query(&self, canister_id: &Principal, method: &str, args: &[u8]) -> Result>; - async fn read_state_canister_info( - &self, - canister_id: &Principal, - prop: &str, - ) -> Result>; + async fn read_state_canister_info(&self, canister_id: &Principal, prop: &str) -> Result>; async fn clone_with_identity(&self, identity: Arc) -> Result>; diff --git a/crates/dscvr-canister-agent/src/agent_impl/embedded_canister_impl.rs b/crates/dscvr-canister-agent/src/agent_impl/embedded_canister_impl.rs index 7cc6bd8..df08f41 100644 --- a/crates/dscvr-canister-agent/src/agent_impl/embedded_canister_impl.rs +++ b/crates/dscvr-canister-agent/src/agent_impl/embedded_canister_impl.rs @@ -26,14 +26,13 @@ where State: std::marker::Send + 'static, { async fn update(&self, canister_id: &Principal, method: &str, args: &[u8]) -> Result> { - let method: &CanisterUpdateMethod = - self.canister.update_methods.get(method).ok_or_else(|| { - format!( - "Canister {} does not have an update method named {}", - canister_id, method - ) - .into_instrumented_error() - })?; + let method: &CanisterUpdateMethod = self.canister.update_methods.get(method).ok_or_else(|| { + format!( + "Canister {} does not have an update method named {}", + canister_id, method + ) + .into_instrumented_error() + })?; let mut locked_state: std::sync::MutexGuard = self.state.lock().expect("valid"); let system = Edge::new_with_caller_and_time(self.caller, None); @@ -47,27 +46,21 @@ where } async fn query(&self, canister_id: &Principal, method: &str, args: &[u8]) -> Result> { - let method: &CanisterMethod = - self.canister.query_methods.get(method).ok_or_else(|| { - format!( - "Canister {} does not have an query method named {}", - canister_id, method - ) - .into_instrumented_error() - })?; + let method: &CanisterMethod = self.canister.query_methods.get(method).ok_or_else(|| { + format!( + "Canister {} does not have an query method named {}", + canister_id, method + ) + .into_instrumented_error() + })?; let locked_state: std::sync::MutexGuard = self.state.lock().expect("valid"); let system = Edge::new_with_caller_and_time(self.caller, None); - method(ImmutableContext::new(&locked_state, &system), args) - .map_err(|e| e.into_instrumented_error()) + method(ImmutableContext::new(&locked_state, &system), args).map_err(|e| e.into_instrumented_error()) } - async fn read_state_canister_info( - &self, - _canister_id: &Principal, - _prop: &str, - ) -> Result> { + async fn read_state_canister_info(&self, _canister_id: &Principal, _prop: &str) -> Result> { todo!(); } diff --git a/crates/dscvr-canister-agent/src/agent_impl/replica_impl.rs b/crates/dscvr-canister-agent/src/agent_impl/replica_impl.rs index a0c4148..a50dc5a 100644 --- a/crates/dscvr-canister-agent/src/agent_impl/replica_impl.rs +++ b/crates/dscvr-canister-agent/src/agent_impl/replica_impl.rs @@ -32,12 +32,7 @@ impl WrappedAgent { #[async_trait::async_trait] impl AgentImpl for WrappedAgent { async fn query(&self, canister_id: &Principal, method: &str, args: &[u8]) -> Result> { - Ok(self - .agent - .query(canister_id, method) - .with_arg(args) - .call() - .await?) + Ok(self.agent.query(canister_id, method).with_arg(args).call().await?) } async fn update(&self, canister_id: &Principal, method: &str, args: &[u8]) -> Result> { @@ -50,9 +45,7 @@ impl AgentImpl for WrappedAgent { } fn get_principal(&self) -> Result { - self.agent - .get_principal() - .map_err(|e| e.into_instrumented_error()) + self.agent.get_principal().map_err(|e| e.into_instrumented_error()) } async fn clone_with_identity(&self, identity: Arc) -> Result> { @@ -71,11 +64,7 @@ impl AgentImpl for WrappedAgent { Ok(agent) } - async fn read_state_canister_info( - &self, - canister_id: &Principal, - prop: &str, - ) -> Result> { + async fn read_state_canister_info(&self, canister_id: &Principal, prop: &str) -> Result> { Ok(self .agent .read_state_canister_info(canister_id.to_owned(), prop) @@ -83,20 +72,14 @@ impl AgentImpl for WrappedAgent { } } -pub async fn new>( - identity: Arc, - url: U, -) -> Result> { +pub async fn new>(identity: Arc, url: U) -> Result> { let url_string: String = url.into(); let agent = Agent::builder() .with_transport(ReqwestHttpReplicaV2Transport::create(url_string.clone())?) .with_arc_identity(identity) .build()?; - let agent = Arc::new(WrappedAgent { - agent, - url: url_string, - }); + let agent = Arc::new(WrappedAgent { agent, url: url_string }); agent.fetch_root_key().await?; diff --git a/crates/dscvr-canister-agent/src/agent_impl/state_machine_impl.rs b/crates/dscvr-canister-agent/src/agent_impl/state_machine_impl.rs index 5bb1280..c2e8551 100644 --- a/crates/dscvr-canister-agent/src/agent_impl/state_machine_impl.rs +++ b/crates/dscvr-canister-agent/src/agent_impl/state_machine_impl.rs @@ -18,12 +18,7 @@ impl AgentImpl for WrappedStateMachine { async fn query(&self, canister_id: &Principal, method: &str, args: &[u8]) -> Result> { let state_machine = self.machine.lock().unwrap(); match state_machine - .query_call( - canister_id.to_owned(), - self.caller.to_owned(), - method, - args.to_owned(), - ) + .query_call(canister_id.to_owned(), self.caller.to_owned(), method, args.to_owned()) .map_err(|e| e.to_string().into_instrumented_error())? { WasmResult::Reply(reply) => Ok(reply), @@ -34,12 +29,7 @@ impl AgentImpl for WrappedStateMachine { async fn update(&self, canister_id: &Principal, method: &str, args: &[u8]) -> Result> { let state_machine = self.machine.lock().unwrap(); match state_machine - .update_call( - canister_id.to_owned(), - self.caller.to_owned(), - method, - args.to_owned(), - ) + .update_call(canister_id.to_owned(), self.caller.to_owned(), method, args.to_owned()) .map_err(|e| e.to_string().into_instrumented_error())? { WasmResult::Reply(reply) => Ok(reply), @@ -59,20 +49,12 @@ impl AgentImpl for WrappedStateMachine { })) } - async fn read_state_canister_info( - &self, - _canister_id: &Principal, - _prop: &str, - ) -> Result> { + async fn read_state_canister_info(&self, _canister_id: &Principal, _prop: &str) -> Result> { unimplemented!() } } -pub fn new( - caller: Principal, - wasm: Vec, - init_arguments: Vec, -) -> Result<(Arc, Principal)> { +pub fn new(caller: Principal, wasm: Vec, init_arguments: Vec) -> Result<(Arc, Principal)> { // TODO: for multi-canister WrappedStateMachine needs to be a singleton let machine = Arc::new(Mutex::new(StateMachine::new( &std::env::var("STATE_MACHINE_BINARY_PATH").expect("valid state machine binary path"), diff --git a/crates/dscvr-canister-agent/src/lib.rs b/crates/dscvr-canister-agent/src/lib.rs index bb7ec0b..447ed1a 100644 --- a/crates/dscvr-canister-agent/src/lib.rs +++ b/crates/dscvr-canister-agent/src/lib.rs @@ -84,18 +84,6 @@ impl CanisterAgent { }) } - pub async fn new_replica( - caller: Arc, - replica: &str, - canister_id: &str, - ) -> Result { - let agent = Self { - agent: agent_impl::replica_impl::new(caller, replica).await?, - canister_id: Principal::from_text(canister_id)?, - }; - Ok(agent) - } - pub fn new_from_agent(agent: Agent, canister_id: Principal) -> Self where Agent: AgentImpl + 'static, diff --git a/crates/dscvr-canister-agent/src/rust_canister_agent.rs b/crates/dscvr-canister-agent/src/rust_canister_agent.rs new file mode 100644 index 0000000..a3fcc48 --- /dev/null +++ b/crates/dscvr-canister-agent/src/rust_canister_agent.rs @@ -0,0 +1,461 @@ +// Based on Dfinity's rust bindings generator: +// https://github.com/dfinity/candid/blob/master/rust/candid/src/bindings/rust.rs + +use candid::bindings::analysis::chase_actor; +use candid::bindings::analysis::infer_rec; +use candid::bindings::rust::TypePath; +use candid::parser::typing::CheckFileOptions; +use candid::parser::typing::CheckFileResult; +use candid::types::Field; +use candid::types::FuncMode; +use candid::types::Function; +use candid::types::Label; +use candid::types::Type; +use candid::types::TypeInner; +use candid::TypeEnv; +use convert_case::Case; +use convert_case::Casing; +use instrumented_error::Result; +use quote::__private::TokenStream; +use quote::format_ident; +use quote::quote; +use std::collections::BTreeSet; +use std::io::Write; +use std::path::Path; +use std::path::PathBuf; +use syn::Ident; + +fn is_tuple(fs: &[candid::types::Field]) -> bool { + if fs.is_empty() { + return false; + } + !fs.iter().enumerate().any(|(i, field)| field.id.get_id() != (i as u32)) +} + +fn q_ident(id: &str) -> (Ident, bool) { + if id.is_empty() + || id.starts_with(|c: char| !c.is_ascii_alphabetic() && c != '_') + || id.chars().any(|c| !c.is_ascii_alphanumeric() && c != '_') + { + (format_ident!("_{}_", candid::idl_hash(id)), true) + } else if ["crate", "self", "super", "Self"].contains(&id) { + (format_ident!("_{}", id), true) + } else { + (format_ident!("{}", id), false) + } +} + +fn q_field_name(id: &str) -> TokenStream { + let (ident, is_rename) = q_ident(id); + if is_rename { + let id_escape_debug = id.escape_debug().to_string(); + quote!( + #[serde(rename=#id_escape_debug)] + #ident + ) + } else { + quote!(#ident) + } +} + +fn q_label(id: &Label) -> TokenStream { + match id { + Label::Named(str) => q_field_name(str), + Label::Id(n) | Label::Unnamed(n) => { + let ident = format_ident!("_{}_", n); + quote!(#ident) + } + } +} + +fn q_record_field(field: &candid::types::Field, recs: &BTreeSet<&str>) -> TokenStream { + let field_name = q_label(&field.id); + let type_ = q_ty(&field.ty, recs); + quote!(pub #field_name : #type_) +} + +fn q_record_fields(fs: &[candid::types::Field], recs: &BTreeSet<&str>, make_pub: bool) -> TokenStream { + if is_tuple(fs) { + let fields = fs.iter().map(|f| q_ty(&f.ty, recs)); + // We want to make fields on a tuple public + // However `q_record_fields` can be called + // from multiple paths. + // Valid candid: + // type ServiceControllers = vec record { + // principal; + // vec ServiceControllerKind; + // } + // In rust translates to + // type ServiceControllers = Vec<(...)> + // Here we cannot make fields of the tuple pub + // + // Valid candid: + // type TxLogSerializedEntry = record { + // u64; + // ByteBuf; + // } + // Rust equivalent + // pub struct TxLogSerializedEntry = (u64, ByteBuf); + // Here we need to make tuple entrants pub + // Or we have no way to access the members / instantiate + // objects. + if make_pub { + quote!((#(pub #fields),*)) + } else { + quote!((#(#fields),*)) + } + } else { + let fields = fs.iter().map(|f| q_record_field(f, recs)); + quote!({#(#fields),*}) + } +} + +fn q_variant_field(field: &candid::types::Field, recs: &BTreeSet<&str>) -> TokenStream { + match &field.ty.as_ref() { + TypeInner::Null => q_label(&field.id), + TypeInner::Record(fs) => { + let label = q_label(&field.id); + let fields = q_record_fields(fs, recs, false); + quote!(#label #fields) + } + _ => { + let label = q_label(&field.id); + let field = q_ty(&field.ty, recs); + quote!(#label(#field)) + } + } +} + +fn q_ty(ty: &Type, recs: &BTreeSet<&str>) -> TokenStream { + use TypeInner::*; + match ty.as_ref() { + Null => quote!(()), + Bool => quote!(bool), + Nat => quote!(candid::Nat), + Int => quote!(candid::Int), + Nat8 => quote!(u8), + Nat16 => quote!(u16), + Nat32 => quote!(u32), + Nat64 => quote!(u64), + Int8 => quote!(i8), + Int16 => quote!(i16), + Int32 => quote!(i32), + Int64 => quote!(i64), + Float32 => quote!(f32), + Float64 => quote!(f64), + Text => quote!(String), + Reserved => quote!(candid::Reserved), + Empty => quote!(candid::Empty), + Var(ref id) => { + let name = q_ident(id).0; + if recs.contains(id.as_str()) { + quote!(Box<#name>) + } else { + quote!(#name) + } + } + Principal => quote!(candid::Principal), + Opt(ref t) => { + let nested = q_ty(t, recs); + quote!(Option<#nested>) + } + Vec(ref t) => { + let nested = q_ty(t, recs); + quote!(Vec<#nested>) + } + Record(ref fs) => q_record_fields(fs, recs, false), + Variant(_) => unreachable!(), // not possible after rewriting + Func(_) => quote!(candid::Func), + Service(_) => quote!(candid::Service), + Class(_, _) => unreachable!(), + Knot(_) | Unknown => unreachable!(), + Future => unreachable!(), + } +} + +fn q_function(id: &str, func: &Function) -> TokenStream { + let name = q_ident(id).0; + let empty = BTreeSet::new(); + let func_args = func.args.iter().enumerate().map(|(i, ty)| { + let arg_ident = format_ident!("arg{i}"); + let type_ = q_ty(ty, &empty); + quote!(#arg_ident: #type_) + }); + let args = [quote!(agent: &dscvr_canister_agent::CanisterAgent)] + .into_iter() + .chain(func_args); + + let rets = func.rets.iter().map(|ty| q_ty(ty, &empty)); + + let arg_names = func.args.iter().enumerate().map(|(i, _ty)| { + let arg_ident = format_ident!("arg{i}"); + quote!(#arg_ident) + }); + + let agent_call: TokenStream = if func.modes.iter().any(|m| m == &FuncMode::Query) { + quote!(agent.query(#id, args).await?.as_slice()) + } else { + quote!(agent.update(#id, args).await?.as_slice()) + }; + + let rets_decode = [agent_call].into_iter().chain(rets.clone()); + + quote!( + #[tracing::instrument(skip_all)] + pub async fn #name(#(#args),*) -> instrumented_error::Result<(#(#rets),*)> { + let args = candid::Encode!(#(&#arg_names),*)?; + Ok(candid::Decode!(#(#rets_decode),*)?) + } + ) +} + +#[tracing::instrument(skip_all)] +fn generate_types(env: &TypeEnv, def_list: &[&str], recs: &BTreeSet<&str>) -> Result { + let mut ret = TokenStream::default(); + let derive = quote!( + #[derive(Debug, Clone, PartialEq, Eq, candid::CandidType, serde::Deserialize, serde::Serialize, deepsize::DeepSizeOf)] + ); + def_list + .iter() + .map(|id| { + let ty = env.find_type(id).expect("type"); + let name = q_ident(id).0; + match ty.as_ref() { + TypeInner::Record(fs) => { + let fields = q_record_fields(fs, recs, true); + let separator = if is_tuple(fs) { quote!(;) } else { quote!() }; + quote!( + #derive + pub struct #name #fields + #separator + ) + } + TypeInner::Variant(fs) => { + if fs.iter().any(|f| f.id.to_string() == "Ok" || f.id.to_string() == "Err") { + let rets = fs.iter().map(|f| q_ty(&f.ty, &BTreeSet::default())); + quote!( + pub type #name = std::result::Result<#(#rets),*>; + ) + } else { + let fields = fs.iter().map(|f| q_variant_field(f, recs)); + quote!( + #derive + pub enum #name { + #(#fields,)* + } + ) + } + } + _ => { + let field = q_ty(ty, recs); + if recs.contains(id) { + // unit tuple struct + quote!( + #derive + pub struct #name(pub #field); + ) + } else { + // type alias + quote!(type #name = #field;) + } + } + } + }) + .for_each(|tokens| ret.extend(tokens)); + Ok(ret) +} + +fn path_to_var(path: &[TypePath]) -> String { + let name: Vec = path + .iter() + .map(|node| match node { + TypePath::Id(id) => id.to_string(), + TypePath::RecordField(f) | TypePath::VariantField(f) => f.to_string().to_case(Case::Title), + TypePath::Opt => "Inner".to_owned(), + TypePath::Vec => "Item".to_owned(), + TypePath::Func(id) => id.to_string(), + TypePath::Init => "Init".to_owned(), + }) + .collect(); + name.join("") +} + +// Convert structural typing to nominal typing to fit Rust's type system +fn nominalize(env: &mut TypeEnv, path: &mut Vec, t: Type) -> Type { + match t.as_ref() { + TypeInner::Opt(ty) => { + path.push(TypePath::Opt); + let ty = nominalize(env, path, ty.to_owned()); + path.pop(); + TypeInner::Opt(ty).into() + } + TypeInner::Vec(ty) => { + path.push(TypePath::Opt); + let ty = nominalize(env, path, ty.to_owned()); + path.pop(); + TypeInner::Vec(ty).into() + } + TypeInner::Record(fs) => { + if matches!( + path.last(), + None | Some(TypePath::VariantField(_)) | Some(TypePath::Id(_)) + ) || is_tuple(fs) + { + let fs: Vec<_> = fs + .iter() + .map(|Field { id, ty }| { + path.push(TypePath::RecordField(id.to_string())); + let ty = nominalize(env, path, ty.to_owned()); + path.pop(); + Field { id: id.to_owned(), ty } + }) + .collect(); + TypeInner::Record(fs).into() + } else { + let new_var = path_to_var(path); + let ty = nominalize( + env, + &mut vec![TypePath::Id(new_var.clone())], + TypeInner::Record(fs.to_owned()).into(), + ); + env.0.insert(new_var.clone(), ty); + TypeInner::Var(new_var).into() + } + } + TypeInner::Variant(fs) => match path.last() { + None | Some(TypePath::Id(_)) => { + let fs: Vec<_> = fs + .iter() + .map(|Field { id, ty }| { + path.push(TypePath::VariantField(id.to_string())); + let ty = nominalize(env, path, ty.to_owned()); + path.pop(); + Field { id: id.to_owned(), ty } + }) + .collect(); + TypeInner::Variant(fs).into() + } + Some(_) => { + let new_var = path_to_var(path); + let ty = nominalize( + env, + &mut vec![TypePath::Id(new_var.clone())], + TypeInner::Variant(fs.to_owned()).into(), + ); + env.0.insert(new_var.clone(), ty); + TypeInner::Var(new_var).into() + } + }, + TypeInner::Func(func) => TypeInner::Func(Function { + modes: func.modes.clone(), + args: func + .args + .iter() + .enumerate() + .map(|(i, ty)| { + path.push(TypePath::Func(format!("arg{}", i))); + let ty = nominalize(env, path, ty.to_owned()); + path.pop(); + ty + }) + .collect(), + rets: func + .rets + .iter() + .enumerate() + .map(|(i, ty)| { + path.push(TypePath::Func(format!("ret{}", i))); + let ty = nominalize(env, path, ty.to_owned()); + path.pop(); + ty + }) + .collect(), + }) + .into(), + TypeInner::Service(serv) => TypeInner::Service( + serv.iter() + .map(|(meth, ty)| { + path.push(TypePath::Id(meth.to_string())); + let ty = nominalize(env, path, ty.to_owned()); + path.pop(); + (meth.to_owned(), ty) + }) + .collect(), + ) + .into(), + TypeInner::Class(args, ty) => TypeInner::Class( + args.iter() + .map(|ty| { + path.push(TypePath::Init); + let ty = nominalize(env, path, ty.to_owned()); + path.pop(); + ty + }) + .collect(), + nominalize(env, path, ty.to_owned()), + ) + .into(), + _ => t, + } +} + +fn nominalize_all(env: &TypeEnv, actor: &Option) -> (TypeEnv, Option) { + let mut res = TypeEnv(Default::default()); + for (id, ty) in env.0.iter() { + let ty = nominalize(&mut res, &mut vec![TypePath::Id(id.clone())], ty.clone()); + res.0.insert(id.to_string(), ty); + } + let actor = actor.as_ref().map(|ty| nominalize(&mut res, &mut vec![], ty.clone())); + (res, actor) +} + +#[tracing::instrument(skip(tokens))] +fn generate_file(path: &Path, tokens: TokenStream) -> Result<()> { + let mut file = std::fs::File::create(path)?; + file.write_all(b"// @generated\n")?; + file.write_all(b"#![allow(unused)]\n")?; + file.write_all(b"#![allow(non_camel_case_types)]\n")?; + file.write_all(b"#![allow(clippy::upper_case_acronyms)]\n")?; + // TODO: the vec_box should not be needed + file.write_all(b"#![allow(clippy::vec_box)]\n")?; + file.write_all(b"#![allow(clippy::large_enum_variant)]\n")?; + file.write_all(b"use candid::{Encode, Decode};\n")?; + + let tokens_string = tokens.to_string(); + let syn_file = syn::parse_file(&tokens_string)?; + file.write_all(prettyplease::unparse(&syn_file).as_bytes())?; + + Ok(()) +} + +#[tracing::instrument] +pub fn generate(did: &Path, output: &Path) -> Result> { + let CheckFileResult { types, actor, imports } = candid::parser::typing::check_file_with_options( + did, + &CheckFileOptions { + pretty_errors: false, + combine_actors: true, + }, + )?; + let (env, actor) = nominalize_all(&types, &actor); + let def_list: Vec<_> = if let Some(actor) = &actor { + chase_actor(&env, actor).unwrap() + } else { + env.0.iter().map(|pair| pair.0.as_ref()).collect() + }; + let recs = infer_rec(&env, &def_list)?; + let mut tokens = generate_types(&env, &def_list, &recs)?; + + if let Some(actor) = actor { + let serv = env.as_service(&actor).unwrap(); + serv.iter() + .map(|(id, func)| { + let func = env.as_func(func).unwrap(); + q_function(id, func) + }) + .for_each(|f| tokens.extend(f)); + } + + generate_file(output, tokens)?; + Ok(imports) +} diff --git a/crates/dscvr-canister-agent/src/stable_storage_restore_backup.rs b/crates/dscvr-canister-agent/src/stable_storage_restore_backup.rs index 2acf6c6..02cf483 100644 --- a/crates/dscvr-canister-agent/src/stable_storage_restore_backup.rs +++ b/crates/dscvr-canister-agent/src/stable_storage_restore_backup.rs @@ -5,9 +5,7 @@ use async_stream::try_stream; use candid::Encode; use futures::TryStreamExt; use futures::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, SinkExt}; -use ic_canister_stable_storage::{ - data_format::DataFormatType, header::Header, transient::Transient, -}; +use ic_canister_stable_storage::{data_format::DataFormatType, header::Header, transient::Transient}; use instrumented_error::{BoxedInstrumentedError, Result}; use serde_bytes::{ByteBuf, Bytes}; use tokio_retry::strategy::{jitter, ExponentialBackoff}; @@ -57,11 +55,7 @@ impl CanisterAgent { debug!("Fetching {} of {}", offset, len); let bytes = Encode!(&offset, &std::cmp::min(BACKUP_CHUNK_SIZE, len - offset))?; - Ok(Decode!( - self.query("backup_stable_storage", bytes).await?.as_slice(), - ByteBuf - )? - .into_vec()) + Ok(Decode!(self.query("backup_stable_storage", bytes).await?.as_slice(), ByteBuf)?.into_vec()) } /// Backup the stable storage of a canister to a writer @@ -91,11 +85,7 @@ impl CanisterAgent { } item }) - .forward( - (&mut writer) - .into_sink() - .sink_err_into::(), - ) + .forward((&mut writer).into_sink().sink_err_into::()) .await?; let len = len as usize; if total_written != len { @@ -109,11 +99,7 @@ impl CanisterAgent { /// Restore the stable storage of a canister from a reader #[tracing::instrument(skip_all)] - pub async fn restore_stable_storage( - &self, - mut reader: R, - restore_offest: Option, - ) -> Result<()> + pub async fn restore_stable_storage(&self, mut reader: R, restore_offest: Option) -> Result<()> where R: AsyncReadExt + AsyncRead + Unpin + Send + 'static, { @@ -167,26 +153,18 @@ impl CanisterAgent { { let bytes = candid::Encode!(&true)?; - self.update("set_restore_from_stable_storage", bytes) - .await?; + self.update("set_restore_from_stable_storage", bytes).await?; } Ok(()) } - async fn restore( - self: CanisterAgent, - bytes: Arc>, - len: u64, - offset: u64, - ) -> Result<()> { + async fn restore(self: CanisterAgent, bytes: Arc>, len: u64, offset: u64) -> Result<()> { debug!("Restoring {} of {}", offset, len); let ret = { let encoded = candid::Encode!(&offset, &Bytes::new(&bytes[..]))?; - self.update("restore_stable_storage", encoded) - .await - .map(|_| ()) + self.update("restore_stable_storage", encoded).await.map(|_| ()) }; if let Err(e) = ret.as_ref() { @@ -200,10 +178,7 @@ impl CanisterAgent { /// Return the default file name to be used for stable storage backups /// Note: This makes a network call to retrieve the module hash of the canister. - pub async fn get_default_stable_storage_backup_file_name( - &self, - prefix: &str, - ) -> Result { + pub async fn get_default_stable_storage_backup_file_name(&self, prefix: &str) -> Result { let stats = self.canister_stats::().await?; let hash = hex::encode(self.canister_module_hash().await?); let time = OffsetDateTime::from_unix_timestamp_nanos(stats.last_upgraded as i128)?; @@ -211,10 +186,8 @@ impl CanisterAgent { "{}_{}_{}", prefix, &hash[0..5], - time.format(format_description!( - "[year]-[month]-[day]_[hour]-[minute]-[second]" - )) - .unwrap() + time.format(format_description!("[year]-[month]-[day]_[hour]-[minute]-[second]")) + .unwrap() )) } } diff --git a/crates/dscvr-canister-agent/src/stats.rs b/crates/dscvr-canister-agent/src/stats.rs index 016db9c..338c6f1 100644 --- a/crates/dscvr-canister-agent/src/stats.rs +++ b/crates/dscvr-canister-agent/src/stats.rs @@ -12,9 +12,6 @@ impl CanisterAgent { Stats: candid::CandidType, { let bytes = Encode!()?; - Ok(Decode!( - self.query("stats", bytes).await?.as_slice(), - Stats - )?) + Ok(Decode!(self.query("stats", bytes).await?.as_slice(), Stats)?) } } diff --git a/crates/dscvr-canister-agent/src/util.rs b/crates/dscvr-canister-agent/src/util.rs new file mode 100644 index 0000000..90ad054 --- /dev/null +++ b/crates/dscvr-canister-agent/src/util.rs @@ -0,0 +1,22 @@ +use candid::parser::typing::{check_file_with_options, CheckFileOptions}; +use instrumented_error::Result; +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +/// Combines all imported candid files into a single file. +#[tracing::instrument] +pub fn combine_candid_files(path: &Path, output_file: &str) -> Result> { + let candid_path = Path::new(path); + let result = check_file_with_options( + candid_path, + &CheckFileOptions { + pretty_errors: false, + combine_actors: true, + }, + )?; + // export the did to all defined networks + let contents = candid::bindings::candid::compile(&result.types, &result.actor); + std::fs::write(output_file, contents)?; + + Ok(result.imports) +} From 927749d7cb2df97bc18109c7a2ec0580919d6c3e Mon Sep 17 00:00:00 2001 From: Chandra Penke Date: Wed, 6 Mar 2024 15:03:55 -0800 Subject: [PATCH 10/15] chore: fmt --- crates/dscvr-canister-agent/src/agent_impl.rs | 6 ++- .../src/agent_impl/embedded_canister_impl.rs | 39 ++++++++------- .../src/agent_impl/replica_impl.rs | 27 +++++++++-- .../src/agent_impl/state_machine_impl.rs | 26 ++++++++-- .../src/stable_storage_restore_backup.rs | 47 +++++++++++++++---- crates/dscvr-canister-agent/src/stats.rs | 5 +- 6 files changed, 113 insertions(+), 37 deletions(-) diff --git a/crates/dscvr-canister-agent/src/agent_impl.rs b/crates/dscvr-canister-agent/src/agent_impl.rs index 485145d..9425ad2 100644 --- a/crates/dscvr-canister-agent/src/agent_impl.rs +++ b/crates/dscvr-canister-agent/src/agent_impl.rs @@ -15,7 +15,11 @@ pub trait AgentImpl: Sync + Send { async fn query(&self, canister_id: &Principal, method: &str, args: &[u8]) -> Result>; - async fn read_state_canister_info(&self, canister_id: &Principal, prop: &str) -> Result>; + async fn read_state_canister_info( + &self, + canister_id: &Principal, + prop: &str, + ) -> Result>; async fn clone_with_identity(&self, identity: Arc) -> Result>; diff --git a/crates/dscvr-canister-agent/src/agent_impl/embedded_canister_impl.rs b/crates/dscvr-canister-agent/src/agent_impl/embedded_canister_impl.rs index df08f41..7cc6bd8 100644 --- a/crates/dscvr-canister-agent/src/agent_impl/embedded_canister_impl.rs +++ b/crates/dscvr-canister-agent/src/agent_impl/embedded_canister_impl.rs @@ -26,13 +26,14 @@ where State: std::marker::Send + 'static, { async fn update(&self, canister_id: &Principal, method: &str, args: &[u8]) -> Result> { - let method: &CanisterUpdateMethod = self.canister.update_methods.get(method).ok_or_else(|| { - format!( - "Canister {} does not have an update method named {}", - canister_id, method - ) - .into_instrumented_error() - })?; + let method: &CanisterUpdateMethod = + self.canister.update_methods.get(method).ok_or_else(|| { + format!( + "Canister {} does not have an update method named {}", + canister_id, method + ) + .into_instrumented_error() + })?; let mut locked_state: std::sync::MutexGuard = self.state.lock().expect("valid"); let system = Edge::new_with_caller_and_time(self.caller, None); @@ -46,21 +47,27 @@ where } async fn query(&self, canister_id: &Principal, method: &str, args: &[u8]) -> Result> { - let method: &CanisterMethod = self.canister.query_methods.get(method).ok_or_else(|| { - format!( - "Canister {} does not have an query method named {}", - canister_id, method - ) - .into_instrumented_error() - })?; + let method: &CanisterMethod = + self.canister.query_methods.get(method).ok_or_else(|| { + format!( + "Canister {} does not have an query method named {}", + canister_id, method + ) + .into_instrumented_error() + })?; let locked_state: std::sync::MutexGuard = self.state.lock().expect("valid"); let system = Edge::new_with_caller_and_time(self.caller, None); - method(ImmutableContext::new(&locked_state, &system), args).map_err(|e| e.into_instrumented_error()) + method(ImmutableContext::new(&locked_state, &system), args) + .map_err(|e| e.into_instrumented_error()) } - async fn read_state_canister_info(&self, _canister_id: &Principal, _prop: &str) -> Result> { + async fn read_state_canister_info( + &self, + _canister_id: &Principal, + _prop: &str, + ) -> Result> { todo!(); } diff --git a/crates/dscvr-canister-agent/src/agent_impl/replica_impl.rs b/crates/dscvr-canister-agent/src/agent_impl/replica_impl.rs index a50dc5a..a0c4148 100644 --- a/crates/dscvr-canister-agent/src/agent_impl/replica_impl.rs +++ b/crates/dscvr-canister-agent/src/agent_impl/replica_impl.rs @@ -32,7 +32,12 @@ impl WrappedAgent { #[async_trait::async_trait] impl AgentImpl for WrappedAgent { async fn query(&self, canister_id: &Principal, method: &str, args: &[u8]) -> Result> { - Ok(self.agent.query(canister_id, method).with_arg(args).call().await?) + Ok(self + .agent + .query(canister_id, method) + .with_arg(args) + .call() + .await?) } async fn update(&self, canister_id: &Principal, method: &str, args: &[u8]) -> Result> { @@ -45,7 +50,9 @@ impl AgentImpl for WrappedAgent { } fn get_principal(&self) -> Result { - self.agent.get_principal().map_err(|e| e.into_instrumented_error()) + self.agent + .get_principal() + .map_err(|e| e.into_instrumented_error()) } async fn clone_with_identity(&self, identity: Arc) -> Result> { @@ -64,7 +71,11 @@ impl AgentImpl for WrappedAgent { Ok(agent) } - async fn read_state_canister_info(&self, canister_id: &Principal, prop: &str) -> Result> { + async fn read_state_canister_info( + &self, + canister_id: &Principal, + prop: &str, + ) -> Result> { Ok(self .agent .read_state_canister_info(canister_id.to_owned(), prop) @@ -72,14 +83,20 @@ impl AgentImpl for WrappedAgent { } } -pub async fn new>(identity: Arc, url: U) -> Result> { +pub async fn new>( + identity: Arc, + url: U, +) -> Result> { let url_string: String = url.into(); let agent = Agent::builder() .with_transport(ReqwestHttpReplicaV2Transport::create(url_string.clone())?) .with_arc_identity(identity) .build()?; - let agent = Arc::new(WrappedAgent { agent, url: url_string }); + let agent = Arc::new(WrappedAgent { + agent, + url: url_string, + }); agent.fetch_root_key().await?; diff --git a/crates/dscvr-canister-agent/src/agent_impl/state_machine_impl.rs b/crates/dscvr-canister-agent/src/agent_impl/state_machine_impl.rs index c2e8551..5bb1280 100644 --- a/crates/dscvr-canister-agent/src/agent_impl/state_machine_impl.rs +++ b/crates/dscvr-canister-agent/src/agent_impl/state_machine_impl.rs @@ -18,7 +18,12 @@ impl AgentImpl for WrappedStateMachine { async fn query(&self, canister_id: &Principal, method: &str, args: &[u8]) -> Result> { let state_machine = self.machine.lock().unwrap(); match state_machine - .query_call(canister_id.to_owned(), self.caller.to_owned(), method, args.to_owned()) + .query_call( + canister_id.to_owned(), + self.caller.to_owned(), + method, + args.to_owned(), + ) .map_err(|e| e.to_string().into_instrumented_error())? { WasmResult::Reply(reply) => Ok(reply), @@ -29,7 +34,12 @@ impl AgentImpl for WrappedStateMachine { async fn update(&self, canister_id: &Principal, method: &str, args: &[u8]) -> Result> { let state_machine = self.machine.lock().unwrap(); match state_machine - .update_call(canister_id.to_owned(), self.caller.to_owned(), method, args.to_owned()) + .update_call( + canister_id.to_owned(), + self.caller.to_owned(), + method, + args.to_owned(), + ) .map_err(|e| e.to_string().into_instrumented_error())? { WasmResult::Reply(reply) => Ok(reply), @@ -49,12 +59,20 @@ impl AgentImpl for WrappedStateMachine { })) } - async fn read_state_canister_info(&self, _canister_id: &Principal, _prop: &str) -> Result> { + async fn read_state_canister_info( + &self, + _canister_id: &Principal, + _prop: &str, + ) -> Result> { unimplemented!() } } -pub fn new(caller: Principal, wasm: Vec, init_arguments: Vec) -> Result<(Arc, Principal)> { +pub fn new( + caller: Principal, + wasm: Vec, + init_arguments: Vec, +) -> Result<(Arc, Principal)> { // TODO: for multi-canister WrappedStateMachine needs to be a singleton let machine = Arc::new(Mutex::new(StateMachine::new( &std::env::var("STATE_MACHINE_BINARY_PATH").expect("valid state machine binary path"), diff --git a/crates/dscvr-canister-agent/src/stable_storage_restore_backup.rs b/crates/dscvr-canister-agent/src/stable_storage_restore_backup.rs index 02cf483..2acf6c6 100644 --- a/crates/dscvr-canister-agent/src/stable_storage_restore_backup.rs +++ b/crates/dscvr-canister-agent/src/stable_storage_restore_backup.rs @@ -5,7 +5,9 @@ use async_stream::try_stream; use candid::Encode; use futures::TryStreamExt; use futures::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, SinkExt}; -use ic_canister_stable_storage::{data_format::DataFormatType, header::Header, transient::Transient}; +use ic_canister_stable_storage::{ + data_format::DataFormatType, header::Header, transient::Transient, +}; use instrumented_error::{BoxedInstrumentedError, Result}; use serde_bytes::{ByteBuf, Bytes}; use tokio_retry::strategy::{jitter, ExponentialBackoff}; @@ -55,7 +57,11 @@ impl CanisterAgent { debug!("Fetching {} of {}", offset, len); let bytes = Encode!(&offset, &std::cmp::min(BACKUP_CHUNK_SIZE, len - offset))?; - Ok(Decode!(self.query("backup_stable_storage", bytes).await?.as_slice(), ByteBuf)?.into_vec()) + Ok(Decode!( + self.query("backup_stable_storage", bytes).await?.as_slice(), + ByteBuf + )? + .into_vec()) } /// Backup the stable storage of a canister to a writer @@ -85,7 +91,11 @@ impl CanisterAgent { } item }) - .forward((&mut writer).into_sink().sink_err_into::()) + .forward( + (&mut writer) + .into_sink() + .sink_err_into::(), + ) .await?; let len = len as usize; if total_written != len { @@ -99,7 +109,11 @@ impl CanisterAgent { /// Restore the stable storage of a canister from a reader #[tracing::instrument(skip_all)] - pub async fn restore_stable_storage(&self, mut reader: R, restore_offest: Option) -> Result<()> + pub async fn restore_stable_storage( + &self, + mut reader: R, + restore_offest: Option, + ) -> Result<()> where R: AsyncReadExt + AsyncRead + Unpin + Send + 'static, { @@ -153,18 +167,26 @@ impl CanisterAgent { { let bytes = candid::Encode!(&true)?; - self.update("set_restore_from_stable_storage", bytes).await?; + self.update("set_restore_from_stable_storage", bytes) + .await?; } Ok(()) } - async fn restore(self: CanisterAgent, bytes: Arc>, len: u64, offset: u64) -> Result<()> { + async fn restore( + self: CanisterAgent, + bytes: Arc>, + len: u64, + offset: u64, + ) -> Result<()> { debug!("Restoring {} of {}", offset, len); let ret = { let encoded = candid::Encode!(&offset, &Bytes::new(&bytes[..]))?; - self.update("restore_stable_storage", encoded).await.map(|_| ()) + self.update("restore_stable_storage", encoded) + .await + .map(|_| ()) }; if let Err(e) = ret.as_ref() { @@ -178,7 +200,10 @@ impl CanisterAgent { /// Return the default file name to be used for stable storage backups /// Note: This makes a network call to retrieve the module hash of the canister. - pub async fn get_default_stable_storage_backup_file_name(&self, prefix: &str) -> Result { + pub async fn get_default_stable_storage_backup_file_name( + &self, + prefix: &str, + ) -> Result { let stats = self.canister_stats::().await?; let hash = hex::encode(self.canister_module_hash().await?); let time = OffsetDateTime::from_unix_timestamp_nanos(stats.last_upgraded as i128)?; @@ -186,8 +211,10 @@ impl CanisterAgent { "{}_{}_{}", prefix, &hash[0..5], - time.format(format_description!("[year]-[month]-[day]_[hour]-[minute]-[second]")) - .unwrap() + time.format(format_description!( + "[year]-[month]-[day]_[hour]-[minute]-[second]" + )) + .unwrap() )) } } diff --git a/crates/dscvr-canister-agent/src/stats.rs b/crates/dscvr-canister-agent/src/stats.rs index 338c6f1..016db9c 100644 --- a/crates/dscvr-canister-agent/src/stats.rs +++ b/crates/dscvr-canister-agent/src/stats.rs @@ -12,6 +12,9 @@ impl CanisterAgent { Stats: candid::CandidType, { let bytes = Encode!()?; - Ok(Decode!(self.query("stats", bytes).await?.as_slice(), Stats)?) + Ok(Decode!( + self.query("stats", bytes).await?.as_slice(), + Stats + )?) } } From 94622a396da30231909f62c9f7d022e1d098795e Mon Sep 17 00:00:00 2001 From: Chandra Penke Date: Wed, 6 Mar 2024 15:04:24 -0800 Subject: [PATCH 11/15] fix: remove patch from candid --- Cargo.toml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9389c2f..e941a79 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,6 +57,4 @@ tracing-stackdriver = "0.8" tracing-subscriber = { version = "0.3", features = ["env-filter"] } [patch.crates-io] -candid = { git = "https://github.com/dscvr-one/candid.git", rev = "0.9.3-3", features = [ - "deepsize", -] } +candid = { git = "https://github.com/dscvr-one/candid.git", rev = "0.9.3-3" } From b664e5c464333a0c550ad68c99826e497a702f86 Mon Sep 17 00:00:00 2001 From: Chandra Penke Date: Wed, 6 Mar 2024 15:13:47 -0800 Subject: [PATCH 12/15] fix: bring back new_replica method --- crates/dscvr-canister-agent/src/lib.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/crates/dscvr-canister-agent/src/lib.rs b/crates/dscvr-canister-agent/src/lib.rs index 447ed1a..ce179d0 100644 --- a/crates/dscvr-canister-agent/src/lib.rs +++ b/crates/dscvr-canister-agent/src/lib.rs @@ -94,6 +94,18 @@ impl CanisterAgent { } } + pub async fn new_replica( + caller: Arc, + replica: &str, + canister_id: &str, + ) -> Result { + let agent = Self { + agent: agent_impl::replica_impl::new(caller, replica).await?, + canister_id: Principal::from_text(canister_id)?, + }; + Ok(agent) + } + pub async fn clone_with_identity(&self, identity: Arc) -> Result { Ok(Self { agent: self.agent.clone_with_identity(identity).await?, From f0304be7164002873d38315c38f0ec525d485aa5 Mon Sep 17 00:00:00 2001 From: Chandra Penke Date: Wed, 6 Mar 2024 15:53:09 -0800 Subject: [PATCH 13/15] fix: use edge image --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad1c46c..65c15e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,7 @@ jobs: build: runs-on: group: rust-heavy + container: dscvrdocker/be-edge-base-builder steps: - uses: actions/checkout@v3 - uses: actions-rs/toolchain@v1 From a4fbc89b19c5b00e4a9f955893b6c5d4493cab13 Mon Sep 17 00:00:00 2001 From: Chandra Penke Date: Wed, 6 Mar 2024 15:55:04 -0800 Subject: [PATCH 14/15] fix: remove install rust --- .github/workflows/ci.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65c15e1..b92507c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,11 +21,6 @@ jobs: container: dscvrdocker/be-edge-base-builder steps: - uses: actions/checkout@v3 - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: ${{ env.RUST_VERSION }} - - uses: jwlawson/actions-setup-cmake@v1.13 - name: Run run: ./build-scripts/build_and_test.sh env: From 9f76e23b82375b2c9c9d17b1aa453bd09f932943 Mon Sep 17 00:00:00 2001 From: Chandra Penke Date: Wed, 6 Mar 2024 16:19:04 -0800 Subject: [PATCH 15/15] fix: add rust toolchain version --- rust-toolchain.toml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 rust-toolchain.toml diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..6939a60 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.73.0" +components = ["rustfmt", "clippy"] +targets = ["wasm32-unknown-unknown"]