feat(nns): Highly scalable voting (#2932)

This PR adds support for processing neuron following across multiple
messages, so that any number of neurons can vote and have their votes
processed, regardless of the size and complexity of the neuron following
graph or the number of neurons.

rs/nervous_system/long_message/src/lib.rs

@@ -6,6 +6,7 @@ use ic_cdk::query;
 use ic_nervous_system_temporary::Temporary;
 #[cfg(not(target_arch = "wasm32"))]
 use std::cell::{Cell, RefCell};
+use std::fmt::{Debug, Display, Formatter};
 
 #[query(hidden = true)]
 fn __long_message_noop() {
@@ -71,6 +72,21 @@ pub fn is_message_over_threshold(instructions_threshold: u64) -> bool {
     instructions_used >= instructions_threshold
 }
 
+#[derive(Debug)]
+pub struct OverCallContextError {
+    limit: u64,
+}
+
+impl Display for OverCallContextError {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "Call context instruction counter exceeded the limit of {}",
+            self.limit
+        )
+    }
+}
+
 /// Breaks the message into smaller parts if the number of instructions used
 /// exceeds the given threshold.
 /// The upper bound is used to determine the maximum number of instructions
@@ -82,19 +98,20 @@ pub fn is_message_over_threshold(instructions_threshold: u64) -> bool {
 /// # Panics if the number of instructions used exceeds the given panic threshold.
 pub async fn noop_self_call_if_over_instructions(
     message_threshold: u64,
-    panic_threshold: Option<u64>,
-) {
+    call_context_threshold: Option<u64>,
+) -> Result<(), OverCallContextError> {
+    // We may still need a new message context for whatever cleanupis needed, but also we will return
+    // an error if the call context is over the threshold.
+    if is_message_over_threshold(message_threshold) {
+        make_noop_call().await;
+    }
+
     // first we check the upper bound to see if we should panic.
-    if let Some(upper_bound) = panic_threshold {
+    if let Some(upper_bound) = call_context_threshold {
         if is_call_context_over_threshold(upper_bound) {
-            panic!(
-                "Canister call exceeded the limit of {} instructions in the call context.",
-                upper_bound
-            );
+            return Err(OverCallContextError { limit: upper_bound });
         }
     }
 
-    if is_message_over_threshold(message_threshold) {
-        make_noop_call().await;
-    }
+    Ok(())
 }

rs/nervous_system/long_message/tests/test_canisters/long_message_canister.rs

@@ -44,7 +44,9 @@ async fn test_next_message_if_over_instructions(params: BreakMessageParams) {
         // Fib(17) was benchmarked at about 80k instructions
         fib(17);
         if use_break {
-            noop_self_call_if_over_instructions(message_threshold, upper_bound).await;
+            noop_self_call_if_over_instructions(message_threshold, upper_bound)
+                .await
+                .expect("Over upper bound");
         }
     }
 }

rs/nervous_system/long_message/tests/test_message_breaks.rs

@@ -100,17 +100,7 @@ fn test_upper_bound() {
         PrincipalId::new_anonymous(),
     )
     .unwrap_err();
-    assert!(
-        err.contains(
-            format!(
-                "Canister call exceeded the limit of {} instructions in the call context.",
-                700_000
-            )
-            .as_str()
-        ),
-        "Error was: {:?}",
-        err
-    );
+    assert!(err.contains("OverCallContextError"), "Error was: {:?}", err);
 
     update_with_sender::<BreakMessageParams, ()>(
         &state_machine,

rs/nns/common/src/lib.rs

@@ -1,7 +1,6 @@
-use crate::pb::v1::NeuronId;
+use crate::pb::v1::{NeuronId, ProposalId};
 use ic_crypto_sha2::Sha256;
-use ic_stable_structures::storable::Bound;
-use ic_stable_structures::Storable;
+use ic_stable_structures::{storable::Bound, Storable};
 use num_traits::bounds::{LowerBounded, UpperBounded};
 use std::{borrow::Cow, convert::TryInto};
 
@@ -44,7 +43,24 @@ impl Storable for NeuronId {
     }
 
     fn from_bytes(bytes: Cow<[u8]>) -> Self {
-        NeuronId {
+        Self {
+            id: u64::from_bytes(bytes),
+        }
+    }
+
+    const BOUND: Bound = Bound::Bounded {
+        max_size: std::mem::size_of::<u64>() as u32,
+        is_fixed_size: true,
+    };
+}
+
+impl Storable for ProposalId {
+    fn to_bytes(&self) -> Cow<[u8]> {
+        self.id.to_bytes()
+    }
+
+    fn from_bytes(bytes: Cow<[u8]>) -> Self {
+        Self {
             id: u64::from_bytes(bytes),
         }
     }

rs/nns/governance/BUILD.bazel

@@ -37,6 +37,7 @@ DEPENDENCIES = [
     "//rs/nervous_system/common",
     "//rs/nervous_system/governance",
     "//rs/nervous_system/linear_map",
+    "//rs/nervous_system/long_message",
     "//rs/nervous_system/neurons_fund",
     "//rs/nervous_system/proto",
     "//rs/nervous_system/root",

rs/nns/governance/Cargo.toml

@@ -51,6 +51,7 @@ ic-nervous-system-common = { path = "../../nervous_system/common" }
 ic-nervous-system-common-build-metadata = { path = "../../nervous_system/common/build_metadata" }
 ic-nervous-system-governance = { path = "../../nervous_system/governance" }
 ic-nervous-system-linear-map = { path = "../../nervous_system/linear_map" }
+ic-nervous-system-long-message = { path = "../../nervous_system/long_message" }
 ic-nervous-system-root = { path = "../../nervous_system/root" }
 ic-nervous-system-runtime = { path = "../../nervous_system/runtime" }
 ic-nervous-system-proto = { path = "../../nervous_system/proto" }

rs/nns/governance/canbench/canbench_results.yml

@@ -1,127 +1,127 @@
 benches:
   add_neuron_active_maximum:
     total:
-      instructions: 36059517
+      instructions: 36184626
       heap_increase: 1
       stable_memory_increase: 0
     scopes: {}
   add_neuron_active_typical:
     total:
-      instructions: 1830145
+      instructions: 1835593
       heap_increase: 0
       stable_memory_increase: 0
     scopes: {}
   add_neuron_inactive_maximum:
     total:
-      instructions: 96070124
+      instructions: 96163372
       heap_increase: 1
       stable_memory_increase: 0
     scopes: {}
   add_neuron_inactive_typical:
     total:
-      instructions: 7372921
+      instructions: 7371198
       heap_increase: 0
       stable_memory_increase: 0
     scopes: {}
   cascading_vote_all_heap:
     total:
-      instructions: 34047728
+      instructions: 34443866
       heap_increase: 0
-      stable_memory_increase: 0
+      stable_memory_increase: 128
     scopes: {}
   cascading_vote_heap_neurons_stable_index:
     total:
-      instructions: 56554267
+      instructions: 56578763
       heap_increase: 0
-      stable_memory_increase: 0
+      stable_memory_increase: 128
     scopes: {}
   cascading_vote_stable_everything:
     total:
-      instructions: 162950875
+      instructions: 370858158
       heap_increase: 0
-      stable_memory_increase: 0
+      stable_memory_increase: 128
     scopes: {}
   cascading_vote_stable_neurons_with_heap_index:
     total:
-      instructions: 140555145
+      instructions: 348673022
       heap_increase: 0
-      stable_memory_increase: 0
+      stable_memory_increase: 128
     scopes: {}
   centralized_following_all_stable:
     total:
-      instructions: 68428123
+      instructions: 173332865
       heap_increase: 0
-      stable_memory_increase: 0
+      stable_memory_increase: 128
     scopes: {}
   compute_ballots_for_new_proposal_with_stable_neurons:
     total:
-      instructions: 1798483
+      instructions: 1811023
       heap_increase: 0
       stable_memory_increase: 0
     scopes: {}
   draw_maturity_from_neurons_fund_heap:
     total:
-      instructions: 7244019
+      instructions: 7254719
       heap_increase: 0
       stable_memory_increase: 0
     scopes: {}
   draw_maturity_from_neurons_fund_stable:
     total:
-      instructions: 10264384
+      instructions: 10293400
       heap_increase: 0
       stable_memory_increase: 0
     scopes: {}
   list_neurons_ready_to_unstake_maturity_heap:
     total:
-      instructions: 474237
+      instructions: 471240
       heap_increase: 0
       stable_memory_increase: 0
     scopes: {}
   list_neurons_ready_to_unstake_maturity_stable:
     total:
-      instructions: 37739237
+      instructions: 36814392
       heap_increase: 0
       stable_memory_increase: 0
     scopes: {}
   list_ready_to_spawn_neuron_ids_heap:
     total:
-      instructions: 462350
+      instructions: 459353
       heap_increase: 0
       stable_memory_increase: 0
     scopes: {}
   list_ready_to_spawn_neuron_ids_stable:
     total:
-      instructions: 37727938
+      instructions: 36803093
       heap_increase: 0
       stable_memory_increase: 0
     scopes: {}
   neuron_metrics_calculation_heap:
     total:
-      instructions: 529394
+      instructions: 528806
       heap_increase: 0
       stable_memory_increase: 0
     scopes: {}
   neuron_metrics_calculation_stable:
     total:
-      instructions: 1865928
+      instructions: 1837632
       heap_increase: 0
       stable_memory_increase: 0
     scopes: {}
   range_neurons_performance:
     total:
-      instructions: 47978708
+      instructions: 47691642
       heap_increase: 0
       stable_memory_increase: 0
     scopes: {}
   single_vote_all_stable:
     total:
-      instructions: 2690730
+      instructions: 2474894
       heap_increase: 0
-      stable_memory_increase: 0
+      stable_memory_increase: 128
     scopes: {}
   update_recent_ballots_stable_memory:
     total:
-      instructions: 13454675
+      instructions: 236883
       heap_increase: 0
       stable_memory_increase: 0
     scopes: {}

rs/nns/governance/canister/canister.rs

@@ -168,6 +168,7 @@ fn schedule_timers() {
     schedule_prune_following(Duration::from_secs(0));
     schedule_spawn_neurons();
     schedule_unstake_maturity_of_dissolved_neurons();
+    schedule_vote_processing();
 
     // TODO(NNS1-3446): Delete. (This only needs to be run once, but can safely be run multiple times).
     schedule_backfill_voting_power_refreshed_timestamps(Duration::from_secs(0));
@@ -327,6 +328,15 @@ fn schedule_unstake_maturity_of_dissolved_neurons() {
     });
 }
 
+/// The interval at which the voting state machines are processed.
+const VOTE_PROCESSING_INTERVAL: Duration = Duration::from_secs(3);
+
+fn schedule_vote_processing() {
+    ic_cdk_timers::set_timer_interval(VOTE_PROCESSING_INTERVAL, || {
+        spawn(governance_mut().process_voting_state_machines());
+    });
+}
+
 struct CanisterEnv {
     rng: Option<ChaCha20Rng>,
     time_warp: GovTimeWarp,

rs/nns/governance/proto/ic_nns_governance/pb/v1/governance.proto

@@ -3005,3 +3005,13 @@ message ArchivedMonthlyNodeProviderRewards {
     ic_nns_governance.pb.v1.MonthlyNodeProviderRewards rewards = 1;
   }
 }
+
+// Internal type to allow ProposalVotingStateMachine to be stored
+// in stable memory.
+message ProposalVotingStateMachine {
+  ic_nns_common.pb.v1.ProposalId proposal_id = 1;
+  Topic topic = 2;
+  repeated ic_nns_common.pb.v1.NeuronId neurons_to_check_followers = 3;
+  repeated ic_nns_common.pb.v1.NeuronId followers_to_check = 4;
+  map<uint64, Vote> recent_neuron_ballots_to_record = 5;
+}

rs/nns/governance/src/gen/ic_nns_governance.pb.v1.rs

@@ -4757,6 +4757,29 @@ pub mod archived_monthly_node_provider_rewards {
         Version1(V1),
     }
 }
+/// Internal type to allow ProposalVotingStateMachine to be stored
+/// in stable memory.
+#[derive(
+    candid::CandidType,
+    candid::Deserialize,
+    serde::Serialize,
+    comparable::Comparable,
+    Clone,
+    PartialEq,
+    ::prost::Message,
+)]
+pub struct ProposalVotingStateMachine {
+    #[prost(message, optional, tag = "1")]
+    pub proposal_id: ::core::option::Option<::ic_nns_common::pb::v1::ProposalId>,
+    #[prost(enumeration = "Topic", tag = "2")]
+    pub topic: i32,
+    #[prost(message, repeated, tag = "3")]
+    pub neurons_to_check_followers: ::prost::alloc::vec::Vec<::ic_nns_common::pb::v1::NeuronId>,
+    #[prost(message, repeated, tag = "4")]
+    pub followers_to_check: ::prost::alloc::vec::Vec<::ic_nns_common::pb::v1::NeuronId>,
+    #[prost(map = "uint64, enumeration(Vote)", tag = "5")]
+    pub recent_neuron_ballots_to_record: ::std::collections::HashMap<u64, i32>,
+}
 /// Proposal types are organized into topics. Neurons can automatically
 /// vote based on following other neurons, and these follow
 /// relationships are defined per topic.