Use Docker Swarm to manage Ciphernodes

.github/workflows/ecs-deployment.yml

@@ -1,4 +1,4 @@
-name: Build and Deploy Ciphernode to ECS
+name: Build and Deploy Ciphernode 
 
 on:
   push:
@@ -52,7 +52,14 @@ jobs:
         ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
         IMAGE_TAG: ${{ github.sha }}
       run: |
+        CURRENT_IMAGE_ID=$(docker images -q $ECR_REGISTRY/$ECR_REPOSITORY:latest)
         docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f $DOCKERFILE_PATH .
+        if [ -n "$CURRENT_IMAGE_ID" ]; then
+          SHORT_SHA=${CURRENT_IMAGE_ID:0:12}
+          docker tag $CURRENT_IMAGE_ID $ECR_REGISTRY/$ECR_REPOSITORY:$SHORT_SHA
+          docker rmi $ECR_REGISTRY/$ECR_REPOSITORY:latest
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$SHORT_SHA
+        fi
         docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:latest
         docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
         docker push $ECR_REGISTRY/$ECR_REPOSITORY:latest

docker-compose.dev.yml

@@ -1,3 +1,9 @@
 services:
-  ciphernode:
+  cn1:
+    network_mode: "host"
+  cn2:
+    network_mode: "host"
+  cn3:
+    network_mode: "host"
+  aggregator:
     network_mode: "host"

docker-compose.yml

@@ -1,17 +1,94 @@
 services:
-  ciphernode:
+  cn1:
+    container_name: cn1
     build:
       context: .
       dockerfile: ./packages/ciphernode/Dockerfile
     image: ciphernode:latest
     volumes:
-      - ${CONFIG_FILE}:/home/ciphernode/.config/enclave/config.yaml:ro  # Read-only config directory
-      - ${SECRETS_FILE}:/home/ciphernode/secrets/secrets.json:ro  # Read-only secrets directory
-      - ciphernode-data:/home/ciphernode/.local/share/enclave  # Persistent data
+      - ./configs/cn1.yaml:/home/ciphernode/.config/enclave/config.yaml:ro  # Read-only config directory
+      - cn1-data:/home/ciphernode/.local/share/enclave  # Persistent data
+    secrets:
+      - secrets.json
     environment:
       RUST_LOG: "info"
       AGGREGATOR: "false"
-    restart: unless-stopped
+    deploy:
+      replicas: 1
+    networks:
+      - cn1-network
 
+
+  cn2:
+    container_name: cn2
+    build:
+      context: .
+      dockerfile: ./packages/ciphernode/Dockerfile
+    image: ciphernode:latest
+    volumes:
+      - ./configs/cn2.yaml:/home/ciphernode/.config/enclave/config.yaml:ro  # Read-only config directory
+      - cn2-data:/home/ciphernode/.local/share/enclave  # Persistent data
+    secrets:
+      - secrets.json
+    environment:
+      RUST_LOG: "info"
+      AGGREGATOR: "false"
+    deploy:
+      replicas: 1
+    networks:
+      - cn2-network
+
+  cn3:
+    container_name: cn3
+    build:
+      context: .
+      dockerfile: ./packages/ciphernode/Dockerfile
+    image: ciphernode:latest
+    volumes:
+      - ./configs/cn3.yaml:/home/ciphernode/.config/enclave/config.yaml:ro  # Read-only config directory
+      - cn3-data:/home/ciphernode/.local/share/enclave  # Persistent data
+    secrets:
+      - secrets.json
+    environment:
+      RUST_LOG: "info"
+      AGGREGATOR: "false"
+    deploy:
+      replicas: 1
+    networks:
+      - cn3-network
+
+
+  aggregator:
+    container_name: aggregator
+    build:
+      context: .
+      dockerfile: ./packages/ciphernode/Dockerfile
+    image: ciphernode:latest
+    volumes:
+      - ./configs/agg.yaml:/home/ciphernode/.config/enclave/config.yaml:ro  # Read-only config directory
+      - agg-data:/home/ciphernode/.local/share/enclave  # Persistent data
+    secrets:
+      - secrets.json
+    environment:
+      RUST_LOG: "info"
+      AGGREGATOR: "true"
+    deploy:
+      replicas: 1
+    networks:
+      - agg-network
+
+secrets:
+  secrets.json:
+    file: secrets.json
+
 volumes:
-  ciphernode-data:
+  cn1-data:
+  cn2-data:
+  cn3-data:
+  agg-data:
+
+networks:
+  cn1-network:
+  cn2-network:
+  cn3-network:
+  agg-network:

packages/ciphernode/ciphernode-entrypoint.sh

@@ -3,7 +3,7 @@ set -e
 
 # Paths to config and secrets
 CONFIG_FILE="$CONFIG_DIR/config.yaml"
-SECRETS_FILE="$SECRETS_DIR/secrets.json"
+SECRETS_FILE="/run/secrets/secrets.json"
 AGGREGATOR="$AGGREGATOR"
 
 # Ensure required files exist

packages/ciphernode/config/src/app_config.rs

@@ -89,6 +89,8 @@ pub struct AppConfig {
     address: Option<Address>,
     /// A list of libp2p multiaddrs to dial to as peers when joining the network
     peers: Vec<String>,
+    /// The port to use for the quic listener
+    quic_port: u16,
 }
 
 impl Default for AppConfig {
@@ -104,6 +106,7 @@ impl Default for AppConfig {
             peers: vec![], // NOTE: This should remain empty and we should look at config
             // generation via ipns fetch for the latest nodes
             address: None,
+            quic_port: 9091,
         }
     }
 }
@@ -179,6 +182,10 @@ impl AppConfig {
     pub fn peers(&self) -> Vec<String> {
         self.peers.clone()
     }
+
+    pub fn quic_port(&self) -> u16 {
+        self.quic_port
+    }
 }
 
 /// Load the config at the config_file or the default location if not provided

packages/ciphernode/enclave_node/src/aggregator.rs

@@ -85,6 +85,7 @@ pub async fn setup_aggregator(
         bus.clone(),
         config.peers(),
         &cipher,
+        config.quic_port(),
         repositories.libp2pid(),
     )
     .await?;

packages/ciphernode/enclave_node/src/ciphernode.rs

@@ -77,6 +77,7 @@ pub async fn setup_ciphernode(
         bus.clone(),
         config.peers(),
         &cipher,
+        config.quic_port(),
         repositories.libp2pid(),
     )
     .await?;

packages/ciphernode/net/src/network_manager.rs

@@ -72,6 +72,7 @@ impl NetworkManager {
         bus: Addr<EventBus>,
         peers: Vec<String>,
         cipher: &Arc<Cipher>,
+        quic_port: u16,
         repository: Repository<Vec<u8>>,
     ) -> Result<(Addr<Self>, tokio::task::JoinHandle<Result<()>>, String)> {
         info!("Reading from repository");
@@ -93,7 +94,7 @@ impl NetworkManager {
 
         let ed25519_keypair = ed25519::Keypair::try_from_bytes(&mut bytes)?;
         let keypair: libp2p::identity::Keypair = ed25519_keypair.try_into()?;
-        let mut peer = NetworkPeer::new(&keypair, peers, None, "tmp-enclave-gossip-topic")?;
+        let mut peer = NetworkPeer::new(&keypair, peers, Some(quic_port), "tmp-enclave-gossip-topic")?;
         let rx = peer.rx().ok_or(anyhow!("Peer rx already taken"))?;
         let p2p_addr = NetworkManager::setup(bus, peer.tx(), rx);
         let handle = tokio::spawn(async move { Ok(peer.start().await?) });