From c9693205faada1b1c8a4d941138fbbf0f52f9e44 Mon Sep 17 00:00:00 2001 From: gonzalezzfelipe Date: Wed, 13 Nov 2024 16:00:51 -0300 Subject: [PATCH 1/3] chore: Snapshot from S3 --- .github/images/init/dockerfile.init | 2 +- .github/workflows/init.yml | 41 +++++++++++++ bootstrap/stage2/deployment.tf | 20 ++++++ bootstrap/stage2/main.tf | 17 ++++++ crates/operator/src/config.rs | 12 +++- crates/operator/src/custom_resource.rs | 84 +++++++++----------------- playbook/doom-dev/main.tf | 40 +++++++----- 7 files changed, 145 insertions(+), 71 deletions(-) create mode 100644 .github/workflows/init.yml diff --git a/.github/images/init/dockerfile.init b/.github/images/init/dockerfile.init index e8c4860..dc84c7f 100644 --- a/.github/images/init/dockerfile.init +++ b/.github/images/init/dockerfile.init @@ -1,4 +1,4 @@ FROM amazon/aws-cli RUN yum update -y && yum install -y tar gzip -COPY docker/entrypoint.sh /entrypoint.sh +COPY .github/images/init/entrypoint.sh /entrypoint.sh ENTRYPOINT ["sh", "/entrypoint.sh"] diff --git a/.github/workflows/init.yml b/.github/workflows/init.yml new file mode 100644 index 0000000..cee5720 --- /dev/null +++ b/.github/workflows/init.yml @@ -0,0 +1,41 @@ +name: Init + +on: + push: + branches: + - "main" + paths: + - ".github/workflows/init.yml" + - ".github/images/init/**" + workflow_dispatch: {} + +jobs: + build-images: + strategy: + fail-fast: false + matrix: + include: + - context: . + file: .github/images/init/dockerfile.init + endpoint: cardano-scaling/hydra-control-plane + + continue-on-error: true + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2.3.3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: ${{ matrix.context }} + file: ${{ matrix.file }} + platforms: linux/amd64 + push: true + tags: ghcr.io/${{ matrix.endpoint }},ghcr.io/${{ matrix.endpoint }}:${{ github.sha }} diff --git a/bootstrap/stage2/deployment.tf b/bootstrap/stage2/deployment.tf index abc42d8..d2ddfab 100644 --- a/bootstrap/stage2/deployment.tf +++ b/bootstrap/stage2/deployment.tf @@ -106,6 +106,26 @@ resource "kubernetes_deployment_v1" "operator" { value = var.dmtr_port_name } + env { + name = "INIT_IMAGE" + value = var.init_image + } + + env { + name = "BUCKET" + value = var.bucket + } + + env { + name = "INIT_AWS_ACCESS_KEY_ID" + value = var.init_aws_access_key_id + } + + env { + name = "INIT_AWS_SECRET_ACCESS_KEY" + value = var.init_aws_secret_access_key + } + resources { limits = { cpu = var.resources.limits.cpu diff --git a/bootstrap/stage2/main.tf b/bootstrap/stage2/main.tf index f10d97e..85eaa56 100644 --- a/bootstrap/stage2/main.tf +++ b/bootstrap/stage2/main.tf @@ -80,6 +80,23 @@ variable "dmtr_port_name" { type = string } +variable "init_image" { + type = string +} + +variable "bucket" { + type = string + default = "hydradoomsnapshots" +} + +variable "init_aws_access_key_id" { + type = string +} + +variable "init_aws_secret_access_key" { + type = string +} + variable "tolerations" { type = list(object({ effect = string diff --git a/crates/operator/src/config.rs b/crates/operator/src/config.rs index e0eddce..a3e5e13 100644 --- a/crates/operator/src/config.rs +++ b/crates/operator/src/config.rs @@ -12,7 +12,7 @@ pub fn get_config() -> &'static Config { #[derive(Debug, Clone)] pub struct Config { pub image: String, - pub open_head_image: String, + pub init_image: String, pub sidecar_image: String, pub configmap: String, pub secret: String, @@ -24,13 +24,15 @@ pub struct Config { pub dmtr_project_id: String, pub dmtr_api_key: String, pub dmtr_port_name: String, + pub bucket: String, + pub init_aws_access_key_id: String, + pub init_aws_secret_access_key: String, } impl Config { pub fn from_env() -> Self { Self { image: env::var("IMAGE").unwrap_or("ghcr.io/cardano-scaling/hydra-node".into()), - open_head_image: env::var("OPEN_HEAD_IMAGE").expect("Missing OPEN_HEAD_IMAGE env var"), sidecar_image: env::var("SIDECAR_IMAGE").expect("Missing SIDECAR_IMAGE env var"), configmap: env::var("CONFIGMAP").expect("Missing CONFIGMAP env var"), secret: env::var("SECRET").expect("Missing SECRET env var"), @@ -43,6 +45,12 @@ impl Config { dmtr_project_id: env::var("DMTR_PROJECT_ID").expect("Missing DMTR_PROJECT_ID env var."), dmtr_api_key: env::var("DMTR_API_KEY").expect("Missing DMTR_API_KEY env var."), dmtr_port_name: env::var("DMTR_PORT_NAME").expect("Missing DMTR_PORT_NAME env var."), + init_image: env::var("INIT_IMAGE").expect("Missing INIT_IMAGE env var."), + bucket: env::var("BUCKET").expect("Missing BUCKET env var."), + init_aws_access_key_id: env::var("INIT_AWS_ACCESS_KEY_ID") + .expect("Missing INIT_AWS_ACCESS_KEY_ID env var."), + init_aws_secret_access_key: env::var("INIT_AWS_SECRET_ACCESS_KEY") + .expect("Missing INIT_AWS_SECRET_ACCESS_KEY env var."), } } } diff --git a/crates/operator/src/custom_resource.rs b/crates/operator/src/custom_resource.rs index cb26a61..1883441 100644 --- a/crates/operator/src/custom_resource.rs +++ b/crates/operator/src/custom_resource.rs @@ -3,7 +3,7 @@ use k8s_openapi::{ apps::v1::{Deployment, DeploymentSpec}, core::v1::{ ConfigMap, ConfigMapVolumeSource, Container, ContainerPort, EmptyDirVolumeSource, - PodSpec, PodTemplateSpec, ResourceRequirements, SecretVolumeSource, Service, + EnvVar, PodSpec, PodTemplateSpec, ResourceRequirements, SecretVolumeSource, Service, ServicePort, ServiceSpec, Volume, VolumeMount, }, networking::v1::{ @@ -184,7 +184,7 @@ impl HydraDoomNode { "--api-port".to_string(), constants.port.to_string(), "--hydra-signing-key".to_string(), - format!("{}/hydra.sk", constants.data_dir), + format!("{}/keys/hydra.sk", constants.data_dir), "--ledger-protocol-parameters".to_string(), format!("{}/protocol-parameters.json", constants.config_dir), "--persistence-dir".to_string(), @@ -264,8 +264,8 @@ impl HydraDoomNode { Container { name: "sidecar".to_string(), image: Some(config.sidecar_image.clone()), - command: Some(vec!["metrics-exporter".to_string()]), args: Some(vec![ + "metrics-exporter".to_string(), "--host".to_string(), "localhost".to_string(), "--port".to_string(), @@ -283,52 +283,6 @@ impl HydraDoomNode { // Offline is optional. If undefined, the node is presumed to be online. if !self.spec.offline.unwrap_or(false) { - let mut open_head_args = vec![ - "open-head".to_string(), - "--network-id".to_string(), - self.spec.network_id.unwrap_or(0).to_string(), - "--seed-input".to_string(), - self.spec.seed_input.clone(), - "--participant".to_string(), - config.admin_addr.clone(), - "--party-verification-file".to_string(), - format!("{}/hydra.vk", constants.data_dir), - "--cardano-key-file".to_string(), - format!("{}/admin.sk", constants.secret_dir), - "--blockfrost-key".to_string(), - config.blockfrost_key.clone(), - ]; - if !self.spec.commit_inputs.is_empty() { - open_head_args.push("--commit-inputs".to_string()); - open_head_args.extend(self.spec.commit_inputs.clone()); - } - - containers.push(Container { - name: "open-head".to_string(), - image: Some(config.open_head_image.clone()), - command: Some(vec!["open-head".to_string()]), - args: Some(open_head_args), - volume_mounts: Some(vec![ - VolumeMount { - name: "config".to_string(), - mount_path: constants.config_dir.clone(), - ..Default::default() - }, - VolumeMount { - name: "secret".to_string(), - mount_path: constants.secret_dir.clone(), - ..Default::default() - }, - VolumeMount { - name: "data".to_string(), - mount_path: constants.data_dir.clone(), - ..Default::default() - }, - ]), - resources: None, - ..Default::default() - }); - containers.push(Container { name: "dmtrctl".to_string(), image: Some(constants.dmtrctl_image.to_string()), @@ -375,11 +329,33 @@ impl HydraDoomNode { spec: Some(PodSpec { init_containers: Some(vec![Container { name: "init".to_string(), - image: Some(config.image.clone()), - args: Some(vec![ - "gen-hydra-key".to_string(), - "--output-file".to_string(), - format!("{}/hydra", constants.data_dir), + image: Some(config.init_image.clone()), + env: Some(vec![ + EnvVar { + name: "BUCKET".to_string(), + value: Some(config.bucket.clone()), + ..Default::default() + }, + EnvVar { + name: "KEY".to_string(), + value: Some(format!("{}.tar.gz", self.name_any())), + ..Default::default() + }, + EnvVar { + name: "DATA_DIR".to_string(), + value: Some(constants.data_dir.clone()), + ..Default::default() + }, + EnvVar { + name: "AWS_ACCESS_KEY_ID".to_string(), + value: Some(config.init_aws_access_key_id.clone()), + ..Default::default() + }, + EnvVar { + name: "AWS_SECRET_ACCESS_KEY".to_string(), + value: Some(config.init_aws_secret_access_key.clone()), + ..Default::default() + }, ]), volume_mounts: Some(vec![VolumeMount { name: "data".to_string(), diff --git a/playbook/doom-dev/main.tf b/playbook/doom-dev/main.tf index 38b9aa9..bbd1a62 100644 --- a/playbook/doom-dev/main.tf +++ b/playbook/doom-dev/main.tf @@ -44,7 +44,7 @@ variable "external_domain" { } variable "image" { - type = string + type = string } variable "hydra_node_image" { @@ -71,6 +71,14 @@ variable "admin_key" { type = string } +variable "snapshot_aws_access_key_id" { + type = string +} + +variable "snapshot_aws_secret_access_key" { + type = string +} + provider "kubernetes" { config_path = "~/.kube/config" config_context = var.eks_cluster_arn @@ -95,17 +103,21 @@ module "stage2" { protocol_parameters = file("${path.module}/protocol-parameters.json") external_port = 80 - namespace = local.namespace - external_domain = var.external_domain - hydra_node_image = var.hydra_node_image - operator_image = var.image - sidecar_image = var.image - open_head_image = var.image - control_plane_image = var.image - blockfrost_key = var.blockfrost_key - admin_addr = var.admin_addr - dmtr_project_id = var.dmtr_project_id - dmtr_api_key = var.dmtr_api_key - dmtr_port_name = var.dmtr_port_name - hydra_scripts_tx_id = var.hydra_scripts_tx_id + namespace = local.namespace + external_domain = var.external_domain + hydra_node_image = var.hydra_node_image + operator_image = var.image + sidecar_image = var.image + open_head_image = var.image + control_plane_image = var.image + blockfrost_key = var.blockfrost_key + admin_addr = var.admin_addr + dmtr_project_id = var.dmtr_project_id + dmtr_api_key = var.dmtr_api_key + dmtr_port_name = var.dmtr_port_name + hydra_scripts_tx_id = var.hydra_scripts_tx_id + init_aws_access_key_id = var.snapshot_aws_access_key_id + init_aws_secret_access_key = var.snapshot_aws_secret_access_key + init_image = "ghcr.io/demeter-run/doom-patrol-init:b7b4fc499b5274cd71b6b72f93ab4ba8199437fe" + } From 0873112b957b1f65f294b79a7579eac983d26235 Mon Sep 17 00:00:00 2001 From: gonzalezzfelipe Date: Wed, 13 Nov 2024 16:02:08 -0300 Subject: [PATCH 2/3] Rename image --- .github/workflows/init.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/init.yml b/.github/workflows/init.yml index cee5720..15696e1 100644 --- a/.github/workflows/init.yml +++ b/.github/workflows/init.yml @@ -17,7 +17,7 @@ jobs: include: - context: . file: .github/images/init/dockerfile.init - endpoint: cardano-scaling/hydra-control-plane + endpoint: cardano-scaling/hydra-control-plane-init continue-on-error: true runs-on: ubuntu-latest From 67c953978dba124371f76b2aa3bfce8cf8a34c6f Mon Sep 17 00:00:00 2001 From: gonzalezzfelipe Date: Wed, 13 Nov 2024 16:09:10 -0300 Subject: [PATCH 3/3] Add snapshots keys from secrets --- .github/workflows/deploy.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 094cf8b..bc8feb7 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -19,6 +19,8 @@ jobs: TF_VAR_blockfrost_key: ${{ secrets.DEV_BLOCKFROST_KEY }} TF_VAR_dmtr_api_key: ${{ secrets.DEV_DMTR_API_KEY }} TF_VAR_admin_key: ${{ secrets.DEV_HYDRA_ADMIN_KEY }} + TF_VAR_snapshot_aws_access_key_id: ${{ secrets.SNAPSHOT_AWS_ACCESS_KEY_ID }} + TF_VAR_snapshot_aws_secret_access_key: ${{ secrets.SNAPSHOT_AWS_SECRET_ACCESS_KEY }} # Vars TF_VAR_dmtr_port_name: preprod-4raar2