diff --git a/README.md b/README.md index 93d54f8..9a4111d 100644 --- a/README.md +++ b/README.md @@ -3,14 +3,9 @@ This module allows you to create a manually-scalable high-availability nomad cluster on Hetzner Cloud. You only need to provide a API token as variable and a default cluster with 3 servers and 1 client will be created. +## Dependencies +- [jq](https://stedolan.github.io/jq/) + ## Advanced Usage The module is mainly addressed to people who want to test the technology running terraform on their local PC, but it can be used in professional workflows as well. - -### Usage in CI/CD pipelines - -The module creates on startup the CA files as well as a master key for the consul communication. -Generally the folder of these files is created on the initial apply and destroy after a terraform destroy. -Nevertheless there is a condition that checks if these files are already in place during the first apply and dependent on this it will create or will not create new ones. -In addition to this, it is required that you create dummy key pairs for the already existing server and clients. The files can be empty it is just a requirement for them to exist as terraform keeps their reference in the state and tries to read it on plan/destroy/apply. - diff --git a/examples/advanced-setup/main.tf b/examples/advanced-setup/main.tf index 6cede2c..fa1b517 100644 --- a/examples/advanced-setup/main.tf +++ b/examples/advanced-setup/main.tf @@ -1,7 +1,7 @@ module "hetzner-nomad-consul" { source = "../../" hetzner_token = var.hetzner_token - nomad_client_count = 1 + nomad_client_count = 2 } terraform { @@ -73,10 +73,6 @@ output "nomad_address" { value = module.hetzner-nomad-consul.nomad_address } -output "vault_address_http" { - value = module.hetzner-nomad-consul.vault_address_http -} - provider "cloudflare" { api_token = var.cloudflare_token } @@ -100,3 +96,72 @@ resource "cloudflare_record" "traefik" { proxied = true value = local.traefik_ip } + +resource "hcloud_load_balancer" "app_load_balancer" { + name = "api-load-balancer" + load_balancer_type = "lb11" + location = "hel1" +} + +resource "hcloud_load_balancer_network" "app_load_balancer" { + load_balancer_id = hcloud_load_balancer.app_load_balancer.id + network_id = module.hetzner-nomad-consul.network_id +} + +resource "hcloud_load_balancer_service" "app_load_balancer_service_traefik_dashboard" { + load_balancer_id = hcloud_load_balancer.app_load_balancer.id + protocol = "http" + listen_port = 8081 + destination_port = 8081 + http { + sticky_sessions = true + } + health_check { + protocol = "http" + port = 8081 + interval = 10 + timeout = 5 + retries = 3 + http { + path = "/" + status_codes = [ + "2??", + "3??", + ] + } + } +} + +resource "hcloud_load_balancer_service" "app_load_balancer_service_traefik_proxy" { + load_balancer_id = hcloud_load_balancer.app_load_balancer.id + protocol = "http" + listen_port = 80 + destination_port = 8080 + http { + sticky_sessions = true + } + health_check { + protocol = "http" + port = 8081 + interval = 10 + timeout = 5 + retries = 3 + http { + path = "/" + status_codes = [ + "2??", + "3??", + ] + } + } +} + +resource "hcloud_load_balancer_target" "app_load_balancer_target" { + depends_on = [ + hcloud_load_balancer_network.app_load_balancer + ] + type = "label_selector" + load_balancer_id = hcloud_load_balancer.app_load_balancer.id + label_selector = "nomad-client" + use_private_ip = true +} diff --git a/hcloud_firewall.tf b/hcloud_firewall.tf index f4a75fd..9a01814 100644 --- a/hcloud_firewall.tf +++ b/hcloud_firewall.tf @@ -13,8 +13,7 @@ resource "hcloud_firewall" "default" { resource "hcloud_firewall_attachment" "default" { depends_on = [ - hcloud_server.main, - hcloud_server.vault + hcloud_server.main ] firewall_id = hcloud_firewall.default.id label_selectors = [ "nomad-server", "nomad-client", "vault-server" ] diff --git a/hcloud_load_balancer.tf b/hcloud_load_balancer.tf index 0f40a9d..4783913 100644 --- a/hcloud_load_balancer.tf +++ b/hcloud_load_balancer.tf @@ -2,7 +2,7 @@ resource "hcloud_load_balancer" "load_balancer" { depends_on = [ null_resource.fetch_nomad_token ] - name = "my-load-balancer" + name = "nomad-load-balancer" load_balancer_type = "lb11" location = var.hetzner_datacenter } @@ -49,76 +49,4 @@ resource "hcloud_load_balancer_target" "load_balancer_target" { resource "local_file" "load_balancer_ip" { content = hcloud_load_balancer.load_balancer.ipv4 filename = "${path.root}/certs/nomad_address" -} - -resource "hcloud_load_balancer" "app_load_balancer" { - depends_on = [ - null_resource.fetch_nomad_token - ] - name = "my-app-load-balancer" - load_balancer_type = "lb11" - location = var.hetzner_datacenter -} - -resource "hcloud_load_balancer_network" "app_load_balancer" { - load_balancer_id = hcloud_load_balancer.app_load_balancer.id - network_id = hcloud_network.network.id -} - -resource "hcloud_load_balancer_service" "app_load_balancer_service_traefik_dashboard" { - load_balancer_id = hcloud_load_balancer.app_load_balancer.id - protocol = "http" - listen_port = 8081 - destination_port = 8081 - http { - sticky_sessions = true - } - health_check { - protocol = "http" - port = 8081 - interval = 10 - timeout = 5 - retries = 3 - http { - path = "/" - status_codes = [ - "2??", - "3??", - ] - } - } -} - -resource "hcloud_load_balancer_service" "app_load_balancer_service_traefik_proxy" { - load_balancer_id = hcloud_load_balancer.app_load_balancer.id - protocol = "http" - listen_port = 80 - destination_port = 8080 - http { - sticky_sessions = true - } - health_check { - protocol = "http" - port = 8081 - interval = 10 - timeout = 5 - retries = 3 - http { - path = "/" - status_codes = [ - "2??", - "3??", - ] - } - } -} - -resource "hcloud_load_balancer_target" "app_load_balancer_target" { - depends_on = [ - hcloud_load_balancer_network.app_load_balancer - ] - type = "label_selector" - load_balancer_id = hcloud_load_balancer.app_load_balancer.id - label_selector = "nomad-client" - use_private_ip = true } \ No newline at end of file diff --git a/hcloud_server.tf b/hcloud_server.tf index cfd9236..0280e43 100644 --- a/hcloud_server.tf +++ b/hcloud_server.tf @@ -1,7 +1,6 @@ resource "hcloud_server" "main" { depends_on = [ - hcloud_network_subnet.network, - hcloud_server.vault + hcloud_network_subnet.network ] for_each = local.Aggregator_Data name = each.key @@ -20,6 +19,23 @@ resource "hcloud_server" "main" { public_net { ipv6_enabled = false } + + user_data = file("${path.module}/scripts/base_configuration.sh") + + provisioner "remote-exec" { + inline = [ + "echo 'Waiting for cloud-init to complete...'", + "cloud-init status --wait > /dev/null", + "echo 'Completed cloud-init!'", + ] + + connection { + type = "ssh" + host = self.ipv4_address + user = "root" + private_key = tls_private_key.machines.private_key_openssh + } + } } resource "null_resource" "deployment" { @@ -35,26 +51,17 @@ resource "null_resource" "deployment" { } provisioner "file" { - content = tls_private_key.machines.private_key_openssh - destination = "machines.pem" - } - - provisioner "file" { - content = join("\n", [file("${path.module}/scripts/base_configuration.sh"), - each.value.type == "server" ? templatefile("${path.module}/scripts/server_setup.sh", + content = each.value.type == "server" ? templatefile("${path.module}/scripts/server_setup.sh", { - VAULT_IP = hcloud_server.vault.ipv4_address SERVER_COUNT = length(local.Server_Count) IP_RANGE = local.IP_range SERVER_IPs = jsonencode([for key, value in local.Extended_Aggregator_IPs : value.private_ipv4[0] if value.type == "server"]) }) : templatefile("${path.module}/scripts/client_setup.sh", { - VAULT_IP = hcloud_server.vault.ipv4_address SERVER_COUNT = length(local.Server_Count) IP_RANGE = local.IP_range SERVER_IPs = jsonencode([for key, value in local.Extended_Aggregator_IPs : value.private_ipv4[0] if value.type == "server"]) }) - ]) destination = "setup.sh" } @@ -95,7 +102,7 @@ resource "tls_private_key" "machines" { } resource "hcloud_ssh_key" "default" { - name = "Terraform Example" + name = "default" public_key = tls_private_key.machines.public_key_openssh } diff --git a/output.tf b/output.tf index 1362ec1..427a575 100644 --- a/output.tf +++ b/output.tf @@ -22,6 +22,6 @@ output "nomad_address" { value = "http://${hcloud_load_balancer.load_balancer.ipv4}:80" } -output "vault_address_http" { - value = "http://${hcloud_server.vault.ipv4_address}:8200" +output "network_id" { + value = hcloud_network.network.id } diff --git a/scripts/server_setup.sh b/scripts/server_setup.sh index 6e20a5c..564d9f6 100644 --- a/scripts/server_setup.sh +++ b/scripts/server_setup.sh @@ -1,8 +1,3 @@ -# Finally, pull the initial ca token for the servers -cd /root/ -chmod 600 machines.pem -ssh -i machines.pem -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" root@10.0.0.2 cat connect_ca_token > /etc/consul.d/connect_ca_token - # On all servers, edit the configuration file /etc/consul.d/consul.hcl and add the content cat < /etc/consul.d/consul.hcl datacenter = "dc1" @@ -10,18 +5,6 @@ data_dir = "/opt/consul" connect { enabled = true - ca_provider = "vault" - ca_config { - address = "http://${VAULT_IP}:8200" - token = "Your_Vault_Token" - root_pki_path = "connect_root" - intermediate_pki_path = "connect_dc1_inter" - leaf_cert_ttl = "72h" - rotation_period = "2160h" - intermediate_cert_ttl = "8760h" - private_key_type = "rsa" - private_key_bits = 2048 - } } client_addr = "0.0.0.0" ui_config { diff --git a/scripts/vault_setup.sh b/scripts/vault_setup.sh deleted file mode 100644 index 7753035..0000000 --- a/scripts/vault_setup.sh +++ /dev/null @@ -1,143 +0,0 @@ -#!/bin/bash - -wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg -echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list -sudo apt update && sudo apt install jq vault -y - -#openssl req -newkey rsa:4096 -nodes -sha256 -keyout tls.key -x509 -days 3650 -out tls.crt -subj "/O=HashiCorp/CN=Vault" -addext 'subjectAltName = IP:127.0.0.1' -#chown vault: /opt/vault/tls/* -mkdir -p /mnt/vault -chown vault: /mnt/vault -cat </etc/vault.d/vault.hcl -storage "file" { - path = "/mnt/vault/data" -} - -listener "tcp" { - address = "0.0.0.0:8200" - tls_disable = "true" -} - -api_addr = "http://127.0.0.1:8200" -ui = true -EOF - -cat < /etc/systemd/system/consul.service -### BEGIN INIT INFO -# Provides: vault -# Required-Start: $local_fs $remote_fs -# Required-Stop: $local_fs $remote_fs -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Vault server -# Description: Vault secret management tool -### END INIT INFO - -[Unit] -Description=Vault secret management tool -Requires=network-online.target -After=network-online.target - -[Service] -User=vault -Group=vault -PIDFile=/var/run/vault/vault.pid -ExecStart=/usr/bin/vault server -config=/etc/vault.d/vault.hcl -log-level=debug -ExecReload=/bin/kill -HUP $MAINPID -KillMode=process -KillSignal=SIGTERM -Restart=on-failure -RestartSec=42s -LimitMEMLOCK=infinity - -[Install] -WantedBy=multi-user.target -EOF - -systemctl enable vault -systemctl start vault - -sleep 10 -export VAULT_ADDR=http://127.0.0.1:8200 -vault operator init -format=json | jq -r '.root_token, .unseal_keys_b64[]' > vault_keys -vault operator unseal $(cat vault_keys | head -n2 | tail -n1) -vault operator unseal $(cat vault_keys | head -n3 | tail -n1) -vault operator unseal $(cat vault_keys | head -n4 | tail -n1) -sleep 5 -vault login $(cat vault_keys | head -n1 | tail -n1) -vault login -token-only $(cat vault_keys | head -n1 | tail -n1) > vault_token - -## Setup ACL - -# vault secrets enable consul -# vault write consul/config/access address="10.0.0.4:8500" -# vault write consul/roles/my-role \ -# node_identities="server-0:dc1" \ -# node_identities="server-1:dc1" \ -# node_identities="server-2:dc1" -# vault write consul/roles/my-role consul_roles="admin-management" - - -# Setup CA -## https://developer.hashicorp.com/vault/tutorials/secrets-management/pki-engine -vault secrets enable -path=connect_root pki -vault secrets tune -max-lease-ttl=87600h connect_root -vault write -field=certificate connect_root/root/generate/internal \ - common_name="example.com" \ - issuer_name="root-2022" \ - ttl=87600h > root_2022_ca.crt -vault write connect_root/config/urls \ - issuing_certificates="$VAULT_ADDR/v1/connect_root/ca" \ - crl_distribution_points="$VAULT_ADDR/v1/connect_root/crl" - -# Setup Intermediate CA -vault secrets enable -path=connect_dc1_inter pki -vault secrets tune -max-lease-ttl=43800h connect_dc1_inter -vault write -format=json connect_dc1_inter/intermediate/generate/internal \ - common_name="example.com Intermediate Authority" \ - issuer_name="example-dot-com-intermediate" \ - | jq -r '.data.csr' > pki_intermediate.csr -vault write -format=json connect_root/root/sign-intermediate \ - issuer_ref="root-2022" \ - csr=@pki_intermediate.csr \ - format=pem_bundle ttl="43800h" \ - | jq -r '.data.certificate' > intermediate.cert.pem -vault write connect_dc1_inter/intermediate/set-signed certificate=@intermediate.cert.pem - - - -cat < vault-policy-connect-ca.hcl -path "/sys/mounts/connect_root" { - capabilities = [ "read" ] -} - -path "/sys/mounts/connect_dc1_inter" { - capabilities = [ "read" ] -} - -path "/sys/mounts/connect_dc1_inter/tune" { - capabilities = [ "update" ] -} - -path "/connect_root/" { - capabilities = [ "read" ] -} - -path "/connect_root/root/sign-intermediate" { - capabilities = [ "update" ] -} - -path "/connect_dc1_inter/*" { - capabilities = [ "create", "read", "update", "delete", "list" ] -} - -path "auth/token/renew-self" { - capabilities = [ "update" ] -} - -path "auth/token/lookup-self" { - capabilities = [ "read" ] -} -EOF -vault policy write connect-ca vault-policy-connect-ca.hcl -vault token create -policy=connect-ca -format=json | jq -r '.auth.client_token' > connect_ca_token diff --git a/vault.tf b/vault.tf deleted file mode 100644 index e3fc73c..0000000 --- a/vault.tf +++ /dev/null @@ -1,46 +0,0 @@ -resource "hcloud_server" "vault" { - depends_on = [ - hcloud_network_subnet.network - ] - name = "vault-0" - server_type = "cpx11" - image = "ubuntu-20.04" - location = var.hetzner_datacenter - ssh_keys = [hcloud_ssh_key.default.id] - labels = { - "vault-server" = "any" - } - - network { - network_id = hcloud_network.network.id - } - - public_net { - ipv6_enabled = false - } -} - -output "vault_ip" { - value = hcloud_server.vault.ipv4_address -} - -resource "null_resource" "vault_deployment" { - connection { - type = "ssh" - user = "root" - private_key = tls_private_key.machines.private_key_openssh - host = hcloud_server.vault.ipv4_address - } - - provisioner "file" { - content = file("${path.module}/scripts/vault_setup.sh") - destination = "setup.sh" - } - - provisioner "remote-exec" { - inline = [ - "chmod +x setup.sh", - "./setup.sh" - ] - } -} \ No newline at end of file