refactor of tg files

terraform/base-infra-aws/create-infrastructure.tf

@@ -1,13 +1,7 @@
 resource "null_resource" "oauth-app" {
   provisioner "local-exec" {
-    on_failure = continue
-    command = <<EOT
-      curl -s -X POST https://${data.terraform_remote_state.tenant.outputs.gitlab_hostname}/api/v4/applications \
-            -H 'Content-Type: application/json' \
-            -H 'PRIVATE-TOKEN: ${data.terraform_remote_state.tenant.outputs.gitlab_root_token}' \
-            -d '{"name": "oauth-app-kubernetes-${var.environment}", "redirect_uri": "http://localhost:8000", "scopes": "read_api openid" }' \
-            > ${path.module}/oauth-apps/oauth-app-kubernetes-${var.environment}.json
-    EOT
+    interpreter = ["/bin/bash", "-c"]
+    command = "curl -v -X POST https://${local.gitlab_hostname}/api/v4/applications -H 'Content-Type: application/json' -H 'PRIVATE-TOKEN: ${local.gitlab_root_token}' -d '{\"name\": \"oauth-app-kubernetes-${var.environment}\", \"redirect_uri\": \"https://grafana.${aws_route53_zone.public_subdomain.name}/login/gitlab\", \"scopes\": \"read_api openid\" }' > ${path.module}/oauth-apps/oauth-app-kubernetes-${var.environment}.json"
   }
 }
 
@@ -18,14 +12,8 @@ data "local_file" "kubernetes-oauth-app" {
 
 resource "null_resource" "grafana-oauth-app" {
   provisioner "local-exec" {
-    on_failure = continue
-    command = <<EOT
-      curl -s -X POST https://${data.terraform_remote_state.tenant.outputs.gitlab_hostname}/api/v4/applications \
-            -H 'Content-Type: application/json' \
-            -H 'PRIVATE-TOKEN: ${data.terraform_remote_state.tenant.outputs.gitlab_root_token}' \
-            -d '{"name": "oauth-app-grafana-${var.environment}", "redirect_uri": "https://grafana.${aws_route53_zone.public_subdomain.name}/login/gitlab", "scopes": "read_api" }' \
-            > ${path.module}/oauth-apps/oauth-app-grafana-${var.environment}.json
-    EOT
+    interpreter = ["/bin/bash", "-c"]
+    command = "curl -v -X POST https://${local.gitlab_hostname}/api/v4/applications -H 'Content-Type: application/json' -H 'PRIVATE-TOKEN: ${local.gitlab_root_token}' -d '{\"name\": \"oauth-app-grafana-${local.environment}\", \"redirect_uri\": \"https://grafana.${aws_route53_zone.public_subdomain.name}/login/gitlab\", \"scopes\": \"read_api\" }' > ${path.module}/oauth-apps/oauth-app-grafana-${var.environment}.json"
   }
 }
 
@@ -36,14 +24,8 @@ data "local_file" "grafana-oauth-app" {
 
 resource "null_resource" "vault-oauth-app" {
   provisioner "local-exec" {
-    on_failure = continue
-    command = <<EOT
-      curl -s -X POST https://${data.terraform_remote_state.tenant.outputs.gitlab_hostname}/api/v4/applications \
-            -H 'Content-Type: application/json' \
-            -H 'PRIVATE-TOKEN: ${data.terraform_remote_state.tenant.outputs.gitlab_root_token}' \
-            -d '{"name": "oauth-app-vault-${var.environment}", "redirect_uri": "https://vault.${aws_route53_zone.public_subdomain.name}/ui/vault/auth/oidc/oidc/callback", "scopes": "openid" }' \
-            > ${path.module}/oauth-apps/oauth-app-vault-${var.environment}.json
-    EOT
+    interpreter = ["/bin/bash", "-c"]
+    command = "curl -v -X POST https://${local.gitlab_hostname}/api/v4/applications -H 'Content-Type: application/json' -H 'PRIVATE-TOKEN: ${local.gitlab_root_token}' -d '{\"name\": \"oauth-app-vault-${var.environment}\", \"redirect_uri\": \"https://vault.${aws_route53_zone.public_subdomain.name}/ui/vault/auth/oidc/oidc/callback\", \"scopes\": \"openid\" }' > ${path.module}/oauth-apps/oauth-app-vault-${var.environment}.json"
   }
 }
 
@@ -55,19 +37,19 @@ data "local_file" "vault-oauth-app" {
 #creating nexus entries json file for kubespray execution (requires bootstrap version >= v0.5.3)
 resource "local_file" "kubespray_extra_vars" {
   content         = templatefile("${path.module}/templates/extra-vars.json.tpl", {
-    nexus_ip = data.terraform_remote_state.tenant.outputs.nexus_fqdn 
-    nexus_port = data.terraform_remote_state.tenant.outputs.nexus_docker_repo_listening_port
+    nexus_ip = local.nexus_fqdn 
+    nexus_port = local.nexus_docker_repo_listening_port
     apiserver_loadbalancer_domain_name = aws_lb.internal-lb.dns_name
     kube_oidc_enabled = "true"
     kube_oidc_client_id = local.oauth_app_id
-    kube_oidc_url = "https://${data.terraform_remote_state.tenant.outputs.gitlab_hostname}"
+    kube_oidc_url = "https://${local.gitlab_hostname}"
     groups_name = "groups_direct"
   })
   filename        = "${path.module}/extra-vars.json"
 }
 
 data "aws_vpc" "selected" {
-  id = data.terraform_remote_state.tenant.outputs.vpc_id
+  id = local.vpc_id
 }
 
 resource "aws_security_group" "internet" {
@@ -104,36 +86,36 @@ module "aws-iam" {
 }
 
 resource "aws_route53_zone" "main_private" {
-  name = "${var.environment}.${data.terraform_remote_state.tenant.outputs.private_zone_name}"
+  name = "${var.environment}.${local.private_subdomain}"
 
   vpc {
     vpc_id = data.aws_vpc.selected.id
   }
 
-  comment = "Private zone for ${data.terraform_remote_state.tenant.outputs.private_zone_name}"
+  comment = "Private zone for ${local.private_subdomain}"
 
   tags = {
-    "ProductDomain" = data.terraform_remote_state.tenant.outputs.private_zone_name
+    "ProductDomain" = local.private_subdomain
     "Environment"   = var.environment
-    "Description"   = "Private zone for ${data.terraform_remote_state.tenant.outputs.private_zone_name}"
+    "Description"   = "Private zone for ${local.private_subdomain}"
     "ManagedBy"     = "Terraform"
   }
 }
 
 resource "aws_route53_zone" "public_subdomain" {
-  name = "${var.environment}.${data.terraform_remote_state.tenant.outputs.public_zone_name}"
+  name = "${var.environment}.${local.public_subdomain}"
   force_destroy = var.route53_zone_force_destroy
   tags = {
-    "ProductDomain" = data.terraform_remote_state.tenant.outputs.public_zone_name
+    "ProductDomain" = local.public_subdomain
     "Environment"   = var.environment
-    "Description"   = "Public Zone for subdomain ${data.terraform_remote_state.tenant.outputs.public_zone_name}"
+    "Description"   = "Public Zone for subdomain ${local.public_subdomain}"
     "ManagedBy"     = "Terraform"
   }
 }
 
 resource "aws_route53_record" "subdomain-ns" {
   allow_overwrite = true
-  zone_id         = data.terraform_remote_state.tenant.outputs.public_zone_id
+  zone_id         = local.public_zone_id
   name            = aws_route53_zone.public_subdomain.name
   type            = "NS"
   ttl             = "30"
@@ -160,5 +142,4 @@ locals {
   }
   default_tags = merge(local.dynamic_tags, var.custom_tags)
   oauth_app_id = jsondecode(data.local_file.kubernetes-oauth-app.content)["application_id"]
-  tenancy_azs = data.terraform_remote_state.tenant.outputs.availability_zones
 }

terraform/base-infra-aws/k8s-clusters.tf

@@ -24,13 +24,13 @@ module "ubuntu-focal-ami" {
 }
 
 locals {
-  master_node_permutations = {for pair in setproduct(local.tenancy_azs, range(var.kube_master_num / length(local.tenancy_azs))) : "${pair[0]}-${pair[1]}" => pair[0]}
-  worker_node_permutations = {for pair in setproduct(local.tenancy_azs, range(var.kube_worker_num / length(local.tenancy_azs))) : "${pair[0]}-${pair[1]}" => pair[0]}  
+  master_node_permutations = {for pair in setproduct(local.availability_zones, range(var.kube_master_num / length(local.availability_zones))) : "${pair[0]}-${pair[1]}" => pair[0]}
+  worker_node_permutations = {for pair in setproduct(local.availability_zones, range(var.kube_worker_num / length(local.availability_zones))) : "${pair[0]}-${pair[1]}" => pair[0]}  
 
   master_kube_ec2_config = [
     for cluster_ref, az in local.master_node_permutations : 
     {
-      "subnet_id" = data.terraform_remote_state.tenant.outputs.private_subnet_ids["${var.environment}-${az}"]["id"]
+      "subnet_id" = local.private_subnet_ids["${var.environment}-${az}"]["id"]
       "availability_zone" = az
       "ec2_ref" = cluster_ref
       "aws_ami" = var.use_focal_ubuntu ? module.ubuntu-focal-ami.id : module.ubuntu-bionic-ami.id
@@ -45,7 +45,7 @@ locals {
   worker_kube_ec2_config = [
     for cluster_ref, az in local.worker_node_permutations : 
     {
-      "subnet_id" = data.terraform_remote_state.tenant.outputs.private_subnet_ids["${var.environment}-${az}"]["id"]
+      "subnet_id" = local.private_subnet_ids["${var.environment}-${az}"]["id"]
       "availability_zone" = az
       "ec2_ref" = cluster_ref
       "aws_ami" = var.use_focal_ubuntu ? module.ubuntu-focal-ami.id : module.ubuntu-bionic-ami.id

terraform/base-infra-aws/loadbalancers.tf

@@ -4,7 +4,7 @@ locals {
 
 resource "aws_eip" "nlb" {
   for_each = {
-    for az in local.tenancy_azs : data.terraform_remote_state.tenant.outputs.public_subnet_ids["${var.environment}-${az}"]["id"] => data.terraform_remote_state.tenant.outputs.public_subnet_ids["${var.environment}-${az}"]
+    for az in local.availability_zones : local.public_subnet_ids["${var.environment}-${az}"]["id"] => local.public_subnet_ids["${var.environment}-${az}"]
   }
   tags = merge({ Name = "${local.name}-eip-${each.key}" }, local.default_tags)
 }
@@ -14,7 +14,7 @@ resource "aws_lb" "internal-lb" { #  for internal traffic, including kube traffi
   internal           = true
   load_balancer_type = "network"
   enable_cross_zone_load_balancing = true
-  subnets            = [for az in local.tenancy_azs : data.terraform_remote_state.tenant.outputs.private_subnet_ids["${var.environment}-${az}"]["id"]]
+  subnets            = [for az in local.availability_zones : local.private_subnet_ids["${var.environment}-${az}"]["id"]]
   tags = merge({ Name = "${local.name}-internal" }, local.default_tags)
 }
 
@@ -26,7 +26,7 @@ resource "aws_lb" "external-lb" {
   tags = merge({ Name = "${local.name}-public" }, local.default_tags)
   dynamic subnet_mapping {
     for_each = { 
-      for az in local.tenancy_azs : data.terraform_remote_state.tenant.outputs.public_subnet_ids["${var.environment}-${az}"]["id"] => data.terraform_remote_state.tenant.outputs.public_subnet_ids["${var.environment}-${az}"]
+      for az in local.availability_zones : local.public_subnet_ids["${var.environment}-${az}"]["id"] => local.public_subnet_ids["${var.environment}-${az}"]
     }
     content {
       subnet_id = subnet_mapping.key

terraform/base-infra-aws/outputs.tf

@@ -63,8 +63,11 @@ output "perm2" {
 
 output "available_zones" {
   description = "available azs at time of infra build"
-  value       = local.tenancy_azs
+  value       = local.availability_zones
 }
 output "inventory_file_location" {
   value = module.k8s-cluster-main.inventory_file_location
+}
+output "gitlab_hostname" {
+  value = local.gitlab_hostname
 }

terraform/base-infra-aws/terragrunt.hcl

@@ -33,6 +33,20 @@ data "terraform_remote_state" "tenant" {
     key    = "bootstrap/terraform.tfstate"
   }
 }
+locals {
+  public_zone_id = data.terraform_remote_state.tenant.outputs.public_zone_id
+  private_zone_id = data.terraform_remote_state.tenant.outputs.private_zone_id
+  public_subdomain = data.terraform_remote_state.tenant.outputs.public_zone_name
+  private_subdomain = data.terraform_remote_state.tenant.outputs.private_zone_name
+  gitlab_hostname = data.terraform_remote_state.tenant.outputs.gitlab_hostname
+  gitlab_root_token = data.terraform_remote_state.tenant.outputs.gitlab_root_token
+  nexus_fqdn = data.terraform_remote_state.tenant.outputs.nexus_fqdn
+  nexus_docker_repo_listening_port = data.terraform_remote_state.tenant.outputs.nexus_docker_repo_listening_port
+  vpc_id = data.terraform_remote_state.tenant.outputs.vpc_id
+  availability_zones = data.terraform_remote_state.tenant.outputs.availability_zones
+  private_subnet_ids = data.terraform_remote_state.tenant.outputs.private_subnet_ids
+  public_subnet_ids = data.terraform_remote_state.tenant.outputs.public_subnet_ids
+}
 
 EOF
 }

terraform/base-k8s-setup/ext-dns-aws.tf

@@ -12,8 +12,8 @@ resource "helm_release" "external-dns" {
     templatefile("${path.module}/templates/values-external-dns.yaml.tpl", {
         external_dns_iam_access_key = aws_iam_access_key.route53-external-dns.id
         external_dns_iam_secret_key = aws_iam_access_key.route53-external-dns.secret
-        domain = dependency.baseinfra.outputs.public_subdomain
-        internal_domain = dependency.baseinfra.outputs.private_subdomain
+        domain = var.public_subdomain
+        internal_domain = var.private_subdomain
         txt_owner_id = "${var.environment}-${var.client}"
         region = var.region
       })
@@ -57,8 +57,8 @@ resource "aws_iam_user_policy" "route53-external-dns" {
         "route53:ChangeResourceRecordSets"
       ],
       "Resource": [
-        "arn:aws:route53:::hostedzone/${dependency.baseinfra.outputs.public_subdomain_zone_id}",    
-        "arn:aws:route53:::hostedzone/${dependency.baseinfra.outputs.private_zone_id}"
+        "arn:aws:route53:::hostedzone/${var.public_subdomain_zone_id}",    
+        "arn:aws:route53:::hostedzone/${var.private_subdomain_zone_id}"
       ]
     },
     {

terraform/base-k8s-setup/letsencrypt-issuer-aws.tf

@@ -3,7 +3,7 @@ resource "kubectl_manifest" "lets-encrypt-issuer" {
     yaml_body = templatefile("${path.module}/templates/lets-cluster-issuer.yaml.tpl", {
       external_dns_iam_access_key = aws_iam_access_key.route53-external-dns.id
       region = var.region
-      domain = dependency.baseinfra.outputs.public_subdomain
+      domain = var.public_subdomain
       letsencrypt_server = var.letsencrypt_server == "production" ? "https://acme-v02.api.letsencrypt.org/directory" : "https://acme-staging-v02.api.letsencrypt.org/directory"
       letsencrypt_email = var.wso2_email
       secret_name = kubernetes_secret.certmanager-route53-credentials.metadata[0].name
@@ -17,7 +17,7 @@ resource "kubectl_manifest" "lets-encrypt-issuer" {
 
 resource "kubectl_manifest" "lets-encrypt-wildcard-cert" {
     yaml_body = templatefile("${path.module}/templates/lets-wildcard-cert.yaml.tpl", {
-      domain_name = dependency.baseinfra.outputs.public_subdomain
+      domain_name = var.public_subdomain
       secret_name = var.int_wildcard_cert_sec_name
       issuer_name = var.cert_man_letsencrypt_cluster_issuer_name})
     override_namespace = "default"

terraform/base-k8s-setup/main.tf

@@ -28,7 +28,7 @@ resource "helm_release" "deploy_vault" {
     kms_secret_key = var.aws_secret_key,
     kube_engine_path = var.kubernetes_auth_path
     host_name = "vault"
-    domain_name = dependency.baseinfra.outputs.public_subdomain
+    domain_name = var.public_subdomain
   })]
   force_update = true
   cleanup_on_fail = true
@@ -55,7 +55,7 @@ export POD=$(kubectl get pod -l app.kubernetes.io/instance=vault -o jsonpath={.i
 if kubectl exec -ti $POD -c vault -- sh -c 'VAULT_ADDR=http://127.0.0.1:8200 vault status'; then
   echo "vault already initialized"
 else 
-kubectl exec -ti $POD -c vault -- sh -c 'VAULT_ADDR=http://127.0.0.1:8200 vault operator init --key-shares=5 --key-threshold=3 -format json' > ${path.module}/templates/vault_seal_key
+kubectl exec -ti $POD -c vault -- sh -c 'VAULT_ADDR=http://127.0.0.1:8200 vault operator init --key-shares=5 --key-threshold=3 -format json' > ${var.static_files_path_location}/vault_seal_key
 fi
 EOT
     environment = {
@@ -66,7 +66,7 @@ EOT
 }
 
 data "template_file" "vault_key" {
-  template = file("${path.module}/templates/vault_seal_key")
+  template = file("${var.static_files_path_location}/vault_seal_key")
 
   depends_on = [null_resource.initialize-vault]
 }
@@ -75,8 +75,8 @@ resource "null_resource" "tune-secret-engine" {
   provisioner "local-exec" {
     command = <<EOT
 POD=$(kubectl get pod -l app.kubernetes.io/instance=vault -o jsonpath={.items[0].metadata.name})
-kubectl exec -ti $POD -c vault -- sh -c 'VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN=${jsondecode(file("${path.module}/templates/vault_seal_key"))["root_token"]} vault secrets enable --path=secret kv' 
-kubectl exec -ti $POD -c vault -- sh -c 'VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN=${jsondecode(file("${path.module}/templates/vault_seal_key"))["root_token"]} vault secrets tune -default-lease-ttl=2m secret/' 
+kubectl exec -ti $POD -c vault -- sh -c 'VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN=${jsondecode(file("${var.static_files_path_location}/vault_seal_key"))["root_token"]} vault secrets enable --path=secret kv' 
+kubectl exec -ti $POD -c vault -- sh -c 'VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN=${jsondecode(file("${var.static_files_path_location}/vault_seal_key"))["root_token"]} vault secrets tune -default-lease-ttl=2m secret/' 
 EOT
     environment = {
       KUBECONFIG = var.kubeconfig_location
@@ -116,7 +116,7 @@ resource "helm_release" "external-nginx-ingress-controller" {
         ingress_class_name = "nginx-ext"
         http_nodeport_port = 32080
         https_nodeport_port = 32443
-        lb_name            = dependency.baseinfra.outputs.external_load_balancer_dns
+        lb_name            = var.external_load_balancer_dns
         use_proxy_protocol = true
         enable_real_ip     = true
         tls_sec_name = "default/${var.int_wildcard_cert_sec_name}"
@@ -140,7 +140,7 @@ resource "helm_release" "internal-nginx-ingress-controller" {
         http_nodeport_port = 31080
         https_nodeport_port = 31443
         ingress_class_name = "nginx"
-        lb_name            = dependency.baseinfra.outputs.internal_load_balancer_dns
+        lb_name            = var.internal_load_balancer_dns
         use_proxy_protocol = false
         enable_real_ip     = false
         tls_sec_name = "default/${var.int_wildcard_cert_sec_name}"

terraform/base-k8s-setup/terragrunt.hcl

@@ -30,8 +30,29 @@ include "state" {
 include "aws_provider" {
   path = find_in_parent_folders("aws_provider.hcl")
 }
-include "k8s_providers" {
-  path = find_in_parent_folders("k8s_providers.hcl")
+generate "k8s_provider" {
+  path = "k8s_providers.tf"
+
+  if_exists = "overwrite_terragrunt"
+
+  contents = <<EOF
+provider "helm" {
+  alias = "helm-main"
+  kubernetes {
+    config_path = "${local.common_vars.kubeconfig_location}"
+  }
+}
+provider "kubernetes" {
+  alias       = "k8s-main"
+  config_path = "${local.common_vars.kubeconfig_location}"
+}
+
+provider "kubectl" {
+  alias       = "k8s-main"
+  config_path = "${local.common_vars.kubeconfig_location}"
+}
+ 
+EOF
 }
 
 dependency "baseinfra" {
@@ -41,6 +62,14 @@ dependency "baseinfra" {
 locals {
   common_vars = yamldecode(file(find_in_parent_folders("common_vars.yaml")))
 }
+
 inputs = {
-  kubeconfig_location = ${local.common_vars.kubeconfig_location}
+  kubeconfig_location = local.common_vars.kubeconfig_location
+  static_files_path_location = local.common_vars.static_files_path_location
+  private_subdomain_zone_id = dependency.baseinfra.outputs.private_zone_id
+  public_subdomain_zone_id = dependency.baseinfra.outputs.public_subdomain_zone_id
+  private_subdomain = dependency.baseinfra.outputs.private_subdomain
+  public_subdomain = dependency.baseinfra.outputs.public_subdomain
+  external_load_balancer_dns = dependency.baseinfra.outputs.external_load_balancer_dns
+  internal_load_balancer_dns = dependency.baseinfra.outputs.internal_load_balancer_dns
 }

terraform/base-k8s-setup/variables.tf

@@ -117,4 +117,32 @@ variable "longhorn_backup_s3_destroy" {
   description = "destroy s3 backup on destroy of env"
   type        = bool
   default     = false
+}
+variable "public_subdomain_zone_id" {
+  description = "public_subdomain_zone_id"
+  type        = string
+}
+variable "private_subdomain_zone_id" {
+  description = "private_subdomain_zone_id"
+  type        = string
+}
+variable "public_subdomain" {
+  description = "public_subdomain"
+  type        = string
+}
+variable "private_subdomain" {
+  description = "private_subdomain"
+  type        = string
+}
+variable "external_load_balancer_dns" {
+  description = "external_load_balancer_dns"
+  type        = string
+}
+variable "internal_load_balancer_dns" {
+  description = "internal_load_balancer_dns"
+  type        = string
+}
+variable "static_files_path_location" {
+  description = "static_files_path_location"
+  type        = string
 }