Merge pull request #132 from adamrushuk/develop

Finished features for v1.0.0 release

.github/workflows/build.yml

@@ -34,16 +34,21 @@ env:
   ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
 
   # other
-  # prod or staging
-  CERT_API_ENVIRONMENT: prod
+  # prod or staging. "" disables cert-manager annotations
+  CERT_API_ENVIRONMENT: ""
   DEMO_USER_USERNAME: demo_user
   # DEMO_USER_PASSWORD: ${{ secrets.DEMO_USER_PASSWORD }}
   DNS_DOMAIN_NAME: nexus.thehypepipe.co.uk
+  DNS_RG_NAME: rg-dns
   DOCKER_FQDN: docker.thehypepipe.co.uk
   EMAIL_ADDRESS: [email protected]
   ENABLE_TLS_INGRESS: true
   FORCE_TEST_FAIL: false
   HAS_SUBDOMAIN: true
+  K8S_TLS_SECRET_NAME: tls-secret
+  KEY_VAULT_NAME: kv-rush-iz6y
+  KEY_VAULT_CERT_NAME: wildcard-thehypepipe-co-uk
+  KEY_VAULT_RESOURCE_GROUP_NAME: rg-keyvault-acmebot
   LOCATION: uksouth
   ROOT_DOMAIN_NAME: thehypepipe.co.uk
   # NEXUS_ADMIN_PASSWORD: ${{ secrets.NEXUS_ADMIN_PASSWORD }}
@@ -81,17 +86,16 @@ jobs:
       #   ref: develop
 
     # Init tasks - inc Env var concatenation
-    # https://github.community/t5/GitHub-Actions/How-can-we-concatenate-multiple-env-vars-at-workflow-and-job/td-p/48489
+    # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#environment-files
     - name: Init tasks - inc Env var concatenation (Workaround)
       run: |
         chmod -R +x ./scripts/
-        echo ::set-env name=AKS_RG_NAME::${{ env.PREFIX }}-rg-aks-dev-001
-        echo ::set-env name=AKS_CLUSTER_NAME::${{ env.PREFIX }}-aks-001
-        echo ::set-env name=DNS_RG_NAME::${{ env.PREFIX }}-rg-externaldns
-        echo ::set-env name=TERRAFORM_STORAGE_ACCOUNT::${{ env.PREFIX }}sttfstate${{ env.LOCATION }}001
-        echo ::set-env name=TERRAFORM_STORAGE_RG::${{ env.PREFIX }}-rg-tfstate-dev-001
-        echo ::set-env name=VELERO_STORAGE_RG::${{ env.PREFIX }}-rg-velero-dev-001
-        echo ::set-env name=VELERO_STORAGE_ACCOUNT::${{ env.PREFIX }}stbckuksouth001
+        echo "AKS_RG_NAME=${{ env.PREFIX }}-rg-aks-dev-001" >> $GITHUB_ENV
+        echo "AKS_CLUSTER_NAME=${{ env.PREFIX }}-aks-001" >> $GITHUB_ENV
+        echo "TERRAFORM_STORAGE_ACCOUNT=${{ env.PREFIX }}sttfstate${{ env.LOCATION }}001" >> $GITHUB_ENV
+        echo "TERRAFORM_STORAGE_RG=${{ env.PREFIX }}-rg-tfstate-dev-001" >> $GITHUB_ENV
+        echo "VELERO_STORAGE_RG=${{ env.PREFIX }}-rg-velero-dev-001" >> $GITHUB_ENV
+        echo "VELERO_STORAGE_ACCOUNT=${{ env.PREFIX }}stbckuksouth001" >> $GITHUB_ENV
 
     # Show event info
     - name: Show triggered event data
@@ -257,8 +261,8 @@ jobs:
     # Init tasks - inc Env var concatenation
     - name: Init tasks - inc Env var concatenation (Workaround)
       run: |
-        echo "::set-env name=AKS_RG_NAME::${{ env.PREFIX }}-rg-aks-dev-001"
-        echo "::set-env name=AKS_CLUSTER_NAME::${{ env.PREFIX }}-aks-001"
+        echo "AKS_RG_NAME=${{ env.PREFIX }}-rg-aks-dev-001" >> $GITHUB_ENV
+        echo "AKS_CLUSTER_NAME=${{ env.PREFIX }}-aks-001" >> $GITHUB_ENV
 
     # Login
     - name: Login to Azure

.github/workflows/deallocate_aks_vmss.yml

@@ -43,9 +43,8 @@ jobs:
     - name: Init tasks - inc Env var concatenation (Workaround)
       run: |
         chmod -R +x ./scripts/
-        echo ::set-env name=AKS_RG_NAME::${{ env.PREFIX }}-rg-aks-dev-001
-        echo ::set-env name=AKS_CLUSTER_NAME::${{ env.PREFIX }}-aks-001
-
+        echo "AKS_RG_NAME=${{ env.PREFIX }}-rg-aks-dev-001" >> $GITHUB_ENV
+        echo "AKS_CLUSTER_NAME=${{ env.PREFIX }}-aks-001" >> $GITHUB_ENV
     # Show event info
     - name: Show triggered event data
       run: pwsh -command "./scripts/Get-EventData.ps1"

.github/workflows/destroy.yml

@@ -33,13 +33,19 @@ env:
   # prod or staging
   CERT_API_ENVIRONMENT: staging
   DNS_DOMAIN_NAME: nexus.thehypepipe.co.uk
+  DNS_RG_NAME: rg-dns
   EMAIL_ADDRESS: [email protected]
   ENABLE_TLS_INGRESS: true
   FORCE_TEST_FAIL: false
   HAS_SUBDOMAIN: true
+  K8S_TLS_SECRET_NAME: tls-secret
+  KEY_VAULT_NAME: kv-rush-iz6y
+  KEY_VAULT_CERT_NAME: wildcard-thehypepipe-co-uk
+  KEY_VAULT_RESOURCE_GROUP_NAME: rg-keyvault-acmebot
   LOCATION: uksouth
+  ROOT_DOMAIN_NAME: thehypepipe.co.uk
   # STORAGE_KEY: 'env var set by Get-StorageKey.ps1'
-  VELERO_ENABLED: false
+  VELERO_ENABLED: true
 
   # terraform
   TF_IN_AUTOMATION: "true"
@@ -69,16 +75,16 @@ jobs:
       #   ref: develop
 
     # Env var concatenation
-    # https://github.community/t5/GitHub-Actions/How-can-we-concatenate-multiple-env-vars-at-workflow-and-job/td-p/48489
+    # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#environment-files
     - name: Concatenate env vars (Workaround)
       run: |
         chmod -R +x ./scripts/
-        echo ::set-env name=AKS_RG_NAME::${{ env.PREFIX }}-rg-aks-dev-001
-        echo ::set-env name=AKS_CLUSTER_NAME::${{ env.PREFIX }}-aks-001
-        echo ::set-env name=TERRAFORM_STORAGE_ACCOUNT::${{ env.PREFIX }}sttfstate${{ env.LOCATION }}001
-        echo ::set-env name=TERRAFORM_STORAGE_RG::${{ env.PREFIX }}-rg-tfstate-dev-001
-        echo ::set-env name=VELERO_STORAGE_RG::${{ env.PREFIX }}-rg-velero-dev-001
-        echo ::set-env name=VELERO_STORAGE_ACCOUNT::${{ env.PREFIX }}stbckuksouth001
+        echo "AKS_RG_NAME=${{ env.PREFIX }}-rg-aks-dev-001" >> $GITHUB_ENV
+        echo "AKS_CLUSTER_NAME=${{ env.PREFIX }}-aks-001" >> $GITHUB_ENV
+        echo "TERRAFORM_STORAGE_ACCOUNT=${{ env.PREFIX }}sttfstate${{ env.LOCATION }}001" >> $GITHUB_ENV
+        echo "TERRAFORM_STORAGE_RG=${{ env.PREFIX }}-rg-tfstate-dev-001" >> $GITHUB_ENV
+        echo "VELERO_STORAGE_RG=${{ env.PREFIX }}-rg-velero-dev-001" >> $GITHUB_ENV
+        echo "VELERO_STORAGE_ACCOUNT=${{ env.PREFIX }}stbckuksouth001" >> $GITHUB_ENV
 
     # Show event info
     - name: Show triggered event data

.github/workflows/start_aks_vmss.yml

@@ -43,8 +43,8 @@ jobs:
     - name: Init tasks - inc Env var concatenation (Workaround)
       run: |
         chmod -R +x ./scripts/
-        echo ::set-env name=AKS_RG_NAME::${{ env.PREFIX }}-rg-aks-dev-001
-        echo ::set-env name=AKS_CLUSTER_NAME::${{ env.PREFIX }}-aks-001
+        echo "AKS_RG_NAME=${{ env.PREFIX }}-rg-aks-dev-001" >> $GITHUB_ENV
+        echo "AKS_CLUSTER_NAME=${{ env.PREFIX }}-aks-001" >> $GITHUB_ENV
 
     # Show event info
     - name: Show triggered event data

.github/workflows/test.yml

@@ -58,15 +58,15 @@ jobs:
     # https://github.community/t5/GitHub-Actions/How-can-we-concatenate-multiple-env-vars-at-workflow-and-job/td-p/48489
     - name: BASH - Concatenate env vars (Workaround)
       run: |
-        echo ::set-env name=MY_CONCATENATED_BASH_VAR::${{ env.MY_WORKFLOW_VAR }}-${{ env.MY_JOB_VAR }}-bash-stepvalue01
+        echo "MY_CONCATENATED_BASH_VAR=${{ env.MY_WORKFLOW_VAR }}-${{ env.MY_JOB_VAR }}-bash-stepvalue01 >> $GITHUB_ENV
 
     - name: PWSH - Concatenate env vars (Workaround)
       # override the default bash shell, as running on ubuntu
       # https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
       shell: pwsh
       # pwsh requires double quotes when setting env vars
       run: |
-        echo "::set-env name=MY_CONCATENATED_PWSH_VAR::${{ env.MY_WORKFLOW_VAR }}-${{ env.MY_JOB_VAR }}-pwsh-stepvalue01"
+        echo ""MY_CONCATENATED_PWSH_VAR=${{ env.MY_WORKFLOW_VAR }}-${{ env.MY_JOB_VAR }}-pwsh-stepvalue01" >> $GITHUB_ENV
 
     - name: Output event data
       shell: pwsh

aad-pod-identity/README.md

@@ -25,7 +25,7 @@ Before automating the installation and configuration of aad-pod-identity, follow
 
 ### Deploy aad-pod-identity using Helm 3
 
-Repo: [https://github.com/Azure/aad-pod-identity/tree/master/charts/aad-pod-identity](https://github.com/Azure/aad-pod-identity/tree/master/charts/aad-pod-identity)
+Repo: [aad-pod-identity chart](https://github.com/Azure/aad-pod-identity/tree/master/charts/aad-pod-identity)
 
 ```bash
 # Navigate to aad-pod-identity folder
@@ -44,7 +44,7 @@ helm search repo aad-pod-identity --version ^2.0.0
 kubectl create namespace aad-pod-identity
 
 # Install aad-pod-identity
-helm upgrade aad-pod-identity aad-pod-identity/aad-pod-identity --version 2.0.1 --values aad_pod_identity_values.yaml --install --atomic --namespace aad-pod-identity --debug
+helm upgrade aad-pod-identity aad-pod-identity/aad-pod-identity --version 2.0.2 --values aad_pod_identity_values.yaml --set=installCRDs=true --install --atomic --namespace aad-pod-identity --debug
 ```
 
 ### Configure aad-pod-identity
@@ -70,9 +70,20 @@ export IDENTITY_RESOURCE_ID="$(az identity show -g "$AKS_NODE_RESOURCE_GROUP_NAM
 echo "IDENTITY_RESOURCE_ID: $IDENTITY_RESOURCE_ID"
 echo "IDENTITY_CLIENT_ID: $IDENTITY_CLIENT_ID"
 
+# [OPTIONAL] Check assigned VMSS identity
+az vmss identity show -g "$AKS_NODE_RESOURCE_GROUP_NAME" -n "aks-default-39636823-vmss"
+
 # Assign the identity a role
 export IDENTITY_ASSIGNMENT_ID="$(az role assignment create --role Contributor --assignee "$IDENTITY_CLIENT_ID" --scope "$AKS_NODE_RESOURCE_GROUP_ID" --query id -o tsv)"
 
+# Describe AzureIdentity CRDs (they dont have metadata)
+kubectl describe AzureIdentity velero
+kubectl explain --recursive AzureIdentity
+kubectl explain --recursive backups
+kubectl explain --recursive AzureIdentity.spec
+kubectl explain --recursive AzureIdentityBinding
+kubectl explain --recursive AzureIdentityBinding.spec
+
 # Create an AzureIdentity
 cat <<EOF | kubectl apply --namespace aad-pod-identity -f -
 apiVersion: "aadpodidentity.k8s.io/v1"
@@ -116,7 +127,7 @@ kubectl create secret generic --namespace velero velero-credentials --from-file=
 Once `aad-pod-identity` has been configured, and the Velero credentials secret has been populated, install Velero via Helm chart ensuring the aadpodidbinding=$IDENTITY_NAME label has been added to the Velero values.yaml, eg:
 
 ```yaml
-# source: https://github.com/vmware-tanzu/helm-charts/blob/velero-2.12.13/charts/velero/values.yaml#L24
+# source: https://github.com/vmware-tanzu/helm-charts/blob/velero-2.13.3/charts/velero/values.yaml#L27
 podLabels:
   aadpodidbinding: velero
 ```
@@ -131,11 +142,11 @@ $env:KUBE_EDITOR = 'code --wait'
 kubectl get crd
 
 # list velero schedules
-kubectl get schedules.velero.io
+kubectl get schedules.velero.io --namespace velero
 
 # describe velero schedule
-kubectl describe schedules.velero.io/velero-fullbackup
+kubectl describe schedules.velero.io/velero-fullbackup --namespace velero
 
-# edit velero schedule - every 10 mins (*/10 * * * *)
-kubectl edit schedules.velero.io/velero-fullbackup
+# edit velero schedule - every 5 mins (0 */5 * * *)
+kubectl edit schedules.velero.io/velero-fullbackup --namespace velero
 ```

aad-pod-identity/aad_pod_identity_values.yaml

@@ -1,3 +1,5 @@
+# source: https://github.com/Azure/aad-pod-identity/blob/v1.6.3/charts/aad-pod-identity/values.yaml
+
 # Default values for aad-pod-identity-helm.
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
@@ -6,9 +8,13 @@ nameOverride: ""
 fullnameOverride: ""
 
 image:
-  repository: mcr.microsoft.com/k8s/aad-pod-identity
+  repository: mcr.microsoft.com/oss/azure/aad-pod-identity
   imagePullPolicy: Always
 
+# One or more secrets to be used when pulling images
+# imagePullSecrets:
+#   - name: myRegistryKeySecretName
+
 # https://github.com/Azure/aad-pod-identity#4-optional-match-pods-in-the-namespace
 # By default, AAD Pod Identity matches pods to identities across namespaces.
 # To match only pods in the namespace containing AzureIdentity set this to true.
@@ -29,16 +35,17 @@ adminsecret: {}
 #   resourceGroup: <cluster resource group>
 #   vmType: <`standard` for normal virtual machine nodes, and `vmss` for cluster deployed with a virtual machine scale set>
 #   tenantID: <service principal tenant id>
-#   clientID: <service principal client id>
-#   clientSecret: <service principal client secret>
-
+#   clientID: <service principal client id. Set to `msi` when using a User Managed Identity>
+#   clientSecret: <service principal client secret. Set to `msi` when using a User Managed Identity>
+#   useMSI: <set to true when using a User Managed Identity>
+#   userAssignedMSIClientID: <client id for the User Managed Identity>
 # Operation mode for pod-identity. Default is standard mode that has MIC doing identity assignment
 # Allowed values: "standard", "managed"
 operationMode: "standard"
 
 mic:
   image: mic
-  tag: 1.6.2
+  tag: v1.6.3
 
   priorityClassName: ""
 
@@ -111,9 +118,13 @@ mic:
   # Default value is 1s
   updateUserMSIRetryInterval: ""
 
+  # The interval between reconciling identity assignment on Azure based on an existing list of AzureAssignedIdentities
+  # Default value is 3m
+  identityAssignmentReconcileInterval: ""
+
 nmi:
   image: nmi
-  tag: 1.6.2
+  tag: v1.6.3
 
   priorityClassName: ""
 

scripts/Replace-Tokens.ps1

@@ -1,6 +1,6 @@
 # Replace tokens
 param (
-    $TargetFilePattern = './terraform/*.tf',
+    $TargetFilePattern = @('./terraform/*.tf', './terraform/files/*.yaml'),
     $TokenPrefix = '__',
     $TokenSuffix = '__',
 

scripts/storage_key.sh

@@ -17,9 +17,9 @@ echo "FINISHED: $taskMessage."
 taskMessage="Updating workflow env vars"
 echo "STARTED: $taskMessage..."
 
-# https://help.github.com/en/actions/reference/development-tools-for-github-actions#set-an-environment-variable-set-env
-# ::set-env name={name}::{value}
-echo "::set-env name=STORAGE_KEY::$storage_key"
+# # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#environment-files
+# echo "{name}={value}" >> $GITHUB_ENV
+echo "STORAGE_KEY=$storage_key" >> "$GITHUB_ENV"
 
 # Mask sensitive env var
 # https://help.github.com/en/actions/reference/development-tools-for-github-actions#example-masking-an-environment-variable

terraform/aks.tf

@@ -82,36 +82,28 @@ resource "azurerm_kubernetes_cluster" "aks" {
     }
   }
 
-
-  # TODO DELETE SECTION
-  # service_principal block: https://www.terraform.io/docs/providers/azurerm/r/kubernetes_cluster.html#client_id
-  # service_principal {
-  #   client_id     = var.service_principal_client_id
-  #   client_secret = var.service_principal_client_secret
-  # }
-  # TODO DELETE SECTION
-
-
   # managed identity block: https://www.terraform.io/docs/providers/azurerm/r/kubernetes_cluster.html#type-1
   identity {
     type = "SystemAssigned"
   }
 
   # TODO Enable RBAC and AAD auth: https://app.zenhub.com/workspaces/aks-nexus-velero-5e602702ee332f0fc76d35dd/issues/adamrushuk/aks-nexus-velero/105
-  # role_based_access_control {
-  #   enabled = true
-
-  #   azure_active_directory {
-  #     managed = true
-  #     admin_group_object_ids = [
-  #       data.azuread_group.aks.id
-  #     ]
-  #   }
-  # }
+  role_based_access_control {
+    enabled = true
+
+    # azure_active_directory {
+    #   managed = true
+    #   admin_group_object_ids = [
+    #     data.azuread_group.aks.id
+    #   ]
+    # }
+  }
 
   addon_profile {
+    # cannot remove this deprecated block yet, due to this issue:
+    # https://github.com/terraform-providers/terraform-provider-azurerm/issues/7716
     kube_dashboard {
-      enabled = var.aks_dashboard_enabled
+      enabled = false
     }
 
     oms_agent {

terraform/data.tf

@@ -4,3 +4,7 @@ data "azurerm_subscription" "current" {}
 data "azuread_group" "aks" {
   name = var.aad_group_name
 }
+
+data "azurerm_resource_group" "aks_node_rg" {
+  name = azurerm_kubernetes_cluster.aks.node_resource_group
+}

terraform/dns.tf

@@ -1,17 +1,11 @@
 # DNS
-resource "azurerm_resource_group" "dns" {
+data "azurerm_resource_group" "dns" {
   name     = var.dns_resource_group_name
-  location = var.location
-  tags     = var.tags
-
-  lifecycle {
-    ignore_changes = [tags]
-  }
 }
 
-resource "azurerm_dns_zone" "dns" {
+data "azurerm_dns_zone" "dns" {
   name                = var.dns_zone_name
-  resource_group_name = azurerm_resource_group.dns.name
+  resource_group_name = data.azurerm_resource_group.dns.name
 }
 
 
@@ -48,7 +42,7 @@ resource "azuread_service_principal_password" "aks_dns_sp" {
 resource "azurerm_role_assignment" "aks_dns_sp_to_rg" {
   principal_id                     = azuread_service_principal.aks_dns_sp.id
   role_definition_name             = "Reader"
-  scope                            = azurerm_dns_zone.dns.id
+  scope                            = data.azurerm_dns_zone.dns.id
   skip_service_principal_aad_check = true
   depends_on                       = [azuread_service_principal_password.aks_dns_sp]
 }
@@ -57,7 +51,7 @@ resource "azurerm_role_assignment" "aks_dns_sp_to_rg" {
 resource "azurerm_role_assignment" "aks_dns_sp_to_zone" {
   principal_id                     = azuread_service_principal.aks_dns_sp.id
   role_definition_name             = "Contributor"
-  scope                            = azurerm_resource_group.dns.id
+  scope                            = data.azurerm_resource_group.dns.id
   skip_service_principal_aad_check = true
   depends_on                       = [azuread_service_principal_password.aks_dns_sp]
 }
@@ -77,7 +71,7 @@ resource "kubernetes_secret" "external_dns" {
   "aadClientSecret": "${random_string.aks_dns_sp.result}",
   "tenantId": "${data.azurerm_subscription.current.tenant_id}",
   "subscriptionId": "${data.azurerm_subscription.current.subscription_id}",
-  "resourceGroup": "${azurerm_resource_group.dns.name}"
+  "resourceGroup": "${data.azurerm_resource_group.dns.name}"
 }
 EOT
   }