Merge pull request #160 from uktrade/fetch-external-buckets

Added ability for dag processors and tasks to fetch from external buckets
uktrade · Nov 20, 2024 · 4bb25b3 · 4bb25b3
2 parents 393eedd + 1c87290
commit 4bb25b3
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 2 deletions.
diff --git a/infra/airflow_dag_processor.tf b/infra/airflow_dag_processor.tf
@@ -200,10 +200,28 @@ data "aws_iam_policy_document" "airflow_team" {
       actions = [
         "sts:AssumeRole",
       ]
-
       resources = var.airflow_dag_processors[count.index].assume_roles
     }
+  }
+
+  dynamic "statement" {
+    for_each = length(var.airflow_dag_processors[count.index].buckets) > 0 ? [1] : []
+    content {
+      actions = [
+        "s3:ListBucket",
+      ]
+      resources = var.airflow_dag_processors[count.index].buckets
+    }
+  }
 
+  dynamic "statement" {
+    for_each = length(var.airflow_dag_processors[count.index].buckets) > 0 ? [1] : []
+    content {
+      actions = [
+        "s3:GetObject",
+      ]
+      resources = [for s in var.airflow_dag_processors[count.index].buckets : "${s}/*"]
+    }
   }
 
   statement {

diff --git a/infra/main.tf b/infra/main.tf
@@ -180,7 +180,7 @@ variable "airflow_on" {
 variable "airflow_db_instance_class" {}
 variable "airflow_domain" {}
 variable "airflow_dag_processors" {
-  type    = list(object({ name = string, assume_roles = list(string) }))
+  type    = list(object({ name = string, assume_roles = list(string), buckets = list(string) }))
   default = []
 }
 variable "airflow_bucket_infix" {}