From 61171358ab9dd044f57c7600e8d79336ee1ce8f1 Mon Sep 17 00:00:00 2001
From: "john.green" <john.green@engineering.digital.dwp.gov.uk>
Date: Mon, 21 Nov 2022 09:18:13 +0000
Subject: [PATCH 1/3] added new dynamodb row for trainign data

---
 data-egress.tf | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/data-egress.tf b/data-egress.tf
index 838d000..9e70103 100644
--- a/data-egress.tf
+++ b/data-egress.tf
@@ -1394,6 +1394,27 @@ resource "aws_dynamodb_table_item" "ap_ml_preprocessed_data" {
   ITEM
 }
 
+resource "aws_dynamodb_table_item" "ap_ml_preprocessed_training_data" {
+  table_name = aws_dynamodb_table.data_egress.name
+  hash_key   = aws_dynamodb_table.data_egress.hash_key
+  range_key  = aws_dynamodb_table.data_egress.range_key
+
+  item = <<ITEM
+  {
+    "source_prefix":                {"S":    "preprocessed_training_data/journal/$TODAYS_DATE/*"},
+    "pipeline_name":                {"S":    "ap_preproc_journal"},
+    "recipient_name":               {"S":    "ap_preproc_journal_training"},
+    "transfer_type":                {"S":    "S3"},
+    "source_bucket":                {"S":    "${data.terraform_remote_state.common.outputs.published_bucket.id}"},
+    "destination_bucket":           {"S":    "${data.terraform_remote_state.common.outputs.dataworks_model_published_bucket.id}"},
+    "destination_prefix":           {"S":    "/preprocessed_training_data/journal/$TODAYS_DATE/"},
+    "decrypt":                      {"bool": true},
+    "rewrap_datakey":               {"bool": false},
+    "encrypting_key_ssm_parm_name": {"S":    ""}
+  }
+  ITEM
+}
+
 resource "aws_acm_certificate" "data_egress_server" {
   certificate_authority_arn = data.terraform_remote_state.certificate_authority.outputs.root_ca.arn
   domain_name               = "${local.data_egress_server_name}.${local.env_prefix[local.environment]}dataworks.dwp.gov.uk"

From 024594016a1b0663829f5b3c2ac7040c47a15f13 Mon Sep 17 00:00:00 2001
From: "john.green" <john.green@engineering.digital.dwp.gov.uk>
Date: Mon, 21 Nov 2022 10:54:24 +0000
Subject: [PATCH 2/3] Updated root folder for ap_ml on dynamodb/iam for ap
 machine learning projects

---
 data-egress.tf     | 8 ++++----
 data-egress_iam.tf | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/data-egress.tf b/data-egress.tf
index 9e70103..3f37261 100644
--- a/data-egress.tf
+++ b/data-egress.tf
@@ -1380,13 +1380,13 @@ resource "aws_dynamodb_table_item" "ap_ml_preprocessed_data" {
 
   item = <<ITEM
   {
-    "source_prefix":                {"S":    "preprocessed_csv/journal/$TODAYS_DATE/*"},
+    "source_prefix":                {"S":    "ap_ml/preprocessed_data/journal/$TODAYS_DATE/*"},
     "pipeline_name":                {"S":    "ap_preproc_journal"},
     "recipient_name":               {"S":    "ap_preproc_journal"},
     "transfer_type":                {"S":    "S3"},
     "source_bucket":                {"S":    "${data.terraform_remote_state.common.outputs.published_bucket.id}"},
     "destination_bucket":           {"S":    "${data.terraform_remote_state.common.outputs.dataworks_model_published_bucket.id}"},
-    "destination_prefix":           {"S":    "/preprocessed/journal/$TODAYS_DATE/"},
+    "destination_prefix":           {"S":    "ap_ml/preprocessed_data/journal/$TODAYS_DATE/"},
     "decrypt":                      {"bool": true},
     "rewrap_datakey":               {"bool": false},
     "encrypting_key_ssm_parm_name": {"S":    ""}
@@ -1401,13 +1401,13 @@ resource "aws_dynamodb_table_item" "ap_ml_preprocessed_training_data" {
 
   item = <<ITEM
   {
-    "source_prefix":                {"S":    "preprocessed_training_data/journal/$TODAYS_DATE/*"},
+    "source_prefix":                {"S":    "ap_ml/preprocessed_training_data/journal/$TODAYS_DATE/*"},
     "pipeline_name":                {"S":    "ap_preproc_journal"},
     "recipient_name":               {"S":    "ap_preproc_journal_training"},
     "transfer_type":                {"S":    "S3"},
     "source_bucket":                {"S":    "${data.terraform_remote_state.common.outputs.published_bucket.id}"},
     "destination_bucket":           {"S":    "${data.terraform_remote_state.common.outputs.dataworks_model_published_bucket.id}"},
-    "destination_prefix":           {"S":    "/preprocessed_training_data/journal/$TODAYS_DATE/"},
+    "destination_prefix":           {"S":    "ap_ml/preprocessed_training_data/journal/$TODAYS_DATE/"},
     "decrypt":                      {"bool": true},
     "rewrap_datakey":               {"bool": false},
     "encrypting_key_ssm_parm_name": {"S":    ""}
diff --git a/data-egress_iam.tf b/data-egress_iam.tf
index e41b121..a526a52 100644
--- a/data-egress_iam.tf
+++ b/data-egress_iam.tf
@@ -141,7 +141,7 @@ data "aws_iam_policy_document" "data_egress_server_task" {
       "${data.terraform_remote_state.common.outputs.published_bucket.arn}/dataegress/best-start/*",
       "${data.terraform_remote_state.common.outputs.published_bucket.arn}/dataegress/ers/*",
       "${data.terraform_remote_state.common.outputs.published_bucket.arn}/dataegress/RIS_DSP_Manual/*",
-      "${data.terraform_remote_state.common.outputs.published_bucket.arn}/preprocessed_csv/*"
+      "${data.terraform_remote_state.common.outputs.published_bucket.arn}/ap_ml/*"
     ]
   }
 

From ed9689c9f1c73f9437925088b01ca845b6c7b2f2 Mon Sep 17 00:00:00 2001
From: "john.green" <john.green@engineering.digital.dwp.gov.uk>
Date: Mon, 21 Nov 2022 11:28:11 +0000
Subject: [PATCH 3/3] change directory order model variable before date

---
 data-egress.tf | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/data-egress.tf b/data-egress.tf
index 3f37261..6a23fcf 100644
--- a/data-egress.tf
+++ b/data-egress.tf
@@ -1380,13 +1380,13 @@ resource "aws_dynamodb_table_item" "ap_ml_preprocessed_data" {
 
   item = <<ITEM
   {
-    "source_prefix":                {"S":    "ap_ml/preprocessed_data/journal/$TODAYS_DATE/*"},
+    "source_prefix":                {"S":    "ap_ml/journal/preprocessed_data/$TODAYS_DATE/*"},
     "pipeline_name":                {"S":    "ap_preproc_journal"},
     "recipient_name":               {"S":    "ap_preproc_journal"},
     "transfer_type":                {"S":    "S3"},
     "source_bucket":                {"S":    "${data.terraform_remote_state.common.outputs.published_bucket.id}"},
     "destination_bucket":           {"S":    "${data.terraform_remote_state.common.outputs.dataworks_model_published_bucket.id}"},
-    "destination_prefix":           {"S":    "ap_ml/preprocessed_data/journal/$TODAYS_DATE/"},
+    "destination_prefix":           {"S":    "ap_ml/journal/preprocessed_data/$TODAYS_DATE/"},
     "decrypt":                      {"bool": true},
     "rewrap_datakey":               {"bool": false},
     "encrypting_key_ssm_parm_name": {"S":    ""}
@@ -1401,13 +1401,13 @@ resource "aws_dynamodb_table_item" "ap_ml_preprocessed_training_data" {
 
   item = <<ITEM
   {
-    "source_prefix":                {"S":    "ap_ml/preprocessed_training_data/journal/$TODAYS_DATE/*"},
+    "source_prefix":                {"S":    "ap_ml/journal/preprocessed_training_data/$TODAYS_DATE/*"},
     "pipeline_name":                {"S":    "ap_preproc_journal"},
     "recipient_name":               {"S":    "ap_preproc_journal_training"},
     "transfer_type":                {"S":    "S3"},
     "source_bucket":                {"S":    "${data.terraform_remote_state.common.outputs.published_bucket.id}"},
     "destination_bucket":           {"S":    "${data.terraform_remote_state.common.outputs.dataworks_model_published_bucket.id}"},
-    "destination_prefix":           {"S":    "ap_ml/preprocessed_training_data/journal/$TODAYS_DATE/"},
+    "destination_prefix":           {"S":    "ap_ml/journal/preprocessed_training_data/$TODAYS_DATE/"},
     "decrypt":                      {"bool": true},
     "rewrap_datakey":               {"bool": false},
     "encrypting_key_ssm_parm_name": {"S":    ""}