From 04c979f684c09163f263b2679a0b6a59b71433dc Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Wed, 20 Dec 2023 15:48:47 -0600 Subject: [PATCH 1/5] ...and get dev working again too --- infra/mdf/dev/main.tf | 2 ++ infra/mdf/dev/variables.tf | 9 +++++++++ infra/mdf/modules/lambdas/main.tf | 8 ++++---- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/infra/mdf/dev/main.tf b/infra/mdf/dev/main.tf index 43c020a..7b9825b 100644 --- a/infra/mdf/dev/main.tf +++ b/infra/mdf/dev/main.tf @@ -36,6 +36,8 @@ module "lambdas" { namespace = var.namespace lambda_execution_role_arn = module.permissions.submit_lambda_invoke_arn ecr_repos = var.ecr_repos + resource_tags = var.resource_tags + } module "dynamodb" { diff --git a/infra/mdf/dev/variables.tf b/infra/mdf/dev/variables.tf index 5867565..c59d4e6 100644 --- a/infra/mdf/dev/variables.tf +++ b/infra/mdf/dev/variables.tf @@ -47,4 +47,13 @@ variable "ecr_repos" { "status" = "557062710055.dkr.ecr.us-east-1.amazonaws.com/mdf-lambdas/status" "auth" = "557062710055.dkr.ecr.us-east-1.amazonaws.com/mdf-lambdas/auth" } +} + +variable "resource_tags" { + type = map(string) + default = { + "Owner" = "MDF" + "Environment" = "Development" + "Project" = "MDF Connect" + } } \ No newline at end of file diff --git a/infra/mdf/modules/lambdas/main.tf b/infra/mdf/modules/lambdas/main.tf index 7178d2e..f42f25e 100644 --- a/infra/mdf/modules/lambdas/main.tf +++ b/infra/mdf/modules/lambdas/main.tf @@ -27,7 +27,7 @@ resource "aws_lambda_function" "mdf-connect-auth" { resource "aws_cloudwatch_log_group" "auth_log_group" { - name = "/aws/lambda/${local.auth_function_name}" + name = "/aws/lambda/${local.auth_function_name}-${var.env}" retention_in_days = 5 tags = var.resource_tags } @@ -48,7 +48,7 @@ resource "aws_lambda_function" "mdf-connect-submit" { } resource "aws_cloudwatch_log_group" "submit_log_group" { - name = "/aws/lambda/${local.submit_function_name}" + name = "/aws/lambda/${local.submit_function_name}-${var.env}" retention_in_days = 5 tags = var.resource_tags } @@ -69,7 +69,7 @@ resource "aws_lambda_function" "mdf-connect-status" { } resource "aws_cloudwatch_log_group" "status_log_group" { - name = "/aws/lambda/${local.status_function_name}" + name = "/aws/lambda/${local.status_function_name}-${var.env}" retention_in_days = 5 tags = var.resource_tags } @@ -92,7 +92,7 @@ resource "aws_lambda_function" "mdf-connect-submissions" { } resource "aws_cloudwatch_log_group" "submissions_log_group" { - name = "/aws/lambda/${local.submissions_function_name}" + name = "/aws/lambda/${local.submissions_function_name}--${var.env}" retention_in_days = 5 tags = var.resource_tags } From a8633398411e4909d4d1f285db0d6afdb3f553dd Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Wed, 20 Dec 2023 20:31:53 -0600 Subject: [PATCH 2/5] Clean up automate resoures --- .gitignore | 4 +- automate/create_new_flow.py | 90 +++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 automate/create_new_flow.py diff --git a/.gitignore b/.gitignore index f83f4e3..175c9cb 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,8 @@ prototypes/forge_sandbox.ipynb travis.tar .mdfsecrets +.mdfsecrets.* aws/python .idea/ -secrets.env \ No newline at end of file +secrets.env + diff --git a/automate/create_new_flow.py b/automate/create_new_flow.py new file mode 100644 index 0000000..48cb383 --- /dev/null +++ b/automate/create_new_flow.py @@ -0,0 +1,90 @@ +import json +import sys + +import globus_sdk + +import minimus_mdf_flow +from globus_auth_manager import GlobusAuthManager +from globus_automate_flow import GlobusAutomateFlow, GlobusAutomateFlowDef + +if len(sys.argv) > 1: + secrets_file = f".mdfsecrets.{sys.argv[1]}" +else: + secrets_file = ".mdfsecrets" + +with open(secrets_file, 'r') as f: + globus_secrets = json.load(f) + +smtp_send_credentials = [{ + "credential_type": "smtp", + "credential_value": { + "hostname": globus_secrets['smtp_hostname'], + "username": globus_secrets['smtp_user'], + "password": globus_secrets['smtp_pass'] + } +}] + +conf_client = globus_sdk.ConfidentialAppAuthClient( + globus_secrets['API_CLIENT_ID'], globus_secrets['API_CLIENT_SECRET'] +) +cc_authorizer = globus_sdk.ClientCredentialsAuthorizer(conf_client, globus_sdk.FlowsClient.scopes.manage_flows) + +flows_client = globus_sdk.FlowsClient(authorizer=cc_authorizer) + +with open("mdf_flow_config.json") as f: + config = json.load(f) + +description = "MDF Connect Flow deployed manually" +globus_auth = GlobusAuthManager(globus_secrets['API_CLIENT_ID'], globus_secrets['API_CLIENT_SECRET']) +flow_def=minimus_mdf_flow.flow_def( + smtp_send_credentials=smtp_send_credentials, + sender_email=config['sender_email'], + flow_permissions=config['flow_permissions'], + description=description, + administered_by=[ + 'urn:globus:groups:id:5fc63928-3752-11e8-9c6f-0e00fd09bf20' # MDF Connect Admins + ]) + +flow_def.title = "Production MDF Ingest Flow" +mdf_flow = GlobusAutomateFlow.from_flow_def(client=flows_client, + flow_def=flow_def, + globus_auth=globus_auth) + +print(mdf_flow) + +mdf_flow.save_flow("mdf_flow_info.prod.json") + +print("MDF Flow deployed", mdf_flow) +submitting_user_scope_id = mdf_flow.get_scope_id_for_runAs_role('SubmittingUser')['scopes'][0]['id'] +print(f"RunAs Dependent scope ID = {submitting_user_scope_id}") + +submitting_user_scope_uri = mdf_flow.get_scope_uri_for_runAs_role('SubmittingUser') +print(f"RunAs Dependent Scope URI (will appear in the dict of dependent scopes in the authorizer) = {submitting_user_scope_uri}") + +print("PUT this to https://auth.globus.org/v2/api/scopes/:SCOPE_ID where SCOPE_ID is the ID of your API's scope") +connect_scope_def = { + "scope": { + "name": "MDF Connect", + "description": "Submit data to MDF Connect", + "scope_suffix": "connect", + "dependent_scopes": [ + { + "scope": "80fa5a88-ae26-4db7-be3a-c5f4cf4ac8d2", + "optional": False, + "requires_refresh_token": False + }, + { + "scope": "0b21a92f-2fed-4b2d-a481-50a58cc796b9", + "optional": False, + "requires_refresh_token": True + }, + { + "optional": False, + "requires_refresh_token": True, + "scope": submitting_user_scope_id + } + ] + } +} + +print(json.dumps(connect_scope_def)) \ No newline at end of file From 0553d391afb37b3a940bd51385b61b06c1166a9a Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Thu, 21 Dec 2023 09:57:08 -0600 Subject: [PATCH 3/5] Make submissions code tolerant of old dynamo records --- aws/submissions.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/aws/submissions.py b/aws/submissions.py index 0761a6b..6533391 100644 --- a/aws/submissions.py +++ b/aws/submissions.py @@ -1,6 +1,8 @@ import json import os +from globus_sdk import GlobusAPIError + from dynamo_manager import DynamoManager from automate_manager import AutomateManager from utils import get_secret @@ -19,8 +21,20 @@ def format_status_record(status:dict, automate_manager:AutomateManager) -> dict: status["submitter"], status["submission_time"]) - automate_status = automate_manager.get_status(status['action_id']) + automate_status = { + "status": "Unknown", + "details": { + "description": "Unknown" + } + } + if 'action-id' in status: + try: + automate_status = automate_manager.get_status(status['action_id']) + except GlobusAPIError: + automate_status["details"]['description'] = "Flow not found" + else: + automate_status["details"]['description'] = "Submission prior to GlobusAutomate" return { "source_id": status["source_id"], From 2fe2850f047542f55b715f93c5f9154f41631ca9 Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Thu, 21 Dec 2023 10:46:32 -0600 Subject: [PATCH 4/5] Handle unknown status --- aws/submissions.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/aws/submissions.py b/aws/submissions.py index 6533391..3acd928 100644 --- a/aws/submissions.py +++ b/aws/submissions.py @@ -10,7 +10,8 @@ status_codes = { "SUCCEEDED": "S", "ACTIVE": "P", - "FAILED": "F" + "FAILED": "F", + "UNKNOWN": "U" } def format_status_record(status:dict, automate_manager:AutomateManager) -> dict: @@ -22,7 +23,7 @@ def format_status_record(status:dict, automate_manager:AutomateManager) -> dict: status["submission_time"]) automate_status = { - "status": "Unknown", + "status": "UNKNOWN", "details": { "description": "Unknown" } From 9331df24cf69cadca957aa6a03536e0c6e0ed27c Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Thu, 21 Dec 2023 11:40:25 -0600 Subject: [PATCH 5/5] Allow for transfering data from legacay dynamo tables --- infra/mdf/dev/main.tf | 3 +++ infra/mdf/modules/dynamo/main.tf | 8 +++----- infra/mdf/modules/dynamo/variables.tf | 10 ++++++++++ infra/mdf/modules/permissions/main.tf | 5 ++++- infra/mdf/modules/permissions/variables.tf | 5 +++++ infra/mdf/prod/main.tf | 2 ++ 6 files changed, 27 insertions(+), 6 deletions(-) diff --git a/infra/mdf/dev/main.tf b/infra/mdf/dev/main.tf index 7b9825b..bf4906a 100644 --- a/infra/mdf/dev/main.tf +++ b/infra/mdf/dev/main.tf @@ -45,6 +45,8 @@ module "dynamodb" { env = var.env namespace = var.namespace env_vars = var.env_vars + resource_tags = var.resource_tags + dynamodb_write_capacity = 20 } module "permissions" { @@ -53,6 +55,7 @@ module "permissions" { namespace = var.namespace mdf_secrets_arn = var.mdf_secrets_arn dynamo_db_arn = module.dynamodb.dynamodb_arn + legacy_table_arn = "arn:aws:dynamodb:us-east-1:557062710055:table/dev-status-0.4" } module "api_gateway" { diff --git a/infra/mdf/modules/dynamo/main.tf b/infra/mdf/modules/dynamo/main.tf index 090eab3..29aef28 100644 --- a/infra/mdf/modules/dynamo/main.tf +++ b/infra/mdf/modules/dynamo/main.tf @@ -1,8 +1,9 @@ + resource "aws_dynamodb_table" "dynamodb-table" { name = "${var.namespace}-${var.env}" billing_mode = "PROVISIONED" read_capacity = 5 - write_capacity = 5 + write_capacity = var.dynamodb_write_capacity hash_key = "source_id" range_key = "version" attribute { @@ -21,8 +22,5 @@ resource "aws_dynamodb_table" "dynamodb-table" { enabled = false } - tags = { - Name = var.namespace - Environment = var.env - } + tags = var.resource_tags } diff --git a/infra/mdf/modules/dynamo/variables.tf b/infra/mdf/modules/dynamo/variables.tf index 9a3caab..92478ad 100644 --- a/infra/mdf/modules/dynamo/variables.tf +++ b/infra/mdf/modules/dynamo/variables.tf @@ -11,4 +11,14 @@ variable "namespace" { variable "env_vars" { description = "Set of environment variables for the functions." type = map(string) +} + +variable "resource_tags" { + description = "Tags to apply to all resources." + type = map(string) +} + +variable "dynamodb_write_capacity" { + type = number + description = "The write capacity for the DynamoDB table." } \ No newline at end of file diff --git a/infra/mdf/modules/permissions/main.tf b/infra/mdf/modules/permissions/main.tf index f40f0ea..9663ca2 100644 --- a/infra/mdf/modules/permissions/main.tf +++ b/infra/mdf/modules/permissions/main.tf @@ -75,7 +75,10 @@ resource "aws_iam_policy" "lambda_dynamodb_policy" { "dynamodb:Scan", ], Effect = "Allow", - Resource = [ var.dynamo_db_arn ] + Resource = [ + var.dynamo_db_arn, + var.legacy_table_arn + ] }, ], }) diff --git a/infra/mdf/modules/permissions/variables.tf b/infra/mdf/modules/permissions/variables.tf index b77a988..1e09a56 100644 --- a/infra/mdf/modules/permissions/variables.tf +++ b/infra/mdf/modules/permissions/variables.tf @@ -16,4 +16,9 @@ variable "mdf_secrets_arn" { variable "dynamo_db_arn" { type = string description = "ARN of the DynamoDB table" +} + +variable "legacy_table_arn" { + type = string + description = "ARN of the legacy DynamoDB table" } \ No newline at end of file diff --git a/infra/mdf/prod/main.tf b/infra/mdf/prod/main.tf index 384755c..c88a17e 100644 --- a/infra/mdf/prod/main.tf +++ b/infra/mdf/prod/main.tf @@ -44,6 +44,7 @@ module "dynamodb" { env = var.env namespace = var.namespace env_vars = var.env_vars + resource_tags = var.resource_tags } module "permissions" { @@ -52,6 +53,7 @@ module "permissions" { namespace = var.namespace mdf_secrets_arn = var.mdf_secrets_arn dynamo_db_arn = module.dynamodb.dynamodb_arn + legacy_table_arn = "arn:aws:dynamodb:us-east-1:557062710055:table/prod-status-alpha-1" } module "api_gateway" {