diff --git a/automate/deploy_mdf_flow.py b/automate/deploy_mdf_flow.py
index 03fbcdd7..14528e64 100644
--- a/automate/deploy_mdf_flow.py
+++ b/automate/deploy_mdf_flow.py
@@ -58,8 +58,8 @@
globus_auth=globus_auth)
mdf_flow.update_flow(flow_def=minimus_mdf_flow.flow_def(
- smtp_send_credentials=smtp_send_credentials,
sender_email=config['sender_email'],
+ admin_email=config['admin_email'],
flow_permissions=config['flow_permissions'],
description=description,
administered_by=[
diff --git a/automate/mdf_dev_flow_config.json b/automate/mdf_dev_flow_config.json
index be2d32c9..4714434a 100644
--- a/automate/mdf_dev_flow_config.json
+++ b/automate/mdf_dev_flow_config.json
@@ -6,5 +6,6 @@
"urn:globus:groups:id:5fc63928-3752-11e8-9c6f-0e00fd09bf20"
],
- "sender_email": "materialsdatafacility@uchicago.edu"
+ "sender_email": "materialsdatafacility@uchicago.edu",
+ "admin_email": "bengal1@illinois.edu"
}
diff --git a/automate/mdf_prod_flow_config.json b/automate/mdf_prod_flow_config.json
index be2d32c9..a0666a4a 100644
--- a/automate/mdf_prod_flow_config.json
+++ b/automate/mdf_prod_flow_config.json
@@ -6,5 +6,7 @@
"urn:globus:groups:id:5fc63928-3752-11e8-9c6f-0e00fd09bf20"
],
- "sender_email": "materialsdatafacility@uchicago.edu"
+ "sender_email": "materialsdatafacility@uchicago.edu",
+ "admin_email": "blaiszik@uchicago.edu"
+
}
diff --git a/automate/minimus_mdf_flow.py b/automate/minimus_mdf_flow.py
index c0aefd27..7625c804 100644
--- a/automate/minimus_mdf_flow.py
+++ b/automate/minimus_mdf_flow.py
@@ -2,8 +2,40 @@
from globus_automate_flow import GlobusAutomateFlowDef
+def email_submission_to_admin(sender_email, admin_email):
+ return {
+ "EmailSubmission": {
+ "Type": "Action",
+ "ActionUrl": "https://actions.globus.org/notification/notify",
+ "ExceptionOnActionFailure": True,
+ "Parameters": {
+ "body_mimetype": "text/html",
+ "sender": sender_email,
+ "destination": admin_email,
+ "subject": "Materials Data Facility Dataset Submission",
+ "body_template": "A new dataset has been submitted. $submission_json",
+ "body_variables": {
+ "$submission_json.=": "$.dataset_mdata"
+ },
+ "notification_method": "any",
+ "notification_priority": "high",
+ "send_credentials": [
+ {
+ "credential_method": "email",
+ "credential_type": "ses",
+ "credential_value.$": "$._private_email_credentials",
+ }
+ ],
+ "__Private_Parameters": ["send_credentials"],
+ },
+ "Next": "Check Metadata Only"
+ },
+ }
+
+
+
def check_update_metadata_only():
- return{
+ return {
"Check Metadata Only": {
"Comment": "Checks whether flow just updates the metadata",
"Type": "Choice",
@@ -11,10 +43,10 @@ def check_update_metadata_only():
{
"Variable": "$.update_metadata_only",
"BooleanEquals": True,
- "Next": "ChooseCuration"
+ "Next": "ChooseCuration",
}
],
- "Default": "CreateDestinationDir"
+ "Default": "CreateDestinationDir",
}
}
@@ -22,7 +54,7 @@ def check_update_metadata_only():
def file_transfer_steps():
"""
Steps to transfer user data to MDF repository:
- * Add a temporary write permission to the repo so we can execute the transfer
+ * Add a temporary write permission to the repo, so we can execute the transfer
as the submitting user
* Execute the transfer using runAs
* Remove the temporary write permission
@@ -41,11 +73,10 @@ def file_transfer_steps():
"Catch": [
{
"ErrorEquals": ["ActionFailedException", "States.Runtime"],
- "Next": "FailUserPermission"
+ "Next": "FailUserPermission",
}
],
-
- "Next": "UserPermissions"
+ "Next": "UserPermissions",
},
"UserPermissions": {
"Comment": "Temporarily add write permissions for the submitting user",
@@ -58,17 +89,16 @@ def file_transfer_steps():
"path.$": "$.user_transfer_inputs.transfer_items[0].destination_path",
"principal_type": "identity",
"principal.$": "$.user_transfer_inputs.submitting-user-id",
- "permissions": "rw"
+ "permissions": "rw",
},
"ResultPath": "$.UserPermissionResult",
"Catch": [
{
"ErrorEquals": ["ActionFailedException", "States.Runtime"],
- "Next": "FailUserPermission"
+ "Next": "FailUserPermission",
}
],
-
- "Next": "UserTransfer"
+ "Next": "UserTransfer",
},
"UserTransfer": {
"Comment": "Copy from user's endpoint to organization's dataset destination",
@@ -80,10 +110,10 @@ def file_transfer_steps():
"source_endpoint_id.$": "$.user_transfer_inputs.source_endpoint_id",
"destination_endpoint_id.$": "$.user_transfer_inputs.destination_endpoint_id",
"label.$": "$.user_transfer_inputs.label",
- "transfer_items.$": "$.user_transfer_inputs.transfer_items"
+ "transfer_items.$": "$.user_transfer_inputs.transfer_items",
},
"ResultPath": "$.UserTransferResult",
- "Next": "UndoUserPermissions"
+ "Next": "UndoUserPermissions",
},
"UndoUserPermissions": {
"Comment": "Remove temporary write permissions for the submitting user",
@@ -93,16 +123,16 @@ def file_transfer_steps():
"Parameters": {
"operation": "DELETE",
"endpoint_id.$": "$.user_transfer_inputs.destination_endpoint_id",
- "rule_id.$": "$.UserPermissionResult.details.access_id"
+ "rule_id.$": "$.UserPermissionResult.details.access_id",
},
"ResultPath": "$.UndoUserPermissionResult",
"Catch": [
{
"ErrorEquals": ["ActionFailedException", "States.Runtime"],
- "Next": "FailUserPermission"
+ "Next": "FailUserPermission",
}
],
- "Next": "CheckUserTransfer"
+ "Next": "CheckUserTransfer",
},
"CheckUserTransfer": {
"Type": "Choice",
@@ -110,33 +140,33 @@ def file_transfer_steps():
{
"Variable": "$.UserTransferResult.status",
"StringEquals": "SUCCEEDED",
- "Next": "ChooseCuration"
+ "Next": "ChooseCuration",
}
],
- "Default": "FailUserTransfer"
+ "Default": "FailUserTransfer",
},
"FailUserTransfer": {
"Type": "ExpressionEval",
"Parameters": {
"title": "MDF Submission Failed",
- "message.=": "'Your MDF submission ' + `$.source_id` + ' failed during transfer, please try again or contact materialsdatafacility@uchicago.edu:\n' + `$.UserTransferResult.details`"
+ "message.=": "'Your MDF submission ' + `$.source_id` + ' failed during transfer, please try again or contact materialsdatafacility@uchicago.edu:\n' + `$.UserTransferResult.details`",
},
"ResultPath": "$.FinalState",
- "Next": "NotifyUserEnd"
+ "Next": "NotifyUserEnd",
},
"FailUserPermission": {
"Type": "ExpressionEval",
"Parameters": {
"title": "MDF Permission Settings Failed",
- "message.=": "'Your MDF submission ' + `$.source_id` + ' failed to transfer, please try again or contact materialsdatafacility@uchicago.edu:\n' + `$.UserPermissionResult.details`"
+ "message.=": "'Your MDF submission ' + `$.source_id` + ' failed to transfer, please try again or contact materialsdatafacility@uchicago.edu:\n' + `$.UserPermissionResult.details`",
},
"ResultPath": "$.FinalState",
- "Next": "NotifyUserEnd"
+ "Next": "NotifyUserEnd",
},
}
-def curation_steps(sender_email):
+def curation_steps(sender_email, admin_email):
"""
Steps for allowing an administrator to curate submissions:
* Check to see if curation has been requested
@@ -151,10 +181,10 @@ def curation_steps(sender_email):
{
"Variable": "$.curation_input",
"BooleanEquals": False,
- "Next": "NeedDOI"
+ "Next": "NeedDOI",
}
],
- "Default": "SendCurationEmail"
+ "Default": "SendCurationEmail",
},
"SendCurationEmail": {
"Type": "Action",
@@ -164,26 +194,24 @@ def curation_steps(sender_email):
"Parameters": {
"body_mimetype": "text/html",
"sender": sender_email,
- "destination": "blaiszik@uchicago.edu",
- # "destination": "materialsdatafacility@uchicago.edu",
+ "destination": admin_email,
"subject": "Materials Data Facility Dataset Curation Request",
"body_template": "Please either Approve or Deny the dataset publication request here: $landing_page_url",
"body_variables": {
"landing_page_url.=": "'https://actions.globus.org/weboption/landing_page/' + `$._context.action_id`"
- }, "notification_method": "any",
+ },
+ "notification_method": "any",
"notification_priority": "high",
"send_credentials": [
{
"credential_method": "email",
"credential_type": "ses",
- "credential_value.$": "$._private_email_credentials"
+ "credential_value.$": "$._private_email_credentials",
}
],
- "__Private_Parameters": [
- "send_credentials"
- ]
+ "__Private_Parameters": ["send_credentials"],
},
- "Next": "CurateSubmission"
+ "Next": "CurateSubmission",
},
"CurateSubmission": {
"Type": "Action",
@@ -198,24 +226,25 @@ def curation_steps(sender_email):
"header_icon_link": "https://materialsdatafacility.org",
"header_text": "Curate an MDF Dataset",
"page_title": "Materials Data Facility",
- "preamble_text.=": "'A new dataset has been submitted. ' + `$.mdf_portal_link` +' Please review it to allow processing to continue.'"},
+ "preamble_text.=": "'A new dataset has been submitted. ' + `$.mdf_portal_link` +' Please review it to allow processing to continue.'",
+ },
"options": [
{
"name": "accepted",
"description": "Accept dataset",
"url_suffix.=": "`$._context.action_id` + '_approve'",
- "completed_message": " Curation Complete - Accepted
The dataset has been accepted for publication and processing will proceed."
+ "completed_message": " Curation Complete - Accepted
The dataset has been accepted for publication and processing will proceed.",
},
{
"name": "rejected",
"description": "Reject Dataset",
"url_suffix.=": "`$._context.action_id` + '_deny'",
- "completed_message": " Curation Complete - Rejected
The dataset has been rejected and will not proceed."
- }
- ]
+ "completed_message": " Curation Complete - Rejected
The dataset has been rejected and will not proceed.",
+ },
+ ],
},
"WaitTime": 86400,
- "Next": "ChooseAcceptance"
+ "Next": "ChooseAcceptance",
},
"ChooseAcceptance": {
"Type": "Choice",
@@ -223,27 +252,25 @@ def curation_steps(sender_email):
{
"Variable": "$.CurateResult.details.name",
"StringEquals": "accepted",
- "Next": "NeedDOI"
+ "Next": "NeedDOI",
},
{
"Variable": "$.CurateResult.details.name",
"StringEquals": "rejected",
- "Next": "FailCuration"
- }
+ "Next": "FailCuration",
+ },
],
- "Default": "ExceptionState"
+ "Default": "ExceptionState",
},
-
"FailCuration": {
"Type": "ExpressionEval",
"Parameters": {
"title": "MDF Submission Rejected",
- "message.=": "'Your submission (' + `$.source_id` + ') was rejected by a curator and did not complete the publication process. The curator gave the following reason: '+ `$.CurateResult.details.output.CurationResult.details.parameters.user_input`"
+ "message.=": "'Your submission (' + `$.source_id` + ') was rejected by a curator and did not complete the publication process. The curator gave the following reason: '+ `$.CurateResult.details.output.CurationResult.details.parameters.user_input`",
},
"ResultPath": "$.FinalState",
- "Next": "NotifyUserEnd"
+ "Next": "NotifyUserEnd",
},
-
}
@@ -259,13 +286,9 @@ def mint_doi_steps():
"Comment": "Checks whether flow needs to mint a DOI",
"Type": "Choice",
"Choices": [
- {
- "Variable": "$.mint_doi",
- "BooleanEquals": True,
- "Next": "MintDOI"
- }
+ {"Variable": "$.mint_doi", "BooleanEquals": True, "Next": "MintDOI"}
],
- "Default": "SearchIngest"
+ "Default": "SearchIngest",
},
"MintDOI": {
"Type": "Action",
@@ -285,22 +308,19 @@ def mint_doi_steps():
"titles.$": "$.dataset_mdata.dc.titles",
"publisher.$": "$.dataset_mdata.dc.publisher",
"publicationYear.$": "$.dataset_mdata.dc.publicationYear",
- "url.$": "$.mdf_portal_link"
- }
+ "url.$": "$.mdf_portal_link",
+ },
},
- "__Private_Parameters": [
- "username",
- "password"
- ]
+ "__Private_Parameters": ["username", "password"],
},
- "Next": "AddDoiToSearchRecord"
+ "Next": "AddDoiToSearchRecord",
},
"AddDoiToSearchRecord": {
"Type": "ExpressionEval",
"ResultPath": "$.dataset_mdata.dc.identifier",
"Parameters": {
"identifierType": "DOI",
- "identifier.$": "$.DoiResult.details.data.attributes.doi"
+ "identifier.$": "$.DoiResult.details.data.attributes.doi",
},
"Next": "SearchIngest",
},
@@ -318,12 +338,10 @@ def search_ingest_steps():
"Parameters": {
"search_index.$": "$.search_index",
"subject.$": "$.dataset_mdata.mdf.versioned_source_id",
- "visible_to": [
- "public"
- ],
- "content.$": "$.dataset_mdata"
+ "visible_to": ["public"],
+ "content.$": "$.dataset_mdata",
},
- "Next": "SubmissionSuccess"
+ "Next": "SubmissionSuccess",
},
}
@@ -338,10 +356,10 @@ def notify_user_steps(sender_email):
"Type": "ExpressionEval",
"Parameters": {
"title": "Dataset Accepted for Publication in the Materials Data Facility",
- "message.=": "'Publication succeeded! Your publication (' + `$.dataset_mdata.mdf.source_id`+ ') can be viewed at this link: ' + `$.mdf_portal_link`"
+ "message.=": "'Publication succeeded! Your publication (' + `$.dataset_mdata.mdf.source_id`+ ') can be viewed at this link: ' + `$.mdf_portal_link`",
},
"ResultPath": "$.FinalState",
- "Next": "NotifyUserEnd"
+ "Next": "NotifyUserEnd",
},
"NotifyUserEnd": {
"Type": "Action",
@@ -356,16 +374,14 @@ def notify_user_steps(sender_email):
{
"credential_method": "email",
"credential_type": "ses",
- "credential_value.$": "$._private_email_credentials"
+ "credential_value.$": "$._private_email_credentials",
}
],
- "__Private_Parameters": [
- "send_credentials"
- ]
+ "__Private_Parameters": ["send_credentials"],
},
"ResultPath": "$.NotifyUserResult",
"WaitTime": 86400,
- "Next": "EndSubmission"
+ "Next": "EndSubmission",
},
}
@@ -374,7 +390,7 @@ def exception_state(sender_email):
"""
Handle any general exceptions that occur as a result of the flow
"""
- return{
+ return {
"ExceptionState": {
"Type": "Action",
"ActionUrl": "https://actions.globus.org/notification/notify",
@@ -385,28 +401,31 @@ def exception_state(sender_email):
"destination.$": "$.submitting_user_email",
"subject": "Submission Failed to Ingest",
"body_template.=": "'Submission ' + `$.source_id` + ' fatally errored processing in Flow '+ `$._context.action_id` + '. Please review the Flow log for details about this exception.'",
-
"notification_method": "any",
"notification_priority": "high",
"send_credentials": [
{
"credential_method": "email",
"credential_type": "ses",
- "credential_value.$": "$._private_email_credentials"
+ "credential_value.$": "$._private_email_credentials",
}
],
- "__Private_Parameters": [
- "send_credentials"
- ]
+ "__Private_Parameters": ["send_credentials"],
},
"ResultPath": "$.ExceptionNotifyResult",
"WaitTime": 86400,
- "Next": "NotifyUserEnd"
+ "Next": "NotifyUserEnd",
},
}
-def flow_def(smtp_send_credentials, sender_email, flow_permissions, administered_by, description=""):
+def flow_def(
+ sender_email,
+ admin_email,
+ flow_permissions,
+ administered_by,
+ description="",
+):
return GlobusAutomateFlowDef(
title="MDF Ingest Flow",
subtitle="Ingest Materials Data Facility Submissions",
@@ -415,25 +434,19 @@ def flow_def(smtp_send_credentials, sender_email, flow_permissions, administered
runnable_by=flow_permissions,
administered_by=administered_by,
input_schema={},
-
flow_definition={
"StartAt": "StartSubmission",
"States": {
- "StartSubmission": {
- "Type": "Pass",
- "Next": "Check Metadata Only"
- },
+ "StartSubmission": {"Type": "Pass", "Next": "EmailSubmission"},
+ **email_submission_to_admin(sender_email, admin_email),
**check_update_metadata_only(),
**file_transfer_steps(),
- **curation_steps(sender_email),
+ **curation_steps(sender_email, admin_email),
**mint_doi_steps(),
**search_ingest_steps(),
**notify_user_steps(sender_email),
**exception_state(sender_email),
-
- "EndSubmission": {
- "Type": "Pass",
- "End": True
- }
- }
- })
+ "EndSubmission": {"Type": "Pass", "End": True},
+ },
+ },
+ )