Merge pull request #59 from materials-data-facility/moc-dev

Remove feedstock backup, attempt to address GDrive errors again
materials-data-facility · Nov 10, 2020 · 9ec5b61 · 9ec5b61
2 parents 1d52010 + 43b2890
commit 9ec5b61
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 20 deletions.
diff --git a/mdf_connect_server/config/dev.py b/mdf_connect_server/config/dev.py
@@ -30,8 +30,6 @@
     "BACKUP_HOST": "https://data.materialsdatafacility.org",
     "BACKUP_FEEDSTOCK": "/mdf_connect/dev/feedstock/",
 
-    # "GDRIVE_EP": "6ab13202-7c99-4e44-b0ff-04b8fbd77c97",
-
     "DEFAULT_CLEANUP": True,
     "FINAL_CLEANUP": True,
 

diff --git a/mdf_connect_server/processor/processor.py b/mdf_connect_server/processor/processor.py
@@ -517,23 +517,8 @@ def submission_driver(metadata, sub_conf, source_id, access_token, user_id):
             utils.complete_submission(source_id)
             return
 
-        # Back up feedstock
-        source_feed_loc = "globus://{}{}".format(CONFIG["LOCAL_EP"], feedstock_file)
-        backup_feed_loc = "globus://{}{}".format(CONFIG["BACKUP_EP"],
-                                                 os.path.join(CONFIG["BACKUP_FEEDSTOCK"],
-                                                              source_id + "_final.json"))
-        try:
-            feed_backup_res = list(utils.backup_data(mdf_transfer_client, source_feed_loc,
-                                                     backup_feed_loc, acl=None))[-1]
-            if not feed_backup_res.get(backup_feed_loc, {}).get("success"):
-                raise ValueError(feed_backup_res.get(backup_feed_loc, {}).get("error"))
-        except Exception as e:
-            utils.update_status(source_id, "ingest_search", "R",
-                                text=("Feedstock backup failed: {}".format(str(e))),
-                                except_on_fail=True)
-        else:
-            utils.update_status(source_id, "ingest_search", "S", except_on_fail=True)
-            os.remove(feedstock_file)
+        utils.update_status(source_id, "ingest_search", "S", except_on_fail=True)
+        os.remove(feedstock_file)
         service_res["mdf_search"] = "This dataset was ingested to MDF Search."
 
     # Move files to data_destinations

diff --git a/mdf_connect_server/utils/utils.py b/mdf_connect_server/utils/utils.py
@@ -820,6 +820,11 @@ def backup_data(transfer_client, storage_loc, backup_locs, acl=None,
         raise ValueError("Storage location '{}' (from '{}') is not a Globus Endpoint and cannot "
                          "be directly published from or backed up from"
                          .format(norm_store, storage_loc))
+    # If the storage is the MDF GDrive EP, the data_client and data_user aren't necessary
+    # because the MDF client has read access already.
+    elif storage_info.netloc == CONFIG["GDRIVE_EP"]:
+        data_client = None
+        data_user = None
 
     for backup in backup_locs:
         error = ""
@@ -1392,7 +1397,8 @@ def cancel_submission(source_id, wait=True):
     old_status_code = old_read_table("status", source_id)["status"]["code"]
     new_status_code = old_status_code.replace("z", "X").replace("W", "X") \
                                      .replace("T", "X").replace("P", "X")
-    update_res = modify_status_entry(source_id, {"code": new_status_code})
+    update_res = modify_status_entry(source_id, {"code": new_status_code,
+                                                 "active": False})
     if not update_res["success"]:
         return {
             "success": False,