diff --git a/docs/articles/data_storage.rst b/docs/articles/data_storage.rst index 177249ea7..14fb6bf28 100644 --- a/docs/articles/data_storage.rst +++ b/docs/articles/data_storage.rst @@ -198,61 +198,78 @@ Putting all the migration pieces together ----------------------------------------- .. code-block:: - import os - import synapseutils - import synapseclient - - my_synapse_project_or_folder_to_migrate = "syn123" - - external_bucket_name = "my-external-synapse-bucket" - external_bucket_base_key = "path/within/bucket/" - - # # a path on disk where this utility can create a sqlite database to store its index. - # # nothing needs to exist at this path, but it must be a valid path on a volume with sufficient - # # disk space to store a meta data listing of all the contents in the indexed entity. - # # a rough rule of thumb is 100kB per 1000 entities indexed. - db_path = os.path.expanduser(f"~/synapseMigration/{my_synapse_project_or_folder_to_migrate}_my.db") - - syn = synapseclient.Synapse() - - # # Log-in with ~.synapseConfig `authToken` - syn.login() - - # The project or folder I want to migrate everything to this S3 storage location - project_or_folder = syn.get(my_synapse_project_or_folder_to_migrate) - - project_or_folder, storage_location, project_setting = syn.create_s3_storage_location( - # Despite the KW argument name, this can be a project or folder - folder=project_or_folder, - bucket_name=external_bucket_name, - base_key=external_bucket_base_key, - ) - - # The id of the destination storage location being migrated to - storage_location_id = storage_location["storageLocationId"] - print( - f"Indexing: {project_or_folder.id} for migration to storage_id: {storage_location_id} at: {db_path}" - ) - - result = synapseutils.index_files_for_migration( - syn, - project_or_folder.id, - storage_location_id, - db_path, - file_version_strategy="all", - ) - - print(f"Indexing result: {result.get_counts_by_status()}") - - print("Migrating files...") - - result = synapseutils.migrate_indexed_files( - syn, - db_path, - force=True, - ) + import os + import synapseutils + import synapseclient + + my_synapse_project_or_folder_to_migrate = "syn123" + + external_bucket_name = "my-external-synapse-bucket" + external_bucket_base_key = "path/within/bucket/" + + my_user_id = "1234" + + # # a path on disk where this utility can create a sqlite database to store its index. + # # nothing needs to exist at this path, but it must be a valid path on a volume with sufficient + # # disk space to store a meta data listing of all the contents in the indexed entity. + # # a rough rule of thumb is 100kB per 1000 entities indexed. + db_path = os.path.expanduser( + f"~/synapseMigration/{my_synapse_project_or_folder_to_migrate}_my.db" + ) + + syn = synapseclient.Synapse() + + # # Log-in with ~.synapseConfig `authToken` + syn.login() + + # The project or folder I want to migrate everything to this S3 storage location + project_or_folder = syn.get(my_synapse_project_or_folder_to_migrate) + + project_or_folder, storage_location, project_setting = syn.create_s3_storage_location( + # Despite the KW argument name, this can be a project or folder + folder=project_or_folder, + bucket_name=external_bucket_name, + base_key=external_bucket_base_key, + ) + + # The id of the destination storage location being migrated to + storage_location_id = storage_location["storageLocationId"] + print( + f"Indexing: {project_or_folder.id} for migration to storage_id: {storage_location_id} at: {db_path}" + ) + + try: + result = synapseutils.index_files_for_migration( + syn, + project_or_folder.id, + storage_location_id, + db_path, + file_version_strategy="all", + ) + + print(f"Indexing result: {result.get_counts_by_status()}") + + print("Migrating files...") + + result = synapseutils.migrate_indexed_files( + syn, + db_path, + force=True, + ) + + print(f"Migration result: {result.get_counts_by_status()}") + syn.sendMessage( + userIds=[my_user_id], + messageSubject=f"Migration success for {project_or_folder.id}", + messageBody=f"Migration result: {result.get_counts_by_status()}", + ) + except Exception as e: + syn.sendMessage( + userIds=[my_user_id], + messageSubject=f"Migration failed for {project_or_folder.id}", + messageBody=f"Migration failed with error: {e}", + ) - print(f"Migration result: {result.get_counts_by_status()}") The result of running this should look like .. code-block::