Skip to content

Commit

Permalink
Update script to include sending an email on migration finish
Browse files Browse the repository at this point in the history
  • Loading branch information
BryanFauble authored Nov 27, 2023
1 parent 55e1df3 commit 56d66b8
Showing 1 changed file with 71 additions and 54 deletions.
125 changes: 71 additions & 54 deletions docs/articles/data_storage.rst
Original file line number Diff line number Diff line change
Expand Up @@ -198,61 +198,78 @@ Putting all the migration pieces together
-----------------------------------------
.. code-block::
import os
import synapseutils
import synapseclient
my_synapse_project_or_folder_to_migrate = "syn123"
external_bucket_name = "my-external-synapse-bucket"
external_bucket_base_key = "path/within/bucket/"
# # a path on disk where this utility can create a sqlite database to store its index.
# # nothing needs to exist at this path, but it must be a valid path on a volume with sufficient
# # disk space to store a meta data listing of all the contents in the indexed entity.
# # a rough rule of thumb is 100kB per 1000 entities indexed.
db_path = os.path.expanduser(f"~/synapseMigration/{my_synapse_project_or_folder_to_migrate}_my.db")
syn = synapseclient.Synapse()
# # Log-in with ~.synapseConfig `authToken`
syn.login()
# The project or folder I want to migrate everything to this S3 storage location
project_or_folder = syn.get(my_synapse_project_or_folder_to_migrate)
project_or_folder, storage_location, project_setting = syn.create_s3_storage_location(
# Despite the KW argument name, this can be a project or folder
folder=project_or_folder,
bucket_name=external_bucket_name,
base_key=external_bucket_base_key,
)
# The id of the destination storage location being migrated to
storage_location_id = storage_location["storageLocationId"]
print(
f"Indexing: {project_or_folder.id} for migration to storage_id: {storage_location_id} at: {db_path}"
)
result = synapseutils.index_files_for_migration(
syn,
project_or_folder.id,
storage_location_id,
db_path,
file_version_strategy="all",
)
print(f"Indexing result: {result.get_counts_by_status()}")
print("Migrating files...")
result = synapseutils.migrate_indexed_files(
syn,
db_path,
force=True,
)
import os
import synapseutils
import synapseclient
my_synapse_project_or_folder_to_migrate = "syn123"
external_bucket_name = "my-external-synapse-bucket"
external_bucket_base_key = "path/within/bucket/"
my_user_id = "1234"
# # a path on disk where this utility can create a sqlite database to store its index.
# # nothing needs to exist at this path, but it must be a valid path on a volume with sufficient
# # disk space to store a meta data listing of all the contents in the indexed entity.
# # a rough rule of thumb is 100kB per 1000 entities indexed.
db_path = os.path.expanduser(
f"~/synapseMigration/{my_synapse_project_or_folder_to_migrate}_my.db"
)
syn = synapseclient.Synapse()
# # Log-in with ~.synapseConfig `authToken`
syn.login()
# The project or folder I want to migrate everything to this S3 storage location
project_or_folder = syn.get(my_synapse_project_or_folder_to_migrate)
project_or_folder, storage_location, project_setting = syn.create_s3_storage_location(
# Despite the KW argument name, this can be a project or folder
folder=project_or_folder,
bucket_name=external_bucket_name,
base_key=external_bucket_base_key,
)
# The id of the destination storage location being migrated to
storage_location_id = storage_location["storageLocationId"]
print(
f"Indexing: {project_or_folder.id} for migration to storage_id: {storage_location_id} at: {db_path}"
)
try:
result = synapseutils.index_files_for_migration(
syn,
project_or_folder.id,
storage_location_id,
db_path,
file_version_strategy="all",
)
print(f"Indexing result: {result.get_counts_by_status()}")
print("Migrating files...")
result = synapseutils.migrate_indexed_files(
syn,
db_path,
force=True,
)
print(f"Migration result: {result.get_counts_by_status()}")
syn.sendMessage(
userIds=[my_user_id],
messageSubject=f"Migration success for {project_or_folder.id}",
messageBody=f"Migration result: {result.get_counts_by_status()}",
)
except Exception as e:
syn.sendMessage(
userIds=[my_user_id],
messageSubject=f"Migration failed for {project_or_folder.id}",
messageBody=f"Migration failed with error: {e}",
)
print(f"Migration result: {result.get_counts_by_status()}")
The result of running this should look like
.. code-block::
Expand Down

0 comments on commit 56d66b8

Please sign in to comment.