Add retry in copy (#310)

Signed-off-by: wayblink <[email protected]>
zilliztech · Mar 7, 2024 · 8c34a47 · 8c34a47
1 parent 86ed608
commit 8c34a47
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 6 deletions.
diff --git a/core/backup_impl_create_backup.go b/core/backup_impl_create_backup.go
@@ -747,9 +747,11 @@ func (b *BackupContext) copySegments(ctx context.Context, segments []*backuppb.S
 						return err
 					}
 
-					err = b.getStorageClient().Copy(ctx, b.milvusBucketName, b.backupBucketName, binlog.GetLogPath(), targetPath)
+					err = retry.Do(ctx, func() error {
+						return b.getStorageClient().Copy(ctx, b.milvusBucketName, b.backupBucketName, binlog.GetLogPath(), targetPath)
+					}, retry.Sleep(2*time.Second), retry.Attempts(5))
 					if err != nil {
-						log.Info("Fail to copy file",
+						log.Info("Fail to copy file after retry",
 							zap.Error(err),
 							zap.String("from", binlog.GetLogPath()),
 							zap.String("to", targetPath))
@@ -795,9 +797,11 @@ func (b *BackupContext) copySegments(ctx context.Context, segments []*backuppb.S
 							zap.String("file", binlog.GetLogPath()))
 						return errors.New("Binlog file not exist " + binlog.GetLogPath())
 					}
-					err = b.getStorageClient().Copy(ctx, b.milvusBucketName, b.backupBucketName, binlog.GetLogPath(), targetPath)
+					err = retry.Do(ctx, func() error {
+						return b.getStorageClient().Copy(ctx, b.milvusBucketName, b.backupBucketName, binlog.GetLogPath(), targetPath)
+					}, retry.Sleep(2*time.Second), retry.Attempts(5))
 					if err != nil {
-						log.Info("Fail to copy file",
+						log.Info("Fail to copy file after retry",
 							zap.Error(err),
 							zap.String("from", binlog.GetLogPath()),
 							zap.String("to", targetPath))

diff --git a/core/backup_impl_restore_backup.go b/core/backup_impl_restore_backup.go
@@ -623,9 +623,11 @@ func (b *BackupContext) restorePartition(ctx context.Context, targetDBName, targ
 					realFiles[i] = file
 				} else {
 					log.Debug("Copy temporary restore file", zap.String("from", file), zap.String("to", tempDir+file))
-					err := b.getStorageClient().Copy(ctx, backupBucketName, b.milvusBucketName, file, tempDir+file)
+					err = retry.Do(ctx, func() error {
+						return b.getStorageClient().Copy(ctx, backupBucketName, b.milvusBucketName, file, tempDir+file)
+					}, retry.Sleep(2*time.Second), retry.Attempts(5))
 					if err != nil {
-						log.Error("fail to copy backup date from backup bucket to restore target milvus bucket", zap.Error(err))
+						log.Error("fail to copy backup date from backup bucket to restore target milvus bucket after retry", zap.Error(err))
 						return err
 					}
 					realFiles[i] = tempDir + file