From 8c34a47ef189cd6683fcaa7803e9aaef9074fa56 Mon Sep 17 00:00:00 2001 From: wayblink Date: Thu, 7 Mar 2024 19:55:30 +0800 Subject: [PATCH] Add retry in copy (#310) Signed-off-by: wayblink --- core/backup_impl_create_backup.go | 12 ++++++++---- core/backup_impl_restore_backup.go | 6 ++++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/core/backup_impl_create_backup.go b/core/backup_impl_create_backup.go index 4f42e8d..da0738a 100644 --- a/core/backup_impl_create_backup.go +++ b/core/backup_impl_create_backup.go @@ -747,9 +747,11 @@ func (b *BackupContext) copySegments(ctx context.Context, segments []*backuppb.S return err } - err = b.getStorageClient().Copy(ctx, b.milvusBucketName, b.backupBucketName, binlog.GetLogPath(), targetPath) + err = retry.Do(ctx, func() error { + return b.getStorageClient().Copy(ctx, b.milvusBucketName, b.backupBucketName, binlog.GetLogPath(), targetPath) + }, retry.Sleep(2*time.Second), retry.Attempts(5)) if err != nil { - log.Info("Fail to copy file", + log.Info("Fail to copy file after retry", zap.Error(err), zap.String("from", binlog.GetLogPath()), zap.String("to", targetPath)) @@ -795,9 +797,11 @@ func (b *BackupContext) copySegments(ctx context.Context, segments []*backuppb.S zap.String("file", binlog.GetLogPath())) return errors.New("Binlog file not exist " + binlog.GetLogPath()) } - err = b.getStorageClient().Copy(ctx, b.milvusBucketName, b.backupBucketName, binlog.GetLogPath(), targetPath) + err = retry.Do(ctx, func() error { + return b.getStorageClient().Copy(ctx, b.milvusBucketName, b.backupBucketName, binlog.GetLogPath(), targetPath) + }, retry.Sleep(2*time.Second), retry.Attempts(5)) if err != nil { - log.Info("Fail to copy file", + log.Info("Fail to copy file after retry", zap.Error(err), zap.String("from", binlog.GetLogPath()), zap.String("to", targetPath)) diff --git a/core/backup_impl_restore_backup.go b/core/backup_impl_restore_backup.go index 3048cd6..22dceec 100644 --- a/core/backup_impl_restore_backup.go +++ b/core/backup_impl_restore_backup.go @@ -623,9 +623,11 @@ func (b *BackupContext) restorePartition(ctx context.Context, targetDBName, targ realFiles[i] = file } else { log.Debug("Copy temporary restore file", zap.String("from", file), zap.String("to", tempDir+file)) - err := b.getStorageClient().Copy(ctx, backupBucketName, b.milvusBucketName, file, tempDir+file) + err = retry.Do(ctx, func() error { + return b.getStorageClient().Copy(ctx, backupBucketName, b.milvusBucketName, file, tempDir+file) + }, retry.Sleep(2*time.Second), retry.Attempts(5)) if err != nil { - log.Error("fail to copy backup date from backup bucket to restore target milvus bucket", zap.Error(err)) + log.Error("fail to copy backup date from backup bucket to restore target milvus bucket after retry", zap.Error(err)) return err } realFiles[i] = tempDir + file