From d2ccc0b80fd8ee5561a8fde7806b88cf008c9578 Mon Sep 17 00:00:00 2001 From: Jimmy Moore Date: Tue, 26 Nov 2024 09:48:55 +0000 Subject: [PATCH] Added throttle on s3 grab concurrency Signed-off-by: Jimmy Moore --- pkg/storage/device/device.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/storage/device/device.go b/pkg/storage/device/device.go index bf3352b..49b971f 100644 --- a/pkg/storage/device/device.go +++ b/pkg/storage/device/device.go @@ -34,6 +34,7 @@ const ( ) var syncConcurrency = map[int]int{storage.BlockTypeAny: 10} +var syncGrabConcurrency = 100 type Device struct { Provider storage.Provider @@ -376,9 +377,12 @@ func NewDeviceWithLoggingMetrics(ds *config.DeviceSchema, log types.Logger, met var wg sync.WaitGroup + concurrency := make(chan bool, syncGrabConcurrency) + // Pull these blocks in parallel for _, as := range startConfig.AlternateSources { wg.Add(1) + concurrency <- true go func(a packets.AlternateSource) { buffer := make([]byte, a.Length) n, err := s3dest.ReadAt(buffer, a.Offset) @@ -396,6 +400,7 @@ func NewDeviceWithLoggingMetrics(ds *config.DeviceSchema, log types.Logger, met if err != nil || n != int(a.Length) { panic(fmt.Sprintf("sync.start unable to write data to device from S3. %v", err)) } + <-concurrency wg.Done() }(as) }