fix(s3fs): Consolidate retry logic for rate limits (#865)

fsspec · Mar 21, 2024 · 11ec756 · 11ec756
1 parent efbe1e4
commit 11ec756
Showing 1 changed file with 5 additions and 2 deletions.
diff --git a/s3fs/core.py b/s3fs/core.py
@@ -118,10 +118,13 @@ async def _error_wrapper(func, *, args=(), kwargs=None, retries):
         except ClientError as e:
             logger.debug("Client error (maybe retryable): %s", e)
             err = e
+            wait_time = min(1.7**i * 0.1, 15)
             if "SlowDown" in str(e):
-                await asyncio.sleep(min(1.7**i * 0.1, 15))
+                await asyncio.sleep(wait_time)
+            elif "reduce your request rate" in str(e):
+                await asyncio.sleep(wait_time)
             elif "XAmzContentSHA256Mismatch" in str(e):
-                await asyncio.sleep(min(1.7**i * 0.1, 15))
+                await asyncio.sleep(wait_time)
             else:
                 break
         except Exception as e: