Move retry arguments to settings module

onegram/session.py

@@ -9,8 +9,7 @@
 from requests import HTTPError
 from sessionlib import Session
 from sessionlib import sessionaware as _sessionaware
-from tenacity import retry, retry_if_exception_type
-from tenacity import wait_chain, wait_fixed
+from tenacity import retry, retry_never
 from urllib3.exceptions import InsecureRequestWarning
 from urllib3.util import parse_url
 
@@ -103,14 +102,19 @@ def request(self, method, url, *a, **kw):
         def _after_request_attempt(func, trial_number, *a, **kw):
             self.logger.warning(f'RETRY {trial_number} attempt(s) ...')
 
-        @retry(wait=wait_chain(wait_fixed(60), wait_fixed(15)),
-               retry=retry_if_exception_type(RateLimitedError),
-               after=_after_request_attempt)
+        if self.settings.get('RETRY_ENABLED'):
+            retry_kw = {'after': _after_request_attempt}
+            retry_kw.update(self.settings.get('RETRY_SETTINGS', {}))
+        else:
+            retry_kw = {'retry': retry_never}
+
+        @retry(**retry_kw)
         def _request():
             with self.rate_limiter:
                 self.logger.info(f'{method} "{url}"')
                 response = self._requests.request(method, url, *a, **kw)
                 return validate_response(self, response)
+
         return _request()
 
 

onegram/settings.py

@@ -3,7 +3,12 @@
 from pathlib import Path
 from decouple import config
 
+from tenacity import retry_if_exception_type
+from tenacity import wait_chain, wait_fixed, wait_random
+from tenacity import stop_after_delay
+
 from .utils import head_tail, choices, repeat
+from .exceptions import RateLimitedError
 
 
 CURRENT_DIR = Path.cwd()
@@ -16,6 +21,9 @@
 # Uncomment to set a custom User-Agent
 # USER_AGENT = None
 
+# Uncomment to set proxies
+# PROXIES = {'http': '<proxy>', 'https': '<proxy>'}
+
 VERIFY_SSL = config('VERIFY_SSL', default=True, cast=bool)
 
 # Limits requests per second
@@ -42,3 +50,10 @@
     'explore': repeat(24),
     'explore_tag': choices(range(1, 13)),
 }
+
+RETRY_ENABLED = True
+RETRY_SETTINGS = {
+    'wait': wait_chain(wait_fixed(60), wait_fixed(15) + wait_random(0, 2)),
+    'retry': retry_if_exception_type(RateLimitedError),
+    'stop': stop_after_delay(60 * 20),
+}