[CLOUDOPS-548] Allow retry on openstack HttpException

requirements.txt

@@ -20,3 +20,4 @@ parse
 tooz # Apache-2.0
 sherlock>=0.4.1 # MIT
 kubernetes # Apache-2.0
+tenacity

staffeln/common/openstack.py

@@ -4,10 +4,33 @@
 from oslo_log import log
 
 from staffeln.common import auth
+from staffeln import conf
 from staffeln.i18n import _
 
+import tenacity
+
+CONF = conf.CONF
 LOG = log.getLogger(__name__)
 
+class RetryHTTPError(tenacity.retry_if_exception):
+    """Retry strategy that retries if the exception is an ``HTTPError`` with
+    a abnormal status code.
+    """
+
+    def __init__(self):
+        def is_http_error(exception):
+            # Make sure we don't retry on 404, as not found could be an
+            # expected status.
+            result = (isinstance(exception, exceptions.HttpException) and
+                      exception.status_code != 404)
+            if result:
+                LOG.debug(f"Getting HttpException {exception} (status "
+                          f"code: {exception.status_code}), "
+                          "retry till timeout...")
+            return result
+
+        super().__init__(predicate=is_http_error)
+
 
 class OpenstackSDK:
     def __init__(self):
@@ -26,6 +49,11 @@ def set_project(self, project):
         self.conn = self.conn_list[project_id]
 
     # user
+    @tenacity.retry(
+        retry=RetryHTTPError(),
+        wait=tenacity.wait_exponential(max=30),
+        reraise=True,
+        stop=tenacity.stop_after_delay(CONF.conductor.retry_timeout))
     def get_user_id(self):
         user_name = self.conn.config.auth["username"]
         if "user_domain_id" in self.conn.config.auth:
@@ -38,15 +66,30 @@ def get_user_id(self):
             user = self.conn.get_user(name_or_id=user_name)
         return user.id
 
+    @tenacity.retry(
+        retry=RetryHTTPError(),
+        wait=tenacity.wait_exponential(max=30),
+        reraise=True,
+        stop=tenacity.stop_after_delay(CONF.conductor.retry_timeout))
     def get_role_assignments(self, project_id, user_id=None):
         filters = {"project": project_id}
         if user_id:
             filters["user"] = user_id
         return self.conn.list_role_assignments(filters=filters)
 
+    @tenacity.retry(
+        retry=RetryHTTPError(),
+        wait=tenacity.wait_exponential(max=30),
+        reraise=True,
+        stop=tenacity.stop_after_delay(CONF.conductor.retry_timeout))
     def get_user(self, user_id):
         return self.conn.get_user(name_or_id=user_id)
 
+    @tenacity.retry(
+        retry=RetryHTTPError(),
+        wait=tenacity.wait_exponential(max=30),
+        reraise=True,
+        stop=tenacity.stop_after_delay(CONF.conductor.retry_timeout))
     def get_project_member_emails(self, project_id):
         members = self.get_role_assignments(project_id)
         emails = []
@@ -63,9 +106,19 @@ def get_project_member_emails(self, project_id):
                         emails.append(user.email)
         return emails
 
+    @tenacity.retry(
+        retry=RetryHTTPError(),
+        wait=tenacity.wait_exponential(max=30),
+        reraise=True,
+        stop=tenacity.stop_after_delay(CONF.conductor.retry_timeout))
     def get_projects(self):
         return self.conn.list_projects()
 
+    @tenacity.retry(
+        retry=RetryHTTPError(),
+        wait=tenacity.wait_exponential(max=30),
+        reraise=True,
+        stop=tenacity.stop_after_delay(CONF.conductor.retry_timeout))
     def get_servers(self, project_id=None, all_projects=True, details=True):
         if project_id is not None:
             return self.conn.compute.servers(
@@ -76,9 +129,19 @@ def get_servers(self, project_id=None, all_projects=True, details=True):
         else:
             return self.conn.compute.servers(details=details, all_projects=all_projects)
 
+    @tenacity.retry(
+        retry=RetryHTTPError(),
+        wait=tenacity.wait_exponential(max=30),
+        reraise=True,
+        stop=tenacity.stop_after_delay(CONF.conductor.retry_timeout))
     def get_volume(self, uuid, project_id):
         return self.conn.get_volume_by_id(uuid)
 
+    @tenacity.retry(
+        retry=RetryHTTPError(),
+        wait=tenacity.wait_exponential(max=30),
+        reraise=True,
+        stop=tenacity.stop_after_delay(CONF.conductor.retry_timeout))
     def get_backup(self, uuid, project_id=None):
         try:
             return self.conn.get_volume_backup(uuid)
@@ -102,6 +165,11 @@ def create_backup(
             incremental=incremental,
         )
 
+    @tenacity.retry(
+        retry=RetryHTTPError(),
+        wait=tenacity.wait_exponential(max=30),
+        reraise=True,
+        stop=tenacity.stop_after_delay(CONF.conductor.retry_timeout))
     def delete_backup(self, uuid, project_id=None, force=False):
         # Note(Alex): v3 is not supporting force delete?
         # conn.block_storage.delete_backup(
@@ -115,11 +183,21 @@ def delete_backup(self, uuid, project_id=None, force=False):
         except exceptions.ResourceNotFound:
             return None
 
+    @tenacity.retry(
+        retry=RetryHTTPError(),
+        wait=tenacity.wait_exponential(max=30),
+        reraise=True,
+        stop=tenacity.stop_after_delay(CONF.conductor.retry_timeout))
     def get_backup_quota(self, project_id):
         # quota = conn.get_volume_quotas(project_id)
         quota = self._get_volume_quotas(project_id)
         return quota.backups
 
+    @tenacity.retry(
+        retry=RetryHTTPError(),
+        wait=tenacity.wait_exponential(max=30),
+        reraise=True,
+        stop=tenacity.stop_after_delay(CONF.conductor.retry_timeout))
     def get_backup_gigabytes_quota(self, project_id):
         # quota = conn.get_volume_quotas(project_id)
         quota = self._get_volume_quotas(project_id)

staffeln/conf/conductor.py

@@ -72,6 +72,12 @@
         min=0,
         help=_("Number of incremental backups between full backups."),
     ),
+    cfg.IntOpt(
+        "retry_timeout",
+        default=300,
+        min=1,
+        help=_("The timeout for retry, the unit is one second."),
+    ),
 ]
 
 rotation_opts = [

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py3,linters
+envlist = py3,py38,py39,py310,linters
 skipsdist = True
 sitepackages = False
 skip_missing_interpreters = True
@@ -14,15 +14,15 @@ setenv =
 deps =
     -r{toxinidir}/test-requirements.txt
     -r{toxinidir}/requirements.txt
-    -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
 
 install_commands =
     pip install {opts} {packages}
 
 
 [testenv:{py3,py38,py39,py310}]
 basepython = python3
-deps = -r{toxinidir}/test-requirements.txt
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
 commands = stestr run --slowest {posargs}
 
 [testenv:cover]