Fix authorization issue
In one period of tasks, it switches projects several times using connect_as_project().
It fails to perform identity:list_projects action when openstack connection handler is connected as a project in which staffeln user has no admin role.
- Before cluster-scope actions, connect as staffeln default project (typically admin project) in which staffeln user has admin role.
- To avoid auth failure with rarely happening token rotation, added a decorate to retry connection.
diff --git a/staffeln/common/openstack.py b/staffeln/common/openstack.py
index 75f2379..c2c2e15 100644
--- a/staffeln/common/openstack.py
+++ b/staffeln/common/openstack.py
@@ -1,6 +1,10 @@
from openstack import exceptions
from openstack import proxy
+from oslo_log import log
from staffeln.common import auth
+from staffeln.i18n import _
+
+LOG = log.getLogger(__name__)
class OpenstackSDK():
@@ -11,13 +15,16 @@
def set_project(self, project):
+ LOG.debug(_("Connect as project %s" % project.get('name')))
project_id = project.get('id')
- if project_id in self.conn_list:
- self.conn = self.conn_list[project_id]
- else:
+ if project_id not in self.conn_list:
+ LOG.debug(_("Initiate connection for project %s" % project.get('name')))
conn = self.conn.connect_as_project(project)
- self.conn = conn
+ self.conn_list[project_id] = conn
+ LOG.debug(_("Connect as project %s" % project.get('name')))
+ self.conn = self.conn_list[project_id]
+
# user
def get_user_id(self):
diff --git a/staffeln/conductor/backup.py b/staffeln/conductor/backup.py
index d5807a4..1ba0ecf 100755
--- a/staffeln/conductor/backup.py
+++ b/staffeln/conductor/backup.py
@@ -5,6 +5,7 @@
from staffeln.conductor import result
from openstack.exceptions import ResourceNotFound as OpenstackResourceNotFound
from openstack.exceptions import SDKException as OpenstackSDKException
+from openstack.exceptions import HttpException as OpenstackHttpException
from oslo_log import log
from staffeln.common import context
from staffeln import objects
@@ -23,15 +24,31 @@
)
+def retry_auth(func):
+ """Decorator to reconnect openstack and avoid token rotation"""
+ def wrapper(self, *args, **kwargs):
+ try:
+ return func(self, *args, **kwargs)
+ except OpenstackHttpException as ex:
+ if ex.status_code == 403:
+ LOG.warn(_("Token has been expired or rotated!"))
+ self.refresh_openstacksdk()
+ return func(self, *args, **kwargs)
+ return wrapper
+
+
class Backup(object):
"""Implmentations of the queue with the sql."""
def __init__(self):
self.ctx = context.make_context()
self.result = result.BackupResult()
- self.openstacksdk = openstack.OpenstackSDK()
+ self.refresh_openstacksdk()
self.project_list = {}
+ def refresh_openstacksdk(self):
+ self.openstacksdk = openstack.OpenstackSDK()
+
def publish_backup_result(self):
self.result.publish()
@@ -188,6 +205,7 @@
for project in projects:
self.project_list[project.id] = project
+ @retry_auth
def check_instance_volumes(self):
"""Get the list of all the volumes from the project using openstacksdk
Function first list all the servers in the project and get the volumes
diff --git a/staffeln/conductor/manager.py b/staffeln/conductor/manager.py
index 785af67..5388df5 100755
--- a/staffeln/conductor/manager.py
+++ b/staffeln/conductor/manager.py
@@ -97,6 +97,7 @@
# Refresh the task queue
def _update_task_queue(self):
LOG.info(_("Updating backup task queue..."))
+ self.controller.refresh_openstacksdk()
self.controller.refresh_backup_result()
current_tasks = self.controller.get_queues()
self.controller.create_queue(current_tasks)
@@ -160,6 +161,7 @@
@periodics.periodic(spacing=retention_service_period, run_immediately=True)
def rotation_tasks():
+ self.controller.refresh_openstacksdk()
# 1. get the list of backups to remove based on the retention time
if not self.get_backup_list(): return
# 2. get project list