Merge pull request #83 from vexxhost/improve-delete-backup
Improve delete backup and email
diff --git a/staffeln/conductor/backup.py b/staffeln/conductor/backup.py
index 7b8e523..565e09d 100755
--- a/staffeln/conductor/backup.py
+++ b/staffeln/conductor/backup.py
@@ -70,8 +70,19 @@
def refresh_openstacksdk(self):
self.openstacksdk = openstack.OpenstackSDK()
- def publish_backup_result(self):
- self.result.publish()
+ def publish_backup_result(self, purge_on_success=False):
+ for project_id, project_name in self.result.project_list:
+ try:
+ publish_result = self.result.publish(project_id, project_name)
+ if publish_result and purge_on_success:
+ # Purge backup queue tasks
+ self.purge_backups(project_id)
+ except Exception as ex: # pylint: disable=W0703
+ LOG.warn(
+ "Failed to publish backup result or "
+ f"purge backup tasks for project {project_id} "
+ f"{str(ex)}"
+ )
def refresh_backup_result(self):
self.result.initialize()
@@ -144,13 +155,20 @@
except OpenstackResourceNotFound:
return False
- def purge_backups(self):
+ def purge_backups(self, project_id=None):
+ LOG.info(f"Start pruge backup tasks for project {project_id}")
# TODO make all this in a single DB command
success_tasks = self.get_queues(
- filters={"backup_status": constants.BACKUP_COMPLETED}
+ filters={
+ "backup_status": constants.BACKUP_COMPLETED,
+ "project_id": project_id,
+ }
)
failed_tasks = self.get_queues(
- filters={"backup_status": constants.BACKUP_FAILED}
+ filters={
+ "backup_status": constants.BACKUP_FAILED,
+ "project_id": project_id,
+ }
)
for queue in success_tasks:
LOG.info("Start purge completed tasks.")
@@ -207,11 +225,13 @@
return backup_object.delete_backup()
if backup["status"] in ("available"):
self.openstacksdk.delete_backup(backup_object.backup_id)
- backup_object.delete_backup()
+ # Don't remove backup until it's officially removed from Cinder
+ # backup_object.delete_backup()
elif backup["status"] in ("error", "error_restoring"):
# Try to remove it from cinder
self.openstacksdk.delete_backup(backup_object.backup_id)
- backup_object.delete_backup()
+ # Don't remove backup until it's officially removed from Cinder
+ # backup_object.delete_backup()
else: # "deleting", "restoring"
LOG.info(
_(
@@ -259,22 +279,20 @@
)
return backup_object.delete_backup()
- self.openstacksdk.delete_backup(uuid=backup_object.backup_id)
- backup_object.delete_backup()
-
- # TODO(ricolin) should check if backup delete completed
-
+ self.openstacksdk.delete_backup(uuid=backup_object.backup_id, force=True)
+ # Don't remove backup until it's officially removed from Cinder
+ # backup_object.delete_backup()
except Exception as e:
if skip_inc_err and "Incremental backups exist for this backup" in str(e):
- pass
+ LOG.debug(str(e))
else:
- LOG.warn(
+ LOG.info(
_(
- f"Backup {backup_object.backup_id} deletion failed. "
- f"Please check into the exception: {str(e)}."
+ f"Backup {backup_object.backup_id} deletion failed."
"Skip this backup from remove now and will retry later."
)
)
+ LOG.debug(f"deletion failed {str(e)}")
# Don't remove backup object, keep it and retry on next periodic task
# backup_object.delete_backup()
diff --git a/staffeln/conductor/manager.py b/staffeln/conductor/manager.py
index 4f99ecc..d8ead77 100755
--- a/staffeln/conductor/manager.py
+++ b/staffeln/conductor/manager.py
@@ -138,9 +138,7 @@
constants.BACKUP_FAILED,
):
LOG.info(_("Reporting finished backup tasks..."))
- self.controller.publish_backup_result()
- # Purge backup queue tasks
- self.controller.purge_backups()
+ self.controller.publish_backup_result(purge_on_success=True)
return
def backup_engine(self, backup_service_period):
diff --git a/staffeln/conductor/result.py b/staffeln/conductor/result.py
index aaca100..3b49a05 100644
--- a/staffeln/conductor/result.py
+++ b/staffeln/conductor/result.py
@@ -22,118 +22,118 @@
def add_project(self, project_id, project_name):
self.project_list.add((project_id, project_name))
- def send_result_email(self):
- subject = "Staffeln Backup result"
+ def send_result_email(self, subject=None, receiver=None):
+ if not CONF.notification.sender_email:
+ LOG.info(
+ "Directly record report in log as sender email "
+ f"are not configed. Report: {self.content}"
+ )
+ return
+ if not subject:
+ subject = "Staffeln Backup result"
+ if len(CONF.notification.receiver) != 0:
+ # Found receiver in config, override report receiver.
+ receiver = CONF.notification.receiver
try:
- if len(CONF.notification.receiver) == 0:
- LOG.info(
- "Directly record report in log as no receiver "
- "email provided. Report: %s" % self.content
- )
- return
smtp_profile = {
"src_email": CONF.notification.sender_email,
"src_name": "Staffeln",
"src_pwd": CONF.notification.sender_pwd,
- "dest_email": CONF.notification.receiver,
+ "dest_email": receiver,
"subject": subject,
"content": self.content,
"smtp_server_domain": CONF.notification.smtp_server_domain,
"smtp_server_port": CONF.notification.smtp_server_port,
}
email.send(smtp_profile)
- LOG.info(_("Backup result email sent"))
+ LOG.info(_(f"Backup result email sent to {receiver}"))
except Exception as e:
- LOG.error(
+ LOG.warn(
_(
- "Backup result email send failed. Please check email configuration. %s"
- % (str(e))
+ f"Backup result email send to {receiver} failed. "
+ f"Please check email configuration. {str(e)}"
)
)
raise
- def publish(self):
+ def publish(self, project_id=None, project_name=None):
# 1. get quota
self.content = "<h3>${TIME}</h3><br>"
self.content = self.content.replace("${TIME}", xtime.get_current_strtime())
backup_mgt = backup.Backup()
- project_success = {}
- project_failed = {}
success_tasks = backup_mgt.get_queues(
- filters={"backup_status": constants.BACKUP_COMPLETED}
+ filters={
+ "backup_status": constants.BACKUP_COMPLETED,
+ "project_id": project_id,
+ }
)
- for task in success_tasks:
- if task.project_id in project_success:
- project_success[task.project_id].append(task)
- else:
- project_success[task.project_id] = [task]
failed_tasks = backup_mgt.get_queues(
- filters={"backup_status": constants.BACKUP_FAILED}
+ filters={
+ "backup_status": constants.BACKUP_FAILED,
+ "project_id": project_id,
+ }
)
- for task in failed_tasks:
- if task.project_id in project_failed:
- project_failed[task.project_id].append(task)
- else:
- project_failed[task.project_id] = [task]
+ if not success_tasks and not failed_tasks:
+ return False
html = ""
- for project_id, project_name in self.project_list:
- quota = backup_mgt.get_backup_quota(project_id)
+ quota = backup_mgt.get_backup_quota(project_id)
- html += (
- "<h3>Project: ${PROJECT}</h3><h3>Quota Usage</h3>"
- "<FONT COLOR=${QUOTA_COLLOR}><h4>Limit: ${QUOTA_LIMIT}, In Use: "
- "${QUOTA_IN_USE}, Reserved: ${QUOTA_RESERVED}, Total "
- "rate: ${QUOTA_USAGE}</h4></FONT>"
- "<h3>Success List</h3>"
- "<FONT COLOR=GREEN><h4>${SUCCESS_VOLUME_LIST}</h4></FONT><br>"
- "<h3>Failed List</h3>"
- "<FONT COLOR=RED><h4>${FAILED_VOLUME_LIST}</h4></FONT><br>"
+ html += (
+ "<h3>Project: ${PROJECT} (ID: ${PROJECT_ID})</h3><h3>Quota Usage</h3>"
+ "<FONT COLOR=${QUOTA_COLLOR}><h4>Limit: ${QUOTA_LIMIT}, In Use: "
+ "${QUOTA_IN_USE}, Reserved: ${QUOTA_RESERVED}, Total "
+ "rate: ${QUOTA_USAGE}</h4></FONT>"
+ "<h3>Success List</h3>"
+ "<FONT COLOR=GREEN><h4>${SUCCESS_VOLUME_LIST}</h4></FONT><br>"
+ "<h3>Failed List</h3>"
+ "<FONT COLOR=RED><h4>${FAILED_VOLUME_LIST}</h4></FONT><br>"
+ )
+
+ if success_tasks:
+ success_volumes = "<br>".join(
+ [
+ (
+ f"Volume ID: {str(e.volume_id)}, Backup ID: {str(e.backup_id)}, "
+ f"Backup mode: {'Incremental' if e.incremental else 'Full'}, "
+ f"Created at: {str(e.created_at)}, Last updated at: "
+ f"{str(e.updated_at)}"
+ )
+ for e in success_tasks
+ ]
)
-
- if project_id in project_success:
- success_volumes = "<br>".join(
- [
- (
- f"Volume ID: {str(e.volume_id)}, Backup ID: {str(e.backup_id)}, "
- f"Backup mode: {'Incremental' if e.incremental else 'Full'}, "
- f"Created at: {str(e.created_at)}, Last updated at: "
- f"{str(e.updated_at)}"
- )
- for e in project_success[project_id]
- ]
- )
- else:
- success_volumes = "<br>"
- if project_id in project_failed:
- failed_volumes = "<br>".join(
- [
- (
- f"Volume ID: {str(e.volume_id)}, Reason: {str(e.reason)}, "
- f"Created at: {str(e.created_at)}, Last updated at: "
- f"{str(e.updated_at)}"
- )
- for e in project_failed[project_id]
- ]
- )
- else:
- failed_volumes = "<br>"
- quota_usage = (quota["in_use"] + quota["reserved"]) / quota["limit"]
- if quota_usage > 0.8:
- quota_color = "RED"
- elif quota_usage > 0.5:
- quota_color = "YALLOW"
- else:
- quota_color = "GREEN"
- html = html.replace("${QUOTA_USAGE}", str(quota_usage))
- html = html.replace("${QUOTA_COLLOR}", quota_color)
- html = html.replace("${QUOTA_LIMIT}", str(quota["limit"]))
- html = html.replace("${QUOTA_IN_USE}", str(quota["in_use"]))
- html = html.replace("${QUOTA_RESERVED}", str(quota["reserved"]))
- html = html.replace("${SUCCESS_VOLUME_LIST}", success_volumes)
- html = html.replace("${FAILED_VOLUME_LIST}", failed_volumes)
- html = html.replace("${PROJECT}", project_name)
- if html == "":
- return
+ else:
+ success_volumes = "<br>"
+ if failed_tasks:
+ failed_volumes = "<br>".join(
+ [
+ (
+ f"Volume ID: {str(e.volume_id)}, Reason: {str(e.reason)}, "
+ f"Created at: {str(e.created_at)}, Last updated at: "
+ f"{str(e.updated_at)}"
+ )
+ for e in failed_tasks
+ ]
+ )
+ else:
+ failed_volumes = "<br>"
+ quota_usage = (quota["in_use"] + quota["reserved"]) / quota["limit"]
+ if quota_usage > 0.8:
+ quota_color = "RED"
+ elif quota_usage > 0.5:
+ quota_color = "YALLOW"
+ else:
+ quota_color = "GREEN"
+ html = html.replace("${QUOTA_USAGE}", str(quota_usage))
+ html = html.replace("${QUOTA_COLLOR}", quota_color)
+ html = html.replace("${QUOTA_LIMIT}", str(quota["limit"]))
+ html = html.replace("${QUOTA_IN_USE}", str(quota["in_use"]))
+ html = html.replace("${QUOTA_RESERVED}", str(quota["reserved"]))
+ html = html.replace("${SUCCESS_VOLUME_LIST}", success_volumes)
+ html = html.replace("${FAILED_VOLUME_LIST}", failed_volumes)
+ html = html.replace("${PROJECT}", project_name)
+ html = html.replace("${PROJECT_ID}", project_id)
self.content += html
- self.send_result_email()
+ subject = f"Staffeln Backup result: {project_id}"
+ self.send_result_email(subject=subject, receiver=f"{project_name}@cgm.com")
+ return True