Refact report function to use report timestamp
diff --git a/requirements.txt b/requirements.txt
index 103f767..8a52b2b 100755
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,6 +3,8 @@
# process, which may cause wedges in the gate later.
pbr>=2.0 # Apache-2.0
+
+alembic>=1.4.2 # MIT
flask
cotyledon>=1.3.0 #Apache-2.0
futurist>=1.8.0 # Apache-2.0
diff --git a/staffeln/conductor/manager.py b/staffeln/conductor/manager.py
index dabcca0..34f1af3 100755
--- a/staffeln/conductor/manager.py
+++ b/staffeln/conductor/manager.py
@@ -10,6 +10,7 @@
from staffeln.common import constants, context, lock
from staffeln.common import time as xtime
from staffeln.conductor import backup as backup_controller
+from staffeln import objects
from staffeln.i18n import _
from tooz import coordination
@@ -131,16 +132,25 @@
def _report_backup_result(self):
report_period = CONF.conductor.report_period
threshold_strtime = timeutils.utcnow() - timedelta(seconds=report_period)
- filters = {"created_at__lt": threshold_strtime.astimezone()}
- old_tasks = self.controller.get_queues(filters=filters)
- for task in old_tasks:
- if task.backup_status in (
- constants.BACKUP_COMPLETED,
- constants.BACKUP_FAILED,
- ):
- LOG.info(_("Reporting finished backup tasks..."))
- self.controller.publish_backup_result(purge_on_success=True)
- return
+
+ filters = {"created_at__gt": threshold_strtime.astimezone()}
+ report_tss = objects.ReportTimestamp.list( # pylint: disable=E1120
+ context=self.ctx, filters=filters
+ )
+ # If there are no reports that generated within report_period seconds,
+ # generate and publish one.
+ if not report_tss:
+ LOG.info(_("Reporting finished backup tasks..."))
+ self.controller.publish_backup_result(purge_on_success=True)
+
+ # Purge records that live longer than 10 report cycles
+ threshold_strtime = timeutils.utcnow() - timedelta(seconds=report_period*10)
+ filters = {"created_at__lt": threshold_strtime.astimezone()}
+ old_report_tss = objects.ReportTimestamp.list( # pylint: disable=E1120
+ context=self.ctx, filters=filters
+ )
+ for report_ts in old_report_tss:
+ report_ts.delete()
def backup_engine(self, backup_service_period):
LOG.info("Backup manager started %s" % str(time.time()))
diff --git a/staffeln/conductor/result.py b/staffeln/conductor/result.py
index 88173ca..bfcbf8e 100644
--- a/staffeln/conductor/result.py
+++ b/staffeln/conductor/result.py
@@ -2,8 +2,10 @@
# This should be upgraded by integrating with mail server to send batch
import staffeln.conf
from oslo_log import log
+from oslo_utils import timeutils
from staffeln.common import constants, email
from staffeln.common import time as xtime
+from staffeln import objects
from staffeln.i18n import _
CONF = staffeln.conf.CONF
@@ -27,7 +29,7 @@
"Directly record report in log as sender email "
f"are not configed. Report: {self.content}"
)
- return
+ return True
if not subject:
subject = "Staffeln Backup result"
if len(CONF.notification.receiver) != 0:
@@ -67,6 +69,7 @@
}
email.send(smtp_profile)
LOG.info(_(f"Backup result email sent to {receiver}"))
+ return True
except Exception as e:
LOG.warn(
_(
@@ -76,10 +79,17 @@
)
raise
+ def create_report_record(self):
+ sender = CONF.notification.sender_email \
+ if CONF.notification.sender_email else "RecordInLog"
+ report_ts = objects.ReportTimestamp(self.backup_mgt.ctx)
+ report_ts.sender = sender
+ report_ts.created_at = timeutils.utcnow()
+ return report_ts.create()
+
def publish(self, project_id=None, project_name=None):
# 1. get quota
- self.content = "<h3>${TIME}</h3><br>"
- self.content = self.content.replace("${TIME}", xtime.get_current_strtime())
+ self.content = f"<h3>{xtime.get_current_strtime()}</h3><br>"
success_tasks = self.backup_mgt.get_queues(
filters={
"backup_status": constants.BACKUP_COMPLETED,
@@ -95,20 +105,16 @@
if not success_tasks and not failed_tasks:
return False
+ # Geneerate HTML Content
html = ""
- quota = self.backup_mgt.get_backup_quota(project_id)
-
- html += (
- "<h3>Project: ${PROJECT} (ID: ${PROJECT_ID})</h3><h3>Quota Usage</h3>"
- "<FONT COLOR=${QUOTA_COLLOR}><h4>Limit: ${QUOTA_LIMIT}, In Use: "
- "${QUOTA_IN_USE}, Reserved: ${QUOTA_RESERVED}, Total "
- "rate: ${QUOTA_USAGE}</h4></FONT>"
- "<h3>Success List</h3>"
- "<FONT COLOR=GREEN><h4>${SUCCESS_VOLUME_LIST}</h4></FONT><br>"
- "<h3>Failed List</h3>"
- "<FONT COLOR=RED><h4>${FAILED_VOLUME_LIST}</h4></FONT><br>"
- )
-
+ quota = self.backup_mgt.get_backup_gigabytes_quota(project_id)
+ quota_usage = (quota["in_use"] + quota["reserved"]) / quota["limit"]
+ if quota_usage > 0.8:
+ quota_color = "RED"
+ elif quota_usage > 0.5:
+ quota_color = "YALLOW"
+ else:
+ quota_color = "GREEN"
if success_tasks:
success_volumes = "<br>".join(
[
@@ -136,23 +142,22 @@
)
else:
failed_volumes = "<br>"
- quota_usage = (quota["in_use"] + quota["reserved"]) / quota["limit"]
- if quota_usage > 0.8:
- quota_color = "RED"
- elif quota_usage > 0.5:
- quota_color = "YALLOW"
- else:
- quota_color = "GREEN"
- html = html.replace("${QUOTA_USAGE}", str(quota_usage))
- html = html.replace("${QUOTA_COLLOR}", quota_color)
- html = html.replace("${QUOTA_LIMIT}", str(quota["limit"]))
- html = html.replace("${QUOTA_IN_USE}", str(quota["in_use"]))
- html = html.replace("${QUOTA_RESERVED}", str(quota["reserved"]))
- html = html.replace("${SUCCESS_VOLUME_LIST}", success_volumes)
- html = html.replace("${FAILED_VOLUME_LIST}", failed_volumes)
- html = html.replace("${PROJECT}", project_name)
- html = html.replace("${PROJECT_ID}", project_id)
+ html += (
+ f"<h3>Project: {project_name} (ID: {project_id})</h3>"
+ "<h3>Quota Usage (Backup Gigabytes)</h3>"
+ f"<FONT COLOR={quota_color}><h4>Limit: {str(quota['limit'])} GB, In Use: "
+ f"{str(quota['in_use'])} GB, Reserved: {str(quota['reserved'])} GB, Total "
+ f"rate: {str(quota_usage)}</h4></FONT>"
+ "<h3>Success List</h3>"
+ f"<FONT COLOR=GREEN><h4>{success_volumes}</h4></FONT><br>"
+ "<h3>Failed List</h3>"
+ f"<FONT COLOR=RED><h4>{failed_volumes}</h4></FONT><br>"
+ )
self.content += html
subject = f"Staffeln Backup result: {project_id}"
- self.send_result_email(project_id, subject=subject, project_name=project_name)
+ reported = self.send_result_email(project_id, subject=subject,
+ project_name=project_name)
+ if reported:
+ # Record success report
+ self.create_report_record()
return True
diff --git a/staffeln/objects/queue.py b/staffeln/objects/queue.py
index e709ce1..ea88b4d 100644
--- a/staffeln/objects/queue.py
+++ b/staffeln/objects/queue.py
@@ -1,3 +1,5 @@
+from oslo_versionedobjects import fields as ovoo_fields
+
from staffeln.db import api as db_api
from staffeln.objects import base
from staffeln.objects import fields as sfeild
@@ -7,9 +9,10 @@
class Queue(
base.StaffelnPersistentObject, base.StaffelnObject, base.StaffelnObjectDictCompat
):
- VERSION = "1.1"
+ VERSION = "1.2"
# Version 1.0: Initial version
# Version 1.1: Add 'incremental' and 'reason' field
+ # Version 1.2: Add 'created_at' field
dbapi = db_api.get_instance()
@@ -24,6 +27,7 @@
"instance_name": sfeild.StringField(),
"incremental": sfeild.BooleanField(),
"reason": sfeild.StringField(nullable=True),
+ "created_at": ovoo_fields.DateTimeField(),
}
@base.remotable_classmethod