Decide backup method per volume
diff --git a/staffeln/common/openstack.py b/staffeln/common/openstack.py
index 7fb86e7..71bafc9 100644
--- a/staffeln/common/openstack.py
+++ b/staffeln/common/openstack.py
@@ -56,7 +56,15 @@
except exceptions.ResourceNotFound:
return None
- def create_backup(self, volume_id, project_id, force=True, wait=False, name=None, incremental=False):
+ def create_backup(
+ self,
+ volume_id,
+ project_id,
+ force=True,
+ wait=False,
+ name=None,
+ incremental=False,
+ ):
# return conn.block_storage.create_backup(
# volume_id=queue.volume_id, force=True, project_id=queue.project_id, name="name"
# )
@@ -65,7 +73,7 @@
force=force,
wait=wait,
name=name,
- incremental=incremental
+ incremental=incremental,
)
def delete_backup(self, uuid, project_id=None, force=False):
diff --git a/staffeln/conductor/backup.py b/staffeln/conductor/backup.py
index ad2d86b..7d6aa6b 100755
--- a/staffeln/conductor/backup.py
+++ b/staffeln/conductor/backup.py
@@ -17,7 +17,14 @@
BackupMapping = collections.namedtuple(
"BackupMapping",
- ["volume_id", "backup_id", "project_id", "instance_id", "backup_completed"],
+ [
+ "volume_id",
+ "backup_id",
+ "project_id",
+ "instance_id",
+ "backup_completed",
+ "incremental",
+ ],
)
QueueMapping = collections.namedtuple(
@@ -30,6 +37,7 @@
"backup_status",
"instance_name",
"volume_name",
+ "incremental",
],
)
@@ -67,9 +75,9 @@
def refresh_backup_result(self):
self.result.initialize()
- def get_backups(self, filters=None):
+ def get_backups(self, filters=None, **kwargs):
return objects.Volume.list( # pylint: disable=E1120
- context=self.ctx, filters=filters
+ context=self.ctx, filters=filters, **kwargs
)
def get_backup_quota(self, project_id):
@@ -82,15 +90,12 @@
)
return queues
- def create_queue(self, old_tasks, incremental):
+ def create_queue(self, old_tasks):
"""
Create the queue of all the volumes for backup
:param old_tasks: Task list not completed in the previous cycle
:type: List<Class objects.Queue>
-
- :param incremental: If backup is incremental or full
- :type: bool
"""
# 1. get the old task list, not finished in the last cycle
# and keep till now
@@ -102,7 +107,7 @@
queue_list = self.check_instance_volumes()
for queue in queue_list:
if queue.volume_id not in old_task_volume_list:
- self._volume_queue(queue, incremental)
+ self._volume_queue(queue)
# Backup the volumes attached to which has a specific metadata
def filter_by_server_metadata(self, metadata):
@@ -241,6 +246,42 @@
for project in projects:
self.project_list[project.id] = project
+ def _is_incremental(self, volume_id):
+ """
+ Decide the backup method based on the backup history
+
+ It queries to select the last N backups from backup table and
+ decide backup type as full if there is no full backup.
+ N equals to CONF.conductor.full_backup_depth.
+
+ :param volume_id: Target volume id
+ :type: uuid string
+
+ :return: if backup method is incremental or not
+ :return type: bool
+ """
+ # select *from backup order by Id DESC LIMIT 2;
+ try:
+ backups = self.get_backups(
+ filters={"volume_id__eq": volume_id},
+ limit=CONF.conductor.full_backup_depth,
+ sort_key="id",
+ sort_dir="desc",
+ )
+ for bk in backups:
+ if bk.incremental:
+ continue
+ else:
+ return True
+ except Exception as e:
+ LOG.debug(
+ _(
+ "Failed to get backup history to decide backup method. Reason: %s"
+ % str(e)
+ )
+ )
+ return False
+
def check_instance_volumes(self):
"""
Retrieves volume list to backup
@@ -274,7 +315,7 @@
for volume in server.attached_volumes:
if not self.filter_by_volume_status(volume["id"], project.id):
continue
- if "name" not in volume:
+ if "name" not in volume or not volume["name"]:
volume_name = volume["id"]
else:
volume_name = volume["name"][:100]
@@ -289,21 +330,18 @@
# volume_name for forming backup_name
instance_name=server.name[:100],
volume_name=volume_name,
+ incremental=self._is_incremental(volume["id"]),
)
)
return queues_map
- def _volume_queue(self, task, incremental):
+ def _volume_queue(self, task):
"""
Commits one backup task to tasbk queue db table
:param task: One backup task
:type: QueueMapping
-
- :incremental: If backup task is incremental or full
- :type: bool
"""
-
volume_queue = objects.Queue(self.ctx)
volume_queue.backup_id = task.backup_id
volume_queue.volume_id = task.volume_id
@@ -314,8 +352,8 @@
volume_queue.volume_name = task.volume_name
# NOTE(Oleks): Backup mode is inherited from backup service.
# Need to keep and navigate backup mode history, to decide a different mode per volume
- volume_queue.incremental = incremental
- volume_queue.create()
+ volume_queue.incremental = task.incremental
+ return volume_queue.create()
def create_volume_backup(self, task):
"""Initiate the backup of the volume
@@ -345,17 +383,18 @@
self.process_non_existing_backup(task)
return
self.openstacksdk.set_project(self.project_list[project_id])
+ backup_method = "Incremental" if task.incremental else "Full"
LOG.info(
_(
- ("Backup (name: %s) for volume %s creating in project %s")
- % (backup_name, task.volume_id, project_id)
+ ("%s Backup (name: %s) for volume %s creating in project %s")
+ % (backup_method, backup_name, task.volume_id, project_id)
)
)
volume_backup = self.openstacksdk.create_backup(
volume_id=task.volume_id,
project_id=project_id,
name=backup_name,
- incremental=task.incremental
+ incremental=task.incremental,
)
task.backup_id = volume_backup.id
except OpenstackSDKException as error:
@@ -429,9 +468,12 @@
backup_id=task.backup_id,
instance_id=task.instance_id,
backup_completed=1,
+ incremental=task.incremental,
)
)
- self.result.add_success_backup(task.project_id, task.volume_id, task.backup_id, task.incremental)
+ self.result.add_success_backup(
+ task.project_id, task.volume_id, task.backup_id, task.incremental
+ )
# 2. remove from the task list
task.delete_queue()
# 3. TODO(Alex): notify via email
@@ -485,4 +527,5 @@
volume_backup.instance_id = task.instance_id
volume_backup.project_id = task.project_id
volume_backup.backup_completed = task.backup_completed
+ volume_backup.incremental = task.incremental
volume_backup.create()
diff --git a/staffeln/conductor/manager.py b/staffeln/conductor/manager.py
index 9c58001..bd00256 100755
--- a/staffeln/conductor/manager.py
+++ b/staffeln/conductor/manager.py
@@ -95,24 +95,6 @@
return True
return False
- def _is_incremental(self):
- """
- Determine backup mode, whether full or incremental backup
-
- :return: If backup will be incremental or not
- :rtype: bool
- """
-
- LOG.debug(_("Inc count is %s" % self.inc_count))
- if self.inc_count == CONF.conductor.full_backup_depth:
- LOG.info(_("Full backup!"))
- self.inc_count = 0
- return False
- else:
- LOG.info(_("Incremental backup!"))
- self.inc_count += 1
- return True
-
# Create backup generators
def _process_todo_tasks(self):
LOG.info(_("Creating new backup generators..."))
@@ -129,13 +111,13 @@
self.controller.refresh_openstacksdk()
self.controller.refresh_backup_result()
current_tasks = self.controller.get_queues()
- self.controller.create_queue(current_tasks, incremental=self._is_incremental())
+ self.controller.create_queue(current_tasks)
def _report_backup_result(self):
self.controller.publish_backup_result()
def backup_engine(self, backup_service_period):
- LOG.info("backing... %s" % str(time.time()))
+ LOG.info("Backup manager started %s" % str(time.time()))
LOG.info("%s periodics" % self.name)
@periodics.periodic(spacing=backup_service_period, run_immediately=True)
diff --git a/staffeln/db/sqlalchemy/alembic/scriptpy.mako b/staffeln/db/sqlalchemy/alembic/script.py.mako
similarity index 100%
rename from staffeln/db/sqlalchemy/alembic/scriptpy.mako
rename to staffeln/db/sqlalchemy/alembic/script.py.mako
diff --git a/staffeln/db/sqlalchemy/alembic/versions/ebdbed01e9a7_added_incremental_field.py b/staffeln/db/sqlalchemy/alembic/versions/ebdbed01e9a7_added_incremental_field.py
new file mode 100644
index 0000000..e57cd2c
--- /dev/null
+++ b/staffeln/db/sqlalchemy/alembic/versions/ebdbed01e9a7_added_incremental_field.py
@@ -0,0 +1,21 @@
+"""Added incremental field
+
+Revision ID: ebdbed01e9a7
+Revises: 041d9a0f1159
+Create Date: 2022-11-02 01:06:46.021902
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'ebdbed01e9a7'
+down_revision = '041d9a0f1159'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.add_column('backup_data', sa.Column('incremental', sa.Boolean(), nullable=True))
+ op.add_column('queue_data', sa.Column('incremental', sa.Boolean(), nullable=True))
+ # ### end Alembic commands ###
diff --git a/staffeln/db/sqlalchemy/models.py b/staffeln/db/sqlalchemy/models.py
index 6c3deb7..0198ec2 100644
--- a/staffeln/db/sqlalchemy/models.py
+++ b/staffeln/db/sqlalchemy/models.py
@@ -4,7 +4,7 @@
import urllib.parse as urlparse
from oslo_db.sqlalchemy import models
-from sqlalchemy import Column, Integer, String, UniqueConstraint
+from sqlalchemy import Boolean, Column, Integer, String, UniqueConstraint
from sqlalchemy.ext.declarative import declarative_base
from staffeln import conf
@@ -29,7 +29,6 @@
def save(self, session=None):
import staffeln.db.sqlalchemy.api as db_api
-
if session is None:
session = db_api.get_session()
@@ -53,6 +52,7 @@
volume_id = Column(String(100))
instance_id = Column(String(100))
backup_completed = Column(Integer())
+ incremental = Column(Boolean, default=False)
class Queue_data(Base):
@@ -68,3 +68,4 @@
instance_id = Column(String(100))
volume_name = Column(String(100))
instance_name = Column(String(100))
+ incremental = Column(Boolean, default=False)
diff --git a/staffeln/objects/queue.py b/staffeln/objects/queue.py
index 21c041c..5c3c1f9 100644
--- a/staffeln/objects/queue.py
+++ b/staffeln/objects/queue.py
@@ -21,7 +21,7 @@
"backup_status": sfeild.IntegerField(),
"volume_name": sfeild.StringField(),
"instance_name": sfeild.StringField(),
- "incremental": sfeild.BooleanField(default=False),
+ "incremental": sfeild.BooleanField(),
}
@base.remotable_classmethod
@@ -50,7 +50,7 @@
"""Create a :class:`Backup_data` record in the DB"""
values = self.obj_get_changes()
db_queue = self.dbapi.create_queue(values)
- self._from_db_object(self, db_queue)
+ return self._from_db_object(self, db_queue)
@base.remotable
def save(self):
diff --git a/staffeln/objects/volume.py b/staffeln/objects/volume.py
index 0680c78..1f0bdb6 100644
--- a/staffeln/objects/volume.py
+++ b/staffeln/objects/volume.py
@@ -18,15 +18,16 @@
"project_id": sfeild.UUIDField(),
"volume_id": sfeild.UUIDField(),
"backup_completed": sfeild.IntegerField(),
+ "incremental": sfeild.BooleanField(),
}
@base.remotable_classmethod
- def list(cls, context, filters=None): # pylint: disable=E0213
+ def list(cls, context, filters=None, **kwargs): # pylint: disable=E0213
"""Return a list of :class:`Backup` objects.
:param filters: dict mapping the filter to a value.
"""
- db_backups = cls.dbapi.get_backup_list(context, filters=filters)
+ db_backups = cls.dbapi.get_backup_list(context, filters=filters, **kwargs)
return [cls._from_db_object(cls(context), obj) for obj in db_backups]