Provide backup name when create volume backup
The name format will be follow:
$INSTANCE_NAME[:100]-$VOLUME_NAME[:100]-$TIMESTAMP
Also provides db upgrade schema to add instance_name and volume_name to
queue_data table.
Note: upgrade schema with alembic is reference from Magnum.
diff --git a/README.md b/README.md
index d5cdcac..f707938 100755
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
## Project Description
-This solution is a volume-level scheduled backup to implement a non-intrusive automatic backup for Openstack VMs.
+This solution is a volume-level scheduled backup to implement a non-intrusive automatic backup for Openstack VMs.
All volumes attached to the specified VMs are backed up periodically.
diff --git a/staffeln/cmd/dbmanage.py b/staffeln/cmd/dbmanage.py
index 2331261..0ea259e 100644
--- a/staffeln/cmd/dbmanage.py
+++ b/staffeln/cmd/dbmanage.py
@@ -17,12 +17,20 @@
def create_schema():
migration.create_schema()
+ @staticmethod
+ def do_upgrade():
+ migration.upgrade(CONF.command.revision)
+
def add_command_parsers(subparsers):
parser = subparsers.add_parser("create_schema", help="Create the database schema.")
parser.set_defaults(func=DBCommand.create_schema)
+ parser = subparsers.add_parser("upgrade", help="Upgrade the database schema.")
+ parser.add_argument("revision", nargs="?")
+ parser.set_defaults(func=DBCommand.do_upgrade)
+
command_opt = cfg.SubCommandOpt(
"command", title="Command", help="Available commands", handler=add_command_parsers
@@ -39,10 +47,12 @@
valid_commands = set(
[
"create_schema",
+ "do_upgrade",
]
)
if not set(sys.argv).intersection(valid_commands):
sys.argv.append("create_schema")
+ sys.argv.append("do_upgrade")
service.prepare_service(sys.argv)
CONF.command.func()
diff --git a/staffeln/common/openstack.py b/staffeln/common/openstack.py
index 01e5ff5..5c40467 100644
--- a/staffeln/common/openstack.py
+++ b/staffeln/common/openstack.py
@@ -56,14 +56,15 @@
except exceptions.ResourceNotFound:
return None
- def create_backup(self, volume_id, project_id, force=True, wait=False):
+ def create_backup(self, volume_id, project_id, force=True, wait=False, name=None):
# return conn.block_storage.create_backup(
- # volume_id=queue.volume_id, force=True, project_id=queue.project_id,
+ # volume_id=queue.volume_id, force=True, project_id=queue.project_id, name="name"
# )
return self.conn.create_volume_backup(
volume_id=volume_id,
force=force,
wait=wait,
+ name=name,
)
def delete_backup(self, uuid, project_id=None, force=False):
diff --git a/staffeln/conductor/backup.py b/staffeln/conductor/backup.py
index 70db2b5..37a1fbe 100755
--- a/staffeln/conductor/backup.py
+++ b/staffeln/conductor/backup.py
@@ -1,4 +1,5 @@
import collections
+from datetime import datetime
import parse
import staffeln.conf
@@ -21,7 +22,15 @@
QueueMapping = collections.namedtuple(
"QueueMapping",
- ["volume_id", "backup_id", "project_id", "instance_id", "backup_status"],
+ [
+ "volume_id",
+ "backup_id",
+ "project_id",
+ "instance_id",
+ "backup_status",
+ "instance_name",
+ "volume_name",
+ ],
)
@@ -261,6 +270,10 @@
backup_id="NULL",
instance_id=server.id,
backup_status=constants.BACKUP_PLANNED,
+ # Only keep the last 100 chars of instance_name and
+ # volume_name for forming backup_name
+ instance_name=server.name[:100],
+ volume_name=volume["name"][:100],
)
)
return queues_map
@@ -274,6 +287,8 @@
volume_queue.instance_id = task.instance_id
volume_queue.project_id = task.project_id
volume_queue.backup_status = task.backup_status
+ volume_queue.instance_name = task.instance_name
+ volume_queue.volume_name = task.volume_name
volume_queue.create()
def create_volume_backup(self, queue):
@@ -284,6 +299,16 @@
backup_status and backup_id in the queue table.
"""
project_id = queue.project_id
+ timestamp = int(datetime.now().timestamp())
+ # Backup name allows max 255 chars of string
+ backup_name = ("%(instance_name)s-%(volume_name)s-%(timestamp)s") % {
+ "instance_name": queue.instance_name,
+ "volume_name": queue.volume_name,
+ "timestamp": timestamp,
+ }
+
+ # Make sure we don't exceed max size of backup_name
+ backup_name = backup_name[:255]
if queue.backup_id == "NULL":
try:
# NOTE(Alex): no need to wait because we have a cycle time out
@@ -296,20 +321,22 @@
self.openstacksdk.set_project(self.project_list[project_id])
LOG.info(
_(
- "Backup for volume %s creating in project %s"
- % (queue.volume_id, project_id)
+ ("Backup (name: %s) for volume %s creating in project %s")
+ % (backup_name, queue.volume_id, project_id)
)
)
volume_backup = self.openstacksdk.create_backup(
- volume_id=queue.volume_id, project_id=project_id
+ volume_id=queue.volume_id,
+ project_id=project_id,
+ name=backup_name,
)
queue.backup_id = volume_backup.id
queue.backup_status = constants.BACKUP_WIP
queue.save()
except OpenstackSDKException as error:
reason = _(
- "Backup creation for the volume %s failled. %s"
- % (queue.volume_id, str(error))
+ "Backup (name: %s) creation for the volume %s failled. %s"
+ % (backup_name, queue.volume_id, str(error))
)
LOG.info(reason)
self.result.add_failed_backup(project_id, queue.volume_id, reason)
@@ -321,8 +348,8 @@
# Added extra exception as OpenstackSDKException does not handle the keystone unauthourized issue.
except Exception as error:
reason = _(
- "Backup creation for the volume %s failled. %s"
- % (queue.volume_id, str(error))
+ "Backup (name: %s) creation for the volume %s failled. %s"
+ % (backup_name, queue.volume_id, str(error))
)
LOG.error(reason)
self.result.add_failed_backup(project_id, queue.volume_id, reason)
diff --git a/staffeln/db/migration.py b/staffeln/db/migration.py
index c75952d..113116e 100644
--- a/staffeln/db/migration.py
+++ b/staffeln/db/migration.py
@@ -18,3 +18,8 @@
def create_schema():
return get_backend().create_schema()
+
+
+def upgrade(version=None):
+ """Migrate the database to `version` or the most recent version."""
+ return get_backend().upgrade(version)
diff --git a/staffeln/db/sqlalchemy/alembic.ini b/staffeln/db/sqlalchemy/alembic.ini
new file mode 100644
index 0000000..a768980
--- /dev/null
+++ b/staffeln/db/sqlalchemy/alembic.ini
@@ -0,0 +1,54 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = %(here)s/alembic
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+#sqlalchemy.url = driver://user:pass@localhost/dbname
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S
diff --git a/staffeln/db/sqlalchemy/alembic/README b/staffeln/db/sqlalchemy/alembic/README
new file mode 100644
index 0000000..42fcedd
--- /dev/null
+++ b/staffeln/db/sqlalchemy/alembic/README
@@ -0,0 +1,5 @@
+Please see https://alembic.readthedocs.org/en/latest/index.html for general documentation
+
+Upgrade can be performed by:
+$ staffeln-dbmanage upgrade
+$ staffeln-dbmanage upgrade head
diff --git a/staffeln/db/sqlalchemy/alembic/env.py b/staffeln/db/sqlalchemy/alembic/env.py
new file mode 100644
index 0000000..71461fe
--- /dev/null
+++ b/staffeln/db/sqlalchemy/alembic/env.py
@@ -0,0 +1,52 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from logging import config as log_config
+
+from alembic import context
+from staffeln.db.sqlalchemy import api as sqla_api
+from staffeln.db.sqlalchemy import models
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+log_config.fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+target_metadata = models.Base.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_online():
+ """Run migrations in 'online' mode.
+
+ In this scenario we need to create an Engine
+ and associate a connection with the context.
+
+ """
+ engine = sqla_api.get_engine()
+ with engine.connect() as connection:
+ context.configure(connection=connection, target_metadata=target_metadata)
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+run_migrations_online()
diff --git a/staffeln/db/sqlalchemy/alembic/scriptpy.mako b/staffeln/db/sqlalchemy/alembic/scriptpy.mako
new file mode 100644
index 0000000..3b1c960
--- /dev/null
+++ b/staffeln/db/sqlalchemy/alembic/scriptpy.mako
@@ -0,0 +1,18 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision}
+Create Date: ${create_date}
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+def upgrade():
+ ${upgrades if upgrades else "pass"}
diff --git a/staffeln/db/sqlalchemy/alembic/versions/041d9a0f1159_backup_add_names.py b/staffeln/db/sqlalchemy/alembic/versions/041d9a0f1159_backup_add_names.py
new file mode 100644
index 0000000..c6869b2
--- /dev/null
+++ b/staffeln/db/sqlalchemy/alembic/versions/041d9a0f1159_backup_add_names.py
@@ -0,0 +1,34 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""Add volume_name and instance_name to queue_data
+
+Revision ID: 041d9a0f1159
+Revises:
+Create Date: 2022-06-14 20:28:40
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "041d9a0f1159"
+down_revision = ""
+
+import sqlalchemy as sa # noqa: E402
+from alembic import op # noqa: E402
+
+
+def upgrade():
+ op.add_column(
+ "queue_data", sa.Column("volume_name", sa.String(length=100), nullable=True)
+ )
+ op.add_column(
+ "queue_data", sa.Column("instance_name", sa.String(length=100), nullable=True)
+ )
diff --git a/staffeln/db/sqlalchemy/migration.py b/staffeln/db/sqlalchemy/migration.py
index 4c46672..3a34c2b 100644
--- a/staffeln/db/sqlalchemy/migration.py
+++ b/staffeln/db/sqlalchemy/migration.py
@@ -1,6 +1,32 @@
+import os
+
+import staffeln.conf
+from oslo_db.sqlalchemy.migration_cli import manager
from staffeln.db.sqlalchemy import api as sqla_api
from staffeln.db.sqlalchemy import models
+CONF = staffeln.conf.CONF
+_MANAGER = None
+
+
+def get_manager():
+ global _MANAGER
+ if not _MANAGER:
+ alembic_path = os.path.abspath(
+ os.path.join(os.path.dirname(__file__), "alembic.ini")
+ )
+ migrate_path = os.path.abspath(
+ os.path.join(os.path.dirname(__file__), "alembic")
+ )
+ migration_config = {
+ "alembic_ini_path": alembic_path,
+ "alembic_repo_path": migrate_path,
+ "db_url": CONF.database.connection,
+ }
+ _MANAGER = manager.MigrationManager(migration_config)
+
+ return _MANAGER
+
def create_schema(config=None, engine=None):
"""Create database schema from models description/
@@ -11,3 +37,14 @@
engine = sqla_api.get_engine()
models.Base.metadata.create_all(engine)
+
+
+def upgrade(version):
+ """Used for upgrading database.
+
+ :param version: Desired database version
+ :type version: string
+ """
+ version = version or "head"
+
+ get_manager().upgrade(version)
diff --git a/staffeln/db/sqlalchemy/models.py b/staffeln/db/sqlalchemy/models.py
index a93bd6e..6c3deb7 100644
--- a/staffeln/db/sqlalchemy/models.py
+++ b/staffeln/db/sqlalchemy/models.py
@@ -66,3 +66,5 @@
volume_id = Column(String(100))
backup_status = Column(Integer())
instance_id = Column(String(100))
+ volume_name = Column(String(100))
+ instance_name = Column(String(100))
diff --git a/staffeln/objects/queue.py b/staffeln/objects/queue.py
index 8bdebc7..561637c 100644
--- a/staffeln/objects/queue.py
+++ b/staffeln/objects/queue.py
@@ -18,6 +18,8 @@
"volume_id": sfeild.UUIDField(),
"instance_id": sfeild.StringField(),
"backup_status": sfeild.IntegerField(),
+ "volume_name": sfeild.StringField(),
+ "instance_name": sfeild.StringField(),
}
@base.remotable_classmethod