Added database modules, migration entrypoint.
diff --git a/etc/staffeln/staffeln.conf b/etc/staffeln/staffeln.conf
old mode 100755
new mode 100644
index e69de29..da23a79
--- a/etc/staffeln/staffeln.conf
+++ b/etc/staffeln/staffeln.conf
@@ -0,0 +1,22 @@
+[conductor]
+# backup_peroid = 1
+# workers = 1
+
+[database]
+backend = sqlalchemy
+connection = "mysql://root:password@localhost:3306/staffeln"
+mysql_engine = InnoDB
+# mysql_sql_mode = TRADITIONAL
+# idle_timeout = 3600
+# min_pool_size = 1
+# max_pool_size = 5
+# max_retries = 10
+# retry_interval = 10
+
+[api]
+; host = 0.0.0.0
+; port = 1234
+# enabled_ssl = false
+# ca_file = <None>
+# ssl_cert_file = <None>
+# ssl_key_file = <None>
diff --git a/requirements.txt b/requirements.txt
index 2fce69b..f0b6bc7 100755
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,4 +10,5 @@
oslo.db>=5.0.0
oslo.config>=8.1.0
oslo.log>=4.4.0 # Apache-2.0
-openstacksdk>0.28.0
\ No newline at end of file
+openstacksdk>0.28.0
+pymysql
\ No newline at end of file
diff --git a/setup.cfg b/setup.cfg
index 2aee593..5466daa 100755
--- a/setup.cfg
+++ b/setup.cfg
@@ -31,5 +31,8 @@
console_scripts =
staffeln-api = staffeln.cmd.api:main
staffeln-conductor = staffeln.cmd.conductor:main
+ staffeln-db-manage = staffeln.cmd.dbmanage:main
wsgi_scripts =
staffeln-api-wsgi = staffeln.api:app
+staffeln.database.migration_backend =
+ sqlalchemy = staffeln.db.sqlalchemy.migration
\ No newline at end of file
diff --git a/staffeln/cmd/dbmanage.py b/staffeln/cmd/dbmanage.py
new file mode 100644
index 0000000..89b69ba
--- /dev/null
+++ b/staffeln/cmd/dbmanage.py
@@ -0,0 +1,52 @@
+"""
+Run storage database migration.
+"""
+
+import sys
+
+from oslo_config import cfg
+
+from staffeln.common import service
+from staffeln import conf
+from staffeln.db import migration
+
+
+CONF = conf.CONF
+
+
+class DBCommand(object):
+
+ @staticmethod
+ def create_schema():
+ migration.create_schema()
+
+
+def add_command_parsers(subparsers):
+
+ parser = subparsers.add_parser(
+ 'create_schema',
+ help="Create the database schema.")
+ parser.set_defaults(func=DBCommand.create_schema)
+
+
+command_opt = cfg.SubCommandOpt('command',
+ title='Command',
+ help='Available commands',
+ handler=add_command_parsers)
+
+
+def register_sub_command_opts():
+ cfg.CONF.register_cli_opt(command_opt)
+
+
+def main():
+ register_sub_command_opts()
+
+ valid_commands = set([
+ 'create_schema',
+ ])
+ if not set(sys.argv).intersection(valid_commands):
+ sys.argv.append('create_schema')
+
+ service.prepare_service(sys.argv)
+ CONF.command.func()
diff --git a/staffeln/conductor/backup.py b/staffeln/conductor/backup.py
index ba43eb9..ea3059a 100755
--- a/staffeln/conductor/backup.py
+++ b/staffeln/conductor/backup.py
@@ -11,4 +11,4 @@
def backup_volumes_in_project(conn, project_name):
# conn.list_servers()
- pass
\ No newline at end of file
+ pass
diff --git a/staffeln/conductor/manager.py b/staffeln/conductor/manager.py
index a33c800..85b8f78 100755
--- a/staffeln/conductor/manager.py
+++ b/staffeln/conductor/manager.py
@@ -50,7 +50,8 @@
for project in projects:
print("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<Project>>>>>>>>>>>>>>>>>>>>>>>>>")
print(project.id)
- servers = conn.list_servers(all_projects=True, filters={"project_id": project.id})
+ servers = conn.list_servers(all_projects=True, filters={
+ "project_id": project.id})
for server in servers:
if not backup.check_vm_backup_metadata(server.metadata):
continue
diff --git a/staffeln/conf/__init__.py b/staffeln/conf/__init__.py
index c934096..8f82b29 100755
--- a/staffeln/conf/__init__.py
+++ b/staffeln/conf/__init__.py
@@ -2,9 +2,12 @@
from staffeln.conf import api
from staffeln.conf import conductor
-
+from staffeln.conf import database
+from staffeln.conf import paths
CONF = cfg.CONF
api.register_opts(CONF)
conductor.register_opts(CONF)
+database.register_opts(CONF)
+paths.register_opts(CONF)
diff --git a/staffeln/conf/database.py b/staffeln/conf/database.py
new file mode 100644
index 0000000..bd8a6df
--- /dev/null
+++ b/staffeln/conf/database.py
@@ -0,0 +1,27 @@
+from oslo_config import cfg
+from oslo_db import options as oslo_db_options
+
+_DEFAULT_SQL_CONNECTION = 'mysql+pymysql://root:password@localhost:3306/staffeln'
+
+database = cfg.OptGroup(
+ 'database',
+ title='Database options',
+ help='Options under this group are used for defining database.'
+)
+
+SQL_OPTS = [
+ cfg.StrOpt('mysql_engine',
+ default='InnoDB',
+ help='MySQL engine to use.'
+ ),
+]
+
+
+def register_opts(conf):
+ oslo_db_options.set_defaults(conf, connection=_DEFAULT_SQL_CONNECTION)
+ conf.register_group(database)
+ conf.register_opts(SQL_OPTS, group=database)
+
+
+def list_opts():
+ return [(database, SQL_OPTS)]
diff --git a/staffeln/db/__init__.py b/staffeln/db/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/staffeln/db/__init__.py
diff --git a/staffeln/db/api.py b/staffeln/db/api.py
new file mode 100644
index 0000000..f2b0285
--- /dev/null
+++ b/staffeln/db/api.py
@@ -0,0 +1,34 @@
+"""Base classes for storage engines"""
+
+import abc
+from oslo_config import cfg
+from oslo_db import api as db_api
+
+_BACKEND_MAPPING = {'sqlalchemy': 'staffeln.db.sqlalchemy.api'}
+IMPL = db_api.DBAPI.from_config(
+ cfg.CONF, backend_mapping=_BACKEND_MAPPING, lazy=True)
+
+
+def get_instance():
+ """Return a DB API instance."""
+ return IMPL
+
+
+class BaseConnection(object, metaclass=abc.ABCMeta):
+ """Base class for storage system connections."""
+
+ @abc.abstractmethod
+ def create_backup(self, values):
+ """Create new backup.
+
+ :param values: A dict containing several items used to add
+ the backup. For example:
+
+ ::
+
+ {
+ 'uuid': short_id.generate_uuid(),
+ 'volume_name': 'Dummy',
+ }
+ :returns: A backup
+ """
diff --git a/staffeln/db/migration.py b/staffeln/db/migration.py
new file mode 100644
index 0000000..bbe05f4
--- /dev/null
+++ b/staffeln/db/migration.py
@@ -0,0 +1,23 @@
+"""Database setup and migration commands."""
+
+
+from oslo_config import cfg
+from stevedore import driver
+import staffeln.conf
+
+CONF = staffeln.conf.CONF
+
+_IMPL = None
+
+
+def get_backend():
+ global _IMPL
+ if not _IMPL:
+ # cfg.CONF.import_opt('backend', 'oslo_db.options', group='database')
+ _IMPL = driver.DriverManager(
+ "staffeln.database.migration_backend", CONF.database.backend).driver
+ return _IMPL
+
+
+def create_schema():
+ return get_backend().create_schema()
diff --git a/staffeln/db/sqlalchemy/__init__.py b/staffeln/db/sqlalchemy/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/staffeln/db/sqlalchemy/__init__.py
diff --git a/staffeln/db/sqlalchemy/alembic.ini b/staffeln/db/sqlalchemy/alembic.ini
new file mode 100644
index 0000000..5dcef31
--- /dev/null
+++ b/staffeln/db/sqlalchemy/alembic.ini
@@ -0,0 +1,54 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = %(here)s/alembic
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+#sqlalchemy.url = driver://user:pass@localhost/dbname
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S
\ No newline at end of file
diff --git a/staffeln/db/sqlalchemy/api.py b/staffeln/db/sqlalchemy/api.py
new file mode 100644
index 0000000..5921417
--- /dev/null
+++ b/staffeln/db/sqlalchemy/api.py
@@ -0,0 +1,77 @@
+"""SQLAlchemy storage backend."""
+
+import collections
+import datetime
+import operator
+
+from oslo_config import cfg
+from oslo_db import exception as db_exc
+from oslo_db.sqlalchemy import session as db_session
+from oslo_db.sqlalchemy import utils as db_utils
+from oslo_utils import timeutils
+from sqlalchemy.inspection import inspect
+from sqlalchemy.orm import exc
+
+
+from staffeln.i18n import _
+from staffeln.common import config
+from staffeln.db import api
+from staffeln.db.sqlalchemy import models
+from staffeln.common import short_id
+
+CONF = cfg.CONF
+
+_FACADE = None
+
+
+def _create_facade_lazily():
+ global _FACADE
+ if _FACADE is None:
+ _FACADE = db_session.EngineFacade.from_config(CONF)
+ return _FACADE
+
+
+def get_engine():
+ facade = _create_facade_lazily()
+ return facade.get_engine()
+
+
+def get_session(**kwargs):
+ facade = _create_facade_lazily()
+ return facade.get_session(**kwargs)
+
+
+def model_query(model, *args, **kwargs):
+ session = kwargs.get('session') or get_session()
+ query = session.query(model, *args)
+ return query
+
+
+class Connection(api.BaseConnection):
+ """SQLAlchemy connection."""
+
+ def __init__(self):
+ super(Connection, self).__init__()
+
+ def _get_relationships(model):
+ return inspect(model).relationships
+
+ def _create(self, model, values):
+ obj = model()
+
+ cleaned_values = {k: v for k, v in values.items()
+ if k not in self._get_relationships(model)}
+ obj.update(cleaned_values)
+ obj.save()
+ return obj
+
+ def create_backup(self, values):
+ # ensure uuid are present for new backup
+ if not values.get('uuid'):
+ values['uuid'] = short_id.generate_id()
+
+ try:
+ backup_data = self._create(models.Backup_data, values)
+ except db_exc.DBDuplicateEntry:
+ pass
+ return goal
diff --git a/staffeln/db/sqlalchemy/migration.py b/staffeln/db/sqlalchemy/migration.py
new file mode 100644
index 0000000..e0d0375
--- /dev/null
+++ b/staffeln/db/sqlalchemy/migration.py
@@ -0,0 +1,27 @@
+import os
+
+import alembic
+from alembic import config as alembic_config
+import alembic.migration as alembic_migration
+from oslo_db import exception as db_exec
+
+from staffeln.i18n import _
+from staffeln.db.sqlalchemy import api as sqla_api
+from staffeln.db.sqlalchemy import models
+
+
+def _alembic_config():
+ path = os.path.join(os.path.dirname(__file__), 'alembic.ini')
+ config = alembic_config.Config(path)
+ return config
+
+
+def create_schema(config=None, engine=None):
+ """Create database schema from models description/
+
+ Can be used for initial installation.
+ """
+ if engine is None:
+ engine = sqla_api.get_engine()
+
+ models.Base.metadata.create_all(engine)
diff --git a/staffeln/db/sqlalchemy/models.py b/staffeln/db/sqlalchemy/models.py
new file mode 100644
index 0000000..df79194
--- /dev/null
+++ b/staffeln/db/sqlalchemy/models.py
@@ -0,0 +1,66 @@
+"""
+SQLAlchemy models for staffeln service
+"""
+
+from oslo_db.sqlalchemy import models
+from oslo_serialization import jsonutils
+from sqlalchemy import Boolean
+from sqlalchemy import Column
+from sqlalchemy import DateTime
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import Float
+from sqlalchemy import ForeignKey
+from sqlalchemy import Integer
+from sqlalchemy import LargeBinary
+from sqlalchemy import Numeric
+from sqlalchemy import orm
+from sqlalchemy import String
+from sqlalchemy import Text
+from sqlalchemy.types import TypeDecorator, TEXT
+from sqlalchemy import UniqueConstraint
+import urllib.parse as urlparse
+from staffeln import conf
+
+CONF = conf.CONF
+
+
+def table_args():
+ engine_name = urlparse.urlparse(CONF.database.connection).scheme
+ if engine_name == 'mysql':
+ return {'mysql_engine': CONF.database.mysql_engine,
+ 'mysql_charset': "utf8"}
+ return None
+
+
+class StaffelnBase(models.SoftDeleteMixin, models.TimestampMixin, models.ModelBase):
+ metadata = None
+
+ def as_dict(self):
+ d = {}
+ for c in self.__table__.columns:
+ d[c.name] = self[c.name]
+ return d
+
+ def save(self, session=None):
+ import staffeln.db.sqlalchemy.api as db_api
+
+ if session is None:
+ session = db_api.get_session()
+
+ super(StaffelnBase, self).save(session)
+
+
+Base = declarative_base(cls=StaffelnBase)
+
+
+class Backup_data(Base):
+ """Represent the backup_data"""
+
+ __tablename__ = 'backup_data'
+ __table_args__ = (
+ UniqueConstraint('uuid', name='unique_backup0uuid'),
+ table_args()
+ )
+ id = Column(Integer, primary_key=True, autoincrement=True)
+ uuid = Column(String(36))
+ volume_name = Column(String(36))
diff --git a/tox.ini b/tox.ini
index 5ed45fa..9676ac4 100755
--- a/tox.ini
+++ b/tox.ini
@@ -1,53 +1,53 @@
[tox]
-minversion = 3.2.0
-envlist = py37,pep8
+envlist = py3,pep8
skipsdist = True
-ignore_basepython_conflict = true
-
+sitepackages = False
+skip_missing_interpreters = True
[testenv]
-basepython = python3
-usedevelop = True
setenv =
+ VIRTUAL_ENV={envdir}
PYTHONWARNINGS=default::DeprecationWarning
- OS_STDOUT_CAPTURE=1
- OS_STDERR_CAPTURE=1
- OS_TEST_TIMEOUT=60
-deps = -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
- -r{toxinidir}/test-requirements.txt
-commands = stestr run {posargs}
+ PYTHONHASHSEED=0
+ TERM=linux
-[testenv:lower-constraints]
-deps = -c{toxinidir}/lower-constraints.txt
- -r{toxinidir}/test-requirements.txt
+deps =
+ flake8
+ -r{toxinidir}/test-requirements.txt
+ -r{toxinidir}/requirements.txt
+ -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
+
+install_commands =
+ pip install {opts} {packages}
+
+
+[testenv:py3]
+basepython = python3
+deps = -r{toxinidir}/test-requirements.txt
+commands = stestr run --slowest {posargs}
[testenv:pep8]
-commands = flake8 {posargs}
-
-[testenv:venv]
-commands = {posargs}
+commands =
+ flake8
[testenv:cover]
+basepython = python3
+deps = -r{toxinidir}/requirements.txt
+ -r{toxinidir}/test-requirements.txt
setenv =
- VIRTUAL_ENV={envdir}
- PYTHON=coverage run --source staffeln --parallel-mode
+ {[testenv]setenv}
+ PYTHON=coverage run
commands =
- stestr run {posargs}
+ coverage erase
+ stestr run --slowest {posargs}
coverage combine
coverage html -d cover
coverage xml -o cover/coverage.xml
+ coverage report
-[testenv:docs]
-deps = -r{toxinidir}/doc/requirements.txt
-commands = sphinx-build -W -b html doc/source doc/build/html
-[testenv:releasenotes]
-deps = {[testenv:docs]deps}
-commands =
- sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html
-
-[testenv:debug]
-commands = oslo_debug_helper {posargs}
+[testenv:venv]
+commands = {posargs}
[flake8]
# E123, E125 skipped as they are invalid PEP-8.