Integrate openstacksdk for backup
diff --git a/.gitignore b/.gitignore
index 4cbb844..73d1eaf 100755
--- a/.gitignore
+++ b/.gitignore
@@ -59,3 +59,4 @@
# Files created by releasenotes build
releasenotes/build
+.idea/
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 4537d1c..2fce69b 100755
--- a/requirements.txt
+++ b/requirements.txt
@@ -9,5 +9,5 @@
gunicorn
oslo.db>=5.0.0
oslo.config>=8.1.0
-oslo.service>=2.5.0
oslo.log>=4.4.0 # Apache-2.0
+openstacksdk>0.28.0
\ No newline at end of file
diff --git a/staffeln/api/__init__.py b/staffeln/api/__init__.py
new file mode 100755
index 0000000..e69de29
--- /dev/null
+++ b/staffeln/api/__init__.py
diff --git a/staffeln/cmd/conductor.py b/staffeln/cmd/conductor.py
index 2dd731d..cbe01a0 100755
--- a/staffeln/cmd/conductor.py
+++ b/staffeln/cmd/conductor.py
@@ -4,7 +4,7 @@
from cotyledon import oslo_config_glue
from staffeln.common import service
-from staffeln import conductor
+from staffeln.conductor import manager
import staffeln.conf
@@ -15,7 +15,9 @@
service.prepare_service()
sm = cotyledon.ServiceManager()
- sm.add(conductor.BackupService,
- workers=CONF.conductor.workers, args=(CONF,))
+ sm.add(manager.BackupManager,
+ workers=CONF.conductor.backup_workers, args=(CONF,))
+ # sm.add(manager.RotationManager,
+ # workers=CONF.conductor.rotation_workers, args=(CONF,))
oslo_config_glue.setup(sm, CONF)
sm.run()
diff --git a/staffeln/common/auth.py b/staffeln/common/auth.py
new file mode 100755
index 0000000..5177098
--- /dev/null
+++ b/staffeln/common/auth.py
@@ -0,0 +1,5 @@
+import openstack
+
+
+def create_connection():
+ return openstack.connect(cloud="envvars")
\ No newline at end of file
diff --git a/staffeln/conductor/__init__.py b/staffeln/conductor/__init__.py
index 94ff7c7..e69de29 100755
--- a/staffeln/conductor/__init__.py
+++ b/staffeln/conductor/__init__.py
@@ -1,37 +0,0 @@
-import cotyledon
-from futurist import periodics
-from oslo_log import log
-import staffeln.conf
-import sys
-import threading
-
-
-LOG = log.getLogger(__name__)
-CONF = staffeln.conf.CONF
-
-
-class BackupService(cotyledon.Service):
- name = "conductor"
-
- def __init__(self, worker_id, conf):
- super(BackupService, self).__init__(worker_id)
- self._shutdown = threading.Event()
- self.conf = conf
- LOG.error("%s init" % self.name)
-
- def run(self):
- LOG.error("%s run" % self.name)
- self._shutdown.wait()
- interval = CONF.conductor.backup_period
- @periodics.periodic(spacing=interval, run_immediately=True)
- def backup_engine():
- print("echo")
- pass
-
- def terminate(self):
- LOG.error("%s terminate" % self.name)
- self._shutdown.set()
- sys.exit(42)
-
- def reload(self):
- LOG.error("%s reload" % self.name)
diff --git a/staffeln/conductor/backup.py b/staffeln/conductor/backup.py
new file mode 100755
index 0000000..ba43eb9
--- /dev/null
+++ b/staffeln/conductor/backup.py
@@ -0,0 +1,14 @@
+import staffeln.conf
+
+
+CONF = staffeln.conf.CONF
+
+
+def check_vm_backup_metadata(metadata):
+ if not CONF.conductor.backup_metadata_key in metadata:
+ return False
+ return metadata[CONF.conductor.backup_metadata_key].lower() in ['true']
+
+def backup_volumes_in_project(conn, project_name):
+ # conn.list_servers()
+ pass
\ No newline at end of file
diff --git a/staffeln/conductor/manage.py b/staffeln/conductor/manage.py
deleted file mode 100755
index 0e7174d..0000000
--- a/staffeln/conductor/manage.py
+++ /dev/null
@@ -1,37 +0,0 @@
-import cotyledon
-from futurist import periodics
-from oslo_log import log
-import staffeln.conf
-import sys
-import threading
-
-
-LOG = log.getLogger(__name__)
-CONF = staffeln.conf.CONF
-
-
-class BackupService(cotyledon.Service):
- name = "conductor"
-
- def __init__(self, worker_id, conf):
- super(BackupService, self).__init__(worker_id)
- self._shutdown = threading.Event()
- self.conf = conf
- LOG.error("%s init" % self.name)
-
- def run(self):
- LOG.error("%s run" % self.name)
- self._shutdown.wait()
- interval = CONF.conductor.backup_period
- @periodics.periodic(spacing=interval, run_immediately=True)
- def backup_engine():
- print("echo")
- pass
-
- def terminate(self):
- LOG.error("%s terminate" % self.name)
- self._shutdown.set()
- sys.exit(42)
-
- def reload(self):
- LOG.error("%s reload" % self.name)
diff --git a/staffeln/conductor/manager.py b/staffeln/conductor/manager.py
new file mode 100755
index 0000000..a33c800
--- /dev/null
+++ b/staffeln/conductor/manager.py
@@ -0,0 +1,97 @@
+import cotyledon
+from futurist import periodics
+from oslo_log import log
+import staffeln.conf
+import sys
+import threading
+import time
+
+from staffeln.common import auth
+from staffeln.conductor import backup
+
+
+LOG = log.getLogger(__name__)
+CONF = staffeln.conf.CONF
+
+
+class BackupManager(cotyledon.Service):
+ name = "Staffeln conductor backup controller"
+
+ def __init__(self, worker_id, conf):
+ super(BackupManager, self).__init__(worker_id)
+ self._shutdown = threading.Event()
+ self.conf = conf
+ LOG.info("%s init" % self.name)
+
+ def run(self):
+ LOG.info("%s run" % self.name)
+ periodic_callables = [
+ (self.backup_engine, (), {}),
+ ]
+ periodic_worker = periodics.PeriodicWorker(periodic_callables)
+ periodic_thread = threading.Thread(
+ target=periodic_worker.start)
+ periodic_thread.daemon = True
+ periodic_thread.start()
+
+ def terminate(self):
+ LOG.info("%s terminate" % self.name)
+ super(BackupManager, self).terminate()
+
+ def reload(self):
+ LOG.info("%s reload" % self.name)
+
+ @periodics.periodic(spacing=CONF.conductor.backup_period, run_immediately=True)
+ def backup_engine(self):
+ print("backing... %s" % str(time.time()))
+ LOG.info("%s periodics" % self.name)
+ conn = auth.create_connection()
+ projects = conn.list_projects()
+ for project in projects:
+ print("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<Project>>>>>>>>>>>>>>>>>>>>>>>>>")
+ print(project.id)
+ servers = conn.list_servers(all_projects=True, filters={"project_id": project.id})
+ for server in servers:
+ if not backup.check_vm_backup_metadata(server.metadata):
+ continue
+ for volume in server.volumes:
+ print("<<<<<<<<<<<Volume>>>>>>>>>>")
+ print(volume)
+ # 1 backup volume
+ conn.create_volume_backup(volume_id=volume.id, force=True)
+ # 2 store backup_id in the database
+
+
+class RotationManager(cotyledon.Service):
+ name = "Staffeln conductor rotation controller"
+
+ def __init__(self, worker_id, conf):
+ super(RotationManager, self).__init__(worker_id)
+ self._shutdown = threading.Event()
+ self.conf = conf
+ LOG.info("%s init" % self.name)
+
+ def run(self):
+ LOG.info("%s run" % self.name)
+ interval = CONF.conductor.rotation_period
+
+ periodic_callables = [
+ (self.rotation_engine, (), {}),
+ ]
+ periodic_worker = periodics.PeriodicWorker(periodic_callables)
+ periodic_thread = threading.Thread(
+ target=periodic_worker.start)
+ periodic_thread.daemon = True
+ periodic_thread.start()
+
+ def terminate(self):
+ LOG.info("%s terminate" % self.name)
+ super(RotationManager, self).terminate()
+
+ def reload(self):
+ LOG.info("%s reload" % self.name)
+
+ @periodics.periodic(spacing=CONF.conductor.rotation_period, run_immediately=True)
+ def rotation_engine(self):
+ print("rotating... %s" % str(time.time()))
+ LOG.info("%s rotation_engine" % self.name)
diff --git a/staffeln/conf/api.py b/staffeln/conf/api.py
index 36baed2..56dab61 100755
--- a/staffeln/conf/api.py
+++ b/staffeln/conf/api.py
@@ -15,7 +15,7 @@
),
cfg.PortOpt(
'port',
- default=8774,
+ default=8808,
help='Staffeln API listens on this port number for incoming requests.'
),
cfg.BoolOpt(
diff --git a/staffeln/conf/conductor.py b/staffeln/conf/conductor.py
index ae89d38..57f9f6f 100755
--- a/staffeln/conf/conductor.py
+++ b/staffeln/conf/conductor.py
@@ -9,20 +9,29 @@
backup_opts = [
cfg.IntOpt(
- 'workers',
+ 'backup_workers',
default=1,
- help='The maximum number of conductor processes to '
+ help='The maximum number of backup processes to '
'fork and run. Default to number of CPUs on the host.'),
cfg.IntOpt(
'backup_period',
- default=1,
+ default=10,
min=1,
help='The time of bakup period, the unit is one minute.'),
+ cfg.StrOpt(
+ 'backup_metadata_key',
+ default="test",
+ help='The key string of metadata the VM, which requres back up, has'),
]
rotation_opts = [
cfg.IntOpt(
- 'rotation_default_period',
+ 'rotation_workers',
+ default=1,
+ help='The maximum number of rotation processes to '
+ 'fork and run. Default to number of CPUs on the host.'),
+ cfg.IntOpt(
+ 'rotation_period',
default=1,
min=1,
help='The time of rotation period, the unit is one day.'),
@@ -34,7 +43,7 @@
def register_opts(conf):
conf.register_group(conductor_group)
conf.register_opts(backup_opts, group=conductor_group)
- conf.register_opts(rotation_opts)
+ conf.register_opts(rotation_opts, group=conductor_group)
def list_opts():