DO-NOT-MERGE: sdn-base to 9.8-stable 46e2320 with only f5_openstack_agent directory

f5_openstack_agent/__init__.py

@@ -1 +1,2 @@
+# actually patched to 46e2320 of 9.8-stable althougn it is 5.0.0 here
 __version__ = "5.0.0"

f5_openstack_agent/lbaasv2/drivers/bigip/agent.py

@@ -16,14 +16,14 @@
 
 import errno
 import inspect
+import service_launcher
 import sys
 
 import f5_openstack_agent.lbaasv2.drivers.bigip.exceptions as exceptions
 
 try:
     from oslo_config import cfg
     from oslo_log import log as oslo_logging
-    from oslo_service import service
 except ImportError as CriticalError:
     frame = inspect.getframeinfo(inspect.currentframe())
     CriticalError = \
@@ -77,14 +77,12 @@ def main():
     """F5 LBaaS agent for OpenStack."""
     cfg.CONF.register_opts(OPTS)
     cfg.CONF.register_opts(manager.OPTS)
-    # pzhang(NOTE): may not be used anywhere, needs to check
     cfg.CONF.register_opts(INTERFACE_OPTS)
 
     config.register_agent_state_opts_helper(cfg.CONF)
     config.register_root_helper(cfg.CONF)
 
     common_config.init(sys.argv[1:])
-    # alias for common_config.setup_logging()...
     config.setup_logging()
 
     mgr = manager.LbaasAgentManager(cfg.CONF)
@@ -94,7 +92,18 @@ def main():
         topic=f5constants.TOPIC_LOADBALANCER_AGENT_V2,
         manager=mgr
     )
-    service.launch(cfg.CONF, svc).wait()
+
+    def handler(*args):
+        LOG.info("receive signal to refresh ESD files!")
+        if mgr.lbdriver.esd_processor:
+            mgr.lbdriver.init_esd()
+            LOG.info("ESD has been refreshed")
+
+    service_launch = service_launcher.F5ServiceLauncher(cfg.CONF)
+
+    service_launch.signal_handler.add_handler('SIGUSR1', handler)
+    service_launch.launch_service(svc)
+    service_launch.wait()
 
 
 if __name__ == '__main__':

f5_openstack_agent/lbaasv2/drivers/bigip/agent_manager.py

@@ -129,6 +129,16 @@
         default=60,
         help=(
             'Amount of time to wait for a errored service to become active')
+    ),
+    cfg.BoolOpt(
+        'password_cipher_mode',
+        default=False,
+        help='The flag indicating the password is plain text or not.'
+    ),
+    cfg.BoolOpt(
+        'esd_auto_refresh',
+        default=True,
+        help='Enable ESD file periodic refresh'
     )
 ]
 
@@ -324,6 +334,9 @@ def __init__(self, conf):
                 self._report_state)
             heartbeat.start(interval=report_interval)
 
+        if self.lbdriver:
+            self.lbdriver.connect()
+
     def _load_driver(self, conf):
         self.lbdriver = None
 
@@ -489,6 +502,13 @@ def connect_driver(self, context):
         if self.lbdriver:
             self.lbdriver.connect()
 
+    @periodic_task.periodic_task(spacing=PERIODIC_TASK_INTERVAL)
+    def refresh_esd(self, context):
+        """Refresh ESD files."""
+        if self.lbdriver.esd_processor and self.conf.esd_auto_refresh:
+            LOG.info("refresh ESD files automatically")
+            self.lbdriver.init_esd()
+
     @periodic_task.periodic_task(spacing=PERIODIC_TASK_INTERVAL)
     def recover_errored_devices(self, context):
         """Try to reconnect to errored devices."""

f5_openstack_agent/lbaasv2/drivers/bigip/esd_filehandler.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 #
 
+import copy
 import glob
 import json
 import os
@@ -31,56 +32,50 @@
 LOG = logging.getLogger(__name__)
 
 
-class EsdJSONValidation(object):
-    """Class reads the json file(s)
+class EsdTagProcessor(object):
+    """Class processes json dictionary
 
-    It checks and parses the content of json file(s) to a dictionary
+    It checks compares the tags from esdjson dictionary to list of valid tags
     """
-    def __init__(self, esddir):
-        self.esdJSONFileList = glob.glob(os.path.join(esddir, '*.json'))
-        self.esdJSONDict = {}
-
-    def read_json(self):
-        for fileList in self.esdJSONFileList:
+    _instance = None
+    esd_dict = {}
+
+    def __new__(cls, *args, **kwargs):
+        if not isinstance(cls._instance, cls):
+            cls._instance = object.__new__(cls, *args, **kwargs)
+        return cls._instance
+
+    def read_json(self, esddir):
+        esdJSONFileList = glob.glob(os.path.join(esddir, '*.json'))
+        esdJSONDict = {}
+        for fileList in esdJSONFileList:
             try:
                 with open(fileList) as json_file:
                     # Reading each file to a dictionary
                     fileJSONDict = json.load(json_file)
                     # Combine all dictionaries to one
-                    self.esdJSONDict.update(fileJSONDict)
+                    esdJSONDict.update(fileJSONDict)
 
             except ValueError as err:
                 LOG.error('ESD JSON File is invalid: %s', err)
                 raise f5_ex.esdJSONFileInvalidException()
 
-        return self.esdJSONDict
-
-
-class EsdTagProcessor(EsdJSONValidation):
-    """Class processes json dictionary
-
-    It checks compares the tags from esdjson dictionary to list of valid tags
-    """
-    def __init__(self, esddir):
-        super(EsdTagProcessor, self).__init__(esddir)
-
-    # this function will return intersection of known valid esd tags
-    # and the ones that user provided
-    def valid_tag_key_subset(self):
-        self.validtags = list(set(self.esdJSONDict.keys()) &
-                              set(self.valid_esd_tags.keys()))
-        if not self.validtags:
-            LOG.error("Intersect of valid esd tags and user esd tags is empty")
-
-        if set(self.validtags) != set(self.esdJSONDict.keys()):
-            LOG.error("invalid tags in the user esd tags")
+        return esdJSONDict
 
-    def process_esd(self, bigips):
+    def process_esd(self, bigips, esddir):
+        esd_json_backup = {}
+        esd_json_backup = copy.deepcopy(self.esd_dict)
         try:
-            dict = self.read_json()
+            dict = self.read_json(esddir)
             self.esd_dict = self.verify_esd_dict(bigips, dict)
         except f5_ex.esdJSONFileInvalidException:
-            self.esd_dict = {}
+            # if error happens, we set backup esd.
+            self.esd_dict = copy.deepcopy(esd_json_backup)
+            LOG.warning(
+                "ESD JSON File is invalid, "
+                "ESD JSON Dict is restored to %s",
+                esd_json_backup
+            )
             raise
 
     def get_esd(self, name):

f5_openstack_agent/lbaasv2/drivers/bigip/icontrol_driver.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 #
 
+import base64
 import datetime
 import hashlib
 import json
@@ -334,7 +335,7 @@ class iControlDriver(LBaaSBaseDriver):
     def __init__(self, conf, registerOpts=True):
         # The registerOpts parameter allows a test to
         # turn off config option handling so that it can
-        # set the options manually instead. """
+        # set the options manually instead.
         super(iControlDriver, self).__init__(conf)
         self.conf = conf
         if registerOpts:
@@ -368,6 +369,7 @@ def __init__(self, conf, registerOpts=True):
 
         # to store the verified esd names
         self.esd_names = []
+        self.esd_processor = None
 
         # service component managers
         self.tenant_manager = None
@@ -389,6 +391,12 @@ def __init__(self, conf, registerOpts=True):
         self.pool_manager = resource_helper.BigIPResourceHelper(
             resource_helper.ResourceType.pool)
 
+        if self.conf.password_cipher_mode:
+            self.conf.icontrol_password = \
+                base64.b64decode(self.conf.icontrol_password)
+            if self.conf.os_password:
+                self.conf.os_password = base64.b64decode(self.conf.os_password)
+
         try:
 
             # debug logging of service requests recieved by driver
@@ -493,6 +501,11 @@ def _init_bigip_managers(self):
         self.system_helper = SystemHelper()
         self.lbaas_builder = LBaaSBuilder(self.conf, self)
 
+        # Set esd_processor object as soon as ServiceModelAdapter and
+        # LBaaSBuilder class instantiated, otherwise manager RPC exception
+        # will break setting esd_porcessor procedure.
+        self.init_esd()
+
         if self.conf.f5_global_routed_mode:
             self.network_builder = None
         else:
@@ -668,7 +681,7 @@ def _init_errored_bigips(self):
             raise
 
     def _open_bigip(self, hostname):
-        # Open bigip connection """
+        # Open bigip connection
         try:
             bigip = self.__bigips[hostname]
             if bigip.status not in ['creating', 'error']:
@@ -824,27 +837,37 @@ def _post_init(self):
         if self.network_builder:
             self.network_builder.post_init()
 
-        # read enhanced services definitions
+        self._set_agent_status(False)
+
+    def init_esd(self):
+        # read all esd file from esd dir
+        # init esd object in lbaas_builder
+        # init esd object in service_adapter
         esd_dir = os.path.join(self.get_config_dir(), 'esd')
-        esd = EsdTagProcessor(esd_dir)
+        # EsdTagProcessor is a singleton, so nothing new
+        self.esd_processor = EsdTagProcessor()
         try:
-            esd.process_esd(self.get_all_bigips())
-            self.lbaas_builder.init_esd(esd)
-            self.service_adapter.init_esd(esd)
-
-            LOG.debug('esd details here after process_esd(): ')
-            LOG.debug(esd)
-            self.esd_names = esd.esd_dict.keys() or []
-            LOG.debug('##### self.esd_names obtainded here:')
-            LOG.debug(self.esd_names)
+            self.esd_processor.process_esd(self.get_all_bigips(), esd_dir)
+            self.lbaas_builder.init_esd(self.esd_processor)
+            self.service_adapter.init_esd(self.esd_processor)
+
         except f5ex.esdJSONFileInvalidException as err:
             LOG.error("unable to initialize ESD. Error: %s.", err.message)
-        self._set_agent_status(False)
+        except IOError as ioe:
+            LOG.error("unable to process ESD file. Error: %s.", ioe.message)
+        except Exception as exc:
+            LOG.error("unknown Error happens. Error: %s.", exc.message)
+
+        LOG.debug('ESD details here after process_esd(): ')
+        LOG.debug(self.esd_processor)
+        self.esd_names = self.esd_processor.esd_dict.keys() or []
+        LOG.debug('self.esd_names obtainded here:')
+        LOG.debug(self.esd_names)
 
     def _validate_ha(self, bigip):
         # if there was only one address supplied and
         # this is not a standalone device, get the
-        # devices trusted by this device. """
+        # devices trusted by this device.
         device_group_name = None
         if self.conf.f5_ha_type == 'standalone':
             if len(self.hostnames) != 1: