CumulusNetworks-ifupdown2/ifupdown/scheduler.py

#!/usr/bin/python
#
# Copyright 2014 Cumulus Networks, Inc. All rights reserved.
# Author: Roopa Prabhu, roopa@cumulusnetworks.com
#
# ifaceScheduler --
#    interface scheduler
#

from statemanager import *
import ifupdown.ifupdownflags as ifupdownflags
from iface import *
from graph import *
from collections import deque
from collections import OrderedDict
import logging
import traceback
import sys
from graph import *
from collections import deque
from threading import *
from ifupdownbase import *
from ifupdown.utils import utils
from sets import Set

class ifaceSchedulerFlags():
    """ Enumerates scheduler flags """

    INORDER = 0x1
    POSTORDER = 0x2

class ifaceScheduler():
    """ scheduler functions to schedule configuration of interfaces.

    supports scheduling of interfaces serially in plain interface list
    or dependency graph format.

    """

    _STATE_CHECK = True

    _SCHED_STATUS = True

    @classmethod
    def get_sched_status(cls):
        return cls._SCHED_STATUS

    @classmethod
    def set_sched_status(cls, state):
        cls._SCHED_STATUS = state

    @classmethod
    def run_iface_op(cls, ifupdownobj, ifaceobj, op, cenv=None):
        """ Runs sub operation on an interface """
        ifacename = ifaceobj.name

        if ifupdownobj.type and ifupdownobj.type != ifaceobj.type:
            return

        if not ifupdownobj.flags.ADDONS_ENABLE: return
        if op == 'query-checkcurr':
            query_ifaceobj=ifupdownobj.create_n_save_ifaceobjcurr(ifaceobj)
            # If not type bridge vlan and the object does not exist,
            # mark not found and return
            if (not ifupdownobj.link_exists(ifaceobj.name) and
                ifaceobj.type != ifaceType.BRIDGE_VLAN):
                query_ifaceobj.set_state_n_status(ifaceState.from_str(op),
                                                  ifaceStatus.NOTFOUND)
                return
        for mname in ifupdownobj.module_ops.get(op):
            m = ifupdownobj.modules.get(mname)
            err = 0
            try:
                if hasattr(m, 'run'):
                    msg = ('%s: %s : running module %s' %(ifacename, op, mname))
                    if op == 'query-checkcurr':
                        # Dont check curr if the interface object was 
                        # auto generated
                        if (ifaceobj.priv_flags and
                            ifaceobj.priv_flags.NOCONFIG):
                            continue
                        ifupdownobj.logger.debug(msg)
                        m.run(ifaceobj, op, query_ifaceobj,
                              ifaceobj_getfunc=ifupdownobj.get_ifaceobjs)
                    else:
                        ifupdownobj.logger.debug(msg)
                        m.run(ifaceobj, op,
                              ifaceobj_getfunc=ifupdownobj.get_ifaceobjs)
            except Exception, e:
                if not ifupdownobj.ignore_error(str(e)):
                   err = 1
                   ifupdownobj.logger.error(str(e))
                # Continue with rest of the modules
                pass
            finally:
                if err or ifaceobj.status == ifaceStatus.ERROR:
                    ifaceobj.set_state_n_status(ifaceState.from_str(op),
                                                ifaceStatus.ERROR)
                    if 'up' in  op or 'down' in op or 'query-checkcurr' in op:
                        cls.set_sched_status(False)
                else:
                    # Mark success only if the interface was not already
                    # marked with error
                    status = (ifaceobj.status
                              if ifaceobj.status == ifaceStatus.ERROR
                              else ifaceStatus.SUCCESS)
                    ifaceobj.set_state_n_status(ifaceState.from_str(op),
                                                status)

        if ifupdownobj.config.get('addon_scripts_support', '0') == '1':
            # execute /etc/network/ scripts 
            for mname in ifupdownobj.script_ops.get(op, []):
                ifupdownobj.logger.debug('%s: %s : running script %s'
                    %(ifacename, op, mname))
                try:
                    utils.exec_command(mname, env=cenv)
                except Exception, e:
                    ifupdownobj.log_error('%s: %s %s' % (ifacename, op, str(e)))

    @classmethod
    def run_iface_list_ops(cls, ifupdownobj, ifaceobjs, ops):
        """ Runs all operations on a list of interface
            configurations for the same interface
        """

        # minor optimization. If operation is 'down', proceed only
        # if interface exists in the system
        ifacename = ifaceobjs[0].name
        ifupdownobj.logger.info('%s: running ops ...' %ifacename)
        if ('down' in ops[0] and
                ifaceobjs[0].type != ifaceType.BRIDGE_VLAN and
                not ifupdownobj.link_exists(ifacename)):
            ifupdownobj.logger.debug('%s: does not exist' %ifacename)
            # run posthook before you get out of here, so that
            # appropriate cleanup is done
            posthookfunc = ifupdownobj.sched_hooks.get('posthook')
            if posthookfunc:
                for ifaceobj in ifaceobjs:
                    ifaceobj.status = ifaceStatus.SUCCESS
                    posthookfunc(ifupdownobj, ifaceobj, 'down')
            return 
        for op in ops:
            # first run ifupdownobj handlers. This is good enough
            # for the first object in the list
            handler = ifupdownobj.ops_handlers.get(op)
            if handler:
                try:
                    handler(ifupdownobj, ifaceobjs[0])
                except Exception, e:
                    if not ifupdownobj.link_master_slave_ignore_error(str(e)):
                       ifupdownobj.logger.warn('%s: %s'
                                   %(ifaceobjs[0].name, str(e)))
                    pass
            for ifaceobj in ifaceobjs:
                cls.run_iface_op(ifupdownobj, ifaceobj, op,
                    cenv=ifupdownobj.generate_running_env(ifaceobj, op)
                        if ifupdownobj.config.get('addon_scripts_support',
                            '0') == '1' else None)
        posthookfunc = ifupdownobj.sched_hooks.get('posthook')
        if posthookfunc:
            try:
                [posthookfunc(ifupdownobj, ifaceobj, ops[0])
                    for ifaceobj in ifaceobjs]
            except Exception, e:
                ifupdownobj.logger.warn('%s' %str(e))
                pass

    @classmethod
    def _check_upperifaces(cls, ifupdownobj, ifaceobj, ops, parent,
                           followdependents=False):
        """ Check if upperifaces are hanging off us and help caller decide
        if he can proceed with the ops on this device

        Returns True or False indicating the caller to proceed with the
        operation.
        """
        # proceed only for down operation
        if 'down' not in ops[0]:
            return True

        if (ifupdownobj.flags.SCHED_SKIP_CHECK_UPPERIFACES):
            return True

        if (ifupdownflags.flags.FORCE or
                not ifupdownobj.flags.ADDONS_ENABLE or
                (not ifupdownobj.is_ifaceobj_noconfig(ifaceobj) and
                ifupdownobj.config.get('warn_on_ifdown', '0') == '0' and
                not ifupdownflags.flags.ALL)):
            return True

        ulist = ifaceobj.upperifaces
        if not ulist:
            return True

        # Get the list of upper ifaces other than the parent
        tmpulist = ([u for u in ulist if u != parent] if parent
                    else ulist)
        if not tmpulist:
            return True
        # XXX: This is expensive. Find a cheaper way to do this.
        # if any of the upperdevs are present,
        # return false to the caller to skip this interface
        for u in tmpulist:
            if ifupdownobj.link_exists(u):
                if not ifupdownflags.flags.ALL:
                    if ifupdownobj.is_ifaceobj_noconfig(ifaceobj):
                        ifupdownobj.logger.info('%s: skipping interface down,'
                            %ifaceobj.name + ' upperiface %s still around ' %u)
                    else:
                        ifupdownobj.logger.warn('%s: skipping interface down,'
                            %ifaceobj.name + ' upperiface %s still around ' %u)
                return False
        return True

    @classmethod
    def run_iface_graph(cls, ifupdownobj, ifacename, ops, parent=None,
                        order=ifaceSchedulerFlags.POSTORDER,
                        followdependents=True):
        """ runs interface by traversing all nodes rooted at itself """

        # Each ifacename can have a list of iface objects
        ifaceobjs = ifupdownobj.get_ifaceobjs(ifacename)
        if not ifaceobjs:
            raise Exception('%s: not found' %ifacename)

        # Check state of the dependent. If it is already brought up, return
        if (cls._STATE_CHECK and
            (ifaceobjs[0].state == ifaceState.from_str(ops[-1]))):
            ifupdownobj.logger.debug('%s: already processed' %ifacename)
            return

        for ifaceobj in ifaceobjs:
            if not cls._check_upperifaces(ifupdownobj, ifaceobj,
                                          ops, parent, followdependents):
               return

        # If inorder, run the iface first and then its dependents
        if order == ifaceSchedulerFlags.INORDER:
            cls.run_iface_list_ops(ifupdownobj, ifaceobjs, ops)

        for ifaceobj in ifaceobjs:
            # Run lowerifaces or dependents
            dlist = ifaceobj.lowerifaces
            if dlist:
                ifupdownobj.logger.debug('%s: found dependents %s'
                            %(ifacename, str(dlist)))
                try:
                    if not followdependents:
                        # XXX: this is yet another extra step,
                        # but is needed for interfaces that are
                        # implicit dependents. even though we are asked to
                        # not follow dependents, we must follow the ones
                        # that dont have user given config. Because we own them
                        new_dlist = [d for d in dlist
                                    if ifupdownobj.is_iface_noconfig(d)]
                        if new_dlist:
                            cls.run_iface_list(ifupdownobj, new_dlist, ops,
                                           ifacename, order, followdependents,
                                           continueonfailure=False)
                    else:
                        cls.run_iface_list(ifupdownobj, dlist, ops,
                                            ifacename, order,
                                            followdependents,
                                            continueonfailure=False)
                except Exception, e:
                    if (ifupdownobj.ignore_error(str(e))):
                        pass
                    else:
                        # Dont bring the iface up if children did not come up
                        ifaceobj.set_state_n_status(ifaceState.NEW,
                                                ifaceStatus.ERROR)
                        raise
        if order == ifaceSchedulerFlags.POSTORDER:
            cls.run_iface_list_ops(ifupdownobj, ifaceobjs, ops)

    @classmethod
    def run_iface_list(cls, ifupdownobj, ifacenames,
                       ops, parent=None, order=ifaceSchedulerFlags.POSTORDER,
                       followdependents=True, continueonfailure=True):
        """ Runs interface list """

        for ifacename in ifacenames:
            try:
              cls.run_iface_graph(ifupdownobj, ifacename, ops, parent,
                      order, followdependents)
            except Exception, e:
                if continueonfailure:
                    if ifupdownobj.logger.isEnabledFor(logging.DEBUG):
                        traceback.print_tb(sys.exc_info()[2])
                    ifupdownobj.logger.error('%s : %s' %(ifacename, str(e)))
                    pass
                else:
                    if (ifupdownobj.ignore_error(str(e))):
                        pass
                    else:
                        raise Exception('%s : (%s)' %(ifacename, str(e)))

    @classmethod
    def run_iface_graph_upper(cls, ifupdownobj, ifacename, ops, parent=None,
                        followdependents=True, skip_root=False):
        """ runs interface by traversing all nodes rooted at itself """

        # Each ifacename can have a list of iface objects
        ifaceobjs = ifupdownobj.get_ifaceobjs(ifacename)
        if not ifaceobjs:
            raise Exception('%s: not found' %ifacename)

        if (cls._STATE_CHECK and
            (ifaceobjs[0].state == ifaceState.from_str(ops[-1]))):
            ifupdownobj.logger.debug('%s: already processed' %ifacename)
            return

        if not skip_root:
            # run the iface first and then its upperifaces
            cls.run_iface_list_ops(ifupdownobj, ifaceobjs, ops)
        for ifaceobj in ifaceobjs:
            # Run upperifaces
            ulist = ifaceobj.upperifaces
            if ulist:
                ifupdownobj.logger.debug('%s: found upperifaces %s'
                                            %(ifacename, str(ulist)))
                try:
                    cls.run_iface_list_upper(ifupdownobj, ulist, ops,
                                            ifacename,
                                            followdependents,
                                            continueonfailure=True)
                except Exception, e:
                    if (ifupdownobj.ignore_error(str(e))):
                        pass
                    else:
                        raise

    @classmethod
    def run_iface_list_upper(cls, ifupdownobj, ifacenames,
                       ops, parent=None, followdependents=True,
                       continueonfailure=True, skip_root=False):
        """ Runs interface list """

        for ifacename in ifacenames:
            try:
              cls.run_iface_graph_upper(ifupdownobj, ifacename, ops, parent,
                      followdependents, skip_root)
            except Exception, e:
                if ifupdownobj.logger.isEnabledFor(logging.DEBUG):
                    traceback.print_tb(sys.exc_info()[2])
                ifupdownobj.logger.warn('%s : %s' %(ifacename, str(e)))
                pass

    @classmethod
    def _get_valid_upperifaces(cls, ifupdownobj, ifacenames,
                               allupperifacenames):
        """ Recursively find valid upperifaces

        valid upperifaces are:
            - An upperiface which had no user config (example builtin
              interfaces. usually vlan interfaces.)
            - or had config and previously up
            - and interface currently does not exist
            - or is a bridge (because if your upperiface was a bridge
            - u will have to execute up on the bridge
              to enslave the port and apply bridge attributes to the port) """

        upperifacenames = []
        for ifacename in ifacenames:
            # get upperifaces
            ifaceobj = ifupdownobj.get_ifaceobj_first(ifacename)
            if not ifaceobj:
               continue
            ulist = Set(ifaceobj.upperifaces).difference(upperifacenames)
            nulist = []
            for u in ulist:
                uifaceobj = ifupdownobj.get_ifaceobj_first(u)
                if not uifaceobj:
                   continue
                has_config = not (uifaceobj.priv_flags and
                                  uifaceobj.priv_flags.NOCONFIG)
                if (((has_config and ifupdownobj.get_ifaceobjs_saved(u)) or
                     not has_config) and (not ifupdownobj.link_exists(u)
                         # Do this always for a bridge. Note that this is
                         # not done for a vlan aware bridge because,
                         # in the vlan aware bridge case, the bridge module
                         # applies the bridge port configuration on the port
                         # when up is scheduled on the port.
                         or (uifaceobj.link_kind == ifaceLinkKind.BRIDGE))):
                     nulist.append(u)
            upperifacenames.extend(nulist)
        allupperifacenames.extend(upperifacenames)
        if upperifacenames:
            cls._get_valid_upperifaces(ifupdownobj, upperifacenames,
                                       allupperifacenames)
        return

    @classmethod
    def run_upperifaces(cls, ifupdownobj, ifacenames, ops,
                        continueonfailure=True):
        """ Run through valid upperifaces """ 
        upperifaces = []

        cls._get_valid_upperifaces(ifupdownobj, ifacenames, upperifaces)
        if not upperifaces:
           return
        # dump valid upperifaces
        ifupdownobj.logger.debug(upperifaces)
        for u in upperifaces:
            try:
                ifaceobjs = ifupdownobj.get_ifaceobjs(u)
                if not ifaceobjs:
                   continue
                cls.run_iface_list_ops(ifupdownobj, ifaceobjs, ops)
            except Exception, e:
                if continueonfailure:
                    ifupdownobj.logger.warn('%s' %str(e))

    @classmethod
    def _dump_dependency_info(cls, ifupdownobj, ifacenames,
                              dependency_graph=None, indegrees=None):
        ifupdownobj.logger.info('{\n')
        ifupdownobj.logger.info('\nifaceobjs:')
        for iname in ifacenames:
	        iobjs = ifupdownobj.get_ifaceobjs(iname)
	        for iobj in iobjs:
		        iobj.dump(ifupdownobj.logger)
        if (dependency_graph):
            ifupdownobj.logger.info('\nDependency Graph:')
            ifupdownobj.logger.info(dependency_graph)
	    if (indegrees):
		    ifupdownobj.logger.info('\nIndegrees:')
		    ifupdownobj.logger.info(indegrees)
	    ifupdownobj.logger.info('}\n')

    @classmethod
    def get_sorted_iface_list(cls, ifupdownobj, ifacenames, ops,
                              dependency_graph, indegrees=None):
        if len(ifacenames) == 1:
            return ifacenames
        # Get a sorted list of all interfaces
        if not indegrees:
            indegrees = OrderedDict()
            for ifacename in dependency_graph.keys():
                indegrees[ifacename] = ifupdownobj.get_iface_refcnt(ifacename)

        #cls._dump_dependency_info(ifupdownobj, ifacenames,
        #                          dependency_graph, indegrees)

        ifacenames_all_sorted = graph.topological_sort_graphs_all(
                                        dependency_graph, indegrees)
        # if ALL was set, return all interfaces
        if ifupdownflags.flags.ALL:
            return ifacenames_all_sorted

        # else return ifacenames passed as argument in sorted order
        ifacenames_sorted = []
        [ifacenames_sorted.append(ifacename)
                        for ifacename in ifacenames_all_sorted
                            if ifacename in ifacenames]
        return ifacenames_sorted

    @classmethod
    def sched_ifaces(cls, ifupdownobj, ifacenames, ops,
                dependency_graph=None, indegrees=None,
                order=ifaceSchedulerFlags.POSTORDER,
                followdependents=True, skipupperifaces=False, sort=False):
        """ runs interface configuration modules on interfaces passed as
            argument. Runs topological sort on interface dependency graph.

        Args:
            **ifupdownobj** (object): ifupdownMain object 

            **ifacenames** (list): list of interface names

            **ops** : list of operations to perform eg ['pre-up', 'up', 'post-up']

            **dependency_graph** (dict): dependency graph in adjacency list format

        Kwargs:
            **indegrees** (dict): indegree array of the dependency graph

            **order** (int): ifaceSchedulerFlags (POSTORDER, INORDER)

            **followdependents** (bool): follow dependent interfaces if true

            **sort** (bool): sort ifacelist in the case where ALL is not set

        """
        #
        # Algo:
        # if ALL/auto interfaces are specified,
        #   - walk the dependency tree in postorder or inorder depending
        #     on the operation.
        #     (This is to run interfaces correctly in order)
        # else:
        #   - sort iface list if the ifaces belong to a "class"
        #   - else just run iface list in the order they were specified
        #
        # Run any upperifaces if available
        #
        followupperifaces = False
        run_queue = []
        skip_ifacesort = int(ifupdownobj.config.get('skip_ifacesort', '0'))
        if not skip_ifacesort and not indegrees:
            indegrees = OrderedDict()
            for ifacename in dependency_graph.keys():
                indegrees[ifacename] = ifupdownobj.get_iface_refcnt(ifacename)

        if not ifupdownflags.flags.ALL:
            if 'up' in ops[0]:
                # If there is any interface that does not exist, maybe it
                # is a logical interface and we have to followupperifaces
                # when it comes up, so lets get that list.
                if any([True for i in ifacenames
                        if ifupdownobj.must_follow_upperifaces(i)]):
                    followupperifaces = (True if
                                    [i for i in ifacenames
                                        if not ifupdownobj.link_exists(i)]
                                        else False)
            # sort interfaces only if the caller asked to sort
            # and skip_ifacesort is not on.
            if not skip_ifacesort and sort:
                run_queue = cls.get_sorted_iface_list(ifupdownobj, ifacenames,
                                    ops, dependency_graph, indegrees)
                if run_queue and 'up' in ops[0]:
                    run_queue.reverse()
        else:
            # if -a is set, we pick the interfaces
            # that have no parents and use a sorted list of those
            if not skip_ifacesort:
                sorted_ifacenames = cls.get_sorted_iface_list(ifupdownobj,
                                            ifacenames, ops, dependency_graph,
                                            indegrees)
                if sorted_ifacenames:
                    # pick interfaces that user asked
                    # and those that dont have any dependents first
                    [run_queue.append(ifacename)
                        for ifacename in sorted_ifacenames
                            if ifacename in ifacenames and
                            not indegrees.get(ifacename)]
                    ifupdownobj.logger.debug('graph roots (interfaces that ' +
                            'dont have dependents):' + ' %s' %str(run_queue))
                else:
                    ifupdownobj.logger.warn('interface sort returned None')

        # If queue not present, just run interfaces that were asked by the
        # user
        if not run_queue:
            run_queue = list(ifacenames)
            # if we are taking the order of interfaces as specified
            # in the interfaces file, we should reverse the list if we
            # want to down. This can happen if 'skip_ifacesort'
            # is been specified.
            if 'down' in ops[0]:
                run_queue.reverse()

        # run interface list
        cls.run_iface_list(ifupdownobj, run_queue, ops,
                           parent=None, order=order,
                           followdependents=followdependents)
        if not cls.get_sched_status():
            return

        if (not skipupperifaces and
                ifupdownobj.config.get('skip_upperifaces', '0') == '0' and
                ((not ifupdownflags.flags.ALL and followdependents) or
                followupperifaces) and
                'up' in ops[0]):
            # If user had given a set of interfaces to bring up
            # try and execute 'up' on the upperifaces
            ifupdownobj.logger.info('running upperifaces (parent interfaces) ' +
                                    'if available ..')
            try:
                # upperiface bring up is best effort.
                # eg case: if we are bringing up a bridge port
                # this section does an 'ifup on the bridge'
                # so that the recently up'ed bridge port gets enslaved
                # to the bridge. But the up on the bridge may
                # throw out more errors if the bridge is not
                # in the correct state. Lets not surprise
                # the user with such errors when he has
                # only requested to bring up the bridge port.
                cls._STATE_CHECK = False
                ifupdownflags.flags.IGNORE_ERRORS = True
                cls.run_upperifaces(ifupdownobj, ifacenames, ops)
            finally:
                ifupdownflags.flags.IGNORE_ERRORS = False
                cls._STATE_CHECK = True
                # upperiface bringup is best effort, so dont propagate errors
                # reset scheduler status to True
                cls.set_sched_status(True)