Merge remote-tracking branch 'origin/master' into py3-f-strings

octodns/provider/azuredns.py

@@ -722,14 +722,7 @@ class AzureProvider(BaseProvider):
                                                    ar.value))
                            for ar in azrecord.txt_records]}
 
-    def _data_for_dynamic(self, azrecord):
+    def _get_geo_endpoints(self, root_profile):
-        default = set()
-        pools = defaultdict(lambda: {'fallback': None, 'values': []})
-        rules = []
-        typ = _parse_azure_type(azrecord.type)
-
-        # top level profile
-        root_profile = self._get_tm_profile_by_id(azrecord.target_resource.id)
         if root_profile.traffic_routing_method != 'Geographic':
             # This record does not use geo fencing, so we skip the Geographic
             # profile hop; let's pretend to be a geo-profile's only endpoint
@@ -738,116 +731,146 @@ class AzureProvider(BaseProvider):
                 target_resource_id=root_profile.id
             )
             geo_ep.target_resource = root_profile
-            endpoints = [geo_ep]
+            return [geo_ep]
-        else:
-            endpoints = root_profile.endpoints
 
-        for geo_ep in endpoints:
+        return root_profile.endpoints
+
+    def _get_rule_endpoints(self, geo_ep):
+        if geo_ep.target_resource_id and \
+           geo_ep.target_resource.traffic_routing_method == 'Priority':
+            return sorted(
+                geo_ep.target_resource.endpoints, key=lambda e: e.priority)
+        else:
+            # this geo directly points to a pool containing the default
+            # so we skip the Priority profile hop and directly use an
+            # external endpoint or Weighted profile
+            # let's pretend to be a Priority profile's only endpoint
+            return [geo_ep]
+
+    def _get_pool_endpoints(self, rule_ep):
+        if rule_ep.target_resource_id:
+            # third (and last) level weighted RR profile
+            return rule_ep.target_resource.endpoints
+        else:
+            # single-value pool, so we skip the Weighted profile hop and
+            # directly use an external endpoint; let's pretend to be a
+            # Weighted profile's only endpoint
+            return [rule_ep]
+
+    def _populate_geos(self, geo_map, name, fqdn):
+        if 'GEO-ME' in geo_map:
+            # Azure treats Middle East as a separate group, but its part of
+            # Asia in octoDNS, so we need to remove GEO-ME if GEO-AS is also
+            # in the list. Throw exception otherwise, which should not happen
+            # if the profile was generated by octoDNS.
+            if 'GEO-AS' not in geo_map:
+                msg = f'Profile={name} for record {fqdn}: Middle East ' \
+                    '(GEO-ME) is not supported by octoDNS. It needs to be ' \
+                    'either paired with Asia (GEO-AS) or expanded  into ' \
+                    'individual list of countries.'
+                raise AzureException(msg)
+            geo_map.remove('GEO-ME')
+
+        geos = []
+        for code in geo_map:
+            if code.startswith('GEO-'):
+                # continent
+                if code == 'GEO-AP':
+                    # Azure uses Australia/Pacific (AP) instead of Oceania
+                    # https://docs.microsoft.com/en-us/azure/traffic-manager/
+                    #                      traffic-manager-geographic-regions
+                    geos.append('OC')
+                else:
+                    geos.append(code[len('GEO-'):])
+            elif '-' in code:
+                # state
+                country, province = code.split('-', 1)
+                country = GeoCodes.country_to_code(country)
+                geos.append(f'{country}-{province}')
+            elif code == 'WORLD':
+                geos.append(code)
+            else:
+                # country
+                geos.append(GeoCodes.country_to_code(code))
+
+        return geos
+
+    def _populate_pool_values(self, rule_ep, typ, defaults):
+        values = []
+        for pool_ep in self._get_pool_endpoints(rule_ep):
+            val = pool_ep.target
+            if typ == 'CNAME':
+                val = _check_endswith_dot(val)
+
+            ep_name = pool_ep.name
+            if ep_name.endswith('--default--'):
+                defaults.add(val)
+                ep_name = ep_name[:-len('--default--')]
+
+            values.append({
+                'value': val,
+                'weight': pool_ep.weight or 1,
+            })
+
+        return values
+
+    def _populate_pools(self, geo_ep, typ, defaults, pools):
+        rule_endpoints = self._get_rule_endpoints(geo_ep)
+        rule_pool = None
+        pool = None
+        for rule_ep in rule_endpoints:
+            pool_name = rule_ep.name
+
+            # last/default pool
+            if pool_name.endswith('--default--'):
+                defaults.add(rule_ep.target)
+                if pool_name == '--default--':
+                    # this should be the last one, so let's break here
+                    break
+                # last pool is a single value pool and its value is same as
+                # record's default value
+                pool_name = pool_name[:-len('--default--')]
+
+            # set first priority endpoint as the rule's primary pool
+            if rule_pool is None:
+                rule_pool = pool_name
+
+            if pool:
+                # set current pool as fallback of the previous pool
+                pool['fallback'] = pool_name
+
+            if pool_name in pools:
+                # we've already populated this and subsequent pools
+                break
+
+            # populate the pool from Weighted profile
+            # these should be leaf node entries with no further nesting
+            pool = pools[pool_name]
+            pool['values'] = self._populate_pool_values(rule_ep, typ, defaults)
+
+        return rule_pool
+
+    def _data_for_dynamic(self, azrecord):
+        typ = _parse_azure_type(azrecord.type)
+        defaults = set()
+        pools = defaultdict(lambda: {'fallback': None, 'values': []})
+        rules = []
+
+        # top level profile
+        root_profile = self._get_tm_profile_by_id(azrecord.target_resource.id)
+
+        # construct rules and, in turn, pools
+        for geo_ep in self._get_geo_endpoints(root_profile):
             rule = {}
 
             # resolve list of regions
             geo_map = list(geo_ep.geo_mapping or [])
             if geo_map and geo_map != ['WORLD']:
-                if 'GEO-ME' in geo_map:
+                rule['geos'] = self._populate_geos(
-                    # Azure treats Middle East as a separate group, but
+                    geo_map, root_profile.name, azrecord.fqdn)
-                    # its part of Asia in octoDNS, so we need to remove GEO-ME
-                    # if GEO-AS is also in the list
-                    # Throw exception otherwise, it should not happen if the
-                    # profile was generated by octoDNS
-                    if 'GEO-AS' not in geo_map:
-                        raise AzureException(f'Profile={root_profile.name} '
-                                             f'for record {azrecord.fqdn}: '
-                                             'Middle East (GEO-ME) is not '
-                                             'supported by octoDNS. It needs '
-                                             'to be either paired with Asia '
-                                             '(GEO-AS) or expanded into '
-                                             'individual list of countries.')
-                    geo_map.remove('GEO-ME')
-                geos = rule.setdefault('geos', [])
-                for code in geo_map:
-                    if code.startswith('GEO-'):
-                        # continent
-                        if code == 'GEO-AP':
-                            # Azure uses Australia/Pacific (AP) instead of
-                            # Oceania https://docs.microsoft.com/en-us/azure/
-                            #                                traffic-manager/
-                            #              traffic-manager-geographic-regions
-                            geos.append('OC')
-                        else:
-                            geos.append(code[len('GEO-'):])
-                    elif '-' in code:
-                        # state
-                        country, province = code.split('-', 1)
-                        country = GeoCodes.country_to_code(country)
-                        geos.append(f'{country}-{province}')
-                    elif code == 'WORLD':
-                        geos.append(code)
-                    else:
-                        # country
-                        geos.append(GeoCodes.country_to_code(code))
 
-            # build fallback chain from second level priority profile
+            # build pool fallback chain from second level priority profile
-            if geo_ep.target_resource_id and \
+            rule['pool'] = self._populate_pools(geo_ep, typ, defaults, pools)
-               geo_ep.target_resource.traffic_routing_method == 'Priority':
-                rule_endpoints = geo_ep.target_resource.endpoints
-                rule_endpoints.sort(key=lambda e: e.priority)
-            else:
-                # this geo directly points to a pool containing the default
-                # so we skip the Priority profile hop and directly use an
-                # external endpoint or Weighted profile
-                # let's pretend to be a Priority profile's only endpoint
-                rule_endpoints = [geo_ep]
-
-            pool = None
-            for rule_ep in rule_endpoints:
-                pool_name = rule_ep.name
-
-                # last/default pool
-                if pool_name.endswith('--default--'):
-                    default.add(rule_ep.target)
-                    if pool_name == '--default--':
-                        # this should be the last one, so let's break here
-                        break
-                    # last pool is a single value pool and its value is same
-                    # as record's default value
-                    pool_name = pool_name[:-len('--default--')]
-
-                # set first priority endpoint as the rule's primary pool
-                if 'pool' not in rule:
-                    rule['pool'] = pool_name
-
-                if pool:
-                    # set current pool as fallback of the previous pool
-                    pool['fallback'] = pool_name
-
-                if pool_name in pools:
-                    # we've already populated this and subsequent pools
-                    break
-
-                # populate the pool from Weighted profile
-                # these should be leaf node entries with no further nesting
-                pool = pools[pool_name]
-                endpoints = []
-
-                if rule_ep.target_resource_id:
-                    # third (and last) level weighted RR profile
-                    endpoints = rule_ep.target_resource.endpoints
-                else:
-                    # single-value pool, so we skip the Weighted profile hop
-                    # and directly use an external endpoint; let's pretend to
-                    # be a Weighted profile's only endpoint
-                    endpoints = [rule_ep]
-
-                for pool_ep in endpoints:
-                    val = pool_ep.target
-                    if typ == 'CNAME':
-                        val = _check_endswith_dot(val)
-                    pool['values'].append({
-                        'value': val,
-                        'weight': pool_ep.weight or 1,
-                    })
-                    if pool_ep.name.endswith('--default--'):
-                        default.add(val)
 
             rules.append(rule)
 
@@ -859,7 +882,7 @@ class AzureProvider(BaseProvider):
                 rules.append({'pool': rule['pool']})
 
         # Order and convert to a list
-        default = sorted(default)
+        defaults = sorted(defaults)
 
         data = {
             'dynamic': {
@@ -869,9 +892,9 @@ class AzureProvider(BaseProvider):
         }
 
         if typ == 'CNAME':
-            data['value'] = _check_endswith_dot(default[0])
+            data['value'] = _check_endswith_dot(defaults[0])
         else:
-            data['values'] = default
+            data['values'] = defaults
 
         return data
 
@@ -977,17 +1000,171 @@ class AzureProvider(BaseProvider):
 
         return profile
 
-    def _generate_traffic_managers(self, record):
+    def _make_azure_geos(self, rule_geos):
-        traffic_managers = []
+        geos = []
-        pools = record.dynamic.pools
+        for geo in rule_geos:
-        rules = record.dynamic.rules
+            if '-' in geo:
-        typ = record._type
+                # country/state
+                geos.append(geo.split('-', 1)[-1])
+            else:
+                # continent
+                if geo == 'AS':
+                    # Middle East is part of Asia in octoDNS, but Azure treats
+                    # it as a separate "group", so let's add it in the list of
+                    # geo mappings. We will drop it when we later parse the
+                    # list of regions.
+                    geos.append('GEO-ME')
+                elif geo == 'OC':
+                    # Azure uses Australia/Pacific (AP) instead of Oceania
+                    geo = 'AP'
 
-        if typ == 'CNAME':
+                geos.append(f'GEO-{geo}')
+
+        return geos
+
+    def _make_pool_profile(self, pool, record, defaults):
+        pool_name = pool._id
+        default_seen = False
+
+        endpoints = []
+        for val in pool.data['values']:
+            target = val['value']
+            # strip trailing dot from CNAME value
+            if record._type == 'CNAME':
+                target = target[:-1]
+            ep_name = f'{pool_name}--{target}'
+            # Endpoint names cannot have colons, drop them from IPv6 addresses
+            ep_name = ep_name.replace(':', '-')
+            if target in defaults:
+                # mark default
+                ep_name += '--default--'
+                default_seen = True
+            endpoints.append(Endpoint(
+                name=ep_name,
+                target=target,
+                weight=val.get('weight', 1),
+            ))
+
+        pool_profile = self._generate_tm_profile(
+            'Weighted', endpoints, record, pool_name)
+
+        return pool_profile, default_seen
+
+    def _make_pool(self, pool, priority, pool_profiles, record, defaults,
+                   traffic_managers):
+        pool_name = pool._id
+        pool_values = pool.data['values']
+        default_seen = False
+
+        if len(pool_values) > 1:
+            # create Weighted profile for multi-value pool
+            pool_profile = pool_profiles.get(pool_name)
+            # TODO: what if a cached pool_profile had seen the default
+            if pool_profile is None:
+                pool_profile, default_seen = self._make_pool_profile(
+                    pool, record, defaults)
+                traffic_managers.append(pool_profile)
+                pool_profiles[pool_name] = pool_profile
+
+            # append pool to endpoint list of fallback rule profile
+            return Endpoint(
+                name=pool_name,
+                target_resource_id=pool_profile.id,
+                priority=priority,
+            ), default_seen
+        else:
+            # Skip Weighted profile hop for single-value pool; append its
+            # value as an external endpoint to fallback rule profile
+            target = pool_values[0]['value']
+            if record._type == 'CNAME':
+                target = target[:-1]
+            ep_name = pool_name
+            if target in defaults:
+                # mark default
+                ep_name += '--default--'
+                default_seen = True
+            return Endpoint(
+                name=ep_name,
+                target=target,
+                priority=priority,
+            ), default_seen
+
+    def _make_rule_profile(self, rule_endpoints, rule_name, record, geos,
+                           traffic_managers):
+        if len(rule_endpoints) > 1:
+            # create rule profile with fallback chain
+            rule_profile = self._generate_tm_profile(
+                'Priority', rule_endpoints, record, rule_name)
+            traffic_managers.append(rule_profile)
+
+            # append rule profile to top-level geo profile
+            return Endpoint(
+                name=rule_name,
+                target_resource_id=rule_profile.id,
+                geo_mapping=geos,
+            )
+        else:
+            # Priority profile has only one endpoint; skip the hop and append
+            # its only endpoint to the top-level profile
+            rule_ep = rule_endpoints[0]
+            if rule_ep.target_resource_id:
+                # point directly to the Weighted pool profile
+                return Endpoint(
+                    name=rule_ep.name,
+                    target_resource_id=rule_ep.target_resource_id,
+                    geo_mapping=geos,
+                )
+            else:
+                # just add the value of single-value pool
+                return Endpoint(
+                    name=rule_ep.name,
+                    target=rule_ep.target,
+                    geo_mapping=geos,
+                )
+
+    def _make_rule(self, pool_name, pool_profiles, record, geos,
+                   traffic_managers):
+        endpoints = []
+        rule_name = pool_name
+
+        if record._type == 'CNAME':
             defaults = [record.value[:-1]]
         else:
             defaults = record.values
-        profile = self._generate_tm_profile
+
+        priority = 1
+        default_seen = False
+
+        while pool_name:
+            # iterate until we reach end of fallback chain
+            pool = record.dynamic.pools[pool_name]
+
+            rule_ep, saw_default = self._make_pool(
+                pool, priority, pool_profiles, record, defaults,
+                traffic_managers
+            )
+            endpoints.append(rule_ep)
+            if saw_default:
+                default_seen = True
+
+            priority += 1
+            pool_name = pool.data.get('fallback')
+
+        # append default endpoint unless it is already included in last pool
+        # of rule profile
+        if not default_seen:
+            endpoints.append(Endpoint(
+                name='--default--',
+                target=defaults[0],
+                priority=priority,
+            ))
+
+        return self._make_rule_profile(
+            endpoints, rule_name, record, geos, traffic_managers
+        )
+
+    def _make_geo_rules(self, record):
+        rules = record.dynamic.rules
 
         # a pool can be re-used only with a world pool, record the pool
         # to later consolidate it with a geo pool if one exists since we
@@ -996,152 +1173,45 @@ class AzureProvider(BaseProvider):
         for rule in rules:
             if not rule.data.get('geos', []):
                 world_pool = rule.data['pool']
-        world_seen = False
 
+        traffic_managers = []
         geo_endpoints = []
         pool_profiles = {}
+        world_seen = False
+
+        for rule in rules:
+            rule = rule.data
+            pool_name = rule['pool']
+            rule_geos = rule.get('geos', [])
 
-        for rule in record.dynamic.rules:
-            pool_name = rule.data['pool']
             if pool_name == world_pool and world_seen:
                 # this world pool is already mentioned in another geo rule
                 continue
 
             # Prepare the list of Traffic manager geos
-            rule_geos = rule.data.get('geos', [])
+            geos = self._make_azure_geos(rule_geos)
-            geos = []
-            for geo in rule_geos:
-                if '-' in geo:
-                    # country/state
-                    geos.append(geo.split('-', 1)[-1])
-                else:
-                    # continent
-                    if geo == 'AS':
-                        # Middle East is part of Asia in octoDNS, but
-                        # Azure treats it as a separate "group", so let's
-                        # add it in the list of geo mappings. We will drop
-                        # it when we later parse the list of regions.
-                        geos.append('GEO-ME')
-                    elif geo == 'OC':
-                        # Azure uses Australia/Pacific (AP) instead of
-                        # Oceania
-                        geo = 'AP'
-
-                    geos.append(f'GEO-{geo}')
             if not geos or pool_name == world_pool:
                 geos.append('WORLD')
                 world_seen = True
 
-            rule_endpoints = []
+            geo_endpoints.append(self._make_rule(
-            priority = 1
+                pool_name, pool_profiles, record, geos, traffic_managers
-            default_seen = False
+            ))
 
-            while pool_name:
+        return geo_endpoints, traffic_managers
-                # iterate until we reach end of fallback chain
-                pool = pools[pool_name].data
-                if len(pool['values']) > 1:
-                    # create Weighted profile for multi-value pool
-                    pool_profile = pool_profiles.get(pool_name)
-                    if pool_profile is None:
-                        endpoints = []
-                        for val in pool['values']:
-                            target = val['value']
-                            # strip trailing dot from CNAME value
-                            if typ == 'CNAME':
-                                target = target[:-1]
-                            ep_name = f'{pool_name}--{target}'
-                            # Endpoint names cannot have colons, drop them
-                            # from IPv6 addresses
-                            ep_name = ep_name.replace(':', '-')
-                            if target in defaults:
-                                # mark default
-                                ep_name += '--default--'
-                                default_seen = True
-                            endpoints.append(Endpoint(
-                                name=ep_name,
-                                target=target,
-                                weight=val.get('weight', 1),
-                            ))
-                        pool_profile = profile(
-                            'Weighted', endpoints, record, pool_name)
-                        traffic_managers.append(pool_profile)
-                        pool_profiles[pool_name] = pool_profile
 
-                    # append pool to endpoint list of fallback rule profile
+    def _generate_traffic_managers(self, record):
-                    rule_endpoints.append(Endpoint(
+        geo_endpoints, traffic_managers = self._make_geo_rules(record)
-                        name=pool_name,
-                        target_resource_id=pool_profile.id,
-                        priority=priority,
-                    ))
-                else:
-                    # Skip Weighted profile hop for single-value pool
-                    # append its value as an external endpoint to fallback
-                    # rule profile
-                    target = pool['values'][0]['value']
-                    if typ == 'CNAME':
-                        target = target[:-1]
-                    ep_name = pool_name
-                    if target in defaults:
-                        # mark default
-                        ep_name += '--default--'
-                        default_seen = True
-                    rule_endpoints.append(Endpoint(
-                        name=ep_name,
-                        target=target,
-                        priority=priority,
-                    ))
-
-                priority += 1
-                pool_name = pool.get('fallback')
-
-            # append default endpoint unless it is already included in
-            # last pool of rule profile
-            if not default_seen:
-                rule_endpoints.append(Endpoint(
-                    name='--default--',
-                    target=defaults[0],
-                    priority=priority,
-                ))
-
-            if len(rule_endpoints) > 1:
-                # create rule profile with fallback chain
-                rule_profile = profile(
-                    'Priority', rule_endpoints, record, rule.data['pool'])
-                traffic_managers.append(rule_profile)
-
-                # append rule profile to top-level geo profile
-                geo_endpoints.append(Endpoint(
-                    name=rule.data['pool'],
-                    target_resource_id=rule_profile.id,
-                    geo_mapping=geos,
-                ))
-            else:
-                # Priority profile has only one endpoint; skip the hop and
-                # append its only endpoint to the top-level profile
-                rule_ep = rule_endpoints[0]
-                if rule_ep.target_resource_id:
-                    # point directly to the Weighted pool profile
-                    geo_endpoints.append(Endpoint(
-                        name=rule_ep.name,
-                        target_resource_id=rule_ep.target_resource_id,
-                        geo_mapping=geos,
-                    ))
-                else:
-                    # just add the value of single-value pool
-                    geo_endpoints.append(Endpoint(
-                        name=rule_ep.name,
-                        target=rule_ep.target,
-                        geo_mapping=geos,
-                    ))
 
         if len(geo_endpoints) == 1 and \
            geo_endpoints[0].geo_mapping == ['WORLD'] and \
            geo_endpoints[0].target_resource_id:
-            # Single WORLD rule does not require a Geographic profile, use
+            # Single WORLD rule does not require a Geographic profile, use the
-            # the target profile as the root profile
+            # target profile (which is at the end) as the root profile
             self._convert_tm_to_root(traffic_managers[-1], record)
         else:
-            geo_profile = profile('Geographic', geo_endpoints, record)
+            geo_profile = self._generate_tm_profile(
+                'Geographic', geo_endpoints, record)
             traffic_managers.append(geo_profile)
 
         return traffic_managers