From 33945ab84a7a2e0d68f39c48f2b87ca16e736e18 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Thu, 16 Sep 2021 14:22:53 -0700 Subject: [PATCH 1/2] Code refactor of Azure DNS provider --- octodns/provider/azuredns.py | 542 ++++++++++++++++++++--------------- 1 file changed, 304 insertions(+), 238 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 0b4e1c9..73e4a53 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -722,14 +722,7 @@ class AzureProvider(BaseProvider): ar.value)) for ar in azrecord.txt_records]} - def _data_for_dynamic(self, azrecord): - default = set() - pools = defaultdict(lambda: {'fallback': None, 'values': []}) - rules = [] - typ = _parse_azure_type(azrecord.type) - - # top level profile - root_profile = self._get_tm_profile_by_id(azrecord.target_resource.id) + def _get_geo_endpoints(self, root_profile): if root_profile.traffic_routing_method != 'Geographic': # This record does not use geo fencing, so we skip the Geographic # profile hop; let's pretend to be a geo-profile's only endpoint @@ -738,116 +731,146 @@ class AzureProvider(BaseProvider): target_resource_id=root_profile.id ) geo_ep.target_resource = root_profile - endpoints = [geo_ep] - else: - endpoints = root_profile.endpoints + return [geo_ep] - for geo_ep in endpoints: + return root_profile.endpoints + + def _get_rule_endpoints(self, geo_ep): + if geo_ep.target_resource_id and \ + geo_ep.target_resource.traffic_routing_method == 'Priority': + return sorted( + geo_ep.target_resource.endpoints, key=lambda e: e.priority) + else: + # this geo directly points to a pool containing the default + # so we skip the Priority profile hop and directly use an + # external endpoint or Weighted profile + # let's pretend to be a Priority profile's only endpoint + return [geo_ep] + + def _get_pool_endpoints(self, rule_ep): + if rule_ep.target_resource_id: + # third (and last) level weighted RR profile + return rule_ep.target_resource.endpoints + else: + # single-value pool, so we skip the Weighted profile hop and + # directly use an external endpoint; let's pretend to be a + # Weighted profile's only endpoint + return [rule_ep] + + def _populate_geos(self, geo_map, name, fqdn): + if 'GEO-ME' in geo_map: + # Azure treats Middle East as a separate group, but its part of + # Asia in octoDNS, so we need to remove GEO-ME if GEO-AS is also + # in the list. Throw exception otherwise, which should not happen + # if the profile was generated by octoDNS. + if 'GEO-AS' not in geo_map: + msg = 'Profile={} for record {}: Middle East (GEO-ME) is ' \ + 'not supported by octoDNS. It needs to be either ' \ + 'paired with Asia (GEO-AS) or expanded into individual' \ + 'list of countries.'.format(name, fqdn) + raise AzureException(msg) + geo_map.remove('GEO-ME') + + geos = [] + for code in geo_map: + if code.startswith('GEO-'): + # continent + if code == 'GEO-AP': + # Azure uses Australia/Pacific (AP) instead of Oceania + # https://docs.microsoft.com/en-us/azure/traffic-manager/ + # traffic-manager-geographic-regions + geos.append('OC') + else: + geos.append(code[len('GEO-'):]) + elif '-' in code: + # state + country, province = code.split('-', 1) + country = GeoCodes.country_to_code(country) + geos.append('{}-{}'.format(country, province)) + elif code == 'WORLD': + geos.append(code) + else: + # country + geos.append(GeoCodes.country_to_code(code)) + + return geos + + def _populate_pool_values(self, rule_ep, typ, defaults): + values = [] + for pool_ep in self._get_pool_endpoints(rule_ep): + val = pool_ep.target + if typ == 'CNAME': + val = _check_endswith_dot(val) + + ep_name = pool_ep.name + if ep_name.endswith('--default--'): + defaults.add(val) + ep_name = ep_name[:-len('--default--')] + + values.append({ + 'value': val, + 'weight': pool_ep.weight or 1, + }) + + return values + + def _populate_pools(self, geo_ep, typ, defaults, pools): + rule_endpoints = self._get_rule_endpoints(geo_ep) + rule_pool = None + pool = None + for rule_ep in rule_endpoints: + pool_name = rule_ep.name + + # last/default pool + if pool_name.endswith('--default--'): + defaults.add(rule_ep.target) + if pool_name == '--default--': + # this should be the last one, so let's break here + break + # last pool is a single value pool and its value is same as + # record's default value + pool_name = pool_name[:-len('--default--')] + + # set first priority endpoint as the rule's primary pool + if rule_pool is None: + rule_pool = pool_name + + if pool: + # set current pool as fallback of the previous pool + pool['fallback'] = pool_name + + if pool_name in pools: + # we've already populated this and subsequent pools + break + + # populate the pool from Weighted profile + # these should be leaf node entries with no further nesting + pool = pools[pool_name] + pool['values'] = self._populate_pool_values(rule_ep, typ, defaults) + + return rule_pool + + def _data_for_dynamic(self, azrecord): + typ = _parse_azure_type(azrecord.type) + defaults = set() + pools = defaultdict(lambda: {'fallback': None, 'values': []}) + rules = [] + + # top level profile + root_profile = self._get_tm_profile_by_id(azrecord.target_resource.id) + + # construct rules and, in turn, pools + for geo_ep in self._get_geo_endpoints(root_profile): rule = {} # resolve list of regions geo_map = list(geo_ep.geo_mapping or []) if geo_map and geo_map != ['WORLD']: - if 'GEO-ME' in geo_map: - # Azure treats Middle East as a separate group, but - # its part of Asia in octoDNS, so we need to remove GEO-ME - # if GEO-AS is also in the list - # Throw exception otherwise, it should not happen if the - # profile was generated by octoDNS - if 'GEO-AS' not in geo_map: - msg = 'Profile={} for record {}: '.format( - root_profile.name, azrecord.fqdn) - msg += 'Middle East (GEO-ME) is not supported by ' + \ - 'octoDNS. It needs to be either paired ' + \ - 'with Asia (GEO-AS) or expanded into ' + \ - 'individual list of countries.' - raise AzureException(msg) - geo_map.remove('GEO-ME') - geos = rule.setdefault('geos', []) - for code in geo_map: - if code.startswith('GEO-'): - # continent - if code == 'GEO-AP': - # Azure uses Australia/Pacific (AP) instead of - # Oceania https://docs.microsoft.com/en-us/azure/ - # traffic-manager/ - # traffic-manager-geographic-regions - geos.append('OC') - else: - geos.append(code[len('GEO-'):]) - elif '-' in code: - # state - country, province = code.split('-', 1) - country = GeoCodes.country_to_code(country) - geos.append('{}-{}'.format(country, province)) - elif code == 'WORLD': - geos.append(code) - else: - # country - geos.append(GeoCodes.country_to_code(code)) + rule['geos'] = self._populate_geos( + geo_map, root_profile.name, azrecord.fqdn) - # build fallback chain from second level priority profile - if geo_ep.target_resource_id and \ - geo_ep.target_resource.traffic_routing_method == 'Priority': - rule_endpoints = geo_ep.target_resource.endpoints - rule_endpoints.sort(key=lambda e: e.priority) - else: - # this geo directly points to a pool containing the default - # so we skip the Priority profile hop and directly use an - # external endpoint or Weighted profile - # let's pretend to be a Priority profile's only endpoint - rule_endpoints = [geo_ep] - - pool = None - for rule_ep in rule_endpoints: - pool_name = rule_ep.name - - # last/default pool - if pool_name.endswith('--default--'): - default.add(rule_ep.target) - if pool_name == '--default--': - # this should be the last one, so let's break here - break - # last pool is a single value pool and its value is same - # as record's default value - pool_name = pool_name[:-len('--default--')] - - # set first priority endpoint as the rule's primary pool - if 'pool' not in rule: - rule['pool'] = pool_name - - if pool: - # set current pool as fallback of the previous pool - pool['fallback'] = pool_name - - if pool_name in pools: - # we've already populated this and subsequent pools - break - - # populate the pool from Weighted profile - # these should be leaf node entries with no further nesting - pool = pools[pool_name] - endpoints = [] - - if rule_ep.target_resource_id: - # third (and last) level weighted RR profile - endpoints = rule_ep.target_resource.endpoints - else: - # single-value pool, so we skip the Weighted profile hop - # and directly use an external endpoint; let's pretend to - # be a Weighted profile's only endpoint - endpoints = [rule_ep] - - for pool_ep in endpoints: - val = pool_ep.target - if typ == 'CNAME': - val = _check_endswith_dot(val) - pool['values'].append({ - 'value': val, - 'weight': pool_ep.weight or 1, - }) - if pool_ep.name.endswith('--default--'): - default.add(val) + # build pool fallback chain from second level priority profile + rule['pool'] = self._populate_pools(geo_ep, typ, defaults, pools) rules.append(rule) @@ -859,7 +882,7 @@ class AzureProvider(BaseProvider): rules.append({'pool': rule['pool']}) # Order and convert to a list - default = sorted(default) + defaults = sorted(defaults) data = { 'dynamic': { @@ -869,9 +892,9 @@ class AzureProvider(BaseProvider): } if typ == 'CNAME': - data['value'] = _check_endswith_dot(default[0]) + data['value'] = _check_endswith_dot(defaults[0]) else: - data['values'] = default + data['values'] = defaults return data @@ -978,17 +1001,171 @@ class AzureProvider(BaseProvider): return profile + def _generate_geos(self, rule_geos): + geos = [] + for geo in rule_geos: + if '-' in geo: + # country/state + geos.append(geo.split('-', 1)[-1]) + else: + # continent + if geo == 'AS': + # Middle East is part of Asia in octoDNS, but Azure treats + # it as a separate "group", so let's add it in the list of + # geo mappings. We will drop it when we later parse the + # list of regions. + geos.append('GEO-ME') + elif geo == 'OC': + # Azure uses Australia/Pacific (AP) instead of Oceania + geo = 'AP' + + geos.append('GEO-{}'.format(geo)) + + return geos + + def _generate_pool_profile(self, pool, record, defaults): + pool_name = pool._id + default_seen = False + + endpoints = [] + for val in pool.data['values']: + target = val['value'] + # strip trailing dot from CNAME value + if record._type == 'CNAME': + target = target[:-1] + ep_name = '{}--{}'.format(pool_name, target) + # Endpoint names cannot have colons, drop them from IPv6 addresses + ep_name = ep_name.replace(':', '-') + if target in defaults: + # mark default + ep_name += '--default--' + default_seen = True + endpoints.append(Endpoint( + name=ep_name, + target=target, + weight=val.get('weight', 1), + )) + + pool_profile = self._generate_tm_profile( + 'Weighted', endpoints, record, pool_name) + + return pool_profile, default_seen + + def _generate_pool(self, pool, priority, pool_profiles, record, defaults, + traffic_managers): + pool_name = pool._id + pool_values = pool.data['values'] + default_seen = False + + if len(pool_values) > 1: + # create Weighted profile for multi-value pool + pool_profile = pool_profiles.get(pool_name) + # TODO: what if a cached pool_profile had seen the default + if pool_profile is None: + pool_profile, default_seen = self._generate_pool_profile( + pool, record, defaults) + traffic_managers.append(pool_profile) + pool_profiles[pool_name] = pool_profile + + # append pool to endpoint list of fallback rule profile + return Endpoint( + name=pool_name, + target_resource_id=pool_profile.id, + priority=priority, + ), default_seen + else: + # Skip Weighted profile hop for single-value pool; append its + # value as an external endpoint to fallback rule profile + target = pool_values[0]['value'] + if record._type == 'CNAME': + target = target[:-1] + ep_name = pool_name + if target in defaults: + # mark default + ep_name += '--default--' + default_seen = True + return Endpoint( + name=ep_name, + target=target, + priority=priority, + ), default_seen + + def _pool_fallback_chain(self, pool_name, record, defaults, pool_profiles, + traffic_managers): + rule_endpoints = [] + + priority = 1 + default_seen = False + + while pool_name: + # iterate until we reach end of fallback chain + pool = record.dynamic.pools[pool_name] + + rule_ep, saw_default = self._generate_pool( + pool, priority, pool_profiles, record, defaults, + traffic_managers + ) + rule_endpoints.append(rule_ep) + if saw_default: + default_seen = True + + priority += 1 + pool_name = pool.data.get('fallback') + + # append default endpoint unless it is already included in last pool + # of rule profile + if not default_seen: + rule_endpoints.append(Endpoint( + name='--default--', + target=defaults[0], + priority=priority, + )) + + return rule_endpoints + + def _geo_endpoint(self, rule_name, rule_endpoints, record, geos, + traffic_managers): + if len(rule_endpoints) > 1: + # create rule profile with fallback chain + rule_profile = self._generate_tm_profile( + 'Priority', rule_endpoints, record, rule_name) + traffic_managers.append(rule_profile) + + # append rule profile to top-level geo profile + return Endpoint( + name=rule_name, + target_resource_id=rule_profile.id, + geo_mapping=geos, + ) + else: + # Priority profile has only one endpoint; skip the hop and append + # its only endpoint to the top-level profile + rule_ep = rule_endpoints[0] + if rule_ep.target_resource_id: + # point directly to the Weighted pool profile + return Endpoint( + name=rule_ep.name, + target_resource_id=rule_ep.target_resource_id, + geo_mapping=geos, + ) + else: + # just add the value of single-value pool + return Endpoint( + name=rule_ep.name, + target=rule_ep.target, + geo_mapping=geos, + ) + def _generate_traffic_managers(self, record): traffic_managers = [] - pools = record.dynamic.pools rules = record.dynamic.rules typ = record._type + profile = self._generate_tm_profile if typ == 'CNAME': defaults = [record.value[:-1]] else: defaults = record.values - profile = self._generate_tm_profile # a pool can be re-used only with a world pool, record the pool # to later consolidate it with a geo pool if one exists since we @@ -1002,138 +1179,27 @@ class AzureProvider(BaseProvider): geo_endpoints = [] pool_profiles = {} - for rule in record.dynamic.rules: - pool_name = rule.data['pool'] + for rule in rules: + rule = rule.data + pool_name = rule['pool'] + rule_geos = rule.get('geos', []) + if pool_name == world_pool and world_seen: # this world pool is already mentioned in another geo rule continue # Prepare the list of Traffic manager geos - rule_geos = rule.data.get('geos', []) - geos = [] - for geo in rule_geos: - if '-' in geo: - # country/state - geos.append(geo.split('-', 1)[-1]) - else: - # continent - if geo == 'AS': - # Middle East is part of Asia in octoDNS, but - # Azure treats it as a separate "group", so let's - # add it in the list of geo mappings. We will drop - # it when we later parse the list of regions. - geos.append('GEO-ME') - elif geo == 'OC': - # Azure uses Australia/Pacific (AP) instead of - # Oceania - geo = 'AP' - - geos.append('GEO-{}'.format(geo)) + geos = self._generate_geos(rule_geos) if not geos or pool_name == world_pool: geos.append('WORLD') world_seen = True - rule_endpoints = [] - priority = 1 - default_seen = False + rule_endpoints = self._pool_fallback_chain( + pool_name, record, defaults, pool_profiles, traffic_managers) - while pool_name: - # iterate until we reach end of fallback chain - pool = pools[pool_name].data - if len(pool['values']) > 1: - # create Weighted profile for multi-value pool - pool_profile = pool_profiles.get(pool_name) - if pool_profile is None: - endpoints = [] - for val in pool['values']: - target = val['value'] - # strip trailing dot from CNAME value - if typ == 'CNAME': - target = target[:-1] - ep_name = '{}--{}'.format(pool_name, target) - # Endpoint names cannot have colons, drop them - # from IPv6 addresses - ep_name = ep_name.replace(':', '-') - if target in defaults: - # mark default - ep_name += '--default--' - default_seen = True - endpoints.append(Endpoint( - name=ep_name, - target=target, - weight=val.get('weight', 1), - )) - pool_profile = profile( - 'Weighted', endpoints, record, pool_name) - traffic_managers.append(pool_profile) - pool_profiles[pool_name] = pool_profile - - # append pool to endpoint list of fallback rule profile - rule_endpoints.append(Endpoint( - name=pool_name, - target_resource_id=pool_profile.id, - priority=priority, - )) - else: - # Skip Weighted profile hop for single-value pool - # append its value as an external endpoint to fallback - # rule profile - target = pool['values'][0]['value'] - if typ == 'CNAME': - target = target[:-1] - ep_name = pool_name - if target in defaults: - # mark default - ep_name += '--default--' - default_seen = True - rule_endpoints.append(Endpoint( - name=ep_name, - target=target, - priority=priority, - )) - - priority += 1 - pool_name = pool.get('fallback') - - # append default endpoint unless it is already included in - # last pool of rule profile - if not default_seen: - rule_endpoints.append(Endpoint( - name='--default--', - target=defaults[0], - priority=priority, - )) - - if len(rule_endpoints) > 1: - # create rule profile with fallback chain - rule_profile = profile( - 'Priority', rule_endpoints, record, rule.data['pool']) - traffic_managers.append(rule_profile) - - # append rule profile to top-level geo profile - geo_endpoints.append(Endpoint( - name=rule.data['pool'], - target_resource_id=rule_profile.id, - geo_mapping=geos, - )) - else: - # Priority profile has only one endpoint; skip the hop and - # append its only endpoint to the top-level profile - rule_ep = rule_endpoints[0] - if rule_ep.target_resource_id: - # point directly to the Weighted pool profile - geo_endpoints.append(Endpoint( - name=rule_ep.name, - target_resource_id=rule_ep.target_resource_id, - geo_mapping=geos, - )) - else: - # just add the value of single-value pool - geo_endpoints.append(Endpoint( - name=rule_ep.name, - target=rule_ep.target, - geo_mapping=geos, - )) + geo_endpoints.append(self._geo_endpoint( + pool_name, rule_endpoints, record, geos, traffic_managers + )) if len(geo_endpoints) == 1 and \ geo_endpoints[0].geo_mapping == ['WORLD'] and \ From 9636d108d6710f4d7ad06b62cab486fe2202446f Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Thu, 16 Sep 2021 16:06:31 -0700 Subject: [PATCH 2/2] Bit more refactor of Azure DNS provider --- octodns/provider/azuredns.py | 116 ++++++++++++++++++----------------- 1 file changed, 60 insertions(+), 56 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 73e4a53..33f9393 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -1001,7 +1001,7 @@ class AzureProvider(BaseProvider): return profile - def _generate_geos(self, rule_geos): + def _make_azure_geos(self, rule_geos): geos = [] for geo in rule_geos: if '-' in geo: @@ -1023,7 +1023,7 @@ class AzureProvider(BaseProvider): return geos - def _generate_pool_profile(self, pool, record, defaults): + def _make_pool_profile(self, pool, record, defaults): pool_name = pool._id default_seen = False @@ -1051,8 +1051,8 @@ class AzureProvider(BaseProvider): return pool_profile, default_seen - def _generate_pool(self, pool, priority, pool_profiles, record, defaults, - traffic_managers): + def _make_pool(self, pool, priority, pool_profiles, record, defaults, + traffic_managers): pool_name = pool._id pool_values = pool.data['values'] default_seen = False @@ -1062,7 +1062,7 @@ class AzureProvider(BaseProvider): pool_profile = pool_profiles.get(pool_name) # TODO: what if a cached pool_profile had seen the default if pool_profile is None: - pool_profile, default_seen = self._generate_pool_profile( + pool_profile, default_seen = self._make_pool_profile( pool, record, defaults) traffic_managers.append(pool_profile) pool_profiles[pool_name] = pool_profile @@ -1090,41 +1090,8 @@ class AzureProvider(BaseProvider): priority=priority, ), default_seen - def _pool_fallback_chain(self, pool_name, record, defaults, pool_profiles, - traffic_managers): - rule_endpoints = [] - - priority = 1 - default_seen = False - - while pool_name: - # iterate until we reach end of fallback chain - pool = record.dynamic.pools[pool_name] - - rule_ep, saw_default = self._generate_pool( - pool, priority, pool_profiles, record, defaults, - traffic_managers - ) - rule_endpoints.append(rule_ep) - if saw_default: - default_seen = True - - priority += 1 - pool_name = pool.data.get('fallback') - - # append default endpoint unless it is already included in last pool - # of rule profile - if not default_seen: - rule_endpoints.append(Endpoint( - name='--default--', - target=defaults[0], - priority=priority, - )) - - return rule_endpoints - - def _geo_endpoint(self, rule_name, rule_endpoints, record, geos, - traffic_managers): + def _make_rule_profile(self, rule_endpoints, rule_name, record, geos, + traffic_managers): if len(rule_endpoints) > 1: # create rule profile with fallback chain rule_profile = self._generate_tm_profile( @@ -1156,17 +1123,50 @@ class AzureProvider(BaseProvider): geo_mapping=geos, ) - def _generate_traffic_managers(self, record): - traffic_managers = [] - rules = record.dynamic.rules - typ = record._type - profile = self._generate_tm_profile + def _make_rule(self, pool_name, pool_profiles, record, geos, + traffic_managers): + endpoints = [] + rule_name = pool_name - if typ == 'CNAME': + if record._type == 'CNAME': defaults = [record.value[:-1]] else: defaults = record.values + priority = 1 + default_seen = False + + while pool_name: + # iterate until we reach end of fallback chain + pool = record.dynamic.pools[pool_name] + + rule_ep, saw_default = self._make_pool( + pool, priority, pool_profiles, record, defaults, + traffic_managers + ) + endpoints.append(rule_ep) + if saw_default: + default_seen = True + + priority += 1 + pool_name = pool.data.get('fallback') + + # append default endpoint unless it is already included in last pool + # of rule profile + if not default_seen: + endpoints.append(Endpoint( + name='--default--', + target=defaults[0], + priority=priority, + )) + + return self._make_rule_profile( + endpoints, rule_name, record, geos, traffic_managers + ) + + def _make_geo_rules(self, record): + rules = record.dynamic.rules + # a pool can be re-used only with a world pool, record the pool # to later consolidate it with a geo pool if one exists since we # can't have multiple endpoints with the same target in ATM @@ -1174,10 +1174,11 @@ class AzureProvider(BaseProvider): for rule in rules: if not rule.data.get('geos', []): world_pool = rule.data['pool'] - world_seen = False + traffic_managers = [] geo_endpoints = [] pool_profiles = {} + world_seen = False for rule in rules: rule = rule.data @@ -1189,26 +1190,29 @@ class AzureProvider(BaseProvider): continue # Prepare the list of Traffic manager geos - geos = self._generate_geos(rule_geos) + geos = self._make_azure_geos(rule_geos) if not geos or pool_name == world_pool: geos.append('WORLD') world_seen = True - rule_endpoints = self._pool_fallback_chain( - pool_name, record, defaults, pool_profiles, traffic_managers) - - geo_endpoints.append(self._geo_endpoint( - pool_name, rule_endpoints, record, geos, traffic_managers + geo_endpoints.append(self._make_rule( + pool_name, pool_profiles, record, geos, traffic_managers )) + return geo_endpoints, traffic_managers + + def _generate_traffic_managers(self, record): + geo_endpoints, traffic_managers = self._make_geo_rules(record) + if len(geo_endpoints) == 1 and \ geo_endpoints[0].geo_mapping == ['WORLD'] and \ geo_endpoints[0].target_resource_id: - # Single WORLD rule does not require a Geographic profile, use - # the target profile as the root profile + # Single WORLD rule does not require a Geographic profile, use the + # target profile (which is at the end) as the root profile self._convert_tm_to_root(traffic_managers[-1], record) else: - geo_profile = profile('Geographic', geo_endpoints, record) + geo_profile = self._generate_tm_profile( + 'Geographic', geo_endpoints, record) traffic_managers.append(geo_profile) return traffic_managers