From c71784b0e832850f1c535485994976d9d48b29da Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Mon, 12 Apr 2021 08:06:59 +0200 Subject: [PATCH 01/58] initial work on Hetzner provider - implemented HetznerClient API class - tested manually, lacking formal tests --- octodns/provider/hetzner.py | 91 +++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 octodns/provider/hetzner.py diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py new file mode 100644 index 0000000..17dc046 --- /dev/null +++ b/octodns/provider/hetzner.py @@ -0,0 +1,91 @@ +# +# +# + +from requests import Session + + +class HetznerClientException(Exception): + pass + + +class HetznerClientNotFound(HetznerClientException): + + def __init__(self): + super(HetznerClientNotFound, self).__init__('Not Found') + + +class HetznerClient(object): + BASE_URL = 'https://dns.hetzner.com/api/v1' + + def __init__(self, token): + session = Session() + session.headers.update({'Auth-API-Token': token}) + self._session = session + + def _do(self, method, path, params=None, data=None): + url = self.BASE_URL + path + response = self._session.request(method, url, params=params, json=data) + if response.status_code == 404: + raise HetznerClientNotFound() + response.raise_for_status() + return response.json() + + def _do_with_pagination(self, method, path, key, params=None, data=None, + per_page=100): + pagination_params = {'page': 1, 'per_page': per_page} + if params is not None: + params = {**params, **pagination_params} + else: + params = pagination_params + + items = [] + while True: + response = self._do(method, path, params, data) + items += response[key] + if response['meta']['pagination']['page'] >= \ + response['meta']['pagination']['last_page']: + break + params['page'] += 1 + return items + + def zones_get(self, name=None, search_name=None): + params = {'name': name, 'search_name': search_name} + return self._do_with_pagination('GET', '/zones', 'zones', + params=params) + + def zone_get(self, zone_id): + return self._do('GET', '/zones/' + zone_id)['zone'] + + def zone_create(self, name, ttl=None): + data = {'name': name, 'ttl': ttl} + return self._do('POST', '/zones', data=data)['zone'] + + def zone_update(self, zone_id, name, ttl=None): + data = {'name': name, 'ttl': ttl} + return self._do('PUT', '/zones/' + zone_id, data=data)['zone'] + + def zone_delete(self, zone_id): + return self._do('DELETE', '/zones/' + zone_id) + + def zone_records_get(self, zone_id): + params = {'zone_id': zone_id} + # No need to handle pagination as it returns all records by default. + return self._do('GET', '/records', params=params)['records'] + + def zone_record_get(self, record_id): + return self._do('GET', '/records/' + record_id)['record'] + + def zone_record_create(self, zone_id, name, _type, value, ttl=None): + data = {'name': name, 'ttl': ttl, 'type': _type, 'value': value, + 'zone_id': zone_id} + return self._do('POST', '/records', data=data)['record'] + + def zone_record_update(self, zone_id, record_id, name, _type, value, + ttl=None): + data = {'name': name, 'ttl': ttl, 'type': _type, 'value': value, + 'zone_id': zone_id} + return self._do('PUT', '/records/' + record_id, data=data)['record'] + + def zone_record_delete(self, zone_id, record_id): + return self._do('DELETE', '/records/' + record_id) From c8e91c1e11b86106645ac67055b34eec4b6724f4 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Mon, 12 Apr 2021 21:08:53 +0200 Subject: [PATCH 02/58] added HetznerClient docstring --- octodns/provider/hetzner.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 17dc046..1002e7d 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -16,6 +16,15 @@ class HetznerClientNotFound(HetznerClientException): class HetznerClient(object): + ''' + Hetzner DNS Public API v1 client class. + + Zone and Record resources are (almost) fully supported, even if unnecessary + to future-proof this client. Bulk Record create/update is not supported. + + No support for Primary Servers. + ''' + BASE_URL = 'https://dns.hetzner.com/api/v1' def __init__(self, token): From 8a9743b36ec56b81429b0bf9481c0f674a9834a9 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Tue, 13 Apr 2021 08:08:48 +0200 Subject: [PATCH 03/58] added HetznerClient._replace_at to address the "@"/"" record name problem --- octodns/provider/hetzner.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 1002e7d..2e069d0 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -77,24 +77,34 @@ class HetznerClient(object): def zone_delete(self, zone_id): return self._do('DELETE', '/zones/' + zone_id) + def _replace_at(self, record): + record['name'] = '' if record['name'] == '@' else record['name'] + return record + def zone_records_get(self, zone_id): params = {'zone_id': zone_id} # No need to handle pagination as it returns all records by default. - return self._do('GET', '/records', params=params)['records'] + return [ + self._replace_at(record) + for record in self._do('GET', '/records', params=params)['records'] + ] def zone_record_get(self, record_id): - return self._do('GET', '/records/' + record_id)['record'] + record = self._do('GET', '/records/' + record_id)['record'] + return self._replace_at(record) def zone_record_create(self, zone_id, name, _type, value, ttl=None): - data = {'name': name, 'ttl': ttl, 'type': _type, 'value': value, + data = {'name': name or '@', 'ttl': ttl, 'type': _type, 'value': value, 'zone_id': zone_id} - return self._do('POST', '/records', data=data)['record'] + record = self._do('POST', '/records', data=data)['record'] + return self._replace_at(record) def zone_record_update(self, zone_id, record_id, name, _type, value, ttl=None): - data = {'name': name, 'ttl': ttl, 'type': _type, 'value': value, + data = {'name': name or '@', 'ttl': ttl, 'type': _type, 'value': value, 'zone_id': zone_id} - return self._do('PUT', '/records/' + record_id, data=data)['record'] + record = self._do('PUT', '/records/' + record_id, data=data)['record'] + return self._replace_at(record) def zone_record_delete(self, zone_id, record_id): return self._do('DELETE', '/records/' + record_id) From f507349ce506af5554de338a0eb56977b8632a60 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Wed, 14 Apr 2021 07:57:23 +0200 Subject: [PATCH 04/58] implemented HetznerProvider --- octodns/provider/hetzner.py | 254 ++++++++++++++++++++++++++++++++++++ 1 file changed, 254 insertions(+) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 2e069d0..570cefe 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -2,8 +2,17 @@ # # +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +import logging +from collections import defaultdict + from requests import Session +from ..record import Record +from .base import BaseProvider + class HetznerClientException(Exception): pass @@ -108,3 +117,248 @@ class HetznerClient(object): def zone_record_delete(self, zone_id, record_id): return self._do('DELETE', '/records/' + record_id) + + +class HetznerProvider(BaseProvider): + ''' + Hetzner DNS provider using API v1 + + hetzner: + class: octodns.provider.hetzner.HetznerProvider + # Your Hetzner API token (required) + token: foo + ''' + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = False + SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'SRV', 'TXT')) + + def __init__(self, id, token, *args, **kwargs): + self.log = logging.getLogger('HetznerProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, token=***', id) + super(HetznerProvider, self).__init__(id, *args, **kwargs) + self._client = HetznerClient(token) + + self._zone_records = {} + + def _append_dot(self, value): + return value if value[-1] == '.' else '{}.'.format(value) + + def _data_for_multiple(self, _type, records): + values = [record['value'].replace(';', '\\;') for record in records] + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + + def _data_for_CAA(self, _type, records): + values = [] + for record in records: + value_without_spaces = record['value'].replace(' ', '') + flags = value_without_spaces[0] + tag = value_without_spaces[1:].split('"')[0] + value = record['value'].split('"')[1] + values.append({ + 'flags': int(flags), + 'tag': tag, + 'value': value, + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + def _data_for_CNAME(self, _type, records): + record = records[0] + return { + 'ttl': record['ttl'], + 'type': _type, + 'value': self._append_dot(record['value']) + } + + def _data_for_MX(self, _type, records): + values = [] + for record in records: + value_stripped_split = record['value'].strip().split(' ') + preference = value_stripped_split[0] + exchange = value_stripped_split[-1] + values.append({ + 'preference': int(preference), + 'exchange': self._append_dot(exchange) + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + def _data_for_NS(self, _type, records): + values = [] + for record in records: + values.append(self._append_dot(record['value'])) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values, + } + + def _data_for_SRV(self, _type, records): + values = [] + for record in records: + value_stripped = record['value'].strip() + priority = value_stripped.split(' ')[0] + weight = value_stripped[len(priority):].strip().split(' ')[0] + target = value_stripped.split(' ')[-1] + port = value_stripped[:-len(target)].strip().split(' ')[-1] + values.append({ + 'port': int(port), + 'priority': int(priority), + 'target': self._append_dot(target), + 'weight': int(weight) + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + _data_for_TXT = _data_for_multiple + + def zone_records(self, zone): + if zone.name not in self._zone_records: + try: + zone_id = self._client.zones_get(name=zone.name[:-1])[0]['id'] + self._zone_records[zone.name] = \ + self._client.zone_records_get(zone_id) + except HetznerClientNotFound: + return [] + + return self._zone_records[zone.name] + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + + values = defaultdict(lambda: defaultdict(list)) + for record in self.zone_records(zone): + _type = record['type'] + if _type not in self.SUPPORTS: + self.log.warning('populate: skipping unsupported %s record', + _type) + continue + values[record['name']][record['type']].append(record) + + before = len(zone.records) + for name, types in values.items(): + for _type, records in types.items(): + data_for = getattr(self, '_data_for_{}'.format(_type)) + record = Record.new(zone, name, data_for(_type, records), + source=self, lenient=lenient) + zone.add_record(record, lenient=lenient) + + exists = zone.name in self._zone_records + self.log.info('populate: found %s records, exists=%s', + len(zone.records) - before, exists) + return exists + + def _params_for_multiple(self, record): + for value in record.values: + yield { + 'value': value.replace('\\;', ';'), + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + _params_for_A = _params_for_multiple + _params_for_AAAA = _params_for_multiple + + def _params_for_CAA(self, record): + for value in record.values: + data = '{} {} "{}"'.format(value.flags, value.tag, value.value) + yield { + 'value': data, + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + def _params_for_single(self, record): + yield { + 'value': record.value, + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + _params_for_CNAME = _params_for_single + + def _params_for_MX(self, record): + for value in record.values: + data = '{} {}'.format(value.preference, value.exchange) + yield { + 'value': data, + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + _params_for_NS = _params_for_multiple + + def _params_for_SRV(self, record): + for value in record.values: + data = '{} {} {} {}'.format(value.priority, value.weight, + value.port, value.target) + yield { + 'value': data, + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + _params_for_TXT = _params_for_multiple + + def _apply_Create(self, zone_id, change): + new = change.new + params_for = getattr(self, '_params_for_{}'.format(new._type)) + for params in params_for(new): + self._client.zone_record_create(zone_id, params['name'], + params['type'], params['value'], + params['ttl']) + + def _apply_Update(self, zone_id, change): + # It's way simpler to delete-then-recreate than to update + self._apply_Delete(zone_id, change) + self._apply_Create(zone_id, change) + + def _apply_Delete(self, zone_id, change): + existing = change.existing + zone = existing.zone + for record in self.zone_records(zone): + if existing.name == record['name'] and \ + existing._type == record['type']: + self._client.zone_record_delete(zone_id, record['id']) + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + + zone_name = desired.name[:-1] + try: + zone_id = self._client.zones_get(name=zone_name)[0]['id'] + except HetznerClientNotFound: + self.log.debug('_apply: no matching zone, creating domain') + zone_id = self._client.zone_create(zone_name)['id'] + + for change in changes: + class_name = change.__class__.__name__ + getattr(self, '_apply_{}'.format(class_name))(zone_id, change) + + # Clear out the cache if any + self._zone_records.pop(desired.name, None) From 8d1dd926ea6a8241928140e62258221f8636b782 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Wed, 14 Apr 2021 07:59:00 +0200 Subject: [PATCH 05/58] added Hetzner provider to README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 0a26da9..42c91a0 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,7 @@ The above command pulled the existing data out of Route53 and placed the results | [EnvVarSource](/octodns/source/envvar.py) | | TXT | No | read-only environment variable injection | | [GandiProvider](/octodns/provider/gandi.py) | | A, AAAA, ALIAS, CAA, CNAME, DNAME, MX, NS, PTR, SPF, SRV, SSHFP, TXT | No | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | +| [HetznerProvider](/octodns/provider/hetzner.py) | | A, AAAA, CAA, CNAME, MX, NS, SRV, TXT | No | | | [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | | | [Ns1Provider](/octodns/provider/ns1.py) | ns1-python | All | Yes | Missing `NA` geo target | | [OVH](/octodns/provider/ovh.py) | ovh | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT, DKIM | No | | From 1a2cb50c6326b22a870694c51374bd2bb05556fa Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Wed, 21 Apr 2021 07:52:38 +0200 Subject: [PATCH 06/58] fixed potential KeyError when record ttl field is missing --- octodns/provider/hetzner.py | 38 ++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 570cefe..17b1141 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -5,10 +5,9 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -import logging from collections import defaultdict - from requests import Session +import logging from ..record import Record from .base import BaseProvider @@ -24,6 +23,12 @@ class HetznerClientNotFound(HetznerClientException): super(HetznerClientNotFound, self).__init__('Not Found') +class HetznerClientUnauthorized(HetznerClientException): + + def __init__(self): + super(HetznerClientUnauthorized, self).__init__('Unauthorized') + + class HetznerClient(object): ''' Hetzner DNS Public API v1 client class. @@ -44,6 +49,8 @@ class HetznerClient(object): def _do(self, method, path, params=None, data=None): url = self.BASE_URL + path response = self._session.request(method, url, params=params, json=data) + if response.status_code == 401: + raise HetznerClientUnauthorized() if response.status_code == 404: raise HetznerClientNotFound() response.raise_for_status() @@ -139,14 +146,22 @@ class HetznerProvider(BaseProvider): self._client = HetznerClient(token) self._zone_records = {} + self._zone_metadata = {} def _append_dot(self, value): - return value if value[-1] == '.' else '{}.'.format(value) + if value == '@' or value[-1] == '.': + return value + return '{}.'.format(value) + + def _record_ttl(self, record): + if 'ttl' in record: + return record['ttl'] + return self._zone_metadata[record['zone_id']]['ttl'] def _data_for_multiple(self, _type, records): values = [record['value'].replace(';', '\\;') for record in records] return { - 'ttl': records[0]['ttl'], + 'ttl': self._record_ttl(records[0]), 'type': _type, 'values': values } @@ -167,7 +182,7 @@ class HetznerProvider(BaseProvider): 'value': value, }) return { - 'ttl': records[0]['ttl'], + 'ttl': self._record_ttl(records[0]), 'type': _type, 'values': values } @@ -175,7 +190,7 @@ class HetznerProvider(BaseProvider): def _data_for_CNAME(self, _type, records): record = records[0] return { - 'ttl': record['ttl'], + 'ttl': self._record_ttl(record), 'type': _type, 'value': self._append_dot(record['value']) } @@ -191,7 +206,7 @@ class HetznerProvider(BaseProvider): 'exchange': self._append_dot(exchange) }) return { - 'ttl': records[0]['ttl'], + 'ttl': self._record_ttl(records[0]), 'type': _type, 'values': values } @@ -201,7 +216,7 @@ class HetznerProvider(BaseProvider): for record in records: values.append(self._append_dot(record['value'])) return { - 'ttl': records[0]['ttl'], + 'ttl': self._record_ttl(records[0]), 'type': _type, 'values': values, } @@ -221,7 +236,7 @@ class HetznerProvider(BaseProvider): 'weight': int(weight) }) return { - 'ttl': records[0]['ttl'], + 'ttl': self._record_ttl(records[0]), 'type': _type, 'values': values } @@ -231,9 +246,10 @@ class HetznerProvider(BaseProvider): def zone_records(self, zone): if zone.name not in self._zone_records: try: - zone_id = self._client.zones_get(name=zone.name[:-1])[0]['id'] + zone_metadata = self._client.zones_get(name=zone.name[:-1])[0] + self._zone_metadata[zone_metadata['id']] = zone_metadata self._zone_records[zone.name] = \ - self._client.zone_records_get(zone_id) + self._client.zone_records_get(zone_metadata['id']) except HetznerClientNotFound: return [] From ab436af92d42643eac3796ee58a8109be737ace9 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Wed, 21 Apr 2021 07:55:18 +0200 Subject: [PATCH 07/58] added populate() tests --- tests/fixtures/hetzner-records.json | 223 +++++++++++++++++++++++++ tests/fixtures/hetzner-zones.json | 43 +++++ tests/test_octodns_provider_hetzner.py | 84 ++++++++++ 3 files changed, 350 insertions(+) create mode 100644 tests/fixtures/hetzner-records.json create mode 100644 tests/fixtures/hetzner-zones.json create mode 100644 tests/test_octodns_provider_hetzner.py diff --git a/tests/fixtures/hetzner-records.json b/tests/fixtures/hetzner-records.json new file mode 100644 index 0000000..bbafdcb --- /dev/null +++ b/tests/fixtures/hetzner-records.json @@ -0,0 +1,223 @@ +{ + "records": [ + { + "id": "SOA", + "type": "SOA", + "name": "@", + "value": "hydrogen.ns.hetzner.com. dns.hetzner.com. 1 86400 10800 3600000 3600", + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "NS:sub:0", + "type": "NS", + "name": "sub", + "value": "6.2.3.4", + "ttl": 3600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "NS:sub:1", + "type": "NS", + "name": "sub", + "value": "7.2.3.4", + "ttl": 3600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "SRV:_srv._tcp:0", + "type": "SRV", + "name": "_srv._tcp", + "value": "10 20 30 foo-1.unit.tests", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "SRV:_srv._tcp:1", + "type": "SRV", + "name": "_srv._tcp", + "value": "12 20 30 foo-2.unit.tests", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "TXT:txt:0", + "type": "TXT", + "name": "txt", + "value": "\"Bah bah black sheep\"", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "TXT:txt:1", + "type": "TXT", + "name": "txt", + "value": "\"have you any wool.\"", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "A:@:0", + "type": "A", + "name": "@", + "value": "1.2.3.4", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "A:@:1", + "type": "A", + "name": "@", + "value": "1.2.3.5", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "A:www:0", + "type": "A", + "name": "www", + "value": "2.2.3.6", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "MX:mx:0", + "type": "MX", + "name": "mx", + "value": "10 smtp-4.unit.tests", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "MX:mx:1", + "type": "MX", + "name": "mx", + "value": "20 smtp-2.unit.tests", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "MX:mx:2", + "type": "MX", + "name": "mx", + "value": "30 smtp-3.unit.tests", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "MX:mx:3", + "type": "MX", + "name": "mx", + "value": "40 smtp-1.unit.tests", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "AAAA:aaaa:0", + "type": "AAAA", + "name": "aaaa", + "value": "2601:644:500:e210:62f8:1dff:feb8:947a", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "CNAME:cname:0", + "type": "CNAME", + "name": "cname", + "value": "unit.tests", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "A:www.sub:0", + "type": "A", + "name": "www.sub", + "value": "2.2.3.6", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "TXT:txt:2", + "type": "TXT", + "name": "txt", + "value": "v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "CAA:@:0", + "type": "CAA", + "name": "@", + "value": "0 issue \"ca.unit.tests\"", + "ttl": 3600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "CNAME:included:0", + "type": "CNAME", + "name": "included", + "value": "unit.tests", + "ttl": 3600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "SRV:_imap._tcp:0", + "type": "SRV", + "name": "_imap._tcp", + "value": "0 0 0 .", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "SRV:_pop3._tcp:0", + "type": "SRV", + "name": "_pop3._tcp", + "value": "0 0 0 .", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + } + ] +} diff --git a/tests/fixtures/hetzner-zones.json b/tests/fixtures/hetzner-zones.json new file mode 100644 index 0000000..4d9b897 --- /dev/null +++ b/tests/fixtures/hetzner-zones.json @@ -0,0 +1,43 @@ +{ + "zones": [ + { + "id": "unit.tests", + "name": "unit.tests", + "ttl": 3600, + "registrar": "", + "legacy_dns_host": "", + "legacy_ns": [], + "ns": [], + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "verified": "", + "modified": "0000-00-00 00:00:00.000 +0000 UTC", + "project": "", + "owner": "", + "permission": "", + "zone_type": { + "id": "", + "name": "", + "description": "", + "prices": null + }, + "status": "verified", + "paused": false, + "is_secondary_dns": false, + "txt_verification": { + "name": "", + "token": "" + }, + "records_count": null + } + ], + "meta": { + "pagination": { + "page": 1, + "per_page": 100, + "previous_page": 1, + "next_page": 1, + "last_page": 1, + "total_entries": 1 + } + } +} diff --git a/tests/test_octodns_provider_hetzner.py b/tests/test_octodns_provider_hetzner.py new file mode 100644 index 0000000..bf619fb --- /dev/null +++ b/tests/test_octodns_provider_hetzner.py @@ -0,0 +1,84 @@ +# +# +# + + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from os.path import dirname, join +from requests import HTTPError +from requests_mock import ANY, mock as requests_mock +from six import text_type +from unittest import TestCase + +from octodns.provider.hetzner import HetznerProvider +from octodns.provider.yaml import YamlProvider +from octodns.zone import Zone + + +class TestdHetznerProvider(TestCase): + expected = Zone('unit.tests.', []) + source = YamlProvider('test', join(dirname(__file__), 'config')) + source.populate(expected) + + def test_populate(self): + provider = HetznerProvider('test', 'token') + + # Bad auth + with requests_mock() as mock: + mock.get(ANY, status_code=401, + text='{"message":"Invalid authentication credentials"}') + + with self.assertRaises(Exception) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals('Unauthorized', text_type(ctx.exception)) + + # General error + with requests_mock() as mock: + mock.get(ANY, status_code=502, text='Things caught fire') + + with self.assertRaises(HTTPError) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(502, ctx.exception.response.status_code) + + # Non-existent zone doesn't populate anything + with requests_mock() as mock: + mock.get(ANY, status_code=404, + text='{"zone":{"id":"","name":"","ttl":0,"registrar":"",' + '"legacy_dns_host":"","legacy_ns":null,"ns":null,' + '"created":"","verified":"","modified":"","project":"",' + '"owner":"","permission":"","zone_type":{"id":"",' + '"name":"","description":"","prices":null},"status":"",' + '"paused":false,"is_secondary_dns":false,' + '"txt_verification":{"name":"","token":""},' + '"records_count":0},"error":{' + '"message":"zone not found","code":404}}') + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(set(), zone.records) + + # No diffs == no changes + with requests_mock() as mock: + base = 'https://dns.hetzner.com/api/v1' + with open('tests/fixtures/hetzner-zones.json') as fh: + mock.get('{}/zones'.format(base), text=fh.read()) + with open('tests/fixtures/hetzner-records.json') as fh: + mock.get('{}/records'.format(base), text=fh.read()) + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(13, len(zone.records)) + changes = self.expected.changes(zone, provider) + self.assertEquals(0, len(changes)) + + # 2nd populate makes no network calls/all from cache + again = Zone('unit.tests.', []) + provider.populate(again) + self.assertEquals(13, len(again.records)) + + # bust the cache + del provider._zone_records[zone.name] From b55a647e6e915ea5395711152ce3f2745a376062 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 26 Apr 2021 06:36:05 -0700 Subject: [PATCH 08/58] Remove redudant pools_seen.add(pool) call --- octodns/record/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 8c8760a..135baf8 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -628,7 +628,6 @@ class _DynamicMixin(object): if pool not in pools: reasons.append('rule {} undefined pool "{}"' .format(rule_num, pool)) - pools_seen.add(pool) elif pool in pools_seen and geos: reasons.append('rule {} invalid, target pool "{}" ' 'reused'.format(rule_num, pool)) From 078576520d67e7b6360f5f1ef1166ee6b3f0f2d2 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 26 Apr 2021 08:34:02 -0700 Subject: [PATCH 09/58] Rework NS1 pool handling to support fallback-only pools --- octodns/provider/ns1.py | 63 +++++++++++++++++++----------- tests/test_octodns_provider_ns1.py | 15 +++++-- 2 files changed, 51 insertions(+), 27 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 6cea185..7724fcb 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -406,7 +406,7 @@ class Ns1Provider(BaseProvider): for piece in note.split(' '): try: k, v = piece.split(':', 1) - data[k] = v + data[k] = v if v != '' else None except ValueError: pass return data @@ -479,31 +479,45 @@ class Ns1Provider(BaseProvider): # region. pools = defaultdict(lambda: {'fallback': None, 'values': []}) for answer in record['answers']: - # region (group name in the UI) is the pool name - pool_name = answer['region'] - # Get the actual pool name by removing the type - pool_name = self._parse_dynamic_pool_name(pool_name) - pool = pools[pool_name] - meta = answer['meta'] + notes = self._parse_notes(meta.get('note', '')) + value = text_type(answer['answer'][0]) - if meta['priority'] == 1: - # priority 1 means this answer is part of the pools own values - value_dict = { - 'value': value, - 'weight': int(meta.get('weight', 1)), - } - # If we have the original pool name and the catchall pool name - # in the answers, they point at the same pool. Add values only - # once - if value_dict not in pool['values']: - pool['values'].append(value_dict) + if notes.get('from', False) == '--default--': + # It's a final/default value, record it and move on + default.add(value) + continue + + # NS1 pool names can be found in notes > v0.9.11, in order to allow + # us to find fallback-only pools/values. Before that we used + # `region` (group name in the UI) and only paid attention to + # priority=1 (first level) + notes_pool_name = notes.get('pool', None) + if notes_pool_name is None: + # < v0.9.11 + if meta['priority'] != 1: + # Ignore all but priority 1 + continue + # And use region's pool name as the pool name + pool_name = self._parse_dynamic_pool_name(answer['region']) else: - # It's a fallback, we only care about it if it's a - # final/default - notes = self._parse_notes(meta.get('note', '')) - if notes.get('from', False) == '--default--': - default.add(value) + # > v0.9.11, use the notes-based name and consider all values + pool_name = notes_pool_name + + pool = pools[pool_name] + value_dict = { + 'value': value, + 'weight': int(meta.get('weight', 1)), + } + if value_dict not in pool['values']: + # If we haven't seen this value before add it to the pool + pool['values'].append(value_dict) + + # If there's a fallback recorded in the value for its pool go ahead + # and use it, another v0.9.11 thing + fallback = notes.get('fallback', None) + if fallback is not None: + pool['fallback'] = fallback # The regions objects map to rules, but it's a bit fuzzy since they're # tied to pools on the NS1 side, e.g. we can only have 1 rule per pool, @@ -978,12 +992,15 @@ class Ns1Provider(BaseProvider): seen.add(current_pool_name) pool = pools[current_pool_name] for answer in pool_answers[current_pool_name]: + fallback = pool.data['fallback'] answer = { 'answer': answer['answer'], 'meta': { 'priority': priority, 'note': self._encode_notes({ 'from': pool_label, + 'pool': current_pool_name, + 'fallback': fallback or '', }), 'up': { 'feed': answer['feed_id'], diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 00b068b..b7270fb 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -1117,14 +1117,21 @@ class TestNs1ProviderDynamic(TestCase): # finally has a catchall. Those are examples of the two ways pools get # expanded. # - # lhr splits in two, with a region and country. + # lhr splits in two, with a region and country and includes a fallback + # + # All values now include their own `pool:` name # # well as both lhr georegion (for contients) and country. The first is # an example of a repeated target pool in a rule (only allowed when the # 2nd is a catchall.) - self.assertEquals(['from:--default--', 'from:iad__catchall', - 'from:iad__country', 'from:iad__georegion', - 'from:lhr__country', 'from:lhr__georegion'], + self.assertEquals(['fallback: from:iad__catchall pool:iad', + 'fallback: from:iad__country pool:iad', + 'fallback: from:iad__georegion pool:iad', + 'fallback: from:lhr__country pool:iad', + 'fallback: from:lhr__georegion pool:iad', + 'fallback:iad from:lhr__country pool:lhr', + 'fallback:iad from:lhr__georegion pool:lhr', + 'from:--default--'], sorted(notes.keys())) # All the iad's should match (after meta and region were removed) From fe74c462138a951577ce4693e8eafa5e32764557 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Mon, 26 Apr 2021 19:02:44 +0200 Subject: [PATCH 10/58] made hetzner.HetznerProvider._do Mock-able for testing purposes making it a wrapper for _do_raw --- octodns/provider/hetzner.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 17b1141..78be756 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -46,7 +46,7 @@ class HetznerClient(object): session.headers.update({'Auth-API-Token': token}) self._session = session - def _do(self, method, path, params=None, data=None): + def _do_raw(self, method, path, params=None, data=None): url = self.BASE_URL + path response = self._session.request(method, url, params=params, json=data) if response.status_code == 401: @@ -54,7 +54,10 @@ class HetznerClient(object): if response.status_code == 404: raise HetznerClientNotFound() response.raise_for_status() - return response.json() + return response + + def _do(self, method, path, params=None, data=None): + return self._do_raw(method, path, params, data).json() def _do_with_pagination(self, method, path, key, params=None, data=None, per_page=100): From 612738b327b26b417a683ec5dd0c3c55291b6bd4 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Mon, 26 Apr 2021 19:03:25 +0200 Subject: [PATCH 11/58] renamed TestdHetznerProvider -> TestHetznerProvider (missing "d") --- tests/test_octodns_provider_hetzner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_hetzner.py b/tests/test_octodns_provider_hetzner.py index bf619fb..a8a67fb 100644 --- a/tests/test_octodns_provider_hetzner.py +++ b/tests/test_octodns_provider_hetzner.py @@ -17,7 +17,7 @@ from octodns.provider.yaml import YamlProvider from octodns.zone import Zone -class TestdHetznerProvider(TestCase): +class TestHetznerProvider(TestCase): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) From fbd838990364dec250d3aa58e3ed1b441f853ef4 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 26 Apr 2021 17:10:22 -0700 Subject: [PATCH 12/58] Tests for new-style ns1 data_for_dynamic_A fallback only pools --- octodns/provider/ns1.py | 2 +- tests/test_octodns_provider_ns1.py | 109 +++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 7724fcb..52fb3d2 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -542,7 +542,7 @@ class Ns1Provider(BaseProvider): rules[rule_order] = rule # The group notes field in the UI is a `note` on the region here, - # that's where we can find our pool's fallback. + # that's where we can find our pool's fallback in < v0.9.11 anyway if 'fallback' in notes: # set the fallback pool name pools[pool_name]['fallback'] = notes['fallback'] diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index b7270fb..db713eb 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -1478,6 +1478,115 @@ class TestNs1ProviderDynamic(TestCase): self.assertTrue( 'OC-{}'.format(c) in data4['dynamic']['rules'][0]['geos']) + # Test out fallback only pools and new-style notes + ns1_record = { + 'answers': [{ + 'answer': ['1.1.1.1'], + 'meta': { + 'priority': 1, + 'note': 'from:one__country pool:one fallback:two', + }, + 'region': 'one_country', + }, { + 'answer': ['2.2.2.2'], + 'meta': { + 'priority': 2, + 'note': 'from:one__country pool:two fallback:three', + }, + 'region': 'one_country', + }, { + 'answer': ['3.3.3.3'], + 'meta': { + 'priority': 3, + 'note': 'from:one__country pool:three fallback:', + }, + 'region': 'one_country', + }, { + 'answer': ['5.5.5.5'], + 'meta': { + 'priority': 4, + 'note': 'from:--default--', + }, + 'region': 'one_country', + }, { + 'answer': ['4.4.4.4'], + 'meta': { + 'priority': 1, + 'note': 'from:four__country pool:four fallback:', + }, + 'region': 'four_country', + }, { + 'answer': ['5.5.5.5'], + 'meta': { + 'priority': 2, + 'note': 'from:--default--', + }, + 'region': 'four_country', + }], + 'domain': 'unit.tests', + 'filters': filters, + 'regions': { + 'one__country': { + 'meta': { + 'note': 'rule-order:1 fallback:two', + 'country': ['CA'], + 'us_state': ['OR'], + }, + }, + 'four__country': { + 'meta': { + 'note': 'rule-order:2', + 'country': ['CA'], + 'us_state': ['OR'], + }, + }, + catchall_pool_name: { + 'meta': { + 'note': 'rule-order:3', + }, + } + }, + 'tier': 3, + 'ttl': 42, + } + data = provider._data_for_dynamic_A('A', ns1_record) + self.assertEquals({ + 'dynamic': { + 'pools': { + 'four': { + 'fallback': None, + 'values': [{'value': '4.4.4.4', 'weight': 1}] + }, + 'one': { + 'fallback': 'two', + 'values': [{'value': '1.1.1.1', 'weight': 1}] + }, + 'three': { + 'fallback': None, + 'values': [{'value': '3.3.3.3', 'weight': 1}] + }, + 'two': { + 'fallback': 'three', + 'values': [{'value': '2.2.2.2', 'weight': 1}] + }, + }, + 'rules': [{ + '_order': '1', + 'geos': ['NA-CA', 'NA-US-OR'], + 'pool': 'one' + }, { + '_order': '2', + 'geos': ['NA-CA', 'NA-US-OR'], + 'pool': 'four' + }, { + '_order': '3', 'pool': 'iad'} + ] + }, + 'ttl': 42, + 'type': 'A', + 'values': ['5.5.5.5'] + }, data) + @patch('ns1.rest.records.Records.retrieve') @patch('ns1.rest.zones.Zones.retrieve') @patch('octodns.provider.ns1.Ns1Provider._monitors_for') From 0de9efd03268ae0d447997fe1960dc314880348d Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Tue, 27 Apr 2021 07:31:05 +0200 Subject: [PATCH 13/58] removed unused HetznerClient methods to fix imparial coverage --- octodns/provider/hetzner.py | 70 +++++++++---------------------------- 1 file changed, 17 insertions(+), 53 deletions(-) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 78be756..0be01ea 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -32,11 +32,6 @@ class HetznerClientUnauthorized(HetznerClientException): class HetznerClient(object): ''' Hetzner DNS Public API v1 client class. - - Zone and Record resources are (almost) fully supported, even if unnecessary - to future-proof this client. Bulk Record create/update is not supported. - - No support for Primary Servers. ''' BASE_URL = 'https://dns.hetzner.com/api/v1' @@ -46,8 +41,8 @@ class HetznerClient(object): session.headers.update({'Auth-API-Token': token}) self._session = session - def _do_raw(self, method, path, params=None, data=None): - url = self.BASE_URL + path + def _do(self, method, path, params=None, data=None): + url = '{}{}'.format(self.BASE_URL, path) response = self._session.request(method, url, params=params, json=data) if response.status_code == 401: raise HetznerClientUnauthorized() @@ -56,20 +51,15 @@ class HetznerClient(object): response.raise_for_status() return response - def _do(self, method, path, params=None, data=None): - return self._do_raw(method, path, params, data).json() - - def _do_with_pagination(self, method, path, key, params=None, data=None, - per_page=100): - pagination_params = {'page': 1, 'per_page': per_page} - if params is not None: - params = {**params, **pagination_params} - else: - params = pagination_params + def _do_json(self, method, path, params=None, data=None): + return self._do(method, path, params, data).json() + def _do_json_paginate(self, method, path, key, params=None, data=None, + per_page=100): items = [] + params = {**{'page': 1, 'per_page': per_page}, **params} while True: - response = self._do(method, path, params, data) + response = self._do_json(method, path, params, data) items += response[key] if response['meta']['pagination']['page'] >= \ response['meta']['pagination']['last_page']: @@ -79,54 +69,28 @@ class HetznerClient(object): def zones_get(self, name=None, search_name=None): params = {'name': name, 'search_name': search_name} - return self._do_with_pagination('GET', '/zones', 'zones', - params=params) - - def zone_get(self, zone_id): - return self._do('GET', '/zones/' + zone_id)['zone'] + return self._do_json_paginate('GET', '/zones', 'zones', params=params) def zone_create(self, name, ttl=None): data = {'name': name, 'ttl': ttl} - return self._do('POST', '/zones', data=data)['zone'] - - def zone_update(self, zone_id, name, ttl=None): - data = {'name': name, 'ttl': ttl} - return self._do('PUT', '/zones/' + zone_id, data=data)['zone'] - - def zone_delete(self, zone_id): - return self._do('DELETE', '/zones/' + zone_id) - - def _replace_at(self, record): - record['name'] = '' if record['name'] == '@' else record['name'] - return record + return self._do_json('POST', '/zones', data=data)['zone'] def zone_records_get(self, zone_id): params = {'zone_id': zone_id} # No need to handle pagination as it returns all records by default. - return [ - self._replace_at(record) - for record in self._do('GET', '/records', params=params)['records'] - ] - - def zone_record_get(self, record_id): - record = self._do('GET', '/records/' + record_id)['record'] - return self._replace_at(record) + records = self._do_json('GET', '/records', params=params)['records'] + for record in records: + if record['name'] == '@': + record['name'] = '' + return records def zone_record_create(self, zone_id, name, _type, value, ttl=None): data = {'name': name or '@', 'ttl': ttl, 'type': _type, 'value': value, 'zone_id': zone_id} - record = self._do('POST', '/records', data=data)['record'] - return self._replace_at(record) - - def zone_record_update(self, zone_id, record_id, name, _type, value, - ttl=None): - data = {'name': name or '@', 'ttl': ttl, 'type': _type, 'value': value, - 'zone_id': zone_id} - record = self._do('PUT', '/records/' + record_id, data=data)['record'] - return self._replace_at(record) + self._do('POST', '/records', data=data) def zone_record_delete(self, zone_id, record_id): - return self._do('DELETE', '/records/' + record_id) + self._do('DELETE', '/records/{}'.format(record_id)) class HetznerProvider(BaseProvider): From 192231109181dac8f61c1b41be156ba4c53cbac9 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Tue, 27 Apr 2021 07:53:17 +0200 Subject: [PATCH 14/58] WIP added TestHetznerProvider.test_apply --- tests/test_octodns_provider_hetzner.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_hetzner.py b/tests/test_octodns_provider_hetzner.py index a8a67fb..0c18731 100644 --- a/tests/test_octodns_provider_hetzner.py +++ b/tests/test_octodns_provider_hetzner.py @@ -6,13 +6,15 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +from mock import Mock from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock from six import text_type from unittest import TestCase -from octodns.provider.hetzner import HetznerProvider +from octodns.provider.hetzner import HetznerClientNotFound, \ + HetznerProvider from octodns.provider.yaml import YamlProvider from octodns.zone import Zone @@ -82,3 +84,24 @@ class TestHetznerProvider(TestCase): # bust the cache del provider._zone_records[zone.name] + + def test_apply(self): + provider = HetznerProvider('test', 'token') + + resp = Mock() + resp.json = Mock() + provider._client._do = Mock(return_value=resp) + + # non-existent domain, create everything + resp.json.side_effect = [ + HetznerClientNotFound, # no zone in populate + HetznerClientNotFound, # no zone during apply + {"zone": {"id": "string"}} + ] + plan = provider.plan(self.expected) + + # No root NS, no ignored, no excluded, no unsupported + n = len(self.expected.records) - 9 + self.assertEquals(n, len(plan.changes)) + self.assertEquals(n, provider.apply(plan)) + self.assertFalse(plan.exists) From a0c4e9ecd76af5f2a830b8cb430468b147e070f3 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Tue, 27 Apr 2021 19:56:39 +0200 Subject: [PATCH 15/58] simplified HetznerClient by removing unused pagination handling --- octodns/provider/hetzner.py | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 0be01ea..d396e18 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -30,10 +30,6 @@ class HetznerClientUnauthorized(HetznerClientException): class HetznerClient(object): - ''' - Hetzner DNS Public API v1 client class. - ''' - BASE_URL = 'https://dns.hetzner.com/api/v1' def __init__(self, token): @@ -54,22 +50,9 @@ class HetznerClient(object): def _do_json(self, method, path, params=None, data=None): return self._do(method, path, params, data).json() - def _do_json_paginate(self, method, path, key, params=None, data=None, - per_page=100): - items = [] - params = {**{'page': 1, 'per_page': per_page}, **params} - while True: - response = self._do_json(method, path, params, data) - items += response[key] - if response['meta']['pagination']['page'] >= \ - response['meta']['pagination']['last_page']: - break - params['page'] += 1 - return items - - def zones_get(self, name=None, search_name=None): - params = {'name': name, 'search_name': search_name} - return self._do_json_paginate('GET', '/zones', 'zones', params=params) + def zone_get(self, name): + params = {'name': name} + return self._do_json('GET', '/zones', params)['zones'][0] def zone_create(self, name, ttl=None): data = {'name': name, 'ttl': ttl} @@ -77,7 +60,6 @@ class HetznerClient(object): def zone_records_get(self, zone_id): params = {'zone_id': zone_id} - # No need to handle pagination as it returns all records by default. records = self._do_json('GET', '/records', params=params)['records'] for record in records: if record['name'] == '@': From d4c6836d0bb8b0d8bd6b3fb58ba9abb2378f706a Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Tue, 27 Apr 2021 19:58:07 +0200 Subject: [PATCH 16/58] implemented HetznerProvider.zone_metadata as the equivalent to zone_records for zone metadata --- octodns/provider/hetzner.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index d396e18..53914f5 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -96,16 +96,28 @@ class HetznerProvider(BaseProvider): self._zone_records = {} self._zone_metadata = {} + self._zone_name_to_id = {} def _append_dot(self, value): if value == '@' or value[-1] == '.': return value return '{}.'.format(value) + def zone_metadata(self, zone_id=None, zone_name=None): + if zone_name is not None: + if zone_name in self._zone_name_to_id: + zone_id = self._zone_name_to_id[zone_name] + else: + zone = self._client.zone_get(name=zone_name[:-1]) + zone_id = zone['id'] + self._zone_name_to_id[zone_name] = zone_id + self._zone_metadata[zone_id] = zone + + return self._zone_metadata[zone_id] + def _record_ttl(self, record): - if 'ttl' in record: - return record['ttl'] - return self._zone_metadata[record['zone_id']]['ttl'] + default_ttl = self.zone_metadata(zone_id=record['zone_id'])['ttl'] + return record['ttl'] if 'ttl' in record else default_ttl def _data_for_multiple(self, _type, records): values = [record['value'].replace(';', '\\;') for record in records] @@ -195,10 +207,9 @@ class HetznerProvider(BaseProvider): def zone_records(self, zone): if zone.name not in self._zone_records: try: - zone_metadata = self._client.zones_get(name=zone.name[:-1])[0] - self._zone_metadata[zone_metadata['id']] = zone_metadata + zone_id = self.zone_metadata(zone_name=zone.name)['id'] self._zone_records[zone.name] = \ - self._client.zone_records_get(zone_metadata['id']) + self._client.zone_records_get(zone_id) except HetznerClientNotFound: return [] @@ -314,12 +325,11 @@ class HetznerProvider(BaseProvider): self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, len(changes)) - zone_name = desired.name[:-1] try: - zone_id = self._client.zones_get(name=zone_name)[0]['id'] + zone_id = self.zone_metadata(zone_name=desired.name)['id'] except HetznerClientNotFound: self.log.debug('_apply: no matching zone, creating domain') - zone_id = self._client.zone_create(zone_name)['id'] + zone_id = self._client.zone_create(desired.name[:-1])['id'] for change in changes: class_name = change.__class__.__name__ From 6bf24d867894c8df992b8491762a9217e563d7d5 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Tue, 27 Apr 2021 19:59:00 +0200 Subject: [PATCH 17/58] implemented TestHetznerProvider.test_apply --- tests/test_octodns_provider_hetzner.py | 230 ++++++++++++++++++++++++- 1 file changed, 229 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_hetzner.py b/tests/test_octodns_provider_hetzner.py index 0c18731..02da32e 100644 --- a/tests/test_octodns_provider_hetzner.py +++ b/tests/test_octodns_provider_hetzner.py @@ -6,13 +6,14 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from mock import Mock +from mock import Mock, call from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock from six import text_type from unittest import TestCase +from octodns.record import Record from octodns.provider.hetzner import HetznerClientNotFound, \ HetznerProvider from octodns.provider.yaml import YamlProvider @@ -105,3 +106,230 @@ class TestHetznerProvider(TestCase): self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) + + provider._client._do.assert_has_calls([ + # created the zone + call('POST', '/zones', None, { + 'name': 'unit.tests', + 'ttl': None, + }), + # created all the records with their expected data + call('POST', '/records', data={ + 'name': '@', + 'ttl': 300, + 'type': 'A', + 'value': '1.2.3.4', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': '@', + 'ttl': 300, + 'type': 'A', + 'value': '1.2.3.5', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': '@', + 'ttl': 3600, + 'type': 'CAA', + 'value': '0 issue "ca.unit.tests"', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': '_imap._tcp', + 'ttl': 600, + 'type': 'SRV', + 'value': '0 0 0 .', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': '_pop3._tcp', + 'ttl': 600, + 'type': 'SRV', + 'value': '0 0 0 .', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': '_srv._tcp', + 'ttl': 600, + 'type': 'SRV', + 'value': '10 20 30 foo-1.unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': '_srv._tcp', + 'ttl': 600, + 'type': 'SRV', + 'value': '12 20 30 foo-2.unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'aaaa', + 'ttl': 600, + 'type': 'AAAA', + 'value': '2601:644:500:e210:62f8:1dff:feb8:947a', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'cname', + 'ttl': 300, + 'type': 'CNAME', + 'value': 'unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'included', + 'ttl': 3600, + 'type': 'CNAME', + 'value': 'unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'mx', + 'ttl': 300, + 'type': 'MX', + 'value': '10 smtp-4.unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'mx', + 'ttl': 300, + 'type': 'MX', + 'value': '20 smtp-2.unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'mx', + 'ttl': 300, + 'type': 'MX', + 'value': '30 smtp-3.unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'mx', + 'ttl': 300, + 'type': 'MX', + 'value': '40 smtp-1.unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'sub', + 'ttl': 3600, + 'type': 'NS', + 'value': '6.2.3.4.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'sub', + 'ttl': 3600, + 'type': 'NS', + 'value': '7.2.3.4.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'txt', + 'ttl': 600, + 'type': 'TXT', + 'value': 'Bah bah black sheep', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'txt', + 'ttl': 600, + 'type': 'TXT', + 'value': 'have you any wool.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'txt', + 'ttl': 600, + 'type': 'TXT', + 'value': 'v=DKIM1;k=rsa;s=email;h=sha256;' + 'p=A/kinda+of/long/string+with+numb3rs', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'www', + 'ttl': 300, + 'type': 'A', + 'value': '2.2.3.6', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'www.sub', + 'ttl': 300, + 'type': 'A', + 'value': '2.2.3.6', + 'zone_id': 'unit.tests', + }), + ]) + self.assertEquals(24, provider._client._do.call_count) + + provider._client._do.reset_mock() + + # delete 1 and update 1 + provider._client.zone_get = Mock(return_value={ + 'id': 'unit.tests', + 'name': 'unit.tests', + 'ttl': 3600, + }) + provider._client.zone_records_get = Mock(return_value=[ + { + 'type': 'A', + 'id': 'one', + 'created': '0000-00-00T00:00:00Z', + 'modified': '0000-00-00T00:00:00Z', + 'zone_id': 'unit.tests', + 'name': 'www', + 'value': '1.2.3.4', + 'ttl': 300, + }, + { + 'type': 'A', + 'id': 'two', + 'created': '0000-00-00T00:00:00Z', + 'modified': '0000-00-00T00:00:00Z', + 'zone_id': 'unit.tests', + 'name': 'www', + 'value': '2.2.3.4', + 'ttl': 300, + }, + { + 'type': 'A', + 'id': 'three', + 'created': '0000-00-00T00:00:00Z', + 'modified': '0000-00-00T00:00:00Z', + 'zone_id': 'unit.tests', + 'name': 'ttl', + 'value': '3.2.3.4', + 'ttl': 600, + }, + ]) + + # Domain exists, we don't care about return + resp.json.side_effect = ['{}'] + + wanted = Zone('unit.tests.', []) + wanted.add_record(Record.new(wanted, 'ttl', { + 'ttl': 300, + 'type': 'A', + 'value': '3.2.3.4', + })) + + plan = provider.plan(wanted) + self.assertTrue(plan.exists) + self.assertEquals(2, len(plan.changes)) + self.assertEquals(2, provider.apply(plan)) + # recreate for update, and delete for the 2 parts of the other + provider._client._do.assert_has_calls([ + call('POST', '/records', data={ + 'name': 'ttl', + 'ttl': 300, + 'type': 'A', + 'value': '3.2.3.4', + 'zone_id': 'unit.tests', + }), + call('DELETE', '/records/one'), + call('DELETE', '/records/two'), + call('DELETE', '/records/three'), + ], any_order=True) From b3c394a5e0056e5c0f4382e89e7854b4e02c02e4 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Tue, 27 Apr 2021 19:59:26 +0200 Subject: [PATCH 18/58] minor correctness tweaks --- tests/test_octodns_provider_hetzner.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_octodns_provider_hetzner.py b/tests/test_octodns_provider_hetzner.py index 02da32e..4167944 100644 --- a/tests/test_octodns_provider_hetzner.py +++ b/tests/test_octodns_provider_hetzner.py @@ -66,7 +66,7 @@ class TestHetznerProvider(TestCase): # No diffs == no changes with requests_mock() as mock: - base = 'https://dns.hetzner.com/api/v1' + base = provider._client.BASE_URL with open('tests/fixtures/hetzner-zones.json') as fh: mock.get('{}/zones'.format(base), text=fh.read()) with open('tests/fixtures/hetzner-records.json') as fh: @@ -93,11 +93,17 @@ class TestHetznerProvider(TestCase): resp.json = Mock() provider._client._do = Mock(return_value=resp) + domain_after_creation = {'zone': { + 'id': 'unit.tests', + 'name': 'unit.tests', + 'ttl': 3600, + }} + # non-existent domain, create everything resp.json.side_effect = [ HetznerClientNotFound, # no zone in populate HetznerClientNotFound, # no zone during apply - {"zone": {"id": "string"}} + domain_after_creation, ] plan = provider.plan(self.expected) From a564270ef5cf8030e55e3a2434ad2e7b51c9c370 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 29 Apr 2021 09:12:35 -0700 Subject: [PATCH 19/58] Add a blurb on pip installing a sha --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index cd9b884..5eaca41 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,14 @@ $ pip install octodns $ mkdir config ``` +#### Installing a specific commit SHA + +If you'd like to install a version that has not yet been released in a repetable/safe manner you can do the following. In general octoDNS is fairly stable inbetween releases thanks to the plan and apply process, but care should be taken regardless. + +```shell +$ pip install -e git+https://git@github.com/github/octodns.git@#egg=octodns +``` + ### Config We start by creating a config file to tell OctoDNS about our providers and the zone(s) we want it to manage. Below we're setting up a `YamlProvider` to source records from our config files and both a `Route53Provider` and `DynProvider` to serve as the targets for those records. You can have any number of zones set up and any number of sources of data and targets for records for each. You can also have multiple config files, that make use of separate accounts and each manage a distinct set of zones. A good example of this this might be `./config/staging.yaml` & `./config/production.yaml`. We'll focus on a `config/production.yaml`. From ad37b997739fc76c370aba1983f156bd8e32c03a Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 29 Apr 2021 09:12:38 -0700 Subject: [PATCH 20/58] Add shell code block types --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5eaca41..4ab3c6b 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ It is similar to [Netflix/denominator](https://github.com/Netflix/denominator). Running through the following commands will install the latest release of OctoDNS and set up a place for your config files to live. To determine if provider specific requirements are necessary see the [Supported providers table](#supported-providers) below. -``` +```shell $ mkdir dns $ cd dns $ virtualenv env @@ -121,7 +121,7 @@ Further information can be found in [Records Documentation](/docs/records.md). We're ready to do a dry-run with our new setup to see what changes it would make. Since we're pretending here we'll act like there are no existing records for `example.com.` in our accounts on either provider. -``` +```shell $ octodns-sync --config-file=./config/production.yaml ... ******************************************************************************** @@ -145,7 +145,7 @@ There will be other logging information presented on the screen, but successful Now it's time to tell OctoDNS to make things happen. We'll invoke it again with the same options and add a `--doit` on the end to tell it this time we actually want it to try and make the specified changes. -``` +```shell $ octodns-sync --config-file=./config/production.yaml --doit ... ``` @@ -176,7 +176,7 @@ If that goes smoothly, you again see the expected changes, and verify them with Very few situations will involve starting with a blank slate which is why there's tooling built in to pull existing data out of providers into a matching config file. -``` +```shell $ octodns-dump --config-file=config/production.yaml --output-dir=tmp/ example.com. route53 2017-03-15T13:33:34 INFO Manager __init__: config_file=tmp/production.yaml 2017-03-15T13:33:34 INFO Manager dump: zone=example.com., sources=('route53',) From 58c7f431e86ce441767ab1f5a7f6dee7837465e8 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 30 Apr 2021 14:43:23 -0700 Subject: [PATCH 21/58] v0.9.12 version bump and CHANGELOG update --- CHANGELOG.md | 25 ++++++++++++++++++++++++- octodns/__init__.py | 2 +- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb68e25..1d16544 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,29 @@ +## v0.9.12 - 2021-04-30 - Enough time has passed + +#### Noteworthy changes + +* Formal Python 2.7 support removed, deps and tooling were becoming + unmaintainable +* octodns/octodns move, from github/octodns, more to come + +#### Stuff + +* ZoneFileSource supports specifying an extension & no files end in . to better + support Windows +* LOC record type support added +* Support for pre-release versions of PowerDNS +* PowerDNS delete before create which allows A <-> CNAME etc. +* Improved validation of fqdn's in ALIAS, CNAME, etc. +* Transip support for NS records +* Support for sending plan output to a file +* DNSimple uses zone api rather than domain to support non-registered stuff, + e.g. reverse zones. +* Support for fallback-only dynamic pools and related fixes to NS1 provider +* Initial Hetzner provider + ## v0.9.11 - 2020-11-05 - We still don't know edition -#### Noteworthy changtes +#### Noteworthy changes * ALIAS records only allowed at the root of zones - see `leient` in record docs for work-arounds if you really need them. diff --git a/octodns/__init__.py b/octodns/__init__.py index 3fcdaa1..1885d42 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -3,4 +3,4 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -__VERSION__ = '0.9.11' +__VERSION__ = '0.9.12' From 40569945d2bc0fb89735fd97e3cdbbd00b1379c1 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Fri, 30 Apr 2021 16:53:37 -0700 Subject: [PATCH 22/58] Add support for dynamic CNAME records in NS1 --- octodns/provider/ns1.py | 24 +++++- tests/test_octodns_provider_ns1.py | 118 +++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+), 4 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 6cea185..4da0d33 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -588,16 +588,22 @@ class Ns1Provider(BaseProvider): rules = list(rules.values()) rules.sort(key=lambda r: (r['_order'], r['pool'])) - return { + data = { 'dynamic': { 'pools': pools, 'rules': rules, }, 'ttl': record['ttl'], 'type': _type, - 'values': sorted(default), } + if _type == 'CNAME': + data['value'] = default[0] + else: + data['values'] = default + + return data + def _data_for_A(self, _type, record): if record.get('tier', 1) > 1: # Advanced record, see if it's first answer has a note @@ -646,6 +652,10 @@ class Ns1Provider(BaseProvider): } def _data_for_CNAME(self, _type, record): + if record.get('tier', 1) > 1: + # Advanced dynamic record + return self._data_for_dynamic_A(_type, record) + try: value = record['short_answers'][0] except IndexError: @@ -1114,10 +1124,14 @@ class Ns1Provider(BaseProvider): 'feed_id': feed_id, }) + if record._type == 'CNAME': + default_values = [record.value] + else: + default_values = record.values default_answers = [{ 'answer': [v], 'weight': 1, - } for v in record.values] + } for v in default_values] # Build our list of answers # The regions dictionary built above already has the required pool @@ -1171,8 +1185,10 @@ class Ns1Provider(BaseProvider): values = [(v.flags, v.tag, v.value) for v in record.values] return {'answers': values, 'ttl': record.ttl}, None - # TODO: dynamic CNAME support def _params_for_CNAME(self, record): + if getattr(record, 'dynamic', False): + return self._params_for_dynamic_A(record) + return {'answers': [record.value], 'ttl': record.ttl}, None _params_for_ALIAS = _params_for_CNAME diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 00b068b..c16a573 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -1270,6 +1270,63 @@ class TestNs1ProviderDynamic(TestCase): params, _ = provider._params_for_geo_A(record) self.assertEquals([], params['filters']) + @patch('octodns.provider.ns1.Ns1Provider._monitor_sync') + @patch('octodns.provider.ns1.Ns1Provider._monitors_for') + def test_params_for_dynamic_CNAME(self, monitors_for_mock, + monitor_sync_mock): + provider = Ns1Provider('test', 'api-key') + + # pre-fill caches to avoid extranious calls (things we're testing + # elsewhere) + provider._client._datasource_id = 'foo' + provider._client._feeds_for_monitors = { + 'mon-id': 'feed-id', + } + + # provider._params_for_A() calls provider._monitors_for() and + # provider._monitor_sync(). Mock their return values so that we don't + # make NS1 API calls during tests + monitors_for_mock.reset_mock() + monitor_sync_mock.reset_mock() + monitors_for_mock.side_effect = [{ + 'iad.unit.tests.': 'mid-1', + }] + monitor_sync_mock.side_effect = [ + ('mid-1', 'fid-1'), + ] + + record = Record.new(self.zone, 'foo', { + 'dynamic': { + 'pools': { + 'iad': { + 'values': [{ + 'value': 'iad.unit.tests.', + }], + }, + }, + 'rules': [{ + 'pool': 'iad', + }], + }, + 'octodns': { + 'healthcheck': { + 'host': 'send.me', + 'path': '/_ping', + 'port': 80, + 'protocol': 'HTTP', + } + }, + 'ttl': 32, + 'type': 'CNAME', + 'value': 'value.unit.tests.', + 'meta': {}, + }) + ret, _ = provider._params_for_CNAME(record) + + # Check if the default value was correctly read and populated + # All other dynamic record test cases are covered by dynamic_A tests + self.assertEquals(ret['answers'][1]['answer'][0], 'value.unit.tests.') + def test_data_for_dynamic_A(self): provider = Ns1Provider('test', 'api-key') @@ -1471,6 +1528,67 @@ class TestNs1ProviderDynamic(TestCase): self.assertTrue( 'OC-{}'.format(c) in data4['dynamic']['rules'][0]['geos']) + def test_data_for_dynamic_CNAME(self): + provider = Ns1Provider('test', 'api-key') + + # Test out a small setup that just covers default value validation + # Everything else is same as dynamic A whose tests will cover all + # other options and test cases + # Not testing for geo/region specific cases + filters = provider._get_updated_filter_chain(False, False) + catchall_pool_name = 'iad__catchall' + ns1_record = { + 'answers': [{ + 'answer': ['2.3.4.5.unit.tests.'], + 'meta': { + 'priority': 1, + 'weight': 12, + 'note': 'from:{}'.format(catchall_pool_name), + }, + 'region': catchall_pool_name, + }, { + 'answer': ['1.2.3.4.unit.tests.'], + 'meta': { + 'priority': 2, + 'note': 'from:--default--', + }, + 'region': catchall_pool_name, + }], + 'domain': 'foo.unit.tests', + 'filters': filters, + 'regions': { + catchall_pool_name: { + 'meta': { + 'note': 'rule-order:1', + }, + } + }, + 'tier': 3, + 'ttl': 42, + 'type': 'CNAME', + } + data = provider._data_for_CNAME('CNAME', ns1_record) + self.assertEquals({ + 'dynamic': { + 'pools': { + 'iad': { + 'fallback': None, + 'values': [{ + 'value': '2.3.4.5.unit.tests.', + 'weight': 12, + }], + }, + }, + 'rules': [{ + '_order': '1', + 'pool': 'iad', + }], + }, + 'ttl': 42, + 'type': 'CNAME', + 'value': '1.2.3.4.unit.tests.', + }, data) + @patch('ns1.rest.records.Records.retrieve') @patch('ns1.rest.zones.Zones.retrieve') @patch('octodns.provider.ns1.Ns1Provider._monitors_for') From 15eb23eeb672f743cb21724eddb2fa394c931aea Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Fri, 30 Apr 2021 20:26:11 -0700 Subject: [PATCH 23/58] Trim trailing dot from CNAME answers for NS1 monitors --- octodns/provider/ns1.py | 11 +++- tests/test_octodns_provider_ns1.py | 87 +++++++++++++++++++----------- 2 files changed, 64 insertions(+), 34 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 4da0d33..6ade3b9 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -832,6 +832,10 @@ class Ns1Provider(BaseProvider): # This monitor does not belong to this record config = monitor['config'] value = config['host'] + if record._type == 'CNAME': + # Append a trailing dot for CNAME records so that + # lookup by a CNAME answer works + value = value + '.' monitors[value] = monitor return monitors @@ -882,6 +886,10 @@ class Ns1Provider(BaseProvider): host = record.fqdn[:-1] _type = record._type + if _type == 'CNAME': + # NS1 does not accept a host value with a trailing dot + value = value[:-1] + ret = { 'active': True, 'config': { @@ -1266,8 +1274,7 @@ class Ns1Provider(BaseProvider): extra.append(Update(record, record)) continue - for have in self._monitors_for(record).values(): - value = have['config']['host'] + for value, have in self._monitors_for(record).items(): expected = self._monitor_gen(record, value) # TODO: find values which have missing monitors if not self._monitor_is_match(expected, have): diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index c16a573..a5c41a6 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -578,6 +578,34 @@ class TestNs1ProviderDynamic(TestCase): 'meta': {}, }) + def cname_record(self): + return Record.new(self.zone, 'foo', { + 'dynamic': { + 'pools': { + 'iad': { + 'values': [{ + 'value': 'iad.unit.tests.', + }], + }, + }, + 'rules': [{ + 'pool': 'iad', + }], + }, + 'octodns': { + 'healthcheck': { + 'host': 'send.me', + 'path': '/_ping', + 'port': 80, + 'protocol': 'HTTP', + } + }, + 'ttl': 33, + 'type': 'CNAME', + 'value': 'value.unit.tests.', + 'meta': {}, + }) + def test_notes(self): provider = Ns1Provider('test', 'api-key') @@ -609,6 +637,12 @@ class TestNs1ProviderDynamic(TestCase): }, 'notes': 'host:unit.tests type:A', } + monitor_five = { + 'config': { + 'host': 'iad.unit.tests', + }, + 'notes': 'host:foo.unit.tests type:CNAME', + } provider._client._monitors_cache = { 'one': monitor_one, 'two': { @@ -624,6 +658,7 @@ class TestNs1ProviderDynamic(TestCase): 'notes': 'host:other.unit.tests type:A', }, 'four': monitor_four, + 'five': monitor_five, } # Would match, but won't get there b/c it's not dynamic @@ -641,6 +676,11 @@ class TestNs1ProviderDynamic(TestCase): '2.3.4.5': monitor_four, }, provider._monitors_for(self.record())) + # Check match for CNAME values + self.assertEquals({ + 'iad.unit.tests.': monitor_five, + }, provider._monitors_for(self.cname_record())) + def test_uuid(self): # Just a smoke test/for coverage provider = Ns1Provider('test', 'api-key') @@ -728,6 +768,14 @@ class TestNs1ProviderDynamic(TestCase): # No http response expected self.assertFalse('rules' in monitor) + def test_monitor_gen_CNAME(self): + provider = Ns1Provider('test', 'api-key') + + value = 'iad.unit.tests.' + record = self.cname_record() + monitor = provider._monitor_gen(record, value) + self.assertEquals(value[:-1], monitor['config']['host']) + def test_monitor_is_match(self): provider = Ns1Provider('test', 'api-key') @@ -1295,32 +1343,7 @@ class TestNs1ProviderDynamic(TestCase): ('mid-1', 'fid-1'), ] - record = Record.new(self.zone, 'foo', { - 'dynamic': { - 'pools': { - 'iad': { - 'values': [{ - 'value': 'iad.unit.tests.', - }], - }, - }, - 'rules': [{ - 'pool': 'iad', - }], - }, - 'octodns': { - 'healthcheck': { - 'host': 'send.me', - 'path': '/_ping', - 'port': 80, - 'protocol': 'HTTP', - } - }, - 'ttl': 32, - 'type': 'CNAME', - 'value': 'value.unit.tests.', - 'meta': {}, - }) + record = self.cname_record() ret, _ = provider._params_for_CNAME(record) # Check if the default value was correctly read and populated @@ -1539,7 +1562,7 @@ class TestNs1ProviderDynamic(TestCase): catchall_pool_name = 'iad__catchall' ns1_record = { 'answers': [{ - 'answer': ['2.3.4.5.unit.tests.'], + 'answer': ['iad.unit.tests.'], 'meta': { 'priority': 1, 'weight': 12, @@ -1547,7 +1570,7 @@ class TestNs1ProviderDynamic(TestCase): }, 'region': catchall_pool_name, }, { - 'answer': ['1.2.3.4.unit.tests.'], + 'answer': ['value.unit.tests.'], 'meta': { 'priority': 2, 'note': 'from:--default--', @@ -1564,7 +1587,7 @@ class TestNs1ProviderDynamic(TestCase): } }, 'tier': 3, - 'ttl': 42, + 'ttl': 43, 'type': 'CNAME', } data = provider._data_for_CNAME('CNAME', ns1_record) @@ -1574,7 +1597,7 @@ class TestNs1ProviderDynamic(TestCase): 'iad': { 'fallback': None, 'values': [{ - 'value': '2.3.4.5.unit.tests.', + 'value': 'iad.unit.tests.', 'weight': 12, }], }, @@ -1584,9 +1607,9 @@ class TestNs1ProviderDynamic(TestCase): 'pool': 'iad', }], }, - 'ttl': 42, + 'ttl': 43, 'type': 'CNAME', - 'value': '1.2.3.4.unit.tests.', + 'value': 'value.unit.tests.', }, data) @patch('ns1.rest.records.Records.retrieve') From 3de5cd27404f5fae35feb14e7e4f01addace2528 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Fri, 30 Apr 2021 20:38:08 -0700 Subject: [PATCH 24/58] More future proof index lookup --- tests/test_octodns_provider_ns1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index a5c41a6..c14173b 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -1348,7 +1348,7 @@ class TestNs1ProviderDynamic(TestCase): # Check if the default value was correctly read and populated # All other dynamic record test cases are covered by dynamic_A tests - self.assertEquals(ret['answers'][1]['answer'][0], 'value.unit.tests.') + self.assertEquals(ret['answers'][-1]['answer'][0], 'value.unit.tests.') def test_data_for_dynamic_A(self): provider = Ns1Provider('test', 'api-key') From 6d7cab43e881247201ea77cd5ef281fb925674a6 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Mon, 3 May 2021 10:59:12 -0700 Subject: [PATCH 25/58] Rename data/params for dynamic methods --- octodns/provider/ns1.py | 14 +++++++------- tests/test_octodns_provider_ns1.py | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 6ade3b9..38f9da3 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -464,10 +464,10 @@ class Ns1Provider(BaseProvider): pass return pool_name - def _data_for_dynamic_A(self, _type, record): + def _data_for_dynamic(self, _type, record): # First make sure we have the expected filters config if not self._valid_filter_config(record['filters'], record['domain']): - self.log.error('_data_for_dynamic_A: %s %s has unsupported ' + self.log.error('_data_for_dynamic: %s %s has unsupported ' 'filters', record['domain'], _type) raise Ns1Exception('Unrecognized advanced record') @@ -613,7 +613,7 @@ class Ns1Provider(BaseProvider): first_answer_note = '' # If that note includes a `from` (pool name) it's a dynamic record if 'from:' in first_answer_note: - return self._data_for_dynamic_A(_type, record) + return self._data_for_dynamic(_type, record) # If not it's an old geo record return self._data_for_geo_A(_type, record) @@ -654,7 +654,7 @@ class Ns1Provider(BaseProvider): def _data_for_CNAME(self, _type, record): if record.get('tier', 1) > 1: # Advanced dynamic record - return self._data_for_dynamic_A(_type, record) + return self._data_for_dynamic(_type, record) try: value = record['short_answers'][0] @@ -1031,7 +1031,7 @@ class Ns1Provider(BaseProvider): } answers.append(answer) - def _params_for_dynamic_A(self, record): + def _params_for_dynamic(self, record): pools = record.dynamic.pools # Convert rules to regions @@ -1168,7 +1168,7 @@ class Ns1Provider(BaseProvider): def _params_for_A(self, record): if getattr(record, 'dynamic', False): - return self._params_for_dynamic_A(record) + return self._params_for_dynamic(record) elif hasattr(record, 'geo'): return self._params_for_geo_A(record) @@ -1195,7 +1195,7 @@ class Ns1Provider(BaseProvider): def _params_for_CNAME(self, record): if getattr(record, 'dynamic', False): - return self._params_for_dynamic_A(record) + return self._params_for_dynamic(record) return {'answers': [record.value], 'ttl': record.ttl}, None diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index c14173b..4d6f02a 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -1290,7 +1290,7 @@ class TestNs1ProviderDynamic(TestCase): ('mid-2', 'fid-2'), ('mid-3', 'fid-3'), ] - # This indirectly calls into _params_for_dynamic_A and tests the + # This indirectly calls into _params_for_dynamic and tests the # handling to get there record = self.record() ret, _ = provider._params_for_A(record) @@ -1350,7 +1350,7 @@ class TestNs1ProviderDynamic(TestCase): # All other dynamic record test cases are covered by dynamic_A tests self.assertEquals(ret['answers'][-1]['answer'][0], 'value.unit.tests.') - def test_data_for_dynamic_A(self): + def test_data_for_dynamic(self): provider = Ns1Provider('test', 'api-key') # Unexpected filters throws an error @@ -1359,7 +1359,7 @@ class TestNs1ProviderDynamic(TestCase): 'filters': [], } with self.assertRaises(Ns1Exception) as ctx: - provider._data_for_dynamic_A('A', ns1_record) + provider._data_for_dynamic('A', ns1_record) self.assertEquals('Unrecognized advanced record', text_type(ctx.exception)) @@ -1371,7 +1371,7 @@ class TestNs1ProviderDynamic(TestCase): 'regions': {}, 'ttl': 42, } - data = provider._data_for_dynamic_A('A', ns1_record) + data = provider._data_for_dynamic('A', ns1_record) self.assertEquals({ 'dynamic': { 'pools': {}, @@ -1476,7 +1476,7 @@ class TestNs1ProviderDynamic(TestCase): 'tier': 3, 'ttl': 42, } - data = provider._data_for_dynamic_A('A', ns1_record) + data = provider._data_for_dynamic('A', ns1_record) self.assertEquals({ 'dynamic': { 'pools': { @@ -1520,7 +1520,7 @@ class TestNs1ProviderDynamic(TestCase): }, data) # Same answer if we go through _data_for_A which out sources the job to - # _data_for_dynamic_A + # _data_for_dynamic data2 = provider._data_for_A('A', ns1_record) self.assertEquals(data, data2) @@ -1531,7 +1531,7 @@ class TestNs1ProviderDynamic(TestCase): ns1_record['regions'][old_style_catchall_pool_name] = \ ns1_record['regions'][catchall_pool_name] del ns1_record['regions'][catchall_pool_name] - data3 = provider._data_for_dynamic_A('A', ns1_record) + data3 = provider._data_for_dynamic('A', ns1_record) self.assertEquals(data, data2) # Oceania test cases From ebfb9355b136dfce46db0905bd6c4e714d6563b7 Mon Sep 17 00:00:00 2001 From: omar Date: Mon, 10 May 2021 19:32:38 -0700 Subject: [PATCH 26/58] Update the AzureProvider to support azure-mgmt-dns 8.0.0 and azure-identity. --- README.md | 2 +- octodns/provider/azuredns.py | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 4ab3c6b..0a04e27 100644 --- a/README.md +++ b/README.md @@ -192,7 +192,7 @@ The above command pulled the existing data out of Route53 and placed the results | Provider | Requirements | Record Support | Dynamic | Notes | |--|--|--|--|--| -| [AzureProvider](/octodns/provider/azuredns.py) | azure-mgmt-dns | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | No | | +| [AzureProvider](/octodns/provider/azuredns.py) | azure-identity, azure-mgmt-dns | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | No | | | [Akamai](/octodns/provider/edgedns.py) | edgegrid-python | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | | | [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, LOC, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [ConstellixProvider](/octodns/provider/constellix.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 2fca5af..282077f 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -5,7 +5,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from azure.common.credentials import ServicePrincipalCredentials +from azure.identity import ClientSecretCredential from azure.mgmt.dns import DnsManagementClient from azure.mgmt.dns.models import ARecord, AaaaRecord, CaaRecord, \ @@ -71,10 +71,10 @@ class _AzureRecord(object): '''Constructor for _AzureRecord. Notes on Azure records: An Azure record set has the form - RecordSet(name=<...>, type=<...>, arecords=[...], aaaa_records, ..) + RecordSet(name=<...>, type=<...>, a_records=[...], aaaa_records, ..) When constructing an azure record as done in self._apply_Create, the argument parameters for an A record would be - parameters={'ttl': , 'arecords': [ARecord(),]}. + parameters={'ttl': , 'a_records': [ARecord(),]}. As another example for CNAME record: parameters={'ttl': , 'cname_record': CnameRecord()}. @@ -263,7 +263,7 @@ def _parse_azure_type(string): def _check_for_alias(azrecord): - if (azrecord.target_resource.id and not azrecord.arecords and not + if (azrecord.target_resource.id and not azrecord.a_records and not azrecord.cname_record): return True return False @@ -343,14 +343,14 @@ class AzureProvider(BaseProvider): @property def _dns_client(self): if self.__dns_client is None: - credentials = ServicePrincipalCredentials( - self._dns_client_client_id, - secret=self._dns_client_key, - tenant=self._dns_client_directory_id + credential = ClientSecretCredential( + client_id=self._dns_client_client_id, + client_secret=self._dns_client_key, + tenant_id=self._dns_client_directory_id ) self.__dns_client = DnsManagementClient( - credentials, - self._dns_client_subscription_id + credential=credential, + subscription_id=self._dns_client_subscription_id ) return self.__dns_client @@ -452,7 +452,7 @@ class AzureProvider(BaseProvider): return exists def _data_for_A(self, azrecord): - return {'values': [ar.ipv4_address for ar in azrecord.arecords]} + return {'values': [ar.ipv4_address for ar in azrecord.a_records]} def _data_for_AAAA(self, azrecord): return {'values': [ar.ipv6_address for ar in azrecord.aaaa_records]} From 93de918e01b0c97a89ba8ff74f926534c966863b Mon Sep 17 00:00:00 2001 From: omar Date: Mon, 10 May 2021 20:38:30 -0700 Subject: [PATCH 27/58] Fix lint, requirements.txt, and all the tests but one. --- octodns/provider/azuredns.py | 3 ++- requirements.txt | 5 +++-- tests/test_octodns_provider_azuredns.py | 28 ++++++++++++------------- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 282077f..7a1883f 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -71,7 +71,8 @@ class _AzureRecord(object): '''Constructor for _AzureRecord. Notes on Azure records: An Azure record set has the form - RecordSet(name=<...>, type=<...>, a_records=[...], aaaa_records, ..) + RecordSet(name=<...>, type=<...>, a_records=[...], + aaaa_records=[...], ...) When constructing an azure record as done in self._apply_Create, the argument parameters for an A record would be parameters={'ttl': , 'a_records': [ARecord(),]}. diff --git a/requirements.txt b/requirements.txt index 933ac60..54bd2a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ PyYaml==5.4 -azure-common==1.1.25 -azure-mgmt-dns==3.0.0 +azure-common==1.1.27 +azure-identity==1.5.0 +azure-mgmt-dns==8.0.0 boto3==1.15.9 botocore==1.18.9 dnspython==1.16.0 diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 9523b51..d9b2b5a 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -152,8 +152,8 @@ _base0.zone_name = 'unit.tests' _base0.relative_record_set_name = '@' _base0.record_type = 'A' _base0.params['ttl'] = 0 -_base0.params['arecords'] = [ARecord(ipv4_address='1.2.3.4'), - ARecord(ipv4_address='10.10.10.10')] +_base0.params['a_records'] = [ARecord(ipv4_address='1.2.3.4'), + ARecord(ipv4_address='10.10.10.10')] azure_records.append(_base0) _base1 = _AzureRecord('TestAzure', octo_records[1]) @@ -161,8 +161,8 @@ _base1.zone_name = 'unit.tests' _base1.relative_record_set_name = 'a' _base1.record_type = 'A' _base1.params['ttl'] = 1 -_base1.params['arecords'] = [ARecord(ipv4_address='1.2.3.4'), - ARecord(ipv4_address='1.1.1.1')] +_base1.params['a_records'] = [ARecord(ipv4_address='1.2.3.4'), + ARecord(ipv4_address='1.1.1.1')] azure_records.append(_base1) _base2 = _AzureRecord('TestAzure', octo_records[2]) @@ -170,7 +170,7 @@ _base2.zone_name = 'unit.tests' _base2.relative_record_set_name = 'aa' _base2.record_type = 'A' _base2.params['ttl'] = 9001 -_base2.params['arecords'] = ARecord(ipv4_address='1.2.4.3') +_base2.params['a_records'] = ARecord(ipv4_address='1.2.4.3') azure_records.append(_base2) _base3 = _AzureRecord('TestAzure', octo_records[3]) @@ -178,7 +178,7 @@ _base3.zone_name = 'unit.tests' _base3.relative_record_set_name = 'aaa' _base3.record_type = 'A' _base3.params['ttl'] = 2 -_base3.params['arecords'] = ARecord(ipv4_address='1.1.1.3') +_base3.params['a_records'] = ARecord(ipv4_address='1.1.1.3') azure_records.append(_base3) _base4 = _AzureRecord('TestAzure', octo_records[4]) @@ -366,7 +366,7 @@ class Test_CheckAzureAlias(TestCase): alias_record = type('C', (object,), {}) alias_record.target_resource = type('C', (object,), {}) alias_record.target_resource.id = "/subscriptions/x/resourceGroups/y/z" - alias_record.arecords = None + alias_record.a_records = None alias_record.cname_record = None self.assertEquals(_check_for_alias(alias_record), True) @@ -377,7 +377,7 @@ class TestAzureDnsProvider(TestCase): return self._get_provider('mock_spc', 'mock_dns_client') @patch('octodns.provider.azuredns.DnsManagementClient') - @patch('octodns.provider.azuredns.ServicePrincipalCredentials') + @patch('octodns.provider.azuredns.ClientSecretCredential') def _get_provider(self, mock_spc, mock_dns_client): '''Returns a mock AzureProvider object to use in testing. @@ -399,12 +399,12 @@ class TestAzureDnsProvider(TestCase): provider = self._get_provider() rs = [] - recordSet = RecordSet(arecords=[ARecord(ipv4_address='1.1.1.1')]) + recordSet = RecordSet(a_records=[ARecord(ipv4_address='1.1.1.1')]) recordSet.name, recordSet.ttl, recordSet.type = 'a1', 0, 'A' recordSet.target_resource = SubResource() rs.append(recordSet) - recordSet = RecordSet(arecords=[ARecord(ipv4_address='1.1.1.1'), - ARecord(ipv4_address='2.2.2.2')]) + recordSet = RecordSet(a_records=[ARecord(ipv4_address='1.1.1.1'), + ARecord(ipv4_address='2.2.2.2')]) recordSet.name, recordSet.ttl, recordSet.type = 'a2', 1, 'A' recordSet.target_resource = SubResource() rs.append(recordSet) @@ -575,11 +575,11 @@ class TestAzureDnsProvider(TestCase): provider = self._get_provider() rs = [] - recordSet = RecordSet(arecords=[ARecord(ipv4_address='1.1.1.1')]) + recordSet = RecordSet(a_records=[ARecord(ipv4_address='1.1.1.1')]) recordSet.name, recordSet.ttl, recordSet.type = 'a1', 0, 'A' rs.append(recordSet) - recordSet = RecordSet(arecords=[ARecord(ipv4_address='1.1.1.1'), - ARecord(ipv4_address='2.2.2.2')]) + recordSet = RecordSet(a_records=[ARecord(ipv4_address='1.1.1.1'), + ARecord(ipv4_address='2.2.2.2')]) recordSet.name, recordSet.ttl, recordSet.type = 'a2', 1, 'A' rs.append(recordSet) From 758c7fab61209084dc96cc845b08fa154835943c Mon Sep 17 00:00:00 2001 From: omar Date: Mon, 10 May 2021 20:49:28 -0700 Subject: [PATCH 28/58] Fix the last test. --- octodns/provider/azuredns.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 7a1883f..83cfeb0 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -102,8 +102,7 @@ class _AzureRecord(object): return # Refer to function docstring for key_name and class_name. - format_u_s = '' if record._type == 'A' else '_' - key_name = '{}{}records'.format(self.record_type, format_u_s).lower() + key_name = '{}_records'.format(self.record_type).lower() if record._type == 'CNAME': key_name = key_name[:len(key_name) - 1] azure_class = self.TYPE_MAP[self.record_type] From d619025040dc1e8087290e70554f9ccc9c028b78 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Tue, 11 May 2021 15:39:00 -0700 Subject: [PATCH 29/58] Support dynamic records in Azure DNS --- README.md | 2 +- octodns/provider/azuredns.py | 630 +++++++++++++++-- requirements.txt | 1 + tests/test_octodns_provider_azuredns.py | 887 +++++++++++++++++++++++- 4 files changed, 1439 insertions(+), 81 deletions(-) diff --git a/README.md b/README.md index 0a04e27..8d59cd0 100644 --- a/README.md +++ b/README.md @@ -192,7 +192,7 @@ The above command pulled the existing data out of Route53 and placed the results | Provider | Requirements | Record Support | Dynamic | Notes | |--|--|--|--|--| -| [AzureProvider](/octodns/provider/azuredns.py) | azure-identity, azure-mgmt-dns | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | No | | +| [AzureProvider](/octodns/provider/azuredns.py) | azure-identity, azure-mgmt-dns, azure-mgmt-trafficmanager | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | Yes (CNAMEs only) | | | [Akamai](/octodns/provider/edgedns.py) | edgegrid-python | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | | | [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, LOC, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [ConstellixProvider](/octodns/provider/constellix.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 83cfeb0..4d3dcc5 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -5,18 +5,28 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +from collections import defaultdict + from azure.identity import ClientSecretCredential +from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient +from azure.mgmt.trafficmanager import TrafficManagerManagementClient from azure.mgmt.dns.models import ARecord, AaaaRecord, CaaRecord, \ CnameRecord, MxRecord, SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone +from azure.mgmt.trafficmanager.models import Profile, DnsConfig, \ + MonitorConfig, Endpoint, MonitorConfigCustomHeadersItem import logging from functools import reduce -from ..record import Record +from ..record import Record, Update, GeoCodes from .base import BaseProvider +class AzureException(Exception): + pass + + def escape_semicolon(s): assert s return s.replace(';', '\\;') @@ -67,7 +77,8 @@ class _AzureRecord(object): 'TXT': TxtRecord } - def __init__(self, resource_group, record, delete=False): + def __init__(self, resource_group, record, delete=False, + traffic_manager=None): '''Constructor for _AzureRecord. Notes on Azure records: An Azure record set has the form @@ -94,9 +105,11 @@ class _AzureRecord(object): self.log = logging.getLogger('AzureRecord') self.resource_group = resource_group - self.zone_name = record.zone.name[:len(record.zone.name) - 1] + self.zone_name = record.zone.name[:-1] self.relative_record_set_name = record.name or '@' self.record_type = record._type + self._record = record + self.traffic_manager = traffic_manager if delete: return @@ -104,11 +117,11 @@ class _AzureRecord(object): # Refer to function docstring for key_name and class_name. key_name = '{}_records'.format(self.record_type).lower() if record._type == 'CNAME': - key_name = key_name[:len(key_name) - 1] + key_name = key_name[:-1] azure_class = self.TYPE_MAP[self.record_type] - self.params = getattr(self, '_params_for_{}'.format(record._type)) - self.params = self.params(record.data, key_name, azure_class) + params_for = getattr(self, '_params_for_{}'.format(record._type)) + self.params = params_for(record.data, key_name, azure_class) self.params['ttl'] = record.ttl def _params_for_A(self, data, key_name, azure_class): @@ -139,6 +152,9 @@ class _AzureRecord(object): return {key_name: params} def _params_for_CNAME(self, data, key_name, azure_class): + if self._record.dynamic and self.traffic_manager: + return {'target_resource': self.traffic_manager} + return {key_name: azure_class(cname=data['value'])} def _params_for_MX(self, data, key_name, azure_class): @@ -227,25 +243,6 @@ class _AzureRecord(object): (parse_dict(self.params) == parse_dict(b.params)) & \ (self.relative_record_set_name == b.relative_record_set_name) - def __str__(self): - '''String representation of an _AzureRecord. - :type return: str - ''' - string = 'Zone: {}; '.format(self.zone_name) - string += 'Name: {}; '.format(self.relative_record_set_name) - string += 'Type: {}; '.format(self.record_type) - if not hasattr(self, 'params'): - return string - string += 'Ttl: {}; '.format(self.params['ttl']) - for char in self.params: - if char != 'ttl': - try: - for rec in self.params[char]: - string += 'Record: {}; '.format(rec.__dict__) - except: - string += 'Record: {}; '.format(self.params[char].__dict__) - return string - def _check_endswith_dot(string): return string if string.endswith('.') else string + '.' @@ -259,14 +256,88 @@ def _parse_azure_type(string): :type return: str ''' - return string.split('/')[len(string.split('/')) - 1] + return string.split('/')[-1] -def _check_for_alias(azrecord): - if (azrecord.target_resource.id and not azrecord.a_records and not - azrecord.cname_record): - return True - return False +def _traffic_manager_suffix(record): + return record.fqdn[:-1].replace('.', '-') + + +def _get_monitor(record): + monitor = MonitorConfig( + protocol=record.healthcheck_protocol, + port=record.healthcheck_port, + path=record.healthcheck_path, + ) + host = record.healthcheck_host + if host: + monitor.custom_headers = [MonitorConfigCustomHeadersItem( + name='Host', value=host + )] + return monitor + + +def _profile_is_match(have, desired): + if have is None or desired is None: + return False + + # compare basic attributes + if have.name != desired.name or \ + have.traffic_routing_method != desired.traffic_routing_method or \ + have.dns_config.ttl != desired.dns_config.ttl or \ + len(have.endpoints) != len(desired.endpoints): + return False + + # compare monitoring configuration + monitor_have = have.monitor_config + monitor_desired = desired.monitor_config + if monitor_have.protocol != monitor_desired.protocol or \ + monitor_have.port != monitor_desired.port or \ + monitor_have.path != monitor_desired.path or \ + monitor_have.custom_headers != monitor_desired.custom_headers: + return False + + # compare endpoints + method = have.traffic_routing_method + if method == 'Priority': + have_endpoints = sorted(have.endpoints, key=lambda e: e.priority) + desired_endpoints = sorted(desired.endpoints, + key=lambda e: e.priority) + elif method == 'Weighted': + have_endpoints = sorted(have.endpoints, key=lambda e: e.target) + desired_endpoints = sorted(desired.endpoints, key=lambda e: e.target) + else: + have_endpoints = have.endpoints + desired_endpoints = desired.endpoints + endpoints = zip(have_endpoints, desired_endpoints) + for have_endpoint, desired_endpoint in endpoints: + if have_endpoint.name != desired_endpoint.name or \ + have_endpoint.type != desired_endpoint.type: + return False + target_type = have_endpoint.type.split('/')[-1] + if target_type == 'externalEndpoints': + # compare value, weight, priority + if have_endpoint.target != desired_endpoint.target: + return False + if method == 'Weighted' and \ + have_endpoint.weight != desired_endpoint.weight: + return False + elif target_type == 'nestedEndpoints': + # compare targets + if have_endpoint.target_resource_id != \ + desired_endpoint.target_resource_id: + return False + # compare geos + if method == 'Geographic': + have_geos = sorted(have_endpoint.geo_mapping) + desired_geos = sorted(desired_endpoint.geo_mapping) + if have_geos != desired_geos: + return False + else: + # unexpected, give up + return False + + return True class AzureProvider(BaseProvider): @@ -318,7 +389,7 @@ class AzureProvider(BaseProvider): possible to also hard-code into the config file: eg, resource_group. ''' SUPPORTS_GEO = False - SUPPORTS_DYNAMIC = False + SUPPORTS_DYNAMIC = True SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', 'TXT')) @@ -336,24 +407,45 @@ class AzureProvider(BaseProvider): self._dns_client_directory_id = directory_id self._dns_client_subscription_id = sub_id self.__dns_client = None + self.__tm_client = None self._resource_group = resource_group self._azure_zones = set() + self._traffic_managers = dict() @property def _dns_client(self): if self.__dns_client is None: - credential = ClientSecretCredential( - client_id=self._dns_client_client_id, - client_secret=self._dns_client_key, - tenant_id=self._dns_client_directory_id - ) + # Azure's logger spits out a lot of debug messages at 'INFO' + # level, override it by re-assigning `info` method to `debug` + # (ugly hack until I find a better way) + logger_name = 'azure.core.pipeline.policies.http_logging_policy' + logger = logging.getLogger(logger_name) + logger.info = logger.debug self.__dns_client = DnsManagementClient( - credential=credential, - subscription_id=self._dns_client_subscription_id + credential=ClientSecretCredential( + client_id=self._dns_client_client_id, + client_secret=self._dns_client_key, + tenant_id=self._dns_client_directory_id, + logger=logger, + ), + subscription_id=self._dns_client_subscription_id, ) return self.__dns_client + @property + def _tm_client(self): + if self.__tm_client is None: + self.__tm_client = TrafficManagerManagementClient( + ServicePrincipalCredentials( + self._dns_client_client_id, + secret=self._dns_client_key, + tenant=self._dns_client_directory_id, + ), + self._dns_client_subscription_id, + ) + return self.__tm_client + def _populate_zones(self): self.log.debug('azure_zones: loading') list_zones = self._dns_client.zones.list_by_resource_group @@ -388,6 +480,42 @@ class AzureProvider(BaseProvider): # Else return nothing (aka false) return + def _populate_traffic_managers(self): + self.log.debug('traffic managers: loading') + list_profiles = self._tm_client.profiles.list_by_resource_group + for profile in list_profiles(self._resource_group): + self._traffic_managers[profile.id] = profile + # link nested profiles in advance for convenience + for _, profile in self._traffic_managers.items(): + self._populate_nested_profiles(profile) + + def _populate_nested_profiles(self, profile): + for ep in profile.endpoints: + target_id = ep.target_resource_id + if target_id and target_id in self._traffic_managers: + target = self._traffic_managers[target_id] + ep.target_resource = self._populate_nested_profiles(target) + return profile + + def _get_tm_profile_by_id(self, resource_id): + if not self._traffic_managers: + self._populate_traffic_managers() + return self._traffic_managers.get(resource_id) + + def _profile_name_to_id(self, name): + return '/subscriptions/' + self._dns_client_subscription_id + \ + '/resourceGroups/' + self._resource_group + \ + '/providers/Microsoft.Network/trafficManagerProfiles/' + \ + name + + def _get_tm_profile_by_name(self, name): + profile_id = self._profile_name_to_id(name) + return self._get_tm_profile_by_id(profile_id) + + def _get_tm_for_dynamic_record(self, record): + name = _traffic_manager_suffix(record) + return self._get_tm_profile_by_name(name) + def populate(self, zone, target=False, lenient=False): '''Required function of manager.py to collect records from zone. @@ -417,40 +545,35 @@ class AzureProvider(BaseProvider): exists = False before = len(zone.records) - zone_name = zone.name[:len(zone.name) - 1] + zone_name = zone.name[:-1] self._populate_zones() - self._check_zone(zone_name) - _records = [] records = self._dns_client.record_sets.list_by_dns_zone if self._check_zone(zone_name): exists = True for azrecord in records(self._resource_group, zone_name): - if _parse_azure_type(azrecord.type) in self.SUPPORTS: - _records.append(azrecord) - for azrecord in _records: - record_name = azrecord.name if azrecord.name != '@' else '' typ = _parse_azure_type(azrecord.type) + if typ not in self.SUPPORTS: + continue - if typ in ['A', 'CNAME']: - if _check_for_alias(azrecord): - self.log.debug( - 'Skipping - ALIAS. zone=%s record=%s, type=%s', - zone_name, record_name, typ) # pragma: no cover - continue # pragma: no cover - - data = getattr(self, '_data_for_{}'.format(typ)) - data = data(azrecord) - data['type'] = typ - data['ttl'] = azrecord.ttl - record = Record.new(zone, record_name, data, source=self) - + record = self._populate_record(zone, azrecord, lenient) zone.add_record(record, lenient=lenient) self.log.info('populate: found %s records, exists=%s', len(zone.records) - before, exists) return exists + def _populate_record(self, zone, azrecord, lenient=False): + record_name = azrecord.name if azrecord.name != '@' else '' + typ = _parse_azure_type(azrecord.type) + + data_for = getattr(self, '_data_for_{}'.format(typ)) + data = data_for(azrecord) + data['type'] = typ + data['ttl'] = azrecord.ttl + return Record.new(zone, record_name, data, source=self, + lenient=lenient) + def _data_for_A(self, azrecord): return {'values': [ar.ipv4_address for ar in azrecord.a_records]} @@ -470,6 +593,9 @@ class AzureProvider(BaseProvider): :type return: dict ''' + if azrecord.cname_record is None and azrecord.target_resource.id: + return self._data_for_dynamic(azrecord) + return {'value': _check_endswith_dot(azrecord.cname_record.cname)} def _data_for_MX(self, azrecord): @@ -495,6 +621,322 @@ class AzureProvider(BaseProvider): ar.value)) for ar in azrecord.txt_records]} + def _data_for_dynamic(self, azrecord): + default = set() + pools = defaultdict(lambda: {'fallback': None, 'values': []}) + rules = [] + + # top level geo profile + geo_profile = self._get_tm_profile_by_id(azrecord.target_resource.id) + for geo_ep in geo_profile.endpoints: + rule = {} + + # resolve list of regions + geo_map = list(geo_ep.geo_mapping) + if geo_map != ['WORLD']: + if 'GEO-ME' in geo_map: + # Azure treats Middle East as a separate group, but + # its part of Asia in octoDNS, so we need to remove GEO-ME + # if GEO-AS is also in the list + # Throw exception otherwise, it should not happen if the + # profile was generated by octoDNS + if 'GEO-AS' not in geo_map: + msg = '_data_for_dynamic: Profile={}: '.format( + geo_profile.name) + msg += 'Middle East (GEO-ME) is not supported by ' + \ + 'octoDNS. It needs to be either paired ' + \ + 'with Asia (GEO-AS) or expanded into ' + \ + 'individual list of countries.' + raise AzureException(msg) + geo_map.remove('GEO-ME') + geos = rule.setdefault('geos', []) + for code in geo_map: + if code.startswith('GEO-'): + geos.append(code[len('GEO-'):]) + elif '-' in code: + country, province = code.split('-', 1) + country = GeoCodes.country_to_code(country) + geos.append('{}-{}'.format(country, province)) + else: + geos.append(GeoCodes.country_to_code(code)) + + # second level priority profile + pool = None + rule_endpoints = geo_ep.target_resource.endpoints + rule_endpoints = sorted(rule_endpoints, key=lambda e: e.priority) + for rule_ep in rule_endpoints: + pool_name = rule_ep.name + + # third (and last) level weighted RR profile + # these should be leaf node profiles with no further nesting + pool_profile = rule_ep.target_resource + + # last/default pool + if pool_name == '--default--': + for pool_ep in pool_profile.endpoints: + default.add(pool_ep.target) + # this should be the last one, so let's break here + break + + # set first priority endpoint as the rule's primary pool + if 'pool' not in rule: + rule['pool'] = pool_name + + if pool: + # set current pool as fallback of the previous pool + pool['fallback'] = pool_name + + pool = pools[pool_name] + for pool_ep in pool_profile.endpoints: + val = pool_ep.target + value_dict = { + 'value': _check_endswith_dot(val), + 'weight': pool_ep.weight, + } + if value_dict not in pool['values']: + pool['values'].append(value_dict) + + if 'pool' not in rule or not default: + # this will happen if the priority profile does not have + # enough endpoints + msg = 'Expected at least 2 endpoints in {}, got {}'.format( + geo_ep.target_resource.name, len(rule_endpoints) + ) + raise AzureException(msg) + rules.append(rule) + + # Order and convert to a list + default = sorted(default) + + data = { + 'dynamic': { + 'pools': pools, + 'rules': rules, + }, + 'value': _check_endswith_dot(default[0]), + } + + return data + + def _extra_changes(self, existing, desired, changes): + changed = set() + + # Abort if there are non-CNAME dynamic records + for change in changes: + record = change.record + changed.add(record) + typ = record._type + dynamic = getattr(record, 'dynamic', False) + if dynamic and typ != 'CNAME': + msg = '{}: Dynamic records in Azure must be of type CNAME' + msg = msg.format(record.fqdn) + raise AzureException(msg) + + log = self.log.info + extra = [] + for record in desired.records: + if not getattr(record, 'dynamic', False): + # Already changed, or not dynamic, no need to check it + continue + + # let's walk through and show what will be changed even if + # the record is already be in list of changes + added = (record in changed) + + active = set() + profiles = self._generate_traffic_managers(record) + for profile in profiles: + name = profile.name + active.add(name) + existing_profile = self._get_tm_profile_by_name(name) + if not _profile_is_match(existing_profile, profile): + log('_extra_changes: Profile name=%s will be synced', + name) + if not added: + extra.append(Update(record, record)) + added = True + + existing_profiles = self._find_traffic_managers(record) + for name in existing_profiles - active: + log('_extra_changes: Profile name=%s will be destroyed', name) + if not added: + extra.append(Update(record, record)) + added = True + + return extra + + def _generate_tm_profile(self, name, routing, endpoints, record): + # set appropriate endpoint types + endpoint_type_prefix = 'Microsoft.Network/trafficManagerProfiles/' + for ep in endpoints: + if ep.target_resource_id: + ep.type = endpoint_type_prefix + 'nestedEndpoints' + elif ep.target: + ep.type = endpoint_type_prefix + 'externalEndpoints' + else: + msg = ('_generate_tm_profile: Invalid endpoint {} ' + + 'in profile {}, needs to have either target or ' + + 'target_resource_id').format(ep.name, name) + raise AzureException(msg) + + # build and return + return Profile( + id=self._profile_name_to_id(name), + name=name, + traffic_routing_method=routing, + dns_config=DnsConfig( + relative_name=name, + ttl=record.ttl, + ), + monitor_config=_get_monitor(record), + endpoints=endpoints, + location='global', + ) + + def _generate_traffic_managers(self, record): + traffic_managers = [] + pools = record.dynamic.pools + + tm_suffix = _traffic_manager_suffix(record) + profile = self._generate_tm_profile + + # construct the default pool that will be used at the end of + # all rules + target = record.value[:-1] + default_endpoints = [Endpoint( + name=target, + target=target, + weight=1, + )] + default_profile_name = 'default--{}'.format(tm_suffix) + default_profile = profile(default_profile_name, 'Weighted', + default_endpoints, record) + traffic_managers.append(default_profile) + + geo_endpoints = [] + + for rule in record.dynamic.rules: + pool_name = rule.data['pool'] + rule_endpoints = [] + priority = 1 + + while pool_name: + # iterate until we reach end of fallback chain + pool = pools[pool_name].data + profile_name = 'pool-{}--{}'.format(pool_name, tm_suffix) + endpoints = [] + for val in pool['values']: + target = val['value'] + # strip trailing dot from CNAME value + target = target[:-1] + endpoints.append(Endpoint( + name=target, + target=target, + weight=val.get('weight', 1), + )) + pool_profile = profile(profile_name, 'Weighted', endpoints, + record) + traffic_managers.append(pool_profile) + + # append pool to endpoint list of fallback rule profile + rule_endpoints.append(Endpoint( + name=pool_name, + target_resource_id=pool_profile.id, + priority=priority, + )) + + priority += 1 + pool_name = pool.get('fallback') + + # append default profile to the end + rule_endpoints.append(Endpoint( + name='--default--', + target_resource_id=default_profile.id, + priority=priority, + )) + # create rule profile with fallback chain + rule_profile_name = 'rule-{}--{}'.format(rule.data['pool'], + tm_suffix) + rule_profile = profile(rule_profile_name, 'Priority', + rule_endpoints, record) + traffic_managers.append(rule_profile) + + # append rule profile to top-level geo profile + rule_geos = rule.data.get('geos', []) + geos = [] + if len(rule_geos) > 0: + for geo in rule_geos: + if '-' in geo: + geos.append(geo.split('-', 1)[-1]) + else: + geos.append('GEO-{}'.format(geo)) + if geo == 'AS': + # Middle East is part of Asia in octoDNS, but + # Azure treats it as a separate "group", so let's + # add it in the list of geo mappings. We will drop + # it when we later parse the list of regions. + geos.append('GEO-ME') + else: + geos.append('WORLD') + geo_endpoints.append(Endpoint( + name='rule-{}'.format(rule.data['pool']), + target_resource_id=rule_profile.id, + geo_mapping=geos, + )) + + geo_profile = profile(tm_suffix, 'Geographic', geo_endpoints, record) + traffic_managers.append(geo_profile) + + return traffic_managers + + def _sync_traffic_managers(self, record): + desired_profiles = self._generate_traffic_managers(record) + seen = set() + + tm_sync = self._tm_client.profiles.create_or_update + populate = self._populate_nested_profiles + + for desired in desired_profiles: + name = desired.name + if name in seen: + continue + + existing = self._get_tm_profile_by_name(name) + if not _profile_is_match(existing, desired): + self.log.info( + '_sync_traffic_managers: Syncing profile=%s', name) + profile = tm_sync(self._resource_group, name, desired) + self._traffic_managers[profile.id] = populate(profile) + else: + self.log.debug( + '_sync_traffic_managers: Skipping profile=%s: up to date', + name) + seen.add(name) + + return seen + + def _find_traffic_managers(self, record): + tm_suffix = _traffic_manager_suffix(record) + + profiles = set() + for profile_id in self._traffic_managers: + # match existing profiles with record's suffix + name = profile_id.split('/')[-1] + if name == tm_suffix or \ + name.endswith('--{}'.format(tm_suffix)): + profiles.add(name) + + return profiles + + def _traffic_managers_gc(self, record, active_profiles): + existing_profiles = self._find_traffic_managers(record) + + # delete unused profiles + for profile_name in existing_profiles - active_profiles: + self.log.info('_traffic_managers_gc: Deleting profile=%s', + profile_name) + self._tm_client.profiles.delete(self._resource_group, profile_name) + def _apply_Create(self, change): '''A record from change must be created. @@ -503,7 +945,15 @@ class AzureProvider(BaseProvider): :type return: void ''' - ar = _AzureRecord(self._resource_group, change.new) + record = change.new + + dynamic = getattr(record, 'dynamic', False) + if dynamic: + self._sync_traffic_managers(record) + + profile = self._get_tm_for_dynamic_record(record) + ar = _AzureRecord(self._resource_group, record, + traffic_manager=profile) create = self._dns_client.record_sets.create_or_update create(resource_group_name=ar.resource_group, @@ -512,17 +962,71 @@ class AzureProvider(BaseProvider): record_type=ar.record_type, parameters=ar.params) - self.log.debug('* Success Create/Update: {}'.format(ar)) + self.log.debug('* Success Create: {}'.format(record)) - _apply_Update = _apply_Create + def _apply_Update(self, change): + '''A record from change must be created. + + :param change: a change object + :type change: octodns.record.Change + + :type return: void + ''' + existing = change.existing + new = change.new + existing_is_dynamic = getattr(existing, 'dynamic', False) + new_is_dynamic = getattr(new, 'dynamic', False) + + update_record = True + + if new_is_dynamic: + active = self._sync_traffic_managers(new) + # only TTL is configured in record, everything else goes inside + # traffic managers, so no need to update if TTL is unchanged + # and existing record is already aliased to its traffic manager + if existing.ttl == new.ttl and existing_is_dynamic: + update_record = False + + if update_record: + profile = self._get_tm_for_dynamic_record(new) + ar = _AzureRecord(self._resource_group, new, + traffic_manager=profile) + update = self._dns_client.record_sets.create_or_update + + update(resource_group_name=ar.resource_group, + zone_name=ar.zone_name, + relative_record_set_name=ar.relative_record_set_name, + record_type=ar.record_type, + parameters=ar.params) + + if new_is_dynamic: + # let's cleanup unused traffic managers + self._traffic_managers_gc(new, active) + elif existing_is_dynamic: + # cleanup traffic managers when a dynamic record gets + # changed to a simple record + self._traffic_managers_gc(existing, set()) + + self.log.debug('* Success Update: {}'.format(new)) def _apply_Delete(self, change): - ar = _AzureRecord(self._resource_group, change.existing, delete=True) + '''A record from change must be deleted. + + :param change: a change object + :type change: octodns.record.Change + + :type return: void + ''' + record = change.record + ar = _AzureRecord(self._resource_group, record, delete=True) delete = self._dns_client.record_sets.delete delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, ar.record_type) + if getattr(record, 'dynamic', False): + self._traffic_managers_gc(record, set()) + self.log.debug('* Success Delete: {}'.format(ar)) def _apply(self, plan): diff --git a/requirements.txt b/requirements.txt index 54bd2a3..143ba67 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ PyYaml==5.4 azure-common==1.1.27 azure-identity==1.5.0 azure-mgmt-dns==8.0.0 +azure-mgmt-trafficmanager==0.51.0 boto3==1.15.9 botocore==1.18.9 dnspython==1.16.0 diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index d9b2b5a..5655719 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -5,19 +5,23 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from octodns.record import Create, Delete, Record +from octodns.record import Create, Update, Delete, Record from octodns.provider.azuredns import _AzureRecord, AzureProvider, \ - _check_endswith_dot, _parse_azure_type, _check_for_alias + _check_endswith_dot, _parse_azure_type, _traffic_manager_suffix, \ + _get_monitor, _profile_is_match, AzureException from octodns.zone import Zone from octodns.provider.base import Plan from azure.mgmt.dns.models import ARecord, AaaaRecord, CaaRecord, \ CnameRecord, MxRecord, SrvRecord, NsRecord, PtrRecord, TxtRecord, \ RecordSet, SoaRecord, SubResource, Zone as AzureZone +from azure.mgmt.trafficmanager.models import Profile, DnsConfig, \ + MonitorConfig, Endpoint, MonitorConfigCustomHeadersItem from msrestazure.azure_exceptions import CloudError +from six import text_type from unittest import TestCase -from mock import Mock, patch +from mock import Mock, patch, call zone = Zone(name='unit.tests.', sub_zones=[]) @@ -343,6 +347,43 @@ class Test_AzureRecord(TestCase): assert(azure_records[i]._equals(octo)) +class Test_DynamicAzureRecord(TestCase): + def test_azure_record(self): + tm_profile = Profile() + data = { + 'ttl': 60, + 'type': 'CNAME', + 'value': 'default.unit.tests.', + 'dynamic': { + 'pools': { + 'one': { + 'values': [ + {'value': 'one.unit.tests.', 'weight': 1} + ], + 'fallback': 'two', + }, + 'two': { + 'values': [ + {'value': 'two.unit.tests.', 'weight': 1} + ], + }, + }, + 'rules': [ + {'geos': ['AF'], 'pool': 'one'}, + {'pool': 'two'}, + ], + } + } + octo_record = Record.new(zone, 'foo', data) + azure_record = _AzureRecord('TestAzure', octo_record, + traffic_manager=tm_profile) + self.assertEqual(azure_record.zone_name, zone.name[:-1]) + self.assertEqual(azure_record.relative_record_set_name, 'foo') + self.assertEqual(azure_record.record_type, 'CNAME') + self.assertEqual(azure_record.params['ttl'], 60) + self.assertEqual(azure_record.params['target_resource'], tm_profile) + + class Test_ParseAzureType(TestCase): def test_parse_azure_type(self): for expected, test in [['A', 'Microsoft.Network/dnszones/A'], @@ -361,40 +402,369 @@ class Test_CheckEndswithDot(TestCase): self.assertEquals(expected, _check_endswith_dot(test)) -class Test_CheckAzureAlias(TestCase): - def test_check_for_alias(self): - alias_record = type('C', (object,), {}) - alias_record.target_resource = type('C', (object,), {}) - alias_record.target_resource.id = "/subscriptions/x/resourceGroups/y/z" - alias_record.a_records = None - alias_record.cname_record = None +class Test_TrafficManagerSuffix(TestCase): + def test_traffic_manager_suffix(self): + test = Record.new(zone, 'foo', data={ + 'ttl': 60, 'type': 'CNAME', 'value': 'default.unit.tests.', + }) + self.assertEqual(_traffic_manager_suffix(test), 'foo-unit-tests') - self.assertEquals(_check_for_alias(alias_record), True) + +class Test_GetMonitor(TestCase): + def test_get_monitor(self): + record = Record.new(zone, 'foo', data={ + 'type': 'CNAME', 'ttl': 60, 'value': 'default.unit.tests.', + 'octodns': { + 'healthcheck': { + 'path': '/_ping', + 'port': 4443, + 'protocol': 'HTTPS', + } + }, + }) + + monitor = _get_monitor(record) + self.assertEqual(monitor.protocol, 'HTTPS') + self.assertEqual(monitor.port, 4443) + self.assertEqual(monitor.path, '/_ping') + headers = monitor.custom_headers + self.assertIsInstance(headers, list) + self.assertEquals(len(headers), 1) + headers = headers[0] + self.assertEqual(headers.name, 'Host') + self.assertEqual(headers.value, record.healthcheck_host) + + # test TCP monitor + record._octodns['healthcheck']['protocol'] = 'TCP' + monitor = _get_monitor(record) + self.assertEqual(monitor.protocol, 'TCP') + self.assertIsNone(monitor.custom_headers) + + +class Test_ProfileIsMatch(TestCase): + def test_profile_is_match(self): + is_match = _profile_is_match + + self.assertFalse(is_match(None, Profile())) + + # Profile object builder with default property values that can be + # overridden for testing below + def profile( + name = 'foo-unit-tests', + ttl = 60, + method = 'Geographic', + monitor_proto = 'HTTPS', + monitor_port = 4443, + monitor_path = '/_ping', + endpoints = 1, + endpoint_name = 'name', + endpoint_type = 'profile/nestedEndpoints', + target = 'target.unit.tests', + target_id = 'resource/id', + geos = ['GEO-AF'], + weight = 1, + priority = 1, + ): + dns = DnsConfig(ttl=ttl) + return Profile( + name=name, traffic_routing_method=method, dns_config=dns, + monitor_config=MonitorConfig( + protocol=monitor_proto, + port=monitor_port, + path=monitor_path, + ), + endpoints=[Endpoint( + name=endpoint_name, + type=endpoint_type, + target=target, + target_resource_id=target_id, + geo_mapping=geos, + weight=weight, + priority=priority, + )] + [Endpoint()] * (endpoints - 1), + ) + + self.assertTrue(is_match(profile(), profile())) + + self.assertFalse(is_match(profile(), profile(name='two'))) + self.assertFalse(is_match(profile(), profile(endpoints=2))) + self.assertFalse(is_match(profile(), profile(monitor_proto='HTTP'))) + self.assertFalse(is_match(profile(), profile(endpoint_name='a'))) + self.assertFalse(is_match(profile(), profile(endpoint_type='b'))) + self.assertFalse( + is_match(profile(endpoint_type='b'), profile(endpoint_type='b')) + ) + self.assertFalse(is_match(profile(), profile(target_id='rsrc/id2'))) + self.assertFalse(is_match(profile(), profile(geos=['IN']))) + + def wprofile(**kwargs): + kwargs['method'] = 'Weighted' + kwargs['endpoint_type'] = 'profile/externalEndpoints' + return profile(**kwargs) + + self.assertFalse(is_match(wprofile(), wprofile(target='bar.unit'))) + self.assertFalse(is_match(wprofile(), wprofile(weight=3))) class TestAzureDnsProvider(TestCase): def _provider(self): return self._get_provider('mock_spc', 'mock_dns_client') + @patch('octodns.provider.azuredns.TrafficManagerManagementClient') @patch('octodns.provider.azuredns.DnsManagementClient') @patch('octodns.provider.azuredns.ClientSecretCredential') - def _get_provider(self, mock_spc, mock_dns_client): + @patch('octodns.provider.azuredns.ServicePrincipalCredentials') + def _get_provider(self, mock_spc, mock_css, mock_dns_client, + mock_tm_client): '''Returns a mock AzureProvider object to use in testing. :param mock_spc: placeholder :type mock_spc: str :param mock_dns_client: placeholder :type mock_dns_client: str + :param mock_tm_client: placeholder + :type mock_tm_client: str :type return: AzureProvider ''' provider = AzureProvider('mock_id', 'mock_client', 'mock_key', 'mock_directory', 'mock_sub', 'mock_rg' ) + # Fetch the client to force it to load the creds provider._dns_client + + # set critical functions to return properly + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = [] + tm_sync = provider._tm_client.profiles.create_or_update + + def side_effect(rg, name, profile): + return profile + + tm_sync.side_effect = side_effect + return provider + def _get_dynamic_record(self, zone): + return Record.new(zone, 'foo', data={ + 'type': 'CNAME', + 'ttl': 60, + 'value': 'default.unit.tests.', + 'dynamic': { + 'pools': { + 'one': { + 'values': [ + {'value': 'one.unit.tests.', 'weight': 11}, + ], + 'fallback': 'two', + }, + 'two': { + 'values': [ + {'value': 'two1.unit.tests.', 'weight': 3}, + {'value': 'two2.unit.tests.', 'weight': 4}, + ], + 'fallback': 'three', + }, + 'three': { + 'values': [ + {'value': 'three.unit.tests.', 'weight': 13}, + ], + }, + }, + 'rules': [ + {'geos': ['AF', 'EU-DE', 'NA-US-CA'], 'pool': 'one'}, + {'pool': 'three'}, + ], + }, + 'octodns': { + 'healthcheck': { + 'path': '/_ping', + 'port': 4443, + 'protocol': 'HTTPS', + } + }, + }) + + def _get_tm_profiles(self, provider): + sub = provider._dns_client_subscription_id + rg = provider._resource_group + base_id = '/subscriptions/' + sub + \ + '/resourceGroups/' + rg + \ + '/providers/Microsoft.Network/trafficManagerProfiles/' + suffix = 'foo-unit-tests' + id_format = base_id + '{}--' + suffix + name_format = '{}--' + suffix + + dns = DnsConfig(ttl=60) + header = MonitorConfigCustomHeadersItem(name='Host', + value='foo.unit.tests') + monitor = MonitorConfig(protocol='HTTPS', port=4443, path='/_ping', + custom_headers=[header]) + external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' + + return [ + Profile( + id=id_format.format('default'), + name=name_format.format('default'), + traffic_routing_method='Weighted', + dns_config=dns, + monitor_config=monitor, + endpoints=[ + Endpoint( + name='default.unit.tests', + type=external, + target='default.unit.tests', + weight=1, + ), + ], + ), + Profile( + id=id_format.format('pool-one'), + name=name_format.format('pool-one'), + traffic_routing_method='Weighted', + dns_config=dns, + monitor_config=monitor, + endpoints=[ + Endpoint( + name='one.unit.tests', + type=external, + target='one.unit.tests', + weight=11, + ), + ], + ), + Profile( + id=id_format.format('pool-two'), + name=name_format.format('pool-two'), + traffic_routing_method='Weighted', + dns_config=dns, + monitor_config=monitor, + endpoints=[ + Endpoint( + name='two1.unit.tests', + type=external, + target='two1.unit.tests', + weight=3, + ), + Endpoint( + name='two2.unit.tests', + type=external, + target='two2.unit.tests', + weight=4, + ), + ], + ), + Profile( + id=id_format.format('pool-three'), + name=name_format.format('pool-three'), + traffic_routing_method='Weighted', + dns_config=dns, + monitor_config=monitor, + endpoints=[ + Endpoint( + name='three.unit.tests', + type=external, + target='three.unit.tests', + weight=13, + ), + ], + ), + Profile( + id=id_format.format('rule-one'), + name=name_format.format('rule-one'), + traffic_routing_method='Priority', + dns_config=dns, + monitor_config=monitor, + endpoints=[ + Endpoint( + name='one', + type=nested, + target_resource_id=id_format.format('pool-one'), + priority=1, + ), + Endpoint( + name='two', + type=nested, + target_resource_id=id_format.format('pool-two'), + priority=2, + ), + Endpoint( + name='three', + type=nested, + target_resource_id=id_format.format('pool-three'), + priority=3, + ), + Endpoint( + name='--default--', + type=nested, + target_resource_id=id_format.format('default'), + priority=4, + ), + ], + ), + Profile( + id=id_format.format('rule-three'), + name=name_format.format('rule-three'), + traffic_routing_method='Priority', + dns_config=dns, + monitor_config=monitor, + endpoints=[ + Endpoint( + name='three', + type=nested, + target_resource_id=id_format.format('pool-three'), + priority=1, + ), + Endpoint( + name='--default--', + type=nested, + target_resource_id=id_format.format('default'), + priority=2, + ), + ], + ), + Profile( + id=base_id + suffix, + name=suffix, + traffic_routing_method='Geographic', + dns_config=dns, + monitor_config=monitor, + endpoints=[ + Endpoint( + geo_mapping=['GEO-AF', 'DE', 'US-CA'], + name='rule-one', + type=nested, + target_resource_id=id_format.format('rule-one'), + ), + Endpoint( + geo_mapping=['WORLD'], + name='rule-three', + type=nested, + target_resource_id=id_format.format('rule-three'), + ), + ], + ), + ] + + def _get_dynamic_package(self): + '''Convenience function to setup a sample dynamic record. + ''' + provider = self._get_provider() + + # setup traffic manager profiles + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = self._get_tm_profiles(provider) + + # setup zone with dynamic record + zone = Zone(name='unit.tests.', sub_zones=[]) + record = self._get_dynamic_record(zone) + zone.add_record(record) + + # return everything + return provider, zone, record + def test_populate_records(self): provider = self._get_provider() @@ -510,6 +880,121 @@ class TestAzureDnsProvider(TestCase): self.assertEquals(len(zone.records), 17) self.assertTrue(exists) + def test_populate_dynamic(self): + # Middle east without Asia raises exception + provider, zone, record = self._get_dynamic_package() + tm_suffix = _traffic_manager_suffix(record) + tm_id = provider._profile_name_to_id + tm_list = provider._tm_client.profiles.list_by_resource_group + rule_name = 'rule-one--{}'.format(tm_suffix) + nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' + tm_list.return_value = [ + Profile( + id=tm_id(tm_suffix), + name=tm_suffix, + traffic_routing_method='Geographic', + endpoints=[ + Endpoint( + geo_mapping=['GEO-ME'], + ), + ], + ), + ] + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=tm_id(tm_suffix)), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + with self.assertRaises(AzureException) as ctx: + provider._populate_record(zone, azrecord) + self.assertTrue(text_type(ctx).startswith( + 'Middle East (GEO-ME) is not supported' + )) + + # empty priority profile raises exception + provider, zone, record = self._get_dynamic_package() + tm_list = provider._tm_client.profiles.list_by_resource_group + rule_name = 'rule-one--{}'.format(tm_suffix) + nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' + tm_list.return_value = [ + Profile( + id=tm_id(rule_name), + name=rule_name, + traffic_routing_method='Priority', + endpoints=[], + ), + Profile( + id=tm_id(tm_suffix), + name=tm_suffix, + traffic_routing_method='Geographic', + endpoints=[ + Endpoint( + geo_mapping=['WORLD'], + name='rule-one', + type=nested, + target_resource_id=tm_id(rule_name), + ), + ], + ), + ] + with self.assertRaises(AzureException) as ctx: + provider._populate_record(zone, azrecord) + self.assertTrue(text_type(ctx).startswith( + 'Expected at least 2 endpoints' + )) + + # valid set of profiles produce expected dynamic record + provider, zone, record = self._get_dynamic_package() + root_profile_id = provider._profile_name_to_id( + _traffic_manager_suffix(record) + ) + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=root_profile_id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + + record = provider._populate_record(zone, azrecord) + self.assertEqual(record.name, 'foo') + self.assertEqual(record.ttl, 60) + self.assertEqual(record.value, 'default.unit.tests.') + self.assertEqual(record.dynamic._data(), { + 'pools': { + 'one': { + 'values': [ + {'value': 'one.unit.tests.', 'weight': 11}, + ], + 'fallback': 'two', + }, + 'two': { + 'values': [ + {'value': 'two1.unit.tests.', 'weight': 3}, + {'value': 'two2.unit.tests.', 'weight': 4}, + ], + 'fallback': 'three', + }, + 'three': { + 'values': [ + {'value': 'three.unit.tests.', 'weight': 13}, + ], + 'fallback': None, + }, + }, + 'rules': [ + {'geos': ['AF', 'EU-DE', 'NA-US-CA'], 'pool': 'one'}, + {'pool': 'three'}, + ], + }) + + # valid profiles with Middle East test case + geo_profile = provider._get_tm_for_dynamic_record(record) + geo_profile.endpoints[0].geo_mapping.extend(['GEO-ME', 'GEO-AS']) + record = provider._populate_record(zone, azrecord) + self.assertIn('AS', record.dynamic.rules[0].data['geos']) + self.assertNotIn('ME', record.dynamic.rules[0].data['geos']) + def test_populate_zone(self): provider = self._get_provider() @@ -541,20 +1026,388 @@ class TestAzureDnsProvider(TestCase): None ) + def test_extra_changes(self): + provider, existing, record = self._get_dynamic_package() + + # test simple records produce no extra changes + desired = Zone(name=existing.name, sub_zones=[]) + desired.add_record(Record.new(desired, 'simple', data={ + 'type': record._type, + 'ttl': record.ttl, + 'value': record.value, + })) + extra = provider._extra_changes(desired, desired, []) + self.assertEqual(len(extra), 0) + + # test an unchanged dynamic record produces no extra changes + desired.add_record(record) + extra = provider._extra_changes(existing, desired, []) + self.assertEqual(len(extra), 0) + + # test unused TM produces the extra change for clean up + sample_profile = self._get_tm_profiles(provider)[0] + tm_id = provider._profile_name_to_id + root_profile_name = _traffic_manager_suffix(record) + extra_profile = Profile( + id=tm_id('random--{}'.format(root_profile_name)), + name='random--{}'.format(root_profile_name), + traffic_routing_method='Weighted', + dns_config=sample_profile.dns_config, + monitor_config=sample_profile.monitor_config, + endpoints=sample_profile.endpoints, + ) + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value.append(extra_profile) + provider._populate_traffic_managers() + extra = provider._extra_changes(existing, desired, []) + self.assertEqual(len(extra), 1) + extra = extra[0] + self.assertIsInstance(extra, Update) + self.assertEqual(extra.new, record) + desired._remove_record(record) + tm_list.return_value.pop() + + # test new dynamic record does not produce an extra change for it + new_dynamic = Record.new(desired, record.name + '2', data={ + 'type': record._type, + 'ttl': record.ttl, + 'value': record.value, + 'dynamic': record.dynamic._data(), + 'octodns': record._octodns, + }) + # test change in healthcheck by using a different port number + update_dynamic = Record.new(desired, record.name, data={ + 'type': record._type, + 'ttl': record.ttl, + 'value': record.value, + 'dynamic': record.dynamic._data(), + 'octodns': { + 'healthcheck': { + 'path': '/_ping', + 'port': 443, + 'protocol': 'HTTPS', + }, + }, + }) + desired.add_record(new_dynamic) + desired.add_record(update_dynamic) + changes = [Create(new_dynamic)] + extra = provider._extra_changes(existing, desired, changes) + # implicitly asserts that new_dynamic was not added to extra changes + # as it was already in the `changes` list + self.assertEqual(len(extra), 1) + extra = extra[0] + self.assertIsInstance(extra, Update) + self.assertEqual(extra.new, update_dynamic) + + # test non-CNAME dynamic record throws exception + a_dynamic = Record.new(desired, record.name + '3', data={ + 'type': 'A', + 'ttl': record.ttl, + 'values': ['1.1.1.1'], + 'dynamic': { + 'pools': { + 'one': {'values': [{'value': '2.2.2.2'}]}, + }, + 'rules': [ + {'pool': 'one'}, + ], + }, + }) + desired.add_record(a_dynamic) + changes.append(Create(a_dynamic)) + with self.assertRaises(AzureException): + provider._extra_changes(existing, desired, changes) + + def test_generate_tm_profile(self): + provider, zone, record = self._get_dynamic_package() + profile_gen = provider._generate_tm_profile + + name = 'foobar' + routing = 'Priority' + endpoints = [ + Endpoint(target='one.unit.tests'), + Endpoint(target_resource_id='/s/1/rg/foo/tm/foobar2'), + Endpoint(name='invalid'), + ] + + # invalid endpoint raises exception + with self.assertRaises(AzureException): + profile_gen(name, routing, endpoints, record) + + # regular test + endpoints.pop() + profile = profile_gen(name, routing, endpoints, record) + + # implicitly tests _profile_name_to_id + sub = provider._dns_client_subscription_id + rg = provider._resource_group + expected_id = '/subscriptions/' + sub + \ + '/resourceGroups/' + rg + \ + '/providers/Microsoft.Network/trafficManagerProfiles/' + name + self.assertEqual(profile.id, expected_id) + self.assertEqual(profile.name, name) + self.assertEqual(profile.traffic_routing_method, routing) + self.assertEqual(profile.dns_config.ttl, record.ttl) + self.assertEqual(len(profile.endpoints), len(endpoints)) + + self.assertEqual( + profile.endpoints[0].type, + 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + ) + self.assertEqual( + profile.endpoints[1].type, + 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' + ) + + def test_generate_traffic_managers(self): + provider, zone, record = self._get_dynamic_package() + profiles = provider._generate_traffic_managers(record) + deduped = [] + seen = set() + for profile in profiles: + if profile.name not in seen: + deduped.append(profile) + seen.add(profile.name) + + # check that every profile is a match with what we expect + expected_profiles = self._get_tm_profiles(provider) + self.assertEqual(len(expected_profiles), len(deduped)) + for have, expected in zip(deduped, expected_profiles): + self.assertTrue(_profile_is_match(have, expected)) + + # check Asia/Middle East test case + record.dynamic._data()['rules'][0]['geos'].append('AS') + profiles = provider._generate_traffic_managers(record) + geo_profile_name = _traffic_manager_suffix(record) + geo_profile = next( + profile + for profile in profiles + if profile.name == geo_profile_name + ) + self.assertIn('GEO-ME', geo_profile.endpoints[0].geo_mapping) + self.assertIn('GEO-AS', geo_profile.endpoints[0].geo_mapping) + + def test_sync_traffic_managers(self): + provider, zone, record = self._get_dynamic_package() + provider._populate_traffic_managers() + + tm_sync = provider._tm_client.profiles.create_or_update + + suffix = 'foo-unit-tests' + expected_seen = { + suffix, 'default--{}'.format(suffix), + 'rule-one--{}'.format(suffix), 'rule-three--{}'.format(suffix), + 'pool-one--{}'.format(suffix), 'pool-two--{}'.format(suffix), + 'pool-three--{}'.format(suffix), + } + + # test no change + seen = provider._sync_traffic_managers(record) + self.assertEqual(seen, expected_seen) + tm_sync.assert_not_called() + + # test that changing weight causes update API call + dynamic = record.dynamic._data() + dynamic['pools']['one']['values'][0]['weight'] = 14 + data = { + 'type': 'CNAME', + 'ttl': record.ttl, + 'value': record.value, + 'dynamic': dynamic, + 'octodns': record._octodns, + } + new_record = Record.new(zone, record.name, data) + tm_sync.reset_mock() + seen2 = provider._sync_traffic_managers(new_record) + self.assertEqual(seen2, expected_seen) + tm_sync.assert_called_once() + + # test that new profile was successfully inserted in cache + new_profile = provider._get_tm_profile_by_name( + 'pool-one--{}'.format(suffix) + ) + self.assertEqual(new_profile.endpoints[0].weight, 14) + + def test_find_traffic_managers(self): + provider, zone, record = self._get_dynamic_package() + + # insert a non-matching profile + sample_profile = self._get_tm_profiles(provider)[0] + # dummy record for generating suffix + record2 = Record.new(zone, record.name + '2', data={ + 'type': record._type, + 'ttl': record.ttl, + 'value': record.value, + }) + suffix2 = _traffic_manager_suffix(record2) + tm_id = provider._profile_name_to_id + extra_profile = Profile( + id=tm_id('random--{}'.format(suffix2)), + name='random--{}'.format(suffix2), + traffic_routing_method='Weighted', + dns_config=sample_profile.dns_config, + monitor_config=sample_profile.monitor_config, + endpoints=sample_profile.endpoints, + ) + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value.append(extra_profile) + provider._populate_traffic_managers() + + # implicitly asserts that non-matching profile is not included + suffix = _traffic_manager_suffix(record) + self.assertEqual(provider._find_traffic_managers(record), { + suffix, 'default--{}'.format(suffix), + 'rule-one--{}'.format(suffix), 'rule-three--{}'.format(suffix), + 'pool-one--{}'.format(suffix), 'pool-two--{}'.format(suffix), + 'pool-three--{}'.format(suffix), + }) + + def test_traffic_manager_gc(self): + provider, zone, record = self._get_dynamic_package() + provider._populate_traffic_managers() + + profiles = provider._find_traffic_managers(record) + profile_delete_mock = provider._tm_client.profiles.delete + + provider._traffic_managers_gc(record, profiles) + profile_delete_mock.assert_not_called() + + profile_delete_mock.reset_mock() + remove = list(profiles)[3] + profiles.discard(remove) + + provider._traffic_managers_gc(record, profiles) + profile_delete_mock.assert_has_calls( + [call(provider._resource_group, remove)] + ) + def test_apply(self): provider = self._get_provider() - changes = [] - deletes = [] - for i in octo_records: - changes.append(Create(i)) - deletes.append(Delete(i)) + half = int(len(octo_records) / 2) + changes = [Create(r) for r in octo_records[:half]] + \ + [Update(r, r) for r in octo_records[half:]] + deletes = [Delete(r) for r in octo_records] self.assertEquals(19, provider.apply(Plan(None, zone, changes, True))) self.assertEquals(19, provider.apply(Plan(zone, zone, deletes, True))) + def test_apply_create_dynamic(self): + provider = self._get_provider() + + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = [] + + tm_sync = provider._tm_client.profiles.create_or_update + + zone = Zone(name='unit.tests.', sub_zones=[]) + record = self._get_dynamic_record(zone) + + profiles = self._get_tm_profiles(provider) + + provider._apply_Create(Create(record)) + # create was called as many times as number of profiles required for + # the dynamic record + self.assertEqual(tm_sync.call_count, len(profiles)) + + create = provider._dns_client.record_sets.create_or_update + create.assert_called_once() + + def test_apply_update_dynamic(self): + # existing is simple, new is dynamic + provider = self._get_provider() + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = [] + profiles = self._get_tm_profiles(provider) + dynamic_record = self._get_dynamic_record(zone) + simple_record = Record.new(zone, dynamic_record.name, data={ + 'type': 'CNAME', + 'ttl': 3600, + 'value': 'cname.unit.tests.', + }) + change = Update(simple_record, dynamic_record) + provider._apply_Update(change) + tm_sync, dns_update, tm_delete = ( + provider._tm_client.profiles.create_or_update, + provider._dns_client.record_sets.create_or_update, + provider._tm_client.profiles.delete + ) + self.assertEqual(tm_sync.call_count, len(profiles)) + dns_update.assert_called_once() + tm_delete.assert_not_called() + + # existing is dynamic, new is simple + provider, existing, dynamic_record = self._get_dynamic_package() + profiles = self._get_tm_profiles(provider) + change = Update(dynamic_record, simple_record) + provider._apply_Update(change) + tm_sync, dns_update, tm_delete = ( + provider._tm_client.profiles.create_or_update, + provider._dns_client.record_sets.create_or_update, + provider._tm_client.profiles.delete + ) + tm_sync.assert_not_called() + dns_update.assert_called_once() + self.assertEqual(tm_delete.call_count, len(profiles)) + + # both are dynamic, healthcheck port is changed + provider, existing, dynamic_record = self._get_dynamic_package() + profiles = self._get_tm_profiles(provider) + dynamic_record2 = self._get_dynamic_record(existing) + dynamic_record2._octodns['healthcheck']['port'] += 1 + change = Update(dynamic_record, dynamic_record2) + provider._apply_Update(change) + tm_sync, dns_update, tm_delete = ( + provider._tm_client.profiles.create_or_update, + provider._dns_client.record_sets.create_or_update, + provider._tm_client.profiles.delete + ) + self.assertEqual(tm_sync.call_count, len(profiles)) + dns_update.assert_not_called() + tm_delete.assert_not_called() + + # both are dynamic, extra profile should be deleted + provider, existing, dynamic_record = self._get_dynamic_package() + sample_profile = self._get_tm_profiles(provider)[0] + tm_id = provider._profile_name_to_id + root_profile_name = _traffic_manager_suffix(dynamic_record) + extra_profile = Profile( + id=tm_id('random--{}'.format(root_profile_name)), + name='random--{}'.format(root_profile_name), + traffic_routing_method='Weighted', + dns_config=sample_profile.dns_config, + monitor_config=sample_profile.monitor_config, + endpoints=sample_profile.endpoints, + ) + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value.append(extra_profile) + change = Update(dynamic_record, dynamic_record) + provider._apply_Update(change) + tm_sync, dns_update, tm_delete = ( + provider._tm_client.profiles.create_or_update, + provider._dns_client.record_sets.create_or_update, + provider._tm_client.profiles.delete + ) + tm_sync.assert_not_called() + dns_update.assert_not_called() + tm_delete.assert_called_once() + + def test_apply_delete_dynamic(self): + provider, existing, record = self._get_dynamic_package() + provider._populate_traffic_managers() + profiles = self._get_tm_profiles(provider) + change = Delete(record) + provider._apply_Delete(change) + dns_delete, tm_delete = ( + provider._dns_client.record_sets.delete, + provider._tm_client.profiles.delete + ) + dns_delete.assert_called_once() + self.assertEqual(tm_delete.call_count, len(profiles)) + def test_create_zone(self): provider = self._get_provider() From 72b0438b7fb120e1155bf9934ddde34e9f28e267 Mon Sep 17 00:00:00 2001 From: Steven Hollingsworth Date: Wed, 12 May 2021 09:40:06 -0700 Subject: [PATCH 30/58] Updated doc to reflect config file location for zone wide lenient flag --- docs/records.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/records.md b/docs/records.md index e39a85d..c6b2a77 100644 --- a/docs/records.md +++ b/docs/records.md @@ -114,8 +114,7 @@ If you'd like to enable lenience for a whole zone you can do so with the followi ```yaml non-compliant-zone.com.: - octodns: - lenient: true + lenient: true sources: - route53 targets: From 9b5c8be01ece38e740fb7ee77aefec783812c852 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Wed, 12 May 2021 10:02:04 -0700 Subject: [PATCH 31/58] optimize by not creating traffic manager for single-value pools If single-value pools have a weight defined, it will be lost by this optimization. Next time octodns-sync is run, it will show an update for setting the weight on remote. To overcome this, this commit includes a change to Record object that ignores the weight in single-value pools. --- octodns/provider/azuredns.py | 84 ++++++++++--------- octodns/record/__init__.py | 4 + tests/test_octodns_provider_azuredns.py | 105 +++++++----------------- tests/test_octodns_record.py | 1 + 4 files changed, 79 insertions(+), 115 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 4d3dcc5..a792b9b 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -667,14 +667,9 @@ class AzureProvider(BaseProvider): for rule_ep in rule_endpoints: pool_name = rule_ep.name - # third (and last) level weighted RR profile - # these should be leaf node profiles with no further nesting - pool_profile = rule_ep.target_resource - # last/default pool if pool_name == '--default--': - for pool_ep in pool_profile.endpoints: - default.add(pool_ep.target) + default.add(rule_ep.target) # this should be the last one, so let's break here break @@ -687,11 +682,20 @@ class AzureProvider(BaseProvider): pool['fallback'] = pool_name pool = pools[pool_name] - for pool_ep in pool_profile.endpoints: + endpoints = [] + # these should be leaf node entries with no further nesting + if rule_ep.target_resource_id: + # third (and last) level weighted RR profile + endpoints = rule_ep.target_resource.endpoints + else: + # single-value pool + endpoints = [rule_ep] + + for pool_ep in endpoints: val = pool_ep.target value_dict = { 'value': _check_endswith_dot(val), - 'weight': pool_ep.weight, + 'weight': pool_ep.weight or 1, } if value_dict not in pool['values']: pool['values'].append(value_dict) @@ -703,6 +707,7 @@ class AzureProvider(BaseProvider): geo_ep.target_resource.name, len(rule_endpoints) ) raise AzureException(msg) + rules.append(rule) # Order and convert to a list @@ -800,19 +805,6 @@ class AzureProvider(BaseProvider): tm_suffix = _traffic_manager_suffix(record) profile = self._generate_tm_profile - # construct the default pool that will be used at the end of - # all rules - target = record.value[:-1] - default_endpoints = [Endpoint( - name=target, - target=target, - weight=1, - )] - default_profile_name = 'default--{}'.format(tm_suffix) - default_profile = profile(default_profile_name, 'Weighted', - default_endpoints, record) - traffic_managers.append(default_profile) - geo_endpoints = [] for rule in record.dynamic.rules: @@ -824,26 +816,36 @@ class AzureProvider(BaseProvider): # iterate until we reach end of fallback chain pool = pools[pool_name].data profile_name = 'pool-{}--{}'.format(pool_name, tm_suffix) - endpoints = [] - for val in pool['values']: - target = val['value'] - # strip trailing dot from CNAME value - target = target[:-1] - endpoints.append(Endpoint( - name=target, - target=target, - weight=val.get('weight', 1), - )) - pool_profile = profile(profile_name, 'Weighted', endpoints, - record) - traffic_managers.append(pool_profile) + if len(pool['values']) > 1: + # create Weighted profile for multi-value pool + endpoints = [] + for val in pool['values']: + target = val['value'] + # strip trailing dot from CNAME value + target = target[:-1] + endpoints.append(Endpoint( + name=target, + target=target, + weight=val.get('weight', 1), + )) + pool_profile = profile(profile_name, 'Weighted', endpoints, + record) + traffic_managers.append(pool_profile) - # append pool to endpoint list of fallback rule profile - rule_endpoints.append(Endpoint( - name=pool_name, - target_resource_id=pool_profile.id, - priority=priority, - )) + # append pool to endpoint list of fallback rule profile + rule_endpoints.append(Endpoint( + name=pool_name, + target_resource_id=pool_profile.id, + priority=priority, + )) + else: + # add single-value pool as an external endpoint + target = pool['values'][0]['value'][:-1] + rule_endpoints.append(Endpoint( + name=pool_name, + target=target, + priority=priority, + )) priority += 1 pool_name = pool.get('fallback') @@ -851,7 +853,7 @@ class AzureProvider(BaseProvider): # append default profile to the end rule_endpoints.append(Endpoint( name='--default--', - target_resource_id=default_profile.id, + target=record.value[:-1], priority=priority, )) # create rule profile with fallback chain diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 135baf8..cc503d6 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -429,6 +429,10 @@ class _DynamicPool(object): ] values.sort(key=lambda d: d['value']) + # normalize weight of a single-value pool + if len(values) == 1: + values[0]['weight'] = 1 + fallback = data.get('fallback', None) self.data = { 'fallback': fallback if fallback != 'default' else None, diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 5655719..fe867fe 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -574,7 +574,7 @@ class TestAzureDnsProvider(TestCase): }, 'rules': [ {'geos': ['AF', 'EU-DE', 'NA-US-CA'], 'pool': 'one'}, - {'pool': 'three'}, + {'pool': 'two'}, ], }, 'octodns': { @@ -605,36 +605,6 @@ class TestAzureDnsProvider(TestCase): nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' return [ - Profile( - id=id_format.format('default'), - name=name_format.format('default'), - traffic_routing_method='Weighted', - dns_config=dns, - monitor_config=monitor, - endpoints=[ - Endpoint( - name='default.unit.tests', - type=external, - target='default.unit.tests', - weight=1, - ), - ], - ), - Profile( - id=id_format.format('pool-one'), - name=name_format.format('pool-one'), - traffic_routing_method='Weighted', - dns_config=dns, - monitor_config=monitor, - endpoints=[ - Endpoint( - name='one.unit.tests', - type=external, - target='one.unit.tests', - weight=11, - ), - ], - ), Profile( id=id_format.format('pool-two'), name=name_format.format('pool-two'), @@ -656,21 +626,6 @@ class TestAzureDnsProvider(TestCase): ), ], ), - Profile( - id=id_format.format('pool-three'), - name=name_format.format('pool-three'), - traffic_routing_method='Weighted', - dns_config=dns, - monitor_config=monitor, - endpoints=[ - Endpoint( - name='three.unit.tests', - type=external, - target='three.unit.tests', - weight=13, - ), - ], - ), Profile( id=id_format.format('rule-one'), name=name_format.format('rule-one'), @@ -680,8 +635,8 @@ class TestAzureDnsProvider(TestCase): endpoints=[ Endpoint( name='one', - type=nested, - target_resource_id=id_format.format('pool-one'), + type=external, + target='one.unit.tests', priority=1, ), Endpoint( @@ -692,37 +647,43 @@ class TestAzureDnsProvider(TestCase): ), Endpoint( name='three', - type=nested, - target_resource_id=id_format.format('pool-three'), + type=external, + target='three.unit.tests', priority=3, ), Endpoint( name='--default--', - type=nested, - target_resource_id=id_format.format('default'), + type=external, + target='default.unit.tests', priority=4, ), ], ), Profile( - id=id_format.format('rule-three'), - name=name_format.format('rule-three'), + id=id_format.format('rule-two'), + name=name_format.format('rule-two'), traffic_routing_method='Priority', dns_config=dns, monitor_config=monitor, endpoints=[ Endpoint( - name='three', + name='two', type=nested, - target_resource_id=id_format.format('pool-three'), + target_resource_id=id_format.format('pool-two'), priority=1, ), Endpoint( - name='--default--', - type=nested, - target_resource_id=id_format.format('default'), + name='three', + type=external, + target='three.unit.tests', priority=2, ), + Endpoint( + name='--default--', + type=external, + target='default.unit.tests', + priority=3, + ), ], ), Profile( @@ -740,9 +701,9 @@ class TestAzureDnsProvider(TestCase): ), Endpoint( geo_mapping=['WORLD'], - name='rule-three', + name='rule-two', type=nested, - target_resource_id=id_format.format('rule-three'), + target_resource_id=id_format.format('rule-two'), ), ], ), @@ -964,7 +925,7 @@ class TestAzureDnsProvider(TestCase): 'pools': { 'one': { 'values': [ - {'value': 'one.unit.tests.', 'weight': 11}, + {'value': 'one.unit.tests.', 'weight': 1}, ], 'fallback': 'two', }, @@ -977,14 +938,14 @@ class TestAzureDnsProvider(TestCase): }, 'three': { 'values': [ - {'value': 'three.unit.tests.', 'weight': 13}, + {'value': 'three.unit.tests.', 'weight': 1}, ], 'fallback': None, }, }, 'rules': [ {'geos': ['AF', 'EU-DE', 'NA-US-CA'], 'pool': 'one'}, - {'pool': 'three'}, + {'pool': 'two'}, ], }) @@ -1196,10 +1157,8 @@ class TestAzureDnsProvider(TestCase): suffix = 'foo-unit-tests' expected_seen = { - suffix, 'default--{}'.format(suffix), - 'rule-one--{}'.format(suffix), 'rule-three--{}'.format(suffix), - 'pool-one--{}'.format(suffix), 'pool-two--{}'.format(suffix), - 'pool-three--{}'.format(suffix), + suffix, 'pool-two--{}'.format(suffix), + 'rule-one--{}'.format(suffix), 'rule-two--{}'.format(suffix), } # test no change @@ -1209,7 +1168,7 @@ class TestAzureDnsProvider(TestCase): # test that changing weight causes update API call dynamic = record.dynamic._data() - dynamic['pools']['one']['values'][0]['weight'] = 14 + dynamic['pools']['two']['values'][0]['weight'] = 14 data = { 'type': 'CNAME', 'ttl': record.ttl, @@ -1225,7 +1184,7 @@ class TestAzureDnsProvider(TestCase): # test that new profile was successfully inserted in cache new_profile = provider._get_tm_profile_by_name( - 'pool-one--{}'.format(suffix) + 'pool-two--{}'.format(suffix) ) self.assertEqual(new_profile.endpoints[0].weight, 14) @@ -1257,10 +1216,8 @@ class TestAzureDnsProvider(TestCase): # implicitly asserts that non-matching profile is not included suffix = _traffic_manager_suffix(record) self.assertEqual(provider._find_traffic_managers(record), { - suffix, 'default--{}'.format(suffix), - 'rule-one--{}'.format(suffix), 'rule-three--{}'.format(suffix), - 'pool-one--{}'.format(suffix), 'pool-two--{}'.format(suffix), - 'pool-three--{}'.format(suffix), + suffix, 'pool-two--{}'.format(suffix), + 'rule-one--{}'.format(suffix), 'rule-two--{}'.format(suffix), }) def test_traffic_manager_gc(self): diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index ce40b9b..3bb5d24 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -3013,6 +3013,7 @@ class TestDynamicRecords(TestCase): 'pools': { 'one': { 'values': [{ + 'weight': 10, 'value': '3.3.3.3', }], }, From 5df2077ed02124487b7a69be024ddb2a0e6a9386 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Fri, 14 May 2021 23:21:26 -0700 Subject: [PATCH 32/58] clarify the limitations and caveats of azure dynamic records --- README.md | 2 +- octodns/provider/azuredns.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8d59cd0..2ce9445 100644 --- a/README.md +++ b/README.md @@ -192,7 +192,7 @@ The above command pulled the existing data out of Route53 and placed the results | Provider | Requirements | Record Support | Dynamic | Notes | |--|--|--|--|--| -| [AzureProvider](/octodns/provider/azuredns.py) | azure-identity, azure-mgmt-dns, azure-mgmt-trafficmanager | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | Yes (CNAMEs only) | | +| [AzureProvider](/octodns/provider/azuredns.py) | azure-identity, azure-mgmt-dns, azure-mgmt-trafficmanager | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | Alpha (CNAMEs only) | | | [Akamai](/octodns/provider/edgedns.py) | edgegrid-python | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | | | [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, LOC, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [ConstellixProvider](/octodns/provider/constellix.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index a792b9b..8474014 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -387,6 +387,9 @@ class AzureProvider(BaseProvider): The first four variables above can be hidden in environment variables and octoDNS will automatically search for them in the shell. It is possible to also hard-code into the config file: eg, resource_group. + + Please read https://github.com/octodns/octodns/pull/706 for an overview + of how dynamic records are designed and caveats of using them. ''' SUPPORTS_GEO = False SUPPORTS_DYNAMIC = True From 1b5bf75c585a3a79c72ee655cbb383b8b962df72 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Mon, 17 May 2021 13:16:00 -0700 Subject: [PATCH 33/58] drop method names from exceptions --- octodns/provider/azuredns.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 8474014..5c2d23c 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -644,8 +644,8 @@ class AzureProvider(BaseProvider): # Throw exception otherwise, it should not happen if the # profile was generated by octoDNS if 'GEO-AS' not in geo_map: - msg = '_data_for_dynamic: Profile={}: '.format( - geo_profile.name) + msg = 'Profile={} for record {}: '.format( + geo_profile.name, azrecord.fqdn) msg += 'Middle East (GEO-ME) is not supported by ' + \ 'octoDNS. It needs to be either paired ' + \ 'with Asia (GEO-AS) or expanded into ' + \ @@ -782,9 +782,9 @@ class AzureProvider(BaseProvider): elif ep.target: ep.type = endpoint_type_prefix + 'externalEndpoints' else: - msg = ('_generate_tm_profile: Invalid endpoint {} ' + - 'in profile {}, needs to have either target or ' + - 'target_resource_id').format(ep.name, name) + msg = ('Invalid endpoint {} in profile {}, needs to have' + + 'either target or target_resource_id').format( + ep.name, name) raise AzureException(msg) # build and return @@ -1032,7 +1032,7 @@ class AzureProvider(BaseProvider): if getattr(record, 'dynamic', False): self._traffic_managers_gc(record, set()) - self.log.debug('* Success Delete: {}'.format(ar)) + self.log.debug('* Success Delete: {}'.format(record)) def _apply(self, plan): '''Required function of manager.py to actually apply a record change. From cca288faa6463e2abfd572612ff1209e8d1053c5 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Mon, 17 May 2021 13:19:43 -0700 Subject: [PATCH 34/58] log warning when non-1 weight is set for single-value pools --- octodns/record/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index cc503d6..45cea51 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -417,6 +417,7 @@ class _ValueMixin(object): class _DynamicPool(object): + log = getLogger('_DynamicPool') def __init__(self, _id, data): self._id = _id @@ -431,7 +432,12 @@ class _DynamicPool(object): # normalize weight of a single-value pool if len(values) == 1: - values[0]['weight'] = 1 + weight = data['values'][0].get('weight', 1) + if weight != 1: + self.log.warn( + 'Using weight=1 instead of %s for single-value pool %s', + weight, _id) + values[0]['weight'] = 1 fallback = data.get('fallback', None) self.data = { From e1d262a3013942e2a614622ce6d819deab3909c9 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 17 May 2021 17:06:40 -0700 Subject: [PATCH 35/58] Add a validation requiring single value weight=1 --- octodns/record/__init__.py | 4 ++++ tests/test_octodns_record.py | 29 ++++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 135baf8..f960998 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -573,6 +573,10 @@ class _DynamicMixin(object): reasons.append('missing value in pool "{}" ' 'value {}'.format(_id, value_num)) + if len(values) == 1 and values[0].get('weight', 1) != 1: + reasons.append('pool "{}" has single value with ' + 'weight!=1'.format(_id)) + fallback = pool.get('fallback', None) if fallback is not None: if fallback in pools: diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index ce40b9b..5a4704a 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -3412,7 +3412,7 @@ class TestDynamicRecords(TestCase): self.assertEquals(['pool "one" is missing values'], ctx.exception.reasons) - # pool valu not a dict + # pool value not a dict a_data = { 'dynamic': { 'pools': { @@ -3596,6 +3596,33 @@ class TestDynamicRecords(TestCase): self.assertEquals(['invalid weight "foo" in pool "three" value 2'], ctx.exception.reasons) + # single value with weight!=1 + a_data = { + 'dynamic': { + 'pools': { + 'one': { + 'values': [{ + 'weight': 12, + 'value': '6.6.6.6', + }], + }, + }, + 'rules': [{ + 'pool': 'one', + }], + }, + 'ttl': 60, + 'type': 'A', + 'values': [ + '1.1.1.1', + '2.2.2.2', + ], + } + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, 'bad', a_data) + self.assertEquals(['pool "one" has single value with weight!=1'], + ctx.exception.reasons) + # invalid fallback a_data = { 'dynamic': { From 753a337ecc604d837f018ac7f29313fd2e083b36 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 17 May 2021 17:49:31 -0700 Subject: [PATCH 36/58] Fix weights on new Azure dynamic record tests --- tests/test_octodns_provider_azuredns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index fe867fe..609fc5c 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -555,7 +555,7 @@ class TestAzureDnsProvider(TestCase): 'pools': { 'one': { 'values': [ - {'value': 'one.unit.tests.', 'weight': 11}, + {'value': 'one.unit.tests.', 'weight': 1}, ], 'fallback': 'two', }, @@ -568,7 +568,7 @@ class TestAzureDnsProvider(TestCase): }, 'three': { 'values': [ - {'value': 'three.unit.tests.', 'weight': 13}, + {'value': 'three.unit.tests.', 'weight': 1}, ], }, }, From b02d5d0a2de8ff61271458da9d570283e325aff3 Mon Sep 17 00:00:00 2001 From: Tom Kaminski Date: Mon, 17 May 2021 19:29:10 -0500 Subject: [PATCH 37/58] Do not trigger change for health checks on cname dynamic records --- octodns/provider/route53.py | 11 +- tests/test_octodns_provider_route53.py | 157 +++++++++++++++++++++++++ 2 files changed, 167 insertions(+), 1 deletion(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 0d5bab9..a3b3a57 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -1253,7 +1253,16 @@ class Route53Provider(BaseProvider): return self._gen_mods('DELETE', existing_records, existing_rrsets) def _extra_changes_update_needed(self, record, rrset): - healthcheck_host = record.healthcheck_host + value = rrset['ResourceRecords'][0]['Value'] + + try: + ip_address(text_type(value)) + # We're working with an IP + healthcheck_host = record.healthcheck_host + except (AddressValueError, ValueError): + # This isn't an IP, host is the value + healthcheck_host = value + healthcheck_path = record.healthcheck_path healthcheck_protocol = record.healthcheck_protocol healthcheck_port = record.healthcheck_port diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index a2b61e7..1e7210d 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -1962,6 +1962,163 @@ class TestRoute53Provider(TestCase): self.assertEquals(1, len(extra)) stubber.assert_no_pending_responses() + def test_extra_change_dynamic_has_health_check_cname(self): + provider, stubber = self._get_stubbed_provider() + + list_hosted_zones_resp = { + 'HostedZones': [{ + 'Name': 'unit.tests.', + 'Id': 'z42', + 'CallerReference': 'abc', + }], + 'Marker': 'm', + 'IsTruncated': False, + 'MaxItems': '100', + } + stubber.add_response('list_hosted_zones', list_hosted_zones_resp, {}) + + # record with geo and no health check returns change + desired = Zone('unit.tests.', []) + record = Record.new(desired, 'cname', { + 'ttl': 30, + 'type': 'CNAME', + 'value': 'cname.unit.tests.', + 'dynamic': { + 'pools': { + 'one': { + 'values': [{ + 'value': 'one.cname.unit.tests.', + }], + }, + }, + 'rules': [{ + 'pool': 'one', + }], + }, + }) + desired.add_record(record) + list_resource_record_sets_resp = { + 'ResourceRecordSets': [{ + # Not dynamic value and other name + 'Name': 'unit.tests.', + 'Type': 'CNAME', + 'GeoLocation': { + 'CountryCode': '*', + }, + 'ResourceRecords': [{ + 'Value': 'cname.unit.tests.', + }], + 'TTL': 61, + # All the non-matches have a different Id so we'll fail if they + # match + 'HealthCheckId': '33', + }, { + # Not dynamic value, matching name, other type + 'Name': 'cname.unit.tests.', + 'Type': 'AAAA', + 'ResourceRecords': [{ + 'Value': '2001:0db8:3c4d:0015:0000:0000:1a2f:1a4b' + }], + 'TTL': 61, + 'HealthCheckId': '33', + }, { + # default value pool + 'Name': '_octodns-default-value.cname.unit.tests.', + 'Type': 'CNAME', + 'GeoLocation': { + 'CountryCode': '*', + }, + 'ResourceRecords': [{ + 'Value': 'cname.unit.tests.', + }], + 'TTL': 61, + 'HealthCheckId': '33', + }, { + # different record + 'Name': '_octodns-two-value.other.unit.tests.', + 'Type': 'CNAME', + 'GeoLocation': { + 'CountryCode': '*', + }, + 'ResourceRecords': [{ + 'Value': 'cname.unit.tests.', + }], + 'TTL': 61, + 'HealthCheckId': '33', + }, { + # same everything, but different type + 'Name': '_octodns-one-value.cname.unit.tests.', + 'Type': 'AAAA', + 'ResourceRecords': [{ + 'Value': '2001:0db8:3c4d:0015:0000:0000:1a2f:1a4b' + }], + 'TTL': 61, + 'HealthCheckId': '33', + }, { + # same everything, sub + 'Name': '_octodns-one-value.sub.cname.unit.tests.', + 'Type': 'CNAME', + 'ResourceRecords': [{ + 'Value': 'cname.unit.tests.', + }], + 'TTL': 61, + 'HealthCheckId': '33', + }, { + # match + 'Name': '_octodns-one-value.cname.unit.tests.', + 'Type': 'CNAME', + 'ResourceRecords': [{ + 'Value': 'one.cname.unit.tests.', + }], + 'TTL': 61, + 'HealthCheckId': '42', + }], + 'IsTruncated': False, + 'MaxItems': '100', + } + stubber.add_response('list_resource_record_sets', + list_resource_record_sets_resp, + {'HostedZoneId': 'z42'}) + + stubber.add_response('list_health_checks', { + 'HealthChecks': [{ + 'Id': '42', + 'CallerReference': self.caller_ref, + 'HealthCheckConfig': { + 'Type': 'HTTPS', + 'FullyQualifiedDomainName': 'one.cname.unit.tests.', + 'ResourcePath': '/_dns', + 'Type': 'HTTPS', + 'Port': 443, + 'MeasureLatency': True, + 'RequestInterval': 10, + }, + 'HealthCheckVersion': 2, + }], + 'IsTruncated': False, + 'MaxItems': '100', + 'Marker': '', + }) + extra = provider._extra_changes(desired=desired, changes=[]) + self.assertEquals(0, len(extra)) + stubber.assert_no_pending_responses() + + # change b/c of healthcheck path + record._octodns['healthcheck'] = { + 'path': '/_ready' + } + extra = provider._extra_changes(desired=desired, changes=[]) + self.assertEquals(1, len(extra)) + stubber.assert_no_pending_responses() + + # no change b/c healthcheck host ignored for dynamic cname + record._octodns['healthcheck'] = { + 'host': 'foo.bar.io' + } + extra = provider._extra_changes(desired=desired, changes=[]) + self.assertEquals(0, len(extra)) + stubber.assert_no_pending_responses() + def _get_test_plan(self, max_changes): provider = Route53Provider('test', 'abc', '123', max_changes) From 5f57b52d07f5a75777e36837a76828c3af9c48a3 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Tue, 18 May 2021 10:12:59 -0700 Subject: [PATCH 38/58] map Oceania to Australia/Pacific in Azure --- octodns/provider/azuredns.py | 21 +++++++++++++++++++-- tests/test_octodns_provider_azuredns.py | 6 +++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 5c2d23c..8eeba02 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -655,12 +655,22 @@ class AzureProvider(BaseProvider): geos = rule.setdefault('geos', []) for code in geo_map: if code.startswith('GEO-'): - geos.append(code[len('GEO-'):]) + # continent + if code == 'GEO-AP': + # Azure uses Australia/Pacific (AP) instead of + # Oceania https://docs.microsoft.com/en-us/azure/ + # traffic-manager/ + # traffic-manager-geographic-regions + geos.append('OC') + else: + geos.append(code[len('GEO-'):]) elif '-' in code: + # state country, province = code.split('-', 1) country = GeoCodes.country_to_code(country) geos.append('{}-{}'.format(country, province)) else: + # country geos.append(GeoCodes.country_to_code(code)) # second level priority profile @@ -872,15 +882,22 @@ class AzureProvider(BaseProvider): if len(rule_geos) > 0: for geo in rule_geos: if '-' in geo: + # country or state geos.append(geo.split('-', 1)[-1]) else: - geos.append('GEO-{}'.format(geo)) + # continent if geo == 'AS': # Middle East is part of Asia in octoDNS, but # Azure treats it as a separate "group", so let's # add it in the list of geo mappings. We will drop # it when we later parse the list of regions. geos.append('GEO-ME') + elif geo == 'OC': + # Azure uses Australia/Pacific (AP) instead of + # Oceania + geo = 'AP' + + geos.append('GEO-{}'.format(geo)) else: geos.append('WORLD') geo_endpoints.append(Endpoint( diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 609fc5c..e8e5281 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -573,7 +573,7 @@ class TestAzureDnsProvider(TestCase): }, }, 'rules': [ - {'geos': ['AF', 'EU-DE', 'NA-US-CA'], 'pool': 'one'}, + {'geos': ['AF', 'EU-DE', 'NA-US-CA', 'OC'], 'pool': 'one'}, {'pool': 'two'}, ], }, @@ -694,7 +694,7 @@ class TestAzureDnsProvider(TestCase): monitor_config=monitor, endpoints=[ Endpoint( - geo_mapping=['GEO-AF', 'DE', 'US-CA'], + geo_mapping=['GEO-AF', 'DE', 'US-CA', 'GEO-AP'], name='rule-one', type=nested, target_resource_id=id_format.format('rule-one'), @@ -944,7 +944,7 @@ class TestAzureDnsProvider(TestCase): }, }, 'rules': [ - {'geos': ['AF', 'EU-DE', 'NA-US-CA'], 'pool': 'one'}, + {'geos': ['AF', 'EU-DE', 'NA-US-CA', 'OC'], 'pool': 'one'}, {'pool': 'two'}, ], }) From 62122e442931cf30f7b98cd91280419a3c58fac7 Mon Sep 17 00:00:00 2001 From: Tom Kaminski Date: Thu, 20 May 2021 15:24:34 -0500 Subject: [PATCH 39/58] More explicit check for CNAME records when checking for extra updates --- octodns/provider/route53.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index a3b3a57..ad59eb0 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -1253,15 +1253,11 @@ class Route53Provider(BaseProvider): return self._gen_mods('DELETE', existing_records, existing_rrsets) def _extra_changes_update_needed(self, record, rrset): - value = rrset['ResourceRecords'][0]['Value'] - - try: - ip_address(text_type(value)) - # We're working with an IP + if record._type == 'CNAME': + # For CNAME, healthcheck host by default points to the CNAME value + healthcheck_host = rrset['ResourceRecords'][0]['Value'] + else: healthcheck_host = record.healthcheck_host - except (AddressValueError, ValueError): - # This isn't an IP, host is the value - healthcheck_host = value healthcheck_path = record.healthcheck_path healthcheck_protocol = record.healthcheck_protocol From eb14873abbd6acbf08ce8ee08eaf70aa691343a7 Mon Sep 17 00:00:00 2001 From: Sham Date: Mon, 24 May 2021 19:01:17 -0700 Subject: [PATCH 40/58] Allow the option to not pass Host header in healthchecks --- octodns/provider/ns1.py | 4 +++- octodns/provider/route53.py | 2 +- tests/test_octodns_provider_ns1.py | 4 ++++ tests/test_octodns_provider_route53.py | 30 ++++++++++++++++++++++++++ 4 files changed, 38 insertions(+), 2 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index a910459..8408682 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -929,7 +929,9 @@ class Ns1Provider(BaseProvider): if record.healthcheck_protocol != 'TCP': # IF it's HTTP we need to send the request string path = record.healthcheck_path - host = record.healthcheck_host + # if host header is explicitly set to null in the yaml, + # just pass the domain (value) as the host header + host = record.healthcheck_host or value request = r'GET {path} HTTP/1.0\r\nHost: {host}\r\n' \ r'User-agent: NS1\r\n\r\n'.format(path=path, host=host) ret['config']['send'] = request diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index ad59eb0..0331847 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -1131,7 +1131,7 @@ class Route53Provider(BaseProvider): 'Type': healthcheck_protocol, } if healthcheck_protocol != 'TCP': - config['FullyQualifiedDomainName'] = healthcheck_host + config['FullyQualifiedDomainName'] = healthcheck_host or value config['ResourcePath'] = healthcheck_path if value: config['IPAddress'] = value diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index f54c7bd..4ae4757 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -757,6 +757,10 @@ class TestNs1ProviderDynamic(TestCase): self.assertFalse(monitor['config']['ssl']) self.assertEquals('host:unit.tests type:A', monitor['notes']) + record._octodns['healthcheck']['host'] = None + monitor = provider._monitor_gen(record, value) + self.assertTrue(r'\nHost: 3.4.5.6\r' in monitor['config']['send']) + record._octodns['healthcheck']['protocol'] = 'HTTPS' monitor = provider._monitor_gen(record, value) self.assertTrue(monitor['config']['ssl']) diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 1e7210d..1bf3332 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -1166,6 +1166,31 @@ class TestRoute53Provider(TestCase): }) stubber.add_response('change_tags_for_resource', {}) + health_check_config = { + 'EnableSNI': False, + 'FailureThreshold': 6, + 'FullyQualifiedDomainName': '4.2.3.4', + 'IPAddress': '4.2.3.4', + 'MeasureLatency': True, + 'Port': 8080, + 'RequestInterval': 10, + 'ResourcePath': '/_status', + 'Type': 'HTTP' + } + stubber.add_response('create_health_check', { + 'HealthCheck': { + 'Id': '43', + 'CallerReference': self.caller_ref, + 'HealthCheckConfig': health_check_config, + 'HealthCheckVersion': 1, + }, + 'Location': 'http://url', + }, { + 'CallerReference': ANY, + 'HealthCheckConfig': health_check_config, + }) + stubber.add_response('change_tags_for_resource', {}) + record = Record.new(self.expected, '', { 'ttl': 61, 'type': 'A', @@ -1191,6 +1216,11 @@ class TestRoute53Provider(TestCase): # when allowed to create we do id = provider.get_health_check_id(record, value, True) self.assertEquals('42', id) + + # when allowed to create and when host is None + record._octodns['healthcheck']['host'] = None + id = provider.get_health_check_id(record, value, True) + self.assertEquals('43', id) stubber.assert_no_pending_responses() # A CNAME style healthcheck, without a value From f00766a77909c216249b72dcb4396ced16909ac2 Mon Sep 17 00:00:00 2001 From: Stan Brinkerhoff Date: Wed, 26 May 2021 22:45:07 -0400 Subject: [PATCH 41/58] Add support for ALIAS type --- octodns/provider/ultra.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/octodns/provider/ultra.py b/octodns/provider/ultra.py index 03b70de..f65c519 100644 --- a/octodns/provider/ultra.py +++ b/octodns/provider/ultra.py @@ -36,12 +36,12 @@ class UltraProvider(BaseProvider): ''' Neustar UltraDNS provider - Documentation for Ultra REST API requires a login: - https://portal.ultradns.com/static/docs/REST-API_User_Guide.pdf - Implemented to the May 20, 2020 version of the document (dated on page ii) - Also described as Version 2.83.0 (title page) + Documentation for Ultra REST API: + https://ultra-portalstatic.ultradns.com/static/docs/REST-API_User_Guide.pdf + Implemented to the May 26, 2021 version of the document (dated on page ii) + Also described as Version 3.18.0 (title page) - Tested against 3.0.0-20200627220036.81047f5 + Tested against 3.20.1-20210521075351.36b9297 As determined by querying https://api.ultradns.com/version ultra: @@ -57,6 +57,7 @@ class UltraProvider(BaseProvider): RECORDS_TO_TYPE = { 'A (1)': 'A', 'AAAA (28)': 'AAAA', + 'APEXALIAS (65282)': 'ALIAS', 'CAA (257)': 'CAA', 'CNAME (5)': 'CNAME', 'MX (15)': 'MX', @@ -72,6 +73,7 @@ class UltraProvider(BaseProvider): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False TIMEOUT = 5 + ZONE_REQUEST_LIMIT = 1000 def _request(self, method, path, params=None, data=None, json=None, json_response=True): @@ -151,7 +153,7 @@ class UltraProvider(BaseProvider): def zones(self): if self._zones is None: offset = 0 - limit = 100 + limit = self.ZONE_REQUEST_LIMIT zones = [] paging = True while paging: @@ -401,8 +403,15 @@ class UltraProvider(BaseProvider): def _gen_data(self, record): zone_name = self._remove_prefix(record.fqdn, record.name + '.') + + # UltraDNS treats the `APEXALIAS` type as the octodns `ALIAS`. + if record._type == "ALIAS": + record_type = "APEXALIAS" + else: + record_type = record._type + path = '/v2/zones/{}/rrsets/{}/{}'.format(zone_name, - record._type, + record_type, record.fqdn) contents_for = getattr(self, '_contents_for_{}'.format(record._type)) return path, contents_for(record) @@ -444,7 +453,12 @@ class UltraProvider(BaseProvider): existing._type == self.RECORDS_TO_TYPE[record['rrtype']]: zone_name = self._remove_prefix(existing.fqdn, existing.name + '.') - path = '/v2/zones/{}/rrsets/{}/{}'.format(zone_name, + + # UltraDNS treats the `APEXALIAS` type as the octodns `ALIAS`. + if existing._type == "ALIAS": + existing._type = "APEXALIAS" + + path = '/v2/zones/{}/rrsets/{}/{ }'.format(zone_name, existing._type, existing.fqdn) self._delete(path, json_response=False) From 4b6fd8b4a1e19fe889943aab705ecf23b6c19495 Mon Sep 17 00:00:00 2001 From: sbrinkerhoff Date: Thu, 27 May 2021 17:59:10 -0400 Subject: [PATCH 42/58] Remove type mutation Co-authored-by: Ross McFarland --- octodns/provider/ultra.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/octodns/provider/ultra.py b/octodns/provider/ultra.py index f65c519..a2eae3c 100644 --- a/octodns/provider/ultra.py +++ b/octodns/provider/ultra.py @@ -455,10 +455,11 @@ class UltraProvider(BaseProvider): existing.name + '.') # UltraDNS treats the `APEXALIAS` type as the octodns `ALIAS`. - if existing._type == "ALIAS": - existing._type = "APEXALIAS" + existing_type = existing._type + if existing_type == "ALIAS": + existing_type = "APEXALIAS" path = '/v2/zones/{}/rrsets/{}/{ }'.format(zone_name, - existing._type, + existing_type, existing.fqdn) self._delete(path, json_response=False) From e816e8e49b6c467cdf6a911b9d0a27bd2c8424e6 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 27 May 2021 15:19:33 -0700 Subject: [PATCH 43/58] Fixed extraneous whitespace in format in UltraDNS --- octodns/provider/ultra.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/ultra.py b/octodns/provider/ultra.py index a2eae3c..55c2c34 100644 --- a/octodns/provider/ultra.py +++ b/octodns/provider/ultra.py @@ -459,7 +459,7 @@ class UltraProvider(BaseProvider): if existing_type == "ALIAS": existing_type = "APEXALIAS" - path = '/v2/zones/{}/rrsets/{}/{ }'.format(zone_name, + path = '/v2/zones/{}/rrsets/{}/{}'.format(zone_name, existing_type, existing.fqdn) self._delete(path, json_response=False) From 60f916cd2400a27f23fb365f90978e6968cc088a Mon Sep 17 00:00:00 2001 From: Stan Brinkerhoff Date: Fri, 28 May 2021 00:52:48 -0400 Subject: [PATCH 44/58] add contents for alias method, test for alias --- octodns/provider/ultra.py | 4 +++- tests/test_octodns_provider_ultra.py | 15 ++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/octodns/provider/ultra.py b/octodns/provider/ultra.py index 55c2c34..bc855d4 100644 --- a/octodns/provider/ultra.py +++ b/octodns/provider/ultra.py @@ -73,7 +73,7 @@ class UltraProvider(BaseProvider): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False TIMEOUT = 5 - ZONE_REQUEST_LIMIT = 1000 + ZONE_REQUEST_LIMIT = 100 def _request(self, method, path, params=None, data=None, json=None, json_response=True): @@ -213,6 +213,7 @@ class UltraProvider(BaseProvider): _data_for_PTR = _data_for_single _data_for_CNAME = _data_for_single + _data_for_ALIAS = _data_for_single def _data_for_CAA(self, _type, records): return { @@ -376,6 +377,7 @@ class UltraProvider(BaseProvider): } _contents_for_PTR = _contents_for_CNAME + _contents_for_ALIAS = _contents_for_CNAME def _contents_for_SRV(self, record): return { diff --git a/tests/test_octodns_provider_ultra.py b/tests/test_octodns_provider_ultra.py index b6d1017..52e0307 100644 --- a/tests/test_octodns_provider_ultra.py +++ b/tests/test_octodns_provider_ultra.py @@ -274,7 +274,7 @@ class TestUltraProvider(TestCase): self.assertTrue(provider.populate(zone)) self.assertEquals('octodns1.test.', zone.name) - self.assertEquals(11, len(zone.records)) + self.assertEquals(12, len(zone.records)) self.assertEquals(4, mock.call_count) def test_apply(self): @@ -352,8 +352,8 @@ class TestUltraProvider(TestCase): })) plan = provider.plan(wanted) - self.assertEquals(10, len(plan.changes)) - self.assertEquals(10, provider.apply(plan)) + self.assertEquals(11, len(plan.changes)) + self.assertEquals(11, provider.apply(plan)) self.assertTrue(plan.exists) provider._request.assert_has_calls([ @@ -492,6 +492,15 @@ class TestUltraProvider(TestCase): Record.new(zone, 'txt', {'ttl': 60, 'type': 'TXT', 'values': ['abc', 'def']})), + + # ALIAS + ('', 'ALIAS', + '/v2/zones/unit.tests./rrsets/APEXALIAS/unit.tests.', + {'ttl': 60, 'rdata': ['target.unit.tests.']}, + Record.new(zone, '', + {'ttl': 60, 'type': 'ALIAS', + 'value': 'target.unit.tests.'})), + ): # Validate path and payload based on record meet expectations path, payload = provider._gen_data(expected_record) From 6e71f2df76d454578bac1a84bf8920058350743f Mon Sep 17 00:00:00 2001 From: Stan Brinkerhoff Date: Fri, 28 May 2021 00:54:57 -0400 Subject: [PATCH 45/58] update test record --- tests/fixtures/ultra-records-page-2.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/fixtures/ultra-records-page-2.json b/tests/fixtures/ultra-records-page-2.json index abdc44f..274d95e 100644 --- a/tests/fixtures/ultra-records-page-2.json +++ b/tests/fixtures/ultra-records-page-2.json @@ -32,7 +32,16 @@ "rdata": [ "www.octodns1.test." ] + }, + { + "ownerName": "host1.octodns1.test.", + "rrtype": "RRSET (70)", + "ttl": 3600, + "rdata": [ + "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855" + ] } + ], "resultInfo": { "totalCount": 13, From e97675fe3ddaa02ec4baa145fc7254f9b616fc28 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Fri, 28 May 2021 12:48:09 -0700 Subject: [PATCH 46/58] check dns config when comparing profiles for equality --- octodns/provider/azuredns.py | 10 +++++++++- tests/test_octodns_provider_azuredns.py | 20 +++++++++++++------- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 8eeba02..65d9967 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -284,10 +284,18 @@ def _profile_is_match(have, desired): # compare basic attributes if have.name != desired.name or \ have.traffic_routing_method != desired.traffic_routing_method or \ - have.dns_config.ttl != desired.dns_config.ttl or \ len(have.endpoints) != len(desired.endpoints): return False + # compare dns config + dns_have = have.dns_config + dns_desired = desired.dns_config + if dns_have.ttl != dns_desired.ttl or \ + dns_have.relative_name is None or \ + dns_desired.relative_name is None or \ + dns_have.relative_name != dns_desired.relative_name: + return False + # compare monitoring configuration monitor_have = have.monitor_config monitor_desired = desired.monitor_config diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index e8e5281..0a1c366 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -453,6 +453,7 @@ class Test_ProfileIsMatch(TestCase): name = 'foo-unit-tests', ttl = 60, method = 'Geographic', + dns_name = None, monitor_proto = 'HTTPS', monitor_port = 4443, monitor_path = '/_ping', @@ -465,7 +466,7 @@ class Test_ProfileIsMatch(TestCase): weight = 1, priority = 1, ): - dns = DnsConfig(ttl=ttl) + dns = DnsConfig(relative_name=(dns_name or name), ttl=ttl) return Profile( name=name, traffic_routing_method=method, dns_config=dns, monitor_config=MonitorConfig( @@ -488,6 +489,7 @@ class Test_ProfileIsMatch(TestCase): self.assertFalse(is_match(profile(), profile(name='two'))) self.assertFalse(is_match(profile(), profile(endpoints=2))) + self.assertFalse(is_match(profile(), profile(dns_name='two'))) self.assertFalse(is_match(profile(), profile(monitor_proto='HTTP'))) self.assertFalse(is_match(profile(), profile(endpoint_name='a'))) self.assertFalse(is_match(profile(), profile(endpoint_type='b'))) @@ -596,7 +598,6 @@ class TestAzureDnsProvider(TestCase): id_format = base_id + '{}--' + suffix name_format = '{}--' + suffix - dns = DnsConfig(ttl=60) header = MonitorConfigCustomHeadersItem(name='Host', value='foo.unit.tests') monitor = MonitorConfig(protocol='HTTPS', port=4443, path='/_ping', @@ -604,12 +605,12 @@ class TestAzureDnsProvider(TestCase): external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' - return [ + profiles = [ Profile( id=id_format.format('pool-two'), name=name_format.format('pool-two'), traffic_routing_method='Weighted', - dns_config=dns, + dns_config=DnsConfig(ttl=60), monitor_config=monitor, endpoints=[ Endpoint( @@ -630,7 +631,7 @@ class TestAzureDnsProvider(TestCase): id=id_format.format('rule-one'), name=name_format.format('rule-one'), traffic_routing_method='Priority', - dns_config=dns, + dns_config=DnsConfig(ttl=60), monitor_config=monitor, endpoints=[ Endpoint( @@ -663,7 +664,7 @@ class TestAzureDnsProvider(TestCase): id=id_format.format('rule-two'), name=name_format.format('rule-two'), traffic_routing_method='Priority', - dns_config=dns, + dns_config=DnsConfig(ttl=60), monitor_config=monitor, endpoints=[ Endpoint( @@ -690,7 +691,7 @@ class TestAzureDnsProvider(TestCase): id=base_id + suffix, name=suffix, traffic_routing_method='Geographic', - dns_config=dns, + dns_config=DnsConfig(ttl=60), monitor_config=monitor, endpoints=[ Endpoint( @@ -709,6 +710,11 @@ class TestAzureDnsProvider(TestCase): ), ] + for profile in profiles: + profile.dns_config.relative_name = profile.name + + return profiles + def _get_dynamic_package(self): '''Convenience function to setup a sample dynamic record. ''' From 18c0e5675978507063178cf9a83622ae25fe054b Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Fri, 28 May 2021 12:59:27 -0700 Subject: [PATCH 47/58] log.debug the reason for profile mismatch --- octodns/provider/azuredns.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 65d9967..689088c 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -281,11 +281,20 @@ def _profile_is_match(have, desired): if have is None or desired is None: return False + log = logging.getLogger('azuredns._profile_is_match').debug + + def false(have, desired, name=None): + prefix = 'profile={}'.format(name) if name else '' + attr = have.__class__.__name__ + log('%s have.%s = %s', prefix, attr, have) + log('%s desired.%s = %s', prefix, attr, desired) + return False + # compare basic attributes if have.name != desired.name or \ have.traffic_routing_method != desired.traffic_routing_method or \ len(have.endpoints) != len(desired.endpoints): - return False + return false(have, desired) # compare dns config dns_have = have.dns_config @@ -294,7 +303,7 @@ def _profile_is_match(have, desired): dns_have.relative_name is None or \ dns_desired.relative_name is None or \ dns_have.relative_name != dns_desired.relative_name: - return False + return false(dns_have, dns_desired, have.name) # compare monitoring configuration monitor_have = have.monitor_config @@ -303,7 +312,7 @@ def _profile_is_match(have, desired): monitor_have.port != monitor_desired.port or \ monitor_have.path != monitor_desired.path or \ monitor_have.custom_headers != monitor_desired.custom_headers: - return False + return false(monitor_have, monitor_desired, have.name) # compare endpoints method = have.traffic_routing_method @@ -321,26 +330,26 @@ def _profile_is_match(have, desired): for have_endpoint, desired_endpoint in endpoints: if have_endpoint.name != desired_endpoint.name or \ have_endpoint.type != desired_endpoint.type: - return False + return false(have_endpoint, desired_endpoint, have.name) target_type = have_endpoint.type.split('/')[-1] if target_type == 'externalEndpoints': # compare value, weight, priority if have_endpoint.target != desired_endpoint.target: - return False + return false(have_endpoint, desired_endpoint, have.name) if method == 'Weighted' and \ have_endpoint.weight != desired_endpoint.weight: - return False + return false(have_endpoint, desired_endpoint, have.name) elif target_type == 'nestedEndpoints': # compare targets if have_endpoint.target_resource_id != \ desired_endpoint.target_resource_id: - return False + return false(have_endpoint, desired_endpoint, have.name) # compare geos if method == 'Geographic': have_geos = sorted(have_endpoint.geo_mapping) desired_geos = sorted(desired_endpoint.geo_mapping) if have_geos != desired_geos: - return False + return false(have_endpoint, desired_endpoint, have.name) else: # unexpected, give up return False From aaffdb1388a0569710ca5f0063236d977fdeb9f8 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Fri, 28 May 2021 13:23:35 -0700 Subject: [PATCH 48/58] minimize Azure Traffic Manager hops --- octodns/provider/azuredns.py | 253 ++++++---- tests/test_octodns_provider_azuredns.py | 585 ++++++++++++++++++------ 2 files changed, 622 insertions(+), 216 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 689088c..777e15b 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -646,14 +646,23 @@ class AzureProvider(BaseProvider): pools = defaultdict(lambda: {'fallback': None, 'values': []}) rules = [] - # top level geo profile - geo_profile = self._get_tm_profile_by_id(azrecord.target_resource.id) - for geo_ep in geo_profile.endpoints: + # top level profile + root_profile = self._get_tm_profile_by_id(azrecord.target_resource.id) + if root_profile.traffic_routing_method != 'Geographic': + # This record does not use geo fencing, so we skip the Geographic + # profile hop; let's pretend to be a geo-profile's only endpoint + geo_ep = Endpoint(target_resource_id=root_profile.id) + geo_ep.target_resource = root_profile + endpoints = [geo_ep] + else: + endpoints = root_profile.endpoints + + for geo_ep in endpoints: rule = {} # resolve list of regions - geo_map = list(geo_ep.geo_mapping) - if geo_map != ['WORLD']: + geo_map = list(geo_ep.geo_mapping or []) + if geo_map and geo_map != ['WORLD']: if 'GEO-ME' in geo_map: # Azure treats Middle East as a separate group, but # its part of Asia in octoDNS, so we need to remove GEO-ME @@ -662,7 +671,7 @@ class AzureProvider(BaseProvider): # profile was generated by octoDNS if 'GEO-AS' not in geo_map: msg = 'Profile={} for record {}: '.format( - geo_profile.name, azrecord.fqdn) + root_profile.name, azrecord.fqdn) msg += 'Middle East (GEO-ME) is not supported by ' + \ 'octoDNS. It needs to be either paired ' + \ 'with Asia (GEO-AS) or expanded into ' + \ @@ -690,18 +699,35 @@ class AzureProvider(BaseProvider): # country geos.append(GeoCodes.country_to_code(code)) - # second level priority profile + # build fallback chain from second level priority profile + if geo_ep.target_resource_id: + target = geo_ep.target_resource + if target.traffic_routing_method == 'Priority': + rule_endpoints = target.endpoints + rule_endpoints.sort(key=lambda e: e.priority) + else: + # Weighted + geo_ep.name = target.endpoints[0].name.split('--', 1)[0] + rule_endpoints = [geo_ep] + else: + # this geo directly points to the default, so we skip the + # Priority profile hop and directly use an external endpoint; + # let's pretend to be a Priority profile's only endpoint + rule_endpoints = [geo_ep] + pool = None - rule_endpoints = geo_ep.target_resource.endpoints - rule_endpoints = sorted(rule_endpoints, key=lambda e: e.priority) for rule_ep in rule_endpoints: pool_name = rule_ep.name # last/default pool - if pool_name == '--default--': + if pool_name.endswith('--default--'): default.add(rule_ep.target) - # this should be the last one, so let's break here - break + if pool_name == '--default--': + # this should be the last one, so let's break here + break + # last pool is a single value pool and its value is same + # as record's default value + pool_name = pool_name[:-len('--default--')] # set first priority endpoint as the rule's primary pool if 'pool' not in rule: @@ -711,32 +737,32 @@ class AzureProvider(BaseProvider): # set current pool as fallback of the previous pool pool['fallback'] = pool_name + if pool_name in pools: + # we've already populated the pool + continue + + # populate the pool from Weighted profile + # these should be leaf node entries with no further nesting pool = pools[pool_name] endpoints = [] - # these should be leaf node entries with no further nesting + if rule_ep.target_resource_id: # third (and last) level weighted RR profile endpoints = rule_ep.target_resource.endpoints else: - # single-value pool + # single-value pool, so we skip the Weighted profile hop + # and directly use an external endpoint; let's pretend to + # be a Weighted profile's only endpoint endpoints = [rule_ep] for pool_ep in endpoints: val = pool_ep.target - value_dict = { + pool['values'].append({ 'value': _check_endswith_dot(val), 'weight': pool_ep.weight or 1, - } - if value_dict not in pool['values']: - pool['values'].append(value_dict) - - if 'pool' not in rule or not default: - # this will happen if the priority profile does not have - # enough endpoints - msg = 'Expected at least 2 endpoints in {}, got {}'.format( - geo_ep.target_resource.name, len(rule_endpoints) - ) - raise AzureException(msg) + }) + if pool_ep.name.endswith('--default--'): + default.add(val) rules.append(rule) @@ -809,7 +835,7 @@ class AzureProvider(BaseProvider): elif ep.target: ep.type = endpoint_type_prefix + 'externalEndpoints' else: - msg = ('Invalid endpoint {} in profile {}, needs to have' + + msg = ('Invalid endpoint {} in profile {}, needs to have ' + 'either target or target_resource_id').format( ep.name, name) raise AzureException(msg) @@ -828,39 +854,83 @@ class AzureProvider(BaseProvider): location='global', ) + def _update_tm_name(self, profile, new_name): + profile.name = new_name + profile.id = self._profile_name_to_id(new_name) + profile.dns_config.relative_name = new_name + + return profile + def _generate_traffic_managers(self, record): traffic_managers = [] pools = record.dynamic.pools + default = record.value[:-1] tm_suffix = _traffic_manager_suffix(record) profile = self._generate_tm_profile geo_endpoints = [] + pool_profiles = {} for rule in record.dynamic.rules: + # Prepare the list of Traffic manager geos + rule_geos = rule.data.get('geos', []) + geos = [] + for geo in rule_geos: + if '-' in geo: + # country/state + geos.append(geo.split('-', 1)[-1]) + else: + # continent + if geo == 'AS': + # Middle East is part of Asia in octoDNS, but + # Azure treats it as a separate "group", so let's + # add it in the list of geo mappings. We will drop + # it when we later parse the list of regions. + geos.append('GEO-ME') + elif geo == 'OC': + # Azure uses Australia/Pacific (AP) instead of + # Oceania + geo = 'AP' + + geos.append('GEO-{}'.format(geo)) + if not geos: + geos.append('WORLD') + pool_name = rule.data['pool'] rule_endpoints = [] priority = 1 + default_seen = False while pool_name: # iterate until we reach end of fallback chain + default_seen = False pool = pools[pool_name].data - profile_name = 'pool-{}--{}'.format(pool_name, tm_suffix) if len(pool['values']) > 1: # create Weighted profile for multi-value pool - endpoints = [] - for val in pool['values']: - target = val['value'] - # strip trailing dot from CNAME value - target = target[:-1] - endpoints.append(Endpoint( - name=target, - target=target, - weight=val.get('weight', 1), - )) - pool_profile = profile(profile_name, 'Weighted', endpoints, - record) - traffic_managers.append(pool_profile) + pool_profile = pool_profiles.get(pool_name) + if pool_profile is None: + endpoints = [] + for val in pool['values']: + target = val['value'] + # strip trailing dot from CNAME value + target = target[:-1] + ep_name = '{}--{}'.format(pool_name, target) + if target == default: + # mark default + ep_name += '--default--' + default_seen = True + endpoints.append(Endpoint( + name=ep_name, + target=target, + weight=val.get('weight', 1), + )) + profile_name = 'pool-{}--{}'.format( + pool_name, tm_suffix) + pool_profile = profile(profile_name, 'Weighted', + endpoints, record) + traffic_managers.append(pool_profile) + pool_profiles[pool_name] = pool_profile # append pool to endpoint list of fallback rule profile rule_endpoints.append(Endpoint( @@ -869,8 +939,15 @@ class AzureProvider(BaseProvider): priority=priority, )) else: - # add single-value pool as an external endpoint + # Skip Weighted profile hop for single-value pool + # append its value as an external endpoint to fallback + # rule profile target = pool['values'][0]['value'][:-1] + ep_name = pool_name + if target == default: + # mark default + ep_name += '--default--' + default_seen = True rule_endpoints.append(Endpoint( name=pool_name, target=target, @@ -880,51 +957,61 @@ class AzureProvider(BaseProvider): priority += 1 pool_name = pool.get('fallback') - # append default profile to the end - rule_endpoints.append(Endpoint( - name='--default--', - target=record.value[:-1], - priority=priority, - )) - # create rule profile with fallback chain - rule_profile_name = 'rule-{}--{}'.format(rule.data['pool'], - tm_suffix) - rule_profile = profile(rule_profile_name, 'Priority', - rule_endpoints, record) - traffic_managers.append(rule_profile) + # append default endpoint unless it is already included in + # last pool of rule profile + if not default_seen: + rule_endpoints.append(Endpoint( + name='--default--', + target=default, + priority=priority, + )) - # append rule profile to top-level geo profile - rule_geos = rule.data.get('geos', []) - geos = [] - if len(rule_geos) > 0: - for geo in rule_geos: - if '-' in geo: - # country or state - geos.append(geo.split('-', 1)[-1]) - else: - # continent - if geo == 'AS': - # Middle East is part of Asia in octoDNS, but - # Azure treats it as a separate "group", so let's - # add it in the list of geo mappings. We will drop - # it when we later parse the list of regions. - geos.append('GEO-ME') - elif geo == 'OC': - # Azure uses Australia/Pacific (AP) instead of - # Oceania - geo = 'AP' + if len(rule_endpoints) > 1: + # create rule profile with fallback chain + rule_profile_name = 'rule-{}--{}'.format( + rule.data['pool'], tm_suffix) + rule_profile = profile(rule_profile_name, 'Priority', + rule_endpoints, record) + traffic_managers.append(rule_profile) - geos.append('GEO-{}'.format(geo)) + # append rule profile to top-level geo profile + geo_endpoints.append(Endpoint( + name='rule-{}'.format(rule.data['pool']), + target_resource_id=rule_profile.id, + geo_mapping=geos, + )) else: - geos.append('WORLD') - geo_endpoints.append(Endpoint( - name='rule-{}'.format(rule.data['pool']), - target_resource_id=rule_profile.id, - geo_mapping=geos, - )) + # Priority profile has only one endpoint; skip the hop and + # append its only endpoint to the top-level profile + rule_ep = rule_endpoints[0] + if rule_ep.target_resource_id: + # point directly to the Weighted pool profile + geo_endpoints.append(Endpoint( + name='rule-{}'.format(rule.data['pool']), + target_resource_id=rule_ep.target_resource_id, + geo_mapping=geos, + )) + else: + # just add the value of single-value pool + geo_endpoints.append(Endpoint( + name=rule_ep.name + '--default--', + target=rule_ep.target, + geo_mapping=geos, + )) - geo_profile = profile(tm_suffix, 'Geographic', geo_endpoints, record) - traffic_managers.append(geo_profile) + if len(geo_endpoints) == 1 and \ + geo_endpoints[0].geo_mapping == ['WORLD'] and \ + geo_endpoints[0].target_resource_id: + # Single WORLD rule does not require a Geographic profile, use + # the target profile as the root profile + target_profile_id = geo_endpoints[0].target_resource_id + profile_map = dict((tm.id, tm) for tm in traffic_managers) + target_profile = profile_map[target_profile_id] + self._update_tm_name(target_profile, tm_suffix) + else: + geo_profile = profile(tm_suffix, 'Geographic', geo_endpoints, + record) + traffic_managers.append(geo_profile) return traffic_managers diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 0a1c366..2c998b6 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -614,13 +614,13 @@ class TestAzureDnsProvider(TestCase): monitor_config=monitor, endpoints=[ Endpoint( - name='two1.unit.tests', + name='two--two1.unit.tests', type=external, target='two1.unit.tests', weight=3, ), Endpoint( - name='two2.unit.tests', + name='two--two2.unit.tests', type=external, target='two2.unit.tests', weight=4, @@ -847,121 +847,6 @@ class TestAzureDnsProvider(TestCase): self.assertEquals(len(zone.records), 17) self.assertTrue(exists) - def test_populate_dynamic(self): - # Middle east without Asia raises exception - provider, zone, record = self._get_dynamic_package() - tm_suffix = _traffic_manager_suffix(record) - tm_id = provider._profile_name_to_id - tm_list = provider._tm_client.profiles.list_by_resource_group - rule_name = 'rule-one--{}'.format(tm_suffix) - nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' - tm_list.return_value = [ - Profile( - id=tm_id(tm_suffix), - name=tm_suffix, - traffic_routing_method='Geographic', - endpoints=[ - Endpoint( - geo_mapping=['GEO-ME'], - ), - ], - ), - ] - azrecord = RecordSet( - ttl=60, - target_resource=SubResource(id=tm_id(tm_suffix)), - ) - azrecord.name = record.name or '@' - azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) - with self.assertRaises(AzureException) as ctx: - provider._populate_record(zone, azrecord) - self.assertTrue(text_type(ctx).startswith( - 'Middle East (GEO-ME) is not supported' - )) - - # empty priority profile raises exception - provider, zone, record = self._get_dynamic_package() - tm_list = provider._tm_client.profiles.list_by_resource_group - rule_name = 'rule-one--{}'.format(tm_suffix) - nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' - tm_list.return_value = [ - Profile( - id=tm_id(rule_name), - name=rule_name, - traffic_routing_method='Priority', - endpoints=[], - ), - Profile( - id=tm_id(tm_suffix), - name=tm_suffix, - traffic_routing_method='Geographic', - endpoints=[ - Endpoint( - geo_mapping=['WORLD'], - name='rule-one', - type=nested, - target_resource_id=tm_id(rule_name), - ), - ], - ), - ] - with self.assertRaises(AzureException) as ctx: - provider._populate_record(zone, azrecord) - self.assertTrue(text_type(ctx).startswith( - 'Expected at least 2 endpoints' - )) - - # valid set of profiles produce expected dynamic record - provider, zone, record = self._get_dynamic_package() - root_profile_id = provider._profile_name_to_id( - _traffic_manager_suffix(record) - ) - azrecord = RecordSet( - ttl=60, - target_resource=SubResource(id=root_profile_id), - ) - azrecord.name = record.name or '@' - azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) - - record = provider._populate_record(zone, azrecord) - self.assertEqual(record.name, 'foo') - self.assertEqual(record.ttl, 60) - self.assertEqual(record.value, 'default.unit.tests.') - self.assertEqual(record.dynamic._data(), { - 'pools': { - 'one': { - 'values': [ - {'value': 'one.unit.tests.', 'weight': 1}, - ], - 'fallback': 'two', - }, - 'two': { - 'values': [ - {'value': 'two1.unit.tests.', 'weight': 3}, - {'value': 'two2.unit.tests.', 'weight': 4}, - ], - 'fallback': 'three', - }, - 'three': { - 'values': [ - {'value': 'three.unit.tests.', 'weight': 1}, - ], - 'fallback': None, - }, - }, - 'rules': [ - {'geos': ['AF', 'EU-DE', 'NA-US-CA', 'OC'], 'pool': 'one'}, - {'pool': 'two'}, - ], - }) - - # valid profiles with Middle East test case - geo_profile = provider._get_tm_for_dynamic_record(record) - geo_profile.endpoints[0].geo_mapping.extend(['GEO-ME', 'GEO-AS']) - record = provider._populate_record(zone, azrecord) - self.assertIn('AS', record.dynamic.rules[0].data['geos']) - self.assertNotIn('ME', record.dynamic.rules[0].data['geos']) - def test_populate_zone(self): provider = self._get_provider() @@ -1114,6 +999,7 @@ class TestAzureDnsProvider(TestCase): '/providers/Microsoft.Network/trafficManagerProfiles/' + name self.assertEqual(profile.id, expected_id) self.assertEqual(profile.name, name) + self.assertEqual(profile.name, profile.dns_config.relative_name) self.assertEqual(profile.traffic_routing_method, routing) self.assertEqual(profile.dns_config.ttl, record.ttl) self.assertEqual(len(profile.endpoints), len(endpoints)) @@ -1127,33 +1013,451 @@ class TestAzureDnsProvider(TestCase): 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' ) - def test_generate_traffic_managers(self): + def test_dynamic_record(self): provider, zone, record = self._get_dynamic_package() profiles = provider._generate_traffic_managers(record) - deduped = [] - seen = set() - for profile in profiles: - if profile.name not in seen: - deduped.append(profile) - seen.add(profile.name) # check that every profile is a match with what we expect expected_profiles = self._get_tm_profiles(provider) - self.assertEqual(len(expected_profiles), len(deduped)) - for have, expected in zip(deduped, expected_profiles): + self.assertEqual(len(expected_profiles), len(profiles)) + for have, expected in zip(profiles, expected_profiles): self.assertTrue(_profile_is_match(have, expected)) + # check that dynamic record is populated back from profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[-1].id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + + def test_generate_traffic_managers_middle_east(self): # check Asia/Middle East test case + provider, zone, record = self._get_dynamic_package() record.dynamic._data()['rules'][0]['geos'].append('AS') profiles = provider._generate_traffic_managers(record) - geo_profile_name = _traffic_manager_suffix(record) - geo_profile = next( - profile - for profile in profiles - if profile.name == geo_profile_name + self.assertIn('GEO-ME', profiles[-1].endpoints[0].geo_mapping) + self.assertIn('GEO-AS', profiles[-1].endpoints[0].geo_mapping) + + def test_populate_dynamic_middle_east(self): + # Middle east without Asia raises exception + provider, zone, record = self._get_dynamic_package() + tm_suffix = _traffic_manager_suffix(record) + tm_id = provider._profile_name_to_id + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = [ + Profile( + id=tm_id(tm_suffix), + name=tm_suffix, + traffic_routing_method='Geographic', + endpoints=[ + Endpoint( + geo_mapping=['GEO-ME'], + ), + ], + ), + ] + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=tm_id(tm_suffix)), ) - self.assertIn('GEO-ME', geo_profile.endpoints[0].geo_mapping) - self.assertIn('GEO-AS', geo_profile.endpoints[0].geo_mapping) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + with self.assertRaises(AzureException) as ctx: + provider._populate_record(zone, azrecord) + self.assertTrue(text_type(ctx).startswith( + 'Middle East (GEO-ME) is not supported' + )) + + # valid profiles with Middle East test case + provider, zone, record = self._get_dynamic_package() + geo_profile = provider._get_tm_for_dynamic_record(record) + geo_profile.endpoints[0].geo_mapping.extend(['GEO-ME', 'GEO-AS']) + record = provider._populate_record(zone, azrecord) + self.assertIn('AS', record.dynamic.rules[0].data['geos']) + self.assertNotIn('ME', record.dynamic.rules[0].data['geos']) + + def test_dynamic_no_geo(self): + # test that traffic managers are generated as expected + provider, zone, record = self._get_dynamic_package() + external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + + record = Record.new(zone, 'foo', data={ + 'type': 'CNAME', + 'ttl': 60, + 'value': 'default.unit.tests.', + 'dynamic': { + 'pools': { + 'one': { + 'values': [ + {'value': 'one.unit.tests.'}, + ], + }, + }, + 'rules': [ + {'pool': 'one'}, + ], + } + }) + profiles = provider._generate_traffic_managers(record) + + self.assertEqual(len(profiles), 1) + self.assertTrue(_profile_is_match(profiles[0], Profile( + name='foo-unit-tests', + traffic_routing_method='Priority', + dns_config=DnsConfig( + relative_name='foo-unit-tests', ttl=60), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='one', + type=external, + target='one.unit.tests', + priority=1, + ), + Endpoint( + name='--default--', + type=external, + target='default.unit.tests', + priority=2, + ), + ], + ))) + + # test that same record gets populated back from traffic managers + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[0].id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + + def test_dynamic_fallback_is_default(self): + # test that traffic managers are generated as expected + provider, zone, record = self._get_dynamic_package() + external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + + record = Record.new(zone, 'foo', data={ + 'type': 'CNAME', + 'ttl': 60, + 'value': 'default.unit.tests.', + 'dynamic': { + 'pools': { + 'def': { + 'values': [ + {'value': 'default.unit.tests.'}, + ], + }, + }, + 'rules': [ + {'geos': ['AF'], 'pool': 'def'}, + ], + } + }) + profiles = provider._generate_traffic_managers(record) + + self.assertEqual(len(profiles), 1) + self.assertTrue(_profile_is_match(profiles[0], Profile( + name='foo-unit-tests', + traffic_routing_method='Geographic', + dns_config=DnsConfig( + relative_name='foo-unit-tests', ttl=60), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='def--default--', + type=external, + target='default.unit.tests', + geo_mapping=['GEO-AF'], + ), + ], + ))) + + # test that same record gets populated back from traffic managers + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[0].id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + + def test_dynamic_pool_contains_default(self): + # test that traffic managers are generated as expected + provider, zone, record = self._get_dynamic_package() + tm_id = provider._profile_name_to_id + external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' + + record = Record.new(zone, 'foo', data={ + 'type': 'CNAME', + 'ttl': 60, + 'value': 'default.unit.tests.', + 'dynamic': { + 'pools': { + 'rr': { + 'values': [ + {'value': 'one.unit.tests.'}, + {'value': 'two.unit.tests.'}, + {'value': 'default.unit.tests.'}, + {'value': 'final.unit.tests.'}, + ], + }, + }, + 'rules': [ + {'geos': ['AF'], 'pool': 'rr'}, + ], + } + }) + profiles = provider._generate_traffic_managers(record) + + self.assertEqual(len(profiles), 2) + self.assertTrue(_profile_is_match(profiles[0], Profile( + name='pool-rr--foo-unit-tests', + traffic_routing_method='Weighted', + dns_config=DnsConfig( + relative_name='pool-rr--foo-unit-tests', ttl=60), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='rr--one.unit.tests', + type=external, + target='one.unit.tests', + weight=1, + ), + Endpoint( + name='rr--two.unit.tests', + type=external, + target='two.unit.tests', + weight=1, + ), + Endpoint( + name='rr--default.unit.tests--default--', + type=external, + target='default.unit.tests', + weight=1, + ), + Endpoint( + name='rr--final.unit.tests', + type=external, + target='final.unit.tests', + weight=1, + ), + ], + ))) + self.assertTrue(_profile_is_match(profiles[1], Profile( + name='foo-unit-tests', + traffic_routing_method='Geographic', + dns_config=DnsConfig( + relative_name='foo-unit-tests', ttl=60), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='rule-rr', + type=nested, + target_resource_id=tm_id('pool-rr--foo-unit-tests'), + geo_mapping=['GEO-AF'], + ), + ], + ))) + + # test that same record gets populated back from traffic managers + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[1].id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + + def test_dynamic_pool_contains_default_no_geo(self): + # test that traffic managers are generated as expected + provider, zone, record = self._get_dynamic_package() + external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + + record = Record.new(zone, 'foo', data={ + 'type': 'CNAME', + 'ttl': 60, + 'value': 'default.unit.tests.', + 'dynamic': { + 'pools': { + 'rr': { + 'values': [ + {'value': 'one.unit.tests.'}, + {'value': 'two.unit.tests.'}, + {'value': 'default.unit.tests.'}, + {'value': 'final.unit.tests.'}, + ], + }, + }, + 'rules': [ + {'pool': 'rr'}, + ], + } + }) + profiles = provider._generate_traffic_managers(record) + + self.assertEqual(len(profiles), 1) + self.assertTrue(_profile_is_match(profiles[0], Profile( + name='foo-unit-tests', + traffic_routing_method='Weighted', + dns_config=DnsConfig( + relative_name='foo-unit-tests', ttl=60), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='rr--one.unit.tests', + type=external, + target='one.unit.tests', + weight=1, + ), + Endpoint( + name='rr--two.unit.tests', + type=external, + target='two.unit.tests', + weight=1, + ), + Endpoint( + name='rr--default.unit.tests--default--', + type=external, + target='default.unit.tests', + weight=1, + ), + Endpoint( + name='rr--final.unit.tests', + type=external, + target='final.unit.tests', + weight=1, + ), + ], + ))) + + # test that same record gets populated back from traffic managers + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[0].id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + + def test_dynamic_last_pool_contains_default_no_geo(self): + # test that traffic managers are generated as expected + provider, zone, record = self._get_dynamic_package() + tm_id = provider._profile_name_to_id + external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' + + record = Record.new(zone, 'foo', data={ + 'type': 'CNAME', + 'ttl': 60, + 'value': 'default.unit.tests.', + 'dynamic': { + 'pools': { + 'cloud': { + 'values': [ + {'value': 'cloud.unit.tests.'}, + ], + 'fallback': 'rr', + }, + 'rr': { + 'values': [ + {'value': 'one.unit.tests.'}, + {'value': 'two.unit.tests.'}, + {'value': 'default.unit.tests.'}, + {'value': 'final.unit.tests.'}, + ], + }, + }, + 'rules': [ + {'pool': 'cloud'}, + ], + } + }) + profiles = provider._generate_traffic_managers(record) + + self.assertEqual(len(profiles), 2) + self.assertTrue(_profile_is_match(profiles[0], Profile( + name='pool-rr--foo-unit-tests', + traffic_routing_method='Weighted', + dns_config=DnsConfig( + relative_name='pool-rr--foo-unit-tests', ttl=60), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='rr--one.unit.tests', + type=external, + target='one.unit.tests', + weight=1, + ), + Endpoint( + name='rr--two.unit.tests', + type=external, + target='two.unit.tests', + weight=1, + ), + Endpoint( + name='rr--default.unit.tests--default--', + type=external, + target='default.unit.tests', + weight=1, + ), + Endpoint( + name='rr--final.unit.tests', + type=external, + target='final.unit.tests', + weight=1, + ), + ], + ))) + self.assertTrue(_profile_is_match(profiles[1], Profile( + name='foo-unit-tests', + traffic_routing_method='Priority', + dns_config=DnsConfig( + relative_name='foo-unit-tests', ttl=60), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='cloud', + type=external, + target='cloud.unit.tests', + priority=1, + ), + Endpoint( + name='rr', + type=nested, + target_resource_id=tm_id('pool-rr--foo-unit-tests'), + priority=2, + ), + ], + ))) + + # test that same record gets populated back from traffic managers + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[1].id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) def test_sync_traffic_managers(self): provider, zone, record = self._get_dynamic_package() @@ -1194,6 +1498,21 @@ class TestAzureDnsProvider(TestCase): ) self.assertEqual(new_profile.endpoints[0].weight, 14) + @patch( + 'octodns.provider.azuredns.AzureProvider._generate_traffic_managers') + def test_sync_traffic_managers_duplicate(self, mock_gen_tms): + provider, zone, record = self._get_dynamic_package() + tm_sync = provider._tm_client.profiles.create_or_update + + # change and duplicate profiles + profile = self._get_tm_profiles(provider)[0] + profile.name = 'changing_this_to_trigger_sync' + mock_gen_tms.return_value = [profile, profile] + provider._sync_traffic_managers(record) + + # it should only be called once for duplicate profiles + tm_sync.assert_called_once() + def test_find_traffic_managers(self): provider, zone, record = self._get_dynamic_package() From f90d261133d11e10aff322a9526493315b503584 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Wed, 9 Jun 2021 15:12:04 -0700 Subject: [PATCH 49/58] drop Azure DNS env vars in scripts --- script/coverage | 4 ++++ script/test | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/script/coverage b/script/coverage index bd6e4c9..db8e219 100755 --- a/script/coverage +++ b/script/coverage @@ -25,6 +25,10 @@ export DYN_CUSTOMER= export DYN_PASSWORD= export DYN_USERNAME= export GOOGLE_APPLICATION_CREDENTIALS= +export ARM_CLIENT_ID= +export ARM_CLIENT_SECRET= +export ARM_TENANT_ID= +export ARM_SUBSCRIPTION_ID= # Don't allow disabling coverage grep -r -I --line-number "# pragma: +no.*cover" octodns && { diff --git a/script/test b/script/test index 41edfd8..98bae20 100755 --- a/script/test +++ b/script/test @@ -25,5 +25,9 @@ export DYN_CUSTOMER= export DYN_PASSWORD= export DYN_USERNAME= export GOOGLE_APPLICATION_CREDENTIALS= +export ARM_CLIENT_ID= +export ARM_CLIENT_SECRET= +export ARM_TENANT_ID= +export ARM_SUBSCRIPTION_ID= nosetests "$@" From 5a943f0b13c08796137637dd55149fc2878ddc72 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Wed, 9 Jun 2021 16:01:18 -0700 Subject: [PATCH 50/58] handle broken alias to ATM for dynamic records --- octodns/provider/azuredns.py | 9 +++++++-- tests/test_octodns_provider_azuredns.py | 21 +++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 777e15b..082fd42 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -613,8 +613,13 @@ class AzureProvider(BaseProvider): :type return: dict ''' - if azrecord.cname_record is None and azrecord.target_resource.id: - return self._data_for_dynamic(azrecord) + if azrecord.cname_record is None: + if azrecord.target_resource.id: + return self._data_for_dynamic(azrecord) + else: + # dynamic record alias is broken, return dummy value and apply + # will likely overwrite/fix it + return {'value': 'iam.invalid.'} return {'value': _check_endswith_dot(azrecord.cname_record.cname)} diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 2c998b6..0332a72 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -1677,6 +1677,27 @@ class TestAzureDnsProvider(TestCase): dns_update.assert_not_called() tm_delete.assert_called_once() + # both are dynamic but alias is broken + provider, existing, record1 = self._get_dynamic_package() + azrecord = RecordSet( + ttl=record1.ttl, target_resource=SubResource(id=None)) + azrecord.name = record1.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record1._type) + + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.value, 'iam.invalid.') + + change = Update(record2, record1) + provider._apply_Update(change) + tm_sync, dns_update, tm_delete = ( + provider._tm_client.profiles.create_or_update, + provider._dns_client.record_sets.create_or_update, + provider._tm_client.profiles.delete + ) + tm_sync.assert_not_called() + dns_update.assert_called_once() + tm_delete.assert_not_called() + def test_apply_delete_dynamic(self): provider, existing, record = self._get_dynamic_package() provider._populate_traffic_managers() From 7e0c6296fbcd62243f0b0d673b177e9c6a548ea1 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Wed, 9 Jun 2021 17:25:06 -0700 Subject: [PATCH 51/58] ensure dynamic records map to unique ATM profiles --- octodns/provider/azuredns.py | 77 ++++++++----- tests/test_octodns_provider_azuredns.py | 145 ++++++++++++++++-------- 2 files changed, 147 insertions(+), 75 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 082fd42..bd555a7 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -259,8 +259,21 @@ def _parse_azure_type(string): return string.split('/')[-1] -def _traffic_manager_suffix(record): - return record.fqdn[:-1].replace('.', '-') +def _root_traffic_manager_name(record): + # ATM names can only have letters, numbers and hyphens + # replace dots with double hyphens to ensure unique mapping, + # hoping that real life FQDNs won't have double hyphens + return record.fqdn[:-1].replace('.', '--') + + +def _rule_traffic_manager_name(pool, record): + prefix = _root_traffic_manager_name(record) + return '{}-rule-{}'.format(prefix, pool) + + +def _pool_traffic_manager_name(pool, record): + prefix = _root_traffic_manager_name(record) + return '{}-pool-{}'.format(prefix, pool) def _get_monitor(record): @@ -533,7 +546,7 @@ class AzureProvider(BaseProvider): return self._get_tm_profile_by_id(profile_id) def _get_tm_for_dynamic_record(self, record): - name = _traffic_manager_suffix(record) + name = _root_traffic_manager_name(record) return self._get_tm_profile_by_name(name) def populate(self, zone, target=False, lenient=False): @@ -799,6 +812,7 @@ class AzureProvider(BaseProvider): raise AzureException(msg) log = self.log.info + seen_profiles = {} extra = [] for record in desired.records: if not getattr(record, 'dynamic', False): @@ -813,6 +827,16 @@ class AzureProvider(BaseProvider): profiles = self._generate_traffic_managers(record) for profile in profiles: name = profile.name + if name in seen_profiles: + # exit if a possible collision is detected, even though + # we've tried to ensure unique mapping + msg = 'Collision in Traffic Manager names detected' + msg = '{}: {} and {} both want to use {}'.format( + msg, seen_profiles[name], record.fqdn, name) + raise AzureException(msg) + else: + seen_profiles[name] = record.fqdn + active.add(name) existing_profile = self._get_tm_profile_by_name(name) if not _profile_is_match(existing_profile, profile): @@ -831,7 +855,14 @@ class AzureProvider(BaseProvider): return extra - def _generate_tm_profile(self, name, routing, endpoints, record): + def _generate_tm_profile(self, routing, endpoints, record, label=None): + # figure out profile name and Traffic Manager FQDN + name = _root_traffic_manager_name(record) + if routing == 'Weighted': + name = _pool_traffic_manager_name(label, record) + elif routing == 'Priority': + name = _rule_traffic_manager_name(label, record) + # set appropriate endpoint types endpoint_type_prefix = 'Microsoft.Network/trafficManagerProfiles/' for ep in endpoints: @@ -859,10 +890,10 @@ class AzureProvider(BaseProvider): location='global', ) - def _update_tm_name(self, profile, new_name): - profile.name = new_name - profile.id = self._profile_name_to_id(new_name) - profile.dns_config.relative_name = new_name + def _convert_tm_to_root(self, profile, record): + profile.name = _root_traffic_manager_name(record) + profile.id = self._profile_name_to_id(profile.name) + profile.dns_config.relative_name = profile.name return profile @@ -871,7 +902,6 @@ class AzureProvider(BaseProvider): pools = record.dynamic.pools default = record.value[:-1] - tm_suffix = _traffic_manager_suffix(record) profile = self._generate_tm_profile geo_endpoints = [] @@ -930,10 +960,8 @@ class AzureProvider(BaseProvider): target=target, weight=val.get('weight', 1), )) - profile_name = 'pool-{}--{}'.format( - pool_name, tm_suffix) - pool_profile = profile(profile_name, 'Weighted', - endpoints, record) + pool_profile = profile( + 'Weighted', endpoints, record, pool_name) traffic_managers.append(pool_profile) pool_profiles[pool_name] = pool_profile @@ -973,10 +1001,8 @@ class AzureProvider(BaseProvider): if len(rule_endpoints) > 1: # create rule profile with fallback chain - rule_profile_name = 'rule-{}--{}'.format( - rule.data['pool'], tm_suffix) - rule_profile = profile(rule_profile_name, 'Priority', - rule_endpoints, record) + rule_profile = profile( + 'Priority', rule_endpoints, record, rule.data['pool']) traffic_managers.append(rule_profile) # append rule profile to top-level geo profile @@ -1009,13 +1035,9 @@ class AzureProvider(BaseProvider): geo_endpoints[0].target_resource_id: # Single WORLD rule does not require a Geographic profile, use # the target profile as the root profile - target_profile_id = geo_endpoints[0].target_resource_id - profile_map = dict((tm.id, tm) for tm in traffic_managers) - target_profile = profile_map[target_profile_id] - self._update_tm_name(target_profile, tm_suffix) + self._convert_tm_to_root(traffic_managers[-1], record) else: - geo_profile = profile(tm_suffix, 'Geographic', geo_endpoints, - record) + geo_profile = profile('Geographic', geo_endpoints, record) traffic_managers.append(geo_profile) return traffic_managers @@ -1047,14 +1069,15 @@ class AzureProvider(BaseProvider): return seen def _find_traffic_managers(self, record): - tm_suffix = _traffic_manager_suffix(record) + tm_prefix = _root_traffic_manager_name(record) profiles = set() for profile_id in self._traffic_managers: - # match existing profiles with record's suffix + # match existing profiles with record's prefix name = profile_id.split('/')[-1] - if name == tm_suffix or \ - name.endswith('--{}'.format(tm_suffix)): + if name == tm_prefix or \ + name.startswith('{}-pool-'.format(tm_prefix)) or \ + name.startswith('{}-rule-'.format(tm_prefix)): profiles.add(name) return profiles diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 0332a72..c7840d6 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -7,7 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from octodns.record import Create, Update, Delete, Record from octodns.provider.azuredns import _AzureRecord, AzureProvider, \ - _check_endswith_dot, _parse_azure_type, _traffic_manager_suffix, \ + _check_endswith_dot, _parse_azure_type, _root_traffic_manager_name, \ _get_monitor, _profile_is_match, AzureException from octodns.zone import Zone from octodns.provider.base import Plan @@ -402,12 +402,12 @@ class Test_CheckEndswithDot(TestCase): self.assertEquals(expected, _check_endswith_dot(test)) -class Test_TrafficManagerSuffix(TestCase): - def test_traffic_manager_suffix(self): +class Test_RootTrafficManagerName(TestCase): + def test_root_traffic_manager_name(self): test = Record.new(zone, 'foo', data={ 'ttl': 60, 'type': 'CNAME', 'value': 'default.unit.tests.', }) - self.assertEqual(_traffic_manager_suffix(test), 'foo-unit-tests') + self.assertEqual(_root_traffic_manager_name(test), 'foo--unit--tests') class Test_GetMonitor(TestCase): @@ -594,9 +594,9 @@ class TestAzureDnsProvider(TestCase): base_id = '/subscriptions/' + sub + \ '/resourceGroups/' + rg + \ '/providers/Microsoft.Network/trafficManagerProfiles/' - suffix = 'foo-unit-tests' - id_format = base_id + '{}--' + suffix - name_format = '{}--' + suffix + prefix = 'foo--unit--tests' + name_format = prefix + '-{}' + id_format = base_id + name_format header = MonitorConfigCustomHeadersItem(name='Host', value='foo.unit.tests') @@ -688,8 +688,8 @@ class TestAzureDnsProvider(TestCase): ], ), Profile( - id=base_id + suffix, - name=suffix, + id=base_id + prefix, + name=prefix, traffic_routing_method='Geographic', dns_config=DnsConfig(ttl=60), monitor_config=monitor, @@ -899,10 +899,10 @@ class TestAzureDnsProvider(TestCase): # test unused TM produces the extra change for clean up sample_profile = self._get_tm_profiles(provider)[0] tm_id = provider._profile_name_to_id - root_profile_name = _traffic_manager_suffix(record) + root_profile_name = _root_traffic_manager_name(record) extra_profile = Profile( - id=tm_id('random--{}'.format(root_profile_name)), - name='random--{}'.format(root_profile_name), + id=tm_id('{}-pool-random'.format(root_profile_name)), + name='{}-pool-random'.format(root_profile_name), traffic_routing_method='Weighted', dns_config=sample_profile.dns_config, monitor_config=sample_profile.monitor_config, @@ -968,14 +968,40 @@ class TestAzureDnsProvider(TestCase): }) desired.add_record(a_dynamic) changes.append(Create(a_dynamic)) - with self.assertRaises(AzureException): + with self.assertRaises(AzureException) as ctx: provider._extra_changes(existing, desired, changes) + self.assertTrue(text_type(ctx).endswith( + 'must be of type CNAME' + )) + desired._remove_record(a_dynamic) + + # test colliding ATM names throws exception + record1 = Record.new(desired, 'sub.www', data={ + 'type': record._type, + 'ttl': record.ttl, + 'value': record.value, + 'dynamic': record.dynamic._data(), + }) + record2 = Record.new(desired, 'sub--www', data={ + 'type': record._type, + 'ttl': record.ttl, + 'value': record.value, + 'dynamic': record.dynamic._data(), + }) + desired.add_record(record1) + desired.add_record(record2) + changes = [Create(record1), Create(record2)] + with self.assertRaises(AzureException) as ctx: + provider._extra_changes(existing, desired, changes) + self.assertTrue(text_type(ctx).startswith( + 'Collision in Traffic Manager' + )) def test_generate_tm_profile(self): provider, zone, record = self._get_dynamic_package() profile_gen = provider._generate_tm_profile - name = 'foobar' + label = 'foobar' routing = 'Priority' endpoints = [ Endpoint(target='one.unit.tests'), @@ -985,20 +1011,22 @@ class TestAzureDnsProvider(TestCase): # invalid endpoint raises exception with self.assertRaises(AzureException): - profile_gen(name, routing, endpoints, record) + profile_gen(routing, endpoints, record, label) # regular test endpoints.pop() - profile = profile_gen(name, routing, endpoints, record) + profile = profile_gen(routing, endpoints, record, label) # implicitly tests _profile_name_to_id sub = provider._dns_client_subscription_id rg = provider._resource_group + expected_name = 'foo--unit--tests-rule-foobar' expected_id = '/subscriptions/' + sub + \ '/resourceGroups/' + rg + \ - '/providers/Microsoft.Network/trafficManagerProfiles/' + name + '/providers/Microsoft.Network/trafficManagerProfiles/' + \ + expected_name self.assertEqual(profile.id, expected_id) - self.assertEqual(profile.name, name) + self.assertEqual(profile.name, expected_name) self.assertEqual(profile.name, profile.dns_config.relative_name) self.assertEqual(profile.traffic_routing_method, routing) self.assertEqual(profile.dns_config.ttl, record.ttl) @@ -1044,7 +1072,7 @@ class TestAzureDnsProvider(TestCase): def test_populate_dynamic_middle_east(self): # Middle east without Asia raises exception provider, zone, record = self._get_dynamic_package() - tm_suffix = _traffic_manager_suffix(record) + tm_suffix = _root_traffic_manager_name(record) tm_id = provider._profile_name_to_id tm_list = provider._tm_client.profiles.list_by_resource_group tm_list.return_value = [ @@ -1105,10 +1133,10 @@ class TestAzureDnsProvider(TestCase): self.assertEqual(len(profiles), 1) self.assertTrue(_profile_is_match(profiles[0], Profile( - name='foo-unit-tests', + name='foo--unit--tests', traffic_routing_method='Priority', dns_config=DnsConfig( - relative_name='foo-unit-tests', ttl=60), + relative_name='foo--unit--tests', ttl=60), monitor_config=_get_monitor(record), endpoints=[ Endpoint( @@ -1164,10 +1192,10 @@ class TestAzureDnsProvider(TestCase): self.assertEqual(len(profiles), 1) self.assertTrue(_profile_is_match(profiles[0], Profile( - name='foo-unit-tests', + name='foo--unit--tests', traffic_routing_method='Geographic', dns_config=DnsConfig( - relative_name='foo-unit-tests', ttl=60), + relative_name='foo--unit--tests', ttl=60), monitor_config=_get_monitor(record), endpoints=[ Endpoint( @@ -1222,10 +1250,10 @@ class TestAzureDnsProvider(TestCase): self.assertEqual(len(profiles), 2) self.assertTrue(_profile_is_match(profiles[0], Profile( - name='pool-rr--foo-unit-tests', + name='foo--unit--tests-pool-rr', traffic_routing_method='Weighted', dns_config=DnsConfig( - relative_name='pool-rr--foo-unit-tests', ttl=60), + relative_name='foo--unit--tests-pool-rr', ttl=60), monitor_config=_get_monitor(record), endpoints=[ Endpoint( @@ -1255,16 +1283,16 @@ class TestAzureDnsProvider(TestCase): ], ))) self.assertTrue(_profile_is_match(profiles[1], Profile( - name='foo-unit-tests', + name='foo--unit--tests', traffic_routing_method='Geographic', dns_config=DnsConfig( - relative_name='foo-unit-tests', ttl=60), + relative_name='foo--unit--tests', ttl=60), monitor_config=_get_monitor(record), endpoints=[ Endpoint( name='rule-rr', type=nested, - target_resource_id=tm_id('pool-rr--foo-unit-tests'), + target_resource_id=tm_id(profiles[0].name), geo_mapping=['GEO-AF'], ), ], @@ -1311,10 +1339,10 @@ class TestAzureDnsProvider(TestCase): self.assertEqual(len(profiles), 1) self.assertTrue(_profile_is_match(profiles[0], Profile( - name='foo-unit-tests', + name='foo--unit--tests', traffic_routing_method='Weighted', dns_config=DnsConfig( - relative_name='foo-unit-tests', ttl=60), + relative_name='foo--unit--tests', ttl=60), monitor_config=_get_monitor(record), endpoints=[ Endpoint( @@ -1393,10 +1421,10 @@ class TestAzureDnsProvider(TestCase): self.assertEqual(len(profiles), 2) self.assertTrue(_profile_is_match(profiles[0], Profile( - name='pool-rr--foo-unit-tests', + name='foo--unit--tests-pool-rr', traffic_routing_method='Weighted', dns_config=DnsConfig( - relative_name='pool-rr--foo-unit-tests', ttl=60), + relative_name='foo--unit--tests-pool-rr', ttl=60), monitor_config=_get_monitor(record), endpoints=[ Endpoint( @@ -1426,10 +1454,10 @@ class TestAzureDnsProvider(TestCase): ], ))) self.assertTrue(_profile_is_match(profiles[1], Profile( - name='foo-unit-tests', + name='foo--unit--tests', traffic_routing_method='Priority', dns_config=DnsConfig( - relative_name='foo-unit-tests', ttl=60), + relative_name='foo--unit--tests', ttl=60), monitor_config=_get_monitor(record), endpoints=[ Endpoint( @@ -1441,7 +1469,7 @@ class TestAzureDnsProvider(TestCase): Endpoint( name='rr', type=nested, - target_resource_id=tm_id('pool-rr--foo-unit-tests'), + target_resource_id=tm_id(profiles[0].name), priority=2, ), ], @@ -1459,16 +1487,37 @@ class TestAzureDnsProvider(TestCase): record2 = provider._populate_record(zone, azrecord) self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + def test_dynamic_unique_traffic_managers(self): + record = self._get_dynamic_record(zone) + data = { + 'type': record._type, + 'ttl': record.ttl, + 'value': record.value, + 'dynamic': record.dynamic._data() + } + record_names = [ + 'www.foo', 'www-foo' + ] + provider = self._get_provider() + + seen = set() + for name in record_names: + record = Record.new(zone, name, data=data) + tms = provider._generate_traffic_managers(record) + for tm in tms: + self.assertNotIn(tm.name, seen) + seen.add(tm.name) + def test_sync_traffic_managers(self): provider, zone, record = self._get_dynamic_package() provider._populate_traffic_managers() tm_sync = provider._tm_client.profiles.create_or_update - suffix = 'foo-unit-tests' + prefix = 'foo--unit--tests' expected_seen = { - suffix, 'pool-two--{}'.format(suffix), - 'rule-one--{}'.format(suffix), 'rule-two--{}'.format(suffix), + prefix, '{}-pool-two'.format(prefix), + '{}-rule-one'.format(prefix), '{}-rule-two'.format(prefix), } # test no change @@ -1494,7 +1543,7 @@ class TestAzureDnsProvider(TestCase): # test that new profile was successfully inserted in cache new_profile = provider._get_tm_profile_by_name( - 'pool-two--{}'.format(suffix) + '{}-pool-two'.format(prefix) ) self.assertEqual(new_profile.endpoints[0].weight, 14) @@ -1524,11 +1573,11 @@ class TestAzureDnsProvider(TestCase): 'ttl': record.ttl, 'value': record.value, }) - suffix2 = _traffic_manager_suffix(record2) + prefix2 = _root_traffic_manager_name(record2) tm_id = provider._profile_name_to_id extra_profile = Profile( - id=tm_id('random--{}'.format(suffix2)), - name='random--{}'.format(suffix2), + id=tm_id('{}-pool-random'.format(prefix2)), + name='{}-pool-random'.format(prefix2), traffic_routing_method='Weighted', dns_config=sample_profile.dns_config, monitor_config=sample_profile.monitor_config, @@ -1539,10 +1588,10 @@ class TestAzureDnsProvider(TestCase): provider._populate_traffic_managers() # implicitly asserts that non-matching profile is not included - suffix = _traffic_manager_suffix(record) + prefix = _root_traffic_manager_name(record) self.assertEqual(provider._find_traffic_managers(record), { - suffix, 'pool-two--{}'.format(suffix), - 'rule-one--{}'.format(suffix), 'rule-two--{}'.format(suffix), + prefix, '{}-pool-two'.format(prefix), + '{}-rule-one'.format(prefix), '{}-rule-two'.format(prefix), }) def test_traffic_manager_gc(self): @@ -1655,10 +1704,10 @@ class TestAzureDnsProvider(TestCase): provider, existing, dynamic_record = self._get_dynamic_package() sample_profile = self._get_tm_profiles(provider)[0] tm_id = provider._profile_name_to_id - root_profile_name = _traffic_manager_suffix(dynamic_record) + root_profile_name = _root_traffic_manager_name(dynamic_record) extra_profile = Profile( - id=tm_id('random--{}'.format(root_profile_name)), - name='random--{}'.format(root_profile_name), + id=tm_id('{}-pool-random'.format(root_profile_name)), + name='{}-pool-random'.format(root_profile_name), traffic_routing_method='Weighted', dns_config=sample_profile.dns_config, monitor_config=sample_profile.monitor_config, From 6734448462b79977ef0d0fa1391c716ad63bee6c Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Fri, 11 Jun 2021 11:43:35 -0700 Subject: [PATCH 52/58] log warning when dynamic CNAME has broken alias --- octodns/provider/azuredns.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index bd555a7..0fd3dae 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -632,6 +632,10 @@ class AzureProvider(BaseProvider): else: # dynamic record alias is broken, return dummy value and apply # will likely overwrite/fix it + self.log.warn('_data_for_CNAME: Missing Traffic Manager ' + 'alias for dynamic CNAME record %s, forcing ' + 're-link by setting an invalid value', + azrecord.fqdn) return {'value': 'iam.invalid.'} return {'value': _check_endswith_dot(azrecord.cname_record.cname)} From fec0aea14426b7f8cb9845063ec72bd4d89f6f8d Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Mon, 14 Jun 2021 13:56:19 -0700 Subject: [PATCH 53/58] minor clean up of azuredns tests --- tests/test_octodns_provider_azuredns.py | 26 ++++++++++++------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index c7840d6..81a3084 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -1109,7 +1109,7 @@ class TestAzureDnsProvider(TestCase): def test_dynamic_no_geo(self): # test that traffic managers are generated as expected - provider, zone, record = self._get_dynamic_package() + provider = self._get_provider() external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' record = Record.new(zone, 'foo', data={ @@ -1159,7 +1159,7 @@ class TestAzureDnsProvider(TestCase): tm_list.return_value = profiles azrecord = RecordSet( ttl=60, - target_resource=SubResource(id=profiles[0].id), + target_resource=SubResource(id=profiles[-1].id), ) azrecord.name = record.name or '@' azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) @@ -1168,7 +1168,7 @@ class TestAzureDnsProvider(TestCase): def test_dynamic_fallback_is_default(self): # test that traffic managers are generated as expected - provider, zone, record = self._get_dynamic_package() + provider = self._get_provider() external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' record = Record.new(zone, 'foo', data={ @@ -1212,7 +1212,7 @@ class TestAzureDnsProvider(TestCase): tm_list.return_value = profiles azrecord = RecordSet( ttl=60, - target_resource=SubResource(id=profiles[0].id), + target_resource=SubResource(id=profiles[-1].id), ) azrecord.name = record.name or '@' azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) @@ -1221,8 +1221,7 @@ class TestAzureDnsProvider(TestCase): def test_dynamic_pool_contains_default(self): # test that traffic managers are generated as expected - provider, zone, record = self._get_dynamic_package() - tm_id = provider._profile_name_to_id + provider = self._get_provider() external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' @@ -1292,7 +1291,7 @@ class TestAzureDnsProvider(TestCase): Endpoint( name='rule-rr', type=nested, - target_resource_id=tm_id(profiles[0].name), + target_resource_id=profiles[0].id, geo_mapping=['GEO-AF'], ), ], @@ -1303,7 +1302,7 @@ class TestAzureDnsProvider(TestCase): tm_list.return_value = profiles azrecord = RecordSet( ttl=60, - target_resource=SubResource(id=profiles[1].id), + target_resource=SubResource(id=profiles[-1].id), ) azrecord.name = record.name or '@' azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) @@ -1312,7 +1311,7 @@ class TestAzureDnsProvider(TestCase): def test_dynamic_pool_contains_default_no_geo(self): # test that traffic managers are generated as expected - provider, zone, record = self._get_dynamic_package() + provider = self._get_provider() external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' record = Record.new(zone, 'foo', data={ @@ -1377,7 +1376,7 @@ class TestAzureDnsProvider(TestCase): tm_list.return_value = profiles azrecord = RecordSet( ttl=60, - target_resource=SubResource(id=profiles[0].id), + target_resource=SubResource(id=profiles[-1].id), ) azrecord.name = record.name or '@' azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) @@ -1386,8 +1385,7 @@ class TestAzureDnsProvider(TestCase): def test_dynamic_last_pool_contains_default_no_geo(self): # test that traffic managers are generated as expected - provider, zone, record = self._get_dynamic_package() - tm_id = provider._profile_name_to_id + provider = self._get_provider() external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' @@ -1469,7 +1467,7 @@ class TestAzureDnsProvider(TestCase): Endpoint( name='rr', type=nested, - target_resource_id=tm_id(profiles[0].name), + target_resource_id=profiles[0].id, priority=2, ), ], @@ -1480,7 +1478,7 @@ class TestAzureDnsProvider(TestCase): tm_list.return_value = profiles azrecord = RecordSet( ttl=60, - target_resource=SubResource(id=profiles[1].id), + target_resource=SubResource(id=profiles[-1].id), ) azrecord.name = record.name or '@' azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) From 725189af49791686542c5f580b2c30660ea653bb Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Mon, 14 Jun 2021 13:58:20 -0700 Subject: [PATCH 54/58] Handle re-used pools in Azure DNS dynamic records --- octodns/provider/azuredns.py | 28 ++++++++++- tests/test_octodns_provider_azuredns.py | 67 +++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 2 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 0fd3dae..93478b8 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -717,6 +717,8 @@ class AzureProvider(BaseProvider): country, province = code.split('-', 1) country = GeoCodes.country_to_code(country) geos.append('{}-{}'.format(country, province)) + elif code == 'WORLD': + geos.append(code) else: # country geos.append(GeoCodes.country_to_code(code)) @@ -788,6 +790,13 @@ class AzureProvider(BaseProvider): rules.append(rule) + # add separate rule for re-used world pool + for rule in list(rules): + geos = rule.get('geos', []) + if len(geos) > 1 and 'WORLD' in geos: + geos.remove('WORLD') + rules.append({'pool': rule['pool']}) + # Order and convert to a list default = sorted(default) @@ -904,14 +913,29 @@ class AzureProvider(BaseProvider): def _generate_traffic_managers(self, record): traffic_managers = [] pools = record.dynamic.pools + rules = record.dynamic.rules default = record.value[:-1] profile = self._generate_tm_profile + # a pool can be re-used only with a world pool, record the pool + # to later consolidate it with a geo pool if one exists since we + # can't have multiple endpoints with the same target in ATM + world_pool = None + for rule in rules: + if not rule.data.get('geos', []): + world_pool = rule.data['pool'] + world_seen = False + geo_endpoints = [] pool_profiles = {} for rule in record.dynamic.rules: + pool_name = rule.data['pool'] + if pool_name == world_pool and world_seen: + # this world pool is already mentioned in another geo rule + continue + # Prepare the list of Traffic manager geos rule_geos = rule.data.get('geos', []) geos = [] @@ -933,10 +957,10 @@ class AzureProvider(BaseProvider): geo = 'AP' geos.append('GEO-{}'.format(geo)) - if not geos: + if not geos or pool_name == world_pool: geos.append('WORLD') + world_seen = True - pool_name = rule.data['pool'] rule_endpoints = [] priority = 1 default_seen = False diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 81a3084..56a783d 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -1506,6 +1506,73 @@ class TestAzureDnsProvider(TestCase): self.assertNotIn(tm.name, seen) seen.add(tm.name) + def test_dynamic_reused_pool(self): + # test that traffic managers are generated as expected + provider = self._get_provider() + nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' + + record = Record.new(zone, 'foo', data={ + 'type': 'CNAME', + 'ttl': 60, + 'value': 'default.unit.tests.', + 'dynamic': { + 'pools': { + 'iad': { + 'values': [ + {'value': 'iad.unit.tests.'}, + ], + 'fallback': 'lhr', + }, + 'lhr': { + 'values': [ + {'value': 'lhr.unit.tests.'}, + ], + }, + }, + 'rules': [ + {'geos': ['EU'], 'pool': 'iad'}, + {'geos': ['EU-GB'], 'pool': 'lhr'}, + {'pool': 'lhr'}, + ], + } + }) + profiles = provider._generate_traffic_managers(record) + + self.assertEqual(len(profiles), 3) + self.assertTrue(_profile_is_match(profiles[-1], Profile( + name='foo--unit--tests', + traffic_routing_method='Geographic', + dns_config=DnsConfig( + relative_name='foo--unit--tests', ttl=record.ttl), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='rule-iad', + type=nested, + target_resource_id=profiles[0].id, + geo_mapping=['GEO-EU'], + ), + Endpoint( + name='rule-lhr', + type=nested, + target_resource_id=profiles[1].id, + geo_mapping=['GB', 'WORLD'], + ), + ], + ))) + + # test that same record gets populated back from traffic managers + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[-1].id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + def test_sync_traffic_managers(self): provider, zone, record = self._get_dynamic_package() provider._populate_traffic_managers() From a31fa189c131b8e45b5cddb309945e1a4bb7e3d7 Mon Sep 17 00:00:00 2001 From: Ella <72365100+eilla1@users.noreply.github.com> Date: Wed, 16 Jun 2021 00:32:43 -0700 Subject: [PATCH 55/58] fix: update syntax for commands for example: `$ octodns-sync --config-file=./config/production.yaml` was throwing an error. Then, when i did `$ octodns-dump`, it prompted me to put it in this format: `$ octodns-sync --config-file CONFIG_FILE` (without the = sign) --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2ce9445..eb7789b 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ Further information can be found in [Records Documentation](/docs/records.md). We're ready to do a dry-run with our new setup to see what changes it would make. Since we're pretending here we'll act like there are no existing records for `example.com.` in our accounts on either provider. ```shell -$ octodns-sync --config-file=./config/production.yaml +$ octodns-sync --config-file ./config/production.yaml ... ******************************************************************************** * example.com. @@ -146,7 +146,7 @@ There will be other logging information presented on the screen, but successful Now it's time to tell OctoDNS to make things happen. We'll invoke it again with the same options and add a `--doit` on the end to tell it this time we actually want it to try and make the specified changes. ```shell -$ octodns-sync --config-file=./config/production.yaml --doit +$ octodns-sync --config-file ./config/production.yaml --doit ... ``` @@ -177,7 +177,7 @@ If that goes smoothly, you again see the expected changes, and verify them with Very few situations will involve starting with a blank slate which is why there's tooling built in to pull existing data out of providers into a matching config file. ```shell -$ octodns-dump --config-file=config/production.yaml --output-dir=tmp/ example.com. route53 +$ octodns-dump --config-file ./config/production.yaml --output-dir tmp/ example.com. route53 2017-03-15T13:33:34 INFO Manager __init__: config_file=tmp/production.yaml 2017-03-15T13:33:34 INFO Manager dump: zone=example.com., sources=('route53',) 2017-03-15T13:33:36 INFO Route53Provider[route53] populate: found 64 records From fb805ced62e649dd42529081055cf38bcb616bc8 Mon Sep 17 00:00:00 2001 From: Ella <72365100+eilla1@users.noreply.github.com> Date: Wed, 16 Jun 2021 00:39:01 -0700 Subject: [PATCH 56/58] chore: add alt text to images --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index eb7789b..3ad16d3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ - +OctoDNS Logo ## DNS as code - Tools for managing DNS across multiple providers @@ -158,17 +158,17 @@ In the above case we manually ran OctoDNS from the command line. That works and The first step is to create a PR with your changes. -![](/docs/assets/pr.png) +![GitHub user interface of a pull request](/docs/assets/pr.png) Assuming the code tests and config validation statuses are green the next step is to do a noop deploy and verify that the changes OctoDNS plans to make are the ones you expect. -![](/docs/assets/noop.png) +![Output of a noop deployment command](/docs/assets/noop.png) After that comes a set of reviews. One from a teammate who should have full context on what you're trying to accomplish and visibility in to the changes you're making to do it. The other is from a member of the team here at GitHub that owns DNS, mostly as a sanity check and to make sure that best practices are being followed. As much of that as possible is baked into `octodns-validate`. After the reviews it's time to branch deploy the change. -![](/docs/assets/deploy.png) +![Output of a deployment command](/docs/assets/deploy.png) If that goes smoothly, you again see the expected changes, and verify them with `dig` and/or `octodns-report` you're good to hit the merge button. If there are problems you can quickly do a `.deploy dns/master` to go back to the previous state. From 501ae886738522b13e3c9e8c5f564c48d5c135d8 Mon Sep 17 00:00:00 2001 From: Ella <72365100+eilla1@users.noreply.github.com> Date: Wed, 16 Jun 2021 15:25:14 -0700 Subject: [PATCH 57/58] chore: use = --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3ad16d3..91e04eb 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ Further information can be found in [Records Documentation](/docs/records.md). We're ready to do a dry-run with our new setup to see what changes it would make. Since we're pretending here we'll act like there are no existing records for `example.com.` in our accounts on either provider. ```shell -$ octodns-sync --config-file ./config/production.yaml +$ octodns-sync --config-file=./config/production.yaml ... ******************************************************************************** * example.com. @@ -146,7 +146,7 @@ There will be other logging information presented on the screen, but successful Now it's time to tell OctoDNS to make things happen. We'll invoke it again with the same options and add a `--doit` on the end to tell it this time we actually want it to try and make the specified changes. ```shell -$ octodns-sync --config-file ./config/production.yaml --doit +$ octodns-sync --config-file=./config/production.yaml --doit ... ``` @@ -177,7 +177,7 @@ If that goes smoothly, you again see the expected changes, and verify them with Very few situations will involve starting with a blank slate which is why there's tooling built in to pull existing data out of providers into a matching config file. ```shell -$ octodns-dump --config-file ./config/production.yaml --output-dir tmp/ example.com. route53 +$ octodns-dump --config-file=config/production.yaml --output-dir=tmp/ example.com. route53 2017-03-15T13:33:34 INFO Manager __init__: config_file=tmp/production.yaml 2017-03-15T13:33:34 INFO Manager dump: zone=example.com., sources=('route53',) 2017-03-15T13:33:36 INFO Route53Provider[route53] populate: found 64 records From 794fca0ee765b8ae8912ef557c66484f8253959f Mon Sep 17 00:00:00 2001 From: Iordanis Grigoriou Date: Thu, 17 Jun 2021 16:36:46 +0200 Subject: [PATCH 58/58] Fix typo --- octodns/provider/dyn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/dyn.py b/octodns/provider/dyn.py index da56a2e..2fbcadb 100644 --- a/octodns/provider/dyn.py +++ b/octodns/provider/dyn.py @@ -604,7 +604,7 @@ class DynProvider(BaseProvider): return record - def _is_traffic_director_dyanmic(self, td, rulesets): + def _is_traffic_director_dynamic(self, td, rulesets): for ruleset in rulesets: try: pieces = ruleset.label.split(':') @@ -632,7 +632,7 @@ class DynProvider(BaseProvider): continue # critical to call rulesets once, each call loads them :-( rulesets = td.rulesets - if self._is_traffic_director_dyanmic(td, rulesets): + if self._is_traffic_director_dynamic(td, rulesets): record = \ self._populate_dynamic_traffic_director(zone, fqdn, _type, td,