From c71784b0e832850f1c535485994976d9d48b29da Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Mon, 12 Apr 2021 08:06:59 +0200 Subject: [PATCH 01/18] initial work on Hetzner provider - implemented HetznerClient API class - tested manually, lacking formal tests --- octodns/provider/hetzner.py | 91 +++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 octodns/provider/hetzner.py diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py new file mode 100644 index 0000000..17dc046 --- /dev/null +++ b/octodns/provider/hetzner.py @@ -0,0 +1,91 @@ +# +# +# + +from requests import Session + + +class HetznerClientException(Exception): + pass + + +class HetznerClientNotFound(HetznerClientException): + + def __init__(self): + super(HetznerClientNotFound, self).__init__('Not Found') + + +class HetznerClient(object): + BASE_URL = 'https://dns.hetzner.com/api/v1' + + def __init__(self, token): + session = Session() + session.headers.update({'Auth-API-Token': token}) + self._session = session + + def _do(self, method, path, params=None, data=None): + url = self.BASE_URL + path + response = self._session.request(method, url, params=params, json=data) + if response.status_code == 404: + raise HetznerClientNotFound() + response.raise_for_status() + return response.json() + + def _do_with_pagination(self, method, path, key, params=None, data=None, + per_page=100): + pagination_params = {'page': 1, 'per_page': per_page} + if params is not None: + params = {**params, **pagination_params} + else: + params = pagination_params + + items = [] + while True: + response = self._do(method, path, params, data) + items += response[key] + if response['meta']['pagination']['page'] >= \ + response['meta']['pagination']['last_page']: + break + params['page'] += 1 + return items + + def zones_get(self, name=None, search_name=None): + params = {'name': name, 'search_name': search_name} + return self._do_with_pagination('GET', '/zones', 'zones', + params=params) + + def zone_get(self, zone_id): + return self._do('GET', '/zones/' + zone_id)['zone'] + + def zone_create(self, name, ttl=None): + data = {'name': name, 'ttl': ttl} + return self._do('POST', '/zones', data=data)['zone'] + + def zone_update(self, zone_id, name, ttl=None): + data = {'name': name, 'ttl': ttl} + return self._do('PUT', '/zones/' + zone_id, data=data)['zone'] + + def zone_delete(self, zone_id): + return self._do('DELETE', '/zones/' + zone_id) + + def zone_records_get(self, zone_id): + params = {'zone_id': zone_id} + # No need to handle pagination as it returns all records by default. + return self._do('GET', '/records', params=params)['records'] + + def zone_record_get(self, record_id): + return self._do('GET', '/records/' + record_id)['record'] + + def zone_record_create(self, zone_id, name, _type, value, ttl=None): + data = {'name': name, 'ttl': ttl, 'type': _type, 'value': value, + 'zone_id': zone_id} + return self._do('POST', '/records', data=data)['record'] + + def zone_record_update(self, zone_id, record_id, name, _type, value, + ttl=None): + data = {'name': name, 'ttl': ttl, 'type': _type, 'value': value, + 'zone_id': zone_id} + return self._do('PUT', '/records/' + record_id, data=data)['record'] + + def zone_record_delete(self, zone_id, record_id): + return self._do('DELETE', '/records/' + record_id) From c8e91c1e11b86106645ac67055b34eec4b6724f4 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Mon, 12 Apr 2021 21:08:53 +0200 Subject: [PATCH 02/18] added HetznerClient docstring --- octodns/provider/hetzner.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 17dc046..1002e7d 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -16,6 +16,15 @@ class HetznerClientNotFound(HetznerClientException): class HetznerClient(object): + ''' + Hetzner DNS Public API v1 client class. + + Zone and Record resources are (almost) fully supported, even if unnecessary + to future-proof this client. Bulk Record create/update is not supported. + + No support for Primary Servers. + ''' + BASE_URL = 'https://dns.hetzner.com/api/v1' def __init__(self, token): From 8a9743b36ec56b81429b0bf9481c0f674a9834a9 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Tue, 13 Apr 2021 08:08:48 +0200 Subject: [PATCH 03/18] added HetznerClient._replace_at to address the "@"/"" record name problem --- octodns/provider/hetzner.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 1002e7d..2e069d0 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -77,24 +77,34 @@ class HetznerClient(object): def zone_delete(self, zone_id): return self._do('DELETE', '/zones/' + zone_id) + def _replace_at(self, record): + record['name'] = '' if record['name'] == '@' else record['name'] + return record + def zone_records_get(self, zone_id): params = {'zone_id': zone_id} # No need to handle pagination as it returns all records by default. - return self._do('GET', '/records', params=params)['records'] + return [ + self._replace_at(record) + for record in self._do('GET', '/records', params=params)['records'] + ] def zone_record_get(self, record_id): - return self._do('GET', '/records/' + record_id)['record'] + record = self._do('GET', '/records/' + record_id)['record'] + return self._replace_at(record) def zone_record_create(self, zone_id, name, _type, value, ttl=None): - data = {'name': name, 'ttl': ttl, 'type': _type, 'value': value, + data = {'name': name or '@', 'ttl': ttl, 'type': _type, 'value': value, 'zone_id': zone_id} - return self._do('POST', '/records', data=data)['record'] + record = self._do('POST', '/records', data=data)['record'] + return self._replace_at(record) def zone_record_update(self, zone_id, record_id, name, _type, value, ttl=None): - data = {'name': name, 'ttl': ttl, 'type': _type, 'value': value, + data = {'name': name or '@', 'ttl': ttl, 'type': _type, 'value': value, 'zone_id': zone_id} - return self._do('PUT', '/records/' + record_id, data=data)['record'] + record = self._do('PUT', '/records/' + record_id, data=data)['record'] + return self._replace_at(record) def zone_record_delete(self, zone_id, record_id): return self._do('DELETE', '/records/' + record_id) From f507349ce506af5554de338a0eb56977b8632a60 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Wed, 14 Apr 2021 07:57:23 +0200 Subject: [PATCH 04/18] implemented HetznerProvider --- octodns/provider/hetzner.py | 254 ++++++++++++++++++++++++++++++++++++ 1 file changed, 254 insertions(+) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 2e069d0..570cefe 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -2,8 +2,17 @@ # # +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +import logging +from collections import defaultdict + from requests import Session +from ..record import Record +from .base import BaseProvider + class HetznerClientException(Exception): pass @@ -108,3 +117,248 @@ class HetznerClient(object): def zone_record_delete(self, zone_id, record_id): return self._do('DELETE', '/records/' + record_id) + + +class HetznerProvider(BaseProvider): + ''' + Hetzner DNS provider using API v1 + + hetzner: + class: octodns.provider.hetzner.HetznerProvider + # Your Hetzner API token (required) + token: foo + ''' + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = False + SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'SRV', 'TXT')) + + def __init__(self, id, token, *args, **kwargs): + self.log = logging.getLogger('HetznerProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, token=***', id) + super(HetznerProvider, self).__init__(id, *args, **kwargs) + self._client = HetznerClient(token) + + self._zone_records = {} + + def _append_dot(self, value): + return value if value[-1] == '.' else '{}.'.format(value) + + def _data_for_multiple(self, _type, records): + values = [record['value'].replace(';', '\\;') for record in records] + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + + def _data_for_CAA(self, _type, records): + values = [] + for record in records: + value_without_spaces = record['value'].replace(' ', '') + flags = value_without_spaces[0] + tag = value_without_spaces[1:].split('"')[0] + value = record['value'].split('"')[1] + values.append({ + 'flags': int(flags), + 'tag': tag, + 'value': value, + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + def _data_for_CNAME(self, _type, records): + record = records[0] + return { + 'ttl': record['ttl'], + 'type': _type, + 'value': self._append_dot(record['value']) + } + + def _data_for_MX(self, _type, records): + values = [] + for record in records: + value_stripped_split = record['value'].strip().split(' ') + preference = value_stripped_split[0] + exchange = value_stripped_split[-1] + values.append({ + 'preference': int(preference), + 'exchange': self._append_dot(exchange) + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + def _data_for_NS(self, _type, records): + values = [] + for record in records: + values.append(self._append_dot(record['value'])) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values, + } + + def _data_for_SRV(self, _type, records): + values = [] + for record in records: + value_stripped = record['value'].strip() + priority = value_stripped.split(' ')[0] + weight = value_stripped[len(priority):].strip().split(' ')[0] + target = value_stripped.split(' ')[-1] + port = value_stripped[:-len(target)].strip().split(' ')[-1] + values.append({ + 'port': int(port), + 'priority': int(priority), + 'target': self._append_dot(target), + 'weight': int(weight) + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + _data_for_TXT = _data_for_multiple + + def zone_records(self, zone): + if zone.name not in self._zone_records: + try: + zone_id = self._client.zones_get(name=zone.name[:-1])[0]['id'] + self._zone_records[zone.name] = \ + self._client.zone_records_get(zone_id) + except HetznerClientNotFound: + return [] + + return self._zone_records[zone.name] + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + + values = defaultdict(lambda: defaultdict(list)) + for record in self.zone_records(zone): + _type = record['type'] + if _type not in self.SUPPORTS: + self.log.warning('populate: skipping unsupported %s record', + _type) + continue + values[record['name']][record['type']].append(record) + + before = len(zone.records) + for name, types in values.items(): + for _type, records in types.items(): + data_for = getattr(self, '_data_for_{}'.format(_type)) + record = Record.new(zone, name, data_for(_type, records), + source=self, lenient=lenient) + zone.add_record(record, lenient=lenient) + + exists = zone.name in self._zone_records + self.log.info('populate: found %s records, exists=%s', + len(zone.records) - before, exists) + return exists + + def _params_for_multiple(self, record): + for value in record.values: + yield { + 'value': value.replace('\\;', ';'), + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + _params_for_A = _params_for_multiple + _params_for_AAAA = _params_for_multiple + + def _params_for_CAA(self, record): + for value in record.values: + data = '{} {} "{}"'.format(value.flags, value.tag, value.value) + yield { + 'value': data, + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + def _params_for_single(self, record): + yield { + 'value': record.value, + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + _params_for_CNAME = _params_for_single + + def _params_for_MX(self, record): + for value in record.values: + data = '{} {}'.format(value.preference, value.exchange) + yield { + 'value': data, + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + _params_for_NS = _params_for_multiple + + def _params_for_SRV(self, record): + for value in record.values: + data = '{} {} {} {}'.format(value.priority, value.weight, + value.port, value.target) + yield { + 'value': data, + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + _params_for_TXT = _params_for_multiple + + def _apply_Create(self, zone_id, change): + new = change.new + params_for = getattr(self, '_params_for_{}'.format(new._type)) + for params in params_for(new): + self._client.zone_record_create(zone_id, params['name'], + params['type'], params['value'], + params['ttl']) + + def _apply_Update(self, zone_id, change): + # It's way simpler to delete-then-recreate than to update + self._apply_Delete(zone_id, change) + self._apply_Create(zone_id, change) + + def _apply_Delete(self, zone_id, change): + existing = change.existing + zone = existing.zone + for record in self.zone_records(zone): + if existing.name == record['name'] and \ + existing._type == record['type']: + self._client.zone_record_delete(zone_id, record['id']) + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + + zone_name = desired.name[:-1] + try: + zone_id = self._client.zones_get(name=zone_name)[0]['id'] + except HetznerClientNotFound: + self.log.debug('_apply: no matching zone, creating domain') + zone_id = self._client.zone_create(zone_name)['id'] + + for change in changes: + class_name = change.__class__.__name__ + getattr(self, '_apply_{}'.format(class_name))(zone_id, change) + + # Clear out the cache if any + self._zone_records.pop(desired.name, None) From 8d1dd926ea6a8241928140e62258221f8636b782 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Wed, 14 Apr 2021 07:59:00 +0200 Subject: [PATCH 05/18] added Hetzner provider to README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 0a26da9..42c91a0 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,7 @@ The above command pulled the existing data out of Route53 and placed the results | [EnvVarSource](/octodns/source/envvar.py) | | TXT | No | read-only environment variable injection | | [GandiProvider](/octodns/provider/gandi.py) | | A, AAAA, ALIAS, CAA, CNAME, DNAME, MX, NS, PTR, SPF, SRV, SSHFP, TXT | No | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | +| [HetznerProvider](/octodns/provider/hetzner.py) | | A, AAAA, CAA, CNAME, MX, NS, SRV, TXT | No | | | [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | | | [Ns1Provider](/octodns/provider/ns1.py) | ns1-python | All | Yes | Missing `NA` geo target | | [OVH](/octodns/provider/ovh.py) | ovh | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT, DKIM | No | | From 1a2cb50c6326b22a870694c51374bd2bb05556fa Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Wed, 21 Apr 2021 07:52:38 +0200 Subject: [PATCH 06/18] fixed potential KeyError when record ttl field is missing --- octodns/provider/hetzner.py | 38 ++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 570cefe..17b1141 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -5,10 +5,9 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -import logging from collections import defaultdict - from requests import Session +import logging from ..record import Record from .base import BaseProvider @@ -24,6 +23,12 @@ class HetznerClientNotFound(HetznerClientException): super(HetznerClientNotFound, self).__init__('Not Found') +class HetznerClientUnauthorized(HetznerClientException): + + def __init__(self): + super(HetznerClientUnauthorized, self).__init__('Unauthorized') + + class HetznerClient(object): ''' Hetzner DNS Public API v1 client class. @@ -44,6 +49,8 @@ class HetznerClient(object): def _do(self, method, path, params=None, data=None): url = self.BASE_URL + path response = self._session.request(method, url, params=params, json=data) + if response.status_code == 401: + raise HetznerClientUnauthorized() if response.status_code == 404: raise HetznerClientNotFound() response.raise_for_status() @@ -139,14 +146,22 @@ class HetznerProvider(BaseProvider): self._client = HetznerClient(token) self._zone_records = {} + self._zone_metadata = {} def _append_dot(self, value): - return value if value[-1] == '.' else '{}.'.format(value) + if value == '@' or value[-1] == '.': + return value + return '{}.'.format(value) + + def _record_ttl(self, record): + if 'ttl' in record: + return record['ttl'] + return self._zone_metadata[record['zone_id']]['ttl'] def _data_for_multiple(self, _type, records): values = [record['value'].replace(';', '\\;') for record in records] return { - 'ttl': records[0]['ttl'], + 'ttl': self._record_ttl(records[0]), 'type': _type, 'values': values } @@ -167,7 +182,7 @@ class HetznerProvider(BaseProvider): 'value': value, }) return { - 'ttl': records[0]['ttl'], + 'ttl': self._record_ttl(records[0]), 'type': _type, 'values': values } @@ -175,7 +190,7 @@ class HetznerProvider(BaseProvider): def _data_for_CNAME(self, _type, records): record = records[0] return { - 'ttl': record['ttl'], + 'ttl': self._record_ttl(record), 'type': _type, 'value': self._append_dot(record['value']) } @@ -191,7 +206,7 @@ class HetznerProvider(BaseProvider): 'exchange': self._append_dot(exchange) }) return { - 'ttl': records[0]['ttl'], + 'ttl': self._record_ttl(records[0]), 'type': _type, 'values': values } @@ -201,7 +216,7 @@ class HetznerProvider(BaseProvider): for record in records: values.append(self._append_dot(record['value'])) return { - 'ttl': records[0]['ttl'], + 'ttl': self._record_ttl(records[0]), 'type': _type, 'values': values, } @@ -221,7 +236,7 @@ class HetznerProvider(BaseProvider): 'weight': int(weight) }) return { - 'ttl': records[0]['ttl'], + 'ttl': self._record_ttl(records[0]), 'type': _type, 'values': values } @@ -231,9 +246,10 @@ class HetznerProvider(BaseProvider): def zone_records(self, zone): if zone.name not in self._zone_records: try: - zone_id = self._client.zones_get(name=zone.name[:-1])[0]['id'] + zone_metadata = self._client.zones_get(name=zone.name[:-1])[0] + self._zone_metadata[zone_metadata['id']] = zone_metadata self._zone_records[zone.name] = \ - self._client.zone_records_get(zone_id) + self._client.zone_records_get(zone_metadata['id']) except HetznerClientNotFound: return [] From ab436af92d42643eac3796ee58a8109be737ace9 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Wed, 21 Apr 2021 07:55:18 +0200 Subject: [PATCH 07/18] added populate() tests --- tests/fixtures/hetzner-records.json | 223 +++++++++++++++++++++++++ tests/fixtures/hetzner-zones.json | 43 +++++ tests/test_octodns_provider_hetzner.py | 84 ++++++++++ 3 files changed, 350 insertions(+) create mode 100644 tests/fixtures/hetzner-records.json create mode 100644 tests/fixtures/hetzner-zones.json create mode 100644 tests/test_octodns_provider_hetzner.py diff --git a/tests/fixtures/hetzner-records.json b/tests/fixtures/hetzner-records.json new file mode 100644 index 0000000..bbafdcb --- /dev/null +++ b/tests/fixtures/hetzner-records.json @@ -0,0 +1,223 @@ +{ + "records": [ + { + "id": "SOA", + "type": "SOA", + "name": "@", + "value": "hydrogen.ns.hetzner.com. dns.hetzner.com. 1 86400 10800 3600000 3600", + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "NS:sub:0", + "type": "NS", + "name": "sub", + "value": "6.2.3.4", + "ttl": 3600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "NS:sub:1", + "type": "NS", + "name": "sub", + "value": "7.2.3.4", + "ttl": 3600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "SRV:_srv._tcp:0", + "type": "SRV", + "name": "_srv._tcp", + "value": "10 20 30 foo-1.unit.tests", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "SRV:_srv._tcp:1", + "type": "SRV", + "name": "_srv._tcp", + "value": "12 20 30 foo-2.unit.tests", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "TXT:txt:0", + "type": "TXT", + "name": "txt", + "value": "\"Bah bah black sheep\"", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "TXT:txt:1", + "type": "TXT", + "name": "txt", + "value": "\"have you any wool.\"", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "A:@:0", + "type": "A", + "name": "@", + "value": "1.2.3.4", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "A:@:1", + "type": "A", + "name": "@", + "value": "1.2.3.5", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "A:www:0", + "type": "A", + "name": "www", + "value": "2.2.3.6", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "MX:mx:0", + "type": "MX", + "name": "mx", + "value": "10 smtp-4.unit.tests", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "MX:mx:1", + "type": "MX", + "name": "mx", + "value": "20 smtp-2.unit.tests", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "MX:mx:2", + "type": "MX", + "name": "mx", + "value": "30 smtp-3.unit.tests", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "MX:mx:3", + "type": "MX", + "name": "mx", + "value": "40 smtp-1.unit.tests", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "AAAA:aaaa:0", + "type": "AAAA", + "name": "aaaa", + "value": "2601:644:500:e210:62f8:1dff:feb8:947a", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "CNAME:cname:0", + "type": "CNAME", + "name": "cname", + "value": "unit.tests", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "A:www.sub:0", + "type": "A", + "name": "www.sub", + "value": "2.2.3.6", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "TXT:txt:2", + "type": "TXT", + "name": "txt", + "value": "v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "CAA:@:0", + "type": "CAA", + "name": "@", + "value": "0 issue \"ca.unit.tests\"", + "ttl": 3600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "CNAME:included:0", + "type": "CNAME", + "name": "included", + "value": "unit.tests", + "ttl": 3600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "SRV:_imap._tcp:0", + "type": "SRV", + "name": "_imap._tcp", + "value": "0 0 0 .", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "SRV:_pop3._tcp:0", + "type": "SRV", + "name": "_pop3._tcp", + "value": "0 0 0 .", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + } + ] +} diff --git a/tests/fixtures/hetzner-zones.json b/tests/fixtures/hetzner-zones.json new file mode 100644 index 0000000..4d9b897 --- /dev/null +++ b/tests/fixtures/hetzner-zones.json @@ -0,0 +1,43 @@ +{ + "zones": [ + { + "id": "unit.tests", + "name": "unit.tests", + "ttl": 3600, + "registrar": "", + "legacy_dns_host": "", + "legacy_ns": [], + "ns": [], + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "verified": "", + "modified": "0000-00-00 00:00:00.000 +0000 UTC", + "project": "", + "owner": "", + "permission": "", + "zone_type": { + "id": "", + "name": "", + "description": "", + "prices": null + }, + "status": "verified", + "paused": false, + "is_secondary_dns": false, + "txt_verification": { + "name": "", + "token": "" + }, + "records_count": null + } + ], + "meta": { + "pagination": { + "page": 1, + "per_page": 100, + "previous_page": 1, + "next_page": 1, + "last_page": 1, + "total_entries": 1 + } + } +} diff --git a/tests/test_octodns_provider_hetzner.py b/tests/test_octodns_provider_hetzner.py new file mode 100644 index 0000000..bf619fb --- /dev/null +++ b/tests/test_octodns_provider_hetzner.py @@ -0,0 +1,84 @@ +# +# +# + + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from os.path import dirname, join +from requests import HTTPError +from requests_mock import ANY, mock as requests_mock +from six import text_type +from unittest import TestCase + +from octodns.provider.hetzner import HetznerProvider +from octodns.provider.yaml import YamlProvider +from octodns.zone import Zone + + +class TestdHetznerProvider(TestCase): + expected = Zone('unit.tests.', []) + source = YamlProvider('test', join(dirname(__file__), 'config')) + source.populate(expected) + + def test_populate(self): + provider = HetznerProvider('test', 'token') + + # Bad auth + with requests_mock() as mock: + mock.get(ANY, status_code=401, + text='{"message":"Invalid authentication credentials"}') + + with self.assertRaises(Exception) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals('Unauthorized', text_type(ctx.exception)) + + # General error + with requests_mock() as mock: + mock.get(ANY, status_code=502, text='Things caught fire') + + with self.assertRaises(HTTPError) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(502, ctx.exception.response.status_code) + + # Non-existent zone doesn't populate anything + with requests_mock() as mock: + mock.get(ANY, status_code=404, + text='{"zone":{"id":"","name":"","ttl":0,"registrar":"",' + '"legacy_dns_host":"","legacy_ns":null,"ns":null,' + '"created":"","verified":"","modified":"","project":"",' + '"owner":"","permission":"","zone_type":{"id":"",' + '"name":"","description":"","prices":null},"status":"",' + '"paused":false,"is_secondary_dns":false,' + '"txt_verification":{"name":"","token":""},' + '"records_count":0},"error":{' + '"message":"zone not found","code":404}}') + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(set(), zone.records) + + # No diffs == no changes + with requests_mock() as mock: + base = 'https://dns.hetzner.com/api/v1' + with open('tests/fixtures/hetzner-zones.json') as fh: + mock.get('{}/zones'.format(base), text=fh.read()) + with open('tests/fixtures/hetzner-records.json') as fh: + mock.get('{}/records'.format(base), text=fh.read()) + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(13, len(zone.records)) + changes = self.expected.changes(zone, provider) + self.assertEquals(0, len(changes)) + + # 2nd populate makes no network calls/all from cache + again = Zone('unit.tests.', []) + provider.populate(again) + self.assertEquals(13, len(again.records)) + + # bust the cache + del provider._zone_records[zone.name] From fe74c462138a951577ce4693e8eafa5e32764557 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Mon, 26 Apr 2021 19:02:44 +0200 Subject: [PATCH 08/18] made hetzner.HetznerProvider._do Mock-able for testing purposes making it a wrapper for _do_raw --- octodns/provider/hetzner.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 17b1141..78be756 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -46,7 +46,7 @@ class HetznerClient(object): session.headers.update({'Auth-API-Token': token}) self._session = session - def _do(self, method, path, params=None, data=None): + def _do_raw(self, method, path, params=None, data=None): url = self.BASE_URL + path response = self._session.request(method, url, params=params, json=data) if response.status_code == 401: @@ -54,7 +54,10 @@ class HetznerClient(object): if response.status_code == 404: raise HetznerClientNotFound() response.raise_for_status() - return response.json() + return response + + def _do(self, method, path, params=None, data=None): + return self._do_raw(method, path, params, data).json() def _do_with_pagination(self, method, path, key, params=None, data=None, per_page=100): From 612738b327b26b417a683ec5dd0c3c55291b6bd4 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Mon, 26 Apr 2021 19:03:25 +0200 Subject: [PATCH 09/18] renamed TestdHetznerProvider -> TestHetznerProvider (missing "d") --- tests/test_octodns_provider_hetzner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_hetzner.py b/tests/test_octodns_provider_hetzner.py index bf619fb..a8a67fb 100644 --- a/tests/test_octodns_provider_hetzner.py +++ b/tests/test_octodns_provider_hetzner.py @@ -17,7 +17,7 @@ from octodns.provider.yaml import YamlProvider from octodns.zone import Zone -class TestdHetznerProvider(TestCase): +class TestHetznerProvider(TestCase): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) From 0de9efd03268ae0d447997fe1960dc314880348d Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Tue, 27 Apr 2021 07:31:05 +0200 Subject: [PATCH 10/18] removed unused HetznerClient methods to fix imparial coverage --- octodns/provider/hetzner.py | 70 +++++++++---------------------------- 1 file changed, 17 insertions(+), 53 deletions(-) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 78be756..0be01ea 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -32,11 +32,6 @@ class HetznerClientUnauthorized(HetznerClientException): class HetznerClient(object): ''' Hetzner DNS Public API v1 client class. - - Zone and Record resources are (almost) fully supported, even if unnecessary - to future-proof this client. Bulk Record create/update is not supported. - - No support for Primary Servers. ''' BASE_URL = 'https://dns.hetzner.com/api/v1' @@ -46,8 +41,8 @@ class HetznerClient(object): session.headers.update({'Auth-API-Token': token}) self._session = session - def _do_raw(self, method, path, params=None, data=None): - url = self.BASE_URL + path + def _do(self, method, path, params=None, data=None): + url = '{}{}'.format(self.BASE_URL, path) response = self._session.request(method, url, params=params, json=data) if response.status_code == 401: raise HetznerClientUnauthorized() @@ -56,20 +51,15 @@ class HetznerClient(object): response.raise_for_status() return response - def _do(self, method, path, params=None, data=None): - return self._do_raw(method, path, params, data).json() - - def _do_with_pagination(self, method, path, key, params=None, data=None, - per_page=100): - pagination_params = {'page': 1, 'per_page': per_page} - if params is not None: - params = {**params, **pagination_params} - else: - params = pagination_params + def _do_json(self, method, path, params=None, data=None): + return self._do(method, path, params, data).json() + def _do_json_paginate(self, method, path, key, params=None, data=None, + per_page=100): items = [] + params = {**{'page': 1, 'per_page': per_page}, **params} while True: - response = self._do(method, path, params, data) + response = self._do_json(method, path, params, data) items += response[key] if response['meta']['pagination']['page'] >= \ response['meta']['pagination']['last_page']: @@ -79,54 +69,28 @@ class HetznerClient(object): def zones_get(self, name=None, search_name=None): params = {'name': name, 'search_name': search_name} - return self._do_with_pagination('GET', '/zones', 'zones', - params=params) - - def zone_get(self, zone_id): - return self._do('GET', '/zones/' + zone_id)['zone'] + return self._do_json_paginate('GET', '/zones', 'zones', params=params) def zone_create(self, name, ttl=None): data = {'name': name, 'ttl': ttl} - return self._do('POST', '/zones', data=data)['zone'] - - def zone_update(self, zone_id, name, ttl=None): - data = {'name': name, 'ttl': ttl} - return self._do('PUT', '/zones/' + zone_id, data=data)['zone'] - - def zone_delete(self, zone_id): - return self._do('DELETE', '/zones/' + zone_id) - - def _replace_at(self, record): - record['name'] = '' if record['name'] == '@' else record['name'] - return record + return self._do_json('POST', '/zones', data=data)['zone'] def zone_records_get(self, zone_id): params = {'zone_id': zone_id} # No need to handle pagination as it returns all records by default. - return [ - self._replace_at(record) - for record in self._do('GET', '/records', params=params)['records'] - ] - - def zone_record_get(self, record_id): - record = self._do('GET', '/records/' + record_id)['record'] - return self._replace_at(record) + records = self._do_json('GET', '/records', params=params)['records'] + for record in records: + if record['name'] == '@': + record['name'] = '' + return records def zone_record_create(self, zone_id, name, _type, value, ttl=None): data = {'name': name or '@', 'ttl': ttl, 'type': _type, 'value': value, 'zone_id': zone_id} - record = self._do('POST', '/records', data=data)['record'] - return self._replace_at(record) - - def zone_record_update(self, zone_id, record_id, name, _type, value, - ttl=None): - data = {'name': name or '@', 'ttl': ttl, 'type': _type, 'value': value, - 'zone_id': zone_id} - record = self._do('PUT', '/records/' + record_id, data=data)['record'] - return self._replace_at(record) + self._do('POST', '/records', data=data) def zone_record_delete(self, zone_id, record_id): - return self._do('DELETE', '/records/' + record_id) + self._do('DELETE', '/records/{}'.format(record_id)) class HetznerProvider(BaseProvider): From 192231109181dac8f61c1b41be156ba4c53cbac9 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Tue, 27 Apr 2021 07:53:17 +0200 Subject: [PATCH 11/18] WIP added TestHetznerProvider.test_apply --- tests/test_octodns_provider_hetzner.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_hetzner.py b/tests/test_octodns_provider_hetzner.py index a8a67fb..0c18731 100644 --- a/tests/test_octodns_provider_hetzner.py +++ b/tests/test_octodns_provider_hetzner.py @@ -6,13 +6,15 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +from mock import Mock from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock from six import text_type from unittest import TestCase -from octodns.provider.hetzner import HetznerProvider +from octodns.provider.hetzner import HetznerClientNotFound, \ + HetznerProvider from octodns.provider.yaml import YamlProvider from octodns.zone import Zone @@ -82,3 +84,24 @@ class TestHetznerProvider(TestCase): # bust the cache del provider._zone_records[zone.name] + + def test_apply(self): + provider = HetznerProvider('test', 'token') + + resp = Mock() + resp.json = Mock() + provider._client._do = Mock(return_value=resp) + + # non-existent domain, create everything + resp.json.side_effect = [ + HetznerClientNotFound, # no zone in populate + HetznerClientNotFound, # no zone during apply + {"zone": {"id": "string"}} + ] + plan = provider.plan(self.expected) + + # No root NS, no ignored, no excluded, no unsupported + n = len(self.expected.records) - 9 + self.assertEquals(n, len(plan.changes)) + self.assertEquals(n, provider.apply(plan)) + self.assertFalse(plan.exists) From a0c4e9ecd76af5f2a830b8cb430468b147e070f3 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Tue, 27 Apr 2021 19:56:39 +0200 Subject: [PATCH 12/18] simplified HetznerClient by removing unused pagination handling --- octodns/provider/hetzner.py | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 0be01ea..d396e18 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -30,10 +30,6 @@ class HetznerClientUnauthorized(HetznerClientException): class HetznerClient(object): - ''' - Hetzner DNS Public API v1 client class. - ''' - BASE_URL = 'https://dns.hetzner.com/api/v1' def __init__(self, token): @@ -54,22 +50,9 @@ class HetznerClient(object): def _do_json(self, method, path, params=None, data=None): return self._do(method, path, params, data).json() - def _do_json_paginate(self, method, path, key, params=None, data=None, - per_page=100): - items = [] - params = {**{'page': 1, 'per_page': per_page}, **params} - while True: - response = self._do_json(method, path, params, data) - items += response[key] - if response['meta']['pagination']['page'] >= \ - response['meta']['pagination']['last_page']: - break - params['page'] += 1 - return items - - def zones_get(self, name=None, search_name=None): - params = {'name': name, 'search_name': search_name} - return self._do_json_paginate('GET', '/zones', 'zones', params=params) + def zone_get(self, name): + params = {'name': name} + return self._do_json('GET', '/zones', params)['zones'][0] def zone_create(self, name, ttl=None): data = {'name': name, 'ttl': ttl} @@ -77,7 +60,6 @@ class HetznerClient(object): def zone_records_get(self, zone_id): params = {'zone_id': zone_id} - # No need to handle pagination as it returns all records by default. records = self._do_json('GET', '/records', params=params)['records'] for record in records: if record['name'] == '@': From d4c6836d0bb8b0d8bd6b3fb58ba9abb2378f706a Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Tue, 27 Apr 2021 19:58:07 +0200 Subject: [PATCH 13/18] implemented HetznerProvider.zone_metadata as the equivalent to zone_records for zone metadata --- octodns/provider/hetzner.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index d396e18..53914f5 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -96,16 +96,28 @@ class HetznerProvider(BaseProvider): self._zone_records = {} self._zone_metadata = {} + self._zone_name_to_id = {} def _append_dot(self, value): if value == '@' or value[-1] == '.': return value return '{}.'.format(value) + def zone_metadata(self, zone_id=None, zone_name=None): + if zone_name is not None: + if zone_name in self._zone_name_to_id: + zone_id = self._zone_name_to_id[zone_name] + else: + zone = self._client.zone_get(name=zone_name[:-1]) + zone_id = zone['id'] + self._zone_name_to_id[zone_name] = zone_id + self._zone_metadata[zone_id] = zone + + return self._zone_metadata[zone_id] + def _record_ttl(self, record): - if 'ttl' in record: - return record['ttl'] - return self._zone_metadata[record['zone_id']]['ttl'] + default_ttl = self.zone_metadata(zone_id=record['zone_id'])['ttl'] + return record['ttl'] if 'ttl' in record else default_ttl def _data_for_multiple(self, _type, records): values = [record['value'].replace(';', '\\;') for record in records] @@ -195,10 +207,9 @@ class HetznerProvider(BaseProvider): def zone_records(self, zone): if zone.name not in self._zone_records: try: - zone_metadata = self._client.zones_get(name=zone.name[:-1])[0] - self._zone_metadata[zone_metadata['id']] = zone_metadata + zone_id = self.zone_metadata(zone_name=zone.name)['id'] self._zone_records[zone.name] = \ - self._client.zone_records_get(zone_metadata['id']) + self._client.zone_records_get(zone_id) except HetznerClientNotFound: return [] @@ -314,12 +325,11 @@ class HetznerProvider(BaseProvider): self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, len(changes)) - zone_name = desired.name[:-1] try: - zone_id = self._client.zones_get(name=zone_name)[0]['id'] + zone_id = self.zone_metadata(zone_name=desired.name)['id'] except HetznerClientNotFound: self.log.debug('_apply: no matching zone, creating domain') - zone_id = self._client.zone_create(zone_name)['id'] + zone_id = self._client.zone_create(desired.name[:-1])['id'] for change in changes: class_name = change.__class__.__name__ From 6bf24d867894c8df992b8491762a9217e563d7d5 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Tue, 27 Apr 2021 19:59:00 +0200 Subject: [PATCH 14/18] implemented TestHetznerProvider.test_apply --- tests/test_octodns_provider_hetzner.py | 230 ++++++++++++++++++++++++- 1 file changed, 229 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_hetzner.py b/tests/test_octodns_provider_hetzner.py index 0c18731..02da32e 100644 --- a/tests/test_octodns_provider_hetzner.py +++ b/tests/test_octodns_provider_hetzner.py @@ -6,13 +6,14 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from mock import Mock +from mock import Mock, call from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock from six import text_type from unittest import TestCase +from octodns.record import Record from octodns.provider.hetzner import HetznerClientNotFound, \ HetznerProvider from octodns.provider.yaml import YamlProvider @@ -105,3 +106,230 @@ class TestHetznerProvider(TestCase): self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) + + provider._client._do.assert_has_calls([ + # created the zone + call('POST', '/zones', None, { + 'name': 'unit.tests', + 'ttl': None, + }), + # created all the records with their expected data + call('POST', '/records', data={ + 'name': '@', + 'ttl': 300, + 'type': 'A', + 'value': '1.2.3.4', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': '@', + 'ttl': 300, + 'type': 'A', + 'value': '1.2.3.5', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': '@', + 'ttl': 3600, + 'type': 'CAA', + 'value': '0 issue "ca.unit.tests"', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': '_imap._tcp', + 'ttl': 600, + 'type': 'SRV', + 'value': '0 0 0 .', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': '_pop3._tcp', + 'ttl': 600, + 'type': 'SRV', + 'value': '0 0 0 .', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': '_srv._tcp', + 'ttl': 600, + 'type': 'SRV', + 'value': '10 20 30 foo-1.unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': '_srv._tcp', + 'ttl': 600, + 'type': 'SRV', + 'value': '12 20 30 foo-2.unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'aaaa', + 'ttl': 600, + 'type': 'AAAA', + 'value': '2601:644:500:e210:62f8:1dff:feb8:947a', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'cname', + 'ttl': 300, + 'type': 'CNAME', + 'value': 'unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'included', + 'ttl': 3600, + 'type': 'CNAME', + 'value': 'unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'mx', + 'ttl': 300, + 'type': 'MX', + 'value': '10 smtp-4.unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'mx', + 'ttl': 300, + 'type': 'MX', + 'value': '20 smtp-2.unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'mx', + 'ttl': 300, + 'type': 'MX', + 'value': '30 smtp-3.unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'mx', + 'ttl': 300, + 'type': 'MX', + 'value': '40 smtp-1.unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'sub', + 'ttl': 3600, + 'type': 'NS', + 'value': '6.2.3.4.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'sub', + 'ttl': 3600, + 'type': 'NS', + 'value': '7.2.3.4.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'txt', + 'ttl': 600, + 'type': 'TXT', + 'value': 'Bah bah black sheep', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'txt', + 'ttl': 600, + 'type': 'TXT', + 'value': 'have you any wool.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'txt', + 'ttl': 600, + 'type': 'TXT', + 'value': 'v=DKIM1;k=rsa;s=email;h=sha256;' + 'p=A/kinda+of/long/string+with+numb3rs', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'www', + 'ttl': 300, + 'type': 'A', + 'value': '2.2.3.6', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'www.sub', + 'ttl': 300, + 'type': 'A', + 'value': '2.2.3.6', + 'zone_id': 'unit.tests', + }), + ]) + self.assertEquals(24, provider._client._do.call_count) + + provider._client._do.reset_mock() + + # delete 1 and update 1 + provider._client.zone_get = Mock(return_value={ + 'id': 'unit.tests', + 'name': 'unit.tests', + 'ttl': 3600, + }) + provider._client.zone_records_get = Mock(return_value=[ + { + 'type': 'A', + 'id': 'one', + 'created': '0000-00-00T00:00:00Z', + 'modified': '0000-00-00T00:00:00Z', + 'zone_id': 'unit.tests', + 'name': 'www', + 'value': '1.2.3.4', + 'ttl': 300, + }, + { + 'type': 'A', + 'id': 'two', + 'created': '0000-00-00T00:00:00Z', + 'modified': '0000-00-00T00:00:00Z', + 'zone_id': 'unit.tests', + 'name': 'www', + 'value': '2.2.3.4', + 'ttl': 300, + }, + { + 'type': 'A', + 'id': 'three', + 'created': '0000-00-00T00:00:00Z', + 'modified': '0000-00-00T00:00:00Z', + 'zone_id': 'unit.tests', + 'name': 'ttl', + 'value': '3.2.3.4', + 'ttl': 600, + }, + ]) + + # Domain exists, we don't care about return + resp.json.side_effect = ['{}'] + + wanted = Zone('unit.tests.', []) + wanted.add_record(Record.new(wanted, 'ttl', { + 'ttl': 300, + 'type': 'A', + 'value': '3.2.3.4', + })) + + plan = provider.plan(wanted) + self.assertTrue(plan.exists) + self.assertEquals(2, len(plan.changes)) + self.assertEquals(2, provider.apply(plan)) + # recreate for update, and delete for the 2 parts of the other + provider._client._do.assert_has_calls([ + call('POST', '/records', data={ + 'name': 'ttl', + 'ttl': 300, + 'type': 'A', + 'value': '3.2.3.4', + 'zone_id': 'unit.tests', + }), + call('DELETE', '/records/one'), + call('DELETE', '/records/two'), + call('DELETE', '/records/three'), + ], any_order=True) From b3c394a5e0056e5c0f4382e89e7854b4e02c02e4 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Tue, 27 Apr 2021 19:59:26 +0200 Subject: [PATCH 15/18] minor correctness tweaks --- tests/test_octodns_provider_hetzner.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_octodns_provider_hetzner.py b/tests/test_octodns_provider_hetzner.py index 02da32e..4167944 100644 --- a/tests/test_octodns_provider_hetzner.py +++ b/tests/test_octodns_provider_hetzner.py @@ -66,7 +66,7 @@ class TestHetznerProvider(TestCase): # No diffs == no changes with requests_mock() as mock: - base = 'https://dns.hetzner.com/api/v1' + base = provider._client.BASE_URL with open('tests/fixtures/hetzner-zones.json') as fh: mock.get('{}/zones'.format(base), text=fh.read()) with open('tests/fixtures/hetzner-records.json') as fh: @@ -93,11 +93,17 @@ class TestHetznerProvider(TestCase): resp.json = Mock() provider._client._do = Mock(return_value=resp) + domain_after_creation = {'zone': { + 'id': 'unit.tests', + 'name': 'unit.tests', + 'ttl': 3600, + }} + # non-existent domain, create everything resp.json.side_effect = [ HetznerClientNotFound, # no zone in populate HetznerClientNotFound, # no zone during apply - {"zone": {"id": "string"}} + domain_after_creation, ] plan = provider.plan(self.expected) From a564270ef5cf8030e55e3a2434ad2e7b51c9c370 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 29 Apr 2021 09:12:35 -0700 Subject: [PATCH 16/18] Add a blurb on pip installing a sha --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index cd9b884..5eaca41 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,14 @@ $ pip install octodns $ mkdir config ``` +#### Installing a specific commit SHA + +If you'd like to install a version that has not yet been released in a repetable/safe manner you can do the following. In general octoDNS is fairly stable inbetween releases thanks to the plan and apply process, but care should be taken regardless. + +```shell +$ pip install -e git+https://git@github.com/github/octodns.git@#egg=octodns +``` + ### Config We start by creating a config file to tell OctoDNS about our providers and the zone(s) we want it to manage. Below we're setting up a `YamlProvider` to source records from our config files and both a `Route53Provider` and `DynProvider` to serve as the targets for those records. You can have any number of zones set up and any number of sources of data and targets for records for each. You can also have multiple config files, that make use of separate accounts and each manage a distinct set of zones. A good example of this this might be `./config/staging.yaml` & `./config/production.yaml`. We'll focus on a `config/production.yaml`. From ad37b997739fc76c370aba1983f156bd8e32c03a Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 29 Apr 2021 09:12:38 -0700 Subject: [PATCH 17/18] Add shell code block types --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5eaca41..4ab3c6b 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ It is similar to [Netflix/denominator](https://github.com/Netflix/denominator). Running through the following commands will install the latest release of OctoDNS and set up a place for your config files to live. To determine if provider specific requirements are necessary see the [Supported providers table](#supported-providers) below. -``` +```shell $ mkdir dns $ cd dns $ virtualenv env @@ -121,7 +121,7 @@ Further information can be found in [Records Documentation](/docs/records.md). We're ready to do a dry-run with our new setup to see what changes it would make. Since we're pretending here we'll act like there are no existing records for `example.com.` in our accounts on either provider. -``` +```shell $ octodns-sync --config-file=./config/production.yaml ... ******************************************************************************** @@ -145,7 +145,7 @@ There will be other logging information presented on the screen, but successful Now it's time to tell OctoDNS to make things happen. We'll invoke it again with the same options and add a `--doit` on the end to tell it this time we actually want it to try and make the specified changes. -``` +```shell $ octodns-sync --config-file=./config/production.yaml --doit ... ``` @@ -176,7 +176,7 @@ If that goes smoothly, you again see the expected changes, and verify them with Very few situations will involve starting with a blank slate which is why there's tooling built in to pull existing data out of providers into a matching config file. -``` +```shell $ octodns-dump --config-file=config/production.yaml --output-dir=tmp/ example.com. route53 2017-03-15T13:33:34 INFO Manager __init__: config_file=tmp/production.yaml 2017-03-15T13:33:34 INFO Manager dump: zone=example.com., sources=('route53',) From 58c7f431e86ce441767ab1f5a7f6dee7837465e8 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 30 Apr 2021 14:43:23 -0700 Subject: [PATCH 18/18] v0.9.12 version bump and CHANGELOG update --- CHANGELOG.md | 25 ++++++++++++++++++++++++- octodns/__init__.py | 2 +- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb68e25..1d16544 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,29 @@ +## v0.9.12 - 2021-04-30 - Enough time has passed + +#### Noteworthy changes + +* Formal Python 2.7 support removed, deps and tooling were becoming + unmaintainable +* octodns/octodns move, from github/octodns, more to come + +#### Stuff + +* ZoneFileSource supports specifying an extension & no files end in . to better + support Windows +* LOC record type support added +* Support for pre-release versions of PowerDNS +* PowerDNS delete before create which allows A <-> CNAME etc. +* Improved validation of fqdn's in ALIAS, CNAME, etc. +* Transip support for NS records +* Support for sending plan output to a file +* DNSimple uses zone api rather than domain to support non-registered stuff, + e.g. reverse zones. +* Support for fallback-only dynamic pools and related fixes to NS1 provider +* Initial Hetzner provider + ## v0.9.11 - 2020-11-05 - We still don't know edition -#### Noteworthy changtes +#### Noteworthy changes * ALIAS records only allowed at the root of zones - see `leient` in record docs for work-arounds if you really need them. diff --git a/octodns/__init__.py b/octodns/__init__.py index 3fcdaa1..1885d42 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -3,4 +3,4 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -__VERSION__ = '0.9.11' +__VERSION__ = '0.9.12'