From be9dbd8ce3eb0de6249cd1e170551d1f68f0072c Mon Sep 17 00:00:00 2001 From: Rhosyn Celyn Date: Tue, 23 Apr 2019 16:25:00 +0100 Subject: [PATCH 01/13] Initial commit for Mythic API --- octodns/provider/mythicbeasts.py | 394 +++++++++++++++++++++++++++++++ 1 file changed, 394 insertions(+) create mode 100644 octodns/provider/mythicbeasts.py diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py new file mode 100644 index 0000000..ed3fed8 --- /dev/null +++ b/octodns/provider/mythicbeasts.py @@ -0,0 +1,394 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from requests import HTTPError, Session +from logging import getLogger + +from ..record import Create, Record +from .base import BaseProvider + +import re + +import pprint +import sys + +def add_trailing_dot(s): + assert s + assert s[-1] != '.' + return s + '.' + + +def remove_trailing_dot(s): + assert s + assert s[-1] == '.' + return s[:-1] + + +class MythicBeastsProvider(BaseProvider): + ''' + Mythic Beasts DNS API Provider + + mythicbeasts: + class: octodns.provider.mythicbeasts.MythicBeastsProvider + zones: + my-zone: 'password' + + ''' + + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = False + SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', + 'SRV', 'TXT')) + BASE = 'https://dnsapi.mythic-beasts.com/' + TIMEOUT = 15 + + def __init__(self, id, passwords, *args, **kwargs): + self.log = getLogger('MythicBeastsProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, registered zones; %s', id, passwords.keys()) + super(MythicBeastsProvider, self).__init__(id, *args, **kwargs) + + self._passwords = passwords + sess = Session() + self._sess = sess + + def _request(self, method, path, data=None): + self.log.debug('_request: method=%s, path=%s', method, path) + + url = self.BASE + resp = self._sess.request(method, url, data=data, timeout=self.TIMEOUT) + self.log.debug('_request: status=%d', resp.status_code) + resp.raise_for_status() + return resp + + def _post(self, data=None): + return self._request('POST', self.BASE, data=data) + + def records(self, zone): + return self._post({ + 'domain': remove_trailing_dot(zone), + 'password': self._passwords[zone], + 'command': 'LIST', + }) + + def _data_for_single(self, _type, data): + return { + 'type': _type, + 'value': data['raw_values'][0]['value'], + 'ttl': data['raw_values'][0]['ttl'] + } + + def _data_for_multiple(self, _type, data): + return { + 'type': _type, + 'values': [raw_values['value'] for raw_values in data['raw_values']], + 'ttl': max([raw_values['ttl'] for raw_values in data['raw_values']]), + } + + def _data_for_MX(self, _type, data): + ttl = max([raw_values['ttl'] for raw_values in data['raw_values']]) + values = [] + + for raw_value in [raw_values['value'] for raw_values in data['raw_values']]: + match = re.match('^([0-9]+)\s+(\S+)$', raw_value, re.IGNORECASE) + + if match is not None: + exchange = match.group(2) + + if not exchange.endswith('.'): + exchange = '{}.{}'.format(exchange, data['zone']) + + values.append({ + 'preference': match.group(1), + 'exchange': exchange, + }) + + return { + 'type': _type, + 'values': values, + 'ttl': ttl, + } + + def _data_for_CNAME(self, _type, data): + ttl = data['raw_values'][0]['ttl'] + value = data['raw_values'][0]['value'] + if not value.endswith('.'): + value = '{}.{}'.format(value, data['zone']) + + return self._data_for_single(_type, {'raw_values': [ {'value': value, 'ttl': ttl} ]}) + + def _data_for_ANAME(self, _type, data): + ttl = data['raw_values'][0]['ttl'] + value = data['raw_values'][0]['value'] + return self._data_for_single('ALIAS', {'raw_values': [ {'value': value, 'ttl': ttl} ]}) + + + def _data_for_SRV(self, _type, data): + ttl = data['raw_values'][0]['ttl'] + raw_value = data['raw_values'][0]['value'] + + match = re.match('^([0-9]+)\s+([0-9]+)\s+([0-9]+)\s+(\S+)$', raw_value, re.IGNORECASE) + + if match is not None: + target = match.group(4) + if not target.endswith('.'): + target = '{}.{}'.format(target, data['zone']) + + value = { + 'priority': match.group(1), + 'weight': match.group(2), + 'port': match.group(3), + 'target': target, + } + + return self._data_for_single('SRV', {'raw_values': [ {'value': value, 'ttl': ttl} ]}) + + def _data_for_SSHFP(self, _type, data): + ttl = data['raw_values'][0]['ttl'] + raw_value = data['raw_values'][0]['value'] + + match = re.match('^([0-9]+)\s+([0-9]+)\s+(\S+)$', raw_value, re.IGNORECASE) + + if match is not None: + value = { + 'algorithm': match.group(1), + 'fingerprint_type': match.group(2), + 'fingerprint': match.group(3), + } + + return self._data_for_single('SSHFP', {'raw_values': [ {'value': value, 'ttl': ttl} ]}) + + + # TODO fix bug with CAA output from API + ''' + def _data_for_CAA(self, _type, data): + ttl = data['raw_values'][0]['ttl'] + + match = re.match('^()$', re.IGNORECASE) + + value = { + 'flags': + 'tag': + 'value': + } + value = data['raw_values'][0]['value'] + return self._data_for_single('ALIAS', {'raw_values': [ {'value': value, 'ttl': ttl} ]}) + ''' + + + _data_for_NS = _data_for_multiple + _data_for_TXT = _data_for_multiple + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + + resp = None + try: + resp = self.records(zone.name) + except HTTPError as e: + if e.response.status_code == 401: + # Nicer error message for auth problems + raise Exception('Mythic Beasts authentication problem with %s'.format(zone.name)) + elif e.response.status_code == 422: + # 422 means mythicbeasts doesn't know anything about the requested + # domain. We'll just ignore it here and leave the zone + # untouched. + raise + else: + # just re-throw + raise + + before = len(zone.records) + exists = False + data = dict() + + if resp: + exists = True + for line in resp.content.splitlines(): + match = re.match('^(\S+)\s+(\S+)\s+(\S+)\s+(.*)$', line, re.IGNORECASE) + + if match is not None: + if match.group(1) == '@': + _name = '' + else: + _name = match.group(1) + + _type = match.group(3) + _ttl = int(match.group(2)) + _value = match.group(4).strip() + + if _type == 'SOA': + continue + + try: + if getattr(self, '_data_for_{}'.format(_type)) is not None: + + if _type not in data: + data[_type] = dict() + + if _name not in data[_type]: + data[_type][_name] = { + 'raw_values': [{'value': _value, 'ttl': _ttl}], + 'name': _name, + 'zone': zone.name, + } + + else: + data[_type][_name].get('raw_values').append( + {'value': _value, 'ttl': _ttl} + ) + except AttributeError as error: + self.log.debug('skipping {} as not supported', _type) + continue + + + for _type in data: + for _name in data[_type]: + data_for = getattr(self, '_data_for_{}'.format(_type)) + + record = Record.new( + zone, + _name, + data_for(_type, data[_type][_name]), + source=self, + ) + zone.add_record(record, lenient=lenient) + + + self.log.debug('populate: found %s records, exists=%s', + len(zone.records) - before, exists) + return exists + + + def _compile_commands(self, action, change): + commands = [] + + record = None + + if action == 'ADD': + record = change.new + + elif action == 'DELETE': + record = change.existing + + zone = record.zone + hostname = remove_trailing_dot(record.fqdn) + ttl = record.ttl + _type = record._type + + if hostname == '': + hostname = '@' + if _type == 'ALIAS': + _type = 'ANAME' + + + if hasattr(record, 'values'): + values = record.values + else: + values = [record.value] + + + base = '{} {} {} {}'.format(action, hostname, ttl, _type) + + if re.match('[A]{1,4}', _type) is not None: + for value in values: + commands.append('{} {}'.format(base, value)) + + elif _type == 'SSHFP': + data = values[0].data + commands.append('{} {} {} {}'.format( + base, data['algorithm'], data['fingerprint_type'], data['fingerprint'])) + + elif _type == 'SRV': + data = values[0].data + commands.append('{} {} {} {} {}'.format( + base, data['priority'], data['weight'], data['port'], data['target'])) + + elif _type == 'MX': + for value in values: + data = value.data + commands.append('{} {} {}'.format( + base, data['preference'], data['exchange'])) + + else: + try: + if getattr(self, '_data_for_{}'.format(_type)) is not None: + commands.append('{} {}'.format( + base, values[0])) + except AttributeError as error: + self.log.debug('skipping {} as not supported', _type) + pass + + return commands + + def _apply_Create(self, change): + + pp = pprint.PrettyPrinter(depth=10, stream=sys.stderr) + + zone = change.new.zone + commands = self._compile_commands('ADD', change) + pp.pprint(commands) + + for command in commands: + self._post({ + 'domain': remove_trailing_dot(zone.name), + 'origin': '.', + 'password': self._passwords[zone.name], + 'command': command, + }) + pp.pprint({ + 'domain': remove_trailing_dot(zone.name), + 'origin': '.', + 'password': self._passwords[zone.name], + 'command': command, + }) + return True + + def _apply_Update(self, change): + self._apply_Delete(change) + self._apply_Create(change) + + def _apply_Delete(self, change): + + pp = pprint.PrettyPrinter(depth=10, stream=sys.stderr) + + zone = change.existing.zone + commands = self._compile_commands('DELETE', change) + pp.pprint(commands) + + for command in commands: + self._post({ + 'domain': remove_trailing_dot(zone.name), + 'origin': '.', + 'password': self._passwords[zone.name], + 'command': command, + }) + pp.pprint({ + 'domain': remove_trailing_dot(zone.name), + 'origin': '.', + 'password': self._passwords[zone.name], + 'command': command, + }) + return True + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + + domain_name = desired.name + + for change in changes: + class_name = change.__class__.__name__ + getattr(self, '_apply_{}'.format(class_name))(change) + + + From b670a0695f22425c1deed8e32ac3f8b12dc5193c Mon Sep 17 00:00:00 2001 From: Rhosyn Celyn Date: Wed, 1 May 2019 15:49:33 +0100 Subject: [PATCH 02/13] Updated tweaks and bug fixes for ANAME/ALIAS --- octodns/provider/mythicbeasts.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py index ed3fed8..f416a1b 100644 --- a/octodns/provider/mythicbeasts.py +++ b/octodns/provider/mythicbeasts.py @@ -43,7 +43,8 @@ class MythicBeastsProvider(BaseProvider): SUPPORTS_DYNAMIC = False SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'SRV', 'TXT')) - BASE = 'https://dnsapi.mythic-beasts.com/' + #BASE = 'https://dnsapi.mythic-beasts.com/' + BASE = 'https://cwningen.dev.mythic-beasts.com/customer/primarydnsapi' TIMEOUT = 15 def __init__(self, id, passwords, *args, **kwargs): @@ -71,6 +72,7 @@ class MythicBeastsProvider(BaseProvider): return self._post({ 'domain': remove_trailing_dot(zone), 'password': self._passwords[zone], + 'showall': 0, 'command': 'LIST', }) @@ -195,7 +197,7 @@ class MythicBeastsProvider(BaseProvider): except HTTPError as e: if e.response.status_code == 401: # Nicer error message for auth problems - raise Exception('Mythic Beasts authentication problem with %s'.format(zone.name)) + raise Exception('Mythic Beasts authentication problem with {}'.format(zone.name)) elif e.response.status_code == 422: # 422 means mythicbeasts doesn't know anything about the requested # domain. We'll just ignore it here and leave the zone From 3f4afbac27e2d69e2384cf21b20158685466460c Mon Sep 17 00:00:00 2001 From: Rhosyn Celyn Date: Thu, 9 May 2019 16:45:19 +0100 Subject: [PATCH 03/13] Working initial implementation for Mythic Beasts DNS API --- octodns/provider/mythicbeasts.py | 58 ++++++++++---------------------- 1 file changed, 18 insertions(+), 40 deletions(-) diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py index f416a1b..d85e133 100644 --- a/octodns/provider/mythicbeasts.py +++ b/octodns/provider/mythicbeasts.py @@ -13,7 +13,6 @@ from .base import BaseProvider import re -import pprint import sys def add_trailing_dot(s): @@ -42,9 +41,8 @@ class MythicBeastsProvider(BaseProvider): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', - 'SRV', 'TXT')) - #BASE = 'https://dnsapi.mythic-beasts.com/' - BASE = 'https://cwningen.dev.mythic-beasts.com/customer/primarydnsapi' + 'SRV', 'SSHFP', 'CAA', 'TXT')) + BASE = 'https://dnsapi.mythic-beasts.com/' TIMEOUT = 15 def __init__(self, id, passwords, *args, **kwargs): @@ -61,8 +59,10 @@ class MythicBeastsProvider(BaseProvider): url = self.BASE resp = self._sess.request(method, url, data=data, timeout=self.TIMEOUT) - self.log.debug('_request: status=%d', resp.status_code) - resp.raise_for_status() + self.log.debug('_request: status=%d data=%s', resp.status_code, resp.text) + if resp.status_code != 200: + self.log.info('request failed: %s, response %s', data, resp.text) + resp.raise_for_status() return resp def _post(self, data=None): @@ -164,21 +164,19 @@ class MythicBeastsProvider(BaseProvider): return self._data_for_single('SSHFP', {'raw_values': [ {'value': value, 'ttl': ttl} ]}) - # TODO fix bug with CAA output from API - ''' def _data_for_CAA(self, _type, data): ttl = data['raw_values'][0]['ttl'] + raw_value = data['raw_values'][0]['value'] - match = re.match('^()$', re.IGNORECASE) + match = re.match('^([0-9]+)\s+(issue|issuewild|iodef)\s+(\S+)$', raw_value, re.IGNORECASE) - value = { - 'flags': - 'tag': - 'value': - } - value = data['raw_values'][0]['value'] - return self._data_for_single('ALIAS', {'raw_values': [ {'value': value, 'ttl': ttl} ]}) - ''' + if match is not None: + value = { + 'flags': match.group(1), + 'tag': match.group(2), + 'value': match.group(3), + } + return self._data_for_single('CAA', {'raw_values': [ {'value': value, 'ttl': ttl} ]}) _data_for_NS = _data_for_multiple @@ -255,11 +253,13 @@ class MythicBeastsProvider(BaseProvider): for _name in data[_type]: data_for = getattr(self, '_data_for_{}'.format(_type)) + self.log.debug('record: {}, {}'.format(_type, data[_type][_name])) + record = Record.new( zone, _name, data_for(_type, data[_type][_name]), - source=self, + source=self ) zone.add_record(record, lenient=lenient) @@ -285,8 +285,6 @@ class MythicBeastsProvider(BaseProvider): ttl = record.ttl _type = record._type - if hostname == '': - hostname = '@' if _type == 'ALIAS': _type = 'ANAME' @@ -331,12 +329,8 @@ class MythicBeastsProvider(BaseProvider): return commands def _apply_Create(self, change): - - pp = pprint.PrettyPrinter(depth=10, stream=sys.stderr) - zone = change.new.zone commands = self._compile_commands('ADD', change) - pp.pprint(commands) for command in commands: self._post({ @@ -345,12 +339,6 @@ class MythicBeastsProvider(BaseProvider): 'password': self._passwords[zone.name], 'command': command, }) - pp.pprint({ - 'domain': remove_trailing_dot(zone.name), - 'origin': '.', - 'password': self._passwords[zone.name], - 'command': command, - }) return True def _apply_Update(self, change): @@ -358,12 +346,8 @@ class MythicBeastsProvider(BaseProvider): self._apply_Create(change) def _apply_Delete(self, change): - - pp = pprint.PrettyPrinter(depth=10, stream=sys.stderr) - zone = change.existing.zone commands = self._compile_commands('DELETE', change) - pp.pprint(commands) for command in commands: self._post({ @@ -372,12 +356,6 @@ class MythicBeastsProvider(BaseProvider): 'password': self._passwords[zone.name], 'command': command, }) - pp.pprint({ - 'domain': remove_trailing_dot(zone.name), - 'origin': '.', - 'password': self._passwords[zone.name], - 'command': command, - }) return True def _apply(self, plan): From 47f76d153564285feb794c96d3290a2fd0166d73 Mon Sep 17 00:00:00 2001 From: Rhosyn Celyn Date: Thu, 9 May 2019 18:45:57 +0100 Subject: [PATCH 04/13] Linting and clean up --- octodns/provider/mythicbeasts.py | 190 +++++++++++++++++++------------ 1 file changed, 119 insertions(+), 71 deletions(-) diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py index d85e133..17ec574 100644 --- a/octodns/provider/mythicbeasts.py +++ b/octodns/provider/mythicbeasts.py @@ -5,26 +5,31 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +import re + from requests import HTTPError, Session from logging import getLogger -from ..record import Create, Record +from ..record import Record from .base import BaseProvider -import re -import sys - -def add_trailing_dot(s): - assert s - assert s[-1] != '.' - return s + '.' +def add_trailing_dot(value): + ''' + Add trailing dots to values + ''' + assert value + assert value[-1] != '.' + return value + '.' -def remove_trailing_dot(s): - assert s - assert s[-1] == '.' - return s[:-1] +def remove_trailing_dot(value): + ''' + Remove trailing dots from values + ''' + assert value + assert value[-1] == '.' + return value[:-1] class MythicBeastsProvider(BaseProvider): @@ -35,7 +40,6 @@ class MythicBeastsProvider(BaseProvider): class: octodns.provider.mythicbeasts.MythicBeastsProvider zones: my-zone: 'password' - ''' SUPPORTS_GEO = False @@ -45,10 +49,13 @@ class MythicBeastsProvider(BaseProvider): BASE = 'https://dnsapi.mythic-beasts.com/' TIMEOUT = 15 - def __init__(self, id, passwords, *args, **kwargs): - self.log = getLogger('MythicBeastsProvider[{}]'.format(id)) - self.log.debug('__init__: id=%s, registered zones; %s', id, passwords.keys()) - super(MythicBeastsProvider, self).__init__(id, *args, **kwargs) + def __init__(self, identifier, passwords, *args, **kwargs): + self.log = getLogger('MythicBeastsProvider[{}]'.format(identifier)) + self.log.debug( + '__init__: id=%s, registered zones; %s', + identifier, + passwords.keys()) + super(MythicBeastsProvider, self).__init__(identifier, *args, **kwargs) self._passwords = passwords sess = Session() @@ -59,7 +66,10 @@ class MythicBeastsProvider(BaseProvider): url = self.BASE resp = self._sess.request(method, url, data=data, timeout=self.TIMEOUT) - self.log.debug('_request: status=%d data=%s', resp.status_code, resp.text) + self.log.debug( + '_request: status=%d data=%s', + resp.status_code, + resp.text) if resp.status_code != 200: self.log.info('request failed: %s, response %s', data, resp.text) resp.raise_for_status() @@ -76,26 +86,32 @@ class MythicBeastsProvider(BaseProvider): 'command': 'LIST', }) - def _data_for_single(self, _type, data): + @staticmethod + def _data_for_single(_type, data): return { 'type': _type, 'value': data['raw_values'][0]['value'], 'ttl': data['raw_values'][0]['ttl'] } - def _data_for_multiple(self, _type, data): + @staticmethod + def _data_for_multiple(_type, data): return { 'type': _type, - 'values': [raw_values['value'] for raw_values in data['raw_values']], - 'ttl': max([raw_values['ttl'] for raw_values in data['raw_values']]), + 'values': + [raw_values['value'] for raw_values in data['raw_values']], + 'ttl': + max([raw_values['ttl'] for raw_values in data['raw_values']]), } - def _data_for_MX(self, _type, data): + @staticmethod + def _data_for_MX(_type, data): ttl = max([raw_values['ttl'] for raw_values in data['raw_values']]) values = [] - for raw_value in [raw_values['value'] for raw_values in data['raw_values']]: - match = re.match('^([0-9]+)\s+(\S+)$', raw_value, re.IGNORECASE) + for raw_value in \ + [raw_values['value'] for raw_values in data['raw_values']]: + match = re.match('^([0-9]+)\\s+(\\S+)$', raw_value, re.IGNORECASE) if match is not None: exchange = match.group(2) @@ -114,25 +130,38 @@ class MythicBeastsProvider(BaseProvider): 'ttl': ttl, } - def _data_for_CNAME(self, _type, data): + @staticmethod + def _data_for_CNAME(_type, data): ttl = data['raw_values'][0]['ttl'] value = data['raw_values'][0]['value'] if not value.endswith('.'): value = '{}.{}'.format(value, data['zone']) - return self._data_for_single(_type, {'raw_values': [ {'value': value, 'ttl': ttl} ]}) + return MythicBeastsProvider._data_for_single( + _type, + {'raw_values': [ + {'value': value, 'ttl': ttl} + ]}) - def _data_for_ANAME(self, _type, data): + @staticmethod + def _data_for_ANAME(_type, data): ttl = data['raw_values'][0]['ttl'] value = data['raw_values'][0]['value'] - return self._data_for_single('ALIAS', {'raw_values': [ {'value': value, 'ttl': ttl} ]}) - + return MythicBeastsProvider._data_for_single( + 'ALIAS', + {'raw_values': [ + {'value': value, 'ttl': ttl} + ]}) - def _data_for_SRV(self, _type, data): + @staticmethod + def _data_for_SRV(_type, data): ttl = data['raw_values'][0]['ttl'] raw_value = data['raw_values'][0]['value'] - match = re.match('^([0-9]+)\s+([0-9]+)\s+([0-9]+)\s+(\S+)$', raw_value, re.IGNORECASE) + match = re.match( + '^([0-9]+)\\s+([0-9]+)\\s+([0-9]+)\\s+(\\S+)$', + raw_value, + re.IGNORECASE) if match is not None: target = match.group(4) @@ -146,13 +175,21 @@ class MythicBeastsProvider(BaseProvider): 'target': target, } - return self._data_for_single('SRV', {'raw_values': [ {'value': value, 'ttl': ttl} ]}) + return MythicBeastsProvider._data_for_single( + 'SRV', + {'raw_values': [ + {'value': value, 'ttl': ttl} + ]}) - def _data_for_SSHFP(self, _type, data): + @staticmethod + def _data_for_SSHFP(_type, data): ttl = data['raw_values'][0]['ttl'] raw_value = data['raw_values'][0]['value'] - match = re.match('^([0-9]+)\s+([0-9]+)\s+(\S+)$', raw_value, re.IGNORECASE) + match = re.match( + '^([0-9]+)\\s+([0-9]+)\\s+(\\S+)$', + raw_value, + re.IGNORECASE) if match is not None: value = { @@ -161,14 +198,21 @@ class MythicBeastsProvider(BaseProvider): 'fingerprint': match.group(3), } - return self._data_for_single('SSHFP', {'raw_values': [ {'value': value, 'ttl': ttl} ]}) - + return MythicBeastsProvider._data_for_single( + 'SSHFP', + {'raw_values': [ + {'value': value, 'ttl': ttl} + ]}) - def _data_for_CAA(self, _type, data): + @staticmethod + def _data_for_CAA(_type, data): ttl = data['raw_values'][0]['ttl'] raw_value = data['raw_values'][0]['value'] - match = re.match('^([0-9]+)\s+(issue|issuewild|iodef)\s+(\S+)$', raw_value, re.IGNORECASE) + match = re.match( + '^([0-9]+)\\s+(issue|issuewild|iodef)\\s+(\\S+)$', + raw_value, + re.IGNORECASE) if match is not None: value = { @@ -176,15 +220,17 @@ class MythicBeastsProvider(BaseProvider): 'tag': match.group(2), 'value': match.group(3), } - return self._data_for_single('CAA', {'raw_values': [ {'value': value, 'ttl': ttl} ]}) - + return MythicBeastsProvider._data_for_single( + 'CAA', + {'raw_values': [ + {'value': value, 'ttl': ttl} + ]}) _data_for_NS = _data_for_multiple _data_for_TXT = _data_for_multiple _data_for_A = _data_for_multiple _data_for_AAAA = _data_for_multiple - def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) @@ -192,15 +238,12 @@ class MythicBeastsProvider(BaseProvider): resp = None try: resp = self.records(zone.name) - except HTTPError as e: - if e.response.status_code == 401: + except HTTPError as err: + if err.response.status_code == 401: # Nicer error message for auth problems - raise Exception('Mythic Beasts authentication problem with {}'.format(zone.name)) - elif e.response.status_code == 422: - # 422 means mythicbeasts doesn't know anything about the requested - # domain. We'll just ignore it here and leave the zone - # untouched. - raise + raise Exception( + 'Mythic Beasts authentication problem with {}'.format( + zone.name)) else: # just re-throw raise @@ -212,7 +255,10 @@ class MythicBeastsProvider(BaseProvider): if resp: exists = True for line in resp.content.splitlines(): - match = re.match('^(\S+)\s+(\S+)\s+(\S+)\s+(.*)$', line, re.IGNORECASE) + match = re.match( + '^(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(.*)$', + line, + re.IGNORECASE) if match is not None: if match.group(1) == '@': @@ -244,16 +290,18 @@ class MythicBeastsProvider(BaseProvider): data[_type][_name].get('raw_values').append( {'value': _value, 'ttl': _ttl} ) - except AttributeError as error: + except AttributeError: self.log.debug('skipping {} as not supported', _type) continue - for _type in data: for _name in data[_type]: data_for = getattr(self, '_data_for_{}'.format(_type)) - self.log.debug('record: {}, {}'.format(_type, data[_type][_name])) + self.log.debug( + 'record: %s,\t%s', + _type, + data[_type][_name]) record = Record.new( zone, @@ -263,12 +311,10 @@ class MythicBeastsProvider(BaseProvider): ) zone.add_record(record, lenient=lenient) - self.log.debug('populate: found %s records, exists=%s', - len(zone.records) - before, exists) + len(zone.records) - before, exists) return exists - def _compile_commands(self, action, change): commands = [] @@ -280,7 +326,6 @@ class MythicBeastsProvider(BaseProvider): elif action == 'DELETE': record = change.existing - zone = record.zone hostname = remove_trailing_dot(record.fqdn) ttl = record.ttl _type = record._type @@ -288,13 +333,11 @@ class MythicBeastsProvider(BaseProvider): if _type == 'ALIAS': _type = 'ANAME' - if hasattr(record, 'values'): values = record.values else: values = [record.value] - base = '{} {} {} {}'.format(action, hostname, ttl, _type) if re.match('[A]{1,4}', _type) is not None: @@ -304,25 +347,35 @@ class MythicBeastsProvider(BaseProvider): elif _type == 'SSHFP': data = values[0].data commands.append('{} {} {} {}'.format( - base, data['algorithm'], data['fingerprint_type'], data['fingerprint'])) + base, + data['algorithm'], + data['fingerprint_type'], + data['fingerprint'] + )) elif _type == 'SRV': data = values[0].data commands.append('{} {} {} {} {}'.format( - base, data['priority'], data['weight'], data['port'], data['target'])) - + base, + data['priority'], + data['weight'], + data['port'], + data['target'])) + elif _type == 'MX': for value in values: data = value.data commands.append('{} {} {}'.format( - base, data['preference'], data['exchange'])) + base, + data['preference'], + data['exchange'])) else: try: if getattr(self, '_data_for_{}'.format(_type)) is not None: commands.append('{} {}'.format( - base, values[0])) - except AttributeError as error: + base, values[0])) + except AttributeError: self.log.debug('skipping {} as not supported', _type) pass @@ -364,11 +417,6 @@ class MythicBeastsProvider(BaseProvider): self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, len(changes)) - domain_name = desired.name - for change in changes: class_name = change.__class__.__name__ getattr(self, '_apply_{}'.format(class_name))(change) - - - From 2b6d86fb4f3281d52b117efa873a28f146411a97 Mon Sep 17 00:00:00 2001 From: Rhosyn Celyn Date: Thu, 9 May 2019 21:47:17 +0100 Subject: [PATCH 05/13] E125 fix --- octodns/provider/mythicbeasts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py index 17ec574..dd11573 100644 --- a/octodns/provider/mythicbeasts.py +++ b/octodns/provider/mythicbeasts.py @@ -110,7 +110,7 @@ class MythicBeastsProvider(BaseProvider): values = [] for raw_value in \ - [raw_values['value'] for raw_values in data['raw_values']]: + [raw_values['value'] for raw_values in data['raw_values']]: match = re.match('^([0-9]+)\\s+(\\S+)$', raw_value, re.IGNORECASE) if match is not None: From fd63150cac52ba28b305cfeaa4f0b6f2419c2acd Mon Sep 17 00:00:00 2001 From: Rhosyn Celyn Date: Mon, 13 May 2019 16:51:38 +0100 Subject: [PATCH 06/13] Added tests, clean up and small modifications --- octodns/provider/mythicbeasts.py | 274 ++++++++------- tests/fixtures/mythicbeasts-list.txt | 25 ++ tests/test_octodns_provider_mythicbeasts.py | 356 ++++++++++++++++++++ 3 files changed, 517 insertions(+), 138 deletions(-) create mode 100644 tests/fixtures/mythicbeasts-list.txt create mode 100644 tests/test_octodns_provider_mythicbeasts.py diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py index dd11573..ca03b37 100644 --- a/octodns/provider/mythicbeasts.py +++ b/octodns/provider/mythicbeasts.py @@ -7,7 +7,7 @@ from __future__ import absolute_import, division, print_function, \ import re -from requests import HTTPError, Session +from requests import Session from logging import getLogger from ..record import Record @@ -18,8 +18,8 @@ def add_trailing_dot(value): ''' Add trailing dots to values ''' - assert value - assert value[-1] != '.' + assert value, 'Missing value' + assert value[-1] != '.', 'Value already has trailing dot' return value + '.' @@ -27,8 +27,8 @@ def remove_trailing_dot(value): ''' Remove trailing dots from values ''' - assert value - assert value[-1] == '.' + assert value, 'Missing value' + assert value[-1] == '.', 'Value already missing trailing dot' return value[:-1] @@ -44,13 +44,15 @@ class MythicBeastsProvider(BaseProvider): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False - SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NS', 'SRV', 'SSHFP', 'CAA', 'TXT')) BASE = 'https://dnsapi.mythic-beasts.com/' - TIMEOUT = 15 def __init__(self, identifier, passwords, *args, **kwargs): self.log = getLogger('MythicBeastsProvider[{}]'.format(identifier)) + + assert isinstance(passwords, dict), 'Missing passwords' + self.log.debug( '__init__: id=%s, registered zones; %s', identifier, @@ -62,23 +64,28 @@ class MythicBeastsProvider(BaseProvider): self._sess = sess def _request(self, method, path, data=None): - self.log.debug('_request: method=%s, path=%s', method, path) + self.log.debug('_request: method=%s, path=%s data=%s', + method, path, data) - url = self.BASE - resp = self._sess.request(method, url, data=data, timeout=self.TIMEOUT) + resp = self._sess.request(method, path, data=data) self.log.debug( '_request: status=%d data=%s', resp.status_code, - resp.text) - if resp.status_code != 200: - self.log.info('request failed: %s, response %s', data, resp.text) - resp.raise_for_status() + resp.text[:20]) + + if resp.status_code == 401: + raise Exception('Mythic Beasts unauthorized for domain: {}' + .format(data['domain'])) + resp.raise_for_status() return resp def _post(self, data=None): return self._request('POST', self.BASE, data=data) def records(self, zone): + assert zone in self._passwords, 'Missing password for domain: {}' \ + .format(remove_trailing_dot(zone)) + return self._post({ 'domain': remove_trailing_dot(zone), 'password': self._passwords[zone], @@ -113,16 +120,17 @@ class MythicBeastsProvider(BaseProvider): [raw_values['value'] for raw_values in data['raw_values']]: match = re.match('^([0-9]+)\\s+(\\S+)$', raw_value, re.IGNORECASE) - if match is not None: - exchange = match.group(2) + assert match is not None, 'Unable to parse MX data' - if not exchange.endswith('.'): - exchange = '{}.{}'.format(exchange, data['zone']) + exchange = match.group(2) - values.append({ - 'preference': match.group(1), - 'exchange': exchange, - }) + if not exchange.endswith('.'): + exchange = '{}.{}'.format(exchange, data['zone']) + + values.append({ + 'preference': match.group(1), + 'exchange': exchange, + }) return { 'type': _type, @@ -155,54 +163,61 @@ class MythicBeastsProvider(BaseProvider): @staticmethod def _data_for_SRV(_type, data): - ttl = data['raw_values'][0]['ttl'] - raw_value = data['raw_values'][0]['value'] + ttl = max([raw_values['ttl'] for raw_values in data['raw_values']]) + values = [] - match = re.match( - '^([0-9]+)\\s+([0-9]+)\\s+([0-9]+)\\s+(\\S+)$', - raw_value, - re.IGNORECASE) + for raw_value in \ + [raw_values['value'] for raw_values in data['raw_values']]: + + match = re.match( + '^([0-9]+)\\s+([0-9]+)\\s+([0-9]+)\\s+(\\S+)$', + raw_value, + re.IGNORECASE) + + assert match is not None, 'Unable to parse SRV data' - if match is not None: target = match.group(4) if not target.endswith('.'): target = '{}.{}'.format(target, data['zone']) - value = { + values.append({ 'priority': match.group(1), 'weight': match.group(2), 'port': match.group(3), 'target': target, - } + }) - return MythicBeastsProvider._data_for_single( - 'SRV', - {'raw_values': [ - {'value': value, 'ttl': ttl} - ]}) + return { + 'type': _type, + 'values': values, + 'ttl': ttl, + } @staticmethod def _data_for_SSHFP(_type, data): - ttl = data['raw_values'][0]['ttl'] - raw_value = data['raw_values'][0]['value'] + ttl = max([raw_values['ttl'] for raw_values in data['raw_values']]) + values = [] - match = re.match( - '^([0-9]+)\\s+([0-9]+)\\s+(\\S+)$', - raw_value, - re.IGNORECASE) + for raw_value in \ + [raw_values['value'] for raw_values in data['raw_values']]: + match = re.match( + '^([0-9]+)\\s+([0-9]+)\\s+(\\S+)$', + raw_value, + re.IGNORECASE) - if match is not None: - value = { + assert match is not None, 'Unable to parse SSHFP data' + + values.append({ 'algorithm': match.group(1), 'fingerprint_type': match.group(2), 'fingerprint': match.group(3), - } + }) - return MythicBeastsProvider._data_for_single( - 'SSHFP', - {'raw_values': [ - {'value': value, 'ttl': ttl} - ]}) + return { + 'type': _type, + 'values': values, + 'ttl': ttl, + } @staticmethod def _data_for_CAA(_type, data): @@ -214,17 +229,17 @@ class MythicBeastsProvider(BaseProvider): raw_value, re.IGNORECASE) - if match is not None: - value = { - 'flags': match.group(1), - 'tag': match.group(2), - 'value': match.group(3), - } - return MythicBeastsProvider._data_for_single( - 'CAA', - {'raw_values': [ - {'value': value, 'ttl': ttl} - ]}) + assert match is not None, 'Unable to parse CAA data' + + value = { + 'flags': match.group(1), + 'tag': match.group(2), + 'value': match.group(3), + } + + return MythicBeastsProvider._data_for_single( + 'CAA', + {'raw_values': [{'value': value, 'ttl': ttl}]}) _data_for_NS = _data_for_multiple _data_for_TXT = _data_for_multiple @@ -235,84 +250,69 @@ class MythicBeastsProvider(BaseProvider): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) - resp = None - try: - resp = self.records(zone.name) - except HTTPError as err: - if err.response.status_code == 401: - # Nicer error message for auth problems - raise Exception( - 'Mythic Beasts authentication problem with {}'.format( - zone.name)) - else: - # just re-throw - raise + resp = self.records(zone.name) before = len(zone.records) exists = False data = dict() - if resp: - exists = True - for line in resp.content.splitlines(): - match = re.match( - '^(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(.*)$', - line, - re.IGNORECASE) + exists = True + for line in resp.content.splitlines(): + match = re.match( + '^(\\S+)\\s+(\\d+)\\s+(\\S+)\\s+(.*)$', + line, + re.IGNORECASE) - if match is not None: - if match.group(1) == '@': - _name = '' - else: - _name = match.group(1) + if match is None: + self.log.debug('failed to match line: %s', line) + continue - _type = match.group(3) - _ttl = int(match.group(2)) - _value = match.group(4).strip() + if match.group(1) == '@': + _name = '' + else: + _name = match.group(1) - if _type == 'SOA': - continue + _type = match.group(3) + _ttl = int(match.group(2)) + _value = match.group(4).strip() - try: - if getattr(self, '_data_for_{}'.format(_type)) is not None: + if _type == 'TXT': + _value = _value.replace(';', '\\;') - if _type not in data: - data[_type] = dict() + if hasattr(self, '_data_for_{}'.format(_type)): - if _name not in data[_type]: - data[_type][_name] = { - 'raw_values': [{'value': _value, 'ttl': _ttl}], - 'name': _name, - 'zone': zone.name, - } + if _type not in data: + data[_type] = dict() - else: - data[_type][_name].get('raw_values').append( - {'value': _value, 'ttl': _ttl} - ) - except AttributeError: - self.log.debug('skipping {} as not supported', _type) - continue + if _name not in data[_type]: + data[_type][_name] = { + 'raw_values': [{'value': _value, 'ttl': _ttl}], + 'name': _name, + 'zone': zone.name, + } - for _type in data: - for _name in data[_type]: - data_for = getattr(self, '_data_for_{}'.format(_type)) - - self.log.debug( - 'record: %s,\t%s', - _type, - data[_type][_name]) - - record = Record.new( - zone, - _name, - data_for(_type, data[_type][_name]), - source=self + else: + data[_type][_name].get('raw_values').append( + {'value': _value, 'ttl': _ttl} ) - zone.add_record(record, lenient=lenient) + else: + self.log.debug('skipping %s as not supported', _type) + + for _type in data: + for _name in data[_type]: + data_for = getattr(self, '_data_for_{}'.format(_type)) + + record = Record.new( + zone, + _name, + data_for(_type, data[_type][_name]), + source=self + ) + zone.add_record(record, lenient=lenient) self.log.debug('populate: found %s records, exists=%s', len(zone.records) - before, exists) + return exists def _compile_commands(self, action, change): @@ -322,8 +322,7 @@ class MythicBeastsProvider(BaseProvider): if action == 'ADD': record = change.new - - elif action == 'DELETE': + else: record = change.existing hostname = remove_trailing_dot(record.fqdn) @@ -354,13 +353,14 @@ class MythicBeastsProvider(BaseProvider): )) elif _type == 'SRV': - data = values[0].data - commands.append('{} {} {} {} {}'.format( - base, - data['priority'], - data['weight'], - data['port'], - data['target'])) + for value in values: + data = value.data + commands.append('{} {} {} {} {}'.format( + base, + data['priority'], + data['weight'], + data['port'], + data['target'])) elif _type == 'MX': for value in values: @@ -371,13 +371,11 @@ class MythicBeastsProvider(BaseProvider): data['exchange'])) else: - try: - if getattr(self, '_data_for_{}'.format(_type)) is not None: - commands.append('{} {}'.format( - base, values[0])) - except AttributeError: - self.log.debug('skipping {} as not supported', _type) - pass + if hasattr(self, '_data_for_{}'.format(_type)): + commands.append('{} {}'.format( + base, values[0])) + else: + self.log.debug('skipping %s as not supported', _type) return commands diff --git a/tests/fixtures/mythicbeasts-list.txt b/tests/fixtures/mythicbeasts-list.txt new file mode 100644 index 0000000..ed4ea4c --- /dev/null +++ b/tests/fixtures/mythicbeasts-list.txt @@ -0,0 +1,25 @@ +@ 3600 NS 6.2.3.4. +@ 3600 NS 7.2.3.4. +@ 300 A 1.2.3.4 +@ 300 A 1.2.3.5 +@ 3600 SSHFP 1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73 +@ 3600 SSHFP 1 1 7491973e5f8b39d5327cd4e08bc81b05f7710b49 +@ 3600 CAA 0 issue ca.unit.tests +_srv._tcp 600 SRV 10 20 30 foo-1.unit.tests. +_srv._tcp 600 SRV 12 20 30 foo-2.unit.tests. +aaaa 600 AAAA 2601:644:500:e210:62f8:1dff:feb8:947a +cname 300 CNAME unit.tests. +excluded 300 CNAME unit.tests. +ignored 300 A 9.9.9.9 +included 3600 CNAME unit.tests. +mx 300 MX 10 smtp-4.unit.tests. +mx 300 MX 20 smtp-2.unit.tests. +mx 300 MX 30 smtp-3.unit.tests. +mx 300 MX 40 smtp-1.unit.tests. +sub 3600 NS 6.2.3.4. +sub 3600 NS 7.2.3.4. +txt 600 TXT "Bah bah black sheep" +txt 600 TXT "have you any wool." +txt 600 TXT "v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs" +www 300 A 2.2.3.6 +www.sub 300 A 2.2.3.6 diff --git a/tests/test_octodns_provider_mythicbeasts.py b/tests/test_octodns_provider_mythicbeasts.py new file mode 100644 index 0000000..a1ef5d5 --- /dev/null +++ b/tests/test_octodns_provider_mythicbeasts.py @@ -0,0 +1,356 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from os.path import dirname, join + +from requests_mock import ANY, mock as requests_mock +from unittest import TestCase + +from octodns.provider.mythicbeasts import MythicBeastsProvider, \ + add_trailing_dot, remove_trailing_dot +from octodns.provider.yaml import YamlProvider +from octodns.zone import Zone +from octodns.record import Create, Update, Delete, Record + + +class TestMythicBeastsProvider(TestCase): + expected = Zone('unit.tests.', []) + source = YamlProvider('test_expected', join(dirname(__file__), 'config')) + source.populate(expected) + + # Dump anything we don't support from expected + for record in list(expected.records): + if record._type not in MythicBeastsProvider.SUPPORTS: + expected._remove_record(record) + + def test_trailing_dot(self): + with self.assertRaises(AssertionError) as err: + add_trailing_dot('unit.tests.') + self.assertEquals('Value already has trailing dot', + err.exception.message) + + with self.assertRaises(AssertionError) as err: + remove_trailing_dot('unit.tests') + self.assertEquals('Value already missing trailing dot', + err.exception.message) + + self.assertEquals(add_trailing_dot('unit.tests'), 'unit.tests.') + self.assertEquals(remove_trailing_dot('unit.tests.'), 'unit.tests') + + def test_data_for_single(self): + test_data = { + 'raw_values': [{'value': 'a:a::c', 'ttl': 0}], + 'zone': 'unit.tests.', + } + test_single = MythicBeastsProvider._data_for_single('', test_data) + self.assertTrue(isinstance(test_single, dict)) + self.assertEquals('a:a::c', test_single['value']) + + def test_data_for_multiple(self): + test_data = { + 'raw_values': [ + {'value': 'b:b::d', 'ttl': 60}, + {'value': 'a:a::c', 'ttl': 60}], + 'zone': 'unit.tests.', + } + test_multiple = MythicBeastsProvider._data_for_multiple('', test_data) + self.assertTrue(isinstance(test_multiple, dict)) + self.assertEquals(2, len(test_multiple['values'])) + + def test_data_for_MX(self): + test_data = { + 'raw_values': [ + {'value': '10 un.unit', 'ttl': 60}, + {'value': '20 dau.unit', 'ttl': 60}, + {'value': '30 tri.unit', 'ttl': 60}], + 'zone': 'unit.tests.', + } + test_MX = MythicBeastsProvider._data_for_MX('', test_data) + self.assertTrue(isinstance(test_MX, dict)) + self.assertEquals(3, len(test_MX['values'])) + + with self.assertRaises(AssertionError) as err: + test_MX = MythicBeastsProvider._data_for_MX( + '', + {'raw_values': [{'value': '', 'ttl': 0}]} + ) + self.assertEquals('Unable to parse MX data', + err.exception.message) + + def test_data_for_CNAME(self): + test_data = { + 'raw_values': [{'value': 'cname', 'ttl': 60}], + 'zone': 'unit.tests.', + } + test_cname = MythicBeastsProvider._data_for_CNAME('', test_data) + self.assertTrue(isinstance(test_cname, dict)) + self.assertEquals('cname.unit.tests.', test_cname['value']) + + def test_data_for_ANAME(self): + test_data = { + 'raw_values': [{'value': 'aname', 'ttl': 60}], + 'zone': 'unit.tests.', + } + test_aname = MythicBeastsProvider._data_for_ANAME('', test_data) + self.assertTrue(isinstance(test_aname, dict)) + self.assertEquals('aname', test_aname['value']) + + def test_data_for_SRV(self): + test_data = { + 'raw_values': [ + {'value': '10 20 30 un.srv.unit', 'ttl': 60}, + {'value': '20 30 40 dau.srv.unit', 'ttl': 60}, + {'value': '30 30 50 tri.srv.unit', 'ttl': 60}], + 'zone': 'unit.tests.', + } + test_SRV = MythicBeastsProvider._data_for_SRV('', test_data) + self.assertTrue(isinstance(test_SRV, dict)) + self.assertEquals(3, len(test_SRV['values'])) + + with self.assertRaises(AssertionError) as err: + test_SRV = MythicBeastsProvider._data_for_SRV( + '', + {'raw_values': [{'value': '', 'ttl': 0}]} + ) + self.assertEquals('Unable to parse SRV data', + err.exception.message) + + def test_data_for_SSHFP(self): + test_data = { + 'raw_values': [ + {'value': '1 1 0123456789abcdef', 'ttl': 60}, + {'value': '1 2 0123456789abcdef', 'ttl': 60}, + {'value': '2 3 0123456789abcdef', 'ttl': 60}], + 'zone': 'unit.tests.', + } + test_SSHFP = MythicBeastsProvider._data_for_SSHFP('', test_data) + self.assertTrue(isinstance(test_SSHFP, dict)) + self.assertEquals(3, len(test_SSHFP['values'])) + + with self.assertRaises(AssertionError) as err: + test_SSHFP = MythicBeastsProvider._data_for_SSHFP( + '', + {'raw_values': [{'value': '', 'ttl': 0}]} + ) + self.assertEquals('Unable to parse SSHFP data', + err.exception.message) + + def test_data_for_CAA(self): + test_data = { + 'raw_values': [{'value': '1 issue letsencrypt.org', 'ttl': 60}], + 'zone': 'unit.tests.', + } + test_CAA = MythicBeastsProvider._data_for_CAA('', test_data) + self.assertTrue(isinstance(test_CAA, dict)) + self.assertEquals(3, len(test_CAA['value'])) + + with self.assertRaises(AssertionError) as err: + test_CAA = MythicBeastsProvider._data_for_CAA( + '', + {'raw_values': [{'value': '', 'ttl': 0}]} + ) + self.assertEquals('Unable to parse CAA data', + err.exception.message) + + def test_alias_command_generation(self): + zone = Zone('unit.tests.', []) + zone.add_record(Record.new(zone, 'prawf', { + 'ttl': 60, + 'type': 'ALIAS', + 'value': 'alias.unit.tests.', + })) + with requests_mock() as mock: + mock.post(ANY, status_code=200, text='') + + provider = MythicBeastsProvider('test', { + 'unit.tests.': 'mypassword' + }) + + plan = provider.plan(zone) + change = plan.changes[0] + + command = provider._compile_commands('ADD', change) + self.assertEquals( + ['ADD prawf.unit.tests 60 ANAME alias.unit.tests.'], + command + ) + + def test_txt_command_generation(self): + zone = Zone('unit.tests.', []) + zone.add_record(Record.new(zone, 'prawf', { + 'ttl': 60, + 'type': 'TXT', + 'value': 'prawf prawf dyma prawf', + })) + with requests_mock() as mock: + mock.post(ANY, status_code=200, text='') + + provider = MythicBeastsProvider('test', { + 'unit.tests.': 'mypassword' + }) + + plan = provider.plan(zone) + change = plan.changes[0] + + command = provider._compile_commands('ADD', change) + self.assertEquals( + ['ADD prawf.unit.tests 60 TXT prawf prawf dyma prawf'], + command + ) + + def test_command_generation(self): + class FakeChangeRecord(object): + def __init__(self): + self.__fqdn = 'prawf.unit.tests.' + self._type = 'NOOP' + self.value = 'prawf' + self.ttl = 60 + + @property + def new(self): + return self + + @property + def existing(self): + return self + + @property + def fqdn(self): + return self.__fqdn + + with requests_mock() as mock: + mock.post(ANY, status_code=200, text='') + + provider = MythicBeastsProvider('test', { + 'unit.tests.': 'mypassword' + }) + record = FakeChangeRecord() + command = provider._compile_commands('ADD', record) + self.assertEquals([], command) + + def test_populate(self): + provider = None + + # Null passwords dict + with self.assertRaises(AssertionError) as err: + provider = MythicBeastsProvider('test', None) + self.assertEquals('Missing passwords', err.exception.message) + + # Missing password + with requests_mock() as mock: + mock.post(ANY, status_code=401, text='ERR Not authenticated') + + with self.assertRaises(AssertionError) as err: + provider = MythicBeastsProvider('test', dict()) + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals( + 'Missing password for domain: unit.tests', + err.exception.message) + + # Failed authentication + with requests_mock() as mock: + mock.post(ANY, status_code=401, text='ERR Not authenticated') + + with self.assertRaises(Exception) as err: + provider = MythicBeastsProvider('test', { + 'unit.tests.': 'mypassword' + }) + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals( + 'Mythic Beasts unauthorized for domain: unit.tests', + err.exception.message) + + # Check unmatched lines are ignored + test_data = 'This should not match' + with requests_mock() as mock: + mock.post(ANY, status_code=200, text=test_data) + + provider = MythicBeastsProvider('test', { + 'unit.tests.': 'mypassword' + }) + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(0, len(zone.records)) + + # Check unsupported records are skipped + test_data = '@ 60 NOOP prawf\n@ 60 SPF prawf prawf prawf' + with requests_mock() as mock: + mock.post(ANY, status_code=200, text=test_data) + + provider = MythicBeastsProvider('test', { + 'unit.tests.': 'mypassword' + }) + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(0, len(zone.records)) + + # Check no changes between what we support and what's parsed + # from the unit.tests. config YAML. Also make sure we see the same + # for both after we've thrown away records we don't support + with requests_mock() as mock: + with open('tests/fixtures/mythicbeasts-list.txt') as file_handle: + mock.post(ANY, status_code=200, text=file_handle.read()) + + provider = MythicBeastsProvider('test', { + 'unit.tests.': 'mypassword' + }) + zone = Zone('unit.tests.', []) + provider.populate(zone) + + self.assertEquals(15, len(zone.records)) + self.assertEquals(15, len(self.expected.records)) + changes = self.expected.changes(zone, provider) + self.assertEquals(0, len(changes)) + + def test_apply(self): + provider = MythicBeastsProvider('test', { + 'unit.tests.': 'mypassword' + }) + zone = Zone('unit.tests.', []) + + # Create blank zone + with requests_mock() as mock: + mock.post(ANY, status_code=200, text='') + provider.populate(zone) + + self.assertEquals(0, len(zone.records)) + + # Check deleting and adding/changing test record + existing = 'prawf 300 TXT prawf prawf prawf\ndileu 300 TXT dileu' + + with requests_mock() as mock: + mock.post(ANY, status_code=200, text=existing) + + # Mash up a new zone with records so a plan + # is generated with changes and applied. For some reason + # passing self.expected, or just changing each record's zone + # doesn't work. Nor does this without a single add_record after + wanted = Zone('unit.tests.', []) + for record in list(self.expected.records): + data = {'type': record._type} + data.update(record.data) + wanted.add_record(Record.new(wanted, record.name, data)) + + wanted.add_record(Record.new(wanted, 'prawf', { + 'ttl': 60, + 'type': 'TXT', + 'value': 'prawf yw e', + })) + + plan = provider.plan(wanted) + + # Octo ignores NS records (15-1) + self.assertEquals(1, len(filter(lambda u: isinstance(u, Update), + plan.changes))) + self.assertEquals(1, len(filter(lambda d: isinstance(d, Delete), + plan.changes))) + self.assertEquals(14, len(filter(lambda c: isinstance(c, Create), + plan.changes))) + self.assertEquals(16, provider.apply(plan)) + self.assertTrue(plan.exists) From 57b03971d8c1c9b7f1f23b4fc592669eea53bd64 Mon Sep 17 00:00:00 2001 From: cwningen Date: Tue, 14 May 2019 14:46:49 +0100 Subject: [PATCH 07/13] Update octodns/provider/mythicbeasts.py Seems fair to me! I think a lot of the suggestions you've mentioned are obvious ones that have been lost on me from being very confused trying to understand the available objects Co-Authored-By: Ross McFarland --- octodns/provider/mythicbeasts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py index ca03b37..3d1f5ec 100644 --- a/octodns/provider/mythicbeasts.py +++ b/octodns/provider/mythicbeasts.py @@ -339,7 +339,7 @@ class MythicBeastsProvider(BaseProvider): base = '{} {} {} {}'.format(action, hostname, ttl, _type) - if re.match('[A]{1,4}', _type) is not None: + if _type in ('A', 'AAAA'): for value in values: commands.append('{} {}'.format(base, value)) From d6fb3310d53b3daa0c5cfd48f77f208d8ec08192 Mon Sep 17 00:00:00 2001 From: Rhosyn Celyn Date: Tue, 14 May 2019 16:12:53 +0100 Subject: [PATCH 08/13] Applied suggested modifications --- octodns/provider/mythicbeasts.py | 118 ++++++++++++-------- tests/test_octodns_provider_mythicbeasts.py | 11 +- 2 files changed, 75 insertions(+), 54 deletions(-) diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py index 3d1f5ec..af07ddb 100644 --- a/octodns/provider/mythicbeasts.py +++ b/octodns/provider/mythicbeasts.py @@ -32,16 +32,59 @@ def remove_trailing_dot(value): return value[:-1] +class MythicBeastsUnauthorizedException(Exception): + def __init__(self, zone, *args): + self.zone = zone + self.message = 'Mythic Beasts unauthorized for zone: {}'.format( + self.zone + ) + + super(MythicBeastsUnauthorizedException, self).__init__( + self.message, self.zone, *args) + + class MythicBeastsProvider(BaseProvider): ''' Mythic Beasts DNS API Provider - mythicbeasts: - class: octodns.provider.mythicbeasts.MythicBeastsProvider - zones: - my-zone: 'password' + Config settings: + + --- + providers: + config: + ... + mythicbeasts: + class: octodns.provider.mythicbeasts.MythicBeastsProvider + passwords: + my.domain.: 'password' + + zones: + my.domain.: + targets: + - mythic ''' + RE_MX = re.compile(r'^(?P[0-9]+)\s+(?P\S+)$', + re.IGNORECASE) + + RE_SRV = re.compile(r'^(?P[0-9]+)\s+(?P[0-9]+)\s+' + r'(?P[0-9]+)\s+(?P\S+)$', + re.IGNORECASE) + + RE_SSHFP = re.compile(r'^(?P[0-9]+)\s+' + r'(?P[0-9]+)\s+' + r'(?P\S+)$', + re.IGNORECASE) + + RE_CAA = re.compile(r'^(?P[0-9]+)\s+' + r'(?Pissue|issuewild|iodef)\s+' + r'(?P\S+)$', + re.IGNORECASE) + + RE_POPLINE = re.compile(r'^(?P\S+)\s+(?P\d+)\s+' + r'(?P\S+)\s+(?P.*)$', + re.IGNORECASE) + SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NS', @@ -51,7 +94,7 @@ class MythicBeastsProvider(BaseProvider): def __init__(self, identifier, passwords, *args, **kwargs): self.log = getLogger('MythicBeastsProvider[{}]'.format(identifier)) - assert isinstance(passwords, dict), 'Missing passwords' + assert isinstance(passwords, dict), 'Passwords must be a dictionary' self.log.debug( '__init__: id=%s, registered zones; %s', @@ -74,8 +117,7 @@ class MythicBeastsProvider(BaseProvider): resp.text[:20]) if resp.status_code == 401: - raise Exception('Mythic Beasts unauthorized for domain: {}' - .format(data['domain'])) + raise MythicBeastsUnauthorizedException(data['domain']) resp.raise_for_status() return resp @@ -118,17 +160,17 @@ class MythicBeastsProvider(BaseProvider): for raw_value in \ [raw_values['value'] for raw_values in data['raw_values']]: - match = re.match('^([0-9]+)\\s+(\\S+)$', raw_value, re.IGNORECASE) + match = MythicBeastsProvider.RE_MX.match(raw_value) assert match is not None, 'Unable to parse MX data' - exchange = match.group(2) + exchange = match.group('exchange') if not exchange.endswith('.'): exchange = '{}.{}'.format(exchange, data['zone']) values.append({ - 'preference': match.group(1), + 'preference': match.group('preference'), 'exchange': exchange, }) @@ -169,21 +211,18 @@ class MythicBeastsProvider(BaseProvider): for raw_value in \ [raw_values['value'] for raw_values in data['raw_values']]: - match = re.match( - '^([0-9]+)\\s+([0-9]+)\\s+([0-9]+)\\s+(\\S+)$', - raw_value, - re.IGNORECASE) + match = MythicBeastsProvider.RE_SRV.match(raw_value) assert match is not None, 'Unable to parse SRV data' - target = match.group(4) + target = match.group('target') if not target.endswith('.'): target = '{}.{}'.format(target, data['zone']) values.append({ - 'priority': match.group(1), - 'weight': match.group(2), - 'port': match.group(3), + 'priority': match.group('priority'), + 'weight': match.group('weight'), + 'port': match.group('port'), 'target': target, }) @@ -200,17 +239,14 @@ class MythicBeastsProvider(BaseProvider): for raw_value in \ [raw_values['value'] for raw_values in data['raw_values']]: - match = re.match( - '^([0-9]+)\\s+([0-9]+)\\s+(\\S+)$', - raw_value, - re.IGNORECASE) + match = MythicBeastsProvider.RE_SSHFP.match(raw_value) assert match is not None, 'Unable to parse SSHFP data' values.append({ - 'algorithm': match.group(1), - 'fingerprint_type': match.group(2), - 'fingerprint': match.group(3), + 'algorithm': match.group('algorithm'), + 'fingerprint_type': match.group('fingerprint_type'), + 'fingerprint': match.group('fingerprint'), }) return { @@ -224,17 +260,14 @@ class MythicBeastsProvider(BaseProvider): ttl = data['raw_values'][0]['ttl'] raw_value = data['raw_values'][0]['value'] - match = re.match( - '^([0-9]+)\\s+(issue|issuewild|iodef)\\s+(\\S+)$', - raw_value, - re.IGNORECASE) + match = MythicBeastsProvider.RE_CAA.match(raw_value) assert match is not None, 'Unable to parse CAA data' value = { - 'flags': match.group(1), - 'tag': match.group(2), - 'value': match.group(3), + 'flags': match.group('flags'), + 'tag': match.group('tag'), + 'value': match.group('value'), } return MythicBeastsProvider._data_for_single( @@ -258,10 +291,7 @@ class MythicBeastsProvider(BaseProvider): exists = True for line in resp.content.splitlines(): - match = re.match( - '^(\\S+)\\s+(\\d+)\\s+(\\S+)\\s+(.*)$', - line, - re.IGNORECASE) + match = MythicBeastsProvider.RE_POPLINE.match(line) if match is None: self.log.debug('failed to match line: %s', line) @@ -270,11 +300,11 @@ class MythicBeastsProvider(BaseProvider): if match.group(1) == '@': _name = '' else: - _name = match.group(1) + _name = match.group('name') - _type = match.group(3) - _ttl = int(match.group(2)) - _value = match.group(4).strip() + _type = match.group('type') + _ttl = int(match.group('ttl')) + _value = match.group('value').strip() if _type == 'TXT': _value = _value.replace(';', '\\;') @@ -318,13 +348,7 @@ class MythicBeastsProvider(BaseProvider): def _compile_commands(self, action, change): commands = [] - record = None - - if action == 'ADD': - record = change.new - else: - record = change.existing - + record = change.record hostname = remove_trailing_dot(record.fqdn) ttl = record.ttl _type = record._type diff --git a/tests/test_octodns_provider_mythicbeasts.py b/tests/test_octodns_provider_mythicbeasts.py index a1ef5d5..5deba3b 100644 --- a/tests/test_octodns_provider_mythicbeasts.py +++ b/tests/test_octodns_provider_mythicbeasts.py @@ -211,11 +211,7 @@ class TestMythicBeastsProvider(TestCase): self.ttl = 60 @property - def new(self): - return self - - @property - def existing(self): + def record(self): return self @property @@ -238,7 +234,8 @@ class TestMythicBeastsProvider(TestCase): # Null passwords dict with self.assertRaises(AssertionError) as err: provider = MythicBeastsProvider('test', None) - self.assertEquals('Missing passwords', err.exception.message) + self.assertEquals('Passwords must be a dictionary', + err.exception.message) # Missing password with requests_mock() as mock: @@ -263,7 +260,7 @@ class TestMythicBeastsProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals( - 'Mythic Beasts unauthorized for domain: unit.tests', + 'Mythic Beasts unauthorized for zone: unit.tests', err.exception.message) # Check unmatched lines are ignored From 9b8e74c5dd448012a417eaebee602b69db736946 Mon Sep 17 00:00:00 2001 From: Rhosyn Celyn Date: Fri, 14 Jun 2019 15:09:41 +0100 Subject: [PATCH 09/13] Small clean up to populate and exceptions for requests --- octodns/provider/mythicbeasts.py | 31 +++++++++++++++++---- tests/test_octodns_provider_mythicbeasts.py | 21 ++++++++++++++ 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py index af07ddb..8bd2979 100644 --- a/octodns/provider/mythicbeasts.py +++ b/octodns/provider/mythicbeasts.py @@ -13,6 +13,8 @@ from logging import getLogger from ..record import Record from .base import BaseProvider +from collections import defaultdict + def add_trailing_dot(value): ''' @@ -43,6 +45,19 @@ class MythicBeastsUnauthorizedException(Exception): self.message, self.zone, *args) +class MythicBeastsRecordException(Exception): + def __init__(self, zone, command, *args): + self.zone = zone + self.command = command + self.message = 'Mythic Beasts could not action command: {} {}'.format( + self.zone, + self.command, + ) + + super(MythicBeastsRecordException, self).__init__( + self.message, self.zone, self.command, *args) + + class MythicBeastsProvider(BaseProvider): ''' Mythic Beasts DNS API Provider @@ -118,6 +133,12 @@ class MythicBeastsProvider(BaseProvider): if resp.status_code == 401: raise MythicBeastsUnauthorizedException(data['domain']) + + if resp.status_code == 400: + raise MythicBeastsRecordException( + data['domain'], + data['command'] + ) resp.raise_for_status() return resp @@ -287,7 +308,11 @@ class MythicBeastsProvider(BaseProvider): before = len(zone.records) exists = False - data = dict() + data = defaultdict(lambda: defaultdict(lambda: { + 'raw_values': [], + 'name': None, + 'zone': None, + })) exists = True for line in resp.content.splitlines(): @@ -310,10 +335,6 @@ class MythicBeastsProvider(BaseProvider): _value = _value.replace(';', '\\;') if hasattr(self, '_data_for_{}'.format(_type)): - - if _type not in data: - data[_type] = dict() - if _name not in data[_type]: data[_type][_name] = { 'raw_values': [{'value': _value, 'ttl': _ttl}], diff --git a/tests/test_octodns_provider_mythicbeasts.py b/tests/test_octodns_provider_mythicbeasts.py index 5deba3b..2a3b85b 100644 --- a/tests/test_octodns_provider_mythicbeasts.py +++ b/tests/test_octodns_provider_mythicbeasts.py @@ -318,6 +318,27 @@ class TestMythicBeastsProvider(TestCase): self.assertEquals(0, len(zone.records)) + # Record change failed + with requests_mock() as mock: + mock.post(ANY, status_code=200, text='') + provider.populate(zone) + zone.add_record(Record.new(zone, 'prawf', { + 'ttl': 300, + 'type': 'TXT', + 'value': 'prawf', + })) + plan = provider.plan(zone) + + with requests_mock() as mock: + mock.post(ANY, status_code=400, text='NADD 300 TXT prawf') + + with self.assertRaises(Exception) as err: + provider.apply(plan) + self.assertEquals( + 'Mythic Beasts could not action command: unit.tests ' + 'ADD prawf.unit.tests 300 TXT prawf', + err.exception.message) + # Check deleting and adding/changing test record existing = 'prawf 300 TXT prawf prawf prawf\ndileu 300 TXT dileu' From 9dc44c3f2c5f40541b845d8bc2e18bd2126ec792 Mon Sep 17 00:00:00 2001 From: Rhosyn Celyn Date: Mon, 17 Jun 2019 16:34:58 +0100 Subject: [PATCH 10/13] Clean up and simplification of tests and command generation, bug fix for multiple sub domain NS records and handling of creation/deletion --- octodns/provider/mythicbeasts.py | 17 ++--- tests/test_octodns_provider_mythicbeasts.py | 79 ++++++++++++++------- 2 files changed, 59 insertions(+), 37 deletions(-) diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py index 8bd2979..3470ff1 100644 --- a/octodns/provider/mythicbeasts.py +++ b/octodns/provider/mythicbeasts.py @@ -366,10 +366,9 @@ class MythicBeastsProvider(BaseProvider): return exists - def _compile_commands(self, action, change): + def _compile_commands(self, action, record): commands = [] - record = change.record hostname = remove_trailing_dot(record.fqdn) ttl = record.ttl _type = record._type @@ -384,11 +383,7 @@ class MythicBeastsProvider(BaseProvider): base = '{} {} {} {}'.format(action, hostname, ttl, _type) - if _type in ('A', 'AAAA'): - for value in values: - commands.append('{} {}'.format(base, value)) - - elif _type == 'SSHFP': + if _type == 'SSHFP': data = values[0].data commands.append('{} {} {} {}'.format( base, @@ -417,8 +412,8 @@ class MythicBeastsProvider(BaseProvider): else: if hasattr(self, '_data_for_{}'.format(_type)): - commands.append('{} {}'.format( - base, values[0])) + for value in values: + commands.append('{} {}'.format(base, value)) else: self.log.debug('skipping %s as not supported', _type) @@ -426,7 +421,7 @@ class MythicBeastsProvider(BaseProvider): def _apply_Create(self, change): zone = change.new.zone - commands = self._compile_commands('ADD', change) + commands = self._compile_commands('ADD', change.new) for command in commands: self._post({ @@ -443,7 +438,7 @@ class MythicBeastsProvider(BaseProvider): def _apply_Delete(self, change): zone = change.existing.zone - commands = self._compile_commands('DELETE', change) + commands = self._compile_commands('DELETE', change.existing) for command in commands: self._post({ diff --git a/tests/test_octodns_provider_mythicbeasts.py b/tests/test_octodns_provider_mythicbeasts.py index 2a3b85b..143d1c7 100644 --- a/tests/test_octodns_provider_mythicbeasts.py +++ b/tests/test_octodns_provider_mythicbeasts.py @@ -156,32 +156,39 @@ class TestMythicBeastsProvider(TestCase): self.assertEquals('Unable to parse CAA data', err.exception.message) - def test_alias_command_generation(self): + def test_command_generation(self): zone = Zone('unit.tests.', []) - zone.add_record(Record.new(zone, 'prawf', { + zone.add_record(Record.new(zone, 'prawf-alias', { 'ttl': 60, 'type': 'ALIAS', 'value': 'alias.unit.tests.', })) - with requests_mock() as mock: - mock.post(ANY, status_code=200, text='') - - provider = MythicBeastsProvider('test', { - 'unit.tests.': 'mypassword' - }) - - plan = provider.plan(zone) - change = plan.changes[0] - - command = provider._compile_commands('ADD', change) - self.assertEquals( - ['ADD prawf.unit.tests 60 ANAME alias.unit.tests.'], - command - ) - - def test_txt_command_generation(self): - zone = Zone('unit.tests.', []) - zone.add_record(Record.new(zone, 'prawf', { + zone.add_record(Record.new(zone, 'prawf-ns', { + 'ttl': 300, + 'type': 'NS', + 'values': [ + 'alias.unit.tests.', + 'alias2.unit.tests.', + ], + })) + zone.add_record(Record.new(zone, 'prawf-a', { + 'ttl': 60, + 'type': 'A', + 'values': [ + '1.2.3.4', + '5.6.7.8', + ], + })) + zone.add_record(Record.new(zone, 'prawf-aaaa', { + 'ttl': 60, + 'type': 'AAAA', + 'values': [ + 'a:a::a', + 'b:b::b', + 'c:c::c:c', + ], + })) + zone.add_record(Record.new(zone, 'prawf-txt', { 'ttl': 60, 'type': 'TXT', 'value': 'prawf prawf dyma prawf', @@ -194,15 +201,35 @@ class TestMythicBeastsProvider(TestCase): }) plan = provider.plan(zone) - change = plan.changes[0] + changes = plan.changes + generated_commands = [] + + for change in changes: + generated_commands.extend( + provider._compile_commands('ADD', change.new) + ) + + expected_commands = [ + 'ADD prawf-alias.unit.tests 60 ANAME alias.unit.tests.', + 'ADD prawf-ns.unit.tests 300 NS alias.unit.tests.', + 'ADD prawf-ns.unit.tests 300 NS alias2.unit.tests.', + 'ADD prawf-a.unit.tests 60 A 1.2.3.4', + 'ADD prawf-a.unit.tests 60 A 5.6.7.8', + 'ADD prawf-aaaa.unit.tests 60 AAAA a:a::a', + 'ADD prawf-aaaa.unit.tests 60 AAAA b:b::b', + 'ADD prawf-aaaa.unit.tests 60 AAAA c:c::c:c', + 'ADD prawf-txt.unit.tests 60 TXT prawf prawf dyma prawf', + ] + + generated_commands.sort() + expected_commands.sort() - command = provider._compile_commands('ADD', change) self.assertEquals( - ['ADD prawf.unit.tests 60 TXT prawf prawf dyma prawf'], - command + generated_commands, + expected_commands ) - def test_command_generation(self): + def test_fake_command_generation(self): class FakeChangeRecord(object): def __init__(self): self.__fqdn = 'prawf.unit.tests.' From 402f645acd43a9b48359cac0bbf05b01e129907a Mon Sep 17 00:00:00 2001 From: Rhosyn Celyn Date: Mon, 17 Jun 2019 18:59:16 +0100 Subject: [PATCH 11/13] Additional test on compile_commands for deletion --- tests/test_octodns_provider_mythicbeasts.py | 30 +++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/test_octodns_provider_mythicbeasts.py b/tests/test_octodns_provider_mythicbeasts.py index 143d1c7..8181562 100644 --- a/tests/test_octodns_provider_mythicbeasts.py +++ b/tests/test_octodns_provider_mythicbeasts.py @@ -229,6 +229,36 @@ class TestMythicBeastsProvider(TestCase): expected_commands ) + # Now test deletion + existing = 'prawf-txt 300 TXT prawf prawf dyma prawf\n' \ + 'prawf-a 60 A 1.2.3.4' + + with requests_mock() as mock: + mock.post(ANY, status_code=200, text=existing) + wanted = Zone('unit.tests.', []) + + plan = provider.plan(wanted) + changes = plan.changes + generated_commands = [] + + for change in changes: + generated_commands.extend( + provider._compile_commands('DELETE', change.existing) + ) + + expected_commands = [ + 'DELETE prawf-a.unit.tests 60 A 1.2.3.4', + 'DELETE prawf-txt.unit.tests 300 TXT prawf prawf dyma prawf', + ] + + generated_commands.sort() + expected_commands.sort() + + self.assertEquals( + generated_commands, + expected_commands + ) + def test_fake_command_generation(self): class FakeChangeRecord(object): def __init__(self): From ddbad2498f6a02ea0edbecc16ecea2bf70b7fab6 Mon Sep 17 00:00:00 2001 From: Rhosyn Celyn Date: Tue, 2 Jul 2019 13:25:30 +0100 Subject: [PATCH 12/13] Fixes for escaping TXT properly, extra test coverage --- octodns/provider/mythicbeasts.py | 22 +++++++++++++++++---- tests/test_octodns_provider_mythicbeasts.py | 20 +++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py index 3470ff1..17029db 100644 --- a/octodns/provider/mythicbeasts.py +++ b/octodns/provider/mythicbeasts.py @@ -174,6 +174,19 @@ class MythicBeastsProvider(BaseProvider): max([raw_values['ttl'] for raw_values in data['raw_values']]), } + @staticmethod + def _data_for_TXT(_type, data): + return { + 'type': _type, + 'values': + [ + str(raw_values['value']).replace(';', '\\;') + for raw_values in data['raw_values'] + ], + 'ttl': + max([raw_values['ttl'] for raw_values in data['raw_values']]), + } + @staticmethod def _data_for_MX(_type, data): ttl = max([raw_values['ttl'] for raw_values in data['raw_values']]) @@ -296,7 +309,6 @@ class MythicBeastsProvider(BaseProvider): {'raw_values': [{'value': value, 'ttl': ttl}]}) _data_for_NS = _data_for_multiple - _data_for_TXT = _data_for_multiple _data_for_A = _data_for_multiple _data_for_AAAA = _data_for_multiple @@ -331,9 +343,6 @@ class MythicBeastsProvider(BaseProvider): _ttl = int(match.group('ttl')) _value = match.group('value').strip() - if _type == 'TXT': - _value = _value.replace(';', '\\;') - if hasattr(self, '_data_for_{}'.format(_type)): if _name not in data[_type]: data[_type][_name] = { @@ -383,6 +392,11 @@ class MythicBeastsProvider(BaseProvider): base = '{} {} {} {}'.format(action, hostname, ttl, _type) + # Unescape TXT records + if _type == 'TXT': + values = [value.replace('\\;', ';') for value in values] + + # Handle specific types or default if _type == 'SSHFP': data = values[0].data commands.append('{} {} {} {}'.format( diff --git a/tests/test_octodns_provider_mythicbeasts.py b/tests/test_octodns_provider_mythicbeasts.py index 8181562..5acbc55 100644 --- a/tests/test_octodns_provider_mythicbeasts.py +++ b/tests/test_octodns_provider_mythicbeasts.py @@ -61,6 +61,18 @@ class TestMythicBeastsProvider(TestCase): self.assertTrue(isinstance(test_multiple, dict)) self.assertEquals(2, len(test_multiple['values'])) + def test_data_for_txt(self): + test_data = { + 'raw_values': [ + {'value': 'v=DKIM1; k=rsa; p=prawf', 'ttl': 60}, + {'value': 'prawf prawf dyma prawf', 'ttl': 300}], + 'zone': 'unit.tests.', + } + test_txt = MythicBeastsProvider._data_for_TXT('', test_data) + self.assertTrue(isinstance(test_txt, dict)) + self.assertEquals(2, len(test_txt['values'])) + self.assertEquals('v=DKIM1\\; k=rsa\\; p=prawf', test_txt['values'][0]) + def test_data_for_MX(self): test_data = { 'raw_values': [ @@ -193,6 +205,11 @@ class TestMythicBeastsProvider(TestCase): 'type': 'TXT', 'value': 'prawf prawf dyma prawf', })) + zone.add_record(Record.new(zone, 'prawf-txt2', { + 'ttl': 60, + 'type': 'TXT', + 'value': 'v=DKIM1\\; k=rsa\\; p=prawf', + })) with requests_mock() as mock: mock.post(ANY, status_code=200, text='') @@ -219,6 +236,7 @@ class TestMythicBeastsProvider(TestCase): 'ADD prawf-aaaa.unit.tests 60 AAAA b:b::b', 'ADD prawf-aaaa.unit.tests 60 AAAA c:c::c:c', 'ADD prawf-txt.unit.tests 60 TXT prawf prawf dyma prawf', + 'ADD prawf-txt2.unit.tests 60 TXT v=DKIM1; k=rsa; p=prawf', ] generated_commands.sort() @@ -231,6 +249,7 @@ class TestMythicBeastsProvider(TestCase): # Now test deletion existing = 'prawf-txt 300 TXT prawf prawf dyma prawf\n' \ + 'prawf-txt2 300 TXT v=DKIM1; k=rsa; p=prawf\n' \ 'prawf-a 60 A 1.2.3.4' with requests_mock() as mock: @@ -249,6 +268,7 @@ class TestMythicBeastsProvider(TestCase): expected_commands = [ 'DELETE prawf-a.unit.tests 60 A 1.2.3.4', 'DELETE prawf-txt.unit.tests 300 TXT prawf prawf dyma prawf', + 'DELETE prawf-txt2.unit.tests 300 TXT v=DKIM1; k=rsa; p=prawf', ] generated_commands.sort() From 6f39ecf261eba24b71dd07e144958361ed855af7 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 19 Jul 2019 08:10:14 -0700 Subject: [PATCH 13/13] Add MythicBeastsProvider to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 163c723..aa9950e 100644 --- a/README.md +++ b/README.md @@ -179,6 +179,7 @@ The above command pulled the existing data out of Route53 and placed the results | [DynProvider](/octodns/provider/dyn.py) | dyn | All | Both | | | [EtcHostsProvider](/octodns/provider/etc_hosts.py) | | A, AAAA, ALIAS, CNAME | No | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | +| [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | | | [Ns1Provider](/octodns/provider/ns1.py) | nsone | All | Partial Geo | No health checking for GeoDNS | | [OVH](/octodns/provider/ovh.py) | ovh | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT, DKIM | No | | | [PowerDnsProvider](/octodns/provider/powerdns.py) | | All | No | |