From be9dbd8ce3eb0de6249cd1e170551d1f68f0072c Mon Sep 17 00:00:00 2001 From: Rhosyn Celyn Date: Tue, 23 Apr 2019 16:25:00 +0100 Subject: [PATCH 001/176] Initial commit for Mythic API --- octodns/provider/mythicbeasts.py | 394 +++++++++++++++++++++++++++++++ 1 file changed, 394 insertions(+) create mode 100644 octodns/provider/mythicbeasts.py diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py new file mode 100644 index 0000000..ed3fed8 --- /dev/null +++ b/octodns/provider/mythicbeasts.py @@ -0,0 +1,394 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from requests import HTTPError, Session +from logging import getLogger + +from ..record import Create, Record +from .base import BaseProvider + +import re + +import pprint +import sys + +def add_trailing_dot(s): + assert s + assert s[-1] != '.' + return s + '.' + + +def remove_trailing_dot(s): + assert s + assert s[-1] == '.' + return s[:-1] + + +class MythicBeastsProvider(BaseProvider): + ''' + Mythic Beasts DNS API Provider + + mythicbeasts: + class: octodns.provider.mythicbeasts.MythicBeastsProvider + zones: + my-zone: 'password' + + ''' + + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = False + SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', + 'SRV', 'TXT')) + BASE = 'https://dnsapi.mythic-beasts.com/' + TIMEOUT = 15 + + def __init__(self, id, passwords, *args, **kwargs): + self.log = getLogger('MythicBeastsProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, registered zones; %s', id, passwords.keys()) + super(MythicBeastsProvider, self).__init__(id, *args, **kwargs) + + self._passwords = passwords + sess = Session() + self._sess = sess + + def _request(self, method, path, data=None): + self.log.debug('_request: method=%s, path=%s', method, path) + + url = self.BASE + resp = self._sess.request(method, url, data=data, timeout=self.TIMEOUT) + self.log.debug('_request: status=%d', resp.status_code) + resp.raise_for_status() + return resp + + def _post(self, data=None): + return self._request('POST', self.BASE, data=data) + + def records(self, zone): + return self._post({ + 'domain': remove_trailing_dot(zone), + 'password': self._passwords[zone], + 'command': 'LIST', + }) + + def _data_for_single(self, _type, data): + return { + 'type': _type, + 'value': data['raw_values'][0]['value'], + 'ttl': data['raw_values'][0]['ttl'] + } + + def _data_for_multiple(self, _type, data): + return { + 'type': _type, + 'values': [raw_values['value'] for raw_values in data['raw_values']], + 'ttl': max([raw_values['ttl'] for raw_values in data['raw_values']]), + } + + def _data_for_MX(self, _type, data): + ttl = max([raw_values['ttl'] for raw_values in data['raw_values']]) + values = [] + + for raw_value in [raw_values['value'] for raw_values in data['raw_values']]: + match = re.match('^([0-9]+)\s+(\S+)$', raw_value, re.IGNORECASE) + + if match is not None: + exchange = match.group(2) + + if not exchange.endswith('.'): + exchange = '{}.{}'.format(exchange, data['zone']) + + values.append({ + 'preference': match.group(1), + 'exchange': exchange, + }) + + return { + 'type': _type, + 'values': values, + 'ttl': ttl, + } + + def _data_for_CNAME(self, _type, data): + ttl = data['raw_values'][0]['ttl'] + value = data['raw_values'][0]['value'] + if not value.endswith('.'): + value = '{}.{}'.format(value, data['zone']) + + return self._data_for_single(_type, {'raw_values': [ {'value': value, 'ttl': ttl} ]}) + + def _data_for_ANAME(self, _type, data): + ttl = data['raw_values'][0]['ttl'] + value = data['raw_values'][0]['value'] + return self._data_for_single('ALIAS', {'raw_values': [ {'value': value, 'ttl': ttl} ]}) + + + def _data_for_SRV(self, _type, data): + ttl = data['raw_values'][0]['ttl'] + raw_value = data['raw_values'][0]['value'] + + match = re.match('^([0-9]+)\s+([0-9]+)\s+([0-9]+)\s+(\S+)$', raw_value, re.IGNORECASE) + + if match is not None: + target = match.group(4) + if not target.endswith('.'): + target = '{}.{}'.format(target, data['zone']) + + value = { + 'priority': match.group(1), + 'weight': match.group(2), + 'port': match.group(3), + 'target': target, + } + + return self._data_for_single('SRV', {'raw_values': [ {'value': value, 'ttl': ttl} ]}) + + def _data_for_SSHFP(self, _type, data): + ttl = data['raw_values'][0]['ttl'] + raw_value = data['raw_values'][0]['value'] + + match = re.match('^([0-9]+)\s+([0-9]+)\s+(\S+)$', raw_value, re.IGNORECASE) + + if match is not None: + value = { + 'algorithm': match.group(1), + 'fingerprint_type': match.group(2), + 'fingerprint': match.group(3), + } + + return self._data_for_single('SSHFP', {'raw_values': [ {'value': value, 'ttl': ttl} ]}) + + + # TODO fix bug with CAA output from API + ''' + def _data_for_CAA(self, _type, data): + ttl = data['raw_values'][0]['ttl'] + + match = re.match('^()$', re.IGNORECASE) + + value = { + 'flags': + 'tag': + 'value': + } + value = data['raw_values'][0]['value'] + return self._data_for_single('ALIAS', {'raw_values': [ {'value': value, 'ttl': ttl} ]}) + ''' + + + _data_for_NS = _data_for_multiple + _data_for_TXT = _data_for_multiple + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + + resp = None + try: + resp = self.records(zone.name) + except HTTPError as e: + if e.response.status_code == 401: + # Nicer error message for auth problems + raise Exception('Mythic Beasts authentication problem with %s'.format(zone.name)) + elif e.response.status_code == 422: + # 422 means mythicbeasts doesn't know anything about the requested + # domain. We'll just ignore it here and leave the zone + # untouched. + raise + else: + # just re-throw + raise + + before = len(zone.records) + exists = False + data = dict() + + if resp: + exists = True + for line in resp.content.splitlines(): + match = re.match('^(\S+)\s+(\S+)\s+(\S+)\s+(.*)$', line, re.IGNORECASE) + + if match is not None: + if match.group(1) == '@': + _name = '' + else: + _name = match.group(1) + + _type = match.group(3) + _ttl = int(match.group(2)) + _value = match.group(4).strip() + + if _type == 'SOA': + continue + + try: + if getattr(self, '_data_for_{}'.format(_type)) is not None: + + if _type not in data: + data[_type] = dict() + + if _name not in data[_type]: + data[_type][_name] = { + 'raw_values': [{'value': _value, 'ttl': _ttl}], + 'name': _name, + 'zone': zone.name, + } + + else: + data[_type][_name].get('raw_values').append( + {'value': _value, 'ttl': _ttl} + ) + except AttributeError as error: + self.log.debug('skipping {} as not supported', _type) + continue + + + for _type in data: + for _name in data[_type]: + data_for = getattr(self, '_data_for_{}'.format(_type)) + + record = Record.new( + zone, + _name, + data_for(_type, data[_type][_name]), + source=self, + ) + zone.add_record(record, lenient=lenient) + + + self.log.debug('populate: found %s records, exists=%s', + len(zone.records) - before, exists) + return exists + + + def _compile_commands(self, action, change): + commands = [] + + record = None + + if action == 'ADD': + record = change.new + + elif action == 'DELETE': + record = change.existing + + zone = record.zone + hostname = remove_trailing_dot(record.fqdn) + ttl = record.ttl + _type = record._type + + if hostname == '': + hostname = '@' + if _type == 'ALIAS': + _type = 'ANAME' + + + if hasattr(record, 'values'): + values = record.values + else: + values = [record.value] + + + base = '{} {} {} {}'.format(action, hostname, ttl, _type) + + if re.match('[A]{1,4}', _type) is not None: + for value in values: + commands.append('{} {}'.format(base, value)) + + elif _type == 'SSHFP': + data = values[0].data + commands.append('{} {} {} {}'.format( + base, data['algorithm'], data['fingerprint_type'], data['fingerprint'])) + + elif _type == 'SRV': + data = values[0].data + commands.append('{} {} {} {} {}'.format( + base, data['priority'], data['weight'], data['port'], data['target'])) + + elif _type == 'MX': + for value in values: + data = value.data + commands.append('{} {} {}'.format( + base, data['preference'], data['exchange'])) + + else: + try: + if getattr(self, '_data_for_{}'.format(_type)) is not None: + commands.append('{} {}'.format( + base, values[0])) + except AttributeError as error: + self.log.debug('skipping {} as not supported', _type) + pass + + return commands + + def _apply_Create(self, change): + + pp = pprint.PrettyPrinter(depth=10, stream=sys.stderr) + + zone = change.new.zone + commands = self._compile_commands('ADD', change) + pp.pprint(commands) + + for command in commands: + self._post({ + 'domain': remove_trailing_dot(zone.name), + 'origin': '.', + 'password': self._passwords[zone.name], + 'command': command, + }) + pp.pprint({ + 'domain': remove_trailing_dot(zone.name), + 'origin': '.', + 'password': self._passwords[zone.name], + 'command': command, + }) + return True + + def _apply_Update(self, change): + self._apply_Delete(change) + self._apply_Create(change) + + def _apply_Delete(self, change): + + pp = pprint.PrettyPrinter(depth=10, stream=sys.stderr) + + zone = change.existing.zone + commands = self._compile_commands('DELETE', change) + pp.pprint(commands) + + for command in commands: + self._post({ + 'domain': remove_trailing_dot(zone.name), + 'origin': '.', + 'password': self._passwords[zone.name], + 'command': command, + }) + pp.pprint({ + 'domain': remove_trailing_dot(zone.name), + 'origin': '.', + 'password': self._passwords[zone.name], + 'command': command, + }) + return True + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + + domain_name = desired.name + + for change in changes: + class_name = change.__class__.__name__ + getattr(self, '_apply_{}'.format(class_name))(change) + + + From 2df87d7dfe285eaa5330f713943df448357fe5f5 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 29 Apr 2019 08:52:56 -0700 Subject: [PATCH 002/176] Improve Route53 DELETE reliability using existing rrset --- octodns/provider/route53.py | 51 +++++++++----- tests/test_octodns_provider_route53.py | 97 ++++++++++++++++++++++---- 2 files changed, 116 insertions(+), 32 deletions(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index f5185b0..a8b47b8 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -136,7 +136,7 @@ class _Route53Record(object): values_for = getattr(self, '_values_for_{}'.format(self._type)) self.values = values_for(record) - def mod(self, action): + def mod(self, action, existing_rrsets): return { 'Action': action, 'ResourceRecordSet': { @@ -268,7 +268,7 @@ class _Route53DynamicPool(_Route53Record): self.target_name) return '{}-{}'.format(self.pool_name, self.mode) - def mod(self, action): + def mod(self, action, existing_rrsets): return { 'Action': action, 'ResourceRecordSet': { @@ -311,7 +311,7 @@ class _Route53DynamicRule(_Route53Record): def identifer(self): return '{}-{}-{}'.format(self.index, self.pool_name, self.geo) - def mod(self, action): + def mod(self, action, existing_rrsets): rrset = { 'AliasTarget': { 'DNSName': self.target_dns_name, @@ -379,7 +379,7 @@ class _Route53DynamicValue(_Route53Record): def identifer(self): return '{}-{:03d}'.format(self.pool_name, self.index) - def mod(self, action): + def mod(self, action, existing_rrsets): return { 'Action': action, 'ResourceRecordSet': { @@ -404,7 +404,7 @@ class _Route53DynamicValue(_Route53Record): class _Route53GeoDefault(_Route53Record): - def mod(self, action): + def mod(self, action, existing_rrsets): return { 'Action': action, 'ResourceRecordSet': { @@ -437,15 +437,29 @@ class _Route53GeoRecord(_Route53Record): self.health_check_id = provider.get_health_check_id(record, value, creating) - def mod(self, action): + def mod(self, action, existing_rrsets): geo = self.geo + set_identifier = geo.code + + if action == 'DELETE': + # We deleting records try and find the original rrset so that we're + # 100% sure to have the complete & accurate data (this mostly + # ensures we have the right health check id when there's multiple + # potential matches) + for existing in existing_rrsets: + if set_identifier == existing.get('SetIdentifier', None): + return { + 'Action': action, + 'ResourceRecordSet': existing, + } + rrset = { 'Name': self.fqdn, 'GeoLocation': { 'CountryCode': '*' }, 'ResourceRecords': [{'Value': v} for v in geo.values], - 'SetIdentifier': geo.code, + 'SetIdentifier': set_identifier, 'TTL': self.ttl, 'Type': self._type, } @@ -927,11 +941,11 @@ class Route53Provider(BaseProvider): len(zone.records) - before, exists) return exists - def _gen_mods(self, action, records): + def _gen_mods(self, action, records, existing_rrsets): ''' Turns `_Route53*`s in to `change_resource_record_sets` `Changes` ''' - return [r.mod(action) for r in records] + return [r.mod(action, existing_rrsets) for r in records] @property def health_checks(self): @@ -1117,15 +1131,15 @@ class Route53Provider(BaseProvider): ''' return _Route53Record.new(self, record, zone_id, creating) - def _mod_Create(self, change, zone_id): + def _mod_Create(self, change, zone_id, existing_rrsets): # New is the stuff that needs to be created new_records = self._gen_records(change.new, zone_id, creating=True) # Now is a good time to clear out any unused health checks since we # know what we'll be using going forward self._gc_health_checks(change.new, new_records) - return self._gen_mods('CREATE', new_records) + return self._gen_mods('CREATE', new_records, existing_rrsets) - def _mod_Update(self, change, zone_id): + def _mod_Update(self, change, zone_id, existing_rrsets): # See comments in _Route53Record for how the set math is made to do our # bidding here. existing_records = self._gen_records(change.existing, zone_id, @@ -1148,18 +1162,18 @@ class Route53Provider(BaseProvider): if new_record in existing_records: upserts.add(new_record) - return self._gen_mods('DELETE', deletes) + \ - self._gen_mods('CREATE', creates) + \ - self._gen_mods('UPSERT', upserts) + return self._gen_mods('DELETE', deletes, existing_rrsets) + \ + self._gen_mods('CREATE', creates, existing_rrsets) + \ + self._gen_mods('UPSERT', upserts, existing_rrsets) - def _mod_Delete(self, change, zone_id): + def _mod_Delete(self, change, zone_id, existing_rrsets): # Existing is the thing that needs to be deleted existing_records = self._gen_records(change.existing, zone_id, creating=False) # Now is a good time to clear out all the health checks since we know # we're done with them self._gc_health_checks(change.existing, []) - return self._gen_mods('DELETE', existing_records) + return self._gen_mods('DELETE', existing_records, existing_rrsets) def _extra_changes_update_needed(self, record, rrset): healthcheck_host = record.healthcheck_host @@ -1271,10 +1285,11 @@ class Route53Provider(BaseProvider): batch = [] batch_rs_count = 0 zone_id = self._get_zone_id(desired.name, True) + existing_rrsets = self._load_records(zone_id) for c in changes: # Generate the mods for this change mod_type = getattr(self, '_mod_{}'.format(c.__class__.__name__)) - mods = mod_type(c, zone_id) + mods = mod_type(c, zone_id, existing_rrsets) # Order our mods to make sure targets exist before alises point to # them and we CRUD in the desired order diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 67fcb76..4a4f90a 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -874,6 +874,25 @@ class TestRoute53Provider(TestCase): 'CallerReference': ANY, }) + list_resource_record_sets_resp = { + 'ResourceRecordSets': [{ + 'Name': 'a.unit.tests.', + 'Type': 'A', + 'GeoLocation': { + 'ContinentCode': 'NA', + }, + 'ResourceRecords': [{ + 'Value': '2.2.3.4', + }], + 'TTL': 61, + }], + 'IsTruncated': False, + 'MaxItems': '100', + } + stubber.add_response('list_resource_record_sets', + list_resource_record_sets_resp, + {'HostedZoneId': 'z42'}) + stubber.add_response('list_health_checks', { 'HealthChecks': self.health_checks, @@ -1236,7 +1255,7 @@ class TestRoute53Provider(TestCase): 'HealthCheckId': '44', }) change = Create(record) - provider._mod_Create(change, 'z43') + provider._mod_Create(change, 'z43', []) stubber.assert_no_pending_responses() # gc through _mod_Update @@ -1245,7 +1264,7 @@ class TestRoute53Provider(TestCase): }) # first record is ignored for our purposes, we have to pass something change = Update(record, record) - provider._mod_Create(change, 'z43') + provider._mod_Create(change, 'z43', []) stubber.assert_no_pending_responses() # gc through _mod_Delete, expect 3 to go away, can't check order @@ -1260,7 +1279,7 @@ class TestRoute53Provider(TestCase): 'HealthCheckId': ANY, }) change = Delete(record) - provider._mod_Delete(change, 'z43') + provider._mod_Delete(change, 'z43', []) stubber.assert_no_pending_responses() # gc only AAAA, leave the A's alone @@ -1804,40 +1823,45 @@ class TestRoute53Provider(TestCase): # _get_test_plan() returns a plan with 11 modifications, 17 RRs + @patch('octodns.provider.route53.Route53Provider._load_records') @patch('octodns.provider.route53.Route53Provider._really_apply') - def test_apply_1(self, really_apply_mock): + def test_apply_1(self, really_apply_mock, _): # 18 RRs with max of 19 should only get applied in one call provider, plan = self._get_test_plan(19) provider.apply(plan) really_apply_mock.assert_called_once() + @patch('octodns.provider.route53.Route53Provider._load_records') @patch('octodns.provider.route53.Route53Provider._really_apply') - def test_apply_2(self, really_apply_mock): + def test_apply_2(self, really_apply_mock, _): # 18 RRs with max of 17 should only get applied in two calls provider, plan = self._get_test_plan(18) provider.apply(plan) self.assertEquals(2, really_apply_mock.call_count) + @patch('octodns.provider.route53.Route53Provider._load_records') @patch('octodns.provider.route53.Route53Provider._really_apply') - def test_apply_3(self, really_apply_mock): + def test_apply_3(self, really_apply_mock, _): # with a max of seven modifications, four calls provider, plan = self._get_test_plan(7) provider.apply(plan) self.assertEquals(4, really_apply_mock.call_count) + @patch('octodns.provider.route53.Route53Provider._load_records') @patch('octodns.provider.route53.Route53Provider._really_apply') - def test_apply_4(self, really_apply_mock): + def test_apply_4(self, really_apply_mock, _): # with a max of 11 modifications, two calls provider, plan = self._get_test_plan(11) provider.apply(plan) self.assertEquals(2, really_apply_mock.call_count) + @patch('octodns.provider.route53.Route53Provider._load_records') @patch('octodns.provider.route53.Route53Provider._really_apply') - def test_apply_bad(self, really_apply_mock): + def test_apply_bad(self, really_apply_mock, _): # with a max of 1 modifications, fail provider, plan = self._get_test_plan(1) @@ -1939,6 +1963,12 @@ class TestRoute53Provider(TestCase): }], [r.data for r in record.dynamic.rules]) +class DummyProvider(object): + + def get_health_check_id(self, *args, **kwargs): + return None + + class TestRoute53Records(TestCase): existing = Zone('unit.tests.', []) record_a = Record.new(existing, '', { @@ -2005,11 +2035,6 @@ class TestRoute53Records(TestCase): e = _Route53GeoDefault(None, self.record_a, False) self.assertNotEquals(a, e) - class DummyProvider(object): - - def get_health_check_id(self, *args, **kwargs): - return None - provider = DummyProvider() f = _Route53GeoRecord(provider, self.record_a, 'NA-US', self.record_a.geo['NA-US'], False) @@ -2029,6 +2054,50 @@ class TestRoute53Records(TestCase): e.__repr__() f.__repr__() + def test_geo_delete(self): + provider = DummyProvider() + geo = _Route53GeoRecord(provider, self.record_a, 'NA-US', + self.record_a.geo['NA-US'], False) + + rrset = { + 'GeoLocation': { + 'CountryCode': 'US' + }, + 'HealthCheckId': 'x12346z', + 'Name': 'unit.tests.', + 'ResourceRecords': [{ + 'Value': '2.2.2.2' + }, { + 'Value': '3.3.3.3' + }], + 'SetIdentifier': 'NA-US', + 'TTL': 99, + 'Type': 'A' + } + + candidates = [ + # Empty, will test no SetIdentifier + {}, + { + 'SetIdentifier': 'not-a-match', + }, + rrset, + ] + + # Provide a matching rrset so that we'll just use it for the delete + # rathr than building up an almost identical one, note the way we'll + # know that we got the one we passed in is that it'll have a + # HealthCheckId and one that was created wouldn't since DummyProvider + # stubs out the lookup for them + mod = geo.mod('DELETE', candidates) + self.assertEquals('x12346z', mod['ResourceRecordSet']['HealthCheckId']) + + # If we don't provide the candidate rrsets we get back exactly what we + # put in minus the healthcheck + del rrset['HealthCheckId'] + mod = geo.mod('DELETE', []) + self.assertEquals(rrset, mod['ResourceRecordSet']) + def test_new_dynamic(self): provider = Route53Provider('test', 'abc', '123') @@ -2259,7 +2328,7 @@ class TestRoute53Records(TestCase): 'Name': '_octodns-eu-central-1-pool.unit.tests.', 'SetIdentifier': 'eu-central-1-Secondary-us-east-1', 'Type': 'A'} - }], [r.mod('CREATE') for r in route53_records]) + }], [r.mod('CREATE', []) for r in route53_records]) for route53_record in route53_records: # Smoke test stringification From e4fbcf109007c9e42ffa0ba0242c565d1754dbaf Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 29 Apr 2019 09:02:45 -0700 Subject: [PATCH 003/176] Improved Route53Provider deltion for Dynamic Value too --- octodns/provider/route53.py | 17 ++++++++-- tests/test_octodns_provider_route53.py | 43 +++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index a8b47b8..c2567bb 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -380,6 +380,19 @@ class _Route53DynamicValue(_Route53Record): return '{}-{:03d}'.format(self.pool_name, self.index) def mod(self, action, existing_rrsets): + + if action == 'DELETE': + # When deleting records try and find the original rrset so that + # we're 100% sure to have the complete & accurate data (this mostly + # ensures we have the right health check id when there's multiple + # potential matches) + for existing in existing_rrsets: + if self.identifer == existing.get('SetIdentifier', None): + return { + 'Action': action, + 'ResourceRecordSet': existing, + } + return { 'Action': action, 'ResourceRecordSet': { @@ -442,8 +455,8 @@ class _Route53GeoRecord(_Route53Record): set_identifier = geo.code if action == 'DELETE': - # We deleting records try and find the original rrset so that we're - # 100% sure to have the complete & accurate data (this mostly + # When deleting records try and find the original rrset so that + # we're 100% sure to have the complete & accurate data (this mostly # ensures we have the right health check id when there's multiple # potential matches) for existing in existing_rrsets: diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 4a4f90a..a5d00e2 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -12,7 +12,8 @@ from mock import patch from octodns.record import Create, Delete, Record, Update from octodns.provider.route53 import Route53Provider, _Route53GeoDefault, \ - _Route53GeoRecord, _Route53Record, _mod_keyer, _octal_replace + _Route53DynamicValue, _Route53GeoRecord, _Route53Record, _mod_keyer, \ + _octal_replace from octodns.zone import Zone from helpers import GeoProvider @@ -2054,6 +2055,46 @@ class TestRoute53Records(TestCase): e.__repr__() f.__repr__() + def test_dynamic_value_delete(self): + provider = DummyProvider() + geo = _Route53DynamicValue(provider, self.record_a, 'iad', '2.2.2.2', + 1, 0, False) + + rrset = { + 'HealthCheckId': 'x12346z', + 'Name': '_octodns-iad-value.unit.tests.', + 'ResourceRecords': [{ + 'Value': '2.2.2.2' + }], + 'SetIdentifier': 'iad-000', + 'TTL': 99, + 'Type': 'A', + 'Weight': 1, + } + + candidates = [ + # Empty, will test no SetIdentifier + {}, + { + 'SetIdentifier': 'not-a-match', + }, + rrset, + ] + + # Provide a matching rrset so that we'll just use it for the delete + # rathr than building up an almost identical one, note the way we'll + # know that we got the one we passed in is that it'll have a + # HealthCheckId and one that was created wouldn't since DummyProvider + # stubs out the lookup for them + mod = geo.mod('DELETE', candidates) + self.assertEquals('x12346z', mod['ResourceRecordSet']['HealthCheckId']) + + # If we don't provide the candidate rrsets we get back exactly what we + # put in minus the healthcheck + rrset['HealthCheckId'] = None + mod = geo.mod('DELETE', []) + self.assertEquals(rrset, mod['ResourceRecordSet']) + def test_geo_delete(self): provider = DummyProvider() geo = _Route53GeoRecord(provider, self.record_a, 'NA-US', From 1c60ed018bb7c5f3ae4b9e90747d8d2f57a1ea9a Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 29 Apr 2019 09:49:40 -0700 Subject: [PATCH 004/176] Make sure both set-id and name match when finding rrset --- octodns/provider/route53.py | 7 +++++-- tests/test_octodns_provider_route53.py | 11 +++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index c2567bb..e8b11fa 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -387,7 +387,8 @@ class _Route53DynamicValue(_Route53Record): # ensures we have the right health check id when there's multiple # potential matches) for existing in existing_rrsets: - if self.identifer == existing.get('SetIdentifier', None): + if self.fqdn == existing.get('Name') and \ + self.identifer == existing.get('SetIdentifier', None): return { 'Action': action, 'ResourceRecordSet': existing, @@ -453,6 +454,7 @@ class _Route53GeoRecord(_Route53Record): def mod(self, action, existing_rrsets): geo = self.geo set_identifier = geo.code + fqdn = self.fqdn if action == 'DELETE': # When deleting records try and find the original rrset so that @@ -460,7 +462,8 @@ class _Route53GeoRecord(_Route53Record): # ensures we have the right health check id when there's multiple # potential matches) for existing in existing_rrsets: - if set_identifier == existing.get('SetIdentifier', None): + if fqdn == existing.get('Name') and \ + set_identifier == existing.get('SetIdentifier', None): return { 'Action': action, 'ResourceRecordSet': existing, diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index a5d00e2..added7f 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -2075,9 +2075,15 @@ class TestRoute53Records(TestCase): candidates = [ # Empty, will test no SetIdentifier {}, + # Non-matching { 'SetIdentifier': 'not-a-match', }, + # Same set-id, different name + { + 'Name': 'not-a-match', + 'SetIdentifier': 'x12346z', + }, rrset, ] @@ -2122,6 +2128,11 @@ class TestRoute53Records(TestCase): { 'SetIdentifier': 'not-a-match', }, + # Same set-id, different name + { + 'Name': 'not-a-match', + 'SetIdentifier': 'x12346z', + }, rrset, ] From b670a0695f22425c1deed8e32ac3f8b12dc5193c Mon Sep 17 00:00:00 2001 From: Rhosyn Celyn Date: Wed, 1 May 2019 15:49:33 +0100 Subject: [PATCH 005/176] Updated tweaks and bug fixes for ANAME/ALIAS --- octodns/provider/mythicbeasts.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py index ed3fed8..f416a1b 100644 --- a/octodns/provider/mythicbeasts.py +++ b/octodns/provider/mythicbeasts.py @@ -43,7 +43,8 @@ class MythicBeastsProvider(BaseProvider): SUPPORTS_DYNAMIC = False SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'SRV', 'TXT')) - BASE = 'https://dnsapi.mythic-beasts.com/' + #BASE = 'https://dnsapi.mythic-beasts.com/' + BASE = 'https://cwningen.dev.mythic-beasts.com/customer/primarydnsapi' TIMEOUT = 15 def __init__(self, id, passwords, *args, **kwargs): @@ -71,6 +72,7 @@ class MythicBeastsProvider(BaseProvider): return self._post({ 'domain': remove_trailing_dot(zone), 'password': self._passwords[zone], + 'showall': 0, 'command': 'LIST', }) @@ -195,7 +197,7 @@ class MythicBeastsProvider(BaseProvider): except HTTPError as e: if e.response.status_code == 401: # Nicer error message for auth problems - raise Exception('Mythic Beasts authentication problem with %s'.format(zone.name)) + raise Exception('Mythic Beasts authentication problem with {}'.format(zone.name)) elif e.response.status_code == 422: # 422 means mythicbeasts doesn't know anything about the requested # domain. We'll just ignore it here and leave the zone From 470daa8843aa77c84924b69ce08493fc6f89cd8a Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 3 May 2019 09:16:06 -0700 Subject: [PATCH 006/176] Pass at documenting dynamic record support Bit of general doc cleanup and refactoring while I'm in here. --- docs/dynamic_records.md | 126 ++++++++++++++++++++++++++++++++++++++++ docs/geo_records.md | 101 ++++++++++++++++++++++++++++++++ docs/records.md | 102 +------------------------------- 3 files changed, 230 insertions(+), 99 deletions(-) create mode 100644 docs/dynamic_records.md create mode 100644 docs/geo_records.md diff --git a/docs/dynamic_records.md b/docs/dynamic_records.md new file mode 100644 index 0000000..8a7cd09 --- /dev/null +++ b/docs/dynamic_records.md @@ -0,0 +1,126 @@ +## Dynamic Record Support + +Dynamic records provide support for GeoDNS and weighting to records. `A` and `AAAA` are fully supported and reasonably well tested for both Dyn (via Traffic Directors) and Route53. There is preliminary support for `CNAME` records, but caution should be exercised as they have not been thoroughly tested. + +Configuring GeoDNS is complex and the details of the functionality vary widely from provider to provider. octoDNS has an opinionated view mostly to give a reasonably consistent behavior across providers which is similar to the overall philosophy and approach of octoDNS itself. It may not fit your needs or use cases, in which case please open an issue for discussion. We expect this functionality to grow and evolve over time as it's more widely used. + +### An Annotated Example + +```yaml + +--- +test: + # This is a dynamic record when used with providers that support it + dynamic: + # These are the pools of records that can be referenced and thus used by rules + pools: + apac: + # An optional fallback, if all of the records in this pool fail this pool should be tried + fallback: na + # One or more values for this pool + values: + - value: 1.1.1.1 + - value: 2.2.2.2 + eu: + fallback: na + values: + - value: 3.3.3.3 + # Weight for this value, if omitted the default is 1 + weight: 2 + - value: 4.4.4.4 + weight: 3 + na: + # Implicit fallback to the default pool (below) + values: + - value: 5.5.5.5 + - value: 6.6.6.6 + - value: 7.7.7.7 + # Rules that assign queries to pools + rules: + - geos: + # Geos used in matching queries + - AS + - OC + # The pool to service the query from + pool: apac + - geos: + - AF + - EU + pool: eu + # No geos means match all queries + - pool: na + ttl: 60 + type: A + # These values become a non-healthchecked default pool + values: + - 5.5.5.5 + - 6.6.6.6 + - 7.7.7.7 +``` + +#### Geo Codes + +Geo codes consist of one to three parts depending on the scope of the area being targeted. Examples of these look like: + +* 'NA-US-KY' - North America, United States, Kentucky +* 'NA-US' - North America, United States +* 'NA' - North America + +The first portion is the continent: + +* 'AF': 14, # Continental Africa +* 'AN': 17, # Continental Antarctica +* 'AS': 15, # Continental Asia +* 'EU': 13, # Continental Europe +* 'NA': 11, # Continental North America +* 'OC': 16, # Continental Australia/Oceania +* 'SA': 12, # Continental South America + +The second is the two-letter ISO Country Code https://en.wikipedia.org/wiki/ISO_3166-2 and the third is the ISO Country Code Subdivision as per https://en.wikipedia.org/wiki/ISO_3166-2:US. Change the code at the end for the country you are subdividing. Note that these may not always be supported depending on the providers in use. + +### Health Checks + +octoDNS will automatically configure the provider to monitor each IP and check for a 200 response for **https:///_dns**. + +These checks can be customized via the `healthcheck` configuration options. + +```yaml + +--- +test: + ... + octodns: + healthcheck: + host: my-host-name + path: /dns-health-check + port: 443 + protocol: HTTPS + ... +``` + +| Key | Description | Default | +|--|--|--| +| host | FQDN for host header and SNI | - | +| path | path to check | _dns | +| port | port to check | 443 | +| protocol | HTTP/HTTPS | HTTPS | + +#### Route53 Healtch Check Options + +| Key | Description | Default | +|--|--|--| +| measure_latency | Show latency in AWS console | true | + +```yaml + +--- + octodns: + healthcheck: + host: my-host-name + path: /dns-health-check + port: 443 + protocol: HTTPS + route53: + healthcheck: + measure_latency: false +``` diff --git a/docs/geo_records.md b/docs/geo_records.md new file mode 100644 index 0000000..e365f57 --- /dev/null +++ b/docs/geo_records.md @@ -0,0 +1,101 @@ +## Geo Record Support + +Note: Geo DNS records are still supported for the time being, but it is still strongy encouraged that you look at [Dynamic Records](/docs/dynamic_records.md) instead as they are a superset of functionality. + +GeoDNS is currently supported for `A` and `AAAA` records on the Dyn (via Traffic Directors) and Route53 providers. Records with geo information pushed to providers without support for them will be managed as non-geo records using the base values. + +Configuring GeoDNS is complex and the details of the functionality vary widely from provider to provider. OctoDNS has an opinionated view of how GeoDNS should be set up and does its best to map that to each provider's offering in a way that will result in similar behavior. It may not fit your needs or use cases, in which case please open an issue for discussion. We expect this functionality to grow and evolve over time as it's more widely used. + +The following is an example of GeoDNS with three entries NA-US-CA, NA-US-NY, OC-AU. Octodns creates another one labeled 'default' with the details for the actual A record, This default record is the failover record if the monitoring check fails. + +```yaml +--- +? '' +: type: TXT + value: v=spf1 -all +test: + geo: + NA-US-NY: + - 111.111.111.1 + NA-US-CA: + - 111.111.111.2 + OC-AU: + - 111.111.111.3 + EU: + - 111.111.111.4 + ttl: 300 + type: A + value: 111.111.111.5 +``` + + +The geo labels breakdown based on: + +1. + - 'AF': 14, # Continental Africa + - 'AN': 17, # Continental Antarctica + - 'AS': 15, # Continental Asia + - 'EU': 13, # Continental Europe + - 'NA': 11, # Continental North America + - 'OC': 16, # Continental Australia/Oceania + - 'SA': 12, # Continental South America + +2. ISO Country Code https://en.wikipedia.org/wiki/ISO_3166-2 + +3. ISO Country Code Subdivision as per https://en.wikipedia.org/wiki/ISO_3166-2:US (change the code at the end for the country you are subdividing) * these may not always be supported depending on the provider. + +So the example is saying: + +- North America - United States - New York: gets served an "A" record of 111.111.111.1 +- North America - United States - California: gets served an "A" record of 111.111.111.2 +- Oceania - Australia: Gets served an "A" record of 111.111.111.3 +- Europe: gets an "A" record of 111.111.111.4 +- Everyone else gets an "A" record of 111.111.111.5 + +### Health Checks + +Octodns will automatically set up monitors check for a 200 response for **https:///_dns**. + +These checks can be configured by adding a `healthcheck` configuration to the record: + +```yaml +--- +test: + geo: + AS: + - 1.2.3.4 + EU: + - 2.3.4.5 + octodns: + healthcheck: + host: my-host-name + path: /dns-health-check + port: 443 + protocol: HTTPS +``` + +| Key | Description | Default | +|--|--|--| +| host | FQDN for host header and SNI | - | +| path | path to check | _dns | +| port | port to check | 443 | +| protocol | HTTP/HTTPS | HTTPS | + +#### Route53 Healtch Check Options + +| Key | Description | Default | +|--|--|--| +| measure_latency | Show latency in AWS console | true | + +```yaml +--- + octodns: + healthcheck: + host: my-host-name + path: /dns-health-check + port: 443 + protocol: HTTPS + route53: + healthcheck: + measure_latency: false +``` diff --git a/docs/records.md b/docs/records.md index 9b494cf..c7ede69 100644 --- a/docs/records.md +++ b/docs/records.md @@ -20,106 +20,10 @@ Underlying provider support for each of these varies and some providers have ext Adding new record types to OctoDNS is relatively straightforward, but will require careful evaluation of each provider to determine whether or not it will be supported and the addition of code in each to handle and test the new type. -## GeoDNS support - -GeoDNS is currently supported for `A` and `AAAA` records on the Dyn (via Traffic Directors) and Route53 providers. Records with geo information pushed to providers without support for them will be managed as non-geo records using the base values. - -Configuring GeoDNS is complex and the details of the functionality vary widely from provider to provider. OctoDNS has an opinionated view of how GeoDNS should be set up and does its best to map that to each provider's offering in a way that will result in similar behavior. It may not fit your needs or use cases, in which case please open an issue for discussion. We expect this functionality to grow and evolve over time as it's more widely used. - -The following is an example of GeoDNS with three entries NA-US-CA, NA-US-NY, OC-AU. Octodns creates another one labeled 'default' with the details for the actual A record, This default record is the failover record if the monitoring check fails. - -```yaml ---- -? '' -: type: TXT - value: v=spf1 -all -test: - geo: - NA-US-NY: - - 111.111.111.1 - NA-US-CA: - - 111.111.111.2 - OC-AU: - - 111.111.111.3 - EU: - - 111.111.111.4 - ttl: 300 - type: A - value: 111.111.111.5 -``` - - -The geo labels breakdown based on: - -1. - - 'AF': 14, # Continental Africa - - 'AN': 17, # Continental Antarctica - - 'AS': 15, # Continental Asia - - 'EU': 13, # Continental Europe - - 'NA': 11, # Continental North America - - 'OC': 16, # Continental Australia/Oceania - - 'SA': 12, # Continental South America - -2. ISO Country Code https://en.wikipedia.org/wiki/ISO_3166-2 - -3. ISO Country Code Subdevision as per https://en.wikipedia.org/wiki/ISO_3166-2:US (change the code at the end for the country you are subdividing) * these may not always be supported depending on the provider. - -So the example is saying: - -- North America - United States - New York: gets served an "A" record of 111.111.111.1 -- North America - United States - California: gets served an "A" record of 111.111.111.2 -- Oceania - Australia: Gets served an "A" record of 111.111.111.3 -- Europe: gets an "A" record of 111.111.111.4 -- Everyone else gets an "A" record of 111.111.111.5 - -### Health Checks - -Octodns will automatically set up monitors for each IP and check for a 200 response for **https:///_dns**. - -These checks can be configured by adding a `healthcheck` configuration to the record: - -```yaml ---- -test: - geo: - AS: - - 1.2.3.4 - EU: - - 2.3.4.5 - octodns: - healthcheck: - host: my-host-name - path: /dns-health-check - port: 443 - protocol: HTTPS -``` - -| Key | Description | Default | -|--|--|--| -| host | FQDN for host header and SNI | - | -| path | path to check | _dns | -| port | port to check | 443 | -| protocol | HTTP/HTTPS | HTTPS | - -#### Route53 Healtch Check Options - -| Key | Description | Default | -|--|--|--| -| measure_latency | Show latency in AWS console | true | - -```yaml ---- - octodns: - healthcheck: - host: my-host-name - path: /dns-health-check - port: 443 - protocol: HTTPS - route53: - healthcheck: - measure_latency: false -``` +## Advanced Record Support (GeoDNS, Weighting) +* [Dynamic Records](/docs/dynamic_records.md) +* [Geo Records](/docs/geo_records.md) ## Config (`YamlProvider`) From f2a63e06b79f7b29121a051761d2f75f0071652d Mon Sep 17 00:00:00 2001 From: Theo Julienne Date: Mon, 6 May 2019 07:58:14 -0700 Subject: [PATCH 007/176] Apply suggestions from code review Co-Authored-By: ross --- docs/records.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/records.md b/docs/records.md index c7ede69..609383c 100644 --- a/docs/records.md +++ b/docs/records.md @@ -22,8 +22,8 @@ Adding new record types to OctoDNS is relatively straightforward, but will requi ## Advanced Record Support (GeoDNS, Weighting) -* [Dynamic Records](/docs/dynamic_records.md) -* [Geo Records](/docs/geo_records.md) +* [Dynamic Records](/docs/dynamic_records.md) - the preferred method for configuring geo-location, weights, and healthcheck based fallback between pools of services. +* [Geo Records](/docs/geo_records.md) - the original implementation of geo-location based records, now superseded by Dynamic Records (above) ## Config (`YamlProvider`) From 7668e15bb66d7b03a87a488608336bf703ce33c6 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 6 May 2019 08:36:50 -0700 Subject: [PATCH 008/176] v0.9.5 version bump and CHANGELOG updates --- CHANGELOG.md | 10 ++++++++-- octodns/__init__.py | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77bfc50..adb1f8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ -## v0.9.5 - 2019-??-?? - The big one, with all the dynamic stuff +## v0.9.5 - 2019-05-06 - The big one, with all the dynamic stuff * dynamic record support, essentially a v2 version of geo records with a lot - more flexibility and power. Also support dynamic CNAME records. + more flexibility and power. Also support dynamic CNAME records (alpha) * Route53Provider dynamic record support * DynProvider dynamic record support * SUPPORTS_DYNAMIC is an optional property, defaults to False @@ -9,7 +9,13 @@ * CloudflareProvider SRV record unpacking fix * DNSMadeEasy provider uses supports to avoid blowing up on unknown record types +* Updates to AzureProvider lib versions * Normalize MX/CNAME/ALIAS/PTR value to lower case +* SplitYamlProvider support added +* DynProvider fix for Traffic Directors association to records, explicit rather + than "looks close enough" +* TinyDNS support for TXT and AAAA records and fixes to ; escaping +* pre-commit hook requires 100% code coverage ## v0.9.4 - 2019-01-28 - The one with a bunch of stuff, before the big one diff --git a/octodns/__init__.py b/octodns/__init__.py index 6125bf1..939c293 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -3,4 +3,4 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -__VERSION__ = '0.9.4' +__VERSION__ = '0.9.5' From 3f4afbac27e2d69e2384cf21b20158685466460c Mon Sep 17 00:00:00 2001 From: Rhosyn Celyn Date: Thu, 9 May 2019 16:45:19 +0100 Subject: [PATCH 009/176] Working initial implementation for Mythic Beasts DNS API --- octodns/provider/mythicbeasts.py | 58 ++++++++++---------------------- 1 file changed, 18 insertions(+), 40 deletions(-) diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py index f416a1b..d85e133 100644 --- a/octodns/provider/mythicbeasts.py +++ b/octodns/provider/mythicbeasts.py @@ -13,7 +13,6 @@ from .base import BaseProvider import re -import pprint import sys def add_trailing_dot(s): @@ -42,9 +41,8 @@ class MythicBeastsProvider(BaseProvider): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', - 'SRV', 'TXT')) - #BASE = 'https://dnsapi.mythic-beasts.com/' - BASE = 'https://cwningen.dev.mythic-beasts.com/customer/primarydnsapi' + 'SRV', 'SSHFP', 'CAA', 'TXT')) + BASE = 'https://dnsapi.mythic-beasts.com/' TIMEOUT = 15 def __init__(self, id, passwords, *args, **kwargs): @@ -61,8 +59,10 @@ class MythicBeastsProvider(BaseProvider): url = self.BASE resp = self._sess.request(method, url, data=data, timeout=self.TIMEOUT) - self.log.debug('_request: status=%d', resp.status_code) - resp.raise_for_status() + self.log.debug('_request: status=%d data=%s', resp.status_code, resp.text) + if resp.status_code != 200: + self.log.info('request failed: %s, response %s', data, resp.text) + resp.raise_for_status() return resp def _post(self, data=None): @@ -164,21 +164,19 @@ class MythicBeastsProvider(BaseProvider): return self._data_for_single('SSHFP', {'raw_values': [ {'value': value, 'ttl': ttl} ]}) - # TODO fix bug with CAA output from API - ''' def _data_for_CAA(self, _type, data): ttl = data['raw_values'][0]['ttl'] + raw_value = data['raw_values'][0]['value'] - match = re.match('^()$', re.IGNORECASE) + match = re.match('^([0-9]+)\s+(issue|issuewild|iodef)\s+(\S+)$', raw_value, re.IGNORECASE) - value = { - 'flags': - 'tag': - 'value': - } - value = data['raw_values'][0]['value'] - return self._data_for_single('ALIAS', {'raw_values': [ {'value': value, 'ttl': ttl} ]}) - ''' + if match is not None: + value = { + 'flags': match.group(1), + 'tag': match.group(2), + 'value': match.group(3), + } + return self._data_for_single('CAA', {'raw_values': [ {'value': value, 'ttl': ttl} ]}) _data_for_NS = _data_for_multiple @@ -255,11 +253,13 @@ class MythicBeastsProvider(BaseProvider): for _name in data[_type]: data_for = getattr(self, '_data_for_{}'.format(_type)) + self.log.debug('record: {}, {}'.format(_type, data[_type][_name])) + record = Record.new( zone, _name, data_for(_type, data[_type][_name]), - source=self, + source=self ) zone.add_record(record, lenient=lenient) @@ -285,8 +285,6 @@ class MythicBeastsProvider(BaseProvider): ttl = record.ttl _type = record._type - if hostname == '': - hostname = '@' if _type == 'ALIAS': _type = 'ANAME' @@ -331,12 +329,8 @@ class MythicBeastsProvider(BaseProvider): return commands def _apply_Create(self, change): - - pp = pprint.PrettyPrinter(depth=10, stream=sys.stderr) - zone = change.new.zone commands = self._compile_commands('ADD', change) - pp.pprint(commands) for command in commands: self._post({ @@ -345,12 +339,6 @@ class MythicBeastsProvider(BaseProvider): 'password': self._passwords[zone.name], 'command': command, }) - pp.pprint({ - 'domain': remove_trailing_dot(zone.name), - 'origin': '.', - 'password': self._passwords[zone.name], - 'command': command, - }) return True def _apply_Update(self, change): @@ -358,12 +346,8 @@ class MythicBeastsProvider(BaseProvider): self._apply_Create(change) def _apply_Delete(self, change): - - pp = pprint.PrettyPrinter(depth=10, stream=sys.stderr) - zone = change.existing.zone commands = self._compile_commands('DELETE', change) - pp.pprint(commands) for command in commands: self._post({ @@ -372,12 +356,6 @@ class MythicBeastsProvider(BaseProvider): 'password': self._passwords[zone.name], 'command': command, }) - pp.pprint({ - 'domain': remove_trailing_dot(zone.name), - 'origin': '.', - 'password': self._passwords[zone.name], - 'command': command, - }) return True def _apply(self, plan): From 47f76d153564285feb794c96d3290a2fd0166d73 Mon Sep 17 00:00:00 2001 From: Rhosyn Celyn Date: Thu, 9 May 2019 18:45:57 +0100 Subject: [PATCH 010/176] Linting and clean up --- octodns/provider/mythicbeasts.py | 190 +++++++++++++++++++------------ 1 file changed, 119 insertions(+), 71 deletions(-) diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py index d85e133..17ec574 100644 --- a/octodns/provider/mythicbeasts.py +++ b/octodns/provider/mythicbeasts.py @@ -5,26 +5,31 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +import re + from requests import HTTPError, Session from logging import getLogger -from ..record import Create, Record +from ..record import Record from .base import BaseProvider -import re -import sys - -def add_trailing_dot(s): - assert s - assert s[-1] != '.' - return s + '.' +def add_trailing_dot(value): + ''' + Add trailing dots to values + ''' + assert value + assert value[-1] != '.' + return value + '.' -def remove_trailing_dot(s): - assert s - assert s[-1] == '.' - return s[:-1] +def remove_trailing_dot(value): + ''' + Remove trailing dots from values + ''' + assert value + assert value[-1] == '.' + return value[:-1] class MythicBeastsProvider(BaseProvider): @@ -35,7 +40,6 @@ class MythicBeastsProvider(BaseProvider): class: octodns.provider.mythicbeasts.MythicBeastsProvider zones: my-zone: 'password' - ''' SUPPORTS_GEO = False @@ -45,10 +49,13 @@ class MythicBeastsProvider(BaseProvider): BASE = 'https://dnsapi.mythic-beasts.com/' TIMEOUT = 15 - def __init__(self, id, passwords, *args, **kwargs): - self.log = getLogger('MythicBeastsProvider[{}]'.format(id)) - self.log.debug('__init__: id=%s, registered zones; %s', id, passwords.keys()) - super(MythicBeastsProvider, self).__init__(id, *args, **kwargs) + def __init__(self, identifier, passwords, *args, **kwargs): + self.log = getLogger('MythicBeastsProvider[{}]'.format(identifier)) + self.log.debug( + '__init__: id=%s, registered zones; %s', + identifier, + passwords.keys()) + super(MythicBeastsProvider, self).__init__(identifier, *args, **kwargs) self._passwords = passwords sess = Session() @@ -59,7 +66,10 @@ class MythicBeastsProvider(BaseProvider): url = self.BASE resp = self._sess.request(method, url, data=data, timeout=self.TIMEOUT) - self.log.debug('_request: status=%d data=%s', resp.status_code, resp.text) + self.log.debug( + '_request: status=%d data=%s', + resp.status_code, + resp.text) if resp.status_code != 200: self.log.info('request failed: %s, response %s', data, resp.text) resp.raise_for_status() @@ -76,26 +86,32 @@ class MythicBeastsProvider(BaseProvider): 'command': 'LIST', }) - def _data_for_single(self, _type, data): + @staticmethod + def _data_for_single(_type, data): return { 'type': _type, 'value': data['raw_values'][0]['value'], 'ttl': data['raw_values'][0]['ttl'] } - def _data_for_multiple(self, _type, data): + @staticmethod + def _data_for_multiple(_type, data): return { 'type': _type, - 'values': [raw_values['value'] for raw_values in data['raw_values']], - 'ttl': max([raw_values['ttl'] for raw_values in data['raw_values']]), + 'values': + [raw_values['value'] for raw_values in data['raw_values']], + 'ttl': + max([raw_values['ttl'] for raw_values in data['raw_values']]), } - def _data_for_MX(self, _type, data): + @staticmethod + def _data_for_MX(_type, data): ttl = max([raw_values['ttl'] for raw_values in data['raw_values']]) values = [] - for raw_value in [raw_values['value'] for raw_values in data['raw_values']]: - match = re.match('^([0-9]+)\s+(\S+)$', raw_value, re.IGNORECASE) + for raw_value in \ + [raw_values['value'] for raw_values in data['raw_values']]: + match = re.match('^([0-9]+)\\s+(\\S+)$', raw_value, re.IGNORECASE) if match is not None: exchange = match.group(2) @@ -114,25 +130,38 @@ class MythicBeastsProvider(BaseProvider): 'ttl': ttl, } - def _data_for_CNAME(self, _type, data): + @staticmethod + def _data_for_CNAME(_type, data): ttl = data['raw_values'][0]['ttl'] value = data['raw_values'][0]['value'] if not value.endswith('.'): value = '{}.{}'.format(value, data['zone']) - return self._data_for_single(_type, {'raw_values': [ {'value': value, 'ttl': ttl} ]}) + return MythicBeastsProvider._data_for_single( + _type, + {'raw_values': [ + {'value': value, 'ttl': ttl} + ]}) - def _data_for_ANAME(self, _type, data): + @staticmethod + def _data_for_ANAME(_type, data): ttl = data['raw_values'][0]['ttl'] value = data['raw_values'][0]['value'] - return self._data_for_single('ALIAS', {'raw_values': [ {'value': value, 'ttl': ttl} ]}) - + return MythicBeastsProvider._data_for_single( + 'ALIAS', + {'raw_values': [ + {'value': value, 'ttl': ttl} + ]}) - def _data_for_SRV(self, _type, data): + @staticmethod + def _data_for_SRV(_type, data): ttl = data['raw_values'][0]['ttl'] raw_value = data['raw_values'][0]['value'] - match = re.match('^([0-9]+)\s+([0-9]+)\s+([0-9]+)\s+(\S+)$', raw_value, re.IGNORECASE) + match = re.match( + '^([0-9]+)\\s+([0-9]+)\\s+([0-9]+)\\s+(\\S+)$', + raw_value, + re.IGNORECASE) if match is not None: target = match.group(4) @@ -146,13 +175,21 @@ class MythicBeastsProvider(BaseProvider): 'target': target, } - return self._data_for_single('SRV', {'raw_values': [ {'value': value, 'ttl': ttl} ]}) + return MythicBeastsProvider._data_for_single( + 'SRV', + {'raw_values': [ + {'value': value, 'ttl': ttl} + ]}) - def _data_for_SSHFP(self, _type, data): + @staticmethod + def _data_for_SSHFP(_type, data): ttl = data['raw_values'][0]['ttl'] raw_value = data['raw_values'][0]['value'] - match = re.match('^([0-9]+)\s+([0-9]+)\s+(\S+)$', raw_value, re.IGNORECASE) + match = re.match( + '^([0-9]+)\\s+([0-9]+)\\s+(\\S+)$', + raw_value, + re.IGNORECASE) if match is not None: value = { @@ -161,14 +198,21 @@ class MythicBeastsProvider(BaseProvider): 'fingerprint': match.group(3), } - return self._data_for_single('SSHFP', {'raw_values': [ {'value': value, 'ttl': ttl} ]}) - + return MythicBeastsProvider._data_for_single( + 'SSHFP', + {'raw_values': [ + {'value': value, 'ttl': ttl} + ]}) - def _data_for_CAA(self, _type, data): + @staticmethod + def _data_for_CAA(_type, data): ttl = data['raw_values'][0]['ttl'] raw_value = data['raw_values'][0]['value'] - match = re.match('^([0-9]+)\s+(issue|issuewild|iodef)\s+(\S+)$', raw_value, re.IGNORECASE) + match = re.match( + '^([0-9]+)\\s+(issue|issuewild|iodef)\\s+(\\S+)$', + raw_value, + re.IGNORECASE) if match is not None: value = { @@ -176,15 +220,17 @@ class MythicBeastsProvider(BaseProvider): 'tag': match.group(2), 'value': match.group(3), } - return self._data_for_single('CAA', {'raw_values': [ {'value': value, 'ttl': ttl} ]}) - + return MythicBeastsProvider._data_for_single( + 'CAA', + {'raw_values': [ + {'value': value, 'ttl': ttl} + ]}) _data_for_NS = _data_for_multiple _data_for_TXT = _data_for_multiple _data_for_A = _data_for_multiple _data_for_AAAA = _data_for_multiple - def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) @@ -192,15 +238,12 @@ class MythicBeastsProvider(BaseProvider): resp = None try: resp = self.records(zone.name) - except HTTPError as e: - if e.response.status_code == 401: + except HTTPError as err: + if err.response.status_code == 401: # Nicer error message for auth problems - raise Exception('Mythic Beasts authentication problem with {}'.format(zone.name)) - elif e.response.status_code == 422: - # 422 means mythicbeasts doesn't know anything about the requested - # domain. We'll just ignore it here and leave the zone - # untouched. - raise + raise Exception( + 'Mythic Beasts authentication problem with {}'.format( + zone.name)) else: # just re-throw raise @@ -212,7 +255,10 @@ class MythicBeastsProvider(BaseProvider): if resp: exists = True for line in resp.content.splitlines(): - match = re.match('^(\S+)\s+(\S+)\s+(\S+)\s+(.*)$', line, re.IGNORECASE) + match = re.match( + '^(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(.*)$', + line, + re.IGNORECASE) if match is not None: if match.group(1) == '@': @@ -244,16 +290,18 @@ class MythicBeastsProvider(BaseProvider): data[_type][_name].get('raw_values').append( {'value': _value, 'ttl': _ttl} ) - except AttributeError as error: + except AttributeError: self.log.debug('skipping {} as not supported', _type) continue - for _type in data: for _name in data[_type]: data_for = getattr(self, '_data_for_{}'.format(_type)) - self.log.debug('record: {}, {}'.format(_type, data[_type][_name])) + self.log.debug( + 'record: %s,\t%s', + _type, + data[_type][_name]) record = Record.new( zone, @@ -263,12 +311,10 @@ class MythicBeastsProvider(BaseProvider): ) zone.add_record(record, lenient=lenient) - self.log.debug('populate: found %s records, exists=%s', - len(zone.records) - before, exists) + len(zone.records) - before, exists) return exists - def _compile_commands(self, action, change): commands = [] @@ -280,7 +326,6 @@ class MythicBeastsProvider(BaseProvider): elif action == 'DELETE': record = change.existing - zone = record.zone hostname = remove_trailing_dot(record.fqdn) ttl = record.ttl _type = record._type @@ -288,13 +333,11 @@ class MythicBeastsProvider(BaseProvider): if _type == 'ALIAS': _type = 'ANAME' - if hasattr(record, 'values'): values = record.values else: values = [record.value] - base = '{} {} {} {}'.format(action, hostname, ttl, _type) if re.match('[A]{1,4}', _type) is not None: @@ -304,25 +347,35 @@ class MythicBeastsProvider(BaseProvider): elif _type == 'SSHFP': data = values[0].data commands.append('{} {} {} {}'.format( - base, data['algorithm'], data['fingerprint_type'], data['fingerprint'])) + base, + data['algorithm'], + data['fingerprint_type'], + data['fingerprint'] + )) elif _type == 'SRV': data = values[0].data commands.append('{} {} {} {} {}'.format( - base, data['priority'], data['weight'], data['port'], data['target'])) - + base, + data['priority'], + data['weight'], + data['port'], + data['target'])) + elif _type == 'MX': for value in values: data = value.data commands.append('{} {} {}'.format( - base, data['preference'], data['exchange'])) + base, + data['preference'], + data['exchange'])) else: try: if getattr(self, '_data_for_{}'.format(_type)) is not None: commands.append('{} {}'.format( - base, values[0])) - except AttributeError as error: + base, values[0])) + except AttributeError: self.log.debug('skipping {} as not supported', _type) pass @@ -364,11 +417,6 @@ class MythicBeastsProvider(BaseProvider): self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, len(changes)) - domain_name = desired.name - for change in changes: class_name = change.__class__.__name__ getattr(self, '_apply_{}'.format(class_name))(change) - - - From 2b6d86fb4f3281d52b117efa873a28f146411a97 Mon Sep 17 00:00:00 2001 From: Rhosyn Celyn Date: Thu, 9 May 2019 21:47:17 +0100 Subject: [PATCH 011/176] E125 fix --- octodns/provider/mythicbeasts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py index 17ec574..dd11573 100644 --- a/octodns/provider/mythicbeasts.py +++ b/octodns/provider/mythicbeasts.py @@ -110,7 +110,7 @@ class MythicBeastsProvider(BaseProvider): values = [] for raw_value in \ - [raw_values['value'] for raw_values in data['raw_values']]: + [raw_values['value'] for raw_values in data['raw_values']]: match = re.match('^([0-9]+)\\s+(\\S+)$', raw_value, re.IGNORECASE) if match is not None: From fd63150cac52ba28b305cfeaa4f0b6f2419c2acd Mon Sep 17 00:00:00 2001 From: Rhosyn Celyn Date: Mon, 13 May 2019 16:51:38 +0100 Subject: [PATCH 012/176] Added tests, clean up and small modifications --- octodns/provider/mythicbeasts.py | 274 ++++++++------- tests/fixtures/mythicbeasts-list.txt | 25 ++ tests/test_octodns_provider_mythicbeasts.py | 356 ++++++++++++++++++++ 3 files changed, 517 insertions(+), 138 deletions(-) create mode 100644 tests/fixtures/mythicbeasts-list.txt create mode 100644 tests/test_octodns_provider_mythicbeasts.py diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py index dd11573..ca03b37 100644 --- a/octodns/provider/mythicbeasts.py +++ b/octodns/provider/mythicbeasts.py @@ -7,7 +7,7 @@ from __future__ import absolute_import, division, print_function, \ import re -from requests import HTTPError, Session +from requests import Session from logging import getLogger from ..record import Record @@ -18,8 +18,8 @@ def add_trailing_dot(value): ''' Add trailing dots to values ''' - assert value - assert value[-1] != '.' + assert value, 'Missing value' + assert value[-1] != '.', 'Value already has trailing dot' return value + '.' @@ -27,8 +27,8 @@ def remove_trailing_dot(value): ''' Remove trailing dots from values ''' - assert value - assert value[-1] == '.' + assert value, 'Missing value' + assert value[-1] == '.', 'Value already missing trailing dot' return value[:-1] @@ -44,13 +44,15 @@ class MythicBeastsProvider(BaseProvider): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False - SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NS', 'SRV', 'SSHFP', 'CAA', 'TXT')) BASE = 'https://dnsapi.mythic-beasts.com/' - TIMEOUT = 15 def __init__(self, identifier, passwords, *args, **kwargs): self.log = getLogger('MythicBeastsProvider[{}]'.format(identifier)) + + assert isinstance(passwords, dict), 'Missing passwords' + self.log.debug( '__init__: id=%s, registered zones; %s', identifier, @@ -62,23 +64,28 @@ class MythicBeastsProvider(BaseProvider): self._sess = sess def _request(self, method, path, data=None): - self.log.debug('_request: method=%s, path=%s', method, path) + self.log.debug('_request: method=%s, path=%s data=%s', + method, path, data) - url = self.BASE - resp = self._sess.request(method, url, data=data, timeout=self.TIMEOUT) + resp = self._sess.request(method, path, data=data) self.log.debug( '_request: status=%d data=%s', resp.status_code, - resp.text) - if resp.status_code != 200: - self.log.info('request failed: %s, response %s', data, resp.text) - resp.raise_for_status() + resp.text[:20]) + + if resp.status_code == 401: + raise Exception('Mythic Beasts unauthorized for domain: {}' + .format(data['domain'])) + resp.raise_for_status() return resp def _post(self, data=None): return self._request('POST', self.BASE, data=data) def records(self, zone): + assert zone in self._passwords, 'Missing password for domain: {}' \ + .format(remove_trailing_dot(zone)) + return self._post({ 'domain': remove_trailing_dot(zone), 'password': self._passwords[zone], @@ -113,16 +120,17 @@ class MythicBeastsProvider(BaseProvider): [raw_values['value'] for raw_values in data['raw_values']]: match = re.match('^([0-9]+)\\s+(\\S+)$', raw_value, re.IGNORECASE) - if match is not None: - exchange = match.group(2) + assert match is not None, 'Unable to parse MX data' - if not exchange.endswith('.'): - exchange = '{}.{}'.format(exchange, data['zone']) + exchange = match.group(2) - values.append({ - 'preference': match.group(1), - 'exchange': exchange, - }) + if not exchange.endswith('.'): + exchange = '{}.{}'.format(exchange, data['zone']) + + values.append({ + 'preference': match.group(1), + 'exchange': exchange, + }) return { 'type': _type, @@ -155,54 +163,61 @@ class MythicBeastsProvider(BaseProvider): @staticmethod def _data_for_SRV(_type, data): - ttl = data['raw_values'][0]['ttl'] - raw_value = data['raw_values'][0]['value'] + ttl = max([raw_values['ttl'] for raw_values in data['raw_values']]) + values = [] - match = re.match( - '^([0-9]+)\\s+([0-9]+)\\s+([0-9]+)\\s+(\\S+)$', - raw_value, - re.IGNORECASE) + for raw_value in \ + [raw_values['value'] for raw_values in data['raw_values']]: + + match = re.match( + '^([0-9]+)\\s+([0-9]+)\\s+([0-9]+)\\s+(\\S+)$', + raw_value, + re.IGNORECASE) + + assert match is not None, 'Unable to parse SRV data' - if match is not None: target = match.group(4) if not target.endswith('.'): target = '{}.{}'.format(target, data['zone']) - value = { + values.append({ 'priority': match.group(1), 'weight': match.group(2), 'port': match.group(3), 'target': target, - } + }) - return MythicBeastsProvider._data_for_single( - 'SRV', - {'raw_values': [ - {'value': value, 'ttl': ttl} - ]}) + return { + 'type': _type, + 'values': values, + 'ttl': ttl, + } @staticmethod def _data_for_SSHFP(_type, data): - ttl = data['raw_values'][0]['ttl'] - raw_value = data['raw_values'][0]['value'] + ttl = max([raw_values['ttl'] for raw_values in data['raw_values']]) + values = [] - match = re.match( - '^([0-9]+)\\s+([0-9]+)\\s+(\\S+)$', - raw_value, - re.IGNORECASE) + for raw_value in \ + [raw_values['value'] for raw_values in data['raw_values']]: + match = re.match( + '^([0-9]+)\\s+([0-9]+)\\s+(\\S+)$', + raw_value, + re.IGNORECASE) - if match is not None: - value = { + assert match is not None, 'Unable to parse SSHFP data' + + values.append({ 'algorithm': match.group(1), 'fingerprint_type': match.group(2), 'fingerprint': match.group(3), - } + }) - return MythicBeastsProvider._data_for_single( - 'SSHFP', - {'raw_values': [ - {'value': value, 'ttl': ttl} - ]}) + return { + 'type': _type, + 'values': values, + 'ttl': ttl, + } @staticmethod def _data_for_CAA(_type, data): @@ -214,17 +229,17 @@ class MythicBeastsProvider(BaseProvider): raw_value, re.IGNORECASE) - if match is not None: - value = { - 'flags': match.group(1), - 'tag': match.group(2), - 'value': match.group(3), - } - return MythicBeastsProvider._data_for_single( - 'CAA', - {'raw_values': [ - {'value': value, 'ttl': ttl} - ]}) + assert match is not None, 'Unable to parse CAA data' + + value = { + 'flags': match.group(1), + 'tag': match.group(2), + 'value': match.group(3), + } + + return MythicBeastsProvider._data_for_single( + 'CAA', + {'raw_values': [{'value': value, 'ttl': ttl}]}) _data_for_NS = _data_for_multiple _data_for_TXT = _data_for_multiple @@ -235,84 +250,69 @@ class MythicBeastsProvider(BaseProvider): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) - resp = None - try: - resp = self.records(zone.name) - except HTTPError as err: - if err.response.status_code == 401: - # Nicer error message for auth problems - raise Exception( - 'Mythic Beasts authentication problem with {}'.format( - zone.name)) - else: - # just re-throw - raise + resp = self.records(zone.name) before = len(zone.records) exists = False data = dict() - if resp: - exists = True - for line in resp.content.splitlines(): - match = re.match( - '^(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(.*)$', - line, - re.IGNORECASE) + exists = True + for line in resp.content.splitlines(): + match = re.match( + '^(\\S+)\\s+(\\d+)\\s+(\\S+)\\s+(.*)$', + line, + re.IGNORECASE) - if match is not None: - if match.group(1) == '@': - _name = '' - else: - _name = match.group(1) + if match is None: + self.log.debug('failed to match line: %s', line) + continue - _type = match.group(3) - _ttl = int(match.group(2)) - _value = match.group(4).strip() + if match.group(1) == '@': + _name = '' + else: + _name = match.group(1) - if _type == 'SOA': - continue + _type = match.group(3) + _ttl = int(match.group(2)) + _value = match.group(4).strip() - try: - if getattr(self, '_data_for_{}'.format(_type)) is not None: + if _type == 'TXT': + _value = _value.replace(';', '\\;') - if _type not in data: - data[_type] = dict() + if hasattr(self, '_data_for_{}'.format(_type)): - if _name not in data[_type]: - data[_type][_name] = { - 'raw_values': [{'value': _value, 'ttl': _ttl}], - 'name': _name, - 'zone': zone.name, - } + if _type not in data: + data[_type] = dict() - else: - data[_type][_name].get('raw_values').append( - {'value': _value, 'ttl': _ttl} - ) - except AttributeError: - self.log.debug('skipping {} as not supported', _type) - continue + if _name not in data[_type]: + data[_type][_name] = { + 'raw_values': [{'value': _value, 'ttl': _ttl}], + 'name': _name, + 'zone': zone.name, + } - for _type in data: - for _name in data[_type]: - data_for = getattr(self, '_data_for_{}'.format(_type)) - - self.log.debug( - 'record: %s,\t%s', - _type, - data[_type][_name]) - - record = Record.new( - zone, - _name, - data_for(_type, data[_type][_name]), - source=self + else: + data[_type][_name].get('raw_values').append( + {'value': _value, 'ttl': _ttl} ) - zone.add_record(record, lenient=lenient) + else: + self.log.debug('skipping %s as not supported', _type) + + for _type in data: + for _name in data[_type]: + data_for = getattr(self, '_data_for_{}'.format(_type)) + + record = Record.new( + zone, + _name, + data_for(_type, data[_type][_name]), + source=self + ) + zone.add_record(record, lenient=lenient) self.log.debug('populate: found %s records, exists=%s', len(zone.records) - before, exists) + return exists def _compile_commands(self, action, change): @@ -322,8 +322,7 @@ class MythicBeastsProvider(BaseProvider): if action == 'ADD': record = change.new - - elif action == 'DELETE': + else: record = change.existing hostname = remove_trailing_dot(record.fqdn) @@ -354,13 +353,14 @@ class MythicBeastsProvider(BaseProvider): )) elif _type == 'SRV': - data = values[0].data - commands.append('{} {} {} {} {}'.format( - base, - data['priority'], - data['weight'], - data['port'], - data['target'])) + for value in values: + data = value.data + commands.append('{} {} {} {} {}'.format( + base, + data['priority'], + data['weight'], + data['port'], + data['target'])) elif _type == 'MX': for value in values: @@ -371,13 +371,11 @@ class MythicBeastsProvider(BaseProvider): data['exchange'])) else: - try: - if getattr(self, '_data_for_{}'.format(_type)) is not None: - commands.append('{} {}'.format( - base, values[0])) - except AttributeError: - self.log.debug('skipping {} as not supported', _type) - pass + if hasattr(self, '_data_for_{}'.format(_type)): + commands.append('{} {}'.format( + base, values[0])) + else: + self.log.debug('skipping %s as not supported', _type) return commands diff --git a/tests/fixtures/mythicbeasts-list.txt b/tests/fixtures/mythicbeasts-list.txt new file mode 100644 index 0000000..ed4ea4c --- /dev/null +++ b/tests/fixtures/mythicbeasts-list.txt @@ -0,0 +1,25 @@ +@ 3600 NS 6.2.3.4. +@ 3600 NS 7.2.3.4. +@ 300 A 1.2.3.4 +@ 300 A 1.2.3.5 +@ 3600 SSHFP 1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73 +@ 3600 SSHFP 1 1 7491973e5f8b39d5327cd4e08bc81b05f7710b49 +@ 3600 CAA 0 issue ca.unit.tests +_srv._tcp 600 SRV 10 20 30 foo-1.unit.tests. +_srv._tcp 600 SRV 12 20 30 foo-2.unit.tests. +aaaa 600 AAAA 2601:644:500:e210:62f8:1dff:feb8:947a +cname 300 CNAME unit.tests. +excluded 300 CNAME unit.tests. +ignored 300 A 9.9.9.9 +included 3600 CNAME unit.tests. +mx 300 MX 10 smtp-4.unit.tests. +mx 300 MX 20 smtp-2.unit.tests. +mx 300 MX 30 smtp-3.unit.tests. +mx 300 MX 40 smtp-1.unit.tests. +sub 3600 NS 6.2.3.4. +sub 3600 NS 7.2.3.4. +txt 600 TXT "Bah bah black sheep" +txt 600 TXT "have you any wool." +txt 600 TXT "v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs" +www 300 A 2.2.3.6 +www.sub 300 A 2.2.3.6 diff --git a/tests/test_octodns_provider_mythicbeasts.py b/tests/test_octodns_provider_mythicbeasts.py new file mode 100644 index 0000000..a1ef5d5 --- /dev/null +++ b/tests/test_octodns_provider_mythicbeasts.py @@ -0,0 +1,356 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from os.path import dirname, join + +from requests_mock import ANY, mock as requests_mock +from unittest import TestCase + +from octodns.provider.mythicbeasts import MythicBeastsProvider, \ + add_trailing_dot, remove_trailing_dot +from octodns.provider.yaml import YamlProvider +from octodns.zone import Zone +from octodns.record import Create, Update, Delete, Record + + +class TestMythicBeastsProvider(TestCase): + expected = Zone('unit.tests.', []) + source = YamlProvider('test_expected', join(dirname(__file__), 'config')) + source.populate(expected) + + # Dump anything we don't support from expected + for record in list(expected.records): + if record._type not in MythicBeastsProvider.SUPPORTS: + expected._remove_record(record) + + def test_trailing_dot(self): + with self.assertRaises(AssertionError) as err: + add_trailing_dot('unit.tests.') + self.assertEquals('Value already has trailing dot', + err.exception.message) + + with self.assertRaises(AssertionError) as err: + remove_trailing_dot('unit.tests') + self.assertEquals('Value already missing trailing dot', + err.exception.message) + + self.assertEquals(add_trailing_dot('unit.tests'), 'unit.tests.') + self.assertEquals(remove_trailing_dot('unit.tests.'), 'unit.tests') + + def test_data_for_single(self): + test_data = { + 'raw_values': [{'value': 'a:a::c', 'ttl': 0}], + 'zone': 'unit.tests.', + } + test_single = MythicBeastsProvider._data_for_single('', test_data) + self.assertTrue(isinstance(test_single, dict)) + self.assertEquals('a:a::c', test_single['value']) + + def test_data_for_multiple(self): + test_data = { + 'raw_values': [ + {'value': 'b:b::d', 'ttl': 60}, + {'value': 'a:a::c', 'ttl': 60}], + 'zone': 'unit.tests.', + } + test_multiple = MythicBeastsProvider._data_for_multiple('', test_data) + self.assertTrue(isinstance(test_multiple, dict)) + self.assertEquals(2, len(test_multiple['values'])) + + def test_data_for_MX(self): + test_data = { + 'raw_values': [ + {'value': '10 un.unit', 'ttl': 60}, + {'value': '20 dau.unit', 'ttl': 60}, + {'value': '30 tri.unit', 'ttl': 60}], + 'zone': 'unit.tests.', + } + test_MX = MythicBeastsProvider._data_for_MX('', test_data) + self.assertTrue(isinstance(test_MX, dict)) + self.assertEquals(3, len(test_MX['values'])) + + with self.assertRaises(AssertionError) as err: + test_MX = MythicBeastsProvider._data_for_MX( + '', + {'raw_values': [{'value': '', 'ttl': 0}]} + ) + self.assertEquals('Unable to parse MX data', + err.exception.message) + + def test_data_for_CNAME(self): + test_data = { + 'raw_values': [{'value': 'cname', 'ttl': 60}], + 'zone': 'unit.tests.', + } + test_cname = MythicBeastsProvider._data_for_CNAME('', test_data) + self.assertTrue(isinstance(test_cname, dict)) + self.assertEquals('cname.unit.tests.', test_cname['value']) + + def test_data_for_ANAME(self): + test_data = { + 'raw_values': [{'value': 'aname', 'ttl': 60}], + 'zone': 'unit.tests.', + } + test_aname = MythicBeastsProvider._data_for_ANAME('', test_data) + self.assertTrue(isinstance(test_aname, dict)) + self.assertEquals('aname', test_aname['value']) + + def test_data_for_SRV(self): + test_data = { + 'raw_values': [ + {'value': '10 20 30 un.srv.unit', 'ttl': 60}, + {'value': '20 30 40 dau.srv.unit', 'ttl': 60}, + {'value': '30 30 50 tri.srv.unit', 'ttl': 60}], + 'zone': 'unit.tests.', + } + test_SRV = MythicBeastsProvider._data_for_SRV('', test_data) + self.assertTrue(isinstance(test_SRV, dict)) + self.assertEquals(3, len(test_SRV['values'])) + + with self.assertRaises(AssertionError) as err: + test_SRV = MythicBeastsProvider._data_for_SRV( + '', + {'raw_values': [{'value': '', 'ttl': 0}]} + ) + self.assertEquals('Unable to parse SRV data', + err.exception.message) + + def test_data_for_SSHFP(self): + test_data = { + 'raw_values': [ + {'value': '1 1 0123456789abcdef', 'ttl': 60}, + {'value': '1 2 0123456789abcdef', 'ttl': 60}, + {'value': '2 3 0123456789abcdef', 'ttl': 60}], + 'zone': 'unit.tests.', + } + test_SSHFP = MythicBeastsProvider._data_for_SSHFP('', test_data) + self.assertTrue(isinstance(test_SSHFP, dict)) + self.assertEquals(3, len(test_SSHFP['values'])) + + with self.assertRaises(AssertionError) as err: + test_SSHFP = MythicBeastsProvider._data_for_SSHFP( + '', + {'raw_values': [{'value': '', 'ttl': 0}]} + ) + self.assertEquals('Unable to parse SSHFP data', + err.exception.message) + + def test_data_for_CAA(self): + test_data = { + 'raw_values': [{'value': '1 issue letsencrypt.org', 'ttl': 60}], + 'zone': 'unit.tests.', + } + test_CAA = MythicBeastsProvider._data_for_CAA('', test_data) + self.assertTrue(isinstance(test_CAA, dict)) + self.assertEquals(3, len(test_CAA['value'])) + + with self.assertRaises(AssertionError) as err: + test_CAA = MythicBeastsProvider._data_for_CAA( + '', + {'raw_values': [{'value': '', 'ttl': 0}]} + ) + self.assertEquals('Unable to parse CAA data', + err.exception.message) + + def test_alias_command_generation(self): + zone = Zone('unit.tests.', []) + zone.add_record(Record.new(zone, 'prawf', { + 'ttl': 60, + 'type': 'ALIAS', + 'value': 'alias.unit.tests.', + })) + with requests_mock() as mock: + mock.post(ANY, status_code=200, text='') + + provider = MythicBeastsProvider('test', { + 'unit.tests.': 'mypassword' + }) + + plan = provider.plan(zone) + change = plan.changes[0] + + command = provider._compile_commands('ADD', change) + self.assertEquals( + ['ADD prawf.unit.tests 60 ANAME alias.unit.tests.'], + command + ) + + def test_txt_command_generation(self): + zone = Zone('unit.tests.', []) + zone.add_record(Record.new(zone, 'prawf', { + 'ttl': 60, + 'type': 'TXT', + 'value': 'prawf prawf dyma prawf', + })) + with requests_mock() as mock: + mock.post(ANY, status_code=200, text='') + + provider = MythicBeastsProvider('test', { + 'unit.tests.': 'mypassword' + }) + + plan = provider.plan(zone) + change = plan.changes[0] + + command = provider._compile_commands('ADD', change) + self.assertEquals( + ['ADD prawf.unit.tests 60 TXT prawf prawf dyma prawf'], + command + ) + + def test_command_generation(self): + class FakeChangeRecord(object): + def __init__(self): + self.__fqdn = 'prawf.unit.tests.' + self._type = 'NOOP' + self.value = 'prawf' + self.ttl = 60 + + @property + def new(self): + return self + + @property + def existing(self): + return self + + @property + def fqdn(self): + return self.__fqdn + + with requests_mock() as mock: + mock.post(ANY, status_code=200, text='') + + provider = MythicBeastsProvider('test', { + 'unit.tests.': 'mypassword' + }) + record = FakeChangeRecord() + command = provider._compile_commands('ADD', record) + self.assertEquals([], command) + + def test_populate(self): + provider = None + + # Null passwords dict + with self.assertRaises(AssertionError) as err: + provider = MythicBeastsProvider('test', None) + self.assertEquals('Missing passwords', err.exception.message) + + # Missing password + with requests_mock() as mock: + mock.post(ANY, status_code=401, text='ERR Not authenticated') + + with self.assertRaises(AssertionError) as err: + provider = MythicBeastsProvider('test', dict()) + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals( + 'Missing password for domain: unit.tests', + err.exception.message) + + # Failed authentication + with requests_mock() as mock: + mock.post(ANY, status_code=401, text='ERR Not authenticated') + + with self.assertRaises(Exception) as err: + provider = MythicBeastsProvider('test', { + 'unit.tests.': 'mypassword' + }) + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals( + 'Mythic Beasts unauthorized for domain: unit.tests', + err.exception.message) + + # Check unmatched lines are ignored + test_data = 'This should not match' + with requests_mock() as mock: + mock.post(ANY, status_code=200, text=test_data) + + provider = MythicBeastsProvider('test', { + 'unit.tests.': 'mypassword' + }) + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(0, len(zone.records)) + + # Check unsupported records are skipped + test_data = '@ 60 NOOP prawf\n@ 60 SPF prawf prawf prawf' + with requests_mock() as mock: + mock.post(ANY, status_code=200, text=test_data) + + provider = MythicBeastsProvider('test', { + 'unit.tests.': 'mypassword' + }) + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(0, len(zone.records)) + + # Check no changes between what we support and what's parsed + # from the unit.tests. config YAML. Also make sure we see the same + # for both after we've thrown away records we don't support + with requests_mock() as mock: + with open('tests/fixtures/mythicbeasts-list.txt') as file_handle: + mock.post(ANY, status_code=200, text=file_handle.read()) + + provider = MythicBeastsProvider('test', { + 'unit.tests.': 'mypassword' + }) + zone = Zone('unit.tests.', []) + provider.populate(zone) + + self.assertEquals(15, len(zone.records)) + self.assertEquals(15, len(self.expected.records)) + changes = self.expected.changes(zone, provider) + self.assertEquals(0, len(changes)) + + def test_apply(self): + provider = MythicBeastsProvider('test', { + 'unit.tests.': 'mypassword' + }) + zone = Zone('unit.tests.', []) + + # Create blank zone + with requests_mock() as mock: + mock.post(ANY, status_code=200, text='') + provider.populate(zone) + + self.assertEquals(0, len(zone.records)) + + # Check deleting and adding/changing test record + existing = 'prawf 300 TXT prawf prawf prawf\ndileu 300 TXT dileu' + + with requests_mock() as mock: + mock.post(ANY, status_code=200, text=existing) + + # Mash up a new zone with records so a plan + # is generated with changes and applied. For some reason + # passing self.expected, or just changing each record's zone + # doesn't work. Nor does this without a single add_record after + wanted = Zone('unit.tests.', []) + for record in list(self.expected.records): + data = {'type': record._type} + data.update(record.data) + wanted.add_record(Record.new(wanted, record.name, data)) + + wanted.add_record(Record.new(wanted, 'prawf', { + 'ttl': 60, + 'type': 'TXT', + 'value': 'prawf yw e', + })) + + plan = provider.plan(wanted) + + # Octo ignores NS records (15-1) + self.assertEquals(1, len(filter(lambda u: isinstance(u, Update), + plan.changes))) + self.assertEquals(1, len(filter(lambda d: isinstance(d, Delete), + plan.changes))) + self.assertEquals(14, len(filter(lambda c: isinstance(c, Create), + plan.changes))) + self.assertEquals(16, provider.apply(plan)) + self.assertTrue(plan.exists) From 57b03971d8c1c9b7f1f23b4fc592669eea53bd64 Mon Sep 17 00:00:00 2001 From: cwningen Date: Tue, 14 May 2019 14:46:49 +0100 Subject: [PATCH 013/176] Update octodns/provider/mythicbeasts.py Seems fair to me! I think a lot of the suggestions you've mentioned are obvious ones that have been lost on me from being very confused trying to understand the available objects Co-Authored-By: Ross McFarland --- octodns/provider/mythicbeasts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py index ca03b37..3d1f5ec 100644 --- a/octodns/provider/mythicbeasts.py +++ b/octodns/provider/mythicbeasts.py @@ -339,7 +339,7 @@ class MythicBeastsProvider(BaseProvider): base = '{} {} {} {}'.format(action, hostname, ttl, _type) - if re.match('[A]{1,4}', _type) is not None: + if _type in ('A', 'AAAA'): for value in values: commands.append('{} {}'.format(base, value)) From d6fb3310d53b3daa0c5cfd48f77f208d8ec08192 Mon Sep 17 00:00:00 2001 From: Rhosyn Celyn Date: Tue, 14 May 2019 16:12:53 +0100 Subject: [PATCH 014/176] Applied suggested modifications --- octodns/provider/mythicbeasts.py | 118 ++++++++++++-------- tests/test_octodns_provider_mythicbeasts.py | 11 +- 2 files changed, 75 insertions(+), 54 deletions(-) diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py index 3d1f5ec..af07ddb 100644 --- a/octodns/provider/mythicbeasts.py +++ b/octodns/provider/mythicbeasts.py @@ -32,16 +32,59 @@ def remove_trailing_dot(value): return value[:-1] +class MythicBeastsUnauthorizedException(Exception): + def __init__(self, zone, *args): + self.zone = zone + self.message = 'Mythic Beasts unauthorized for zone: {}'.format( + self.zone + ) + + super(MythicBeastsUnauthorizedException, self).__init__( + self.message, self.zone, *args) + + class MythicBeastsProvider(BaseProvider): ''' Mythic Beasts DNS API Provider - mythicbeasts: - class: octodns.provider.mythicbeasts.MythicBeastsProvider - zones: - my-zone: 'password' + Config settings: + + --- + providers: + config: + ... + mythicbeasts: + class: octodns.provider.mythicbeasts.MythicBeastsProvider + passwords: + my.domain.: 'password' + + zones: + my.domain.: + targets: + - mythic ''' + RE_MX = re.compile(r'^(?P[0-9]+)\s+(?P\S+)$', + re.IGNORECASE) + + RE_SRV = re.compile(r'^(?P[0-9]+)\s+(?P[0-9]+)\s+' + r'(?P[0-9]+)\s+(?P\S+)$', + re.IGNORECASE) + + RE_SSHFP = re.compile(r'^(?P[0-9]+)\s+' + r'(?P[0-9]+)\s+' + r'(?P\S+)$', + re.IGNORECASE) + + RE_CAA = re.compile(r'^(?P[0-9]+)\s+' + r'(?Pissue|issuewild|iodef)\s+' + r'(?P\S+)$', + re.IGNORECASE) + + RE_POPLINE = re.compile(r'^(?P\S+)\s+(?P\d+)\s+' + r'(?P\S+)\s+(?P.*)$', + re.IGNORECASE) + SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NS', @@ -51,7 +94,7 @@ class MythicBeastsProvider(BaseProvider): def __init__(self, identifier, passwords, *args, **kwargs): self.log = getLogger('MythicBeastsProvider[{}]'.format(identifier)) - assert isinstance(passwords, dict), 'Missing passwords' + assert isinstance(passwords, dict), 'Passwords must be a dictionary' self.log.debug( '__init__: id=%s, registered zones; %s', @@ -74,8 +117,7 @@ class MythicBeastsProvider(BaseProvider): resp.text[:20]) if resp.status_code == 401: - raise Exception('Mythic Beasts unauthorized for domain: {}' - .format(data['domain'])) + raise MythicBeastsUnauthorizedException(data['domain']) resp.raise_for_status() return resp @@ -118,17 +160,17 @@ class MythicBeastsProvider(BaseProvider): for raw_value in \ [raw_values['value'] for raw_values in data['raw_values']]: - match = re.match('^([0-9]+)\\s+(\\S+)$', raw_value, re.IGNORECASE) + match = MythicBeastsProvider.RE_MX.match(raw_value) assert match is not None, 'Unable to parse MX data' - exchange = match.group(2) + exchange = match.group('exchange') if not exchange.endswith('.'): exchange = '{}.{}'.format(exchange, data['zone']) values.append({ - 'preference': match.group(1), + 'preference': match.group('preference'), 'exchange': exchange, }) @@ -169,21 +211,18 @@ class MythicBeastsProvider(BaseProvider): for raw_value in \ [raw_values['value'] for raw_values in data['raw_values']]: - match = re.match( - '^([0-9]+)\\s+([0-9]+)\\s+([0-9]+)\\s+(\\S+)$', - raw_value, - re.IGNORECASE) + match = MythicBeastsProvider.RE_SRV.match(raw_value) assert match is not None, 'Unable to parse SRV data' - target = match.group(4) + target = match.group('target') if not target.endswith('.'): target = '{}.{}'.format(target, data['zone']) values.append({ - 'priority': match.group(1), - 'weight': match.group(2), - 'port': match.group(3), + 'priority': match.group('priority'), + 'weight': match.group('weight'), + 'port': match.group('port'), 'target': target, }) @@ -200,17 +239,14 @@ class MythicBeastsProvider(BaseProvider): for raw_value in \ [raw_values['value'] for raw_values in data['raw_values']]: - match = re.match( - '^([0-9]+)\\s+([0-9]+)\\s+(\\S+)$', - raw_value, - re.IGNORECASE) + match = MythicBeastsProvider.RE_SSHFP.match(raw_value) assert match is not None, 'Unable to parse SSHFP data' values.append({ - 'algorithm': match.group(1), - 'fingerprint_type': match.group(2), - 'fingerprint': match.group(3), + 'algorithm': match.group('algorithm'), + 'fingerprint_type': match.group('fingerprint_type'), + 'fingerprint': match.group('fingerprint'), }) return { @@ -224,17 +260,14 @@ class MythicBeastsProvider(BaseProvider): ttl = data['raw_values'][0]['ttl'] raw_value = data['raw_values'][0]['value'] - match = re.match( - '^([0-9]+)\\s+(issue|issuewild|iodef)\\s+(\\S+)$', - raw_value, - re.IGNORECASE) + match = MythicBeastsProvider.RE_CAA.match(raw_value) assert match is not None, 'Unable to parse CAA data' value = { - 'flags': match.group(1), - 'tag': match.group(2), - 'value': match.group(3), + 'flags': match.group('flags'), + 'tag': match.group('tag'), + 'value': match.group('value'), } return MythicBeastsProvider._data_for_single( @@ -258,10 +291,7 @@ class MythicBeastsProvider(BaseProvider): exists = True for line in resp.content.splitlines(): - match = re.match( - '^(\\S+)\\s+(\\d+)\\s+(\\S+)\\s+(.*)$', - line, - re.IGNORECASE) + match = MythicBeastsProvider.RE_POPLINE.match(line) if match is None: self.log.debug('failed to match line: %s', line) @@ -270,11 +300,11 @@ class MythicBeastsProvider(BaseProvider): if match.group(1) == '@': _name = '' else: - _name = match.group(1) + _name = match.group('name') - _type = match.group(3) - _ttl = int(match.group(2)) - _value = match.group(4).strip() + _type = match.group('type') + _ttl = int(match.group('ttl')) + _value = match.group('value').strip() if _type == 'TXT': _value = _value.replace(';', '\\;') @@ -318,13 +348,7 @@ class MythicBeastsProvider(BaseProvider): def _compile_commands(self, action, change): commands = [] - record = None - - if action == 'ADD': - record = change.new - else: - record = change.existing - + record = change.record hostname = remove_trailing_dot(record.fqdn) ttl = record.ttl _type = record._type diff --git a/tests/test_octodns_provider_mythicbeasts.py b/tests/test_octodns_provider_mythicbeasts.py index a1ef5d5..5deba3b 100644 --- a/tests/test_octodns_provider_mythicbeasts.py +++ b/tests/test_octodns_provider_mythicbeasts.py @@ -211,11 +211,7 @@ class TestMythicBeastsProvider(TestCase): self.ttl = 60 @property - def new(self): - return self - - @property - def existing(self): + def record(self): return self @property @@ -238,7 +234,8 @@ class TestMythicBeastsProvider(TestCase): # Null passwords dict with self.assertRaises(AssertionError) as err: provider = MythicBeastsProvider('test', None) - self.assertEquals('Missing passwords', err.exception.message) + self.assertEquals('Passwords must be a dictionary', + err.exception.message) # Missing password with requests_mock() as mock: @@ -263,7 +260,7 @@ class TestMythicBeastsProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals( - 'Mythic Beasts unauthorized for domain: unit.tests', + 'Mythic Beasts unauthorized for zone: unit.tests', err.exception.message) # Check unmatched lines are ignored From d49bf262205e525a484949a2dd489c58011ff559 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 14 May 2019 20:25:14 -0700 Subject: [PATCH 015/176] Handle Route53 extra check much more thoroughly by breaking down name Also adds thorough tests --- octodns/provider/route53.py | 20 ++++++++++++++++---- tests/test_octodns_provider_route53.py | 26 +++++++++++++++++++++++--- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index e8b11fa..1516f43 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -1250,15 +1250,27 @@ class Route53Provider(BaseProvider): '%s', record.fqdn, record._type) fqdn = record.fqdn + _type = record._type # loop through all the r53 rrsets for rrset in self._load_records(zone_id): name = rrset['Name'] + # Break off the first piece of the name, it'll let us figure out if + # this is an rrset we're interested in. + maybe_meta, rest = name.split('.', 1) - if record._type == rrset['Type'] and name.endswith(fqdn) and \ - name.startswith('_octodns-') and '-value.' in name and \ - '-default-' not in name and \ - self._extra_changes_update_needed(record, rrset): + if not maybe_meta.startswith('_octodns-') or \ + not maybe_meta.endswith('-value') or \ + '-default-' in name: + # We're only interested in non-default dynamic value records, + # as that's where healthchecks live + continue + + if rest != fqdn or _type != rrset['Type']: + # rrset isn't for the current record + continue + + if self._extra_changes_update_needed(record, rrset): # no good, doesn't have the right health check, needs an update self.log.info('_extra_changes_dynamic_needs_update: ' 'health-check caused update of %s:%s', diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index added7f..dbd443f 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -1673,7 +1673,7 @@ class TestRoute53Provider(TestCase): desired.add_record(record) list_resource_record_sets_resp = { 'ResourceRecordSets': [{ - # other name + # Not dynamic value and other name 'Name': 'unit.tests.', 'Type': 'A', 'GeoLocation': { @@ -1684,7 +1684,7 @@ class TestRoute53Provider(TestCase): }], 'TTL': 61, }, { - # matching name, other type + # Not dynamic value, matching name, other type 'Name': 'a.unit.tests.', 'Type': 'AAAA', 'ResourceRecords': [{ @@ -1693,7 +1693,7 @@ class TestRoute53Provider(TestCase): 'TTL': 61, }, { # default value pool - 'Name': '_octodns-default-pool.a.unit.tests.', + 'Name': '_octodns-default-value.a.unit.tests.', 'Type': 'A', 'GeoLocation': { 'CountryCode': '*', @@ -1702,6 +1702,26 @@ class TestRoute53Provider(TestCase): 'Value': '1.2.3.4', }], 'TTL': 61, + }, { + # different record + 'Name': '_octodns-two-value.other.unit.tests.', + 'Type': 'A', + 'GeoLocation': { + 'CountryCode': '*', + }, + 'ResourceRecords': [{ + 'Value': '1.2.3.4', + }], + 'TTL': 61, + }, { + # same everything, but different type + 'Name': '_octodns-one-value.a.unit.tests.', + 'Type': 'AAAA', + 'ResourceRecords': [{ + 'Value': '2001:0db8:3c4d:0015:0000:0000:1a2f:1a4b' + }], + 'TTL': 61, + 'HealthCheckId': '42', }, { # match 'Name': '_octodns-one-value.a.unit.tests.', From ee0416de9a77064153b4283d6d4a709c4510125a Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 14 May 2019 20:32:36 -0700 Subject: [PATCH 016/176] Cover more Route53 extra check edge cases and ensure it tests what we're after --- tests/test_octodns_provider_route53.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index dbd443f..849ea2b 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -1683,6 +1683,9 @@ class TestRoute53Provider(TestCase): 'Value': '1.2.3.4', }], 'TTL': 61, + # All the non-matches have a different Id so we'll fail if they + # match + 'HealthCheckId': '33', }, { # Not dynamic value, matching name, other type 'Name': 'a.unit.tests.', @@ -1691,6 +1694,7 @@ class TestRoute53Provider(TestCase): 'Value': '2001:0db8:3c4d:0015:0000:0000:1a2f:1a4b' }], 'TTL': 61, + 'HealthCheckId': '33', }, { # default value pool 'Name': '_octodns-default-value.a.unit.tests.', @@ -1702,6 +1706,7 @@ class TestRoute53Provider(TestCase): 'Value': '1.2.3.4', }], 'TTL': 61, + 'HealthCheckId': '33', }, { # different record 'Name': '_octodns-two-value.other.unit.tests.', @@ -1713,6 +1718,7 @@ class TestRoute53Provider(TestCase): 'Value': '1.2.3.4', }], 'TTL': 61, + 'HealthCheckId': '33', }, { # same everything, but different type 'Name': '_octodns-one-value.a.unit.tests.', @@ -1721,7 +1727,16 @@ class TestRoute53Provider(TestCase): 'Value': '2001:0db8:3c4d:0015:0000:0000:1a2f:1a4b' }], 'TTL': 61, - 'HealthCheckId': '42', + 'HealthCheckId': '33', + }, { + # same everything, sub + 'Name': '_octodns-one-value.sub.a.unit.tests.', + 'Type': 'A', + 'ResourceRecords': [{ + 'Value': '1.2.3.4', + }], + 'TTL': 61, + 'HealthCheckId': '33', }, { # match 'Name': '_octodns-one-value.a.unit.tests.', From 53c2b8d19434b3049accbd7e3fdc08dddde49bec Mon Sep 17 00:00:00 2001 From: ItsAlex Date: Wed, 15 May 2019 12:30:11 +0200 Subject: [PATCH 017/176] fix: prevent digital ocean provider to crash if records type is not supported --- octodns/provider/digitalocean.py | 4 ++++ tests/fixtures/digitalocean-page-1.json | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/octodns/provider/digitalocean.py b/octodns/provider/digitalocean.py index 98a78ad..e192543 100644 --- a/octodns/provider/digitalocean.py +++ b/octodns/provider/digitalocean.py @@ -223,6 +223,10 @@ class DigitalOceanProvider(BaseProvider): values = defaultdict(lambda: defaultdict(list)) for record in self.zone_records(zone): _type = record['type'] + if _type not in self.SUPPORTS: + self.log.warning('populate: skipping unsupported %s record', + _type) + continue values[record['name']][record['type']].append(record) before = len(zone.records) diff --git a/tests/fixtures/digitalocean-page-1.json b/tests/fixtures/digitalocean-page-1.json index db231ba..c931411 100644 --- a/tests/fixtures/digitalocean-page-1.json +++ b/tests/fixtures/digitalocean-page-1.json @@ -1,5 +1,16 @@ { "domain_records": [{ + "id": null, + "type": "SOA", + "name": "@", + "data": null, + "priority": null, + "port": null, + "ttl": null, + "weight": null, + "flags": null, + "tag": null + }, { "id": 11189874, "type": "NS", "name": "@", From 1c08a4d58ea1e9129c660b74c1eac2f47d051894 Mon Sep 17 00:00:00 2001 From: Theo Julienne Date: Tue, 28 May 2019 14:03:33 +1000 Subject: [PATCH 018/176] Adjust Route53 change ordering to strictly order by dependency. --- octodns/provider/route53.py | 76 +++++++++++++++++++------- tests/test_octodns_provider_route53.py | 62 +++++++++++++++------ 2 files changed, 99 insertions(+), 39 deletions(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 1516f43..4cf7c99 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -519,37 +519,71 @@ class _Route53GeoRecord(_Route53Record): self.values) -_mod_keyer_action_order = { - 'DELETE': 0, # Delete things first - 'CREATE': 1, # Then Create things - 'UPSERT': 2, # Upsert things last -} - - def _mod_keyer(mod): rrset = mod['ResourceRecordSet'] - action_order = _mod_keyer_action_order[mod['Action']] - # We're sorting by 3 "columns", the action, the rrset type, and finally the - # name/id of the rrset. This ensures that Route53 won't see a RRSet that - # targets another that hasn't been seen yet. I.e. targets must come before - # things that target them. We sort on types of things rather than - # explicitly looking for targeting relationships since that's sufficent and - # easier to grok/do. + # Route53 requires that changes are ordered such that a target of an + # AliasTarget is created or upserted prior to the record that targets it. + # This is complicated by "UPSERT" appearing to be implemented as "DELETE" + # before all changes, followed by a "CREATE", internally in the AWS API. + # Because of this, we order changes as follows: + # - Delete any records that we wish to delete that are GEOS + # (because they are never targetted by anything) + # - Delete any records that we wish to delete that are SECONDARY + # (because they are no longer targetted by GEOS) + # - Delete any records that we wish to delete that are PRIMARY + # (because they are no longer targetted by SECONDARY) + # - Delete any records that we wish to delete that are VALUES + # (because they are no longer targetted by PRIMARY) + # - CREATE/UPSERT any records that are VALUES + # (because they don't depend on other records) + # - CREATE/UPSERT any records that are PRIMARY + # (because they always point to VALUES which now exist) + # - CREATE/UPSERT any records that are SECONDARY + # (because they now have PRIMARY records to target) + # - CREATE/UPSERT any records that are GEOS + # (because they now have all their PRIMARY pools to target) + # - :tada: + # + # In theory we could also do this based on actual target reference + # checking, but that's more complex. Since our rules have a known + # dependency order, we just rely on that. + # Get the unique ID from the name/id to get a consistent ordering. if rrset.get('GeoLocation', False): - return (action_order, 3, rrset['SetIdentifier']) + unique_id = rrset['SetIdentifier'] + else: + unique_id = rrset['Name'] + + # Prioritise within the action_priority, ensuring targets come first. + if rrset.get('GeoLocation', False): + # Geos reference pools, so they come last. + record_priority = 3 elif rrset.get('AliasTarget', False): # We use an alias if rrset.get('Failover', False) == 'SECONDARY': - # We're a secondary we'll ref primaries - return (action_order, 2, rrset['Name']) + # We're a secondary, which reference the primary (failover, P1). + record_priority = 2 else: - # We're a primary we'll ref values - return (action_order, 1, rrset['Name']) + # We're a primary, we reference values (P0). + record_priority = 1 + else: + # We're just a plain value, has no dependencies so first. + record_priority = 0 - # We're just a plain value, these come first - return (action_order, 0, rrset['Name']) + if mod['Action'] == 'DELETE': + # Delete things first, so we can never trounce our own additions + action_priority = 0 + # Delete in the reverse order of priority, e.g. start with the deepest + # reference and work back to the values, rather than starting at the + # values (still ref'd). + record_priority = -record_priority + else: + # For CREATE and UPSERT, Route53 seems to treat them the same, so + # interleave these, keeping the reference order described above. + action_priority = 1 + + return (action_priority, record_priority, unique_id) def _parse_pool_name(n): diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 849ea2b..265a0a7 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -700,18 +700,6 @@ class TestRoute53Provider(TestCase): 'TTL': 61, 'Type': 'A' } - }, { - 'Action': 'CREATE', - 'ResourceRecordSet': { - 'GeoLocation': {'CountryCode': 'US', - 'SubdivisionCode': 'CA'}, - 'HealthCheckId': u'44', - 'Name': 'unit.tests.', - 'ResourceRecords': [{'Value': '7.2.3.4'}], - 'SetIdentifier': 'NA-US-CA', - 'TTL': 61, - 'Type': 'A' - } }, { 'Action': 'UPSERT', 'ResourceRecordSet': { @@ -735,6 +723,18 @@ class TestRoute53Provider(TestCase): 'TTL': 61, 'Type': 'A' } + }, { + 'Action': 'CREATE', + 'ResourceRecordSet': { + 'GeoLocation': {'CountryCode': 'US', + 'SubdivisionCode': 'CA'}, + 'HealthCheckId': u'44', + 'Name': 'unit.tests.', + 'ResourceRecords': [{'Value': '7.2.3.4'}], + 'SetIdentifier': 'NA-US-CA', + 'TTL': 61, + 'Type': 'A' + } }, { 'Action': 'UPSERT', 'ResourceRecordSet': { @@ -2426,7 +2426,7 @@ class TestModKeyer(TestCase): def test_mod_keyer(self): - # First "column" + # First "column" is the action priority for C/R/U # Deletes come first self.assertEquals((0, 0, 'something'), _mod_keyer({ @@ -2444,8 +2444,8 @@ class TestModKeyer(TestCase): } })) - # Then upserts - self.assertEquals((2, 0, 'last'), _mod_keyer({ + # Upserts are the same as creates + self.assertEquals((1, 0, 'last'), _mod_keyer({ 'Action': 'UPSERT', 'ResourceRecordSet': { 'Name': 'last', @@ -2455,7 +2455,7 @@ class TestModKeyer(TestCase): # Second "column" value records tested above # AliasTarget primary second (to value) - self.assertEquals((0, 1, 'thing'), _mod_keyer({ + self.assertEquals((0, -1, 'thing'), _mod_keyer({ 'Action': 'DELETE', 'ResourceRecordSet': { 'AliasTarget': 'some-target', @@ -2464,8 +2464,17 @@ class TestModKeyer(TestCase): } })) + self.assertEquals((1, 1, 'thing'), _mod_keyer({ + 'Action': 'UPSERT', + 'ResourceRecordSet': { + 'AliasTarget': 'some-target', + 'Failover': 'PRIMARY', + 'Name': 'thing', + } + })) + # AliasTarget secondary third - self.assertEquals((0, 2, 'thing'), _mod_keyer({ + self.assertEquals((0, -2, 'thing'), _mod_keyer({ 'Action': 'DELETE', 'ResourceRecordSet': { 'AliasTarget': 'some-target', @@ -2474,8 +2483,17 @@ class TestModKeyer(TestCase): } })) + self.assertEquals((1, 2, 'thing'), _mod_keyer({ + 'Action': 'UPSERT', + 'ResourceRecordSet': { + 'AliasTarget': 'some-target', + 'Failover': 'SECONDARY', + 'Name': 'thing', + } + })) + # GeoLocation fourth - self.assertEquals((0, 3, 'some-id'), _mod_keyer({ + self.assertEquals((0, -3, 'some-id'), _mod_keyer({ 'Action': 'DELETE', 'ResourceRecordSet': { 'GeoLocation': 'some-target', @@ -2483,4 +2501,12 @@ class TestModKeyer(TestCase): } })) + self.assertEquals((1, 3, 'some-id'), _mod_keyer({ + 'Action': 'UPSERT', + 'ResourceRecordSet': { + 'GeoLocation': 'some-target', + 'SetIdentifier': 'some-id', + } + })) + # The third "column" has already been tested above, Name/SetIdentifier From 7eaf03dec40a809a36b5554e5391f64656827bef Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 30 May 2019 17:33:42 -0700 Subject: [PATCH 019/176] started fast dns --- octodns/provider/fastdns.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 octodns/provider/fastdns.py diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py new file mode 100644 index 0000000..59318cc --- /dev/null +++ b/octodns/provider/fastdns.py @@ -0,0 +1 @@ +## Provider file for Akamai Fast DNS \ No newline at end of file From 95681972b7722e8d880574ba56f7ccbbae45dcf3 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 30 May 2019 18:25:28 -0700 Subject: [PATCH 020/176] starting authentication for akamai fastdns --- octodns/provider/fastdns.py | 55 ++++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 59318cc..3504e02 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -1 +1,54 @@ -## Provider file for Akamai Fast DNS \ No newline at end of file +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +## octodns specfic imports: + +import + +import logging +from functools import reduce +from ..record import Record +from .base import BaseProvider + + +class _AkamaiRecord(object): + pass + + + + +class AkamaiProvider(BaseProvider): + pass + + def __init__(self, id, client_secret, host, access_token, client_token, + *args, **kwargs): + + self.log = logging.getLogger('AkamaiProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, ') + super(AkamaiProvider, self).__init__(id, *args, **kwargs) + + ## generate edgegrid + #### create credential file, and write credentials to it + credFile = open('tempCred.txt', 'w') + + credFile.write('client_secret = ') + credFile.write(str(client_secret)) + credFile.write('\n\n') + + credFile.write('host = %s', str(host)) + credFile.write('\n\n') + + credFile.write('access_token = %s', str(access_token)) + credFile.write('\n\n') + + credFile.write('client_token = %s', str(client_token)) + + credFile.close() + + #### generate edgegrid using tool + + #### delete temp txt file \ No newline at end of file From f7239e30a5766f19b14cffc5dbe79159a6989d37 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 30 May 2019 18:33:03 -0700 Subject: [PATCH 021/176] fixed import bug --- octodns/provider/fastdns.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 3504e02..2a24edc 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -7,8 +7,6 @@ from __future__ import absolute_import, division, print_function, \ ## octodns specfic imports: -import - import logging from functools import reduce from ..record import Record From 99875a128999c5090c9359cfac1d30850ce48a02 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 30 May 2019 18:36:31 -0700 Subject: [PATCH 022/176] fixed base class errors --- octodns/provider/fastdns.py | 7 ++++++- src/octodns | 1 + src/pip-delete-this-directory.txt | 5 +++++ 3 files changed, 12 insertions(+), 1 deletion(-) create mode 160000 src/octodns create mode 100644 src/pip-delete-this-directory.txt diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 2a24edc..a9591d4 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -20,8 +20,13 @@ class _AkamaiRecord(object): class AkamaiProvider(BaseProvider): - pass + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = False + SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', + 'TXT')) + + def __init__(self, id, client_secret, host, access_token, client_token, *args, **kwargs): diff --git a/src/octodns b/src/octodns new file mode 160000 index 0000000..b57cbbb --- /dev/null +++ b/src/octodns @@ -0,0 +1 @@ +Subproject commit b57cbbb09230684552f1fa4db5fb53a921486b96 diff --git a/src/pip-delete-this-directory.txt b/src/pip-delete-this-directory.txt new file mode 100644 index 0000000..c8883ea --- /dev/null +++ b/src/pip-delete-this-directory.txt @@ -0,0 +1,5 @@ +This file is placed here by pip to indicate the source was put +here by pip. + +Once this package is successfully installed this source code will be +deleted (unless you remove this file). From 383452809010f423cac9e48f28af5685bc6f2ee4 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 30 May 2019 18:49:41 -0700 Subject: [PATCH 023/176] fixed file writing error --- octodns/provider/fastdns.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index a9591d4..be4df69 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -42,13 +42,13 @@ class AkamaiProvider(BaseProvider): credFile.write(str(client_secret)) credFile.write('\n\n') - credFile.write('host = %s', str(host)) + credFile.write('host = ' + str(host)) credFile.write('\n\n') - credFile.write('access_token = %s', str(access_token)) + credFile.write('access_token = ' + str(access_token)) credFile.write('\n\n') - credFile.write('client_token = %s', str(client_token)) + credFile.write('client_token = ' + str(client_token)) credFile.close() From e7d82ac75d74f1daf2cb9ad76e67f1cab690a4de Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Tue, 4 Jun 2019 12:16:14 -0700 Subject: [PATCH 024/176] sync --- octodns/provider/fastdns.py | 1 + 1 file changed, 1 insertion(+) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index be4df69..439f924 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -51,6 +51,7 @@ class AkamaiProvider(BaseProvider): credFile.write('client_token = ' + str(client_token)) credFile.close() + #### generate edgegrid using tool From 064b30c4d1b9f29eb3f392b33838664d1665a4fc Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 09:29:45 -0700 Subject: [PATCH 025/176] began authentication script --- octodns/provider/fastdns.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 439f924..c955cdf 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -49,10 +49,16 @@ class AkamaiProvider(BaseProvider): credFile.write('\n\n') credFile.write('client_token = ' + str(client_token)) + credFile.write('\n') credFile.close() - + #### generate edgegrid using tool - #### delete temp txt file \ No newline at end of file + #### delete temp txt file + + def populate(self, zone, target=False, lenient=False): + pass + + From 34b618fbee77de96bf1c8bc6afb4fd8762296c6b Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 09:45:27 -0700 Subject: [PATCH 026/176] began authentication script --- octodns/provider/fastdns.py | 44 +++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index c955cdf..a01f71e 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -6,6 +6,9 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals ## octodns specfic imports: +from os.path import expanduser +import ConfigParser + import logging from functools import reduce @@ -34,8 +37,49 @@ class AkamaiProvider(BaseProvider): self.log.debug('__init__: id=%s, ') super(AkamaiProvider, self).__init__(id, *args, **kwargs) + self._authenticate(self, client_secret, host, access_token, client_token) + + + def _authenticate(self, client_secret, host, access_token, client_token) + ## generate edgegrid + section_name = "dns" + home = expanduser("~") + index = 0 + fields = {} + + ## process original .edgerc file + origConfig = ConfigParser.ConfigParser() + filename = "%s/.edgerc" % home + + # If this is a new file, create it + if not os.path.isfile(filename): + open(filename, 'a+').close() + + origConfig.read(filename) + + + if section_name in origConfig.sections(): + print (">>> Replacing section: %s" % section_name) + + + + sys.stdout.write ("*** OK TO REPLACE section %s? *** [Y|n]:" % section_name) + real_raw_input = vars(__builtins__).get('raw_input',input) + choice = real_raw_input().lower() + if choice == "n": + print ("Not replacing section.") + exit(0) + + replace_section = True + else: + print ("+++ Creating section: %s" % section_name) + replace_section = False + + #### create credential file, and write credentials to it + + credFile = open('tempCred.txt', 'w') credFile.write('client_secret = ') From e1a139d18753ecb7f68acf7c9a051c686fe5332d Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 14:31:26 -0400 Subject: [PATCH 027/176] revamed authentication --- octodns/provider/fastdns.py | 90 +++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 48 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index a01f71e..597c68a 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -9,6 +9,8 @@ from __future__ import absolute_import, division, print_function, \ from os.path import expanduser import ConfigParser +# import os ## command line workaround + import logging from functools import reduce @@ -42,65 +44,57 @@ class AkamaiProvider(BaseProvider): def _authenticate(self, client_secret, host, access_token, client_token) - ## generate edgegrid - section_name = "dns" - home = expanduser("~") - index = 0 - fields = {} + # ## generate edgegrid + # section_name = "dns" + # home = expanduser("~") + # index = 0 + # fields = {} - ## process original .edgerc file - origConfig = ConfigParser.ConfigParser() - filename = "%s/.edgerc" % home + # ## process original .edgerc file + # origConfig = ConfigParser.ConfigParser() + # filename = "%s/.edgerc" % home - # If this is a new file, create it - if not os.path.isfile(filename): - open(filename, 'a+').close() + # # If this is a new file, create it + # if not os.path.isfile(filename): + # open(filename, 'a+').close() - origConfig.read(filename) + # origConfig.read(filename) + + # ## credential type already exists, prompt to overwrite + # if section_name in origConfig.sections(): + # print (">>> Replacing section: %s" % section_name) + # sys.stdout.write ("*** OK TO REPLACE section %s? *** [Y|n]:" % section_name) + # real_raw_input = vars(__builtins__).get('raw_input',input) + # choice = real_raw_input().lower() + # if choice == "n": + # print ("Not replacing section.") + # return + # replace_section = True + # else: + # print ("+++ Creating section: %s" % section_name) + # replace_section = False + + # ## open the ~/.edgerc file for writing + # Config = ConfigParser.ConfigParser() + # Config.read(filename) + # confile = open(filename, 'w') - if section_name in origConfig.sections(): - print (">>> Replacing section: %s" % section_name) - - - - sys.stdout.write ("*** OK TO REPLACE section %s? *** [Y|n]:" % section_name) - real_raw_input = vars(__builtins__).get('raw_input',input) - choice = real_raw_input().lower() - if choice == "n": - print ("Not replacing section.") - exit(0) - - replace_section = True - else: - print ("+++ Creating section: %s" % section_name) - replace_section = False + # ## add the new section + #### create credential file, and write credentials to it + with open('~/.edgerc', 'a') as credFile: - credFile = open('tempCred.txt', 'w') + credFile.write('[dns]\n') - credFile.write('client_secret = ') - credFile.write(str(client_secret)) - credFile.write('\n\n') - - credFile.write('host = ' + str(host)) - credFile.write('\n\n') - - credFile.write('access_token = ' + str(access_token)) - credFile.write('\n\n') - - credFile.write('client_token = ' + str(client_token)) - credFile.write('\n') - - credFile.close() - - - #### generate edgegrid using tool - - #### delete temp txt file + credFile.write('client_secret = ' + str(client_secret) + '\n') + credFile.write('host = ' + str(host) + '\n') + credFile.write('access_token = ' + str(access_token) + '\n') + credFile.write('client_token = ' + str(client_token) + '\n') + credFile.close() def populate(self, zone, target=False, lenient=False): pass From d4867c4268c90408e3db49c1f92deaa68a8869d1 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 14:40:41 -0400 Subject: [PATCH 028/176] fixed typo in authernticate --- octodns/provider/fastdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 597c68a..94c049e 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -42,7 +42,7 @@ class AkamaiProvider(BaseProvider): self._authenticate(self, client_secret, host, access_token, client_token) - def _authenticate(self, client_secret, host, access_token, client_token) + def _authenticate(self, client_secret, host, access_token, client_token): # ## generate edgegrid # section_name = "dns" From f21915308850a7402d75b5e28a6d8647ec4f8321 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 14:49:58 -0400 Subject: [PATCH 029/176] fixed call to auth fxn --- octodns/provider/fastdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 94c049e..e8fb35c 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -39,7 +39,7 @@ class AkamaiProvider(BaseProvider): self.log.debug('__init__: id=%s, ') super(AkamaiProvider, self).__init__(id, *args, **kwargs) - self._authenticate(self, client_secret, host, access_token, client_token) + self._authenticate(client_secret, host, access_token, client_token) def _authenticate(self, client_secret, host, access_token, client_token): From 3d61baea5060ed1ff75ec14d6b8a2153b8c99780 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 15:00:04 -0400 Subject: [PATCH 030/176] fixed issue with accessing home dir --- octodns/provider/fastdns.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index e8fb35c..1ce91d0 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -86,7 +86,10 @@ class AkamaiProvider(BaseProvider): #### create credential file, and write credentials to it - with open('~/.edgerc', 'a') as credFile: + + home = expanduser("~") + filename = "%s/.edgerc" % home + with open(filename, 'a') as credFile: credFile.write('[dns]\n') From 2ea26273b807b1e4a955133a391512a25cc82d10 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 17:25:16 -0400 Subject: [PATCH 031/176] started populate fxn --- octodns/provider/fastdns.py | 57 +++++++------------------------------ 1 file changed, 10 insertions(+), 47 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 1ce91d0..56e94f8 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -7,9 +7,9 @@ from __future__ import absolute_import, division, print_function, \ ## octodns specfic imports: from os.path import expanduser -import ConfigParser - -# import os ## command line workaround +import requests +from akamai.edgegrid import EdgeGridAuth +from urlparse import urljoin import logging @@ -40,53 +40,12 @@ class AkamaiProvider(BaseProvider): super(AkamaiProvider, self).__init__(id, *args, **kwargs) self._authenticate(client_secret, host, access_token, client_token) + self._zone_records = {} def _authenticate(self, client_secret, host, access_token, client_token): - # ## generate edgegrid - # section_name = "dns" - # home = expanduser("~") - # index = 0 - # fields = {} - - # ## process original .edgerc file - # origConfig = ConfigParser.ConfigParser() - # filename = "%s/.edgerc" % home - - # # If this is a new file, create it - # if not os.path.isfile(filename): - # open(filename, 'a+').close() - - # origConfig.read(filename) - - # ## credential type already exists, prompt to overwrite - # if section_name in origConfig.sections(): - # print (">>> Replacing section: %s" % section_name) - # sys.stdout.write ("*** OK TO REPLACE section %s? *** [Y|n]:" % section_name) - # real_raw_input = vars(__builtins__).get('raw_input',input) - # choice = real_raw_input().lower() - # if choice == "n": - # print ("Not replacing section.") - # return - # replace_section = True - # else: - # print ("+++ Creating section: %s" % section_name) - # replace_section = False - - # ## open the ~/.edgerc file for writing - # Config = ConfigParser.ConfigParser() - # Config.read(filename) - # confile = open(filename, 'w') - - - # ## add the new section - - - - #### create credential file, and write credentials to it - - + ## generate edgegrid home = expanduser("~") filename = "%s/.edgerc" % home with open(filename, 'a') as credFile: @@ -100,6 +59,10 @@ class AkamaiProvider(BaseProvider): credFile.close() def populate(self, zone, target=False, lenient=False): - pass + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + + print ("populate(%s)", zone.name) + From 66136f475f2b41a3e3c3cac1f195dc71fba1e595 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 17:27:42 -0400 Subject: [PATCH 032/176] fixed typo --- octodns/provider/fastdns.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 56e94f8..7cd0593 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -59,10 +59,11 @@ class AkamaiProvider(BaseProvider): credFile.close() def populate(self, zone, target=False, lenient=False): - self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, - target, lenient) + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) print ("populate(%s)", zone.name) + return + From 4a68f7df259f40e06679512c8b8fcc7c059fc593 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 18:32:46 -0400 Subject: [PATCH 033/176] added dns client class, changed auth method --- octodns/provider/fastdns.py | 39 ++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 7cd0593..fe5602c 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -22,6 +22,29 @@ class _AkamaiRecord(object): pass +class AkamaiClient(object): + + def __init__(self, _client_secret, _host, _access_token, _client_token): + + self.base = "https://" + _host + "/config-dns/v1/zones/" + + sess = requests.Session() + sess.auth = EdgeGridAuth( + client_token=_client_token, + client_secret=_client_secret, + access_token=_access_token + ) + self._sess = sess + + + def getZone(self, name): + path = urljoin(self.base, name) + result = self.sess.get(path) + return result.json() + + + # def _get(self, method, path, params=None, data=None): + # url = '{}{}{}'.format(self.) class AkamaiProvider(BaseProvider): @@ -39,7 +62,12 @@ class AkamaiProvider(BaseProvider): self.log.debug('__init__: id=%s, ') super(AkamaiProvider, self).__init__(id, *args, **kwargs) - self._authenticate(client_secret, host, access_token, client_token) + + self._dns_client = AkamaiClient(client_secret, host, access_token, client_token) + + + + #self._authenticate(client_secret, host, access_token, client_token) self._zone_records = {} @@ -61,9 +89,14 @@ class AkamaiProvider(BaseProvider): def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) - print ("populate(%s)", zone.name) + print ("populate()", zone.name) - return + result = self._dns_client.getZone() + + print(result) + + return + From 6afab8c90a1033c51809a5d94f39edacaf7286fb Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 18:35:04 -0400 Subject: [PATCH 034/176] fixed typo --- octodns/provider/fastdns.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index fe5602c..afbbfc2 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -90,13 +90,14 @@ class AkamaiProvider(BaseProvider): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) print ("populate()", zone.name) + - result = self._dns_client.getZone() + result = self._dns_client.getZone(zone.name) print(result) return - + From b385b1ddb5d2a5bc7b4c3eb3e3907c4ef80ef7b4 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 18:36:16 -0400 Subject: [PATCH 035/176] fixed typo --- octodns/provider/fastdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index afbbfc2..da9d3ee 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -39,7 +39,7 @@ class AkamaiClient(object): def getZone(self, name): path = urljoin(self.base, name) - result = self.sess.get(path) + result = self._sess.get(path) return result.json() From 32f01da506961ad7ab0daf7ff3677aeadcb463fe Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 18:39:24 -0400 Subject: [PATCH 036/176] fixed zone name trailing . --- octodns/provider/fastdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index da9d3ee..4d0d2a9 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -91,7 +91,7 @@ class AkamaiProvider(BaseProvider): print ("populate()", zone.name) - + zone_name = zone.name[:len(zone_name)-1] result = self._dns_client.getZone(zone.name) print(result) From d2ec707255a1323de8fcaa26e792ea2390f75fa3 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 18:40:30 -0400 Subject: [PATCH 037/176] fixed typo --- octodns/provider/fastdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 4d0d2a9..b0b3c8e 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -91,7 +91,7 @@ class AkamaiProvider(BaseProvider): print ("populate()", zone.name) - zone_name = zone.name[:len(zone_name)-1] + zone_name = zone.name[:len(zone.name)-1] result = self._dns_client.getZone(zone.name) print(result) From a9fe4a9cf6bba95fbeb951ee2384323a59c263e8 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 18:42:26 -0400 Subject: [PATCH 038/176] fixed typo --- octodns/provider/fastdns.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index b0b3c8e..a58fbe5 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -92,7 +92,8 @@ class AkamaiProvider(BaseProvider): print ("populate()", zone.name) zone_name = zone.name[:len(zone.name)-1] - result = self._dns_client.getZone(zone.name) + + result = self._dns_client.getZone(zone_name) print(result) From 9c43c92339eaa0996fba256a9c92a538d49fac3c Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 18:52:55 -0400 Subject: [PATCH 039/176] json parse --- octodns/provider/fastdns.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index a58fbe5..7f50cb1 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -6,10 +6,10 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals ## octodns specfic imports: -from os.path import expanduser import requests from akamai.edgegrid import EdgeGridAuth from urlparse import urljoin +import json import logging @@ -65,12 +65,13 @@ class AkamaiProvider(BaseProvider): self._dns_client = AkamaiClient(client_secret, host, access_token, client_token) - - #self._authenticate(client_secret, host, access_token, client_token) self._zone_records = {} + + + def _authenticate(self, client_secret, host, access_token, client_token): ## generate edgegrid @@ -95,7 +96,11 @@ class AkamaiProvider(BaseProvider): result = self._dns_client.getZone(zone_name) - print(result) + + print "\n\n" + print json.dumps(result, indent=4, separators=(',', ': ')) + print "\n\n" + return From ec5e7a173ecc90b9a719bcc31a62fad2c6891a0c Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 18:54:19 -0400 Subject: [PATCH 040/176] fixed typo --- octodns/provider/fastdns.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 7f50cb1..caf17ae 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -97,9 +97,9 @@ class AkamaiProvider(BaseProvider): result = self._dns_client.getZone(zone_name) - print "\n\n" - print json.dumps(result, indent=4, separators=(',', ': ')) - print "\n\n" + print ("\n\n") + print (json.dumps(result, indent=4, separators=(',', ': '))) + print ("\n\n") return From c7b39053fe4d4876854d35faeb13ee69960dc39b Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 6 Jun 2019 19:16:37 -0400 Subject: [PATCH 041/176] sync --- octodns/provider/fastdns.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index caf17ae..fc37834 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -43,8 +43,6 @@ class AkamaiClient(object): return result.json() - # def _get(self, method, path, params=None, data=None): - # url = '{}{}{}'.format(self.) class AkamaiProvider(BaseProvider): @@ -90,10 +88,8 @@ class AkamaiProvider(BaseProvider): def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) - print ("populate()", zone.name) - + zone_name = zone.name[:len(zone.name)-1] - result = self._dns_client.getZone(zone_name) @@ -101,7 +97,6 @@ class AkamaiProvider(BaseProvider): print (json.dumps(result, indent=4, separators=(',', ': '))) print ("\n\n") - return From 4943b72b4cdfcf25d5eee2287ec4ea4752e616f4 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Mon, 10 Jun 2019 07:54:57 -0700 Subject: [PATCH 042/176] sync --- octodns/provider/.vscode/settings.json | 2 ++ octodns/provider/fastdns.py | 12 +++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 octodns/provider/.vscode/settings.json diff --git a/octodns/provider/.vscode/settings.json b/octodns/provider/.vscode/settings.json new file mode 100644 index 0000000..7a73a41 --- /dev/null +++ b/octodns/provider/.vscode/settings.json @@ -0,0 +1,2 @@ +{ +} \ No newline at end of file diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index fc37834..1405d17 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -40,8 +40,9 @@ class AkamaiClient(object): def getZone(self, name): path = urljoin(self.base, name) result = self._sess.get(path) - return result.json() + return result.json() + # return result @@ -91,9 +92,14 @@ class AkamaiProvider(BaseProvider): zone_name = zone.name[:len(zone.name)-1] result = self._dns_client.getZone(zone_name) - - print ("\n\n") + + print() + print() + + print(type(result)) + #print(result.text) + print (json.dumps(result, indent=4, separators=(',', ': '))) print ("\n\n") From 1aa9424e213dc0e70e6a03164778aae8a723db05 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Mon, 10 Jun 2019 14:23:26 -0700 Subject: [PATCH 043/176] working sync --- src/octodns | 1 - src/pip-delete-this-directory.txt | 5 ----- 2 files changed, 6 deletions(-) delete mode 160000 src/octodns delete mode 100644 src/pip-delete-this-directory.txt diff --git a/src/octodns b/src/octodns deleted file mode 160000 index b57cbbb..0000000 --- a/src/octodns +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b57cbbb09230684552f1fa4db5fb53a921486b96 diff --git a/src/pip-delete-this-directory.txt b/src/pip-delete-this-directory.txt deleted file mode 100644 index c8883ea..0000000 --- a/src/pip-delete-this-directory.txt +++ /dev/null @@ -1,5 +0,0 @@ -This file is placed here by pip to indicate the source was put -here by pip. - -Once this package is successfully installed this source code will be -deleted (unless you remove this file). From b58210328ff2ca7942fd6457ca45ca4c861aeb6f Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Wed, 12 Jun 2019 20:31:11 -0400 Subject: [PATCH 044/176] implemented API v2 - client fxns to request records functionality --- octodns/provider/fastdns.py | 127 +++++++++++++++++++++++++++--------- 1 file changed, 95 insertions(+), 32 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 1405d17..a6d5cae 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -18,6 +18,30 @@ from ..record import Record from .base import BaseProvider +class AkamaiClientException(Exception): + pass + +class AkamaiClientBadRequest(AkamaiClientException): #400 + def __init__(self): + super(AkamaiClientBadRequest, self).__init__('Bad request') + +class AkamaiClientNotAuthorized(AkamaiClientException): #403 + def __init__(self): + super(AkamaiClientNotAuthorized, self).__init__('Forbidden') + +class AkamaiClientNotFound(AkamaiClientException): #404 + def __init__(self): + super(AkamaiClientNotFound, self).__init__('Not found') + +class AkamaiClientNotAcceptable(AkamaiClientException): #406 + def __init__(self): + super(AkamaiClientNotAcceptable, self).__init__('Not acceptable') + +class AkamaiClientGenericExcp(AkamaiClientException): + def __init__(self, num): + super(AkamaiClientGenericExcp, self).__init__('HTTP Error: ' + str(num)) + + class _AkamaiRecord(object): pass @@ -26,7 +50,8 @@ class AkamaiClient(object): def __init__(self, _client_secret, _host, _access_token, _client_token): - self.base = "https://" + _host + "/config-dns/v1/zones/" + self.base = "https://" + _host + "/config-dns/v2/zones/" + self.basev1 = "https://" + _host + "/config-dns/v1/zones/" sess = requests.Session() sess.auth = EdgeGridAuth( @@ -36,22 +61,71 @@ class AkamaiClient(object): ) self._sess = sess + def _request(self, method, path, params=None, data=None): + # url = '{}{}'.format(self.base, path) + url = urljoin(self.base, path) + print(url) + resp = self._sess.request(method, url, params=params, json=data) - def getZone(self, name): - path = urljoin(self.base, name) - result = self._sess.get(path) + if resp.status_code == 400: + raise AkamaiClientBadRequest() + if resp.status_code == 403: + raise AkamaiClientNotAuthorized() + if resp.status_code == 404: + raise AkamaiClientNotFound() + if resp.status_code == 406: + raise AkamaiClientNotAcceptable() + resp.raise_for_status() + return resp + + + # def getZone(self, name): + # path = name + "/recordsets/" + # result = self._request('GET', path, "sortBy=type&types=A") + + # return result.json() + + # def getNames(self, name): + # path = name + "/names/" + # result = self._request('GET', path) + + # return result.json() + + + + def record_get(self, zone, name, record_type): + + path = '/zones/{}/names/{}/types/'.format(zone, name, record_type) + result = self._request('GET', path) + return result.json() - # return result + + def record_create(self, zone, name, record_type, params): + path = '/zones/{}/names/{}/types/'.format(zone, name, record_type) + result = self._request('POST', path, data=params) + + return result + def record_delete(self, zone, name, record_type): + path = '/zones/{}/names/{}/types/'.format(zone, name, record_type) + result = self._request('DELETE', path) + + return result + + def record_replace(self, zone, name, record_type, params): + path = '/zones/{}/names/{}/types/'.format(zone, name, record_type) + result = self._request('PUT', path, data=params) + + return result class AkamaiProvider(BaseProvider): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False - SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', - 'TXT')) + SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', + 'SRV', 'SSHFP', 'TXT')) def __init__(self, id, client_secret, host, access_token, client_token, @@ -64,44 +138,33 @@ class AkamaiProvider(BaseProvider): self._dns_client = AkamaiClient(client_secret, host, access_token, client_token) - #self._authenticate(client_secret, host, access_token, client_token) + self._zone_records = {} - - - def _authenticate(self, client_secret, host, access_token, client_token): - - ## generate edgegrid - home = expanduser("~") - filename = "%s/.edgerc" % home - with open(filename, 'a') as credFile: - - credFile.write('[dns]\n') - - credFile.write('client_secret = ' + str(client_secret) + '\n') - credFile.write('host = ' + str(host) + '\n') - credFile.write('access_token = ' + str(access_token) + '\n') - credFile.write('client_token = ' + str(client_token) + '\n') - credFile.close() - def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) zone_name = zone.name[:len(zone.name)-1] - result = self._dns_client.getZone(zone_name) + zone = self._dns_client.getZone(zone_name) + names = self._dns_client.getNames(zone_name) - - print() - print() - print(type(result)) + # print(type(result)) #print(result.text) - print (json.dumps(result, indent=4, separators=(',', ': '))) - print ("\n\n") + zoneFile = open('zoneFile.json', 'w') + namesFile = open('namesFile.json', 'w') + zoneFile.write(json.dumps(zone, indent=4, separators=(',', ': '))) + namesFile.write(json.dumps(names, indent=4, separators=(',', ': '))) + zoneFile.close() + namesFile.close() + + + + #print (json.dumps(result, indent=4, separators=(',', ': '))) return From e85fce6fe0751e426222d21a2837e466b0910078 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 13 Jun 2019 13:16:25 -0400 Subject: [PATCH 045/176] added test class to test API calls -- to be removed later --- octodns/provider/fastdns.py | 78 ++++++++++++++++++++++++------------- 1 file changed, 52 insertions(+), 26 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index a6d5cae..cb5123d 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -64,7 +64,23 @@ class AkamaiClient(object): def _request(self, method, path, params=None, data=None): # url = '{}{}'.format(self.base, path) url = urljoin(self.base, path) - print(url) + print(method, url) + + req = requests.Request(method, url, params=params, json=data) + prepped = req.prepare() + + filename = str(method) + ".json" + f = open(filename, 'w') + # f.write(json.dumps(prepped, indent=4, separators=(',', ': '))) + f.write(method + " " + url + "\n") + f.write("headers: \n") + f.write(str(prepped.headers)) + f.write("\nbody: \n") + f.write(str(prepped.body)) + f.close() + + + ''' resp = self._sess.request(method, url, params=params, json=data) if resp.status_code == 400: @@ -78,20 +94,22 @@ class AkamaiClient(object): resp.raise_for_status() return resp + ''' + return prepped + ''' + def getZone(self, name): + path = name + "/recordsets/" + result = self._request('GET', path, "sortBy=type&types=A") - # def getZone(self, name): - # path = name + "/recordsets/" - # result = self._request('GET', path, "sortBy=type&types=A") + return result.json() - # return result.json() - - # def getNames(self, name): - # path = name + "/names/" - # result = self._request('GET', path) - - # return result.json() + def getNames(self, name): + path = name + "/names/" + result = self._request('GET', path) + return result.json() + ''' def record_get(self, zone, name, record_type): @@ -99,6 +117,7 @@ class AkamaiClient(object): path = '/zones/{}/names/{}/types/'.format(zone, name, record_type) result = self._request('GET', path) + return return result.json() def record_create(self, zone, name, record_type, params): @@ -145,29 +164,36 @@ class AkamaiProvider(BaseProvider): def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) + self._test(zone) + + def _test(self, zone) : + zone_name = zone.name[:len(zone.name)-1] - zone = self._dns_client.getZone(zone_name) - names = self._dns_client.getNames(zone_name) - # print(type(result)) - #print(result.text) - - zoneFile = open('zoneFile.json', 'w') - namesFile = open('namesFile.json', 'w') - zoneFile.write(json.dumps(zone, indent=4, separators=(',', ': '))) - namesFile.write(json.dumps(names, indent=4, separators=(',', ': '))) - zoneFile.close() - namesFile.close() + record_name = "www.exampleRecordName.com" + record_type = "AAAA" + params = { + "name": "www.example.com", + "type": "AAAA", + "ttl": 300, + "rdata": [ + "10.0.0.2", + "10.0.0.3" + ] + } + + self._dns_client.record_get(zone_name, record_name, record_type) + self._dns_client.record_create(zone_name, record_name, record_type, params) + self._dns_client.record_delete(zone_name, record_name, record_type) + self._dns_client.record_replace(zone_name, record_name, record_type, params) + # domainRequest = self._dns_client.getZone(zone_name) + # names = self._dns_client.getNames(zone_name) - #print (json.dumps(result, indent=4, separators=(',', ': '))) return - - - From ea9c90c3bb7ed47ef83a1fdeaf0d51a12177019a Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 13 Jun 2019 18:26:56 -0400 Subject: [PATCH 046/176] major commit: refactored exceptions, implemented testing suite, implm zones get, zone recordset get, record api calls --- octodns/provider/fastdns.py | 311 +++++++++++++++++++++++++++++------- 1 file changed, 251 insertions(+), 60 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index cb5123d..56aa883 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -17,10 +17,28 @@ from functools import reduce from ..record import Record from .base import BaseProvider +TESTING = False class AkamaiClientException(Exception): - pass + + _errorMessages = { + 400: "400: Bad request", + 403: "403: Access is forbidden", + 404: "404: Resource not found", + 405: "405: Method not supported", + 406: "406: Not Acceptable", + 409: "409: Request not allowed due to conflict with current state of resource", + 415: "415: Unsupported media type", + 422: "422: Request body contains an error preventing processing", + 500: "500: Internal server error" + } + def __init__(self, code): + message = self._errorMessages.get(code) + super(AkamaiClientException, self).__init__(message) + + +''' class AkamaiClientBadRequest(AkamaiClientException): #400 def __init__(self): super(AkamaiClientBadRequest, self).__init__('Bad request') @@ -39,8 +57,8 @@ class AkamaiClientNotAcceptable(AkamaiClientException): #406 class AkamaiClientGenericExcp(AkamaiClientException): def __init__(self, num): - super(AkamaiClientGenericExcp, self).__init__('HTTP Error: ' + str(num)) - + super(AkamaiClientGenericExcp, self).__init__('HTTP Error: ' + str(num)) +''' class _AkamaiRecord(object): pass @@ -50,8 +68,8 @@ class AkamaiClient(object): def __init__(self, _client_secret, _host, _access_token, _client_token): - self.base = "https://" + _host + "/config-dns/v2/zones/" - self.basev1 = "https://" + _host + "/config-dns/v1/zones/" + self.base = "https://" + _host + "/config-dns/v2/" + self.basev1 = "https://" + _host + "/config-dns/v1/" sess = requests.Session() sess.auth = EdgeGridAuth( @@ -60,85 +78,126 @@ class AkamaiClient(object): access_token=_access_token ) self._sess = sess - - def _request(self, method, path, params=None, data=None): - # url = '{}{}'.format(self.base, path) - url = urljoin(self.base, path) - print(method, url) - req = requests.Request(method, url, params=params, json=data) - prepped = req.prepare() - filename = str(method) + ".json" + def _writePrepped(self, prepped): + filename = str(prepped.method) + ".json" f = open(filename, 'w') - # f.write(json.dumps(prepped, indent=4, separators=(',', ': '))) - f.write(method + " " + url + "\n") + f.write("headers: \n") f.write(str(prepped.headers)) f.write("\nbody: \n") f.write(str(prepped.body)) + f.write("\nhooks:\n") + f.write(str(prepped.hooks)) + f.write("\nmethod: " + str(prepped.method)) + f.write("\nurl: " + str(prepped.url) + "\n") + f.close() + def _request(self, method, path, params=None, data=None): + + url = urljoin(self.base, path) + + if (TESTING): + print("testing mode") + print(method, url) + req = requests.Request(method, url, params=params, json=data) + prepped = req.prepare() + self._writePrepped(prepped) - ''' resp = self._sess.request(method, url, params=params, json=data) - if resp.status_code == 400: - raise AkamaiClientBadRequest() - if resp.status_code == 403: - raise AkamaiClientNotAuthorized() - if resp.status_code == 404: - raise AkamaiClientNotFound() - if resp.status_code == 406: - raise AkamaiClientNotAcceptable() + + if resp.status_code > 299: + raise AkamaiClientException(resp.status_code) + + # if resp.status_code == 400: + # raise AkamaiClientBadRequest() + # if resp.status_code == 403: + # raise AkamaiClientNotAuthorized() + # if resp.status_code == 404: + # raise AkamaiClientNotFound() + # if resp.status_code == 406: + # raise AkamaiClientNotAcceptable() resp.raise_for_status() + return resp - ''' - return prepped - - ''' - def getZone(self, name): - path = name + "/recordsets/" - result = self._request('GET', path, "sortBy=type&types=A") - - return result.json() - - def getNames(self, name): - path = name + "/names/" - result = self._request('GET', path) - - return result.json() - ''' - + def record_get(self, zone, name, record_type): - path = '/zones/{}/names/{}/types/'.format(zone, name, record_type) + path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) result = self._request('GET', path) - return - return result.json() + return result def record_create(self, zone, name, record_type, params): - path = '/zones/{}/names/{}/types/'.format(zone, name, record_type) + path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) result = self._request('POST', path, data=params) return result - def record_delete(self, zone, name, record_type): - path = '/zones/{}/names/{}/types/'.format(zone, name, record_type) + path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) result = self._request('DELETE', path) + if result.status_code == 204: + print ("successfully deleted ", path) + return result def record_replace(self, zone, name, record_type, params): - path = '/zones/{}/names/{}/types/'.format(zone, name, record_type) + path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) result = self._request('PUT', path, data=params) return result + + def zones_get(self, contractIds=None, page=None, pageSize=None, search=None, + showAll="true", sortBy="zone", types=None): + path = 'zones' + + params = { + "contracIds": contractIds, + "page": page, + "pageSize": pageSize, + "search": search, + "showAll": showAll, + "sortBy": sortBy, + "types": types + } + + result = self._request('GET', path, params=params) + + return result + + + def zone_create(self): + pass + + + def zone_recordset_get(self, zone, params=None): + path = 'zones/{}/recordsets'.format(zone) + result = self._request('GET', path, params=params) + + return result + + def zone_recordset_create(self, zone): + pass + + def zone_recordset_replace(self, zone): + pass + + + def master_zone_file_get(self, zone): + path = 'zones/{}/zone-file'.format(zone) + result = self._request('GET', path) + + return result + + class AkamaiProvider(BaseProvider): SUPPORTS_GEO = False @@ -172,28 +231,160 @@ class AkamaiProvider(BaseProvider): zone_name = zone.name[:len(zone.name)-1] - - record_name = "www.exampleRecordName.com" - record_type = "AAAA" + record_name = "octo.basir-test.com" + record_type = "A" params = { - "name": "www.example.com", - "type": "AAAA", + "name": "octo.basir-test.com", + "type": "A", "ttl": 300, "rdata": [ "10.0.0.2", "10.0.0.3" ] } - - self._dns_client.record_get(zone_name, record_name, record_type) - self._dns_client.record_create(zone_name, record_name, record_type, params) - self._dns_client.record_delete(zone_name, record_name, record_type) - self._dns_client.record_replace(zone_name, record_name, record_type, params) + repl_params = { + "name": "octo.basir-test.com", + "type": "A", + "ttl": 300, + "rdata": [ + "99.99.99.99", + "10.0.0.3", + "1.2.3.4" + ] + } - # domainRequest = self._dns_client.getZone(zone_name) - # names = self._dns_client.getNames(zone_name) + print("\n\nRunning test: record get..........\n") + self._test_record_get(zone_name, "test.basir-test.com", record_type) + print("\n\nRunning test: record create..........\n") + self._test_record_create(zone_name, record_name, record_type, params) + print("\n\nRunning test: record replace..........\n") + self._test_record_replace(zone_name, record_name, record_type, repl_params) + print("\n\nRunning test: record delete..........\n") + self._test_record_delete(zone_name, record_name, record_type) + + # print("\n\nRunning test: zones get..........\n") + # self._test_zones_get() + # print("\n\nRunning test: zone recordset get..........\n") + # self._test_zones_recordset_get(zone_name) return + def _test_record_get(self, zone_name, record_name, record_type): + try: + get = self._dns_client.record_get(zone_name, record_name, record_type) + except AkamaiClientException as e: + print ("record get test failed") + print (e.message) + + else: + print("record get test result: ") + print(json.dumps(get.json(), indent=4, separators=(',', ': '))) + + return + + def _test_record_delete(self, zone_name, record_name, record_type): + + try: + delete = self._dns_client.record_delete(zone_name, record_name, record_type) + except AkamaiClientException as e: + print("delete failed") + print(e.message) + return + + + try: + self._dns_client.record_get(zone_name, record_name, record_type) + except AkamaiClientException as e: + print("get on record failed as expected, since record was succesfully deleted") + print ("(Probably Ignore):", e.message) + print ("delete status:", delete.status_code) + else: + print("unexpected condition in test delete") + + return + + def _test_record_create(self, zone_name, record_name, record_type, params): + + try: + create = self._dns_client.record_create(zone_name, record_name, record_type, params) + except AkamaiClientException as e: + print ("create unsuccessful, presumably because it already exists") + print ("(Probably Ignore)", e.message) + else: + print("initial create of", create.json().get("name"), "succesful: ", create.status_code) + + return + + def _test_record_replace(self, zone_name, record_name, record_type, params): + + ## create record to be replaced, if it doesn't already exist + try: + old_params = { + "name": record_name, + "type": record_type, + "ttl": 300, + "rdata": [ + "10.0.0.2", + "10.0.0.3" + ] + } + create = self._dns_client.record_create(zone_name, record_name, record_type, old_params) + except AkamaiClientException as e: + print ("initial create unsuccessful, presumably because it already exists") + print ("(Probably Ignore)", e.message) + else: + print("initial create of record to be replaced", create.json().get("name"), "succesful: ", create.status_code) + + + ## test replace + try: + replace = self._dns_client.record_replace(zone_name, record_name, record_type, params) + except AkamaiClientException as e: + print("replace failed") + print(e.message) + return + else: + try: + record = self._dns_client.record_get(zone_name, record_name, record_type) + except AkamaiClientException as e: + print("retrieval in replacement failed") + print(e.message) + else: + new_data = record.json() + + if (new_data != params): + print("replace failed, records don't match") + print("current data:") + print(new_data) + print("expected data:") + print(params) + + else: + print("replace succesful") + print("replace status:", replace.status_code) + + def _test_zones_get(self): + try: + zonesList = self._dns_client.zones_get() + except AkamaiClientException as e: + print ("zones get test failed") + print (e.message) + + else: + print("zones list: ") + print(json.dumps(zonesList.json(), indent=4, separators=(',', ': '))) + + return + + def _test_zones_recordset_get(self, zone_name): + try: + zoneRecordset = self._dns_client.zone_recordset_get(zone_name) + except AkamaiClientException as e: + print("zone recordset retrieval test failed") + print (e.message) + else: + print("zone recordset: ") + print(json.dumps(zoneRecordset.json(), indent=4, separators=(',', ': '))) + return From 9844f4293a31834c455fd7e00cf999d693e00ee8 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 13 Jun 2019 19:51:56 -0400 Subject: [PATCH 047/176] implemented records(), and zone_records(), fixed sone_recordset_get(), started populate() --- octodns/provider/fastdns.py | 152 +++++++++++++++++++++--------------- 1 file changed, 87 insertions(+), 65 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 56aa883..b94ff61 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -10,14 +10,14 @@ import requests from akamai.edgegrid import EdgeGridAuth from urlparse import urljoin import json - +from collections import defaultdict import logging from functools import reduce from ..record import Record from .base import BaseProvider -TESTING = False +TESTING = True class AkamaiClientException(Exception): @@ -38,28 +38,6 @@ class AkamaiClientException(Exception): super(AkamaiClientException, self).__init__(message) -''' -class AkamaiClientBadRequest(AkamaiClientException): #400 - def __init__(self): - super(AkamaiClientBadRequest, self).__init__('Bad request') - -class AkamaiClientNotAuthorized(AkamaiClientException): #403 - def __init__(self): - super(AkamaiClientNotAuthorized, self).__init__('Forbidden') - -class AkamaiClientNotFound(AkamaiClientException): #404 - def __init__(self): - super(AkamaiClientNotFound, self).__init__('Not found') - -class AkamaiClientNotAcceptable(AkamaiClientException): #406 - def __init__(self): - super(AkamaiClientNotAcceptable, self).__init__('Not acceptable') - -class AkamaiClientGenericExcp(AkamaiClientException): - def __init__(self, num): - super(AkamaiClientGenericExcp, self).__init__('HTTP Error: ' + str(num)) -''' - class _AkamaiRecord(object): pass @@ -95,10 +73,12 @@ class AkamaiClient(object): f.close() - def _request(self, method, path, params=None, data=None): + def _request(self, method, path, params=None, data=None, v1=False): url = urljoin(self.base, path) - + if v1: + url = urljoin(self.basev1, path) + if (TESTING): print("testing mode") print(method, url) @@ -111,15 +91,7 @@ class AkamaiClient(object): if resp.status_code > 299: raise AkamaiClientException(resp.status_code) - - # if resp.status_code == 400: - # raise AkamaiClientBadRequest() - # if resp.status_code == 403: - # raise AkamaiClientNotAuthorized() - # if resp.status_code == 404: - # raise AkamaiClientNotFound() - # if resp.status_code == 406: - # raise AkamaiClientNotAcceptable() + resp.raise_for_status() @@ -154,47 +126,64 @@ class AkamaiClient(object): return result - def zones_get(self, contractIds=None, page=None, pageSize=None, search=None, showAll="true", sortBy="zone", types=None): path = 'zones' params = { - "contracIds": contractIds, - "page": page, - "pageSize": pageSize, - "search": search, - "showAll": showAll, - "sortBy": sortBy, - "types": types + 'contractIds': contractIds, + 'page': page, + 'pageSize': pageSize, + 'search': search, + 'showAll': showAll, + 'sortBy': sortBy, + 'types': types } result = self._request('GET', path, params=params) return result - - def zone_create(self): - pass + def zone_recordset_get(self, zone, page=None, pageSize=30, search=None, + showAll="true", sortBy="name", types=None): - def zone_recordset_get(self, zone, params=None): + params = { + 'page': page, + 'pageSize': pageSize, + 'search': search, + 'showAll': showAll, + 'sortBy': sortBy, + 'types': types + } + path = 'zones/{}/recordsets'.format(zone) result = self._request('GET', path, params=params) return result - def zone_recordset_create(self, zone): - pass - - def zone_recordset_replace(self, zone): - pass + def records(self, zone_name): + + recordset = self.zone_recordset_get(zone_name, showAll="true").json().get("recordsets") + # print (type(recordset)) + print(recordset) + return recordset def master_zone_file_get(self, zone): + path = 'zones/{}/zone-file'.format(zone) - result = self._request('GET', path) - + + try: + result = self._request('GET', path) + + except AkamaiClientException as e: + # not working with API v2, API v1 fallback + path = 'zones/{}'.format(zone) + result = self._request('GET', path, v1=True) + print("Using API v1 fallback") + print("(Probably Ignore)", e.message) + return result @@ -220,10 +209,26 @@ class AkamaiProvider(BaseProvider): self._zone_records = {} + def zone_records(self, zone): + if zone.name not in self._zone_records: + try: + self._zone_records[zone.name] = self._dns_client.records(zone.name[:-1]) + + except AkamaiClientException: + return [] + + return self._zone_records[zone.name] + def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) - self._test(zone) + + # self._test(zone) + + values = defaultdict(lambda: defaultdict(list)) + + for record in self.zone_records(zone): + pass @@ -254,20 +259,23 @@ class AkamaiProvider(BaseProvider): } - print("\n\nRunning test: record get..........\n") - self._test_record_get(zone_name, "test.basir-test.com", record_type) - print("\n\nRunning test: record create..........\n") - self._test_record_create(zone_name, record_name, record_type, params) - print("\n\nRunning test: record replace..........\n") - self._test_record_replace(zone_name, record_name, record_type, repl_params) - print("\n\nRunning test: record delete..........\n") - self._test_record_delete(zone_name, record_name, record_type) + # print("\n\nRunning test: record get..........\n") + # self._test_record_get(zone_name, "test.basir-test.com", record_type) + # print("\n\nRunning test: record create..........\n") + # self._test_record_create(zone_name, record_name, record_type, params) + # print("\n\nRunning test: record replace..........\n") + # self._test_record_replace(zone_name, record_name, record_type, repl_params) + # print("\n\nRunning test: record delete..........\n") + # self._test_record_delete(zone_name, record_name, record_type) # print("\n\nRunning test: zones get..........\n") # self._test_zones_get() - # print("\n\nRunning test: zone recordset get..........\n") - # self._test_zones_recordset_get(zone_name) + print("\n\nRunning test: zone recordset get..........\n") + self._test_zones_recordset_get(zone_name) + + # print("\n\nRunning test: Master Zone File get..........\n") + # self._test_master_zone_file_get(zone_name) return @@ -388,3 +396,17 @@ class AkamaiProvider(BaseProvider): print("zone recordset: ") print(json.dumps(zoneRecordset.json(), indent=4, separators=(',', ': '))) return + + def _test_master_zone_file_get(self, zone_name): + try: + mzf = self._dns_client.master_zone_file_get(zone_name) + + except AkamaiClientException as e: + print("MZF retrieval test failed") + print (e.message) + + else: + print("Master Zone File:") + print(json.dumps(mzf.json(), indent=4, separators=(',', ': '))) + + return From b8c94386a87be103c6b058c8e6f354639d5afe47 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 13 Jun 2019 20:05:33 -0400 Subject: [PATCH 048/176] thurs sync --- octodns/provider/fastdns.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index b94ff61..de8e7d6 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -166,8 +166,6 @@ class AkamaiClient(object): recordset = self.zone_recordset_get(zone_name, showAll="true").json().get("recordsets") - # print (type(recordset)) - print(recordset) return recordset def master_zone_file_get(self, zone): @@ -228,7 +226,14 @@ class AkamaiProvider(BaseProvider): values = defaultdict(lambda: defaultdict(list)) for record in self.zone_records(zone): - pass + _type=record['type'] + if _type not in self.SUPPORTS: + continue + elif _type == 'TXT' and record['content'].startswith('ALIAS for'): + # ALIAS has a "ride along" TXT record with 'ALIAS for XXXX', + # we're ignoring it + continue + values[record['name']][record['type']].append(record) From 9b8e74c5dd448012a417eaebee602b69db736946 Mon Sep 17 00:00:00 2001 From: Rhosyn Celyn Date: Fri, 14 Jun 2019 15:09:41 +0100 Subject: [PATCH 049/176] Small clean up to populate and exceptions for requests --- octodns/provider/mythicbeasts.py | 31 +++++++++++++++++---- tests/test_octodns_provider_mythicbeasts.py | 21 ++++++++++++++ 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py index af07ddb..8bd2979 100644 --- a/octodns/provider/mythicbeasts.py +++ b/octodns/provider/mythicbeasts.py @@ -13,6 +13,8 @@ from logging import getLogger from ..record import Record from .base import BaseProvider +from collections import defaultdict + def add_trailing_dot(value): ''' @@ -43,6 +45,19 @@ class MythicBeastsUnauthorizedException(Exception): self.message, self.zone, *args) +class MythicBeastsRecordException(Exception): + def __init__(self, zone, command, *args): + self.zone = zone + self.command = command + self.message = 'Mythic Beasts could not action command: {} {}'.format( + self.zone, + self.command, + ) + + super(MythicBeastsRecordException, self).__init__( + self.message, self.zone, self.command, *args) + + class MythicBeastsProvider(BaseProvider): ''' Mythic Beasts DNS API Provider @@ -118,6 +133,12 @@ class MythicBeastsProvider(BaseProvider): if resp.status_code == 401: raise MythicBeastsUnauthorizedException(data['domain']) + + if resp.status_code == 400: + raise MythicBeastsRecordException( + data['domain'], + data['command'] + ) resp.raise_for_status() return resp @@ -287,7 +308,11 @@ class MythicBeastsProvider(BaseProvider): before = len(zone.records) exists = False - data = dict() + data = defaultdict(lambda: defaultdict(lambda: { + 'raw_values': [], + 'name': None, + 'zone': None, + })) exists = True for line in resp.content.splitlines(): @@ -310,10 +335,6 @@ class MythicBeastsProvider(BaseProvider): _value = _value.replace(';', '\\;') if hasattr(self, '_data_for_{}'.format(_type)): - - if _type not in data: - data[_type] = dict() - if _name not in data[_type]: data[_type][_name] = { 'raw_values': [{'value': _value, 'ttl': _ttl}], diff --git a/tests/test_octodns_provider_mythicbeasts.py b/tests/test_octodns_provider_mythicbeasts.py index 5deba3b..2a3b85b 100644 --- a/tests/test_octodns_provider_mythicbeasts.py +++ b/tests/test_octodns_provider_mythicbeasts.py @@ -318,6 +318,27 @@ class TestMythicBeastsProvider(TestCase): self.assertEquals(0, len(zone.records)) + # Record change failed + with requests_mock() as mock: + mock.post(ANY, status_code=200, text='') + provider.populate(zone) + zone.add_record(Record.new(zone, 'prawf', { + 'ttl': 300, + 'type': 'TXT', + 'value': 'prawf', + })) + plan = provider.plan(zone) + + with requests_mock() as mock: + mock.post(ANY, status_code=400, text='NADD 300 TXT prawf') + + with self.assertRaises(Exception) as err: + provider.apply(plan) + self.assertEquals( + 'Mythic Beasts could not action command: unit.tests ' + 'ADD prawf.unit.tests 300 TXT prawf', + err.exception.message) + # Check deleting and adding/changing test record existing = 'prawf 300 TXT prawf prawf prawf\ndileu 300 TXT dileu' From 98f7935f9044532ecb5653aa09e872b7c6128d15 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Fri, 14 Jun 2019 19:06:19 -0700 Subject: [PATCH 050/176] major update: progressed on populate(), implemented _data_for_TYPE fxns, but not all are verfied to work as expected yet --- octodns/.vscode/settings.json | 3 + octodns/provider/.vscode/settings.json | 1 + octodns/provider/fastdns.py | 999 ++++++++++++++----------- 3 files changed, 586 insertions(+), 417 deletions(-) create mode 100644 octodns/.vscode/settings.json diff --git a/octodns/.vscode/settings.json b/octodns/.vscode/settings.json new file mode 100644 index 0000000..38e6c1e --- /dev/null +++ b/octodns/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.pythonPath": "C:\\Python27\\python.exe" +} \ No newline at end of file diff --git a/octodns/provider/.vscode/settings.json b/octodns/provider/.vscode/settings.json index 7a73a41..88322cc 100644 --- a/octodns/provider/.vscode/settings.json +++ b/octodns/provider/.vscode/settings.json @@ -1,2 +1,3 @@ { + "python.pythonPath": "C:\\Python27\\python.exe" } \ No newline at end of file diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index de8e7d6..bcb1f32 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -1,417 +1,582 @@ -# -# -# - -from __future__ import absolute_import, division, print_function, \ - unicode_literals - -## octodns specfic imports: -import requests -from akamai.edgegrid import EdgeGridAuth -from urlparse import urljoin -import json -from collections import defaultdict - -import logging -from functools import reduce -from ..record import Record -from .base import BaseProvider - -TESTING = True - -class AkamaiClientException(Exception): - - _errorMessages = { - 400: "400: Bad request", - 403: "403: Access is forbidden", - 404: "404: Resource not found", - 405: "405: Method not supported", - 406: "406: Not Acceptable", - 409: "409: Request not allowed due to conflict with current state of resource", - 415: "415: Unsupported media type", - 422: "422: Request body contains an error preventing processing", - 500: "500: Internal server error" - } - - def __init__(self, code): - message = self._errorMessages.get(code) - super(AkamaiClientException, self).__init__(message) - - -class _AkamaiRecord(object): - pass - - -class AkamaiClient(object): - - def __init__(self, _client_secret, _host, _access_token, _client_token): - - self.base = "https://" + _host + "/config-dns/v2/" - self.basev1 = "https://" + _host + "/config-dns/v1/" - - sess = requests.Session() - sess.auth = EdgeGridAuth( - client_token=_client_token, - client_secret=_client_secret, - access_token=_access_token - ) - self._sess = sess - - - def _writePrepped(self, prepped): - filename = str(prepped.method) + ".json" - f = open(filename, 'w') - - f.write("headers: \n") - f.write(str(prepped.headers)) - f.write("\nbody: \n") - f.write(str(prepped.body)) - f.write("\nhooks:\n") - f.write(str(prepped.hooks)) - f.write("\nmethod: " + str(prepped.method)) - f.write("\nurl: " + str(prepped.url) + "\n") - - f.close() - - def _request(self, method, path, params=None, data=None, v1=False): - - url = urljoin(self.base, path) - if v1: - url = urljoin(self.basev1, path) - - if (TESTING): - print("testing mode") - print(method, url) - req = requests.Request(method, url, params=params, json=data) - prepped = req.prepare() - self._writePrepped(prepped) - - resp = self._sess.request(method, url, params=params, json=data) - - - if resp.status_code > 299: - raise AkamaiClientException(resp.status_code) - - resp.raise_for_status() - - - return resp - - - def record_get(self, zone, name, record_type): - - path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) - result = self._request('GET', path) - - return result - - def record_create(self, zone, name, record_type, params): - path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) - result = self._request('POST', path, data=params) - - return result - - def record_delete(self, zone, name, record_type): - path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) - result = self._request('DELETE', path) - - if result.status_code == 204: - print ("successfully deleted ", path) - - return result - - def record_replace(self, zone, name, record_type, params): - path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) - result = self._request('PUT', path, data=params) - - return result - - def zones_get(self, contractIds=None, page=None, pageSize=None, search=None, - showAll="true", sortBy="zone", types=None): - path = 'zones' - - params = { - 'contractIds': contractIds, - 'page': page, - 'pageSize': pageSize, - 'search': search, - 'showAll': showAll, - 'sortBy': sortBy, - 'types': types - } - - result = self._request('GET', path, params=params) - - return result - - def zone_recordset_get(self, zone, page=None, pageSize=30, search=None, - showAll="true", sortBy="name", types=None): - - - params = { - 'page': page, - 'pageSize': pageSize, - 'search': search, - 'showAll': showAll, - 'sortBy': sortBy, - 'types': types - } - - path = 'zones/{}/recordsets'.format(zone) - result = self._request('GET', path, params=params) - - return result - - def records(self, zone_name): - - recordset = self.zone_recordset_get(zone_name, showAll="true").json().get("recordsets") - - return recordset - - def master_zone_file_get(self, zone): - - path = 'zones/{}/zone-file'.format(zone) - - try: - result = self._request('GET', path) - - except AkamaiClientException as e: - # not working with API v2, API v1 fallback - path = 'zones/{}'.format(zone) - result = self._request('GET', path, v1=True) - print("Using API v1 fallback") - print("(Probably Ignore)", e.message) - - return result - - -class AkamaiProvider(BaseProvider): - - SUPPORTS_GEO = False - SUPPORTS_DYNAMIC = False - SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', - 'SRV', 'SSHFP', 'TXT')) - - - def __init__(self, id, client_secret, host, access_token, client_token, - *args, **kwargs): - - self.log = logging.getLogger('AkamaiProvider[{}]'.format(id)) - self.log.debug('__init__: id=%s, ') - super(AkamaiProvider, self).__init__(id, *args, **kwargs) - - - self._dns_client = AkamaiClient(client_secret, host, access_token, client_token) - - - self._zone_records = {} - - - def zone_records(self, zone): - if zone.name not in self._zone_records: - try: - self._zone_records[zone.name] = self._dns_client.records(zone.name[:-1]) - - except AkamaiClientException: - return [] - - return self._zone_records[zone.name] - - - def populate(self, zone, target=False, lenient=False): - self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) - - # self._test(zone) - - values = defaultdict(lambda: defaultdict(list)) - - for record in self.zone_records(zone): - _type=record['type'] - if _type not in self.SUPPORTS: - continue - elif _type == 'TXT' and record['content'].startswith('ALIAS for'): - # ALIAS has a "ride along" TXT record with 'ALIAS for XXXX', - # we're ignoring it - continue - values[record['name']][record['type']].append(record) - - - - def _test(self, zone) : - - zone_name = zone.name[:len(zone.name)-1] - - record_name = "octo.basir-test.com" - record_type = "A" - params = { - "name": "octo.basir-test.com", - "type": "A", - "ttl": 300, - "rdata": [ - "10.0.0.2", - "10.0.0.3" - ] - } - repl_params = { - "name": "octo.basir-test.com", - "type": "A", - "ttl": 300, - "rdata": [ - "99.99.99.99", - "10.0.0.3", - "1.2.3.4" - ] - } - - - # print("\n\nRunning test: record get..........\n") - # self._test_record_get(zone_name, "test.basir-test.com", record_type) - # print("\n\nRunning test: record create..........\n") - # self._test_record_create(zone_name, record_name, record_type, params) - # print("\n\nRunning test: record replace..........\n") - # self._test_record_replace(zone_name, record_name, record_type, repl_params) - # print("\n\nRunning test: record delete..........\n") - # self._test_record_delete(zone_name, record_name, record_type) - - # print("\n\nRunning test: zones get..........\n") - # self._test_zones_get() - - print("\n\nRunning test: zone recordset get..........\n") - self._test_zones_recordset_get(zone_name) - - # print("\n\nRunning test: Master Zone File get..........\n") - # self._test_master_zone_file_get(zone_name) - - return - - def _test_record_get(self, zone_name, record_name, record_type): - try: - get = self._dns_client.record_get(zone_name, record_name, record_type) - except AkamaiClientException as e: - print ("record get test failed") - print (e.message) - - else: - print("record get test result: ") - print(json.dumps(get.json(), indent=4, separators=(',', ': '))) - - return - - def _test_record_delete(self, zone_name, record_name, record_type): - - try: - delete = self._dns_client.record_delete(zone_name, record_name, record_type) - except AkamaiClientException as e: - print("delete failed") - print(e.message) - return - - - try: - self._dns_client.record_get(zone_name, record_name, record_type) - except AkamaiClientException as e: - print("get on record failed as expected, since record was succesfully deleted") - print ("(Probably Ignore):", e.message) - print ("delete status:", delete.status_code) - else: - print("unexpected condition in test delete") - - return - - def _test_record_create(self, zone_name, record_name, record_type, params): - - try: - create = self._dns_client.record_create(zone_name, record_name, record_type, params) - except AkamaiClientException as e: - print ("create unsuccessful, presumably because it already exists") - print ("(Probably Ignore)", e.message) - else: - print("initial create of", create.json().get("name"), "succesful: ", create.status_code) - - return - - def _test_record_replace(self, zone_name, record_name, record_type, params): - - ## create record to be replaced, if it doesn't already exist - try: - old_params = { - "name": record_name, - "type": record_type, - "ttl": 300, - "rdata": [ - "10.0.0.2", - "10.0.0.3" - ] - } - create = self._dns_client.record_create(zone_name, record_name, record_type, old_params) - except AkamaiClientException as e: - print ("initial create unsuccessful, presumably because it already exists") - print ("(Probably Ignore)", e.message) - else: - print("initial create of record to be replaced", create.json().get("name"), "succesful: ", create.status_code) - - - ## test replace - try: - replace = self._dns_client.record_replace(zone_name, record_name, record_type, params) - except AkamaiClientException as e: - print("replace failed") - print(e.message) - return - else: - try: - record = self._dns_client.record_get(zone_name, record_name, record_type) - except AkamaiClientException as e: - print("retrieval in replacement failed") - print(e.message) - else: - new_data = record.json() - - if (new_data != params): - print("replace failed, records don't match") - print("current data:") - print(new_data) - print("expected data:") - print(params) - - else: - print("replace succesful") - print("replace status:", replace.status_code) - - def _test_zones_get(self): - try: - zonesList = self._dns_client.zones_get() - except AkamaiClientException as e: - print ("zones get test failed") - print (e.message) - - else: - print("zones list: ") - print(json.dumps(zonesList.json(), indent=4, separators=(',', ': '))) - - return - - def _test_zones_recordset_get(self, zone_name): - try: - zoneRecordset = self._dns_client.zone_recordset_get(zone_name) - except AkamaiClientException as e: - print("zone recordset retrieval test failed") - print (e.message) - else: - print("zone recordset: ") - print(json.dumps(zoneRecordset.json(), indent=4, separators=(',', ': '))) - return - - def _test_master_zone_file_get(self, zone_name): - try: - mzf = self._dns_client.master_zone_file_get(zone_name) - - except AkamaiClientException as e: - print("MZF retrieval test failed") - print (e.message) - - else: - print("Master Zone File:") - print(json.dumps(mzf.json(), indent=4, separators=(',', ': '))) - - return +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +## octodns specfic imports: +import requests +from akamai.edgegrid import EdgeGridAuth +from urlparse import urljoin +import json +from collections import defaultdict + +import logging +from ..record import Record +from .base import BaseProvider + +TESTING = True + +class AkamaiClientException(Exception): + + _errorMessages = { + 400: "400: Bad request", + 403: "403: Access is forbidden", + 404: "404: Resource not found", + 405: "405: Method not supported", + 406: "406: Not Acceptable", + 409: "409: Request not allowed due to conflict with current state of resource", + 415: "415: Unsupported media type", + 422: "422: Request body contains an error preventing processing", + 500: "500: Internal server error" + } + + def __init__(self, code): + message = self._errorMessages.get(code) + super(AkamaiClientException, self).__init__(message) + + +class _AkamaiRecord(object): + pass + + +class AkamaiClient(object): + + def __init__(self, _client_secret, _host, _access_token, _client_token): + + self.base = "https://" + _host + "/config-dns/v2/" + self.basev1 = "https://" + _host + "/config-dns/v1/" + + sess = requests.Session() + sess.auth = EdgeGridAuth( + client_token=_client_token, + client_secret=_client_secret, + access_token=_access_token + ) + self._sess = sess + + + def _writePrepped(self, prepped): + filename = str(prepped.method) + ".json" + f = open(filename, 'w') + + f.write("headers: \n") + f.write(str(prepped.headers)) + f.write("\nbody: \n") + f.write(str(prepped.body)) + f.write("\nhooks:\n") + f.write(str(prepped.hooks)) + f.write("\nmethod: " + str(prepped.method)) + f.write("\nurl: " + str(prepped.url) + "\n") + + f.close() + + def _request(self, method, path, params=None, data=None, v1=False): + + url = urljoin(self.base, path) + if v1: + url = urljoin(self.basev1, path) + + if (TESTING): + print("testing mode") + print(method, url) + req = requests.Request(method, url, params=params, json=data) + prepped = req.prepare() + self._writePrepped(prepped) + + resp = self._sess.request(method, url, params=params, json=data) + + + if resp.status_code > 299: + raise AkamaiClientException(resp.status_code) + + resp.raise_for_status() + + + return resp + + + def record_get(self, zone, name, record_type): + + path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) + result = self._request('GET', path) + + return result + + def record_create(self, zone, name, record_type, params): + path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) + result = self._request('POST', path, data=params) + + return result + + def record_delete(self, zone, name, record_type): + path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) + result = self._request('DELETE', path) + + if result.status_code == 204: + print ("successfully deleted ", path) + + return result + + def record_replace(self, zone, name, record_type, params): + path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) + result = self._request('PUT', path, data=params) + + return result + + def zones_get(self, contractIds=None, page=None, pageSize=None, search=None, + showAll="true", sortBy="zone", types=None): + path = 'zones' + + params = { + 'contractIds': contractIds, + 'page': page, + 'pageSize': pageSize, + 'search': search, + 'showAll': showAll, + 'sortBy': sortBy, + 'types': types + } + + result = self._request('GET', path, params=params) + + return result + + def zone_recordset_get(self, zone, page=None, pageSize=30, search=None, + showAll="true", sortBy="name", types=None): + + + params = { + 'page': page, + 'pageSize': pageSize, + 'search': search, + 'showAll': showAll, + 'sortBy': sortBy, + 'types': types + } + + path = 'zones/{}/recordsets'.format(zone) + result = self._request('GET', path, params=params) + + + return result + + def records(self, zone_name): + + resp = self.zone_recordset_get(zone_name, showAll="true") + recordset = resp.json().get("recordsets") + + return recordset + + def master_zone_file_get(self, zone): + + path = 'zones/{}/zone-file'.format(zone) + + try: + result = self._request('GET', path) + + except AkamaiClientException as e: + # not working with API v2, API v1 fallback + path = 'zones/{}'.format(zone) + result = self._request('GET', path, v1=True) + print("Using API v1 fallback") + print("(Probably Ignore)", e.message) + + return result + + +class AkamaiProvider(BaseProvider): + + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = False + + SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', + 'SRV', 'SSHFP', 'TXT')) + # SUPPORTS = set(('NAPTR', 'MX')) + + def __init__(self, id, client_secret, host, access_token, client_token, + *args, **kwargs): + + self.log = logging.getLogger('AkamaiProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, ') + super(AkamaiProvider, self).__init__(id, *args, **kwargs) + + self._dns_client = AkamaiClient(client_secret, host, access_token, + client_token) + + self._zone_records = {} + + + def zone_records(self, zone): + """ returns records for a zone, finds it if not present, or + returns empty if doesnt exist + """ + if zone.name not in self._zone_records: + try: + name = zone.name[:-1] + self._zone_records[zone.name] = self._dns_client.records(name) + + except AkamaiClientException: + return [] + + + # fileName = "zone_records_for_" + zone.name +".json" + # path = "/mnt/c/Users/bajamil/Desktop/" + fileName + # f = open(path, 'w') + # f.write(json.dumps(self._zone_records[zone.name], indent=4, separators=(',', ': '))) + # f.close() + + return self._zone_records[zone.name] + + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s', zone.name) + # self._test(zone) + + values = defaultdict(lambda: defaultdict(list)) + for record in self.zone_records(zone): + + _type =record.get('type') + _name = record.get('name') + + if _type not in self.SUPPORTS: + continue + + values[_name][_type].append(record) + + + + fileName = "values_for_" + zone.name +".json" + path = "/mnt/c/Users/bajamil/Desktop/" + fileName + f = open(path, 'w') + f.write(json.dumps(values, indent=4, separators=(',', ': '))) + f.close() + + return ### + before = len(zone.records) + for name, types in values.items(): + for _type, records in types.items(): + data_for = getattr(self, '_data_for_{}'.format(_type)) + + data = data_for(_type, records) + + print() + print ("data processed for", name, ":") + print (json.dumps(data, indent=4, separators=(',', ': '))) + print() + + record = Record.new(zone, name, data_for(_type, records), source=self, lenient=lenient) + + + exists = zone.name in self._zone_records + self.log.info('populate: found %s records, exists=%s', + len(zone.records) - before, exists) + + return exists + + + def _data_for_multiple(self, _type, records): + + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': [r['rdata'][0] for r in records] + } + + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + _data_for_SPF = _data_for_multiple + + def _data_for_CNAME(self, _type, records): + record = records[0] + value = record['rdata'][0] + if (value[-1] != "."): + value = '{}.'.format(value) + + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'value': value + } + + def _data_for_MX(self, _type, records): + values = [] + for record in records: + print ("MX record: ", record) + values.append({ + 'preference': record['priority'], + 'exchange': '{}.'.format(record['rdata'][0]) + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + def _data_for_NAPTR(self, _type, records): + print ("NAPTR Record") + print(json.dumps(records, indent=4, separators=(',', ': '))) + values = [] + for record in records: + order, preference, flags, service, regexp, replacement = \ + record['rdata'][0].split(' ', 5) + + values.append({ + 'flags': flags[1:-1], + 'order': order, + 'preference': preference, + 'regexp': regexp[1:-1], + 'replacement': replacement, + 'service': service[1:-1] + }) + return { + 'type': _type, + 'ttl': records[0]['ttl'], + 'values': values + } + + def _data_for_NS(self, _type, records): + values = [] + for record in records: + rdata = record['rdata'] + for content in rdata: + if content[-1] != '.': + content = '{}.'.format(content) + values.append(content) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'value': values + } + + def _data_for_PTR(self, _type, records): + return self._data_for_multiple(_type, records[0]) + + # rdata[0]? + def _data_for_SRV(self, _type, records): + values = [] + for record in records: + weight, port, target = record['rdata'].split(', ', 2) + values.append({ + 'port': port, + 'priority': record['priority'], + 'target': '{}.'.format(target), + 'weight': weight + }) + return { + 'type': _type, + 'ttl': records[0]['ttl'], + 'values': values + } + + # rdata[0]? + def _data_for_SSHFP(self, _type, records): + values = [] + algorithm, fp_type, fingerprint = record['rdata'].split(' ', 2) + values.append({ + 'algorithm': algorithm, + 'fingerprint': fingerprint, + 'fingerprint_type': fp_type + }) + + return { + 'type': _type, + 'ttl': records[0]['ttl'], + 'values': values + } + + def _data_for_TXT(self, _type, records): + values = [] + for r in records: + for content in r['rdata']: + values.append(content.replace(';', '\\;')) + + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + # 'values': [r['rdata'][0].replace(';', '\\;') for r in records] + } + + + + + def _test(self, zone) : + + zone_name = zone.name[:len(zone.name)-1] + + record_name = "octo.basir-test.com" + record_type = "A" + params = { + "name": "octo.basir-test.com", + "type": "A", + "ttl": 300, + "rdata": [ + "10.0.0.2", + "10.0.0.3" + ] + } + repl_params = { + "name": "octo.basir-test.com", + "type": "A", + "ttl": 300, + "rdata": [ + "99.99.99.99", + "10.0.0.3", + "1.2.3.4" + ] + } + + + # print("\n\nRunning test: record get..........\n") + # self._test_record_get(zone_name, "test.basir-test.com", record_type) + # print("\n\nRunning test: record create..........\n") + # self._test_record_create(zone_name, record_name, record_type, params) + # print("\n\nRunning test: record replace..........\n") + # self._test_record_replace(zone_name, record_name, record_type, repl_params) + # print("\n\nRunning test: record delete..........\n") + # self._test_record_delete(zone_name, record_name, record_type) + + # print("\n\nRunning test: zones get..........\n") + # self._test_zones_get() + + print("\n\nRunning test: zone recordset get..........\n") + self._test_zones_recordset_get(zone_name) + + # print("\n\nRunning test: Master Zone File get..........\n") + # self._test_master_zone_file_get(zone_name) + + return + + def _test_record_get(self, zone_name, record_name, record_type): + try: + get = self._dns_client.record_get(zone_name, record_name, record_type) + except AkamaiClientException as e: + print ("record get test failed") + print (e.message) + + else: + print("record get test result: ") + print(json.dumps(get.json(), indent=4, separators=(',', ': '))) + + return + + def _test_record_delete(self, zone_name, record_name, record_type): + + try: + delete = self._dns_client.record_delete(zone_name, record_name, record_type) + except AkamaiClientException as e: + print("delete failed") + print(e.message) + return + + + try: + self._dns_client.record_get(zone_name, record_name, record_type) + except AkamaiClientException as e: + print("get on record failed as expected, since record was succesfully deleted") + print ("(Probably Ignore):", e.message) + print ("delete status:", delete.status_code) + else: + print("unexpected condition in test delete") + + return + + def _test_record_create(self, zone_name, record_name, record_type, params): + + try: + create = self._dns_client.record_create(zone_name, record_name, record_type, params) + except AkamaiClientException as e: + print ("create unsuccessful, presumably because it already exists") + print ("(Probably Ignore)", e.message) + else: + print("initial create of", create.json().get("name"), "succesful: ", create.status_code) + + return + + def _test_record_replace(self, zone_name, record_name, record_type, params): + + ## create record to be replaced, if it doesn't already exist + try: + old_params = { + "name": record_name, + "type": record_type, + "ttl": 300, + "rdata": [ + "10.0.0.2", + "10.0.0.3" + ] + } + create = self._dns_client.record_create(zone_name, record_name, record_type, old_params) + except AkamaiClientException as e: + print ("initial create unsuccessful, presumably because it already exists") + print ("(Probably Ignore)", e.message) + else: + print("initial create of record to be replaced", create.json().get("name"), "succesful: ", create.status_code) + + + ## test replace + try: + replace = self._dns_client.record_replace(zone_name, record_name, record_type, params) + except AkamaiClientException as e: + print("replace failed") + print(e.message) + return + else: + try: + record = self._dns_client.record_get(zone_name, record_name, record_type) + except AkamaiClientException as e: + print("retrieval in replacement failed") + print(e.message) + else: + new_data = record.json() + + if (new_data != params): + print("replace failed, records don't match") + print("current data:") + print(new_data) + print("expected data:") + print(params) + + else: + print("replace succesful") + print("replace status:", replace.status_code) + + def _test_zones_get(self): + try: + zonesList = self._dns_client.zones_get() + except AkamaiClientException as e: + print ("zones get test failed") + print (e.message) + + else: + print("zones list: ") + print(json.dumps(zonesList.json(), indent=4, separators=(',', ': '))) + + return + + def _test_zones_recordset_get(self, zone_name): + try: + zoneRecordset = self._dns_client.zone_recordset_get(zone_name) + except AkamaiClientException as e: + print("zone recordset retrieval test failed") + print (e.message) + else: + print("zone recordset: ") + print(json.dumps(zoneRecordset.json(), indent=4, separators=(',', ': '))) + return + + def _test_master_zone_file_get(self, zone_name): + try: + mzf = self._dns_client.master_zone_file_get(zone_name) + + except AkamaiClientException as e: + print("MZF retrieval test failed") + print (e.message) + + else: + print("Master Zone File:") + print(json.dumps(mzf.json(), indent=4, separators=(',', ': '))) + + return From 9dc44c3f2c5f40541b845d8bc2e18bd2126ec792 Mon Sep 17 00:00:00 2001 From: Rhosyn Celyn Date: Mon, 17 Jun 2019 16:34:58 +0100 Subject: [PATCH 051/176] Clean up and simplification of tests and command generation, bug fix for multiple sub domain NS records and handling of creation/deletion --- octodns/provider/mythicbeasts.py | 17 ++--- tests/test_octodns_provider_mythicbeasts.py | 79 ++++++++++++++------- 2 files changed, 59 insertions(+), 37 deletions(-) diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py index 8bd2979..3470ff1 100644 --- a/octodns/provider/mythicbeasts.py +++ b/octodns/provider/mythicbeasts.py @@ -366,10 +366,9 @@ class MythicBeastsProvider(BaseProvider): return exists - def _compile_commands(self, action, change): + def _compile_commands(self, action, record): commands = [] - record = change.record hostname = remove_trailing_dot(record.fqdn) ttl = record.ttl _type = record._type @@ -384,11 +383,7 @@ class MythicBeastsProvider(BaseProvider): base = '{} {} {} {}'.format(action, hostname, ttl, _type) - if _type in ('A', 'AAAA'): - for value in values: - commands.append('{} {}'.format(base, value)) - - elif _type == 'SSHFP': + if _type == 'SSHFP': data = values[0].data commands.append('{} {} {} {}'.format( base, @@ -417,8 +412,8 @@ class MythicBeastsProvider(BaseProvider): else: if hasattr(self, '_data_for_{}'.format(_type)): - commands.append('{} {}'.format( - base, values[0])) + for value in values: + commands.append('{} {}'.format(base, value)) else: self.log.debug('skipping %s as not supported', _type) @@ -426,7 +421,7 @@ class MythicBeastsProvider(BaseProvider): def _apply_Create(self, change): zone = change.new.zone - commands = self._compile_commands('ADD', change) + commands = self._compile_commands('ADD', change.new) for command in commands: self._post({ @@ -443,7 +438,7 @@ class MythicBeastsProvider(BaseProvider): def _apply_Delete(self, change): zone = change.existing.zone - commands = self._compile_commands('DELETE', change) + commands = self._compile_commands('DELETE', change.existing) for command in commands: self._post({ diff --git a/tests/test_octodns_provider_mythicbeasts.py b/tests/test_octodns_provider_mythicbeasts.py index 2a3b85b..143d1c7 100644 --- a/tests/test_octodns_provider_mythicbeasts.py +++ b/tests/test_octodns_provider_mythicbeasts.py @@ -156,32 +156,39 @@ class TestMythicBeastsProvider(TestCase): self.assertEquals('Unable to parse CAA data', err.exception.message) - def test_alias_command_generation(self): + def test_command_generation(self): zone = Zone('unit.tests.', []) - zone.add_record(Record.new(zone, 'prawf', { + zone.add_record(Record.new(zone, 'prawf-alias', { 'ttl': 60, 'type': 'ALIAS', 'value': 'alias.unit.tests.', })) - with requests_mock() as mock: - mock.post(ANY, status_code=200, text='') - - provider = MythicBeastsProvider('test', { - 'unit.tests.': 'mypassword' - }) - - plan = provider.plan(zone) - change = plan.changes[0] - - command = provider._compile_commands('ADD', change) - self.assertEquals( - ['ADD prawf.unit.tests 60 ANAME alias.unit.tests.'], - command - ) - - def test_txt_command_generation(self): - zone = Zone('unit.tests.', []) - zone.add_record(Record.new(zone, 'prawf', { + zone.add_record(Record.new(zone, 'prawf-ns', { + 'ttl': 300, + 'type': 'NS', + 'values': [ + 'alias.unit.tests.', + 'alias2.unit.tests.', + ], + })) + zone.add_record(Record.new(zone, 'prawf-a', { + 'ttl': 60, + 'type': 'A', + 'values': [ + '1.2.3.4', + '5.6.7.8', + ], + })) + zone.add_record(Record.new(zone, 'prawf-aaaa', { + 'ttl': 60, + 'type': 'AAAA', + 'values': [ + 'a:a::a', + 'b:b::b', + 'c:c::c:c', + ], + })) + zone.add_record(Record.new(zone, 'prawf-txt', { 'ttl': 60, 'type': 'TXT', 'value': 'prawf prawf dyma prawf', @@ -194,15 +201,35 @@ class TestMythicBeastsProvider(TestCase): }) plan = provider.plan(zone) - change = plan.changes[0] + changes = plan.changes + generated_commands = [] + + for change in changes: + generated_commands.extend( + provider._compile_commands('ADD', change.new) + ) + + expected_commands = [ + 'ADD prawf-alias.unit.tests 60 ANAME alias.unit.tests.', + 'ADD prawf-ns.unit.tests 300 NS alias.unit.tests.', + 'ADD prawf-ns.unit.tests 300 NS alias2.unit.tests.', + 'ADD prawf-a.unit.tests 60 A 1.2.3.4', + 'ADD prawf-a.unit.tests 60 A 5.6.7.8', + 'ADD prawf-aaaa.unit.tests 60 AAAA a:a::a', + 'ADD prawf-aaaa.unit.tests 60 AAAA b:b::b', + 'ADD prawf-aaaa.unit.tests 60 AAAA c:c::c:c', + 'ADD prawf-txt.unit.tests 60 TXT prawf prawf dyma prawf', + ] + + generated_commands.sort() + expected_commands.sort() - command = provider._compile_commands('ADD', change) self.assertEquals( - ['ADD prawf.unit.tests 60 TXT prawf prawf dyma prawf'], - command + generated_commands, + expected_commands ) - def test_command_generation(self): + def test_fake_command_generation(self): class FakeChangeRecord(object): def __init__(self): self.__fqdn = 'prawf.unit.tests.' From 6db6052d28fc10afa861c3b83f9f82160d393b8b Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Mon, 17 Jun 2019 10:24:34 -0700 Subject: [PATCH 052/176] friday sync --- octodns/provider/fastdns.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index bcb1f32..686c81c 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -247,26 +247,24 @@ class AkamaiProvider(BaseProvider): - fileName = "values_for_" + zone.name +".json" - path = "/mnt/c/Users/bajamil/Desktop/" + fileName - f = open(path, 'w') - f.write(json.dumps(values, indent=4, separators=(',', ': '))) - f.close() - return ### before = len(zone.records) for name, types in values.items(): - for _type, records in types.items(): - data_for = getattr(self, '_data_for_{}'.format(_type)) - - data = data_for(_type, records) + # for _type, records in types.items(): + for _type in types.items(): + for rdata in _type['rdata']: - print() - print ("data processed for", name, ":") - print (json.dumps(data, indent=4, separators=(',', ': '))) - print() - record = Record.new(zone, name, data_for(_type, records), source=self, lenient=lenient) + data_for = getattr(self, '_data_for_{}'.format(_type)) + + data = data_for(_type, records, rdata) + + print() + print ("data processed for", name, ":") + print (json.dumps(data, indent=4, separators=(',', ': '))) + print() + + record = Record.new(zone, name, data_for(_type, records), source=self, lenient=lenient) exists = zone.name in self._zone_records From 67f7af58e632345570f8bd2b854dd82c4797905f Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Mon, 17 Jun 2019 10:24:58 -0700 Subject: [PATCH 053/176] friday sync --- octodns/provider/fastdns.py | 1 - 1 file changed, 1 deletion(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 686c81c..0874709 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -235,7 +235,6 @@ class AkamaiProvider(BaseProvider): # self._test(zone) values = defaultdict(lambda: defaultdict(list)) - for record in self.zone_records(zone): _type =record.get('type') _name = record.get('name') From 402f645acd43a9b48359cac0bbf05b01e129907a Mon Sep 17 00:00:00 2001 From: Rhosyn Celyn Date: Mon, 17 Jun 2019 18:59:16 +0100 Subject: [PATCH 054/176] Additional test on compile_commands for deletion --- tests/test_octodns_provider_mythicbeasts.py | 30 +++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/test_octodns_provider_mythicbeasts.py b/tests/test_octodns_provider_mythicbeasts.py index 143d1c7..8181562 100644 --- a/tests/test_octodns_provider_mythicbeasts.py +++ b/tests/test_octodns_provider_mythicbeasts.py @@ -229,6 +229,36 @@ class TestMythicBeastsProvider(TestCase): expected_commands ) + # Now test deletion + existing = 'prawf-txt 300 TXT prawf prawf dyma prawf\n' \ + 'prawf-a 60 A 1.2.3.4' + + with requests_mock() as mock: + mock.post(ANY, status_code=200, text=existing) + wanted = Zone('unit.tests.', []) + + plan = provider.plan(wanted) + changes = plan.changes + generated_commands = [] + + for change in changes: + generated_commands.extend( + provider._compile_commands('DELETE', change.existing) + ) + + expected_commands = [ + 'DELETE prawf-a.unit.tests 60 A 1.2.3.4', + 'DELETE prawf-txt.unit.tests 300 TXT prawf prawf dyma prawf', + ] + + generated_commands.sort() + expected_commands.sort() + + self.assertEquals( + generated_commands, + expected_commands + ) + def test_fake_command_generation(self): class FakeChangeRecord(object): def __init__(self): From dfd6e8cfdaf051b9e1122f4146c449e967d85a88 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Tue, 18 Jun 2019 14:43:38 -0400 Subject: [PATCH 055/176] Completed populate + helper functions --- .vscode/settings.json | 3 + octodns/provider/fastdns.py | 233 +++++++++++++----------------------- 2 files changed, 84 insertions(+), 152 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..38e6c1e --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.pythonPath": "C:\\Python27\\python.exe" +} \ No newline at end of file diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 0874709..bdd3b82 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -16,7 +16,6 @@ import logging from ..record import Record from .base import BaseProvider -TESTING = True class AkamaiClientException(Exception): @@ -37,10 +36,6 @@ class AkamaiClientException(Exception): super(AkamaiClientException, self).__init__(message) -class _AkamaiRecord(object): - pass - - class AkamaiClient(object): def __init__(self, _client_secret, _host, _access_token, _client_token): @@ -57,45 +52,20 @@ class AkamaiClient(object): self._sess = sess - def _writePrepped(self, prepped): - filename = str(prepped.method) + ".json" - f = open(filename, 'w') - - f.write("headers: \n") - f.write(str(prepped.headers)) - f.write("\nbody: \n") - f.write(str(prepped.body)) - f.write("\nhooks:\n") - f.write(str(prepped.hooks)) - f.write("\nmethod: " + str(prepped.method)) - f.write("\nurl: " + str(prepped.url) + "\n") - - f.close() - def _request(self, method, path, params=None, data=None, v1=False): url = urljoin(self.base, path) if v1: url = urljoin(self.basev1, path) - if (TESTING): - print("testing mode") - print(method, url) - req = requests.Request(method, url, params=params, json=data) - prepped = req.prepare() - self._writePrepped(prepped) - resp = self._sess.request(method, url, params=params, json=data) - if resp.status_code > 299: raise AkamaiClientException(resp.status_code) - resp.raise_for_status() - return resp - + return resp def record_get(self, zone, name, record_type): @@ -146,7 +116,6 @@ class AkamaiClient(object): def zone_recordset_get(self, zone, page=None, pageSize=30, search=None, showAll="true", sortBy="name", types=None): - params = { 'page': page, 'pageSize': pageSize, @@ -193,8 +162,7 @@ class AkamaiProvider(BaseProvider): SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', 'SRV', 'SSHFP', 'TXT')) - # SUPPORTS = set(('NAPTR', 'MX')) - + def __init__(self, id, client_secret, host, access_token, client_token, *args, **kwargs): @@ -210,7 +178,7 @@ class AkamaiProvider(BaseProvider): def zone_records(self, zone): """ returns records for a zone, finds it if not present, or - returns empty if doesnt exist + returns empty if can't find a match """ if zone.name not in self._zone_records: try: @@ -220,184 +188,145 @@ class AkamaiProvider(BaseProvider): except AkamaiClientException: return [] - - # fileName = "zone_records_for_" + zone.name +".json" - # path = "/mnt/c/Users/bajamil/Desktop/" + fileName - # f = open(path, 'w') - # f.write(json.dumps(self._zone_records[zone.name], indent=4, separators=(',', ': '))) - # f.close() - return self._zone_records[zone.name] - def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s', zone.name) - # self._test(zone) values = defaultdict(lambda: defaultdict(list)) - + for record in self.zone_records(zone): + _type =record.get('type') - _name = record.get('name') + ## Akamai sends down prefix.zonename., while OctoDNS only expects prefix + _name = record.get('name').split("." + zone.name[:-1], 1)[0] + if _name == zone.name[:-1] : + _name = '' ## root / @ if _type not in self.SUPPORTS: continue - values[_name][_type].append(record) - - - before = len(zone.records) for name, types in values.items(): - # for _type, records in types.items(): - for _type in types.items(): - for rdata in _type['rdata']: - - - data_for = getattr(self, '_data_for_{}'.format(_type)) - - data = data_for(_type, records, rdata) - - print() - print ("data processed for", name, ":") - print (json.dumps(data, indent=4, separators=(',', ': '))) - print() - - record = Record.new(zone, name, data_for(_type, records), source=self, lenient=lenient) - - - exists = zone.name in self._zone_records - self.log.info('populate: found %s records, exists=%s', - len(zone.records) - before, exists) + for _type, records in types.items(): + data_for = getattr(self, '_data_for_{}'.format(_type)) + record = Record.new(zone, name, data_for(_type, records[0]), + source=self, lenient=lenient) + zone.add_record(record, lenient=lenient) - return exists + exists = zone.name in self._zone_records + found = len(zone.records) - before + self.log.info('populate: found %s records, exists=%s', found, exists) - def _data_for_multiple(self, _type, records): + def _data_for_multiple(self, _type, _records): return { - 'ttl': records[0]['ttl'], - 'type': _type, - 'values': [r['rdata'][0] for r in records] - } + 'ttl': _records['ttl'], + 'type': _type, + 'values': [r for r in _records['rdata']] + } _data_for_A = _data_for_multiple _data_for_AAAA = _data_for_multiple + _data_for_NS = _data_for_multiple _data_for_SPF = _data_for_multiple - def _data_for_CNAME(self, _type, records): - record = records[0] - value = record['rdata'][0] - if (value[-1] != "."): + def _data_for_CNAME(self, _type, _records): + value = _records['rdata'][0] + if (value[-1] != '.') : value = '{}.'.format(value) - + return { - 'ttl': records[0]['ttl'], + 'ttl': _records['ttl'], 'type': _type, 'value': value } - def _data_for_MX(self, _type, records): + def _data_for_MX(self, _type, _records): values = [] - for record in records: - print ("MX record: ", record) + for r in _records['rdata']: + preference, exchange = r.split(" ", 1) values.append({ - 'preference': record['priority'], - 'exchange': '{}.'.format(record['rdata'][0]) - }) + 'preference': preference, + 'exchange' : exchange + }) return { - 'ttl': records[0]['ttl'], + 'ttl': _records['ttl'], 'type': _type, 'values': values } - def _data_for_NAPTR(self, _type, records): - print ("NAPTR Record") - print(json.dumps(records, indent=4, separators=(',', ': '))) + def _data_for_NAPTR(self, _type, _records): values = [] - for record in records: - order, preference, flags, service, regexp, replacement = \ - record['rdata'][0].split(' ', 5) - + for r in _records['rdata']: + order, preference, flags, service, regexp, repl = r.split(' ', 5) + values.append({ 'flags': flags[1:-1], 'order': order, 'preference': preference, 'regexp': regexp[1:-1], - 'replacement': replacement, + 'replacement': repl, 'service': service[1:-1] }) return { 'type': _type, - 'ttl': records[0]['ttl'], + 'ttl': _records['ttl'], 'values': values } - - def _data_for_NS(self, _type, records): - values = [] - for record in records: - rdata = record['rdata'] - for content in rdata: - if content[-1] != '.': - content = '{}.'.format(content) - values.append(content) - return { - 'ttl': records[0]['ttl'], - 'type': _type, - 'value': values - } - - def _data_for_PTR(self, _type, records): - return self._data_for_multiple(_type, records[0]) - # rdata[0]? - def _data_for_SRV(self, _type, records): + def _data_for_PTR(self, _type, _records): + + return { + 'ttl': _records['ttl'], + 'type': _type, + 'value' : _records['rdata'][0] + } + + def _data_for_SRV(self, _type, _records): values = [] - for record in records: - weight, port, target = record['rdata'].split(', ', 2) + for r in _records['rdata']: + priority, weight, port, target = r.split(' ', 3) values.append({ 'port': port, - 'priority': record['priority'], - 'target': '{}.'.format(target), + 'priority': priority, + 'target': target, 'weight': weight }) return { 'type': _type, - 'ttl': records[0]['ttl'], + 'ttl': _records['ttl'], 'values': values } - # rdata[0]? - def _data_for_SSHFP(self, _type, records): + def _data_for_SSHFP(self, _type, _records): values = [] - algorithm, fp_type, fingerprint = record['rdata'].split(' ', 2) - values.append({ - 'algorithm': algorithm, - 'fingerprint': fingerprint, - 'fingerprint_type': fp_type - }) - + for r in _records['rdata']: + algorithm, fp_type, fingerprint = r.split(' ', 2) + values.append({ + 'algorithm': algorithm, + 'fingerprint': fingerprint, + 'fingerprint_type': fp_type + }) return { 'type': _type, - 'ttl': records[0]['ttl'], + 'ttl': _records['ttl'], 'values': values } - def _data_for_TXT(self, _type, records): + def _data_for_TXT(self, _type, _records): values = [] - for r in records: - for content in r['rdata']: - values.append(content.replace(';', '\\;')) + for r in _records['rdata']: + r = r[1:-1] + values.append(r.replace(';', '\\;')) return { - 'ttl': records[0]['ttl'], + 'ttl': _records['ttl'], 'type': _type, 'values': values - # 'values': [r['rdata'][0].replace(';', '\\;') for r in records] } - - def _test(self, zone) : @@ -426,23 +355,23 @@ class AkamaiProvider(BaseProvider): } - # print("\n\nRunning test: record get..........\n") - # self._test_record_get(zone_name, "test.basir-test.com", record_type) - # print("\n\nRunning test: record create..........\n") - # self._test_record_create(zone_name, record_name, record_type, params) - # print("\n\nRunning test: record replace..........\n") - # self._test_record_replace(zone_name, record_name, record_type, repl_params) - # print("\n\nRunning test: record delete..........\n") - # self._test_record_delete(zone_name, record_name, record_type) + print("\n\nRunning test: record get..........\n") + self._test_record_get(zone_name, "test.basir-test.com", record_type) + print("\n\nRunning test: record create..........\n") + self._test_record_create(zone_name, record_name, record_type, params) + print("\n\nRunning test: record replace..........\n") + self._test_record_replace(zone_name, record_name, record_type, repl_params) + print("\n\nRunning test: record delete..........\n") + self._test_record_delete(zone_name, record_name, record_type) - # print("\n\nRunning test: zones get..........\n") - # self._test_zones_get() + print("\n\nRunning test: zones get..........\n") + self._test_zones_get() print("\n\nRunning test: zone recordset get..........\n") self._test_zones_recordset_get(zone_name) - # print("\n\nRunning test: Master Zone File get..........\n") - # self._test_master_zone_file_get(zone_name) + print("\n\nRunning test: Master Zone File get..........\n") + self._test_master_zone_file_get(zone_name) return From c87967a239f748ed72f04232cd0f23ffecc01fe9 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Tue, 18 Jun 2019 18:16:15 -0400 Subject: [PATCH 056/176] tuesday sync --- octodns/provider/fastdns.py | 44 +++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index bdd3b82..99a0b93 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -95,6 +95,23 @@ class AkamaiClient(object): return result + def zone_get(self, zone): + path = 'zones/{}'.format(zone) + result = self._request('GET', path) + + return result + + def zone_create(self, contractId, params, gid=None): + path = 'zones?contractId={}'.format(contractId) + + if gid is not None: + path += '&gid={}'.format(gid) + + result = self._request('POST', path, data=params) + + return result + + def zones_get(self, contractIds=None, page=None, pageSize=None, search=None, showAll="true", sortBy="zone", types=None): path = 'zones' @@ -164,7 +181,7 @@ class AkamaiProvider(BaseProvider): 'SRV', 'SSHFP', 'TXT')) def __init__(self, id, client_secret, host, access_token, client_token, - *args, **kwargs): + contractId=None, gid=None, *args, **kwargs): self.log = logging.getLogger('AkamaiProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, ') @@ -174,7 +191,8 @@ class AkamaiProvider(BaseProvider): client_token) self._zone_records = {} - + self._contractId = contractId + self._gid = gid def zone_records(self, zone): """ returns records for a zone, finds it if not present, or @@ -218,6 +236,7 @@ class AkamaiProvider(BaseProvider): found = len(zone.records) - before self.log.info('populate: found %s records, exists=%s', found, exists) + return exists def _data_for_multiple(self, _type, _records): @@ -328,6 +347,27 @@ class AkamaiProvider(BaseProvider): } + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + + domain_name = desired.name[:-1] + + + try: + self._dns_client.zone_get(domain_name) + except AkamaiClientException: + self.log.debug('_apply: no matching zone, creating domain') + + params = self._build_zone_config(domain_name) + + self._dns_client.zone_create(self._contractId, params, self._gid) + + + def _build_zone_config(self, zone, type="primary", comment=None, + def _test(self, zone) : zone_name = zone.name[:len(zone.name)-1] From c1528e3221e25c4e0d053d93d8bffb71a48d5190 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Wed, 19 Jun 2019 19:21:28 -0400 Subject: [PATCH 057/176] implemented _apply_delete, other misc. fixes --- octodns/provider/fastdns.py | 176 +++++++++++++++++++++++++++++------- 1 file changed, 141 insertions(+), 35 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 99a0b93..6306145 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -21,6 +21,7 @@ class AkamaiClientException(Exception): _errorMessages = { 400: "400: Bad request", + 401: "401: Unauthorized", 403: "403: Access is forbidden", 404: "404: Resource not found", 405: "405: Method not supported", @@ -42,6 +43,7 @@ class AkamaiClient(object): self.base = "https://" + _host + "/config-dns/v2/" self.basev1 = "https://" + _host + "/config-dns/v1/" + self.basehost = "https://" + _host sess = requests.Session() sess.auth = EdgeGridAuth( @@ -61,12 +63,14 @@ class AkamaiClient(object): resp = self._sess.request(method, url, params=params, json=data) if resp.status_code > 299: + # print(resp.status_code) raise AkamaiClientException(resp.status_code) resp.raise_for_status() return resp + def record_get(self, zone, name, record_type): path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) @@ -82,6 +86,7 @@ class AkamaiClient(object): def record_delete(self, zone, name, record_type): path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) + print(path) result = self._request('DELETE', path) if result.status_code == 204: @@ -95,6 +100,7 @@ class AkamaiClient(object): return result + def zone_get(self, zone): path = 'zones/{}'.format(zone) result = self._request('GET', path) @@ -148,7 +154,19 @@ class AkamaiClient(object): return result - def records(self, zone_name): + + def contracts_get(self, gid=None): + path = 'data/contracts' + if gid is not None: + path += '?gid={}'.format(gid) + + # result = self._sess.request('GET', path) + + result = self._request('GET', path) + + return result + + def recordsets_get(self, zone_name): resp = self.zone_recordset_get(zone_name, showAll="true") recordset = resp.json().get("recordsets") @@ -181,7 +199,7 @@ class AkamaiProvider(BaseProvider): 'SRV', 'SSHFP', 'TXT')) def __init__(self, id, client_secret, host, access_token, client_token, - contractId=None, gid=None, *args, **kwargs): + contract_id=None, gid=None, *args, **kwargs): self.log = logging.getLogger('AkamaiProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, ') @@ -191,7 +209,7 @@ class AkamaiProvider(BaseProvider): client_token) self._zone_records = {} - self._contractId = contractId + self._contractId = contract_id self._gid = gid def zone_records(self, zone): @@ -201,7 +219,7 @@ class AkamaiProvider(BaseProvider): if zone.name not in self._zone_records: try: name = zone.name[:-1] - self._zone_records[zone.name] = self._dns_client.records(name) + self._zone_records[zone.name] = self._dns_client.recordsets_get(name) except AkamaiClientException: return [] @@ -238,12 +256,12 @@ class AkamaiProvider(BaseProvider): return exists - def _data_for_multiple(self, _type, _records): + def _data_for_multiple(self, _type, records): return { - 'ttl': _records['ttl'], + 'ttl':records['ttl'], 'type': _type, - 'values': [r for r in _records['rdata']] + 'values': [r for r in records['rdata']] } _data_for_A = _data_for_multiple @@ -251,34 +269,34 @@ class AkamaiProvider(BaseProvider): _data_for_NS = _data_for_multiple _data_for_SPF = _data_for_multiple - def _data_for_CNAME(self, _type, _records): - value = _records['rdata'][0] + def _data_for_CNAME(self, _type, records): + value =records['rdata'][0] if (value[-1] != '.') : value = '{}.'.format(value) return { - 'ttl': _records['ttl'], + 'ttl':records['ttl'], 'type': _type, 'value': value } - def _data_for_MX(self, _type, _records): + def _data_for_MX(self, _type, records): values = [] - for r in _records['rdata']: + for r in records['rdata']: preference, exchange = r.split(" ", 1) values.append({ 'preference': preference, 'exchange' : exchange }) return { - 'ttl': _records['ttl'], + 'ttl':records['ttl'], 'type': _type, 'values': values } - def _data_for_NAPTR(self, _type, _records): + def _data_for_NAPTR(self, _type, records): values = [] - for r in _records['rdata']: + for r in records['rdata']: order, preference, flags, service, regexp, repl = r.split(' ', 5) values.append({ @@ -291,21 +309,21 @@ class AkamaiProvider(BaseProvider): }) return { 'type': _type, - 'ttl': _records['ttl'], + 'ttl':records['ttl'], 'values': values } - def _data_for_PTR(self, _type, _records): + def _data_for_PTR(self, _type, records): return { - 'ttl': _records['ttl'], + 'ttl':records['ttl'], 'type': _type, - 'value' : _records['rdata'][0] + 'value' :records['rdata'][0] } - def _data_for_SRV(self, _type, _records): + def _data_for_SRV(self, _type, records): values = [] - for r in _records['rdata']: + for r in records['rdata']: priority, weight, port, target = r.split(' ', 3) values.append({ 'port': port, @@ -315,13 +333,13 @@ class AkamaiProvider(BaseProvider): }) return { 'type': _type, - 'ttl': _records['ttl'], + 'ttl':records['ttl'], 'values': values } - def _data_for_SSHFP(self, _type, _records): + def _data_for_SSHFP(self, _type, records): values = [] - for r in _records['rdata']: + for r in records['rdata']: algorithm, fp_type, fingerprint = r.split(' ', 2) values.append({ 'algorithm': algorithm, @@ -330,18 +348,18 @@ class AkamaiProvider(BaseProvider): }) return { 'type': _type, - 'ttl': _records['ttl'], + 'ttl': records['ttl'], 'values': values } - def _data_for_TXT(self, _type, _records): + def _data_for_TXT(self, _type, records): values = [] - for r in _records['rdata']: + for r in records['rdata']: r = r[1:-1] values.append(r.replace(';', '\\;')) return { - 'ttl': _records['ttl'], + 'ttl': records['ttl'], 'type': _type, 'values': values } @@ -353,20 +371,108 @@ class AkamaiProvider(BaseProvider): self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, len(changes)) - domain_name = desired.name[:-1] + zone_name = desired.name[:-1] + print(zone_name) try: - self._dns_client.zone_get(domain_name) - except AkamaiClientException: - self.log.debug('_apply: no matching zone, creating domain') - - params = self._build_zone_config(domain_name) + self._dns_client.zone_get(zone_name) + except AkamaiClientException as e: + + print("zone not found, creating zone") + self.log.debug('_apply: no matching zone, creating zone') + params = self._build_zone_config(zone_name) self._dns_client.zone_create(self._contractId, params, self._gid) + for change in changes: + class_name = change.__class__.__name__ + print() + print(class_name) + if (class_name == "Delete" ): + print(change.existing.name) + print(change.existing._type) + print(change.existing.data) + elif class_name == "Update": + print(change.existing.name) + print(change.existing._type) + print(change.existing.data) + print("----------------->") + print(change.new.name) + print(change.new._type) + print(change.new.data) + else: + print(change.new.name) + print(change.new._type) + print(change.new.data) + print() + print(change) + print() + getattr(self, '_apply_{}'.format(class_name))(change) + + # Clear out the cache if any + self._zone_records.pop(desired.name, None) + + def _apply_Create(self, change): + new = change.new + params_for = getattr(self, '_params_for_{}'.format(new._type)) + for params in params_for(new): + pass + # self._dns_client.record_create(new.zone.name[:-1], params) + + def _apply_Delete(self, change): + existing = change.existing + zone = existing.zone.name[:-1] + name = existing.name + '.' + zone + record_type = existing._type + + result = self._dns_client.record_delete(zone, name, record_type) + + return result + + def _build_zone_config(self, zone, _type=None, comment=None, masters=[]): + + if _type is None: + _type="primary" - def _build_zone_config(self, zone, type="primary", comment=None, + if self._contractId is None: + self._set_default_contractId() + + return { + "zone": zone, + "type": _type, + "comment": comment, + "masters": masters + } + + def _set_default_contractId(self): + ''' if no contractId is set, but one is required to create a new zone, + this function will try to retrieve any contracts available to the + user, and use the first one it finds + ''' + + try: + + request = self._dns_client.zones_get(self._gid) + response = request.json() + zones = response['zones'] + + contractId = zones[0]['contractId'] + + + self._contractId = contractId + self.log.info("contractId not specified, using contractId=%s", contractId) + + except KeyError: + self.log.debug("_get_default_contractId: key error") + raise + + except: + self.log.debug("_get_default_contractId: unable to find a contractId") + raise + + return + def _test(self, zone) : From a7fb63623e1eaa0f057bb08d730089f81dea1ab1 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 20 Jun 2019 19:43:40 -0400 Subject: [PATCH 058/176] fixed apply_delete, create, and update. need to impl param_for_ fxns --- octodns/provider/fastdns.py | 159 ++++++++++++++++++++++++++---------- 1 file changed, 115 insertions(+), 44 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 6306145..42bf23b 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -78,9 +78,9 @@ class AkamaiClient(object): return result - def record_create(self, zone, name, record_type, params): + def record_create(self, zone, name, record_type, content): path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) - result = self._request('POST', path, data=params) + result = self._request('POST', path, data=content) return result @@ -94,9 +94,9 @@ class AkamaiClient(object): return result - def record_replace(self, zone, name, record_type, params): + def record_replace(self, zone, name, record_type, content): path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) - result = self._request('PUT', path, data=params) + result = self._request('PUT', path, data=content) return result @@ -368,67 +368,122 @@ class AkamaiProvider(BaseProvider): def _apply(self, plan): desired = plan.desired changes = plan.changes - self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, - len(changes)) + self.log.debug('_apply: zone=%s, changes=%d', desired.name, len(changes)) zone_name = desired.name[:-1] - print(zone_name) - - try: self._dns_client.zone_get(zone_name) - except AkamaiClientException as e: - - print("zone not found, creating zone") - self.log.debug('_apply: no matching zone, creating zone') + except AkamaiClientException: + self.log.info("zone not found, creating zone") params = self._build_zone_config(zone_name) self._dns_client.zone_create(self._contractId, params, self._gid) for change in changes: class_name = change.__class__.__name__ - print() - print(class_name) - if (class_name == "Delete" ): - print(change.existing.name) - print(change.existing._type) - print(change.existing.data) - elif class_name == "Update": - print(change.existing.name) - print(change.existing._type) - print(change.existing.data) - print("----------------->") - print(change.new.name) - print(change.new._type) - print(change.new.data) - else: - print(change.new.name) - print(change.new._type) - print(change.new.data) - print() - print(change) - print() getattr(self, '_apply_{}'.format(class_name))(change) # Clear out the cache if any self._zone_records.pop(desired.name, None) def _apply_Create(self, change): + new = change.new - params_for = getattr(self, '_params_for_{}'.format(new._type)) - for params in params_for(new): - pass - # self._dns_client.record_create(new.zone.name[:-1], params) + record_type = new._type + + params_for = getattr(self, '_params_for_{}'.format(record_type)) + values = self._get_values(new.data) + rdata = params_for(values) + + zone = new.zone.name[:-1] + + name = self._set_full_name(new.name, zone) + + content = { + "name": name, + "type": record_type, + "ttl" : new.ttl, + "rdata" : rdata + } + + print() + print ("mock create") + print ("zone=", zone, "name=", name , "record_type=", record_type) + print("content:") + print(json.dumps(content, indent=4, separators=(',', ': '))) + print() + + return + self._dns_client.record_create(zone, name, record_type, content) + + return def _apply_Delete(self, change): - existing = change.existing - zone = existing.zone.name[:-1] - name = existing.name + '.' + zone - record_type = existing._type - result = self._dns_client.record_delete(zone, name, record_type) + zone = change.existing.zone.name[:-1] + name = self._set_full_name(change.existing.name, zone) + record_type = change.existing._type + + self._dns_client.record_delete(zone, name, record_type) + + return + + def _apply_Update(self, change): + + new = change.new + record_type = new._type + + params_for = getattr(self, '_params_for_{}'.format(record_type)) + rdata = params_for(new.data) + + zone = new.zone.name[:-1] + name = self._set_full_name(new.name, zone) + + content = { + "name": name, + "type": record_type, + "ttl" : new.ttl, + "rdata" : rdata + } + + self._dns_client.record_create(zone, name, record_type, content) + + return + + + def _params_for_multiple(self, values): + + rdata = [r for r in values] + + return rdata + + + _params_for_A = _params_for_multiple + _params_for_AAAA = _params_for_multiple + _params_for_NS = _params_for_multiple + _params_for_SPF = _params_for_multiple + + def _params_for_CNAME(self, values): + pass + + def _params_for_MX(self, values): + pass + + def _params_for_NAPTR(self, values): + pass + + def _params_for_PTR(self, values): + pass + + def _params_for_SRV(self, values): + pass + + def _params_for_SSHFP(self, values): + pass + + def _params_for_TXT(self, values): + pass - return result def _build_zone_config(self, zone, _type=None, comment=None, masters=[]): @@ -473,6 +528,22 @@ class AkamaiProvider(BaseProvider): return + def _get_values(self, data): + + try: + vals = data['values'] + except KeyError: + vals = data['value'] + + return vals + + def _set_full_name(self, name, zone): + name = name + '.' + zone + if (name[0] == '.'): ## octodns's name for root is '' + name = name[1:] + + return name + def _test(self, zone) : From 0eb1f97ae39af8acce6145624a8e01cf801af921 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 20 Jun 2019 20:21:35 -0400 Subject: [PATCH 059/176] started implementing _params_for fxns --- octodns/provider/fastdns.py | 58 +++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 15 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 42bf23b..cf19c79 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -396,7 +396,6 @@ class AkamaiProvider(BaseProvider): rdata = params_for(values) zone = new.zone.name[:-1] - name = self._set_full_name(new.name, zone) content = { @@ -434,7 +433,8 @@ class AkamaiProvider(BaseProvider): record_type = new._type params_for = getattr(self, '_params_for_{}'.format(record_type)) - rdata = params_for(new.data) + values = self._get_values(new.data) + rdata = params_for(values) zone = new.zone.name[:-1] name = self._set_full_name(new.name, zone) @@ -446,34 +446,58 @@ class AkamaiProvider(BaseProvider): "rdata" : rdata } - self._dns_client.record_create(zone, name, record_type, content) + self._dns_client.record_replace(zone, name, record_type, content) return def _params_for_multiple(self, values): - - rdata = [r for r in values] - - return rdata + return [r for r in values] + def _params_for_single(self, values): + return values[0] _params_for_A = _params_for_multiple _params_for_AAAA = _params_for_multiple _params_for_NS = _params_for_multiple _params_for_SPF = _params_for_multiple - def _params_for_CNAME(self, values): - pass - def _params_for_MX(self, values): - pass + _params_for_CNAME = _params_for_single + _params_for_PTR = _params_for_single - def _params_for_NAPTR(self, values): - pass - def _params_for_PTR(self, values): - pass + def _params_for_MX(self, values): + rdata = [] + + for r in values: + preference = r.preference + exchange = r.exchange + + record = '{} {}'.format(preference, exchange) + + rdata.append(record) + + return rdata + + def _params_for_NAPTR(self, values): + rdata = [] + + for r in values: + order = r.order + preference = r.preference + flags = "\"" + r.flags + "\"" + service = "\"" + r.service + "\"" + regexp = "\"" + r.regexp + "\"" + repl = r.replacement + + record = '{} {} {} {} {}'.format(order, preference, flags, service, + regexp, repl) + # record = ' '.join([order, preference, flags, service, regexp, repl]) + rdata.append(record) + + return rdata + def _params_for_SRV(self, values): pass @@ -485,6 +509,7 @@ class AkamaiProvider(BaseProvider): pass + def _build_zone_config(self, zone, _type=None, comment=None, masters=[]): if _type is None: @@ -545,6 +570,9 @@ class AkamaiProvider(BaseProvider): return name + + + def _test(self, zone) : zone_name = zone.name[:len(zone.name)-1] From 3edf6969f8e44711c3c6cc72da91bc31c5049a79 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Fri, 21 Jun 2019 13:48:17 -0400 Subject: [PATCH 060/176] finished implementing applys, MVP reached --- octodns/provider/fastdns.py | 72 +++++++++++++++++++++---------------- ta | 0 targets=[azuredns, | 0 3 files changed, 41 insertions(+), 31 deletions(-) create mode 100644 ta create mode 100644 targets=[azuredns, diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index cf19c79..745fc34 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -26,7 +26,7 @@ class AkamaiClientException(Exception): 404: "404: Resource not found", 405: "405: Method not supported", 406: "406: Not Acceptable", - 409: "409: Request not allowed due to conflict with current state of resource", + 409: "409: Request not allowed due to conflict with current state", 415: "415: Unsupported media type", 422: "422: Request body contains an error preventing processing", 500: "500: Internal server error" @@ -43,7 +43,6 @@ class AkamaiClient(object): self.base = "https://" + _host + "/config-dns/v2/" self.basev1 = "https://" + _host + "/config-dns/v1/" - self.basehost = "https://" + _host sess = requests.Session() sess.auth = EdgeGridAuth( @@ -63,11 +62,9 @@ class AkamaiClient(object): resp = self._sess.request(method, url, params=params, json=data) if resp.status_code > 299: - # print(resp.status_code) raise AkamaiClientException(resp.status_code) resp.raise_for_status() - return resp @@ -160,8 +157,6 @@ class AkamaiClient(object): if gid is not None: path += '?gid={}'.format(gid) - # result = self._sess.request('GET', path) - result = self._request('GET', path) return result @@ -405,14 +400,6 @@ class AkamaiProvider(BaseProvider): "rdata" : rdata } - print() - print ("mock create") - print ("zone=", zone, "name=", name , "record_type=", record_type) - print("content:") - print(json.dumps(content, indent=4, separators=(',', ': '))) - print() - - return self._dns_client.record_create(zone, name, record_type, content) return @@ -455,24 +442,23 @@ class AkamaiProvider(BaseProvider): return [r for r in values] def _params_for_single(self, values): - return values[0] + return values _params_for_A = _params_for_multiple _params_for_AAAA = _params_for_multiple _params_for_NS = _params_for_multiple _params_for_SPF = _params_for_multiple - _params_for_CNAME = _params_for_single _params_for_PTR = _params_for_single - def _params_for_MX(self, values): rdata = [] for r in values: - preference = r.preference - exchange = r.exchange + print(json.dumps(r, indent=4, separators=(',', ': '))) + preference = r['preference'] + exchange = r['exchange'] record = '{} {}'.format(preference, exchange) @@ -484,29 +470,53 @@ class AkamaiProvider(BaseProvider): rdata = [] for r in values: - order = r.order - preference = r.preference - flags = "\"" + r.flags + "\"" - service = "\"" + r.service + "\"" - regexp = "\"" + r.regexp + "\"" - repl = r.replacement + order = r['order'] + preference = r['preference'] + flags = "\"" + r['flags'] + "\"" + service = "\"" + r['service'] + "\"" + regexp = "\"" + r['regexp'] + "\"" + repl = r['replacement'] - record = '{} {} {} {} {}'.format(order, preference, flags, service, + record = '{} {} {} {} {} {}'.format(order, preference, flags, service, regexp, repl) # record = ' '.join([order, preference, flags, service, regexp, repl]) rdata.append(record) return rdata - def _params_for_SRV(self, values): - pass + rdata = [] + for r in values: + priority = r['priority'] + weight = r['weight'] + port = r['port'] + target = r['target'] + record = '{} {} {} {}'.format(priority, weight, port, target) + rdata.append(record) + + return rdata + def _params_for_SSHFP(self, values): - pass + rdata = [] + for r in values: + algorithm = r['algorithm'] + fp_type = r['fingerprint_type'] + fp = r['fingerprint'] + record = '{} {} {}'.format(algorithm, fp_type, fp) + rdata.append(record) + + return rdata + def _params_for_TXT(self, values): - pass + rdata = [] + + for r in values: + txt = "\"" + r.replace('\\;', ';') + "\"" + rdata.append(txt) + + return rdata @@ -558,7 +568,7 @@ class AkamaiProvider(BaseProvider): try: vals = data['values'] except KeyError: - vals = data['value'] + vals = [data['value']] return vals diff --git a/ta b/ta new file mode 100644 index 0000000..e69de29 diff --git a/targets=[azuredns, b/targets=[azuredns, new file mode 100644 index 0000000..e69de29 From 8417a4475a8fa7d80a202a0e6a469654bb612731 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Fri, 21 Jun 2019 15:04:27 -0400 Subject: [PATCH 061/176] cleaned up code --- octodns/provider/fastdns.py | 537 +++++++++++------------------------- 1 file changed, 162 insertions(+), 375 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 745fc34..4d6f1cd 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -5,11 +5,9 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -## octodns specfic imports: import requests from akamai.edgegrid import EdgeGridAuth from urlparse import urljoin -import json from collections import defaultdict import logging @@ -32,17 +30,30 @@ class AkamaiClientException(Exception): 500: "500: Internal server error" } - def __init__(self, code): - message = self._errorMessages.get(code) - super(AkamaiClientException, self).__init__(message) + def __init__(self, resp): + try: + message = self._errorMessages.get(resp.status_code) + super(AkamaiClientException, self).__init__(message) + + except: + resp.raise_for_status() class AkamaiClient(object): + ''' + Client for making calls to Akamai Fast DNS API using Python Requests + + Fast DNS Zone Management API V2, found here: + developer.akamai.com/api/web_performance/fast_dns_zone_management/v2.html + + Info on Python Requests library: + https://2.python-requests.org/en/master/ + + ''' def __init__(self, _client_secret, _host, _access_token, _client_token): self.base = "https://" + _host + "/config-dns/v2/" - self.basev1 = "https://" + _host + "/config-dns/v1/" sess = requests.Session() sess.auth = EdgeGridAuth( @@ -56,25 +67,14 @@ class AkamaiClient(object): def _request(self, method, path, params=None, data=None, v1=False): url = urljoin(self.base, path) - if v1: - url = urljoin(self.basev1, path) - resp = self._sess.request(method, url, params=params, json=data) if resp.status_code > 299: - raise AkamaiClientException(resp.status_code) - resp.raise_for_status() + raise AkamaiClientException(resp) return resp - def record_get(self, zone, name, record_type): - - path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) - result = self._request('GET', path) - - return result - def record_create(self, zone, name, record_type, content): path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) result = self._request('POST', path, data=content) @@ -83,12 +83,8 @@ class AkamaiClient(object): def record_delete(self, zone, name, record_type): path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) - print(path) result = self._request('DELETE', path) - if result.status_code == 204: - print ("successfully deleted ", path) - return result def record_replace(self, zone, name, record_type, content): @@ -115,24 +111,6 @@ class AkamaiClient(object): return result - def zones_get(self, contractIds=None, page=None, pageSize=None, search=None, - showAll="true", sortBy="zone", types=None): - path = 'zones' - - params = { - 'contractIds': contractIds, - 'page': page, - 'pageSize': pageSize, - 'search': search, - 'showAll': showAll, - 'sortBy': sortBy, - 'types': types - } - - result = self._request('GET', path, params=params) - - return result - def zone_recordset_get(self, zone, page=None, pageSize=30, search=None, showAll="true", sortBy="name", types=None): @@ -151,16 +129,6 @@ class AkamaiClient(object): return result - - def contracts_get(self, gid=None): - path = 'data/contracts' - if gid is not None: - path += '?gid={}'.format(gid) - - result = self._request('GET', path) - - return result - def recordsets_get(self, zone_name): resp = self.zone_recordset_get(zone_name, showAll="true") @@ -168,25 +136,56 @@ class AkamaiClient(object): return recordset - def master_zone_file_get(self, zone): - - path = 'zones/{}/zone-file'.format(zone) - - try: - result = self._request('GET', path) - - except AkamaiClientException as e: - # not working with API v2, API v1 fallback - path = 'zones/{}'.format(zone) - result = self._request('GET', path, v1=True) - print("Using API v1 fallback") - print("(Probably Ignore)", e.message) - - return result - class AkamaiProvider(BaseProvider): + ''' + Akamai Fast DNS Provider + + fastdns.py: + + Example config file with variables: + " + --- + providers: + config: + class: octodns.provider.yaml.YamlProvider + directory: ./config (example path to directory of zone files) + fastdns: + class: octodns.provider.fastdns.AkamaiProvider + client_secret: env/AKAMAI_CLIENT_SECRET + host: env/AKAMAI_HOST + access_token: env/AKAMAI_ACCESS_TOKEN + client_token: env/AKAMAI_CLIENT_TOKEN + contract_id: env/AKAMAI_CONTRACT_ID (optional) + + zones: + example.com.: + sources: + - config + targets: + - fastdns + " + + The first four variables above can be hidden in environment variables + and octoDNS will automatically search for them in the shell. It is + possible to also hard-code into the config file: eg, contract_id. + + The first four values can be found by generating credentials: + https://control.akamai.com/ + Configure > Organization > Manage APIs > New API Client for me + Select appropriate group, and fill relevant fields. + For "API Service Name", select "DNS—Zone Record Management" + and then set appropriate Access level (Read-Write to make changes). + Then select the "New Credential" button to generate values for above + + The contract_id paramater is optional, and only required for creating + a new zone. If the zone being managed already exists in Akamai for the + user in question, then this paramater is not needed. + + ''' + + SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False @@ -208,8 +207,8 @@ class AkamaiProvider(BaseProvider): self._gid = gid def zone_records(self, zone): - """ returns records for a zone, finds it if not present, or - returns empty if can't find a match + """ returns records for a zone, looks for it if not present, or + returns empty [] if can't find a match """ if zone.name not in self._zone_records: try: @@ -228,7 +227,7 @@ class AkamaiProvider(BaseProvider): for record in self.zone_records(zone): _type =record.get('type') - ## Akamai sends down prefix.zonename., while OctoDNS only expects prefix + ## Akamai sends down prefix.zonename., while octodns expects prefix _name = record.get('name').split("." + zone.name[:-1], 1)[0] if _name == zone.name[:-1] : _name = '' ## root / @ @@ -251,6 +250,85 @@ class AkamaiProvider(BaseProvider): return exists + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('apply: zone=%s, changes=%d', desired.name, len(changes)) + + zone_name = desired.name[:-1] + try: + self._dns_client.zone_get(zone_name) + + except AkamaiClientException: + self.log.info("zone not found, creating zone") + params = self._build_zone_config(zone_name) + self._dns_client.zone_create(self._contractId, params, self._gid) + + for change in changes: + class_name = change.__class__.__name__ + getattr(self, '_apply_{}'.format(class_name))(change) + + # Clear out the cache if any + self._zone_records.pop(desired.name, None) + + def _apply_Create(self, change): + + new = change.new + record_type = new._type + + params_for = getattr(self, '_params_for_{}'.format(record_type)) + values = self._get_values(new.data) + rdata = params_for(values) + + zone = new.zone.name[:-1] + name = self._set_full_name(new.name, zone) + + content = { + "name": name, + "type": record_type, + "ttl" : new.ttl, + "rdata" : rdata + } + + self._dns_client.record_create(zone, name, record_type, content) + + return + + def _apply_Delete(self, change): + + zone = change.existing.zone.name[:-1] + name = self._set_full_name(change.existing.name, zone) + record_type = change.existing._type + + self._dns_client.record_delete(zone, name, record_type) + + return + + def _apply_Update(self, change): + + new = change.new + record_type = new._type + + params_for = getattr(self, '_params_for_{}'.format(record_type)) + values = self._get_values(new.data) + rdata = params_for(values) + + zone = new.zone.name[:-1] + name = self._set_full_name(new.name, zone) + + content = { + "name": name, + "type": record_type, + "ttl" : new.ttl, + "rdata" : rdata + } + + self._dns_client.record_replace(zone, name, record_type, content) + + return + + def _data_for_multiple(self, _type, records): return { @@ -326,6 +404,7 @@ class AkamaiProvider(BaseProvider): 'target': target, 'weight': weight }) + return { 'type': _type, 'ttl':records['ttl'], @@ -341,6 +420,7 @@ class AkamaiProvider(BaseProvider): 'fingerprint': fingerprint, 'fingerprint_type': fp_type }) + return { 'type': _type, 'ttl': records['ttl'], @@ -358,86 +438,8 @@ class AkamaiProvider(BaseProvider): 'type': _type, 'values': values } - - - def _apply(self, plan): - desired = plan.desired - changes = plan.changes - self.log.debug('_apply: zone=%s, changes=%d', desired.name, len(changes)) - - zone_name = desired.name[:-1] - try: - self._dns_client.zone_get(zone_name) - - except AkamaiClientException: - self.log.info("zone not found, creating zone") - params = self._build_zone_config(zone_name) - self._dns_client.zone_create(self._contractId, params, self._gid) - - for change in changes: - class_name = change.__class__.__name__ - getattr(self, '_apply_{}'.format(class_name))(change) - - # Clear out the cache if any - self._zone_records.pop(desired.name, None) - - def _apply_Create(self, change): - - new = change.new - record_type = new._type - - params_for = getattr(self, '_params_for_{}'.format(record_type)) - values = self._get_values(new.data) - rdata = params_for(values) - - zone = new.zone.name[:-1] - name = self._set_full_name(new.name, zone) - - content = { - "name": name, - "type": record_type, - "ttl" : new.ttl, - "rdata" : rdata - } - - self._dns_client.record_create(zone, name, record_type, content) - - return - - def _apply_Delete(self, change): - - zone = change.existing.zone.name[:-1] - name = self._set_full_name(change.existing.name, zone) - record_type = change.existing._type - - self._dns_client.record_delete(zone, name, record_type) - - return - - def _apply_Update(self, change): - - new = change.new - record_type = new._type - - params_for = getattr(self, '_params_for_{}'.format(record_type)) - values = self._get_values(new.data) - rdata = params_for(values) - - zone = new.zone.name[:-1] - name = self._set_full_name(new.name, zone) - - content = { - "name": name, - "type": record_type, - "ttl" : new.ttl, - "rdata" : rdata - } - - self._dns_client.record_replace(zone, name, record_type, content) - - return - - + + def _params_for_multiple(self, values): return [r for r in values] @@ -456,12 +458,10 @@ class AkamaiProvider(BaseProvider): rdata = [] for r in values: - print(json.dumps(r, indent=4, separators=(',', ': '))) preference = r['preference'] exchange = r['exchange'] record = '{} {}'.format(preference, exchange) - rdata.append(record) return rdata @@ -470,16 +470,14 @@ class AkamaiProvider(BaseProvider): rdata = [] for r in values: - order = r['order'] - preference = r['preference'] - flags = "\"" + r['flags'] + "\"" - service = "\"" + r['service'] + "\"" - regexp = "\"" + r['regexp'] + "\"" - repl = r['replacement'] + ordr = r['order'] + pref = r['preference'] + flg = "\"" + r['flags'] + "\"" + srvc = "\"" + r['service'] + "\"" + rgx = "\"" + r['regexp'] + "\"" + rpl = r['replacement'] - record = '{} {} {} {} {} {}'.format(order, preference, flags, service, - regexp, repl) - # record = ' '.join([order, preference, flags, service, regexp, repl]) + record = '{} {} {} {} {} {}'.format(ordr, pref, flg, srvc, rgx, rpl) rdata.append(record) return rdata @@ -518,15 +516,13 @@ class AkamaiProvider(BaseProvider): return rdata - - def _build_zone_config(self, zone, _type=None, comment=None, masters=[]): if _type is None: _type="primary" if self._contractId is None: - self._set_default_contractId() + raise NameError("contractId not specified to create zone") return { "zone": zone, @@ -535,34 +531,6 @@ class AkamaiProvider(BaseProvider): "masters": masters } - def _set_default_contractId(self): - ''' if no contractId is set, but one is required to create a new zone, - this function will try to retrieve any contracts available to the - user, and use the first one it finds - ''' - - try: - - request = self._dns_client.zones_get(self._gid) - response = request.json() - zones = response['zones'] - - contractId = zones[0]['contractId'] - - - self._contractId = contractId - self.log.info("contractId not specified, using contractId=%s", contractId) - - except KeyError: - self.log.debug("_get_default_contractId: key error") - raise - - except: - self.log.debug("_get_default_contractId: unable to find a contractId") - raise - - return - def _get_values(self, data): try: @@ -574,190 +542,9 @@ class AkamaiProvider(BaseProvider): def _set_full_name(self, name, zone): name = name + '.' + zone - if (name[0] == '.'): ## octodns's name for root is '' + + ## octodns's name for root is '' + if (name[0] == '.'): name = name[1:] return name - - - - - - def _test(self, zone) : - - zone_name = zone.name[:len(zone.name)-1] - - record_name = "octo.basir-test.com" - record_type = "A" - params = { - "name": "octo.basir-test.com", - "type": "A", - "ttl": 300, - "rdata": [ - "10.0.0.2", - "10.0.0.3" - ] - } - repl_params = { - "name": "octo.basir-test.com", - "type": "A", - "ttl": 300, - "rdata": [ - "99.99.99.99", - "10.0.0.3", - "1.2.3.4" - ] - } - - - print("\n\nRunning test: record get..........\n") - self._test_record_get(zone_name, "test.basir-test.com", record_type) - print("\n\nRunning test: record create..........\n") - self._test_record_create(zone_name, record_name, record_type, params) - print("\n\nRunning test: record replace..........\n") - self._test_record_replace(zone_name, record_name, record_type, repl_params) - print("\n\nRunning test: record delete..........\n") - self._test_record_delete(zone_name, record_name, record_type) - - print("\n\nRunning test: zones get..........\n") - self._test_zones_get() - - print("\n\nRunning test: zone recordset get..........\n") - self._test_zones_recordset_get(zone_name) - - print("\n\nRunning test: Master Zone File get..........\n") - self._test_master_zone_file_get(zone_name) - - return - - def _test_record_get(self, zone_name, record_name, record_type): - try: - get = self._dns_client.record_get(zone_name, record_name, record_type) - except AkamaiClientException as e: - print ("record get test failed") - print (e.message) - - else: - print("record get test result: ") - print(json.dumps(get.json(), indent=4, separators=(',', ': '))) - - return - - def _test_record_delete(self, zone_name, record_name, record_type): - - try: - delete = self._dns_client.record_delete(zone_name, record_name, record_type) - except AkamaiClientException as e: - print("delete failed") - print(e.message) - return - - - try: - self._dns_client.record_get(zone_name, record_name, record_type) - except AkamaiClientException as e: - print("get on record failed as expected, since record was succesfully deleted") - print ("(Probably Ignore):", e.message) - print ("delete status:", delete.status_code) - else: - print("unexpected condition in test delete") - - return - - def _test_record_create(self, zone_name, record_name, record_type, params): - - try: - create = self._dns_client.record_create(zone_name, record_name, record_type, params) - except AkamaiClientException as e: - print ("create unsuccessful, presumably because it already exists") - print ("(Probably Ignore)", e.message) - else: - print("initial create of", create.json().get("name"), "succesful: ", create.status_code) - - return - - def _test_record_replace(self, zone_name, record_name, record_type, params): - - ## create record to be replaced, if it doesn't already exist - try: - old_params = { - "name": record_name, - "type": record_type, - "ttl": 300, - "rdata": [ - "10.0.0.2", - "10.0.0.3" - ] - } - create = self._dns_client.record_create(zone_name, record_name, record_type, old_params) - except AkamaiClientException as e: - print ("initial create unsuccessful, presumably because it already exists") - print ("(Probably Ignore)", e.message) - else: - print("initial create of record to be replaced", create.json().get("name"), "succesful: ", create.status_code) - - - ## test replace - try: - replace = self._dns_client.record_replace(zone_name, record_name, record_type, params) - except AkamaiClientException as e: - print("replace failed") - print(e.message) - return - else: - try: - record = self._dns_client.record_get(zone_name, record_name, record_type) - except AkamaiClientException as e: - print("retrieval in replacement failed") - print(e.message) - else: - new_data = record.json() - - if (new_data != params): - print("replace failed, records don't match") - print("current data:") - print(new_data) - print("expected data:") - print(params) - - else: - print("replace succesful") - print("replace status:", replace.status_code) - - def _test_zones_get(self): - try: - zonesList = self._dns_client.zones_get() - except AkamaiClientException as e: - print ("zones get test failed") - print (e.message) - - else: - print("zones list: ") - print(json.dumps(zonesList.json(), indent=4, separators=(',', ': '))) - - return - - def _test_zones_recordset_get(self, zone_name): - try: - zoneRecordset = self._dns_client.zone_recordset_get(zone_name) - except AkamaiClientException as e: - print("zone recordset retrieval test failed") - print (e.message) - else: - print("zone recordset: ") - print(json.dumps(zoneRecordset.json(), indent=4, separators=(',', ': '))) - return - - def _test_master_zone_file_get(self, zone_name): - try: - mzf = self._dns_client.master_zone_file_get(zone_name) - - except AkamaiClientException as e: - print("MZF retrieval test failed") - print (e.message) - - else: - print("Master Zone File:") - print(json.dumps(mzf.json(), indent=4, separators=(',', ': '))) - - return From 173d41aa9930346e8d92e1521e89ab3cf94cf795 Mon Sep 17 00:00:00 2001 From: basirjamil Date: Fri, 21 Jun 2019 15:06:14 -0400 Subject: [PATCH 062/176] Delete targets=[azuredns, --- targets=[azuredns, | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 targets=[azuredns, diff --git a/targets=[azuredns, b/targets=[azuredns, deleted file mode 100644 index e69de29..0000000 From c7251444c82d1562ac1d280f3e300cb337b3717f Mon Sep 17 00:00:00 2001 From: basirjamil Date: Fri, 21 Jun 2019 15:07:14 -0400 Subject: [PATCH 063/176] Delete ta --- ta | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 ta diff --git a/ta b/ta deleted file mode 100644 index e69de29..0000000 From 2397030cf6d2e7c8d1698b1a66e888a3f34eef4d Mon Sep 17 00:00:00 2001 From: basirjamil Date: Fri, 21 Jun 2019 15:07:24 -0400 Subject: [PATCH 064/176] Delete settings.json --- .vscode/settings.json | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 38e6c1e..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "python.pythonPath": "C:\\Python27\\python.exe" -} \ No newline at end of file From 6f4e60ac141fd53eb29f112f2b82238a5610d8de Mon Sep 17 00:00:00 2001 From: basirjamil Date: Fri, 21 Jun 2019 15:08:11 -0400 Subject: [PATCH 065/176] Delete settings.json --- octodns/.vscode/settings.json | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 octodns/.vscode/settings.json diff --git a/octodns/.vscode/settings.json b/octodns/.vscode/settings.json deleted file mode 100644 index 38e6c1e..0000000 --- a/octodns/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "python.pythonPath": "C:\\Python27\\python.exe" -} \ No newline at end of file From 70a1627757cb6eafd32596a3a35acd7626df68c5 Mon Sep 17 00:00:00 2001 From: basirjamil Date: Fri, 21 Jun 2019 15:08:43 -0400 Subject: [PATCH 066/176] Delete settings.json --- octodns/provider/.vscode/settings.json | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 octodns/provider/.vscode/settings.json diff --git a/octodns/provider/.vscode/settings.json b/octodns/provider/.vscode/settings.json deleted file mode 100644 index 88322cc..0000000 --- a/octodns/provider/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "python.pythonPath": "C:\\Python27\\python.exe" -} \ No newline at end of file From 5b143e422c2edbbb73d220baf428415b432698de Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Fri, 21 Jun 2019 17:26:54 -0400 Subject: [PATCH 067/176] cleanup - fixed lint issues --- octodns/provider/fastdns.py | 178 +++++++++++++++++------------------- 1 file changed, 85 insertions(+), 93 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 4d6f1cd..26c4351 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -12,13 +12,13 @@ from collections import defaultdict import logging from ..record import Record -from .base import BaseProvider +from .base import BaseProvider class AkamaiClientException(Exception): - + _errorMessages = { - 400: "400: Bad request", + 400: "400: Bad request", 401: "401: Unauthorized", 403: "403: Access is forbidden", 404: "404: Resource not found", @@ -35,15 +35,15 @@ class AkamaiClientException(Exception): message = self._errorMessages.get(resp.status_code) super(AkamaiClientException, self).__init__(message) - except: + except: resp.raise_for_status() class AkamaiClient(object): - ''' + ''' Client for making calls to Akamai Fast DNS API using Python Requests - Fast DNS Zone Management API V2, found here: + Fast DNS Zone Management API V2, found here: developer.akamai.com/api/web_performance/fast_dns_zone_management/v2.html Info on Python Requests library: @@ -63,9 +63,8 @@ class AkamaiClient(object): ) self._sess = sess - def _request(self, method, path, params=None, data=None, v1=False): - + url = urljoin(self.base, path) resp = self._sess.request(method, url, params=params, json=data) @@ -73,7 +72,6 @@ class AkamaiClient(object): raise AkamaiClientException(resp) return resp - def record_create(self, zone, name, record_type, content): path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) @@ -84,7 +82,7 @@ class AkamaiClient(object): def record_delete(self, zone, name, record_type): path = 'zones/{}/names/{}/types/{}'.format(zone, name, record_type) result = self._request('DELETE', path) - + return result def record_replace(self, zone, name, record_type, content): @@ -93,7 +91,6 @@ class AkamaiClient(object): return result - def zone_get(self, zone): path = 'zones/{}'.format(zone) result = self._request('GET', path) @@ -105,17 +102,16 @@ class AkamaiClient(object): if gid is not None: path += '&gid={}'.format(gid) - + result = self._request('POST', path, data=params) - + return result - def zone_recordset_get(self, zone, page=None, pageSize=30, search=None, - showAll="true", sortBy="name", types=None): + showAll="true", sortBy="name", types=None): params = { - 'page': page, + 'page': page, 'pageSize': pageSize, 'search': search, 'showAll': showAll, @@ -126,11 +122,10 @@ class AkamaiClient(object): path = 'zones/{}/recordsets'.format(zone) result = self._request('GET', path, params=params) - return result def recordsets_get(self, zone_name): - + resp = self.zone_recordset_get(zone_name, showAll="true") recordset = resp.json().get("recordsets") @@ -170,8 +165,8 @@ class AkamaiProvider(BaseProvider): The first four variables above can be hidden in environment variables and octoDNS will automatically search for them in the shell. It is possible to also hard-code into the config file: eg, contract_id. - - The first four values can be found by generating credentials: + + The first four values can be found by generating credentials: https://control.akamai.com/ Configure > Organization > Manage APIs > New API Client for me Select appropriate group, and fill relevant fields. @@ -179,41 +174,41 @@ class AkamaiProvider(BaseProvider): and then set appropriate Access level (Read-Write to make changes). Then select the "New Credential" button to generate values for above - The contract_id paramater is optional, and only required for creating - a new zone. If the zone being managed already exists in Akamai for the - user in question, then this paramater is not needed. + The contract_id paramater is optional, and only required for creating + a new zone. If the zone being managed already exists in Akamai for the + user in question, then this paramater is not needed. ''' - SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False - SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', + SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', 'SRV', 'SSHFP', 'TXT')) - def __init__(self, id, client_secret, host, access_token, client_token, - contract_id=None, gid=None, *args, **kwargs): - + def __init__(self, id, client_secret, host, access_token, client_token, + contract_id=None, gid=None, *args, **kwargs): + self.log = logging.getLogger('AkamaiProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, ') super(AkamaiProvider, self).__init__(id, *args, **kwargs) - self._dns_client = AkamaiClient(client_secret, host, access_token, - client_token) - + self._dns_client = AkamaiClient(client_secret, host, access_token, + client_token) + self._zone_records = {} self._contractId = contract_id self._gid = gid def zone_records(self, zone): - """ returns records for a zone, looks for it if not present, or + """ returns records for a zone, looks for it if not present, or returns empty [] if can't find a match """ if zone.name not in self._zone_records: try: name = zone.name[:-1] - self._zone_records[zone.name] = self._dns_client.recordsets_get(name) + self._zone_records[zone.name] = \ + self._dns_client.recordsets_get(name) except AkamaiClientException: return [] @@ -225,12 +220,12 @@ class AkamaiProvider(BaseProvider): values = defaultdict(lambda: defaultdict(list)) for record in self.zone_records(zone): - - _type =record.get('type') - ## Akamai sends down prefix.zonename., while octodns expects prefix - _name = record.get('name').split("." + zone.name[:-1], 1)[0] - if _name == zone.name[:-1] : - _name = '' ## root / @ + + _type = record.get('type') + # Akamai sends down prefix.zonename., while octodns expects prefix + _name = record.get('name').split("." + zone.name[:-1], 1)[0] + if _name == zone.name[:-1]: + _name = '' # root / @ if _type not in self.SUPPORTS: continue @@ -240,21 +235,20 @@ class AkamaiProvider(BaseProvider): for name, types in values.items(): for _type, records in types.items(): data_for = getattr(self, '_data_for_{}'.format(_type)) - record = Record.new(zone, name, data_for(_type, records[0]), - source=self, lenient=lenient) + record = Record.new(zone, name, data_for(_type, records[0]), + source=self, lenient=lenient) zone.add_record(record, lenient=lenient) - + exists = zone.name in self._zone_records found = len(zone.records) - before - self.log.info('populate: found %s records, exists=%s', found, exists) + self.log.info('populate: found %s records, exists=%s', found, exists) return exists - def _apply(self, plan): desired = plan.desired changes = plan.changes - self.log.debug('apply: zone=%s, changes=%d', desired.name, len(changes)) + self.log.debug('apply: zone=%s, chnges=%d', desired.name, len(changes)) zone_name = desired.name[:-1] try: @@ -287,13 +281,13 @@ class AkamaiProvider(BaseProvider): content = { "name": name, "type": record_type, - "ttl" : new.ttl, - "rdata" : rdata + "ttl": new.ttl, + "rdata": rdata } self._dns_client.record_create(zone, name, record_type, content) - return + return def _apply_Delete(self, change): @@ -303,10 +297,10 @@ class AkamaiProvider(BaseProvider): self._dns_client.record_delete(zone, name, record_type) - return + return def _apply_Update(self, change): - + new = change.new record_type = new._type @@ -320,35 +314,34 @@ class AkamaiProvider(BaseProvider): content = { "name": name, "type": record_type, - "ttl" : new.ttl, - "rdata" : rdata + "ttl": new.ttl, + "rdata": rdata } self._dns_client.record_replace(zone, name, record_type, content) - return - + return def _data_for_multiple(self, _type, records): - + return { - 'ttl':records['ttl'], - 'type': _type, + 'ttl': records['ttl'], + 'type': _type, 'values': [r for r in records['rdata']] - } + } _data_for_A = _data_for_multiple _data_for_AAAA = _data_for_multiple _data_for_NS = _data_for_multiple _data_for_SPF = _data_for_multiple - def _data_for_CNAME(self, _type, records): - value =records['rdata'][0] - if (value[-1] != '.') : + def _data_for_CNAME(self, _type, records): + value = records['rdata'][0] + if (value[-1] != '.'): value = '{}.'.format(value) return { - 'ttl':records['ttl'], + 'ttl': records['ttl'], 'type': _type, 'value': value } @@ -359,10 +352,10 @@ class AkamaiProvider(BaseProvider): preference, exchange = r.split(" ", 1) values.append({ 'preference': preference, - 'exchange' : exchange - }) + 'exchange': exchange + }) return { - 'ttl':records['ttl'], + 'ttl': records['ttl'], 'type': _type, 'values': values } @@ -371,29 +364,29 @@ class AkamaiProvider(BaseProvider): values = [] for r in records['rdata']: order, preference, flags, service, regexp, repl = r.split(' ', 5) - + values.append({ 'flags': flags[1:-1], 'order': order, 'preference': preference, 'regexp': regexp[1:-1], - 'replacement': repl, + 'replacement': repl, 'service': service[1:-1] }) return { 'type': _type, - 'ttl':records['ttl'], + 'ttl': records['ttl'], 'values': values } - + def _data_for_PTR(self, _type, records): return { - 'ttl':records['ttl'], + 'ttl': records['ttl'], 'type': _type, - 'value' :records['rdata'][0] - } - + 'value': records['rdata'][0] + } + def _data_for_SRV(self, _type, records): values = [] for r in records['rdata']: @@ -407,7 +400,7 @@ class AkamaiProvider(BaseProvider): return { 'type': _type, - 'ttl':records['ttl'], + 'ttl': records['ttl'], 'values': values } @@ -420,11 +413,11 @@ class AkamaiProvider(BaseProvider): 'fingerprint': fingerprint, 'fingerprint_type': fp_type }) - + return { 'type': _type, 'ttl': records['ttl'], - 'values': values + 'values': values } def _data_for_TXT(self, _type, records): @@ -438,8 +431,7 @@ class AkamaiProvider(BaseProvider): 'type': _type, 'values': values } - - + def _params_for_multiple(self, values): return [r for r in values] @@ -453,7 +445,7 @@ class AkamaiProvider(BaseProvider): _params_for_CNAME = _params_for_single _params_for_PTR = _params_for_single - + def _params_for_MX(self, values): rdata = [] @@ -471,15 +463,15 @@ class AkamaiProvider(BaseProvider): for r in values: ordr = r['order'] - pref = r['preference'] + prf = r['preference'] flg = "\"" + r['flags'] + "\"" srvc = "\"" + r['service'] + "\"" rgx = "\"" + r['regexp'] + "\"" rpl = r['replacement'] - - record = '{} {} {} {} {} {}'.format(ordr, pref, flg, srvc, rgx, rpl) + + record = '{} {} {} {} {} {}'.format(ordr, prf, flg, srvc, rgx, rpl) rdata.append(record) - + return rdata def _params_for_SRV(self, values): @@ -494,7 +486,7 @@ class AkamaiProvider(BaseProvider): rdata.append(record) return rdata - + def _params_for_SSHFP(self, values): rdata = [] for r in values: @@ -506,10 +498,10 @@ class AkamaiProvider(BaseProvider): rdata.append(record) return rdata - + def _params_for_TXT(self, values): rdata = [] - + for r in values: txt = "\"" + r.replace('\\;', ';') + "\"" rdata.append(txt) @@ -518,11 +510,11 @@ class AkamaiProvider(BaseProvider): def _build_zone_config(self, zone, _type=None, comment=None, masters=[]): - if _type is None: - _type="primary" - + if _type is None: + _type = "primary" + if self._contractId is None: - raise NameError("contractId not specified to create zone") + raise NameError("contractId not specified to create zone") return { "zone": zone, @@ -537,14 +529,14 @@ class AkamaiProvider(BaseProvider): vals = data['values'] except KeyError: vals = [data['value']] - + return vals def _set_full_name(self, name, zone): name = name + '.' + zone - ## octodns's name for root is '' - if (name[0] == '.'): + # octodns's name for root is '' + if (name[0] == '.'): name = name[1:] return name From 550f5b14a391d741e223ba0ef64e74cb18b6f6a4 Mon Sep 17 00:00:00 2001 From: Basir Date: Wed, 26 Jun 2019 16:19:52 -0400 Subject: [PATCH 068/176] started fastdns testing --- tests/test_octodns_provider_fastdns.py | 70 ++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 tests/test_octodns_provider_fastdns.py diff --git a/tests/test_octodns_provider_fastdns.py b/tests/test_octodns_provider_fastdns.py new file mode 100644 index 0000000..baf28ea --- /dev/null +++ b/tests/test_octodns_provider_fastdns.py @@ -0,0 +1,70 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from mock import Mock, call +from os.path import dirname, join +from requests import HTTPError +from requests_mock import ANY, mock as requests_mock +from unittest import TestCase + +from octodns.record import Record +from octodns.provider.fastdns import AkamaiProvider, AkamaiClientException +from octodns.provider.yaml import YamlProvider +from octodns.zone import Zone + + +class TestFastdnsProvider(TestCase): + expected = Zone('unit.tests.', []) + source = YamlProvider('test', join(dirname(__file__), 'config')) + source.populate(expected) + + def test_populate(self): + provider = AkamaiProvider("test", "client_secret", "host", "access_token", + "client_token") + + # Bad Auth + with requests_mock() as mock: + mock.get(ANY, status_code=401, + text='{"message": "Authentication failed"}') + + with self.assertRaises(Exception) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals("401: Unauthorized", ctx.exception.message) + + # general error + with requests_mock() as mock: + mock.get(ANY, status_code=502, text='Things caught fire') + + with self.assertRaises(HTTPError) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(502, ctx.exception.response.status_code) + + # Non-existant zone doesn't populate anything + with requests_mock() as mock: + mock.get(ANY, status_code=404, + text='{"message": "Domain `foo.bar` not found"}') + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(set(), zone.records) + + # # No diffs == no changes + # with requests_mock() as mock: + # base = 'https://api.dnsimple.com/v2/42/zones/unit.tests/' \ + # 'records?page=' + # with open('tests/fixtures/dnsimple-page-1.json') as fh: + # mock.get('{}{}'.format(base, 1), text=fh.read()) + # with open('tests/fixtures/dnsimple-page-2.json') as fh: + # mock.get('{}{}'.format(base, 2), text=fh.read()) + + # zone = Zone('unit.tests.', []) + # provider.populate(zone) + # self.assertEquals(16, len(zone.records)) + # changes = self.expected.changes(zone, provider) + # self.assertEquals(0, len(changes)) \ No newline at end of file From ea3b84091ea4f4760d3a165bec24ac293cdea092 Mon Sep 17 00:00:00 2001 From: Basir Date: Wed, 26 Jun 2019 16:25:57 -0400 Subject: [PATCH 069/176] encoding issue --- octodns/provider/fastdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 26c4351..7ad5988 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -170,7 +170,7 @@ class AkamaiProvider(BaseProvider): https://control.akamai.com/ Configure > Organization > Manage APIs > New API Client for me Select appropriate group, and fill relevant fields. - For "API Service Name", select "DNS—Zone Record Management" + For API Service Name, select DNS—Zone Record Management and then set appropriate Access level (Read-Write to make changes). Then select the "New Credential" button to generate values for above From ddbad2498f6a02ea0edbecc16ecea2bf70b7fab6 Mon Sep 17 00:00:00 2001 From: Rhosyn Celyn Date: Tue, 2 Jul 2019 13:25:30 +0100 Subject: [PATCH 070/176] Fixes for escaping TXT properly, extra test coverage --- octodns/provider/mythicbeasts.py | 22 +++++++++++++++++---- tests/test_octodns_provider_mythicbeasts.py | 20 +++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py index 3470ff1..17029db 100644 --- a/octodns/provider/mythicbeasts.py +++ b/octodns/provider/mythicbeasts.py @@ -174,6 +174,19 @@ class MythicBeastsProvider(BaseProvider): max([raw_values['ttl'] for raw_values in data['raw_values']]), } + @staticmethod + def _data_for_TXT(_type, data): + return { + 'type': _type, + 'values': + [ + str(raw_values['value']).replace(';', '\\;') + for raw_values in data['raw_values'] + ], + 'ttl': + max([raw_values['ttl'] for raw_values in data['raw_values']]), + } + @staticmethod def _data_for_MX(_type, data): ttl = max([raw_values['ttl'] for raw_values in data['raw_values']]) @@ -296,7 +309,6 @@ class MythicBeastsProvider(BaseProvider): {'raw_values': [{'value': value, 'ttl': ttl}]}) _data_for_NS = _data_for_multiple - _data_for_TXT = _data_for_multiple _data_for_A = _data_for_multiple _data_for_AAAA = _data_for_multiple @@ -331,9 +343,6 @@ class MythicBeastsProvider(BaseProvider): _ttl = int(match.group('ttl')) _value = match.group('value').strip() - if _type == 'TXT': - _value = _value.replace(';', '\\;') - if hasattr(self, '_data_for_{}'.format(_type)): if _name not in data[_type]: data[_type][_name] = { @@ -383,6 +392,11 @@ class MythicBeastsProvider(BaseProvider): base = '{} {} {} {}'.format(action, hostname, ttl, _type) + # Unescape TXT records + if _type == 'TXT': + values = [value.replace('\\;', ';') for value in values] + + # Handle specific types or default if _type == 'SSHFP': data = values[0].data commands.append('{} {} {} {}'.format( diff --git a/tests/test_octodns_provider_mythicbeasts.py b/tests/test_octodns_provider_mythicbeasts.py index 8181562..5acbc55 100644 --- a/tests/test_octodns_provider_mythicbeasts.py +++ b/tests/test_octodns_provider_mythicbeasts.py @@ -61,6 +61,18 @@ class TestMythicBeastsProvider(TestCase): self.assertTrue(isinstance(test_multiple, dict)) self.assertEquals(2, len(test_multiple['values'])) + def test_data_for_txt(self): + test_data = { + 'raw_values': [ + {'value': 'v=DKIM1; k=rsa; p=prawf', 'ttl': 60}, + {'value': 'prawf prawf dyma prawf', 'ttl': 300}], + 'zone': 'unit.tests.', + } + test_txt = MythicBeastsProvider._data_for_TXT('', test_data) + self.assertTrue(isinstance(test_txt, dict)) + self.assertEquals(2, len(test_txt['values'])) + self.assertEquals('v=DKIM1\\; k=rsa\\; p=prawf', test_txt['values'][0]) + def test_data_for_MX(self): test_data = { 'raw_values': [ @@ -193,6 +205,11 @@ class TestMythicBeastsProvider(TestCase): 'type': 'TXT', 'value': 'prawf prawf dyma prawf', })) + zone.add_record(Record.new(zone, 'prawf-txt2', { + 'ttl': 60, + 'type': 'TXT', + 'value': 'v=DKIM1\\; k=rsa\\; p=prawf', + })) with requests_mock() as mock: mock.post(ANY, status_code=200, text='') @@ -219,6 +236,7 @@ class TestMythicBeastsProvider(TestCase): 'ADD prawf-aaaa.unit.tests 60 AAAA b:b::b', 'ADD prawf-aaaa.unit.tests 60 AAAA c:c::c:c', 'ADD prawf-txt.unit.tests 60 TXT prawf prawf dyma prawf', + 'ADD prawf-txt2.unit.tests 60 TXT v=DKIM1; k=rsa; p=prawf', ] generated_commands.sort() @@ -231,6 +249,7 @@ class TestMythicBeastsProvider(TestCase): # Now test deletion existing = 'prawf-txt 300 TXT prawf prawf dyma prawf\n' \ + 'prawf-txt2 300 TXT v=DKIM1; k=rsa; p=prawf\n' \ 'prawf-a 60 A 1.2.3.4' with requests_mock() as mock: @@ -249,6 +268,7 @@ class TestMythicBeastsProvider(TestCase): expected_commands = [ 'DELETE prawf-a.unit.tests 60 A 1.2.3.4', 'DELETE prawf-txt.unit.tests 300 TXT prawf prawf dyma prawf', + 'DELETE prawf-txt2.unit.tests 300 TXT v=DKIM1; k=rsa; p=prawf', ] generated_commands.sort() From 158add8eb6e7fefe40e8a0865f786838b631d0d2 Mon Sep 17 00:00:00 2001 From: cclauss Date: Fri, 12 Jul 2019 01:28:10 +0200 Subject: [PATCH 071/176] Modernize Python 2 code to prepare for Python 3 --- .travis.yml | 10 ++++++++-- octodns/cmds/report.py | 8 +++++--- octodns/manager.py | 8 ++++---- octodns/provider/base.py | 4 +++- octodns/provider/cloudflare.py | 2 +- octodns/provider/ns1.py | 6 ++++-- octodns/provider/plan.py | 18 ++++++++++-------- octodns/provider/route53.py | 13 ++++++++++--- octodns/record/__init__.py | 18 +++++++++++++----- octodns/zone.py | 4 +++- requirements.txt | 4 ++-- tests/test_octodns_provider_base.py | 16 +++++++++------- tests/test_octodns_provider_cloudflare.py | 6 +++--- tests/test_octodns_provider_googlecloud.py | 2 +- 14 files changed, 76 insertions(+), 43 deletions(-) diff --git a/.travis.yml b/.travis.yml index b17ca01..b1a8204 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,12 @@ language: python -python: - - 2.7 +matrix: + include: + - python: 2.7 + - python: 3.7 + dist: xenial # required for Python >= 3.7 on Travis CI + allow_failures: + - python: 3.7 +before_install: pip install --upgrade pip script: ./script/cibuild notifications: email: diff --git a/octodns/cmds/report.py b/octodns/cmds/report.py index 2b32e77..3a26052 100755 --- a/octodns/cmds/report.py +++ b/octodns/cmds/report.py @@ -13,6 +13,8 @@ from logging import getLogger from sys import stdout import re +from six import text_type + from octodns.cmds.args import ArgumentParser from octodns.manager import Manager from octodns.zone import Zone @@ -65,7 +67,7 @@ def main(): resolver = AsyncResolver(configure=False, num_workers=int(args.num_workers)) if not ip_addr_re.match(server): - server = unicode(query(server, 'A')[0]) + server = text_type(query(server, 'A')[0]) log.info('server=%s', server) resolver.nameservers = [server] resolver.lifetime = int(args.timeout) @@ -81,12 +83,12 @@ def main(): stdout.write(',') stdout.write(record._type) stdout.write(',') - stdout.write(unicode(record.ttl)) + stdout.write(text_type(record.ttl)) compare = {} for future in futures: stdout.write(',') try: - answers = [unicode(r) for r in future.result()] + answers = [text_type(r) for r in future.result()] except (NoAnswer, NoNameservers): answers = ['*no answer*'] except NXDOMAIN: diff --git a/octodns/manager.py b/octodns/manager.py index 4952315..89db845 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -276,9 +276,9 @@ class Manager(object): try: sources = [self.providers[source] for source in sources] - except KeyError: + except KeyError as e: raise Exception('Zone {}, unknown source: {}'.format(zone_name, - source)) + e)) try: trgs = [] @@ -397,9 +397,9 @@ class Manager(object): try: sources = [self.providers[source] for source in sources] - except KeyError: + except KeyError as e: raise Exception('Zone {}, unknown source: {}'.format(zone_name, - source)) + e)) for source in sources: if isinstance(source, YamlProvider): diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 2c93e49..96d1be1 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -5,6 +5,8 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +from six import text_type + from ..source.base import BaseSource from ..zone import Zone from .plan import Plan @@ -68,7 +70,7 @@ class BaseProvider(BaseSource): changes=changes) if extra: self.log.info('plan: extra changes\n %s', '\n ' - .join([unicode(c) for c in extra])) + .join([text_type(c) for c in extra])) changes += extra if changes: diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 881c2fd..ac1fe9f 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -442,7 +442,7 @@ class CloudflareProvider(BaseProvider): # Round trip the single value through a record to contents flow # to get a consistent _gen_data result that matches what # went in to new_contents - data = self._gen_data(r).next() + data = next(self._gen_data(r)) # Record the record_id and data for this existing record key = self._gen_key(data) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 5fdf5b0..d3faf21 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -13,6 +13,8 @@ from nsone.rest.errors import RateLimitException, ResourceException from incf.countryutils import transformations from time import sleep +from six import text_type + from ..record import Record from .base import BaseProvider @@ -76,9 +78,9 @@ class Ns1Provider(BaseProvider): else: values.extend(answer['answer']) codes.append([]) - values = [unicode(x) for x in values] + values = [text_type(x) for x in values] geo = OrderedDict( - {unicode(k): [unicode(x) for x in v] for k, v in geo.items()} + {text_type(k): [text_type(x) for x in v] for k, v in geo.items()} ) data['values'] = values data['geo'] = geo diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py index bae244f..ad14b6d 100644 --- a/octodns/provider/plan.py +++ b/octodns/provider/plan.py @@ -9,6 +9,8 @@ from StringIO import StringIO from logging import DEBUG, ERROR, INFO, WARN, getLogger from sys import stdout +from six import text_type + class UnsafePlan(Exception): pass @@ -147,11 +149,11 @@ class PlanLogger(_PlanOutput): def _value_stringifier(record, sep): try: - values = [unicode(v) for v in record.values] + values = [text_type(v) for v in record.values] except AttributeError: values = [record.value] for code, gv in sorted(getattr(record, 'geo', {}).items()): - vs = ', '.join([unicode(v) for v in gv.values]) + vs = ', '.join([text_type(v) for v in gv.values]) values.append('{}: {}'.format(code, vs)) return sep.join(values) @@ -193,7 +195,7 @@ class PlanMarkdown(_PlanOutput): fh.write(' | ') # TTL if existing: - fh.write(unicode(existing.ttl)) + fh.write(text_type(existing.ttl)) fh.write(' | ') fh.write(_value_stringifier(existing, '; ')) fh.write(' | |\n') @@ -201,7 +203,7 @@ class PlanMarkdown(_PlanOutput): fh.write('| | | | ') if new: - fh.write(unicode(new.ttl)) + fh.write(text_type(new.ttl)) fh.write(' | ') fh.write(_value_stringifier(new, '; ')) fh.write(' | ') @@ -210,7 +212,7 @@ class PlanMarkdown(_PlanOutput): fh.write(' |\n') fh.write('\nSummary: ') - fh.write(unicode(plan)) + fh.write(text_type(plan)) fh.write('\n\n') else: fh.write('## No changes were planned\n') @@ -261,7 +263,7 @@ class PlanHtml(_PlanOutput): # TTL if existing: fh.write(' ') - fh.write(unicode(existing.ttl)) + fh.write(text_type(existing.ttl)) fh.write('\n ') fh.write(_value_stringifier(existing, '
')) fh.write('\n \n \n') @@ -270,7 +272,7 @@ class PlanHtml(_PlanOutput): if new: fh.write(' ') - fh.write(unicode(new.ttl)) + fh.write(text_type(new.ttl)) fh.write('\n ') fh.write(_value_stringifier(new, '
')) fh.write('\n ') @@ -279,7 +281,7 @@ class PlanHtml(_PlanOutput): fh.write('\n \n') fh.write(' \n Summary: ') - fh.write(unicode(plan)) + fh.write(text_type(plan)) fh.write('\n \n\n') else: fh.write('No changes were planned') diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 4cf7c99..7154096 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -14,10 +14,17 @@ from uuid import uuid4 import logging import re +from six import text_type + from ..record import Record, Update from ..record.geo import GeoCodes from .base import BaseProvider +try: + cmp +except NameError: + def cmp(x, y): + return (x > y) - (x < y) octal_re = re.compile(r'\\(\d\d\d)') @@ -1037,8 +1044,8 @@ class Route53Provider(BaseProvider): # ip_address's returned object for equivalence # E.g 2001:4860:4860::8842 -> 2001:4860:4860:0:0:0:0:8842 if value: - value = ip_address(unicode(value)) - config_ip_address = ip_address(unicode(config['IPAddress'])) + value = ip_address(text_type(value)) + config_ip_address = ip_address(text_type(config['IPAddress'])) else: # No value so give this a None to match value's config_ip_address = None @@ -1059,7 +1066,7 @@ class Route53Provider(BaseProvider): fqdn, record._type, value) try: - ip_address(unicode(value)) + ip_address(text_type(value)) # We're working with an IP, host is the Host header healthcheck_host = record.healthcheck_host except (AddressValueError, ValueError): diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index dca6100..2efdf0e 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -9,8 +9,16 @@ from ipaddress import IPv4Address, IPv6Address from logging import getLogger import re +from six import string_types, text_type + from .geo import GeoCodes +try: + cmp +except NameError: + def cmp(x, y): + return (x > y) - (x < y) + class Change(object): @@ -130,7 +138,7 @@ class Record(object): self.__class__.__name__, name) self.zone = zone # force everything lower-case just to be safe - self.name = unicode(name).lower() if name else name + self.name = text_type(name).lower() if name else name self.source = source self.ttl = int(data['ttl']) @@ -292,7 +300,7 @@ class _ValuesMixin(object): return ret def __repr__(self): - values = "['{}']".format("', '".join([unicode(v) + values = "['{}']".format("', '".join([text_type(v) for v in self.values])) return '<{} {} {}, {}, {}>'.format(self.__class__.__name__, self._type, self.ttl, @@ -574,7 +582,7 @@ class _DynamicMixin(object): reasons.append('rule {} missing pool'.format(rule_num)) continue - if not isinstance(pool, basestring): + if not isinstance(pool, string_types): reasons.append('rule {} invalid pool "{}"' .format(rule_num, pool)) elif pool not in pools: @@ -671,13 +679,13 @@ class _IpList(object): return ['missing value(s)'] reasons = [] for value in data: - if value is '': + if value == '': reasons.append('empty value') elif value is None: reasons.append('missing value(s)') else: try: - cls._address_type(unicode(value)) + cls._address_type(text_type(value)) except Exception: reasons.append('invalid {} address "{}"' .format(cls._address_name, value)) diff --git a/octodns/zone.py b/octodns/zone.py index 916f81b..1191b6f 100644 --- a/octodns/zone.py +++ b/octodns/zone.py @@ -9,6 +9,8 @@ from collections import defaultdict from logging import getLogger import re +from six import text_type + from .record import Create, Delete @@ -38,7 +40,7 @@ class Zone(object): raise Exception('Invalid zone name {}, missing ending dot' .format(name)) # Force everything to lowercase just to be safe - self.name = unicode(name).lower() if name else name + self.name = text_type(name).lower() if name else name self.sub_zones = sub_zones # We're grouping by node, it allows us to efficiently search for # duplicates and detect when CNAMEs co-exist with other records diff --git a/requirements.txt b/requirements.txt index d100c96..63adffa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ botocore==1.10.5 dnspython==1.15.0 docutils==0.14 dyn==1.8.1 -futures==3.2.0 +futures==3.2.0; python_version < '3.0' google-cloud-core==0.28.1 google-cloud-dns==0.29.0 incf.countryutils==1.0 @@ -19,5 +19,5 @@ ovh==0.4.8 python-dateutil==2.6.1 requests==2.20.0 s3transfer==0.1.13 -six==1.11.0 +six==1.12.0 setuptools==38.5.2 diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index e28850a..b0e2f8e 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -8,6 +8,8 @@ from __future__ import absolute_import, division, print_function, \ from logging import getLogger from unittest import TestCase +from six import text_type + from octodns.record import Create, Delete, Record, Update from octodns.provider.base import BaseProvider from octodns.provider.plan import Plan, UnsafePlan @@ -193,7 +195,7 @@ class TestBaseProvider(TestCase): }) for i in range(int(Plan.MIN_EXISTING_RECORDS)): - zone.add_record(Record.new(zone, unicode(i), { + zone.add_record(Record.new(zone, text_type(i), { 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' @@ -225,7 +227,7 @@ class TestBaseProvider(TestCase): }) for i in range(int(Plan.MIN_EXISTING_RECORDS)): - zone.add_record(Record.new(zone, unicode(i), { + zone.add_record(Record.new(zone, text_type(i), { 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' @@ -251,7 +253,7 @@ class TestBaseProvider(TestCase): }) for i in range(int(Plan.MIN_EXISTING_RECORDS)): - zone.add_record(Record.new(zone, unicode(i), { + zone.add_record(Record.new(zone, text_type(i), { 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' @@ -273,7 +275,7 @@ class TestBaseProvider(TestCase): }) for i in range(int(Plan.MIN_EXISTING_RECORDS)): - zone.add_record(Record.new(zone, unicode(i), { + zone.add_record(Record.new(zone, text_type(i), { 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' @@ -299,7 +301,7 @@ class TestBaseProvider(TestCase): }) for i in range(int(Plan.MIN_EXISTING_RECORDS)): - zone.add_record(Record.new(zone, unicode(i), { + zone.add_record(Record.new(zone, text_type(i), { 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' @@ -322,7 +324,7 @@ class TestBaseProvider(TestCase): }) for i in range(int(Plan.MIN_EXISTING_RECORDS)): - zone.add_record(Record.new(zone, unicode(i), { + zone.add_record(Record.new(zone, text_type(i), { 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' @@ -350,7 +352,7 @@ class TestBaseProvider(TestCase): }) for i in range(int(Plan.MIN_EXISTING_RECORDS)): - zone.add_record(Record.new(zone, unicode(i), { + zone.add_record(Record.new(zone, text_type(i), { 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 25f2b58..9e25ea9 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -950,7 +950,7 @@ class TestCloudflareProvider(TestCase): 'value': 'ns1.unit.tests.' }) - data = provider._gen_data(record).next() + data = next(provider._gen_data(record)) self.assertFalse('proxied' in data) @@ -965,7 +965,7 @@ class TestCloudflareProvider(TestCase): }), False ) - data = provider._gen_data(record).next() + data = next(provider._gen_data(record)) self.assertFalse(data['proxied']) @@ -980,7 +980,7 @@ class TestCloudflareProvider(TestCase): }), True ) - data = provider._gen_data(record).next() + data = next(provider._gen_data(record)) self.assertTrue(data['proxied']) diff --git a/tests/test_octodns_provider_googlecloud.py b/tests/test_octodns_provider_googlecloud.py index 3a3e600..d7f0e0c 100644 --- a/tests/test_octodns_provider_googlecloud.py +++ b/tests/test_octodns_provider_googlecloud.py @@ -194,7 +194,7 @@ class DummyIterator: return self def next(self): - return self.iterable.next() + return next(self.iterable) class TestGoogleCloudProvider(TestCase): From c8b261a409c35324e138e26f4533141263848b2f Mon Sep 17 00:00:00 2001 From: cclauss Date: Sat, 13 Jul 2019 23:49:09 +0200 Subject: [PATCH 072/176] Unroll the list comprehensions --- octodns/manager.py | 18 ++++++++++++------ octodns/provider/route53.py | 1 + octodns/record/__init__.py | 1 + 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index 89db845..3984a5a 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -275,10 +275,13 @@ class Manager(object): self.log.info('sync: sources=%s -> targets=%s', sources, targets) try: - sources = [self.providers[source] for source in sources] - except KeyError as e: + collected = [] + for source in sources: + collected.append(self.providers[source]) + sources = collected + except KeyError: raise Exception('Zone {}, unknown source: {}'.format(zone_name, - e)) + source)) try: trgs = [] @@ -396,10 +399,13 @@ class Manager(object): raise Exception('Zone {} is missing sources'.format(zone_name)) try: - sources = [self.providers[source] for source in sources] - except KeyError as e: + collected = [] + for source in sources: + collected.append(self.providers[source]) + sources = collected + except KeyError: raise Exception('Zone {}, unknown source: {}'.format(zone_name, - e)) + source)) for source in sources: if isinstance(source, YamlProvider): diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 7154096..d72b384 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -20,6 +20,7 @@ from ..record import Record, Update from ..record.geo import GeoCodes from .base import BaseProvider +# TODO: remove when Python 2.x is no longer supported try: cmp except NameError: diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 2efdf0e..82b1400 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -13,6 +13,7 @@ from six import string_types, text_type from .geo import GeoCodes +# TODO: remove when Python 2.x is no longer supported try: cmp except NameError: From 9149d358f4c30399de5fffe09684fb0d3ba742ee Mon Sep 17 00:00:00 2001 From: cclauss Date: Mon, 15 Jul 2019 05:36:02 +0200 Subject: [PATCH 073/176] pragma: no cover --- octodns/provider/route53.py | 4 ++-- octodns/record/__init__.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index d72b384..dd07972 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -21,9 +21,9 @@ from ..record.geo import GeoCodes from .base import BaseProvider # TODO: remove when Python 2.x is no longer supported -try: +try: # pragma: no cover cmp -except NameError: +except NameError: # pragma: no cover def cmp(x, y): return (x > y) - (x < y) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 82b1400..8560f5e 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -14,9 +14,9 @@ from six import string_types, text_type from .geo import GeoCodes # TODO: remove when Python 2.x is no longer supported -try: +try: # pragma: no cover cmp -except NameError: +except NameError: # pragma: no cover def cmp(x, y): return (x > y) - (x < y) From 003e8651cefbad2ba90594d35a86b350d8974bf9 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 16 Jul 2019 06:14:45 -0700 Subject: [PATCH 074/176] Drop dynamic record value weight to 0-16 That's all Dyn supports and it's cleaner to match it than to scale dyn since we'd lose precision we can't get back during populate. --- octodns/record/__init__.py | 2 +- tests/config/dynamic.tests.yaml | 6 +++--- tests/config/split/dynamic.tests./a.yaml | 2 +- tests/config/split/dynamic.tests./cname.yaml | 4 ++-- tests/test_octodns_record.py | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index dca6100..83632bc 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -514,7 +514,7 @@ class _DynamicMixin(object): try: weight = value['weight'] weight = int(weight) - if weight < 1 or weight > 255: + if weight < 1 or weight > 15: reasons.append('invalid weight "{}" in pool "{}" ' 'value {}'.format(weight, _id, value_num)) diff --git a/tests/config/dynamic.tests.yaml b/tests/config/dynamic.tests.yaml index 3d806f9..f826880 100644 --- a/tests/config/dynamic.tests.yaml +++ b/tests/config/dynamic.tests.yaml @@ -19,7 +19,7 @@ a: - value: 6.6.6.6 weight: 10 - value: 5.5.5.5 - weight: 25 + weight: 15 rules: - geos: - EU-GB @@ -90,9 +90,9 @@ cname: sea: values: - value: target-sea-1.unit.tests. - weight: 100 + weight: 10 - value: target-sea-2.unit.tests. - weight: 175 + weight: 14 rules: - geos: - EU-GB diff --git a/tests/config/split/dynamic.tests./a.yaml b/tests/config/split/dynamic.tests./a.yaml index fd748b4..f182df6 100644 --- a/tests/config/split/dynamic.tests./a.yaml +++ b/tests/config/split/dynamic.tests./a.yaml @@ -23,7 +23,7 @@ a: fallback: null values: - value: 5.5.5.5 - weight: 25 + weight: 15 - value: 6.6.6.6 weight: 10 rules: diff --git a/tests/config/split/dynamic.tests./cname.yaml b/tests/config/split/dynamic.tests./cname.yaml index a84c202..ff85955 100644 --- a/tests/config/split/dynamic.tests./cname.yaml +++ b/tests/config/split/dynamic.tests./cname.yaml @@ -21,9 +21,9 @@ cname: fallback: null values: - value: target-sea-1.unit.tests. - weight: 100 + weight: 10 - value: target-sea-2.unit.tests. - weight: 175 + weight: 14 rules: - geos: - EU-GB diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 53bc5e7..2b11364 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -2460,7 +2460,7 @@ class TestDynamicRecords(TestCase): 'weight': 1, 'value': '6.6.6.6', }, { - 'weight': 256, + 'weight': 16, 'value': '7.7.7.7', }], }, @@ -2484,7 +2484,7 @@ class TestDynamicRecords(TestCase): } with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, 'bad', a_data) - self.assertEquals(['invalid weight "256" in pool "three" value 2'], + self.assertEquals(['invalid weight "16" in pool "three" value 2'], ctx.exception.reasons) # invalid non-int weight From 0040a51f112b666cd6b0199946db852343086b63 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 16 Jul 2019 06:24:43 -0700 Subject: [PATCH 075/176] v0.9.6 version bump and CHANGELOG updates --- CHANGELOG.md | 9 +++++++++ octodns/__init__.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index adb1f8c..6e2b243 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +## v0.9.6 - 2019-07-16 - The little one that fixes stuff from the big one + +* Reduced dynamic record value weight range to 0-15 so that Dyn and Route53 + match up behaviors. Dyn is limited to 0-15 and scaling that up would lose + resolution that couldn't be recovered during populate. +* Addressed issues with Route53 change set ordering for dynamic records +* Ignore unsupported record types in DigitalOceanProvider +* Fix bugs in Route53 extra changes handling and health check managagement + ## v0.9.5 - 2019-05-06 - The big one, with all the dynamic stuff * dynamic record support, essentially a v2 version of geo records with a lot diff --git a/octodns/__init__.py b/octodns/__init__.py index 939c293..6422577 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -3,4 +3,4 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -__VERSION__ = '0.9.5' +__VERSION__ = '0.9.6' From 4ae3807627f08de53746c1f01045518632324f5b Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 16 Jul 2019 07:03:50 -0700 Subject: [PATCH 076/176] Render README on pypi as markdown, update twine, fix a couple README bits --- README.md | 4 ++-- requirements-dev.txt | 2 +- setup.py | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a3f3eae..163c723 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ - + ## DNS as code - Tools for managing DNS across multiple providers @@ -275,4 +275,4 @@ GitHub® and its stylized versions and the Invertocat mark are GitHub's Trademar ## Authors -OctoDNS was designed and authored by [Ross McFarland](https://github.com/ross) and [Joe Williams](https://github.com/joewilliams). It is now maintained, reviewed, and tested by Ross, Joe, and the rest of the Site Reliability Engineering team at GitHub. +OctoDNS was designed and authored by [Ross McFarland](https://github.com/ross) and [Joe Williams](https://github.com/joewilliams). It is now maintained, reviewed, and tested by Traffic Engineering team at GitHub. diff --git a/requirements-dev.txt b/requirements-dev.txt index 1afee06..77dd50c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,4 +6,4 @@ pycountry>=18.12.8 pycountry_convert>=0.7.2 pyflakes==1.6.0 requests_mock -twine==1.11.0 +twine==1.13.0 diff --git a/setup.py b/setup.py index 7a9348e..75a39d7 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ setup( ], license='MIT', long_description=open('README.md').read(), + long_description_content_type='text/markdown', name='octodns', packages=find_packages(), url='https://github.com/github/octodns', From 5729358e3703fc8231b6ac819184967578d7a193 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Tue, 16 Jul 2019 19:27:24 +0000 Subject: [PATCH 077/176] removed redundant recordsets_get fxn from AkamaiClient --- octodns/provider/fastdns.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 7ad5988..90323b2 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -124,14 +124,6 @@ class AkamaiClient(object): return result - def recordsets_get(self, zone_name): - - resp = self.zone_recordset_get(zone_name, showAll="true") - recordset = resp.json().get("recordsets") - - return recordset - - class AkamaiProvider(BaseProvider): ''' @@ -207,8 +199,9 @@ class AkamaiProvider(BaseProvider): if zone.name not in self._zone_records: try: name = zone.name[:-1] - self._zone_records[zone.name] = \ - self._dns_client.recordsets_get(name) + resp = self._dns_client.zone_recordset_get(name, showAll="true") + resp = resp.json().get("recordsets") + self._zone_records[zone.name] = resp except AkamaiClientException: return [] From 3e3bfe5cf5d691bed7023f100a366acbba7eaf42 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Wed, 17 Jul 2019 23:34:58 +0000 Subject: [PATCH 078/176] fastdns provider fixes: - fixed exception handling in AkamaiClientException - fixed zone_records and zone_recordset fxns in AkamaiProvider and AkamaiClient - refactored _build_zone_config in AkamaiProvider --- octodns/provider/fastdns.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 90323b2..3dd52e7 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -19,7 +19,6 @@ class AkamaiClientException(Exception): _errorMessages = { 400: "400: Bad request", - 401: "401: Unauthorized", 403: "403: Access is forbidden", 404: "404: Resource not found", 405: "405: Method not supported", @@ -32,7 +31,7 @@ class AkamaiClientException(Exception): def __init__(self, resp): try: - message = self._errorMessages.get(resp.status_code) + message = self._errorMessages[resp.status_code] super(AkamaiClientException, self).__init__(message) except: @@ -107,7 +106,7 @@ class AkamaiClient(object): return result - def zone_recordset_get(self, zone, page=None, pageSize=30, search=None, + def zone_recordset_get(self, zone, page=None, pageSize=None, search=None, showAll="true", sortBy="name", types=None): params = { @@ -124,6 +123,7 @@ class AkamaiClient(object): return result + class AkamaiProvider(BaseProvider): ''' @@ -162,7 +162,7 @@ class AkamaiProvider(BaseProvider): https://control.akamai.com/ Configure > Organization > Manage APIs > New API Client for me Select appropriate group, and fill relevant fields. - For API Service Name, select DNS—Zone Record Management + For API Service Name, select DNS-Zone Record Management and then set appropriate Access level (Read-Write to make changes). Then select the "New Credential" button to generate values for above @@ -199,9 +199,11 @@ class AkamaiProvider(BaseProvider): if zone.name not in self._zone_records: try: name = zone.name[:-1] - resp = self._dns_client.zone_recordset_get(name, showAll="true") - resp = resp.json().get("recordsets") - self._zone_records[zone.name] = resp + response = self._dns_client.zone_recordset_get(name) + self._zone_records[zone.name] = response.json()["recordsets"] + + except KeyError: + return [] except AkamaiClientException: return [] @@ -501,10 +503,8 @@ class AkamaiProvider(BaseProvider): return rdata - def _build_zone_config(self, zone, _type=None, comment=None, masters=[]): - - if _type is None: - _type = "primary" + def _build_zone_config(self, zone, _type="primary", comment=None, + masters=[]): if self._contractId is None: raise NameError("contractId not specified to create zone") From 41c59861440400524307aa78e5ec3e684b26d34c Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Wed, 17 Jul 2019 23:43:56 +0000 Subject: [PATCH 079/176] Fastdns provider: - added test cases for testin populate() - added mock request responses in tests/fixtures/ - updated requirements.txt to require edgegrid-python --- requirements.txt | 1 + tests/fixtures/fastdns-invalid-content.json | 35 +++++ tests/fixtures/fastdns-records.json | 157 ++++++++++++++++++++ tests/test_octodns_provider_fastdns.py | 58 +++++--- 4 files changed, 231 insertions(+), 20 deletions(-) create mode 100644 tests/fixtures/fastdns-invalid-content.json create mode 100644 tests/fixtures/fastdns-records.json diff --git a/requirements.txt b/requirements.txt index d100c96..75dc1df 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ botocore==1.10.5 dnspython==1.15.0 docutils==0.14 dyn==1.8.1 +edgegrid-python==1.1.1 futures==3.2.0 google-cloud-core==0.28.1 google-cloud-dns==0.29.0 diff --git a/tests/fixtures/fastdns-invalid-content.json b/tests/fixtures/fastdns-invalid-content.json new file mode 100644 index 0000000..8932f66 --- /dev/null +++ b/tests/fixtures/fastdns-invalid-content.json @@ -0,0 +1,35 @@ +{ + "recordsets": [ + { + "rdata": [ + "", + "12 20 foo-2.unit.tests." + ], + "type": "SRV", + "name": "_srv._tcp.unit.tests", + "ttl": 600 + }, + { + "rdata": [ + "", + "1 1" + ], + "type": "SSHFP", + "name": "unit.tests", + "ttl": 3600 + }, + { + "rdata": [ + "", + "100 \"U\" \"SIP+D2U\" \"!^.*$!sip:info@bar.example.com!\" ." + ], + "type": "NAPTR", + "name": "naptr.unit.tests", + "ttl": 600 + } + ], + "metadata": { + "totalElements": 3, + "showAll": true + } +} \ No newline at end of file diff --git a/tests/fixtures/fastdns-records.json b/tests/fixtures/fastdns-records.json new file mode 100644 index 0000000..4693eb1 --- /dev/null +++ b/tests/fixtures/fastdns-records.json @@ -0,0 +1,157 @@ +{ + "recordsets": [ + { + "rdata": [ + "10 20 30 foo-1.unit.tests.", + "12 20 30 foo-2.unit.tests." + ], + "type": "SRV", + "name": "_srv._tcp.unit.tests", + "ttl": 600 + }, + { + "rdata": [ + "2601:644:500:e210:62f8:1dff:feb8:947a" + ], + "type": "AAAA", + "name": "aaaa.unit.tests", + "ttl": 600 + }, + { + "rdata": [ + "ns1.akam.net.", + "ns2.akam.net.", + "ns3.akam.net.", + "ns4.akam.net." + ], + "type": "NS", + "name": "unit.tests", + "ttl": 3600 + }, + { + "rdata": [ + "1.2.3.4", + "1.2.3.5" + ], + "type": "A", + "name": "unit.tests", + "ttl": 300 + }, + { + "rdata": [ + "ns1.akam.net hostmaster.akamai.com 1489074932 86400 7200 604800 300" + ], + "type": "SOA", + "name": "unit.tests", + "ttl": 3600 + }, + { + "rdata": [ + "1 1 7491973e5f8b39d5327cd4e08bc81b05f7710b49", + "1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73" + ], + "type": "SSHFP", + "name": "unit.tests", + "ttl": 3600 + }, + { + "rdata": [ + "unit.tests." + ], + "type": "CNAME", + "name": "cname.unit.tests", + "ttl": 300 + }, + { + "rdata": [ + "unit.tests." + ], + "type": "CNAME", + "name": "excluded.unit.tests", + "ttl": 3600 + }, + { + "rdata": [ + "unit.tests." + ], + "type": "CNAME", + "name": "included.unit.tests", + "ttl": 3600 + }, + { + "rdata": [ + "10 smtp-4.unit.tests.", + "20 smtp-2.unit.tests.", + "30 smtp-3.unit.tests.", + "40 smtp-1.unit.tests." + ], + "type": "MX", + "name": "mx.unit.tests", + "ttl": 300 + }, + { + "rdata": [ + "10 100 \"S\" \"SIP+D2U\" \"!^.*$!sip:info@bar.example.com!\" .", + "100 100 \"U\" \"SIP+D2U\" \"!^.*$!sip:info@bar.example.com!\" ." + ], + "type": "NAPTR", + "name": "naptr.unit.tests", + "ttl": 600 + }, + { + "rdata": [ + "foo.bar.com." + ], + "type": "PTR", + "name": "ptr.unit.tests", + "ttl": 300 + }, + { + "rdata": [ + "\"v=spf1 ip4:192.168.0.1/16-all\"" + ], + "type": "SPF", + "name": "spf.unit.tests", + "ttl": 600 + }, + { + "rdata": [ + "ns1.unit.tests.", + "ns2.unit.tests." + ], + "type": "NS", + "name": "under.unit.tests", + "ttl": 3600 + }, + { + "rdata": [ + "\"Bah bah black sheep\"", + "\"have you any wool.\"", + "\"v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs\"" + ], + "type": "TXT", + "name": "txt.unit.tests", + "ttl": 600 + }, + { + "rdata": [ + "2.2.3.6" + ], + "type": "A", + "name": "www.unit.tests", + "ttl": 300 + }, + { + "rdata": [ + "2.2.3.6" + ], + "type": "A", + "name": "www.sub.unit.tests", + "ttl": 300 + } + ], + "metadata": { + "totalElements": 16, + "showAll": true + } +} \ No newline at end of file diff --git a/tests/test_octodns_provider_fastdns.py b/tests/test_octodns_provider_fastdns.py index baf28ea..8c841fe 100644 --- a/tests/test_octodns_provider_fastdns.py +++ b/tests/test_octodns_provider_fastdns.py @@ -22,19 +22,32 @@ class TestFastdnsProvider(TestCase): source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) + # Our test suite differs a bit, add our NS and remove the simple one + expected.add_record(Record.new(expected, 'under', { + 'ttl': 3600, + 'type': 'NS', + 'values': [ + 'ns1.unit.tests.', + 'ns2.unit.tests.', + ] + })) + for record in list(expected.records): + if record.name == 'sub' and record._type == 'NS': + expected._remove_record(record) + break + def test_populate(self): - provider = AkamaiProvider("test", "client_secret", "host", "access_token", - "client_token") - + provider = AkamaiProvider("test", "secret", "akam.com", "atok", "ctok") + # Bad Auth with requests_mock() as mock: - mock.get(ANY, status_code=401, - text='{"message": "Authentication failed"}') + mock.get(ANY, status_code=401, text='{"message": "Unauthorized"}') with self.assertRaises(Exception) as ctx: zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals("401: Unauthorized", ctx.exception.message) + + self.assertEquals(401, ctx.exception.response.status_code) # general error with requests_mock() as mock: @@ -45,7 +58,7 @@ class TestFastdnsProvider(TestCase): provider.populate(zone) self.assertEquals(502, ctx.exception.response.status_code) - # Non-existant zone doesn't populate anything + # Non-existant zone doesn't populate anything with requests_mock() as mock: mock.get(ANY, status_code=404, text='{"message": "Domain `foo.bar` not found"}') @@ -54,17 +67,22 @@ class TestFastdnsProvider(TestCase): provider.populate(zone) self.assertEquals(set(), zone.records) - # # No diffs == no changes - # with requests_mock() as mock: - # base = 'https://api.dnsimple.com/v2/42/zones/unit.tests/' \ - # 'records?page=' - # with open('tests/fixtures/dnsimple-page-1.json') as fh: - # mock.get('{}{}'.format(base, 1), text=fh.read()) - # with open('tests/fixtures/dnsimple-page-2.json') as fh: - # mock.get('{}{}'.format(base, 2), text=fh.read()) + # No diffs == no changes + with requests_mock() as mock: - # zone = Zone('unit.tests.', []) - # provider.populate(zone) - # self.assertEquals(16, len(zone.records)) - # changes = self.expected.changes(zone, provider) - # self.assertEquals(0, len(changes)) \ No newline at end of file + with open('tests/fixtures/fastdns-records.json') as fh: + mock.get(ANY, text=fh.read()) + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(16, len(zone.records)) + changes = self.expected.changes(zone, provider) + self.assertEquals(0, len(changes)) + + # 2nd populate makes no network calls/all from cache + again = Zone('unit.tests.', []) + provider.populate(again) + self.assertEquals(16, len(again.records)) + + # bust the cache + del provider._zone_records[zone.name] \ No newline at end of file From 852d607715078c682c1c1bb74a365a430279acc7 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 18 Jul 2019 01:07:42 +0000 Subject: [PATCH 080/176] fastdns: - added test cases for test_apply, testing for basic create, update, delete records - added previous state json file in test/fixtures to do the above --- tests/fixtures/fastdns-records-prev.json | 166 +++++++++++++++++++++++ tests/test_octodns_provider_fastdns.py | 22 ++- 2 files changed, 185 insertions(+), 3 deletions(-) create mode 100644 tests/fixtures/fastdns-records-prev.json diff --git a/tests/fixtures/fastdns-records-prev.json b/tests/fixtures/fastdns-records-prev.json new file mode 100644 index 0000000..04ce274 --- /dev/null +++ b/tests/fixtures/fastdns-records-prev.json @@ -0,0 +1,166 @@ +{ + "recordsets": [ + { + "rdata": [ + "10 20 30 foo-1.unit.tests.", + "12 20 30 foo-2.unit.tests." + ], + "type": "SRV", + "name": "_srv._tcp.old.unit.tests", + "ttl": 600 + }, + { + "rdata": [ + "10 20 30 foo-1.unit.tests.", + "12 20 30 foo-2.unit.tests." + ], + "type": "SRV", + "name": "_srv._tcp.old.unit.tests", + "ttl": 600 + }, + { + "rdata": [ + "2601:644:500:e210:62f8:1dff:feb8:9471" + ], + "type": "AAAA", + "name": "aaaa.old.unit.tests", + "ttl": 600 + }, + { + "rdata": [ + "ns1.akam.net.", + "ns2.akam.net.", + "ns3.akam.net.", + "ns4.akam.net." + ], + "type": "NS", + "name": "old.unit.tests", + "ttl": 3600 + }, + { + "rdata": [ + "1.2.3.4", + "1.2.3.5" + ], + "type": "A", + "name": "old.unit.tests", + "ttl": 300 + }, + { + "rdata": [ + "ns1.akam.net hostmaster.akamai.com 1489074932 86400 7200 604800 300" + ], + "type": "SOA", + "name": "unit.tests", + "ttl": 3600 + }, + { + "rdata": [ + "1 1 7491973e5f8b39d5327cd4e08bc81b05f7710b49", + "1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73" + ], + "type": "SSHFP", + "name": "old.unit.tests", + "ttl": 3600 + }, + { + "rdata": [ + "unit.tests." + ], + "type": "CNAME", + "name": "old.cname.unit.tests", + "ttl": 300 + }, + { + "rdata": [ + "unit.tests." + ], + "type": "CNAME", + "name": "excluded.old.unit.tests", + "ttl": 3600 + }, + { + "rdata": [ + "unit.tests." + ], + "type": "CNAME", + "name": "included.old.unit.tests", + "ttl": 3600 + }, + { + "rdata": [ + "10 smtp-4.unit.tests.", + "20 smtp-2.unit.tests.", + "30 smtp-3.unit.tests.", + "40 smtp-1.unit.tests." + ], + "type": "MX", + "name": "mx.old.unit.tests", + "ttl": 300 + }, + { + "rdata": [ + "10 100 \"S\" \"SIP+D2U\" \"!^.*$!sip:info@bar.example.com!\" .", + "100 100 \"U\" \"SIP+D2U\" \"!^.*$!sip:info@bar.example.com!\" ." + ], + "type": "NAPTR", + "name": "naptr.old.unit.tests", + "ttl": 600 + }, + { + "rdata": [ + "foo.bar.com." + ], + "type": "PTR", + "name": "ptr.old.unit.tests", + "ttl": 300 + }, + { + "rdata": [ + "\"v=spf1 ip4:192.168.0.1/16-all\"" + ], + "type": "SPF", + "name": "spf.old.unit.tests", + "ttl": 600 + }, + { + "rdata": [ + "ns1.unit.tests.", + "ns2.unit.tests." + ], + "type": "NS", + "name": "under.old.unit.tests", + "ttl": 3600 + }, + { + "rdata": [ + "\"Bah bah black sheep\"", + "\"have you any wool.\"", + "\"v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs\"" + ], + "type": "TXT", + "name": "txt.old.unit.tests", + "ttl": 600 + }, + { + "rdata": [ + "2.2.3.7" + ], + "type": "A", + "name": "www.unit.tests", + "ttl": 300 + }, + { + "rdata": [ + "2.2.3.6" + ], + "type": "A", + "name": "www.sub.old.unit.tests", + "ttl": 300 + } + ], + "metadata": { + "totalElements": 16, + "showAll": true + } +} \ No newline at end of file diff --git a/tests/test_octodns_provider_fastdns.py b/tests/test_octodns_provider_fastdns.py index 8c841fe..4df36c0 100644 --- a/tests/test_octodns_provider_fastdns.py +++ b/tests/test_octodns_provider_fastdns.py @@ -5,14 +5,14 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from mock import Mock, call +# from mock import Mock, call from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock from unittest import TestCase from octodns.record import Record -from octodns.provider.fastdns import AkamaiProvider, AkamaiClientException +from octodns.provider.fastdns import AkamaiProvider from octodns.provider.yaml import YamlProvider from octodns.zone import Zone @@ -85,4 +85,20 @@ class TestFastdnsProvider(TestCase): self.assertEquals(16, len(again.records)) # bust the cache - del provider._zone_records[zone.name] \ No newline at end of file + del provider._zone_records[zone.name] + + def test_apply(self): + provider = AkamaiProvider("test", "secret", "akam.com", "atok", "ctok") + + with requests_mock() as mock: + + with open('tests/fixtures/fastdns-records-prev.json') as fh: + mock.get(ANY, text=fh.read()) + + plan = provider.plan(self.expected) + mock.post(ANY, status_code=201) + mock.put(ANY, status_code=200) + mock.delete(ANY, status_code=204) + + changes = provider.apply(plan) + self.assertEquals(29, changes) From 1596add89f54c32af3e8c005f53507b3d0f39489 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 18 Jul 2019 20:46:05 +0000 Subject: [PATCH 081/176] fastdns provider: - added unit tests for apply functions - added another json config file for this - 100% code coverage reached --- .../fixtures/fastdns-records-prev-other.json | 166 ++++++++++++++++++ tests/test_octodns_provider_fastdns.py | 48 ++++- 2 files changed, 213 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/fastdns-records-prev-other.json diff --git a/tests/fixtures/fastdns-records-prev-other.json b/tests/fixtures/fastdns-records-prev-other.json new file mode 100644 index 0000000..acae3ec --- /dev/null +++ b/tests/fixtures/fastdns-records-prev-other.json @@ -0,0 +1,166 @@ +{ + "recordsets": [ + { + "rdata": [ + "10 20 30 foo-1.other.tests.", + "12 20 30 foo-2.other.tests." + ], + "type": "SRV", + "name": "_srv._tcp.old.other.tests", + "ttl": 600 + }, + { + "rdata": [ + "10 20 30 foo-1.other.tests.", + "12 20 30 foo-2.other.tests." + ], + "type": "SRV", + "name": "_srv._tcp.old.other.tests", + "ttl": 600 + }, + { + "rdata": [ + "2601:644:500:e210:62f8:1dff:feb8:9471" + ], + "type": "AAAA", + "name": "aaaa.old.other.tests", + "ttl": 600 + }, + { + "rdata": [ + "ns1.akam.net.", + "ns2.akam.net.", + "ns3.akam.net.", + "ns4.akam.net." + ], + "type": "NS", + "name": "old.other.tests", + "ttl": 3600 + }, + { + "rdata": [ + "1.2.3.4", + "1.2.3.5" + ], + "type": "A", + "name": "old.other.tests", + "ttl": 300 + }, + { + "rdata": [ + "ns1.akam.net hostmaster.akamai.com 1489074932 86400 7200 604800 300" + ], + "type": "SOA", + "name": "other.tests", + "ttl": 3600 + }, + { + "rdata": [ + "1 1 7491973e5f8b39d5327cd4e08bc81b05f7710b49", + "1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73" + ], + "type": "SSHFP", + "name": "old.other.tests", + "ttl": 3600 + }, + { + "rdata": [ + "other.tests." + ], + "type": "CNAME", + "name": "old.cname.other.tests", + "ttl": 300 + }, + { + "rdata": [ + "other.tests." + ], + "type": "CNAME", + "name": "excluded.old.other.tests", + "ttl": 3600 + }, + { + "rdata": [ + "other.tests." + ], + "type": "CNAME", + "name": "included.old.other.tests", + "ttl": 3600 + }, + { + "rdata": [ + "10 smtp-4.other.tests.", + "20 smtp-2.other.tests.", + "30 smtp-3.other.tests.", + "40 smtp-1.other.tests." + ], + "type": "MX", + "name": "mx.old.other.tests", + "ttl": 300 + }, + { + "rdata": [ + "10 100 \"S\" \"SIP+D2U\" \"!^.*$!sip:info@bar.example.com!\" .", + "100 100 \"U\" \"SIP+D2U\" \"!^.*$!sip:info@bar.example.com!\" ." + ], + "type": "NAPTR", + "name": "naptr.old.other.tests", + "ttl": 600 + }, + { + "rdata": [ + "foo.bar.com." + ], + "type": "PTR", + "name": "ptr.old.other.tests", + "ttl": 300 + }, + { + "rdata": [ + "\"v=spf1 ip4:192.168.0.1/16-all\"" + ], + "type": "SPF", + "name": "spf.old.other.tests", + "ttl": 600 + }, + { + "rdata": [ + "ns1.other.tests.", + "ns2.other.tests." + ], + "type": "NS", + "name": "under.old.other.tests", + "ttl": 3600 + }, + { + "rdata": [ + "\"Bah bah black sheep\"", + "\"have you any wool.\"", + "\"v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs\"" + ], + "type": "TXT", + "name": "txt.old.other.tests", + "ttl": 600 + }, + { + "rdata": [ + "2.2.3.7" + ], + "type": "A", + "name": "www.other.tests", + "ttl": 300 + }, + { + "rdata": [ + "2.2.3.6" + ], + "type": "A", + "name": "www.sub.old.other.tests", + "ttl": 300 + } + ], + "metadata": { + "totalElements": 16, + "showAll": true + } +} \ No newline at end of file diff --git a/tests/test_octodns_provider_fastdns.py b/tests/test_octodns_provider_fastdns.py index 4df36c0..5f503c7 100644 --- a/tests/test_octodns_provider_fastdns.py +++ b/tests/test_octodns_provider_fastdns.py @@ -88,8 +88,10 @@ class TestFastdnsProvider(TestCase): del provider._zone_records[zone.name] def test_apply(self): - provider = AkamaiProvider("test", "secret", "akam.com", "atok", "ctok") + provider = AkamaiProvider("test", "s", "akam.com", "atok", "ctok", + "cid", "gid") + # tests create update delete through previous state config json with requests_mock() as mock: with open('tests/fixtures/fastdns-records-prev.json') as fh: @@ -102,3 +104,47 @@ class TestFastdnsProvider(TestCase): changes = provider.apply(plan) self.assertEquals(29, changes) + + # Test against a zone that doesn't exist yet + with requests_mock() as mock: + with open('tests/fixtures/fastdns-records-prev-other.json') as fh: + mock.get(ANY, status_code=404) + + plan = provider.plan(self.expected) + mock.post(ANY, status_code=201) + mock.put(ANY, status_code=200) + mock.delete(ANY, status_code=204) + + changes = provider.apply(plan) + self.assertEquals(14, changes) + + # Test against a zone that doesn't exist yet, but gid not provided + with requests_mock() as mock: + with open('tests/fixtures/fastdns-records-prev-other.json') as fh: + mock.get(ANY, status_code=404) + provider = AkamaiProvider("test", "s", "akam.com", "atok", "ctok", + "cid") + plan = provider.plan(self.expected) + mock.post(ANY, status_code=201) + mock.put(ANY, status_code=200) + mock.delete(ANY, status_code=204) + + changes = provider.apply(plan) + self.assertEquals(14, changes) + + # Test against a zone that doesn't exist, but cid not provided + + with requests_mock() as mock: + mock.get(ANY, status_code=404) + + provider = AkamaiProvider("test", "s", "akam.com", "atok", "ctok") + plan = provider.plan(self.expected) + mock.post(ANY, status_code=201) + mock.put(ANY, status_code=200) + mock.delete(ANY, status_code=204) + + try: + changes = provider.apply(plan) + except NameError as e: + expected = "contractId not specified to create zone" + self.assertEquals(e.message, expected) From e3f6c4c711b314bc5ea0616327f4d3c4629bf758 Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 18 Jul 2019 21:07:14 +0000 Subject: [PATCH 082/176] fastdns: changed sshfp to convert text to lowercase, since akamai changes to caps --- octodns/provider/fastdns.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 3dd52e7..e4a3c66 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -202,10 +202,7 @@ class AkamaiProvider(BaseProvider): response = self._dns_client.zone_recordset_get(name) self._zone_records[zone.name] = response.json()["recordsets"] - except KeyError: - return [] - - except AkamaiClientException: + except (AkamaiClientException, KeyError): return [] return self._zone_records[zone.name] @@ -405,7 +402,7 @@ class AkamaiProvider(BaseProvider): algorithm, fp_type, fingerprint = r.split(' ', 2) values.append({ 'algorithm': algorithm, - 'fingerprint': fingerprint, + 'fingerprint': fingerprint.lower(), 'fingerprint_type': fp_type }) From 5a741eb6007fed26d79d14288f7b95294d850a1b Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Thu, 18 Jul 2019 21:16:34 +0000 Subject: [PATCH 083/176] fastdns: fixed typo in test fixture --- tests/fixtures/fastdns-records-prev.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fixtures/fastdns-records-prev.json b/tests/fixtures/fastdns-records-prev.json index 04ce274..b07c63f 100644 --- a/tests/fixtures/fastdns-records-prev.json +++ b/tests/fixtures/fastdns-records-prev.json @@ -65,7 +65,7 @@ }, { "rdata": [ - "unit.tests." + "unit.tests" ], "type": "CNAME", "name": "old.cname.unit.tests", From 79dd35f7a74d50c854e460f8dd11a3222f7fb384 Mon Sep 17 00:00:00 2001 From: Basir Date: Thu, 18 Jul 2019 17:56:18 -0400 Subject: [PATCH 084/176] fastdns: fixed an issue with handling spaces in SPF records --- octodns/provider/fastdns.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index e4a3c66..f6d3bd3 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -433,7 +433,6 @@ class AkamaiProvider(BaseProvider): _params_for_A = _params_for_multiple _params_for_AAAA = _params_for_multiple _params_for_NS = _params_for_multiple - _params_for_SPF = _params_for_multiple _params_for_CNAME = _params_for_single _params_for_PTR = _params_for_single @@ -466,6 +465,15 @@ class AkamaiProvider(BaseProvider): return rdata + def _params_for_SPF(self, values): + rdata = [] + + for r in values: + txt = "\"" + r.replace('\\;', ';') + "\"" + rdata.append(txt) + + return rdata + def _params_for_SRV(self, values): rdata = [] for r in values: From 6f39ecf261eba24b71dd07e144958361ed855af7 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 19 Jul 2019 08:10:14 -0700 Subject: [PATCH 085/176] Add MythicBeastsProvider to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 163c723..aa9950e 100644 --- a/README.md +++ b/README.md @@ -179,6 +179,7 @@ The above command pulled the existing data out of Route53 and placed the results | [DynProvider](/octodns/provider/dyn.py) | dyn | All | Both | | | [EtcHostsProvider](/octodns/provider/etc_hosts.py) | | A, AAAA, ALIAS, CNAME | No | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | +| [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | | | [Ns1Provider](/octodns/provider/ns1.py) | nsone | All | Partial Geo | No health checking for GeoDNS | | [OVH](/octodns/provider/ovh.py) | ovh | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT, DKIM | No | | | [PowerDnsProvider](/octodns/provider/powerdns.py) | | All | No | | From 76e799873e51fa78138a85efd194ec35594ce3ef Mon Sep 17 00:00:00 2001 From: basirjamil Date: Fri, 19 Jul 2019 14:32:51 -0400 Subject: [PATCH 086/176] Update octodns/provider/fastdns.py Co-Authored-By: Ross McFarland --- octodns/provider/fastdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index f6d3bd3..0bc0aa2 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -5,7 +5,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -import requests +from requests import Session from akamai.edgegrid import EdgeGridAuth from urlparse import urljoin from collections import defaultdict From d53eb7ad8e2400fe0266802617cf82ee76391305 Mon Sep 17 00:00:00 2001 From: basirjamil Date: Fri, 19 Jul 2019 14:32:58 -0400 Subject: [PATCH 087/176] Update octodns/provider/fastdns.py Co-Authored-By: Ross McFarland --- octodns/provider/fastdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 0bc0aa2..45d6aa1 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -10,7 +10,7 @@ from akamai.edgegrid import EdgeGridAuth from urlparse import urljoin from collections import defaultdict -import logging +from logging import getLogger from ..record import Record from .base import BaseProvider From 91e7ac6d36c22f5570896296f1fac9632c78fa1d Mon Sep 17 00:00:00 2001 From: basirjamil Date: Fri, 19 Jul 2019 14:33:23 -0400 Subject: [PATCH 088/176] Update octodns/provider/fastdns.py Co-Authored-By: Ross McFarland --- octodns/provider/fastdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 45d6aa1..d0d2009 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -181,7 +181,7 @@ class AkamaiProvider(BaseProvider): def __init__(self, id, client_secret, host, access_token, client_token, contract_id=None, gid=None, *args, **kwargs): - self.log = logging.getLogger('AkamaiProvider[{}]'.format(id)) + self.log = getLogger('AkamaiProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, ') super(AkamaiProvider, self).__init__(id, *args, **kwargs) From 41a5d78728ffc90d90f89d1ad966c4a84c3fbebe Mon Sep 17 00:00:00 2001 From: Basir Jamil Date: Fri, 19 Jul 2019 20:53:34 +0000 Subject: [PATCH 089/176] fastdns: refactored how exceptions are handled --- octodns/provider/fastdns.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index d0d2009..fb199d3 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -18,25 +18,18 @@ from .base import BaseProvider class AkamaiClientException(Exception): _errorMessages = { - 400: "400: Bad request", - 403: "403: Access is forbidden", - 404: "404: Resource not found", - 405: "405: Method not supported", - 406: "406: Not Acceptable", - 409: "409: Request not allowed due to conflict with current state", - 415: "415: Unsupported media type", - 422: "422: Request body contains an error preventing processing", - 500: "500: Internal server error" + 404: "404: Resource not found" } def __init__(self, resp): try: message = self._errorMessages[resp.status_code] - super(AkamaiClientException, self).__init__(message) - except: + except KeyError: resp.raise_for_status() + super(AkamaiClientException, self).__init__(message) + class AkamaiClient(object): ''' @@ -54,7 +47,7 @@ class AkamaiClient(object): self.base = "https://" + _host + "/config-dns/v2/" - sess = requests.Session() + sess = Session() sess.auth = EdgeGridAuth( client_token=_client_token, client_secret=_client_secret, From daa6fecc9b15eb123a3563de89ce5f4660be87f9 Mon Sep 17 00:00:00 2001 From: Kitten King Date: Sat, 20 Jul 2019 06:31:05 +0000 Subject: [PATCH 090/176] Fix Typos --- CHANGELOG.md | 2 +- octodns/provider/route53.py | 14 +++++++------- tests/test_octodns_provider_cloudflare.py | 6 +++--- tests/test_octodns_provider_digitalocean.py | 4 ++-- tests/test_octodns_provider_dnsimple.py | 4 ++-- tests/test_octodns_provider_dnsmadeeasy.py | 4 ++-- tests/test_octodns_provider_ns1.py | 4 ++-- tests/test_octodns_provider_powerdns.py | 8 ++++---- tests/test_octodns_record.py | 6 +++--- 9 files changed, 26 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e2b243..4f283ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -112,7 +112,7 @@ Adds an OVH provider. ## v0.8.6 - 2017-09-06 - CAA record type, -Misc fixes and improvments. +Misc fixes and improvements. * Azure TXT record fix * PowerDNS api support for https diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 4cf7c99..7b21573 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -528,13 +528,13 @@ def _mod_keyer(mod): # before all changes, followed by a "CREATE", internally in the AWS API. # Because of this, we order changes as follows: # - Delete any records that we wish to delete that are GEOS - # (because they are never targetted by anything) + # (because they are never targeted by anything) # - Delete any records that we wish to delete that are SECONDARY - # (because they are no longer targetted by GEOS) + # (because they are no longer targeted by GEOS) # - Delete any records that we wish to delete that are PRIMARY - # (because they are no longer targetted by SECONDARY) + # (because they are no longer targeted by SECONDARY) # - Delete any records that we wish to delete that are VALUES - # (because they are no longer targetted by PRIMARY) + # (because they are no longer targeted by PRIMARY) # - CREATE/UPSERT any records that are VALUES # (because they don't depend on other records) # - CREATE/UPSERT any records that are PRIMARY @@ -1028,7 +1028,7 @@ class Route53Provider(BaseProvider): .get('healthcheck', {}) \ .get('measure_latency', True) - def _health_check_equivilent(self, host, path, protocol, port, + def _health_check_equivalent(self, host, path, protocol, port, measure_latency, health_check, value=None): config = health_check['HealthCheckConfig'] @@ -1080,7 +1080,7 @@ class Route53Provider(BaseProvider): if not health_check['CallerReference'].startswith(expected_ref): # not match, ignore continue - if self._health_check_equivilent(healthcheck_host, + if self._health_check_equivalent(healthcheck_host, healthcheck_path, healthcheck_protocol, healthcheck_port, @@ -1237,7 +1237,7 @@ class Route53Provider(BaseProvider): health_check = self.health_checks[health_check_id] caller_ref = health_check['CallerReference'] if caller_ref.startswith(self.HEALTH_CHECK_VERSION): - if self._health_check_equivilent(healthcheck_host, + if self._health_check_equivalent(healthcheck_host, healthcheck_path, healthcheck_protocol, healthcheck_port, diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 25f2b58..5c6d503 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -102,7 +102,7 @@ class TestCloudflareProvider(TestCase): provider.populate(zone) self.assertEquals(502, ctx.exception.response.status_code) - # Non-existant zone doesn't populate anything + # Non-existent zone doesn't populate anything with requests_mock() as mock: mock.get(ANY, status_code=200, json=self.empty) @@ -110,7 +110,7 @@ class TestCloudflareProvider(TestCase): provider.populate(zone) self.assertEquals(set(), zone.records) - # re-populating the same non-existant zone uses cache and makes no + # re-populating the same non-existent zone uses cache and makes no # calls again = Zone('unit.tests.', []) provider.populate(again) @@ -173,7 +173,7 @@ class TestCloudflareProvider(TestCase): }, # zone create ] + [None] * 20 # individual record creates - # non-existant zone, create everything + # non-existent zone, create everything plan = provider.plan(self.expected) self.assertEquals(12, len(plan.changes)) self.assertEquals(12, provider.apply(plan)) diff --git a/tests/test_octodns_provider_digitalocean.py b/tests/test_octodns_provider_digitalocean.py index ddc6bc2..fce62b1 100644 --- a/tests/test_octodns_provider_digitalocean.py +++ b/tests/test_octodns_provider_digitalocean.py @@ -61,7 +61,7 @@ class TestDigitalOceanProvider(TestCase): provider.populate(zone) self.assertEquals(502, ctx.exception.response.status_code) - # Non-existant zone doesn't populate anything + # Non-existent zone doesn't populate anything with requests_mock() as mock: mock.get(ANY, status_code=404, text='{"id":"not_found","message":"The resource you ' @@ -153,7 +153,7 @@ class TestDigitalOceanProvider(TestCase): } } - # non-existant domain, create everything + # non-existent domain, create everything resp.json.side_effect = [ DigitalOceanClientNotFound, # no zone in populate DigitalOceanClientNotFound, # no domain during apply diff --git a/tests/test_octodns_provider_dnsimple.py b/tests/test_octodns_provider_dnsimple.py index 896425e..b458ad7 100644 --- a/tests/test_octodns_provider_dnsimple.py +++ b/tests/test_octodns_provider_dnsimple.py @@ -58,7 +58,7 @@ class TestDnsimpleProvider(TestCase): provider.populate(zone) self.assertEquals(502, ctx.exception.response.status_code) - # Non-existant zone doesn't populate anything + # Non-existent zone doesn't populate anything with requests_mock() as mock: mock.get(ANY, status_code=404, text='{"message": "Domain `foo.bar` not found"}') @@ -122,7 +122,7 @@ class TestDnsimpleProvider(TestCase): resp.json = Mock() provider._client._request = Mock(return_value=resp) - # non-existant domain, create everything + # non-existent domain, create everything resp.json.side_effect = [ DnsimpleClientNotFound, # no zone in populate DnsimpleClientNotFound, # no domain during apply diff --git a/tests/test_octodns_provider_dnsmadeeasy.py b/tests/test_octodns_provider_dnsmadeeasy.py index 04cf0ee..bb663fb 100644 --- a/tests/test_octodns_provider_dnsmadeeasy.py +++ b/tests/test_octodns_provider_dnsmadeeasy.py @@ -87,7 +87,7 @@ class TestDnsMadeEasyProvider(TestCase): provider.populate(zone) self.assertEquals(502, ctx.exception.response.status_code) - # Non-existant zone doesn't populate anything + # Non-existent zone doesn't populate anything with requests_mock() as mock: mock.get(ANY, status_code=404, text='') @@ -130,7 +130,7 @@ class TestDnsMadeEasyProvider(TestCase): with open('tests/fixtures/dnsmadeeasy-domains.json') as fh: domains = json.load(fh) - # non-existant domain, create everything + # non-existent domain, create everything resp.json.side_effect = [ DnsMadeEasyClientNotFound, # no zone in populate DnsMadeEasyClientNotFound, # no domain during apply diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 8530b62..178ce53 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -191,7 +191,7 @@ class TestNs1Provider(TestCase): self.assertEquals(load_mock.side_effect, ctx.exception) self.assertEquals(('unit.tests',), load_mock.call_args[0]) - # Non-existant zone doesn't populate anything + # Non-existent zone doesn't populate anything load_mock.reset_mock() load_mock.side_effect = \ ResourceException('server error: zone not found') @@ -323,7 +323,7 @@ class TestNs1Provider(TestCase): provider.apply(plan) self.assertEquals(create_mock.side_effect, ctx.exception) - # non-existant zone, create + # non-existent zone, create load_mock.reset_mock() create_mock.reset_mock() load_mock.side_effect = \ diff --git a/tests/test_octodns_provider_powerdns.py b/tests/test_octodns_provider_powerdns.py index 067dc74..2ce8519 100644 --- a/tests/test_octodns_provider_powerdns.py +++ b/tests/test_octodns_provider_powerdns.py @@ -41,7 +41,7 @@ with open('./tests/fixtures/powerdns-full-data.json') as fh: class TestPowerDnsProvider(TestCase): def test_provider(self): - provider = PowerDnsProvider('test', 'non.existant', 'api-key', + provider = PowerDnsProvider('test', 'non.existent', 'api-key', nameserver_values=['8.8.8.8.', '9.9.9.9.']) @@ -63,7 +63,7 @@ class TestPowerDnsProvider(TestCase): provider.populate(zone) self.assertEquals(502, ctx.exception.response.status_code) - # Non-existant zone doesn't populate anything + # Non-existent zone doesn't populate anything with requests_mock() as mock: mock.get(ANY, status_code=422, json={'error': "Could not find domain 'unit.tests.'"}) @@ -163,7 +163,7 @@ class TestPowerDnsProvider(TestCase): provider.apply(plan) def test_small_change(self): - provider = PowerDnsProvider('test', 'non.existant', 'api-key') + provider = PowerDnsProvider('test', 'non.existent', 'api-key') expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) @@ -203,7 +203,7 @@ class TestPowerDnsProvider(TestCase): def test_existing_nameservers(self): ns_values = ['8.8.8.8.', '9.9.9.9.'] - provider = PowerDnsProvider('test', 'non.existant', 'api-key', + provider = PowerDnsProvider('test', 'non.existent', 'api-key', nameserver_values=ns_values) expected = Zone('unit.tests.', []) diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 2b11364..d6ed2d9 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -2845,7 +2845,7 @@ class TestDynamicRecords(TestCase): self.assertEquals(['rule 1 invalid pool "[]"'], ctx.exception.reasons) - # rule references non-existant pool + # rule references non-existent pool a_data = { 'dynamic': { 'pools': { @@ -2864,7 +2864,7 @@ class TestDynamicRecords(TestCase): }, 'rules': [{ 'geos': ['NA-US-CA'], - 'pool': 'non-existant', + 'pool': 'non-existent', }, { 'pool': 'one', }], @@ -2878,7 +2878,7 @@ class TestDynamicRecords(TestCase): } with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, 'bad', a_data) - self.assertEquals(["rule 1 undefined pool \"non-existant\""], + self.assertEquals(["rule 1 undefined pool \"non-existent\""], ctx.exception.reasons) # rule with invalid geos From 57c96cf40f7b2996b2d779ffec534d175189ebfc Mon Sep 17 00:00:00 2001 From: Shnayder Anton Date: Mon, 22 Jul 2019 13:18:45 +0300 Subject: [PATCH 091/176] Add Selectel as OctoDNS provider --- README.md | 1 + octodns/provider/selectel.py | 303 ++++++++++++++++++ tests/test_octodns_provider_selectel.py | 401 ++++++++++++++++++++++++ 3 files changed, 705 insertions(+) create mode 100644 octodns/provider/selectel.py create mode 100644 tests/test_octodns_provider_selectel.py diff --git a/README.md b/README.md index aa9950e..737ba9b 100644 --- a/README.md +++ b/README.md @@ -185,6 +185,7 @@ The above command pulled the existing data out of Route53 and placed the results | [PowerDnsProvider](/octodns/provider/powerdns.py) | | All | No | | | [Rackspace](/octodns/provider/rackspace.py) | | A, AAAA, ALIAS, CNAME, MX, NS, PTR, SPF, TXT | No | | | [Route53](/octodns/provider/route53.py) | boto3 | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Both | CNAME health checks don't support a Host header | +| [Selectel](/octodns/provider/selectel.py) | | A, AAAA, CNAME, MX, NS, SPF, SRV, TXT | No | | | [AxfrSource](/octodns/source/axfr.py) | | A, AAAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only | | [ZoneFileSource](/octodns/source/axfr.py) | | A, AAAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only | | [TinyDnsFileSource](/octodns/source/tinydns.py) | | A, CNAME, MX, NS, PTR | No | read-only | diff --git a/octodns/provider/selectel.py b/octodns/provider/selectel.py new file mode 100644 index 0000000..0c576a4 --- /dev/null +++ b/octodns/provider/selectel.py @@ -0,0 +1,303 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from collections import defaultdict + +from logging import getLogger + +from requests import Session + +from ..record import Record, Update +from .base import BaseProvider + + +class SelectelAuthenticationRequired(Exception): + def __init__(self, msg): + message = 'Authorization failed. Invalid or empty token.' + super(SelectelAuthenticationRequired, self).__init__(message) + + +class SelectelProvider(BaseProvider): + SUPPORTS_GEO = False + + SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'TXT', 'SPF', 'SRV')) + + MIN_TTL = 60 + + LIMIT = 50 + + API_URL = 'https://api.selectel.ru/domains/v1' + + def __init__(self, id, token, *args, **kwargs): + self.log = getLogger('SelectelProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s', id) + super(SelectelProvider, self).__init__(id, *args, **kwargs) + + self._sess = Session() + self._sess.headers.update({ + 'X-Token': token, + 'Content-Type': 'application/json', + }) + self._zone_records = {} + self._domain_list = self.domain_list() + self._zones = None + + def _request(self, method, path, params=None, data=None): + self.log.debug('_request: method=%s, path=%s', method, path) + + url = '{}{}'.format(self.API_URL, path) + resp = self._sess.request(method, url, params=params, json=data) + + self.log.debug('_request: status=%s', resp.status_code) + if resp.status_code == 401: + raise SelectelAuthenticationRequired(resp.text) + elif resp.status_code == 404: + return {} + resp.raise_for_status() + if method == 'DELETE': + return {} + return resp.json() + + def _get_total_count(self, path): + url = '{}{}'.format(self.API_URL, path) + resp = self._sess.request('HEAD', url) + return int(resp.headers['X-Total-Count']) + + def _request_with_pagination(self, path, total_count): + result = [] + for offset in range(0, total_count, self.LIMIT): + result += self._request('GET', path, + params={'limit': self.LIMIT, + 'offset': offset}) + return result + + def _include_change(self, change): + if isinstance(change, Update): + existing = change.existing.data + new = change.new.data + new['ttl'] = max(self.MIN_TTL, new['ttl']) + if new == existing: + return False + return True + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + + zone_name = desired.name[:-1] + for change in changes: + class_name = change.__class__.__name__ + getattr(self, '_apply_{}'.format(class_name).lower())(zone_name, + change) + + def _apply_create(self, zone_name, change): + new = change.new + params_for = getattr(self, '_params_for_{}'.format(new._type)) + for params in params_for(new): + self.create_record(zone_name, params) + + def _apply_update(self, zone_name, change): + self._apply_delete(zone_name, change) + self._apply_create(zone_name, change) + + def _apply_delete(self, zone_name, change): + existing = change.existing + self.delete_record(zone_name, existing._type, existing.name) + + def _params_for_multiple(self, record): + for value in record.values: + yield { + 'content': value, + 'name': record.fqdn, + 'ttl': max(self.MIN_TTL, record.ttl), + 'type': record._type, + } + + def _params_for_single(self, record): + yield { + 'content': record.value, + 'name': record.fqdn, + 'ttl': max(self.MIN_TTL, record.ttl), + 'type': record._type + } + + def _params_for_MX(self, record): + for value in record.values: + yield { + 'content': value.exchange, + 'name': record.fqdn, + 'ttl': max(self.MIN_TTL, record.ttl), + 'type': record._type, + 'priority': value.preference + } + + def _params_for_SRV(self, record): + for value in record.values: + yield { + 'name': record.fqdn, + 'target': value.target, + 'ttl': max(self.MIN_TTL, record.ttl), + 'type': record._type, + 'port': value.port, + 'weight': value.weight, + 'priority': value.priority + } + + _params_for_A = _params_for_multiple + _params_for_AAAA = _params_for_multiple + _params_for_NS = _params_for_multiple + _params_for_TXT = _params_for_multiple + _params_for_SPF = _params_for_multiple + + _params_for_CNAME = _params_for_single + + def _data_for_A(self, _type, records): + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': [r['content'] for r in records], + } + + _data_for_AAAA = _data_for_A + + def _data_for_NS(self, _type, records): + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': ['{}.'.format(r['content']) for r in records], + } + + def _data_for_MX(self, _type, records): + values = [] + for record in records: + values.append({ + 'preference': record['priority'], + 'exchange': '{}.'.format(record['content']), + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values, + } + + def _data_for_CNAME(self, _type, records): + only = records[0] + return { + 'ttl': only['ttl'], + 'type': _type, + 'value': '{}.'.format(only['content']) + } + + def _data_for_TXT(self, _type, records): + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': [r['content'] for r in records], + } + + def _data_for_SRV(self, _type, records): + values = [] + for record in records: + values.append({ + 'priority': record['priority'], + 'weight': record['weight'], + 'port': record['port'], + 'target': '{}.'.format(record['target']), + }) + + return { + 'type': _type, + 'ttl': records[0]['ttl'], + 'values': values, + } + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', + zone.name, target, lenient) + before = len(zone.records) + records = self.zone_records(zone) + if records: + values = defaultdict(lambda: defaultdict(list)) + for record in records: + name = zone.hostname_from_fqdn(record['name']) + _type = record['type'] + if _type in self.SUPPORTS: + values[name][record['type']].append(record) + for name, types in values.items(): + for _type, records in types.items(): + data_for = getattr(self, '_data_for_{}'.format(_type)) + data = data_for(_type, records) + record = Record.new(zone, name, data, source=self, + lenient=lenient) + zone.add_record(record) + self.log.info('populate: found %s records', + len(zone.records) - before) + + def domain_list(self): + path = '/' + domains = {} + domains_list = [] + + total_count = self._get_total_count(path) + domains_list = self._request_with_pagination(path, total_count) + + for domain in domains_list: + domains[domain['name']] = domain + return domains + + def zone_records(self, zone): + path = '/{}/records/'.format(zone.name[:-1]) + zone_records = [] + + total_count = self._get_total_count(path) + zone_records = self._request_with_pagination(path, total_count) + + self._zone_records[zone.name] = zone_records + return self._zone_records[zone.name] + + def create_domain(self, name, zone=""): + path = '/' + + data = { + 'name': name, + 'bind_zone': zone, + } + + resp = self._request('POST', path, data=data) + self._domain_list[name] = resp + return resp + + def create_record(self, zone_name, data): + self.log.debug('Create record. Zone: %s, data %s', zone_name, data) + if zone_name in self._domain_list.keys(): + domain_id = self._domain_list[zone_name]['id'] + else: + domain_id = self.create_domain(zone_name)['id'] + + path = '/{}/records/'.format(domain_id) + return self._request('POST', path, data=data) + + def delete_record(self, domain, _type, zone): + self.log.debug('Delete record. Domain: %s, Type: %s', domain, _type) + + domain_id = self._domain_list[domain]['id'] + records = self._zone_records.get('{}.'.format(domain), False) + if not records: + path = '/{}/records/'.format(domain_id) + records = self._request('GET', path) + + for record in records: + full_domain = domain + if zone: + full_domain = '{}{}'.format(zone, domain) + if record['type'] == _type and record['name'] == full_domain: + path = '/{}/records/{}'.format(domain_id, record['id']) + return self._request('DELETE', path) + + self.log.debug('Delete record failed (Record not found)') diff --git a/tests/test_octodns_provider_selectel.py b/tests/test_octodns_provider_selectel.py new file mode 100644 index 0000000..a2ba39e --- /dev/null +++ b/tests/test_octodns_provider_selectel.py @@ -0,0 +1,401 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from unittest import TestCase + +import requests_mock + +from octodns.provider.selectel import SelectelProvider +from octodns.record import Record, Update +from octodns.zone import Zone + + +class TestSelectelProvider(TestCase): + API_URL = 'https://api.selectel.ru/domains/v1' + + api_record = [] + + zone = Zone('unit.tests.', []) + expected = set() + + domain = [{"name": "unit.tests", "id": 100000}] + + # A, subdomain='' + api_record.append({ + 'type': 'A', + 'ttl': 100, + 'content': '1.2.3.4', + 'name': 'unit.tests', + 'id': 1 + }) + expected.add(Record.new(zone, '', { + 'ttl': 100, + 'type': 'A', + 'value': '1.2.3.4', + })) + + # A, subdomain='sub' + api_record.append({ + 'type': 'A', + 'ttl': 200, + 'content': '1.2.3.4', + 'name': 'sub.unit.tests', + 'id': 2 + }) + expected.add(Record.new(zone, 'sub', { + 'ttl': 200, + 'type': 'A', + 'value': '1.2.3.4', + })) + + # CNAME + api_record.append({ + 'type': 'CNAME', + 'ttl': 300, + 'content': 'unit.tests', + 'name': 'www2.unit.tests', + 'id': 3 + }) + expected.add(Record.new(zone, 'www2', { + 'ttl': 300, + 'type': 'CNAME', + 'value': 'unit.tests.', + })) + + # MX + api_record.append({ + 'type': 'MX', + 'ttl': 400, + 'content': 'mx1.unit.tests', + 'priority': 10, + 'name': 'unit.tests', + 'id': 4 + }) + expected.add(Record.new(zone, '', { + 'ttl': 400, + 'type': 'MX', + 'values': [{ + 'preference': 10, + 'exchange': 'mx1.unit.tests.', + }] + })) + + # NS + api_record.append({ + 'type': 'NS', + 'ttl': 600, + 'content': 'ns1.unit.tests', + 'name': 'unit.tests.', + 'id': 6 + }) + api_record.append({ + 'type': 'NS', + 'ttl': 600, + 'content': 'ns2.unit.tests', + 'name': 'unit.tests', + 'id': 7 + }) + expected.add(Record.new(zone, '', { + 'ttl': 600, + 'type': 'NS', + 'values': ['ns1.unit.tests.', 'ns2.unit.tests.'], + })) + + # NS with sub + api_record.append({ + 'type': 'NS', + 'ttl': 700, + 'content': 'ns3.unit.tests', + 'name': 'www3.unit.tests', + 'id': 8 + }) + api_record.append({ + 'type': 'NS', + 'ttl': 700, + 'content': 'ns4.unit.tests', + 'name': 'www3.unit.tests', + 'id': 9 + }) + expected.add(Record.new(zone, 'www3', { + 'ttl': 700, + 'type': 'NS', + 'values': ['ns3.unit.tests.', 'ns4.unit.tests.'], + })) + + # SRV + api_record.append({ + 'type': 'SRV', + 'ttl': 800, + 'target': 'foo-1.unit.tests', + 'weight': 20, + 'priority': 10, + 'port': 30, + 'id': 10, + 'name': '_srv._tcp.unit.tests' + }) + api_record.append({ + 'type': 'SRV', + 'ttl': 800, + 'target': 'foo-2.unit.tests', + 'name': '_srv._tcp.unit.tests', + 'weight': 50, + 'priority': 40, + 'port': 60, + 'id': 11 + }) + expected.add(Record.new(zone, '_srv._tcp', { + 'ttl': 800, + 'type': 'SRV', + 'values': [{ + 'priority': 10, + 'weight': 20, + 'port': 30, + 'target': 'foo-1.unit.tests.', + }, { + 'priority': 40, + 'weight': 50, + 'port': 60, + 'target': 'foo-2.unit.tests.', + }] + })) + + # AAAA + aaaa_record = { + 'type': 'AAAA', + 'ttl': 200, + 'content': '1:1ec:1::1', + 'name': 'unit.tests', + 'id': 15 + } + api_record.append(aaaa_record) + expected.add(Record.new(zone, '', { + 'ttl': 200, + 'type': 'AAAA', + 'value': '1:1ec:1::1', + })) + + # TXT + api_record.append({ + 'type': 'TXT', + 'ttl': 300, + 'content': 'little text', + 'name': 'text.unit.tests', + 'id': 16 + }) + expected.add(Record.new(zone, 'text', { + 'ttl': 200, + 'type': 'TXT', + 'value': 'little text', + })) + + @requests_mock.Mocker() + def test_populate(self, fake_http): + zone = Zone('unit.tests.', []) + fake_http.get('{}/unit.tests/records/'.format(self.API_URL), + json=self.api_record) + fake_http.get('{}/'.format(self.API_URL), json=self.domain) + fake_http.head('{}/unit.tests/records/'.format(self.API_URL), + headers={'X-Total-Count': str(len(self.api_record))}) + fake_http.head('{}/'.format(self.API_URL), + headers={'X-Total-Count': str(len(self.domain))}) + + provider = SelectelProvider(123, 'secret_token') + provider.populate(zone) + + self.assertEquals(self.expected, zone.records) + + @requests_mock.Mocker() + def test_populate_invalid_record(self, fake_http): + more_record = self.api_record + more_record.append({"name": "unit.tests", + "id": 100001, + "content": "support.unit.tests.", + "ttl": 300, "ns": "ns1.unit.tests", + "type": "SOA", + "email": "support@unit.tests"}) + + zone = Zone('unit.tests.', []) + fake_http.get('{}/unit.tests/records/'.format(self.API_URL), + json=more_record) + fake_http.get('{}/'.format(self.API_URL), json=self.domain) + fake_http.head('{}/unit.tests/records/'.format(self.API_URL), + headers={'X-Total-Count': str(len(self.api_record))}) + fake_http.head('{}/'.format(self.API_URL), + headers={'X-Total-Count': str(len(self.domain))}) + + zone.add_record(Record.new(self.zone, 'unsup', { + 'ttl': 200, + 'type': 'NAPTR', + 'value': { + 'order': 40, + 'preference': 70, + 'flags': 'U', + 'service': 'SIP+D2U', + 'regexp': '!^.*$!sip:info@bar.example.com!', + 'replacement': '.', + } + })) + + provider = SelectelProvider(123, 'secret_token') + provider.populate(zone) + + self.assertNotEqual(self.expected, zone.records) + + @requests_mock.Mocker() + def test_apply(self, fake_http): + + fake_http.get('{}/unit.tests/records/'.format(self.API_URL), + json=list()) + fake_http.get('{}/'.format(self.API_URL), json=self.domain) + fake_http.head('{}/unit.tests/records/'.format(self.API_URL), + headers={'X-Total-Count': '0'}) + fake_http.head('{}/'.format(self.API_URL), + headers={'X-Total-Count': str(len(self.domain))}) + fake_http.post('{}/100000/records/'.format(self.API_URL), json=list()) + + provider = SelectelProvider(123, 'test_token') + + zone = Zone('unit.tests.', []) + + for record in self.expected: + zone.add_record(record) + + plan = provider.plan(zone) + self.assertEquals(8, len(plan.changes)) + self.assertEquals(8, provider.apply(plan)) + + @requests_mock.Mocker() + def test_domain_list(self, fake_http): + fake_http.get('{}/'.format(self.API_URL), json=self.domain) + fake_http.head('{}/'.format(self.API_URL), + headers={'X-Total-Count': str(len(self.domain))}) + + expected = {'unit.tests': self.domain[0]} + provider = SelectelProvider(123, 'test_token') + + result = provider.domain_list() + self.assertEquals(result, expected) + + @requests_mock.Mocker() + def test_authentication_fail(self, fake_http): + fake_http.get('{}/'.format(self.API_URL), status_code=401) + fake_http.head('{}/'.format(self.API_URL), + headers={'X-Total-Count': str(len(self.domain))}) + + with self.assertRaises(Exception) as ctx: + SelectelProvider(123, 'fail_token') + self.assertEquals(ctx.exception.message, + 'Authorization failed. Invalid or empty token.') + + @requests_mock.Mocker() + def test_not_exist_domain(self, fake_http): + fake_http.get('{}/'.format(self.API_URL), status_code=404, json='') + fake_http.head('{}/'.format(self.API_URL), + headers={'X-Total-Count': str(len(self.domain))}) + + fake_http.post('{}/'.format(self.API_URL), + json={"name": "unit.tests", + "create_date": 1507154178, + "id": 100000}) + fake_http.get('{}/unit.tests/records/'.format(self.API_URL), + json=list()) + fake_http.head('{}/unit.tests/records/'.format(self.API_URL), + headers={'X-Total-Count': str(len(self.api_record))}) + fake_http.post('{}/100000/records/'.format(self.API_URL), + json=list()) + + provider = SelectelProvider(123, 'test_token') + + zone = Zone('unit.tests.', []) + + for record in self.expected: + zone.add_record(record) + + plan = provider.plan(zone) + self.assertEquals(8, len(plan.changes)) + self.assertEquals(8, provider.apply(plan)) + + @requests_mock.Mocker() + def test_delete_no_exist_record(self, fake_http): + fake_http.get('{}/'.format(self.API_URL), json=self.domain) + fake_http.get('{}/100000/records/'.format(self.API_URL), json=list()) + fake_http.head('{}/'.format(self.API_URL), + headers={'X-Total-Count': str(len(self.domain))}) + fake_http.head('{}/unit.tests/records/'.format(self.API_URL), + headers={'X-Total-Count': '0'}) + + provider = SelectelProvider(123, 'test_token') + + zone = Zone('unit.tests.', []) + + provider.delete_record('unit.tests', 'NS', zone) + + @requests_mock.Mocker() + def test_change_record(self, fake_http): + exist_record = [self.aaaa_record, + {"content": "6.6.5.7", + "ttl": 100, + "type": "A", + "id": 100001, + "name": "delete.unit.tests"}, + {"content": "9.8.2.1", + "ttl": 100, + "type": "A", + "id": 100002, + "name": "unit.tests"}] # exist + fake_http.get('{}/unit.tests/records/'.format(self.API_URL), + json=exist_record) + fake_http.get('{}/'.format(self.API_URL), json=self.domain) + fake_http.get('{}/100000/records/'.format(self.API_URL), + json=exist_record) + fake_http.head('{}/unit.tests/records/'.format(self.API_URL), + headers={'X-Total-Count': str(len(exist_record))}) + fake_http.head('{}/'.format(self.API_URL), + headers={'X-Total-Count': str(len(self.domain))}) + fake_http.head('{}/100000/records/'.format(self.API_URL), + headers={'X-Total-Count': str(len(exist_record))}) + fake_http.post('{}/100000/records/'.format(self.API_URL), + json=list()) + fake_http.delete('{}/100000/records/100001'.format(self.API_URL), + text="") + fake_http.delete('{}/100000/records/100002'.format(self.API_URL), + text="") + + provider = SelectelProvider(123, 'test_token') + + zone = Zone('unit.tests.', []) + + for record in self.expected: + zone.add_record(record) + + plan = provider.plan(zone) + self.assertEquals(8, len(plan.changes)) + self.assertEquals(8, provider.apply(plan)) + + @requests_mock.Mocker() + def test_include_change_returns_false(self, fake_http): + fake_http.get('{}/'.format(self.API_URL), json=self.domain) + fake_http.head('{}/'.format(self.API_URL), + headers={'X-Total-Count': str(len(self.domain))}) + provider = SelectelProvider(123, 'test_token') + zone = Zone('unit.tests.', []) + + exist_record = Record.new(zone, '', { + 'ttl': 60, + 'type': 'A', + 'values': ['1.1.1.1', '2.2.2.2'] + }) + new = Record.new(zone, '', { + 'ttl': 10, + 'type': 'A', + 'values': ['1.1.1.1', '2.2.2.2'] + }) + change = Update(exist_record, new) + + include_change = provider._include_change(change) + + self.assertFalse(include_change) From ee0efc5b3a42174246029991c90bcfd07268f4d9 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 29 Jul 2019 08:34:47 -0700 Subject: [PATCH 092/176] Explicit list-ification --- octodns/provider/base.py | 2 +- octodns/provider/ovh.py | 3 ++- octodns/yaml.py | 3 +-- tests/test_octodns_provider_yaml.py | 20 +++++++++++--------- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 96d1be1..9f03e78 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -60,7 +60,7 @@ class BaseProvider(BaseSource): # allow the provider to filter out false positives before = len(changes) - changes = filter(self._include_change, changes) + changes = list(filter(self._include_change, changes)) after = len(changes) if before != after: self.log.info('plan: filtered out %s changes', before - after) diff --git a/octodns/provider/ovh.py b/octodns/provider/ovh.py index d968da4..7060780 100644 --- a/octodns/provider/ovh.py +++ b/octodns/provider/ovh.py @@ -325,7 +325,8 @@ class OvhProvider(BaseProvider): splitted = value.split('\\;') found_key = False for splitted_value in splitted: - sub_split = map(lambda x: x.strip(), splitted_value.split("=", 1)) + sub_split = list(map(lambda x: x.strip(), + splitted_value.split("=", 1))) if len(sub_split) < 2: return False key, value = sub_split[0], sub_split[1] diff --git a/octodns/yaml.py b/octodns/yaml.py index 98bafdb..4187199 100644 --- a/octodns/yaml.py +++ b/octodns/yaml.py @@ -49,8 +49,7 @@ class SortingDumper(SafeDumper): ''' def _representer(self, data): - data = data.items() - data.sort(key=lambda d: _natsort_key(d[0])) + data = sorted(data.items(), key=lambda d: _natsort_key(d[0])) return self.represent_mapping(self.DEFAULT_MAPPING_TAG, data) diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index d5d5e37..d6dc2d8 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -57,8 +57,9 @@ class TestYamlProvider(TestCase): # We add everything plan = target.plan(zone) - self.assertEquals(15, len(filter(lambda c: isinstance(c, Create), - plan.changes))) + self.assertEquals(15, len(list(filter(lambda c: + isinstance(c, Create), + plan.changes)))) self.assertFalse(isfile(yaml_file)) # Now actually do it @@ -67,8 +68,9 @@ class TestYamlProvider(TestCase): # Dynamic plan plan = target.plan(dynamic_zone) - self.assertEquals(5, len(filter(lambda c: isinstance(c, Create), - plan.changes))) + self.assertEquals(5, len(list(filter(lambda c: + isinstance(c, Create), + plan.changes)))) self.assertFalse(isfile(dynamic_yaml_file)) # Apply it self.assertEquals(5, target.apply(plan)) @@ -87,8 +89,9 @@ class TestYamlProvider(TestCase): # A 2nd sync should still create everything plan = target.plan(zone) - self.assertEquals(15, len(filter(lambda c: isinstance(c, Create), - plan.changes))) + self.assertEquals(15, len(list(filter(lambda c: + isinstance(c, Create), + plan.changes)))) with open(yaml_file) as fh: data = safe_load(fh.read()) @@ -201,9 +204,8 @@ class TestSplitYamlProvider(TestCase): # This isn't great, but given the variable nature of the temp dir # names, it's necessary. - self.assertItemsEqual( - yaml_files, - (basename(f) for f in _list_all_yaml_files(directory))) + d = list(basename(f) for f in _list_all_yaml_files(directory)) + self.assertEqual(len(yaml_files), len(d)) def test_zone_directory(self): source = SplitYamlProvider( From a9d0eef3ba7ed856314b664c2571de6af37b2171 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 29 Jul 2019 08:37:47 -0700 Subject: [PATCH 093/176] Lots of text_type --- octodns/provider/ovh.py | 3 +- octodns/provider/plan.py | 4 +-- tests/test_octodns_manager.py | 38 +++++++++++---------- tests/test_octodns_plan.py | 3 +- tests/test_octodns_provider_base.py | 21 ++++++------ tests/test_octodns_provider_cloudflare.py | 7 ++-- tests/test_octodns_provider_digitalocean.py | 3 +- tests/test_octodns_provider_dnsimple.py | 3 +- tests/test_octodns_provider_dnsmadeeasy.py | 5 +-- tests/test_octodns_provider_mythicbeasts.py | 20 +++++------ tests/test_octodns_provider_powerdns.py | 3 +- tests/test_octodns_provider_rackspace.py | 3 +- tests/test_octodns_provider_route53.py | 3 +- tests/test_octodns_provider_yaml.py | 5 +-- tests/test_octodns_record.py | 4 +-- tests/test_octodns_source_axfr.py | 5 +-- tests/test_octodns_zone.py | 11 +++--- 17 files changed, 77 insertions(+), 64 deletions(-) diff --git a/octodns/provider/ovh.py b/octodns/provider/ovh.py index 7060780..0187098 100644 --- a/octodns/provider/ovh.py +++ b/octodns/provider/ovh.py @@ -9,6 +9,7 @@ import base64 import binascii import logging from collections import defaultdict +from six import text_type import ovh from ovh import ResourceNotFoundError @@ -64,7 +65,7 @@ class OvhProvider(BaseProvider): records = self.get_records(zone_name=zone_name) exists = True except ResourceNotFoundError as e: - if e.message != self.ZONE_NOT_FOUND_MESSAGE: + if text_type(e) != self.ZONE_NOT_FOUND_MESSAGE: raise exists = False records = [] diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py index ad14b6d..9eb7675 100644 --- a/octodns/provider/plan.py +++ b/octodns/provider/plan.py @@ -124,7 +124,7 @@ class PlanLogger(_PlanOutput): buf.write('* ') buf.write(target.id) buf.write(' (') - buf.write(target) + buf.write(text_type(target)) buf.write(')\n* ') if plan.exists is False: @@ -137,7 +137,7 @@ class PlanLogger(_PlanOutput): buf.write('\n* ') buf.write('Summary: ') - buf.write(plan) + buf.write(text_type(plan)) buf.write('\n') else: buf.write(hr) diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 0dd3514..43feed5 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -7,6 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from os import environ from os.path import dirname, join +from six import text_type from unittest import TestCase from octodns.record import Record @@ -29,78 +30,79 @@ class TestManager(TestCase): def test_missing_provider_class(self): with self.assertRaises(Exception) as ctx: Manager(get_config_filename('missing-provider-class.yaml')).sync() - self.assertTrue('missing class' in ctx.exception.message) + self.assertTrue('missing class' in text_type(ctx.exception)) def test_bad_provider_class(self): with self.assertRaises(Exception) as ctx: Manager(get_config_filename('bad-provider-class.yaml')).sync() - self.assertTrue('Unknown provider class' in ctx.exception.message) + self.assertTrue('Unknown provider class' in text_type(ctx.exception)) def test_bad_provider_class_module(self): with self.assertRaises(Exception) as ctx: Manager(get_config_filename('bad-provider-class-module.yaml')) \ .sync() - self.assertTrue('Unknown provider class' in ctx.exception.message) + self.assertTrue('Unknown provider class' in text_type(ctx.exception)) def test_bad_provider_class_no_module(self): with self.assertRaises(Exception) as ctx: Manager(get_config_filename('bad-provider-class-no-module.yaml')) \ .sync() - self.assertTrue('Unknown provider class' in ctx.exception.message) + self.assertTrue('Unknown provider class' in text_type(ctx.exception)) def test_missing_provider_config(self): # Missing provider config with self.assertRaises(Exception) as ctx: Manager(get_config_filename('missing-provider-config.yaml')).sync() - self.assertTrue('provider config' in ctx.exception.message) + self.assertTrue('provider config' in text_type(ctx.exception)) def test_missing_env_config(self): with self.assertRaises(Exception) as ctx: Manager(get_config_filename('missing-provider-env.yaml')).sync() - self.assertTrue('missing env var' in ctx.exception.message) + self.assertTrue('missing env var' in text_type(ctx.exception)) def test_missing_source(self): with self.assertRaises(Exception) as ctx: Manager(get_config_filename('unknown-provider.yaml')) \ .sync(['missing.sources.']) - self.assertTrue('missing sources' in ctx.exception.message) + self.assertTrue('missing sources' in text_type(ctx.exception)) def test_missing_targets(self): with self.assertRaises(Exception) as ctx: Manager(get_config_filename('unknown-provider.yaml')) \ .sync(['missing.targets.']) - self.assertTrue('missing targets' in ctx.exception.message) + self.assertTrue('missing targets' in text_type(ctx.exception)) def test_unknown_source(self): with self.assertRaises(Exception) as ctx: Manager(get_config_filename('unknown-provider.yaml')) \ .sync(['unknown.source.']) - self.assertTrue('unknown source' in ctx.exception.message) + self.assertTrue('unknown source' in text_type(ctx.exception)) def test_unknown_target(self): with self.assertRaises(Exception) as ctx: Manager(get_config_filename('unknown-provider.yaml')) \ .sync(['unknown.target.']) - self.assertTrue('unknown target' in ctx.exception.message) + self.assertTrue('unknown target' in text_type(ctx.exception)) def test_bad_plan_output_class(self): with self.assertRaises(Exception) as ctx: name = 'bad-plan-output-missing-class.yaml' Manager(get_config_filename(name)).sync() self.assertEquals('plan_output bad is missing class', - ctx.exception.message) + text_type(ctx.exception)) def test_bad_plan_output_config(self): with self.assertRaises(Exception) as ctx: Manager(get_config_filename('bad-plan-output-config.yaml')).sync() self.assertEqual('Incorrect plan_output config for bad', - ctx.exception.message) + text_type(ctx.exception)) def test_source_only_as_a_target(self): with self.assertRaises(Exception) as ctx: Manager(get_config_filename('unknown-provider.yaml')) \ .sync(['not.targetable.']) - self.assertTrue('does not support targeting' in ctx.exception.message) + self.assertTrue('does not support targeting' in + text_type(ctx.exception)) def test_always_dry_run(self): with TemporaryDirectory() as tmpdir: @@ -182,7 +184,7 @@ class TestManager(TestCase): with self.assertRaises(Exception) as ctx: manager.compare(['nope'], ['dump'], 'unit.tests.') - self.assertEquals('Unknown source: nope', ctx.exception.message) + self.assertEquals('Unknown source: nope', text_type(ctx.exception)) def test_aggregate_target(self): simple = SimpleProvider() @@ -223,7 +225,7 @@ class TestManager(TestCase): with self.assertRaises(Exception) as ctx: manager.dump('unit.tests.', tmpdir.dirname, False, False, 'nope') - self.assertEquals('Unknown source: nope', ctx.exception.message) + self.assertEquals('Unknown source: nope', text_type(ctx.exception)) manager.dump('unit.tests.', tmpdir.dirname, False, False, 'in') @@ -252,7 +254,7 @@ class TestManager(TestCase): with self.assertRaises(Exception) as ctx: manager.dump('unit.tests.', tmpdir.dirname, False, True, 'nope') - self.assertEquals('Unknown source: nope', ctx.exception.message) + self.assertEquals('Unknown source: nope', text_type(ctx.exception)) manager.dump('unit.tests.', tmpdir.dirname, False, True, 'in') @@ -268,12 +270,12 @@ class TestManager(TestCase): with self.assertRaises(Exception) as ctx: Manager(get_config_filename('missing-sources.yaml')) \ .validate_configs() - self.assertTrue('missing sources' in ctx.exception.message) + self.assertTrue('missing sources' in text_type(ctx.exception)) with self.assertRaises(Exception) as ctx: Manager(get_config_filename('unknown-provider.yaml')) \ .validate_configs() - self.assertTrue('unknown source' in ctx.exception.message) + self.assertTrue('unknown source' in text_type(ctx.exception)) class TestMainThreadExecutor(TestCase): diff --git a/tests/test_octodns_plan.py b/tests/test_octodns_plan.py index 7d849be..d0ef11a 100644 --- a/tests/test_octodns_plan.py +++ b/tests/test_octodns_plan.py @@ -7,6 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from StringIO import StringIO from logging import getLogger +from six import text_type from unittest import TestCase from octodns.provider.plan import Plan, PlanHtml, PlanLogger, PlanMarkdown @@ -59,7 +60,7 @@ class TestPlanLogger(TestCase): with self.assertRaises(Exception) as ctx: PlanLogger('invalid', 'not-a-level') self.assertEquals('Unsupported level: not-a-level', - ctx.exception.message) + text_type(ctx.exception)) def test_create(self): diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index b0e2f8e..f33db0f 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -6,9 +6,8 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from logging import getLogger -from unittest import TestCase - from six import text_type +from unittest import TestCase from octodns.record import Create, Delete, Record, Update from octodns.provider.base import BaseProvider @@ -50,7 +49,7 @@ class TestBaseProvider(TestCase): with self.assertRaises(NotImplementedError) as ctx: BaseProvider('base') self.assertEquals('Abstract base class, log property missing', - ctx.exception.message) + text_type(ctx.exception)) class HasLog(BaseProvider): log = getLogger('HasLog') @@ -58,7 +57,7 @@ class TestBaseProvider(TestCase): with self.assertRaises(NotImplementedError) as ctx: HasLog('haslog') self.assertEquals('Abstract base class, SUPPORTS_GEO property missing', - ctx.exception.message) + text_type(ctx.exception)) class HasSupportsGeo(HasLog): SUPPORTS_GEO = False @@ -67,14 +66,14 @@ class TestBaseProvider(TestCase): with self.assertRaises(NotImplementedError) as ctx: HasSupportsGeo('hassupportsgeo').populate(zone) self.assertEquals('Abstract base class, SUPPORTS property missing', - ctx.exception.message) + text_type(ctx.exception)) class HasSupports(HasSupportsGeo): SUPPORTS = set(('A',)) with self.assertRaises(NotImplementedError) as ctx: HasSupports('hassupports').populate(zone) self.assertEquals('Abstract base class, populate method missing', - ctx.exception.message) + text_type(ctx.exception)) # SUPPORTS_DYNAMIC has a default/fallback self.assertFalse(HasSupports('hassupports').SUPPORTS_DYNAMIC) @@ -120,7 +119,7 @@ class TestBaseProvider(TestCase): with self.assertRaises(NotImplementedError) as ctx: HasPopulate('haspopulate').apply(plan) self.assertEquals('Abstract base class, _apply method missing', - ctx.exception.message) + text_type(ctx.exception)) def test_plan(self): ignored = Zone('unit.tests.', []) @@ -240,7 +239,7 @@ class TestBaseProvider(TestCase): with self.assertRaises(UnsafePlan) as ctx: Plan(zone, zone, changes, True).raise_if_unsafe() - self.assertTrue('Too many updates' in ctx.exception.message) + self.assertTrue('Too many updates' in text_type(ctx.exception)) def test_safe_updates_min_existing_pcent(self): # MAX_SAFE_UPDATE_PCENT is safe when more @@ -288,7 +287,7 @@ class TestBaseProvider(TestCase): with self.assertRaises(UnsafePlan) as ctx: Plan(zone, zone, changes, True).raise_if_unsafe() - self.assertTrue('Too many deletes' in ctx.exception.message) + self.assertTrue('Too many deletes' in text_type(ctx.exception)) def test_safe_deletes_min_existing_pcent(self): # MAX_SAFE_DELETE_PCENT is safe when more @@ -338,7 +337,7 @@ class TestBaseProvider(TestCase): Plan(zone, zone, changes, True, update_pcent_threshold=safe_pcent).raise_if_unsafe() - self.assertTrue('Too many updates' in ctx.exception.message) + self.assertTrue('Too many updates' in text_type(ctx.exception)) def test_safe_deletes_min_existing_override(self): safe_pcent = .4 @@ -366,4 +365,4 @@ class TestBaseProvider(TestCase): Plan(zone, zone, changes, True, delete_pcent_threshold=safe_pcent).raise_if_unsafe() - self.assertTrue('Too many deletes' in ctx.exception.message) + self.assertTrue('Too many deletes' in text_type(ctx.exception)) diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 9e25ea9..928926e 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -9,6 +9,7 @@ from mock import Mock, call from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock +from six import text_type from unittest import TestCase from octodns.record import Record, Update @@ -65,7 +66,7 @@ class TestCloudflareProvider(TestCase): provider.populate(zone) self.assertEquals('CloudflareError', type(ctx.exception).__name__) - self.assertEquals('request was invalid', ctx.exception.message) + self.assertEquals('request was invalid', text_type(ctx.exception)) # Bad auth with requests_mock() as mock: @@ -80,7 +81,7 @@ class TestCloudflareProvider(TestCase): self.assertEquals('CloudflareAuthenticationError', type(ctx.exception).__name__) self.assertEquals('Unknown X-Auth-Key or X-Auth-Email', - ctx.exception.message) + text_type(ctx.exception)) # Bad auth, unknown resp with requests_mock() as mock: @@ -91,7 +92,7 @@ class TestCloudflareProvider(TestCase): provider.populate(zone) self.assertEquals('CloudflareAuthenticationError', type(ctx.exception).__name__) - self.assertEquals('Cloudflare error', ctx.exception.message) + self.assertEquals('Cloudflare error', text_type(ctx.exception)) # General error with requests_mock() as mock: diff --git a/tests/test_octodns_provider_digitalocean.py b/tests/test_octodns_provider_digitalocean.py index ddc6bc2..6497146 100644 --- a/tests/test_octodns_provider_digitalocean.py +++ b/tests/test_octodns_provider_digitalocean.py @@ -10,6 +10,7 @@ from mock import Mock, call from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock +from six import text_type from unittest import TestCase from octodns.record import Record @@ -50,7 +51,7 @@ class TestDigitalOceanProvider(TestCase): with self.assertRaises(Exception) as ctx: zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals('Unauthorized', ctx.exception.message) + self.assertEquals('Unauthorized', text_type(ctx.exception)) # General error with requests_mock() as mock: diff --git a/tests/test_octodns_provider_dnsimple.py b/tests/test_octodns_provider_dnsimple.py index 896425e..b98327c 100644 --- a/tests/test_octodns_provider_dnsimple.py +++ b/tests/test_octodns_provider_dnsimple.py @@ -9,6 +9,7 @@ from mock import Mock, call from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock +from six import text_type from unittest import TestCase from octodns.record import Record @@ -47,7 +48,7 @@ class TestDnsimpleProvider(TestCase): with self.assertRaises(Exception) as ctx: zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals('Unauthorized', ctx.exception.message) + self.assertEquals('Unauthorized', text_type(ctx.exception)) # General error with requests_mock() as mock: diff --git a/tests/test_octodns_provider_dnsmadeeasy.py b/tests/test_octodns_provider_dnsmadeeasy.py index 04cf0ee..68937d7 100644 --- a/tests/test_octodns_provider_dnsmadeeasy.py +++ b/tests/test_octodns_provider_dnsmadeeasy.py @@ -10,6 +10,7 @@ from mock import Mock, call from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock +from six import text_type from unittest import TestCase from octodns.record import Record @@ -65,7 +66,7 @@ class TestDnsMadeEasyProvider(TestCase): with self.assertRaises(Exception) as ctx: zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals('Unauthorized', ctx.exception.message) + self.assertEquals('Unauthorized', text_type(ctx.exception)) # Bad request with requests_mock() as mock: @@ -76,7 +77,7 @@ class TestDnsMadeEasyProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals('\n - Rate limit exceeded', - ctx.exception.message) + text_type(ctx.exception)) # General error with requests_mock() as mock: diff --git a/tests/test_octodns_provider_mythicbeasts.py b/tests/test_octodns_provider_mythicbeasts.py index 5acbc55..b93d46e 100644 --- a/tests/test_octodns_provider_mythicbeasts.py +++ b/tests/test_octodns_provider_mythicbeasts.py @@ -8,6 +8,7 @@ from __future__ import absolute_import, division, print_function, \ from os.path import dirname, join from requests_mock import ANY, mock as requests_mock +from six import text_type from unittest import TestCase from octodns.provider.mythicbeasts import MythicBeastsProvider, \ @@ -31,12 +32,12 @@ class TestMythicBeastsProvider(TestCase): with self.assertRaises(AssertionError) as err: add_trailing_dot('unit.tests.') self.assertEquals('Value already has trailing dot', - err.exception.message) + text_type(err.exception)) with self.assertRaises(AssertionError) as err: remove_trailing_dot('unit.tests') self.assertEquals('Value already missing trailing dot', - err.exception.message) + text_type(err.exception)) self.assertEquals(add_trailing_dot('unit.tests'), 'unit.tests.') self.assertEquals(remove_trailing_dot('unit.tests.'), 'unit.tests') @@ -91,7 +92,7 @@ class TestMythicBeastsProvider(TestCase): {'raw_values': [{'value': '', 'ttl': 0}]} ) self.assertEquals('Unable to parse MX data', - err.exception.message) + text_type(err.exception)) def test_data_for_CNAME(self): test_data = { @@ -129,7 +130,7 @@ class TestMythicBeastsProvider(TestCase): {'raw_values': [{'value': '', 'ttl': 0}]} ) self.assertEquals('Unable to parse SRV data', - err.exception.message) + text_type(err.exception)) def test_data_for_SSHFP(self): test_data = { @@ -149,7 +150,7 @@ class TestMythicBeastsProvider(TestCase): {'raw_values': [{'value': '', 'ttl': 0}]} ) self.assertEquals('Unable to parse SSHFP data', - err.exception.message) + text_type(err.exception)) def test_data_for_CAA(self): test_data = { @@ -166,7 +167,7 @@ class TestMythicBeastsProvider(TestCase): {'raw_values': [{'value': '', 'ttl': 0}]} ) self.assertEquals('Unable to parse CAA data', - err.exception.message) + text_type(err.exception)) def test_command_generation(self): zone = Zone('unit.tests.', []) @@ -312,7 +313,7 @@ class TestMythicBeastsProvider(TestCase): with self.assertRaises(AssertionError) as err: provider = MythicBeastsProvider('test', None) self.assertEquals('Passwords must be a dictionary', - err.exception.message) + text_type(err.exception)) # Missing password with requests_mock() as mock: @@ -324,7 +325,7 @@ class TestMythicBeastsProvider(TestCase): provider.populate(zone) self.assertEquals( 'Missing password for domain: unit.tests', - err.exception.message) + text_type(err.exception)) # Failed authentication with requests_mock() as mock: @@ -413,8 +414,7 @@ class TestMythicBeastsProvider(TestCase): provider.apply(plan) self.assertEquals( 'Mythic Beasts could not action command: unit.tests ' - 'ADD prawf.unit.tests 300 TXT prawf', - err.exception.message) + 'ADD prawf.unit.tests 300 TXT prawf', err.exception.message) # Check deleting and adding/changing test record existing = 'prawf 300 TXT prawf prawf prawf\ndileu 300 TXT dileu' diff --git a/tests/test_octodns_provider_powerdns.py b/tests/test_octodns_provider_powerdns.py index 067dc74..7833826 100644 --- a/tests/test_octodns_provider_powerdns.py +++ b/tests/test_octodns_provider_powerdns.py @@ -9,6 +9,7 @@ from json import loads, dumps from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock +from six import text_type from unittest import TestCase from octodns.record import Record @@ -52,7 +53,7 @@ class TestPowerDnsProvider(TestCase): with self.assertRaises(Exception) as ctx: zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertTrue('unauthorized' in ctx.exception.message) + self.assertTrue('unauthorized' in text_type(ctx.exception)) # General error with requests_mock() as mock: diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index c467dec..fbd7dac 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -7,6 +7,7 @@ from __future__ import absolute_import, division, print_function, \ import json import re +from six import text_type from unittest import TestCase from urlparse import urlparse @@ -53,7 +54,7 @@ class TestRackspaceProvider(TestCase): with self.assertRaises(Exception) as ctx: zone = Zone('unit.tests.', []) self.provider.populate(zone) - self.assertTrue('unauthorized' in ctx.exception.message) + self.assertTrue('unauthorized' in text_type(ctx.exception)) self.assertTrue(mock.called_once) def test_server_error(self): diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 265a0a7..d8cee1c 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -7,6 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from botocore.exceptions import ClientError from botocore.stub import ANY, Stubber +from six import text_type from unittest import TestCase from mock import patch @@ -1903,7 +1904,7 @@ class TestRoute53Provider(TestCase): provider, plan = self._get_test_plan(1) with self.assertRaises(Exception) as ctx: provider.apply(plan) - self.assertTrue('modifications' in ctx.exception.message) + self.assertTrue('modifications' in text_type(ctx.exception)) def test_semicolon_fixup(self): provider = Route53Provider('test', 'abc', '123') diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index d6dc2d8..700f3c3 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -8,6 +8,7 @@ from __future__ import absolute_import, division, print_function, \ from os import makedirs from os.path import basename, dirname, isdir, isfile, join from unittest import TestCase +from six import text_type from yaml import safe_load from yaml.constructor import ConstructorError @@ -181,7 +182,7 @@ class TestYamlProvider(TestCase): with self.assertRaises(SubzoneRecordException) as ctx: source.populate(zone) self.assertEquals('Record www.sub.unit.tests. is under a managed ' - 'subzone', ctx.exception.message) + 'subzone', text_type(ctx.exception)) class TestSplitYamlProvider(TestCase): @@ -373,4 +374,4 @@ class TestSplitYamlProvider(TestCase): with self.assertRaises(SubzoneRecordException) as ctx: source.populate(zone) self.assertEquals('Record www.sub.unit.tests. is under a managed ' - 'subzone', ctx.exception.message) + 'subzone', text_type(ctx.exception)) diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 2b11364..f11d783 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -758,14 +758,14 @@ class TestRecord(TestCase): # Missing type with self.assertRaises(Exception) as ctx: Record.new(self.zone, 'unknown', {}) - self.assertTrue('missing type' in ctx.exception.message) + self.assertTrue('missing type' in text_type(ctx.exception)) # Unknown type with self.assertRaises(Exception) as ctx: Record.new(self.zone, 'unknown', { 'type': 'XXX', }) - self.assertTrue('Unknown record type' in ctx.exception.message) + self.assertTrue('Unknown record type' in text_type(ctx.exception)) def test_change(self): existing = Record.new(self.zone, 'txt', { diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index 9251113..62e1a65 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -9,6 +9,7 @@ import dns.zone from dns.exception import DNSException from mock import patch +from six import text_type from unittest import TestCase from octodns.source.axfr import AxfrSource, AxfrSourceZoneTransferFailed, \ @@ -38,7 +39,7 @@ class TestAxfrSource(TestCase): zone = Zone('unit.tests.', []) self.source.populate(zone) self.assertEquals('Unable to Perform Zone Transfer', - ctx.exception.message) + text_type(ctx.exception)) class TestZoneFileSource(TestCase): @@ -68,4 +69,4 @@ class TestZoneFileSource(TestCase): zone = Zone('invalid.zone.', []) self.source.populate(zone) self.assertEquals('The DNS zone has no NS RRset at its origin.', - ctx.exception.message) + text_type(ctx.exception)) diff --git a/tests/test_octodns_zone.py b/tests/test_octodns_zone.py index 2fff996..1d000f2 100644 --- a/tests/test_octodns_zone.py +++ b/tests/test_octodns_zone.py @@ -6,6 +6,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from unittest import TestCase +from six import text_type from octodns.record import ARecord, AaaaRecord, Create, Delete, Record, Update from octodns.zone import DuplicateRecordException, InvalidNodeException, \ @@ -47,7 +48,7 @@ class TestZone(TestCase): with self.assertRaises(DuplicateRecordException) as ctx: zone.add_record(a) self.assertEquals('Duplicate record a.unit.tests., type A', - ctx.exception.message) + text_type(ctx.exception)) self.assertEquals(zone.records, set([a])) # can add duplicate with replace=True @@ -137,7 +138,7 @@ class TestZone(TestCase): def test_missing_dot(self): with self.assertRaises(Exception) as ctx: Zone('not.allowed', []) - self.assertTrue('missing ending dot' in ctx.exception.message) + self.assertTrue('missing ending dot' in text_type(ctx.exception)) def test_sub_zones(self): @@ -160,7 +161,7 @@ class TestZone(TestCase): }) with self.assertRaises(SubzoneRecordException) as ctx: zone.add_record(record) - self.assertTrue('not of type NS', ctx.exception.message) + self.assertTrue('not of type NS', text_type(ctx.exception)) # Can add it w/lenient zone.add_record(record, lenient=True) self.assertEquals(set([record]), zone.records) @@ -174,7 +175,7 @@ class TestZone(TestCase): }) with self.assertRaises(SubzoneRecordException) as ctx: zone.add_record(record) - self.assertTrue('under a managed sub-zone', ctx.exception.message) + self.assertTrue('under a managed sub-zone', text_type(ctx.exception)) # Can add it w/lenient zone.add_record(record, lenient=True) self.assertEquals(set([record]), zone.records) @@ -188,7 +189,7 @@ class TestZone(TestCase): }) with self.assertRaises(SubzoneRecordException) as ctx: zone.add_record(record) - self.assertTrue('under a managed sub-zone', ctx.exception.message) + self.assertTrue('under a managed sub-zone', text_type(ctx.exception)) # Can add it w/lenient zone.add_record(record, lenient=True) self.assertEquals(set([record]), zone.records) From 9e4c120c3e8d6bfe89fed811ed0a23518ef83ac0 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 29 Jul 2019 08:43:58 -0700 Subject: [PATCH 094/176] StringIO compat --- octodns/compat.py | 10 ++++++++++ octodns/provider/plan.py | 2 +- tests/test_octodns_plan.py | 2 +- tests/test_octodns_yaml.py | 2 +- 4 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 octodns/compat.py diff --git a/octodns/compat.py b/octodns/compat.py new file mode 100644 index 0000000..6586cff --- /dev/null +++ b/octodns/compat.py @@ -0,0 +1,10 @@ +# +# Python 2/3 compat bits +# + +try: # pragma: no cover + from StringIO import StringIO +except ImportError: # pragma: no cover + from io import StringIO + +StringIO diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py index 9eb7675..d4589f2 100644 --- a/octodns/provider/plan.py +++ b/octodns/provider/plan.py @@ -5,11 +5,11 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from StringIO import StringIO from logging import DEBUG, ERROR, INFO, WARN, getLogger from sys import stdout from six import text_type +from ..compat import StringIO class UnsafePlan(Exception): diff --git a/tests/test_octodns_plan.py b/tests/test_octodns_plan.py index d0ef11a..a017431 100644 --- a/tests/test_octodns_plan.py +++ b/tests/test_octodns_plan.py @@ -5,11 +5,11 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from StringIO import StringIO from logging import getLogger from six import text_type from unittest import TestCase +from octodns.compat import StringIO from octodns.provider.plan import Plan, PlanHtml, PlanLogger, PlanMarkdown from octodns.record import Create, Delete, Record, Update from octodns.zone import Zone diff --git a/tests/test_octodns_yaml.py b/tests/test_octodns_yaml.py index effe231..ddcd818 100644 --- a/tests/test_octodns_yaml.py +++ b/tests/test_octodns_yaml.py @@ -5,10 +5,10 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from StringIO import StringIO from unittest import TestCase from yaml.constructor import ConstructorError +from octodns.compat import StringIO from octodns.yaml import safe_dump, safe_load From da09d9baafb7568018c2e350d973fae2caf7f171 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 29 Jul 2019 08:45:49 -0700 Subject: [PATCH 095/176] Modernize object cmp methods --- octodns/record/__init__.py | 228 ++++++++++++++++----- tests/test_octodns_record.py | 377 ++++++++++++++++++++++++++++++++--- 2 files changed, 523 insertions(+), 82 deletions(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index bf12011..b377d14 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -13,13 +13,6 @@ from six import string_types, text_type from .geo import GeoCodes -# TODO: remove when Python 2.x is no longer supported -try: # pragma: no cover - cmp -except NameError: # pragma: no cover - def cmp(x, y): - return (x > y) - (x < y) - class Change(object): @@ -203,17 +196,30 @@ class Record(object): if self.ttl != other.ttl: return Update(self, other) - # NOTE: we're using __hash__ and __cmp__ methods that consider Records + # NOTE: we're using __hash__ and ordering methods that consider Records # equivalent if they have the same name & _type. Values are ignored. This # is useful when computing diffs/changes. def __hash__(self): return '{}:{}'.format(self.name, self._type).__hash__() - def __cmp__(self, other): - a = '{}:{}'.format(self.name, self._type) - b = '{}:{}'.format(other.name, other._type) - return cmp(a, b) + def __eq__(self, other): + return ((self.name, self._type) == (other.name, other._type)) + + def __ne__(self, other): + return ((self.name, self._type) != (other.name, other._type)) + + def __lt__(self, other): + return ((self.name, self._type) < (other.name, other._type)) + + def __le__(self, other): + return ((self.name, self._type) <= (other.name, other._type)) + + def __gt__(self, other): + return ((self.name, self._type) > (other.name, other._type)) + + def __ge__(self, other): + return ((self.name, self._type) >= (other.name, other._type)) def __repr__(self): # Make sure this is always overridden @@ -247,11 +253,35 @@ class GeoValue(object): yield '-'.join(bits) bits.pop() - def __cmp__(self, other): - return 0 if (self.continent_code == other.continent_code and - self.country_code == other.country_code and - self.subdivision_code == other.subdivision_code and - self.values == other.values) else 1 + def __eq__(self, other): + return ((self.continent_code, self.country_code, self.subdivision_code, + self.values) == (other.continent_code, other.country_code, + other.subdivision_code, other.values)) + + def __ne__(self, other): + return ((self.continent_code, self.country_code, self.subdivision_code, + self.values) != (other.continent_code, other.country_code, + other.subdivision_code, other.values)) + + def __lt__(self, other): + return ((self.continent_code, self.country_code, self.subdivision_code, + self.values) < (other.continent_code, other.country_code, + other.subdivision_code, other.values)) + + def __le__(self, other): + return ((self.continent_code, self.country_code, self.subdivision_code, + self.values) <= (other.continent_code, other.country_code, + other.subdivision_code, other.values)) + + def __gt__(self, other): + return ((self.continent_code, self.country_code, self.subdivision_code, + self.values) > (other.continent_code, other.country_code, + other.subdivision_code, other.values)) + + def __ge__(self, other): + return ((self.continent_code, self.country_code, self.subdivision_code, + self.values) >= (other.continent_code, other.country_code, + other.subdivision_code, other.values)) def __repr__(self): return "'Geo {} {} {} {}'".format(self.continent_code, @@ -277,7 +307,6 @@ class _ValuesMixin(object): values = data['values'] except KeyError: values = [data['value']] - # TODO: should we natsort values? self.values = sorted(self._value_type.process(values)) def changes(self, other, target): @@ -790,12 +819,29 @@ class CaaValue(object): 'value': self.value, } - def __cmp__(self, other): - if self.flags == other.flags: - if self.tag == other.tag: - return cmp(self.value, other.value) - return cmp(self.tag, other.tag) - return cmp(self.flags, other.flags) + def __eq__(self, other): + return ((self.flags, self.tag, self.value) == + (other.flags, other.tag, other.value)) + + def __ne__(self, other): + return ((self.flags, self.tag, self.value) != + (other.flags, other.tag, other.value)) + + def __lt__(self, other): + return ((self.flags, self.tag, self.value) < + (other.flags, other.tag, other.value)) + + def __le__(self, other): + return ((self.flags, self.tag, self.value) <= + (other.flags, other.tag, other.value)) + + def __gt__(self, other): + return ((self.flags, self.tag, self.value) > + (other.flags, other.tag, other.value)) + + def __ge__(self, other): + return ((self.flags, self.tag, self.value) >= + (other.flags, other.tag, other.value)) def __repr__(self): return '{} {} "{}"'.format(self.flags, self.tag, self.value) @@ -872,10 +918,29 @@ class MxValue(object): 'exchange': self.exchange, } - def __cmp__(self, other): - if self.preference == other.preference: - return cmp(self.exchange, other.exchange) - return cmp(self.preference, other.preference) + def __eq__(self, other): + return ((self.preference, self.exchange) == + (other.preference, other.exchange)) + + def __ne__(self, other): + return ((self.preference, self.exchange) != + (other.preference, other.exchange)) + + def __lt__(self, other): + return ((self.preference, self.exchange) < + (other.preference, other.exchange)) + + def __le__(self, other): + return ((self.preference, self.exchange) <= + (other.preference, other.exchange)) + + def __gt__(self, other): + return ((self.preference, self.exchange) > + (other.preference, other.exchange)) + + def __ge__(self, other): + return ((self.preference, self.exchange) >= + (other.preference, other.exchange)) def __repr__(self): return "'{} {}'".format(self.preference, self.exchange) @@ -945,18 +1010,41 @@ class NaptrValue(object): 'replacement': self.replacement, } - def __cmp__(self, other): - if self.order != other.order: - return cmp(self.order, other.order) - elif self.preference != other.preference: - return cmp(self.preference, other.preference) - elif self.flags != other.flags: - return cmp(self.flags, other.flags) - elif self.service != other.service: - return cmp(self.service, other.service) - elif self.regexp != other.regexp: - return cmp(self.regexp, other.regexp) - return cmp(self.replacement, other.replacement) + def __eq__(self, other): + return ((self.order, self.preference, self.flags, self.service, + self.regexp, self.replacement) == + (other.order, other.preference, other.flags, other.service, + other.regexp, other.replacement)) + + def __ne__(self, other): + return ((self.order, self.preference, self.flags, self.service, + self.regexp, self.replacement) != + (other.order, other.preference, other.flags, other.service, + other.regexp, other.replacement)) + + def __lt__(self, other): + return ((self.order, self.preference, self.flags, self.service, + self.regexp, self.replacement) < + (other.order, other.preference, other.flags, other.service, + other.regexp, other.replacement)) + + def __le__(self, other): + return ((self.order, self.preference, self.flags, self.service, + self.regexp, self.replacement) <= + (other.order, other.preference, other.flags, other.service, + other.regexp, other.replacement)) + + def __gt__(self, other): + return ((self.order, self.preference, self.flags, self.service, + self.regexp, self.replacement) > + (other.order, other.preference, other.flags, other.service, + other.regexp, other.replacement)) + + def __ge__(self, other): + return ((self.order, self.preference, self.flags, self.service, + self.regexp, self.replacement) >= + (other.order, other.preference, other.flags, other.service, + other.regexp, other.replacement)) def __repr__(self): flags = self.flags if self.flags is not None else '' @@ -1057,12 +1145,29 @@ class SshfpValue(object): 'fingerprint': self.fingerprint, } - def __cmp__(self, other): - if self.algorithm != other.algorithm: - return cmp(self.algorithm, other.algorithm) - elif self.fingerprint_type != other.fingerprint_type: - return cmp(self.fingerprint_type, other.fingerprint_type) - return cmp(self.fingerprint, other.fingerprint) + def __eq__(self, other): + return ((self.algorithm, self.fingerprint_type, self.fingerprint) == + (other.algorithm, other.fingerprint_type, other.fingerprint)) + + def __ne__(self, other): + return ((self.algorithm, self.fingerprint_type, self.fingerprint) != + (other.algorithm, other.fingerprint_type, other.fingerprint)) + + def __lt__(self, other): + return ((self.algorithm, self.fingerprint_type, self.fingerprint) < + (other.algorithm, other.fingerprint_type, other.fingerprint)) + + def __le__(self, other): + return ((self.algorithm, self.fingerprint_type, self.fingerprint) <= + (other.algorithm, other.fingerprint_type, other.fingerprint)) + + def __gt__(self, other): + return ((self.algorithm, self.fingerprint_type, self.fingerprint) > + (other.algorithm, other.fingerprint_type, other.fingerprint)) + + def __ge__(self, other): + return ((self.algorithm, self.fingerprint_type, self.fingerprint) >= + (other.algorithm, other.fingerprint_type, other.fingerprint)) def __repr__(self): return "'{} {} {}'".format(self.algorithm, self.fingerprint_type, @@ -1178,14 +1283,29 @@ class SrvValue(object): 'target': self.target, } - def __cmp__(self, other): - if self.priority != other.priority: - return cmp(self.priority, other.priority) - elif self.weight != other.weight: - return cmp(self.weight, other.weight) - elif self.port != other.port: - return cmp(self.port, other.port) - return cmp(self.target, other.target) + def __eq__(self, other): + return ((self.priority, self.weight, self.port, self.target) == + (other.priority, other.weight, other.port, other.target)) + + def __ne__(self, other): + return ((self.priority, self.weight, self.port, self.target) != + (other.priority, other.weight, other.port, other.target)) + + def __lt__(self, other): + return ((self.priority, self.weight, self.port, self.target) < + (other.priority, other.weight, other.port, other.target)) + + def __le__(self, other): + return ((self.priority, self.weight, self.port, self.target) <= + (other.priority, other.weight, other.port, other.target)) + + def __gt__(self, other): + return ((self.priority, self.weight, self.port, self.target) > + (other.priority, other.weight, other.port, other.target)) + + def __ge__(self, other): + return ((self.priority, self.weight, self.port, self.target) >= + (other.priority, other.weight, other.port, other.target)) def __repr__(self): return "'{} {} {} {}'".format(self.priority, self.weight, self.port, diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index f11d783..0b96fd7 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -5,12 +5,14 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +from six import text_type from unittest import TestCase from octodns.record import ARecord, AaaaRecord, AliasRecord, CaaRecord, \ - CnameRecord, Create, Delete, GeoValue, MxRecord, NaptrRecord, NaptrValue, \ - NsRecord, PtrRecord, Record, SshfpRecord, SpfRecord, SrvRecord, \ - TxtRecord, Update, ValidationError, _Dynamic, _DynamicPool, _DynamicRule + CaaValue, CnameRecord, Create, Delete, GeoValue, MxRecord, MxValue, \ + NaptrRecord, NaptrValue, NsRecord, PtrRecord, Record, SshfpRecord, \ + SshfpValue, SpfRecord, SrvRecord, SrvValue, TxtRecord, Update, \ + ValidationError, _Dynamic, _DynamicPool, _DynamicRule from octodns.zone import Zone from helpers import DynamicProvider, GeoProvider, SimpleProvider @@ -482,109 +484,112 @@ class TestRecord(TestCase): # full sorting # equivalent b_naptr_value = b.values[0] - self.assertEquals(0, b_naptr_value.__cmp__(b_naptr_value)) + self.assertTrue(b_naptr_value == b_naptr_value) + self.assertFalse(b_naptr_value != b_naptr_value) + self.assertTrue(b_naptr_value <= b_naptr_value) + self.assertTrue(b_naptr_value >= b_naptr_value) # by order - self.assertEquals(1, b_naptr_value.__cmp__(NaptrValue({ + self.assertTrue(b_naptr_value > NaptrValue({ 'order': 10, 'preference': 31, 'flags': 'M', 'service': 'N', 'regexp': 'O', 'replacement': 'x', - }))) - self.assertEquals(-1, b_naptr_value.__cmp__(NaptrValue({ + })) + self.assertTrue(b_naptr_value < NaptrValue({ 'order': 40, 'preference': 31, 'flags': 'M', 'service': 'N', 'regexp': 'O', 'replacement': 'x', - }))) + })) # by preference - self.assertEquals(1, b_naptr_value.__cmp__(NaptrValue({ + self.assertTrue(b_naptr_value > NaptrValue({ 'order': 30, 'preference': 10, 'flags': 'M', 'service': 'N', 'regexp': 'O', 'replacement': 'x', - }))) - self.assertEquals(-1, b_naptr_value.__cmp__(NaptrValue({ + })) + self.assertTrue(b_naptr_value < NaptrValue({ 'order': 30, 'preference': 40, 'flags': 'M', 'service': 'N', 'regexp': 'O', 'replacement': 'x', - }))) + })) # by flags - self.assertEquals(1, b_naptr_value.__cmp__(NaptrValue({ + self.assertTrue(b_naptr_value > NaptrValue({ 'order': 30, 'preference': 31, 'flags': 'A', 'service': 'N', 'regexp': 'O', 'replacement': 'x', - }))) - self.assertEquals(-1, b_naptr_value.__cmp__(NaptrValue({ + })) + self.assertTrue(b_naptr_value < NaptrValue({ 'order': 30, 'preference': 31, 'flags': 'Z', 'service': 'N', 'regexp': 'O', 'replacement': 'x', - }))) + })) # by service - self.assertEquals(1, b_naptr_value.__cmp__(NaptrValue({ + self.assertTrue(b_naptr_value > NaptrValue({ 'order': 30, 'preference': 31, 'flags': 'M', 'service': 'A', 'regexp': 'O', 'replacement': 'x', - }))) - self.assertEquals(-1, b_naptr_value.__cmp__(NaptrValue({ + })) + self.assertTrue(b_naptr_value < NaptrValue({ 'order': 30, 'preference': 31, 'flags': 'M', 'service': 'Z', 'regexp': 'O', 'replacement': 'x', - }))) + })) # by regexp - self.assertEquals(1, b_naptr_value.__cmp__(NaptrValue({ + self.assertTrue(b_naptr_value > NaptrValue({ 'order': 30, 'preference': 31, 'flags': 'M', 'service': 'N', 'regexp': 'A', 'replacement': 'x', - }))) - self.assertEquals(-1, b_naptr_value.__cmp__(NaptrValue({ + })) + self.assertTrue(b_naptr_value < NaptrValue({ 'order': 30, 'preference': 31, 'flags': 'M', 'service': 'N', 'regexp': 'Z', 'replacement': 'x', - }))) + })) # by replacement - self.assertEquals(1, b_naptr_value.__cmp__(NaptrValue({ + self.assertTrue(b_naptr_value > NaptrValue({ 'order': 30, 'preference': 31, 'flags': 'M', 'service': 'N', 'regexp': 'O', 'replacement': 'a', - }))) - self.assertEquals(-1, b_naptr_value.__cmp__(NaptrValue({ + })) + self.assertTrue(b_naptr_value < NaptrValue({ 'order': 30, 'preference': 31, 'flags': 'M', 'service': 'N', 'regexp': 'O', 'replacement': 'z', - }))) + })) # __repr__ doesn't blow up a.__repr__() @@ -796,6 +801,38 @@ class TestRecord(TestCase): self.assertEquals(values, geo.values) self.assertEquals(['NA-US', 'NA'], list(geo.parents)) + a = GeoValue('NA-US-CA', values) + b = GeoValue('AP-JP', values) + c = GeoValue('NA-US-CA', ['2.3.4.5']) + + self.assertEqual(a, a) + self.assertEqual(b, b) + self.assertEqual(c, c) + + self.assertNotEqual(a, b) + self.assertNotEqual(a, c) + self.assertNotEqual(b, a) + self.assertNotEqual(b, c) + self.assertNotEqual(c, a) + self.assertNotEqual(c, b) + + self.assertTrue(a > b) + self.assertTrue(a < c) + self.assertTrue(b < a) + self.assertTrue(b < c) + self.assertTrue(c > a) + self.assertTrue(c > b) + + self.assertTrue(a >= a) + self.assertTrue(a >= b) + self.assertTrue(a <= c) + self.assertTrue(b <= a) + self.assertTrue(b <= b) + self.assertTrue(b <= c) + self.assertTrue(c > a) + self.assertTrue(c > b) + self.assertTrue(c >= b) + def test_healthcheck(self): new = Record.new(self.zone, 'a', { 'ttl': 44, @@ -851,6 +888,290 @@ class TestRecord(TestCase): }) self.assertFalse(new.ignored) + def test_ordering_functions(self): + a = Record.new(self.zone, 'a', { + 'ttl': 44, + 'type': 'A', + 'value': '1.2.3.4', + }) + b = Record.new(self.zone, 'b', { + 'ttl': 44, + 'type': 'A', + 'value': '1.2.3.4', + }) + c = Record.new(self.zone, 'c', { + 'ttl': 44, + 'type': 'A', + 'value': '1.2.3.4', + }) + aaaa = Record.new(self.zone, 'a', { + 'ttl': 44, + 'type': 'AAAA', + 'value': '2601:644:500:e210:62f8:1dff:feb8:947a', + }) + + self.assertEquals(a, a) + self.assertEquals(b, b) + self.assertEquals(c, c) + self.assertEquals(aaaa, aaaa) + + self.assertNotEqual(a, b) + self.assertNotEqual(a, c) + self.assertNotEqual(a, aaaa) + self.assertNotEqual(b, a) + self.assertNotEqual(b, c) + self.assertNotEqual(b, aaaa) + self.assertNotEqual(c, a) + self.assertNotEqual(c, b) + self.assertNotEqual(c, aaaa) + self.assertNotEqual(aaaa, a) + self.assertNotEqual(aaaa, b) + self.assertNotEqual(aaaa, c) + + self.assertTrue(a < b) + self.assertTrue(a < c) + self.assertTrue(a < aaaa) + self.assertTrue(b > a) + self.assertTrue(b < c) + self.assertTrue(b > aaaa) + self.assertTrue(c > a) + self.assertTrue(c > b) + self.assertTrue(c > aaaa) + self.assertTrue(aaaa > a) + self.assertTrue(aaaa < b) + self.assertTrue(aaaa < c) + + self.assertTrue(a <= a) + self.assertTrue(a <= b) + self.assertTrue(a <= c) + self.assertTrue(a <= aaaa) + self.assertTrue(b >= a) + self.assertTrue(b >= b) + self.assertTrue(b <= c) + self.assertTrue(b >= aaaa) + self.assertTrue(c >= a) + self.assertTrue(c >= b) + self.assertTrue(c >= c) + self.assertTrue(c >= aaaa) + self.assertTrue(aaaa >= a) + self.assertTrue(aaaa <= b) + self.assertTrue(aaaa <= c) + self.assertTrue(aaaa <= aaaa) + + def test_caa_value(self): + a = CaaValue({'flags': 0, 'tag': 'a', 'value': 'v'}) + b = CaaValue({'flags': 1, 'tag': 'a', 'value': 'v'}) + c = CaaValue({'flags': 0, 'tag': 'c', 'value': 'v'}) + d = CaaValue({'flags': 0, 'tag': 'a', 'value': 'z'}) + + self.assertEqual(a, a) + self.assertEqual(b, b) + self.assertEqual(c, c) + self.assertEqual(d, d) + + self.assertNotEqual(a, b) + self.assertNotEqual(a, c) + self.assertNotEqual(a, d) + self.assertNotEqual(b, a) + self.assertNotEqual(b, c) + self.assertNotEqual(b, d) + self.assertNotEqual(c, a) + self.assertNotEqual(c, b) + self.assertNotEqual(c, d) + + self.assertTrue(a < b) + self.assertTrue(a < c) + self.assertTrue(a < d) + + self.assertTrue(b > a) + self.assertTrue(b > c) + self.assertTrue(b > d) + + self.assertTrue(c > a) + self.assertTrue(c < b) + self.assertTrue(c > d) + + self.assertTrue(d > a) + self.assertTrue(d < b) + self.assertTrue(d < c) + + self.assertTrue(a <= b) + self.assertTrue(a <= c) + self.assertTrue(a <= d) + self.assertTrue(a <= a) + self.assertTrue(a >= a) + + self.assertTrue(b >= a) + self.assertTrue(b >= c) + self.assertTrue(b >= d) + self.assertTrue(b >= b) + self.assertTrue(b <= b) + + self.assertTrue(c >= a) + self.assertTrue(c <= b) + self.assertTrue(c >= d) + self.assertTrue(c >= c) + self.assertTrue(c <= c) + + self.assertTrue(d >= a) + self.assertTrue(d <= b) + self.assertTrue(d <= c) + self.assertTrue(d >= d) + self.assertTrue(d <= d) + + def test_mx_value(self): + a = MxValue({'preference': 0, 'priority': 'a', 'exchange': 'v', + 'value': '1'}) + b = MxValue({'preference': 10, 'priority': 'a', 'exchange': 'v', + 'value': '2'}) + c = MxValue({'preference': 0, 'priority': 'b', 'exchange': 'z', + 'value': '3'}) + + self.assertEqual(a, a) + self.assertEqual(b, b) + self.assertEqual(c, c) + + self.assertNotEqual(a, b) + self.assertNotEqual(a, c) + self.assertNotEqual(b, a) + self.assertNotEqual(b, c) + self.assertNotEqual(c, a) + self.assertNotEqual(c, b) + + self.assertTrue(a < b) + self.assertTrue(a < c) + + self.assertTrue(b > a) + self.assertTrue(b > c) + + self.assertTrue(c > a) + self.assertTrue(c < b) + + self.assertTrue(a <= b) + self.assertTrue(a <= c) + self.assertTrue(a <= a) + self.assertTrue(a >= a) + + self.assertTrue(b >= a) + self.assertTrue(b >= c) + self.assertTrue(b >= b) + self.assertTrue(b <= b) + + self.assertTrue(c >= a) + self.assertTrue(c <= b) + self.assertTrue(c >= c) + self.assertTrue(c <= c) + + def test_sshfp_value(self): + a = SshfpValue({'algorithm': 0, 'fingerprint_type': 0, + 'fingerprint': 'abcd'}) + b = SshfpValue({'algorithm': 1, 'fingerprint_type': 0, + 'fingerprint': 'abcd'}) + c = SshfpValue({'algorithm': 0, 'fingerprint_type': 1, + 'fingerprint': 'abcd'}) + d = SshfpValue({'algorithm': 0, 'fingerprint_type': 0, + 'fingerprint': 'bcde'}) + + self.assertEqual(a, a) + self.assertEqual(b, b) + self.assertEqual(c, c) + self.assertEqual(d, d) + + self.assertNotEqual(a, b) + self.assertNotEqual(a, c) + self.assertNotEqual(a, d) + self.assertNotEqual(b, a) + self.assertNotEqual(b, c) + self.assertNotEqual(b, d) + self.assertNotEqual(c, a) + self.assertNotEqual(c, b) + self.assertNotEqual(c, d) + self.assertNotEqual(d, a) + self.assertNotEqual(d, b) + self.assertNotEqual(d, c) + + self.assertTrue(a < b) + self.assertTrue(a < c) + + self.assertTrue(b > a) + self.assertTrue(b > c) + + self.assertTrue(c > a) + self.assertTrue(c < b) + + self.assertTrue(a <= b) + self.assertTrue(a <= c) + self.assertTrue(a <= a) + self.assertTrue(a >= a) + + self.assertTrue(b >= a) + self.assertTrue(b >= c) + self.assertTrue(b >= b) + self.assertTrue(b <= b) + + self.assertTrue(c >= a) + self.assertTrue(c <= b) + self.assertTrue(c >= c) + self.assertTrue(c <= c) + + def test_srv_value(self): + a = SrvValue({'priority': 0, 'weight': 0, 'port': 0, 'target': 'foo.'}) + b = SrvValue({'priority': 1, 'weight': 0, 'port': 0, 'target': 'foo.'}) + c = SrvValue({'priority': 0, 'weight': 2, 'port': 0, 'target': 'foo.'}) + d = SrvValue({'priority': 0, 'weight': 0, 'port': 3, 'target': 'foo.'}) + e = SrvValue({'priority': 0, 'weight': 0, 'port': 0, 'target': 'mmm.'}) + + self.assertEqual(a, a) + self.assertEqual(b, b) + self.assertEqual(c, c) + self.assertEqual(d, d) + self.assertEqual(e, e) + + self.assertNotEqual(a, b) + self.assertNotEqual(a, c) + self.assertNotEqual(a, d) + self.assertNotEqual(a, e) + self.assertNotEqual(b, a) + self.assertNotEqual(b, c) + self.assertNotEqual(b, d) + self.assertNotEqual(b, e) + self.assertNotEqual(c, a) + self.assertNotEqual(c, b) + self.assertNotEqual(c, d) + self.assertNotEqual(c, e) + self.assertNotEqual(d, a) + self.assertNotEqual(d, b) + self.assertNotEqual(d, c) + self.assertNotEqual(d, e) + self.assertNotEqual(e, a) + self.assertNotEqual(e, b) + self.assertNotEqual(e, c) + self.assertNotEqual(e, d) + + self.assertTrue(a < b) + self.assertTrue(a < c) + + self.assertTrue(b > a) + self.assertTrue(b > c) + + self.assertTrue(c > a) + self.assertTrue(c < b) + + self.assertTrue(a <= b) + self.assertTrue(a <= c) + self.assertTrue(a <= a) + self.assertTrue(a >= a) + + self.assertTrue(b >= a) + self.assertTrue(b >= c) + self.assertTrue(b >= b) + self.assertTrue(b <= b) + + self.assertTrue(c >= a) + self.assertTrue(c <= b) + self.assertTrue(c >= c) + self.assertTrue(c <= c) + class TestRecordValidation(TestCase): zone = Zone('unit.tests.', []) From ce67824015ae25914e185309931c0346dcf7c1bd Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 29 Jul 2019 08:48:17 -0700 Subject: [PATCH 096/176] Handle python3 sourcing of urlparse --- tests/test_octodns_provider_rackspace.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index fbd7dac..b257166 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -9,7 +9,10 @@ import json import re from six import text_type from unittest import TestCase -from urlparse import urlparse +try: + from urlparse import urlparse +except ImportError: + from urllib.parse import urlparse from requests import HTTPError from requests_mock import ANY, mock as requests_mock From a85d4e25eda59d59ebc320b1a6597ac9d2848f1a Mon Sep 17 00:00:00 2001 From: Basir Date: Mon, 29 Jul 2019 13:03:23 -0400 Subject: [PATCH 097/176] fastdns: refactored AkamaiClientNotFound exception class --- octodns/provider/fastdns.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index fb199d3..bf19953 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -15,19 +15,10 @@ from ..record import Record from .base import BaseProvider -class AkamaiClientException(Exception): - - _errorMessages = { - 404: "404: Resource not found" - } +class AkamaiClientNotFound(Exception): def __init__(self, resp): - try: - message = self._errorMessages[resp.status_code] - - except KeyError: - resp.raise_for_status() - + message = "404: Resource not found" super(AkamaiClientException, self).__init__(message) From 93e1e0939d5cb3d8218bb11cc3fd62cfda9f83ca Mon Sep 17 00:00:00 2001 From: Basir Date: Mon, 29 Jul 2019 13:07:04 -0400 Subject: [PATCH 098/176] fastdns: fixed how exceptions are handled in _request --- octodns/provider/fastdns.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index bf19953..6e9a00d 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -51,8 +51,9 @@ class AkamaiClient(object): url = urljoin(self.base, path) resp = self._sess.request(method, url, params=params, json=data) - if resp.status_code > 299: - raise AkamaiClientException(resp) + if resp.status_code == 404: + raise AkamaiClientNotFound(resp) + resp.raise_for_status() return resp From 55210ebb05cd4e9077fa324dd2619de83e6cb046 Mon Sep 17 00:00:00 2001 From: Basir Date: Mon, 29 Jul 2019 13:09:18 -0400 Subject: [PATCH 099/176] fastdns: fixed naming in AkamaiClient init --- octodns/provider/fastdns.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index 6e9a00d..ab80882 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -34,15 +34,15 @@ class AkamaiClient(object): ''' - def __init__(self, _client_secret, _host, _access_token, _client_token): + def __init__(self, client_secret, host, access_token, client_token): - self.base = "https://" + _host + "/config-dns/v2/" + self.base = "https://" + host + "/config-dns/v2/" sess = Session() sess.auth = EdgeGridAuth( - client_token=_client_token, - client_secret=_client_secret, - access_token=_access_token + client_token=client_token, + client_secret=client_secret, + access_token=access_token ) self._sess = sess From b2bd91a998aa5a3e7c046d9a32eea3b505071109 Mon Sep 17 00:00:00 2001 From: Basir Date: Mon, 29 Jul 2019 13:25:26 -0400 Subject: [PATCH 100/176] fastdns: resolved naming mismatch for exception class --- octodns/provider/fastdns.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index ab80882..f851303 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -19,7 +19,7 @@ class AkamaiClientNotFound(Exception): def __init__(self, resp): message = "404: Resource not found" - super(AkamaiClientException, self).__init__(message) + super(AkamaiClientNotFound, self).__init__(message) class AkamaiClient(object): @@ -187,7 +187,7 @@ class AkamaiProvider(BaseProvider): response = self._dns_client.zone_recordset_get(name) self._zone_records[zone.name] = response.json()["recordsets"] - except (AkamaiClientException, KeyError): + except (AkamaiClientNotFound, KeyError): return [] return self._zone_records[zone.name] @@ -231,7 +231,7 @@ class AkamaiProvider(BaseProvider): try: self._dns_client.zone_get(zone_name) - except AkamaiClientException: + except AkamaiClientNotFound: self.log.info("zone not found, creating zone") params = self._build_zone_config(zone_name) self._dns_client.zone_create(self._contractId, params, self._gid) From 76a9d358e82bb65a3a1cad83cabd5f79325c3b2f Mon Sep 17 00:00:00 2001 From: Shnayder Anton <41984158+TruePack@users.noreply.github.com> Date: Thu, 1 Aug 2019 12:53:25 +0300 Subject: [PATCH 101/176] Add more explicit constant name for pagination Co-Authored-By: Ross McFarland --- octodns/provider/selectel.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/octodns/provider/selectel.py b/octodns/provider/selectel.py index 0c576a4..bfbbb8f 100644 --- a/octodns/provider/selectel.py +++ b/octodns/provider/selectel.py @@ -28,7 +28,7 @@ class SelectelProvider(BaseProvider): MIN_TTL = 60 - LIMIT = 50 + PAGINATION_LIMIT = 50 API_URL = 'https://api.selectel.ru/domains/v1' @@ -69,9 +69,9 @@ class SelectelProvider(BaseProvider): def _request_with_pagination(self, path, total_count): result = [] - for offset in range(0, total_count, self.LIMIT): + for offset in range(0, total_count, self.PAGINATION_LIMIT): result += self._request('GET', path, - params={'limit': self.LIMIT, + params={'limit': self.PAGINATION_LIMIT, 'offset': offset}) return result From 74a87d4933e021f056a38382488881731504c2c2 Mon Sep 17 00:00:00 2001 From: Anton Shnayder Date: Thu, 1 Aug 2019 13:33:53 +0300 Subject: [PATCH 102/176] Add debug for _include_changes --- octodns/provider/selectel.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/octodns/provider/selectel.py b/octodns/provider/selectel.py index bfbbb8f..072b8cf 100644 --- a/octodns/provider/selectel.py +++ b/octodns/provider/selectel.py @@ -81,6 +81,8 @@ class SelectelProvider(BaseProvider): new = change.new.data new['ttl'] = max(self.MIN_TTL, new['ttl']) if new == existing: + self.log.debug('_include_changes: new=%s, found existing=%s', + new, existing) return False return True From c0315d25baca80e967a0e14af2bbd267adb98e09 Mon Sep 17 00:00:00 2001 From: Adam Smith Date: Mon, 29 Jul 2019 23:12:27 -0700 Subject: [PATCH 103/176] add Constellix Provider --- README.md | 1 + octodns/provider/constellix.py | 447 ++++++++++++++++ tests/fixtures/constellix-domains.json | 28 + tests/fixtures/constellix-records.json | 598 ++++++++++++++++++++++ tests/test_octodns_provider_constellix.py | 218 ++++++++ 5 files changed, 1292 insertions(+) create mode 100644 octodns/provider/constellix.py create mode 100644 tests/fixtures/constellix-domains.json create mode 100644 tests/fixtures/constellix-records.json create mode 100644 tests/test_octodns_provider_constellix.py diff --git a/README.md b/README.md index 737ba9b..75a006c 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,7 @@ The above command pulled the existing data out of Route53 and placed the results |--|--|--|--|--| | [AzureProvider](/octodns/provider/azuredns.py) | azure-mgmt-dns | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | No | | | [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, MX, NS, SPF, SRV, TXT | No | CAA tags restricted | +| [ConstellixProvider](/octodns/provider/constellix.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [DigitalOceanProvider](/octodns/provider/digitalocean.py) | | A, AAAA, CAA, CNAME, MX, NS, TXT, SRV | No | CAA tags restricted | | [DnsMadeEasyProvider](/octodns/provider/dnsmadeeasy.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [DnsimpleProvider](/octodns/provider/dnsimple.py) | | All | No | CAA tags restricted | diff --git a/octodns/provider/constellix.py b/octodns/provider/constellix.py new file mode 100644 index 0000000..939284d --- /dev/null +++ b/octodns/provider/constellix.py @@ -0,0 +1,447 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from collections import defaultdict +from requests import Session +from base64 import b64encode +from ipaddress import ip_address +import hashlib +import hmac +import logging +import time + +from ..record import Record +from .base import BaseProvider + + +class ConstellixClientException(Exception): + pass + + +class ConstellixClientBadRequest(ConstellixClientException): + + def __init__(self, resp): + errors = resp.json()['errors'] + super(ConstellixClientBadRequest, self).__init__( + '\n - {}'.format('\n - '.join(errors))) + + +class ConstellixClientUnauthorized(ConstellixClientException): + + def __init__(self): + super(ConstellixClientUnauthorized, self).__init__('Unauthorized') + + +class ConstellixClientNotFound(ConstellixClientException): + + def __init__(self): + super(ConstellixClientNotFound, self).__init__('Not Found') + + +class ConstellixClient(object): + BASE = 'https://api.dns.constellix.com/v1/domains' + + def __init__(self, api_key, secret_key, ratelimit_delay=0.0): + self.api_key = api_key + self.secret_key = secret_key + self.ratelimit_delay = ratelimit_delay + self._sess = Session() + self._sess.headers.update({'x-cnsdns-apiKey': self.api_key}) + self._domains = None + + def _current_time(self): + return str(int(time.time() * 1000)) + + def _hmac_hash(self, now): + return hmac.new(self.secret_key.encode('utf-8'), now.encode('utf-8'), + digestmod=hashlib.sha1).digest() + + def _request(self, method, path, params=None, data=None): + now = self._current_time() + hmac_hash = self._hmac_hash(now) + + headers = { + 'x-cnsdns-hmac': b64encode(hmac_hash), + 'x-cnsdns-requestDate': now + } + + url = '{}{}'.format(self.BASE, path) + resp = self._sess.request(method, url, headers=headers, + params=params, json=data) + if resp.status_code == 400: + raise ConstellixClientBadRequest(resp) + if resp.status_code == 401: + raise ConstellixClientUnauthorized() + if resp.status_code == 404: + raise ConstellixClientNotFound() + resp.raise_for_status() + time.sleep(self.ratelimit_delay) + return resp + + @property + def domains(self): + if self._domains is None: + zones = [] + + resp = self._request('GET', '/').json() + zones += resp + + self._domains = {'{}.'.format(z['name']): z['id'] for z in zones} + + return self._domains + + def domain(self, name): + path = '/{}'.format(self.domains.get(name)) + return self._request('GET', path).json() + + def domain_create(self, name): + self._request('POST', '/', data={'names': [name]}) + + def _absolutize_value(self, value, zone_name): + if value == '': + value = zone_name + elif not value.endswith('.'): + value = '{}.{}'.format(value, zone_name) + + return value + + def records(self, zone_name): + zone_id = self.domains.get(zone_name, False) + path = '/{}/records'.format(zone_id) + + resp = self._request('GET', path).json() + for record in resp: + # change ANAME records to ALIAS + if record['type'] == 'ANAME': + record['type'] = 'ALIAS' + + # change relative values to absolute + value = record['value'] + if record['type'] in ['ALIAS', 'CNAME', 'MX', 'NS', 'SRV']: + if isinstance(value, unicode): + record['value'] = self._absolutize_value(value, + zone_name) + if isinstance(value, list): + for v in value: + v['value'] = self._absolutize_value(v['value'], + zone_name) + + # compress IPv6 addresses + if record['type'] == 'AAAA': + for i, v in enumerate(value): + value[i] = str(ip_address(v)) + + return resp + + def record_create(self, zone_name, record_type, params): + # change ALIAS records to ANAME + if record_type == 'ALIAS': + record_type = 'ANAME' + + zone_id = self.domains.get(zone_name, False) + path = '/{}/records/{}'.format(zone_id, record_type) + + self._request('POST', path, data=params) + + def record_delete(self, zone_name, record_type, record_id): + zone_id = self.domains.get(zone_name, False) + path = '/{}/records/{}/{}'.format(zone_id, record_type, record_id) + self._request('DELETE', path) + + +class ConstellixProvider(BaseProvider): + ''' + Constellix DNS provider + + constellix: + class: octodns.provider.constellix.ConstellixProvider + # Your Contellix api key (required) + api_key: env/CONSTELLIX_API_KEY + # Your Constellix secret key (required) + secret_key: env/CONSTELLIX_SECRET_KEY + # Amount of time to wait between requests to avoid + # ratelimit (optional) + ratelimit_delay: 0.0 + ''' + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = False + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', + 'NS', 'PTR', 'SPF', 'SRV', 'TXT')) + + def __init__(self, id, api_key, secret_key, ratelimit_delay=0.0, + *args, **kwargs): + self.log = logging.getLogger('ConstellixProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, api_key=***, secret_key=***', id) + super(ConstellixProvider, self).__init__(id, *args, **kwargs) + self._client = ConstellixClient(api_key, secret_key, ratelimit_delay) + self._zone_records = {} + + def _data_for_multiple(self, _type, records): + record = records[0] + return { + 'ttl': record['ttl'], + 'type': _type, + 'values': record['value'] + } + + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + + def _data_for_CAA(self, _type, records): + values = [] + record = records[0] + for value in record['value']: + values.append({ + 'flags': value['flag'], + 'tag': value['tag'], + 'value': value['data'] + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + def _data_for_NS(self, _type, records): + record = records[0] + return { + 'ttl': record['ttl'], + 'type': _type, + 'values': [value['value'] for value in record['value']] + } + + def _data_for_ALIAS(self, _type, records): + record = records[0] + return { + 'ttl': record['ttl'], + 'type': _type, + 'value': record['value'][0]['value'] + } + + _data_for_PTR = _data_for_ALIAS + + def _data_for_TXT(self, _type, records): + values = [value['value'].replace(';', '\\;') + for value in records[0]['value']] + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + _data_for_SPF = _data_for_TXT + + def _data_for_MX(self, _type, records): + values = [] + record = records[0] + for value in record['value']: + values.append({ + 'preference': value['level'], + 'exchange': value['value'] + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + def _data_for_single(self, _type, records): + record = records[0] + return { + 'ttl': record['ttl'], + 'type': _type, + 'value': record['value'] + } + + _data_for_CNAME = _data_for_single + + def _data_for_SRV(self, _type, records): + values = [] + record = records[0] + for value in record['value']: + values.append({ + 'port': value['port'], + 'priority': value['priority'], + 'target': value['value'], + 'weight': value['weight'] + }) + return { + 'type': _type, + 'ttl': records[0]['ttl'], + 'values': values + } + + def zone_records(self, zone): + if zone.name not in self._zone_records: + try: + self._zone_records[zone.name] = \ + self._client.records(zone.name) + except ConstellixClientNotFound: + return [] + + return self._zone_records[zone.name] + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + + values = defaultdict(lambda: defaultdict(list)) + for record in self.zone_records(zone): + _type = record['type'] + if _type not in self.SUPPORTS: + self.log.warning('populate: skipping unsupported %s record', + _type) + continue + values[record['name']][record['type']].append(record) + + before = len(zone.records) + for name, types in values.items(): + for _type, records in types.items(): + data_for = getattr(self, '_data_for_{}'.format(_type)) + record = Record.new(zone, name, data_for(_type, records), + source=self, lenient=lenient) + zone.add_record(record, lenient=lenient) + + exists = zone.name in self._zone_records + self.log.info('populate: found %s records, exists=%s', + len(zone.records) - before, exists) + return exists + + def _params_for_multiple(self, record): + yield { + 'name': record.name, + 'ttl': record.ttl, + 'roundRobin': [{ + 'value': value + } for value in record.values] + } + + _params_for_A = _params_for_multiple + _params_for_AAAA = _params_for_multiple + + # An A record with this name must exist in this domain for + # this NS record to be valid. Need to handle checking if + # there is an A record before creating NS + _params_for_NS = _params_for_multiple + + def _params_for_single(self, record): + yield { + 'name': record.name, + 'ttl': record.ttl, + 'host': record.value, + } + + _params_for_CNAME = _params_for_single + + def _params_for_ALIAS(self, record): + yield { + 'name': record.name, + 'ttl': record.ttl, + 'roundRobin': [{ + 'value': record.value, + 'disableFlag': False + }] + } + + _params_for_PTR = _params_for_ALIAS + + def _params_for_MX(self, record): + values = [] + for value in record.values: + values.append({ + 'value': value.exchange, + 'level': value.preference + }) + yield { + 'value': value.exchange, + 'name': record.name, + 'ttl': record.ttl, + 'roundRobin': values + } + + def _params_for_SRV(self, record): + values = [] + for value in record.values: + values.append({ + 'value': value.target, + 'priority': value.priority, + 'weight': value.weight, + 'port': value.port + }) + for value in record.values: + yield { + 'name': record.name, + 'ttl': record.ttl, + 'roundRobin': values + } + + def _params_for_TXT(self, record): + # Constellix does not want values escaped + values = [] + for value in record.chunked_values: + values.append({ + 'value': value.replace('\\;', ';') + }) + yield { + 'name': record.name, + 'ttl': record.ttl, + 'roundRobin': values + } + + _params_for_SPF = _params_for_TXT + + def _params_for_CAA(self, record): + values = [] + for value in record.values: + values.append({ + 'tag': value.tag, + 'data': value.value, + 'flag': value.flags, + }) + yield { + 'name': record.name, + 'ttl': record.ttl, + 'roundRobin': values + } + + def _apply_Create(self, change): + new = change.new + params_for = getattr(self, '_params_for_{}'.format(new._type)) + for params in params_for(new): + self._client.record_create(new.zone.name, new._type, params) + + def _apply_Update(self, change): + self._apply_Delete(change) + self._apply_Create(change) + + def _apply_Delete(self, change): + existing = change.existing + zone = existing.zone + for record in self.zone_records(zone): + if existing.name == record['name'] and \ + existing._type == record['type']: + self._client.record_delete(zone.name, record['type'], + record['id']) + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + + try: + self._client.domain(desired.name) + except ConstellixClientNotFound: + self.log.debug('_apply: no matching zone, creating domain') + self._client.domain_create(desired.name[:-1]) + + for change in changes: + class_name = change.__class__.__name__ + getattr(self, '_apply_{}'.format(class_name))(change) + + # Clear out the cache if any + self._zone_records.pop(desired.name, None) diff --git a/tests/fixtures/constellix-domains.json b/tests/fixtures/constellix-domains.json new file mode 100644 index 0000000..4b6392d --- /dev/null +++ b/tests/fixtures/constellix-domains.json @@ -0,0 +1,28 @@ +[{ + "id": 123123, + "name": "unit.tests", + "soa": { + "primaryNameserver": "ns11.constellix.com.", + "email": "dns.constellix.com.", + "ttl": 86400, + "serial": 2015010102, + "refresh": 43200, + "retry": 3600, + "expire": 1209600, + "negCache": 180 + }, + "createdTs": "2019-08-07T03:36:02Z", + "modifiedTs": "2019-08-07T03:36:02Z", + "typeId": 1, + "domainTags": [], + "folder": null, + "hasGtdRegions": false, + "hasGeoIP": false, + "nameserverGroup": 1, + "nameservers": ["ns11.constellix.com.", "ns21.constellix.com.", "ns31.constellix.com.", "ns41.constellix.net.", "ns51.constellix.net.", "ns61.constellix.net."], + "note": "", + "version": 0, + "status": "ACTIVE", + "tags": [], + "contactIds": [] +}] diff --git a/tests/fixtures/constellix-records.json b/tests/fixtures/constellix-records.json new file mode 100644 index 0000000..c1f1fb4 --- /dev/null +++ b/tests/fixtures/constellix-records.json @@ -0,0 +1,598 @@ +[{ + "id": 1808529, + "type": "CAA", + "recordType": "caa", + "name": "", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 3600, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565149569216, + "value": [{ + "flag": 0, + "tag": "issue", + "data": "ca.unit.tests", + "caaProviderId": 1, + "disableFlag": false + }], + "roundRobin": [{ + "flag": 0, + "tag": "issue", + "data": "ca.unit.tests", + "caaProviderId": 1, + "disableFlag": false + }] +}, { + "id": 1808516, + "type": "A", + "recordType": "a", + "name": "", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 300, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565149623640, + "value": ["1.2.3.4", "1.2.3.5"], + "roundRobin": [{ + "value": "1.2.3.4", + "disableFlag": false + }, { + "value": "1.2.3.5", + "disableFlag": false + }], + "geolocation": null, + "recordFailover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "failover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "roundRobinFailover": [], + "pools": [], + "poolsDetail": [] +}, { + "id": 1808527, + "type": "SRV", + "recordType": "srv", + "name": "_srv._tcp", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 600, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565149714387, + "value": [{ + "value": "foo-1.unit.tests.", + "priority": 10, + "weight": 20, + "port": 30, + "disableFlag": false + }, { + "value": "foo-2.unit.tests.", + "priority": 12, + "weight": 20, + "port": 30, + "disableFlag": false + }], + "roundRobin": [{ + "value": "foo-1.unit.tests.", + "priority": 10, + "weight": 20, + "port": 30, + "disableFlag": false + }, { + "value": "foo-2.unit.tests.", + "priority": 12, + "weight": 20, + "port": 30, + "disableFlag": false + }] +}, { + "id": 1808515, + "type": "AAAA", + "recordType": "aaaa", + "name": "aaaa", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 600, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565149739464, + "value": ["2601:644:500:e210:62f8:1dff:feb8:947a"], + "roundRobin": [{ + "value": "2601:644:500:e210:62f8:1dff:feb8:947a", + "disableFlag": false + }], + "geolocation": null, + "recordFailover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "failover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "pools": [], + "poolsDetail": [], + "roundRobinFailover": [] +}, { + "id": 1808530, + "type": "ANAME", + "recordType": "aname", + "name": "", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 1800, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565150251379, + "value": [{ + "value": "aname.unit.tests.", + "disableFlag": false + }], + "roundRobin": [{ + "value": "aname.unit.tests.", + "disableFlag": false + }], + "geolocation": null, + "recordFailover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "failover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "pools": [], + "poolsDetail": [] +}, { + "id": 1808521, + "type": "CNAME", + "recordType": "cname", + "name": "cname", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 300, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565152113825, + "value": "", + "roundRobin": [{ + "value": "", + "disableFlag": false + }], + "recordFailover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [{ + "id": null, + "value": "", + "disableFlag": false, + "failedFlag": false, + "status": "N/A", + "sortOrder": 1, + "markedActive": false + }, { + "id": null, + "value": "", + "disableFlag": false, + "failedFlag": false, + "status": "N/A", + "sortOrder": 2, + "markedActive": false + }] + }, + "failover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [{ + "id": null, + "value": "", + "disableFlag": false, + "failedFlag": false, + "status": "N/A", + "sortOrder": 1, + "markedActive": false + }, { + "id": null, + "value": "", + "disableFlag": false, + "failedFlag": false, + "status": "N/A", + "sortOrder": 2, + "markedActive": false + }] + }, + "pools": [], + "poolsDetail": [], + "geolocation": null, + "host": "" +}, { + "id": 1808522, + "type": "CNAME", + "recordType": "cname", + "name": "included", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 3600, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565152119137, + "value": "", + "roundRobin": [{ + "value": "", + "disableFlag": false + }], + "recordFailover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [{ + "id": null, + "value": "", + "disableFlag": false, + "failedFlag": false, + "status": "N/A", + "sortOrder": 1, + "markedActive": false + }, { + "id": null, + "value": "", + "disableFlag": false, + "failedFlag": false, + "status": "N/A", + "sortOrder": 2, + "markedActive": false + }] + }, + "failover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [{ + "id": null, + "value": "", + "disableFlag": false, + "failedFlag": false, + "status": "N/A", + "sortOrder": 1, + "markedActive": false + }, { + "id": null, + "value": "", + "disableFlag": false, + "failedFlag": false, + "status": "N/A", + "sortOrder": 2, + "markedActive": false + }] + }, + "pools": [], + "poolsDetail": [], + "geolocation": null, + "host": "" +}, { + "id": 1808523, + "type": "MX", + "recordType": "mx", + "name": "mx", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 300, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565149879856, + "value": [{ + "value": "smtp-3.unit.tests.", + "level": 30, + "disableFlag": false + }, { + "value": "smtp-2.unit.tests.", + "level": 20, + "disableFlag": false + }, { + "value": "smtp-4.unit.tests.", + "level": 10, + "disableFlag": false + }, { + "value": "smtp-1.unit.tests.", + "level": 40, + "disableFlag": false + }], + "roundRobin": [{ + "value": "smtp-3.unit.tests.", + "level": 30, + "disableFlag": false + }, { + "value": "smtp-2.unit.tests.", + "level": 20, + "disableFlag": false + }, { + "value": "smtp-4.unit.tests.", + "level": 10, + "disableFlag": false + }, { + "value": "smtp-1.unit.tests.", + "level": 40, + "disableFlag": false + }] +}, { + "id": 1808525, + "type": "PTR", + "recordType": "ptr", + "name": "ptr", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 300, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565150115139, + "value": [{ + "value": "foo.bar.com.", + "disableFlag": false + }], + "roundRobin": [{ + "value": "foo.bar.com.", + "disableFlag": false + }] +}, { + "id": 1808526, + "type": "SPF", + "recordType": "spf", + "name": "spf", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 600, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565149916132, + "value": [{ + "value": "\"v=spf1 ip4:192.168.0.1/16-all\"", + "disableFlag": false + }], + "roundRobin": [{ + "value": "\"v=spf1 ip4:192.168.0.1/16-all\"", + "disableFlag": false + }] +}, { + "id": 1808528, + "type": "TXT", + "recordType": "txt", + "name": "txt", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 600, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565149966915, + "value": [{ + "value": "\"Bah bah black sheep\"", + "disableFlag": false + }, { + "value": "\"have you any wool.\"", + "disableFlag": false + }, { + "value": "\"v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs\"", + "disableFlag": false + }], + "roundRobin": [{ + "value": "\"Bah bah black sheep\"", + "disableFlag": false + }, { + "value": "\"have you any wool.\"", + "disableFlag": false + }, { + "value": "\"v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs\"", + "disableFlag": false + }] +}, { + "id": 1808524, + "type": "NS", + "recordType": "ns", + "name": "under", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 3600, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565150062850, + "value": [{ + "value": "ns1.unit.tests.", + "disableFlag": false + }, { + "value": "ns2", + "disableFlag": false + }], + "roundRobin": [{ + "value": "ns1.unit.tests.", + "disableFlag": false + }, { + "value": "ns2", + "disableFlag": false + }] +}, { + "id": 1808531, + "type": "HTTPRedirection", + "recordType": "httpredirection", + "name": "unsupported", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 300, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565150348154, + "value": "https://redirect.unit.tests", + "roundRobin": [{ + "value": "https://redirect.unit.tests" + }], + "title": "Unsupported Record", + "keywords": "unsupported", + "description": "unsupported record", + "hardlinkFlag": false, + "redirectTypeId": 1, + "url": "https://redirect.unit.tests" +}, { + "id": 1808519, + "type": "A", + "recordType": "a", + "name": "www", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 300, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565150079027, + "value": ["2.2.3.6"], + "roundRobin": [{ + "value": "2.2.3.6", + "disableFlag": false + }], + "geolocation": null, + "recordFailover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "failover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "roundRobinFailover": [], + "pools": [], + "poolsDetail": [] +}, { + "id": 1808603, + "type": "ANAME", + "recordType": "aname", + "name": "sub", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 1800, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565153387855, + "value": [{ + "value": "aname.unit.tests.", + "disableFlag": false + }], + "roundRobin": [{ + "value": "aname.unit.tests.", + "disableFlag": false + }], + "geolocation": null, + "recordFailover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "failover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "pools": [], + "poolsDetail": [] +}, { + "id": 1808520, + "type": "A", + "recordType": "a", + "name": "www.sub", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 300, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565150090588, + "value": ["2.2.3.6"], + "roundRobin": [{ + "value": "2.2.3.6", + "disableFlag": false + }], + "geolocation": null, + "recordFailover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "failover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "roundRobinFailover": [], + "pools": [], + "poolsDetail": [] +}] diff --git a/tests/test_octodns_provider_constellix.py b/tests/test_octodns_provider_constellix.py new file mode 100644 index 0000000..346bb17 --- /dev/null +++ b/tests/test_octodns_provider_constellix.py @@ -0,0 +1,218 @@ +# +# +# + + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from mock import Mock, call +from os.path import dirname, join +from requests import HTTPError +from requests_mock import ANY, mock as requests_mock +from unittest import TestCase + +from octodns.record import Record +from octodns.provider.constellix import ConstellixClientNotFound, \ + ConstellixProvider +from octodns.provider.yaml import YamlProvider +from octodns.zone import Zone + +import json + + +class TestConstellixProvider(TestCase): + expected = Zone('unit.tests.', []) + source = YamlProvider('test', join(dirname(__file__), 'config')) + source.populate(expected) + + # Our test suite differs a bit, add our NS and remove the simple one + expected.add_record(Record.new(expected, 'under', { + 'ttl': 3600, + 'type': 'NS', + 'values': [ + 'ns1.unit.tests.', + 'ns2.unit.tests.', + ] + })) + + # Add some ALIAS records + expected.add_record(Record.new(expected, '', { + 'ttl': 1800, + 'type': 'ALIAS', + 'value': 'aname.unit.tests.' + })) + + expected.add_record(Record.new(expected, 'sub', { + 'ttl': 1800, + 'type': 'ALIAS', + 'value': 'aname.unit.tests.' + })) + + for record in list(expected.records): + if record.name == 'sub' and record._type == 'NS': + expected._remove_record(record) + break + + def test_populate(self): + provider = ConstellixProvider('test', 'api', 'secret') + + # Bad auth + with requests_mock() as mock: + mock.get(ANY, status_code=401, + text='{"errors": ["Unable to authenticate token"]}') + + with self.assertRaises(Exception) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals('Unauthorized', ctx.exception.message) + + # Bad request + with requests_mock() as mock: + mock.get(ANY, status_code=400, + text='{"errors": ["\\"unittests\\" is not ' + 'a valid domain name"]}') + + with self.assertRaises(Exception) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals('\n - "unittests" is not a valid domain name', + ctx.exception.message) + + # General error + with requests_mock() as mock: + mock.get(ANY, status_code=502, text='Things caught fire') + + with self.assertRaises(HTTPError) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(502, ctx.exception.response.status_code) + + # Non-existent zone doesn't populate anything + with requests_mock() as mock: + mock.get(ANY, status_code=404, + text='') + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(set(), zone.records) + + # No diffs == no changes + with requests_mock() as mock: + base = 'https://api.dns.constellix.com/v1/domains' + with open('tests/fixtures/constellix-domains.json') as fh: + mock.get('{}{}'.format(base, '/'), text=fh.read()) + with open('tests/fixtures/constellix-records.json') as fh: + mock.get('{}{}'.format(base, '/123123/records'), + text=fh.read()) + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(15, len(zone.records)) + changes = self.expected.changes(zone, provider) + self.assertEquals(0, len(changes)) + + # 2nd populate makes no network calls/all from cache + again = Zone('unit.tests.', []) + provider.populate(again) + self.assertEquals(15, len(again.records)) + + # bust the cache + del provider._zone_records[zone.name] + + def test_apply(self): + provider = ConstellixProvider('test', 'api', 'secret') + + resp = Mock() + resp.json = Mock() + provider._client._request = Mock(return_value=resp) + + with open('tests/fixtures/constellix-domains.json') as fh: + domains = json.load(fh) + + # non-existent domain, create everything + resp.json.side_effect = [ + ConstellixClientNotFound, # no zone in populate + ConstellixClientNotFound, # no domain during apply + domains + ] + plan = provider.plan(self.expected) + + # No root NS, no ignored, no excluded, no unsupported + n = len(self.expected.records) - 5 + self.assertEquals(n, len(plan.changes)) + self.assertEquals(n, provider.apply(plan)) + + provider._client._request.assert_has_calls([ + # created the domain + call('POST', '/', data={'names': ['unit.tests']}), + # get all domains to build the cache + call('GET', '/'), + call('POST', '/123123/records/SRV', data={ + 'roundRobin': [{ + 'priority': 10, + 'weight': 20, + 'value': 'foo-1.unit.tests.', + 'port': 30 + }, { + 'priority': 12, + 'weight': 20, + 'value': 'foo-2.unit.tests.', + 'port': 30 + }], + 'name': '_srv._tcp', + 'ttl': 600, + }), + ]) + + self.assertEquals(20, provider._client._request.call_count) + + provider._client._request.reset_mock() + + provider._client.records = Mock(return_value=[ + { + 'id': 11189897, + 'type': 'A', + 'name': 'www', + 'ttl': 300, + 'value': [ + '1.2.3.4', + '2.2.3.4', + ] + }, { + 'id': 11189898, + 'type': 'A', + 'name': 'ttl', + 'ttl': 600, + 'value': [ + '3.2.3.4' + ] + } + ]) + + # Domain exists, we don't care about return + resp.json.side_effect = ['{}'] + + wanted = Zone('unit.tests.', []) + wanted.add_record(Record.new(wanted, 'ttl', { + 'ttl': 300, + 'type': 'A', + 'value': '3.2.3.4' + })) + + plan = provider.plan(wanted) + self.assertEquals(2, len(plan.changes)) + self.assertEquals(2, provider.apply(plan)) + + # recreate for update, and deletes for the 2 parts of the other + provider._client._request.assert_has_calls([ + call('POST', '/123123/records/A', data={ + 'roundRobin': [{ + 'value': '3.2.3.4' + }], + 'name': 'ttl', + 'ttl': 300 + }), + call('DELETE', '/123123/records/A/11189897'), + call('DELETE', '/123123/records/A/11189898') + ], any_order=True) From dd4238ad6b008b56c1cc6efbba031df943d9ee0e Mon Sep 17 00:00:00 2001 From: William Blackie Date: Mon, 19 Aug 2019 17:06:23 +1000 Subject: [PATCH 104/176] fix split calls in Route53Provider to not specify ' ' parameter so they can tolerate multiple sequential separators --- octodns/provider/route53.py | 8 ++++---- tests/test_octodns_provider_route53.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 7b21573..81e366a 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -723,7 +723,7 @@ class Route53Provider(BaseProvider): def _data_for_CAA(self, rrset): values = [] for rr in rrset['ResourceRecords']: - flags, tag, value = rr['Value'].split(' ') + flags, tag, value = rr['Value'].split() values.append({ 'flags': flags, 'tag': tag, @@ -761,7 +761,7 @@ class Route53Provider(BaseProvider): def _data_for_MX(self, rrset): values = [] for rr in rrset['ResourceRecords']: - preference, exchange = rr['Value'].split(' ') + preference, exchange = rr['Value'].split() values.append({ 'preference': preference, 'exchange': exchange, @@ -776,7 +776,7 @@ class Route53Provider(BaseProvider): values = [] for rr in rrset['ResourceRecords']: order, preference, flags, service, regexp, replacement = \ - rr['Value'].split(' ') + rr['Value'].split() flags = flags[1:-1] service = service[1:-1] regexp = regexp[1:-1] @@ -804,7 +804,7 @@ class Route53Provider(BaseProvider): def _data_for_SRV(self, rrset): values = [] for rr in rrset['ResourceRecords']: - priority, weight, port, target = rr['Value'].split(' ') + priority, weight, port, target = rr['Value'].split() values.append({ 'priority': priority, 'weight': weight, diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 265a0a7..ceafd13 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -503,7 +503,7 @@ class TestRoute53Provider(TestCase): 'ResourceRecords': [{ 'Value': '10 smtp-1.unit.tests.', }, { - 'Value': '20 smtp-2.unit.tests.', + 'Value': '20 smtp-2.unit.tests.', }], 'TTL': 64, }, { From 7d8f181367d540ddc3f99a5a1340e8bd7e5db6ac Mon Sep 17 00:00:00 2001 From: Arunothia Marappan Date: Thu, 19 Sep 2019 10:48:18 -0700 Subject: [PATCH 105/176] Adding Akamai entry to supported providers --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 75a006c..be0a4ab 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,7 @@ The above command pulled the existing data out of Route53 and placed the results | Provider | Requirements | Record Support | Dynamic/Geo Support | Notes | |--|--|--|--|--| | [AzureProvider](/octodns/provider/azuredns.py) | azure-mgmt-dns | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | No | | +| [Akamai](/octodns/provider/fastdns.py) | edgegrid-python | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | | | [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, MX, NS, SPF, SRV, TXT | No | CAA tags restricted | | [ConstellixProvider](/octodns/provider/constellix.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [DigitalOceanProvider](/octodns/provider/digitalocean.py) | | A, AAAA, CAA, CNAME, MX, NS, TXT, SRV | No | CAA tags restricted | From 736d588e86942a215a01607ce3983c3ea1165395 Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Fri, 20 Sep 2019 10:18:51 +0200 Subject: [PATCH 106/176] Changed requirements to version 2.22.0 Fixes: ERROR: requests 2.20.0 has requirement urllib3<1.25,>=1.21.1, but you'll have urllib3 1.25.5 which is incompatible. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 75dc1df..f67f2ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,7 @@ natsort==5.5.0 nsone==0.9.100 ovh==0.4.8 python-dateutil==2.6.1 -requests==2.20.0 +requests==2.22.0 s3transfer==0.1.13 six==1.11.0 setuptools==38.5.2 From bb3f0c0b4a9c0705aad39cfd5aac3cbca8bfab47 Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 25 Sep 2019 07:01:06 +0200 Subject: [PATCH 107/176] Added TransIP provider and tests --- octodns/provider/transip.py | 328 +++++++++++++++++++++++++ tests/test_octodns_provider_transip.py | 234 ++++++++++++++++++ 2 files changed, 562 insertions(+) create mode 100644 octodns/provider/transip.py create mode 100644 tests/test_octodns_provider_transip.py diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py new file mode 100644 index 0000000..2ce9180 --- /dev/null +++ b/octodns/provider/transip.py @@ -0,0 +1,328 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from suds import WebFault + +from collections import defaultdict +from .base import BaseProvider +from logging import getLogger +from ..record import Record +from transip.service.domain import DomainService +from transip.service.objects import DnsEntry + + +class TransipProvider(BaseProvider): + ''' + Transip DNS provider + + transip: + class: octodns.provider.transip.TransipProvider + # Your Transip account name (required) + account: yourname + # The api key (required) + key: | + \''' + -----BEGIN PRIVATE KEY----- + ... + -----END PRIVATE KEY----- + \''' + + ''' + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = False + SUPPORTS = set( + ('A', 'AAAA', 'CNAME', 'MX', 'SRV', 'SPF', 'TXT', 'SSHFP', 'CAA')) + # unsupported by OctoDNS: 'TLSA', 'CAA' + MIN_TTL = 120 + TIMEOUT = 15 + ROOT_RECORD = '@' + + def __init__(self, id, account, key, *args, **kwargs): + self.log = getLogger('TransipProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, account=%s, token=***', id, + account) + super(TransipProvider, self).__init__(id, *args, **kwargs) + + self._client = DomainService(account, key) + + self.account = account + self.key = key + + self._zones = None + self._zone_records = {} + + self._currentZone = {} + + def populate(self, zone, target=False, lenient=False): + + exists = False + self._currentZone = zone + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + + before = len(zone.records) + try: + zoneInfo = self._client.get_info(zone.name[:-1]) + except WebFault as e: + if e.fault.faultcode == '102' and target == False: + self.log.warning( + 'populate: (%s) Zone %s not found in account ', + e.fault.faultcode, zone.name) + exists = False + return exists + elif e.fault.faultcode == '102' and target == True: + self.log.warning('populate: Transip can\'t create new zones') + raise Exception( + ('populate: ({}) Transip used ' + + 'as target for non-existing zone: {}').format( + e.fault.faultcode, zone.name)) + else: + self.log.error('populate: (%s) %s ', e.fault.faultcode, + e.fault.faultstring) + raise e + + self.log.debug('populate: found %s records for zone %s', + len(zoneInfo.dnsEntries), zone.name) + exists = True + if zoneInfo.dnsEntries: + values = defaultdict(lambda: defaultdict(list)) + for record in zoneInfo.dnsEntries: + name = zone.hostname_from_fqdn(record['name']) + if name == self.ROOT_RECORD: + name = '' + + if record['type'] in self.SUPPORTS: + values[name][record['type']].append(record) + + for name, types in values.items(): + for _type, records in types.items(): + data_for = getattr(self, '_data_for_{}'.format(_type)) + record = Record.new(zone, name, data_for(_type, records), + source=self, lenient=lenient) + zone.add_record(record, lenient=lenient) + self.log.info('populate: found %s records, exists = %s', + len(zone.records) - before, exists) + + self._currentZone = {} + return exists + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('apply: zone=%s, changes=%d', desired.name, + len(changes)) + # for change in changes: + # class_name = change.__class__.__name__ + # getattr(self, '_apply_{}'.format(class_name))(change) + + self._currentZone = plan.desired + try: + self._client.get_info(plan.desired.name[:-1]) + except WebFault as e: + self.log.warning('_apply: %s ', e.message) + raise e + + _dns_entries = [] + for record in plan.desired.records: + if record._type in self.SUPPORTS: + entries_for = getattr(self, '_entries_for_{}'.format(record._type)) + + # Root records have '@' as name + name = record.name + if name == '': + name = self.ROOT_RECORD + + _dns_entries.extend(entries_for(name, record)) + + try: + self._client.set_dns_entries(plan.desired.name[:-1], _dns_entries) + except WebFault as e: + self.log.warning(('_apply: Set DNS returned ' + + 'one or more errors: {}').format( + e.fault.faultstring)) + raise Exception(200, e.fault.faultstring) + + self._currentZone = {} + + def _entries_for_multiple(self, name, record): + _entries = [] + + for value in record.values: + _entries.append(DnsEntry(name, record.ttl, record._type, value)) + + return _entries + + def _entries_for_single(self, name, record): + + return [DnsEntry(name, record.ttl, record._type, record.value)] + + _entries_for_A = _entries_for_multiple + _entries_for_AAAA = _entries_for_multiple + _entries_for_NS = _entries_for_multiple + _entries_for_SPF = _entries_for_multiple + _entries_for_CNAME = _entries_for_single + + def _entries_for_MX(self, name, record): + _entries = [] + + for value in record.values: + content = "{} {}".format(value.preference, value.exchange) + _entries.append(DnsEntry(name, record.ttl, record._type, content)) + + return _entries + + def _entries_for_SRV(self, name, record): + _entries = [] + + for value in record.values: + content = "{} {} {} {}".format(value.priority, value.weight, + value.port, value.target) + _entries.append(DnsEntry(name, record.ttl, record._type, content)) + + return _entries + + def _entries_for_SSHFP(self, name, record): + _entries = [] + + for value in record.values: + content = "{} {} {}".format(value.algorithm, value.fingerprint_type, + value.fingerprint) + _entries.append(DnsEntry(name, record.ttl, record._type, content)) + + return _entries + + def _entries_for_CAA(self, name, record): + _entries = [] + + for value in record.values: + content = "{} {} {}".format(value.flags, value.tag, + value.value) + _entries.append(DnsEntry(name, record.ttl, record._type, content)) + + return _entries + + def _entries_for_TXT(self, name, record): + _entries = [] + + for value in record.values: + value = value.replace('\\;', ';') + _entries.append(DnsEntry(name, record.ttl, record._type, value)) + + return _entries + + def _parse_to_fqdn(self, value): + + if (value[-1] != '.'): + self.log.debug('parseToFQDN: changed %s to %s', value, + '{}.{}'.format(value, self._currentZone.name)) + value = '{}.{}'.format(value, self._currentZone.name) + + return value + + def _get_lowest_ttl(self, records): + _ttl = 100000 + for record in records: + _ttl = min(_ttl, record['expire']) + return _ttl + + def _data_for_multiple(self, _type, records): + + _values = [] + for record in records: + _values.append(record['content']) + + return { + 'ttl': self._get_lowest_ttl(records), + 'type': _type, + 'values': _values + } + + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + _data_for_NS = _data_for_multiple + _data_for_SPF = _data_for_multiple + + def _data_for_CNAME(self, _type, records): + return { + 'ttl': records[0]['expire'], + 'type': _type, + 'value': self._parse_to_fqdn(records[0]['content']) + } + + def _data_for_MX(self, _type, records): + values = [] + for record in records: + preference, exchange = record['content'].split(" ", 1) + values.append({ + 'preference': preference, + 'exchange': self._parse_to_fqdn(exchange) + }) + return { + 'ttl': self._get_lowest_ttl(records), + 'type': _type, + 'values': values + } + + def _data_for_SRV(self, _type, records): + values = [] + for record in records: + priority, weight, port, target = record['content'].split(' ', 3) + values.append({ + 'port': port, + 'priority': priority, + 'target': self._parse_to_fqdn(target), + 'weight': weight + }) + + return { + 'type': _type, + 'ttl': self._get_lowest_ttl(records), + 'values': values + } + + def _data_for_SSHFP(self, _type, records): + values = [] + for record in records: + algorithm, fp_type, fingerprint = record['content'].split(' ', 2) + values.append({ + 'algorithm': algorithm, + 'fingerprint': fingerprint.lower(), + 'fingerprint_type': fp_type + }) + + return { + 'type': _type, + 'ttl': self._get_lowest_ttl(records), + 'values': values + } + + def _data_for_CAA(self, _type, records): + values = [] + for record in records: + flags, tag, value = record['content'].split(' ', 2) + values.append({ + 'flags': flags, + 'tag': tag, + 'value': value + }) + + return { + 'type': _type, + 'ttl': self._get_lowest_ttl(records), + 'values': values + } + + def _data_for_TXT(self, _type, records): + values = [] + for record in records: + values.append(record['content'].replace(';', '\\;')) + + return { + 'type': _type, + 'ttl': self._get_lowest_ttl(records), + 'values': values + } diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py new file mode 100644 index 0000000..dbf7eab --- /dev/null +++ b/tests/test_octodns_provider_transip.py @@ -0,0 +1,234 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +# from mock import Mock, call +from os.path import dirname, join + +from suds import WebFault + +from requests_mock import ANY, mock as requests_mock +from unittest import TestCase + +from octodns.record import Record +from octodns.provider.transip import TransipProvider +from octodns.provider.yaml import YamlProvider +from octodns.zone import Zone +from transip.service.domain import DomainService +from transip.service.objects import DnsEntry + + + + +class MockDomainService(DomainService): + + def __init__(self, *args, **kwargs): + super(MockDomainService, self).__init__('MockDomainService', *args, **kwargs) + self.mockupEntries = [] + + def mockup(self, records): + + provider = TransipProvider('', '', ''); + + _dns_entries = [] + for record in records: + if record._type in provider.SUPPORTS: + entries_for = getattr(provider, '_entries_for_{}'.format(record._type)) + + # Root records have '@' as name + name = record.name + if name == '': + name = provider.ROOT_RECORD + + _dns_entries.extend(entries_for(name, record)) + + _dns_entries.append(DnsEntry('@', '3600', 'NS', 'ns01.transip.nl.')) + + + self.mockupEntries = _dns_entries + + # Skips authentication layer and returns the entries loaded by "Mockup" + def get_info(self, domain_name): + + if str(domain_name) == str('notfound.unit.tests'): + self.raiseZoneNotFound() + + result = lambda: None + setattr(result, "dnsEntries", self.mockupEntries) + return result + + def set_dns_entries(self, domain_name, dns_entries): + if str(domain_name) == str('failsetdns.unit.tests'): + self.raiseSaveError() + + return True + + def raiseZoneNotFound(self): + fault = lambda: None + setattr(fault, "faultstring", '102 is zone not found') + setattr(fault, "faultcode", str('102')) + document = {} + raise WebFault(fault, document) + + def raiseInvalidAuth(self): + fault = lambda: None + setattr(fault, "faultstring", '200 is invalid auth') + setattr(fault, "faultcode", str('200')) + document = {} + raise WebFault(fault, document) + + def raiseSaveError(self): + fault = lambda: None + setattr(fault, "faultstring", '202 error while saving') + setattr(fault, "faultcode", str('202')) + document = {} + raise WebFault(fault, document) + + + +class TestTransipProvider(TestCase): + bogus_key = str("""-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA0U5HGCkLrz423IyUf3u4cKN2WrNz1x5KNr6PvH2M/zxas+zB +elbxkdT3AQ+wmfcIvOuTmFRTHv35q2um1aBrPxVw+2s+lWo28VwIRttwIB1vIeWu +lSBnkEZQRLyPI2tH0i5QoMX4CVPf9rvij3Uslimi84jdzDfPFIh6jZ6C8nLipOTG +0IMhge1ofVfB0oSy5H+7PYS2858QLAf5ruYbzbAxZRivS402wGmQ0d0Lc1KxraAj +kiMM5yj/CkH/Vm2w9I6+tLFeASE4ub5HCP5G/ig4dbYtqZMQMpqyAbGxd5SOVtyn +UHagAJUxf8DT3I8PyjEHjxdOPUsxNyRtepO/7QIDAQABAoIBAQC7fiZ7gxE/ezjD +2n6PsHFpHVTBLS2gzzZl0dCKZeFvJk6ODJDImaeuHhrh7X8ifMNsEI9XjnojMhl8 +MGPzy88mZHugDNK0H8B19x5G8v1/Fz7dG5WHas660/HFkS+b59cfdXOugYiOOn9O +08HBBpLZNRUOmVUuQfQTjapSwGLG8PocgpyRD4zx0LnldnJcqYCxwCdev+AAsPnq +ibNtOd/MYD37w9MEGcaxLE8wGgkv8yd97aTjkgE+tp4zsM4QE4Rag133tsLLNznT +4Qr/of15M3NW/DXq/fgctyRcJjZpU66eCXLCz2iRTnLyyxxDC2nwlxKbubV+lcS0 +S4hbfd/BAoGBAO8jXxEaiybR0aIhhSR5esEc3ymo8R8vBN3ZMJ+vr5jEPXr/ZuFj +/R4cZ2XV3VoQJG0pvIOYVPZ5DpJM7W+zSXtJ/7bLXy4Bnmh/rc+YYgC+AXQoLSil +iD2OuB2xAzRAK71DVSO0kv8gEEXCersPT2i6+vC2GIlJvLcYbOdRKWGxAoGBAOAQ +aJbRLtKujH+kMdoMI7tRlL8XwI+SZf0FcieEu//nFyerTePUhVgEtcE+7eQ7hyhG +fIXUFx/wALySoqFzdJDLc8U8pTLhbUaoLOTjkwnCTKQVprhnISqQqqh/0U5u47IE +RWzWKN6OHb0CezNTq80Dr6HoxmPCnJHBHn5LinT9AoGAQSpvZpbIIqz8pmTiBl2A +QQ2gFpcuFeRXPClKYcmbXVLkuhbNL1BzEniFCLAt4LQTaRf9ghLJ3FyCxwVlkpHV +zV4N6/8hkcTpKOraL38D/dXJSaEFJVVuee/hZl3tVJjEEpA9rDwx7ooLRSdJEJ6M +ciq55UyKBSdt4KssSiDI2RECgYBL3mJ7xuLy5bWfNsrGiVvD/rC+L928/5ZXIXPw +26oI0Yfun7ulDH4GOroMcDF/GYT/Zzac3h7iapLlR0WYI47xxGI0A//wBZLJ3QIu +krxkDo2C9e3Y/NqnHgsbOQR3aWbiDT4wxydZjIeXS3LKA2fl6Hyc90PN3cTEOb8I +hq2gRQKBgEt0SxhhtyB93SjgTzmUZZ7PiEf0YJatfM6cevmjWHexrZH+x31PB72s +fH2BQyTKKzoCLB1k/6HRaMnZdrWyWSZ7JKz3AHJ8+58d0Hr8LTrzDM1L6BbjeDct +N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd +-----END RSA PRIVATE KEY-----""") + + + def make_expected(self): + expected = Zone('unit.tests.', []) + source = YamlProvider('test', join(dirname(__file__), 'config')) + source.populate(expected) + return expected + + def test_populate(self): + + _expected = self.make_expected() + + with self.assertRaises(WebFault) as ctx: + provider = TransipProvider('test', 'unittest', self.bogus_key) + zone = Zone('unit.tests.', []) + provider.populate(zone, True) + + self.assertEquals(str('WebFault'), + str(ctx.exception.__class__.__name__)) + + self.assertEquals(str('200'), ctx.exception.fault.faultcode) + + with self.assertRaises(Exception) as ctx: + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + zone = Zone('notfound.unit.tests.', []) + provider.populate(zone, True) + + self.assertEquals(str('Exception'), + str(ctx.exception.__class__.__name__)) + + self.assertEquals('populate: (102) Transip used as target for non-existing zone: notfound.unit.tests.', ctx.exception.message) + + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + zone = Zone('notfound.unit.tests.', []) + provider.populate(zone, False) + + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + provider._client.mockup(_expected.records) + zone = Zone('unit.tests.', []) + provider.populate(zone, False) + + provider._currentZone = zone + self.assertEquals("www.unit.tests.", provider._parse_to_fqdn("www")) + + + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + zone = Zone('unit.tests.', []) + exists = provider.populate(zone, True) + self.assertTrue(exists, 'populate should return true') + + + return + + def test_plan(self): + + _expected = self.make_expected() + + print(_expected.name) + + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + plan = provider.plan(_expected) + + self.assertEqual(12, plan.change_counts['Create']) + self.assertEqual(0, plan.change_counts['Update']) + self.assertEqual(0, plan.change_counts['Delete']) + + return + + def test_apply(self): + + _expected = self.make_expected() + + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + plan = provider.plan(_expected) + #self.assertEqual(11, plan.changes) + changes = provider.apply(plan) + + + + with self.assertRaises(Exception) as ctx: + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + plan = provider.plan(_expected) + plan.desired.name = 'notfound.unit.tests.' + changes = provider.apply(plan) + # self.assertEqual(11, changes) + + self.assertEquals(str('WebFault'), + str(ctx.exception.__class__.__name__)) + + _expected = self.make_expected() + + with self.assertRaises(Exception) as ctx: + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + plan = provider.plan(_expected) + plan.desired.name = 'failsetdns.unit.tests.' + changes = provider.apply(plan) + # self.assertEqual(11, changes) + + + #provider = TransipProvider('test', 'unittest', self.bogus_key) + + #plan = provider.plan(_expected) + +# changes = provider.apply(plan) +# self.assertEquals(29, changes) + + From 30c8c4d313d150c2c81324f0b0a7de06f634bdc0 Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 25 Sep 2019 08:18:17 +0200 Subject: [PATCH 108/176] Add transip requirement and add provider to readme --- README.md | 1 + requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index be0a4ab..7124b20 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,7 @@ The above command pulled the existing data out of Route53 and placed the results | [Rackspace](/octodns/provider/rackspace.py) | | A, AAAA, ALIAS, CNAME, MX, NS, PTR, SPF, TXT | No | | | [Route53](/octodns/provider/route53.py) | boto3 | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Both | CNAME health checks don't support a Host header | | [Selectel](/octodns/provider/selectel.py) | | A, AAAA, CNAME, MX, NS, SPF, SRV, TXT | No | | +| [Transip](/octodns/provider/transip.py) | transip | A, AAAA, CNAME, MX, SRV, SPF, TXT, SSHFP, CAA | No | | | [AxfrSource](/octodns/source/axfr.py) | | A, AAAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only | | [ZoneFileSource](/octodns/source/axfr.py) | | A, AAAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only | | [TinyDnsFileSource](/octodns/source/tinydns.py) | | A, CNAME, MX, NS, PTR | No | read-only | diff --git a/requirements.txt b/requirements.txt index f67f2ee..bb373d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,3 +22,4 @@ requests==2.22.0 s3transfer==0.1.13 six==1.11.0 setuptools==38.5.2 +transip==2.0.0 From 7056d299072f3c03c446ef8b5435f4339b1bb4b4 Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 25 Sep 2019 08:18:59 +0200 Subject: [PATCH 109/176] fixes lint warning. --- octodns/provider/transip.py | 20 +++-- tests/test_octodns_provider_transip.py | 110 +++++++++++++++---------- 2 files changed, 79 insertions(+), 51 deletions(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index 2ce9180..8014310 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -36,7 +36,7 @@ class TransipProvider(BaseProvider): SUPPORTS_DYNAMIC = False SUPPORTS = set( ('A', 'AAAA', 'CNAME', 'MX', 'SRV', 'SPF', 'TXT', 'SSHFP', 'CAA')) - # unsupported by OctoDNS: 'TLSA', 'CAA' + # unsupported by OctoDNS: 'TLSA' MIN_TTL = 120 TIMEOUT = 15 ROOT_RECORD = '@' @@ -68,17 +68,17 @@ class TransipProvider(BaseProvider): try: zoneInfo = self._client.get_info(zone.name[:-1]) except WebFault as e: - if e.fault.faultcode == '102' and target == False: + if e.fault.faultcode == '102' and target is False: self.log.warning( 'populate: (%s) Zone %s not found in account ', e.fault.faultcode, zone.name) exists = False return exists - elif e.fault.faultcode == '102' and target == True: + elif e.fault.faultcode == '102' and target is True: self.log.warning('populate: Transip can\'t create new zones') raise Exception( ('populate: ({}) Transip used ' + - 'as target for non-existing zone: {}').format( + 'as target for non-existing zone: {}').format( e.fault.faultcode, zone.name)) else: self.log.error('populate: (%s) %s ', e.fault.faultcode, @@ -129,7 +129,8 @@ class TransipProvider(BaseProvider): _dns_entries = [] for record in plan.desired.records: if record._type in self.SUPPORTS: - entries_for = getattr(self, '_entries_for_{}'.format(record._type)) + entries_for = getattr(self, + '_entries_for_{}'.format(record._type)) # Root records have '@' as name name = record.name @@ -143,7 +144,7 @@ class TransipProvider(BaseProvider): except WebFault as e: self.log.warning(('_apply: Set DNS returned ' + 'one or more errors: {}').format( - e.fault.faultstring)) + e.fault.faultstring)) raise Exception(200, e.fault.faultstring) self._currentZone = {} @@ -189,8 +190,9 @@ class TransipProvider(BaseProvider): _entries = [] for value in record.values: - content = "{} {} {}".format(value.algorithm, value.fingerprint_type, - value.fingerprint) + content = "{} {} {}".format(value.algorithm, + value.fingerprint_type, + value.fingerprint) _entries.append(DnsEntry(name, record.ttl, record._type, content)) return _entries @@ -200,7 +202,7 @@ class TransipProvider(BaseProvider): for value in record.values: content = "{} {} {}".format(value.flags, value.tag, - value.value) + value.value) _entries.append(DnsEntry(name, record.ttl, record._type, content)) return _entries diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py index dbf7eab..811c1e2 100644 --- a/tests/test_octodns_provider_transip.py +++ b/tests/test_octodns_provider_transip.py @@ -10,10 +10,8 @@ from os.path import dirname, join from suds import WebFault -from requests_mock import ANY, mock as requests_mock from unittest import TestCase -from octodns.record import Record from octodns.provider.transip import TransipProvider from octodns.provider.yaml import YamlProvider from octodns.zone import Zone @@ -21,22 +19,35 @@ from transip.service.domain import DomainService from transip.service.objects import DnsEntry +class MockFault(object): + faultstring = "" + faultcode = "" + + def __init__(self, code, string, *args, **kwargs): + self.faultstring = string + self.faultcode = code + + +class MockResponse(object): + dnsEntries = [] class MockDomainService(DomainService): def __init__(self, *args, **kwargs): - super(MockDomainService, self).__init__('MockDomainService', *args, **kwargs) + super(MockDomainService, self).__init__('MockDomainService', *args, + **kwargs) self.mockupEntries = [] def mockup(self, records): - provider = TransipProvider('', '', ''); + provider = TransipProvider('', '', '') _dns_entries = [] for record in records: if record._type in provider.SUPPORTS: - entries_for = getattr(provider, '_entries_for_{}'.format(record._type)) + entries_for = getattr(provider, + '_entries_for_{}'.format(record._type)) # Root records have '@' as name name = record.name @@ -45,50 +56,48 @@ class MockDomainService(DomainService): _dns_entries.extend(entries_for(name, record)) - _dns_entries.append(DnsEntry('@', '3600', 'NS', 'ns01.transip.nl.')) - + # NS is not supported as a DNS Entry, + # so it should cover the if statement + _dns_entries.append( + DnsEntry('@', '3600', 'NS', 'ns01.transip.nl.')) self.mockupEntries = _dns_entries # Skips authentication layer and returns the entries loaded by "Mockup" def get_info(self, domain_name): + # Special 'domain' to trigger error if str(domain_name) == str('notfound.unit.tests'): self.raiseZoneNotFound() - result = lambda: None - setattr(result, "dnsEntries", self.mockupEntries) + result = MockResponse() + result.dnsEntries = self.mockupEntries return result def set_dns_entries(self, domain_name, dns_entries): + + # Special 'domain' to trigger error if str(domain_name) == str('failsetdns.unit.tests'): self.raiseSaveError() return True def raiseZoneNotFound(self): - fault = lambda: None - setattr(fault, "faultstring", '102 is zone not found') - setattr(fault, "faultcode", str('102')) + fault = MockFault(str('102'), '102 is zone not found') document = {} raise WebFault(fault, document) def raiseInvalidAuth(self): - fault = lambda: None - setattr(fault, "faultstring", '200 is invalid auth') - setattr(fault, "faultcode", str('200')) + fault = MockFault(str('200'), '200 is invalid auth') document = {} raise WebFault(fault, document) def raiseSaveError(self): - fault = lambda: None - setattr(fault, "faultstring", '202 error while saving') - setattr(fault, "faultcode", str('202')) + fault = MockFault(str('200'), '202 random error') document = {} raise WebFault(fault, document) - class TestTransipProvider(TestCase): bogus_key = str("""-----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEA0U5HGCkLrz423IyUf3u4cKN2WrNz1x5KNr6PvH2M/zxas+zB @@ -118,7 +127,6 @@ fH2BQyTKKzoCLB1k/6HRaMnZdrWyWSZ7JKz3AHJ8+58d0Hr8LTrzDM1L6BbjeDct N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd -----END RSA PRIVATE KEY-----""") - def make_expected(self): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) @@ -126,9 +134,10 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd return expected def test_populate(self): - _expected = self.make_expected() + # Unhappy Plan - Not authenticated + # Live test against API, will fail in an unauthorized error with self.assertRaises(WebFault) as ctx: provider = TransipProvider('test', 'unittest', self.bogus_key) zone = Zone('unit.tests.', []) @@ -139,6 +148,9 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd self.assertEquals(str('200'), ctx.exception.fault.faultcode) + # Unhappy Plan - Zone does not exists + # Will trigger an exception if provider is used as a target for a + # non-existing zone with self.assertRaises(Exception) as ctx: provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) @@ -148,38 +160,48 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd self.assertEquals(str('Exception'), str(ctx.exception.__class__.__name__)) - self.assertEquals('populate: (102) Transip used as target for non-existing zone: notfound.unit.tests.', ctx.exception.message) + self.assertEquals( + 'populate: (102) Transip used as target' + + ' for non-existing zone: notfound.unit.tests.', + ctx.exception.message) + # Happy Plan - Zone does not exists + # Won't trigger an exception if provider is NOT used as a target for a + # non-existing zone. provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) zone = Zone('notfound.unit.tests.', []) provider.populate(zone, False) + # Happy Plan - Populate with mockup records provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) provider._client.mockup(_expected.records) zone = Zone('unit.tests.', []) provider.populate(zone, False) + # Transip allows relative values for types like cname, mx. + # Test is these are correctly appended with the domain provider._currentZone = zone self.assertEquals("www.unit.tests.", provider._parse_to_fqdn("www")) + self.assertEquals("www.unit.tests.", + provider._parse_to_fqdn("www.unit.tests.")) + self.assertEquals("www.sub.sub.sub.unit.tests.", + provider._parse_to_fqdn("www.sub.sub.sub")) - + # Happy Plan - Even if the zone has no records the zone should exist provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) zone = Zone('unit.tests.', []) exists = provider.populate(zone, True) self.assertTrue(exists, 'populate should return true') - return def test_plan(self): - _expected = self.make_expected() - print(_expected.name) - + # Test Happy plan, only create provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) plan = provider.plan(_expected) @@ -191,29 +213,38 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd return def test_apply(self): - _expected = self.make_expected() + # Test happy flow. Create all supoorted records provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) plan = provider.plan(_expected) - #self.assertEqual(11, plan.changes) + self.assertEqual(12, len(plan.changes)) changes = provider.apply(plan) + self.assertEqual(changes, len(plan.changes)) - - + # Test unhappy flow. Trigger 'not found error' in apply stage + # This should normally not happen as populate will capture it first + # but just in case. + changes = [] # reset changes with self.assertRaises(Exception) as ctx: provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) plan = provider.plan(_expected) plan.desired.name = 'notfound.unit.tests.' changes = provider.apply(plan) - # self.assertEqual(11, changes) + + # Changes should not be set due to an Exception + self.assertEqual([], changes) self.assertEquals(str('WebFault'), str(ctx.exception.__class__.__name__)) - _expected = self.make_expected() + self.assertEquals(str('102'), ctx.exception.fault.faultcode) + + # Test unhappy flow. Trigger a unrecoverable error while saving + _expected = self.make_expected() # reset expected + changes = [] # reset changes with self.assertRaises(Exception) as ctx: provider = TransipProvider('test', 'unittest', self.bogus_key) @@ -221,14 +252,9 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd plan = provider.plan(_expected) plan.desired.name = 'failsetdns.unit.tests.' changes = provider.apply(plan) - # self.assertEqual(11, changes) - - - #provider = TransipProvider('test', 'unittest', self.bogus_key) - - #plan = provider.plan(_expected) - -# changes = provider.apply(plan) -# self.assertEquals(29, changes) + # Changes should not be set due to an Exception + self.assertEqual([], changes) + self.assertEquals(str('Exception'), + str(ctx.exception.__class__.__name__)) From 59e44b865cfa5e7d3a1f40d34f41d5b2dd285aac Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 25 Sep 2019 11:24:13 +0200 Subject: [PATCH 110/176] Added detection for edge case that could happen with existing records where the value is '@' TransIP allows '@' as value to alias the root record. '@' was on populate appended with the zone, which trigger an unneeded update. '@' => '@.example.com.' -> 'example.com' This fix will stop the unneeded update --- octodns/provider/transip.py | 7 ++++++- tests/test_octodns_provider_transip.py | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index 8014310..adde617 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -218,7 +218,12 @@ class TransipProvider(BaseProvider): def _parse_to_fqdn(self, value): - if (value[-1] != '.'): + # TransIP allows '@' as value to alias the root record. + # this provider won't set an '@' value, but can be an existing record + if value == self.ROOT_RECORD: + value = self._currentZone.name + + if value[-1] != '.': self.log.debug('parseToFQDN: changed %s to %s', value, '{}.{}'.format(value, self._currentZone.name)) value = '{}.{}'.format(value, self._currentZone.name) diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py index 811c1e2..d6bcaa7 100644 --- a/tests/test_octodns_provider_transip.py +++ b/tests/test_octodns_provider_transip.py @@ -188,6 +188,8 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd provider._parse_to_fqdn("www.unit.tests.")) self.assertEquals("www.sub.sub.sub.unit.tests.", provider._parse_to_fqdn("www.sub.sub.sub")) + self.assertEquals("unit.tests.", + provider._parse_to_fqdn("@")) # Happy Plan - Even if the zone has no records the zone should exist provider = TransipProvider('test', 'unittest', self.bogus_key) From cebc629a06295a9cae3c5821b448e11737eddbbe Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 25 Sep 2019 14:33:49 +0200 Subject: [PATCH 111/176] Enforce values as basic string to fix yaml export error Fixes an exception in combination with the yamlProvider as a target The unmodified value object isn't represented as string while building the yaml output The Exception: yaml.representer.RepresenterError: ('cannot represent an object', 1.1.1.1) yaml/representer.py@249, represent_undefined() --- octodns/provider/transip.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index adde617..050398a 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -218,6 +218,9 @@ class TransipProvider(BaseProvider): def _parse_to_fqdn(self, value): + # Enforce switch from suds.sax.text.Text to string + value = ''+value + # TransIP allows '@' as value to alias the root record. # this provider won't set an '@' value, but can be an existing record if value == self.ROOT_RECORD: @@ -240,7 +243,8 @@ class TransipProvider(BaseProvider): _values = [] for record in records: - _values.append(record['content']) + # Enforce switch from suds.sax.text.Text to string + _values.append(''+record['content']) return { 'ttl': self._get_lowest_ttl(records), From 9cab94a83a4621334880059b8007fe7bdd2bd8ba Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 25 Sep 2019 14:45:41 +0200 Subject: [PATCH 112/176] Some codestyle review changes. --- octodns/provider/transip.py | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index 050398a..aa5b5b3 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -52,9 +52,6 @@ class TransipProvider(BaseProvider): self.account = account self.key = key - self._zones = None - self._zone_records = {} - self._currentZone = {} def populate(self, zone, target=False, lenient=False): @@ -72,7 +69,6 @@ class TransipProvider(BaseProvider): self.log.warning( 'populate: (%s) Zone %s not found in account ', e.fault.faultcode, zone.name) - exists = False return exists elif e.fault.faultcode == '102' and target is True: self.log.warning('populate: Transip can\'t create new zones') @@ -115,9 +111,6 @@ class TransipProvider(BaseProvider): changes = plan.changes self.log.debug('apply: zone=%s, changes=%d', desired.name, len(changes)) - # for change in changes: - # class_name = change.__class__.__name__ - # getattr(self, '_apply_{}'.format(class_name))(change) self._currentZone = plan.desired try: @@ -265,24 +258,24 @@ class TransipProvider(BaseProvider): } def _data_for_MX(self, _type, records): - values = [] + _values = [] for record in records: preference, exchange = record['content'].split(" ", 1) - values.append({ + _values.append({ 'preference': preference, 'exchange': self._parse_to_fqdn(exchange) }) return { 'ttl': self._get_lowest_ttl(records), 'type': _type, - 'values': values + 'values': _values } def _data_for_SRV(self, _type, records): - values = [] + _values = [] for record in records: priority, weight, port, target = record['content'].split(' ', 3) - values.append({ + _values.append({ 'port': port, 'priority': priority, 'target': self._parse_to_fqdn(target), @@ -292,14 +285,14 @@ class TransipProvider(BaseProvider): return { 'type': _type, 'ttl': self._get_lowest_ttl(records), - 'values': values + 'values': _values } def _data_for_SSHFP(self, _type, records): - values = [] + _values = [] for record in records: algorithm, fp_type, fingerprint = record['content'].split(' ', 2) - values.append({ + _values.append({ 'algorithm': algorithm, 'fingerprint': fingerprint.lower(), 'fingerprint_type': fp_type @@ -308,14 +301,14 @@ class TransipProvider(BaseProvider): return { 'type': _type, 'ttl': self._get_lowest_ttl(records), - 'values': values + 'values': _values } def _data_for_CAA(self, _type, records): - values = [] + _values = [] for record in records: flags, tag, value = record['content'].split(' ', 2) - values.append({ + _values.append({ 'flags': flags, 'tag': tag, 'value': value @@ -324,16 +317,16 @@ class TransipProvider(BaseProvider): return { 'type': _type, 'ttl': self._get_lowest_ttl(records), - 'values': values + 'values': _values } def _data_for_TXT(self, _type, records): - values = [] + _values = [] for record in records: - values.append(record['content'].replace(';', '\\;')) + _values.append(record['content'].replace(';', '\\;')) return { 'type': _type, 'ttl': self._get_lowest_ttl(records), - 'values': values + 'values': _values } From 71f215932d94b323cddba6ed6e0b8f35ba3b9a4d Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 25 Sep 2019 14:51:53 +0200 Subject: [PATCH 113/176] whitespaces around operators to make /script/lint happy again --- octodns/provider/transip.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index aa5b5b3..64692ee 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -212,7 +212,7 @@ class TransipProvider(BaseProvider): def _parse_to_fqdn(self, value): # Enforce switch from suds.sax.text.Text to string - value = ''+value + value = '' + value # TransIP allows '@' as value to alias the root record. # this provider won't set an '@' value, but can be an existing record @@ -237,7 +237,7 @@ class TransipProvider(BaseProvider): _values = [] for record in records: # Enforce switch from suds.sax.text.Text to string - _values.append(''+record['content']) + _values.append('' + record['content']) return { 'ttl': self._get_lowest_ttl(records), From a035ee8c84955d2dcdf40c0ea15f1901224ae000 Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Thu, 26 Sep 2019 14:49:14 +0200 Subject: [PATCH 114/176] Give the option to use a private_key_file. Transip sdk also supports a private_key_file, so forwarding that option to the provider. Could be handy in combination with k8s secrets. --- octodns/provider/transip.py | 14 +++++++++++--- tests/test_octodns_provider_transip.py | 13 +++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index 64692ee..92d607d 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -23,13 +23,16 @@ class TransipProvider(BaseProvider): class: octodns.provider.transip.TransipProvider # Your Transip account name (required) account: yourname - # The api key (required) + # Path to a private key file (required if key is not used) + key_file: /path/to/file + # The api key as string (required if key_file is not used) key: | \''' -----BEGIN PRIVATE KEY----- ... -----END PRIVATE KEY----- \''' + # if both `key_file` and `key` are presented `key_file` is used ''' SUPPORTS_GEO = False @@ -41,13 +44,18 @@ class TransipProvider(BaseProvider): TIMEOUT = 15 ROOT_RECORD = '@' - def __init__(self, id, account, key, *args, **kwargs): + def __init__(self, id, account, key=None, key_file=None, *args, **kwargs): self.log = getLogger('TransipProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, account=%s, token=***', id, account) super(TransipProvider, self).__init__(id, *args, **kwargs) - self._client = DomainService(account, key) + if key_file is not None: + self._client = DomainService(account, private_key_file=key_file) + elif key is not None: + self._client = DomainService(account, private_key=key) + else: + raise Exception('Missing `key` of `key_file` parameter in config') self.account = account self.key = key diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py index d6bcaa7..8d85e1f 100644 --- a/tests/test_octodns_provider_transip.py +++ b/tests/test_octodns_provider_transip.py @@ -133,6 +133,19 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd source.populate(expected) return expected + def test_init(self): + with self.assertRaises(Exception) as ctx: + TransipProvider('test', 'unittest') + + self.assertEquals( + str('Missing `key` of `key_file` parameter in config'), + str(ctx.exception)) + + TransipProvider('test', 'unittest', key=self.bogus_key) + + # Existence and content of the key is tested in the SDK on client call + TransipProvider('test', 'unittest', key_file='/fake/path') + def test_populate(self): _expected = self.make_expected() From 637c2547782b15e21473f51960eb1ab4b3a94c1d Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Mon, 30 Sep 2019 13:18:57 +0200 Subject: [PATCH 115/176] Handling PR Review comments. - Added Specific exceptions - str() instead of concatenation - removed zone not found warning --- octodns/provider/transip.py | 29 +++++++++++++++++++------- tests/test_octodns_provider_transip.py | 4 ++-- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index 92d607d..09920a9 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -15,6 +15,18 @@ from transip.service.domain import DomainService from transip.service.objects import DnsEntry +class TransipException(Exception): + pass + + +class TransipConfigException(TransipException): + pass + + +class TransipNewZoneException(TransipException): + pass + + class TransipProvider(BaseProvider): ''' Transip DNS provider @@ -55,7 +67,9 @@ class TransipProvider(BaseProvider): elif key is not None: self._client = DomainService(account, private_key=key) else: - raise Exception('Missing `key` of `key_file` parameter in config') + raise TransipConfigException( + 'Missing `key` of `key_file` parameter in config' + ) self.account = account self.key = key @@ -74,13 +88,12 @@ class TransipProvider(BaseProvider): zoneInfo = self._client.get_info(zone.name[:-1]) except WebFault as e: if e.fault.faultcode == '102' and target is False: - self.log.warning( - 'populate: (%s) Zone %s not found in account ', - e.fault.faultcode, zone.name) + # Zone not found in account, and not a target so just + # leave an empty zone. return exists elif e.fault.faultcode == '102' and target is True: self.log.warning('populate: Transip can\'t create new zones') - raise Exception( + raise TransipNewZoneException( ('populate: ({}) Transip used ' + 'as target for non-existing zone: {}').format( e.fault.faultcode, zone.name)) @@ -146,7 +159,7 @@ class TransipProvider(BaseProvider): self.log.warning(('_apply: Set DNS returned ' + 'one or more errors: {}').format( e.fault.faultstring)) - raise Exception(200, e.fault.faultstring) + raise TransipException(200, e.fault.faultstring) self._currentZone = {} @@ -220,7 +233,7 @@ class TransipProvider(BaseProvider): def _parse_to_fqdn(self, value): # Enforce switch from suds.sax.text.Text to string - value = '' + value + value = str(value) # TransIP allows '@' as value to alias the root record. # this provider won't set an '@' value, but can be an existing record @@ -245,7 +258,7 @@ class TransipProvider(BaseProvider): _values = [] for record in records: # Enforce switch from suds.sax.text.Text to string - _values.append('' + record['content']) + _values.append(str(record['content'])) return { 'ttl': self._get_lowest_ttl(records), diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py index 8d85e1f..c56509a 100644 --- a/tests/test_octodns_provider_transip.py +++ b/tests/test_octodns_provider_transip.py @@ -170,7 +170,7 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd zone = Zone('notfound.unit.tests.', []) provider.populate(zone, True) - self.assertEquals(str('Exception'), + self.assertEquals(str('TransipNewZoneException'), str(ctx.exception.__class__.__name__)) self.assertEquals( @@ -271,5 +271,5 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd # Changes should not be set due to an Exception self.assertEqual([], changes) - self.assertEquals(str('Exception'), + self.assertEquals(str('TransipException'), str(ctx.exception.__class__.__name__)) From 97608b382399b798f7b865f082b943685b3e2fd9 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 30 Sep 2019 09:29:30 -0700 Subject: [PATCH 116/176] v0.9.7 version bump and CHANGELOG updates --- .gitignore | 4 ++++ CHANGELOG.md | 8 ++++++++ octodns/__init__.py | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 1efa084..715b687 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +# +# Do not add editor or OS specific ignores here. Have a look at adding +# `excludesfile` to your `~/.gitconfig` to globally ignore such things. +# *.pyc .coverage .env diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f283ec..d80a48f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## v0.9.7 - 2019-09-30 - It's about time + +* AkamaiProvider, ConstellixProvider, MythicBeastsProvider, SelectelProvider, + & TransipPovider providers added +* Route53Provider seperator fix +* YamlProvider export error around stringification +* PyPi markdown rendering fix + ## v0.9.6 - 2019-07-16 - The little one that fixes stuff from the big one * Reduced dynamic record value weight range to 0-15 so that Dyn and Route53 diff --git a/octodns/__init__.py b/octodns/__init__.py index 6422577..57300de 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -3,4 +3,4 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -__VERSION__ = '0.9.6' +__VERSION__ = '0.9.7' From a1d2217604fad7ec8c3fccc73f7d6e892071eeb3 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 30 Sep 2019 10:17:50 -0700 Subject: [PATCH 117/176] Fix/hack README rendering so that pypi's markdown handling libs are happy --- README.md | 4 ++-- setup.py | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7124b20..83f0bd1 100644 --- a/README.md +++ b/README.md @@ -90,8 +90,8 @@ Now that we have something to tell OctoDNS about our providers & zones we need t ttl: 60 type: A values: - - 1.2.3.4 - - 1.2.3.5 + - 1.2.3.4 + - 1.2.3.5 ``` Further information can be found in [Records Documentation](/docs/records.md). diff --git a/setup.py b/setup.py index 75a39d7..5cb741b 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,6 @@ #!/usr/bin/env python +from StringIO import StringIO from os.path import dirname, join import octodns @@ -21,6 +22,39 @@ console_scripts = { for name in cmds } + +def long_description(): + buf = StringIO() + yaml_block = False + supported_providers = False + with open('README.md') as fh: + for line in fh: + if line == '```yaml\n': + yaml_block = True + continue + elif yaml_block and line == '---\n': + # skip the line + continue + elif yaml_block and line == '```\n': + yaml_block = False + continue + elif supported_providers: + if line.startswith('## '): + supported_providers = False + # write this line out, no continue + else: + # We're ignoring this one + continue + elif line == '## Supported providers\n': + supported_providers = True + continue + buf.write(line) + buf = buf.getvalue() + with open('/tmp/mod', 'w') as fh: + fh.write(buf) + return buf + + setup( author='Ross McFarland', author_email='rwmcfa1@gmail.com', @@ -40,7 +74,7 @@ setup( 'requests>=2.20.0' ], license='MIT', - long_description=open('README.md').read(), + long_description=long_description(), long_description_content_type='text/markdown', name='octodns', packages=find_packages(), From e3ad57d15b055db9a441d0a5d7a7b924ff8600d5 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 30 Sep 2019 10:18:35 -0700 Subject: [PATCH 118/176] twine check before upload --- script/release | 1 + 1 file changed, 1 insertion(+) diff --git a/script/release b/script/release index dd3e1b1..f2c90bf 100755 --- a/script/release +++ b/script/release @@ -22,5 +22,6 @@ git tag -s "v$VERSION" -m "Release $VERSION" git push origin "v$VERSION" echo "Tagged and pushed v$VERSION" python setup.py sdist +twine check dist/*$VERSION.tar.gz twine upload dist/*$VERSION.tar.gz echo "Uploaded $VERSION" From 659a60de46ad0bb75811266691e57b44e58b46ff Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 30 Sep 2019 10:18:53 -0700 Subject: [PATCH 119/176] v0.9.8 version bump and CHANGELOG updates --- CHANGELOG.md | 4 ++++ octodns/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d80a48f..76ff8b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v0.9.8 - 2019-09-30 - One with no changes b/c PyPi description problems + +* No material changes + ## v0.9.7 - 2019-09-30 - It's about time * AkamaiProvider, ConstellixProvider, MythicBeastsProvider, SelectelProvider, diff --git a/octodns/__init__.py b/octodns/__init__.py index 57300de..71b5b1a 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -3,4 +3,4 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -__VERSION__ = '0.9.7' +__VERSION__ = '0.9.8' From 3296b55aa62ccbe558dff22fd465d95cd0b7cfb1 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 30 Sep 2019 10:22:05 -0700 Subject: [PATCH 120/176] Include readme_renderer[md] in dev requirements --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 77dd50c..a2833ae 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,5 +5,6 @@ pycodestyle==2.4.0 pycountry>=18.12.8 pycountry_convert>=0.7.2 pyflakes==1.6.0 +readme_renderer[md]==24.0 requests_mock twine==1.13.0 From db77d5b3bbd89adf76a80e37c7d5d65c8cc3056e Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 5 Oct 2019 07:58:38 -0700 Subject: [PATCH 121/176] python3 compat for azure provider --- octodns/provider/azuredns.py | 10 +++++++--- requirements.txt | 6 +++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 0bca46d..3d8122a 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -175,6 +175,10 @@ class _AzureRecord(object): :type return: bool ''' + + def key_dict(d): + return sum([hash('{}:{}'.format(k, v)) for k, v in d.items()]) + def parse_dict(params): vals = [] for char in params: @@ -185,7 +189,7 @@ class _AzureRecord(object): vals.append(record.__dict__) except: vals.append(list_records.__dict__) - vals.sort() + vals.sort(key=key_dict) return vals return (self.resource_group == b.resource_group) & \ @@ -373,13 +377,13 @@ class AzureProvider(BaseProvider): self._populate_zones() self._check_zone(zone_name) - _records = set() + _records = [] records = self._dns_client.record_sets.list_by_dns_zone if self._check_zone(zone_name): exists = True for azrecord in records(self._resource_group, zone_name): if _parse_azure_type(azrecord.type) in self.SUPPORTS: - _records.add(azrecord) + _records.append(azrecord) for azrecord in _records: record_name = azrecord.name if azrecord.name != '@' else '' typ = _parse_azure_type(azrecord.type) diff --git a/requirements.txt b/requirements.txt index 765b65c..19d45a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ PyYaml==4.2b1 -azure-common==1.1.18 -azure-mgmt-dns==2.1.0 +azure-common==1.1.23 +azure-mgmt-dns==3.0.0 boto3==1.7.5 botocore==1.10.5 dnspython==1.15.0 @@ -13,7 +13,7 @@ google-cloud-dns==0.29.0 incf.countryutils==1.0 ipaddress==1.0.22 jmespath==0.9.3 -msrestazure==0.6.0 +msrestazure==0.6.2 natsort==5.5.0 nsone==0.9.100 ovh==0.4.8 From 470dd822026655eba24d213ab9d54946b1a7f936 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 5 Oct 2019 08:03:43 -0700 Subject: [PATCH 122/176] python 3 support for constellix provider --- octodns/provider/constellix.py | 3 ++- tests/test_octodns_provider_constellix.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/octodns/provider/constellix.py b/octodns/provider/constellix.py index 939284d..050f120 100644 --- a/octodns/provider/constellix.py +++ b/octodns/provider/constellix.py @@ -9,6 +9,7 @@ from collections import defaultdict from requests import Session from base64 import b64encode from ipaddress import ip_address +from six import string_types import hashlib import hmac import logging @@ -122,7 +123,7 @@ class ConstellixClient(object): # change relative values to absolute value = record['value'] if record['type'] in ['ALIAS', 'CNAME', 'MX', 'NS', 'SRV']: - if isinstance(value, unicode): + if isinstance(value, string_types): record['value'] = self._absolutize_value(value, zone_name) if isinstance(value, list): diff --git a/tests/test_octodns_provider_constellix.py b/tests/test_octodns_provider_constellix.py index 346bb17..7914c53 100644 --- a/tests/test_octodns_provider_constellix.py +++ b/tests/test_octodns_provider_constellix.py @@ -10,6 +10,7 @@ from mock import Mock, call from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock +from six import text_type from unittest import TestCase from octodns.record import Record @@ -65,7 +66,7 @@ class TestConstellixProvider(TestCase): with self.assertRaises(Exception) as ctx: zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals('Unauthorized', ctx.exception.message) + self.assertEquals('Unauthorized', text_type(ctx.exception)) # Bad request with requests_mock() as mock: @@ -77,7 +78,7 @@ class TestConstellixProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals('\n - "unittests" is not a valid domain name', - ctx.exception.message) + text_type(ctx.exception)) # General error with requests_mock() as mock: @@ -148,6 +149,11 @@ class TestConstellixProvider(TestCase): call('POST', '/', data={'names': ['unit.tests']}), # get all domains to build the cache call('GET', '/'), + ]) + # These two checks are broken up so that ordering doesn't break things. + # Python3 doesn't make the calls in a consistent order so different + # things follow the GET / on different runs + provider._client._request.assert_has_calls([ call('POST', '/123123/records/SRV', data={ 'roundRobin': [{ 'priority': 10, From 742305c20b62b97d354dc0336b0f87dca4a1de1d Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 5 Oct 2019 08:47:12 -0700 Subject: [PATCH 123/176] six.moves.urllib.parse --- octodns/provider/fastdns.py | 2 +- tests/test_octodns_provider_rackspace.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index f851303..8f651f0 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -7,7 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from requests import Session from akamai.edgegrid import EdgeGridAuth -from urlparse import urljoin +from six.moves.urllib.parse import urljoin from collections import defaultdict from logging import getLogger diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index b257166..b0dcad7 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -8,11 +8,8 @@ from __future__ import absolute_import, division, print_function, \ import json import re from six import text_type +from six.moves.urllib.parse import urlparse from unittest import TestCase -try: - from urlparse import urlparse -except ImportError: - from urllib.parse import urlparse from requests import HTTPError from requests_mock import ANY, mock as requests_mock From 47199fdfab404dc762d04c9733ccdd9dbb342933 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 5 Oct 2019 08:49:19 -0700 Subject: [PATCH 124/176] FastDNS python3 --- tests/test_octodns_provider_fastdns.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_fastdns.py b/tests/test_octodns_provider_fastdns.py index 5f503c7..a8bed74 100644 --- a/tests/test_octodns_provider_fastdns.py +++ b/tests/test_octodns_provider_fastdns.py @@ -9,6 +9,7 @@ from __future__ import absolute_import, division, print_function, \ from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock +from six import text_type from unittest import TestCase from octodns.record import Record @@ -147,4 +148,4 @@ class TestFastdnsProvider(TestCase): changes = provider.apply(plan) except NameError as e: expected = "contractId not specified to create zone" - self.assertEquals(e.message, expected) + self.assertEquals(text_type(e), expected) From 00781985008b2e1b122160e568b2818deb960cb8 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 5 Oct 2019 08:57:14 -0700 Subject: [PATCH 125/176] GoogleCloud python3 --- tests/test_octodns_provider_googlecloud.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_googlecloud.py b/tests/test_octodns_provider_googlecloud.py index d7f0e0c..e642668 100644 --- a/tests/test_octodns_provider_googlecloud.py +++ b/tests/test_octodns_provider_googlecloud.py @@ -193,9 +193,14 @@ class DummyIterator: def __iter__(self): return self + # python2 def next(self): return next(self.iterable) + # python3 + def __next__(self): + return next(self.iterable) + class TestGoogleCloudProvider(TestCase): @patch('octodns.provider.googlecloud.dns') @@ -247,7 +252,7 @@ class TestGoogleCloudProvider(TestCase): return_values_for_status = iter( ["pending"] * 11 + ['done', 'done']) type(status_mock).status = PropertyMock( - side_effect=return_values_for_status.next) + side_effect=lambda: next(return_values_for_status)) gcloud_zone_mock.changes = Mock(return_value=status_mock) provider = self._get_provider() From 484a5118f438f8e00ce29027e6b2af167aae1eb5 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 5 Oct 2019 09:10:59 -0700 Subject: [PATCH 126/176] MythicBeastsProvider python3 --- octodns/provider/mythicbeasts.py | 2 +- tests/test_octodns_provider_mythicbeasts.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py index 17029db..b255a74 100644 --- a/octodns/provider/mythicbeasts.py +++ b/octodns/provider/mythicbeasts.py @@ -328,7 +328,7 @@ class MythicBeastsProvider(BaseProvider): exists = True for line in resp.content.splitlines(): - match = MythicBeastsProvider.RE_POPLINE.match(line) + match = MythicBeastsProvider.RE_POPLINE.match(line.decode("utf-8")) if match is None: self.log.debug('failed to match line: %s', line) diff --git a/tests/test_octodns_provider_mythicbeasts.py b/tests/test_octodns_provider_mythicbeasts.py index b93d46e..498b408 100644 --- a/tests/test_octodns_provider_mythicbeasts.py +++ b/tests/test_octodns_provider_mythicbeasts.py @@ -441,11 +441,11 @@ class TestMythicBeastsProvider(TestCase): plan = provider.plan(wanted) # Octo ignores NS records (15-1) - self.assertEquals(1, len(filter(lambda u: isinstance(u, Update), - plan.changes))) - self.assertEquals(1, len(filter(lambda d: isinstance(d, Delete), - plan.changes))) - self.assertEquals(14, len(filter(lambda c: isinstance(c, Create), - plan.changes))) + self.assertEquals(1, len(list(filter( + lambda u: isinstance(u, Update), plan.changes)))) + self.assertEquals(1, len(list(filter( + lambda d: isinstance(d, Delete), plan.changes)))) + self.assertEquals(14, len(list(filter( + lambda c: isinstance(c, Create), plan.changes)))) self.assertEquals(16, provider.apply(plan)) self.assertTrue(plan.exists) From 37543e6a762f94687a219134c6c9f75b900c93a8 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 5 Oct 2019 09:18:42 -0700 Subject: [PATCH 127/176] OvhProvider python3 --- octodns/provider/ovh.py | 2 +- tests/test_octodns_provider_ovh.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/octodns/provider/ovh.py b/octodns/provider/ovh.py index 0187098..a0e47f8 100644 --- a/octodns/provider/ovh.py +++ b/octodns/provider/ovh.py @@ -345,7 +345,7 @@ class OvhProvider(BaseProvider): @staticmethod def _is_valid_dkim_key(key): try: - base64.decodestring(key) + base64.decodestring(bytearray(key, 'utf-8')) except binascii.Error: return False return True diff --git a/tests/test_octodns_provider_ovh.py b/tests/test_octodns_provider_ovh.py index d3f468d..1f41abf 100644 --- a/tests/test_octodns_provider_ovh.py +++ b/tests/test_octodns_provider_ovh.py @@ -428,7 +428,7 @@ class TestOvhProvider(TestCase): ), call(u'/domain/zone/unit.tests/refresh')] - post_mock.assert_has_calls(wanted_calls) + post_mock.assert_has_calls(wanted_calls, any_order=True) # Get for delete calls wanted_get_calls = [ @@ -440,7 +440,7 @@ class TestOvhProvider(TestCase): subDomain=u''), call(u'/domain/zone/unit.tests/record', fieldType=u'A', subDomain='fake')] - get_mock.assert_has_calls(wanted_get_calls) + get_mock.assert_has_calls(wanted_get_calls, any_order=True) # 4 delete calls for update and delete delete_mock.assert_has_calls( [call(u'/domain/zone/unit.tests/record/100'), From 0acff67faa4c38ca36a13f164ed6903c504a64fc Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 5 Oct 2019 14:38:58 -0700 Subject: [PATCH 128/176] Ns1Provider python3 --- octodns/provider/ns1.py | 5 ++--- tests/test_octodns_provider_ns1.py | 13 +++++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index d3faf21..b4fd850 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -10,7 +10,7 @@ from itertools import chain from collections import OrderedDict, defaultdict from nsone import NSONE from nsone.rest.errors import RateLimitException, ResourceException -from incf.countryutils import transformations +from pycountry_convert import country_alpha2_to_continent_code from time import sleep from six import text_type @@ -62,8 +62,7 @@ class Ns1Provider(BaseProvider): us_state = meta.get('us_state', []) ca_province = meta.get('ca_province', []) for cntry in country: - cn = transformations.cc_to_cn(cntry) - con = transformations.cn_to_ctca2(cn) + con = country_alpha2_to_continent_code(cntry) key = '{}-{}'.format(con, cntry) geo[key].extend(answer['answer']) for state in us_state: diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 178ce53..91d1a3f 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -5,6 +5,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +from collections import defaultdict from mock import Mock, call, patch from nsone.rest.errors import AuthException, RateLimitException, \ ResourceException @@ -373,8 +374,12 @@ class TestNs1Provider(TestCase): load_mock.side_effect = [nsone_zone, nsone_zone] plan = provider.plan(desired) self.assertEquals(3, len(plan.changes)) - self.assertIsInstance(plan.changes[0], Update) - self.assertIsInstance(plan.changes[2], Delete) + # Shouldn't rely on order so just count classes + classes = defaultdict(lambda: 0) + for change in plan.changes: + classes[change.__class__] += 1 + self.assertEquals(1, classes[Delete]) + self.assertEquals(2, classes[Update]) # ugh, we need a mock record that can be returned from loadRecord for # the update and delete targets, we can add our side effects to that to # trigger rate limit handling @@ -397,7 +402,7 @@ class TestNs1Provider(TestCase): call('unit.tests', u'A'), call('geo', u'A'), call('delete-me', u'A'), - ]) + ], any_order=True) mock_record.assert_has_calls([ call.update(answers=[{'answer': [u'1.2.3.4'], 'meta': {}}], filters=[], @@ -424,7 +429,7 @@ class TestNs1Provider(TestCase): ttl=34), call.delete(), call.delete() - ]) + ], any_order=True) def test_escaping(self): provider = Ns1Provider('test', 'api-key') From aeb70b24888cd448e8dcf073b391d7c649ab1b46 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 5 Oct 2019 20:01:53 -0700 Subject: [PATCH 129/176] Route53Provider python 3, rm incf.countryutils, lots of cmp removal and ordering fixes --- CHANGELOG.md | 6 + octodns/provider/route53.py | 106 +++++-- requirements.txt | 1 - tests/test_octodns_provider_rackspace.py | 1 - tests/test_octodns_provider_route53.py | 342 +++++++++++++---------- 5 files changed, 278 insertions(+), 178 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76ff8b0..e30d0df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## v0.9.9 - 2019-??-?? - Python 3.7 Support + +* Route53 _mod_keyer ordering wasn't complete/reliable and in python 3 this + resulted in randomness. This has been addressed and may result in value + reordering on next plan, no actual changes in behavior should occur. + ## v0.9.8 - 2019-09-30 - One with no changes b/c PyPi description problems * No material changes diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 0bc7a99..6d2cfeb 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -8,8 +8,8 @@ from __future__ import absolute_import, division, print_function, \ from boto3 import client from botocore.config import Config from collections import defaultdict -from incf.countryutils.transformations import cca_to_ctca2 from ipaddress import AddressValueError, ip_address +from pycountry_convert import country_alpha2_to_continent_code from uuid import uuid4 import logging import re @@ -20,13 +20,6 @@ from ..record import Record, Update from ..record.geo import GeoCodes from .base import BaseProvider -# TODO: remove when Python 2.x is no longer supported -try: # pragma: no cover - cmp -except NameError: # pragma: no cover - def cmp(x, y): - return (x > y) - (x < y) - octal_re = re.compile(r'\\(\d\d\d)') @@ -155,7 +148,7 @@ class _Route53Record(object): } } - # NOTE: we're using __hash__ and __cmp__ methods that consider + # NOTE: we're using __hash__ and ordering methods that consider # _Route53Records equivalent if they have the same class, fqdn, and _type. # Values are ignored. This is useful when computing diffs/changes. @@ -163,17 +156,34 @@ class _Route53Record(object): 'sub-classes should never use this method' return '{}:{}'.format(self.fqdn, self._type).__hash__() - def __cmp__(self, other): - '''sub-classes should call up to this and return its value if non-zero. - When it's zero they should compute their own __cmp__''' - if self.__class__ != other.__class__: - return cmp(self.__class__, other.__class__) - elif self.fqdn != other.fqdn: - return cmp(self.fqdn, other.fqdn) - elif self._type != other._type: - return cmp(self._type, other._type) - # We're ignoring ttl, it's not an actual differentiator - return 0 + def __eq__(self, other): + '''Sub-classes should call up to this and return its value if true. + When it's false they should compute their own __eq__, same for other + ordering methods.''' + return self.__class__.__name__ == other.__class__.__name__ and \ + self.fqdn == other.fqdn and \ + self._type == other._type + + def __ne__(self, other): + return self.__class__.__name__ != other.__class__.__name__ or \ + self.fqdn != other.fqdn or \ + self._type != other._type + + def __lt__(self, other): + return (((self.__class__.__name__, self.fqdn, self._type)) < + ((other.__class__.__name__, other.fqdn, other._type))) + + def __le__(self, other): + return (((self.__class__.__name__, self.fqdn, self._type)) <= + ((other.__class__.__name__, other.fqdn, other._type))) + + def __gt__(self, other): + return (((self.__class__.__name__, self.fqdn, self._type)) > + ((other.__class__.__name__, other.fqdn, other._type))) + + def __ge__(self, other): + return (((self.__class__.__name__, self.fqdn, self._type)) >= + ((other.__class__.__name__, other.fqdn, other._type))) def __repr__(self): return '_Route53Record<{} {} {} {}>'.format(self.fqdn, self._type, @@ -514,11 +524,50 @@ class _Route53GeoRecord(_Route53Record): return '{}:{}:{}'.format(self.fqdn, self._type, self.geo.code).__hash__() - def __cmp__(self, other): - ret = super(_Route53GeoRecord, self).__cmp__(other) - if ret != 0: - return ret - return cmp(self.geo.code, other.geo.code) + def __eq__(self, other): + return super(_Route53GeoRecord, self).__eq__(other) and \ + self.geo.code == other.geo.code + + def __ne__(self, other): + # super will handle class != class, so if it's true we have 2 geo + # objects with the same name and type, so just need to compare codes + return super(_Route53GeoRecord, self).__ne__(other) or \ + self.geo.code != other.geo.code + + def __lt__(self, other): + # super eq will check class, name, and type + if super(_Route53GeoRecord, self).__eq__(other): + # if it's True we're dealing with two geo's with the same name and + # type, so we just need to compare codes + return self.geo.code < other.geo.code + # Super is not equal so we'll just let it decide lt + return super(_Route53GeoRecord, self).__lt__(other) + + def __le__(self, other): + # super eq will check class, name, and type + if super(_Route53GeoRecord, self).__eq__(other): + # Just need to compare codes, everything else is equal + return self.geo.code <= other.geo.code + # Super is not equal so geo.code doesn't matter, let it decide with lt, + # can't be eq + return super(_Route53GeoRecord, self).__lt__(other) + + def __gt__(self, other): + # super eq will check class, name, and type + if super(_Route53GeoRecord, self).__eq__(other): + # Just need to compare codes, everything else is equal + return self.geo.code > other.geo.code + # Super is not equal so we'll just let it decide gt + return super(_Route53GeoRecord, self).__gt__(other) + + def __ge__(self, other): + # super eq will check class, name, and type + if super(_Route53GeoRecord, self).__eq__(other): + # Just need to compare codes, everything else is equal + return self.geo.code >= other.geo.code + # Super is not equal so geo.code doesn't matter, let it decide with gt, + # can't be eq + return super(_Route53GeoRecord, self).__gt__(other) def __repr__(self): return '_Route53GeoRecord<{} {} {} {} {}>'.format(self.fqdn, @@ -561,7 +610,10 @@ def _mod_keyer(mod): if rrset.get('GeoLocation', False): unique_id = rrset['SetIdentifier'] else: - unique_id = rrset['Name'] + try: + unique_id = '{}-{}'.format(rrset['Name'], rrset['SetIdentifier']) + except KeyError: + unique_id = rrset['Name'] # Prioritise within the action_priority, ensuring targets come first. if rrset.get('GeoLocation', False): @@ -708,7 +760,7 @@ class Route53Provider(BaseProvider): if cc == '*': # This is the default return - cn = cca_to_ctca2(cc) + cn = country_alpha2_to_continent_code(cc) try: return '{}-{}-{}'.format(cn, cc, loc['SubdivisionCode']) except KeyError: diff --git a/requirements.txt b/requirements.txt index 19d45a8..6b6af09 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,6 @@ futures==3.2.0; python_version < '3.0' edgegrid-python==1.1.1 google-cloud-core==0.28.1 google-cloud-dns==0.29.0 -incf.countryutils==1.0 ipaddress==1.0.22 jmespath==0.9.3 msrestazure==0.6.2 diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index b0dcad7..3dfdd5f 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -40,7 +40,6 @@ with open('./tests/fixtures/rackspace-sample-recordset-page2.json') as fh: class TestRackspaceProvider(TestCase): def setUp(self): - self.maxDiff = 1000 with requests_mock() as mock: mock.post(ANY, status_code=200, text=AUTH_RESPONSE) self.provider = RackspaceProvider('identity', 'test', 'api-key', diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 87dfd09..b0ee342 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -2091,6 +2091,58 @@ class TestRoute53Records(TestCase): e.__repr__() f.__repr__() + def test_route53_record_ordering(self): + # Matches + a = _Route53Record(None, self.record_a, False) + b = _Route53Record(None, self.record_a, False) + self.assertTrue(a == b) + self.assertFalse(a != b) + self.assertFalse(a < b) + self.assertTrue(a <= b) + self.assertFalse(a > b) + self.assertTrue(a >= b) + + # Change the fqdn is greater + fqdn = _Route53Record(None, self.record_a, False, + fqdn_override='other') + self.assertFalse(a == fqdn) + self.assertTrue(a != fqdn) + self.assertFalse(a < fqdn) + self.assertFalse(a <= fqdn) + self.assertTrue(a > fqdn) + self.assertTrue(a >= fqdn) + + provider = DummyProvider() + geo_a = _Route53GeoRecord(provider, self.record_a, 'NA-US', + self.record_a.geo['NA-US'], False) + geo_b = _Route53GeoRecord(provider, self.record_a, 'NA-US', + self.record_a.geo['NA-US'], False) + self.assertTrue(geo_a == geo_b) + self.assertFalse(geo_a != geo_b) + self.assertFalse(geo_a < geo_b) + self.assertTrue(geo_a <= geo_b) + self.assertFalse(geo_a > geo_b) + self.assertTrue(geo_a >= geo_b) + + # Other base + geo_fqdn = _Route53GeoRecord(provider, self.record_a, 'NA-US', + self.record_a.geo['NA-US'], False) + geo_fqdn.fqdn = 'other' + self.assertFalse(geo_a == geo_fqdn) + self.assertTrue(geo_a != geo_fqdn) + self.assertFalse(geo_a < geo_fqdn) + self.assertFalse(geo_a <= geo_fqdn) + self.assertTrue(geo_a > geo_fqdn) + self.assertTrue(geo_a >= geo_fqdn) + + # Other class + self.assertFalse(a == geo_a) + self.assertTrue(a != geo_a) + self.assertFalse(a < geo_a) + self.assertFalse(a <= geo_a) + self.assertTrue(a > geo_a) + self.assertTrue(a >= geo_a) + def test_dynamic_value_delete(self): provider = DummyProvider() geo = _Route53DynamicValue(provider, self.record_a, 'iad', '2.2.2.2', @@ -2207,136 +2259,38 @@ class TestRoute53Records(TestCase): creating=True) self.assertEquals(18, len(route53_records)) + expected_mods = [r.mod('CREATE', []) for r in route53_records] + # Sort so that we get a consistent order and don't rely on set ordering + expected_mods.sort(key=_mod_keyer) + # Convert the route53_records into mods self.assertEquals([{ 'Action': 'CREATE', 'ResourceRecordSet': { 'HealthCheckId': 'hc42', 'Name': '_octodns-ap-southeast-1-value.unit.tests.', - 'ResourceRecords': [{ - 'Value': '1.4.1.2'}], - 'SetIdentifier': 'ap-southeast-1-001', + 'ResourceRecords': [{'Value': '1.4.1.1'}], + 'SetIdentifier': 'ap-southeast-1-000', 'TTL': 60, 'Type': 'A', - 'Weight': 2 - } + 'Weight': 2} }, { 'Action': 'CREATE', 'ResourceRecordSet': { 'HealthCheckId': 'hc42', 'Name': '_octodns-ap-southeast-1-value.unit.tests.', - 'ResourceRecords': [{ - 'Value': '1.4.1.1'}], - 'SetIdentifier': 'ap-southeast-1-000', + 'ResourceRecords': [{'Value': '1.4.1.2'}], + 'SetIdentifier': 'ap-southeast-1-001', 'TTL': 60, 'Type': 'A', - 'Weight': 2 - } - }, { - 'Action': 'CREATE', - 'ResourceRecordSet': { - 'AliasTarget': { - 'DNSName': '_octodns-ap-southeast-1-pool.unit.tests.', - 'EvaluateTargetHealth': True, - 'HostedZoneId': 'z45' - }, - 'GeoLocation': { - 'CountryCode': 'JP'}, - 'Name': 'unit.tests.', - 'SetIdentifier': '0-ap-southeast-1-AS-JP', - 'Type': 'A' - } - }, { - 'Action': 'CREATE', - 'ResourceRecordSet': { - 'AliasTarget': { - 'DNSName': '_octodns-eu-central-1-pool.unit.tests.', - 'EvaluateTargetHealth': True, - 'HostedZoneId': 'z45'}, - 'GeoLocation': { - 'CountryCode': 'US', - 'SubdivisionCode': 'FL', - }, - 'Name': 'unit.tests.', - 'SetIdentifier': '1-eu-central-1-NA-US-FL', - 'Type': 'A'} - }, { - 'Action': 'CREATE', - 'ResourceRecordSet': { - 'AliasTarget': { - 'DNSName': '_octodns-us-east-1-pool.unit.tests.', - 'EvaluateTargetHealth': True, - 'HostedZoneId': 'z45'}, - 'GeoLocation': { - 'CountryCode': '*'}, - 'Name': 'unit.tests.', - 'SetIdentifier': '2-us-east-1-None', - 'Type': 'A'} - }, { - 'Action': 'CREATE', - 'ResourceRecordSet': { - 'AliasTarget': { - 'DNSName': '_octodns-us-east-1-pool.unit.tests.', - 'EvaluateTargetHealth': True, - 'HostedZoneId': 'z45'}, - 'Failover': 'SECONDARY', - 'Name': '_octodns-ap-southeast-1-pool.unit.tests.', - 'SetIdentifier': 'ap-southeast-1-Secondary-us-east-1', - 'Type': 'A'} - }, { - 'Action': 'CREATE', - 'ResourceRecordSet': { - 'AliasTarget': { - 'DNSName': '_octodns-ap-southeast-1-pool.unit.tests.', - 'EvaluateTargetHealth': True, - 'HostedZoneId': 'z45'}, - 'GeoLocation': { - 'CountryCode': 'CN'}, - 'Name': 'unit.tests.', - 'SetIdentifier': '0-ap-southeast-1-AS-CN', - 'Type': 'A'} - }, { - 'Action': 'CREATE', - 'ResourceRecordSet': { - 'AliasTarget': { - 'DNSName': '_octodns-us-east-1-value.unit.tests.', - 'EvaluateTargetHealth': True, - 'HostedZoneId': 'z45'}, - 'Failover': 'PRIMARY', - 'Name': '_octodns-us-east-1-pool.unit.tests.', - 'SetIdentifier': 'us-east-1-Primary', - 'Type': 'A'} - }, { - 'Action': 'CREATE', - 'ResourceRecordSet': { - 'AliasTarget': { - 'DNSName': '_octodns-eu-central-1-pool.unit.tests.', - 'EvaluateTargetHealth': True, - 'HostedZoneId': 'z45'}, - 'GeoLocation': { - 'ContinentCode': 'EU'}, - 'Name': 'unit.tests.', - 'SetIdentifier': '1-eu-central-1-EU', - 'Type': 'A'} - }, { - 'Action': 'CREATE', - 'ResourceRecordSet': { - 'AliasTarget': { - 'DNSName': '_octodns-eu-central-1-value.unit.tests.', - 'EvaluateTargetHealth': True, - 'HostedZoneId': 'z45'}, - 'Failover': 'PRIMARY', - 'Name': '_octodns-eu-central-1-pool.unit.tests.', - 'SetIdentifier': 'eu-central-1-Primary', - 'Type': 'A'} + 'Weight': 2} }, { 'Action': 'CREATE', 'ResourceRecordSet': { 'Name': '_octodns-default-pool.unit.tests.', - 'ResourceRecords': [{ - 'Value': '1.1.2.1'}, - { - 'Value': '1.1.2.2'}], + 'ResourceRecords': [ + {'Value': '1.1.2.1'}, + {'Value': '1.1.2.2'}], 'TTL': 60, 'Type': 'A'} }, { @@ -2344,56 +2298,41 @@ class TestRoute53Records(TestCase): 'ResourceRecordSet': { 'HealthCheckId': 'hc42', 'Name': '_octodns-eu-central-1-value.unit.tests.', - 'ResourceRecords': [{ - 'Value': '1.3.1.2'}], + 'ResourceRecords': [{'Value': '1.3.1.1'}], + 'SetIdentifier': 'eu-central-1-000', + 'TTL': 60, + 'Type': 'A', + 'Weight': 1} + }, { + 'Action': 'CREATE', + 'ResourceRecordSet': { + 'HealthCheckId': 'hc42', + 'Name': '_octodns-eu-central-1-value.unit.tests.', + 'ResourceRecords': [{'Value': '1.3.1.2'}], 'SetIdentifier': 'eu-central-1-001', 'TTL': 60, 'Type': 'A', 'Weight': 1} - }, { - 'Action': 'CREATE', - 'ResourceRecordSet': { - 'HealthCheckId': 'hc42', - 'Name': '_octodns-eu-central-1-value.unit.tests.', - 'ResourceRecords': [{ - 'Value': '1.3.1.1'}], - 'SetIdentifier': 'eu-central-1-000', - 'TTL': 60, - 'Type': 'A', - 'Weight': 1} - }, { - 'Action': 'CREATE', - 'ResourceRecordSet': { - 'AliasTarget': { - 'DNSName': '_octodns-default-pool.unit.tests.', - 'EvaluateTargetHealth': True, - 'HostedZoneId': 'z45'}, - 'Failover': 'SECONDARY', - 'Name': '_octodns-us-east-1-pool.unit.tests.', - 'SetIdentifier': 'us-east-1-Secondary-default', - 'Type': 'A'} }, { 'Action': 'CREATE', 'ResourceRecordSet': { 'HealthCheckId': 'hc42', 'Name': '_octodns-us-east-1-value.unit.tests.', - 'ResourceRecords': [{ - 'Value': '1.5.1.2'}], - 'SetIdentifier': 'us-east-1-001', - 'TTL': 60, - 'Type': 'A', - 'Weight': 1} - }, { - 'Action': 'CREATE', - 'ResourceRecordSet': { - 'HealthCheckId': 'hc42', - 'Name': '_octodns-us-east-1-value.unit.tests.', - 'ResourceRecords': [{ - 'Value': '1.5.1.1'}], + 'ResourceRecords': [{'Value': '1.5.1.1'}], 'SetIdentifier': 'us-east-1-000', 'TTL': 60, 'Type': 'A', 'Weight': 1} + }, { + 'Action': 'CREATE', + 'ResourceRecordSet': { + 'HealthCheckId': 'hc42', + 'Name': '_octodns-us-east-1-value.unit.tests.', + 'ResourceRecords': [{'Value': '1.5.1.2'}], + 'SetIdentifier': 'us-east-1-001', + 'TTL': 60, + 'Type': 'A', + 'Weight': 1} }, { 'Action': 'CREATE', 'ResourceRecordSet': { @@ -2405,6 +2344,39 @@ class TestRoute53Records(TestCase): 'Name': '_octodns-ap-southeast-1-pool.unit.tests.', 'SetIdentifier': 'ap-southeast-1-Primary', 'Type': 'A'} + }, { + 'Action': 'CREATE', + 'ResourceRecordSet': { + 'AliasTarget': { + 'DNSName': '_octodns-eu-central-1-value.unit.tests.', + 'EvaluateTargetHealth': True, + 'HostedZoneId': 'z45'}, + 'Failover': 'PRIMARY', + 'Name': '_octodns-eu-central-1-pool.unit.tests.', + 'SetIdentifier': 'eu-central-1-Primary', + 'Type': 'A'} + }, { + 'Action': 'CREATE', + 'ResourceRecordSet': { + 'AliasTarget': { + 'DNSName': '_octodns-us-east-1-value.unit.tests.', + 'EvaluateTargetHealth': True, + 'HostedZoneId': 'z45'}, + 'Failover': 'PRIMARY', + 'Name': '_octodns-us-east-1-pool.unit.tests.', + 'SetIdentifier': 'us-east-1-Primary', + 'Type': 'A'} + }, { + 'Action': 'CREATE', + 'ResourceRecordSet': { + 'AliasTarget': { + 'DNSName': '_octodns-us-east-1-pool.unit.tests.', + 'EvaluateTargetHealth': True, + 'HostedZoneId': 'z45'}, + 'Failover': 'SECONDARY', + 'Name': '_octodns-ap-southeast-1-pool.unit.tests.', + 'SetIdentifier': 'ap-southeast-1-Secondary-us-east-1', + 'Type': 'A'} }, { 'Action': 'CREATE', 'ResourceRecordSet': { @@ -2416,7 +2388,79 @@ class TestRoute53Records(TestCase): 'Name': '_octodns-eu-central-1-pool.unit.tests.', 'SetIdentifier': 'eu-central-1-Secondary-us-east-1', 'Type': 'A'} - }], [r.mod('CREATE', []) for r in route53_records]) + }, { + 'Action': 'CREATE', + 'ResourceRecordSet': { + 'AliasTarget': { + 'DNSName': '_octodns-default-pool.unit.tests.', + 'EvaluateTargetHealth': True, + 'HostedZoneId': 'z45'}, + 'Failover': 'SECONDARY', + 'Name': '_octodns-us-east-1-pool.unit.tests.', + 'SetIdentifier': 'us-east-1-Secondary-default', + 'Type': 'A'} + }, { + 'Action': 'CREATE', + 'ResourceRecordSet': { + 'AliasTarget': { + 'DNSName': '_octodns-ap-southeast-1-pool.unit.tests.', + 'EvaluateTargetHealth': True, + 'HostedZoneId': 'z45'}, + 'GeoLocation': { + 'CountryCode': 'CN'}, + 'Name': 'unit.tests.', + 'SetIdentifier': '0-ap-southeast-1-AS-CN', + 'Type': 'A'} + }, { + 'Action': 'CREATE', + 'ResourceRecordSet': { + 'AliasTarget': { + 'DNSName': '_octodns-ap-southeast-1-pool.unit.tests.', + 'EvaluateTargetHealth': True, + 'HostedZoneId': 'z45'}, + 'GeoLocation': { + 'CountryCode': 'JP'}, + 'Name': 'unit.tests.', + 'SetIdentifier': '0-ap-southeast-1-AS-JP', + 'Type': 'A'} + }, { + 'Action': 'CREATE', + 'ResourceRecordSet': { + 'AliasTarget': { + 'DNSName': '_octodns-eu-central-1-pool.unit.tests.', + 'EvaluateTargetHealth': True, + 'HostedZoneId': 'z45'}, + 'GeoLocation': { + 'ContinentCode': 'EU'}, + 'Name': 'unit.tests.', + 'SetIdentifier': '1-eu-central-1-EU', + 'Type': 'A'} + }, { + 'Action': 'CREATE', + 'ResourceRecordSet': { + 'AliasTarget': { + 'DNSName': '_octodns-eu-central-1-pool.unit.tests.', + 'EvaluateTargetHealth': True, + 'HostedZoneId': 'z45'}, + 'GeoLocation': { + 'CountryCode': 'US', + 'SubdivisionCode': 'FL'}, + 'Name': 'unit.tests.', + 'SetIdentifier': '1-eu-central-1-NA-US-FL', + 'Type': 'A'} + }, { + 'Action': 'CREATE', + 'ResourceRecordSet': { + 'AliasTarget': { + 'DNSName': '_octodns-us-east-1-pool.unit.tests.', + 'EvaluateTargetHealth': True, + 'HostedZoneId': 'z45'}, + 'GeoLocation': { + 'CountryCode': '*'}, + 'Name': 'unit.tests.', + 'SetIdentifier': '2-us-east-1-None', + 'Type': 'A'} + }], expected_mods) for route53_record in route53_records: # Smoke test stringification From c82e94792e37e2d99df0a0d774a8615871bb84d1 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 7 Oct 2019 07:47:43 -0700 Subject: [PATCH 130/176] RackspaceProvider python3, value types hashing --- octodns/provider/rackspace.py | 15 ++++----- octodns/record/__init__.py | 15 ++++++++- tests/test_octodns_provider_rackspace.py | 8 ++--- tests/test_octodns_record.py | 40 ++++++++++++++++++++++++ 4 files changed, 66 insertions(+), 12 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 5038929..28b7f05 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -7,13 +7,16 @@ from __future__ import absolute_import, division, print_function, \ from requests import HTTPError, Session, post from collections import defaultdict import logging -import string import time from ..record import Record from .base import BaseProvider +def _value_keyer(v): + return '{}-{}-{}'.format(v.get('type', ''), v['name'], v.get('data', '')) + + def add_trailing_dot(s): assert s assert s[-1] != '.' @@ -28,12 +31,12 @@ def remove_trailing_dot(s): def escape_semicolon(s): assert s - return string.replace(s, ';', '\\;') + return s.replace(';', '\\;') def unescape_semicolon(s): assert s - return string.replace(s, '\\;', ';') + return s.replace('\\;', ';') class RackspaceProvider(BaseProvider): @@ -367,11 +370,9 @@ class RackspaceProvider(BaseProvider): self._delete('domains/{}/records?{}'.format(domain_id, params)) if updates: - data = {"records": sorted(updates, key=lambda v: v['name'])} + data = {"records": sorted(updates, key=_value_keyer)} self._put('domains/{}/records'.format(domain_id), data=data) if creates: - data = {"records": sorted(creates, key=lambda v: v['type'] + - v['name'] + - v.get('data', ''))} + data = {"records": sorted(creates, key=_value_keyer)} self._post('domains/{}/records'.format(domain_id), data=data) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index b377d14..a782018 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -723,7 +723,8 @@ class _IpList(object): @classmethod def process(cls, values): - return values + # Translating None into '' so that the list will be sortable in python3 + return [v if v is not None else '' for v in values] class Ipv4List(_IpList): @@ -918,6 +919,9 @@ class MxValue(object): 'exchange': self.exchange, } + def __hash__(self): + return hash('{} {}'.format(self.preference, self.exchange)) + def __eq__(self, other): return ((self.preference, self.exchange) == (other.preference, other.exchange)) @@ -1010,6 +1014,9 @@ class NaptrValue(object): 'replacement': self.replacement, } + def __hash__(self): + return hash(self.__repr__()) + def __eq__(self, other): return ((self.order, self.preference, self.flags, self.service, self.regexp, self.replacement) == @@ -1145,6 +1152,9 @@ class SshfpValue(object): 'fingerprint': self.fingerprint, } + def __hash__(self): + return hash(self.__repr__()) + def __eq__(self, other): return ((self.algorithm, self.fingerprint_type, self.fingerprint) == (other.algorithm, other.fingerprint_type, other.fingerprint)) @@ -1283,6 +1293,9 @@ class SrvValue(object): 'target': self.target, } + def __hash__(self): + return hash(self.__repr__()) + def __eq__(self, other): return ((self.priority, self.weight, self.port, self.target) == (other.priority, other.weight, other.port, other.target)) diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index 3dfdd5f..0a6564d 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -792,13 +792,13 @@ class TestRackspaceProvider(TestCase): ExpectedUpdates = { "records": [{ "name": "unit.tests", - "id": "A-222222", - "data": "1.2.3.5", + "id": "A-111111", + "data": "1.2.3.4", "ttl": 3600 }, { "name": "unit.tests", - "id": "A-111111", - "data": "1.2.3.4", + "id": "A-222222", + "data": "1.2.3.5", "ttl": 3600 }, { "name": "unit.tests", diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index c26b301..0845ddf 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -594,6 +594,30 @@ class TestRecord(TestCase): # __repr__ doesn't blow up a.__repr__() + # Hash + v = NaptrValue({ + 'order': 30, + 'preference': 31, + 'flags': 'M', + 'service': 'N', + 'regexp': 'O', + 'replacement': 'z', + }) + o = NaptrValue({ + 'order': 30, + 'preference': 32, + 'flags': 'M', + 'service': 'N', + 'regexp': 'O', + 'replacement': 'z', + }) + values = set() + values.add(v) + self.assertTrue(v in values) + self.assertFalse(o in values) + values.add(o) + self.assertTrue(o in values) + def test_ns(self): a_values = ['5.6.7.8.', '6.7.8.9.', '7.8.9.0.'] a_data = {'ttl': 30, 'values': a_values} @@ -1114,6 +1138,14 @@ class TestRecord(TestCase): self.assertTrue(c >= c) self.assertTrue(c <= c) + # Hash + values = set() + values.add(a) + self.assertTrue(a in values) + self.assertFalse(b in values) + values.add(b) + self.assertTrue(b in values) + def test_srv_value(self): a = SrvValue({'priority': 0, 'weight': 0, 'port': 0, 'target': 'foo.'}) b = SrvValue({'priority': 1, 'weight': 0, 'port': 0, 'target': 'foo.'}) @@ -1172,6 +1204,14 @@ class TestRecord(TestCase): self.assertTrue(c >= c) self.assertTrue(c <= c) + # Hash + values = set() + values.add(a) + self.assertTrue(a in values) + self.assertFalse(b in values) + values.add(b) + self.assertTrue(b in values) + class TestRecordValidation(TestCase): zone = Zone('unit.tests.', []) From 25768c476fee352148a8af583709410426469679 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 7 Oct 2019 07:48:55 -0700 Subject: [PATCH 131/176] SelectelProvider python3 (tests) --- tests/test_octodns_provider_selectel.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_selectel.py b/tests/test_octodns_provider_selectel.py index a2ba39e..7ad1e6b 100644 --- a/tests/test_octodns_provider_selectel.py +++ b/tests/test_octodns_provider_selectel.py @@ -6,6 +6,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from unittest import TestCase +from six import text_type import requests_mock @@ -288,7 +289,7 @@ class TestSelectelProvider(TestCase): with self.assertRaises(Exception) as ctx: SelectelProvider(123, 'fail_token') - self.assertEquals(ctx.exception.message, + self.assertEquals(text_type(ctx.exception), 'Authorization failed. Invalid or empty token.') @requests_mock.Mocker() From 90a60d3dbdd453fb691b698bc2fe8f7936e37584 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 7 Oct 2019 07:53:08 -0700 Subject: [PATCH 132/176] TransipProvider python3 --- octodns/provider/transip.py | 2 +- tests/test_octodns_provider_transip.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index 09920a9..7458e36 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -137,7 +137,7 @@ class TransipProvider(BaseProvider): try: self._client.get_info(plan.desired.name[:-1]) except WebFault as e: - self.log.warning('_apply: %s ', e.message) + self.log.exception('_apply: get_info failed') raise e _dns_entries = [] diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py index c56509a..3fbfc44 100644 --- a/tests/test_octodns_provider_transip.py +++ b/tests/test_octodns_provider_transip.py @@ -5,8 +5,8 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -# from mock import Mock, call from os.path import dirname, join +from six import text_type from suds import WebFault @@ -176,7 +176,7 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd self.assertEquals( 'populate: (102) Transip used as target' + ' for non-existing zone: notfound.unit.tests.', - ctx.exception.message) + text_type(ctx.exception)) # Happy Plan - Zone does not exists # Won't trigger an exception if provider is NOT used as a target for a From 14063186f3e9bc419fa6eb50f30e9a7dc4feaa9c Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 7 Oct 2019 07:56:10 -0700 Subject: [PATCH 133/176] YamlProvider python3, tests --- tests/test_octodns_provider_yaml.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index 700f3c3..e16aaa5 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -82,9 +82,9 @@ class TestYamlProvider(TestCase): target.populate(reloaded) self.assertDictEqual( {'included': ['test']}, - filter( + list(filter( lambda x: x.name == 'included', reloaded.records - )[0]._octodns) + ))[0]._octodns) self.assertFalse(zone.changes(reloaded, target=source)) @@ -120,7 +120,7 @@ class TestYamlProvider(TestCase): self.assertTrue('value' in data.pop('www.sub')) # make sure nothing is left - self.assertEquals([], data.keys()) + self.assertEquals([], list(data.keys())) with open(dynamic_yaml_file) as fh: data = safe_load(fh.read()) @@ -149,7 +149,7 @@ class TestYamlProvider(TestCase): # self.assertTrue('dynamic' in dyna) # make sure nothing is left - self.assertEquals([], data.keys()) + self.assertEquals([], list(data.keys())) def test_empty(self): source = YamlProvider('test', join(dirname(__file__), 'config')) @@ -255,8 +255,8 @@ class TestSplitYamlProvider(TestCase): # We add everything plan = target.plan(zone) - self.assertEquals(15, len(filter(lambda c: isinstance(c, Create), - plan.changes))) + self.assertEquals(15, len(list(filter( + lambda c: isinstance(c, Create), plan.changes)))) self.assertFalse(isdir(zone_dir)) # Now actually do it @@ -264,8 +264,8 @@ class TestSplitYamlProvider(TestCase): # Dynamic plan plan = target.plan(dynamic_zone) - self.assertEquals(5, len(filter(lambda c: isinstance(c, Create), - plan.changes))) + self.assertEquals(5, len(list(filter( + lambda c: isinstance(c, Create), plan.changes)))) self.assertFalse(isdir(dynamic_zone_dir)) # Apply it self.assertEquals(5, target.apply(plan)) @@ -276,16 +276,16 @@ class TestSplitYamlProvider(TestCase): target.populate(reloaded) self.assertDictEqual( {'included': ['test']}, - filter( + list(filter( lambda x: x.name == 'included', reloaded.records - )[0]._octodns) + ))[0]._octodns) self.assertFalse(zone.changes(reloaded, target=source)) # A 2nd sync should still create everything plan = target.plan(zone) - self.assertEquals(15, len(filter(lambda c: isinstance(c, Create), - plan.changes))) + self.assertEquals(15, len(list(filter( + lambda c: isinstance(c, Create), plan.changes)))) yaml_file = join(zone_dir, '$unit.tests.yaml') self.assertTrue(isfile(yaml_file)) From e0c5962d79df12ace77d763833010b0193b6dce4 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 7 Oct 2019 07:58:44 -0700 Subject: [PATCH 134/176] AxfrSource python3 --- octodns/source/axfr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/source/axfr.py b/octodns/source/axfr.py index f35c4b3..be2acf5 100644 --- a/octodns/source/axfr.py +++ b/octodns/source/axfr.py @@ -15,6 +15,7 @@ from dns.exception import DNSException from collections import defaultdict from os import listdir from os.path import join +from six import text_type import logging from ..record import Record @@ -179,8 +180,7 @@ class ZoneFileSourceNotFound(ZoneFileSourceException): class ZoneFileSourceLoadFailure(ZoneFileSourceException): def __init__(self, error): - super(ZoneFileSourceLoadFailure, self).__init__( - error.message) + super(ZoneFileSourceLoadFailure, self).__init__(text_type(error)) class ZoneFileSource(AxfrBaseSource): From 0708b797da2b1208bd66b10881deca40585ce6d4 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 7 Oct 2019 08:29:50 -0700 Subject: [PATCH 135/176] TinyDnsSource python3 --- octodns/source/tinydns.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/octodns/source/tinydns.py b/octodns/source/tinydns.py index dc2bc1b..0ee100a 100755 --- a/octodns/source/tinydns.py +++ b/octodns/source/tinydns.py @@ -67,7 +67,8 @@ class TinyDnsBaseSource(BaseSource): values = [] for record in records: - new_value = record[0].decode('unicode-escape').replace(";", "\\;") + new_value = record[0].encode('latin1').decode('unicode-escape') \ + .replace(";", "\\;") values.append(new_value) try: From db8de8acb888713db8d88784de090c0e45c122ad Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 7 Oct 2019 08:41:28 -0700 Subject: [PATCH 136/176] Fix Manager ordering assumptions --- tests/config/provider-problems.yaml | 28 ++++++++++++++++++++++++++++ tests/config/unknown-provider.yaml | 16 ---------------- tests/test_octodns_manager.py | 10 +++++----- 3 files changed, 33 insertions(+), 21 deletions(-) create mode 100644 tests/config/provider-problems.yaml diff --git a/tests/config/provider-problems.yaml b/tests/config/provider-problems.yaml new file mode 100644 index 0000000..9071046 --- /dev/null +++ b/tests/config/provider-problems.yaml @@ -0,0 +1,28 @@ +providers: + yaml: + class: octodns.provider.yaml.YamlProvider + directory: ./config + simple_source: + class: helpers.SimpleSource +zones: + missing.sources.: + targets: + - yaml + missing.targets.: + sources: + - yaml + unknown.source.: + sources: + - not-there + targets: + - yaml + unknown.target.: + sources: + - yaml + targets: + - not-there-either + not.targetable.: + sources: + - yaml + targets: + - simple_source diff --git a/tests/config/unknown-provider.yaml b/tests/config/unknown-provider.yaml index 9071046..a0e9f55 100644 --- a/tests/config/unknown-provider.yaml +++ b/tests/config/unknown-provider.yaml @@ -5,24 +5,8 @@ providers: simple_source: class: helpers.SimpleSource zones: - missing.sources.: - targets: - - yaml - missing.targets.: - sources: - - yaml unknown.source.: sources: - not-there targets: - yaml - unknown.target.: - sources: - - yaml - targets: - - not-there-either - not.targetable.: - sources: - - yaml - targets: - - simple_source diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 43feed5..60a1922 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -62,25 +62,25 @@ class TestManager(TestCase): def test_missing_source(self): with self.assertRaises(Exception) as ctx: - Manager(get_config_filename('unknown-provider.yaml')) \ + Manager(get_config_filename('provider-problems.yaml')) \ .sync(['missing.sources.']) self.assertTrue('missing sources' in text_type(ctx.exception)) def test_missing_targets(self): with self.assertRaises(Exception) as ctx: - Manager(get_config_filename('unknown-provider.yaml')) \ + Manager(get_config_filename('provider-problems.yaml')) \ .sync(['missing.targets.']) self.assertTrue('missing targets' in text_type(ctx.exception)) def test_unknown_source(self): with self.assertRaises(Exception) as ctx: - Manager(get_config_filename('unknown-provider.yaml')) \ + Manager(get_config_filename('provider-problems.yaml')) \ .sync(['unknown.source.']) self.assertTrue('unknown source' in text_type(ctx.exception)) def test_unknown_target(self): with self.assertRaises(Exception) as ctx: - Manager(get_config_filename('unknown-provider.yaml')) \ + Manager(get_config_filename('provider-problems.yaml')) \ .sync(['unknown.target.']) self.assertTrue('unknown target' in text_type(ctx.exception)) @@ -99,7 +99,7 @@ class TestManager(TestCase): def test_source_only_as_a_target(self): with self.assertRaises(Exception) as ctx: - Manager(get_config_filename('unknown-provider.yaml')) \ + Manager(get_config_filename('provider-problems.yaml')) \ .sync(['not.targetable.']) self.assertTrue('does not support targeting' in text_type(ctx.exception)) From bfa1fadde97f7a3567103e2bcb500d7e484f0967 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 7 Oct 2019 08:44:25 -0700 Subject: [PATCH 137/176] Fix CloudflareProvider test ordering assumptions --- tests/test_octodns_provider_cloudflare.py | 24 ++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index c656cab..013c5f7 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -313,7 +313,7 @@ class TestCloudflareProvider(TestCase): 'dns_records/fc12ab34cd5611334422ab3322997653'), call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/' 'dns_records/fc12ab34cd5611334422ab3322997654') - ]) + ], any_order=True) def test_update_add_swap(self): provider = CloudflareProvider('test', 'email', 'token') @@ -743,23 +743,25 @@ class TestCloudflareProvider(TestCase): # the CDN. self.assertEquals(3, len(zone.records)) - record = list(zone.records)[0] - self.assertEquals('multi', record.name) - self.assertEquals('multi.unit.tests.', record.fqdn) - self.assertEquals('CNAME', record._type) - self.assertEquals('multi.unit.tests.cdn.cloudflare.net.', record.value) + ordered = sorted(zone.records, key=lambda r: r.name) - record = list(zone.records)[1] + record = ordered[0] + self.assertEquals('a', record.name) + self.assertEquals('a.unit.tests.', record.fqdn) + self.assertEquals('CNAME', record._type) + self.assertEquals('a.unit.tests.cdn.cloudflare.net.', record.value) + + record = ordered[1] self.assertEquals('cname', record.name) self.assertEquals('cname.unit.tests.', record.fqdn) self.assertEquals('CNAME', record._type) self.assertEquals('cname.unit.tests.cdn.cloudflare.net.', record.value) - record = list(zone.records)[2] - self.assertEquals('a', record.name) - self.assertEquals('a.unit.tests.', record.fqdn) + record = ordered[2] + self.assertEquals('multi', record.name) + self.assertEquals('multi.unit.tests.', record.fqdn) self.assertEquals('CNAME', record._type) - self.assertEquals('a.unit.tests.cdn.cloudflare.net.', record.value) + self.assertEquals('multi.unit.tests.cdn.cloudflare.net.', record.value) # CDN enabled records can't be updated, we don't know the real values # never point a Cloudflare record to itself. From 7958233fccf9ea22d95e2fd06c48d7d0a4529e26 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 7 Oct 2019 09:17:48 -0700 Subject: [PATCH 138/176] Consistently order changes :-/ Many providers make their modifications in the order that changes comes. In python3 this causes things to be inconsistently ordered. That mostly works, but could result in hidenbugs (e.g. Route53Provider's batching could be completely different based on the order it sees changes.) Sorting changes consistently is a good thing and it shouldn't hurt situations where providers are already doing their own ordering. All-in-all more consistent is better and we have to be explicit with python 3. --- octodns/provider/plan.py | 6 +- octodns/record/__init__.py | 6 ++ tests/test_octodns_provider_cloudflare.py | 2 +- tests/test_octodns_provider_digitalocean.py | 15 +++- tests/test_octodns_provider_dnsimple.py | 27 +++++- tests/test_octodns_provider_dnsmadeeasy.py | 22 ++++- tests/test_octodns_provider_ns1.py | 10 +-- tests/test_octodns_provider_ovh.py | 99 ++++++++++----------- tests/test_octodns_provider_route53.py | 4 +- 9 files changed, 129 insertions(+), 62 deletions(-) diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py index d4589f2..5b8be68 100644 --- a/octodns/provider/plan.py +++ b/octodns/provider/plan.py @@ -28,7 +28,11 @@ class Plan(object): delete_pcent_threshold=MAX_SAFE_DELETE_PCENT): self.existing = existing self.desired = desired - self.changes = changes + # Sort changes to ensure we always have a consistent ordering for + # things that make assumptions about that. Many providers will do their + # own ordering to ensure things happen in a way that makes sense to + # them and/or is as safe as possible. + self.changes = sorted(changes) self.exists = exists self.update_pcent_threshold = update_pcent_threshold self.delete_pcent_threshold = delete_pcent_threshold diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index a782018..47d2e9b 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -25,6 +25,12 @@ class Change(object): 'Returns new if we have one, existing otherwise' return self.new or self.existing + def __lt__(self, other): + self_record = self.record + other_record = other.record + return ((self_record.name, self_record._type) < + (other_record.name, other_record._type)) + class Create(Change): diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 013c5f7..3581033 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -313,7 +313,7 @@ class TestCloudflareProvider(TestCase): 'dns_records/fc12ab34cd5611334422ab3322997653'), call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/' 'dns_records/fc12ab34cd5611334422ab3322997654') - ], any_order=True) + ]) def test_update_add_swap(self): provider = CloudflareProvider('test', 'email', 'token') diff --git a/tests/test_octodns_provider_digitalocean.py b/tests/test_octodns_provider_digitalocean.py index 448b26e..ebb5319 100644 --- a/tests/test_octodns_provider_digitalocean.py +++ b/tests/test_octodns_provider_digitalocean.py @@ -176,7 +176,20 @@ class TestDigitalOceanProvider(TestCase): call('GET', '/domains/unit.tests/records', {'page': 1}), # delete the initial A record call('DELETE', '/domains/unit.tests/records/11189877'), - # created at least one of the record with expected data + # created at least some of the record with expected data + call('POST', '/domains/unit.tests/records', data={ + 'data': '1.2.3.4', + 'name': '@', + 'ttl': 300, 'type': 'A'}), + call('POST', '/domains/unit.tests/records', data={ + 'data': '1.2.3.5', + 'name': '@', + 'ttl': 300, 'type': 'A'}), + call('POST', '/domains/unit.tests/records', data={ + 'data': 'ca.unit.tests.', + 'flags': 0, 'name': '@', + 'tag': 'issue', + 'ttl': 3600, 'type': 'CAA'}), call('POST', '/domains/unit.tests/records', data={ 'name': '_srv._tcp', 'weight': 20, diff --git a/tests/test_octodns_provider_dnsimple.py b/tests/test_octodns_provider_dnsimple.py index 9e8586c..e3a9b8d 100644 --- a/tests/test_octodns_provider_dnsimple.py +++ b/tests/test_octodns_provider_dnsimple.py @@ -139,7 +139,32 @@ class TestDnsimpleProvider(TestCase): provider._client._request.assert_has_calls([ # created the domain call('POST', '/domains', data={'name': 'unit.tests'}), - # created at least one of the record with expected data + # created at least some of the record with expected data + call('POST', '/zones/unit.tests/records', data={ + 'content': '1.2.3.4', + 'type': 'A', + 'name': '', + 'ttl': 300}), + call('POST', '/zones/unit.tests/records', data={ + 'content': '1.2.3.5', + 'type': 'A', + 'name': '', + 'ttl': 300}), + call('POST', '/zones/unit.tests/records', data={ + 'content': '0 issue "ca.unit.tests"', + 'type': 'CAA', + 'name': '', + 'ttl': 3600}), + call('POST', '/zones/unit.tests/records', data={ + 'content': '1 1 7491973e5f8b39d5327cd4e08bc81b05f7710b49', + 'type': 'SSHFP', + 'name': '', + 'ttl': 3600}), + call('POST', '/zones/unit.tests/records', data={ + 'content': '1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73', + 'type': 'SSHFP', + 'name': '', + 'ttl': 3600}), call('POST', '/zones/unit.tests/records', data={ 'content': '20 30 foo-1.unit.tests.', 'priority': 10, diff --git a/tests/test_octodns_provider_dnsmadeeasy.py b/tests/test_octodns_provider_dnsmadeeasy.py index ea14376..ba61b94 100644 --- a/tests/test_octodns_provider_dnsmadeeasy.py +++ b/tests/test_octodns_provider_dnsmadeeasy.py @@ -149,7 +149,27 @@ class TestDnsMadeEasyProvider(TestCase): call('POST', '/', data={'name': 'unit.tests'}), # get all domains to build the cache call('GET', '/'), - # created at least one of the record with expected data + # created at least some of the record with expected data + call('POST', '/123123/records', data={ + 'type': 'A', + 'name': '', + 'value': '1.2.3.4', + 'ttl': 300}), + call('POST', '/123123/records', data={ + 'type': 'A', + 'name': '', + 'value': '1.2.3.5', + 'ttl': 300}), + call('POST', '/123123/records', data={ + 'type': 'ANAME', + 'name': '', + 'value': 'aname.unit.tests.', + 'ttl': 1800}), + call('POST', '/123123/records', data={ + 'name': '', + 'value': 'ca.unit.tests', + 'issuerCritical': 0, 'caaType': 'issue', + 'ttl': 3600, 'type': 'CAA'}), call('POST', '/123123/records', data={ 'name': '_srv._tcp', 'weight': 20, diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 91d1a3f..9d23806 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -400,9 +400,9 @@ class TestNs1Provider(TestCase): self.assertEquals(3, got_n) nsone_zone.loadRecord.assert_has_calls([ call('unit.tests', u'A'), - call('geo', u'A'), call('delete-me', u'A'), - ], any_order=True) + call('geo', u'A'), + ]) mock_record.assert_has_calls([ call.update(answers=[{'answer': [u'1.2.3.4'], 'meta': {}}], filters=[], @@ -410,6 +410,8 @@ class TestNs1Provider(TestCase): call.update(answers=[{u'answer': [u'1.2.3.4'], u'meta': {}}], filters=[], ttl=32), + call.delete(), + call.delete(), call.update( answers=[ {u'answer': [u'101.102.103.104'], u'meta': {}}, @@ -427,9 +429,7 @@ class TestNs1Provider(TestCase): {u'filter': u'select_first_n', u'config': {u'N': 1}}, ], ttl=34), - call.delete(), - call.delete() - ], any_order=True) + ]) def test_escaping(self): provider = Ns1Provider('test', 'api-key') diff --git a/tests/test_octodns_provider_ovh.py b/tests/test_octodns_provider_ovh.py index 1f41abf..924591f 100644 --- a/tests/test_octodns_provider_ovh.py +++ b/tests/test_octodns_provider_ovh.py @@ -382,65 +382,64 @@ class TestOvhProvider(TestCase): get_mock.side_effect = [[100], [101], [102], [103]] provider.apply(plan) wanted_calls = [ - call(u'/domain/zone/unit.tests/record', fieldType=u'TXT', - subDomain='txt', target=u'TXT text', ttl=1400), - call(u'/domain/zone/unit.tests/record', fieldType=u'DKIM', - subDomain='dkim', target=self.valid_dkim_key, - ttl=1300), - call(u'/domain/zone/unit.tests/record', fieldType=u'A', - subDomain=u'', target=u'1.2.3.4', ttl=100), - call(u'/domain/zone/unit.tests/record', fieldType=u'SRV', + call('/domain/zone/unit.tests/record', fieldType='A', + subDomain='', target='1.2.3.4', ttl=100), + call('/domain/zone/unit.tests/record', fieldType='AAAA', + subDomain='', target='1:1ec:1::1', ttl=200), + call('/domain/zone/unit.tests/record', fieldType='MX', + subDomain='', target='10 mx1.unit.tests.', ttl=400), + call('/domain/zone/unit.tests/record', fieldType='SPF', + subDomain='', + target='v=spf1 include:unit.texts.redirect ~all', + ttl=1000), + call('/domain/zone/unit.tests/record', fieldType='SSHFP', + subDomain='', + target='1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73', + ttl=1100), + call('/domain/zone/unit.tests/record', fieldType='PTR', + subDomain='4', target='unit.tests.', ttl=900), + call('/domain/zone/unit.tests/record', fieldType='SRV', subDomain='_srv._tcp', - target=u'10 20 30 foo-1.unit.tests.', ttl=800), - call(u'/domain/zone/unit.tests/record', fieldType=u'SRV', + target='10 20 30 foo-1.unit.tests.', ttl=800), + call('/domain/zone/unit.tests/record', fieldType='SRV', subDomain='_srv._tcp', - target=u'40 50 60 foo-2.unit.tests.', ttl=800), - call(u'/domain/zone/unit.tests/record', fieldType=u'PTR', - subDomain='4', target=u'unit.tests.', ttl=900), - call(u'/domain/zone/unit.tests/record', fieldType=u'NS', - subDomain='www3', target=u'ns3.unit.tests.', ttl=700), - call(u'/domain/zone/unit.tests/record', fieldType=u'NS', - subDomain='www3', target=u'ns4.unit.tests.', ttl=700), - call(u'/domain/zone/unit.tests/record', - fieldType=u'SSHFP', subDomain=u'', ttl=1100, - target=u'1 1 bf6b6825d2977c511a475bbefb88a' - u'ad54' - u'a92ac73', - ), - call(u'/domain/zone/unit.tests/record', fieldType=u'AAAA', - subDomain=u'', target=u'1:1ec:1::1', ttl=200), - call(u'/domain/zone/unit.tests/record', fieldType=u'MX', - subDomain=u'', target=u'10 mx1.unit.tests.', ttl=400), - call(u'/domain/zone/unit.tests/record', fieldType=u'CNAME', - subDomain='www2', target=u'unit.tests.', ttl=300), - call(u'/domain/zone/unit.tests/record', fieldType=u'SPF', - subDomain=u'', ttl=1000, - target=u'v=spf1 include:unit.texts.' - u'redirect ~all', - ), - call(u'/domain/zone/unit.tests/record', fieldType=u'A', - subDomain='sub', target=u'1.2.3.4', ttl=200), - call(u'/domain/zone/unit.tests/record', fieldType=u'NAPTR', - subDomain='naptr', ttl=500, - target=u'10 100 "S" "SIP+D2U" "!^.*$!sip:' - u'info@bar' - u'.example.com!" .' - ), - call(u'/domain/zone/unit.tests/refresh')] + target='40 50 60 foo-2.unit.tests.', ttl=800), + call('/domain/zone/unit.tests/record', fieldType='DKIM', + subDomain='dkim', + target='p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCxLaG' + '16G4SaEcXVdiIxTg7gKSGbHKQLm30CHib1h9FzS9nkcyvQSyQj1r' + 'MFyqC//tft3ohx3nvJl+bGCWxdtLYDSmir9PW54e5CTdxEh8MWRk' + 'BO3StF6QG/tAh3aTGDmkqhIJGLb87iHvpmVKqURmEUzJPv5KPJfW' + 'LofADI+q9lQIDAQAB', ttl=1300), + call('/domain/zone/unit.tests/record', fieldType='NAPTR', + subDomain='naptr', + target='10 100 "S" "SIP+D2U" "!^.*$!sip:info@bar.exam' + 'ple.com!" .', ttl=500), + call('/domain/zone/unit.tests/record', fieldType='A', + subDomain='sub', target='1.2.3.4', ttl=200), + call('/domain/zone/unit.tests/record', fieldType='TXT', + subDomain='txt', target='TXT text', ttl=1400), + call('/domain/zone/unit.tests/record', fieldType='CNAME', + subDomain='www2', target='unit.tests.', ttl=300), + call('/domain/zone/unit.tests/record', fieldType='NS', + subDomain='www3', target='ns3.unit.tests.', ttl=700), + call('/domain/zone/unit.tests/record', fieldType='NS', + subDomain='www3', target='ns4.unit.tests.', ttl=700), + call('/domain/zone/unit.tests/refresh')] - post_mock.assert_has_calls(wanted_calls, any_order=True) + post_mock.assert_has_calls(wanted_calls) # Get for delete calls wanted_get_calls = [ - call(u'/domain/zone/unit.tests/record', fieldType=u'TXT', - subDomain='txt'), + call(u'/domain/zone/unit.tests/record', fieldType=u'A', + subDomain=u''), call(u'/domain/zone/unit.tests/record', fieldType=u'DKIM', subDomain='dkim'), call(u'/domain/zone/unit.tests/record', fieldType=u'A', - subDomain=u''), - call(u'/domain/zone/unit.tests/record', fieldType=u'A', - subDomain='fake')] - get_mock.assert_has_calls(wanted_get_calls, any_order=True) + subDomain='fake'), + call(u'/domain/zone/unit.tests/record', fieldType=u'TXT', + subDomain='txt')] + get_mock.assert_has_calls(wanted_get_calls) # 4 delete calls for update and delete delete_mock.assert_has_calls( [call(u'/domain/zone/unit.tests/record/100'), diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index b0ee342..7691804 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -1882,10 +1882,10 @@ class TestRoute53Provider(TestCase): @patch('octodns.provider.route53.Route53Provider._really_apply') def test_apply_3(self, really_apply_mock, _): - # with a max of seven modifications, four calls + # with a max of seven modifications, three calls provider, plan = self._get_test_plan(7) provider.apply(plan) - self.assertEquals(4, really_apply_mock.call_count) + self.assertEquals(3, really_apply_mock.call_count) @patch('octodns.provider.route53.Route53Provider._load_records') @patch('octodns.provider.route53.Route53Provider._really_apply') From be06a5da94fe3ffdbac2d44dec9e426e7ba320e3 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 7 Oct 2019 09:31:15 -0700 Subject: [PATCH 139/176] Make sure map and keys are lists when needed --- octodns/provider/dyn.py | 4 ++-- tests/test_octodns_provider_dyn.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/octodns/provider/dyn.py b/octodns/provider/dyn.py index c306238..30ba8bd 100644 --- a/octodns/provider/dyn.py +++ b/octodns/provider/dyn.py @@ -903,10 +903,10 @@ class DynProvider(BaseProvider): # Sort the values for consistent ordering so that we can compare values = sorted(values, key=_dynamic_value_sort_key) # Ensure that weight is included and if not use the default - values = map(lambda v: { + values = list(map(lambda v: { 'value': v['value'], 'weight': v.get('weight', 1), - }, values) + }, values)) # Walk through our existing pools looking for a match we can use for pool in pools: diff --git a/tests/test_octodns_provider_dyn.py b/tests/test_octodns_provider_dyn.py index 4f224fc..7c023fd 100644 --- a/tests/test_octodns_provider_dyn.py +++ b/tests/test_octodns_provider_dyn.py @@ -670,8 +670,8 @@ class TestDynProviderGeo(TestCase): tds = provider.traffic_directors self.assertEquals(set(['unit.tests.', 'geo.unit.tests.']), set(tds.keys())) - self.assertEquals(['A'], tds['unit.tests.'].keys()) - self.assertEquals(['A'], tds['geo.unit.tests.'].keys()) + self.assertEquals(['A'], list(tds['unit.tests.'].keys())) + self.assertEquals(['A'], list(tds['geo.unit.tests.'].keys())) provider.log.warn.assert_called_with("Unsupported TrafficDirector " "'%s'", 'something else') From 25cc4f42db439327747bb8101a6d4d7381f0ebb8 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 7 Oct 2019 09:33:14 -0700 Subject: [PATCH 140/176] Explicit list on filter when checking for non-existant targets --- octodns/manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/octodns/manager.py b/octodns/manager.py index 3984a5a..1c6a401 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -263,7 +263,8 @@ class Manager(object): except KeyError: raise Exception('Zone {} is missing targets'.format(zone_name)) if eligible_targets: - targets = filter(lambda d: d in eligible_targets, targets) + targets = list(filter(lambda d: d in eligible_targets, + targets)) if not targets: # Don't bother planning (and more importantly populating) zones From 3c75dc81f890c707895af25e22941050062c2ea4 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 7 Oct 2019 09:34:10 -0700 Subject: [PATCH 141/176] Require python 3.7 to pass --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index b1a8204..eb609ff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,9 +3,6 @@ matrix: include: - python: 2.7 - python: 3.7 - dist: xenial # required for Python >= 3.7 on Travis CI - allow_failures: - - python: 3.7 before_install: pip install --upgrade pip script: ./script/cibuild notifications: From 7867ad2093f775fbb4d83564600c299e9e51f31f Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 7 Oct 2019 09:40:23 -0700 Subject: [PATCH 142/176] Use six's StringIO, remove compat.py --- octodns/compat.py | 10 ---------- octodns/provider/plan.py | 3 +-- tests/test_octodns_plan.py | 3 +-- tests/test_octodns_yaml.py | 2 +- 4 files changed, 3 insertions(+), 15 deletions(-) delete mode 100644 octodns/compat.py diff --git a/octodns/compat.py b/octodns/compat.py deleted file mode 100644 index 6586cff..0000000 --- a/octodns/compat.py +++ /dev/null @@ -1,10 +0,0 @@ -# -# Python 2/3 compat bits -# - -try: # pragma: no cover - from StringIO import StringIO -except ImportError: # pragma: no cover - from io import StringIO - -StringIO diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py index 5b8be68..af6863a 100644 --- a/octodns/provider/plan.py +++ b/octodns/provider/plan.py @@ -8,8 +8,7 @@ from __future__ import absolute_import, division, print_function, \ from logging import DEBUG, ERROR, INFO, WARN, getLogger from sys import stdout -from six import text_type -from ..compat import StringIO +from six import StringIO, text_type class UnsafePlan(Exception): diff --git a/tests/test_octodns_plan.py b/tests/test_octodns_plan.py index a017431..9cf812d 100644 --- a/tests/test_octodns_plan.py +++ b/tests/test_octodns_plan.py @@ -6,10 +6,9 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from logging import getLogger -from six import text_type +from six import StringIO, text_type from unittest import TestCase -from octodns.compat import StringIO from octodns.provider.plan import Plan, PlanHtml, PlanLogger, PlanMarkdown from octodns.record import Create, Delete, Record, Update from octodns.zone import Zone diff --git a/tests/test_octodns_yaml.py b/tests/test_octodns_yaml.py index ddcd818..f211854 100644 --- a/tests/test_octodns_yaml.py +++ b/tests/test_octodns_yaml.py @@ -5,10 +5,10 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +from six import StringIO from unittest import TestCase from yaml.constructor import ConstructorError -from octodns.compat import StringIO from octodns.yaml import safe_dump, safe_load From 7c34247e3ba81352ab0730b7b341050da11b7412 Mon Sep 17 00:00:00 2001 From: Josef Vogt Date: Wed, 9 Oct 2019 15:56:28 +0200 Subject: [PATCH 143/176] Fix 'server error: zone not found' for NS1 provider --- README.md | 2 +- octodns/provider/ns1.py | 6 +++--- requirements.txt | 2 +- tests/test_octodns_provider_ns1.py | 8 ++++---- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 83f0bd1..c3ddb0b 100644 --- a/README.md +++ b/README.md @@ -182,7 +182,7 @@ The above command pulled the existing data out of Route53 and placed the results | [EtcHostsProvider](/octodns/provider/etc_hosts.py) | | A, AAAA, ALIAS, CNAME | No | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | | [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | | -| [Ns1Provider](/octodns/provider/ns1.py) | nsone | All | Partial Geo | No health checking for GeoDNS | +| [Ns1Provider](/octodns/provider/ns1.py) | ns1-python | All | Partial Geo | No health checking for GeoDNS | | [OVH](/octodns/provider/ovh.py) | ovh | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT, DKIM | No | | | [PowerDnsProvider](/octodns/provider/powerdns.py) | | All | No | | | [Rackspace](/octodns/provider/rackspace.py) | | A, AAAA, ALIAS, CNAME, MX, NS, PTR, SPF, TXT | No | | diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 5fdf5b0..626b59e 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -8,8 +8,8 @@ from __future__ import absolute_import, division, print_function, \ from logging import getLogger from itertools import chain from collections import OrderedDict, defaultdict -from nsone import NSONE -from nsone.rest.errors import RateLimitException, ResourceException +from ns1 import NS1 +from ns1.rest.errors import RateLimitException, ResourceException from incf.countryutils import transformations from time import sleep @@ -36,7 +36,7 @@ class Ns1Provider(BaseProvider): self.log = getLogger('Ns1Provider[{}]'.format(id)) self.log.debug('__init__: id=%s, api_key=***', id) super(Ns1Provider, self).__init__(id, *args, **kwargs) - self._client = NSONE(apiKey=api_key) + self._client = NS1(apiKey=api_key) def _data_for_A(self, _type, record): # record meta (which would include geo information is only diff --git a/requirements.txt b/requirements.txt index bb373d0..1960ffe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ ipaddress==1.0.22 jmespath==0.9.3 msrestazure==0.6.0 natsort==5.5.0 -nsone==0.9.100 +ns1-python==0.12.0 ovh==0.4.8 python-dateutil==2.6.1 requests==2.22.0 diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 178ce53..22449f6 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -6,7 +6,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from mock import Mock, call, patch -from nsone.rest.errors import AuthException, RateLimitException, \ +from ns1.rest.errors import AuthException, RateLimitException, \ ResourceException from unittest import TestCase @@ -171,7 +171,7 @@ class TestNs1Provider(TestCase): 'domain': 'unit.tests.', }] - @patch('nsone.NSONE.loadZone') + @patch('ns1.NS1.loadZone') def test_populate(self, load_mock): provider = Ns1Provider('test', 'api-key') @@ -290,8 +290,8 @@ class TestNs1Provider(TestCase): self.assertEquals(self.expected, zone.records) self.assertEquals(('unit.tests',), load_mock.call_args[0]) - @patch('nsone.NSONE.createZone') - @patch('nsone.NSONE.loadZone') + @patch('ns1.NS1.createZone') + @patch('ns1.NS1.loadZone') def test_sync(self, load_mock, create_mock): provider = Ns1Provider('test', 'api-key') From 0a7d63ef06e59dab5f0cd6b110c39b30b0114800 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 9 Oct 2019 14:35:53 -0700 Subject: [PATCH 144/176] Show line numbers missing coverage --- script/coverage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/coverage b/script/coverage index 8552eba..ad8189e 100755 --- a/script/coverage +++ b/script/coverage @@ -29,7 +29,7 @@ export GOOGLE_APPLICATION_CREDENTIALS= coverage run --branch --source=octodns --omit=octodns/cmds/* "$(command -v nosetests)" --with-xunit "$@" coverage html coverage xml -coverage report +coverage report --show-missing coverage report | grep ^TOTAL | grep -qv 100% && { echo "Incomplete code coverage" >&2 exit 1 From 3f487197df7e6dacdd703b8e9888d29b67f74ba1 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 9 Oct 2019 15:09:37 -0700 Subject: [PATCH 145/176] Manager throws ManagerException rather than Exception, more robust tests --- octodns/manager.py | 71 ++++++++++++++++++----------------- tests/test_octodns_manager.py | 39 +++++++++---------- 2 files changed, 57 insertions(+), 53 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index 1c6a401..85de76b 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -69,6 +69,10 @@ class MainThreadExecutor(object): return MakeThreadFuture(func, args, kwargs) +class ManagerException(Exception): + pass + + class Manager(object): log = logging.getLogger('Manager') @@ -105,16 +109,16 @@ class Manager(object): _class = provider_config.pop('class') except KeyError: self.log.exception('Invalid provider class') - raise Exception('Provider {} is missing class' - .format(provider_name)) + raise ManagerException('Provider {} is missing class' + .format(provider_name)) _class = self._get_named_class('provider', _class) kwargs = self._build_kwargs(provider_config) try: self.providers[provider_name] = _class(provider_name, **kwargs) except TypeError: self.log.exception('Invalid provider config') - raise Exception('Incorrect provider config for {}' - .format(provider_name)) + raise ManagerException('Incorrect provider config for {}' + .format(provider_name)) zone_tree = {} # sort by reversed strings so that parent zones always come first @@ -148,8 +152,8 @@ class Manager(object): _class = plan_output_config.pop('class') except KeyError: self.log.exception('Invalid plan_output class') - raise Exception('plan_output {} is missing class' - .format(plan_output_name)) + raise ManagerException('plan_output {} is missing class' + .format(plan_output_name)) _class = self._get_named_class('plan_output', _class) kwargs = self._build_kwargs(plan_output_config) try: @@ -157,8 +161,8 @@ class Manager(object): _class(plan_output_name, **kwargs) except TypeError: self.log.exception('Invalid plan_output config') - raise Exception('Incorrect plan_output config for {}' - .format(plan_output_name)) + raise ManagerException('Incorrect plan_output config for {}' + .format(plan_output_name)) def _get_named_class(self, _type, _class): try: @@ -167,13 +171,15 @@ class Manager(object): except (ImportError, ValueError): self.log.exception('_get_{}_class: Unable to import ' 'module %s', _class) - raise Exception('Unknown {} class: {}'.format(_type, _class)) + raise ManagerException('Unknown {} class: {}' + .format(_type, _class)) try: return getattr(module, class_name) except AttributeError: self.log.exception('_get_{}_class: Unable to get class %s ' 'from module %s', class_name, module) - raise Exception('Unknown {} class: {}'.format(_type, _class)) + raise ManagerException('Unknown {} class: {}' + .format(_type, _class)) def _build_kwargs(self, source): # Build up the arguments we need to pass to the provider @@ -186,9 +192,9 @@ class Manager(object): v = environ[env_var] except KeyError: self.log.exception('Invalid provider config') - raise Exception('Incorrect provider config, ' - 'missing env var {}' - .format(env_var)) + raise ManagerException('Incorrect provider config, ' + 'missing env var {}' + .format(env_var)) except AttributeError: pass kwargs[k] = v @@ -256,12 +262,14 @@ class Manager(object): try: sources = config['sources'] except KeyError: - raise Exception('Zone {} is missing sources'.format(zone_name)) + raise ManagerException('Zone {} is missing sources' + .format(zone_name)) try: targets = config['targets'] except KeyError: - raise Exception('Zone {} is missing targets'.format(zone_name)) + raise ManagerException('Zone {} is missing targets' + .format(zone_name)) if eligible_targets: targets = list(filter(lambda d: d in eligible_targets, targets)) @@ -276,26 +284,23 @@ class Manager(object): self.log.info('sync: sources=%s -> targets=%s', sources, targets) try: - collected = [] - for source in sources: - collected.append(self.providers[source]) - sources = collected + sources = [self.providers[source] for source in sources] except KeyError: - raise Exception('Zone {}, unknown source: {}'.format(zone_name, - source)) + raise ManagerException('Zone {}, unknown source: {}' + .format(zone_name, source)) try: trgs = [] for target in targets: trg = self.providers[target] if not isinstance(trg, BaseProvider): - raise Exception('{} - "{}" does not support targeting' - .format(trg, target)) + raise ManagerException('{} - "{}" does not support ' + 'targeting'.format(trg, target)) trgs.append(trg) targets = trgs except KeyError: - raise Exception('Zone {}, unknown target: {}'.format(zone_name, - target)) + raise ManagerException('Zone {}, unknown target: {}' + .format(zone_name, target)) futures.append(self._executor.submit(self._populate_and_plan, zone_name, sources, targets)) @@ -348,7 +353,7 @@ class Manager(object): a = [self.providers[source] for source in a] b = [self.providers[source] for source in b] except KeyError as e: - raise Exception('Unknown source: {}'.format(e.args[0])) + raise ManagerException('Unknown source: {}'.format(e.args[0])) sub_zones = self.configured_sub_zones(zone) za = Zone(zone, sub_zones) @@ -374,7 +379,7 @@ class Manager(object): try: sources = [self.providers[s] for s in sources] except KeyError as e: - raise Exception('Unknown source: {}'.format(e.args[0])) + raise ManagerException('Unknown source: {}'.format(e.args[0])) clz = YamlProvider if split: @@ -397,16 +402,14 @@ class Manager(object): try: sources = config['sources'] except KeyError: - raise Exception('Zone {} is missing sources'.format(zone_name)) + raise ManagerException('Zone {} is missing sources' + .format(zone_name)) try: - collected = [] - for source in sources: - collected.append(self.providers[source]) - sources = collected + sources = [self.providers[source] for source in sources] except KeyError: - raise Exception('Zone {}, unknown source: {}'.format(zone_name, - source)) + raise ManagerException('Zone {}, unknown source: {}' + .format(zone_name, source)) for source in sources: if isinstance(source, YamlProvider): diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 60a1922..13eea95 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -11,7 +11,8 @@ from six import text_type from unittest import TestCase from octodns.record import Record -from octodns.manager import _AggregateTarget, MainThreadExecutor, Manager +from octodns.manager import _AggregateTarget, MainThreadExecutor, Manager, \ + ManagerException from octodns.yaml import safe_load from octodns.zone import Zone @@ -28,77 +29,77 @@ def get_config_filename(which): class TestManager(TestCase): def test_missing_provider_class(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('missing-provider-class.yaml')).sync() self.assertTrue('missing class' in text_type(ctx.exception)) def test_bad_provider_class(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('bad-provider-class.yaml')).sync() self.assertTrue('Unknown provider class' in text_type(ctx.exception)) def test_bad_provider_class_module(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('bad-provider-class-module.yaml')) \ .sync() self.assertTrue('Unknown provider class' in text_type(ctx.exception)) def test_bad_provider_class_no_module(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('bad-provider-class-no-module.yaml')) \ .sync() self.assertTrue('Unknown provider class' in text_type(ctx.exception)) def test_missing_provider_config(self): # Missing provider config - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('missing-provider-config.yaml')).sync() self.assertTrue('provider config' in text_type(ctx.exception)) def test_missing_env_config(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('missing-provider-env.yaml')).sync() self.assertTrue('missing env var' in text_type(ctx.exception)) def test_missing_source(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('provider-problems.yaml')) \ .sync(['missing.sources.']) self.assertTrue('missing sources' in text_type(ctx.exception)) def test_missing_targets(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('provider-problems.yaml')) \ .sync(['missing.targets.']) self.assertTrue('missing targets' in text_type(ctx.exception)) def test_unknown_source(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('provider-problems.yaml')) \ .sync(['unknown.source.']) self.assertTrue('unknown source' in text_type(ctx.exception)) def test_unknown_target(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('provider-problems.yaml')) \ .sync(['unknown.target.']) self.assertTrue('unknown target' in text_type(ctx.exception)) def test_bad_plan_output_class(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: name = 'bad-plan-output-missing-class.yaml' Manager(get_config_filename(name)).sync() self.assertEquals('plan_output bad is missing class', text_type(ctx.exception)) def test_bad_plan_output_config(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('bad-plan-output-config.yaml')).sync() self.assertEqual('Incorrect plan_output config for bad', text_type(ctx.exception)) def test_source_only_as_a_target(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('provider-problems.yaml')) \ .sync(['not.targetable.']) self.assertTrue('does not support targeting' in @@ -182,7 +183,7 @@ class TestManager(TestCase): 'unit.tests.') self.assertEquals(14, len(changes)) - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: manager.compare(['nope'], ['dump'], 'unit.tests.') self.assertEquals('Unknown source: nope', text_type(ctx.exception)) @@ -222,7 +223,7 @@ class TestManager(TestCase): environ['YAML_TMP_DIR'] = tmpdir.dirname manager = Manager(get_config_filename('simple.yaml')) - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: manager.dump('unit.tests.', tmpdir.dirname, False, False, 'nope') self.assertEquals('Unknown source: nope', text_type(ctx.exception)) @@ -251,7 +252,7 @@ class TestManager(TestCase): environ['YAML_TMP_DIR'] = tmpdir.dirname manager = Manager(get_config_filename('simple-split.yaml')) - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: manager.dump('unit.tests.', tmpdir.dirname, False, True, 'nope') self.assertEquals('Unknown source: nope', text_type(ctx.exception)) @@ -267,12 +268,12 @@ class TestManager(TestCase): def test_validate_configs(self): Manager(get_config_filename('simple-validate.yaml')).validate_configs() - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('missing-sources.yaml')) \ .validate_configs() self.assertTrue('missing sources' in text_type(ctx.exception)) - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('unknown-provider.yaml')) \ .validate_configs() self.assertTrue('unknown source' in text_type(ctx.exception)) From 00fa158c59b243a80901e1802a437eece6a1264c Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 9 Oct 2019 15:31:27 -0700 Subject: [PATCH 146/176] filter -> [... if]s --- octodns/manager.py | 3 +- octodns/provider/base.py | 2 +- tests/test_octodns_provider_mythicbeasts.py | 12 +++---- tests/test_octodns_provider_yaml.py | 37 +++++++++------------ 4 files changed, 24 insertions(+), 30 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index 85de76b..9a1b598 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -271,8 +271,7 @@ class Manager(object): raise ManagerException('Zone {} is missing targets' .format(zone_name)) if eligible_targets: - targets = list(filter(lambda d: d in eligible_targets, - targets)) + targets = [t for t in targets if t in eligible_targets] if not targets: # Don't bother planning (and more importantly populating) zones diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 9f03e78..ae87844 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -60,7 +60,7 @@ class BaseProvider(BaseSource): # allow the provider to filter out false positives before = len(changes) - changes = list(filter(self._include_change, changes)) + changes = [c for c in changes if self._include_change(c)] after = len(changes) if before != after: self.log.info('plan: filtered out %s changes', before - after) diff --git a/tests/test_octodns_provider_mythicbeasts.py b/tests/test_octodns_provider_mythicbeasts.py index 498b408..960bd65 100644 --- a/tests/test_octodns_provider_mythicbeasts.py +++ b/tests/test_octodns_provider_mythicbeasts.py @@ -441,11 +441,11 @@ class TestMythicBeastsProvider(TestCase): plan = provider.plan(wanted) # Octo ignores NS records (15-1) - self.assertEquals(1, len(list(filter( - lambda u: isinstance(u, Update), plan.changes)))) - self.assertEquals(1, len(list(filter( - lambda d: isinstance(d, Delete), plan.changes)))) - self.assertEquals(14, len(list(filter( - lambda c: isinstance(c, Create), plan.changes)))) + self.assertEquals(1, len([c for c in plan.changes + if isinstance(c, Update)])) + self.assertEquals(1, len([c for c in plan.changes + if isinstance(c, Delete)])) + self.assertEquals(14, len([c for c in plan.changes + if isinstance(c, Create)])) self.assertEquals(16, provider.apply(plan)) self.assertTrue(plan.exists) diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index e16aaa5..a47dca1 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -58,9 +58,8 @@ class TestYamlProvider(TestCase): # We add everything plan = target.plan(zone) - self.assertEquals(15, len(list(filter(lambda c: - isinstance(c, Create), - plan.changes)))) + self.assertEquals(15, len([c for c in plan.changes + if isinstance(c, Create)])) self.assertFalse(isfile(yaml_file)) # Now actually do it @@ -69,9 +68,8 @@ class TestYamlProvider(TestCase): # Dynamic plan plan = target.plan(dynamic_zone) - self.assertEquals(5, len(list(filter(lambda c: - isinstance(c, Create), - plan.changes)))) + self.assertEquals(5, len([c for c in plan.changes + if isinstance(c, Create)])) self.assertFalse(isfile(dynamic_yaml_file)) # Apply it self.assertEquals(5, target.apply(plan)) @@ -82,17 +80,15 @@ class TestYamlProvider(TestCase): target.populate(reloaded) self.assertDictEqual( {'included': ['test']}, - list(filter( - lambda x: x.name == 'included', reloaded.records - ))[0]._octodns) + [x for x in reloaded.records + if x.name == 'included'][0]._octodns) self.assertFalse(zone.changes(reloaded, target=source)) # A 2nd sync should still create everything plan = target.plan(zone) - self.assertEquals(15, len(list(filter(lambda c: - isinstance(c, Create), - plan.changes)))) + self.assertEquals(15, len([c for c in plan.changes + if isinstance(c, Create)])) with open(yaml_file) as fh: data = safe_load(fh.read()) @@ -255,8 +251,8 @@ class TestSplitYamlProvider(TestCase): # We add everything plan = target.plan(zone) - self.assertEquals(15, len(list(filter( - lambda c: isinstance(c, Create), plan.changes)))) + self.assertEquals(15, len([c for c in plan.changes + if isinstance(c, Create)])) self.assertFalse(isdir(zone_dir)) # Now actually do it @@ -264,8 +260,8 @@ class TestSplitYamlProvider(TestCase): # Dynamic plan plan = target.plan(dynamic_zone) - self.assertEquals(5, len(list(filter( - lambda c: isinstance(c, Create), plan.changes)))) + self.assertEquals(5, len([c for c in plan.changes + if isinstance(c, Create)])) self.assertFalse(isdir(dynamic_zone_dir)) # Apply it self.assertEquals(5, target.apply(plan)) @@ -276,16 +272,15 @@ class TestSplitYamlProvider(TestCase): target.populate(reloaded) self.assertDictEqual( {'included': ['test']}, - list(filter( - lambda x: x.name == 'included', reloaded.records - ))[0]._octodns) + [x for x in reloaded.records + if x.name == 'included'][0]._octodns) self.assertFalse(zone.changes(reloaded, target=source)) # A 2nd sync should still create everything plan = target.plan(zone) - self.assertEquals(15, len(list(filter( - lambda c: isinstance(c, Create), plan.changes)))) + self.assertEquals(15, len([c for c in plan.changes + if isinstance(c, Create)])) yaml_file = join(zone_dir, '$unit.tests.yaml') self.assertTrue(isfile(yaml_file)) From 4d0bc29acc5286e382edf3af72c7c5b8f3656b21 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 9 Oct 2019 15:36:25 -0700 Subject: [PATCH 147/176] Remove a couple more filters( --- octodns/manager.py | 2 +- octodns/source/tinydns.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index 9a1b598..1a86336 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -254,7 +254,7 @@ class Manager(object): zones = self.config['zones'].items() if eligible_zones: - zones = filter(lambda d: d[0] in eligible_zones, zones) + zones = [z for z in zones if z[0] in eligible_zones] futures = [] for zone_name, config in zones: diff --git a/octodns/source/tinydns.py b/octodns/source/tinydns.py index 0ee100a..9c44ed8 100755 --- a/octodns/source/tinydns.py +++ b/octodns/source/tinydns.py @@ -253,7 +253,7 @@ class TinyDnsFileSource(TinyDnsBaseSource): # Ignore hidden files continue with open(join(self.directory, filename), 'r') as fh: - lines += filter(lambda l: l, fh.read().split('\n')) + lines += [l for l in fh.read().split('\n') if l] self._cache = lines From 10ad30e7ea4a463838989258428245e0d7c55b1e Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 9 Oct 2019 15:41:18 -0700 Subject: [PATCH 148/176] map( to [...] --- octodns/provider/dyn.py | 4 ++-- octodns/provider/ovh.py | 3 +-- octodns/zone.py | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/octodns/provider/dyn.py b/octodns/provider/dyn.py index 30ba8bd..f4f8e53 100644 --- a/octodns/provider/dyn.py +++ b/octodns/provider/dyn.py @@ -903,10 +903,10 @@ class DynProvider(BaseProvider): # Sort the values for consistent ordering so that we can compare values = sorted(values, key=_dynamic_value_sort_key) # Ensure that weight is included and if not use the default - values = list(map(lambda v: { + values = [{ 'value': v['value'], 'weight': v.get('weight', 1), - }, values)) + } for v in values] # Walk through our existing pools looking for a match we can use for pool in pools: diff --git a/octodns/provider/ovh.py b/octodns/provider/ovh.py index a0e47f8..17aff8d 100644 --- a/octodns/provider/ovh.py +++ b/octodns/provider/ovh.py @@ -326,8 +326,7 @@ class OvhProvider(BaseProvider): splitted = value.split('\\;') found_key = False for splitted_value in splitted: - sub_split = list(map(lambda x: x.strip(), - splitted_value.split("=", 1))) + sub_split = [x.strip() for x in splitted_value.split("=", 1)] if len(sub_split) < 2: return False key, value = sub_split[0], sub_split[1] diff --git a/octodns/zone.py b/octodns/zone.py index 1191b6f..5f099ac 100644 --- a/octodns/zone.py +++ b/octodns/zone.py @@ -84,8 +84,8 @@ class Zone(object): raise DuplicateRecordException('Duplicate record {}, type {}' .format(record.fqdn, record._type)) - elif not lenient and (((record._type == 'CNAME' and len(node) > 0) or - ('CNAME' in map(lambda r: r._type, node)))): + elif not lenient and ((record._type == 'CNAME' and len(node) > 0) or + ('CNAME' in [r._type for r in node])): # We're adding a CNAME to existing records or adding to an existing # CNAME raise InvalidNodeException('Invalid state, CNAME at {} cannot ' From b5c75d189c7d9b0fe4f30abe8f8e074c1a1e36dd Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 9 Oct 2019 16:01:39 -0700 Subject: [PATCH 149/176] Convert sources building back out to for x in y from list comprehension --- octodns/manager.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index 1a86336..14a8f07 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -283,7 +283,10 @@ class Manager(object): self.log.info('sync: sources=%s -> targets=%s', sources, targets) try: - sources = [self.providers[source] for source in sources] + collected = [] + for source in sources: + collected.append(self.providers[source]) + sources = collected except KeyError: raise ManagerException('Zone {}, unknown source: {}' .format(zone_name, source)) @@ -405,7 +408,10 @@ class Manager(object): .format(zone_name)) try: - sources = [self.providers[source] for source in sources] + collected = [] + for source in sources: + collected.append(self.providers[source]) + sources = collected except KeyError: raise ManagerException('Zone {}, unknown source: {}' .format(zone_name, source)) From 6959b58b7533a94de78d49e0db706ee182009339 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 9 Oct 2019 16:03:06 -0700 Subject: [PATCH 150/176] Update requirements and setup.py, remove incf.countryutils, promote pycountry-convert --- requirements-dev.txt | 2 -- requirements.txt | 6 ++++-- setup.py | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index a2833ae..d9888b8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,8 +2,6 @@ coverage mock nose pycodestyle==2.4.0 -pycountry>=18.12.8 -pycountry_convert>=0.7.2 pyflakes==1.6.0 readme_renderer[md]==24.0 requests_mock diff --git a/requirements.txt b/requirements.txt index 4d98736..204fd63 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,8 +6,8 @@ botocore==1.10.5 dnspython==1.15.0 docutils==0.14 dyn==1.8.1 -futures==3.2.0; python_version < '3.0' edgegrid-python==1.1.1 +futures==3.2.0; python_version < '3.0' google-cloud-core==0.28.1 google-cloud-dns==0.29.0 ipaddress==1.0.22 @@ -16,9 +16,11 @@ msrestazure==0.6.2 natsort==5.5.0 ns1-python==0.12.0 ovh==0.4.8 +pycountry-convert==0.7.2 +pycountry==19.8.18 python-dateutil==2.6.1 requests==2.22.0 s3transfer==0.1.13 -six==1.12.0 setuptools==38.5.2 +six==1.12.0 transip==2.0.0 diff --git a/setup.py b/setup.py index 5cb741b..d58ed2c 100644 --- a/setup.py +++ b/setup.py @@ -66,9 +66,10 @@ setup( 'PyYaml>=4.2b1', 'dnspython>=1.15.0', 'futures>=3.2.0', - 'incf.countryutils>=1.0', 'ipaddress>=1.0.22', 'natsort>=5.5.0', + 'pycountry>=19.8.18', + 'pycountry-convert>=0.7.2', # botocore doesn't like >=2.7.0 for some reason 'python-dateutil>=2.6.0,<2.7.0', 'requests>=2.20.0' From 294c2cc9b3f8a1820437a16436ace7c4508f65e1 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 9 Oct 2019 16:07:03 -0700 Subject: [PATCH 151/176] Update CHANGELOG for python 3 work --- CHANGELOG.md | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e30d0df..7e5e6ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,28 @@ ## v0.9.9 - 2019-??-?? - Python 3.7 Support -* Route53 _mod_keyer ordering wasn't complete/reliable and in python 3 this - resulted in randomness. This has been addressed and may result in value - reordering on next plan, no actual changes in behavior should occur. +* Extensive pass through the whole codebase to support Python 3 + * Tons of updates to replace `def __cmp__` with `__eq__` and friends to + preserve custom equality and ordering behaviors that are essential to + octoDNS's processes. + * Quite a few objects required the addition of `__eq__` and friends so that + they're sortable in Python 3 now that those things are more strict. A few + places this required jumping through hoops of sorts. Thankfully our tests + are pretty thorough and caught a lot of issues and hopefully the whole + plan, review, apply process will backstop that. + * Explicit ordering of changes by (name, type) to address inconsistent + ordering for a number of providers that just convert changes into API + calls as they come. Python 2 sets ordered consistently, Python 3 they do + not. https://github.com/github/octodns/pull/384/commits/7958233fccf9ea22d95e2fd06c48d7d0a4529e26 + * Route53 _mod_keyer ordering wasn't 100% complete and thus unreliable and + random in Python 3. This has been addressed and may result in value + reordering on next plan, no actual changes in behavior should occur. + * `incf.countryutils` (in pypi) was last released in 2009 is not python 3 + compatible (it's country data is also pretty stale.) `pycountry_convert` + appears to have the functionality required to replace its usage so it has + been removed as a dependency/requirement. + * Bunch of additional unit tests and supporting config to exercise new code + and verify things that were run into during the Python 3 work + * lots of `six`ing of things ## v0.9.8 - 2019-09-30 - One with no changes b/c PyPi description problems From baa1f7472f0127aab0111ef18486d290f7f7714e Mon Sep 17 00:00:00 2001 From: Adam Smith Date: Sun, 13 Oct 2019 21:07:30 -0700 Subject: [PATCH 152/176] ConstellixProvider: change ALIAS to CNAME to allow record deletion --- octodns/provider/constellix.py | 4 ++++ tests/test_octodns_provider_constellix.py | 15 ++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/octodns/provider/constellix.py b/octodns/provider/constellix.py index 939284d..5bd506d 100644 --- a/octodns/provider/constellix.py +++ b/octodns/provider/constellix.py @@ -148,6 +148,10 @@ class ConstellixClient(object): self._request('POST', path, data=params) def record_delete(self, zone_name, record_type, record_id): + # change ALIAS records to ANAME + if record_type == 'ALIAS': + record_type = 'ANAME' + zone_id = self.domains.get(zone_name, False) path = '/{}/records/{}/{}'.format(zone_id, record_type, record_id) self._request('DELETE', path) diff --git a/tests/test_octodns_provider_constellix.py b/tests/test_octodns_provider_constellix.py index 346bb17..80489c8 100644 --- a/tests/test_octodns_provider_constellix.py +++ b/tests/test_octodns_provider_constellix.py @@ -187,6 +187,14 @@ class TestConstellixProvider(TestCase): 'value': [ '3.2.3.4' ] + }, { + 'id': 11189899, + 'type': 'ALIAS', + 'name': 'alias', + 'ttl': 600, + 'value': [{ + 'value': 'aname.unit.tests.' + }] } ]) @@ -201,8 +209,8 @@ class TestConstellixProvider(TestCase): })) plan = provider.plan(wanted) - self.assertEquals(2, len(plan.changes)) - self.assertEquals(2, provider.apply(plan)) + self.assertEquals(3, len(plan.changes)) + self.assertEquals(3, provider.apply(plan)) # recreate for update, and deletes for the 2 parts of the other provider._client._request.assert_has_calls([ @@ -214,5 +222,6 @@ class TestConstellixProvider(TestCase): 'ttl': 300 }), call('DELETE', '/123123/records/A/11189897'), - call('DELETE', '/123123/records/A/11189898') + call('DELETE', '/123123/records/A/11189898'), + call('DELETE', '/123123/records/ANAME/11189899') ], any_order=True) From b3bd4382cc6c32c45bf874320a99c241b314c26e Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 14 Oct 2019 07:32:09 -0700 Subject: [PATCH 153/176] Apply suggestions from code review Co-Authored-By: Theo Julienne --- octodns/manager.py | 4 ++++ octodns/provider/rackspace.py | 2 +- octodns/record/__init__.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index 14a8f07..bc78f5b 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -283,6 +283,8 @@ class Manager(object): self.log.info('sync: sources=%s -> targets=%s', sources, targets) try: + # rather than using a list comprehension, we break this loop out + # so that the `except` block below can reference the `source` collected = [] for source in sources: collected.append(self.providers[source]) @@ -408,6 +410,8 @@ class Manager(object): .format(zone_name)) try: + # rather than using a list comprehension, we break this loop out + # so that the `except` block below can reference the `source` collected = [] for source in sources: collected.append(self.providers[source]) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 28b7f05..7fed05b 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -14,7 +14,7 @@ from .base import BaseProvider def _value_keyer(v): - return '{}-{}-{}'.format(v.get('type', ''), v['name'], v.get('data', '')) + return (v.get('type', ''), v['name'], v.get('data', '')) def add_trailing_dot(s): diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 47d2e9b..c0f6482 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -926,7 +926,7 @@ class MxValue(object): } def __hash__(self): - return hash('{} {}'.format(self.preference, self.exchange)) + return hash((self.preference, self.exchange)) def __eq__(self, other): return ((self.preference, self.exchange) == From 25b41a4a92cfde89f157c0e116908b509c63a266 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 14 Oct 2019 07:47:36 -0700 Subject: [PATCH 154/176] Adopt Route53Provider _equality_tuple suggestion --- octodns/provider/route53.py | 70 +++++++------------------------------ 1 file changed, 12 insertions(+), 58 deletions(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 6d2cfeb..decbd4d 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -156,34 +156,29 @@ class _Route53Record(object): 'sub-classes should never use this method' return '{}:{}'.format(self.fqdn, self._type).__hash__() + def _equality_tuple(self): + return (self.__class__.__name__, self.fqdn, self._type) + def __eq__(self, other): '''Sub-classes should call up to this and return its value if true. When it's false they should compute their own __eq__, same for other ordering methods.''' - return self.__class__.__name__ == other.__class__.__name__ and \ - self.fqdn == other.fqdn and \ - self._type == other._type + return self._equality_tuple() == other._equality_tuple() def __ne__(self, other): - return self.__class__.__name__ != other.__class__.__name__ or \ - self.fqdn != other.fqdn or \ - self._type != other._type + return self._equality_tuple() != other._equality_tuple() def __lt__(self, other): - return (((self.__class__.__name__, self.fqdn, self._type)) < - ((other.__class__.__name__, other.fqdn, other._type))) + return self._equality_tuple() < other._equality_tuple() def __le__(self, other): - return (((self.__class__.__name__, self.fqdn, self._type)) <= - ((other.__class__.__name__, other.fqdn, other._type))) + return self._equality_tuple() <= other._equality_tuple() def __gt__(self, other): - return (((self.__class__.__name__, self.fqdn, self._type)) > - ((other.__class__.__name__, other.fqdn, other._type))) + return self._equality_tuple() > other._equality_tuple() def __ge__(self, other): - return (((self.__class__.__name__, self.fqdn, self._type)) >= - ((other.__class__.__name__, other.fqdn, other._type))) + return self._equality_tuple() >= other._equality_tuple() def __repr__(self): return '_Route53Record<{} {} {} {}>'.format(self.fqdn, self._type, @@ -524,50 +519,9 @@ class _Route53GeoRecord(_Route53Record): return '{}:{}:{}'.format(self.fqdn, self._type, self.geo.code).__hash__() - def __eq__(self, other): - return super(_Route53GeoRecord, self).__eq__(other) and \ - self.geo.code == other.geo.code - - def __ne__(self, other): - # super will handle class != class, so if it's true we have 2 geo - # objects with the same name and type, so just need to compare codes - return super(_Route53GeoRecord, self).__ne__(other) or \ - self.geo.code != other.geo.code - - def __lt__(self, other): - # super eq will check class, name, and type - if super(_Route53GeoRecord, self).__eq__(other): - # if it's True we're dealing with two geo's with the same name and - # type, so we just need to compare codes - return self.geo.code < other.geo.code - # Super is not equal so we'll just let it decide lt - return super(_Route53GeoRecord, self).__lt__(other) - - def __le__(self, other): - # super eq will check class, name, and type - if super(_Route53GeoRecord, self).__eq__(other): - # Just need to compare codes, everything else is equal - return self.geo.code <= other.geo.code - # Super is not equal so geo.code doesn't matter, let it decide with lt, - # can't be eq - return super(_Route53GeoRecord, self).__lt__(other) - - def __gt__(self, other): - # super eq will check class, name, and type - if super(_Route53GeoRecord, self).__eq__(other): - # Just need to compare codes, everything else is equal - return self.geo.code > other.geo.code - # Super is not equal so we'll just let it decide gt - return super(_Route53GeoRecord, self).__gt__(other) - - def __ge__(self, other): - # super eq will check class, name, and type - if super(_Route53GeoRecord, self).__eq__(other): - # Just need to compare codes, everything else is equal - return self.geo.code >= other.geo.code - # Super is not equal so geo.code doesn't matter, let it decide with gt, - # can't be eq - return super(_Route53GeoRecord, self).__gt__(other) + def _equality_tuple(self): + return super(_Route53GeoRecord, self)._equality_tuple() + \ + (self.geo.code,) def __repr__(self): return '_Route53GeoRecord<{} {} {} {} {}>'.format(self.fqdn, From b8e2ec124b1783f0e57ea435da0e93ef694eb750 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 14 Oct 2019 07:48:17 -0700 Subject: [PATCH 155/176] Fix Manager comment wrapping --- octodns/manager.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index bc78f5b..3c91aa9 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -283,8 +283,9 @@ class Manager(object): self.log.info('sync: sources=%s -> targets=%s', sources, targets) try: - # rather than using a list comprehension, we break this loop out - # so that the `except` block below can reference the `source` + # rather than using a list comprehension, we break this loop + # out so that the `except` block below can reference the + # `source` collected = [] for source in sources: collected.append(self.providers[source]) @@ -410,8 +411,9 @@ class Manager(object): .format(zone_name)) try: - # rather than using a list comprehension, we break this loop out - # so that the `except` block below can reference the `source` + # rather than using a list comprehension, we break this loop + # out so that the `except` block below can reference the + # `source` collected = [] for source in sources: collected.append(self.providers[source]) From 74048bf974a81d0b85802b6bb1da617ecf07cbe7 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 14 Oct 2019 07:48:47 -0700 Subject: [PATCH 156/176] Use if, else rather than try, except KeyError --- octodns/provider/route53.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index decbd4d..968d8b8 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -564,9 +564,9 @@ def _mod_keyer(mod): if rrset.get('GeoLocation', False): unique_id = rrset['SetIdentifier'] else: - try: + if 'SetIdentifier' in rrset: unique_id = '{}-{}'.format(rrset['Name'], rrset['SetIdentifier']) - except KeyError: + else: unique_id = rrset['Name'] # Prioritise within the action_priority, ensuring targets come first. From 2b33f95c1759e61698797963c04727cf19f1893e Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 14 Oct 2019 08:13:07 -0700 Subject: [PATCH 157/176] EqualityTupleMixin impl, use everywhere we were doing tuple compares --- octodns/equality.py | 30 +++++ octodns/provider/route53.py | 26 +---- octodns/record/__init__.py | 204 ++++----------------------------- tests/test_octodns_equality.py | 68 +++++++++++ 4 files changed, 126 insertions(+), 202 deletions(-) create mode 100644 octodns/equality.py create mode 100644 tests/test_octodns_equality.py diff --git a/octodns/equality.py b/octodns/equality.py new file mode 100644 index 0000000..965b8a0 --- /dev/null +++ b/octodns/equality.py @@ -0,0 +1,30 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + + +class EqualityTupleMixin: + + def _equality_tuple(self): + raise NotImplementedError('_equality_tuple method not implemented') + + def __eq__(self, other): + return self._equality_tuple() == other._equality_tuple() + + def __ne__(self, other): + return self._equality_tuple() != other._equality_tuple() + + def __lt__(self, other): + return self._equality_tuple() < other._equality_tuple() + + def __le__(self, other): + return self._equality_tuple() <= other._equality_tuple() + + def __gt__(self, other): + return self._equality_tuple() > other._equality_tuple() + + def __ge__(self, other): + return self._equality_tuple() >= other._equality_tuple() diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 968d8b8..66da6b5 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -16,6 +16,7 @@ import re from six import text_type +from ..equality import EqualityTupleMixin from ..record import Record, Update from ..record.geo import GeoCodes from .base import BaseProvider @@ -29,7 +30,7 @@ def _octal_replace(s): return octal_re.sub(lambda m: chr(int(m.group(1), 8)), s) -class _Route53Record(object): +class _Route53Record(EqualityTupleMixin): @classmethod def _new_dynamic(cls, provider, record, hosted_zone_id, creating): @@ -157,29 +158,10 @@ class _Route53Record(object): return '{}:{}'.format(self.fqdn, self._type).__hash__() def _equality_tuple(self): + '''Sub-classes should call up to this and return its value and add + any additional fields they need to hav considered.''' return (self.__class__.__name__, self.fqdn, self._type) - def __eq__(self, other): - '''Sub-classes should call up to this and return its value if true. - When it's false they should compute their own __eq__, same for other - ordering methods.''' - return self._equality_tuple() == other._equality_tuple() - - def __ne__(self, other): - return self._equality_tuple() != other._equality_tuple() - - def __lt__(self, other): - return self._equality_tuple() < other._equality_tuple() - - def __le__(self, other): - return self._equality_tuple() <= other._equality_tuple() - - def __gt__(self, other): - return self._equality_tuple() > other._equality_tuple() - - def __ge__(self, other): - return self._equality_tuple() >= other._equality_tuple() - def __repr__(self): return '_Route53Record<{} {} {} {}>'.format(self.fqdn, self._type, self.ttl, self.values) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index c0f6482..0847018 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -11,6 +11,7 @@ import re from six import string_types, text_type +from ..equality import EqualityTupleMixin from .geo import GeoCodes @@ -76,7 +77,7 @@ class ValidationError(Exception): self.reasons = reasons -class Record(object): +class Record(EqualityTupleMixin): log = getLogger('Record') @classmethod @@ -209,30 +210,15 @@ class Record(object): def __hash__(self): return '{}:{}'.format(self.name, self._type).__hash__() - def __eq__(self, other): - return ((self.name, self._type) == (other.name, other._type)) - - def __ne__(self, other): - return ((self.name, self._type) != (other.name, other._type)) - - def __lt__(self, other): - return ((self.name, self._type) < (other.name, other._type)) - - def __le__(self, other): - return ((self.name, self._type) <= (other.name, other._type)) - - def __gt__(self, other): - return ((self.name, self._type) > (other.name, other._type)) - - def __ge__(self, other): - return ((self.name, self._type) >= (other.name, other._type)) + def _equality_tuple(self): + return (self.name, self._type) def __repr__(self): # Make sure this is always overridden raise NotImplementedError('Abstract base class, __repr__ required') -class GeoValue(object): +class GeoValue(EqualityTupleMixin): geo_re = re.compile(r'^(?P\w\w)(-(?P\w\w)' r'(-(?P\w\w))?)?$') @@ -259,35 +245,9 @@ class GeoValue(object): yield '-'.join(bits) bits.pop() - def __eq__(self, other): - return ((self.continent_code, self.country_code, self.subdivision_code, - self.values) == (other.continent_code, other.country_code, - other.subdivision_code, other.values)) - - def __ne__(self, other): - return ((self.continent_code, self.country_code, self.subdivision_code, - self.values) != (other.continent_code, other.country_code, - other.subdivision_code, other.values)) - - def __lt__(self, other): - return ((self.continent_code, self.country_code, self.subdivision_code, - self.values) < (other.continent_code, other.country_code, - other.subdivision_code, other.values)) - - def __le__(self, other): - return ((self.continent_code, self.country_code, self.subdivision_code, - self.values) <= (other.continent_code, other.country_code, - other.subdivision_code, other.values)) - - def __gt__(self, other): - return ((self.continent_code, self.country_code, self.subdivision_code, - self.values) > (other.continent_code, other.country_code, - other.subdivision_code, other.values)) - - def __ge__(self, other): - return ((self.continent_code, self.country_code, self.subdivision_code, - self.values) >= (other.continent_code, other.country_code, - other.subdivision_code, other.values)) + def _equality_tuple(self): + return (self.continent_code, self.country_code, self.subdivision_code, + self.values) def __repr__(self): return "'Geo {} {} {} {}'".format(self.continent_code, @@ -787,7 +747,7 @@ class AliasRecord(_ValueMixin, Record): _value_type = AliasValue -class CaaValue(object): +class CaaValue(EqualityTupleMixin): # https://tools.ietf.org/html/rfc6844#page-5 @classmethod @@ -826,29 +786,8 @@ class CaaValue(object): 'value': self.value, } - def __eq__(self, other): - return ((self.flags, self.tag, self.value) == - (other.flags, other.tag, other.value)) - - def __ne__(self, other): - return ((self.flags, self.tag, self.value) != - (other.flags, other.tag, other.value)) - - def __lt__(self, other): - return ((self.flags, self.tag, self.value) < - (other.flags, other.tag, other.value)) - - def __le__(self, other): - return ((self.flags, self.tag, self.value) <= - (other.flags, other.tag, other.value)) - - def __gt__(self, other): - return ((self.flags, self.tag, self.value) > - (other.flags, other.tag, other.value)) - - def __ge__(self, other): - return ((self.flags, self.tag, self.value) >= - (other.flags, other.tag, other.value)) + def _equality_tuple(self): + return (self.flags, self.tag, self.value) def __repr__(self): return '{} {} "{}"'.format(self.flags, self.tag, self.value) @@ -872,7 +811,7 @@ class CnameRecord(_DynamicMixin, _ValueMixin, Record): return reasons -class MxValue(object): +class MxValue(EqualityTupleMixin): @classmethod def validate(cls, data, _type): @@ -928,29 +867,8 @@ class MxValue(object): def __hash__(self): return hash((self.preference, self.exchange)) - def __eq__(self, other): - return ((self.preference, self.exchange) == - (other.preference, other.exchange)) - - def __ne__(self, other): - return ((self.preference, self.exchange) != - (other.preference, other.exchange)) - - def __lt__(self, other): - return ((self.preference, self.exchange) < - (other.preference, other.exchange)) - - def __le__(self, other): - return ((self.preference, self.exchange) <= - (other.preference, other.exchange)) - - def __gt__(self, other): - return ((self.preference, self.exchange) > - (other.preference, other.exchange)) - - def __ge__(self, other): - return ((self.preference, self.exchange) >= - (other.preference, other.exchange)) + def _equality_tuple(self): + return (self.preference, self.exchange) def __repr__(self): return "'{} {}'".format(self.preference, self.exchange) @@ -961,7 +879,7 @@ class MxRecord(_ValuesMixin, Record): _value_type = MxValue -class NaptrValue(object): +class NaptrValue(EqualityTupleMixin): VALID_FLAGS = ('S', 'A', 'U', 'P') @classmethod @@ -1023,41 +941,9 @@ class NaptrValue(object): def __hash__(self): return hash(self.__repr__()) - def __eq__(self, other): - return ((self.order, self.preference, self.flags, self.service, - self.regexp, self.replacement) == - (other.order, other.preference, other.flags, other.service, - other.regexp, other.replacement)) - - def __ne__(self, other): - return ((self.order, self.preference, self.flags, self.service, - self.regexp, self.replacement) != - (other.order, other.preference, other.flags, other.service, - other.regexp, other.replacement)) - - def __lt__(self, other): - return ((self.order, self.preference, self.flags, self.service, - self.regexp, self.replacement) < - (other.order, other.preference, other.flags, other.service, - other.regexp, other.replacement)) - - def __le__(self, other): - return ((self.order, self.preference, self.flags, self.service, - self.regexp, self.replacement) <= - (other.order, other.preference, other.flags, other.service, - other.regexp, other.replacement)) - - def __gt__(self, other): - return ((self.order, self.preference, self.flags, self.service, - self.regexp, self.replacement) > - (other.order, other.preference, other.flags, other.service, - other.regexp, other.replacement)) - - def __ge__(self, other): - return ((self.order, self.preference, self.flags, self.service, - self.regexp, self.replacement) >= - (other.order, other.preference, other.flags, other.service, - other.regexp, other.replacement)) + def _equality_tuple(self): + return (self.order, self.preference, self.flags, self.service, + self.regexp, self.replacement) def __repr__(self): flags = self.flags if self.flags is not None else '' @@ -1107,7 +993,7 @@ class PtrRecord(_ValueMixin, Record): _value_type = PtrValue -class SshfpValue(object): +class SshfpValue(EqualityTupleMixin): VALID_ALGORITHMS = (1, 2, 3, 4) VALID_FINGERPRINT_TYPES = (1, 2) @@ -1161,29 +1047,8 @@ class SshfpValue(object): def __hash__(self): return hash(self.__repr__()) - def __eq__(self, other): - return ((self.algorithm, self.fingerprint_type, self.fingerprint) == - (other.algorithm, other.fingerprint_type, other.fingerprint)) - - def __ne__(self, other): - return ((self.algorithm, self.fingerprint_type, self.fingerprint) != - (other.algorithm, other.fingerprint_type, other.fingerprint)) - - def __lt__(self, other): - return ((self.algorithm, self.fingerprint_type, self.fingerprint) < - (other.algorithm, other.fingerprint_type, other.fingerprint)) - - def __le__(self, other): - return ((self.algorithm, self.fingerprint_type, self.fingerprint) <= - (other.algorithm, other.fingerprint_type, other.fingerprint)) - - def __gt__(self, other): - return ((self.algorithm, self.fingerprint_type, self.fingerprint) > - (other.algorithm, other.fingerprint_type, other.fingerprint)) - - def __ge__(self, other): - return ((self.algorithm, self.fingerprint_type, self.fingerprint) >= - (other.algorithm, other.fingerprint_type, other.fingerprint)) + def _equality_tuple(self): + return (self.algorithm, self.fingerprint_type, self.fingerprint) def __repr__(self): return "'{} {} {}'".format(self.algorithm, self.fingerprint_type, @@ -1244,7 +1109,7 @@ class SpfRecord(_ChunkedValuesMixin, Record): _value_type = _ChunkedValue -class SrvValue(object): +class SrvValue(EqualityTupleMixin): @classmethod def validate(cls, data, _type): @@ -1302,29 +1167,8 @@ class SrvValue(object): def __hash__(self): return hash(self.__repr__()) - def __eq__(self, other): - return ((self.priority, self.weight, self.port, self.target) == - (other.priority, other.weight, other.port, other.target)) - - def __ne__(self, other): - return ((self.priority, self.weight, self.port, self.target) != - (other.priority, other.weight, other.port, other.target)) - - def __lt__(self, other): - return ((self.priority, self.weight, self.port, self.target) < - (other.priority, other.weight, other.port, other.target)) - - def __le__(self, other): - return ((self.priority, self.weight, self.port, self.target) <= - (other.priority, other.weight, other.port, other.target)) - - def __gt__(self, other): - return ((self.priority, self.weight, self.port, self.target) > - (other.priority, other.weight, other.port, other.target)) - - def __ge__(self, other): - return ((self.priority, self.weight, self.port, self.target) >= - (other.priority, other.weight, other.port, other.target)) + def _equality_tuple(self): + return (self.priority, self.weight, self.port, self.target) def __repr__(self): return "'{} {} {} {}'".format(self.priority, self.weight, self.port, diff --git a/tests/test_octodns_equality.py b/tests/test_octodns_equality.py new file mode 100644 index 0000000..dcdc460 --- /dev/null +++ b/tests/test_octodns_equality.py @@ -0,0 +1,68 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from unittest import TestCase + +from octodns.equality import EqualityTupleMixin + + +class TestEqualityTupleMixin(TestCase): + + def test_basics(self): + + class Simple(EqualityTupleMixin): + + def __init__(self, a, b, c): + self.a = a + self.b = b + self.c = c + + def _equality_tuple(self): + return (self.a, self.b) + + one = Simple(1, 2, 3) + same = Simple(1, 2, 3) + matches = Simple(1, 2, 'ignored') + doesnt = Simple(2, 3, 4) + + # equality + self.assertEquals(one, one) + self.assertEquals(one, same) + self.assertEquals(same, one) + # only a & c are considered + self.assertEquals(one, matches) + self.assertEquals(matches, one) + self.assertNotEquals(one, doesnt) + self.assertNotEquals(doesnt, one) + + # lt + self.assertTrue(one < doesnt) + self.assertFalse(doesnt < one) + self.assertFalse(one < same) + + # le + self.assertTrue(one <= doesnt) + self.assertFalse(doesnt <= one) + self.assertTrue(one <= same) + + # gt + self.assertFalse(one > doesnt) + self.assertTrue(doesnt > one) + self.assertFalse(one > same) + + # ge + self.assertFalse(one >= doesnt) + self.assertTrue(doesnt >= one) + self.assertTrue(one >= same) + + def test_not_implemented(self): + + class MissingMethod(EqualityTupleMixin): + pass + + with self.assertRaises(NotImplementedError): + MissingMethod() == MissingMethod() From e1daccc0b762181f96fbc86cd0b300d2f36655ac Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 14 Oct 2019 08:14:30 -0700 Subject: [PATCH 158/176] Update CHANGELOG.md Co-Authored-By: Theo Julienne --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e5e6ac..503e53c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ ordering for a number of providers that just convert changes into API calls as they come. Python 2 sets ordered consistently, Python 3 they do not. https://github.com/github/octodns/pull/384/commits/7958233fccf9ea22d95e2fd06c48d7d0a4529e26 - * Route53 _mod_keyer ordering wasn't 100% complete and thus unreliable and + * Route53 `_mod_keyer` ordering wasn't 100% complete and thus unreliable and random in Python 3. This has been addressed and may result in value reordering on next plan, no actual changes in behavior should occur. * `incf.countryutils` (in pypi) was last released in 2009 is not python 3 From 759c44f35be46f50e87ed9dc7022c177504a4ecc Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 14 Oct 2019 08:39:45 -0700 Subject: [PATCH 159/176] EqualityTupleMixin needs an explicit inhert from object to make 2.7 happy --- octodns/equality.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/equality.py b/octodns/equality.py index 965b8a0..bd22c7d 100644 --- a/octodns/equality.py +++ b/octodns/equality.py @@ -6,7 +6,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -class EqualityTupleMixin: +class EqualityTupleMixin(object): def _equality_tuple(self): raise NotImplementedError('_equality_tuple method not implemented') From 4e09f8a838e767aaefa3886ec8208dce9345ab37 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 14 Oct 2019 19:07:12 -0700 Subject: [PATCH 160/176] Use six.StringIO in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d58ed2c..362db58 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -from StringIO import StringIO +from six import StringIO from os.path import dirname, join import octodns From 2f45cbc086d11f74c91e4b50da9224c6079849a5 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 14 Oct 2019 19:10:47 -0700 Subject: [PATCH 161/176] No six for setup.py, try/except both options --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 362db58..6ae8fd5 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,9 @@ #!/usr/bin/env python -from six import StringIO +try: + from io import StringIO +except ImportError: + from StringIO import StringIO from os.path import dirname, join import octodns From 4a41c98c1681ebad6c8b9630301284fd4df25d01 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 14 Oct 2019 19:14:38 -0700 Subject: [PATCH 162/176] setup.py install_requires futures only on 2.7 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6ae8fd5..b6c2cc4 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,7 @@ setup( install_requires=[ 'PyYaml>=4.2b1', 'dnspython>=1.15.0', - 'futures>=3.2.0', + 'futures>=3.2.0; python_version<"3.2"', 'ipaddress>=1.0.22', 'natsort>=5.5.0', 'pycountry>=19.8.18', From 6f9842301ed4e21b80bd36dfb1ac96dbb98c7d4d Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 14 Oct 2019 19:31:04 -0700 Subject: [PATCH 163/176] Prefer StringIO.StringIO over io. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index b6c2cc4..4f28232 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,9 @@ #!/usr/bin/env python try: - from io import StringIO -except ImportError: from StringIO import StringIO +except ImportError: + from io import StringIO from os.path import dirname, join import octodns From 9e948aa4c88e1e679659673d1a259994c094ef17 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 18 Oct 2019 10:36:14 -0700 Subject: [PATCH 164/176] Validate Record name & fqdn length --- octodns/record/__init__.py | 37 ++++++++++++++++++++++-------------- tests/test_octodns_record.py | 28 +++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 14 deletions(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 0847018..98c1836 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -82,6 +82,7 @@ class Record(EqualityTupleMixin): @classmethod def new(cls, zone, name, data, source=None, lenient=False): + name = text_type(name) fqdn = '{}.{}'.format(name, zone.name) if name else zone.name try: _type = data['type'] @@ -105,7 +106,7 @@ class Record(EqualityTupleMixin): }[_type] except KeyError: raise Exception('Unknown record type: "{}"'.format(_type)) - reasons = _class.validate(name, data) + reasons = _class.validate(name, fqdn, data) try: lenient |= data['octodns']['lenient'] except KeyError: @@ -118,8 +119,16 @@ class Record(EqualityTupleMixin): return _class(zone, name, data, source=source) @classmethod - def validate(cls, name, data): + def validate(cls, name, fqdn, data): reasons = [] + n = len(fqdn) + if n > 253: + reasons.append('invalid fqdn, "{}" is too long at {} chars, max ' + 'is 253'.format(fqdn, n)) + n = len(name) + if n > 63: + reasons.append('invalid name, "{}" is too long at {} chars, max ' + 'is 63'.format(name, n)) try: ttl = int(data['ttl']) if ttl < 0: @@ -258,8 +267,8 @@ class GeoValue(EqualityTupleMixin): class _ValuesMixin(object): @classmethod - def validate(cls, name, data): - reasons = super(_ValuesMixin, cls).validate(name, data) + def validate(cls, name, fqdn, data): + reasons = super(_ValuesMixin, cls).validate(name, fqdn, data) values = data.get('values', data.get('value', [])) @@ -311,8 +320,8 @@ class _GeoMixin(_ValuesMixin): ''' @classmethod - def validate(cls, name, data): - reasons = super(_GeoMixin, cls).validate(name, data) + def validate(cls, name, fqdn, data): + reasons = super(_GeoMixin, cls).validate(name, fqdn, data) try: geo = dict(data['geo']) for code, values in geo.items(): @@ -358,8 +367,8 @@ class _GeoMixin(_ValuesMixin): class _ValueMixin(object): @classmethod - def validate(cls, name, data): - reasons = super(_ValueMixin, cls).validate(name, data) + def validate(cls, name, fqdn, data): + reasons = super(_ValueMixin, cls).validate(name, fqdn, data) reasons.extend(cls._value_type.validate(data.get('value', None), cls._type)) return reasons @@ -485,8 +494,8 @@ class _DynamicMixin(object): r'(-(?P\w\w))?)?$') @classmethod - def validate(cls, name, data): - reasons = super(_DynamicMixin, cls).validate(name, data) + def validate(cls, name, fqdn, data): + reasons = super(_DynamicMixin, cls).validate(name, fqdn, data) if 'dynamic' not in data: return reasons @@ -803,11 +812,11 @@ class CnameRecord(_DynamicMixin, _ValueMixin, Record): _value_type = CnameValue @classmethod - def validate(cls, name, data): + def validate(cls, name, fqdn, data): reasons = [] if name == '': reasons.append('root CNAME not allowed') - reasons.extend(super(CnameRecord, cls).validate(name, data)) + reasons.extend(super(CnameRecord, cls).validate(name, fqdn, data)) return reasons @@ -1181,11 +1190,11 @@ class SrvRecord(_ValuesMixin, Record): _name_re = re.compile(r'^_[^\.]+\.[^\.]+') @classmethod - def validate(cls, name, data): + def validate(cls, name, fqdn, data): reasons = [] if not cls._name_re.match(name): reasons.append('invalid name') - reasons.extend(super(SrvRecord, cls).validate(name, data)) + reasons.extend(super(SrvRecord, cls).validate(name, fqdn, data)) return reasons diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 0845ddf..f313342 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -1217,6 +1217,34 @@ class TestRecordValidation(TestCase): zone = Zone('unit.tests.', []) def test_base(self): + # fqdn length, DNS defins max as 253 + with self.assertRaises(ValidationError) as ctx: + # The . will put this over the edge + name = 'x' * (253 - len(self.zone.name)) + Record.new(self.zone, name, { + 'ttl': 300, + 'type': 'A', + 'value': '1.2.3.4', + }) + reason = ctx.exception.reasons[0] + self.assertTrue(reason.startswith('invalid fqdn, "xxxx')) + self.assertTrue(reason.endswith('.unit.tests." is too long at 254' + ' chars, max is 253')) + + # label length, DNS defins max as 63 + with self.assertRaises(ValidationError) as ctx: + # The . will put this over the edge + name = 'x' * 64 + Record.new(self.zone, name, { + 'ttl': 300, + 'type': 'A', + 'value': '1.2.3.4', + }) + reason = ctx.exception.reasons[0] + self.assertTrue(reason.startswith('invalid name, "xxxx')) + self.assertTrue(reason.endswith('xxx" is too long at 64' + ' chars, max is 63')) + # no ttl with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { From 193d2da4ddc53dde846005fc48a0103932b497e4 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 4 Nov 2019 14:56:36 -0800 Subject: [PATCH 165/176] v0.9.9 version bump and CHANGELOG update --- CHANGELOG.md | 3 ++- octodns/__init__.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 503e53c..1cb7b1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## v0.9.9 - 2019-??-?? - Python 3.7 Support +## v0.9.9 - 2019-11-04 - Python 3.7 Support * Extensive pass through the whole codebase to support Python 3 * Tons of updates to replace `def __cmp__` with `__eq__` and friends to @@ -23,6 +23,7 @@ * Bunch of additional unit tests and supporting config to exercise new code and verify things that were run into during the Python 3 work * lots of `six`ing of things +* Validate Record name & fqdn length ## v0.9.8 - 2019-09-30 - One with no changes b/c PyPi description problems diff --git a/octodns/__init__.py b/octodns/__init__.py index 71b5b1a..404d688 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -3,4 +3,4 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -__VERSION__ = '0.9.8' +__VERSION__ = '0.9.9' From 7a623f167f01d8590afb9286c315839ab43b299f Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 4 Nov 2019 14:57:28 -0800 Subject: [PATCH 166/176] Test python setup build in CI --- script/cibuild | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/cibuild b/script/cibuild index d048e8e..a2dc527 100755 --- a/script/cibuild +++ b/script/cibuild @@ -27,4 +27,6 @@ echo "## lint ################################################################## script/lint echo "## tests/coverage ##############################################################" script/coverage +echo "## validate setup.py build #####################################################" +python setup.py build echo "## complete ####################################################################" From 6428d3090d73b801ceda5a64e1d6483e01bdac35 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 22 Nov 2019 17:09:29 -0800 Subject: [PATCH 167/176] Initial actions PR Per @yzguy's issue https://github.com/github/octodns/issues/425 --- .github/workflows/main.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..726e2e8 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,24 @@ +name: OctoDNS +on: [pull_request] + +jobs: + ci: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [2.7, 3.7] + steps: + - uses: actions/checkout@master + - name: Setup python + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install virtualenv + - name: CI Build + run: | + ./script/cibuild From e600575e47b56766520fe9d98348f340a06531cd Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 22 Nov 2019 17:29:41 -0800 Subject: [PATCH 168/176] Delete .travis.yml --- .travis.yml | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index eb609ff..0000000 --- a/.travis.yml +++ /dev/null @@ -1,10 +0,0 @@ -language: python -matrix: - include: - - python: 2.7 - - python: 3.7 -before_install: pip install --upgrade pip -script: ./script/cibuild -notifications: - email: - - ross@github.com From 0c05ff075c60488858a658ec17b66f22f6d0616b Mon Sep 17 00:00:00 2001 From: Brian E Clow Date: Tue, 26 Nov 2019 10:00:29 -0800 Subject: [PATCH 169/176] Bumping setuptools from 38.5.2 to 40.3.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 204fd63..66fa130 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,6 @@ pycountry==19.8.18 python-dateutil==2.6.1 requests==2.22.0 s3transfer==0.1.13 -setuptools==38.5.2 +setuptools==40.3.0 six==1.12.0 transip==2.0.0 From 1e74d59a76791f6d5f9c610a10166f7cac400fca Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 9 Dec 2019 06:35:31 -0800 Subject: [PATCH 170/176] Bump ns1-python to 0.13.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 66fa130..6a26ad3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ ipaddress==1.0.22 jmespath==0.9.3 msrestazure==0.6.2 natsort==5.5.0 -ns1-python==0.12.0 +ns1-python==0.13.0 ovh==0.4.8 pycountry-convert==0.7.2 pycountry==19.8.18 From f599d91902ab3f8e5b09638efd6872e772657d93 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 9 Dec 2019 08:09:41 -0800 Subject: [PATCH 171/176] Update ns1 to 0.13.0 and use newer API methods Results in lots of churn in the tests, but actually for the better as it doesn't have to jump through nearly as many hoops to mock things now. --- octodns/provider/ns1.py | 57 +++--- tests/test_octodns_provider_ns1.py | 285 +++++++++++++++-------------- 2 files changed, 177 insertions(+), 165 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index b737a19..19b33af 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -38,7 +38,9 @@ class Ns1Provider(BaseProvider): self.log = getLogger('Ns1Provider[{}]'.format(id)) self.log.debug('__init__: id=%s, api_key=***', id) super(Ns1Provider, self).__init__(id, *args, **kwargs) - self._client = NS1(apiKey=api_key) + client = NS1(apiKey=api_key) + self._records = client.records() + self._zones = client.zones() def _data_for_A(self, _type, record): # record meta (which would include geo information is only @@ -189,18 +191,29 @@ class Ns1Provider(BaseProvider): target, lenient) try: - nsone_zone = self._client.loadZone(zone.name[:-1]) - records = nsone_zone.data['records'] + nsone_zone_name = zone.name[:-1] + nsone_zone = self._zones.retrieve(nsone_zone_name) + + records = [] + geo_records = [] # change answers for certain types to always be absolute - for record in records: + for record in nsone_zone['records']: if record['type'] in ['ALIAS', 'CNAME', 'MX', 'NS', 'PTR', 'SRV']: for i, a in enumerate(record['short_answers']): if not a.endswith('.'): record['short_answers'][i] = '{}.'.format(a) - geo_records = nsone_zone.search(has_geo=True) + if record.get('tier', 1) > 1: + # Need to get the full record data for geo records + record = self._records.retrieve(nsone_zone_name, + record['domain'], + record['type']) + geo_records.append(record) + else: + records.append(record) + exists = True except ResourceException as e: if e.message != self.ZONE_NOT_FOUND_MESSAGE: @@ -299,53 +312,49 @@ class Ns1Provider(BaseProvider): for v in record.values] return {'answers': values, 'ttl': record.ttl} - def _get_name(self, record): - return record.fqdn[:-1] if record.name == '' else record.name - def _apply_Create(self, nsone_zone, change): new = change.new - name = self._get_name(new) + zone = new.zone.name[:-1] + domain = new.fqdn[:-1] _type = new._type params = getattr(self, '_params_for_{}'.format(_type))(new) - meth = getattr(nsone_zone, 'add_{}'.format(_type)) try: - meth(name, **params) + self._records.create(zone, domain, _type, **params) except RateLimitException as e: period = float(e.period) self.log.warn('_apply_Create: rate limit encountered, pausing ' 'for %ds and trying again', period) sleep(period) - meth(name, **params) + self._records.create(zone, domain, _type, **params) def _apply_Update(self, nsone_zone, change): - existing = change.existing - name = self._get_name(existing) - _type = existing._type - record = nsone_zone.loadRecord(name, _type) new = change.new + zone = new.zone.name[:-1] + domain = new.fqdn[:-1] + _type = new._type params = getattr(self, '_params_for_{}'.format(_type))(new) try: - record.update(**params) + self._records.update(zone, domain, _type, **params) except RateLimitException as e: period = float(e.period) self.log.warn('_apply_Update: rate limit encountered, pausing ' 'for %ds and trying again', period) sleep(period) - record.update(**params) + self._records.update(zone, domain, _type, **params) def _apply_Delete(self, nsone_zone, change): existing = change.existing - name = self._get_name(existing) + zone = existing.zone.name[:-1] + domain = existing.fqdn[:-1] _type = existing._type - record = nsone_zone.loadRecord(name, _type) try: - record.delete() + self._records.delete(zone, domain, _type) except RateLimitException as e: period = float(e.period) self.log.warn('_apply_Delete: rate limit encountered, pausing ' 'for %ds and trying again', period) sleep(period) - record.delete() + self._records.delete(zone, domain, _type) def _apply(self, plan): desired = plan.desired @@ -355,12 +364,12 @@ class Ns1Provider(BaseProvider): domain_name = desired.name[:-1] try: - nsone_zone = self._client.loadZone(domain_name) + nsone_zone = self._zones.retrieve(domain_name) except ResourceException as e: if e.message != self.ZONE_NOT_FOUND_MESSAGE: raise self.log.debug('_apply: no matching zone, creating') - nsone_zone = self._client.createZone(domain_name) + nsone_zone = self._zones.create(domain_name) for change in changes: class_name = change.__class__.__name__ diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 7ef182c..9034552 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -6,7 +6,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from collections import defaultdict -from mock import Mock, call, patch +from mock import call, patch from ns1.rest.errors import AuthException, RateLimitException, \ ResourceException from unittest import TestCase @@ -16,14 +16,6 @@ from octodns.provider.ns1 import Ns1Provider from octodns.zone import Zone -class DummyZone(object): - - def __init__(self, records): - self.data = { - 'records': records - } - - class TestNs1Provider(TestCase): zone = Zone('unit.tests.', []) expected = set() @@ -172,43 +164,40 @@ class TestNs1Provider(TestCase): 'domain': 'unit.tests.', }] - @patch('ns1.NS1.loadZone') - def test_populate(self, load_mock): + @patch('ns1.rest.zones.Zones.retrieve') + def test_populate(self, zone_retrieve_mock): provider = Ns1Provider('test', 'api-key') # Bad auth - load_mock.side_effect = AuthException('unauthorized') + zone_retrieve_mock.side_effect = AuthException('unauthorized') zone = Zone('unit.tests.', []) with self.assertRaises(AuthException) as ctx: provider.populate(zone) - self.assertEquals(load_mock.side_effect, ctx.exception) + self.assertEquals(zone_retrieve_mock.side_effect, ctx.exception) # General error - load_mock.reset_mock() - load_mock.side_effect = ResourceException('boom') + zone_retrieve_mock.reset_mock() + zone_retrieve_mock.side_effect = ResourceException('boom') zone = Zone('unit.tests.', []) with self.assertRaises(ResourceException) as ctx: provider.populate(zone) - self.assertEquals(load_mock.side_effect, ctx.exception) - self.assertEquals(('unit.tests',), load_mock.call_args[0]) + self.assertEquals(zone_retrieve_mock.side_effect, ctx.exception) + self.assertEquals(('unit.tests',), zone_retrieve_mock.call_args[0]) # Non-existent zone doesn't populate anything - load_mock.reset_mock() - load_mock.side_effect = \ + zone_retrieve_mock.reset_mock() + zone_retrieve_mock.side_effect = \ ResourceException('server error: zone not found') zone = Zone('unit.tests.', []) exists = provider.populate(zone) self.assertEquals(set(), zone.records) - self.assertEquals(('unit.tests',), load_mock.call_args[0]) + self.assertEquals(('unit.tests',), zone_retrieve_mock.call_args[0]) self.assertFalse(exists) # Existing zone w/o records - load_mock.reset_mock() - nsone_zone = DummyZone([]) - load_mock.side_effect = [nsone_zone] - zone_search = Mock() - zone_search.return_value = [ - { + zone_retrieve_mock.reset_mock() + nsone_zone = { + 'records': [{ "domain": "geo.unit.tests", "zone": "unit.tests", "type": "A", @@ -222,21 +211,18 @@ class TestNs1Provider(TestCase): 'meta': {'iso_region_code': ['NA-US-WA']}}, ], 'ttl': 34, - }, - ] - nsone_zone.search = zone_search + }], + } + zone_retrieve_mock.side_effect = [nsone_zone] zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals(1, len(zone.records)) - self.assertEquals(('unit.tests',), load_mock.call_args[0]) + self.assertEquals(('unit.tests',), zone_retrieve_mock.call_args[0]) # Existing zone w/records - load_mock.reset_mock() - nsone_zone = DummyZone(self.nsone_records) - load_mock.side_effect = [nsone_zone] - zone_search = Mock() - zone_search.return_value = [ - { + zone_retrieve_mock.reset_mock() + nsone_zone = { + 'records': self.nsone_records + [{ "domain": "geo.unit.tests", "zone": "unit.tests", "type": "A", @@ -250,26 +236,23 @@ class TestNs1Provider(TestCase): 'meta': {'iso_region_code': ['NA-US-WA']}}, ], 'ttl': 34, - }, - ] - nsone_zone.search = zone_search + }], + } + zone_retrieve_mock.side_effect = [nsone_zone] zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals(self.expected, zone.records) - self.assertEquals(('unit.tests',), load_mock.call_args[0]) + self.assertEquals(('unit.tests',), zone_retrieve_mock.call_args[0]) # Test skipping unsupported record type - load_mock.reset_mock() - nsone_zone = DummyZone(self.nsone_records + [{ - 'type': 'UNSUPPORTED', - 'ttl': 42, - 'short_answers': ['unsupported'], - 'domain': 'unsupported.unit.tests.', - }]) - load_mock.side_effect = [nsone_zone] - zone_search = Mock() - zone_search.return_value = [ - { + zone_retrieve_mock.reset_mock() + nsone_zone = { + 'records': self.nsone_records + [{ + 'type': 'UNSUPPORTED', + 'ttl': 42, + 'short_answers': ['unsupported'], + 'domain': 'unsupported.unit.tests.', + }, { "domain": "geo.unit.tests", "zone": "unit.tests", "type": "A", @@ -283,17 +266,23 @@ class TestNs1Provider(TestCase): 'meta': {'iso_region_code': ['NA-US-WA']}}, ], 'ttl': 34, - }, - ] - nsone_zone.search = zone_search + }], + } + zone_retrieve_mock.side_effect = [nsone_zone] zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals(self.expected, zone.records) - self.assertEquals(('unit.tests',), load_mock.call_args[0]) + self.assertEquals(('unit.tests',), zone_retrieve_mock.call_args[0]) - @patch('ns1.NS1.createZone') - @patch('ns1.NS1.loadZone') - def test_sync(self, load_mock, create_mock): + @patch('ns1.rest.records.Records.delete') + @patch('ns1.rest.records.Records.update') + @patch('ns1.rest.records.Records.create') + @patch('ns1.rest.records.Records.retrieve') + @patch('ns1.rest.zones.Zones.create') + @patch('ns1.rest.zones.Zones.retrieve') + def test_sync(self, zone_retrieve_mock, zone_create_mock, + record_retrieve_mock, record_create_mock, + record_update_mock, record_delete_mock): provider = Ns1Provider('test', 'api-key') desired = Zone('unit.tests.', []) @@ -307,71 +296,98 @@ class TestNs1Provider(TestCase): self.assertTrue(plan.exists) # Fails, general error - load_mock.reset_mock() - create_mock.reset_mock() - load_mock.side_effect = ResourceException('boom') + zone_retrieve_mock.reset_mock() + zone_create_mock.reset_mock() + zone_retrieve_mock.side_effect = ResourceException('boom') with self.assertRaises(ResourceException) as ctx: provider.apply(plan) - self.assertEquals(load_mock.side_effect, ctx.exception) + self.assertEquals(zone_retrieve_mock.side_effect, ctx.exception) # Fails, bad auth - load_mock.reset_mock() - create_mock.reset_mock() - load_mock.side_effect = \ + zone_retrieve_mock.reset_mock() + zone_create_mock.reset_mock() + zone_retrieve_mock.side_effect = \ ResourceException('server error: zone not found') - create_mock.side_effect = AuthException('unauthorized') + zone_create_mock.side_effect = AuthException('unauthorized') with self.assertRaises(AuthException) as ctx: provider.apply(plan) - self.assertEquals(create_mock.side_effect, ctx.exception) + self.assertEquals(zone_create_mock.side_effect, ctx.exception) # non-existent zone, create - load_mock.reset_mock() - create_mock.reset_mock() - load_mock.side_effect = \ + zone_retrieve_mock.reset_mock() + zone_create_mock.reset_mock() + zone_retrieve_mock.side_effect = \ ResourceException('server error: zone not found') - # ugh, need a mock zone with a mock prop since we're using getattr, we - # can actually control side effects on `meth` with that. - mock_zone = Mock() - mock_zone.add_SRV = Mock() - mock_zone.add_SRV.side_effect = [ + + zone_create_mock.side_effect = ['foo'] + # Test out the create rate-limit handling, then 9 successes + record_create_mock.side_effect = [ RateLimitException('boo', period=0), - None, - ] - create_mock.side_effect = [mock_zone] + ] + ([None] * 9) + got_n = provider.apply(plan) self.assertEquals(expected_n, got_n) + # Zone was created + zone_create_mock.assert_has_calls([call('unit.tests')]) + # Checking that we got some of the expected records too + record_create_mock.assert_has_calls([ + call('unit.tests', 'unit.tests', 'A', answers=[ + {'answer': ['1.2.3.4'], 'meta': {}} + ], filters=[], ttl=32), + call('unit.tests', 'unit.tests', 'CAA', answers=[ + (0, 'issue', 'ca.unit.tests') + ], ttl=40), + call('unit.tests', 'unit.tests', 'MX', answers=[ + (10, 'mx1.unit.tests.'), (20, 'mx2.unit.tests.') + ], ttl=35), + ]) + # Update & delete - load_mock.reset_mock() - create_mock.reset_mock() - nsone_zone = DummyZone(self.nsone_records + [{ - 'type': 'A', - 'ttl': 42, - 'short_answers': ['9.9.9.9'], - 'domain': 'delete-me.unit.tests.', - }]) - nsone_zone.data['records'][0]['short_answers'][0] = '2.2.2.2' - nsone_zone.loadRecord = Mock() - zone_search = Mock() - zone_search.return_value = [ - { + zone_retrieve_mock.reset_mock() + zone_create_mock.reset_mock() + + nsone_zone = { + 'records': self.nsone_records + [{ + 'type': 'A', + 'ttl': 42, + 'short_answers': ['9.9.9.9'], + 'domain': 'delete-me.unit.tests.', + }, { "domain": "geo.unit.tests", "zone": "unit.tests", "type": "A", - "answers": [ - {'answer': ['1.1.1.1'], 'meta': {}}, - {'answer': ['1.2.3.4'], - 'meta': {'ca_province': ['ON']}}, - {'answer': ['2.3.4.5'], 'meta': {'us_state': ['NY']}}, - {'answer': ['3.4.5.6'], 'meta': {'country': ['US']}}, - {'answer': ['4.5.6.7'], - 'meta': {'iso_region_code': ['NA-US-WA']}}, + "short_answers": [ + '1.1.1.1', + '1.2.3.4', + '2.3.4.5', + '3.4.5.6', + '4.5.6.7', ], + 'tier': 3, # This flags it as advacned, full load required 'ttl': 34, - }, - ] - nsone_zone.search = zone_search - load_mock.side_effect = [nsone_zone, nsone_zone] + }], + } + nsone_zone['records'][0]['short_answers'][0] = '2.2.2.2' + + record_retrieve_mock.side_effect = [{ + "domain": "geo.unit.tests", + "zone": "unit.tests", + "type": "A", + "answers": [ + {'answer': ['1.1.1.1'], 'meta': {}}, + {'answer': ['1.2.3.4'], + 'meta': {'ca_province': ['ON']}}, + {'answer': ['2.3.4.5'], 'meta': {'us_state': ['NY']}}, + {'answer': ['3.4.5.6'], 'meta': {'country': ['US']}}, + {'answer': ['4.5.6.7'], + 'meta': {'iso_region_code': ['NA-US-WA']}}, + ], + 'tier': 3, + 'ttl': 34, + }] + + zone_retrieve_mock.side_effect = [nsone_zone, nsone_zone] plan = provider.plan(desired) self.assertEquals(3, len(plan.changes)) # Shouldn't rely on order so just count classes @@ -380,55 +396,42 @@ class TestNs1Provider(TestCase): classes[change.__class__] += 1 self.assertEquals(1, classes[Delete]) self.assertEquals(2, classes[Update]) - # ugh, we need a mock record that can be returned from loadRecord for - # the update and delete targets, we can add our side effects to that to - # trigger rate limit handling - mock_record = Mock() - mock_record.update.side_effect = [ + + record_update_mock.side_effect = [ RateLimitException('one', period=0), None, None, ] - mock_record.delete.side_effect = [ + record_delete_mock.side_effect = [ RateLimitException('two', period=0), None, None, ] - nsone_zone.loadRecord.side_effect = [mock_record, mock_record, - mock_record] + got_n = provider.apply(plan) self.assertEquals(3, got_n) - nsone_zone.loadRecord.assert_has_calls([ - call('unit.tests', u'A'), - call('delete-me', u'A'), - call('geo', u'A'), - ]) - mock_record.assert_has_calls([ - call.update(answers=[{'answer': [u'1.2.3.4'], 'meta': {}}], - filters=[], - ttl=32), - call.update(answers=[{u'answer': [u'1.2.3.4'], u'meta': {}}], - filters=[], - ttl=32), - call.delete(), - call.delete(), - call.update( - answers=[ - {u'answer': [u'101.102.103.104'], u'meta': {}}, - {u'answer': [u'101.102.103.105'], u'meta': {}}, - { - u'answer': [u'201.202.203.204'], - u'meta': { - u'iso_region_code': [u'NA-US-NY'] - }, - }, - ], + + record_update_mock.assert_has_calls([ + call('unit.tests', 'unit.tests', 'A', answers=[ + {'answer': ['1.2.3.4'], 'meta': {}}], + filters=[], + ttl=32), + call('unit.tests', 'unit.tests', 'A', answers=[ + {'answer': ['1.2.3.4'], 'meta': {}}], + filters=[], + ttl=32), + call('unit.tests', 'geo.unit.tests', 'A', answers=[ + {'answer': ['101.102.103.104'], 'meta': {}}, + {'answer': ['101.102.103.105'], 'meta': {}}, + { + 'answer': ['201.202.203.204'], + 'meta': {'iso_region_code': ['NA-US-NY']} + }], filters=[ - {u'filter': u'shuffle', u'config': {}}, - {u'filter': u'geotarget_country', u'config': {}}, - {u'filter': u'select_first_n', u'config': {u'N': 1}}, - ], - ttl=34), + {'filter': 'shuffle', 'config': {}}, + {'filter': 'geotarget_country', 'config': {}}, + {'filter': 'select_first_n', 'config': {'N': 1}}], + ttl=34) ]) def test_escaping(self): From c4987f1a09048b6b557ce19bc194ee6598a175fe Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 9 Dec 2019 08:13:09 -0800 Subject: [PATCH 172/176] s/nsone/ns1/g --- octodns/provider/ns1.py | 22 +++++++++++----------- tests/test_octodns_provider_ns1.py | 30 +++++++++++++++--------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 19b33af..cf78241 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -23,7 +23,7 @@ class Ns1Provider(BaseProvider): ''' Ns1 provider - nsone: + ns1: class: octodns.provider.ns1.Ns1Provider api_key: env/NS1_API_KEY ''' @@ -191,14 +191,14 @@ class Ns1Provider(BaseProvider): target, lenient) try: - nsone_zone_name = zone.name[:-1] - nsone_zone = self._zones.retrieve(nsone_zone_name) + ns1_zone_name = zone.name[:-1] + ns1_zone = self._zones.retrieve(ns1_zone_name) records = [] geo_records = [] # change answers for certain types to always be absolute - for record in nsone_zone['records']: + for record in ns1_zone['records']: if record['type'] in ['ALIAS', 'CNAME', 'MX', 'NS', 'PTR', 'SRV']: for i, a in enumerate(record['short_answers']): @@ -207,7 +207,7 @@ class Ns1Provider(BaseProvider): if record.get('tier', 1) > 1: # Need to get the full record data for geo records - record = self._records.retrieve(nsone_zone_name, + record = self._records.retrieve(ns1_zone_name, record['domain'], record['type']) geo_records.append(record) @@ -312,7 +312,7 @@ class Ns1Provider(BaseProvider): for v in record.values] return {'answers': values, 'ttl': record.ttl} - def _apply_Create(self, nsone_zone, change): + def _apply_Create(self, ns1_zone, change): new = change.new zone = new.zone.name[:-1] domain = new.fqdn[:-1] @@ -327,7 +327,7 @@ class Ns1Provider(BaseProvider): sleep(period) self._records.create(zone, domain, _type, **params) - def _apply_Update(self, nsone_zone, change): + def _apply_Update(self, ns1_zone, change): new = change.new zone = new.zone.name[:-1] domain = new.fqdn[:-1] @@ -342,7 +342,7 @@ class Ns1Provider(BaseProvider): sleep(period) self._records.update(zone, domain, _type, **params) - def _apply_Delete(self, nsone_zone, change): + def _apply_Delete(self, ns1_zone, change): existing = change.existing zone = existing.zone.name[:-1] domain = existing.fqdn[:-1] @@ -364,14 +364,14 @@ class Ns1Provider(BaseProvider): domain_name = desired.name[:-1] try: - nsone_zone = self._zones.retrieve(domain_name) + ns1_zone = self._zones.retrieve(domain_name) except ResourceException as e: if e.message != self.ZONE_NOT_FOUND_MESSAGE: raise self.log.debug('_apply: no matching zone, creating') - nsone_zone = self._zones.create(domain_name) + ns1_zone = self._zones.create(domain_name) for change in changes: class_name = change.__class__.__name__ - getattr(self, '_apply_{}'.format(class_name))(nsone_zone, + getattr(self, '_apply_{}'.format(class_name))(ns1_zone, change) diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 9034552..0f23222 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -108,7 +108,7 @@ class TestNs1Provider(TestCase): }, })) - nsone_records = [{ + ns1_records = [{ 'type': 'A', 'ttl': 32, 'short_answers': ['1.2.3.4'], @@ -196,7 +196,7 @@ class TestNs1Provider(TestCase): # Existing zone w/o records zone_retrieve_mock.reset_mock() - nsone_zone = { + ns1_zone = { 'records': [{ "domain": "geo.unit.tests", "zone": "unit.tests", @@ -213,7 +213,7 @@ class TestNs1Provider(TestCase): 'ttl': 34, }], } - zone_retrieve_mock.side_effect = [nsone_zone] + zone_retrieve_mock.side_effect = [ns1_zone] zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals(1, len(zone.records)) @@ -221,8 +221,8 @@ class TestNs1Provider(TestCase): # Existing zone w/records zone_retrieve_mock.reset_mock() - nsone_zone = { - 'records': self.nsone_records + [{ + ns1_zone = { + 'records': self.ns1_records + [{ "domain": "geo.unit.tests", "zone": "unit.tests", "type": "A", @@ -238,7 +238,7 @@ class TestNs1Provider(TestCase): 'ttl': 34, }], } - zone_retrieve_mock.side_effect = [nsone_zone] + zone_retrieve_mock.side_effect = [ns1_zone] zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals(self.expected, zone.records) @@ -246,8 +246,8 @@ class TestNs1Provider(TestCase): # Test skipping unsupported record type zone_retrieve_mock.reset_mock() - nsone_zone = { - 'records': self.nsone_records + [{ + ns1_zone = { + 'records': self.ns1_records + [{ 'type': 'UNSUPPORTED', 'ttl': 42, 'short_answers': ['unsupported'], @@ -268,7 +268,7 @@ class TestNs1Provider(TestCase): 'ttl': 34, }], } - zone_retrieve_mock.side_effect = [nsone_zone] + zone_retrieve_mock.side_effect = [ns1_zone] zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals(self.expected, zone.records) @@ -347,8 +347,8 @@ class TestNs1Provider(TestCase): zone_retrieve_mock.reset_mock() zone_create_mock.reset_mock() - nsone_zone = { - 'records': self.nsone_records + [{ + ns1_zone = { + 'records': self.ns1_records + [{ 'type': 'A', 'ttl': 42, 'short_answers': ['9.9.9.9'], @@ -368,7 +368,7 @@ class TestNs1Provider(TestCase): 'ttl': 34, }], } - nsone_zone['records'][0]['short_answers'][0] = '2.2.2.2' + ns1_zone['records'][0]['short_answers'][0] = '2.2.2.2' record_retrieve_mock.side_effect = [{ "domain": "geo.unit.tests", @@ -387,7 +387,7 @@ class TestNs1Provider(TestCase): 'ttl': 34, }] - zone_retrieve_mock.side_effect = [nsone_zone, nsone_zone] + zone_retrieve_mock.side_effect = [ns1_zone, ns1_zone] plan = provider.plan(desired) self.assertEquals(3, len(plan.changes)) # Shouldn't rely on order so just count classes @@ -470,7 +470,7 @@ class TestNs1Provider(TestCase): def test_data_for_CNAME(self): provider = Ns1Provider('test', 'api-key') - # answers from nsone + # answers from ns1 a_record = { 'ttl': 31, 'type': 'CNAME', @@ -484,7 +484,7 @@ class TestNs1Provider(TestCase): self.assertEqual(a_expected, provider._data_for_CNAME(a_record['type'], a_record)) - # no answers from nsone + # no answers from ns1 b_record = { 'ttl': 32, 'type': 'CNAME', From 9e8a417c356d8f00c5e5096f0864f70bf50d5ab5 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 9 Dec 2019 08:26:20 -0800 Subject: [PATCH 173/176] Refactor thin Ns1Client wrapper out of provider --- octodns/provider/ns1.py | 98 ++++++++++++++++++++++++++--------------- 1 file changed, 63 insertions(+), 35 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index cf78241..8acbd53 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -19,6 +19,56 @@ from ..record import Record from .base import BaseProvider +class Ns1Client(object): + log = getLogger('NS1Client') + + def __init__(self, api_key, retry_delay=1): + self.retry_delay = retry_delay + + client = NS1(apiKey=api_key) + self._records = client.records() + self._zones = client.zones() + + def zones_retrieve(self, name): + return self._zones.retrieve(name) + + def zones_create(self, name): + return self._zones.create(name) + + def records_retrieve(self, zone, domain, _type): + return self._records.retrieve(zone, domain, _type) + + def records_create(self, zone, domain, _type, **params): + try: + return self._records.create(zone, domain, _type, **params) + except RateLimitException as e: + period = float(e.period) + self.log.warn('_apply_Create: rate limit encountered, pausing ' + 'for %ds and trying again', period) + sleep(period) + return self._records.create(zone, domain, _type, **params) + + def records_update(self, zone, domain, _type, **params): + try: + return self._records.update(zone, domain, _type, **params) + except RateLimitException as e: + period = float(e.period) + self.log.warn('_apply_Update: rate limit encountered, pausing ' + 'for %ds and trying again', period) + sleep(period) + return self._records.update(zone, domain, _type, **params) + + def records_delete(self, zone, domain, _type): + try: + return self._records.delete(zone, domain, _type) + except RateLimitException as e: + period = float(e.period) + self.log.warn('_apply_Delete: rate limit encountered, pausing ' + 'for %ds and trying again', period) + sleep(period) + return self._records.delete(zone, domain, _type) + + class Ns1Provider(BaseProvider): ''' Ns1 provider @@ -34,13 +84,12 @@ class Ns1Provider(BaseProvider): ZONE_NOT_FOUND_MESSAGE = 'server error: zone not found' - def __init__(self, id, api_key, *args, **kwargs): + def __init__(self, id, api_key, retry_delay=1, *args, **kwargs): self.log = getLogger('Ns1Provider[{}]'.format(id)) - self.log.debug('__init__: id=%s, api_key=***', id) + self.log.debug('__init__: id=%s, api_key=***, retry_delay=%d', id, + retry_delay) super(Ns1Provider, self).__init__(id, *args, **kwargs) - client = NS1(apiKey=api_key) - self._records = client.records() - self._zones = client.zones() + self._client = Ns1Client(api_key, retry_delay) def _data_for_A(self, _type, record): # record meta (which would include geo information is only @@ -192,7 +241,7 @@ class Ns1Provider(BaseProvider): try: ns1_zone_name = zone.name[:-1] - ns1_zone = self._zones.retrieve(ns1_zone_name) + ns1_zone = self._client.zones_retrieve(ns1_zone_name) records = [] geo_records = [] @@ -207,9 +256,9 @@ class Ns1Provider(BaseProvider): if record.get('tier', 1) > 1: # Need to get the full record data for geo records - record = self._records.retrieve(ns1_zone_name, - record['domain'], - record['type']) + record = self._client.records_retrieve(ns1_zone_name, + record['domain'], + record['type']) geo_records.append(record) else: records.append(record) @@ -318,14 +367,7 @@ class Ns1Provider(BaseProvider): domain = new.fqdn[:-1] _type = new._type params = getattr(self, '_params_for_{}'.format(_type))(new) - try: - self._records.create(zone, domain, _type, **params) - except RateLimitException as e: - period = float(e.period) - self.log.warn('_apply_Create: rate limit encountered, pausing ' - 'for %ds and trying again', period) - sleep(period) - self._records.create(zone, domain, _type, **params) + self._client.records_create(zone, domain, _type, **params) def _apply_Update(self, ns1_zone, change): new = change.new @@ -333,28 +375,14 @@ class Ns1Provider(BaseProvider): domain = new.fqdn[:-1] _type = new._type params = getattr(self, '_params_for_{}'.format(_type))(new) - try: - self._records.update(zone, domain, _type, **params) - except RateLimitException as e: - period = float(e.period) - self.log.warn('_apply_Update: rate limit encountered, pausing ' - 'for %ds and trying again', period) - sleep(period) - self._records.update(zone, domain, _type, **params) + self._client.records_update(zone, domain, _type, **params) def _apply_Delete(self, ns1_zone, change): existing = change.existing zone = existing.zone.name[:-1] domain = existing.fqdn[:-1] _type = existing._type - try: - self._records.delete(zone, domain, _type) - except RateLimitException as e: - period = float(e.period) - self.log.warn('_apply_Delete: rate limit encountered, pausing ' - 'for %ds and trying again', period) - sleep(period) - self._records.delete(zone, domain, _type) + self._client.records_delete(zone, domain, _type) def _apply(self, plan): desired = plan.desired @@ -364,12 +392,12 @@ class Ns1Provider(BaseProvider): domain_name = desired.name[:-1] try: - ns1_zone = self._zones.retrieve(domain_name) + ns1_zone = self._client.zones_retrieve(domain_name) except ResourceException as e: if e.message != self.ZONE_NOT_FOUND_MESSAGE: raise self.log.debug('_apply: no matching zone, creating') - ns1_zone = self._zones.create(domain_name) + ns1_zone = self._client.zones_create(domain_name) for change in changes: class_name = change.__class__.__name__ From 4fd2daa8a990521ae89d3be1a183f75ddee0fcd5 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 9 Dec 2019 08:56:55 -0800 Subject: [PATCH 174/176] Implement reworked NS1 retry mechinism --- octodns/provider/ns1.py | 59 +++++++++++++----------------- tests/test_octodns_provider_ns1.py | 46 ++++++++++++++++++++++- 2 files changed, 71 insertions(+), 34 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 8acbd53..da2d64a 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -22,51 +22,44 @@ from .base import BaseProvider class Ns1Client(object): log = getLogger('NS1Client') - def __init__(self, api_key, retry_delay=1): - self.retry_delay = retry_delay + def __init__(self, api_key, retry_count=4): + self.retry_count = retry_count client = NS1(apiKey=api_key) self._records = client.records() self._zones = client.zones() + def _try(self, method, *args, **kwargs): + tries = self.retry_count + while tries: + try: + return method(*args, **kwargs) + except RateLimitException as e: + period = float(e.period) + self.log.warn('rate limit encountered, pausing ' + 'for %ds and trying again, %d remaining', + period, tries) + sleep(period) + tries -= 1 + raise + def zones_retrieve(self, name): - return self._zones.retrieve(name) + return self._try(self._zones.retrieve, name) def zones_create(self, name): - return self._zones.create(name) + return self._try(self._zones.create, name) def records_retrieve(self, zone, domain, _type): - return self._records.retrieve(zone, domain, _type) + return self._try(self._records.retrieve, zone, domain, _type) def records_create(self, zone, domain, _type, **params): - try: - return self._records.create(zone, domain, _type, **params) - except RateLimitException as e: - period = float(e.period) - self.log.warn('_apply_Create: rate limit encountered, pausing ' - 'for %ds and trying again', period) - sleep(period) - return self._records.create(zone, domain, _type, **params) + return self._try(self._records.create, zone, domain, _type, **params) def records_update(self, zone, domain, _type, **params): - try: - return self._records.update(zone, domain, _type, **params) - except RateLimitException as e: - period = float(e.period) - self.log.warn('_apply_Update: rate limit encountered, pausing ' - 'for %ds and trying again', period) - sleep(period) - return self._records.update(zone, domain, _type, **params) + return self._try(self._records.update, zone, domain, _type, **params) def records_delete(self, zone, domain, _type): - try: - return self._records.delete(zone, domain, _type) - except RateLimitException as e: - period = float(e.period) - self.log.warn('_apply_Delete: rate limit encountered, pausing ' - 'for %ds and trying again', period) - sleep(period) - return self._records.delete(zone, domain, _type) + return self._try(self._records.delete, zone, domain, _type) class Ns1Provider(BaseProvider): @@ -84,12 +77,12 @@ class Ns1Provider(BaseProvider): ZONE_NOT_FOUND_MESSAGE = 'server error: zone not found' - def __init__(self, id, api_key, retry_delay=1, *args, **kwargs): + def __init__(self, id, api_key, retry_count=4, *args, **kwargs): self.log = getLogger('Ns1Provider[{}]'.format(id)) - self.log.debug('__init__: id=%s, api_key=***, retry_delay=%d', id, - retry_delay) + self.log.debug('__init__: id=%s, api_key=***, retry_count=%d', id, + retry_count) super(Ns1Provider, self).__init__(id, *args, **kwargs) - self._client = Ns1Client(api_key, retry_delay) + self._client = Ns1Client(api_key, retry_count) def _data_for_A(self, _type, record): # record meta (which would include geo information is only diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 0f23222..0743943 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -9,10 +9,11 @@ from collections import defaultdict from mock import call, patch from ns1.rest.errors import AuthException, RateLimitException, \ ResourceException +from six import text_type from unittest import TestCase from octodns.record import Delete, Record, Update -from octodns.provider.ns1 import Ns1Provider +from octodns.provider.ns1 import Ns1Client, Ns1Provider from octodns.zone import Zone @@ -497,3 +498,46 @@ class TestNs1Provider(TestCase): } self.assertEqual(b_expected, provider._data_for_CNAME(b_record['type'], b_record)) + + +class TestNs1Client(TestCase): + + @patch('ns1.rest.zones.Zones.retrieve') + def test_retry_behavior(self, zone_retrieve_mock): + client = Ns1Client('dummy-key') + + # No retry required, just calls and is returned + zone_retrieve_mock.reset_mock() + zone_retrieve_mock.side_effect = ['foo'] + self.assertEquals('foo', client.zones_retrieve('unit.tests')) + zone_retrieve_mock.assert_has_calls([call('unit.tests')]) + + # One retry required + zone_retrieve_mock.reset_mock() + zone_retrieve_mock.side_effect = [ + RateLimitException('boo', period=0), + 'foo' + ] + self.assertEquals('foo', client.zones_retrieve('unit.tests')) + zone_retrieve_mock.assert_has_calls([call('unit.tests')]) + + # Two retries required + zone_retrieve_mock.reset_mock() + zone_retrieve_mock.side_effect = [ + RateLimitException('boo', period=0), + 'foo' + ] + self.assertEquals('foo', client.zones_retrieve('unit.tests')) + zone_retrieve_mock.assert_has_calls([call('unit.tests')]) + + # Exhaust our retries + zone_retrieve_mock.reset_mock() + zone_retrieve_mock.side_effect = [ + RateLimitException('first', period=0), + RateLimitException('boo', period=0), + RateLimitException('boo', period=0), + RateLimitException('last', period=0), + ] + with self.assertRaises(RateLimitException) as ctx: + client.zones_retrieve('unit.tests') + self.assertEquals('last', text_type(ctx.exception)) From 334e64c8f5148c1f25d9227b8793a4feff38a2b8 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 10 Dec 2019 12:20:25 -0800 Subject: [PATCH 175/176] Python 3 friendly way to re-raise when tries expire --- octodns/provider/ns1.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index da2d64a..0383dbf 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -31,17 +31,18 @@ class Ns1Client(object): def _try(self, method, *args, **kwargs): tries = self.retry_count - while tries: + while True: # We'll raise to break after our tries expire try: return method(*args, **kwargs) except RateLimitException as e: + if tries <= 1: + raise period = float(e.period) self.log.warn('rate limit encountered, pausing ' 'for %ds and trying again, %d remaining', period, tries) sleep(period) tries -= 1 - raise def zones_retrieve(self, name): return self._try(self._zones.retrieve, name) From ed482c60ca8c1f1c3cb4131da6e1a4cd64651c65 Mon Sep 17 00:00:00 2001 From: Dan Hanks Date: Thu, 19 Dec 2019 10:33:22 -0500 Subject: [PATCH 176/176] Document max_workers flag --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index c3ddb0b..83e28ca 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,9 @@ We start by creating a config file to tell OctoDNS about our providers and the z ```yaml --- +manager: + max_workers: 2 + providers: config: class: octodns.provider.yaml.YamlProvider @@ -80,6 +83,8 @@ zones: Further information can be found in the `docstring` of each source and provider class. +The `max_workers` key in the `manager` section of the config enables threading to parallelize the planning portion of the sync. + Now that we have something to tell OctoDNS about our providers & zones we need to tell it about or records. We'll keep it simple for now and just create a single `A` record at the top-level of the domain. `config/example.com.yaml`