From 5158d28b03b950fd06cf5166f769174d568129ec Mon Sep 17 00:00:00 2001 From: mintopia Date: Thu, 10 Sep 2020 23:52:23 +0100 Subject: [PATCH 01/11] Update endpoint for Constellix provider to only include /domains when working on domain and record resources. In order to add support for pools and other API resources from Constellix, we need to update the base URL to not contain domains and instead specify this where it's needed. --- octodns/provider/constellix.py | 13 +++++++------ tests/test_octodns_provider_constellix.py | 14 +++++++------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/octodns/provider/constellix.py b/octodns/provider/constellix.py index 5ca89e1..7829386 100644 --- a/octodns/provider/constellix.py +++ b/octodns/provider/constellix.py @@ -44,7 +44,7 @@ class ConstellixClientNotFound(ConstellixClientException): class ConstellixClient(object): - BASE = 'https://api.dns.constellix.com/v1/domains' + BASE = 'https://api.dns.constellix.com/v1' def __init__(self, api_key, secret_key, ratelimit_delay=0.0): self.api_key = api_key @@ -88,7 +88,7 @@ class ConstellixClient(object): if self._domains is None: zones = [] - resp = self._request('GET', '').json() + resp = self._request('GET', '/domains').json() zones += resp self._domains = {'{}.'.format(z['name']): z['id'] for z in zones} @@ -103,7 +103,7 @@ class ConstellixClient(object): return self._request('GET', path).json() def domain_create(self, name): - resp = self._request('POST', '/', data={'names': [name]}) + resp = self._request('POST', '/domains', data={'names': [name]}) # Add newly created zone to domain cache self._domains['{}.'.format(name)] = resp.json()[0]['id'] @@ -119,7 +119,7 @@ class ConstellixClient(object): zone_id = self.domains.get(zone_name, False) if not zone_id: raise ConstellixClientNotFound() - path = '/{}/records'.format(zone_id) + path = '/domains/{}/records'.format(zone_id) resp = self._request('GET', path).json() for record in resp: @@ -151,7 +151,7 @@ class ConstellixClient(object): record_type = 'ANAME' zone_id = self.domains.get(zone_name, False) - path = '/{}/records/{}'.format(zone_id, record_type) + path = '/domains/{}/records/{}'.format(zone_id, record_type) self._request('POST', path, data=params) @@ -161,7 +161,8 @@ class ConstellixClient(object): record_type = 'ANAME' zone_id = self.domains.get(zone_name, False) - path = '/{}/records/{}/{}'.format(zone_id, record_type, record_id) + path = '/domains/{}/records/{}/{}'.format(zone_id, record_type, + record_id) self._request('DELETE', path) diff --git a/tests/test_octodns_provider_constellix.py b/tests/test_octodns_provider_constellix.py index 151d0d4..52a6b4c 100644 --- a/tests/test_octodns_provider_constellix.py +++ b/tests/test_octodns_provider_constellix.py @@ -144,15 +144,15 @@ class TestConstellixProvider(TestCase): provider._client._request.assert_has_calls([ # get all domains to build the cache - call('GET', ''), + call('GET', '/domains'), # created the domain - call('POST', '/', data={'names': ['unit.tests']}) + call('POST', '/domains', data={'names': ['unit.tests']}) ]) # These two checks are broken up so that ordering doesn't break things. # Python3 doesn't make the calls in a consistent order so different # things follow the GET / on different runs provider._client._request.assert_has_calls([ - call('POST', '/123123/records/SRV', data={ + call('POST', '/domains/123123/records/SRV', data={ 'roundRobin': [{ 'priority': 10, 'weight': 20, @@ -218,14 +218,14 @@ class TestConstellixProvider(TestCase): # recreate for update, and deletes for the 2 parts of the other provider._client._request.assert_has_calls([ - call('POST', '/123123/records/A', data={ + call('POST', '/domains/123123/records/A', data={ 'roundRobin': [{ 'value': '3.2.3.4' }], 'name': 'ttl', 'ttl': 300 }), - call('DELETE', '/123123/records/A/11189897'), - call('DELETE', '/123123/records/A/11189898'), - call('DELETE', '/123123/records/ANAME/11189899') + call('DELETE', '/domains/123123/records/A/11189897'), + call('DELETE', '/domains/123123/records/A/11189898'), + call('DELETE', '/domains/123123/records/ANAME/11189899') ], any_order=True) From ce467587907b151788dac6e201fbd193367153f6 Mon Sep 17 00:00:00 2001 From: mintopia Date: Thu, 1 Oct 2020 00:36:25 +0100 Subject: [PATCH 02/11] Add support for dynamic records to Constellix provider --- octodns/provider/constellix.py | 131 ++++++++- tests/fixtures/constellix-pools.json | 32 +++ tests/fixtures/constellix-records.json | 37 +++ tests/test_octodns_provider_constellix.py | 314 +++++++++++++++++++++- 4 files changed, 501 insertions(+), 13 deletions(-) create mode 100644 tests/fixtures/constellix-pools.json diff --git a/octodns/provider/constellix.py b/octodns/provider/constellix.py index 7829386..cb2a53f 100644 --- a/octodns/provider/constellix.py +++ b/octodns/provider/constellix.py @@ -53,6 +53,7 @@ class ConstellixClient(object): self._sess = Session() self._sess.headers.update({'x-cnsdns-apiKey': self.api_key}) self._domains = None + self._pools = None def _current_time(self): return str(int(time.time() * 1000)) @@ -99,7 +100,7 @@ class ConstellixClient(object): zone_id = self.domains.get(name, False) if not zone_id: raise ConstellixClientNotFound() - path = '/{}'.format(zone_id) + path = '/domains/{}'.format(zone_id) return self._request('GET', path).json() def domain_create(self, name): @@ -165,6 +166,48 @@ class ConstellixClient(object): record_id) self._request('DELETE', path) + def pools(self, pool_type): + if self._pools is None: + self._pools = {} + path = '/pools/{}'.format(pool_type) + response = self._request('GET', path).json() + for pool in response: + self._pools[pool['id']] = pool + return self._pools.values() + + def pool(self, pool_type, pool_name): + pools = self.pools(pool_type) + for pool in pools: + if pool['name'] == pool_name: + return pool + return None + + def pool_by_id(self, pool_type, pool_id): + pools = self.pools(pool_type) + for pool in pools: + if pool['id'] == pool_id: + return pool + + def pool_create(self, data): + path = '/pools/{}'.format(data.get('type')) + # This returns a list of items, we want the first one + response = self._request('POST', path, data=data).json()[0] + + # Invalidate our cache + self._pools = None + return response + + def pool_update(self, pool_id, data): + path = '/pools/{}/{}'.format(data.get('type'), pool_id) + try: + self._request('PUT', path, data=data).json() + + except ConstellixClientBadRequest as e: + message = str(e) + if not message or "no changes to save" not in message: + raise e + return data + class ConstellixProvider(BaseProvider): ''' @@ -181,7 +224,7 @@ class ConstellixProvider(BaseProvider): ratelimit_delay: 0.0 ''' SUPPORTS_GEO = False - SUPPORTS_DYNAMIC = False + SUPPORTS_DYNAMIC = True SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NS', 'PTR', 'SPF', 'SRV', 'TXT')) @@ -195,12 +238,41 @@ class ConstellixProvider(BaseProvider): def _data_for_multiple(self, _type, records): record = records[0] + if record['recordOption'] == 'pools': + return self._data_for_pool(_type, record) return { 'ttl': record['ttl'], 'type': _type, 'values': record['value'] } + def _data_for_pool(self, _type, record): + pool_id = record['pools'][0] + pool = self._client.pool_by_id(_type, pool_id) + pool_name = pool['name'].split(':')[-1] + pools = {} + values = [] + pools[pool_name] = { + 'values': [] + } + for value in pool['values']: + pools[pool_name]['values'].append({ + 'value': value['value'], + 'weight': value['weight'] + }) + values.append(value['value']) + return { + 'ttl': record['ttl'], + 'type': _type, + 'dynamic': { + 'pools': pools, + 'rules': [{ + 'pool': pool_name + }] + }, + 'value': values + } + _data_for_A = _data_for_multiple _data_for_AAAA = _data_for_multiple @@ -421,10 +493,65 @@ class ConstellixProvider(BaseProvider): 'roundRobin': values } + def _handle_pools(self, record): + # If we don't have dynamic, then there's no pools + if not getattr(record, 'dynamic', False): + return None + + # Get our first entry in the rules that references a pool + rules = list(filter( + lambda rule: 'pool' in rule.data, + record.dynamic.rules + )) + + pool_name = rules[0].data.get('pool') + + pool = record.dynamic.pools.get(pool_name) + values = pool.data.get('values') + + # Make a pool name based on zone, record, type and name + pool_name = '{}:{}:{}:{}'.format( + record.zone.name, + record.name, + record._type, + pool_name + ) + + # OK, pool is valid, let's create it or update it + return self._create_update_pool( + pool_name = pool_name, + pool_type = record._type, + ttl = record.ttl, + values = values + ) + + def _create_update_pool(self, pool_name, pool_type, ttl, values): + pool = { + 'name': pool_name, + 'type': pool_type, + 'numReturn': 1, + 'minAvailableFailover': 1, + 'ttl': ttl, + 'values': values + } + existing_pool = self._client.pool(pool_type, pool_name) + if not existing_pool: + return self._client.pool_create(pool) + + pool_id = existing_pool['id'] + updated_pool = self._client.pool_update(pool_id, pool) + updated_pool['id'] = pool_id + return updated_pool + def _apply_Create(self, change): new = change.new params_for = getattr(self, '_params_for_{}'.format(new._type)) + pool = self._handle_pools(new) + for params in params_for(new): + if pool: + params['pools'] = [pool['id']] + params['recordOption'] = 'pools' self._client.record_create(new.zone.name, new._type, params) def _apply_Update(self, change): diff --git a/tests/fixtures/constellix-pools.json b/tests/fixtures/constellix-pools.json new file mode 100644 index 0000000..8fbd491 --- /dev/null +++ b/tests/fixtures/constellix-pools.json @@ -0,0 +1,32 @@ +[ + { + "id": 1808521, + "name": "unit.tests.:www.dynamic:A:two", + "type": "A", + "numReturn": 1, + "minAvailableFailover": 1, + "createdTs": "2020-09-12T00:44:35Z", + "modifiedTs": "2020-09-12T00:44:35Z", + "appliedDomains": [ + { + "id": 123123, + "name": "unit.tests", + "recordOption": "pools" + } + ], + "appliedTemplates": null, + "unlinkedDomains": [], + "unlinkedTemplates": null, + "itoEnabled": false, + "values": [ + { + "value": "1.2.3.4", + "weight": 1 + }, + { + "value": "1.2.3.5", + "weight": 1 + } + ] + } +] \ No newline at end of file diff --git a/tests/fixtures/constellix-records.json b/tests/fixtures/constellix-records.json index c1f1fb4..545eada 100644 --- a/tests/fixtures/constellix-records.json +++ b/tests/fixtures/constellix-records.json @@ -595,4 +595,41 @@ "roundRobinFailover": [], "pools": [], "poolsDetail": [] +}, { + "id": 1808520, + "type": "A", + "recordType": "a", + "name": "www.dynamic", + "recordOption": "pools", + "noAnswer": false, + "note": "", + "ttl": 300, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565150090588, + "value": [], + "roundRobin": [], + "geolocation": null, + "recordFailover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "failover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "roundRobinFailover": [], + "pools": [ + 1808521 + ], + "poolsDetail": [{ + "id": 1808521, + "name": "unit.tests.:www.dynamic:A:two" + }] }] diff --git a/tests/test_octodns_provider_constellix.py b/tests/test_octodns_provider_constellix.py index 52a6b4c..d54d1fe 100644 --- a/tests/test_octodns_provider_constellix.py +++ b/tests/test_octodns_provider_constellix.py @@ -15,7 +15,7 @@ from unittest import TestCase from octodns.record import Record from octodns.provider.constellix import \ - ConstellixProvider + ConstellixProvider, ConstellixClientBadRequest from octodns.provider.yaml import YamlProvider from octodns.zone import Zone @@ -48,6 +48,32 @@ class TestConstellixProvider(TestCase): 'value': 'aname.unit.tests.' })) + # Add a dynamic record + expected.add_record(Record.new(expected, 'www.dynamic', { + 'ttl': 300, + 'type': 'A', + 'value': [ + '1.2.3.4', + '1.2.3.5' + ], + 'dynamic': { + 'pools': { + 'two': { + 'values': [{ + 'value': '1.2.3.4', + 'weight': 1 + }, { + 'value': '1.2.3.5', + 'weight': 1 + }], + }, + }, + 'rules': [{ + 'pool': 'two', + }], + }, + })) + for record in list(expected.records): if record.name == 'sub' and record._type == 'NS': expected._remove_record(record) @@ -98,23 +124,26 @@ class TestConstellixProvider(TestCase): # No diffs == no changes with requests_mock() as mock: - base = 'https://api.dns.constellix.com/v1/domains' + base = 'https://api.dns.constellix.com/v1' with open('tests/fixtures/constellix-domains.json') as fh: - mock.get('{}{}'.format(base, ''), text=fh.read()) + mock.get('{}{}'.format(base, '/domains'), text=fh.read()) with open('tests/fixtures/constellix-records.json') as fh: - mock.get('{}{}'.format(base, '/123123/records'), + mock.get('{}{}'.format(base, '/domains/123123/records'), + text=fh.read()) + with open('tests/fixtures/constellix-pools.json') as fh: + mock.get('{}{}'.format(base, '/pools/A'), text=fh.read()) zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(15, len(zone.records)) + self.assertEquals(16, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) # 2nd populate makes no network calls/all from cache again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(15, len(again.records)) + self.assertEquals(16, len(again.records)) # bust the cache del provider._zone_records[zone.name] @@ -133,6 +162,11 @@ class TestConstellixProvider(TestCase): 'id': 123123, 'name': 'unit.tests' }], # domain created in apply + [], # No pools returned during populate + [{ + "id": 1808520, + "name": "unit.tests.:www.dynamic:A:two", + }] # pool created in apply ] plan = provider.plan(self.expected) @@ -148,6 +182,28 @@ class TestConstellixProvider(TestCase): # created the domain call('POST', '/domains', data={'names': ['unit.tests']}) ]) + + # Check we tried to get our pool + provider._client._request.assert_has_calls([ + # get all pools to build the cache + call('GET', '/pools/A'), + # created the pool + call('POST', '/pools/A', data={ + 'name': 'unit.tests.:www.dynamic:A:two', + 'type': 'A', + 'numReturn': 1, + 'minAvailableFailover': 1, + 'ttl': 300, + 'values': [{ + "value": "1.2.3.4", + "weight": 1 + }, { + "value": "1.2.3.5", + "weight": 1 + }] + }) + ]) + # These two checks are broken up so that ordering doesn't break things. # Python3 doesn't make the calls in a consistent order so different # things follow the GET / on different runs @@ -169,7 +225,7 @@ class TestConstellixProvider(TestCase): }), ]) - self.assertEquals(18, provider._client._request.call_count) + self.assertEquals(21, provider._client._request.call_count) provider._client._request.reset_mock() @@ -179,6 +235,7 @@ class TestConstellixProvider(TestCase): 'type': 'A', 'name': 'www', 'ttl': 300, + 'recordOption': 'roundRobin', 'value': [ '1.2.3.4', '2.2.3.4', @@ -188,6 +245,7 @@ class TestConstellixProvider(TestCase): 'type': 'A', 'name': 'ttl', 'ttl': 600, + 'recordOption': 'roundRobin', 'value': [ '3.2.3.4' ] @@ -196,14 +254,44 @@ class TestConstellixProvider(TestCase): 'type': 'ALIAS', 'name': 'alias', 'ttl': 600, + 'recordOption': 'roundRobin', 'value': [{ 'value': 'aname.unit.tests.' }] + }, { + "id": 1808520, + "type": "A", + "name": "www.dynamic", + "recordOption": "pools", + "ttl": 300, + "value": [], + "pools": [ + 1808521 + ] } ]) + provider._client.pools = Mock(return_value=[{ + "id": 1808521, + "name": "unit.tests.:www.dynamic:A:two", + "type": "A", + "values": [ + { + "value": "1.2.3.4", + "weight": 1 + }, + { + "value": "1.2.3.5", + "weight": 1 + } + ] + }]) + # Domain exists, we don't care about return - resp.json.side_effect = ['{}'] + resp.json.side_effect = [ + ['{}'], + ['{}'], + ] wanted = Zone('unit.tests.', []) wanted.add_record(Record.new(wanted, 'ttl', { @@ -212,9 +300,30 @@ class TestConstellixProvider(TestCase): 'value': '3.2.3.4' })) + wanted.add_record(Record.new(wanted, 'www.dynamic', { + 'ttl': 300, + 'type': 'A', + 'value': [ + '1.2.3.4' + ], + 'dynamic': { + 'pools': { + 'two': { + 'values': [{ + 'value': '1.2.3.4', + 'weight': 1 + }], + }, + }, + 'rules': [{ + 'pool': 'two', + }], + }, + })) + plan = provider.plan(wanted) - self.assertEquals(3, len(plan.changes)) - self.assertEquals(3, provider.apply(plan)) + self.assertEquals(4, len(plan.changes)) + self.assertEquals(4, provider.apply(plan)) # recreate for update, and deletes for the 2 parts of the other provider._client._request.assert_has_calls([ @@ -225,7 +334,190 @@ class TestConstellixProvider(TestCase): 'name': 'ttl', 'ttl': 300 }), + call('PUT', '/pools/A/1808521', data={ + 'name': 'unit.tests.:www.dynamic:A:two', + 'type': 'A', + 'numReturn': 1, + 'minAvailableFailover': 1, + 'ttl': 300, + 'id': 1808521, + 'values': [{ + "value": "1.2.3.4", + "weight": 1 + }] + }), call('DELETE', '/domains/123123/records/A/11189897'), call('DELETE', '/domains/123123/records/A/11189898'), - call('DELETE', '/domains/123123/records/ANAME/11189899') + call('DELETE', '/domains/123123/records/ANAME/11189899'), ], any_order=True) + + def test_dynamic_record_failures(self): + provider = ConstellixProvider('test', 'api', 'secret') + + resp = Mock() + resp.json = Mock() + provider._client._request = Mock(return_value=resp) + + # Let's handle some failures for pools - first if it's not a simple + # weighted pool - we'll be OK as we assume a weight of 1 for all + # entries + provider._client._request.reset_mock() + provider._client.records = Mock(return_value=[ + { + "id": 1808520, + "type": "A", + "name": "www.dynamic", + "recordOption": "pools", + "ttl": 300, + "value": [], + "pools": [ + 1808521 + ] + } + ]) + + provider._client.pools = Mock(return_value=[{ + "id": 1808521, + "name": "unit.tests.:www.dynamic:A:two", + "type": "A", + "values": [ + { + "value": "1.2.3.4", + "weight": 1 + } + ] + }]) + + wanted = Zone('unit.tests.', []) + + resp.json.side_effect = [ + ['{}'], + ['{}'], + ] + wanted.add_record(Record.new(wanted, 'www.dynamic', { + 'ttl': 300, + 'type': 'A', + 'value': [ + '1.2.3.4' + ], + 'dynamic': { + 'pools': { + 'two': { + 'values': [{ + 'value': '1.2.3.4' + }], + }, + }, + 'rules': [{ + 'pool': 'two', + }], + }, + })) + + plan = provider.plan(wanted) + self.assertIsNone(plan) + + def test_dynamic_record_updates(self): + provider = ConstellixProvider('test', 'api', 'secret') + + # Constellix API can return an error if you try and update a pool and + # don't change anything, so let's test we handle it silently + + provider._client.records = Mock(return_value=[ + { + "id": 1808520, + "type": "A", + "name": "www.dynamic", + "recordOption": "pools", + "ttl": 300, + "value": [], + "pools": [ + 1808521 + ] + } + ]) + + provider._client.pools = Mock(return_value=[{ + "id": 1808521, + "name": "unit.tests.:www.dynamic:A:two", + "type": "A", + "values": [ + { + "value": "1.2.3.4", + "weight": 1 + } + ] + }]) + + wanted = Zone('unit.tests.', []) + + wanted.add_record(Record.new(wanted, 'www.dynamic', { + 'ttl': 300, + 'type': 'A', + 'value': [ + '1.2.3.4' + ], + 'dynamic': { + 'pools': { + 'two': { + 'values': [{ + 'value': '1.2.3.5' + }], + }, + }, + 'rules': [{ + 'pool': 'two', + }], + }, + })) + + # Try an error we can handle + with requests_mock() as mock: + mock.get(ANY, status_code=200, + text='{}') + mock.delete(ANY, status_code=200, + text='{}') + mock.put("https://api.dns.constellix.com/v1/pools/A/1808521", + status_code=400, + text='{"errors": [\"no changes to save\"]}') + mock.post(ANY, status_code=200, + text='[{"id": 1234}]') + + plan = provider.plan(wanted) + self.assertEquals(1, len(plan.changes)) + self.assertEquals(1, provider.apply(plan)) + + # Now what happens if an error happens that we can't handle + with requests_mock() as mock: + mock.get(ANY, status_code=200, + text='{}') + mock.delete(ANY, status_code=200, + text='{}') + mock.put("https://api.dns.constellix.com/v1/pools/A/1808521", + status_code=400, + text='{"errors": [\"generic error\"]}') + mock.post(ANY, status_code=200, + text='[{"id": 1234}]') + + plan = provider.plan(wanted) + self.assertEquals(1, len(plan.changes)) + with self.assertRaises(ConstellixClientBadRequest): + provider.apply(plan) + + def test_pools_that_are_notfound(self): + provider = ConstellixProvider('test', 'api', 'secret') + + provider._client.pools = Mock(return_value=[{ + "id": 1808521, + "name": "unit.tests.:www.dynamic:A:two", + "type": "A", + "values": [ + { + "value": "1.2.3.4", + "weight": 1 + } + ] + }]) + + self.assertIsNone(provider._client.pool_by_id('A', 1)) + self.assertIsNone(provider._client.pool('A', 'foobar')) From e89f309179bab3c81b21e6bdf2f0f3999efb6493 Mon Sep 17 00:00:00 2001 From: mintopia Date: Fri, 16 Oct 2020 17:53:07 +0100 Subject: [PATCH 03/11] Ensure pool caching works for Constellix Provider The caching of pools for the Constellix provider will now cache based on the type of pool and the name. Previously it was caching based on the name only. --- octodns/provider/constellix.py | 2 +- tests/test_octodns_provider_constellix.py | 51 +++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/octodns/provider/constellix.py b/octodns/provider/constellix.py index cb2a53f..67f7a6d 100644 --- a/octodns/provider/constellix.py +++ b/octodns/provider/constellix.py @@ -178,7 +178,7 @@ class ConstellixClient(object): def pool(self, pool_type, pool_name): pools = self.pools(pool_type) for pool in pools: - if pool['name'] == pool_name: + if pool['name'] == pool_name and pool['type'] == pool_type: return pool return None diff --git a/tests/test_octodns_provider_constellix.py b/tests/test_octodns_provider_constellix.py index d54d1fe..1ca3179 100644 --- a/tests/test_octodns_provider_constellix.py +++ b/tests/test_octodns_provider_constellix.py @@ -521,3 +521,54 @@ class TestConstellixProvider(TestCase): self.assertIsNone(provider._client.pool_by_id('A', 1)) self.assertIsNone(provider._client.pool('A', 'foobar')) + + def test_pools_are_cached_correctly(self): + provider = ConstellixProvider('test', 'api', 'secret') + + provider._client.pools = Mock(return_value=[{ + "id": 1808521, + "name": "unit.tests.:www.dynamic:A:two", + "type": "A", + "values": [ + { + "value": "1.2.3.4", + "weight": 1 + } + ] + }]) + + found = provider._client.pool('A', 'unit.tests.:www.dynamic:A:two') + self.assertIsNotNone(found) + + not_found = provider._client.pool('AAAA', + 'unit.tests.:www.dynamic:A:two') + self.assertIsNone(not_found) + + provider._client.pools = Mock(return_value=[{ + "id": 42, + "name": "unit.tests.:www.dynamic:A:two", + "type": "A", + "values": [ + { + "value": "1.2.3.4", + "weight": 1 + } + ] + }, { + "id": 451, + "name": "unit.tests.:www.dynamic:A:two", + "type": "AAAA", + "values": [ + { + "value": "1.2.3.4", + "weight": 1 + } + ] + }]) + + a_pool = provider._client.pool('A', 'unit.tests.:www.dynamic:A:two') + self.assertEquals(42, a_pool['id']) + + aaaa_pool = provider._client.pool('AAAA', + 'unit.tests.:www.dynamic:A:two') + self.assertEquals(451, aaaa_pool['id']) From 7c44e5eedc4e4db60fae15298e52d3692b9aee6e Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Wed, 14 Apr 2021 06:59:34 +0200 Subject: [PATCH 04/11] fixed DigitalOcean provider issue with CAA records as reported here: https://github.com/octodns/octodns/pull/691#issuecomment-819228150 --- octodns/provider/digitalocean.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/digitalocean.py b/octodns/provider/digitalocean.py index 6ccee1d..41e6b6b 100644 --- a/octodns/provider/digitalocean.py +++ b/octodns/provider/digitalocean.py @@ -262,7 +262,7 @@ class DigitalOceanProvider(BaseProvider): def _params_for_CAA(self, record): for value in record.values: yield { - 'data': '{}.'.format(value.value), + 'data': '{}'.format(value.value), 'flags': value.flags, 'name': record.name, 'tag': value.tag, From 2351c406f6d20f478c2e16b94829b286b6390aaa Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Wed, 28 Apr 2021 16:26:49 +0200 Subject: [PATCH 05/11] fixed DigitalOcean tests (no need for final dot on CAA records) --- tests/test_octodns_provider_digitalocean.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_digitalocean.py b/tests/test_octodns_provider_digitalocean.py index affd140..4b04d54 100644 --- a/tests/test_octodns_provider_digitalocean.py +++ b/tests/test_octodns_provider_digitalocean.py @@ -186,7 +186,7 @@ class TestDigitalOceanProvider(TestCase): 'name': '@', 'ttl': 300, 'type': 'A'}), call('POST', '/domains/unit.tests/records', data={ - 'data': 'ca.unit.tests.', + 'data': 'ca.unit.tests', 'flags': 0, 'name': '@', 'tag': 'issue', 'ttl': 3600, 'type': 'CAA'}), From f8aa4c8df5938cc80cc5a0944dfab5989c010077 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Wed, 28 Apr 2021 16:30:52 +0200 Subject: [PATCH 06/11] minor correctness change (unnecessary .format() call) --- octodns/provider/digitalocean.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/digitalocean.py b/octodns/provider/digitalocean.py index 41e6b6b..9499009 100644 --- a/octodns/provider/digitalocean.py +++ b/octodns/provider/digitalocean.py @@ -262,7 +262,7 @@ class DigitalOceanProvider(BaseProvider): def _params_for_CAA(self, record): for value in record.values: yield { - 'data': '{}'.format(value.value), + 'data': value.value, 'flags': value.flags, 'name': record.name, 'tag': value.tag, From 47de105a2924ac84507b8bfef26f7f5eb117d2cc Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 23 Aug 2021 12:26:09 -0700 Subject: [PATCH 07/11] POC supports & dynamic checking in _process_desired_zone --- octodns/provider/base.py | 20 ++++++++--- tests/helpers.py | 2 +- tests/test_octodns_provider_base.py | 53 +++++++++++++++++++++++++++-- 3 files changed, 67 insertions(+), 8 deletions(-) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index b636d65..da33aaa 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -50,12 +50,24 @@ class BaseProvider(BaseSource): that are made to have them logged or throw errors depending on the provider configuration. ''' - if self.SUPPORTS_MUTLIVALUE_PTR: - # nothing do here - return desired for record in desired.records: - if record._type == 'PTR' and len(record.values) > 1: + if record._type not in self.SUPPORTS: + msg = '{} records not supported for {}'.format(record._type, + record.fqdn) + fallback = 'omitting record' + self.supports_warn_or_except(msg, fallback) + desired.remove_record(record) + elif getattr(record, 'dynamic', False) and not self.SUPPORTS_DYNAMIC: + msg = 'dynamic records not supported for {}'\ + .format(record.fqdn) + fallback = 'falling back to simple record' + self.supports_warn_or_except(msg, fallback) + record = record.copy() + record.dynamic = None + desired.add_record(record, replace=True) + elif record._type == 'PTR' and len(record.values) > 1 and \ + not self.SUPPORTS_MUTLIVALUE_PTR: # replace with a single-value copy msg = 'multi-value PTR records not supported for {}' \ .format(record.fqdn) diff --git a/tests/helpers.py b/tests/helpers.py index eedfd8b..17b0115 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -107,7 +107,7 @@ class PlannableProvider(BaseProvider): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False - SUPPORTS = set(('A',)) + SUPPORTS = set(('A', 'AAAA', 'TXT')) def __init__(self, *args, **kwargs): super(PlannableProvider, self).__init__(*args, **kwargs) diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index cee7c2c..47cc3b9 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -21,7 +21,10 @@ from octodns.zone import Zone class HelperProvider(BaseProvider): log = getLogger('HelperProvider') - SUPPORTS = set(('A',)) + SUPPORTS = set(('A', 'PTR')) + SUPPORTS_MUTLIVALUE_PTR = False + SUPPORTS_DYNAMIC = False + id = 'test' strict_supports = False @@ -234,6 +237,10 @@ class TestBaseProvider(TestCase): self.assertFalse(plan) def test_process_desired_zone(self): + provider = HelperProvider('test') + + # SUPPORTS_MUTLIVALUE_PTR + provider.SUPPORTS_MUTLIVALUE_PTR = False zone1 = Zone('unit.tests.', []) record1 = Record.new(zone1, 'ptr', { 'type': 'PTR', @@ -242,11 +249,51 @@ class TestBaseProvider(TestCase): }) zone1.add_record(record1) - zone2 = HelperProvider('hasptr')._process_desired_zone(zone1) + zone2 = provider._process_desired_zone(zone1.copy()) record2 = list(zone2.records)[0] - self.assertEqual(len(record2.values), 1) + provider.SUPPORTS_MUTLIVALUE_PTR = True + zone2 = provider._process_desired_zone(zone1.copy()) + record2 = list(zone2.records)[0] + from pprint import pprint + pprint([ + record1, record2 + ]) + self.assertEqual(len(record2.values), 2) + + # SUPPORTS_DYNAMIC + provider.SUPPORTS_DYNAMIC = False + zone1 = Zone('unit.tests.', []) + record1 = Record.new(zone1, 'a', { + 'dynamic': { + 'pools': { + 'one': { + 'values': [{ + 'value': '1.1.1.1', + }], + }, + }, + 'rules': [{ + 'pool': 'one', + }], + }, + 'type': 'A', + 'ttl': 3600, + 'values': ['2.2.2.2'], + }) + self.assertTrue(record1.dynamic) + zone1.add_record(record1) + + zone2 = provider._process_desired_zone(zone1.copy()) + record2 = list(zone2.records)[0] + self.assertFalse(record2.dynamic) + + provider.SUPPORTS_DYNAMIC = True + zone2 = provider._process_desired_zone(zone1.copy()) + record2 = list(zone2.records)[0] + self.assertTrue(record2.dynamic) + def test_safe_none(self): # No changes is safe Plan(None, None, [], True).raise_if_unsafe() From c6e8ee48f53b2accd0688209475450d818e893f4 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 7 Sep 2021 13:24:29 -0700 Subject: [PATCH 08/11] quell lint indention alignment warnings --- octodns/provider/base.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index da33aaa..e0c0db2 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -58,7 +58,8 @@ class BaseProvider(BaseSource): fallback = 'omitting record' self.supports_warn_or_except(msg, fallback) desired.remove_record(record) - elif getattr(record, 'dynamic', False) and not self.SUPPORTS_DYNAMIC: + elif getattr(record, 'dynamic', False) and \ + not self.SUPPORTS_DYNAMIC: msg = 'dynamic records not supported for {}'\ .format(record.fqdn) fallback = 'falling back to simple record' @@ -67,7 +68,7 @@ class BaseProvider(BaseSource): record.dynamic = None desired.add_record(record, replace=True) elif record._type == 'PTR' and len(record.values) > 1 and \ - not self.SUPPORTS_MUTLIVALUE_PTR: + not self.SUPPORTS_MUTLIVALUE_PTR: # replace with a single-value copy msg = 'multi-value PTR records not supported for {}' \ .format(record.fqdn) From bc6c874cc7322fd2c97d765dd75c00443afdae0c Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 7 Sep 2021 13:24:41 -0700 Subject: [PATCH 09/11] transip no longer needs to check supports --- octodns/provider/transip.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index d3e2018..176da88 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -145,16 +145,14 @@ class TransipProvider(BaseProvider): _dns_entries = [] for record in plan.desired.records: - if record._type in self.SUPPORTS: - entries_for = getattr(self, - '_entries_for_{}'.format(record._type)) + entries_for = getattr(self, '_entries_for_{}'.format(record._type)) - # Root records have '@' as name - name = record.name - if name == '': - name = self.ROOT_RECORD + # Root records have '@' as name + name = record.name + if name == '': + name = self.ROOT_RECORD - _dns_entries.extend(entries_for(name, record)) + _dns_entries.extend(entries_for(name, record)) try: self._client.set_dns_entries(plan.desired.name[:-1], _dns_entries) From 52da08dfec3ad6f62c2231844d354ae6c74eccd2 Mon Sep 17 00:00:00 2001 From: apatserkovskyi Date: Thu, 2 Sep 2021 00:01:44 +0300 Subject: [PATCH 10/11] Constellix provider full Dynamic support added --- README.md | 2 +- octodns/provider/constellix.py | 456 +++++++++++--- tests/fixtures/constellix-geofilters.json | 34 ++ tests/fixtures/constellix-pools.json | 30 + tests/fixtures/constellix-records.json | 156 +++-- tests/test_octodns_provider_constellix.py | 703 ++++++++++++++++++++-- 6 files changed, 1213 insertions(+), 168 deletions(-) create mode 100644 tests/fixtures/constellix-geofilters.json diff --git a/README.md b/README.md index 28d9e7f..ebd7111 100644 --- a/README.md +++ b/README.md @@ -195,7 +195,7 @@ The above command pulled the existing data out of Route53 and placed the results | [AzureProvider](/octodns/provider/azuredns.py) | azure-identity, azure-mgmt-dns, azure-mgmt-trafficmanager | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | Alpha (A, AAAA, CNAME) | | | [Akamai](/octodns/provider/edgedns.py) | edgegrid-python | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | | | [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, LOC, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | -| [ConstellixProvider](/octodns/provider/constellix.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | +| [ConstellixProvider](/octodns/provider/constellix.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | Yes | CAA tags restricted | | [DigitalOceanProvider](/octodns/provider/digitalocean.py) | | A, AAAA, CAA, CNAME, MX, NS, TXT, SRV | No | CAA tags restricted | | [DnsMadeEasyProvider](/octodns/provider/dnsmadeeasy.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [DnsimpleProvider](/octodns/provider/dnsimple.py) | | All | No | CAA tags restricted | diff --git a/octodns/provider/constellix.py b/octodns/provider/constellix.py index cc65ae1..3f75650 100644 --- a/octodns/provider/constellix.py +++ b/octodns/provider/constellix.py @@ -9,6 +9,7 @@ from collections import defaultdict from requests import Session from base64 import b64encode from six import string_types +from pycountry_convert import country_alpha2_to_continent_code import hashlib import hmac import logging @@ -53,7 +54,8 @@ class ConstellixClient(object): self._sess = Session() self._sess.headers.update({'x-cnsdns-apiKey': self.api_key}) self._domains = None - self._pools = None + self._pools = {'A': None, 'AAAA': None, 'CNAME': None} + self._geofilters = None def _current_time(self): return str(int(time.time() * 1000)) @@ -100,19 +102,29 @@ class ConstellixClient(object): zone_id = self.domains.get(name, False) if not zone_id: raise ConstellixClientNotFound() - path = '/domains/{}'.format(zone_id) + path = f'/domains/{zone_id}' return self._request('GET', path).json() def domain_create(self, name): resp = self._request('POST', '/domains', data={'names': [name]}) # Add newly created zone to domain cache - self._domains['{}.'.format(name)] = resp.json()[0]['id'] + self._domains[f'{name}.'] = resp.json()[0]['id'] + + def domain_enable_geoip(self, domain_name): + domain = self.domain(domain_name) + if domain['hasGeoIP'] is False: + domain_id = self.domains[domain_name] + self._request( + 'PUT', + f'/domains/{domain_id}', + data={'hasGeoIP': True} + ) def _absolutize_value(self, value, zone_name): if value == '': value = zone_name elif not value.endswith('.'): - value = '{}.{}'.format(value, zone_name) + value = f'{value}.{zone_name}' return value @@ -120,7 +132,7 @@ class ConstellixClient(object): zone_id = self.domains.get(zone_name, False) if not zone_id: raise ConstellixClientNotFound() - path = '/domains/{}/records'.format(zone_id) + path = f'/domains/{zone_id}/records' resp = self._request('GET', path).json() for record in resp: @@ -147,7 +159,7 @@ class ConstellixClient(object): record_type = 'ANAME' zone_id = self.domains.get(zone_name, False) - path = '/domains/{}/records/{}'.format(zone_id, record_type) + path = f'/domains/{zone_id}/records/{record_type}' self._request('POST', path, data=params) @@ -157,18 +169,17 @@ class ConstellixClient(object): record_type = 'ANAME' zone_id = self.domains.get(zone_name, False) - path = '/domains/{}/records/{}/{}'.format(zone_id, record_type, - record_id) + path = f'/domains/{zone_id}/records/{record_type}/{record_id}' self._request('DELETE', path) def pools(self, pool_type): - if self._pools is None: - self._pools = {} - path = '/pools/{}'.format(pool_type) + if self._pools[pool_type] is None: + self._pools[pool_type] = {} + path = f'/pools/{pool_type}' response = self._request('GET', path).json() for pool in response: - self._pools[pool['id']] = pool - return self._pools.values() + self._pools[pool_type][pool['id']] = pool + return self._pools[pool_type].values() def pool(self, pool_type, pool_name): pools = self.pools(pool_type) @@ -186,11 +197,11 @@ class ConstellixClient(object): def pool_create(self, data): path = '/pools/{}'.format(data.get('type')) # This returns a list of items, we want the first one - response = self._request('POST', path, data=data).json()[0] + response = self._request('POST', path, data=data).json() - # Invalidate our cache - self._pools = None - return response + # Update our cache + self._pools[data.get('type')][response[0]['id']] = response[0] + return response[0] def pool_update(self, pool_id, data): path = '/pools/{}/{}'.format(data.get('type'), pool_id) @@ -203,6 +214,63 @@ class ConstellixClient(object): raise e return data + def pool_delete(self, pool_type, pool_id): + path = f'/pools/{pool_type}/{pool_id}' + self._request('DELETE', path) + + # Update our cache + if self._pools[pool_type] is not None: + self._pools[pool_type].pop(pool_id, None) + + def geofilters(self): + if self._geofilters is None: + self._geofilters = {} + path = '/geoFilters' + response = self._request('GET', path).json() + for geofilter in response: + self._geofilters[geofilter['id']] = geofilter + return self._geofilters.values() + + def geofilter(self, geofilter_name): + geofilters = self.geofilters() + for geofilter in geofilters: + if geofilter['name'] == geofilter_name: + return geofilter + return None + + def geofilter_by_id(self, geofilter_id): + geofilters = self.geofilters() + for geofilter in geofilters: + if geofilter['id'] == geofilter_id: + return geofilter + + def geofilter_create(self, data): + path = '/geoFilters' + response = self._request('POST', path, data=data).json() + + # Update our cache + self._geofilters[response[0]['id']] = response[0] + return response[0] + + def geofilter_update(self, geofilter_id, data): + path = f'/geoFilters/{geofilter_id}' + try: + self._request('PUT', path, data=data).json() + + except ConstellixClientBadRequest as e: + message = str(e) + if not message or "no changes to save" not in message: + raise e + return data + + def geofilter_delete(self, geofilter_id): + path = f'/geoFilters/{geofilter_id}' + self._request('DELETE', path) + + # Update our cache + if self._geofilters is not None: + self._geofilters.pop(geofilter_id, None) + class ConstellixProvider(BaseProvider): ''' @@ -225,7 +293,7 @@ class ConstellixProvider(BaseProvider): def __init__(self, id, api_key, secret_key, ratelimit_delay=0.0, *args, **kwargs): - self.log = logging.getLogger('ConstellixProvider[{}]'.format(id)) + self.log = logging.getLogger(f'ConstellixProvider[{id}]') self.log.debug('__init__: id=%s, api_key=***, secret_key=***', id) super(ConstellixProvider, self).__init__(id, *args, **kwargs) self._client = ConstellixClient(api_key, secret_key, ratelimit_delay) @@ -234,39 +302,95 @@ class ConstellixProvider(BaseProvider): def _data_for_multiple(self, _type, records): record = records[0] if record['recordOption'] == 'pools': - return self._data_for_pool(_type, record) + return self._data_for_pool(_type, records) return { 'ttl': record['ttl'], 'type': _type, 'values': record['value'] } - def _data_for_pool(self, _type, record): - pool_id = record['pools'][0] - pool = self._client.pool_by_id(_type, pool_id) - pool_name = pool['name'].split(':')[-1] + def _data_for_pool(self, _type, records): + default_values = [] + fallback_pool_name = None pools = {} - values = [] - pools[pool_name] = { - 'values': [] - } - for value in pool['values']: - pools[pool_name]['values'].append({ - 'value': value['value'], - 'weight': value['weight'] - }) - values.append(value['value']) - return { + rules = [] + + for record in records: + # fetch record pool data + pool_id = record['pools'][0] + pool = self._client.pool_by_id(_type, pool_id) + + geofilter_id = 1 + if 'geolocation' in record.keys() \ + and record['geolocation'] is not None: + # fetch record geofilter data + geofilter_id = record['geolocation']['geoipFilter'] + geofilter = self._client.geofilter_by_id(geofilter_id) + + pool_name = pool['name'].split(':')[-1] + + # fetch default values from the World Default pool + if geofilter_id == 1: + fallback_pool_name = pool_name + for value in pool['values']: + default_values.append(value['value']) + + # populate pools + pools[pool_name] = { + 'fallback': None, + 'values': [] + } + for value in pool['values']: + pools[pool_name]['values'].append({ + 'value': value['value'], + 'weight': value['weight'] + }) + + # populate rules + if geofilter_id == 1: + rules.append({'pool': pool_name}) + else: + geos = [] + + if 'geoipContinents' in geofilter.keys(): + for continent_code in geofilter['geoipContinents']: + geos.append(continent_code) + + if 'geoipCountries' in geofilter.keys(): + for country_code in geofilter['geoipCountries']: + geos.append('{}-{}'.format( + country_alpha2_to_continent_code(country_code), + country_code + )) + + if 'regions' in geofilter.keys(): + for region in geofilter['regions']: + geos.append('{}-{}-{}'.format( + region['continentCode'], + region['countryCode'], + region['regionCode'])) + + rules.append({ + 'pool': pool_name, + 'geos': sorted(geos) + }) + + # set fallback pool + for pool_name in pools: + if pool_name != fallback_pool_name: + pools[pool_name]['fallback'] = fallback_pool_name + + res = { 'ttl': record['ttl'], 'type': _type, 'dynamic': { - 'pools': pools, - 'rules': [{ - 'pool': pool_name - }] + 'pools': dict( + sorted(pools.items(), key=lambda t: t[0])), + 'rules': sorted(rules, key=lambda t: t['pool']) }, - 'value': values + 'values': default_values } + return res _data_for_A = _data_for_multiple _data_for_AAAA = _data_for_multiple @@ -381,7 +505,7 @@ class ConstellixProvider(BaseProvider): before = len(zone.records) for name, types in values.items(): for _type, records in types.items(): - data_for = getattr(self, '_data_for_{}'.format(_type)) + data_for = getattr(self, f'_data_for_{_type}') record = Record.new(zone, name, data_for(_type, records), source=self, lenient=lenient) zone.add_record(record, lenient=lenient) @@ -491,34 +615,70 @@ class ConstellixProvider(BaseProvider): def _handle_pools(self, record): # If we don't have dynamic, then there's no pools if not getattr(record, 'dynamic', False): - return None + return [] - # Get our first entry in the rules that references a pool - rules = list(filter( - lambda rule: 'pool' in rule.data, - record.dynamic.rules - )) + res_pools = [] - pool_name = rules[0].data.get('pool') + for i, rule in enumerate(record.dynamic.rules): + pool_name = rule.data.get('pool') + pool = record.dynamic.pools.get(pool_name) + values = pool.data.get('values') - pool = record.dynamic.pools.get(pool_name) - values = pool.data.get('values') + # Make a pool name based on zone, record, type and name + generated_pool_name = '{}:{}:{}:{}'.format( + record.zone.name, + record.name, + record._type, + pool_name + ) - # Make a pool name based on zone, record, type and name - pool_name = '{}:{}:{}:{}'.format( - record.zone.name, - record.name, - record._type, - pool_name - ) + # OK, pool is valid, let's create it or update it + self.log.debug("Creating pool %s", generated_pool_name) + pool_obj = self._create_update_pool( + pool_name = generated_pool_name, + pool_type = record._type, + ttl = record.ttl, + values = values + ) - # OK, pool is valid, let's create it or update it - return self._create_update_pool( - pool_name = pool_name, - pool_type = record._type, - ttl = record.ttl, - values = values - ) + # Now will crate GeoFilter for the pool + continents = [] + countries = [] + regions = [] + + for geo in rule.data.get('geos', []): + codes = geo.split('-') + n = len(geo) + if n == 2: + continents.append(geo) + elif n == 5: + countries.append(codes[1]) + else: + regions.append({ + 'continentCode': codes[0], + 'countryCode': codes[1], + 'regionCode': codes[2] + }) + + if len(continents) == 0 and \ + len(countries) == 0 and \ + len(regions) == 0: + pool_obj['geofilter'] = 1 + else: + self.log.debug( + "Creating geofilter %s", + generated_pool_name + ) + geofilter_obj = self._create_update_geofilter( + generated_pool_name, + continents, + countries, + regions + ) + pool_obj['geofilter'] = geofilter_obj['id'] + + res_pools.append(pool_obj) + return res_pools def _create_update_pool(self, pool_name, pool_type, ttl, values): pool = { @@ -538,29 +698,173 @@ class ConstellixProvider(BaseProvider): updated_pool['id'] = pool_id return updated_pool - def _apply_Create(self, change): + def _create_update_geofilter( + self, + geofilter_name, + continents, + countries, + regions): + geofilter = { + 'filterRulesLimit': 100, + 'name': geofilter_name, + 'geoipContinents': continents, + 'geoipCountries': countries, + 'regions': regions + } + if len(regions) == 0: + geofilter.pop('regions', None) + + existing_geofilter = self._client.geofilter(geofilter_name) + if not existing_geofilter: + return self._client.geofilter_create(geofilter) + + geofilter_id = existing_geofilter['id'] + updated_geofilter = self._client.geofilter_update( + geofilter_id, geofilter) + updated_geofilter['id'] = geofilter_id + return updated_geofilter + + def _apply_Create(self, change, domain_name): new = change.new params_for = getattr(self, '_params_for_{}'.format(new._type)) - pool = self._handle_pools(new) + pools = self._handle_pools(new) for params in params_for(new): - if pool: - params['pools'] = [pool['id']] + if len(pools) == 0: + self._client.record_create(new.zone.name, new._type, params) + elif len(pools) == 1: + params['pools'] = [pools[0]['id']] params['recordOption'] = 'pools' - self._client.record_create(new.zone.name, new._type, params) + params.pop('roundRobin', None) + self.log.debug( + "Creating record %s %s", + new.zone.name, + new._type + ) + self._client.record_create( + new.zone.name, + new._type, + params + ) + else: + # To use GeoIPFilter feature we need to enable it for domain + self.log.debug("Enabling domain %s geo support", domain_name) + self._client.domain_enable_geoip(domain_name) - def _apply_Update(self, change): - self._apply_Delete(change) - self._apply_Create(change) + # First we need to create World Default (1) Record + for pool in pools: + if pool['geofilter'] != 1: + continue + params['pools'] = [pool['id']] + params['recordOption'] = 'pools' + params['geolocation'] = { + 'geoipUserRegion': [pool['geofilter']] + } + params.pop('roundRobin', None) + self.log.debug( + "Creating record %s %s", + new.zone.name, + new._type) + self._client.record_create( + new.zone.name, + new._type, + params + ) - def _apply_Delete(self, change): + # Now we can create the rest of records + for pool in pools: + if pool['geofilter'] == 1: + continue + params['pools'] = [pool['id']] + params['recordOption'] = 'pools' + params['geolocation'] = { + 'geoipUserRegion': [pool['geofilter']] + } + params.pop('roundRobin', None) + self.log.debug( + "Creating record %s %s", + new.zone.name, + new._type) + self._client.record_create( + new.zone.name, + new._type, + params) + + def _apply_Update(self, change, domain_name): + self._apply_Delete(change, domain_name) + self._apply_Create(change, domain_name) + + def _apply_Delete(self, change, domain_name): existing = change.existing zone = existing.zone + + # if it is dynamic pools record, we need to delete World Default last + world_default_record = None + for record in self.zone_records(zone): if existing.name == record['name'] and \ existing._type == record['type']: - self._client.record_delete(zone.name, record['type'], - record['id']) + + # handle dynamic record + if record['recordOption'] == 'pools': + if record['geolocation'] is None: + world_default_record = record + else: + if record['geolocation']['geoipFilter'] == 1: + world_default_record = record + else: + # delete record + self.log.debug( + "Deleting record %s %s", + zone.name, + record['type']) + self._client.record_delete( + zone.name, + record['type'], + record['id']) + # delete geofilter + self.log.debug( + "Deleting geofilter %s", + zone.name) + self._client.geofilter_delete( + record['geolocation']['geoipFilter']) + + # delete pool + self.log.debug( + "Deleting pool %s %s", + zone.name, + record['type']) + self._client.pool_delete( + record['type'], + record['pools'][0]) + + # for all the rest records + else: + self._client.record_delete( + zone.name, record['type'], record['id']) + # delete World Default + if world_default_record: + # delete record + self.log.debug( + "Deleting record %s %s", + zone.name, + world_default_record['type'] + ) + self._client.record_delete( + zone.name, + world_default_record['type'], + world_default_record['id'] + ) + # delete pool + self.log.debug( + "Deleting pool %s %s", + zone.name, + world_default_record['type'] + ) + self._client.pool_delete( + world_default_record['type'], + world_default_record['pools'][0] + ) def _apply(self, plan): desired = plan.desired @@ -576,7 +880,9 @@ class ConstellixProvider(BaseProvider): for change in changes: class_name = change.__class__.__name__ - getattr(self, '_apply_{}'.format(class_name))(change) + getattr(self, f'_apply_{class_name}')( + change, + desired.name) # Clear out the cache if any self._zone_records.pop(desired.name, None) diff --git a/tests/fixtures/constellix-geofilters.json b/tests/fixtures/constellix-geofilters.json new file mode 100644 index 0000000..eef17a3 --- /dev/null +++ b/tests/fixtures/constellix-geofilters.json @@ -0,0 +1,34 @@ +[ + { + "id": 6303, + "name": "some.other", + "filterRulesLimit": 100, + "createdTs": "2021-08-19T14:47:47Z", + "modifiedTs": "2021-08-19T14:47:47Z", + "geoipContinents": ["AS", "OC"], + "geoipCountries": ["ES", "SE", "UA"], + "regions": [ + { + "continentCode": "NA", + "countryCode": "CA", + "regionCode": "NL" + } + ] + }, + { + "id": 5303, + "name": "unit.tests.:www.dynamic:A:one", + "filterRulesLimit": 100, + "createdTs": "2021-08-19T14:47:47Z", + "modifiedTs": "2021-08-19T14:47:47Z", + "geoipContinents": ["AS", "OC"], + "geoipCountries": ["ES", "SE", "UA"], + "regions": [ + { + "continentCode": "NA", + "countryCode": "CA", + "regionCode": "NL" + } + ] + } +] diff --git a/tests/fixtures/constellix-pools.json b/tests/fixtures/constellix-pools.json index 8fbd491..8d90bd4 100644 --- a/tests/fixtures/constellix-pools.json +++ b/tests/fixtures/constellix-pools.json @@ -28,5 +28,35 @@ "weight": 1 } ] + }, + { + "id": 1808522, + "name": "unit.tests.:www.dynamic:A:one", + "type": "A", + "numReturn": 1, + "minAvailableFailover": 1, + "createdTs": "2020-09-12T00:44:35Z", + "modifiedTs": "2020-09-12T00:44:35Z", + "appliedDomains": [ + { + "id": 123123, + "name": "unit.tests", + "recordOption": "pools" + } + ], + "appliedTemplates": null, + "unlinkedDomains": [], + "unlinkedTemplates": null, + "itoEnabled": false, + "values": [ + { + "value": "1.2.3.6", + "weight": 1 + }, + { + "value": "1.2.3.7", + "weight": 1 + } + ] } ] \ No newline at end of file diff --git a/tests/fixtures/constellix-records.json b/tests/fixtures/constellix-records.json index f509fe7..c5cdf8e 100644 --- a/tests/fixtures/constellix-records.json +++ b/tests/fixtures/constellix-records.json @@ -64,62 +64,6 @@ "roundRobinFailover": [], "pools": [], "poolsDetail": [] -}, { - "id": 1898527, - "type": "SRV", - "recordType": "srv", - "name": "_imap._tcp", - "recordOption": "roundRobin", - "noAnswer": false, - "note": "", - "ttl": 600, - "gtdRegion": 1, - "parentId": 123123, - "parent": "domain", - "source": "Domain", - "modifiedTs": 1565149714387, - "value": [{ - "value": ".", - "priority": 0, - "weight": 0, - "port": 0, - "disableFlag": false - }], - "roundRobin": [{ - "value": ".", - "priority": 0, - "weight": 0, - "port": 0, - "disableFlag": false - }] -}, { - "id": 1898528, - "type": "SRV", - "recordType": "srv", - "name": "_pop3._tcp", - "recordOption": "roundRobin", - "noAnswer": false, - "note": "", - "ttl": 600, - "gtdRegion": 1, - "parentId": 123123, - "parent": "domain", - "source": "Domain", - "modifiedTs": 1565149714387, - "value": [{ - "value": ".", - "priority": 0, - "weight": 0, - "port": 0, - "disableFlag": false - }], - "roundRobin": [{ - "value": ".", - "priority": 0, - "weight": 0, - "port": 0, - "disableFlag": false - }] }, { "id": 1808527, "type": "SRV", @@ -160,6 +104,62 @@ "port": 30, "disableFlag": false }] +}, { + "id": 1808527, + "type": "SRV", + "recordType": "srv", + "name": "_imap._tcp", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 600, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565149714387, + "value": [{ + "value": ".", + "priority": 0, + "weight": 0, + "port": 0, + "disableFlag": false + }], + "roundRobin": [{ + "value": ".", + "priority": 0, + "weight": 0, + "port": 0, + "disableFlag": false + }] +}, { + "id": 1808527, + "type": "SRV", + "recordType": "srv", + "name": "_pop3._tcp", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 600, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565149714387, + "value": [{ + "value": ".", + "priority": 0, + "weight": 0, + "port": 0, + "disableFlag": false + }], + "roundRobin": [{ + "value": ".", + "priority": 0, + "weight": 0, + "port": 0, + "disableFlag": false + }] }, { "id": 1808515, "type": "AAAA", @@ -630,7 +630,9 @@ "modifiedTs": 1565150090588, "value": [], "roundRobin": [], - "geolocation": null, + "geolocation": { + "geoipFilter": 1 + }, "recordFailover": { "disabled": false, "failoverType": 1, @@ -651,4 +653,44 @@ "id": 1808521, "name": "unit.tests.:www.dynamic:A:two" }] +}, +{ + "id": 1808521, + "type": "A", + "recordType": "a", + "name": "www.dynamic", + "recordOption": "pools", + "noAnswer": false, + "note": "", + "ttl": 300, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565150090588, + "value": [], + "roundRobin": [], + "geolocation": { + "geoipFilter": 5303 + }, + "recordFailover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "failover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "roundRobinFailover": [], + "pools": [ + 1808522 + ], + "poolsDetail": [{ + "id": 1808522, + "name": "unit.tests.:www.dynamic:A:one" + }] }] diff --git a/tests/test_octodns_provider_constellix.py b/tests/test_octodns_provider_constellix.py index d15f611..75aa07f 100644 --- a/tests/test_octodns_provider_constellix.py +++ b/tests/test_octodns_provider_constellix.py @@ -42,17 +42,11 @@ class TestConstellixProvider(TestCase): 'value': 'aname.unit.tests.' })) - expected.add_record(Record.new(expected, 'sub', { - 'ttl': 1800, - 'type': 'ALIAS', - 'value': 'aname.unit.tests.' - })) - # Add a dynamic record expected.add_record(Record.new(expected, 'www.dynamic', { 'ttl': 300, 'type': 'A', - 'value': [ + 'values': [ '1.2.3.4', '1.2.3.5' ], @@ -79,6 +73,78 @@ class TestConstellixProvider(TestCase): expected._remove_record(record) break + expected_dynamic = Zone('unit.tests.', []) + source = YamlProvider('test', join(dirname(__file__), 'config')) + source.populate(expected_dynamic) + + # Our test suite differs a bit, add our NS and remove the simple one + expected_dynamic.add_record(Record.new(expected_dynamic, 'under', { + 'ttl': 3600, + 'type': 'NS', + 'values': [ + 'ns1.unit.tests.', + 'ns2.unit.tests.', + ] + })) + + # Add some ALIAS records + expected_dynamic.add_record(Record.new(expected_dynamic, '', { + 'ttl': 1800, + 'type': 'ALIAS', + 'value': 'aname.unit.tests.' + })) + + # Add a dynamic record + expected_dynamic.add_record(Record.new(expected_dynamic, 'www.dynamic', { + 'ttl': 300, + 'type': 'A', + 'values': [ + '1.2.3.4', + '1.2.3.5' + ], + 'dynamic': { + 'pools': { + 'one': { + 'fallback': 'two', + 'values': [{ + 'value': '1.2.3.6', + 'weight': 1 + }, { + 'value': '1.2.3.7', + 'weight': 1 + }], + }, + 'two': { + 'values': [{ + 'value': '1.2.3.4', + 'weight': 1 + }, { + 'value': '1.2.3.5', + 'weight': 1 + }], + }, + }, + 'rules': [{ + 'geos': [ + 'AS', + 'EU-ES', + 'EU-UA', + 'EU-SE', + 'NA-CA-NL', + 'OC' + ], + 'pool': 'one' + }, { + 'pool': 'two', + }], + } + })) + + for record in list(expected_dynamic.records): + if record.name == 'sub' and record._type == 'NS': + expected_dynamic._remove_record(record) + break + def test_populate(self): provider = ConstellixProvider('test', 'api', 'secret') @@ -126,24 +192,28 @@ class TestConstellixProvider(TestCase): with requests_mock() as mock: base = 'https://api.dns.constellix.com/v1' with open('tests/fixtures/constellix-domains.json') as fh: - mock.get('{}{}'.format(base, '/domains'), text=fh.read()) + mock.get('{}{}'.format(base, '/domains'), + text=fh.read()) with open('tests/fixtures/constellix-records.json') as fh: mock.get('{}{}'.format(base, '/domains/123123/records'), text=fh.read()) with open('tests/fixtures/constellix-pools.json') as fh: mock.get('{}{}'.format(base, '/pools/A'), text=fh.read()) + with open('tests/fixtures/constellix-geofilters.json') as fh: + mock.get('{}{}'.format(base, '/geoFilters'), + text=fh.read()) zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(16, len(zone.records)) - changes = self.expected.changes(zone, provider) + self.assertEquals(17, len(zone.records)) + changes = self.expected_dynamic.changes(zone, provider) self.assertEquals(0, len(changes)) # 2nd populate makes no network calls/all from cache again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(16, len(again.records)) + self.assertEquals(17, len(again.records)) # bust the cache del provider._zone_records[zone.name] @@ -225,7 +295,7 @@ class TestConstellixProvider(TestCase): }), ]) - self.assertEquals(21, provider._client._request.call_count) + self.assertEquals(22, provider._client._request.call_count) provider._client._request.reset_mock() @@ -262,6 +332,7 @@ class TestConstellixProvider(TestCase): "id": 1808520, "type": "A", "name": "www.dynamic", + "geolocation": None, "recordOption": "pools", "ttl": 300, "value": [], @@ -289,8 +360,16 @@ class TestConstellixProvider(TestCase): # Domain exists, we don't care about return resp.json.side_effect = [ - ['{}'], - ['{}'], + [], # no domains returned during populate + [{ + 'id': 123123, + 'name': 'unit.tests' + }], # domain created in apply + [], # No pools returned during populate + [{ + "id": 1808521, + "name": "unit.tests.:www.dynamic:A:one" + }] # pool created in apply ] wanted = Zone('unit.tests.', []) @@ -303,7 +382,7 @@ class TestConstellixProvider(TestCase): wanted.add_record(Record.new(wanted, 'www.dynamic', { 'ttl': 300, 'type': 'A', - 'value': [ + 'values': [ '1.2.3.4' ], 'dynamic': { @@ -340,17 +419,393 @@ class TestConstellixProvider(TestCase): 'numReturn': 1, 'minAvailableFailover': 1, 'ttl': 300, - 'id': 1808521, 'values': [{ "value": "1.2.3.4", "weight": 1 - }] + }], + 'id': 1808521, + 'geofilter': 1 }), call('DELETE', '/domains/123123/records/A/11189897'), call('DELETE', '/domains/123123/records/A/11189898'), call('DELETE', '/domains/123123/records/ANAME/11189899'), ], any_order=True) + def test_apply_dunamic(self): + provider = ConstellixProvider('test', 'api', 'secret') + + resp = Mock() + resp.json = Mock() + provider._client._request = Mock(return_value=resp) + + # non-existent domain, create everything + resp.json.side_effect = [ + [], # no domains returned during populate + [{ + 'id': 123123, + 'name': 'unit.tests' + }], # domain created in apply + [], # No pools returned during populate + [{ + "id": 1808521, + "name": "unit.tests.:www.dynamic:A:one" + }], # pool created in apply + [], # no geofilters returned during populate + [{ + "id": 5303, + "name": "unit.tests.:www.dynamic:A:one", + "filterRulesLimit": 100, + "geoipContinents": ["AS", "OC"], + "geoipCountries": ["ES", "SE", "UA"], + "regions": [ + { + "continentCode": "NA", + "countryCode": "CA", + "regionCode": "NL" + } + ] + }], # geofilters created in applly + [{ + "id": 1808520, + "name": "unit.tests.:www.dynamic:A:two", + }], # pool created in apply + { + 'id': 123123, + 'name': 'unit.tests', + 'hasGeoIP': False + }, # domain listed for enabling geo + [] # enabling geo + ] + + plan = provider.plan(self.expected_dynamic) + + # No root NS, no ignored, no excluded, no unsupported + n = len(self.expected_dynamic.records) - 8 + self.assertEquals(n, len(plan.changes)) + self.assertEquals(n, provider.apply(plan)) + + provider._client._request.assert_has_calls([ + # get all domains to build the cache + call('GET', '/domains'), + # created the domain + call('POST', '/domains', data={'names': ['unit.tests']}) + ]) +# + # Check we tried to get our pool + provider._client._request.assert_has_calls([ + call('GET', '/pools/A'), + call('POST', '/pools/A', data={ + 'name': 'unit.tests.:www.dynamic:A:one', + 'type': 'A', + 'numReturn': 1, + 'minAvailableFailover': 1, + 'ttl': 300, + 'values': [{ + 'value': '1.2.3.6', + 'weight': 1 + }, { + 'value': '1.2.3.7', + 'weight': 1}] + }), + call('GET', '/geoFilters'), + call('POST', '/geoFilters', data={ + 'filterRulesLimit': 100, + 'name': 'unit.tests.:www.dynamic:A:one', + 'geoipContinents': ['AS', 'OC'], + 'geoipCountries': ['ES', 'SE', 'UA'], + 'regions': [{ + 'continentCode': 'NA', + 'countryCode': 'CA', + 'regionCode': 'NL'}] + }), + call('POST', '/pools/A', data={ + 'name': 'unit.tests.:www.dynamic:A:two', + 'type': 'A', + 'numReturn': 1, + 'minAvailableFailover': 1, + 'ttl': 300, + 'values': [{ + 'value': '1.2.3.4', + 'weight': 1 + }, { + 'value': '1.2.3.5', + 'weight': 1}] + }) + ]) + + # These two checks are broken up so that ordering doesn't break things. + # Python3 doesn't make the calls in a consistent order so different + # things follow the GET / on different runs + provider._client._request.assert_has_calls([ + call('POST', '/domains/123123/records/SRV', data={ + 'roundRobin': [{ + 'priority': 10, + 'weight': 20, + 'value': 'foo-1.unit.tests.', + 'port': 30 + }, { + 'priority': 12, + 'weight': 20, + 'value': 'foo-2.unit.tests.', + 'port': 30 + }], + 'name': '_srv._tcp', + 'ttl': 600, + }), + ]) + + self.assertEquals(28, provider._client._request.call_count) + + provider._client._request.reset_mock() + + provider._client.records = Mock(return_value=[ + { + 'id': 11189897, + 'type': 'A', + 'name': 'www', + 'ttl': 300, + 'recordOption': 'roundRobin', + 'value': [ + '1.2.3.4', + '2.2.3.4', + ] + }, { + 'id': 11189898, + 'type': 'A', + 'name': 'ttl', + 'ttl': 600, + 'recordOption': 'roundRobin', + 'value': [ + '3.2.3.4' + ] + }, { + 'id': 11189899, + 'type': 'ALIAS', + 'name': 'alias', + 'ttl': 600, + 'recordOption': 'roundRobin', + 'value': [{ + 'value': 'aname.unit.tests.' + }] + }, { + "id": 1808520, + "type": "A", + "name": "www.dynamic", + "geolocation": { + "geoipFilter": 1 + }, + "recordOption": "pools", + "ttl": 300, + "value": [], + "pools": [ + 1808521 + ] + }, { + "id": 1808521, + "type": "A", + "name": "www.dynamic", + "geolocation": { + "geoipFilter": 5303 + }, + "recordOption": "pools", + "ttl": 300, + "value": [], + "pools": [ + 1808522 + ] + } + ]) + + provider._client.pools = Mock(return_value=[ + { + "id": 1808521, + "name": "unit.tests.:www.dynamic:A:two", + "type": "A", + "values": [ + { + "value": "1.2.3.4", + "weight": 1 + }, + { + "value": "1.2.3.5", + "weight": 1 + } + ] + }, + { + "id": 1808522, + "name": "unit.tests.:www.dynamic:A:one", + "type": "A", + "values": [ + { + "value": "1.2.3.6", + "weight": 1 + }, + { + "value": "1.2.3.7", + "weight": 1 + } + ] + } + ]) + + provider._client.geofilters = Mock(return_value=[ + { + "id": 5303, + "name": "unit.tests.:www.dynamic:A:one", + "filterRulesLimit": 100, + "geoipContinents": ["AS", "OC"], + "geoipCountries": ["ES", "SE", "UA"], + "regions": [ + { + "continentCode": "NA", + "countryCode": "CA", + "regionCode": "NL" + } + ] + } + ]) + + # Domain exists, we don't care about return + resp.json.side_effect = [ + [], + [], + [], + [], + { + 'id': 123123, + 'name': 'unit.tests', + 'hasGeoIP': True + } # domain listed for enabling geo + ] + + wanted = Zone('unit.tests.', []) + wanted.add_record(Record.new(wanted, 'ttl', { + 'ttl': 300, + 'type': 'A', + 'value': '3.2.3.4' + })) + + wanted.add_record(Record.new(wanted, 'www.dynamic', { + 'ttl': 300, + 'type': 'A', + 'values': [ + '1.2.3.4' + ], + 'dynamic': { + 'pools': { + 'one': { + 'fallback': 'two', + 'values': [{ + 'value': '1.2.3.6', + 'weight': 1 + }, { + 'value': '1.2.3.7', + 'weight': 1 + }], + }, + 'two': { + 'values': [{ + 'value': '1.2.3.4', + 'weight': 1 + }], + }, + }, + 'rules': [{ + 'geos': [ + 'AS', + 'EU-ES', + 'EU-UA', + 'EU-SE', + 'NA-CA-NL', + 'OC' + ], + 'pool': 'one' + }, { + 'pool': 'two', + }], + }, + })) + + plan = provider.plan(wanted) + self.assertEquals(4, len(plan.changes)) + self.assertEquals(4, provider.apply(plan)) + + # recreate for update, and deletes for the 2 parts of the other + provider._client._request.assert_has_calls([ + call('POST', '/domains/123123/records/A', data={ + 'roundRobin': [{ + 'value': '3.2.3.4' + }], + 'name': 'ttl', + 'ttl': 300 + }), + + call('DELETE', '/domains/123123/records/A/1808521'), + call('DELETE', '/geoFilters/5303'), + call('DELETE', '/pools/A/1808522'), + call('DELETE', '/domains/123123/records/A/1808520'), + call('DELETE', '/pools/A/1808521'), + call('DELETE', '/domains/123123/records/ANAME/11189899'), + + call('PUT', '/pools/A/1808522', data={ + 'name': 'unit.tests.:www.dynamic:A:one', + 'type': 'A', + 'numReturn': 1, + 'minAvailableFailover': 1, + 'ttl': 300, + 'values': [ + {'value': '1.2.3.6', 'weight': 1}, + {'value': '1.2.3.7', 'weight': 1}], + 'id': 1808522, + 'geofilter': 5303 + }), + + call('PUT', '/geoFilters/5303', data={ + 'filterRulesLimit': 100, + 'name': 'unit.tests.:www.dynamic:A:one', + 'geoipContinents': ['AS', 'OC'], + 'geoipCountries': ['ES', 'SE', 'UA'], + 'regions': [{ + 'continentCode': 'NA', + 'countryCode': 'CA', + 'regionCode': 'NL'}], + 'id': 5303 + }), + + call('PUT', '/pools/A/1808521', data={ + 'name': 'unit.tests.:www.dynamic:A:two', + 'type': 'A', + 'numReturn': 1, + 'minAvailableFailover': 1, + 'ttl': 300, + 'values': [{'value': '1.2.3.4', 'weight': 1}], + 'id': 1808521, + 'geofilter': 1 + }), + + call('GET', '/domains/123123'), + call('POST', '/domains/123123/records/A', data={ + 'name': 'www.dynamic', + 'ttl': 300, + 'pools': [1808522], + 'recordOption': 'pools', + 'geolocation': { + 'geoipUserRegion': [5303] + } + }), + + call('POST', '/domains/123123/records/A', data={ + 'name': 'www.dynamic', + 'ttl': 300, + 'pools': [1808522], + 'recordOption': 'pools', + 'geolocation': { + 'geoipUserRegion': [5303] + } + }) + ], any_order=True) + def test_dynamic_record_failures(self): provider = ConstellixProvider('test', 'api', 'secret') @@ -367,6 +822,7 @@ class TestConstellixProvider(TestCase): "id": 1808520, "type": "A", "name": "www.dynamic", + "geolocation": None, "recordOption": "pools", "ttl": 300, "value": [], @@ -388,6 +844,8 @@ class TestConstellixProvider(TestCase): ] }]) + provider._client.geofilters = Mock(return_value=[]) + wanted = Zone('unit.tests.', []) resp.json.side_effect = [ @@ -397,7 +855,7 @@ class TestConstellixProvider(TestCase): wanted.add_record(Record.new(wanted, 'www.dynamic', { 'ttl': 300, 'type': 'A', - 'value': [ + 'values': [ '1.2.3.4' ], 'dynamic': { @@ -428,44 +886,133 @@ class TestConstellixProvider(TestCase): "id": 1808520, "type": "A", "name": "www.dynamic", + "geolocation": { + "geoipFilter": 1 + }, "recordOption": "pools", "ttl": 300, "value": [], "pools": [ 1808521 ] + }, { + "id": 1808521, + "type": "A", + "name": "www.dynamic", + "geolocation": { + "geoipFilter": 5303 + }, + "recordOption": "pools", + "ttl": 300, + "value": [], + "pools": [ + 1808522 + ] } ]) - provider._client.pools = Mock(return_value=[{ - "id": 1808521, - "name": "unit.tests.:www.dynamic:A:two", - "type": "A", - "values": [ - { - "value": "1.2.3.4", - "weight": 1 - } - ] - }]) + provider._client.pools = Mock(return_value=[ + { + "id": 1808521, + "name": "unit.tests.:www.dynamic:A:two", + "type": "A", + "values": [ + { + "value": "1.2.3.4", + "weight": 1 + }, + { + "value": "1.2.3.5", + "weight": 1 + } + ] + }, + { + "id": 1808522, + "name": "unit.tests.:www.dynamic:A:one", + "type": "A", + "values": [ + { + "value": "1.2.3.6", + "weight": 1 + }, + { + "value": "1.2.3.7", + "weight": 1 + } + ] + } + ]) + + provider._client.geofilters = Mock(return_value=[ + { + "id": 6303, + "name": "some.other", + "filterRulesLimit": 100, + "createdTs": "2021-08-19T14:47:47Z", + "modifiedTs": "2021-08-19T14:47:47Z", + "geoipContinents": ["AS", "OC"], + "geoipCountries": ["ES", "SE", "UA"], + "regions": [ + { + "continentCode": "NA", + "countryCode": "CA", + "regionCode": "NL" + } + ] + }, { + "id": 5303, + "name": "unit.tests.:www.dynamic:A:one", + "filterRulesLimit": 100, + "geoipContinents": ["AS", "OC"], + "geoipCountries": ["ES", "SE", "UA"], + "regions": [ + { + "continentCode": "NA", + "countryCode": "CA", + "regionCode": "NL" + } + ] + } + ]) wanted = Zone('unit.tests.', []) wanted.add_record(Record.new(wanted, 'www.dynamic', { 'ttl': 300, 'type': 'A', - 'value': [ + 'values': [ '1.2.3.4' ], 'dynamic': { 'pools': { + 'one': { + 'fallback': 'two', + 'values': [{ + 'value': '1.2.3.6', + 'weight': 1 + }, { + 'value': '1.2.3.7', + 'weight': 1 + }], + }, 'two': { 'values': [{ - 'value': '1.2.3.5' + 'value': '1.2.3.4', + 'weight': 1 }], }, }, 'rules': [{ + 'geos': [ + 'AS', + 'EU-ES', + 'EU-UA', + 'EU-SE', + 'OC' + ], + 'pool': 'one' + }, { 'pool': 'two', }], }, @@ -473,13 +1020,25 @@ class TestConstellixProvider(TestCase): # Try an error we can handle with requests_mock() as mock: - mock.get(ANY, status_code=200, - text='{}') + mock.get( + "https://api.dns.constellix.com/v1/domains", + status_code=200, + text='[{"id": 1234, "name": "unit.tests", "hasGeoIP": true}]') + mock.get( + "https://api.dns.constellix.com/v1/domains/1234", + status_code=200, + text='{"id": 1234, "name": "unit.tests", "hasGeoIP": true}') mock.delete(ANY, status_code=200, text='{}') mock.put("https://api.dns.constellix.com/v1/pools/A/1808521", status_code=400, text='{"errors": [\"no changes to save\"]}') + mock.put("https://api.dns.constellix.com/v1/pools/A/1808522", + status_code=400, + text='{"errors": [\"no changes to save\"]}') + mock.put("https://api.dns.constellix.com/v1/geoFilters/5303", + status_code=400, + text='{"errors": [\"no changes to save\"]}') mock.post(ANY, status_code=200, text='[{"id": 1234}]') @@ -487,13 +1046,87 @@ class TestConstellixProvider(TestCase): self.assertEquals(1, len(plan.changes)) self.assertEquals(1, provider.apply(plan)) + provider._client.geofilters = Mock(return_value=[ + { + "id": 5303, + "name": "unit.tests.:www.dynamic:A:one", + "filterRulesLimit": 100, + "regions": [ + { + "continentCode": "NA", + "countryCode": "CA", + "regionCode": "NL" + } + ] + } + ]) + + plan = provider.plan(wanted) + self.assertEquals(1, len(plan.changes)) + self.assertEquals(1, provider.apply(plan)) + + provider._client.geofilters = Mock(return_value=[ + { + "id": 5303, + "name": "unit.tests.:www.dynamic:A:one", + "filterRulesLimit": 100, + "geoipContinents": ["AS", "OC"], + } + ]) + + plan = provider.plan(wanted) + self.assertEquals(1, len(plan.changes)) + self.assertEquals(1, provider.apply(plan)) + # Now what happens if an error happens that we can't handle + # geofilter case with requests_mock() as mock: - mock.get(ANY, status_code=200, - text='{}') + mock.get( + "https://api.dns.constellix.com/v1/domains", + status_code=200, + text='[{"id": 1234, "name": "unit.tests", "hasGeoIP": true}]') + mock.get( + "https://api.dns.constellix.com/v1/domains/1234", + status_code=200, + text='{"id": 1234, "name": "unit.tests", "hasGeoIP": true}') mock.delete(ANY, status_code=200, text='{}') mock.put("https://api.dns.constellix.com/v1/pools/A/1808521", + status_code=400, + text='{"errors": [\"no changes to save\"]}') + mock.put("https://api.dns.constellix.com/v1/pools/A/1808522", + status_code=400, + text='{"errors": [\"no changes to save\"]}') + mock.put("https://api.dns.constellix.com/v1/geoFilters/5303", + status_code=400, + text='{"errors": [\"generic error\"]}') + mock.post(ANY, status_code=200, + text='[{"id": 1234}]') + + plan = provider.plan(wanted) + self.assertEquals(1, len(plan.changes)) + with self.assertRaises(ConstellixClientBadRequest): + provider.apply(plan) + + # Now what happens if an error happens that we can't handle + with requests_mock() as mock: + mock.get( + "https://api.dns.constellix.com/v1/domains", + status_code=200, + text='[{"id": 1234, "name": "unit.tests", "hasGeoIP": true}]') + mock.get( + "https://api.dns.constellix.com/v1/domains/1234", + status_code=200, + text='{"id": 1234, "name": "unit.tests", "hasGeoIP": true}') + mock.delete(ANY, status_code=200, + text='{}') + mock.put("https://api.dns.constellix.com/v1/pools/A/1808521", + status_code=400, + text='{"errors": [\"generic error\"]}') + mock.put("https://api.dns.constellix.com/v1/pools/A/1808522", + status_code=400, + text='{"errors": [\"generic error\"]}') + mock.put("https://api.dns.constellix.com/v1/geoFilters/5303", status_code=400, text='{"errors": [\"generic error\"]}') mock.post(ANY, status_code=200, From 2cf52180acf489bb67ea6d0dcf1a52f461d44ed0 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 13 Sep 2021 19:09:05 -0700 Subject: [PATCH 11/11] Add lenient config support to validate(_configs) --- octodns/manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/octodns/manager.py b/octodns/manager.py index 104e445..0ce425b 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -552,6 +552,7 @@ class Manager(object): source_zone = source_zone continue + lenient = config.get('lenient', False) try: sources = config['sources'] except KeyError: @@ -572,7 +573,7 @@ class Manager(object): for source in sources: if isinstance(source, YamlProvider): - source.populate(zone) + source.populate(zone, lenient=lenient) # check that processors are in order if any are specified processors = config.get('processors', [])